diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd15dcadea..3b70fc7263 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,11 @@ jobs: run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse-dev - name: Build run: | - go build ./... + CGO_ENABLED=0 go build ./... go install ./tools/build_gcsfuse build_gcsfuse . /tmp ${GITHUB_SHA} - name: Test - run: go test -p 1 -count 1 -v -cover ./... + run: CGO_ENABLED=0 go test -p 1 -count 1 -v -cover ./... lint: name: Lint runs-on: ubuntu-20.04 diff --git a/.gitignore b/.gitignore index f40cd650ac..8656d8cf65 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ _testmain.go # Editors .idea/ +.vscode/ # External folders vendor/ diff --git a/DEBIAN/changelog b/DEBIAN/changelog new file mode 100644 index 0000000000..b3baea62a3 --- /dev/null +++ b/DEBIAN/changelog @@ -0,0 +1,5 @@ +gcsfuse (1.0.0) stable; urgency=medium + + * Package created with dpkg-deb --build + + -- GCSFuse Team Thu, 13 Jul 2023 05:37:50 +0000 diff --git a/DEBIAN/control b/DEBIAN/control new file mode 100644 index 0000000000..37bfff0b34 --- /dev/null +++ b/DEBIAN/control @@ -0,0 +1,12 @@ +Version: 1.0.0 +Source: gcsfuse +Maintainer: GCSFuse Team +Homepage: https://github.com/GoogleCloudPlatform/gcsfuse +Package: gcsfuse +Architecture: all +Depends: fuse +Description: User-space file system for Google Cloud Storage. + GCSFuse is a FUSE adapter that allows you to mount and access Cloud Storage + buckets as local file systems, so applications can read and write objects in + your bucket using standard file system semantics. Cloud Storage FUSE is an + open source product that's supported by Google. diff --git a/DEBIAN/copyright b/DEBIAN/copyright new file mode 100644 index 0000000000..201944d993 --- /dev/null +++ b/DEBIAN/copyright @@ -0,0 +1,23 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: gcsfuse +Upstream-Contact: gcs-fuse-maintainers@google.com + +Files: * +Copyright: Copyright 2020 Google Inc. +License: Apache-2.0 + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the complete text of the Apache version 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/DEBIAN/gcsfuse-docs.docs b/DEBIAN/gcsfuse-docs.docs new file mode 100644 index 0000000000..a2511a80a8 --- /dev/null +++ b/DEBIAN/gcsfuse-docs.docs @@ -0,0 +1,3 @@ +https://cloud.google.com/storage/docs/gcs-fuse +https://github.com/GoogleCloudPlatform/gcsfuse#readme +https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/docs diff --git a/Dockerfile b/Dockerfile index cb12d822db..62dd0aee4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # Mount the gcsfuse to /mnt/gcs: # > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse -FROM golang:1.20.4-alpine as builder +FROM golang:1.20.5-alpine as builder RUN apk add git diff --git a/benchmarks/concurrent_read/job/job.go b/benchmarks/concurrent_read/job/job.go deleted file mode 100644 index d2619b1679..0000000000 --- a/benchmarks/concurrent_read/job/job.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2020 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package job - -import ( - "context" - "fmt" - "io" - "runtime/trace" - "time" - - "github.com/googlecloudplatform/gcsfuse/benchmarks/concurrent_read/readers" - "github.com/googlecloudplatform/gcsfuse/internal/logger" -) - -const ( - KB = 1024 - MB = 1024 * KB -) - -type Job struct { - // Choose from HTTP/1.1, HTTP/2, GRPC - Protocol string - // Max connections for this job - Connections int - // Choose from vendor, google. - Implementation string -} - -type Stats struct { - Job *Job - TotalBytes int64 - TotalFiles int - Mbps []float32 - Duration time.Duration -} - -func (s *Stats) Throughput() float32 { - mbs := float32(s.TotalBytes) / float32(MB) - seconds := float32(s.Duration) / float32(time.Second) - return mbs / seconds -} - -func (s *Stats) Report() { - logger.Infof( - "# TEST READER %s\n"+ - "Protocol: %s (%v connections per host)\n"+ - "Total bytes: %d\n"+ - "Total files: %d\n"+ - "Avg Throughput: %.1f MB/s\n\n", - s.Job.Protocol, - s.Job.Implementation, - s.Job.Connections, - s.TotalBytes, - s.TotalFiles, - s.Throughput(), - ) -} - -func (s *Stats) Query(key string) string { - switch key { - case "Protocol": - return s.Job.Protocol - case "Implementation": - return s.Job.Implementation - case "Connections": - return fmt.Sprintf("%d", s.Job.Connections) - case "TotalBytes (MB)": - return fmt.Sprintf("%d", s.TotalBytes/MB) - case "TotalFiles": - return fmt.Sprintf("%d", s.TotalFiles) - case "Throughput (MB/s)": - return fmt.Sprintf("%.1f", s.Throughput()) - default: - return "" - } -} - -type Client interface { - NewReader(objectName string) (io.ReadCloser, error) -} - -func (job *Job) Run(ctx context.Context, bucketName string, objects []string) (*Stats, error) { - var client Client - var err error - - switch job.Implementation { - case "vendor": - client, err = readers.NewVendorClient(ctx, job.Protocol, job.Connections, bucketName) - case "google": - client, err = readers.NewGoogleClient(ctx, job.Protocol, job.Connections, bucketName) - default: - panic(fmt.Errorf("Unknown reader implementation: %q", job.Implementation)) - } - - if err != nil { - return nil, err - } - stats := job.testReader(ctx, client, objects) - return stats, nil -} - -func (job *Job) testReader(ctx context.Context, client Client, objectNames []string) *Stats { - stats := &Stats{Job: job} - reportDuration := 10 * time.Second - ticker := time.NewTicker(reportDuration) - defer ticker.Stop() - - doneBytes := make(chan int64) - doneFiles := make(chan int) - start := time.Now() - - // run readers concurrently - for _, objectName := range objectNames { - name := objectName - go func() { - region := trace.StartRegion(ctx, "NewReader") - reader, err := client.NewReader(name) - region.End() - if err != nil { - fmt.Printf("Skip %q: %s", name, err) - return - } - defer reader.Close() - - p := make([]byte, 128*1024) - region = trace.StartRegion(ctx, "ReadObject") - for { - n, err := reader.Read(p) - - doneBytes <- int64(n) - if err == io.EOF { - break - } else if err != nil { - panic(fmt.Errorf("read %q fails: %w", name, err)) - } - } - region.End() - doneFiles <- 1 - return - }() - } - - // collect test stats - var lastTotalBytes int64 - for stats.TotalFiles < len(objectNames) { - select { - case b := <-doneBytes: - stats.TotalBytes += b - case f := <-doneFiles: - stats.TotalFiles += f - case <-ticker.C: - readBytes := stats.TotalBytes - lastTotalBytes - lastTotalBytes = stats.TotalBytes - mbps := float32(readBytes/MB) / float32(reportDuration/time.Second) - stats.Mbps = append(stats.Mbps, mbps) - } - } - stats.Duration = time.Since(start) - return stats -} diff --git a/benchmarks/concurrent_read/main.go b/benchmarks/concurrent_read/main.go deleted file mode 100644 index b3375af419..0000000000 --- a/benchmarks/concurrent_read/main.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2020 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Concurrently read objects on GCS provided by stdin. The user must ensure -// (1) all the objects come from the same bucket, and -// (2) the script is authorized to read from the bucket. -// The stdin should contain N lines of object name, in the form of -// "gs://bucket-name/object-name". -// -// This benchmark only tests the internal reader implementation, which -// doesn't have FUSE involved. -// -// Usage Example: -// gsutil ls 'gs://bucket/prefix*' | go run ./benchmarks/concurrent_read/ -// - -package main - -import ( - "bufio" - "context" - "flag" - "fmt" - "io" - "log" - "os" - "runtime/trace" - "strings" - - "github.com/googlecloudplatform/gcsfuse/benchmarks/concurrent_read/job" - "github.com/googlecloudplatform/gcsfuse/internal/perf" -) - -type BenchmarkConfig struct { - // The GCS bucket storing the objects to be read. - Bucket string - // The GCS objects as 'gs://...' to be read from the bucket above. - Objects []string - // Each job reads all the objects. - Jobs []*job.Job -} - -func getJobs() []*job.Job { - return []*job.Job{ - &job.Job{ - Protocol: "HTTP/1.1", - Connections: 50, - Implementation: "vendor", - }, - &job.Job{ - Protocol: "HTTP/2", - Connections: 50, - Implementation: "vendor", - }, - &job.Job{ - Protocol: "HTTP/1.1", - Connections: 50, - Implementation: "google", - }, - &job.Job{ - Protocol: "HTTP/2", - Connections: 50, - Implementation: "google", - }, - } -} - -func run(cfg BenchmarkConfig) { - ctx := context.Background() - - ctx, traceTask := trace.NewTask(ctx, "ReadAllObjects") - defer traceTask.End() - - var statsList []*job.Stats - for _, job := range cfg.Jobs { - stats, err := job.Run(ctx, cfg.Bucket, cfg.Objects) - if err != nil { - fmt.Printf("Job failed: %v", job) - continue - } - stats.Report() - statsList = append(statsList, stats) - } - printSummary(statsList) -} - -func printSummary(statsList []*job.Stats) { - cols := []string{ - "Protocol", - "Implementation", - "Connections", - "TotalBytes (MB)", - "TotalFiles", - "Throughput (MB/s)", - } - for _, col := range cols { - fmt.Printf(" %s |", col) - } - fmt.Println("") - for _, col := range cols { - fmt.Printf(strings.Repeat("-", len(col)+6)) - } - fmt.Println("") - for _, stats := range statsList { - for _, col := range cols { - value := stats.Query(col) - padding := strings.Repeat(" ", len(col)+4-len(value)) - fmt.Printf("%s%s |", padding, value) - } - fmt.Println("") - } -} - -func getLinesFromStdin() (lines []string) { - reader := bufio.NewReader(os.Stdin) - for { - line, err := reader.ReadString('\n') - if err != nil { - if err == io.EOF { - err = nil - break - } - panic(fmt.Errorf("Stdin error: %w", err)) - } - lines = append(lines, line) - } - return -} - -func getObjectNames() (bucketName string, objectNames []string) { - uris := getLinesFromStdin() - for _, uri := range uris { - path := uri[len("gs://"):] - path = strings.TrimRight(path, "\n") - segs := strings.Split(path, "/") - if len(segs) <= 1 { - panic(fmt.Errorf("Not a file name: %q", uri)) - } - - if bucketName == "" { - bucketName = segs[0] - } else if bucketName != segs[0] { - panic(fmt.Errorf("Multiple buckets: %q, %q", bucketName, segs[0])) - } - - objectName := strings.Join(segs[1:], "/") - objectNames = append(objectNames, objectName) - } - return -} - -func main() { - flag.Parse() - - go perf.HandleCPUProfileSignals() - - // Enable trace - f, err := os.Create("/tmp/concurrent_read_trace.out") - if err != nil { - log.Fatalf("failed to create trace output file: %v", err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatalf("failed to close trace file: %v", err) - } - }() - if err := trace.Start(f); err != nil { - log.Fatalf("failed to start trace: %v", err) - } - defer trace.Stop() - - bucketName, objectNames := getObjectNames() - config := BenchmarkConfig{ - Bucket: bucketName, - Objects: objectNames, - Jobs: getJobs(), - } - run(config) - - return -} diff --git a/benchmarks/concurrent_read/readers/google.go b/benchmarks/concurrent_read/readers/google.go deleted file mode 100644 index 38ed90c20e..0000000000 --- a/benchmarks/concurrent_read/readers/google.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2020 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package readers - -import ( - "context" - "errors" - "io" - "net/http" - - "cloud.google.com/go/storage" - "github.com/jacobsa/gcloud/gcs" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - "google.golang.org/api/option" -) - -// Google reader depends on "cloud.google.com/go/storage" -type googleClient struct { - ctx context.Context - bucket *storage.BucketHandle -} - -func NewGoogleClient(ctx context.Context, protocol string, connections int, bucketName string) (*googleClient, error) { - client, err := getStorageClient(ctx, protocol, connections) - if err != nil { - return nil, err - } - bucket := client.Bucket(bucketName) - return &googleClient{ctx, bucket}, nil -} - -func (c *googleClient) NewReader(objectName string) (io.ReadCloser, error) { - return c.bucket.Object(objectName).NewReader(c.ctx) -} - -func getStorageClient(ctx context.Context, protocol string, connections int) (*storage.Client, error) { - if protocol == "GRPC" { - return getGRPCClient() - } - tokenSrc, err := google.DefaultTokenSource(ctx, gcs.Scope_FullControl) - if err != nil { - return nil, err - } - return storage.NewClient( - ctx, - option.WithUserAgent(userAgent), - option.WithHTTPClient(&http.Client{ - Transport: &oauth2.Transport{ - Base: getTransport(protocol, connections), - Source: tokenSrc, - }, - }), - ) -} - -func getGRPCClient() (*storage.Client, error) { - return nil, errors.New("GRPC is not supported") -} diff --git a/benchmarks/concurrent_read/readers/util.go b/benchmarks/concurrent_read/readers/util.go deleted file mode 100644 index 07b2a7a30f..0000000000 --- a/benchmarks/concurrent_read/readers/util.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package readers - -import ( - "crypto/tls" - "fmt" - "net/http" -) - -const userAgent = "gcsfuse/dev Benchmark (concurrent_read)" - -func getTransport(protocol string, connections int) *http.Transport { - switch protocol { - case "HTTP/1.1": - return &http.Transport{ - MaxConnsPerHost: connections, - // This disables HTTP/2 in the transport. - TLSNextProto: make( - map[string]func(string, *tls.Conn) http.RoundTripper, - ), - } - case "HTTP/2": - return http.DefaultTransport.(*http.Transport).Clone() - default: - panic(fmt.Errorf("Unsupported protocol: %q", protocol)) - } -} diff --git a/benchmarks/concurrent_read/readers/vendor.go b/benchmarks/concurrent_read/readers/vendor.go deleted file mode 100644 index 644ea36b9e..0000000000 --- a/benchmarks/concurrent_read/readers/vendor.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package readers - -import ( - "context" - "fmt" - "io" - "net/url" - - "github.com/jacobsa/gcloud/gcs" - "golang.org/x/oauth2/google" -) - -// Vendor reader depends on "github.com/jacobsa/gcloud/gcs" -type vendorClient struct { - ctx context.Context - bucket gcs.Bucket -} - -func NewVendorClient(ctx context.Context, protocol string, connections int, bucketName string) (*vendorClient, error) { - tokenSrc, err := google.DefaultTokenSource(ctx, gcs.Scope_FullControl) - if err != nil { - return nil, err - } - endpoint, _ := url.Parse("https://storage.googleapis.com:443") - config := &gcs.ConnConfig{ - Url: endpoint, - TokenSource: tokenSrc, - UserAgent: userAgent, - Transport: getTransport(protocol, connections), - } - conn, err := gcs.NewConn(config) - if err != nil { - return nil, err - } - bucket, err := conn.OpenBucket( - ctx, - &gcs.OpenBucketOptions{ - Name: bucketName, - }, - ) - if err != nil { - panic(fmt.Errorf("Cannot open bucket %q: %w", bucketName, err)) - } - return &vendorClient{ctx, bucket}, nil -} - -func (c *vendorClient) NewReader(objectName string) (io.ReadCloser, error) { - return c.bucket.NewReader( - c.ctx, - &gcs.ReadObjectRequest{ - Name: objectName, - }, - ) -} diff --git a/docs/semantics.md b/docs/semantics.md index 91b7482ddd..6a47e89f49 100644 --- a/docs/semantics.md +++ b/docs/semantics.md @@ -124,7 +124,7 @@ This is the default behavior, unless a user passes the ```--implicit-dirs``` fla Cloud Storage FUSE supports a flag called ```--implicit-dirs``` that changes the behavior for how pre-existing directory structures, not created by Cloud Storage FUSE, are mounted and visible to Cloud Storage FUSE. When this flag is enabled, name lookup requests from the kernel use the Cloud Storage API's Objects.list operation to search for objects that would implicitly define the existence of a directory with the name in question. -The example above describes how from the local filesystem the user sees only 0.txt, until the user creates A/, A/B/, C/ using mkdir. If instead the ```--implicity-dirs``` flag is passed, you would see the intended directory structure without first having to create the directories A/, A/B/, C/. +The example above describes how from the local filesystem the user sees only 0.txt, until the user creates A/, A/B/, C/ using mkdir. If instead the ```--implicit-dirs``` flag is passed, you would see the intended directory structure without first having to create the directories A/, A/B/, C/. However, implicit directories does have drawbacks: - The feature requires an additional request to Cloud Storage for each name lookup, which may have costs in terms of both charges for operations and latency. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 4015dd11a2..268fdae477 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,4 +1,4 @@ -# Troubleshooting for production issues + # Troubleshooting for production issues This page enumerates some common user facing issues around GCSFuse and also discusses potential solutions to the same. | Issues | Fix | @@ -6,14 +6,14 @@ This page enumerates some common user facing issues around GCSFuse and also disc | Generic Mounting Issue | Most of the common mount point issues are around permissions on both local mount point and the Cloud Storage bucket. It is highly recommended to retry with --foreground --debug_fuse --debug_fs --debug_gcs --debug_http flags which would provide much more detailed logs to understand the errors better and possibly provide a solution. | | Mount successful but files not visible | Try mounting the gcsfuse with --implicit-dir flag. Read the [semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md) to know the reasoning. | | Mount failed with fusermount3 exit status 1 | It comes when the bucket is already mounted in a folder and we try to mount it again. You need to unmount first and then remount. | -| Mount failed with error: Current requires cgo or $USER set in environment | It comes when we try mounting by building the gcsfuse codebase. To fix this, build the gcsfuse package by enabling the CGO_ENABLED flag in the go env and then mount back.
  1. Check the current value using - ```go env``` command.
  2. If it is unset, set this using - ```export CGO_ENABLED=1``` command.
| +| version `GLIBC_x.yz` not found | GCSFuse should not be linking to glibc. Please either `export CGO_ENABLED=0` in your environment or prefix `CGO_ENABLED=0` to any `go build\|run\|test` commands that you're invoking. | | Mount get stuck with error: DefaultTokenSource: google: could not find default credentials | Run ```gcloud auth application-default login``` command to fetch default credentials to the VM. This will fetch the credentials to the following locations:
  1. For linux - $HOME/.config/gcloud/application_default_credentials.json
  2. For windows - %APPDATA%/gcloud/applicateion_default_credentials.json
| | Input/Output Error | It’s a generic error, but the most probable culprit is the bucket not having the right permission for Cloud Storage FUSE to operate on. Ref - [here](https://stackoverflow.com/questions/36382704/gcsfuse-input-output-error) | | Generic NO_PUBKEY Error - while installing Cloud Storage FUSE on ubuntu 22.04 | It happens while running - ```sudo apt-get update``` - working on installing Cloud Storage FUSE. You just have to add the pubkey you get in the error using the below command: ```sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys ``` And then try running ```sudo apt-get update``` | | Cloud Storage FUSE fails with Docker container | Though not tested extensively, the [community](https://stackoverflow.com/questions/65715624/permission-denied-with-gcsfuse-in-unprivileged-ubuntu-based-docker-container) reports that Cloud Storage FUSE works only in privileged mode when used with Docker. There are [solutions](https://cloud.google.com/iam/docs/service-account-overview) which exist and claim to do so without privileged mode, but these are not tested by the Cloud Storage FUSE team | -| daemonize.Run: readFromProcess: sub-process: mountWithArgs: mountWithConn: fs.NewServer: create file system: SetUpBucket: OpenBucket: Bad credentials for bucket BUCKET_NAME: permission denied | Check the bucket name. Make sure it is within your project. Make sure the applied roles on the bucket contain storage.objects.list permission. You can refer to them [here](https://cloud.google.com/storage/docs/access-control/iam-roles). | -| daemonize.Run: readFromProcess: sub-process: mountWithArgs: mountWithConn: fs.NewServer: create file system: SetUpBucket: OpenBucket: Unknown bucket BUCKET_NAME: no such file or directory | Check the bucket name. Make sure the [service account](https://www.google.com/url?q=https://cloud.google.com/iam/docs/service-accounts&sa=D&source=docs&ust=1679992003850814&usg=AOvVaw3nJ6wNQK4FZdgm8gBTS82l) has permissions to access the files. It must at least have the permissions of the Storage Object Viewer role. | -| daemonize.Run: readFromProcess: sub-process: mountWithArgs: mountWithConn: Mount: mount: running fusermount: exit status 1 stderr: /bin/fusermount: fuse device not found, try 'modprobe fuse' first | To run the container locally, add the --privilege flag to the docker run command: ```docker run --privileged gcr.io/PROJECT/my-fs-app ```
  • You must create a local mount directory
  • If you want all the logs from the mount process use the --foreground flag in combination with the mount command: ```gcsfuse --foreground --debug_gcs --debug_fuse $GCSFUSE_BUCKET $MNT_DIR ```
  • Add --debug_http for HTTP request/response debug output.
  • Add --debug_fuse to enable fuse-related debugging output.
  • Add --debug_gcs to print GCS request and timing information.
| +| daemonize.Run: readFromProcess: sub-process: mountWithArgs: mountWithStorageHandle: fs.NewServer: create file system: SetUpBucket: OpenBucket: Bad credentials for bucket BUCKET_NAME: permission denied | Check the bucket name. Make sure it is within your project. Make sure the applied roles on the bucket contain storage.objects.list permission. You can refer to them [here](https://cloud.google.com/storage/docs/access-control/iam-roles). | +| daemonize.Run: readFromProcess: sub-process: mountWithArgs: mountWithStorageHandle: fs.NewServer: create file system: SetUpBucket: OpenBucket: Unknown bucket BUCKET_NAME: no such file or directory | Check the bucket name. Make sure the [service account](https://www.google.com/url?q=https://cloud.google.com/iam/docs/service-accounts&sa=D&source=docs&ust=1679992003850814&usg=AOvVaw3nJ6wNQK4FZdgm8gBTS82l) has permissions to access the files. It must at least have the permissions of the Storage Object Viewer role. | +| daemonize.Run: readFromProcess: sub-process: mountWithArgs: mountWithStorageHandle: Mount: mount: running fusermount: exit status 1 stderr: /bin/fusermount: fuse device not found, try 'modprobe fuse' first | To run the container locally, add the --privilege flag to the docker run command: ```docker run --privileged gcr.io/PROJECT/my-fs-app ```
  • You must create a local mount directory
  • If you want all the logs from the mount process use the --foreground flag in combination with the mount command: ```gcsfuse --foreground --debug_gcs --debug_fuse $GCSFUSE_BUCKET $MNT_DIR ```
  • Add --debug_http for HTTP request/response debug output.
  • Add --debug_fuse to enable fuse-related debugging output.
  • Add --debug_gcs to print GCS request and timing information.
| | Cloud Storage FUSE installation fails with an error at build time. | Only specific OS distributions are currently supported, learn more about [Installing Cloud Storage FUSE](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/installing.md). | | Cloud Storage FUSE not mounting after reboot when entry is present in ```/etc/fstab``` with 1 or 2 as fsck order | Pass [_netdev option](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/mounting.md#persisting-a-mount) in fstab entry (reference issue [here](https://github.com/GoogleCloudPlatform/gcsfuse/issues/1043)). With this option, mount will be attempted on reboot only when network is connected. | | Cloud Storage FUSE get stuck when using it to concurrently work with a large number of opened files (reference issue [here](https://github.com/GoogleCloudPlatform/gcsfuse/issues/1043)) | This happens when gcsfuse is mounted with http1 client (default) and the application using gcsfuse tries to keep more than value of `--max-conns-per-host` number of files opened. You can try (a) Passing a value higher than the number of files you want to keep open to `--max-conns-per-host` flag. (b) Adding some timeout for http client connections using `--http-client-timeout` flag. | diff --git a/flags.go b/flags.go index 5a52e11cf7..74539793f6 100644 --- a/flags.go +++ b/flags.go @@ -140,9 +140,10 @@ func newApp() (app *cli.App) { ///////////////////////// cli.StringFlag{ - Name: "endpoint", - Value: "https://storage.googleapis.com:443", - Usage: "The endpoint to connect to.", + Name: "custom-endpoint", + Usage: "Alternate endpoint for fetching data. Should be used only for testing purposes. " + + "The endpoint should be equivalent to the base endpoint of GCS JSON API (https://storage.googleapis.com/storage/v1). " + + "If not specified GCS endpoint will be used. Auth will be skipped for custom endpoint.", }, cli.StringFlag{ @@ -302,6 +303,11 @@ func newApp() (app *cli.App) { Usage: "The format of the log file: 'text' or 'json'.", }, + cli.BoolFlag{ + Name: "experimental-enable-json-read", + Usage: "By default read flow uses xml api, this flag will enable the json path for read operation.", + }, + ///////////////////////// // Debugging ///////////////////////// @@ -328,9 +334,8 @@ func newApp() (app *cli.App) { }, cli.BoolFlag{ - Name: "debug_http", - Usage: "Dump HTTP requests and responses to/from GCS, " + - "doesn't work when enable-storage-client-library flag is true.", + Name: "debug_http", + Usage: "This flag is currently unused.", }, cli.BoolFlag{ @@ -342,15 +347,6 @@ func newApp() (app *cli.App) { Name: "debug_mutex", Usage: "Print debug messages when a mutex is held too long.", }, - - ///////////////////////// - // Client - ///////////////////////// - - cli.BoolTFlag{ - Name: "enable-storage-client-library", - Usage: "If true, will use go storage client library otherwise jacobsa/gcloud", - }, }, } @@ -373,7 +369,7 @@ type flagStorage struct { RenameDirLimit int64 // GCS - Endpoint *url.URL + CustomEndpoint *url.URL BillingProject string KeyFile string TokenUrl string @@ -398,11 +394,12 @@ type flagStorage struct { EnableNonexistentTypeCache bool // Monitoring & Logging - StackdriverExportInterval time.Duration - OtelCollectorAddress string - LogFile string - LogFormat string - DebugFuseErrors bool + StackdriverExportInterval time.Duration + OtelCollectorAddress string + LogFile string + LogFormat string + ExperimentalEnableJsonRead bool + DebugFuseErrors bool // Debugging DebugFuse bool @@ -411,9 +408,6 @@ type flagStorage struct { DebugHTTP bool DebugInvariants bool DebugMutex bool - - // client - EnableStorageClientLibrary bool } const GCSFUSE_PARENT_PROCESS_DIR = "gcsfuse-parent-process-dir" @@ -490,11 +484,19 @@ func resolvePathForTheFlagsInContext(c *cli.Context) (err error) { // Add the flags accepted by run to the supplied flag set, returning the // variables into which the flags will parse. func populateFlags(c *cli.Context) (flags *flagStorage, err error) { - endpoint, err := url.Parse(c.String("endpoint")) - if err != nil { - fmt.Printf("Could not parse endpoint") - return + customEndpointStr := c.String("custom-endpoint") + var customEndpoint *url.URL + + if customEndpointStr == "" { + customEndpoint = nil + } else { + customEndpoint, err = url.Parse(customEndpointStr) + if err != nil { + err = fmt.Errorf("could not parse custom-endpoint: %w", err) + return + } } + clientProtocolString := strings.ToLower(c.String("client-protocol")) clientProtocol := mountpkg.ClientProtocol(clientProtocolString) flags = &flagStorage{ @@ -513,7 +515,7 @@ func populateFlags(c *cli.Context) (flags *flagStorage, err error) { RenameDirLimit: int64(c.Int("rename-dir-limit")), // GCS, - Endpoint: endpoint, + CustomEndpoint: customEndpoint, BillingProject: c.String("billing-project"), KeyFile: c.String("key-file"), TokenUrl: c.String("token-url"), @@ -538,22 +540,20 @@ func populateFlags(c *cli.Context) (flags *flagStorage, err error) { EnableNonexistentTypeCache: c.Bool("enable-nonexistent-type-cache"), // Monitoring & Logging - StackdriverExportInterval: c.Duration("stackdriver-export-interval"), - OtelCollectorAddress: c.String("experimental-opentelemetry-collector-address"), - LogFile: c.String("log-file"), - LogFormat: c.String("log-format"), + StackdriverExportInterval: c.Duration("stackdriver-export-interval"), + OtelCollectorAddress: c.String("experimental-opentelemetry-collector-address"), + LogFile: c.String("log-file"), + LogFormat: c.String("log-format"), + ExperimentalEnableJsonRead: c.Bool("experimental-enable-json-read"), // Debugging, DebugFuseErrors: c.BoolT("debug_fuse_errors"), DebugFuse: c.Bool("debug_fuse"), DebugGCS: c.Bool("debug_gcs"), - DebugFS: c.Bool("debug_fs"), DebugHTTP: c.Bool("debug_http"), + DebugFS: c.Bool("debug_fs"), DebugInvariants: c.Bool("debug_invariants"), DebugMutex: c.Bool("debug_mutex"), - - // Client, - EnableStorageClientLibrary: c.Bool("enable-storage-client-library"), } // Handle the repeated "-o" flag. diff --git a/flags_test.go b/flags_test.go index a4362e5356..1ff2fa9fad 100644 --- a/flags_test.go +++ b/flags_test.go @@ -47,6 +47,7 @@ func parseArgs(args []string) (flags *flagStorage) { var err error app.Action = func(appCtx *cli.Context) { flags, err = populateFlags(appCtx) + AssertEq(nil, err) } // Simulate argv. @@ -80,6 +81,7 @@ func (t *FlagsTest) Defaults() { ExpectEq(-1, f.EgressBandwidthLimitBytesPerSecond) ExpectEq(-1, f.OpRateLimitHz) ExpectTrue(f.ReuseTokenFromUrl) + ExpectEq(nil, f.CustomEndpoint) // Tuning ExpectEq(4096, f.StatCacheCapacity) @@ -106,11 +108,11 @@ func (t *FlagsTest) Bools() { "reuse-token-from-url", "debug_fuse_errors", "debug_fuse", - "debug_gcs", "debug_http", + "debug_gcs", "debug_invariants", - "enable-storage-client-library", "enable-nonexistent-type-cache", + "experimental-enable-json-read", } var args []string @@ -130,8 +132,8 @@ func (t *FlagsTest) Bools() { ExpectTrue(f.DebugGCS) ExpectTrue(f.DebugHTTP) ExpectTrue(f.DebugInvariants) - ExpectTrue(f.EnableStorageClientLibrary) ExpectTrue(f.EnableNonexistentTypeCache) + ExpectTrue(f.ExperimentalEnableJsonRead) // --foo=false form args = nil @@ -147,7 +149,6 @@ func (t *FlagsTest) Bools() { ExpectFalse(f.DebugGCS) ExpectFalse(f.DebugHTTP) ExpectFalse(f.DebugInvariants) - ExpectFalse(f.EnableStorageClientLibrary) ExpectFalse(f.EnableNonexistentTypeCache) // --foo=true form @@ -164,7 +165,6 @@ func (t *FlagsTest) Bools() { ExpectTrue(f.DebugGCS) ExpectTrue(f.DebugHTTP) ExpectTrue(f.DebugInvariants) - ExpectTrue(f.EnableStorageClientLibrary) ExpectTrue(f.EnableNonexistentTypeCache) } @@ -463,10 +463,3 @@ func (t *FlagsTest) TestValidateFlagsForValidSequentialReadSizeAndHTTP2ClientPro AssertEq(nil, err) } - -func (t *FlagsTest) TestDefaultValueOfEnableStorageClientLibraryFlag() { - var args []string = nil - f := parseArgs(args) - - ExpectTrue(f.EnableStorageClientLibrary) -} diff --git a/go.mod b/go.mod index c0ce91cd8d..d96cba571c 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,18 @@ module github.com/googlecloudplatform/gcsfuse go 1.20 require ( + cloud.google.com/go/compute/metadata v0.2.3 cloud.google.com/go/storage v1.31.0 contrib.go.opencensus.io/exporter/ocagent v0.7.0 contrib.go.opencensus.io/exporter/stackdriver v0.13.11 - github.com/fsouza/fake-gcs-server v1.38.4 + github.com/fsouza/fake-gcs-server v1.40.3 github.com/googleapis/gax-go/v2 v2.11.0 github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f - github.com/jacobsa/fuse v0.0.0-20230509090321-7263f3a2b474 - github.com/jacobsa/gcloud v0.0.0-20230425120041-5ed2958cdfee + github.com/jacobsa/fuse v0.0.0-20230810134708-ab21db1af836 + github.com/jacobsa/gcloud v0.0.0-20230803125757-3196d990d984 github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11 - github.com/jacobsa/ratelimit v0.0.0-20150904001804-f5e47030f3b0 github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3 github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6 @@ -24,6 +24,7 @@ require ( go.opencensus.io v0.24.0 golang.org/x/net v0.10.0 golang.org/x/oauth2 v0.8.0 + golang.org/x/time v0.3.0 google.golang.org/api v0.126.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -31,11 +32,11 @@ require ( require ( cloud.google.com/go v0.110.2 // indirect cloud.google.com/go/compute v1.19.3 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.0 // indirect cloud.google.com/go/monitoring v1.13.0 // indirect cloud.google.com/go/pubsub v1.30.0 // indirect cloud.google.com/go/trace v1.9.0 // indirect + github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/aws/aws-sdk-go v1.44.217 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect @@ -50,7 +51,7 @@ require ( github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/pkg/xattr v0.4.8 // indirect + github.com/pkg/xattr v0.4.9 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/stretchr/testify v1.8.2 // indirect diff --git a/go.sum b/go.sum index f2d810fffb..da7d4d1364 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= @@ -109,8 +111,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsouza/fake-gcs-server v1.38.4 h1:FciRmVB7IC+0TnS2n/Sh12Z+oi0whyWVjTc3oNI2ELg= -github.com/fsouza/fake-gcs-server v1.38.4/go.mod h1:41eZwb5PT2Gyr7KvTkFxciD5otwT72X4DWk7TAXdcuU= +github.com/fsouza/fake-gcs-server v1.40.3 h1:JPCaiXsk9XkHzUqyYM/6MDmpqdwN4C3qSA5iOQPBRrw= +github.com/fsouza/fake-gcs-server v1.40.3/go.mod h1:WmAi3nILMdFDGSC2ppegChf7IMJaqOw1VFu3iFjqAq0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -215,18 +217,16 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f h1:X+tnaqoCcBgAwSTJtoYW6p0qKiuPyMfofEHEFUf2kdU= github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f/go.mod h1:Ip4fOwzCrnDVuluHBd7FXIMb7SHOKfkt9/UDrYSZvqI= -github.com/jacobsa/fuse v0.0.0-20230509090321-7263f3a2b474 h1:6z3Yj4PZKk3n18T2s7RYCa4uBCOpeLoQfDH/mUZTrVo= -github.com/jacobsa/fuse v0.0.0-20230509090321-7263f3a2b474/go.mod h1:XUKuYy1M4vamyxQjW8/WZBTxyZ0NnUiq+kkA+WWOfeI= -github.com/jacobsa/gcloud v0.0.0-20230425120041-5ed2958cdfee h1:1NvpBXX7CiuoK+SdLNMwelLB+2OkJLxhjllc0WgY8sE= -github.com/jacobsa/gcloud v0.0.0-20230425120041-5ed2958cdfee/go.mod h1:CGkT80TfaoPTzQ8My+t2M7PnMDvkwAR36Qm8Mm8HytI= +github.com/jacobsa/fuse v0.0.0-20230810134708-ab21db1af836 h1:Xhn8huWAi1BVXQlpSEO+ZTWmrkaH+FuCJw0KLQtzwOg= +github.com/jacobsa/fuse v0.0.0-20230810134708-ab21db1af836/go.mod h1:XUKuYy1M4vamyxQjW8/WZBTxyZ0NnUiq+kkA+WWOfeI= +github.com/jacobsa/gcloud v0.0.0-20230803125757-3196d990d984 h1:kD9sX/8uHuPQI6OO/VKz/olbTXNkQ4vveSPNdS9AtHw= +github.com/jacobsa/gcloud v0.0.0-20230803125757-3196d990d984/go.mod h1:CGkT80TfaoPTzQ8My+t2M7PnMDvkwAR36Qm8Mm8HytI= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd h1:9GCSedGjMcLZCrusBZuo4tyKLpKUPenUUqi34AkuFmA= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff h1:2xRHTvkpJ5zJmglXLRqHiZQNjUoOkhUyhTAhEQvPAWw= github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff/go.mod h1:gJWba/XXGl0UoOmBQKRWCJdHrr3nE0T65t6ioaj3mLI= github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11 h1:BMb8s3ENQLt5ulwVIHVDWFHp8eIXmbfSExkvdn9qMXI= github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11/go.mod h1:+DBdDyfoO2McrOyDemRBq0q9CMEByef7sYl7JH5Q3BI= -github.com/jacobsa/ratelimit v0.0.0-20150904001804-f5e47030f3b0 h1:6GaIakaFrxn738iBykUc6fyS5sIAKRg/wafwzrzRX30= -github.com/jacobsa/ratelimit v0.0.0-20150904001804-f5e47030f3b0/go.mod h1:5/sdn6lSZE5l3rXMkJGO7Y3MHJImklO43rZx9ouOWYQ= github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb h1:uSWBjJdMf47kQlXMwWEfmc864bA1wAC+Kl3ApryuG9Y= github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb/go.mod h1:ivcmUvxXWjb27NsPEaiYK7AidlZXS7oQ5PowUS9z3I4= github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3 h1:+gHfvQxomE6fI4zg7QYyaGDCnuw2wylD4i6yzrQvAmY= @@ -249,8 +249,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/xattr v0.4.8 h1:3QwVADT+4oUm3zg7MXO/2i/lqnKkQ9viNY8pl5egRDE= -github.com/pkg/xattr v0.4.8/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -485,6 +485,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/internal/fs/inode/file.go b/internal/fs/inode/file.go index 9a227af95b..0dd7941bd4 100644 --- a/internal/fs/inode/file.go +++ b/internal/fs/inode/file.go @@ -200,8 +200,9 @@ func (f *FileInode) openReader(ctx context.Context) (io.ReadCloser, error) { rc, err := f.bucket.NewReader( ctx, &gcs.ReadObjectRequest{ - Name: f.src.Name, - Generation: f.src.Generation, + Name: f.src.Name, + Generation: f.src.Generation, + ReadCompressed: f.src.HasContentEncodingGzip(), }) if err != nil { err = fmt.Errorf("NewReader: %w", err) @@ -638,11 +639,12 @@ func convertObjToMinObject(o *gcs.Object) storage.MinObject { } return storage.MinObject{ - Name: o.Name, - Size: o.Size, - Generation: o.Generation, - MetaGeneration: o.MetaGeneration, - Updated: o.Updated, - Metadata: o.Metadata, + Name: o.Name, + Size: o.Size, + Generation: o.Generation, + MetaGeneration: o.MetaGeneration, + Updated: o.Updated, + Metadata: o.Metadata, + ContentEncoding: o.ContentEncoding, } } diff --git a/internal/fs/inode/file_test.go b/internal/fs/inode/file_test.go index 5b2d48ba59..d1dd52414b 100644 --- a/internal/fs/inode/file_test.go +++ b/internal/fs/inode/file_test.go @@ -866,3 +866,36 @@ func (t *FileTest) TestCreateEmptyTempFileShouldCreateEmptyFile() { AssertEq(nil, err) AssertEq(0, sr.Size) } + +func (t *FileTest) ContentEncodingGzip() { + // Set up an explicit content-encoding on the backing object and re-create the inode. + contentEncoding := "gzip" + t.backingObj.ContentEncoding = contentEncoding + + t.createInode() + + AssertEq(contentEncoding, t.in.Source().ContentEncoding) + AssertTrue(t.in.Source().HasContentEncodingGzip()) +} + +func (t *FileTest) ContentEncodingNone() { + // Set up an explicit content-encoding on the backing object and re-create the inode. + contentEncoding := "" + t.backingObj.ContentEncoding = contentEncoding + + t.createInode() + + AssertEq(contentEncoding, t.in.Source().ContentEncoding) + AssertFalse(t.in.Source().HasContentEncodingGzip()) +} + +func (t *FileTest) ContentEncodingOther() { + // Set up an explicit content-encoding on the backing object and re-create the inode. + contentEncoding := "other" + t.backingObj.ContentEncoding = contentEncoding + + t.createInode() + + AssertEq(contentEncoding, t.in.Source().ContentEncoding) + AssertFalse(t.in.Source().HasContentEncodingGzip()) +} diff --git a/internal/gcsx/bucket_manager.go b/internal/gcsx/bucket_manager.go index a8cfa51c6a..335137778f 100644 --- a/internal/gcsx/bucket_manager.go +++ b/internal/gcsx/bucket_manager.go @@ -20,17 +20,16 @@ import ( "path" "time" - "github.com/googlecloudplatform/gcsfuse/internal/storage" - "github.com/jacobsa/reqtrace" - "golang.org/x/net/context" - "github.com/googlecloudplatform/gcsfuse/internal/canned" "github.com/googlecloudplatform/gcsfuse/internal/logger" "github.com/googlecloudplatform/gcsfuse/internal/monitor" + "github.com/googlecloudplatform/gcsfuse/internal/ratelimit" + "github.com/googlecloudplatform/gcsfuse/internal/storage" "github.com/jacobsa/gcloud/gcs" "github.com/jacobsa/gcloud/gcs/gcscaching" - "github.com/jacobsa/ratelimit" + "github.com/jacobsa/reqtrace" "github.com/jacobsa/timeutil" + "golang.org/x/net/context" ) type BucketConfig struct { @@ -41,7 +40,6 @@ type BucketConfig struct { StatCacheCapacity int StatCacheTTL time.Duration EnableMonitoring bool - EnableStorageClientLibrary bool DebugGCS bool // Files backed by on object of length at least AppendThreshold that have @@ -76,7 +74,6 @@ type BucketManager interface { type bucketManager struct { config BucketConfig - conn *Connection storageHandle storage.StorageHandle // Garbage collector @@ -84,10 +81,9 @@ type bucketManager struct { stopGarbageCollecting func() } -func NewBucketManager(config BucketConfig, conn *Connection, storageHandle storage.StorageHandle) BucketManager { +func NewBucketManager(config BucketConfig, storageHandle storage.StorageHandle) BucketManager { bm := &bucketManager{ config: config, - conn: conn, storageHandle: storageHandle, } bm.gcCtx, bm.stopGarbageCollecting = context.WithCancel(context.Background()) @@ -117,7 +113,7 @@ func setUpRateLimiting( // window of the given size. const window = 8 * time.Hour - opCapacity, err := ratelimit.ChooseTokenBucketCapacity( + opCapacity, err := ratelimit.ChooseLimiterCapacity( opRateLimitHz, window) @@ -126,7 +122,7 @@ func setUpRateLimiting( return } - egressCapacity, err := ratelimit.ChooseTokenBucketCapacity( + egressCapacity, err := ratelimit.ChooseLimiterCapacity( egressBandwidthLimit, window) @@ -152,25 +148,14 @@ func setUpRateLimiting( // // Special case: if the bucket name is canned.FakeBucketName, set up a fake // bucket as described in that package. -func (bm *bucketManager) SetUpGcsBucket(ctx context.Context, name string) (b gcs.Bucket, err error) { - if bm.config.EnableStorageClientLibrary { - b = bm.storageHandle.BucketHandle(name, bm.config.BillingProject) +func (bm *bucketManager) SetUpGcsBucket(name string) (b gcs.Bucket, err error) { + b = bm.storageHandle.BucketHandle(name, bm.config.BillingProject) - if reqtrace.Enabled() { - b = gcs.GetWrappedWithReqtraceBucket(b) - } - if bm.config.DebugGCS { - b = gcs.NewDebugBucket(b, logger.NewDebug("gcs: ")) - } - } else { - logger.Infof("OpenBucket(%q, %q)\n", name, bm.config.BillingProject) - b, err = bm.conn.OpenBucket( - ctx, - &gcs.OpenBucketOptions{ - Name: name, - BillingProject: bm.config.BillingProject, - }, - ) + if reqtrace.Enabled() { + b = gcs.GetWrappedWithReqtraceBucket(b) + } + if bm.config.DebugGCS { + b = gcs.NewDebugBucket(b, logger.NewDebug("gcs: ")) } return } @@ -183,7 +168,7 @@ func (bm *bucketManager) SetUpBucket( if name == canned.FakeBucketName { b = canned.MakeFakeBucket(ctx) } else { - b, err = bm.SetUpGcsBucket(ctx, name) + b, err = bm.SetUpGcsBucket(name) if err != nil { err = fmt.Errorf("OpenBucket: %w", err) return diff --git a/internal/gcsx/bucket_manager_test.go b/internal/gcsx/bucket_manager_test.go index c05ed315b5..35cd732b70 100644 --- a/internal/gcsx/bucket_manager_test.go +++ b/internal/gcsx/bucket_manager_test.go @@ -7,9 +7,7 @@ import ( "github.com/googlecloudplatform/gcsfuse/internal/storage" "github.com/jacobsa/gcloud/gcs" - "github.com/jacobsa/gcloud/gcs/gcsfake" . "github.com/jacobsa/ogletest" - "github.com/jacobsa/timeutil" ) func TestBucketManager(t *testing.T) { RunTests(t) } @@ -56,36 +54,19 @@ func (t *BucketManagerTest) TestNewBucketManagerMethod() { DebugGCS: true, AppendThreshold: 2, TmpObjectPrefix: "TmpObjectPrefix", - EnableStorageClientLibrary: true, } - bm := NewBucketManager(bucketConfig, nil, t.storageHandle) + bm := NewBucketManager(bucketConfig, t.storageHandle) ExpectNe(nil, bm) } -func (t *BucketManagerTest) TestSetupGcsBucketWhenEnableStorageClientLibraryIsTrue() { +func (t *BucketManagerTest) TestSetupGcsBucket() { var bm bucketManager bm.storageHandle = t.storageHandle - bm.config.EnableStorageClientLibrary = true bm.config.DebugGCS = true - bucket, err := bm.SetUpGcsBucket(context.Background(), TestBucketName) - - ExpectNe(nil, bucket) - ExpectEq(nil, err) -} - -func (t *BucketManagerTest) TestSetupGcsBucketWhenEnableStorageClientLibraryIsFalse() { - var bm bucketManager - bm.storageHandle = t.storageHandle - bm.config.EnableStorageClientLibrary = false - bm.config.BillingProject = "BillingProject" - bm.conn = &Connection{ - wrapped: gcsfake.NewConn(timeutil.RealClock()), - } - - bucket, err := bm.SetUpGcsBucket(context.Background(), "fake@bucket") + bucket, err := bm.SetUpGcsBucket(TestBucketName) ExpectNe(nil, bucket) ExpectEq(nil, err) @@ -104,15 +85,11 @@ func (t *BucketManagerTest) TestSetUpBucketMethod() { DebugGCS: true, AppendThreshold: 2, TmpObjectPrefix: "TmpObjectPrefix", - EnableStorageClientLibrary: true, } ctx := context.Background() bm.storageHandle = t.storageHandle bm.config = bucketConfig bm.gcCtx = ctx - bm.conn = &Connection{ - wrapped: gcsfake.NewConn(timeutil.RealClock()), - } bucket, err := bm.SetUpBucket(context.Background(), TestBucketName) @@ -133,15 +110,11 @@ func (t *BucketManagerTest) TestSetUpBucketMethodWhenBucketDoesNotExist() { DebugGCS: true, AppendThreshold: 2, TmpObjectPrefix: "TmpObjectPrefix", - EnableStorageClientLibrary: true, } ctx := context.Background() bm.storageHandle = t.storageHandle bm.config = bucketConfig bm.gcCtx = ctx - bm.conn = &Connection{ - wrapped: gcsfake.NewConn(timeutil.RealClock()), - } bucket, err := bm.SetUpBucket(context.Background(), invalidBucketName) diff --git a/internal/gcsx/connection.go b/internal/gcsx/connection.go deleted file mode 100644 index 87ab6c54d0..0000000000 --- a/internal/gcsx/connection.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2015 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gcsx - -import ( - "fmt" - "strings" - "syscall" - - "github.com/jacobsa/gcloud/gcs" - "golang.org/x/net/context" -) - -type Connection struct { - wrapped gcs.Conn -} - -func NewConnection(cfg *gcs.ConnConfig) (c *Connection, err error) { - wrapped, err := gcs.NewConn(cfg) - if err != nil { - err = fmt.Errorf("Cannot create Conn: %w", err) - return - } - - c = &Connection{ - wrapped: wrapped, - } - return -} - -func (c *Connection) OpenBucket( - ctx context.Context, - options *gcs.OpenBucketOptions) (b gcs.Bucket, err error) { - b, err = c.wrapped.OpenBucket(ctx, options) - - // The gcs.Conn.OpenBucket returns converted errors without the underlying - // googleapi.Error, which is impossible to use errors.As to match the error - // type. To interpret the errors in syscall, here we use string matching. - if err != nil { - if strings.Contains(err.Error(), "Bad credentials") { - return b, fmt.Errorf("Bad credentials for bucket %q: %w", options.Name, syscall.EACCES) - } - if strings.Contains(err.Error(), "Unknown bucket") { - return b, fmt.Errorf("Unknown bucket %q: %w", options.Name, syscall.ENOENT) - } - } - - return -} diff --git a/internal/gcsx/random_reader.go b/internal/gcsx/random_reader.go index 7db7b576ca..4d24e4971b 100644 --- a/internal/gcsx/random_reader.go +++ b/internal/gcsx/random_reader.go @@ -372,6 +372,7 @@ func (rr *randomReader) startRead( Start: uint64(start), Limit: uint64(end), }, + ReadCompressed: rr.object.HasContentEncodingGzip(), }) if err != nil { diff --git a/internal/perms/perms.go b/internal/perms/perms.go index 42834a2c9b..c2386139f4 100644 --- a/internal/perms/perms.go +++ b/internal/perms/perms.go @@ -17,35 +17,28 @@ package perms import ( "fmt" - "os/user" - "strconv" + "os" ) // MyUserAndGroup returns the UID and GID of this process. -func MyUserAndGroup() (uid uint32, gid uint32, err error) { - // Ask for the current user. - user, err := user.Current() - if err != nil { - err = fmt.Errorf("Fetching current user: %w", err) - return - } +func MyUserAndGroup() (uid, gid uint32, err error) { + signed_uid := os.Getuid() + signed_gid := os.Getgid() - // Parse UID. - uid64, err := strconv.ParseUint(user.Uid, 10, 32) - if err != nil { - err = fmt.Errorf("Parsing UID (%s): %w", user.Uid, err) - return - } + // Not sure in what scenarios uid/gid could be returned as negative. The only + // documented scenario at pkg.go.dev/os#Getuid is windows OS. + if signed_gid < 0 || signed_uid < 0 { + err = fmt.Errorf("failed to get uid/gid. UID = %d, GID = %d", signed_uid, signed_gid) + + // An untested improvement idea to fallback here is to invoke os.current.User() + // and use its partial output even when os.current.User() returned error, as + // the partial output would still be useful. - // Parse GID. - gid64, err := strconv.ParseUint(user.Gid, 10, 32) - if err != nil { - err = fmt.Errorf("Parsing GID (%s): %w", user.Gid, err) return } - uid = uint32(uid64) - gid = uint32(gid64) + uid = uint32(signed_uid) + gid = uint32(signed_gid) return } diff --git a/internal/perms/perms_test.go b/internal/perms/perms_test.go new file mode 100644 index 0000000000..217fbd9cb0 --- /dev/null +++ b/internal/perms/perms_test.go @@ -0,0 +1,48 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// System permissions-related code unit tests. +package perms_test + +import ( + "testing" + + "github.com/googlecloudplatform/gcsfuse/internal/perms" + . "github.com/jacobsa/ogletest" +) + +func TestPerms(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type PermsTest struct { +} + +func init() { RegisterTestSuite(&PermsTest{}) } + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *PermsTest) MyUserAndGroupNoError() { + uid, gid, err := perms.MyUserAndGroup() + ExpectEq(err, nil) + + unexpected_id_signed := -1 + unexpected_id := uint32(unexpected_id_signed) + ExpectNe(uid, unexpected_id) + ExpectNe(gid, unexpected_id) +} diff --git a/internal/ratelimit/limiter_capacity.go b/internal/ratelimit/limiter_capacity.go new file mode 100644 index 0000000000..a4c095ebb0 --- /dev/null +++ b/internal/ratelimit/limiter_capacity.go @@ -0,0 +1,83 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "fmt" + "math" + "time" +) + +// Choose a limiter capacity that ensures that the action gated by the +// limiter will be limited to within a few percent of `rateHz * window` +// for any window of the given size. +// +// This is not be possible for all rates and windows. In that case, an error +// will be returned. +func ChooseLimiterCapacity( + rateHz float64, + window time.Duration) (capacity uint64, err error) { + // Check that the input is reasonable. + if rateHz <= 0 || math.IsInf(rateHz, 0) { + err = fmt.Errorf("Illegal rate: %f", rateHz) + return + } + + if window <= 0 { + err = fmt.Errorf("Illegal window: %v", window) + return + } + + // We cannot help but allow the rate to exceed the configured maximum by some + // factor in an arbitrary window, no matter how small we scale the max + // accumulated credit -- the bucket may be full at the start of the window, + // be immediately exhausted, then be repeatedly exhausted just before filling + // throughout the window. + // + // For example: let the window W = 10 seconds, and the bandwidth B = 20 MiB/s. + // Set the max accumulated credit C = W*B/2 = 100 MiB. Then this + // sequence of events is allowed: + // + // * T=0: Allow through 100 MiB. + // * T=4.999999: Allow through nearly 100 MiB. + // * T=9.999999: Allow through nearly 100 MiB. + // + // Above we allow through nearly 300 MiB, exceeding the allowed bytes for the + // window by nearly 50%. Note however that this trend cannot continue into + // the next window, so this must be a transient spike. + // + // In general if we set C <= W*B/N, then we're off by no more than a factor + // of (N+1)/N within any window of size W. + // + // Choose a reasonable N. + const N = 50 // At most 2% error + + w := float64(window) / float64(time.Second) + capacityFloat := math.Floor(w * rateHz / N) + if !(capacityFloat >= 1 && capacityFloat < float64(math.MaxUint64)) { + err = fmt.Errorf( + "Can't use a token bucket to limit to %f Hz over a window of %v "+ + "(result is a capacity of %f)", + rateHz, + window, + capacityFloat) + + return + } + + capacity = uint64(capacityFloat) + + return +} diff --git a/internal/ratelimit/limiter_capacity_test.go b/internal/ratelimit/limiter_capacity_test.go new file mode 100644 index 0000000000..d5dc037b58 --- /dev/null +++ b/internal/ratelimit/limiter_capacity_test.go @@ -0,0 +1,96 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "fmt" + "testing" + "time" + + . "github.com/jacobsa/ogletest" +) + +func TestLimiterCapacity(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type LimiterCapacityTest struct { +} + +func init() { RegisterTestSuite(&LimiterCapacityTest{}) } + +func rateLessThanOrEqualToZero(rate float64) { + _, err := ChooseLimiterCapacity(rate, 30) + + expectedError := fmt.Errorf("Illegal rate: %f", rate) + + AssertEq(expectedError.Error(), err.Error()) +} + +func (t *LimiterCapacityTest) TestRateLessThanZero() { + var negativeRateHz float64 = -1 + + rateLessThanOrEqualToZero(negativeRateHz) +} + +func (t *LimiterCapacityTest) TestRateEqualToZero() { + var zeroRateHz float64 = 0 + + rateLessThanOrEqualToZero(zeroRateHz) +} + +func windowLessThanOrEqualToZero(window time.Duration) { + _, err := ChooseLimiterCapacity(1, window) + + expectedError := fmt.Errorf("Illegal window: %v", window) + + AssertEq(expectedError.Error(), err.Error()) +} + +func (t *LimiterCapacityTest) TestWindowLessThanZero() { + var negativeWindow time.Duration = -1 + + windowLessThanOrEqualToZero(negativeWindow) +} + +func (t *LimiterCapacityTest) TestWindowEqualToZero() { + var zeroWindow time.Duration = 0 + + windowLessThanOrEqualToZero(zeroWindow) +} + +func (t *LimiterCapacityTest) TestCapacityEqualToZero() { + var rate = 0.5 + var window time.Duration = 1 + + capacity, err := ChooseLimiterCapacity(rate, window) + + expectedError := fmt.Errorf( + "Can't use a token bucket to limit to %f Hz over a window of %v (result is a capacity of %f)", rate, window, float64(capacity)) + AssertEq(expectedError.Error(), err.Error()) +} + +func (t *LimiterCapacityTest) TestExpectedCapacity() { + var rate float64 = 20 + var window = 10 * time.Second + + capacity, err := ChooseLimiterCapacity(rate, window) + // capacity = floor((20.0 * 10)/50) = floor(4.0) = 4 + + ExpectEq(nil, err) + ExpectEq(4, capacity) +} diff --git a/internal/ratelimit/throttle.go b/internal/ratelimit/throttle.go new file mode 100644 index 0000000000..4b45da0976 --- /dev/null +++ b/internal/ratelimit/throttle.go @@ -0,0 +1,58 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "golang.org/x/net/context" + "golang.org/x/time/rate" +) + +// A simple interface for limiting the rate of some event. Unlike TokenBucket, +// does not allow the user control over what time means. +// +// Safe for concurrent access. +type Throttle interface { + // Return the maximum number of tokens that can be requested in a call to + // Wait. + Capacity() (c uint64) + + // Acquire the given number of tokens from the underlying token bucket, then + // sleep until when it says to wake. If the context is cancelled before then, + // return early with an error. + // + // REQUIRES: tokens <= capacity + Wait(ctx context.Context, tokens uint64) (err error) +} + +type limiter struct { + *rate.Limiter +} + +func NewThrottle( + rateHz float64, + capacity uint64) (t Throttle) { + t = &limiter{rate.NewLimiter(rate.Limit(rateHz), int(capacity))} + return +} + +func (l *limiter) Capacity() (c uint64) { + return uint64(l.Burst()) +} + +func (l *limiter) Wait( + ctx context.Context, + tokens uint64) (err error) { + return l.WaitN(ctx, int(tokens)) +} diff --git a/internal/ratelimit/throttle_reader_test.go b/internal/ratelimit/throttle_reader_test.go new file mode 100644 index 0000000000..25e612fefe --- /dev/null +++ b/internal/ratelimit/throttle_reader_test.go @@ -0,0 +1,335 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "errors" + "io" + "testing" + + "golang.org/x/net/context" + + . "github.com/jacobsa/ogletest" +) + +func TestThrottledReader(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +// An io.Reader that defers to a function. +type funcReader struct { + f func([]byte) (int, error) +} + +func (fr *funcReader) Read(p []byte) (n int, err error) { + n, err = fr.f(p) + return +} + +// A throttler that defers to a function. +type funcThrottle struct { + f func(context.Context, uint64) error +} + +func (ft *funcThrottle) Capacity() (c uint64) { + return 1024 +} + +func (ft *funcThrottle) Wait( + ctx context.Context, + tokens uint64) (err error) { + err = ft.f(ctx, tokens) + return +} + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type ThrottledReaderTest struct { + ctx context.Context + + wrapped funcReader + throttle funcThrottle + + reader io.Reader +} + +var _ SetUpInterface = &ThrottledReaderTest{} + +func init() { RegisterTestSuite(&ThrottledReaderTest{}) } + +func (t *ThrottledReaderTest) SetUp(ti *TestInfo) { + t.ctx = ti.Ctx + + // Set up the default throttle function. + t.throttle.f = func(ctx context.Context, tokens uint64) (err error) { + return + } + + // Set up the reader. + t.reader = ThrottledReader(t.ctx, &t.wrapped, &t.throttle) +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *ThrottledReaderTest) CallsThrottle() { + const readSize = 17 + AssertLe(readSize, t.throttle.Capacity()) + + // Throttle + var throttleCalled bool + t.throttle.f = func(ctx context.Context, tokens uint64) (err error) { + AssertFalse(throttleCalled) + throttleCalled = true + + AssertEq(t.ctx.Err(), ctx.Err()) + AssertEq(t.ctx.Done(), ctx.Done()) + AssertEq(readSize, tokens) + + err = errors.New("") + return + } + + // Call + _, err := t.reader.Read(make([]byte, readSize)) + + ExpectEq("", err.Error()) + ExpectTrue(throttleCalled) +} + +func (t *ThrottledReaderTest) ThrottleReturnsError() { + // Throttle + expectedErr := errors.New("taco") + t.throttle.f = func(ctx context.Context, tokens uint64) (err error) { + err = expectedErr + return + } + + // Call + n, err := t.reader.Read(make([]byte, 1)) + + ExpectEq(0, n) + ExpectEq(expectedErr, err) +} + +func (t *ThrottledReaderTest) CallsWrapped() { + buf := make([]byte, 16) + AssertLe(len(buf), t.throttle.Capacity()) + + // Wrapped + var readCalled bool + t.wrapped.f = func(p []byte) (n int, err error) { + AssertFalse(readCalled) + readCalled = true + + AssertEq(&buf[0], &p[0]) + AssertEq(len(buf), len(p)) + + err = errors.New("") + return + } + + // Call + _, err := t.reader.Read(buf) + + ExpectEq("", err.Error()) + ExpectTrue(readCalled) +} + +func (t *ThrottledReaderTest) WrappedReturnsError() { + // Wrapped + expectedErr := errors.New("taco") + t.wrapped.f = func(p []byte) (n int, err error) { + n = 11 + err = expectedErr + return + } + + // Call + n, err := t.reader.Read(make([]byte, 16)) + + ExpectEq(11, n) + ExpectEq(expectedErr, err) +} + +func (t *ThrottledReaderTest) WrappedReturnsEOF() { + // Wrapped + t.wrapped.f = func(p []byte) (n int, err error) { + n = 11 + err = io.EOF + return + } + + // Call + n, err := t.reader.Read(make([]byte, 16)) + + ExpectEq(11, n) + ExpectEq(io.EOF, err) +} + +func (t *ThrottledReaderTest) WrappedReturnsFullRead() { + const readSize = 17 + AssertLe(readSize, t.throttle.Capacity()) + + // Wrapped + t.wrapped.f = func(p []byte) (n int, err error) { + n = len(p) + return + } + + // Call + n, err := t.reader.Read(make([]byte, readSize)) + + ExpectEq(nil, err) + ExpectEq(readSize, n) +} + +func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() { + buf := make([]byte, 16) + AssertLe(len(buf), t.throttle.Capacity()) + + // Wrapped + var callCount int + t.wrapped.f = func(p []byte) (n int, err error) { + AssertLt(callCount, 2) + switch callCount { + case 0: + callCount++ + n = 2 + + case 1: + callCount++ + AssertEq(&buf[2], &p[0]) + AssertEq(len(buf)-2, len(p)) + err = errors.New("") + } + + return + } + + // Call + _, err := t.reader.Read(buf) + + ExpectEq("", err.Error()) + ExpectEq(2, callCount) +} + +func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsError() { + // Wrapped + var callCount int + expectedErr := errors.New("taco") + + t.wrapped.f = func(p []byte) (n int, err error) { + AssertLt(callCount, 2) + switch callCount { + case 0: + callCount++ + n = 2 + + case 1: + callCount++ + n = 11 + err = expectedErr + } + + return + } + + // Call + n, err := t.reader.Read(make([]byte, 16)) + + ExpectEq(2+11, n) + ExpectEq(expectedErr, err) +} + +func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsEOF() { + // Wrapped + var callCount int + t.wrapped.f = func(p []byte) (n int, err error) { + AssertLt(callCount, 2) + switch callCount { + case 0: + callCount++ + n = 2 + + case 1: + callCount++ + n = 11 + err = io.EOF + } + + return + } + + // Call + n, err := t.reader.Read(make([]byte, 16)) + + ExpectEq(2+11, n) + ExpectEq(io.EOF, err) +} + +func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSucceedsInFull() { + // Wrapped + var callCount int + t.wrapped.f = func(p []byte) (n int, err error) { + AssertLt(callCount, 2) + switch callCount { + case 0: + callCount++ + n = 2 + + case 1: + callCount++ + n = len(p) + } + + return + } + + // Call + n, err := t.reader.Read(make([]byte, 16)) + + ExpectEq(16, n) + ExpectEq(nil, err) +} + +func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() { + buf := make([]byte, 2048) + AssertGt(len(buf), t.throttle.Capacity()) + + // Wrapped + var readCalled bool + t.wrapped.f = func(p []byte) (n int, err error) { + AssertFalse(readCalled) + readCalled = true + + AssertEq(&buf[0], &p[0]) + ExpectEq(t.throttle.Capacity(), len(p)) + + err = errors.New("") + return + } + + // Call + _, err := t.reader.Read(buf) + + ExpectEq("", err.Error()) + ExpectTrue(readCalled) +} diff --git a/internal/ratelimit/throttle_test.go b/internal/ratelimit/throttle_test.go new file mode 100644 index 0000000000..3a6a7ada67 --- /dev/null +++ b/internal/ratelimit/throttle_test.go @@ -0,0 +1,209 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// It is performing integration tests for throttle.go +// Set up several test cases where we have N goroutines simulating the arrival of +// packets at a given rate, asking a limiter when to admit them. +// limiter can accept number of packets equivalent to capacity. After that, +// it will wait until limiter get space to receive the new packet. +package ratelimit_test + +import ( + cryptorand "crypto/rand" + "io" + "math/rand" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/internal/ratelimit" + "golang.org/x/net/context" + + . "github.com/jacobsa/oglematchers" + . "github.com/jacobsa/ogletest" +) + +func TestThrottle(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Helpers +//////////////////////////////////////////////////////////////////////// + +func makeSeed() (seed int64) { + var buf [8]byte + _, err := io.ReadFull(cryptorand.Reader, buf[:]) + if err != nil { + panic(err) + } + + seed = (int64(buf[0])>>1)<<56 | + int64(buf[1])<<48 | + int64(buf[2])<<40 | + int64(buf[3])<<32 | + int64(buf[4])<<24 | + int64(buf[5])<<16 | + int64(buf[6])<<8 | + int64(buf[7])<<0 + + return +} + +func processArrivals( + ctx context.Context, + throttle ratelimit.Throttle, + arrivalRateHz float64, + d time.Duration) (processed uint64) { + // Set up an independent source of randomness. + randSrc := rand.New(rand.NewSource(makeSeed())) + + // Tick into a channel at a steady rate, buffering over delays caused by the + // limiter. + arrivalPeriod := time.Duration((1.0 / arrivalRateHz) * float64(time.Second)) + ticks := make(chan struct{}, 3*int(float64(d)/float64(arrivalPeriod))) + + go func() { + ticker := time.NewTicker(arrivalPeriod) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + + case <-ticker.C: + select { + case ticks <- struct{}{}: + default: + panic("Buffer exceeded?") + } + } + } + }() + + // Simulate until we're supposed to stop. + for { + // Accumulate a few packets. + toAccumulate := uint64(randSrc.Int63n(5)) + + var accumulated uint64 + for accumulated < toAccumulate { + select { + case <-ctx.Done(): + return + + case <-ticks: + accumulated++ + } + } + + // Wait. + err := throttle.Wait(ctx, accumulated) + if err != nil { + return + } + + processed += accumulated + } +} + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type ThrottleTest struct { +} + +func init() { RegisterTestSuite(&ThrottleTest{}) } + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *ThrottleTest) IntegrationTest() { + runtime.GOMAXPROCS(runtime.NumCPU()) + const perCaseDuration = 1 * time.Second + + // Set up several test cases where we have N goroutines simulating arrival of + // packets at a given rate, asking a limiter when to admit them. + testCases := []struct { + numActors int + arrivalRateHz float64 + limitRateHz float64 + }{ + // Single actor + {1, 150, 200}, + {1, 200, 200}, + {1, 250, 200}, + + // Multiple actors + {4, 150, 200}, + {4, 200, 200}, + {4, 250, 200}, + } + + // Run each test case. + for i, tc := range testCases { + // Create a throttle. + capacity, err := ratelimit.ChooseLimiterCapacity( + tc.limitRateHz, + perCaseDuration) + + AssertEq(nil, err) + + throttle := ratelimit.NewThrottle(tc.limitRateHz, capacity) + + // Start workers. + var wg sync.WaitGroup + var totalProcessed uint64 + + ctx, _ := context.WithDeadline( + context.Background(), + time.Now().Add(perCaseDuration)) + + for i := 0; i < tc.numActors; i++ { + wg.Add(1) + go func() { + defer wg.Done() + processed := processArrivals( + ctx, + throttle, + tc.arrivalRateHz/float64(tc.numActors), + perCaseDuration) + + atomic.AddUint64(&totalProcessed, processed) + }() + } + + // Wait for them all to finish. + wg.Wait() + + // We should have processed about the correct number of arrivals. + smallerRateHz := tc.arrivalRateHz + if smallerRateHz > tc.limitRateHz { + smallerRateHz = tc.limitRateHz + } + + expected := smallerRateHz * (float64(perCaseDuration) / float64(time.Second)) + ExpectThat( + totalProcessed, + AllOf( + GreaterThan(expected*0.90), + LessThan(expected*1.10)), + "Test case %d. expected: %f", + i, + expected) + } +} diff --git a/internal/ratelimit/throttled_bucket.go b/internal/ratelimit/throttled_bucket.go new file mode 100644 index 0000000000..82112ef4b1 --- /dev/null +++ b/internal/ratelimit/throttled_bucket.go @@ -0,0 +1,202 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "io" + + "github.com/jacobsa/gcloud/gcs" + "golang.org/x/net/context" +) + +// Create a bucket that limits the rate at which it calls the wrapped bucket +// using opThrottle, and limits the bandwidth with which it reads from the +// wrapped bucket using egressThrottle. +func NewThrottledBucket( + opThrottle Throttle, + egressThrottle Throttle, + wrapped gcs.Bucket) (b gcs.Bucket) { + b = &throttledBucket{ + opThrottle: opThrottle, + egressThrottle: egressThrottle, + wrapped: wrapped, + } + return +} + +//////////////////////////////////////////////////////////////////////// +// throttledBucket +//////////////////////////////////////////////////////////////////////// + +type throttledBucket struct { + opThrottle Throttle + egressThrottle Throttle + wrapped gcs.Bucket +} + +func (b *throttledBucket) Name() string { + return b.wrapped.Name() +} + +func (b *throttledBucket) NewReader( + ctx context.Context, + req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) { + // Wait for permission to call through. + + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + rc, err = b.wrapped.NewReader(ctx, req) + if err != nil { + return + } + + // Wrap the result in a throttled layer. + rc = &readerCloser{ + Reader: ThrottledReader(ctx, rc, b.egressThrottle), + Closer: rc, + } + + return +} + +func (b *throttledBucket) CreateObject( + ctx context.Context, + req *gcs.CreateObjectRequest) (o *gcs.Object, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + o, err = b.wrapped.CreateObject(ctx, req) + + return +} + +func (b *throttledBucket) CopyObject( + ctx context.Context, + req *gcs.CopyObjectRequest) (o *gcs.Object, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + o, err = b.wrapped.CopyObject(ctx, req) + + return +} + +func (b *throttledBucket) ComposeObjects( + ctx context.Context, + req *gcs.ComposeObjectsRequest) (o *gcs.Object, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + o, err = b.wrapped.ComposeObjects(ctx, req) + + return +} + +func (b *throttledBucket) StatObject( + ctx context.Context, + req *gcs.StatObjectRequest) (o *gcs.Object, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + o, err = b.wrapped.StatObject(ctx, req) + + return +} + +func (b *throttledBucket) ListObjects( + ctx context.Context, + req *gcs.ListObjectsRequest) (listing *gcs.Listing, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + listing, err = b.wrapped.ListObjects(ctx, req) + + return +} + +func (b *throttledBucket) UpdateObject( + ctx context.Context, + req *gcs.UpdateObjectRequest) (o *gcs.Object, err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + o, err = b.wrapped.UpdateObject(ctx, req) + + return +} + +func (b *throttledBucket) DeleteObject( + ctx context.Context, + req *gcs.DeleteObjectRequest) (err error) { + // Wait for permission to call through. + err = b.opThrottle.Wait(ctx, 1) + if err != nil { + return + } + + // Call through. + err = b.wrapped.DeleteObject(ctx, req) + + return +} + +//////////////////////////////////////////////////////////////////////// +// readerCloser +//////////////////////////////////////////////////////////////////////// + +// An io.ReadCloser that forwards read requests to an io.Reader and close +// requests to an io.Closer. +type readerCloser struct { + Reader io.Reader + Closer io.Closer +} + +func (rc *readerCloser) Read(p []byte) (n int, err error) { + n, err = rc.Reader.Read(p) + return +} + +func (rc *readerCloser) Close() (err error) { + err = rc.Closer.Close() + return +} diff --git a/internal/ratelimit/throttled_reader.go b/internal/ratelimit/throttled_reader.go new file mode 100644 index 0000000000..5794a02cbe --- /dev/null +++ b/internal/ratelimit/throttled_reader.go @@ -0,0 +1,66 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "io" + + "golang.org/x/net/context" +) + +// Create a reader that limits the bandwidth of reads made from r according to +// the supplied throttler. Reads are assumed to be made under the supplied +// context. +func ThrottledReader( + ctx context.Context, + r io.Reader, + throttle Throttle) io.Reader { + return &throttledReader{ + ctx: ctx, + wrapped: r, + throttle: throttle, + } +} + +type throttledReader struct { + ctx context.Context + wrapped io.Reader + throttle Throttle +} + +func (tr *throttledReader) Read(p []byte) (n int, err error) { + // We can't serve a read larger than the throttle's capacity. + if uint64(len(p)) > tr.throttle.Capacity() { + p = p[:tr.throttle.Capacity()] + } + + // Wait for permission to continue. + err = tr.throttle.Wait(tr.ctx, uint64(len(p))) + if err != nil { + return + } + + // Serve the full amount we acquired from the throttle (unless we hit an + // early error, including EOF). + for len(p) > 0 && err == nil { + var tmp int + tmp, err = tr.wrapped.Read(p) + + n += tmp + p = p[tmp:] + } + + return +} diff --git a/internal/storage/bucket_handle.go b/internal/storage/bucket_handle.go index 311bd68d21..0526441465 100644 --- a/internal/storage/bucket_handle.go +++ b/internal/storage/bucket_handle.go @@ -64,6 +64,10 @@ func (bh *bucketHandle) NewReader( obj = obj.Generation(req.Generation) } + if req.ReadCompressed { + obj = obj.ReadCompressed(true) + } + // NewRangeReader creates a "storage.Reader" object which is also io.ReadCloser since it contains both Read() and Close() methods present in io.ReadCloser interface. return obj.NewRangeReader(ctx, start, length) } diff --git a/internal/storage/bucket_handle_test.go b/internal/storage/bucket_handle_test.go index 298ae363df..76ccec6ce6 100644 --- a/internal/storage/bucket_handle_test.go +++ b/internal/storage/bucket_handle_test.go @@ -81,7 +81,7 @@ func (t *BucketHandleTest) TestNewReaderMethodWithCompleteRead() { buf := make([]byte, len(ContentInTestObject)) _, err = rc.Read(buf) AssertEq(nil, err) - ExpectEq(string(buf[:]), ContentInTestObject) + ExpectEq(ContentInTestObject, string(buf[:])) } func (t *BucketHandleTest) TestNewReaderMethodWithRangeRead() { @@ -102,7 +102,7 @@ func (t *BucketHandleTest) TestNewReaderMethodWithRangeRead() { buf := make([]byte, limit-start) _, err = rc.Read(buf) AssertEq(nil, err) - ExpectEq(string(buf[:]), ContentInTestObject[start:limit]) + ExpectEq(ContentInTestObject[start:limit], string(buf[:])) } func (t *BucketHandleTest) TestNewReaderMethodWithNilRange() { @@ -117,7 +117,7 @@ func (t *BucketHandleTest) TestNewReaderMethodWithNilRange() { buf := make([]byte, len(ContentInTestObject)) _, err = rc.Read(buf) AssertEq(nil, err) - ExpectEq(string(buf[:]), ContentInTestObject[:]) + ExpectEq(ContentInTestObject, string(buf[:])) } func (t *BucketHandleTest) TestNewReaderMethodWithInValidObject() { @@ -150,7 +150,7 @@ func (t *BucketHandleTest) TestNewReaderMethodWithValidGeneration() { buf := make([]byte, len(ContentInTestObject)) _, err = rc.Read(buf) AssertEq(nil, err) - ExpectEq(string(buf[:]), ContentInTestObject) + ExpectEq(ContentInTestObject, string(buf[:])) } func (t *BucketHandleTest) TestNewReaderMethodWithInvalidGeneration() { @@ -168,6 +168,44 @@ func (t *BucketHandleTest) TestNewReaderMethodWithInvalidGeneration() { AssertEq(nil, rc) } +func (t *BucketHandleTest) TestNewReaderMethodWithCompressionEnabled() { + rc, err := t.bucketHandle.NewReader(context.Background(), + &gcs.ReadObjectRequest{ + Name: TestGzipObjectName, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(len(ContentInTestGzipObjectCompressed)), + }, + ReadCompressed: true, + }) + + AssertEq(nil, err) + defer rc.Close() + buf := make([]byte, len(ContentInTestGzipObjectCompressed)) + _, err = rc.Read(buf) + AssertEq(nil, err) + ExpectEq(ContentInTestGzipObjectCompressed, string(buf)) +} + +func (t *BucketHandleTest) TestNewReaderMethodWithCompressionDisabled() { + rc, err := t.bucketHandle.NewReader(context.Background(), + &gcs.ReadObjectRequest{ + Name: TestGzipObjectName, + Range: &gcs.ByteRange{ + Start: uint64(0), + Limit: uint64(len(ContentInTestGzipObjectCompressed)), + }, + ReadCompressed: false, + }) + + AssertEq(nil, err) + defer rc.Close() + buf := make([]byte, len(ContentInTestGzipObjectDecompressed)) + _, err = rc.Read(buf) + AssertEq(nil, err) + ExpectEq(ContentInTestGzipObjectDecompressed, string(buf)) +} + func (t *BucketHandleTest) TestDeleteObjectMethodWithValidObject() { err := t.bucketHandle.DeleteObject(context.Background(), &gcs.DeleteObjectRequest{ @@ -376,7 +414,7 @@ func (t *BucketHandleTest) TestListObjectMethodWithPrefixObjectExist() { }) AssertEq(nil, err) - AssertEq(3, len(obj.Objects)) + AssertEq(4, len(obj.Objects)) AssertEq(1, len(obj.CollapsedRuns)) AssertEq(TestObjectRootFolderName, obj.Objects[0].Name) AssertEq(TestObjectSubRootFolderName, obj.Objects[1].Name) @@ -413,10 +451,11 @@ func (t *BucketHandleTest) TestListObjectMethodWithIncludeTrailingDelimiterFalse }) AssertEq(nil, err) - AssertEq(2, len(obj.Objects)) + AssertEq(3, len(obj.Objects)) AssertEq(1, len(obj.CollapsedRuns)) AssertEq(TestObjectRootFolderName, obj.Objects[0].Name) AssertEq(TestObjectName, obj.Objects[1].Name) + AssertEq(TestGzipObjectName, obj.Objects[2].Name) AssertEq(TestObjectSubRootFolderName, obj.CollapsedRuns[0]) } @@ -433,62 +472,65 @@ func (t *BucketHandleTest) TestListObjectMethodWithEmptyDelimiter() { }) AssertEq(nil, err) - AssertEq(4, len(obj.Objects)) + AssertEq(5, len(obj.Objects)) AssertEq(TestObjectRootFolderName, obj.Objects[0].Name) AssertEq(TestObjectSubRootFolderName, obj.Objects[1].Name) AssertEq(TestSubObjectName, obj.Objects[2].Name) AssertEq(TestObjectName, obj.Objects[3].Name) + AssertEq(TestGzipObjectName, obj.Objects[4].Name) AssertEq(TestObjectGeneration, obj.Objects[0].Generation) AssertEq(nil, obj.CollapsedRuns) } -// We have 4 objects in fakeserver. +// We have 5 objects in fakeserver. func (t *BucketHandleTest) TestListObjectMethodForMaxResult() { - fourObj, err := t.bucketHandle.ListObjects(context.Background(), + fiveObj, err := t.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "", Delimiter: "", IncludeTrailingDelimiter: false, ContinuationToken: "", - MaxResults: 4, + MaxResults: 5, ProjectionVal: 0, }) - twoObj, err2 := t.bucketHandle.ListObjects(context.Background(), + threeObj, err2 := t.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "gcsfuse/", Delimiter: "/", IncludeTrailingDelimiter: false, ContinuationToken: "", - MaxResults: 2, + MaxResults: 3, ProjectionVal: 0, }) - // Validate that 4 objects are listed when MaxResults is passed 4. + // Validate that 5 objects are listed when MaxResults is passed 5. AssertEq(nil, err) - AssertEq(4, len(fourObj.Objects)) - AssertEq(TestObjectRootFolderName, fourObj.Objects[0].Name) - AssertEq(TestObjectSubRootFolderName, fourObj.Objects[1].Name) - AssertEq(TestSubObjectName, fourObj.Objects[2].Name) - AssertEq(TestObjectName, fourObj.Objects[3].Name) - AssertEq(nil, fourObj.CollapsedRuns) + AssertEq(5, len(fiveObj.Objects)) + AssertEq(TestObjectRootFolderName, fiveObj.Objects[0].Name) + AssertEq(TestObjectSubRootFolderName, fiveObj.Objects[1].Name) + AssertEq(TestSubObjectName, fiveObj.Objects[2].Name) + AssertEq(TestObjectName, fiveObj.Objects[3].Name) + AssertEq(TestGzipObjectName, fiveObj.Objects[4].Name) + AssertEq(nil, fiveObj.CollapsedRuns) // Note: The behavior is different in real GCS storage JSON API. In real API, // only 1 object and 1 collapsedRuns would have been returned if - // IncludeTrailingDelimiter = false and 2 objects and 1 collapsedRuns if + // IncludeTrailingDelimiter = false and 3 objects and 1 collapsedRuns if // IncludeTrailingDelimiter = true. // This is because fake storage doesn't support pagination and hence maxResults // has no affect. AssertEq(nil, err2) - AssertEq(2, len(twoObj.Objects)) - AssertEq(TestObjectRootFolderName, twoObj.Objects[0].Name) - AssertEq(TestObjectName, twoObj.Objects[1].Name) - AssertEq(1, len(twoObj.CollapsedRuns)) + AssertEq(3, len(threeObj.Objects)) + AssertEq(TestObjectRootFolderName, threeObj.Objects[0].Name) + AssertEq(TestObjectName, threeObj.Objects[1].Name) + AssertEq(TestGzipObjectName, threeObj.Objects[2].Name) + AssertEq(1, len(threeObj.CollapsedRuns)) } func (t *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { - // Validate that ee have 4 objects in fakeserver - fourObjWithMaxResults, err := t.bucketHandle.ListObjects(context.Background(), + // Validate that ee have 5 objects in fakeserver + fiveObjWithMaxResults, err := t.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "", Delimiter: "", @@ -498,9 +540,9 @@ func (t *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { ProjectionVal: 0, }) AssertEq(nil, err) - AssertEq(4, len(fourObjWithMaxResults.Objects)) + AssertEq(5, len(fiveObjWithMaxResults.Objects)) - fourObjWithoutMaxResults, err2 := t.bucketHandle.ListObjects(context.Background(), + fiveObjWithoutMaxResults, err2 := t.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "", Delimiter: "", @@ -509,19 +551,20 @@ func (t *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { ProjectionVal: 0, }) - // Validate that all objects (4) are listed when MaxResults is not passed. + // Validate that all objects (5) are listed when MaxResults is not passed. AssertEq(nil, err2) - AssertEq(4, len(fourObjWithoutMaxResults.Objects)) - AssertEq(TestObjectRootFolderName, fourObjWithoutMaxResults.Objects[0].Name) - AssertEq(TestObjectSubRootFolderName, fourObjWithoutMaxResults.Objects[1].Name) - AssertEq(TestSubObjectName, fourObjWithoutMaxResults.Objects[2].Name) - AssertEq(TestObjectName, fourObjWithoutMaxResults.Objects[3].Name) - AssertEq(nil, fourObjWithoutMaxResults.CollapsedRuns) + AssertEq(5, len(fiveObjWithoutMaxResults.Objects)) + AssertEq(TestObjectRootFolderName, fiveObjWithoutMaxResults.Objects[0].Name) + AssertEq(TestObjectSubRootFolderName, fiveObjWithoutMaxResults.Objects[1].Name) + AssertEq(TestSubObjectName, fiveObjWithoutMaxResults.Objects[2].Name) + AssertEq(TestObjectName, fiveObjWithoutMaxResults.Objects[3].Name) + AssertEq(TestGzipObjectName, fiveObjWithoutMaxResults.Objects[4].Name) + AssertEq(nil, fiveObjWithoutMaxResults.CollapsedRuns) } func (t *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { - // Validate that ee have 4 objects in fakeserver - fourObj, err := t.bucketHandle.ListObjects(context.Background(), + // Validate that we have 5 objects in fakeserver + fiveObj, err := t.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "", Delimiter: "", @@ -531,9 +574,9 @@ func (t *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { ProjectionVal: 0, }) AssertEq(nil, err) - AssertEq(4, len(fourObj.Objects)) + AssertEq(5, len(fiveObj.Objects)) - fourObjWithZeroMaxResults, err2 := t.bucketHandle.ListObjects(context.Background(), + fiveObjWithZeroMaxResults, err2 := t.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ Prefix: "", Delimiter: "", @@ -543,15 +586,16 @@ func (t *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { ProjectionVal: 0, }) - // Validate that all objects (4) are listed when MaxResults is 0. This has + // Validate that all objects (5) are listed when MaxResults is 0. This has // same behavior as not passing MaxResults in request. AssertEq(nil, err2) - AssertEq(4, len(fourObjWithZeroMaxResults.Objects)) - AssertEq(TestObjectRootFolderName, fourObjWithZeroMaxResults.Objects[0].Name) - AssertEq(TestObjectSubRootFolderName, fourObjWithZeroMaxResults.Objects[1].Name) - AssertEq(TestSubObjectName, fourObjWithZeroMaxResults.Objects[2].Name) - AssertEq(TestObjectName, fourObjWithZeroMaxResults.Objects[3].Name) - AssertEq(nil, fourObjWithZeroMaxResults.CollapsedRuns) + AssertEq(5, len(fiveObjWithZeroMaxResults.Objects)) + AssertEq(TestObjectRootFolderName, fiveObjWithZeroMaxResults.Objects[0].Name) + AssertEq(TestObjectSubRootFolderName, fiveObjWithZeroMaxResults.Objects[1].Name) + AssertEq(TestSubObjectName, fiveObjWithZeroMaxResults.Objects[2].Name) + AssertEq(TestObjectName, fiveObjWithZeroMaxResults.Objects[3].Name) + AssertEq(TestGzipObjectName, fiveObjWithZeroMaxResults.Objects[4].Name) + AssertEq(nil, fiveObjWithZeroMaxResults.CollapsedRuns) } // FakeGCSServer is not handling ContentType, ContentEncoding, ContentLanguage, CacheControl in updateflow diff --git a/internal/storage/fake_storage_util.go b/internal/storage/fake_storage_util.go index b40f46fa45..9d6801e7e8 100644 --- a/internal/storage/fake_storage_util.go +++ b/internal/storage/fake_storage_util.go @@ -32,6 +32,19 @@ const TestObjectGeneration int64 = 780 const MetaDataValue string = "metaData" const MetaDataKey string = "key" +// Data specific to content-encoding gzip tests +const TestGzipObjectName string = "gcsfuse/test_gzip.txt" + +// ContentInTestGzipObjectCompressed is a gzip-compressed content for gzip tests. +// It was created by uploading a small file to GCS using `gsutil cp -Z` and then +// downloading it as it is (compressed as present on GCS) using go storage client +// library. To view/change it, open it in a gzip.newReader() ur using a gzip plugin +// in the IDE. If you do change it, remember to update ContentInTestGzipObjectDecompressed +// too correspondingly. +const ContentInTestGzipObjectCompressed string = "\x1f\x8b\b\b\x9d\xab\xd5d\x02\xfftmp1bg8d7ug\x00\v\xc9\xc8,\xe6\x02\x00~r\xe2V\x05\x00\x00\x00" +const ContentInTestGzipObjectDecompressed string = "This\n" +const TestGzipObjectGeneration int64 = 781 + type FakeStorage interface { CreateStorageHandle() (sh StorageHandle) @@ -102,6 +115,18 @@ func getTestFakeStorageObject() []fakestorage.Object { } fakeObjects = append(fakeObjects, testSubObject) + testGzipObject := fakestorage.Object{ + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: TestBucketName, + Name: TestGzipObjectName, + Generation: TestGzipObjectGeneration, + Metadata: map[string]string{MetaDataKey: MetaDataValue}, + ContentEncoding: ContentEncodingGzip, + }, + Content: []byte(ContentInTestGzipObjectCompressed), + } + fakeObjects = append(fakeObjects, testGzipObject) + return fakeObjects } diff --git a/internal/storage/object.go b/internal/storage/object.go index c9678d699e..fdce8a29e1 100644 --- a/internal/storage/object.go +++ b/internal/storage/object.go @@ -18,6 +18,8 @@ import ( "time" ) +const ContentEncodingGzip = "gzip" + // MinObject is a record representing subset of properties of a particular // generation object in GCS. // @@ -25,10 +27,15 @@ import ( // // https://cloud.google.com/storage/docs/json_api/v1/objects#resource type MinObject struct { - Name string - Size uint64 - Generation int64 - MetaGeneration int64 - Updated time.Time - Metadata map[string]string + Name string + Size uint64 + Generation int64 + MetaGeneration int64 + Updated time.Time + Metadata map[string]string + ContentEncoding string +} + +func (mo MinObject) HasContentEncodingGzip() bool { + return mo.ContentEncoding == ContentEncodingGzip } diff --git a/internal/storage/object_test.go b/internal/storage/object_test.go new file mode 100644 index 0000000000..03a1dd4564 --- /dev/null +++ b/internal/storage/object_test.go @@ -0,0 +1,54 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "testing" + + . "github.com/jacobsa/ogletest" +) + +func TestObject(t *testing.T) { RunTests(t) } + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type ObjectTest struct { +} + +func init() { RegisterTestSuite(&ObjectTest{}) } + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +func (t *ObjectTest) HasContentEncodingGzipPositive() { + mo := MinObject{} + mo.ContentEncoding = "gzip" + + AssertTrue(mo.HasContentEncodingGzip()) +} + +func (t *ObjectTest) HasContentEncodingGzipNegative() { + encodings := []string{"", "GZIP", "xzip", "zip"} + + for _, encoding := range encodings { + mo := MinObject{} + mo.ContentEncoding = encoding + + AssertFalse(mo.HasContentEncodingGzip()) + } +} diff --git a/internal/storage/storage_handle.go b/internal/storage/storage_handle.go index fe434c5022..4c263c8fd5 100644 --- a/internal/storage/storage_handle.go +++ b/internal/storage/storage_handle.go @@ -15,18 +15,15 @@ package storage import ( - "crypto/tls" "fmt" "net/http" - "time" "cloud.google.com/go/storage" "github.com/googleapis/gax-go/v2" mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount" "github.com/googlecloudplatform/gcsfuse/internal/storage/storageutil" "golang.org/x/net/context" - "golang.org/x/oauth2" - "google.golang.org/api/option" + option "google.golang.org/api/option" ) type StorageHandle interface { @@ -42,57 +39,36 @@ type storageClient struct { client *storage.Client } -type StorageClientConfig struct { - ClientProtocol mountpkg.ClientProtocol - MaxConnsPerHost int - MaxIdleConnsPerHost int - TokenSrc oauth2.TokenSource - HttpClientTimeout time.Duration - MaxRetryDuration time.Duration - RetryMultiplier float64 - UserAgent string -} - // NewStorageHandle returns the handle of Go storage client containing // customized http client. We can configure the http client using the // storageClientConfig parameter. -func NewStorageHandle(ctx context.Context, clientConfig StorageClientConfig) (sh StorageHandle, err error) { - var transport *http.Transport - // Using http1 makes the client more performant. - if clientConfig.ClientProtocol == mountpkg.HTTP1 { - transport = &http.Transport{ - MaxConnsPerHost: clientConfig.MaxConnsPerHost, - MaxIdleConnsPerHost: clientConfig.MaxIdleConnsPerHost, - // This disables HTTP/2 in transport. - TLSNextProto: make( - map[string]func(string, *tls.Conn) http.RoundTripper, - ), - } - } else { - // For http2, change in MaxConnsPerHost doesn't affect the performance. - transport = &http.Transport{ - DisableKeepAlives: true, - MaxConnsPerHost: clientConfig.MaxConnsPerHost, - ForceAttemptHTTP2: true, +func NewStorageHandle(ctx context.Context, clientConfig storageutil.StorageClientConfig) (sh StorageHandle, err error) { + + var clientOpts []option.ClientOption + // Add WithHttpClient option. + if clientConfig.ClientProtocol == mountpkg.HTTP1 || clientConfig.ClientProtocol == mountpkg.HTTP2 { + var httpClient *http.Client + httpClient, err = storageutil.CreateHttpClient(&clientConfig) + if err != nil { + err = fmt.Errorf("while creating http endpoint: %w", err) + return } + + clientOpts = append(clientOpts, option.WithHTTPClient(httpClient)) } - // Custom http client for Go Client. - httpClient := &http.Client{ - Transport: &oauth2.Transport{ - Base: transport, - Source: clientConfig.TokenSrc, - }, - Timeout: clientConfig.HttpClientTimeout, + // Create client with JSON read flow, if EnableJasonRead flag is set. + if clientConfig.ExperimentalEnableJsonRead { + clientOpts = append(clientOpts, storage.WithJSONReads()) } - // Setting UserAgent through RoundTripper middleware - httpClient.Transport = &userAgentRoundTripper{ - wrapped: httpClient.Transport, - UserAgent: clientConfig.UserAgent, + // Add Custom endpoint option. + if clientConfig.CustomEndpoint != nil { + clientOpts = append(clientOpts, option.WithEndpoint(clientConfig.CustomEndpoint.String())) } + var sc *storage.Client - sc, err = storage.NewClient(ctx, option.WithHTTPClient(httpClient)) + sc, err = storage.NewClient(ctx, clientOpts...) if err != nil { err = fmt.Errorf("go storage client creation failed: %w", err) return diff --git a/internal/storage/storage_handle_test.go b/internal/storage/storage_handle_test.go index 841d1b6c07..b69c9cd857 100644 --- a/internal/storage/storage_handle_test.go +++ b/internal/storage/storage_handle_test.go @@ -16,30 +16,18 @@ package storage import ( "context" + "net/url" "testing" - "time" mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount" + "github.com/googlecloudplatform/gcsfuse/internal/storage/storageutil" + "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" - "golang.org/x/oauth2" ) const invalidBucketName string = "will-not-be-present-in-fake-server" const projectID string = "valid-project-id" -func getDefaultStorageClientConfig() (clientConfig StorageClientConfig) { - return StorageClientConfig{ - ClientProtocol: mountpkg.HTTP1, - MaxConnsPerHost: 10, - MaxIdleConnsPerHost: 100, - TokenSrc: oauth2.StaticTokenSource(&oauth2.Token{}), - HttpClientTimeout: 800 * time.Millisecond, - MaxRetryDuration: 30 * time.Second, - RetryMultiplier: 2, - UserAgent: "gcsfuse/unknown (Go version go1.20-pre3 cl/474093167 +a813be86df) (GCP:gcsfuse)", - } -} - func TestStorageHandle(t *testing.T) { RunTests(t) } type StorageHandleTest struct { @@ -61,7 +49,7 @@ func (t *StorageHandleTest) TearDown() { t.fakeStorage.ShutDown() } -func (t *StorageHandleTest) invokeAndVerifyStorageHandle(sc StorageClientConfig) { +func (t *StorageHandleTest) invokeAndVerifyStorageHandle(sc storageutil.StorageClientConfig) { handleCreated, err := NewStorageHandle(context.Background(), sc) AssertEq(nil, err) AssertNe(nil, handleCreated) @@ -98,28 +86,77 @@ func (t *StorageHandleTest) TestBucketHandleWhenBucketDoesNotExistWithNonEmptyBi } func (t *StorageHandleTest) TestNewStorageHandleHttp2Disabled() { - sc := getDefaultStorageClientConfig() // by default http1 enabled + sc := storageutil.GetDefaultStorageClientConfig() // by default http1 enabled t.invokeAndVerifyStorageHandle(sc) } func (t *StorageHandleTest) TestNewStorageHandleHttp2Enabled() { - sc := getDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig() sc.ClientProtocol = mountpkg.HTTP2 t.invokeAndVerifyStorageHandle(sc) } func (t *StorageHandleTest) TestNewStorageHandleWithZeroMaxConnsPerHost() { - sc := getDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig() sc.MaxConnsPerHost = 0 t.invokeAndVerifyStorageHandle(sc) } func (t *StorageHandleTest) TestNewStorageHandleWhenUserAgentIsSet() { - sc := getDefaultStorageClientConfig() + sc := storageutil.GetDefaultStorageClientConfig() sc.UserAgent = "gcsfuse/unknown (Go version go1.20-pre3 cl/474093167 +a813be86df) appName (GPN:Gcsfuse-DLC)" t.invokeAndVerifyStorageHandle(sc) } + +func (t *StorageHandleTest) TestNewStorageHandleWithCustomEndpoint() { + url, err := url.Parse(storageutil.CustomEndpoint) + AssertEq(nil, err) + sc := storageutil.GetDefaultStorageClientConfig() + sc.CustomEndpoint = url + + t.invokeAndVerifyStorageHandle(sc) +} + +// This will fail while fetching the token-source, since key-file doesn't exist. +func (t *StorageHandleTest) TestNewStorageHandleWhenCustomEndpointIsNil() { + sc := storageutil.GetDefaultStorageClientConfig() + sc.CustomEndpoint = nil + + handleCreated, err := NewStorageHandle(context.Background(), sc) + + AssertNe(nil, err) + ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory"))) + AssertEq(nil, handleCreated) +} + +func (t *StorageHandleTest) TestNewStorageHandleWhenKeyFileIsEmpty() { + sc := storageutil.GetDefaultStorageClientConfig() + sc.KeyFile = "" + + t.invokeAndVerifyStorageHandle(sc) +} + +func (t *StorageHandleTest) TestNewStorageHandleWhenReuseTokenUrlFalse() { + sc := storageutil.GetDefaultStorageClientConfig() + sc.ReuseTokenFromUrl = false + + t.invokeAndVerifyStorageHandle(sc) +} + +func (t *StorageHandleTest) TestNewStorageHandleWhenTokenUrlIsSet() { + sc := storageutil.GetDefaultStorageClientConfig() + sc.TokenUrl = storageutil.CustomTokenUrl + + t.invokeAndVerifyStorageHandle(sc) +} + +func (t *StorageHandleTest) TestNewStorageHandleWhenJsonReadEnabled() { + sc := storageutil.GetDefaultStorageClientConfig() + sc.ExperimentalEnableJsonRead = true + + t.invokeAndVerifyStorageHandle(sc) +} diff --git a/internal/storage/storageutil/client.go b/internal/storage/storageutil/client.go new file mode 100644 index 0000000000..3459bab17d --- /dev/null +++ b/internal/storage/storageutil/client.go @@ -0,0 +1,99 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "crypto/tls" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/googlecloudplatform/gcsfuse/internal/auth" + mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount" + "golang.org/x/net/context" + "golang.org/x/oauth2" +) + +type StorageClientConfig struct { + ClientProtocol mountpkg.ClientProtocol + MaxConnsPerHost int + MaxIdleConnsPerHost int + HttpClientTimeout time.Duration + MaxRetryDuration time.Duration + RetryMultiplier float64 + UserAgent string + CustomEndpoint *url.URL + KeyFile string + TokenUrl string + ReuseTokenFromUrl bool + ExperimentalEnableJsonRead bool +} + +func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *http.Client, err error) { + var transport *http.Transport + // Using http1 makes the client more performant. + if storageClientConfig.ClientProtocol == mountpkg.HTTP1 { + transport = &http.Transport{ + MaxConnsPerHost: storageClientConfig.MaxConnsPerHost, + MaxIdleConnsPerHost: storageClientConfig.MaxIdleConnsPerHost, + // This disables HTTP/2 in transport. + TLSNextProto: make( + map[string]func(string, *tls.Conn) http.RoundTripper, + ), + } + } else { + // For http2, change in MaxConnsPerHost doesn't affect the performance. + transport = &http.Transport{ + DisableKeepAlives: true, + MaxConnsPerHost: storageClientConfig.MaxConnsPerHost, + ForceAttemptHTTP2: true, + } + } + + tokenSrc, err := createTokenSource(storageClientConfig) + if err != nil { + err = fmt.Errorf("while fetching tokenSource: %w", err) + return + } + + // Custom http client for Go Client. + httpClient = &http.Client{ + Transport: &oauth2.Transport{ + Base: transport, + Source: tokenSrc, + }, + Timeout: storageClientConfig.HttpClientTimeout, + } + + // Setting UserAgent through RoundTripper middleware + httpClient.Transport = &userAgentRoundTripper{ + wrapped: httpClient.Transport, + UserAgent: storageClientConfig.UserAgent, + } + + return httpClient, err +} + +// It creates dummy token-source in case of non-nil custom url. If the custom-endpoint +// is nil, it creates the token-source from the provided key-file or using ADC search +// order (https://cloud.google.com/docs/authentication/application-default-credentials#order). +func createTokenSource(storageClientConfig *StorageClientConfig) (tokenSrc oauth2.TokenSource, err error) { + if storageClientConfig.CustomEndpoint == nil { + return auth.GetTokenSource(context.Background(), storageClientConfig.KeyFile, storageClientConfig.TokenUrl, storageClientConfig.ReuseTokenFromUrl) + } else { + return oauth2.StaticTokenSource(&oauth2.Token{}), nil + } +} diff --git a/internal/storage/storageutil/client_test.go b/internal/storage/storageutil/client_test.go new file mode 100644 index 0000000000..c345359c50 --- /dev/null +++ b/internal/storage/storageutil/client_test.go @@ -0,0 +1,78 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "net/url" + "testing" + + "github.com/jacobsa/oglematchers" + . "github.com/jacobsa/ogletest" +) + +func TestClient(t *testing.T) { RunTests(t) } + +type clientTest struct { +} + +func init() { RegisterTestSuite(&clientTest{}) } + +func (t *clientTest) TestCreateTokenSrcWithCustomEndpoint() { + url, err := url.Parse(CustomEndpoint) + AssertEq(nil, err) + sc := GetDefaultStorageClientConfig() + sc.CustomEndpoint = url + + tokenSrc, err := createTokenSource(&sc) + + ExpectEq(nil, err) + ExpectNe(nil, &tokenSrc) +} + +func (t *clientTest) TestCreateTokenSrcWhenCustomEndpointIsNil() { + sc := GetDefaultStorageClientConfig() + sc.CustomEndpoint = nil + + // It will try to create the actual auth token and fail since key-file doesn't exist. + tokenSrc, err := createTokenSource(&sc) + + ExpectNe(nil, err) + ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory"))) + ExpectEq(nil, tokenSrc) +} + +func (t *clientTest) TestCreateHttpClientWithHttp1() { + sc := GetDefaultStorageClientConfig() // By default http1 enabled + + // Act: this method add tokenSource and clientOptions. + httpClient, err := CreateHttpClient(&sc) + + ExpectEq(nil, err) + ExpectNe(nil, httpClient) + ExpectNe(nil, httpClient.Transport) + ExpectEq(sc.HttpClientTimeout, httpClient.Timeout) +} + +func (t *clientTest) TestCreateHttpClientWithHttp2() { + sc := GetDefaultStorageClientConfig() + + // Act: this method add tokenSource and clientOptions. + httpClient, err := CreateHttpClient(&sc) + + ExpectEq(nil, err) + ExpectNe(nil, httpClient) + ExpectNe(nil, httpClient.Transport) + ExpectEq(sc.HttpClientTimeout, httpClient.Timeout) +} diff --git a/internal/storage/storageutil/test_util.go b/internal/storage/storageutil/test_util.go new file mode 100644 index 0000000000..56c60a2dc2 --- /dev/null +++ b/internal/storage/storageutil/test_util.go @@ -0,0 +1,45 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storageutil + +import ( + "net/url" + "time" + + mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount" +) + +const CustomEndpoint = "https://localhost:9000" +const DummyKeyFile = "test/test_creds.json" +const CustomTokenUrl = "http://custom-token-url" + +// GetDefaultStorageClientConfig is only for test, making the default endpoint +// non-nil, so that we can create dummy tokenSource while unit test. +func GetDefaultStorageClientConfig() (clientConfig StorageClientConfig) { + return StorageClientConfig{ + ClientProtocol: mountpkg.HTTP1, + MaxConnsPerHost: 10, + MaxIdleConnsPerHost: 100, + HttpClientTimeout: 800 * time.Millisecond, + MaxRetryDuration: 30 * time.Second, + RetryMultiplier: 2, + UserAgent: "gcsfuse/unknown (Go version go1.20-pre3 cl/474093167 +a813be86df) (GCP:gcsfuse)", + CustomEndpoint: &url.URL{}, + KeyFile: DummyKeyFile, + TokenUrl: "", + ReuseTokenFromUrl: true, + ExperimentalEnableJsonRead: false, + } +} diff --git a/internal/storage/user_agent_round_tripper.go b/internal/storage/storageutil/user_agent_round_tripper.go similarity index 98% rename from internal/storage/user_agent_round_tripper.go rename to internal/storage/storageutil/user_agent_round_tripper.go index 78cd36cbcc..fad6855a21 100644 --- a/internal/storage/user_agent_round_tripper.go +++ b/internal/storage/storageutil/user_agent_round_tripper.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package storageutil import "net/http" diff --git a/main.go b/main.go index 40db6513df..939cb6dde0 100644 --- a/main.go +++ b/main.go @@ -20,34 +20,26 @@ package main import ( - "crypto/tls" "fmt" "log" - "net/http" "os" "os/signal" "path" "strings" - "time" - "github.com/googlecloudplatform/gcsfuse/internal/config" - "github.com/googlecloudplatform/gcsfuse/internal/storage" - "golang.org/x/net/context" - "golang.org/x/oauth2" - - "github.com/googlecloudplatform/gcsfuse/internal/auth" "github.com/googlecloudplatform/gcsfuse/internal/canned" - "github.com/googlecloudplatform/gcsfuse/internal/gcsx" + "github.com/googlecloudplatform/gcsfuse/internal/config" "github.com/googlecloudplatform/gcsfuse/internal/locker" "github.com/googlecloudplatform/gcsfuse/internal/logger" "github.com/googlecloudplatform/gcsfuse/internal/monitor" - mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount" "github.com/googlecloudplatform/gcsfuse/internal/perf" + "github.com/googlecloudplatform/gcsfuse/internal/storage" + "github.com/googlecloudplatform/gcsfuse/internal/storage/storageutil" "github.com/jacobsa/daemonize" "github.com/jacobsa/fuse" - "github.com/jacobsa/gcloud/gcs" "github.com/kardianos/osext" "github.com/urfave/cli" + "golang.org/x/net/context" ) //////////////////////////////////////////////////////////////////////// @@ -88,83 +80,20 @@ func getUserAgent(appName string) string { } } -func getConn(flags *flagStorage) (c *gcsx.Connection, err error) { - var tokenSrc oauth2.TokenSource - if flags.Endpoint.Hostname() == "storage.googleapis.com" { - tokenSrc, err = auth.GetTokenSource( - context.Background(), - flags.KeyFile, - flags.TokenUrl, - flags.ReuseTokenFromUrl, - ) - if err != nil { - err = fmt.Errorf("GetTokenSource: %w", err) - return - } - } else { - // Do not use OAuth with non-Google hosts. - tokenSrc = oauth2.StaticTokenSource(&oauth2.Token{}) - } - - // Create the connection. - cfg := &gcs.ConnConfig{ - Url: flags.Endpoint, - TokenSource: tokenSrc, - UserAgent: getUserAgent(flags.AppName), - MaxBackoffSleep: flags.MaxRetrySleep, - } - - // The default HTTP transport uses HTTP/2 with TCP multiplexing, which - // does not create new TCP connections even when the idle connections - // run out. To specify multiple connections per host, HTTP/2 is disabled - // on purpose. - if flags.ClientProtocol == mountpkg.HTTP1 { - cfg.Transport = &http.Transport{ - MaxConnsPerHost: flags.MaxConnsPerHost, - // This disables HTTP/2 in the transport. - TLSNextProto: make( - map[string]func(string, *tls.Conn) http.RoundTripper, - ), - } - } - - if flags.DebugHTTP { - cfg.HTTPDebugLogger = logger.NewDebug("http: ") - } - - if flags.DebugGCS { - cfg.GCSDebugLogger = logger.NewDebug("gcs: ") - } - - return gcsx.NewConnection(cfg) -} - -func getConnWithRetry(flags *flagStorage) (c *gcsx.Connection, err error) { - c, err = getConn(flags) - for delay := 1 * time.Second; delay <= flags.MaxRetrySleep && err != nil; delay = delay/2 + delay { - logger.Infof("Waiting for connection: %v\n", err) - time.Sleep(delay) - c, err = getConn(flags) - } - return -} - func createStorageHandle(flags *flagStorage) (storageHandle storage.StorageHandle, err error) { - tokenSrc, err := auth.GetTokenSource(context.Background(), flags.KeyFile, flags.TokenUrl, true) - if err != nil { - err = fmt.Errorf("get token source: %w", err) - return - } - - storageClientConfig := storage.StorageClientConfig{ - ClientProtocol: flags.ClientProtocol, - MaxConnsPerHost: flags.MaxConnsPerHost, - MaxIdleConnsPerHost: flags.MaxIdleConnsPerHost, - TokenSrc: tokenSrc, - HttpClientTimeout: flags.HttpClientTimeout, - MaxRetryDuration: flags.MaxRetryDuration, - RetryMultiplier: flags.RetryMultiplier, - UserAgent: getUserAgent(flags.AppName), + storageClientConfig := storageutil.StorageClientConfig{ + ClientProtocol: flags.ClientProtocol, + MaxConnsPerHost: flags.MaxConnsPerHost, + MaxIdleConnsPerHost: flags.MaxIdleConnsPerHost, + HttpClientTimeout: flags.HttpClientTimeout, + MaxRetryDuration: flags.MaxRetryDuration, + RetryMultiplier: flags.RetryMultiplier, + UserAgent: getUserAgent(flags.AppName), + CustomEndpoint: flags.CustomEndpoint, + KeyFile: flags.KeyFile, + TokenUrl: flags.TokenUrl, + ReuseTokenFromUrl: flags.ReuseTokenFromUrl, + ExperimentalEnableJsonRead: flags.ExperimentalEnableJsonRead, } storageHandle, err = storage.NewStorageHandle(context.Background(), storageClientConfig) @@ -194,36 +123,29 @@ func mountWithArgs( // // Special case: if we're mounting the fake bucket, we don't need an actual // connection. - var conn *gcsx.Connection var storageHandle storage.StorageHandle if bucketName != canned.FakeBucketName { - mountStatus.Println("Opening GCS connection...") - - if flags.EnableStorageClientLibrary { - storageHandle, err = createStorageHandle(flags) - } else { - conn, err = getConnWithRetry(flags) - } + mountStatus.Println("Creating Storage handle...") + storageHandle, err = createStorageHandle(flags) if err != nil { - err = fmt.Errorf("failed to open connection - getConnWithRetry: %w", err) + err = fmt.Errorf("Failed to create storage handle using createStorageHandle: %w", err) return } } // Mount the file system. logger.Infof("Creating a mount at %q\n", mountPoint) - mfs, err = mountWithConn( + mfs, err = mountWithStorageHandle( context.Background(), bucketName, mountPoint, flags, mountConfig, - conn, storageHandle, mountStatus) if err != nil { - err = fmt.Errorf("mountWithConn: %w", err) + err = fmt.Errorf("mountWithStorageHandle: %w", err) return } diff --git a/main_test.go b/main_test.go index 36dc0b3644..1bfeac0582 100644 --- a/main_test.go +++ b/main_test.go @@ -21,16 +21,6 @@ type MainTest struct { func init() { RegisterTestSuite(&MainTest{}) } -func (t *MainTest) TestCreateStorageHandleEnableStorageClientLibraryIsTrue() { - storageHandle, err := createStorageHandle(&flagStorage{ - EnableStorageClientLibrary: true, - KeyFile: "testdata/test_creds.json", - }) - - ExpectNe(nil, storageHandle) - ExpectEq(nil, err) -} - func (t *MainTest) TestCreateStorageHandle() { flags := &flagStorage{ ClientProtocol: mountpkg.HTTP1, diff --git a/mount.go b/mount.go index 7686282c36..a9fd19be11 100644 --- a/mount.go +++ b/mount.go @@ -34,13 +34,12 @@ import ( // Mount the file system based on the supplied arguments, returning a // fuse.MountedFileSystem that can be joined to wait for unmounting. -func mountWithConn( +func mountWithStorageHandle( ctx context.Context, bucketName string, mountPoint string, flags *flagStorage, mountConfig *config.MountConfig, - conn *gcsx.Connection, storageHandle storage.StorageHandle, status *log.Logger) (mfs *fuse.MountedFileSystem, err error) { // Sanity check: make sure the temporary directory exists and is writable @@ -97,9 +96,8 @@ be interacting with the file system.`) AppendThreshold: 1 << 21, // 2 MiB, a total guess. TmpObjectPrefix: ".gcsfuse_tmp/", DebugGCS: flags.DebugGCS, - EnableStorageClientLibrary: flags.EnableStorageClientLibrary, } - bm := gcsx.NewBucketManager(bucketCfg, conn, storageHandle) + bm := gcsx.NewBucketManager(bucketCfg, storageHandle) // Create a file system server. serverCfg := &fs.ServerConfig{ diff --git a/perfmetrics/scripts/compare_fuse_types_using_fio.py b/perfmetrics/scripts/compare_fuse_types_using_fio.py index 513baa6b31..bb1ad1d7a2 100644 --- a/perfmetrics/scripts/compare_fuse_types_using_fio.py +++ b/perfmetrics/scripts/compare_fuse_types_using_fio.py @@ -61,7 +61,7 @@ def _install_gcsfuse_source(gcs_bucket, gcsfuse_flags) -> None: os.system(f'''git clone {GCSFUSE_REPO} mkdir gcs cd gcsfuse - go run . {gcsfuse_flags} {gcs_bucket} ../gcs + CGO_ENABLED=0 go run . {gcsfuse_flags} {gcs_bucket} ../gcs cd .. ''') diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index 3886b08a20..0f8ce8e3bf 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -6,8 +6,8 @@ echo "Installing git" sudo apt-get install git echo "Installing pip" sudo apt-get install pip -y -echo "Installing go-lang 1.20.4" -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +echo "Installing go-lang 1.20.5" +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin echo "Installing fio" @@ -28,7 +28,7 @@ commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) git checkout $commitId echo "Executing integration tests" -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test +GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test -timeout 24m # Checkout back to master branch to use latest CI test scripts in master. git checkout master @@ -45,7 +45,7 @@ cd "./perfmetrics/scripts/" echo "Mounting gcs bucket" mkdir -p gcs LOG_FILE=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs.txt -GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100 --enable-storage-client-library --debug_fuse --debug_gcs --log-file $LOG_FILE --log-format \"text\" --stackdriver-export-interval=30s" +GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100 --debug_fuse --debug_gcs --log-file $LOG_FILE --log-format \"text\" --stackdriver-export-interval=30s" BUCKET_NAME=periodic-perf-tests MOUNT_POINT=gcs # The VM will itself exit if the gcsfuse mount fails. diff --git a/perfmetrics/scripts/load_tests/python/requirements.txt b/perfmetrics/scripts/load_tests/python/requirements.txt index 0b24497df8..e91abc7100 100644 --- a/perfmetrics/scripts/load_tests/python/requirements.txt +++ b/perfmetrics/scripts/load_tests/python/requirements.txt @@ -18,9 +18,9 @@ cachetools==5.3.0 \ --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 # via google-auth -certifi==2023.5.7 \ - --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ - --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 # via requests charset-normalizer==3.1.0 \ --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ @@ -371,9 +371,9 @@ pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via google-auth -requests==2.30.0 \ - --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ - --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # requests-oauthlib # tensorboard @@ -441,7 +441,7 @@ tensorflow==2.12.0 \ --hash=sha256:c5193ddb3bb5120cb445279beb08ed9e74a85a4eeb2485550d6fb707a89d9a88 \ --hash=sha256:c8001210df7202ef6267150865b0b79f834c3ca69ee3132277de8eeb994dffde \ --hash=sha256:e29fcf6cfd069aefb4b44f357cccbb4415a5a3d7b5b516eaf4450062fe40021e - # via -r ./requirements.in + # via -r requirements.in tensorflow-estimator==2.12.0 \ --hash=sha256:59b191bead4883822de3d63ac02ace11a83bfe6c10d64d0c4dfde75a50e60ca1 # via tensorflow @@ -553,5 +553,6 @@ wrapt==1.14.1 \ # via tensorflow # WARNING: The following packages were not pinned, but pip requires them to be -# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. +# pinned when the requirements file includes hashes and the requirement is not +# satisfied by a package already installed. Consider using the --allow-unsafe flag. # setuptools diff --git a/perfmetrics/scripts/ls_metrics/listing_benchmark.py b/perfmetrics/scripts/ls_metrics/listing_benchmark.py index 6bda6b1496..7c99de82cf 100644 --- a/perfmetrics/scripts/ls_metrics/listing_benchmark.py +++ b/perfmetrics/scripts/ls_metrics/listing_benchmark.py @@ -375,7 +375,7 @@ def _mount_gcs_bucket(bucket_name) -> str: subprocess.call('mkdir {}'.format(gcs_bucket), shell=True) exit_code = subprocess.call( - 'gcsfuse --implicit-dirs --enable-storage-client-library --max-conns-per-host 100 {} {}'.format( + 'gcsfuse --implicit-dirs --max-conns-per-host 100 {} {}'.format( bucket_name, gcs_bucket), shell=True) if exit_code != 0: log.error('Cannot mount the GCS bucket due to exit code %s.\n', exit_code) diff --git a/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh b/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh index 1af6bd1f55..884c633b8d 100644 --- a/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh @@ -1,13 +1,14 @@ #!/bin/bash -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +# Install golang +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz -q rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz export PATH=$PATH:/usr/local/go/bin # Clone and build the gcsfuse master branch. git clone https://github.com/GoogleCloudPlatform/gcsfuse.git cd gcsfuse -go build . +CGO_ENABLED=0 go build . cd - # Create a directory for gcsfuse logs diff --git a/perfmetrics/scripts/ml_tests/run_image_recognition_models.py b/perfmetrics/scripts/ml_tests/run_image_recognition_models.py index 9ceaf7e5ca..34675cccc3 100644 --- a/perfmetrics/scripts/ml_tests/run_image_recognition_models.py +++ b/perfmetrics/scripts/ml_tests/run_image_recognition_models.py @@ -105,7 +105,7 @@ def _run_from_source(gcs_bucket, data_directory_name) -> None: os.system(f'''mkdir {data_directory_name} git clone {GITHUB_REPO} cd gcsfuse - go run . --implicit-dirs --stat-cache-capacity 1000000 --max-conns-per-host 100 --stackdriver-export-interval=60s {gcs_bucket} ../{data_directory_name} + CGO_ENABLED=0 go run . --implicit-dirs --stat-cache-capacity 1000000 --max-conns-per-host 100 --stackdriver-export-interval=60s {gcs_bucket} ../{data_directory_name} cd .. ''') diff --git a/perfmetrics/scripts/ml_tests/setup.sh b/perfmetrics/scripts/ml_tests/setup.sh index f0990cdd63..0f0987f6ee 100644 --- a/perfmetrics/scripts/ml_tests/setup.sh +++ b/perfmetrics/scripts/ml_tests/setup.sh @@ -4,7 +4,7 @@ # >> source setup.sh # Go version to be installed. -GO_VERSION=go1.20.4.linux-amd64.tar.gz +GO_VERSION=go1.20.5.linux-amd64.tar.gz # This function will install the given module/dependency if it's not alredy # installed. diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh index 1d6ec891cc..d31c3cc22c 100644 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh @@ -5,19 +5,19 @@ # and epochs functionality, and runs the model # Install go lang -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin # Clone the repo and build gcsfuse git clone "https://github.com/GoogleCloudPlatform/gcsfuse.git" cd gcsfuse -go build . +CGO_ENABLED=0 go build . cd - # Mount the bucket and run in background so that docker doesn't keep running after resnet_runner.py fails echo "Mounting the bucket" -nohup gcsfuse/gcsfuse --foreground --implicit-dirs --enable-storage-client-library --debug_fuse --debug_gcs --max-conns-per-host 100 --log-format "text" --log-file /home/logs/gcsfuse.log --stackdriver-export-interval 60s ml-models-data-gcsfuse myBucket > /home/output/gcsfuse.out 2> /home/output/gcsfuse.err & +nohup gcsfuse/gcsfuse --foreground --implicit-dirs --debug_fuse --debug_gcs --max-conns-per-host 100 --log-format "text" --log-file /home/logs/gcsfuse.log --stackdriver-export-interval 60s ml-models-data-gcsfuse myBucket > /home/output/gcsfuse.out 2> /home/output/gcsfuse.err & # Install tensorflow model garden library pip3 install --user tf-models-official==2.10.0 diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 49072ef6ef..e6dab6a2e0 100644 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -1,69 +1,102 @@ #!/bin/bash -# Running test only for when PR contains execute-perf-test label +# Running test only for when PR contains execute-perf-test or execute-integration-tests label +readonly EXECUTE_PERF_TEST_LABEL="execute-perf-test" +readonly EXECUTE_INTEGRATION_TEST_LABEL="execute-integration-tests" +readonly INTEGRATION_TEST_EXECUTION_TIME=24m + curl https://api.github.com/repos/GoogleCloudPlatform/gcsfuse/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER >> pr.json -perfTest=$(cat pr.json | grep "execute-perf-test") +perfTest=$(grep "$EXECUTE_PERF_TEST_LABEL" pr.json) +integrationTests=$(grep "$EXECUTE_INTEGRATION_TEST_LABEL" pr.json) rm pr.json perfTestStr="$perfTest" -if [[ "$perfTestStr" != *"execute-perf-test"* ]] +integrationTestsStr="$integrationTests" +if [[ "$perfTestStr" != *"$EXECUTE_PERF_TEST_LABEL"* && "$integrationTestsStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL"* ]] then echo "No need to execute tests" exit 0 fi -# It will take approx 80 minutes to run the script. set -e sudo apt-get update echo Installing git sudo apt-get install git -echo Installing python3-pip -sudo apt-get -y install python3-pip -echo Installing libraries to run python script -pip install google-cloud -pip install google-cloud-vision -pip install google-api-python-client -pip install prettytable -echo Installing go-lang 1.20.4 -wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +echo Installing go-lang 1.20.5 +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin -echo Installing fio -sudo apt-get install fio -y - -# Run on master branch +export CGO_ENABLED=0 cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" -git checkout master -echo Mounting gcs bucket for master branch -mkdir -p gcs -GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100" -BUCKET_NAME=presubmit-perf-tests -MOUNT_POINT=gcs -# The VM will itself exit if the gcsfuse mount fails. -go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT -touch result.txt -# Running FIO test -chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh -./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh -sudo umount gcs - # Fetch PR branch echo '[remote "origin"] fetch = +refs/pull/*/head:refs/remotes/origin/pr/*' >> .git/config -git fetch origin -echo checkout PR branch -git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER +git fetch origin -q + +function execute_perf_test() { + mkdir -p gcs + GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100" + BUCKET_NAME=presubmit-perf-tests + MOUNT_POINT=gcs + # The VM will itself exit if the gcsfuse mount fails. + go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT + # Running FIO test + chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh + ./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh + sudo umount gcs +} + +# execute perf tests. +if [[ "$perfTestStr" == *"$EXECUTE_PERF_TEST_LABEL"* ]]; +then + # Installing requirements + echo Installing python3-pip + sudo apt-get -y install python3-pip + echo Installing libraries to run python script + pip install google-cloud + pip install google-cloud-vision + pip install google-api-python-client + pip install prettytable + echo Installing fio + sudo apt-get install fio -y + + # Executing perf tests for master branch + git checkout master + # Store results + touch result.txt + echo Mounting gcs bucket for master branch and execute tests + execute_perf_test -# Executing integration tests -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test -# Executing perf tests -echo Mounting gcs bucket from pr branch -mkdir -p gcs -# The VM will itself exit if the gcsfuse mount fails. -go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT -# Running FIO test -chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh -./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh -sudo umount gcs + # Executing perf tests for PR branch + echo checkout PR branch + git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER + echo Mounting gcs bucket from pr branch and execute tests + execute_perf_test -echo showing results... -python3 ./perfmetrics/scripts/presubmit/print_results.py + # Show results + echo showing results... + python3 ./perfmetrics/scripts/presubmit/print_results.py +fi + +# Execute integration tests. +if [[ "$integrationTestsStr" == *"$EXECUTE_INTEGRATION_TEST_LABEL"* ]]; +then + echo checkout PR branch + git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER + + # Create bucket for integration tests. + # The prefix for the random string + bucketPrefix="gcsfuse-integration-test-" + # The length of the random string + length=5 + # Generate the random string + random_string=$(tr -dc 'a-z0-9' < /dev/urandom | head -c $length) + BUCKET_NAME=$bucketPrefix$random_string + echo 'bucket name = '$BUCKET_NAME + gcloud alpha storage buckets create gs://$BUCKET_NAME --project=gcs-fuse-test-ml --location=us-west1 --uniform-bucket-level-access + + # Executing integration tests + GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$BUCKET_NAME -timeout $INTEGRATION_TEST_EXECUTION_TIME + + # Delete bucket after testing. + gcloud alpha storage rm --recursive gs://$BUCKET_NAME/ +fi diff --git a/perfmetrics/scripts/requirements.txt b/perfmetrics/scripts/requirements.txt index f96eafb59b..883c3f653b 100644 --- a/perfmetrics/scripts/requirements.txt +++ b/perfmetrics/scripts/requirements.txt @@ -8,9 +8,9 @@ cachetools==5.3.0 \ --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 # via google-auth -certifi==2023.5.7 \ - --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ - --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 # via requests charset-normalizer==3.1.0 \ --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ @@ -261,9 +261,9 @@ pytest==7.3.1 \ --hash=sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362 \ --hash=sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3 # via -r requirements.in -requests==2.30.0 \ - --hash=sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294 \ - --hash=sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # -r requirements.in # google-api-core @@ -277,14 +277,14 @@ six==1.16.0 \ # via # google-auth # google-auth-httplib2 -testresources==2.0.1 \ - --hash=sha256:67a361c3a2412231963b91ab04192209aa91a1aa052f0ab87245dbea889d1282 \ - --hash=sha256:ee9d1982154a1e212d4e4bac6b610800bfb558e4fb853572a827bc14a96e4417 - # via -r requirements.in tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f # via pytest +testresources==2.0.1 \ + --hash=sha256:67a361c3a2412231963b91ab04192209aa91a1aa052f0ab87245dbea889d1282 \ + --hash=sha256:ee9d1982154a1e212d4e4bac6b610800bfb558e4fb853572a827bc14a96e4417 + # via -r requirements.in typing==3.7.4.3 \ --hash=sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9 \ --hash=sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5 diff --git a/tools/build_gcsfuse/main.go b/tools/build_gcsfuse/main.go index bb52ea1191..b1a4022481 100644 --- a/tools/build_gcsfuse/main.go +++ b/tools/build_gcsfuse/main.go @@ -158,6 +158,7 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro fmt.Sprintf("GOROOT=%s", runtime.GOROOT()), fmt.Sprintf("GOPATH=%s", gopath), fmt.Sprintf("GOCACHE=%s", gocache), + "CGO_ENABLED=0", } // Build. diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh new file mode 100644 index 0000000000..18184c7dc0 --- /dev/null +++ b/tools/cd_scripts/e2e_test.sh @@ -0,0 +1,122 @@ +#! /bin/bash +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print commands and their arguments as they are executed. +set -x +# Exit immediately if a command exits with a non-zero status. +set -e + +#details.txt file contains the release version and commit hash of the current release. +gsutil cp gs://gcsfuse-release-packages/version-detail/details.txt . +# Writing VM instance name to details.txt (Format: release-test-) +curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >> details.txt + +# Based on the os type(from vm instance name) in detail.txt, run the following commands to add starterscriptuser +if grep -q ubuntu details.txt || grep -q debian details.txt; +then +# For ubuntu and debian os + sudo adduser --ingroup google-sudoers --disabled-password --home=/home/starterscriptuser --gecos "" starterscriptuser +else +# For rhel and centos + sudo adduser -g google-sudoers --home-dir=/home/starterscriptuser starterscriptuser +fi + +# Run the following as starterscriptuser +sudo -u starterscriptuser bash -c ' +# Exit immediately if a command exits with a non-zero status. +set -e +# Print commands and their arguments as they are executed. +set -x + +#Copy details.txt to starterscriptuser home directory and create logs.txt +cd ~/ +cp /details.txt . +touch logs.txt + +echo User: $USER &>> ~/logs.txt +echo Current Working Directory: $(pwd) &>> ~/logs.txt + +# Based on the os type in detail.txt, run the following commands for setup +if grep -q ubuntu details.txt || grep -q debian details.txt; +then +# For Debian and Ubuntu os + sudo apt update + + #Install fuse + sudo apt install -y fuse + + # download and install gcsfuse deb package + gsutil cp gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/gcsfuse_$(sed -n 1p details.txt)_amd64.deb . + sudo dpkg -i gcsfuse_$(sed -n 1p details.txt)_amd64.deb |& tee -a ~/logs.txt + + # install wget + sudo apt install -y wget + + #install git + sudo apt install -y git + + #install build-essentials + sudo apt install -y build-essential +else +# For rhel and centos + sudo yum makecache + sudo yum -y update + + #Install fuse + sudo yum -y install fuse + + #download and install gcsfuse rpm package + gsutil cp gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/gcsfuse-$(sed -n 1p details.txt)-1.x86_64.rpm . + sudo yum -y localinstall gcsfuse-$(sed -n 1p details.txt)-1.x86_64.rpm + + #install wget + sudo yum -y install wget + + #install git + sudo yum -y install git + + #install Development tools + sudo yum -y install gcc gcc-c++ make +fi + +# install go +wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go_tar.tar.gz +export PATH=${PATH}:/usr/local/go/bin + +#Write gcsfuse and go version to log file +gcsfuse --version |& tee -a ~/logs.txt +go version |& tee -a ~/logs.txt + +# Clone and checkout gcsfuse repo +export PATH=${PATH}:/usr/local/go/bin +git clone https://github.com/googlecloudplatform/gcsfuse |& tee -a ~/logs.txt +cd gcsfuse +git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt + +#run tests with testbucket flag +set +e +GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --testInstalledPackage --timeout=60m &>> ~/logs.txt + +if [ $? -ne 0 ]; +then + echo "Test failures detected" &>> ~/logs.txt +else + touch success.txt + gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ +fi + +gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ +' diff --git a/tools/cd_scripts/install_test.sh b/tools/cd_scripts/install_test.sh new file mode 100644 index 0000000000..e9b3e6f606 --- /dev/null +++ b/tools/cd_scripts/install_test.sh @@ -0,0 +1,107 @@ +#! /bin/bash +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print commands and their arguments as they are executed. +set -x + +#details.txt file contains the release version and commit hash of the current release. +gsutil cp gs://gcsfuse-release-packages/version-detail/details.txt . +# Writing VM instance name to details.txt (Format: release-test-) +curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google" >> details.txt +touch ~/logs.txt + +# Based on the os type(from vm instance name) in detail.txt, run the following commands to install apt-transport-artifact-registry +if grep -q ubuntu details.txt || grep -q debian details.txt; +then +# For ubuntu and debian os + curl https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg | sudo apt-key add - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + echo 'deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main' | sudo tee -a /etc/apt/sources.list.d/artifact-registry.list + sudo apt update + sudo apt install apt-transport-artifact-registry + echo "deb ar+https://us-apt.pkg.dev/projects/gcs-fuse-prod gcsfuse-$(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list.d/artifact-registry.list + sudo apt update + + # Install released gcsfuse version + sudo apt install -y gcsfuse=$(sed -n 1p details.txt) -t gcsfuse-$(lsb_release -cs) |& tee -a ~/logs.txt +else +# For rhel and centos + sudo yum makecache + sudo yum -y install yum-plugin-artifact-registry +sudo tee -a /etc/yum.repos.d/artifact-registry.repo << EOF +[gcsfuse-el7-x86-64] +name=gcsfuse-el7-x86-64 +baseurl=https://asia-yum.pkg.dev/projects/gcs-fuse-prod/gcsfuse-el7-x86-64 +enabled=1 +repo_gpgcheck=0 +gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOF + sudo yum makecache + sudo yum -y --enablerepo=gcsfuse-el7-x86-64 install gcsfuse-$(sed -n 1p details.txt)-1 |& tee -a ~/logs.txt +fi + +# Verify gcsfuse version (successful installation) +gcsfuse --version |& tee version.txt +installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3) +if grep -q $installed_version details.txt; then + echo "GCSFuse latest version installed correctly." &>> ~/logs.txt +else + echo "Failure detected in latest gcsfuse version installation." &>> ~/logs.txt +fi + +# Uninstall gcsfuse latest version and install old version +if grep -q ubuntu details.txt || grep -q debian details.txt; +then + sudo apt remove -y gcsfuse + sudo apt install -y gcsfuse=0.42.5 -t gcsfuse-$(lsb_release -cs) |& tee -a ~/logs.txt +else + sudo yum -y remove gcsfuse + sudo yum -y install gcsfuse-0.42.5-1 |& tee -a ~/logs.txt +fi + +# verify old version installation +gcsfuse --version |& tee version.txt +installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3) +if [ $installed_version == "0.42.5" ]; then + echo "GCSFuse old version (0.42.5) installed successfully" &>> ~/logs.txt +else + echo "Failure detected in GCSFuse old version installation." &>> ~/logs.txt +fi + +# Upgrade gcsfuse to latest version +if grep -q ubuntu details.txt || grep -q debian details.txt; +then + sudo apt install --only-upgrade gcsfuse |& tee -a ~/logs.txt +else + sudo yum -y upgrade gcsfuse |& tee -a ~/logs.txt +fi + +gcsfuse --version |& tee version.txt +installed_version=$(echo $(sed -n 1p version.txt) | cut -d' ' -f3) +if grep -q $installed_version details.txt; then + echo "GCSFuse successfully upgraded to latest version $installed_version." &>> ~/logs.txt +else + echo "Failure detected in upgrading to latest gcsfuse version." &>> ~/logs.txt +fi + +if grep -q Failure ~/logs.txt; then + echo "Test failed" &>> ~/logs.txt ; +else + touch success.txt + gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ ; +fi + +gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p details.txt)/installation-test/$(sed -n 3p details.txt)/ diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index a36d394c97..b638978d4b 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -34,7 +34,7 @@ ARG OS_VERSION ARG OS_NAME # Image with gcsfuse installed and its package (.deb) -FROM golang:1.20.4 as gcsfuse-package +FROM golang:1.20.5 as gcsfuse-package RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler diff --git a/tools/integration_tests/explicit_dir/explicit_dir_test.go b/tools/integration_tests/explicit_dir/explicit_dir_test.go index 06c69ff6de..01e675271b 100644 --- a/tools/integration_tests/explicit_dir/explicit_dir_test.go +++ b/tools/integration_tests/explicit_dir/explicit_dir_test.go @@ -22,7 +22,7 @@ import ( ) func TestMain(m *testing.M) { - flags := [][]string{{"--enable-storage-client-library=true"}, {"--enable-storage-client-library=false"}} + flags := [][]string{{"--implicit-dirs=false"}} implicit_and_explicit_dir_setup.RunTestsForImplicitDirAndExplicitDir(flags, m) } diff --git a/tools/integration_tests/gzip/gzip_test.go b/tools/integration_tests/gzip/gzip_test.go new file mode 100644 index 0000000000..a04eb30f88 --- /dev/null +++ b/tools/integration_tests/gzip/gzip_test.go @@ -0,0 +1,212 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for gzip objects in gcsfuse mounts. +package gzip_test + +import ( + "fmt" + "log" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/gzip/helpers" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const ( + SeqReadSizeMb = 1 + TextContentSize = 10 * 1e6 * SeqReadSizeMb + + TextContentWithContentEncodingWithNoTransformFilename = "textContentWithContentEncodingWithNoTransform.txt" + TextContentWithContentEncodingWithoutNoTransformFilename = "textContentWithContentEncodingWithoutNoTransform.txt" + + GzipContentWithoutContentEncodingFilename = "gzipContentWithoutContentEncoding.txt.gz" + + GzipContentWithContentEncodingWithNoTransformFilename = "gzipContentWithContentEncodingWithNoTransform.txt.gz" + GzipContentWithContentEncodingWithoutNoTransformFilename = "gzipContentWithContentEncodingWithoutNoTransform.txt.gz" + + TextContentWithContentEncodingWithNoTransformToOverwrite = "TextContentWithContentEncodingWithNoTransformToOverwrite.txt" + TextContentWithContentEncodingWithoutNoTransformToOverwrite = "TextContentWithContentEncodingWithoutNoTransformToOverwrite.txt" + + GzipContentWithoutContentEncodingToOverwrite = "GzipContentWithoutContentEncodingToOverwrite.txt.gz" + + GzipContentWithContentEncodingWithNoTransformToOverwrite = "GzipContentWithContentEncodingWithNoTransformToOverwrite.txt.gz" + GzipContentWithContentEncodingWithoutNoTransformToOverwrite = "GzipContentWithContentEncodingWithoutNoTransformToOverwrite.txt.gz" + + TestBucketPrefixPath = "gzip" +) + +var ( + gcsObjectsToBeDeletedEventually []string +) + +func setup_testdata(m *testing.M) error { + fmds := []struct { + filename string + filesize int + keepCacheControlNoTransform bool // if true, no-transform is reset as '' + enableGzipEncodedContent bool // if true, original file content is gzip-encoded + enableGzipContentEncoding bool // if true, the content is uploaded as gsutil cp -Z i.e. with content-encoding: gzip header in GCS + }{ + { + filename: TextContentWithContentEncodingWithNoTransformFilename, + filesize: TextContentSize, + keepCacheControlNoTransform: true, + enableGzipEncodedContent: false, + enableGzipContentEncoding: true, + }, + { + filename: TextContentWithContentEncodingWithoutNoTransformFilename, + filesize: TextContentSize, + keepCacheControlNoTransform: false, + enableGzipEncodedContent: false, + enableGzipContentEncoding: true, + }, + { + filename: GzipContentWithoutContentEncodingFilename, + filesize: TextContentSize, + keepCacheControlNoTransform: true, // it's a don't care in this case + enableGzipEncodedContent: true, + enableGzipContentEncoding: false, + }, { + filename: GzipContentWithContentEncodingWithNoTransformFilename, + filesize: TextContentSize, + keepCacheControlNoTransform: true, + enableGzipEncodedContent: true, + enableGzipContentEncoding: true, + }, { + filename: GzipContentWithContentEncodingWithoutNoTransformFilename, + filesize: TextContentSize, + keepCacheControlNoTransform: false, + enableGzipEncodedContent: true, + enableGzipContentEncoding: true, + }, + { + filename: TextContentWithContentEncodingWithNoTransformToOverwrite, + filesize: TextContentSize, + keepCacheControlNoTransform: true, + enableGzipEncodedContent: false, + enableGzipContentEncoding: true, + }, + { + filename: TextContentWithContentEncodingWithoutNoTransformToOverwrite, + filesize: TextContentSize, + keepCacheControlNoTransform: false, + enableGzipEncodedContent: false, + enableGzipContentEncoding: true, + }, + { + filename: GzipContentWithoutContentEncodingToOverwrite, + filesize: TextContentSize, + keepCacheControlNoTransform: true, // it's a don't care in this case + enableGzipEncodedContent: true, + enableGzipContentEncoding: false, + }, { + filename: GzipContentWithContentEncodingWithNoTransformToOverwrite, + filesize: TextContentSize, + keepCacheControlNoTransform: true, + enableGzipEncodedContent: true, + enableGzipContentEncoding: true, + }, { + filename: GzipContentWithContentEncodingWithoutNoTransformToOverwrite, + filesize: TextContentSize, + keepCacheControlNoTransform: false, + enableGzipEncodedContent: true, + enableGzipContentEncoding: true, + }, + } + + for _, fmd := range fmds { + var localFilePath string + localFilePath, err := helpers.CreateLocalTempFile(fmd.filesize, fmd.enableGzipEncodedContent) + if err != nil { + return err + } + + defer os.Remove(localFilePath) + + // upload to the test-bucket for testing + gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, fmd.filename) + + err = operations.UploadGcsObject(localFilePath, gcsObjectPath, fmd.enableGzipContentEncoding) + if err != nil { + return err + } + + gcsObjectsToBeDeletedEventually = append(gcsObjectsToBeDeletedEventually, gcsObjectPath) + + if !fmd.keepCacheControlNoTransform { + err = operations.ClearCacheControlOnGcsObject(gcsObjectPath) + if err != nil { + return err + } + } + } + + return nil +} + +func destroy_testdata(m *testing.M) error { + for _, gcsObjectPath := range gcsObjectsToBeDeletedEventually { + err := operations.DeleteGcsObject(gcsObjectPath) + if err != nil { + return fmt.Errorf("Failed to delete gcs object gs://%s", gcsObjectPath) + } + } + + return nil +} + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + commonFlags := []string{"--sequential-read-size-mb=" + fmt.Sprint(SeqReadSizeMb), "--implicit-dirs"} + flags := [][]string{commonFlags} + + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + + if setup.TestBucket() == "" && setup.MountedDirectory() != "" { + log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.") + os.Exit(1) + } + + err := setup_testdata(m) + if err != nil { + fmt.Printf("Failed to setup test data: %v", err) + os.Exit(1) + } + + defer func() { + err := destroy_testdata(m) + if err != nil { + fmt.Printf("Failed to destoy gzip test data: %v", err) + } + }() + + // Run tests for mountedDirectory only if --mountedDirectory flag is set. + setup.RunTestsForMountedDirectoryFlag(m) + + // Run tests for testBucket + setup.SetUpTestDirForTestBucketFlag() + + successCode := static_mounting.RunTests(flags, m) + + setup.RemoveBinFileCopiedForTesting() + + os.Exit(successCode) +} diff --git a/tools/integration_tests/gzip/helpers/helpers.go b/tools/integration_tests/gzip/helpers/helpers.go new file mode 100644 index 0000000000..4871768880 --- /dev/null +++ b/tools/integration_tests/gzip/helpers/helpers.go @@ -0,0 +1,179 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helpers + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "io/fs" + "os" + "path" + "strings" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const ( + TempFileStrLine = "This is a test file" + TmpDirectory = "/tmp" +) + +// Creates a temporary file (name-collision-safe) in /tmp with given content size in bytes. +// If gzipCompress is true, output file is a gzip-compressed file. +// contentSize is the size of the uncompressed content. In case gzipCompress is true, the actual output file size will be +// different from contentSize (typically gzip-compressed file size < contentSize). +// Caller is responsible for deleting the created file when done using it. +// Failure cases: +// 1. contentSize <= 0 +// 2. os.CreateTemp() returned error or nil handle +// 3. gzip.NewWriter() returned nil handle +// 4. Failed to write the content to the created temp file +func CreateLocalTempFile(contentSize int, gzipCompress bool) (string, error) { + // fail if contentSize <= 0 + if contentSize <= 0 { + return "", fmt.Errorf("unsupported fileSize: %d", contentSize) + } + + // Create text-content of given size. + // strings.builder is used as opposed to string appends + // as this is much more efficient when multiple concatenations + // are required. + var contentBuilder strings.Builder + const tempStr = TempFileStrLine + "\n" + + for ; contentSize >= len(tempStr); contentSize -= len(tempStr) { + contentBuilder.WriteString(tempStr) + } + + if contentSize > 0 { + contentBuilder.WriteString(tempStr[0:contentSize]) + } + + // reset contentSize + contentSize = contentBuilder.Len() + + // create appropriate name template for temp file + filenameTemplate := "testfile-*.txt" + if gzipCompress { + filenameTemplate += ".gz" + } + + // create a temp file + f, err := os.CreateTemp(TmpDirectory, filenameTemplate) + if err != nil { + return "", err + } else if f == nil { + return "", fmt.Errorf("nil file handle returned from os.CreateTemp") + } + defer operations.CloseFile(f) + filepath := f.Name() + + content := contentBuilder.String() + + if gzipCompress { + w := gzip.NewWriter(f) + if w == nil { + return "", fmt.Errorf("failed to open a gzip writer handle") + } + defer func() { + err := w.Close() + if err != nil { + fmt.Printf("Failed to close file %s: %v", filepath, err) + } + }() + + // write the content created above as gzip + n, err := w.Write([]byte(content)) + if err != nil { + return "", err + } else if n != contentSize { + return "", fmt.Errorf("failed to write to gzip file %s. Content-size: %d bytes, wrote = %d bytes", filepath, contentSize, n) + } + } else { + // write the content created above as text + n, err := f.WriteString(content) + if err != nil { + return "", err + } else if n != contentSize { + return "", fmt.Errorf("failed to write to text file %s. Content-size: %d bytes, wrote = %d bytes", filepath, contentSize, n) + } + } + + return filepath, nil +} + +// Downloads given gzipped GCS object (with path without 'gs://') to local disk. +// Fails if the object doesn't exist or permission to read object is not +// available. +// Uses go storage client library to download object. Use of gsutil/gcloud is not +// possible as they both always read back objects with content-encoding: gzip as +// uncompressed/decompressed irrespective of any argument passed. +func DownloadGzipGcsObjectAsCompressed(bucketName, objPathInBucket string) (string, error) { + gcsObjectPath := path.Join(setup.TestBucket(), objPathInBucket) + gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath) + if err != nil { + return "", fmt.Errorf("failed to get size of gcs object %s: %w", gcsObjectPath, err) + } + + tempfile, err := CreateLocalTempFile(1, false) + if err != nil { + return "", fmt.Errorf("failed to create tempfile for downloading gcs object: %w", err) + } + + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil || client == nil { + return "", fmt.Errorf("failed to create storage client: %w", err) + } + defer client.Close() + + bktName := setup.TestBucket() + bkt := client.Bucket(bktName) + if bkt == nil { + return "", fmt.Errorf("failed to access bucket %s: %w", bktName, err) + } + + obj := bkt.Object(objPathInBucket) + if obj == nil { + return "", fmt.Errorf("failed to access object %s from bucket %s: %w", objPathInBucket, bktName, err) + } + + obj = obj.ReadCompressed(true) + if obj == nil { + return "", fmt.Errorf("failed to access object %s from bucket %s as compressed: %w", objPathInBucket, bktName, err) + } + + r, err := obj.NewReader(ctx) + if r == nil || err != nil { + return "", fmt.Errorf("failed to read object %s from bucket %s: %w", objPathInBucket, bktName, err) + } + defer r.Close() + + gcsObjectData, err := io.ReadAll(r) + if len(gcsObjectData) < gcsObjectSize || err != nil { + return "", fmt.Errorf("failed to read object %s from bucket %s (expected read-size: %d, actual read-size: %d): %w", objPathInBucket, bktName, gcsObjectSize, len(gcsObjectData), err) + } + + err = os.WriteFile(tempfile, gcsObjectData, fs.FileMode(os.O_CREATE|os.O_WRONLY|os.O_TRUNC)) + if err != nil || client == nil { + return "", fmt.Errorf("failed to write to tempfile %s: %w", tempfile, err) + } + + return tempfile, nil +} diff --git a/tools/integration_tests/gzip/read_gzip_test.go b/tools/integration_tests/gzip/read_gzip_test.go new file mode 100644 index 0000000000..1da9b4127c --- /dev/null +++ b/tools/integration_tests/gzip/read_gzip_test.go @@ -0,0 +1,149 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for gzip objects in gcsfuse mounts. +package gzip_test + +import ( + "bytes" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/gzip/helpers" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +// Verify that the passed file exists on the GCS test-bucket and in the mounted bucket +// and its size in the mounted directory matches that of the GCS object. Also verify +// that the passed file in the mounted bucket matches the corresponding +// GCS object in content. +// GCS object. +func verifyFileSizeAndFullFileRead(t *testing.T, filename string) { + mountedFilePath := path.Join(setup.MntDir(), TestBucketPrefixPath, filename) + gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, filename) + gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath) + if err != nil { + t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err) + } + + fi, err := operations.StatFile(mountedFilePath) + if err != nil || fi == nil { + t.Fatalf("Failed to get stat info of mounted file %s: %v\n", mountedFilePath, err) + } + + if (*fi).Size() != int64(gcsObjectSize) { + t.Fatalf("Size of file mounted through gcsfuse (%s, %d) doesn't match the size of the file on GCS (%s, %d)", + mountedFilePath, (*fi).Size(), gcsObjectPath, gcsObjectSize) + } + + localCopy, err := helpers.DownloadGzipGcsObjectAsCompressed(setup.TestBucket(), path.Join(TestBucketPrefixPath, filename)) + if err != nil { + t.Fatalf("failed to download gcs object (gs:/%s) to local-disk: %v", gcsObjectPath, err) + } + + defer operations.RemoveFile(localCopy) + + diff, err := operations.DiffFiles(localCopy, mountedFilePath) + if diff != 0 { + t.Fatalf("Tempfile (%s, download of GCS object %s) didn't match the Mounted local file (%s): %v", localCopy, gcsObjectPath, mountedFilePath, err) + } +} + +// Verify that the passed file exists on the GCS test-bucket and in the mounted bucket +// and its ranged read returns the same size as the requested read size. +func verifyRangedRead(t *testing.T, filename string) { + mountedFilePath := path.Join(setup.MntDir(), TestBucketPrefixPath, filename) + + gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, filename) + gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath) + if err != nil { + t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err) + } + + readSize := int64(gcsObjectSize / 10) + readOffset := int64(readSize / 10) + f, err := os.Open(mountedFilePath) + if err != nil || f == nil { + t.Fatalf("Failed to open local mounted file %s: %v", mountedFilePath, err) + } + + localCopy, err := helpers.DownloadGzipGcsObjectAsCompressed(setup.TestBucket(), path.Join(TestBucketPrefixPath, filename)) + if err != nil { + t.Fatalf("failed to download gcs object (gs:/%s) to local-disk: %v", gcsObjectPath, err) + } + + defer operations.RemoveFile(localCopy) + + for _, offsetMultiplier := range []int64{1, 3, 5, 7, 9} { + buf1, err := operations.ReadChunkFromFile(mountedFilePath, (readSize), offsetMultiplier*(readOffset)) + if err != nil { + t.Fatalf("Failed to read mounted file %s: %v", mountedFilePath, err) + } else if buf1 == nil { + t.Fatalf("Failed to read mounted file %s: buffer returned as nul", mountedFilePath) + } + + buf2, err := operations.ReadChunkFromFile(localCopy, (readSize), offsetMultiplier*(readOffset)) + if err != nil { + t.Fatalf("Failed to read local file %s: %v", localCopy, err) + } else if buf2 == nil { + t.Fatalf("Failed to read local file %s: buffer returned as nul", localCopy) + } + + if !bytes.Equal(buf1, buf2) { + t.Fatalf("Read buffer (of size %d from offset %d) of %s doesn't match that of %s", int64(readSize), offsetMultiplier*int64(readOffset), mountedFilePath, localCopy) + } + } +} + +func TestGzipEncodedTextFileWithNoTransformSizeAndFullFileRead(t *testing.T) { + verifyFileSizeAndFullFileRead(t, TextContentWithContentEncodingWithNoTransformFilename) +} + +func TestGzipEncodedTextFileWithNoTransformRangedRead(t *testing.T) { + verifyRangedRead(t, TextContentWithContentEncodingWithNoTransformFilename) +} + +func TestGzipEncodedTextFileWithoutNoTransformSizeAndFullFileRead(t *testing.T) { + verifyFileSizeAndFullFileRead(t, TextContentWithContentEncodingWithoutNoTransformFilename) +} + +func TestGzipEncodedTextFileWithoutNoTransformRangedRead(t *testing.T) { + verifyRangedRead(t, TextContentWithContentEncodingWithoutNoTransformFilename) +} + +func TestGzipUnencodedGzipFileSizeAndFullFileRead(t *testing.T) { + verifyFileSizeAndFullFileRead(t, GzipContentWithoutContentEncodingFilename) +} + +func TestGzipUnencodedGzipFileRangedRead(t *testing.T) { + verifyRangedRead(t, GzipContentWithoutContentEncodingFilename) +} + +func TestGzipEncodedGzipFileWithNoTransformSizeAndFullFileRead(t *testing.T) { + verifyFileSizeAndFullFileRead(t, GzipContentWithContentEncodingWithNoTransformFilename) +} + +func TestGzipEncodedGzipFileWithNoTransformRangedRead(t *testing.T) { + verifyRangedRead(t, GzipContentWithContentEncodingWithNoTransformFilename) +} + +func TestGzipEncodedGzipFileWithoutNoTransformSizeAndFullFileRead(t *testing.T) { + verifyFileSizeAndFullFileRead(t, GzipContentWithContentEncodingWithoutNoTransformFilename) +} + +func TestGzipEncodedGzipFileWithoutNoTransformRangedRead(t *testing.T) { + verifyRangedRead(t, GzipContentWithContentEncodingWithoutNoTransformFilename) +} diff --git a/tools/integration_tests/gzip/write_gzip_test.go b/tools/integration_tests/gzip/write_gzip_test.go new file mode 100644 index 0000000000..d03e74c6f1 --- /dev/null +++ b/tools/integration_tests/gzip/write_gzip_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for gzip objects in gcsfuse mounts. +package gzip_test + +import ( + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/gzip/helpers" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +// Size of the overwritten content created in bytes. +const OverwittenFileSize = 1000 + +// Verify that the passed file exists on the GCS test-bucket and in the mounted bucket +// and its size in the mounted directory matches that of the GCS object. Also verify +// that the passed file in the mounted bucket matches the corresponding +// GCS object in content. +// GCS object. +func verifyFullFileOverwrite(t *testing.T, filename string) { + mountedFilePath := path.Join(setup.MntDir(), TestBucketPrefixPath, filename) + gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, filename) + gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath) + if err != nil { + t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err) + } + + fi, err := operations.StatFile(mountedFilePath) + if err != nil || fi == nil { + t.Fatalf("Failed to get stat info of mounted file %s: %v\n", mountedFilePath, err) + } + + if (*fi).Size() != int64(gcsObjectSize) { + t.Fatalf("Size of file mounted through gcsfuse (%s, %d) doesn't match the size of the file on GCS (%s, %d)", + mountedFilePath, (*fi).Size(), gcsObjectPath, gcsObjectSize) + } + + // No need to worry about gzipping the overwritten data, because it's + // expensive to invoke a gzip-writer and unnecessary in this case. + // All we are interested in testing is that the content of the overwritten + // gzip file matches in size with that of the source file that was used to + // overwrite it. + tempfile, err := helpers.CreateLocalTempFile(OverwittenFileSize, false) + if err != nil { + t.Fatalf("Failed to create local temp file for overwriting existing gzip object: %v", err) + } + defer operations.RemoveFile(tempfile) + + err = operations.CopyFileAllowOverwrite(tempfile, mountedFilePath) + if err != nil { + t.Fatalf("Failed to copy/overwrite temp file %s to existing gzip object/file %s: %v", tempfile, mountedFilePath, err) + } + + gcsObjectSize, err = operations.GetGcsObjectSize(gcsObjectPath) + if err != nil { + t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err) + } + + if gcsObjectSize != OverwittenFileSize { + t.Fatalf("Size of overwritten gcs object (%s, %d) doesn't match that of the expected overwrite size (%s, %d)", gcsObjectPath, gcsObjectSize, tempfile, OverwittenFileSize) + } +} + +func TestGzipEncodedTextFileWithNoTransformFullFileOverwrite(t *testing.T) { + verifyFullFileOverwrite(t, TextContentWithContentEncodingWithNoTransformToOverwrite) +} + +func TestGzipEncodedTextFileWithoutNoTransformFullFileOverwrite(t *testing.T) { + verifyFullFileOverwrite(t, TextContentWithContentEncodingWithoutNoTransformToOverwrite) +} + +func TestGzipUnencodedGzipFileFullFileOverwrite(t *testing.T) { + verifyFullFileOverwrite(t, GzipContentWithoutContentEncodingToOverwrite) +} + +func TestGzipEncodedGzipFileWithNoTransformFullFileOverwrite(t *testing.T) { + verifyFullFileOverwrite(t, GzipContentWithContentEncodingWithNoTransformToOverwrite) +} + +func TestGzipEncodedGzipFileWithoutNoTransformFullFileOverwrite(t *testing.T) { + verifyFullFileOverwrite(t, GzipContentWithContentEncodingWithoutNoTransformToOverwrite) +} diff --git a/tools/integration_tests/implicit_dir/implicit_dir_test.go b/tools/integration_tests/implicit_dir/implicit_dir_test.go index e710445dee..fd83b37392 100644 --- a/tools/integration_tests/implicit_dir/implicit_dir_test.go +++ b/tools/integration_tests/implicit_dir/implicit_dir_test.go @@ -29,7 +29,7 @@ const NumberOfFilesInExplicitDirInImplicitSubDir = 1 const NumberOfFilesInExplicitDirInImplicitDir = 1 func TestMain(m *testing.M) { - flags := [][]string{{"--implicit-dirs"}, {"--enable-storage-client-library=false", "--implicit-dirs"}} + flags := [][]string{{"--implicit-dirs"}} implicit_and_explicit_dir_setup.RunTestsForImplicitDirAndExplicitDir(flags, m) } diff --git a/tools/integration_tests/list_large_dir/list_large_dir_test.go b/tools/integration_tests/list_large_dir/list_large_dir_test.go index 3a5a24c58a..0189e429ac 100644 --- a/tools/integration_tests/list_large_dir/list_large_dir_test.go +++ b/tools/integration_tests/list_large_dir/list_large_dir_test.go @@ -49,5 +49,7 @@ func TestMain(m *testing.M) { successCode := static_mounting.RunTests(flags, m) + setup.RemoveBinFileCopiedForTesting() + os.Exit(successCode) } diff --git a/tools/integration_tests/operations/file_and_dir_attributes_test.go b/tools/integration_tests/operations/file_and_dir_attributes_test.go new file mode 100644 index 0000000000..66159a40d3 --- /dev/null +++ b/tools/integration_tests/operations/file_and_dir_attributes_test.go @@ -0,0 +1,87 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for file and directory attributes. +package operations_test + +import ( + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const DirAttrTest = "dirAttrTest" +const PrefixFileInDirAttrTest = "fileInDirAttrTest" +const NumberOfFilesInDirAttrTest = 2 +const BytesWrittenInFile = 14 + +func checkIfObjectAttrIsCorrect(objName string, preCreateTime time.Time, postCreateTime time.Time, byteSize int64, t *testing.T) { + oStat, err := os.Stat(objName) + + if err != nil { + t.Errorf("os.Stat error: %s, %v", objName, err) + } + statObjName := path.Join(setup.MntDir(), oStat.Name()) + if objName != statObjName { + t.Errorf("File name not matched in os.Stat, found: %s, expected: %s", statObjName, objName) + } + if (preCreateTime.After(oStat.ModTime())) || (postCreateTime.Before(oStat.ModTime())) { + t.Errorf("File modification time not in the expected time-range") + } + + if oStat.Size() != byteSize { + t.Errorf("File size is not %v bytes, found size: %d bytes", BytesWrittenInFile, oStat.Size()) + } +} + +func TestFileAttributes(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + preCreateTime := time.Now() + fileName := setup.CreateTempFile() + postCreateTime := time.Now() + + // The file size in createTempFile() is BytesWrittenInFile bytes + // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/integration_tests/util/setup/setup.go#L124 + checkIfObjectAttrIsCorrect(fileName, preCreateTime, postCreateTime, BytesWrittenInFile, t) +} + +func TestEmptyDirAttributes(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + preCreateTime := time.Now() + dirName := path.Join(setup.MntDir(), DirAttrTest) + operations.CreateDirectoryWithNFiles(0, dirName, "", t) + postCreateTime := time.Now() + + checkIfObjectAttrIsCorrect(dirName, preCreateTime, postCreateTime, 0, t) +} + +func TestNonEmptyDirAttributes(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + preCreateTime := time.Now() + dirName := path.Join(setup.MntDir(), DirAttrTest) + operations.CreateDirectoryWithNFiles(NumberOfFilesInDirAttrTest, dirName, PrefixFileInDirAttrTest, t) + postCreateTime := time.Now() + + checkIfObjectAttrIsCorrect(dirName, preCreateTime, postCreateTime, 0, t) +} diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index 01200a4a44..8f2ab5a664 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -21,7 +21,9 @@ import ( "testing" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/dynamic_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/persistent_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) @@ -84,10 +86,9 @@ const ContentInFileInDirThreeInCreateThreeLevelDirTest = "Hello world!!" func TestMain(m *testing.M) { setup.ParseSetUpFlags() - flags := [][]string{{"--enable-storage-client-library=true", "--implicit-dirs=true"}, - {"--enable-storage-client-library=false"}, - {"--implicit-dirs=true"}, - {"--implicit-dirs=false"}} + flags := [][]string{{"--implicit-dirs=true"}, + {"--implicit-dirs=false"}, + {"--experimental-enable-json-read=true", "--implicit-dirs=true"}} setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() @@ -108,10 +109,21 @@ func TestMain(m *testing.M) { successCode = only_dir_mounting.RunTests(flags, m) } + if successCode == 0 { + + successCode = persistent_mounting.RunTests(flags, m) + } + + if successCode == 0 { + successCode = dynamic_mounting.RunTests(flags, m) + } + if successCode == 0 { // Test for admin permission on test bucket. successCode = creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(flags, "objectAdmin", m) } + setup.RemoveBinFileCopiedForTesting() + os.Exit(successCode) } diff --git a/tools/integration_tests/read_large_files/concurrent_read_files_test.go b/tools/integration_tests/read_large_files/concurrent_read_files_test.go new file mode 100644 index 0000000000..1eb1c09bce --- /dev/null +++ b/tools/integration_tests/read_large_files/concurrent_read_files_test.go @@ -0,0 +1,83 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_large_files + +import ( + "bytes" + "os" + "path" + "sync" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const FileOne = "fileOne.txt" +const FileTwo = "fileTwo.txt" +const FileThree = "fileThree.txt" +const NumberOfFilesInLocalDiskForConcurrentRead = 3 + +func readFile(fileInLocalDisk string, fileInMntDir string, wg *sync.WaitGroup, t *testing.T) { + // Reduce thread count when it read the file. + defer wg.Done() + + dataInMntDirFile, err := operations.ReadFile(fileInMntDir) + if err != nil { + return + } + + dataInLocalDiskFile, err := operations.ReadFile(fileInLocalDisk) + if err != nil { + return + } + + // Compare actual content and expect content. + if bytes.Equal(dataInLocalDiskFile, dataInMntDirFile) == false { + t.Errorf("Reading incorrect file.") + } +} + +func TestReadFilesConcurrently(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + filesInLocalDisk := [NumberOfFilesInLocalDiskForConcurrentRead]string{FileOne, FileTwo, FileThree} + var filesPathInLocalDisk []string + var filesPathInMntDir []string + + for i := 0; i < NumberOfFilesInLocalDiskForConcurrentRead; i++ { + fileInLocalDisk := path.Join(os.Getenv("HOME"), filesInLocalDisk[i]) + filesPathInLocalDisk = append(filesPathInLocalDisk, fileInLocalDisk) + + file := path.Join(setup.MntDir(), filesInLocalDisk[i]) + filesPathInMntDir = append(filesPathInMntDir, file) + + createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + } + + // For waiting on threads. + var wg sync.WaitGroup + + for i := 0; i < NumberOfFilesInLocalDiskForConcurrentRead; i++ { + // Increment the WaitGroup counter. + wg.Add(1) + // Thread to read file. + go readFile(filesPathInLocalDisk[i], filesPathInMntDir[i], &wg, t) + } + + // Wait on threads to end. + wg.Wait() +} diff --git a/tools/integration_tests/read_large_files/random_read_large_file_test.go b/tools/integration_tests/read_large_files/random_read_large_file_test.go new file mode 100644 index 0000000000..35317e8e0e --- /dev/null +++ b/tools/integration_tests/read_large_files/random_read_large_file_test.go @@ -0,0 +1,59 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_large_files + +import ( + "bytes" + "math/rand" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestReadLargeFileRandomly(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) + file := path.Join(setup.MntDir(), FiveHundredMBFile) + // Create and copy the local file in mountedDirectory. + createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + + for i := 0; i < NumberOfRandomReadCalls; i++ { + offset := rand.Int63n(MaxReadableByteFromFile - MinReadableByteFromFile) + // Randomly read the data from file in mountedDirectory. + content, err := operations.ReadChunkFromFile(file, ChunkSize, offset) + if err != nil { + t.Errorf("Error in reading file: %v", err) + } + + // Read actual content from file located in local disk. + actualContent, err := operations.ReadChunkFromFile(fileInLocalDisk, ChunkSize, offset) + if err != nil { + t.Errorf("Error in reading file: %v", err) + } + + // Compare actual content and expect content. + if bytes.Equal(actualContent, content) == false { + t.Errorf("Error in reading file randomly.") + } + } + + // Removing file after testing. + operations.RemoveFile(fileInLocalDisk) +} diff --git a/tools/integration_tests/read_large_files/read_large_files_test.go b/tools/integration_tests/read_large_files/read_large_files_test.go new file mode 100644 index 0000000000..60c9199623 --- /dev/null +++ b/tools/integration_tests/read_large_files/read_large_files_test.go @@ -0,0 +1,68 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for read large files sequentially and randomly. +package read_large_files + +import ( + "log" + "os" + "strconv" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const OneMB = 1024 * 1024 +const FiveHundredMB = 500 * OneMB +const FiveHundredMBFile = "fiveHundredMBFile.txt" +const ChunkSize = 200 * OneMB +const NumberOfRandomReadCalls = 200 +const MinReadableByteFromFile = 0 +const MaxReadableByteFromFile = 500 * OneMB + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + flags := [][]string{{"--implicit-dirs"}} + + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + + if setup.TestBucket() != "" && setup.MountedDirectory() != "" { + log.Print("Both --testbucket and --mountedDirectory can't be specified at the same time.") + os.Exit(1) + } + + // Run tests for mountedDirectory only if --mountedDirectory flag is set. + setup.RunTestsForMountedDirectoryFlag(m) + + // Run tests for testBucket + setup.SetUpTestDirForTestBucketFlag() + + successCode := static_mounting.RunTests(flags, m) + + setup.RemoveBinFileCopiedForTesting() + + os.Exit(successCode) +} + +func createFileOnDiskAndCopyToMntDir(fileInLocalDisk string, fileInMntDir string, fileSize int, t *testing.T) { + setup.RunScriptForTestData("testdata/write_content_of_fix_size_in_file.sh", fileInLocalDisk, strconv.Itoa(fileSize)) + err := operations.CopyFile(fileInLocalDisk, fileInMntDir) + if err != nil { + t.Errorf("Error in copying file:%v", err) + } +} diff --git a/tools/integration_tests/read_large_files/seq_read_large_file_test.go b/tools/integration_tests/read_large_files/seq_read_large_file_test.go new file mode 100644 index 0000000000..df8a122f69 --- /dev/null +++ b/tools/integration_tests/read_large_files/seq_read_large_file_test.go @@ -0,0 +1,55 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_large_files + +import ( + "bytes" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestReadLargeFileSequentially(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + // Create file of 500 MB with random data in local disk and copy it in mntDir. + fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) + file := path.Join(setup.MntDir(), FiveHundredMBFile) + createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + + // Sequentially read the data from file. + content, err := operations.ReadFileSequentially(file, ChunkSize) + if err != nil { + t.Errorf("Error in reading file: %v", err) + } + + // Read actual content from file located in local disk. + actualContent, err := operations.ReadFile(fileInLocalDisk) + if err != nil { + t.Errorf("Error in reading file: %v", err) + } + + // Compare actual content and expect content. + if bytes.Equal(actualContent, content) == false { + t.Errorf("Error in reading file sequentially.") + } + + // Removing file after testing. + operations.RemoveFile(fileInLocalDisk) +} diff --git a/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh b/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh new file mode 100644 index 0000000000..c060f00b28 --- /dev/null +++ b/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh @@ -0,0 +1,19 @@ +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FILE_PATH=$1 +FILE_SIZE=$2 + +# It will write filesize random data in a file. +head -c $FILE_SIZE $FILE_PATH diff --git a/tools/integration_tests/readonly/readonly_test.go b/tools/integration_tests/readonly/readonly_test.go index 75df9fce8e..49917aa2bf 100644 --- a/tools/integration_tests/readonly/readonly_test.go +++ b/tools/integration_tests/readonly/readonly_test.go @@ -21,6 +21,8 @@ import ( "strings" "testing" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/creds_tests" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/persistent_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) @@ -78,8 +80,19 @@ func TestMain(m *testing.M) { setup.SetUpTestDirForTestBucketFlag() successCode := static_mounting.RunTests(flags, m) + if successCode == 0 { + successCode = persistent_mounting.RunTests(flags, m) + } + + if successCode == 0 { + // Test for viewer permission on test bucket. + successCode = creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(flags, "objectViewer", m) + } + // Delete objects from bucket after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", setup.TestBucket()) + setup.RemoveBinFileCopiedForTesting() + os.Exit(successCode) } diff --git a/tools/integration_tests/rename_dir_limit/move_dir_test.go b/tools/integration_tests/rename_dir_limit/move_dir_test.go new file mode 100644 index 0000000000..84b878c92f --- /dev/null +++ b/tools/integration_tests/rename_dir_limit/move_dir_test.go @@ -0,0 +1,395 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy +// +//of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for move directory. +package rename_dir_limit_test + +import ( + "log" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const SrcMoveDirectory = "srcMoveDir" +const SubSrcMoveDirectory = "subSrcMoveDir" +const SrcMoveFile = "srcMoveFile" +const SrcMoveFileContent = "This is from move file in srcMove directory.\n" +const DestMoveDirectory = "destMoveDir" +const DestNonEmptyMoveDirectory = "destNonEmptyMoveDirectory" +const SubDirInNonEmptyDestMoveDirectory = "subDestMoveDir" +const DestMoveDirectoryNotExist = "notExist" +const NumberOfObjectsInSrcMoveDirectory = 2 +const NumberOfObjectsInNonEmptyDestMoveDirectory = 2 +const DestEmptyMoveDirectory = "destEmptyMoveDirectory" +const EmptySrcDirectoryMoveTest = "emptySrcDirectoryMoveTest" +const NumberOfObjectsInEmptyDestMoveDirectory = 1 + +func checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDirPath string, t *testing.T) { + _, err := os.Stat(srcDirPath) + + if err == nil { + t.Errorf("Directory exist after move operation.") + } +} + +// Create below directory structure. +// srcMoveDir -- Dir +// srcMoveDir/srcMoveFile -- File +// srcMoveDir/subSrcMoveDir -- Dir +func createSrcDirectoryWithObjectsForMoveDirTest(dirPath string, t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + // testBucket/srcMoveDir + err := os.Mkdir(dirPath, setup.FilePermission_0600) + if err != nil { + t.Errorf("Mkdir at %q: %v", dirPath, err) + return + } + + // testBucket/subSrcMoveDir + subDirPath := path.Join(dirPath, SubSrcMoveDirectory) + err = os.Mkdir(subDirPath, setup.FilePermission_0600) + if err != nil { + t.Errorf("Mkdir at %q: %v", subDirPath, err) + return + } + + // testBucket/srcMoveDir/srcMoveFile + filePath := path.Join(dirPath, SrcMoveFile) + + file, err := os.Create(filePath) + if err != nil { + t.Errorf("Error in creating file %v:", err) + } + + // Closing file at the end + defer operations.CloseFile(file) + + err = operations.WriteFile(file.Name(), SrcMoveFileContent) + if err != nil { + t.Errorf("File at %v", err) + } +} + +func checkIfMovedDirectoryHasCorrectData(destDir string, t *testing.T) { + obj, err := os.ReadDir(destDir) + if err != nil { + log.Fatal(err) + } + + // Comparing number of objects in the testBucket - 2 + if len(obj) != NumberOfObjectsInSrcMoveDirectory { + t.Errorf("The number of objects in the current directory doesn't match.") + return + } + + // Comparing first object name and type + // Name - testBucket/destMoveDir/srcMoveFile, Type - file + if obj[0].Name() != SrcMoveFile || obj[0].IsDir() == true { + t.Errorf("Object Listed for bucket directory is incorrect.") + } + + // Comparing second object name and type + // Name - testBucket/destMoveDir/srcMoveDir, Type - dir + if obj[1].Name() != SubSrcMoveDirectory || obj[1].IsDir() != true { + t.Errorf("Object Listed for bucket directory is incorrect.") + } + + destFile := path.Join(destDir, SrcMoveFile) + + content, err := operations.ReadFile(destFile) + if err != nil { + t.Errorf("ReadAll: %v", err) + } + if got, want := string(content), SrcMoveFileContent; got != want { + t.Errorf("File content %q not match %q", got, want) + } +} + +// Move SrcDirectory objects in DestDirectory +// srcMoveDir -- Dir +// srcMoveDir/srcMoveFile -- File +// srcMoveDir/subSrcMoveDir -- Dir + +// destMoveDir -- Dir +// destMoveDir/srcMoveFile -- File +// destMoveDir/subSrcMoveDir -- Dir +func TestMoveDirectoryInNonExistingDirectory(t *testing.T) { + srcDir := path.Join(setup.MntDir(), SrcMoveDirectory) + + createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t) + + destDir := path.Join(setup.MntDir(), DestMoveDirectoryNotExist) + + err := operations.MoveDir(srcDir, destDir) + if err != nil { + t.Errorf("Error in moving directory: %v", err) + } + + checkIfMovedDirectoryHasCorrectData(destDir, t) + checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) +} + +// Move SrcDirectory in DestDirectory +// srcMoveDir -- Dir +// srcMoveDir/srcMoveFile -- File +// srcMoveDir/subSrcMoveDir -- Dir + +// destMoveDir -- Dir +// destMoveDir/srcMoveDir -- Dir +// destMoveDir/srcMoveDir/srcMoveFile -- File +// destMoveDir/srcMoveDir/subSrcMoveDir -- Dir +func TestMoveDirectoryInEmptyDirectory(t *testing.T) { + srcDir := path.Join(setup.MntDir(), SrcMoveDirectory) + + createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t) + + // Create below directory + // destMoveDir -- Dir + destDir := path.Join(setup.MntDir(), DestMoveDirectory) + err := os.Mkdir(destDir, setup.FilePermission_0600) + if err != nil { + t.Errorf("Error in creating directory: %v", err) + } + + err = operations.MoveDir(srcDir, destDir) + if err != nil { + t.Errorf("Error in moving directory: %v", err) + } + + obj, err := os.ReadDir(destDir) + if err != nil { + log.Fatal(err) + } + + // Check if destMoveDirectory has the correct directory copied. + // destMoveDirectory + // destMoveDirectory/srcMoveDirectory + if len(obj) != 1 || obj[0].Name() != SrcMoveDirectory || obj[0].IsDir() != true { + t.Errorf("Error in moving directory.") + return + } + + destSrc := path.Join(destDir, SrcMoveDirectory) + checkIfMovedDirectoryHasCorrectData(destSrc, t) + checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) +} + +func createDestNonEmptyDirectoryForMoveTest(t *testing.T) { + destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory) + operations.CreateDirectoryWithNFiles(0, destDir, "", t) + + destSubDir := path.Join(destDir, SubDirInNonEmptyDestMoveDirectory) + operations.CreateDirectoryWithNFiles(0, destSubDir, "", t) +} + +func TestMoveDirectoryInNonEmptyDirectory(t *testing.T) { + srcDir := path.Join(setup.MntDir(), SrcMoveDirectory) + + createSrcDirectoryWithObjectsForMoveDirTest(srcDir, t) + + // Create below directory + // destMoveDir -- Dir + destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory) + createDestNonEmptyDirectoryForMoveTest(t) + + err := operations.MoveDir(srcDir, destDir) + if err != nil { + t.Errorf("Error in moving directory: %v", err) + } + + obj, err := os.ReadDir(destDir) + if err != nil { + log.Fatal(err) + } + + // Check if destMoveDirectory has the correct directory copied. + // destMoveDirectory + // destMoveDirectory/srcMoveDirectory + // destMoveDirectory/subDestMoveDirectory + if len(obj) != NumberOfObjectsInNonEmptyDestMoveDirectory { + t.Errorf("The number of objects in the current directory doesn't match.") + return + } + + // destMoveDirectory/srcMoveDirectory - Dir + if obj[0].Name() != SrcMoveDirectory || obj[0].IsDir() != true { + t.Errorf("Error in moving directory.") + return + } + + // destMoveDirectory/subDirInNonEmptyDestMoveDirectory - Dir + if obj[1].Name() != SubDirInNonEmptyDestMoveDirectory || obj[1].IsDir() != true { + t.Errorf("Existing object affected.") + return + } + + destSrc := path.Join(destDir, SrcMoveDirectory) + checkIfMovedDirectoryHasCorrectData(destSrc, t) + checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) +} + +func checkIfMovedEmptyDirectoryHasNoData(destSrc string, t *testing.T) { + objs, err := os.ReadDir(destSrc) + if err != nil { + log.Fatal(err) + } + + if len(objs) != 0 { + t.Errorf("Directory has incorrect data.") + } +} + +// Move SrcDirectory in DestDirectory +// emptySrcDirectoryMoveTest + +// destNonEmptyMoveDirectory +// destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory + +// Output +// destNonEmptyMoveDirectory +// destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory +// destNonEmptyMoveDirectory/emptySrcDirectoryMoveTest +func TestMoveEmptyDirectoryInNonEmptyDirectory(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest) + operations.CreateDirectoryWithNFiles(0, srcDir, "", t) + + // Create below directory + // destNonEmptyMoveDirectory -- Dir + // destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory -- Dir + destDir := path.Join(setup.MntDir(), DestNonEmptyMoveDirectory) + createDestNonEmptyDirectoryForMoveTest(t) + + err := operations.MoveDir(srcDir, destDir) + if err != nil { + t.Errorf("Error in moving directory: %v", err) + } + + objs, err := os.ReadDir(destDir) + if err != nil { + log.Fatal(err) + } + + // Check if destMoveDirectory has the correct directory copied. + // destNonEmptyMoveDirectory + // destNonEmptyMoveDirectory/emptyDirectoryMoveTest - Dir + // destNonEmptyMoveDirectory/subDestMoveDirectory - Dir + if len(objs) != NumberOfObjectsInNonEmptyDestMoveDirectory { + t.Errorf("The number of objects in the current directory doesn't match.") + return + } + + // destNonEmptyMoveDirectory/srcMoveDirectory - Dir + if objs[0].Name() != EmptySrcDirectoryMoveTest || objs[0].IsDir() != true { + t.Errorf("Error in moving directory.") + return + } + + // destNonEmptyMoveDirectory/subDirInNonEmptyDestMoveDirectory - Dir + if objs[1].Name() != SubDirInNonEmptyDestMoveDirectory || objs[1].IsDir() != true { + t.Errorf("Existing object affected.") + return + } + + movDirPath := path.Join(destDir, EmptySrcDirectoryMoveTest) + checkIfMovedEmptyDirectoryHasNoData(movDirPath, t) + checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) +} + +// Move SrcDirectory in DestDirectory +// emptySrcDirectoryMoveTest + +// destEmptyMoveDirectory + +// Output +// destEmptyMoveDirectory/emptySrcDirectoryMoveTest +func TestMoveEmptyDirectoryInEmptyDirectory(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest) + operations.CreateDirectoryWithNFiles(0, srcDir, "", t) + + // Create below directory + // destMoveDir -- Dir + destDir := path.Join(setup.MntDir(), DestEmptyMoveDirectory) + operations.CreateDirectoryWithNFiles(0, destDir, "", t) + + err := operations.MoveDir(srcDir, destDir) + if err != nil { + t.Errorf("Error in moving directory: %v", err) + } + + obj, err := os.ReadDir(destDir) + if err != nil { + log.Fatal(err) + } + + // Check if destMoveDirectory has the correct directory copied. + // destEmptyMoveDirectory + // destEmptyMoveDirectory/emptyDirectoryMoveTest + if len(obj) != NumberOfObjectsInEmptyDestMoveDirectory { + t.Errorf("The number of objects in the current directory doesn't match.") + return + } + + // destEmptyMoveDirectory/srcMoveDirectory - Dir + if obj[0].Name() != EmptySrcDirectoryMoveTest || obj[0].IsDir() != true { + t.Errorf("Error in moving directory.") + return + } + + movDirPath := path.Join(destDir, EmptySrcDirectoryMoveTest) + checkIfMovedEmptyDirectoryHasNoData(movDirPath, t) + checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) +} + +// Move SrcDirectory in DestDirectory +// emptySrcDirectoryMoveTest + +// Output +// destMoveDirectoryNotExist +func TestMoveEmptyDirectoryInNonExistingDirectory(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + srcDir := path.Join(setup.MntDir(), EmptySrcDirectoryMoveTest) + operations.CreateDirectoryWithNFiles(0, srcDir, "", t) + + // destMoveDirectoryNotExist -- Dir + destDir := path.Join(setup.MntDir(), DestMoveDirectoryNotExist) + + _, err := os.Stat(destDir) + if err == nil { + t.Errorf("destMoveDirectoryNotExist directory exist.") + } + + err = operations.MoveDir(srcDir, destDir) + if err != nil { + t.Errorf("Error in moving directory: %v", err) + } + + checkIfMovedEmptyDirectoryHasNoData(destDir, t) + checkIfSrcDirectoryGetsRemovedAfterMoveOperation(srcDir, t) +} diff --git a/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go b/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go index c910de2612..83a2ad1bb7 100644 --- a/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go +++ b/tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/persistent_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) @@ -59,5 +60,11 @@ func TestMain(m *testing.M) { successCode = only_dir_mounting.RunTests(flags, m) } + if successCode == 0 { + successCode = persistent_mounting.RunTests(flags, m) + } + + setup.RemoveBinFileCopiedForTesting() + os.Exit(successCode) } diff --git a/tools/integration_tests/run_tests_mounted_directory.sh b/tools/integration_tests/run_tests_mounted_directory.sh old mode 100644 new mode 100755 index 770df4d35c..1ee8910439 --- a/tools/integration_tests/run_tests_mounted_directory.sh +++ b/tools/integration_tests/run_tests_mounted_directory.sh @@ -21,119 +21,213 @@ TEST_BUCKET_NAME=$1 MOUNT_DIR=$2 +export CGO_ENABLED=0 -# Run integration tests for operations directory with static mounting -gcsfuse --enable-storage-client-library=true --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR +# package operations +# Run test with static mounting. (flags: --implicit-dirs=true) +gcsfuse --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -gcsfuse --enable-storage-client-library=false $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs=true) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -gcsfuse --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with static mounting. (flags: --implicit-dirs=false) +gcsfuse --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -gcsfuse --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs=false) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=false +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# Run test with static mounting. (flags: --experimental-enable-json-read --implicit-dirs=true) +gcsfuse --experimental-enable-json-read --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -# Run integration tests for operations with --only-dir mounting. -gcsfuse --only-dir testDir --enable-storage-client-library=true --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --experimental-enable-json-read, --implicit-dirs=true) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=true,experimental_enable_json_read=true GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -gcsfuse --only-dir testDir --enable-storage-client-library=false $TEST_BUCKET_NAME $MOUNT_DIR +# Run tests with static mounting. (flags: --implicit-dirs=true, --only-dir testDir) +gcsfuse --only-dir testDir --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -gcsfuse --only-dir testDir --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR +# Run tests with persistent mounting. (flags: --implicit-dirs=true, --only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=true GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR +# Run tests with static mounting. (flags: --implicit-dirs=false, --only-dir testDir) gcsfuse --only-dir testDir --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -# Run integration tests for readonly directory with static mounting +# Run tests with persistent mounting. (flags: --implicit-dirs=false, --only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=false +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# Run tests with static mounting. (flags: --experimental-enable-json-read, --implicit-dirs=true, --only-dir testDir) +gcsfuse --experimental-enable-json-read --only-dir testDir --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# Run tests with persistent mounting. (flags: --experimental-enable-json-read, --implicit-dirs=true, --only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=true,experimental_enable_json_read=true +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/operations/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# package readonly +# Run tests with static mounting. (flags: --implicit-dirs=true,--o=ro) gcsfuse --o=ro --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs=true,--o=ro) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o ro,implicit_dirs=true +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +sudo umount $MOUNT_DIR + +# Run tests with static mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544) gcsfuse --file-mode=544 --dir-mode=544 --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR -# Run integration tests for readonly with --only-dir mounting. +# Run test with persistent mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o file_mode=544,dir_mode=544,implicit_dirs=true +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +sudo umount $MOUNT_DIR + +# Run tests with static mounting. (flags: --implicit-dirs=true, --o=ro, --only-dir testDir) gcsfuse --only-dir testDir --o=ro --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir sudo umount $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs=true,--o=ro,--only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o ro,only_dir=testDir,implicit_dirs=true +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +sudo umount $MOUNT_DIR + +# Run test with static mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544, --only-dir testDir) gcsfuse --only-dir testDir --file-mode=544 --dir-mode=544 --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir sudo umount $MOUNT_DIR -# Run integration tests for rename_dir_limit directory with static mounting +# Run test with persistent mounting. (flags: --implicit-dirs=true, --file-mode=544, --dir-mode=544, --only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,file_mode=544,dir_mode=544,implicit_dirs=true +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/readonly/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +sudo umount $MOUNT_DIR + +# package rename_dir_limit +# Run tests with static mounting. (flags: --rename-dir-limit=3, --implicit-dirs) gcsfuse --rename-dir-limit=3 --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR +# Run test with persistent mounting. (flags: --rename-dir-limit=3, --implicit-dirs) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o rename_dir_limit=3,implicit_dirs +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# Run tests with static mounting. (flags: --rename-dir-limit=3) gcsfuse --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -# Run integration tests for rename_dir_limit with --only-dir mounting. +# Run test with persistent mounting. (flags: --rename-dir-limit=3) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o rename_dir_limit=3 +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# Run test with static mounting. (flags: --rename-dir-limit=3, --implicit-dirs, --only-dir testDir) gcsfuse --only-dir testDir --rename-dir-limit=3 --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR +# Run test with persistent mounting . (flags: --rename-dir-limit=3, --implicit-dirs) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,rename_dir_limit=3,implicit_dirs +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# Run test with static mounting. (flags: --rename-dir-limit=3, --only-dir testDir) gcsfuse --only-dir testDir --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR sudo umount $MOUNT_DIR -# Run integration tests for implicit_dir directory with static mounting +# Run test with persistent mounting . (flags: --rename-dir-limit=3, --implicit-dirs, --only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,rename_dir_limit=3,implicit_dirs +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/rename_dir_limit/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# package implicit_dir +# Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR -gcsfuse --enable-storage-client-library=false --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR -# Run integration tests for implicit_dir with --only-dir mounting. +# Run tests with static mounting. (flags: --implicit-dirs, --only-dir testDir) gcsfuse --only-dir testDir --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir sudo umount $MOUNT_DIR -gcsfuse --only-dir testDir --enable-storage-client-library=false --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs,--only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/implicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir sudo umount $MOUNT_DIR -# Run integration tests for explicit_dir directory with static mounting -gcsfuse --enable-storage-client-library=true $TEST_BUCKET_NAME $MOUNT_DIR +# package explicit_dir +# Run tests with static mounting. (flags: --implicit-dirs=false) +gcsfuse --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR -gcsfuse --enable-storage-client-library=false $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs=false) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o implicit_dirs=false GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR -# Run integration tests for explicit_dir with --only-dir mounting. -gcsfuse --only-dir testDir --enable-storage-client-library=true $TEST_BUCKET_NAME $MOUNT_DIR +# Run tests with static mounting. (flags: --implicit-dirs=false, --only-dir testDir) +gcsfuse --only-dir testDir --implicit-dirs=false $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir sudo umount $MOUNT_DIR -gcsfuse --only-dir testDir --enable-storage-client-library=false $TEST_BUCKET_NAME $MOUNT_DIR +# Run test with persistent mounting. (flags: --implicit-dirs=false, --only-dir=testDir) +mount.gcsfuse $TEST_BUCKET_NAME $MOUNT_DIR -o only_dir=testDir,implicit_dirs=false GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/explicit_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir sudo umount $MOUNT_DIR -# Run integration tests for list_large_dir directory with static mounting +# package list_large_dir +# Run tests with static mounting. (flags: --implicit-dirs) gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/list_large_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR -# Run integration tests for list_large_dir with --only-dir mounting. -gcsfuse --only-dir testDir --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR -GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/list_large_dir/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME/testDir +# package read_large_files +# Run tests with static mounting. (flags: --implicit-dirs) +gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# package write_large_files +# Run tests with static mounting. (flags: --implicit-dirs) +gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/write_large_files/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR +sudo umount $MOUNT_DIR + +# package gzip +# Run tests with static mounting. (flags: --implicit-dirs) +gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/gzip/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR diff --git a/tools/integration_tests/util/creds_tests/creds.go b/tools/integration_tests/util/creds_tests/creds.go index f1106f7e44..3d98ea4784 100644 --- a/tools/integration_tests/util/creds_tests/creds.go +++ b/tools/integration_tests/util/creds_tests/creds.go @@ -46,7 +46,7 @@ func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(testFlagSet [][] serviceAccount := NameOfServiceAccount + "@" + id + ".iam.gserviceaccount.com" // Create service account - setup.RunScriptForTestData("../util/creds_tests/testdata/create_service_account.sh", NameOfServiceAccount, serviceAccount) + setup.RunScriptForTestData("../util/creds_tests/testdata/create_service_account.sh", NameOfServiceAccount) key_file_path := path.Join(os.Getenv("HOME"), "creds.json") @@ -56,8 +56,8 @@ func RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(testFlagSet [][] // Provide permission to service account for testing. setPermission(permission, serviceAccount) - // Revoke the permission and delete creds and service account after testing. - defer setup.RunScriptForTestData("../util/creds_tests/testdata/revoke_permission_and_delete_service_account_and_creds.sh", serviceAccount, key_file_path) + // Revoke the permission and delete creds after testing. + defer setup.RunScriptForTestData("../util/creds_tests/testdata/revoke_permission_and_creds.sh", serviceAccount, key_file_path) // Without –key-file flag and GOOGLE_APPLICATION_CREDENTIALS // This case will not get covered as gcsfuse internally authenticates from a metadata server on GCE VM. diff --git a/tools/integration_tests/util/creds_tests/testdata/create_key_file.sh b/tools/integration_tests/util/creds_tests/testdata/create_key_file.sh index a7888bce19..62ac78623d 100644 --- a/tools/integration_tests/util/creds_tests/testdata/create_key_file.sh +++ b/tools/integration_tests/util/creds_tests/testdata/create_key_file.sh @@ -14,4 +14,4 @@ KEY_FILE_PATH=$1 SERVICE_ACCOUNT=$2 -gcloud iam service-accounts keys create $KEY_FILE_PATH --iam-account=$SERVICE_ACCOUNT +gcloud iam service-accounts keys create $KEY_FILE_PATH --iam-account=$SERVICE_ACCOUNT 2>&1 | tee ~/key_id.txt diff --git a/tools/integration_tests/util/creds_tests/testdata/create_service_account.sh b/tools/integration_tests/util/creds_tests/testdata/create_service_account.sh index 5b59979f2b..f01dff9503 100644 --- a/tools/integration_tests/util/creds_tests/testdata/create_service_account.sh +++ b/tools/integration_tests/util/creds_tests/testdata/create_service_account.sh @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Create service account if does not exist. SERVICE_ACCOUNT=$1 -SERVICE_ACCOUNT_ID=$2 -# Delete service account if already exist. -gcloud iam service-accounts delete $SERVICE_ACCOUNT_ID -if [ $? -eq 1 ]; then - echo "Service account does not exist." + +gcloud iam service-accounts create $SERVICE_ACCOUNT --description="$SERVICE_ACCOUNT" --display-name="$SERVICE_ACCOUNT" 2>&1 | tee ~/output.txt +if grep "already exists within project" ~/output.txt; then + echo "Service account exist." + rm ~/output.txt +else + rm ~/output.txt + exit 1 fi -gcloud iam service-accounts create $SERVICE_ACCOUNT --description="$SERVICE_ACCOUNT" --display-name="$SERVICE_ACCOUNT" diff --git a/tools/integration_tests/util/creds_tests/testdata/revoke_permission_and_creds.sh b/tools/integration_tests/util/creds_tests/testdata/revoke_permission_and_creds.sh new file mode 100644 index 0000000000..2f84fa0cef --- /dev/null +++ b/tools/integration_tests/util/creds_tests/testdata/revoke_permission_and_creds.sh @@ -0,0 +1,34 @@ +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Delete key file after testing +SERVICE_ACCOUNT=$1 +KEY_FILE=$2 + +gcloud auth revoke $SERVICE_ACCOUNT +# Crete key file output +# e.g. Created key [KEY_ID] of type [json] as [key_file_path] for [service_account] +# Capturing third word from the file to get key-id +# e.g. Capture [KEY_ID] +if [ ! -f "~/key_id.txt" ]; then + echo "file does not exist" +fi +KEY_ID=$(cat ~/key_id.txt | cut -d " " -f 3) +# removing braces +# e.g. capture KEY_ID +KEY_ID=${KEY_ID:1:40} + +gcloud iam service-accounts keys delete $KEY_ID --iam-account=$SERVICE_ACCOUNT +rm ~/key_id.txt +rm $KEY_FILE diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go new file mode 100644 index 0000000000..11e8fee3b5 --- /dev/null +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -0,0 +1,120 @@ +//Copyright 2023 Google Inc. All Rights Reserved. +// +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +package dynamic_mounting + +import ( + "fmt" + "log" + "math/rand" + "path" + "testing" + "time" + + "cloud.google.com/go/compute/metadata" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +const PrefixBucketForDynamicMountingTest = "gcsfuse-dynamic-mounting-test-" +const Charset = "abcdefghijklmnopqrstuvwxyz0123456789" + +var seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano())) +var testBucketForDynamicMounting = PrefixBucketForDynamicMountingTest + generateRandomString(5) + +func mountGcsfuseWithDynamicMounting(flags []string) (err error) { + defaultArg := []string{"--debug_gcs", + "--debug_fs", + "--debug_fuse", + "--log-file=" + setup.LogFile(), + "--log-format=text", + setup.MntDir()} + + for i := 0; i < len(defaultArg); i++ { + flags = append(flags, defaultArg[i]) + } + + err = mounting.MountGcsfuse(setup.BinFile(), flags) + + return err +} + +func runTestsOnGivenMountedTestBucket(bucketName string, flags [][]string, rootMntDir string, m *testing.M) (successCode int) { + for i := 0; i < len(flags); i++ { + if err := mountGcsfuseWithDynamicMounting(flags[i]); err != nil { + setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) + } + + // Changing mntDir to path of bucket mounted in mntDir for testing. + mntDirOfTestBucket := path.Join(setup.MntDir(), bucketName) + + setup.SetMntDir(mntDirOfTestBucket) + + // Running tests on flags. + successCode = setup.ExecuteTest(m) + + // Currently mntDir is mntDir/bucketName. + // Unmounting can happen on rootMntDir. Changing mntDir to rootMntDir for unmounting. + setup.SetMntDir(rootMntDir) + setup.UnMountAndThrowErrorInFailure(flags[i], successCode) + } + return +} + +func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode int) { + rootMntDir := setup.MntDir() + + // In dynamic mounting all the buckets mounted in mntDir which user has permission. + // mntDir - bucket1, bucket2, bucket3, ... + // We will test on passed testBucket and one created bucket. + + // Test on testBucket + successCode = runTestsOnGivenMountedTestBucket(setup.TestBucket(), flags, rootMntDir, m) + + // Test on created bucket. + if successCode == 0 { + successCode = runTestsOnGivenMountedTestBucket(testBucketForDynamicMounting, flags, rootMntDir, m) + } + + // Setting back the original mntDir after testing. + setup.SetMntDir(rootMntDir) + return +} + +func generateRandomString(length int) string { + b := make([]byte, length) + for i := range b { + b[i] = Charset[seededRand.Intn(len(Charset))] + } + return string(b) +} + +func RunTests(flags [][]string, m *testing.M) (successCode int) { + project_id, err := metadata.ProjectID() + if err != nil { + log.Printf("Error in fetching project id: %v", err) + } + + // Create bucket with name gcsfuse-dynamic-mounting-test-xxxxx + setup.RunScriptForTestData("../util/mounting/dynamic_mounting/testdata/create_bucket.sh", testBucketForDynamicMounting, project_id) + + successCode = executeTestsForDynamicMounting(flags, m) + + log.Printf("Test log: %s\n", setup.LogFile()) + + // Deleting bucket after testing. + setup.RunScriptForTestData("../util/mounting/dynamic_mounting/testdata/delete_bucket.sh", testBucketForDynamicMounting) + + return successCode +} diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh b/tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh new file mode 100644 index 0000000000..619657a09e --- /dev/null +++ b/tools/integration_tests/util/mounting/dynamic_mounting/testdata/create_bucket.sh @@ -0,0 +1,28 @@ +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Create bucket for testing. + +BUCKET_NAME=$1 +PROJECT_ID=$2 +gcloud alpha storage buckets create gs://$BUCKET_NAME --project=$PROJECT_ID --location=us-west1 --uniform-bucket-level-access 2> ~/output.txt +if [ $? -eq 1 ]; then + if grep "HTTPError 409" ~/output.txt; then + echo "Bucket already exist." + rm ~/output.txt + else + rm ~/output.txt + exit 1 + fi +fi +rm ~/output.txt diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh b/tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh new file mode 100644 index 0000000000..d4eac2498f --- /dev/null +++ b/tools/integration_tests/util/mounting/dynamic_mounting/testdata/delete_bucket.sh @@ -0,0 +1,18 @@ +# Copyright 2023 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Delete bucket after testing. + +BUCKET_NAME=$1 + +gcloud alpha storage rm --recursive gs://$BUCKET_NAME/ diff --git a/tools/integration_tests/util/mounting/mounting.go b/tools/integration_tests/util/mounting/mounting.go index 131e2b3d6f..68db9791a0 100644 --- a/tools/integration_tests/util/mounting/mounting.go +++ b/tools/integration_tests/util/mounting/mounting.go @@ -24,9 +24,9 @@ import ( "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) -func MountGcsfuse(flags []string) error { +func MountGcsfuse(binaryFile string, flags []string) error { mountCmd := exec.Command( - setup.BinFile(), + binaryFile, flags..., ) diff --git a/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go b/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go index 0b89b74bc8..8f389ea6ff 100644 --- a/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go +++ b/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go @@ -41,7 +41,7 @@ func mountGcsfuseWithOnlyDir(flags []string, dir string) (err error) { flags = append(flags, defaultArg[i]) } - err = mounting.MountGcsfuse(flags) + err = mounting.MountGcsfuse(setup.BinFile(), flags) return err } diff --git a/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go b/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go new file mode 100644 index 0000000000..02a773db45 --- /dev/null +++ b/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go @@ -0,0 +1,90 @@ +//Copyright 2023 Google Inc. All Rights Reserved. +// +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +package persistent_mounting + +import ( + "fmt" + "log" + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +// make e.g --debug_gcs in debug_gcs +func makePersistentMountingArgs(flags []string) (args []string, err error) { + var s string + for i := range flags { + // We are already passing flags with -o flag. + s = strings.Replace(flags[i], "--o=", "", -1) + // e.g. Convert --debug_gcs to __debug_gcs + s = strings.Replace(s, "-", "_", -1) + // e.g. Convert __debug_gcs to debug_gcs + s = strings.Replace(s, "__", "", -1) + args = append(args, s) + } + return +} + +func mountGcsfuseWithPersistentMounting(flags []string) (err error) { + defaultArg := []string{setup.TestBucket(), + setup.MntDir(), + "-o", + "debug_gcs", + "-o", + "debug_fs", + "-o", + "debug_fuse", + "-o", + "log_file=" + setup.LogFile(), + "-o", + "log_format=text", + } + + persistentMountingArgs, err := makePersistentMountingArgs(flags) + if err != nil { + setup.LogAndExit("Error in converting flags for persistent mounting.") + } + + for i := 0; i < len(persistentMountingArgs); i++ { + // e.g. -o flag1, -o flag2, ... + defaultArg = append(defaultArg, "-o", persistentMountingArgs[i]) + } + + err = mounting.MountGcsfuse(setup.SbinFile(), defaultArg) + + return err +} + +func executeTestsForPersistentMounting(flags [][]string, m *testing.M) (successCode int) { + var err error + + for i := 0; i < len(flags); i++ { + if err = mountGcsfuseWithPersistentMounting(flags[i]); err != nil { + setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) + } + setup.ExecuteTestForFlagsSet(flags[i], m) + } + return +} + +func RunTests(flags [][]string, m *testing.M) (successCode int) { + successCode = executeTestsForPersistentMounting(flags, m) + + log.Printf("Test log: %s\n", setup.LogFile()) + + return successCode +} diff --git a/tools/integration_tests/util/mounting/static_mounting/static_mounting.go b/tools/integration_tests/util/mounting/static_mounting/static_mounting.go index fac0ec074a..7bfaf72f25 100644 --- a/tools/integration_tests/util/mounting/static_mounting/static_mounting.go +++ b/tools/integration_tests/util/mounting/static_mounting/static_mounting.go @@ -36,12 +36,12 @@ func mountGcsfuseWithStaticMounting(flags []string) (err error) { flags = append(flags, defaultArg[i]) } - err = mounting.MountGcsfuse(flags) + err = mounting.MountGcsfuse(setup.BinFile(), flags) return err } -func executeTestsForStatingMounting(flags [][]string, m *testing.M) (successCode int) { +func executeTestsForStaticMounting(flags [][]string, m *testing.M) (successCode int) { var err error for i := 0; i < len(flags); i++ { @@ -54,7 +54,7 @@ func executeTestsForStatingMounting(flags [][]string, m *testing.M) (successCode } func RunTests(flags [][]string, m *testing.M) (successCode int) { - successCode = executeTestsForStatingMounting(flags, m) + successCode = executeTestsForStaticMounting(flags, m) log.Printf("Test log: %s\n", setup.LogFile()) diff --git a/tools/integration_tests/util/operations/dir_operations.go b/tools/integration_tests/util/operations/dir_operations.go index 23f3928cc3..346a6470f8 100644 --- a/tools/integration_tests/util/operations/dir_operations.go +++ b/tools/integration_tests/util/operations/dir_operations.go @@ -28,12 +28,36 @@ import ( const FilePermission_0600 = 0600 const FilePermission_0777 = 0777 +func executeCommandForCopyOperation(cmd *exec.Cmd) (err error) { + err = cmd.Run() + if err != nil { + err = fmt.Errorf("Copying dir operation is failed: %v", err) + } + return +} + func CopyDir(srcDirPath string, destDirPath string) (err error) { cmd := exec.Command("cp", "--recursive", srcDirPath, destDirPath) + err = executeCommandForCopyOperation(cmd) + + return +} + +func CopyDirWithRootPermission(srcDirPath string, destDirPath string) (err error) { + cmd := exec.Command("sudo", "cp", "--recursive", srcDirPath, destDirPath) + + err = executeCommandForCopyOperation(cmd) + + return +} + +func MoveDir(srcDirPath string, destDirPath string) (err error) { + cmd := exec.Command("mv", srcDirPath, destDirPath) + err = cmd.Run() if err != nil { - err = fmt.Errorf("Copying dir operation is failed: %v", err) + err = fmt.Errorf("Moving dir operation is failed: %v", err) } return } diff --git a/tools/integration_tests/util/operations/file_operations.go b/tools/integration_tests/util/operations/file_operations.go index be34679ca6..fcff27eba9 100644 --- a/tools/integration_tests/util/operations/file_operations.go +++ b/tools/integration_tests/util/operations/file_operations.go @@ -16,46 +16,68 @@ package operations import ( + "bytes" + "crypto/rand" "fmt" "io" + "io/fs" "log" "os" "os/exec" + "strconv" + "strings" "syscall" ) -func CopyFile(srcFileName string, newFileName string) (err error) { - if _, err = os.Stat(newFileName); err == nil { - err = fmt.Errorf("Copied file %s already present", newFileName) - return +func copyFile(srcFileName, dstFileName string, allowOverwrite bool) (err error) { + if !allowOverwrite { + if _, err = os.Stat(dstFileName); err == nil { + err = fmt.Errorf("destination file %s already present", dstFileName) + return + } } source, err := os.OpenFile(srcFileName, syscall.O_DIRECT, FilePermission_0600) if err != nil { - err = fmt.Errorf("File %s opening error: %v", srcFileName, err) + err = fmt.Errorf("file %s opening error: %v", srcFileName, err) return } // Closing file at the end. defer CloseFile(source) - destination, err := os.OpenFile(newFileName, os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, FilePermission_0600) + var destination *os.File + if allowOverwrite { + destination, err = os.OpenFile(dstFileName, os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT|os.O_TRUNC, FilePermission_0600) + } else { + destination, err = os.OpenFile(dstFileName, os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, FilePermission_0600) + } + if err != nil { - err = fmt.Errorf("Copied file creation error: %v", err) + err = fmt.Errorf("copied file creation error: %v", err) return } + // Closing file at the end. defer CloseFile(destination) // File copying with io.Copy() utility. _, err = io.Copy(destination, source) if err != nil { - err = fmt.Errorf("Error in file copying: %v", err) + err = fmt.Errorf("error in file copying: %v", err) return } return } +func CopyFile(srcFileName, newFileName string) (err error) { + return copyFile(srcFileName, newFileName, false) +} + +func CopyFileAllowOverwrite(srcFileName, newFileName string) (err error) { + return copyFile(srcFileName, newFileName, true) +} + func ReadFile(filePath string) (content []byte, err error) { file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600) if err != nil { @@ -141,3 +163,301 @@ func CloseFile(file *os.File) { log.Printf("error in closing: %v", err) } } + +func RemoveFile(filePath string) { + err := os.Remove(filePath) + if err != nil { + log.Printf("Error in removing file:%v", err) + } +} + +func ReadFileSequentially(filePath string, chunkSize int64) (content []byte, err error) { + chunk := make([]byte, chunkSize) + var offset int64 = 0 + + file, err := os.OpenFile(filePath, os.O_RDONLY|syscall.O_DIRECT, FilePermission_0600) + if err != nil { + log.Printf("Error in opening file:%v", err) + } + + // Closing the file at the end. + defer CloseFile(file) + + for err != io.EOF { + var numberOfBytes int + + // Reading 200 MB chunk sequentially from the file. + numberOfBytes, err = file.ReadAt(chunk, offset) + // If the file reaches the end, write the remaining content in the buffer and return. + if err == io.EOF { + + for i := offset; i < offset+int64(numberOfBytes); i++ { + // Adding remaining bytes. + content = append(content, chunk[i-offset]) + } + err = nil + return + } + if err != nil { + return + } + // Write bytes in the buffer to compare with original content. + content = append(content, chunk...) + + // The number of bytes read is not equal to 200MB. + if int64(numberOfBytes) != chunkSize { + log.Printf("Incorrect number of bytes read from file.") + } + + // The offset will shift to read the next chunk. + offset = offset + chunkSize + } + return +} + +func WriteFileSequentially(filePath string, fileSize int64, chunkSize int64) (err error) { + file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT|os.O_CREATE, FilePermission_0600) + if err != nil { + log.Printf("Error in opening file:%v", err) + } + + // Closing file at the end. + defer CloseFile(file) + + var offset int64 = 0 + + for offset < fileSize { + // Get random chunkSize or remaining filesize data into chunk. + if (fileSize - offset) < chunkSize { + chunkSize = (fileSize - offset) + } + chunk := make([]byte, chunkSize) + _, err = rand.Read(chunk) + if err != nil { + log.Fatalf("error while generating random string: %s", err) + } + + var numberOfBytes int + + // Writes random chunkSize or remaining filesize data into file. + numberOfBytes, err = file.Write(chunk) + err = file.Sync() + if err != nil { + log.Printf("Error in syncing file:%v", err) + } + + if err != nil { + return + } + if int64(numberOfBytes) != chunkSize { + log.Fatalf("Incorrect number of bytes written in the file.") + } + + offset = offset + chunkSize + } + return +} + +func ReadChunkFromFile(filePath string, chunkSize int64, offset int64) (chunk []byte, err error) { + chunk = make([]byte, chunkSize) + + file, err := os.OpenFile(filePath, os.O_RDONLY, FilePermission_0600) + if err != nil { + log.Printf("Error in opening file:%v", err) + return + } + + f, err := os.Stat(filePath) + if err != nil { + log.Printf("Error in stating file:%v", err) + return + } + + // Closing the file at the end. + defer CloseFile(file) + + var numberOfBytes int + + // Reading chunk size randomly from the file. + numberOfBytes, err = file.ReadAt(chunk, offset) + if err == io.EOF { + err = nil + } + if err != nil { + return + } + + // The number of bytes read is not equal to 200MB. + if int64(numberOfBytes) != chunkSize && int64(numberOfBytes) != f.Size()-offset { + log.Printf("Incorrect number of bytes read from file.") + } + + return +} + +// Returns the stats of a file. +// Fails if the passed input is a directory. +func StatFile(file string) (*fs.FileInfo, error) { + fstat, err := os.Stat(file) + if err != nil { + return nil, fmt.Errorf("failed to stat input file %s: %v", file, err) + } else if fstat.IsDir() { + return nil, fmt.Errorf("input file %s is a directory", file) + } + + return &fstat, nil +} + +// Finds if two local files have identical content (equivalnt to binary diff). +// Needs (a) both files to exist, (b)read permission on both the files, (c) both +// inputs to be proper files, i.e. directories not supported. +// Compares file names first. If different, compares sizes next. +// If sizes match, then compares the contents of both the files. +// Not a good idea for very large files as it loads both the files' contents in +// the memory completely. +// Returns 0 if no error and files match. +// Returns 1 if files don't match and captures reason for mismatch in err. +// Returns 2 if any error. +func DiffFiles(filepath1, filepath2 string) (int, error) { + if filepath1 == "" || filepath2 == "" { + return 2, fmt.Errorf("one or both files being diff'ed have empty path") + } else if filepath1 == filepath2 { + return 0, nil + } + + fstat1, err := StatFile(filepath1) + if err != nil { + return 2, err + } + + fstat2, err := StatFile(filepath2) + if err != nil { + return 2, err + } + + file1size := (*fstat1).Size() + file2size := (*fstat2).Size() + if file1size != file2size { + return 1, fmt.Errorf("files don't match in size: %s (%d bytes), %s (%d bytes)", filepath1, file1size, filepath2, file2size) + } + + bytes1, err := ReadFile(filepath1) + if err != nil || bytes1 == nil { + return 2, fmt.Errorf("failed to read file %s", filepath1) + } else if int64(len(bytes1)) != file1size { + return 2, fmt.Errorf("failed to completely read file %s", filepath1) + } + + bytes2, err := ReadFile(filepath2) + if err != nil || bytes2 == nil { + return 2, fmt.Errorf("failed to read file %s", filepath2) + } else if int64(len(bytes2)) != file2size { + return 2, fmt.Errorf("failed to completely read file %s", filepath2) + } + + if !bytes.Equal(bytes1, bytes2) { + return 1, fmt.Errorf("files don't match in content: %s, %s", filepath1, filepath2) + } + + return 0, nil +} + +// Executes any given tool (e.g. gsutil/gcloud) with given args. +func executeToolCommandf(tool string, format string, args ...any) ([]byte, error) { + cmdArgs := tool + " " + fmt.Sprintf(format, args...) + cmd := exec.Command("/bin/bash", "-c", cmdArgs) + + var stdout bytes.Buffer + var stderr bytes.Buffer + + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return stdout.Bytes(), fmt.Errorf("failed command '%s': %v, %s", cmdArgs, err, stderr.String()) + } + + return stdout.Bytes(), nil +} + +// Executes any given gsutil command with given args. +func executeGsutilCommandf(format string, args ...any) ([]byte, error) { + return executeToolCommandf("gsutil", format, args...) +} + +// Returns size of a give GCS object with path (without 'gs://'). +// Fails if the object doesn't exist or permission to read object's metadata is not +// available. +// Uses 'gsutil du -s gs://gcsObjPath'. +// Alternative 'gcloud storage du -s gs://gcsObjPath', but it doesn't work on kokoro VM. +func GetGcsObjectSize(gcsObjPath string) (int, error) { + stdout, err := executeGsutilCommandf("du -s gs://%s", gcsObjPath) + if err != nil { + return 0, err + } + + // The above gcloud command returns output in the following format: + // + // So, we need to pick out only the first string before ' '. + gcsObjectSize, err := strconv.Atoi(strings.TrimSpace(strings.Split(string(stdout), " ")[0])) + if err != nil { + return gcsObjectSize, err + } + + return gcsObjectSize, nil +} + +// Downloads given GCS object (with path without 'gs://') to localPath. +// Fails if the object doesn't exist or permission to read object is not +// available. +// Uses 'gsutil cp gs://gcsObjPath localPath' +// Alternative 'gcloud storage cp gs://gcsObjPath localPath' but it doesn't work on kokoro VM. +func DownloadGcsObject(gcsObjPath, localPath string) error { + _, err := executeGsutilCommandf("cp gs://%s %s", gcsObjPath, localPath) + if err != nil { + return err + } + + return nil +} + +// Uploads given local file to GCS object (with path without 'gs://'). +// Fails if the file doesn't exist or permission to write to object/bucket is not +// available. +// Uses 'gsutil cp localPath gs://gcsObjPath' +// Alternative 'gcloud storage cp localPath gs://gcsObjPath' but it doesn't work on kokoro VM. +func UploadGcsObject(localPath, gcsObjPath string, uploadGzipEncoded bool) error { + var err error + if uploadGzipEncoded { + // Using gsutil instead of `gcloud alpha` here as `gcloud alpha` + // option `-Z` isn't supported on the kokoro VM. + _, err = executeGsutilCommandf("cp -Z %s gs://%s", localPath, gcsObjPath) + } else { + _, err = executeGsutilCommandf("cp %s gs://%s", localPath, gcsObjPath) + } + + return err +} + +// Deletes a given GCS object (with path without 'gs://'). +// Fails if the object doesn't exist or permission to delete object is not +// available. +// Uses 'gsutil rm gs://gcsObjPath' +// Alternative 'gcloud storage rm gs://gcsObjPath' but it doesn't work on kokoro VM. +func DeleteGcsObject(gcsObjPath string) error { + _, err := executeGsutilCommandf("rm gs://%s", gcsObjPath) + return err +} + +// Clears cache-control attributes on given GCS object (with path without 'gs://'). +// Fails if the file doesn't exist or permission to modify object's metadata is not +// available. +// Uses 'gsutil setmeta -h "Cache-Control:" gs://' +// Preferred approach is 'gcloud storage objects update gs://gs://gcsObjPath --cache-control=' ' ' but it doesn't work on kokoro VM. +func ClearCacheControlOnGcsObject(gcsObjPath string) error { + // Using gsutil instead of `gcloud alpha` here as `gcloud alpha` + // implementation for updating object metadata is missing on the kokoro VM. + _, err := executeGsutilCommandf("setmeta -h \"Cache-Control:\" gs://%s ", gcsObjPath) + return err +} diff --git a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go index 77fee597c3..34e8664c42 100644 --- a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go +++ b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go @@ -20,6 +20,7 @@ import ( "path" "testing" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/persistent_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" @@ -58,6 +59,12 @@ func RunTestsForImplicitDirAndExplicitDir(flags [][]string, m *testing.M) { successCode := static_mounting.RunTests(flags, m) + if successCode == 0 { + successCode = persistent_mounting.RunTests(flags, m) + } + + setup.RemoveBinFileCopiedForTesting() + os.Exit(successCode) } diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index 2d81f503d7..9463b5617b 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -38,10 +38,11 @@ const BufferSize = 100 const FilePermission_0600 = 0600 var ( - binFile string - logFile string - testDir string - mntDir string + binFile string + logFile string + testDir string + mntDir string + sbinFile string ) // Run the shell script to prepare the testData in the specified bucket. @@ -82,6 +83,10 @@ func BinFile() string { return binFile } +func SbinFile() string { + return sbinFile +} + func SetTestDir(testDirValue string) { testDir = testDirValue } @@ -142,10 +147,20 @@ func SetUpTestDir() error { return fmt.Errorf("BuildGcsfuse(%q): %w\n", TestDir(), err) } binFile = path.Join(TestDir(), "bin/gcsfuse") + sbinFile = path.Join(TestDir(), "sbin/mount.gcsfuse") + + // mount.gcsfuse will find gcsfuse executable in mentioned locations. + // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/mount_gcsfuse/find.go#L59 + // Copying the executable to /usr/local/bin + err := operations.CopyDirWithRootPermission(binFile, "/usr/local/bin") + if err != nil { + log.Printf("Error in copying bin file:%v", err) + } } else { // when testInstalledPackage flag is set, gcsfuse is preinstalled on the // machine. Hence, here we are overwriting binFile to gcsfuse. binFile = "gcsfuse" + sbinFile = "mount.gcsfuse" } logFile = path.Join(TestDir(), "gcsfuse.log") mntDir = path.Join(TestDir(), "mnt") @@ -157,6 +172,17 @@ func SetUpTestDir() error { return nil } +// Removing bin file after testing. +func RemoveBinFileCopiedForTesting() { + if !TestInstalledPackage() { + cmd := exec.Command("sudo", "rm", "/usr/local/bin/gcsfuse") + err := cmd.Run() + if err != nil { + log.Printf("Error in removing file:%v", err) + } + } +} + func UnMount() error { fusermount, err := exec.LookPath("fusermount") if err != nil { @@ -169,18 +195,14 @@ func UnMount() error { return nil } -func executeTest(m *testing.M) (successCode int) { +func ExecuteTest(m *testing.M) (successCode int) { successCode = m.Run() return successCode } -func ExecuteTestForFlagsSet(flags []string, m *testing.M) (successCode int) { - var err error - - successCode = executeTest(m) - - err = UnMount() +func UnMountAndThrowErrorInFailure(flags []string, successCode int) { + err := UnMount() if err != nil { LogAndExit(fmt.Sprintf("Error in unmounting bucket: %v", err)) } @@ -191,6 +213,13 @@ func ExecuteTestForFlagsSet(flags []string, m *testing.M) (successCode int) { log.Print("Test Fails on " + f) return } +} + +func ExecuteTestForFlagsSet(flags []string, m *testing.M) (successCode int) { + successCode = ExecuteTest(m) + + UnMountAndThrowErrorInFailure(flags, successCode) + return } @@ -216,7 +245,7 @@ func RunTestsForMountedDirectoryFlag(m *testing.M) { // Execute tests for the mounted directory. if *mountedDirectory != "" { mntDir = *mountedDirectory - successCode := executeTest(m) + successCode := ExecuteTest(m) os.Exit(successCode) } } diff --git a/tools/integration_tests/write_large_files/write_large_files_test.go b/tools/integration_tests/write_large_files/write_large_files_test.go new file mode 100644 index 0000000000..21b9f1c96d --- /dev/null +++ b/tools/integration_tests/write_large_files/write_large_files_test.go @@ -0,0 +1,50 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for write large files sequentially and randomly. +package write_large_files + +import ( + "log" + "os" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + flags := [][]string{{"--implicit-dirs"}} + + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + + if setup.TestBucket() != "" && setup.MountedDirectory() != "" { + log.Print("Both --testbucket and --mountedDirectory can't be specified at the same time.") + os.Exit(1) + } + + // Run tests for mountedDirectory only if --mountedDirectory flag is set. + setup.RunTestsForMountedDirectoryFlag(m) + + // Run tests for testBucket + setup.SetUpTestDirForTestBucketFlag() + + successCode := static_mounting.RunTests(flags, m) + + setup.RemoveBinFileCopiedForTesting() + + os.Exit(successCode) +} diff --git a/tools/integration_tests/operations/file_attributes_test.go b/tools/integration_tests/write_large_files/write_one_large_file_sequentially_test.go similarity index 51% rename from tools/integration_tests/operations/file_attributes_test.go rename to tools/integration_tests/write_large_files/write_one_large_file_sequentially_test.go index 8e591a3439..805a3aa585 100644 --- a/tools/integration_tests/operations/file_attributes_test.go +++ b/tools/integration_tests/write_large_files/write_one_large_file_sequentially_test.go @@ -12,40 +12,40 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Provides integration tests for file attributes. -package operations_test +package write_large_files import ( "os" "path" "testing" - "time" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) -func TestFileAttributes(t *testing.T) { +const FiveHundredMB = 500 * 1024 * 1024 +const FiveHundredMBFile = "fiveHundredMBFile.txt" +const ChunkSize = 20 * 1024 * 1024 + +func TestWriteLargeFileSequentially(t *testing.T) { // Clean the mountedDirectory before running test. setup.CleanMntDir() - preCreateTime := time.Now() - fileName := setup.CreateTempFile() - postCreateTime := time.Now() - - fStat, err := os.Stat(fileName) + filePath := path.Join(setup.MntDir(), FiveHundredMBFile) + // Sequentially read the data from file. + err := operations.WriteFileSequentially(filePath, FiveHundredMB, ChunkSize) if err != nil { - t.Errorf("os.Stat error: %s, %v", fileName, err) - } - statFileName := path.Join(setup.MntDir(), fStat.Name()) - if fileName != statFileName { - t.Errorf("File name not matched in os.Stat, found: %s, expected: %s", statFileName, fileName) + t.Errorf("Error in writing file: %v", err) } - if (preCreateTime.After(fStat.ModTime())) || (postCreateTime.Before(fStat.ModTime())) { - t.Errorf("File modification time not in the expected time-range") + + // Check if 500MB data written in the file. + fStat, err := os.Stat(filePath) + if err != nil { + t.Errorf("Error in stating file:%v", err) } - // The file size in createTempFile() is 14 bytes - if fStat.Size() != 14 { - t.Errorf("File size is not 14 bytes, found size: %d bytes", fStat.Size()) + + if fStat.Size() != FiveHundredMB { + t.Errorf("Expecred file size %v found %d", FiveHundredMB, fStat.Size()) } } diff --git a/tools/mount_gcsfuse/main.go b/tools/mount_gcsfuse/main.go index a5d7e75a20..4789b4dc94 100644 --- a/tools/mount_gcsfuse/main.go +++ b/tools/mount_gcsfuse/main.go @@ -83,9 +83,9 @@ func makeGcsfuseArgs( case "implicit_dirs", "foreground", "experimental_local_file_cache", - "enable_storage_client_library", "reuse_token_from_url", - "enable_nonexistent_type_cache": + "enable_nonexistent_type_cache", + "experimental_enable_json_read": if value == "" { value = "true" } @@ -120,8 +120,8 @@ func makeGcsfuseArgs( "experimental_opentelemetry_collector_address", "log_format", "log_file", - "endpoint", - "config_file": + "config_file", + "custom_endpoint": args = append(args, "--"+strings.Replace(name, "_", "-", -1), value) // Special case: support mount-like formatting for gcsfuse debug flags. diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 2eef93278a..535416c056 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -17,7 +17,7 @@ # Copy the gcsfuse packages to the host: # > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output -FROM golang:1.20.4 as builder +FROM golang:1.20.5 as builder RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler @@ -31,6 +31,9 @@ ENV GCSFUSE_PATH "$GOPATH/src/$GCSFUSE_REPO" RUN go get -d ${GCSFUSE_REPO} WORKDIR ${GCSFUSE_PATH} +ARG DEBEMAIL="gcs-fuse-maintainers@google.com" +ARG DEBFULLNAME="GCSFuse Team" + # Build Arg for building through a particular branch/commit. By default, it uses # the tag corresponding to passed GCSFUSE VERSION ARG BRANCH_NAME="v${GCSFUSE_VERSION}" @@ -39,26 +42,36 @@ RUN git checkout "${BRANCH_NAME}" # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile -ARG GCSFUSE_BIN="/gcsfuse" +ARG ARCHITECTURE="amd64" +ARG GCSFUSE_BIN="/gcsfuse_${GCSFUSE_VERSION}_${ARCHITECTURE}" +ARG GCSFUSE_DOC="${GCSFUSE_BIN}/usr/share/doc/gcsfuse" WORKDIR ${GOPATH} RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin +# Creating structure for debian package as we are using 'dpkg-deb --build' to create debian package +RUN mkdir -p ${GCSFUSE_BIN}/DEBIAN && cp $GOPATH/src/$GCSFUSE_REPO/DEBIAN/* ${GCSFUSE_BIN}/DEBIAN/ +RUN mkdir -p ${GCSFUSE_DOC} +RUN mv ${GCSFUSE_BIN}/DEBIAN/copyright ${GCSFUSE_DOC} && \ + mv ${GCSFUSE_BIN}/DEBIAN/changelog ${GCSFUSE_DOC} && \ + mv ${GCSFUSE_BIN}/DEBIAN/gcsfuse-docs.docs ${GCSFUSE_DOC} +# Update gcsfuse version in changelog and control file +RUN sed -i "1s/.*/gcsfuse (${GCSFUSE_VERSION}) stable; urgency=medium/" ${GCSFUSE_DOC}/changelog && \ + sed -i "1s/.*/Version: ${GCSFUSE_VERSION}/" ${GCSFUSE_BIN}/DEBIAN/control +# Compress changelog as required by lintian +RUN gzip -9 -n ${GCSFUSE_DOC}/changelog +# Strip unneeded from binaries as required by lintian +RUN strip --strip-unneeded ${GCSFUSE_BIN}/usr/bin/gcsfuse && \ + strip --strip-unneeded ${GCSFUSE_BIN}/sbin/mount.gcsfuse + ARG GCSFUSE_PKG="/packages" RUN mkdir -p ${GCSFUSE_PKG} WORKDIR ${GCSFUSE_PKG} -RUN fpm \ - -s dir \ - -t deb \ - -n gcsfuse \ - -C ${GCSFUSE_BIN} \ - -v ${GCSFUSE_VERSION} \ - -d fuse \ - --vendor "" \ - --url "https://$GCSFUSE_REPO" \ - --description "A user-space file system for Google Cloud Storage." +# Build the package +RUN dpkg-deb --build ${GCSFUSE_BIN} +RUN mv ${GCSFUSE_BIN}.deb . RUN fpm \ -s dir \ -t rpm \ @@ -67,6 +80,7 @@ RUN fpm \ -v ${GCSFUSE_VERSION} \ -d fuse \ --rpm-digest sha256 \ + --license Apache-2.0 \ --vendor "" \ --url "https://$GCSFUSE_REPO" \ --description "A user-space file system for Google Cloud Storage."