diff --git a/storage/grpc_client.go b/storage/grpc_client.go index 272315adb8eb..352aea596b66 100644 --- a/storage/grpc_client.go +++ b/storage/grpc_client.go @@ -17,7 +17,6 @@ package storage import ( "context" "os" - "strings" gapic "cloud.google.com/go/storage/internal/apiv2" "google.golang.org/api/option" @@ -56,9 +55,7 @@ func defaultGRPCOptions() []option.ClientOption { if host := os.Getenv("STORAGE_EMULATOR_HOST_GRPC"); host != "" { // Strip the scheme from the emulator host. WithEndpoint does not take a // scheme for gRPC. - if strings.Contains(host, "://") { - host = strings.SplitN(host, "://", 2)[1] - } + host = stripScheme(host) defaults = append(defaults, option.WithEndpoint(host), diff --git a/storage/storage.go b/storage/storage.go index 5f650dab4aec..f4fbf5bb9b6a 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -276,10 +276,18 @@ type bucketBoundHostname struct { } func (s pathStyle) host(bucket string) string { + if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" { + return stripScheme(host) + } + return "storage.googleapis.com" } func (s virtualHostedStyle) host(bucket string) string { + if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" { + return bucket + "." + stripScheme(host) + } + return bucket + ".storage.googleapis.com" } @@ -327,6 +335,14 @@ func BucketBoundHostname(hostname string) URLStyle { return bucketBoundHostname{hostname: hostname} } +// Strips the scheme from a host if it contains it +func stripScheme(host string) string { + if strings.Contains(host, "://") { + host = strings.SplitN(host, "://", 2)[1] + } + return host +} + // SignedURLOptions allows you to restrict the access to the signed URL. type SignedURLOptions struct { // GoogleAccessID represents the authorizer of the signed URL generation. @@ -823,7 +839,7 @@ func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) { } encoded := base64.StdEncoding.EncodeToString(b) u.Scheme = "https" - u.Host = "storage.googleapis.com" + u.Host = PathStyle().host(bucket) q := u.Query() q.Set("GoogleAccessId", opts.GoogleAccessID) q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) diff --git a/storage/storage_test.go b/storage/storage_test.go index aceadf470c51..5d6dd065dad3 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -343,6 +343,162 @@ func TestSignedURLV4(t *testing.T) { } } +// TestSignedURL_EmulatorHost tests that SignedURl respects the host set in +// STORAGE_EMULATOR_HOST +func TestSignedURL_EmulatorHost(t *testing.T) { + expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00") + bucketName := "bucket-name" + objectName := "obj-name" + + emulatorHost := os.Getenv("STORAGE_EMULATOR_HOST") + defer os.Setenv("STORAGE_EMULATOR_HOST", emulatorHost) + + tests := []struct { + desc string + emulatorHost string + now time.Time + opts *SignedURLOptions + // Note for future implementors: X-Goog-Signature generated by having + // the client run through its algorithm with pre-defined input and copy + // pasting the output. These tests are not great for testing whether + // the right signature is calculated - instead we rely on the backend + // and integration tests for that. + want string + }{ + { + desc: "SignURLV4 creates link to resources in emulator", + emulatorHost: "localhost:9000", + now: expires.Add(-24 * time.Hour), + opts: &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "POST", + Expires: expires, + Scheme: SigningSchemeV4, + Insecure: true, + }, + want: "http://localhost:9000/" + bucketName + "/" + objectName + + "?X-Goog-Algorithm=GOOG4-RSA-SHA256" + + "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" + + "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" + + "&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" + + "&X-Goog-SignedHeaders=host", + }, + { + desc: "using SigningSchemeV2", + emulatorHost: "localhost:9000", + now: expires.Add(-24 * time.Hour), + opts: &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "POST", + Expires: expires, + Scheme: SigningSchemeV2, + }, + want: "https://localhost:9000/" + bucketName + "/" + objectName + + "?Expires=1033570800" + + "&GoogleAccessId=xxx%40clientid" + + "&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D", + }, + { + desc: "using VirtualHostedStyle", + emulatorHost: "localhost:8000", + now: expires.Add(-24 * time.Hour), + opts: &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "POST", + Expires: expires, + Scheme: SigningSchemeV4, + Style: VirtualHostedStyle(), + Insecure: true, + }, + want: "http://" + bucketName + ".localhost:8000/" + objectName + + "?X-Goog-Algorithm=GOOG4-RSA-SHA256" + + "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" + + "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" + + "&X-Goog-Signature=35e0b9d33901a2518956821175f88c2c4eb3f4461b725af74b37c36d23f8bbe927558ac57b0be40d345f20bca55ba0652d38b7a620f8da68d4f733706ad104da468c3a039459acf35f3022e388760cd49893c998c33fe3ccc8c022d7034ab98bdbdcac4b680bb24ae5ed586a42ee9495a873ffc484e297853a8a3892d0d6385c980cb7e3c5c8bdd4939b4c17105f10fe8b5b9744017bf59431ff176c1550ae1c64ddd6628096eb6895c97c5da4d850aca72c14b7f5018c15b34d4b00ec63ff2ccb688ddbef2d32648e247ffd0137498080f320f293eb811a94fb526227324bbbd01335446388797803e67d802f97b52565deba3d2387ecabf4f3094662236017" + + "&X-Goog-SignedHeaders=host", + }, + { + desc: "using BucketBoundHostname", + emulatorHost: "localhost:8000", + now: expires.Add(-24 * time.Hour), + opts: &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "POST", + Expires: expires, + Scheme: SigningSchemeV4, + Style: BucketBoundHostname("myhost"), + }, + want: "https://" + "myhost/" + objectName + + "?X-Goog-Algorithm=GOOG4-RSA-SHA256" + + "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" + + "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" + + "&X-Goog-Signature=15fe19f6c61bcbdbd6473c32f2bec29caa8a5fa3b2ce32cfb5329a71edaa0d4e5ffe6469f32ed4c23ca2fbed3882fdf1ed107c6a98c2c4995dda6036c64bae51e6cb542c353618f483832aa1f3ef85342ddadd69c13ad4c69fd3f573ea5cf325a58056e3d5a37005217662af63b49fef8688de3c5c7a2f7b43651a030edd0813eb7f7713989a4c29a8add65133ce652895fea9de7dbc6248ee11b4d7c6c1e152df87700100e896e544ba8eeea96584078f56e699665140b750e90550b9b79633f4e7c8409efa807be5670d6e987eeee04a4180be9b9e30bb8557597beaf390a3805cc602c87a3e34800f8bc01449c3dd10ac2f2263e55e55b91e445052548d5e" + + "&X-Goog-SignedHeaders=host", + }, + { + desc: "emulator host specifies scheme", + emulatorHost: "https://localhost:6000", + now: expires.Add(-24 * time.Hour), + opts: &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "POST", + Expires: expires, + Scheme: SigningSchemeV4, + Insecure: true, + }, + want: "http://localhost:6000/" + bucketName + "/" + objectName + + "?X-Goog-Algorithm=GOOG4-RSA-SHA256" + + "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" + + "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" + + "&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" + + "&X-Goog-SignedHeaders=host", + }, + { + desc: "emulator host specifies scheme using SigningSchemeV2", + emulatorHost: "https://localhost:8000", + now: expires.Add(-24 * time.Hour), + opts: &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "POST", + Expires: expires, + Scheme: SigningSchemeV2, + }, + want: "https://localhost:8000/" + bucketName + "/" + objectName + + "?Expires=1033570800" + + "&GoogleAccessId=xxx%40clientid" + + "&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D", + }, + } + oldUTCNow := utcNow + defer func() { + utcNow = oldUTCNow + }() + + for _, test := range tests { + t.Run(test.desc, func(s *testing.T) { + utcNow = func() time.Time { + return test.now + } + + os.Setenv("STORAGE_EMULATOR_HOST", test.emulatorHost) + + got, err := SignedURL(bucketName, objectName, test.opts) + if err != nil { + s.Fatal(err) + } + if got != test.want { + s.Fatalf("\n\tgot:\t%v\n\twant:\t%v", got, test.want) + } + }) + } +} + func TestSignedURL_MissingOptions(t *testing.T) { now, _ := time.Parse(time.RFC3339, "2002-10-01T00:00:00-05:00") expires, _ := time.Parse(time.RFC3339, "2002-10-15T00:00:00-05:00")