Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(storage): respect STORAGE_EMULATOR_HOST in signedURL #5673

Merged
merged 10 commits into from
Mar 11, 2022
Merged

fix(storage): respect STORAGE_EMULATOR_HOST in signedURL #5673

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0bec397
fix: Respect STORAGE_EMULATOR_HOST in signedURL
ChrisBr Jan 29, 2022
7f4327a
update the path for virtual hosted, signv2 and add tests
BrennaEpp Feb 15, 2022
6950b84
add test cases for scheme
BrennaEpp Feb 15, 2022
dc57871
Merge branch 'main' into storage-emulator
BrennaEpp Feb 18, 2022
0921f49
suggestions
BrennaEpp Feb 24, 2022
9540741
Merge branch 'main' into storage-emulator
BrennaEpp Feb 24, 2022
25e3d88
Merge branch 'main' into storage-emulator
tritone Mar 3, 2022
50abe4c
Merge branch 'main' into storage-emulator
BrennaEpp Mar 3, 2022
433b2e4
Merge branch 'main' into storage-emulator
BrennaEpp Mar 10, 2022
796e0a1
Merge branch 'main' into storage-emulator
BrennaEpp Mar 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion storage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,10 +309,26 @@ type bucketBoundHostname struct {
}

func (s pathStyle) host(bucket string) string {
if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" {
BrennaEpp marked this conversation as resolved.
Show resolved Hide resolved
// Strip the scheme from the emulator host
if strings.Contains(host, "://") {
host = strings.SplitN(host, "://", 2)[1]
}
return host
}

return "storage.googleapis.com"
}

func (s virtualHostedStyle) host(bucket string) string {
if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" {
// Strip the scheme from the emulator host
if strings.Contains(host, "://") {
host = strings.SplitN(host, "://", 2)[1]
}
return bucket + "." + host
}

return bucket + ".storage.googleapis.com"
}

Expand All @@ -338,12 +354,16 @@ func (s bucketBoundHostname) path(bucket, object string) string {

// PathStyle is the default style, and will generate a URL of the form
// "storage.googleapis.com/<bucket-name>/<object-name>".
// If STORAGE_EMULATOR_HOST is set, will generate a URL of the form
BrennaEpp marked this conversation as resolved.
Show resolved Hide resolved
// "STORAGE_EMULATOR_HOST/<bucket-name>/<object-name>".
func PathStyle() URLStyle {
return pathStyle{}
}

// VirtualHostedStyle generates a URL relative to the bucket's virtual
// hostname, e.g. "<bucket-name>.storage.googleapis.com/<object-name>".
// If STORAGE_EMULATOR_HOST is set, will generate a URL of the form
// "<bucket-name>.STORAGE_EMULATOR_HOST/<object-name>".
func VirtualHostedStyle() URLStyle {
return virtualHostedStyle{}
}
Expand Down Expand Up @@ -856,7 +876,7 @@ func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) {
}
encoded := base64.StdEncoding.EncodeToString(b)
u.Scheme = "https"
u.Host = "storage.googleapis.com"
u.Host = PathStyle().host(bucket)
q := u.Query()
q.Set("GoogleAccessId", opts.GoogleAccessID)
q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix()))
Expand Down
156 changes: 156 additions & 0 deletions storage/storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,162 @@ func TestSignedURLV4(t *testing.T) {
}
}

// TestSignedURL_EmulatorHost tests that SignedURl respects the host set in
// STORAGE_EMULATOR_HOST
func TestSignedURL_EmulatorHost(t *testing.T) {
expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00")
bucketName := "bucket-name"
objectName := "obj-name"

emulatorHost := os.Getenv("STORAGE_EMULATOR_HOST")
defer os.Setenv("STORAGE_EMULATOR_HOST", emulatorHost)

tests := []struct {
desc string
emulatorHost string
now time.Time
opts *SignedURLOptions
// Note for future implementors: X-Goog-Signature generated by having
// the client run through its algorithm with pre-defined input and copy
// pasting the output. These tests are not great for testing whether
// the right signature is calculated - instead we rely on the backend
// and integration tests for that.
want string
}{
{
desc: "SignURLV4 creates link to resources in emulator",
emulatorHost: "localhost:9000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Insecure: true,
},
want: "http://localhost:9000/" + bucketName + "/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "using SigningSchemeV2",
emulatorHost: "localhost:9000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV2,
},
want: "https://localhost:9000/" + bucketName + "/" + objectName +
"?Expires=1033570800" +
"&GoogleAccessId=xxx%40clientid" +
"&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D",
},
{
desc: "using VirtualHostedStyle",
emulatorHost: "localhost:8000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Style: VirtualHostedStyle(),
Insecure: true,
},
want: "http://" + bucketName + ".localhost:8000/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=35e0b9d33901a2518956821175f88c2c4eb3f4461b725af74b37c36d23f8bbe927558ac57b0be40d345f20bca55ba0652d38b7a620f8da68d4f733706ad104da468c3a039459acf35f3022e388760cd49893c998c33fe3ccc8c022d7034ab98bdbdcac4b680bb24ae5ed586a42ee9495a873ffc484e297853a8a3892d0d6385c980cb7e3c5c8bdd4939b4c17105f10fe8b5b9744017bf59431ff176c1550ae1c64ddd6628096eb6895c97c5da4d850aca72c14b7f5018c15b34d4b00ec63ff2ccb688ddbef2d32648e247ffd0137498080f320f293eb811a94fb526227324bbbd01335446388797803e67d802f97b52565deba3d2387ecabf4f3094662236017" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "using BucketBoundHostname",
emulatorHost: "localhost:8000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Style: BucketBoundHostname("myhost"),
},
want: "https://" + "myhost/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=15fe19f6c61bcbdbd6473c32f2bec29caa8a5fa3b2ce32cfb5329a71edaa0d4e5ffe6469f32ed4c23ca2fbed3882fdf1ed107c6a98c2c4995dda6036c64bae51e6cb542c353618f483832aa1f3ef85342ddadd69c13ad4c69fd3f573ea5cf325a58056e3d5a37005217662af63b49fef8688de3c5c7a2f7b43651a030edd0813eb7f7713989a4c29a8add65133ce652895fea9de7dbc6248ee11b4d7c6c1e152df87700100e896e544ba8eeea96584078f56e699665140b750e90550b9b79633f4e7c8409efa807be5670d6e987eeee04a4180be9b9e30bb8557597beaf390a3805cc602c87a3e34800f8bc01449c3dd10ac2f2263e55e55b91e445052548d5e" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "emulator host specifies scheme",
emulatorHost: "https://localhost:6000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV4,
Insecure: true,
},
want: "http://localhost:6000/" + bucketName + "/" + objectName +
"?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
"&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
"&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
"&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" +
"&X-Goog-SignedHeaders=host",
},
{
desc: "emulator host specifies scheme using SigningSchemeV2",
emulatorHost: "https://localhost:8000",
now: expires.Add(-24 * time.Hour),
opts: &SignedURLOptions{
GoogleAccessID: "xxx@clientid",
PrivateKey: dummyKey("rsa"),
Method: "POST",
Expires: expires,
Scheme: SigningSchemeV2,
},
want: "https://localhost:8000/" + bucketName + "/" + objectName +
"?Expires=1033570800" +
"&GoogleAccessId=xxx%40clientid" +
"&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D",
},
}
oldUTCNow := utcNow
defer func() {
utcNow = oldUTCNow
}()

for _, test := range tests {
t.Run(test.desc, func(s *testing.T) {
utcNow = func() time.Time {
return test.now
}

os.Setenv("STORAGE_EMULATOR_HOST", test.emulatorHost)

got, err := SignedURL(bucketName, objectName, test.opts)
if err != nil {
s.Fatal(err)
}
if got != test.want {
s.Fatalf("\n\tgot:\t%v\n\twant:\t%v", got, test.want)
}
})
}
}

func TestSignedURL_MissingOptions(t *testing.T) {
now, _ := time.Parse(time.RFC3339, "2002-10-01T00:00:00-05:00")
expires, _ := time.Parse(time.RFC3339, "2002-10-15T00:00:00-05:00")
Expand Down