From a3f4891ee60e57cc19929489cae6110b07955216 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Mon, 12 Feb 2024 14:10:17 -0800 Subject: [PATCH] feat: Base TPC Support (#2397) * chore: Update `google-auth-library` to 9.5.0 or later * feat: Base TPC Support * test: `new Storage({universeDomain})` * test: `signingEndpoint` * test: Add conformance tests * fix: Conformance Fixes * fix: More Conformance Fixes * chore: typo * refactor: Use `Storage` Context * refactor: use `hostname` for signing * chore: lint * feat: Add Custom Endpoint Support for `generateSignedPostPolicyV4` * chore: Bump `google-auth-library` 9.6.3+ is required for Storage TPC Support --- conformance-test/test-data/v4SignedUrl.json | 119 +++++++++++++++++++- conformance-test/v4SignedUrl.ts | 65 ++++++++--- package.json | 2 +- src/bucket.ts | 18 ++- src/file.ts | 50 ++++++-- src/nodejs-common/service.ts | 32 ++++-- src/nodejs-common/util.ts | 11 +- src/resumable-upload.ts | 47 ++++++-- src/signer.ts | 97 ++++++++++++---- src/storage.ts | 5 +- test/bucket.ts | 4 + test/file.ts | 21 ++++ test/index.ts | 11 +- test/nodejs-common/service.ts | 19 +--- test/nodejs-common/util.ts | 7 +- test/signer.ts | 63 ++++++++++- 16 files changed, 461 insertions(+), 110 deletions(-) diff --git a/conformance-test/test-data/v4SignedUrl.json b/conformance-test/test-data/v4SignedUrl.json index 23342df8c..86a51e0ba 100644 --- a/conformance-test/test-data/v4SignedUrl.json +++ b/conformance-test/test-data/v4SignedUrl.json @@ -285,6 +285,123 @@ "bucketBoundHostname": "mydomain.tld", "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:mydomain.tld\n\nhost\nUNSIGNED-PAYLOAD", "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nd6c309924b51a5abbe4d6356f7bf29c2120c6b14649b1e97b3bc9309adca7d4b" + }, + { + "description": "Simple GET with hostname", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "hostname": "storage.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple GET with non-default hostname", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "hostname": "localhost:8080", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple GET with endpoint on client", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com:443/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "clientEndpoint": "storage.googleapis.com:443", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Endpoint on client with scheme", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "clientEndpoint": "http://localhost:8080", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Emulator host", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://xyz.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=53b20003ff2c552b3194a6bccc25024663662392554b3334e989e2704f3a0308455eaacf45c335a78c0186a5cf8eef78bf5781a7267465d28a35c9e1291f87ff340e9ee40b3b9bdce70561bf000887ce38ccd7d2445a8749453960a8f11d37576dfd5942f92d6f4527bbeffb90526b5de9653b6ca16136e9f19bcb65d984ddaf22c4ade45d6a168bb4752a43de33ab121206f50d994612824407711bff720cb1b207b61b613c44c85d3ce16dc4fc6eba24e494e176b0780d0ab85a800b13fcbf31434ddf51992efae1efde330ebda0617d1c20078ef22a4f10a7bcbed961237442d9a8db78d7aeb777a4994b50efdd41e07c4966e912a30f92a7426f207e9545", + "emulatorHostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Endpoint on client takes precedence over emulator", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "clientEndpoint": "http://localhost:8080", + "emulatorHostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Hostname takes precendence over endpoint and emulator", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://xyz.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=53b20003ff2c552b3194a6bccc25024663662392554b3334e989e2704f3a0308455eaacf45c335a78c0186a5cf8eef78bf5781a7267465d28a35c9e1291f87ff340e9ee40b3b9bdce70561bf000887ce38ccd7d2445a8749453960a8f11d37576dfd5942f92d6f4527bbeffb90526b5de9653b6ca16136e9f19bcb65d984ddaf22c4ade45d6a168bb4752a43de33ab121206f50d994612824407711bff720cb1b207b61b613c44c85d3ce16dc4fc6eba24e494e176b0780d0ab85a800b13fcbf31434ddf51992efae1efde330ebda0617d1c20078ef22a4f10a7bcbed961237442d9a8db78d7aeb777a4994b50efdd41e07c4966e912a30f92a7426f207e9545", + "emulatorHostname": "http://localhost:9000", + "clientEndpoint": "http://localhost:8080", + "hostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Universe domain", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.domain.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8cd0d479a88fb7d791a2dcc8fc5b5f020ca817eeef5b5a5cb3260eb63cf47ecd271faa238d0fa31efca35bc2a9244bd122178c520749f922c0235726a5a6be099bf4f33a0d54187eee2e0208964c2a13104b03e235cdeb4f07b3eb566b8a33259cf7540a3fe823be601ace2a54a79acd6834cb646380c4cfc7ef0fd95d3ebbc1f97d840f6fe1dceed4269ecb4e91ff7e6633f38adab82049a965968367b9e7c362cec868d804bd42abbb6d2e837ce5d45ee9e1d92c7acc09623acaae3df6128ca15f9f80bb6543944e8c997f691c35113b9e9f44e86fd343524343b08dd8f887685588acc103e0b432f24912e7e1c63e086aeed1890e41b75beb64164fe6bfcf", + "universeDomain": "domain.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Universe domain with virtual hosted style", + "bucket": "test-bucket", + "object": "test-object", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://test-bucket.storage.domain.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=25820e3a60856596cba594511d7d4039239b2728a9738f15d3a7acce8d70aa5435d0c91f99a9318f932afc73355ac562e014cb654e16ed5524b403536f1cba74489701fdc0c088b8826fccf20a648d3b2b704bd6661e01786d4132174c21441d0752be07e8af93e84e24b87799ee91fabef24a0a58d0889263280c3d37423fab677bd4d98469ab01aa36efaad62ff81ca27bf7fc92f14e20faa71e34de9ffbc5eb4ecf1b0361de42270665bb78367bd0a8cc6a604a8e347f0c864754bf14514aac3106fe73572a6c068ce2c380cc2a943b35502093d162ba9ae8de9abbbc9541ef765d5679857a89d36cc01be30cf1e04c4a477bbcd59a02955dcc1a903d8baa", + "universeDomain": "domain.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" } ], "postPolicyV4Tests": [ @@ -578,4 +695,4 @@ } } ] -} \ No newline at end of file +} diff --git a/conformance-test/v4SignedUrl.ts b/conformance-test/v4SignedUrl.ts index 5430689dc..ecf378bd7 100644 --- a/conformance-test/v4SignedUrl.ts +++ b/conformance-test/v4SignedUrl.ts @@ -21,11 +21,7 @@ import * as path from 'path'; import * as sinon from 'sinon'; import * as querystring from 'querystring'; -import { - Storage, - GetSignedUrlConfig, - GenerateSignedPostPolicyV4Options, -} from '../src/'; +import {Storage, GenerateSignedPostPolicyV4Options} from '../src/'; import * as url from 'url'; import {getDirName} from '../src/util.js'; @@ -37,17 +33,23 @@ export enum UrlStyle { interface V4SignedURLTestCase { description: string; + hostname?: string; + emulatorHostname?: string; + clientEndpoint?: string; + universeDomain?: string; bucket: string; object?: string; - urlStyle?: UrlStyle; + urlStyle?: keyof typeof UrlStyle; bucketBoundHostname?: string; - scheme: 'https' | 'http'; + scheme?: 'https' | 'http'; headers?: OutgoingHttpHeaders; queryParameters?: {[key: string]: string}; method: string; expiration: number; timestamp: string; expectedUrl: string; + expectedCanonicalRequest: string; + expectedStringToSign: string; } interface V4SignedPolicyTestCase { @@ -96,7 +98,6 @@ const testFile = fs.readFileSync( 'utf-8' ); -// eslint-disable-next-line @typescript-eslint/no-explicit-any const testCases = JSON.parse(testFile); const v4SignedUrlCases: V4SignedURLTestCase[] = testCases.signingV4Tests; const v4SignedPolicyCases: V4SignedPolicyTestCase[] = @@ -107,18 +108,44 @@ const SERVICE_ACCOUNT = path.join( '../../../conformance-test/fixtures/signing-service-account.json' ); -const storage = new Storage({keyFilename: SERVICE_ACCOUNT}); +let storage: Storage; describe('v4 conformance test', () => { + let fakeTimer: sinon.SinonFakeTimers; + + beforeEach(() => { + delete process.env.STORAGE_EMULATOR_HOST; + }); + + afterEach(() => { + fakeTimer.restore(); + }); + describe('v4 signed url', () => { v4SignedUrlCases.forEach(testCase => { it(testCase.description, async () => { const NOW = new Date(testCase.timestamp); + fakeTimer = sinon.useFakeTimers(NOW); + + if (testCase.emulatorHostname) { + process.env.STORAGE_EMULATOR_HOST = testCase.emulatorHostname; + } + + storage = new Storage({ + keyFilename: SERVICE_ACCOUNT, + apiEndpoint: testCase.clientEndpoint, + universeDomain: testCase.universeDomain, + }); - const fakeTimer = sinon.useFakeTimers(NOW); const bucket = storage.bucket(testCase.bucket); const expires = NOW.valueOf() + testCase.expiration * 1000; const version = 'v4' as const; + const host = testCase.hostname + ? new URL( + (testCase.scheme ? testCase.scheme + '://' : '') + + testCase.hostname + ) + : undefined; const origin = testCase.bucketBoundHostname ? `${testCase.scheme}://${testCase.bucketBoundHostname}` : undefined; @@ -135,7 +162,8 @@ describe('v4 conformance test', () => { cname: bucketBoundHostname, virtualHostedStyle, queryParams, - }; + host, + } as const; let signedUrl: string; if (testCase.object) { @@ -153,7 +181,7 @@ describe('v4 conformance test', () => { [signedUrl] = await file.getSignedUrl({ action, ...baseConfig, - } as GetSignedUrlConfig); + }); } else { // bucket operation const action = ( @@ -178,8 +206,6 @@ describe('v4 conformance test', () => { querystring.parse(actual.search), querystring.parse(expected.search) ); - - fakeTimer.restore(); }); }); }); @@ -189,7 +215,12 @@ describe('v4 conformance test', () => { it(testCase.description, async () => { const input = testCase.policyInput; const NOW = new Date(input.timestamp); - const fakeTimer = sinon.useFakeTimers(NOW); + fakeTimer = sinon.useFakeTimers(NOW); + + storage = new Storage({ + keyFilename: SERVICE_ACCOUNT, + }); + const bucket = storage.bucket(input.bucket); const expires = NOW.valueOf() + input.expiration * 1000; const options: GenerateSignedPostPolicyV4Options = { @@ -237,15 +268,13 @@ describe('v4 conformance test', () => { ); assert.deepStrictEqual(policy.fields, outputFields); - - fakeTimer.restore(); }); }); }); }); function parseUrlStyle( - style?: UrlStyle, + style?: keyof typeof UrlStyle, origin?: string ): {bucketBoundHostname?: string; virtualHostedStyle?: boolean} { if (style === UrlStyle.BUCKET_BOUND_HOSTNAME) { diff --git a/package.json b/package.json index e0ade250d..a0cfc0255 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,7 @@ "ent": "^2.2.0", "fast-xml-parser": "^4.3.0", "gaxios": "^6.0.2", - "google-auth-library": "^9.0.0", + "google-auth-library": "^9.6.3", "mime": "^3.0.0", "mime-types": "^2.0.8", "p-limit": "^3.0.1", diff --git a/src/bucket.ts b/src/bucket.ts index 34df1a64a..c41f1023c 100644 --- a/src/bucket.ts +++ b/src/bucket.ts @@ -367,7 +367,8 @@ export interface GetBucketMetadataOptions { userProject?: string; } -export interface GetBucketSignedUrlConfig { +export interface GetBucketSignedUrlConfig + extends Pick { action: 'list'; version?: 'v2' | 'v4'; cname?: string; @@ -1975,7 +1976,7 @@ class Bucket extends ServiceObject { body.topic = 'projects/{{projectId}}/topics/' + body.topic; } - body.topic = '//pubsub.googleapis.com/' + body.topic; + body.topic = `//pubsub.${this.storage.universeDomain}/` + body.topic; if (!body.payloadFormat) { body.payloadFormat = 'JSON_API_V1'; @@ -3133,17 +3134,24 @@ class Bucket extends ServiceObject { ): void | Promise { const method = BucketActionToHTTPMethod[cfg.action]; - const signConfig = { + const signConfig: SignerGetSignedUrlConfig = { method, expires: cfg.expires, version: cfg.version, cname: cfg.cname, extensionHeaders: cfg.extensionHeaders || {}, queryParams: cfg.queryParams || {}, - } as SignerGetSignedUrlConfig; + host: cfg.host, + signingEndpoint: cfg.signingEndpoint, + }; if (!this.signer) { - this.signer = new URLSigner(this.storage.authClient, this); + this.signer = new URLSigner( + this.storage.authClient, + this, + undefined, + this.storage + ); } this.signer diff --git a/src/file.ts b/src/file.ts index d2274dd57..cb29fc8db 100644 --- a/src/file.ts +++ b/src/file.ts @@ -108,6 +108,11 @@ export interface GenerateSignedPostPolicyV2Options { successRedirect?: string; successStatus?: string; contentLengthRange?: {min?: number; max?: number}; + /** + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string; } export interface PolicyFields { @@ -120,6 +125,11 @@ export interface GenerateSignedPostPolicyV4Options { virtualHostedStyle?: boolean; conditions?: object[]; fields?: PolicyFields; + /** + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string; } export interface GenerateSignedPostPolicyV4Callback { @@ -133,7 +143,8 @@ export interface SignedPostPolicyV4Output { fields: PolicyFields; } -export interface GetSignedUrlConfig { +export interface GetSignedUrlConfig + extends Pick { action: 'read' | 'write' | 'delete' | 'resumable'; version?: 'v2' | 'v4'; virtualHostedStyle?: boolean; @@ -302,7 +313,7 @@ export enum ActionToHTTPMethod { } /** - * @private + * @deprecated - no longer used */ export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com'; @@ -1760,6 +1771,7 @@ class File extends ServiceObject { userProject: options.userProject || this.userProject, retryOptions: retryOptions, params: options?.preconditionOpts || this.instancePreconditionOpts, + universeDomain: this.bucket.storage.universeDomain, [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], }, callback! @@ -2580,7 +2592,7 @@ class File extends ServiceObject { const policyString = JSON.stringify(policy); const policyBase64 = Buffer.from(policyString).toString('base64'); - this.storage.authClient.sign(policyBase64).then( + this.storage.authClient.sign(policyBase64, options.signingEndpoint).then( signature => { callback(null, { string: policyString, @@ -2763,18 +2775,25 @@ class File extends ServiceObject { const policyBase64 = Buffer.from(policyString).toString('base64'); try { - const signature = await this.storage.authClient.sign(policyBase64); + const signature = await this.storage.authClient.sign( + policyBase64, + options.signingEndpoint + ); const signatureHex = Buffer.from(signature, 'base64').toString('hex'); + const universe = this.parent.storage.universeDomain; fields['policy'] = policyBase64; fields['x-goog-signature'] = signatureHex; let url: string; - if (options.virtualHostedStyle) { - url = `https://${this.bucket.name}.storage.googleapis.com/`; + + if (this.storage.customEndpoint) { + url = this.storage.apiEndpoint; + } else if (options.virtualHostedStyle) { + url = `https://${this.bucket.name}.storage.${universe}/`; } else if (options.bucketBoundHostname) { url = `${options.bucketBoundHostname}/`; } else { - url = `${STORAGE_POST_POLICY_BASE_URL}/${this.bucket.name}/`; + url = `https://storage.${universe}/${this.bucket.name}/`; } return { @@ -2828,8 +2847,8 @@ class File extends ServiceObject { * @param {string} [config.version='v2'] The signing version to use, either * 'v2' or 'v4'. * @param {boolean} [config.virtualHostedStyle=false] Use virtual hosted-style - * URLs ('https://mybucket.storage.googleapis.com/...') instead of path-style - * ('https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs + * URLs (e.g. 'https://mybucket.storage.googleapis.com/...') instead of path-style + * (e.g. 'https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs * should generally be preferred instaed of path-style URL. * Currently defaults to `false` for path-style, although this may change in a * future major-version release. @@ -2989,7 +3008,7 @@ class File extends ServiceObject { queryParams['generation'] = this.generation.toString(); } - const signConfig = { + const signConfig: SignerGetSignedUrlConfig = { method, expires: cfg.expires, accessibleAt: cfg.accessibleAt, @@ -2997,7 +3016,8 @@ class File extends ServiceObject { queryParams, contentMd5: cfg.contentMd5, contentType: cfg.contentType, - } as SignerGetSignedUrlConfig; + host: cfg.host, + }; if (cfg.cname) { signConfig.cname = cfg.cname; @@ -3012,7 +3032,12 @@ class File extends ServiceObject { } if (!this.signer) { - this.signer = new URLSigner(this.storage.authClient, this.bucket, this); + this.signer = new URLSigner( + this.storage.authClient, + this.bucket, + this, + this.storage + ); } this.signer @@ -4004,6 +4029,7 @@ class File extends ServiceObject { params: options?.preconditionOpts || this.instancePreconditionOpts, chunkSize: options?.chunkSize, highWaterMark: options?.highWaterMark, + universeDomain: this.bucket.storage.universeDomain, [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], }); diff --git a/src/nodejs-common/service.ts b/src/nodejs-common/service.ts index 4ee305286..0a3111667 100644 --- a/src/nodejs-common/service.ts +++ b/src/nodejs-common/service.ts @@ -13,7 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {AuthClient, GoogleAuth, GoogleAuthOptions} from 'google-auth-library'; +import { + AuthClient, + DEFAULT_UNIVERSE, + GoogleAuth, + GoogleAuthOptions, +} from 'google-auth-library'; import * as r from 'teeny-request'; import * as uuid from 'uuid'; @@ -62,6 +67,11 @@ export interface ServiceConfig { * Reuse an existing `AuthClient` or `GoogleAuth` client instead of creating a new one. */ authClient?: AuthClient | GoogleAuth; + + /** + * Set to true if the endpoint is a custom URL + */ + customEndpoint?: boolean; } export interface ServiceOptions extends Omit { @@ -84,9 +94,10 @@ export class Service { providedUserAgent?: string; makeAuthenticatedRequest: MakeAuthenticatedRequest; authClient: GoogleAuth; - private getCredentials: {}; - readonly apiEndpoint: string; + apiEndpoint: string; timeout?: number; + universeDomain: string; + customEndpoint: boolean; /** * Service is a base class, meant to be inherited from by a "service," like @@ -115,8 +126,10 @@ export class Service { this.projectId = options.projectId || DEFAULT_PROJECT_ID_TOKEN; this.projectIdRequired = config.projectIdRequired !== false; this.providedUserAgent = options.userAgent; + this.universeDomain = options.universeDomain || DEFAULT_UNIVERSE; + this.customEndpoint = config.customEndpoint || false; - const reqCfg = { + this.makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory({ ...config, projectIdRequired: this.projectIdRequired, projectId: this.projectId, @@ -124,13 +137,12 @@ export class Service { credentials: options.credentials, keyFile: options.keyFilename, email: options.email, - token: options.token, - }; - - this.makeAuthenticatedRequest = - util.makeAuthenticatedRequestFactory(reqCfg); + clientOptions: { + universeDomain: options.universeDomain, + ...options.clientOptions, + }, + }); this.authClient = this.makeAuthenticatedRequest.authClient; - this.getCredentials = this.makeAuthenticatedRequest.getCredentials; const isCloudFunctionEnv = !!process.env.FUNCTION_NAME; diff --git a/src/nodejs-common/util.ts b/src/nodejs-common/util.ts index d24194fec..555c5be67 100644 --- a/src/nodejs-common/util.ts +++ b/src/nodejs-common/util.ts @@ -179,7 +179,7 @@ export interface MakeAuthenticatedRequestFactoryConfig /** * A pre-instantiated `AuthClient` or `GoogleAuth` client that should be used. - * A new will be created if this is not set. + * A new client will be created if this is not set. */ authClient?: AuthClient | GoogleAuth; @@ -638,13 +638,12 @@ export class Util { // Use an existing `GoogleAuth` authClient = googleAutoAuthConfig.authClient; } else { - // Pass an `AuthClient` to `GoogleAuth`, if available - const config = { + // Pass an `AuthClient` & `clientOptions` to `GoogleAuth`, if available + authClient = new GoogleAuth({ ...googleAutoAuthConfig, authClient: googleAutoAuthConfig.authClient, - }; - - authClient = new GoogleAuth(config); + clientOptions: googleAutoAuthConfig.clientOptions, + }); } /** diff --git a/src/resumable-upload.ts b/src/resumable-upload.ts index 4184c264b..049e20c43 100644 --- a/src/resumable-upload.ts +++ b/src/resumable-upload.ts @@ -21,7 +21,11 @@ import { GaxiosError, } from 'gaxios'; import * as gaxios from 'gaxios'; -import {GoogleAuth, GoogleAuthOptions} from 'google-auth-library'; +import { + DEFAULT_UNIVERSE, + GoogleAuth, + GoogleAuthOptions, +} from 'google-auth-library'; import {Readable, Writable, WritableOptions} from 'stream'; import AsyncRetry from 'async-retry'; import {RetryOptions, PreconditionOptions} from './storage.js'; @@ -39,7 +43,6 @@ import {getPackageJSON} from './package-json-helper.cjs'; const NOT_FOUND_STATUS_CODE = 404; const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; -const DEFAULT_API_ENDPOINT_REGEX = /.*\.googleapis\.com/; const packageJson = getPackageJSON(); export const PROTOCOL_REGEX = /^(\w*):\/\//; @@ -75,9 +78,10 @@ export interface UploadConfig extends Pick { /** * The API endpoint used for the request. * Defaults to `storage.googleapis.com`. + * * **Warning**: - * If this value does not match the pattern *.googleapis.com, - * an emulator context will be assumed and authentication will be bypassed. + * If this value does not match the current GCP universe an emulator context + * will be assumed and authentication will be bypassed. */ apiEndpoint?: string; @@ -209,6 +213,11 @@ export interface UploadConfig extends Pick { */ public?: boolean; + /** + * The service domain for a given Cloud universe. + */ + universeDomain?: string; + /** * If you already have a resumable URI from a previously-created resumable * upload, just pass it in here and we'll use that. @@ -356,10 +365,34 @@ export class Upload extends Writable { ]; this.authClient = cfg.authClient || new GoogleAuth(cfg.authConfig); - this.apiEndpoint = 'https://storage.googleapis.com'; - if (cfg.apiEndpoint) { + const universe = cfg.universeDomain || DEFAULT_UNIVERSE; + + this.apiEndpoint = `https://storage.${universe}`; + if (cfg.apiEndpoint && cfg.apiEndpoint !== this.apiEndpoint) { this.apiEndpoint = this.sanitizeEndpoint(cfg.apiEndpoint); - if (!DEFAULT_API_ENDPOINT_REGEX.test(cfg.apiEndpoint)) { + + const hostname = new URL(this.apiEndpoint).hostname; + + // check if it is a domain of a known universe + const isDomain = hostname === universe; + const isDefaultUniverseDomain = hostname === DEFAULT_UNIVERSE; + + // check if it is a subdomain of a known universe + // by checking a last (universe's length + 1) of a hostname + const isSubDomainOfUniverse = + hostname.slice(-(universe.length + 1)) === `.${universe}`; + const isSubDomainOfDefaultUniverse = + hostname.slice(-(DEFAULT_UNIVERSE.length + 1)) === + `.${DEFAULT_UNIVERSE}`; + + if ( + !isDomain && + !isDefaultUniverseDomain && + !isSubDomainOfUniverse && + !isSubDomainOfDefaultUniverse + ) { + // a custom, non-universe domain, + // use gaxios this.authClient = gaxios; } } diff --git a/src/signer.ts b/src/signer.ts index 5a50de3f7..879bc4d2a 100644 --- a/src/signer.ts +++ b/src/signer.ts @@ -15,16 +15,20 @@ import * as crypto from 'crypto'; import * as http from 'http'; import * as url from 'url'; -import {ExceptionMessages} from './storage.js'; +import {ExceptionMessages, Storage} from './storage.js'; import {encodeURI, qsStringify, objectEntries, formatAsUTCISO} from './util.js'; +import {GoogleAuth} from 'google-auth-library'; -interface GetCredentialsResponse { - client_email?: string; -} +type GoogleAuthLike = Pick; +/** + * @deprecated Use {@link GoogleAuth} instead + */ export interface AuthClient { sign(blobToSign: string): Promise; - getCredentials(): Promise; + getCredentials(): Promise<{ + client_email?: string; + }>; } export interface BucketI { @@ -50,6 +54,20 @@ export interface GetSignedUrlConfigInternal { contentType?: string; bucket: string; file?: string; + /** + * The host for the generated signed URL + * + * @example + * 'https://localhost:8080/' + */ + host?: string | URL; + /** + * An endpoint for generating the signed URL + * + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string | URL; } interface SignedUrlQuery { @@ -75,6 +93,20 @@ export interface SignerGetSignedUrlConfig { queryParams?: Query; contentMd5?: string; contentType?: string; + /** + * The host for the generated signed URL + * + * @example + * 'https://localhost:8080/' + */ + host?: string | URL; + /** + * An endpoint for generating the signed URL + * + * @example + * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + */ + signingEndpoint?: string | URL; } export type SignerGetSignedUrlResponse = string; @@ -102,20 +134,26 @@ const SEVEN_DAYS = 7 * 24 * 60 * 60; /** * @const {string} - * @private + * @deprecated - unused */ export const PATH_STYLED_HOST = 'https://storage.googleapis.com'; export class URLSigner { - private authClient: AuthClient; - private bucket: BucketI; - private file?: FileI; - - constructor(authClient: AuthClient, bucket: BucketI, file?: FileI) { - this.bucket = bucket; - this.file = file; - this.authClient = authClient; - } + constructor( + private auth: AuthClient | GoogleAuthLike, + private bucket: BucketI, + private file?: FileI, + /** + * A {@link Storage} object. + * + * @privateRemarks + * + * Technically this is a required field, however it would be a breaking change to + * move it before optional properties. In the next major we should refactor the + * constructor of this class to only accept a config object. + */ + private storage: Storage = new Storage() + ) {} getSignedUrl( cfg: SignerGetSignedUrlConfig @@ -137,7 +175,7 @@ export class URLSigner { if (cfg.cname) { customHost = cfg.cname; } else if (isVirtualHostedStyle) { - customHost = `https://${this.bucket.name}.storage.googleapis.com`; + customHost = `https://${this.bucket.name}.storage.${this.storage.universeDomain}`; } const secondsToMilliseconds = 1000; @@ -169,7 +207,10 @@ export class URLSigner { return promise.then(query => { query = Object.assign(query, cfg.queryParams); - const signedUrl = new url.URL(config.cname || PATH_STYLED_HOST); + const signedUrl = new url.URL( + cfg.host?.toString() || config.cname || this.storage.apiEndpoint + ); + signedUrl.pathname = this.getResourcePath( !!config.cname, this.bucket.name, @@ -202,10 +243,13 @@ export class URLSigner { ].join('\n'); const sign = async () => { - const authClient = this.authClient; + const auth = this.auth; try { - const signature = await authClient.sign(blobToSign); - const credentials = await authClient.getCredentials(); + const signature = await auth.sign( + blobToSign, + config.signingEndpoint?.toString() + ); + const credentials = await auth.getCredentials(); return { GoogleAccessId: credentials.client_email!, @@ -240,8 +284,10 @@ export class URLSigner { } const extensionHeaders = Object.assign({}, config.extensionHeaders); - const fqdn = new url.URL(config.cname || PATH_STYLED_HOST); - extensionHeaders.host = fqdn.host; + const fqdn = new url.URL( + config.host?.toString() || config.cname || this.storage.apiEndpoint + ); + extensionHeaders.host = fqdn.hostname; if (config.contentMd5) { extensionHeaders['content-md5'] = config.contentMd5; } @@ -272,7 +318,7 @@ export class URLSigner { const credentialScope = `${datestamp}/auto/storage/goog4_request`; const sign = async () => { - const credentials = await this.authClient.getCredentials(); + const credentials = await this.auth.getCredentials(); const credential = `${credentials.client_email}/${credentialScope}`; const dateISO = formatAsUTCISO( config.accessibleAt ? config.accessibleAt : new Date(), @@ -312,7 +358,10 @@ export class URLSigner { ].join('\n'); try { - const signature = await this.authClient.sign(blobToSign); + const signature = await this.auth.sign( + blobToSign, + config.signingEndpoint?.toString() + ); const signatureHex = Buffer.from(signature, 'base64').toString('hex'); const signedQuery: Query = Object.assign({}, queryParams, { 'X-Goog-Signature': signatureHex, diff --git a/src/storage.ts b/src/storage.ts index e6f251acc..f13c37acc 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -29,6 +29,7 @@ import { CRC32CValidatorGenerator, CRC32C_DEFAULT_VALIDATOR_GENERATOR, } from './crc32c.js'; +import {DEFAULT_UNIVERSE} from 'google-auth-library'; export interface GetServiceAccountOptions { userProject?: string; @@ -692,7 +693,9 @@ export class Storage extends Service { * @param {StorageOptions} [options] Configuration options. */ constructor(options: StorageOptions = {}) { - let apiEndpoint = 'https://storage.googleapis.com'; + const universe = options.universeDomain || DEFAULT_UNIVERSE; + + let apiEndpoint = `https://storage.${universe}`; let customEndpoint = false; // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. diff --git a/test/bucket.ts b/test/bucket.ts index 0c59c3ba9..a9dc42af7 100644 --- a/test/bucket.ts +++ b/test/bucket.ts @@ -54,6 +54,7 @@ import sinon from 'sinon'; import {Transform} from 'stream'; import {IdempotencyStrategy} from '../src/storage.js'; import {convertObjKeysToSnakeCase, getDirName} from '../src/util.js'; +import {DEFAULT_UNIVERSE} from 'google-auth-library'; class FakeFile { calledWith_: IArguments; @@ -204,6 +205,7 @@ describe('Bucket', () => { idempotencyStrategy: IdempotencyStrategy.RetryConditional, }, crc32cGenerator: () => new CRC32C(), + universeDomain: DEFAULT_UNIVERSE, }; const BUCKET_NAME = 'test-bucket'; @@ -2123,8 +2125,10 @@ describe('Bucket', () => { version: 'v4', expires: SIGNED_URL_CONFIG.expires, extensionHeaders: {}, + host: undefined, queryParams: {}, cname: CNAME, + signingEndpoint: undefined, }); done(); } diff --git a/test/file.ts b/test/file.ts index 76d488404..c4098c637 100644 --- a/test/file.ts +++ b/test/file.ts @@ -243,6 +243,7 @@ describe('File', () => { }, idempotencyStrategy: IdempotencyStrategy.RetryConditional, }, + customEndpoint: false, }; BUCKET = new Bucket(STORAGE, 'bucket-name'); @@ -3387,6 +3388,25 @@ describe('File', () => { ); }); + it('should prefer a customEndpoint > virtualHostedStyle, cname', done => { + const customEndpoint = 'https://my-custom-endpoint.com'; + + STORAGE.apiEndpoint = customEndpoint; + STORAGE.customEndpoint = true; + + CONFIG.virtualHostedStyle = true; + CONFIG.bucketBoundHostname = 'http://domain.tld'; + + file.generateSignedPostPolicyV4( + CONFIG, + (err: Error, res: SignedPostPolicyV4Output) => { + assert.ifError(err); + assert(res.url, `https://${BUCKET.name}.storage.googleapis.com/`); + done(); + } + ); + }); + describe('expires', () => { it('should accept Date objects', done => { const expires = new Date(Date.now() + 1000 * 60); @@ -3561,6 +3581,7 @@ describe('File', () => { expires: config.expires, accessibleAt: accessibleAtDate, extensionHeaders: {}, + host: undefined, queryParams: {}, contentMd5: config.contentMd5, contentType: config.contentType, diff --git a/test/index.ts b/test/index.ts index 610cf0d54..e09814ad6 100644 --- a/test/index.ts +++ b/test/index.ts @@ -444,6 +444,14 @@ describe('Storage', () => { ); }); + it('should accept and use a `universeDomain`', () => { + const universeDomain = 'my-universe.com'; + + const storage = new Storage({universeDomain}); + + assert.equal(storage.apiEndpoint, `https://storage.${universeDomain}`); + }); + describe('STORAGE_EMULATOR_HOST', () => { // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. const EMULATOR_HOST = 'https://internal.benchmark.com/path'; @@ -500,8 +508,7 @@ describe('Storage', () => { projectId: PROJECT_ID, }); - const calledWith = storage.calledWith_[0]; - assert.strictEqual(calledWith.customEndpoint, true); + assert.strictEqual(storage.customEndpoint, true); }); }); }); diff --git a/test/nodejs-common/service.ts b/test/nodejs-common/service.ts index 126739784..502c4e541 100644 --- a/test/nodejs-common/service.ts +++ b/test/nodejs-common/service.ts @@ -111,7 +111,9 @@ describe('Service', () => { email: OPTIONS.email, projectIdRequired: CONFIG.projectIdRequired, projectId: OPTIONS.projectId, - token: OPTIONS.token, + clientOptions: { + universeDomain: undefined, + }, }; assert.deepStrictEqual(config, expectedConfig); @@ -193,21 +195,6 @@ describe('Service', () => { assert.strictEqual(service.timeout, timeout); }); - it('should localize the getCredentials method', () => { - function getCredentials() {} - - makeAuthenticatedRequestFactoryOverride = () => { - return { - authClient: {}, - getCredentials, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - }; - - const service = new Service(CONFIG, OPTIONS); - assert.strictEqual(service.getCredentials, getCredentials); - }); - it('should default globalInterceptors to an empty array', () => { assert.deepStrictEqual(service.globalInterceptors, []); }); diff --git a/test/nodejs-common/util.ts b/test/nodejs-common/util.ts index 703f922b6..48d5fd0f1 100644 --- a/test/nodejs-common/util.ts +++ b/test/nodejs-common/util.ts @@ -732,7 +732,11 @@ describe('common/util', () => { sandbox .stub(fakeGoogleAuth, 'GoogleAuth') .callsFake((config_: GoogleAuthOptions) => { - assert.deepStrictEqual(config_, {...config, authClient: undefined}); + assert.deepStrictEqual(config_, { + ...config, + authClient: undefined, + clientOptions: undefined, + }); setImmediate(done); return authClient; }); @@ -745,6 +749,7 @@ describe('common/util', () => { const config: MakeAuthenticatedRequestFactoryConfig = { authClient: customAuthClient, + clientOptions: undefined, }; sandbox diff --git a/test/signer.ts b/test/signer.ts index 12d560477..7d729af15 100644 --- a/test/signer.ts +++ b/test/signer.ts @@ -29,8 +29,9 @@ import { SignerExceptionMessages, } from '../src/signer.js'; import {encodeURI, formatAsUTCISO, qsStringify} from '../src/util.js'; -import {ExceptionMessages} from '../src/storage.js'; +import {ExceptionMessages, Storage} from '../src/storage.js'; import {OutgoingHttpHeaders} from 'http'; +import {GoogleAuth} from 'google-auth-library'; interface SignedUrlArgs { bucket: string; @@ -52,7 +53,7 @@ describe('signer', () => { afterEach(() => sandbox.restore()); describe('URLSigner', () => { - let authClient: AuthClient; + let authClient: GoogleAuth | AuthClient; let bucket: BucketI; let file: FileI; @@ -78,7 +79,7 @@ describe('signer', () => { }); it('should localize authClient', () => { - assert.strictEqual(signer['authClient'], authClient); + assert.strictEqual(signer['auth'], authClient); }); it('should localize bucket', () => { @@ -92,9 +93,12 @@ describe('signer', () => { describe('getSignedUrl', () => { let signer: URLSigner; + let storage: Storage; let CONFIG: SignerGetSignedUrlConfig; + beforeEach(() => { - signer = new URLSigner(authClient, bucket, file); + storage = new Storage(); + signer = new URLSigner(authClient, bucket, file, storage); CONFIG = { method: 'GET', @@ -318,6 +322,17 @@ describe('signer', () => { assert.strictEqual(v2arg.cname, expectedCname); }); + it('should use a universe domain with the virtual host', async () => { + storage.universeDomain = 'my-universe.com'; + + CONFIG.virtualHostedStyle = true; + const expectedCname = `https://${bucket.name}.storage.my-universe.com`; + + await signer.getSignedUrl(CONFIG); + const v2arg = v2.getCall(0).args[0]; + assert.strictEqual(v2arg.cname, expectedCname); + }); + it('should take precedence in cname if both passed', async () => { CONFIG = { virtualHostedStyle: true, @@ -446,10 +461,13 @@ describe('signer', () => { }); describe('blobToSign', () => { - let authClientSign: sinon.SinonStub<[string], Promise>; + let authClientSign: sinon.SinonStub< + [blobToSign: string] & [data: string, endpoint?: string | undefined], + Promise + >; beforeEach(() => { authClientSign = sandbox - .stub(authClient, 'sign') + .stub(authClient, 'sign') .resolves('signature'); }); @@ -460,6 +478,20 @@ describe('signer', () => { assert(blobToSign.startsWith('GET')); }); + it('should sign using the `signingEndpoint` when provided', async () => { + const signingEndpoint = 'https://my-endpoint.com'; + + CONFIG = { + ...CONFIG, + signingEndpoint, + }; + + await signer['getSignedUrlV2'](CONFIG); + + const endpoint = authClientSign.getCall(0).args[1]; + assert.equal(endpoint, signingEndpoint); + }); + it('should sign contentMd5 if given', async () => { CONFIG.contentMd5 = 'md5-hash'; @@ -815,6 +847,25 @@ describe('signer', () => { assert(blobToSign.endsWith(canonicalRequestHash)); }); + it('should sign using the `signingEndpoint` when provided', async () => { + const signingEndpoint = 'https://my-endpoint.com'; + + sinon.stub(signer, 'getCanonicalRequest').returns('canonical-request'); + const authClientSign = sinon + .stub(authClient, 'sign') + .resolves('signature'); + + CONFIG = { + ...CONFIG, + signingEndpoint, + }; + + await signer['getSignedUrlV4'](CONFIG); + + const endpoint = authClientSign.getCall(0).args[1]; + assert.equal(endpoint, signingEndpoint); + }); + it('should compose blobToSign', async () => { const datestamp = formatAsUTCISO(NOW); const credentialScope = `${datestamp}/auto/storage/goog4_request`;