Skip to content

Commit

Permalink
improve(r2): Update Sippy endpoint request payloads to match new sche…
Browse files Browse the repository at this point in the history
…ma (#4928)

* fix(r2): update Sippy API request payloads

* improve(r2): rename sippy flags for clarity

* improve(r2): stricter existence in Sippy
  • Loading branch information
sdnts authored Feb 7, 2024
1 parent cf9c029 commit 4a735c4
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 76 deletions.
5 changes: 5 additions & 0 deletions .changeset/five-cooks-share.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"wrangler": patch
---

Update API calls for Sippy's endpoints
45 changes: 28 additions & 17 deletions packages/wrangler/src/__tests__/r2.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,19 +326,25 @@ describe("r2", () => {
"*/accounts/some-account-id/r2/buckets/testBucket/sippy",
async (request, response, context) => {
expect(await request.json()).toEqual({
access_key: "aws-secret",
bucket: "awsBucket",
key_id: "aws-key",
provider: "AWS",
r2_access_key: "some-secret",
r2_key_id: "some-key",
source: {
provider: "aws",
region: "awsRegion",
bucket: "awsBucket",
accessKeyId: "aws-key",
secretAccessKey: "aws-secret",
},
destination: {
provider: "r2",
accessKeyId: "some-key",
secretAccessKey: "some-secret",
},
});
return response.once(context.json(createFetchResult({})));
}
)
);
await runWrangler(
"r2 bucket sippy enable testBucket --r2-key-id=some-key --r2-secret-access-key=some-secret --provider=AWS --key-id=aws-key --secret-access-key=aws-secret --bucket=awsBucket"
"r2 bucket sippy enable testBucket --r2-access-key-id=some-key --r2-secret-access-key=some-secret --provider=AWS --access-key-id=aws-key --secret-access-key=aws-secret --region=awsRegion --bucket=awsBucket"
);
expect(std.out).toMatchInlineSnapshot(
`"✨ Successfully enabled Sippy on the 'testBucket' bucket."`
Expand All @@ -353,19 +359,24 @@ describe("r2", () => {
"*/accounts/some-account-id/r2/buckets/testBucket/sippy",
async (request, response, context) => {
expect(await request.json()).toEqual({
private_key: "gcs-private-key",
bucket: "gcsBucket",
client_email: "gcs-client-email",
provider: "GCS",
r2_access_key: "some-secret",
r2_key_id: "some-key",
source: {
provider: "gcs",
bucket: "gcsBucket",
clientEmail: "gcs-client-email",
privateKey: "gcs-private-key",
},
destination: {
provider: "r2",
accessKeyId: "some-key",
secretAccessKey: "some-secret",
},
});
return response.once(context.json(createFetchResult({})));
}
)
);
await runWrangler(
"r2 bucket sippy enable testBucket --r2-key-id=some-key --r2-secret-access-key=some-secret --provider=GCS --client-email=gcs-client-email --private-key=gcs-private-key --bucket=gcsBucket"
"r2 bucket sippy enable testBucket --r2-access-key-id=some-key --r2-secret-access-key=some-secret --provider=GCS --client-email=gcs-client-email --private-key=gcs-private-key --bucket=gcsBucket"
);
expect(std.out).toMatchInlineSnapshot(
`"✨ Successfully enabled Sippy on the 'testBucket' bucket."`
Expand Down Expand Up @@ -399,12 +410,12 @@ describe("r2", () => {
--provider [choices: \\"AWS\\", \\"GCS\\"]
--bucket The name of the upstream bucket [string]
--region (AWS provider only) The region of the upstream bucket [string]
--key-id (AWS provider only) The secret access key id for the upstream bucket [string]
--access-key-id (AWS provider only) The secret access key id for the upstream bucket [string]
--secret-access-key (AWS provider only) The secret access key for the upstream bucket [string]
--service-account-key-file (GCS provider only) The path to your Google Cloud service account key JSON file [string]
--client-email (GCS provider only) The client email for your Google Cloud service account key [string]
--private-key (GCS provider only) The private key for your Google Cloud service account key [string]
--r2-key-id The secret access key id for this R2 bucket [string]
--r2-access-key-id The secret access key id for this R2 bucket [string]
--r2-secret-access-key The secret access key for this R2 bucket [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
Expand Down Expand Up @@ -522,7 +533,7 @@ describe("r2", () => {
);
await runWrangler("r2 bucket sippy get testBucket");
expect(std.out).toMatchInlineSnapshot(
`"Sippy upstream bucket: https://storage.googleapis.com/storage/v1/b/testBucket."`
`"Sippy configuration: https://storage.googleapis.com/storage/v1/b/testBucket"`
);
});
});
Expand Down
57 changes: 36 additions & 21 deletions packages/wrangler/src/r2/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -227,14 +227,26 @@ export async function usingLocalBucket<T>(
}
}

type SippyConfig = {
source:
| { provider: "aws"; region: string; bucket: string }
| { provider: "gcs"; bucket: string };
destination: {
provider: "r2";
account: string;
bucket: string;
accessKeyId: string;
};
};

/**
* Retreive the sippy upstream bucket for the bucket with the given name
*/
export async function getR2Sippy(
accountId: string,
bucketName: string,
jurisdiction?: string
): Promise<string> {
): Promise<SippyConfig> {
const headers: HeadersInit = {};
if (jurisdiction !== undefined) {
headers["cf-r2-jurisdiction"] = jurisdiction;
Expand Down Expand Up @@ -263,42 +275,45 @@ export async function deleteR2Sippy(
);
}

export type R2Credentials = {
bucket: string;
r2_key_id: string;
r2_access_key: string;
};

export type SippyPutConfig = R2Credentials &
(
export type SippyPutParams = {
source:
| {
provider: "AWS";
zone: string | undefined;
key_id: string;
access_key: string;
provider: "aws";
region: string;
bucket: string;
accessKeyId: string;
secretAccessKey: string;
}
| {
provider: "GCS";
client_email: string;
private_key: string;
}
);
provider: "gcs";
bucket: string;
clientEmail: string;
privateKey: string;
};
destination: {
provider: "r2";
accessKeyId: string;
secretAccessKey: string;
};
};

/**
* Enable sippy on the bucket with the given name
*/
export async function putR2Sippy(
accountId: string,
bucketName: string,
config: SippyPutConfig,
params: SippyPutParams,
jurisdiction?: string
): Promise<void> {
const headers: HeadersInit = {};
const headers: HeadersInit = {
"Content-Type": "application/json",
};
if (jurisdiction !== undefined) {
headers["cf-r2-jurisdiction"] = jurisdiction;
}
return await fetchResult(
`/accounts/${accountId}/r2/buckets/${bucketName}/sippy`,
{ method: "PUT", body: JSON.stringify(config), headers }
{ method: "PUT", body: JSON.stringify(params), headers }
);
}
110 changes: 72 additions & 38 deletions packages/wrangler/src/r2/sippy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import type {
CommonYargsArgv,
StrictYargsOptionsToInterface,
} from "../yargs-types";
import type { SippyPutConfig } from "./helpers";
import type { SippyPutParams } from "./helpers";

const NO_SUCH_OBJECT_KEY = 10007;
const SIPPY_PROVIDER_CHOICES = ["AWS", "GCS"];
Expand Down Expand Up @@ -38,7 +38,7 @@ export function EnableOptions(yargs: CommonYargsArgv) {
description: "(AWS provider only) The region of the upstream bucket",
string: true,
})
.option("key-id", {
.option("access-key-id", {
description:
"(AWS provider only) The secret access key id for the upstream bucket",
string: true,
Expand All @@ -63,7 +63,7 @@ export function EnableOptions(yargs: CommonYargsArgv) {
"(GCS provider only) The private key for your Google Cloud service account key",
string: true,
})
.option("r2-key-id", {
.option("r2-access-key-id", {
description: "The secret access key id for this R2 bucket",
string: true,
})
Expand Down Expand Up @@ -97,14 +97,17 @@ export async function EnableHandler(
throw new UserError(`Must specify ${args.provider} bucket name.`);
}

if (args.provider == "AWS") {
if (args.provider === "AWS") {
args.region ??= await prompt(
"Enter the AWS region where your S3 bucket is located (example: us-west-2):"
);
args.keyId ??= await prompt(
if (!args.region) {
throw new UserError("Must specify an AWS Region.");

Check warning on line 105 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L105

Added line #L105 was not covered by tests
}
args.accessKeyId ??= await prompt(
"Enter your AWS Access Key ID (requires read and list access):"
);
if (!args.keyId) {
if (!args.accessKeyId) {
throw new UserError("Must specify an AWS Access Key ID.");
}
args.secretAccessKey ??= await prompt(
Expand All @@ -113,7 +116,7 @@ export async function EnableHandler(
if (!args.secretAccessKey) {
throw new UserError("Must specify an AWS Secret Access Key.");
}
} else if (args.provider == "GCS") {
} else if (args.provider === "GCS") {
if (
!(args.clientEmail && args.privateKey) &&
!args.serviceAccountKeyFile
Expand All @@ -129,10 +132,10 @@ export async function EnableHandler(
}
}

args.r2KeyId ??= await prompt(
args.r2AccessKeyId ??= await prompt(
"Enter your R2 Access Key ID (requires read and write access):"
);
if (!args.r2KeyId) {
if (!args.r2AccessKeyId) {
throw new UserError("Must specify an R2 Access Key ID.");
}
args.r2SecretAccessKey ??= await prompt("Enter your R2 Secret Access Key:");
Expand All @@ -141,47 +144,78 @@ export async function EnableHandler(
}
}

let sippyConfig = {
bucket: args.bucket ?? "",
r2_key_id: args.r2KeyId ?? "",
r2_access_key: args.r2SecretAccessKey ?? "",
} as SippyPutConfig;
let sippyConfig: SippyPutParams;

if (args.provider === "AWS") {
if (!args.region) throw new UserError("Error: must provide --region.");
if (!args.bucket) throw new UserError("Error: must provide --bucket.");
if (!args.accessKeyId)
throw new UserError("Error: must provide --access-key-id.");

Check warning on line 153 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L153

Added line #L153 was not covered by tests
if (!args.secretAccessKey)
throw new UserError("Error: must provide --secret-access-key.");

Check warning on line 155 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L155

Added line #L155 was not covered by tests
if (!args.r2AccessKeyId)
throw new UserError("Error: must provide --r2-access-key-id.");

Check warning on line 157 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L157

Added line #L157 was not covered by tests
if (!args.r2SecretAccessKey)
throw new UserError("Error: must provide --r2-secret-access-key.");

Check warning on line 159 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L159

Added line #L159 was not covered by tests

if (args.provider == "AWS") {
if (!(args.keyId && args.secretAccessKey)) {
throw new UserError(
`Error: must provide --key-id and --secret-access-key.`
);
}
sippyConfig = {
...sippyConfig,
provider: "AWS",
zone: args.region,
key_id: args.keyId,
access_key: args.secretAccessKey,
source: {
provider: "aws",
region: args.region,
bucket: args.bucket,
accessKeyId: args.accessKeyId,
secretAccessKey: args.secretAccessKey,
},
destination: {
provider: "r2",
accessKeyId: args.r2AccessKeyId,
secretAccessKey: args.r2SecretAccessKey,
},
};
} else if (args.provider == "GCS") {
} else if (args.provider === "GCS") {
if (args.serviceAccountKeyFile) {
const serviceAccount = JSON.parse(
readFileSync(args.serviceAccountKeyFile)
);
if ("client_email" in serviceAccount && "private_key" in serviceAccount) {
args.clientEmail = serviceAccount["client_email"];
args.privateKey = serviceAccount["private_key"];
args.clientEmail = serviceAccount.client_email;
args.privateKey = serviceAccount.private_key;

Check warning on line 182 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L181-L182

Added lines #L181 - L182 were not covered by tests
}
}
if (!(args.clientEmail && args.privateKey)) {

if (!args.bucket) throw new UserError("Error: must provide --bucket.");
if (!args.clientEmail)
throw new UserError(
`Error: must provide --service-account-key-file or --client-email and --private-key.`
"Error: must provide --service-account-key-file or --client-email."
);
if (!args.privateKey)
throw new UserError(

Check warning on line 192 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L192

Added line #L192 was not covered by tests
"Error: must provide --service-account-key-file or --private-key."
);
}
args.privateKey = args.privateKey.replace(/\\n/g, "\n");

if (!args.r2AccessKeyId)
throw new UserError("Error: must provide --r2-access-key-id.");

Check warning on line 198 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L198

Added line #L198 was not covered by tests
if (!args.r2SecretAccessKey)
throw new UserError("Error: must provide --r2-secret-access-key.");

Check warning on line 200 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L200

Added line #L200 was not covered by tests

sippyConfig = {
...sippyConfig,
provider: "GCS",
client_email: args.clientEmail,
private_key: args.privateKey,
source: {
provider: "gcs",
bucket: args.bucket,
clientEmail: args.clientEmail,
privateKey: args.privateKey,
},
destination: {
provider: "r2",
accessKeyId: args.r2AccessKeyId,
secretAccessKey: args.r2SecretAccessKey,
},
};
} else {
throw new UserError(

Check warning on line 216 in packages/wrangler/src/r2/sippy.ts

View check run for this annotation

Codecov / codecov/patch

packages/wrangler/src/r2/sippy.ts#L215-L216

Added lines #L215 - L216 were not covered by tests
"Error: unrecognized provider. Possible options are AWS & GCS."
);
}

await putR2Sippy(accountId, args.name, sippyConfig, args.jurisdiction);
Expand Down Expand Up @@ -211,14 +245,14 @@ export async function GetHandler(
const accountId = await requireAuth(config);

try {
const sippyBucket = await getR2Sippy(
const sippyConfig = await getR2Sippy(
accountId,
args.name,
args.jurisdiction
);
logger.log(`Sippy upstream bucket: ${sippyBucket}.`);
logger.log("Sippy configuration:", sippyConfig);
} catch (e) {
if (e instanceof APIError && "code" in e && e.code == NO_SUCH_OBJECT_KEY) {
if (e instanceof APIError && "code" in e && e.code === NO_SUCH_OBJECT_KEY) {
logger.log(`No Sippy configuration found for the '${args.name}' bucket.`);
} else {
throw e;
Expand Down

0 comments on commit 4a735c4

Please sign in to comment.