From fa317001afe18327d862c4ab58c27855228f1133 Mon Sep 17 00:00:00 2001 From: Phillip Jones Date: Sat, 9 Nov 2024 13:51:46 -0800 Subject: [PATCH] Added wrangler r2 bucket info command, improved formatting of output for r2 bucket list command --- .changeset/tame-bobcats-suffer.md | 5 + packages/wrangler/src/__tests__/r2.test.ts | 121 +++++++++++++------ packages/wrangler/src/r2/helpers.ts | 134 ++++++++++++++++++++- packages/wrangler/src/r2/index.ts | 43 +++---- packages/wrangler/src/r2/info.ts | 50 ++++++++ packages/wrangler/src/r2/list.ts | 35 ++++++ 6 files changed, 320 insertions(+), 68 deletions(-) create mode 100644 .changeset/tame-bobcats-suffer.md create mode 100644 packages/wrangler/src/r2/info.ts create mode 100644 packages/wrangler/src/r2/list.ts diff --git a/.changeset/tame-bobcats-suffer.md b/.changeset/tame-bobcats-suffer.md new file mode 100644 index 000000000000..e0705ebad8bd --- /dev/null +++ b/.changeset/tame-bobcats-suffer.md @@ -0,0 +1,5 @@ +--- +"wrangler": minor +--- + +Added r2 bucket info command to Wrangler. Improved formatting of r2 bucket list output diff --git a/packages/wrangler/src/__tests__/r2.test.ts b/packages/wrangler/src/__tests__/r2.test.ts index 5eca6e3ff7db..8cd3d01e7893 100644 --- a/packages/wrangler/src/__tests__/r2.test.ts +++ b/packages/wrangler/src/__tests__/r2.test.ts @@ -12,7 +12,6 @@ import { runInTempDir } from "./helpers/run-in-tmp"; import { runWrangler } from "./helpers/run-wrangler"; import type { PutNotificationRequestBody, - R2BucketInfo, R2EventableOperation, R2EventType, } from "../r2/helpers"; @@ -92,14 +91,15 @@ describe("r2", () => { Manage R2 buckets COMMANDS - wrangler r2 bucket create Create a new R2 bucket - wrangler r2 bucket update Update bucket state - wrangler r2 bucket list List R2 buckets - wrangler r2 bucket delete Delete an R2 bucket - wrangler r2 bucket sippy Manage Sippy incremental migration on an R2 bucket - wrangler r2 bucket notification Manage event notification rules for an R2 bucket - wrangler r2 bucket domain Manage custom domains for an R2 bucket - wrangler r2 bucket dev-url Manage public access via the r2.dev URL for an R2 bucket + wrangler r2 bucket create Create a new R2 bucket + wrangler r2 bucket update Update bucket state + wrangler r2 bucket list List R2 buckets + wrangler r2 bucket info Get information about an R2 bucket + wrangler r2 bucket delete Delete an R2 bucket + wrangler r2 bucket sippy Manage Sippy incremental migration on an R2 bucket + wrangler r2 bucket notification Manage event notification rules for an R2 bucket + wrangler r2 bucket domain Manage custom domains for an R2 bucket + wrangler r2 bucket dev-url Manage public access via the r2.dev URL for an R2 bucket GLOBAL FLAGS -j, --experimental-json-config Experimental: support wrangler.json [boolean] @@ -128,14 +128,15 @@ describe("r2", () => { Manage R2 buckets COMMANDS - wrangler r2 bucket create Create a new R2 bucket - wrangler r2 bucket update Update bucket state - wrangler r2 bucket list List R2 buckets - wrangler r2 bucket delete Delete an R2 bucket - wrangler r2 bucket sippy Manage Sippy incremental migration on an R2 bucket - wrangler r2 bucket notification Manage event notification rules for an R2 bucket - wrangler r2 bucket domain Manage custom domains for an R2 bucket - wrangler r2 bucket dev-url Manage public access via the r2.dev URL for an R2 bucket + wrangler r2 bucket create Create a new R2 bucket + wrangler r2 bucket update Update bucket state + wrangler r2 bucket list List R2 buckets + wrangler r2 bucket info Get information about an R2 bucket + wrangler r2 bucket delete Delete an R2 bucket + wrangler r2 bucket sippy Manage Sippy incremental migration on an R2 bucket + wrangler r2 bucket notification Manage event notification rules for an R2 bucket + wrangler r2 bucket domain Manage custom domains for an R2 bucket + wrangler r2 bucket dev-url Manage public access via the r2.dev URL for an R2 bucket GLOBAL FLAGS -j, --experimental-json-config Experimental: support wrangler.json [boolean] @@ -148,9 +149,15 @@ describe("r2", () => { describe("list", () => { it("should list buckets & check request inputs", async () => { - const expectedBuckets: R2BucketInfo[] = [ - { name: "bucket-1-local-once", creation_date: "01-01-2001" }, - { name: "bucket-2-local-once", creation_date: "01-01-2001" }, + const mockBuckets = [ + { + name: "bucket-1-local-once", + creation_date: "01-01-2001", + }, + { + name: "bucket-2-local-once", + creation_date: "01-01-2001", + }, ]; msw.use( http.get( @@ -161,27 +168,65 @@ describe("r2", () => { expect(await request.text()).toEqual(""); return HttpResponse.json( createFetchResult({ - buckets: [ - { - name: "bucket-1-local-once", - creation_date: "01-01-2001", - }, - { - name: "bucket-2-local-once", - creation_date: "01-01-2001", - }, - ], + buckets: mockBuckets, }) ); }, { once: true } ) ); - await runWrangler("r2 bucket list"); - expect(std.err).toMatchInlineSnapshot(`""`); - const buckets = JSON.parse(std.out); - expect(buckets).toEqual(expectedBuckets); + await runWrangler(`r2 bucket list`); + expect(std.out).toMatchInlineSnapshot(` + "Listing buckets... + name: bucket-1-local-once + creation_date: 01-01-2001 + + name: bucket-2-local-once + creation_date: 01-01-2001" + `); + }); + }); + + describe("info", () => { + it("should get information for the given bucket", async () => { + const bucketName = "my-bucket"; + const bucketInfo = { + name: bucketName, + creation_date: "01-01-2001", + location: "WNAM", + storage_class: "Standard", + }; + + msw.use( + http.get( + "*/accounts/:accountId/r2/buckets/:bucketName", + async ({ params }) => { + const { accountId, bucketName: bucketParam } = params; + expect(accountId).toEqual("some-account-id"); + expect(bucketParam).toEqual(bucketName); + return HttpResponse.json( + createFetchResult({ + ...bucketInfo, + }) + ); + }, + { once: true } + ), + http.post("*/graphql", async () => { + return HttpResponse.json(createFetchResult({})); + }) + ); + await runWrangler(`r2 bucket info ${bucketName}`); + expect(std.out).toMatchInlineSnapshot(` + "Getting info for 'my-bucket'... + name: my-bucket + created: 01-01-2001 + location: WNAM + default_storage_class: Standard + object_count: 0 + bucket_size: 0 B" + `); }); }); @@ -475,12 +520,12 @@ binding = \\"testBucket\\"" ); expect(std.out).toMatchInlineSnapshot(` " - wrangler r2 bucket delete + wrangler r2 bucket delete Delete an R2 bucket POSITIONALS - name The name of the bucket to delete [string] [required] + bucket The name of the bucket to delete [string] [required] GLOBAL FLAGS -j, --experimental-json-config Experimental: support wrangler.json [boolean] @@ -515,12 +560,12 @@ binding = \\"testBucket\\"" ); expect(std.out).toMatchInlineSnapshot(` " - wrangler r2 bucket delete + wrangler r2 bucket delete Delete an R2 bucket POSITIONALS - name The name of the bucket to delete [string] [required] + bucket The name of the bucket to delete [string] [required] GLOBAL FLAGS -j, --experimental-json-config Experimental: support wrangler.json [boolean] diff --git a/packages/wrangler/src/r2/helpers.ts b/packages/wrangler/src/r2/helpers.ts index 6e0993efac9e..d99b641857c6 100644 --- a/packages/wrangler/src/r2/helpers.ts +++ b/packages/wrangler/src/r2/helpers.ts @@ -1,5 +1,6 @@ import { Miniflare } from "miniflare"; -import { fetchResult } from "../cfetch"; +import prettyBytes from "pretty-bytes"; +import { fetchGraphqlResult, fetchResult } from "../cfetch"; import { fetchR2Objects } from "../cfetch/internal"; import { getLocalPersistencePath } from "../dev/get-local-persistence-path"; import { buildPersistOptions } from "../dev/miniflare"; @@ -20,6 +21,29 @@ import type { HeadersInit } from "undici"; export interface R2BucketInfo { name: string; creation_date: string; + location?: string; + storage_class?: string; +} + +export interface R2BucketMetrics { + max?: { + objectCount?: number; + payloadSize?: number; + metadataSize?: number; + }; + dimensions: { + datetime?: string; + }; +} + +export interface R2BucketMetricsGraphQLResponse { + data: { + viewer: { + accounts: { + r2StorageAdaptiveGroups?: R2BucketMetrics[]; + }[]; + }; + }; } /** @@ -39,6 +63,114 @@ export async function listR2Buckets( return results.buckets; } +export function tablefromR2BucketsListResponse(buckets: R2BucketInfo[]): { + name: string; + creation_date: string; +}[] { + const rows = []; + for (const bucket of buckets) { + rows.push({ + name: bucket.name, + creation_date: bucket.creation_date, + }); + } + return rows; +} + +export async function getR2Bucket( + accountId: string, + bucketName: string, + jurisdiction?: string +): Promise { + const headers: HeadersInit = {}; + if (jurisdiction !== undefined) { + headers["cf-r2-jurisdiction"] = jurisdiction; + } + const result = await fetchResult( + `/accounts/${accountId}/r2/buckets/${bucketName}`, + { + method: "GET", + headers, + } + ); + return result; +} + +export async function getR2BucketMetrics( + accountId: string, + bucketName: string, + jurisdiction?: string +): Promise<{ objectCount: number; totalSize: string }> { + const today = new Date(); + const yesterday = new Date(new Date(today).setDate(today.getDate() - 1)); + + let fullBucketName = bucketName; + if (jurisdiction) { + fullBucketName = `${jurisdiction}_${bucketName}`; + } + + const storageMetricsQuery = ` + query getR2StorageMetrics($accountTag: String, $filter: R2StorageAdaptiveGroupsFilter_InputObject) { + viewer { + accounts(filter: { accountTag: $accountTag }) { + r2StorageAdaptiveGroups( + limit: 1 + filter: $filter + orderBy: [datetime_DESC] + ) { + max { + objectCount + payloadSize + metadataSize + } + dimensions { + datetime + } + } + } + } + } + `; + + const variables = { + accountTag: accountId, + filter: { + datetime_geq: yesterday.toISOString(), + datetime_leq: today.toISOString(), + bucketName: fullBucketName, + }, + }; + const storageMetricsResult = + await fetchGraphqlResult({ + method: "POST", + body: JSON.stringify({ + query: storageMetricsQuery, + operationName: "getR2StorageMetrics", + variables, + }), + headers: { + "Content-Type": "application/json", + }, + }); + + if (storageMetricsResult) { + const metricsData = + storageMetricsResult.data?.viewer?.accounts[0] + ?.r2StorageAdaptiveGroups?.[0]; + if (metricsData && metricsData.max) { + const objectCount = metricsData.max.objectCount || 0; + const totalSize = + (metricsData.max.payloadSize || 0) + + (metricsData.max.metadataSize || 0); + return { + objectCount, + totalSize: prettyBytes(totalSize), + }; + } + } + return { objectCount: 0, totalSize: "0 B" }; +} + /** * Create a bucket with the given `bucketName` within the account given by `accountId`. * diff --git a/packages/wrangler/src/r2/index.ts b/packages/wrangler/src/r2/index.ts index a37ec0c64786..4d4398b833df 100644 --- a/packages/wrangler/src/r2/index.ts +++ b/packages/wrangler/src/r2/index.ts @@ -18,11 +18,12 @@ import { deleteR2Bucket, deleteR2Object, getR2Object, - listR2Buckets, putR2Object, updateR2BucketStorageClass, usingLocalBucket, } from "./helpers"; +import * as Info from "./info"; +import * as List from "./list"; import * as Notification from "./notification"; import * as PublicDevUrl from "./public-dev-url"; import * as Sippy from "./sippy"; @@ -494,39 +495,23 @@ export function r2(r2Yargs: CommonYargsArgv, subHelp: SubHelp) { r2BucketYargs.command( "list", "List R2 buckets", - (listArgs) => { - return listArgs.option("jurisdiction", { - describe: "The jurisdiction to list", - alias: "J", - requiresArg: true, - type: "string", - }); - }, - async (args) => { - const config = readConfig(args.config, args); - const { jurisdiction } = args; - - const accountId = await requireAuth(config); + List.ListOptions, + List.ListHandler + ); - logger.log( - JSON.stringify( - await listR2Buckets(accountId, jurisdiction), - null, - 2 - ) - ); - await metrics.sendMetricsEvent("list r2 buckets", { - sendMetrics: config.send_metrics, - }); - } + r2BucketYargs.command( + "info ", + "Get information about an R2 bucket", + Info.InfoOptions, + Info.InfoHandler ); r2BucketYargs.command( - "delete ", + "delete ", "Delete an R2 bucket", (yargs) => { return yargs - .positional("name", { + .positional("bucket", { describe: "The name of the bucket to delete", type: "string", demandOption: true, @@ -545,12 +530,12 @@ export function r2(r2Yargs: CommonYargsArgv, subHelp: SubHelp) { const accountId = await requireAuth(config); - let fullBucketName = `${args.name}`; + let fullBucketName = `${args.bucket}`; if (args.jurisdiction !== undefined) { fullBucketName += ` (${args.jurisdiction})`; } logger.log(`Deleting bucket ${fullBucketName}.`); - await deleteR2Bucket(accountId, args.name, args.jurisdiction); + await deleteR2Bucket(accountId, args.bucket, args.jurisdiction); logger.log(`Deleted bucket ${fullBucketName}.`); await metrics.sendMetricsEvent("delete r2 bucket", { sendMetrics: config.send_metrics, diff --git a/packages/wrangler/src/r2/info.ts b/packages/wrangler/src/r2/info.ts new file mode 100644 index 000000000000..543eda4f1868 --- /dev/null +++ b/packages/wrangler/src/r2/info.ts @@ -0,0 +1,50 @@ +import { readConfig } from "../config"; +import { logger } from "../logger"; +import { printWranglerBanner } from "../update-check"; +import { requireAuth } from "../user"; +import formatLabelledValues from "../utils/render-labelled-values"; +import { getR2Bucket, getR2BucketMetrics } from "./helpers"; +import type { + CommonYargsArgv, + StrictYargsOptionsToInterface, +} from "../yargs-types"; + +export function InfoOptions(yargs: CommonYargsArgv) { + return yargs + .positional("bucket", { + describe: "The name of the R2 bucket to get information about", + type: "string", + demandOption: true, + }) + .option("jurisdiction", { + describe: "The jurisdiction where the bucket exists", + alias: "J", + requiresArg: true, + type: "string", + }); +} + +export async function InfoHandler( + args: StrictYargsOptionsToInterface +) { + await printWranglerBanner(); + const config = readConfig(args.config, args); + const accountId = await requireAuth(config); + + const { bucket, jurisdiction } = args; + + logger.log(`Getting info for '${bucket}'...`); + const bucketInfo = await getR2Bucket(accountId, bucket, jurisdiction); + const metrics = await getR2BucketMetrics(accountId, bucket, jurisdiction); + + const output = { + name: bucketInfo.name, + created: bucketInfo.creation_date, + location: bucketInfo.location || "(unknown)", + default_storage_class: bucketInfo.storage_class || "(unknown)", + object_count: metrics.objectCount.toLocaleString(), + bucket_size: metrics.totalSize, + }; + + logger.log(formatLabelledValues(output)); +} diff --git a/packages/wrangler/src/r2/list.ts b/packages/wrangler/src/r2/list.ts new file mode 100644 index 000000000000..08bdd05b7316 --- /dev/null +++ b/packages/wrangler/src/r2/list.ts @@ -0,0 +1,35 @@ +import { readConfig } from "../config"; +import { logger } from "../logger"; +import { printWranglerBanner } from "../update-check"; +import { requireAuth } from "../user"; +import formatLabelledValues from "../utils/render-labelled-values"; +import { listR2Buckets, tablefromR2BucketsListResponse } from "./helpers"; +import type { + CommonYargsArgv, + StrictYargsOptionsToInterface, +} from "../yargs-types"; + +export function ListOptions(yargs: CommonYargsArgv) { + return yargs.option("jurisdiction", { + describe: "The jurisdiction to list", + alias: "J", + requiresArg: true, + type: "string", + }); +} + +export async function ListHandler( + args: StrictYargsOptionsToInterface +) { + await printWranglerBanner(); + const config = readConfig(args.config, args); + const accountId = await requireAuth(config); + + const { jurisdiction } = args; + + logger.log(`Listing buckets...`); + + const buckets = await listR2Buckets(accountId, jurisdiction); + const tableOutput = tablefromR2BucketsListResponse(buckets); + logger.log(tableOutput.map((x) => formatLabelledValues(x)).join("\n\n")); +}