From 3e2349977a70bb404229703c6fac5810972f6a12 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 14 Mar 2023 11:55:08 +0100 Subject: [PATCH] feat: ucan stream consumer store/add and store/remove count (#151) Part of metrics work https://github.com/web3-storage/w3infra/issues/117 Adds system total metrics for `store/add` and `store/remove`. With these, we can track number of CARs that we commited to store, and how many were removed. Other notes: - we will still need to collect the effective number of CARs that are stored given this metrics is just number of signed URLs we provided in `store/add`. - helper functions for database were moved around to be re-used in all tests for metrics --- stacks/ucan-invocation-stack.js | 39 ++++ test/integration.test.js | 4 + ucan-invocation/constants.js | 11 +- .../functions/metrics-store-add-total.js | 46 +++++ .../functions/metrics-store-remove-total.js | 46 +++++ ucan-invocation/tables/metrics.js | 44 +++++ ...s => metrics-store-add-size-total.test.js} | 53 ++---- .../functions/metrics-store-add-total.test.js | 166 ++++++++++++++++++ .../metrics-store-remove-total.test.js | 166 ++++++++++++++++++ ucan-invocation/test/helpers/tables.js | 41 +++++ ucan-invocation/types.ts | 2 + 11 files changed, 573 insertions(+), 45 deletions(-) create mode 100644 ucan-invocation/functions/metrics-store-add-total.js create mode 100644 ucan-invocation/functions/metrics-store-remove-total.js rename ucan-invocation/test/functions/{metrics-size-total.test.js => metrics-store-add-size-total.test.js} (70%) create mode 100644 ucan-invocation/test/functions/metrics-store-add-total.test.js create mode 100644 ucan-invocation/test/functions/metrics-store-remove-total.test.js create mode 100644 ucan-invocation/test/helpers/tables.js diff --git a/stacks/ucan-invocation-stack.js b/stacks/ucan-invocation-stack.js index e5f82bfa..50d43e90 100644 --- a/stacks/ucan-invocation-stack.js +++ b/stacks/ucan-invocation-stack.js @@ -52,6 +52,29 @@ export function UcanInvocationStack({ stack, app }) { } }) + // metrics store/add count + const metricsStoreAddTotalDLQ = new Queue(stack, 'metrics-store-add-total-dlq') + const metricsStoreAddTotalConsumer = new Function(stack, 'metrics-store-add-total-consumer', { + environment: { + TABLE_NAME: adminMetricsTable.tableName + }, + permissions: [adminMetricsTable], + handler: 'functions/metrics-store-add-total.consumer', + deadLetterQueue: metricsStoreAddTotalDLQ.cdk.queue, + }) + + // metrics store/remove count + const metricsStoreRemoveTotalDLQ = new Queue(stack, 'metrics-store-remove-total-dlq') + const metricsStoreRemoveTotalConsumer = new Function(stack, 'metrics-store-remove-total-consumer', { + environment: { + TABLE_NAME: adminMetricsTable.tableName + }, + permissions: [adminMetricsTable], + handler: 'functions/metrics-store-remove-total.consumer', + deadLetterQueue: metricsStoreRemoveTotalDLQ.cdk.queue, + }) + + // metrics store/add size total const metricsStoreAddSizeTotalDLQ = new Queue(stack, 'metrics-store-add-size-total-dlq') const metricsStoreAddSizeTotalConsumer = new Function(stack, 'metrics-store-add-size-total-consumer', { environment: { @@ -83,6 +106,22 @@ export function UcanInvocationStack({ stack, app }) { } }, consumers: { + metricsStoreAddTotalConsumer: { + function: metricsStoreAddTotalConsumer, + cdk: { + eventSource: { + ...(getKinesisEventSourceConfig(stack)) + } + } + }, + metricsStoreRemoveTotalConsumer: { + function: metricsStoreRemoveTotalConsumer, + cdk: { + eventSource: { + ...(getKinesisEventSourceConfig(stack)) + } + } + }, metricsStoreAddSizeTotalConsumer: { function: metricsStoreAddSizeTotalConsumer, // TODO: Set kinesis filters when supported by SST diff --git a/test/integration.test.js b/test/integration.test.js index 78ed9b64..4735487d 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -62,6 +62,7 @@ test('w3infra integration flow', async t => { // Get metrics before upload const beforeOperationMetrics = await getMetrics(t) + const beforeStoreAddTotal = beforeOperationMetrics.find(row => row.name === METRICS_NAMES.STORE_ADD_TOTAL) const beforeStoreAddSizeTotal = beforeOperationMetrics.find(row => row.name === METRICS_NAMES.STORE_ADD_SIZE_TOTAL) const s3Client = getAwsBucketClient() @@ -173,18 +174,21 @@ test('w3infra integration flow', async t => { if (beforeStoreAddSizeTotal && spaceBeforeUploadAddMetrics) { await pWaitFor(async () => { const afterOperationMetrics = await getMetrics(t) + const afterStoreAddTotal = afterOperationMetrics.find(row => row.name === METRICS_NAMES.STORE_ADD_TOTAL) const afterStoreAddSizeTotal = afterOperationMetrics.find(row => row.name === METRICS_NAMES.STORE_ADD_SIZE_TOTAL) const spaceAfterUploadAddMetrics = await getSpaceMetrics(t, spaceDid, SPACE_METRICS_NAMES.UPLOAD_ADD_TOTAL) // If staging accept more broad condition given multiple parallel tests can happen there if (stage === 'staging') { return ( + afterStoreAddTotal?.value >= beforeStoreAddTotal?.value + carSize && afterStoreAddSizeTotal?.value >= beforeStoreAddSizeTotal.value + carSize && spaceAfterUploadAddMetrics?.value >= spaceBeforeUploadAddMetrics?.value + 1 ) } return ( + afterStoreAddTotal?.value === beforeStoreAddTotal?.value + 1 && afterStoreAddSizeTotal?.value === beforeStoreAddSizeTotal.value + carSize && spaceAfterUploadAddMetrics?.value === spaceBeforeUploadAddMetrics?.value + 1 ) diff --git a/ucan-invocation/constants.js b/ucan-invocation/constants.js index c0362d51..5fffdf55 100644 --- a/ucan-invocation/constants.js +++ b/ucan-invocation/constants.js @@ -1,13 +1,20 @@ // UCAN protocol export const STORE_ADD = 'store/add' +export const STORE_REMOVE = 'store/remove' export const UPLOAD_ADD = 'upload/add' // Admin Metrics export const METRICS_NAMES = { - STORE_ADD_SIZE_TOTAL: `${STORE_ADD}-size-total` + UPLOAD_ADD_TOTAL: `${UPLOAD_ADD}-total`, + STORE_ADD_TOTAL: `${STORE_ADD}-total`, + STORE_ADD_SIZE_TOTAL: `${STORE_ADD}-size-total`, + STORE_REMOVE_TOTAL: `${STORE_REMOVE}-total`, } // Spade Metrics export const SPACE_METRICS_NAMES = { - UPLOAD_ADD_TOTAL: `${UPLOAD_ADD}-total` + UPLOAD_ADD_TOTAL: `${UPLOAD_ADD}-total`, + STORE_ADD_TOTAL: `${STORE_ADD}-total`, + STORE_ADD_SIZE_TOTAL: `${STORE_ADD}-size-total`, + STORE_REMOVE_TOTAL: `${STORE_REMOVE}-total`, } diff --git a/ucan-invocation/functions/metrics-store-add-total.js b/ucan-invocation/functions/metrics-store-add-total.js new file mode 100644 index 00000000..205e8e20 --- /dev/null +++ b/ucan-invocation/functions/metrics-store-add-total.js @@ -0,0 +1,46 @@ +import * as Sentry from '@sentry/serverless' + +import { createMetricsTable } from '../tables/metrics.js' +import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' +import { STORE_ADD } from '../constants.js' + +Sentry.AWSLambda.init({ + environment: process.env.SST_STAGE, + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 1.0, +}) + +const AWS_REGION = process.env.AWS_REGION || 'us-west-2' + +/** + * @param {import('aws-lambda').KinesisStreamEvent} event + */ +async function handler(event) { + const ucanInvocations = parseKinesisEvent(event) + + const { + TABLE_NAME: tableName = '', + // set for testing + DYNAMO_DB_ENDPOINT: dbEndpoint, + } = process.env + + await updateStoreAddTotal(ucanInvocations, { + metricsTable: createMetricsTable(AWS_REGION, tableName, { + endpoint: dbEndpoint + }) + }) +} + +/** + * @param {import('../types').UcanInvocation[]} ucanInvocations + * @param {import('../types').TotalSizeCtx} ctx + */ +export async function updateStoreAddTotal (ucanInvocations, ctx) { + const invocationsWithStoreAdd = ucanInvocations.filter( + inv => inv.value.att.find(a => a.can === STORE_ADD) + ).flatMap(inv => inv.value.att) + + await ctx.metricsTable.incrementStoreAddTotal(invocationsWithStoreAdd) +} + +export const consumer = Sentry.AWSLambda.wrapHandler(handler) diff --git a/ucan-invocation/functions/metrics-store-remove-total.js b/ucan-invocation/functions/metrics-store-remove-total.js new file mode 100644 index 00000000..3728d906 --- /dev/null +++ b/ucan-invocation/functions/metrics-store-remove-total.js @@ -0,0 +1,46 @@ +import * as Sentry from '@sentry/serverless' + +import { createMetricsTable } from '../tables/metrics.js' +import { parseKinesisEvent } from '../utils/parse-kinesis-event.js' +import { STORE_REMOVE } from '../constants.js' + +Sentry.AWSLambda.init({ + environment: process.env.SST_STAGE, + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 1.0, +}) + +const AWS_REGION = process.env.AWS_REGION || 'us-west-2' + +/** + * @param {import('aws-lambda').KinesisStreamEvent} event + */ +async function handler(event) { + const ucanInvocations = parseKinesisEvent(event) + + const { + TABLE_NAME: tableName = '', + // set for testing + DYNAMO_DB_ENDPOINT: dbEndpoint, + } = process.env + + await updateStoreRemoveTotal(ucanInvocations, { + metricsTable: createMetricsTable(AWS_REGION, tableName, { + endpoint: dbEndpoint + }) + }) +} + +/** + * @param {import('../types').UcanInvocation[]} ucanInvocations + * @param {import('../types').TotalSizeCtx} ctx + */ +export async function updateStoreRemoveTotal (ucanInvocations, ctx) { + const invocationsWithStoreRemove = ucanInvocations.filter( + inv => inv.value.att.find(a => a.can === STORE_REMOVE) + ).flatMap(inv => inv.value.att) + + await ctx.metricsTable.incrementStoreRemoveTotal(invocationsWithStoreRemove) +} + +export const consumer = Sentry.AWSLambda.wrapHandler(handler) diff --git a/ucan-invocation/tables/metrics.js b/ucan-invocation/tables/metrics.js index b43976b3..e2ad5e18 100644 --- a/ucan-invocation/tables/metrics.js +++ b/ucan-invocation/tables/metrics.js @@ -31,6 +31,28 @@ export function createMetricsTable (region, tableName, options = {}) { }) return { + /** + * Increment total count from store/add operations. + * + * @param {Capabilities} operationsInv + */ + incrementStoreAddTotal: async (operationsInv) => { + const invTotalSize = operationsInv.length + + const updateCmd = new UpdateItemCommand({ + TableName: tableName, + UpdateExpression: `ADD #value :value`, + ExpressionAttributeNames: {'#value': 'value'}, + ExpressionAttributeValues: { + ':value': { N: String(invTotalSize) }, + }, + Key: marshall({ + name: METRICS_NAMES.STORE_ADD_TOTAL + }) + }) + + await dynamoDb.send(updateCmd) + }, /** * Increment total value from new given operations. * @@ -52,6 +74,28 @@ export function createMetricsTable (region, tableName, options = {}) { }) }) + await dynamoDb.send(updateCmd) + }, + /** + * Increment total count from store/remove operations. + * + * @param {Capabilities} operationsInv + */ + incrementStoreRemoveTotal: async (operationsInv) => { + const invTotalSize = operationsInv.length + + const updateCmd = new UpdateItemCommand({ + TableName: tableName, + UpdateExpression: `ADD #value :value`, + ExpressionAttributeNames: {'#value': 'value'}, + ExpressionAttributeValues: { + ':value': { N: String(invTotalSize) }, + }, + Key: marshall({ + name: METRICS_NAMES.STORE_REMOVE_TOTAL + }) + }) + await dynamoDb.send(updateCmd) } } diff --git a/ucan-invocation/test/functions/metrics-size-total.test.js b/ucan-invocation/test/functions/metrics-store-add-size-total.test.js similarity index 70% rename from ucan-invocation/test/functions/metrics-size-total.test.js rename to ucan-invocation/test/functions/metrics-store-add-size-total.test.js index 3e5c516d..d1c6384d 100644 --- a/ucan-invocation/test/functions/metrics-size-total.test.js +++ b/ucan-invocation/test/functions/metrics-store-add-size-total.test.js @@ -1,15 +1,12 @@ import { testConsumer as test } from '../helpers/context.js' -import { customAlphabet } from 'nanoid' -import { CreateTableCommand, GetItemCommand } from '@aws-sdk/client-dynamodb' -import { marshall, unmarshall } from '@aws-sdk/util-dynamodb' import * as Signer from '@ucanto/principal/ed25519' import * as StoreCapabilities from '@web3-storage/capabilities/store' -import { adminMetricsTableProps } from '../../tables/index.js' -import { createDynamodDb, dynamoDBTableConfig } from '../helpers/resources.js' +import { createDynamodDb } from '../helpers/resources.js' import { createSpace } from '../helpers/ucanto.js' import { randomCAR } from '../helpers/random.js' +import { createDynamoAdminMetricsTable, getItemFromTable} from '../helpers/tables.js' import { updateSizeTotal } from '../../functions/metrics-store-add-size-total.js' import { createMetricsTable } from '../../tables/metrics.js' @@ -62,7 +59,9 @@ test('handles a batch of single invocation with store/add', async t => { metricsTable }) - const item = await getItemFromTable(t.context.dynamoClient, tableName, METRICS_NAMES.STORE_ADD_SIZE_TOTAL) + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_SIZE_TOTAL + }) t.truthy(item) t.is(item?.name, METRICS_NAMES.STORE_ADD_SIZE_TOTAL) t.is(item?.value, car.size) @@ -103,7 +102,10 @@ test('handles batch of single invocations with multiple store/add attributes', a metricsTable }) - const item = await getItemFromTable(t.context.dynamoClient, tableName, METRICS_NAMES.STORE_ADD_SIZE_TOTAL) + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_SIZE_TOTAL + }) + t.truthy(item) t.is(item?.name, METRICS_NAMES.STORE_ADD_SIZE_TOTAL) t.is(item?.value, cars.reduce((acc, c) => acc + c.size, 0)) @@ -114,45 +116,10 @@ test('handles batch of single invocations with multiple store/add attributes', a */ async function prepareResources (dynamoClient) { const [ tableName ] = await Promise.all([ - createDynamouploadTable(dynamoClient), + createDynamoAdminMetricsTable(dynamoClient), ]) return { tableName } } - -/** - * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamo - */ -async function createDynamouploadTable(dynamo) { - const id = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 10) - const tableName = id() - - await dynamo.send(new CreateTableCommand({ - TableName: tableName, - ...dynamoDBTableConfig(adminMetricsTableProps), - ProvisionedThroughput: { - ReadCapacityUnits: 1, - WriteCapacityUnits: 1 - } - })) - - return tableName -} - -/** - * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamo - * @param {string} tableName - * @param {string} name - */ -async function getItemFromTable(dynamo, tableName, name) { - const params = { - TableName: tableName, - Key: marshall({ - name, - }) - } - const response = await dynamo.send(new GetItemCommand(params)) - return response?.Item && unmarshall(response?.Item) -} \ No newline at end of file diff --git a/ucan-invocation/test/functions/metrics-store-add-total.test.js b/ucan-invocation/test/functions/metrics-store-add-total.test.js new file mode 100644 index 00000000..2261fcbf --- /dev/null +++ b/ucan-invocation/test/functions/metrics-store-add-total.test.js @@ -0,0 +1,166 @@ +import { testConsumer as test } from '../helpers/context.js' + +import * as Signer from '@ucanto/principal/ed25519' +import * as StoreCapabilities from '@web3-storage/capabilities/store' + +import { createDynamodDb } from '../helpers/resources.js' +import { createSpace } from '../helpers/ucanto.js' +import { randomCAR } from '../helpers/random.js' +import { createDynamoAdminMetricsTable, getItemFromTable} from '../helpers/tables.js' + +import { updateStoreAddTotal } from '../../functions/metrics-store-add-total.js' +import { createMetricsTable } from '../../tables/metrics.js' +import { METRICS_NAMES } from '../../constants.js' + +const REGION = 'us-west-2' + +test.before(async t => { + // Dynamo DB + const { + client: dynamo, + endpoint: dbEndpoint + } = await createDynamodDb({ port: 8000 }) + + t.context.dbEndpoint = dbEndpoint + t.context.dynamoClient = dynamo +}) + +test('handles a batch of single invocation with store/add', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.add.create({ + with: spaceDid, + nb: { + link: car.cid, + size: car.size + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreAddTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_TOTAL + }) + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_ADD_TOTAL) + t.is(item?.value, 1) +}) + +test('handles batch of single invocations with multiple store/add attributes', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + + const cars = await Promise.all( + Array.from({ length: 10 }).map(() => randomCAR(128)) + ) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: cars[0].cid.toString(), + value: { + att: cars.map((car) => StoreCapabilities.add.create({ + with: spaceDid, + nb: { + link: car.cid, + size: car.size + } + })), + aud: uploadService.did(), + iss: alice.did() + }, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreAddTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_ADD_TOTAL) + t.is(item?.value, cars.length) +}) + +test('handles a batch of single invocation without store/add', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.remove.create({ + with: spaceDid, + nb: { + link: car.cid, + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreAddTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_ADD_TOTAL + }) + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_ADD_TOTAL) + t.is(item?.value, 0) +}) + +/** + * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamoClient + */ +async function prepareResources (dynamoClient) { + const [ tableName ] = await Promise.all([ + createDynamoAdminMetricsTable(dynamoClient), + ]) + + return { + tableName + } +} diff --git a/ucan-invocation/test/functions/metrics-store-remove-total.test.js b/ucan-invocation/test/functions/metrics-store-remove-total.test.js new file mode 100644 index 00000000..6ccaf1af --- /dev/null +++ b/ucan-invocation/test/functions/metrics-store-remove-total.test.js @@ -0,0 +1,166 @@ +import { testConsumer as test } from '../helpers/context.js' + +import * as Signer from '@ucanto/principal/ed25519' +import * as StoreCapabilities from '@web3-storage/capabilities/store' + +import { createDynamodDb } from '../helpers/resources.js' +import { createSpace } from '../helpers/ucanto.js' +import { randomCAR } from '../helpers/random.js' +import { createDynamoAdminMetricsTable, getItemFromTable} from '../helpers/tables.js' + +import { updateStoreRemoveTotal } from '../../functions/metrics-store-remove-total.js' +import { createMetricsTable } from '../../tables/metrics.js' +import { METRICS_NAMES } from '../../constants.js' + +const REGION = 'us-west-2' + +test.before(async t => { + // Dynamo DB + const { + client: dynamo, + endpoint: dbEndpoint + } = await createDynamodDb({ port: 8000 }) + + t.context.dbEndpoint = dbEndpoint + t.context.dynamoClient = dynamo +}) + +test('handles a batch of single invocation with store/remove', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.remove.create({ + with: spaceDid, + nb: { + link: car.cid + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreRemoveTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_REMOVE_TOTAL + }) + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_REMOVE_TOTAL) + t.is(item?.value, 1) +}) + +test('handles batch of single invocations with multiple store/remove attributes', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + + const cars = await Promise.all( + Array.from({ length: 10 }).map(() => randomCAR(128)) + ) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: cars[0].cid.toString(), + value: { + att: cars.map((car) => StoreCapabilities.remove.create({ + with: spaceDid, + nb: { + link: car.cid + } + })), + aud: uploadService.did(), + iss: alice.did() + }, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreRemoveTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_REMOVE_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_REMOVE_TOTAL) + t.is(item?.value, cars.length) +}) + +test('handles a batch of single invocation without store/remove', async t => { + const { tableName } = await prepareResources(t.context.dynamoClient) + const uploadService = await Signer.generate() + const alice = await Signer.generate() + const { spaceDid } = await createSpace(alice) + const car = await randomCAR(128) + + const metricsTable = createMetricsTable(REGION, tableName, { + endpoint: t.context.dbEndpoint + }) + + const invocations = [{ + carCid: car.cid.toString(), + value: { + att: [ + StoreCapabilities.add.create({ + with: spaceDid, + nb: { + link: car.cid, + size: car.size + } + }) + ], + aud: uploadService.did(), + iss: alice.did() + }, + ts: Date.now() + }] + + // @ts-expect-error + await updateStoreRemoveTotal(invocations, { + metricsTable + }) + + const item = await getItemFromTable(t.context.dynamoClient, tableName, { + name: METRICS_NAMES.STORE_REMOVE_TOTAL + }) + + t.truthy(item) + t.is(item?.name, METRICS_NAMES.STORE_REMOVE_TOTAL) + t.is(item?.value, 0) +}) + +/** + * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamoClient + */ +async function prepareResources (dynamoClient) { + const [ tableName ] = await Promise.all([ + createDynamoAdminMetricsTable(dynamoClient), + ]) + + return { + tableName + } +} diff --git a/ucan-invocation/test/helpers/tables.js b/ucan-invocation/test/helpers/tables.js new file mode 100644 index 00000000..f149bc10 --- /dev/null +++ b/ucan-invocation/test/helpers/tables.js @@ -0,0 +1,41 @@ +import { customAlphabet } from 'nanoid' + +import { CreateTableCommand, GetItemCommand } from '@aws-sdk/client-dynamodb' +import { marshall, unmarshall } from '@aws-sdk/util-dynamodb' + +import { dynamoDBTableConfig } from './resources.js' + +import { adminMetricsTableProps } from '../../tables/index.js' + +/** + * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamo + */ +export async function createDynamoAdminMetricsTable(dynamo) { + const id = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 10) + const tableName = id() + + await dynamo.send(new CreateTableCommand({ + TableName: tableName, + ...dynamoDBTableConfig(adminMetricsTableProps), + ProvisionedThroughput: { + ReadCapacityUnits: 1, + WriteCapacityUnits: 1 + } + })) + + return tableName +} + +/** + * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamo + * @param {string} tableName + * @param {Record} key + */ +export async function getItemFromTable(dynamo, tableName, key) { + const params = { + TableName: tableName, + Key: marshall(key) + } + const response = await dynamo.send(new GetItemCommand(params)) + return response?.Item && unmarshall(response?.Item) +} diff --git a/ucan-invocation/types.ts b/ucan-invocation/types.ts index 95769d59..296fa1a2 100644 --- a/ucan-invocation/types.ts +++ b/ucan-invocation/types.ts @@ -3,7 +3,9 @@ import { ToString, UnknownLink } from 'multiformats' import { Ability, Capability, Capabilities } from '@ucanto/interface' export interface MetricsTable { + incrementStoreAddTotal: (incrementSizeTotal: Capability[]) => Promise incrementStoreAddSizeTotal: (incrementSizeTotal: Capability[]) => Promise + incrementStoreRemoveTotal: (incrementSizeTotal: Capability[]) => Promise } export interface TotalSizeCtx {