From 9a40509036bd837fa784df7fa76854ee6f98e3f2 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 26 Jul 2023 10:42:55 +0500 Subject: [PATCH 1/4] adds support for updating multiple storage bags using CLI --- storage-node/README.md | 73 +++++------ .../src/commands/leader/update-bag.ts | 57 --------- .../src/commands/leader/update-bags.ts | 117 ++++++++++++++++++ storage-node/src/services/runtime/api.ts | 36 ++++-- .../src/services/runtime/extrinsics.ts | 56 ++++++--- 5 files changed, 223 insertions(+), 116 deletions(-) delete mode 100644 storage-node/src/commands/leader/update-bag.ts create mode 100644 storage-node/src/commands/leader/update-bags.ts diff --git a/storage-node/README.md b/storage-node/README.md index 3911b0df3a..b7eea18319 100644 --- a/storage-node/README.md +++ b/storage-node/README.md @@ -158,8 +158,8 @@ There is also an option to run Colossus as [Docker container](../colossus.Docker * [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) * [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) * [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) -* [`storage-node leader:update-bag`](#storage-node-leaderupdate-bag) * [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) +* [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) * [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) * [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) * [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) @@ -459,22 +459,49 @@ OPTIONS _See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/set-global-uploading-status.ts)_ -## `storage-node leader:update-bag` +## `storage-node leader:update-bag-limit` + +Update StorageBucketsPerBagLimit variable in the Joystream node storage. + +``` +USAGE + $ storage-node leader:update-bag-limit + +OPTIONS + -h, --help show CLI help + -k, --keyFile=keyFile Path to key file to add to the keyring. + -l, --limit=limit (required) New StorageBucketsPerBagLimit value + -m, --dev Use development mode + + -p, --password=password Password to unlock keyfiles. Multiple passwords can be passed, to try against all files. + If not specified a single password can be set in ACCOUNT_PWD environment variable. + + -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment. + + -y, --accountUri=accountUri Account URI (optional). If not specified a single key can be set in ACCOUNT_URI + environment variable. -Add/remove a storage bucket from a bag (adds by default). + --keyStore=keyStore Path to a folder with multiple key files to load into keystore. +``` + +_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/update-bag-limit.ts)_ + +## `storage-node leader:update-bags` + +Add/remove a storage bucket/s from a bag/s. If multiple bags are provided, then the same input bucket ID/s would be added/removed from all bags. ``` USAGE - $ storage-node leader:update-bag + $ storage-node leader:update-bags OPTIONS -a, --add=add - [default: ] Comma separated list of bucket IDs to add to bag + [default: ] Comma separated list of bucket IDs to add to all bag/s -h, --help show CLI help - -i, --bagId=bagId + -i, --bagIds=bagIds (required) Bag ID. Format: {bag_type}:{sub_type}:{id}. - Bag types: 'static', 'dynamic' - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel' @@ -498,7 +525,10 @@ OPTIONS password can be set in ACCOUNT_PWD environment variable. -r, --remove=remove - [default: ] Comma separated list of bucket IDs to remove from bag + [default: ] Comma separated list of bucket IDs to remove from all bag/s + + -s, --updateStrategy=(atomic|force) + [default: atomic] Update strategy to use. Either "atomic" or "force". -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment. @@ -510,34 +540,7 @@ OPTIONS Path to a folder with multiple key files to load into keystore. ``` -_See code: [src/commands/leader/update-bag.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/update-bag.ts)_ - -## `storage-node leader:update-bag-limit` - -Update StorageBucketsPerBagLimit variable in the Joystream node storage. - -``` -USAGE - $ storage-node leader:update-bag-limit - -OPTIONS - -h, --help show CLI help - -k, --keyFile=keyFile Path to key file to add to the keyring. - -l, --limit=limit (required) New StorageBucketsPerBagLimit value - -m, --dev Use development mode - - -p, --password=password Password to unlock keyfiles. Multiple passwords can be passed, to try against all files. - If not specified a single password can be set in ACCOUNT_PWD environment variable. - - -u, --apiUrl=apiUrl [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment. - - -y, --accountUri=accountUri Account URI (optional). If not specified a single key can be set in ACCOUNT_URI - environment variable. - - --keyStore=keyStore Path to a folder with multiple key files to load into keystore. -``` - -_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/update-bag-limit.ts)_ +_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/master/src/commands/leader/update-bags.ts)_ ## `storage-node leader:update-blacklist` diff --git a/storage-node/src/commands/leader/update-bag.ts b/storage-node/src/commands/leader/update-bag.ts deleted file mode 100644 index 7307a6f2a8..0000000000 --- a/storage-node/src/commands/leader/update-bag.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { updateStorageBucketsForBag } from '../../services/runtime/extrinsics' -import LeaderCommandBase from '../../command-base/LeaderCommandBase' -import logger from '../../services/logger' -import ExitCodes from '../../command-base/ExitCodes' -import _ from 'lodash' -import { customFlags } from '../../command-base/CustomFlags' - -/** - * CLI command: - * Updates bags-to-buckets relationships. - * - * @remarks - * Storage working group leader command. Requires storage WG leader priviliges. - * Shell command: "leader:update-bag" - */ -export default class LeaderUpdateBag extends LeaderCommandBase { - static description = 'Add/remove a storage bucket from a bag (adds by default).' - - static flags = { - add: customFlags.integerArr({ - char: 'a', - description: 'Comma separated list of bucket IDs to add to bag', - default: [], - }), - remove: customFlags.integerArr({ - char: 'r', - description: 'Comma separated list of bucket IDs to remove from bag', - default: [], - }), - bagId: customFlags.bagId({ - char: 'i', - required: true, - }), - ...LeaderCommandBase.flags, - } - - async run(): Promise { - const { flags } = this.parse(LeaderUpdateBag) - - logger.info('Updating the bag...') - if (flags.dev) { - await this.ensureDevelopmentChain() - } - - if (_.isEmpty(flags.add) && _.isEmpty(flags.remove)) { - logger.error('No bucket ID provided.') - this.exit(ExitCodes.InvalidParameters) - } - - const account = this.getAccount() - const api = await this.getApi() - - const success = await updateStorageBucketsForBag(api, flags.bagId, account, flags.add, flags.remove) - - this.exitAfterRuntimeCall(success) - } -} diff --git a/storage-node/src/commands/leader/update-bags.ts b/storage-node/src/commands/leader/update-bags.ts new file mode 100644 index 0000000000..94fb049d35 --- /dev/null +++ b/storage-node/src/commands/leader/update-bags.ts @@ -0,0 +1,117 @@ +import { flags } from '@oclif/command' +import _ from 'lodash' +import { customFlags } from '../../command-base/CustomFlags' +import ExitCodes from '../../command-base/ExitCodes' +import LeaderCommandBase from '../../command-base/LeaderCommandBase' +import logger from '../../services/logger' +import { updateStorageBucketsForBags } from '../../services/runtime/extrinsics' + +/** + * CLI command: + * Updates bags-to-buckets relationships. + * + * @remarks + * Storage working group leader command. Requires storage WG leader priviliges. + * Shell command: "leader:update-bag" + */ +export default class LeaderUpdateBag extends LeaderCommandBase { + static description = + `Add/remove a storage bucket/s from a bag/s. If multiple bags are ` + + `provided, then the same input bucket ID/s would be added/removed from all bags.` + + static flags = { + add: customFlags.integerArr({ + char: 'a', + description: 'Comma separated list of bucket IDs to add to all bag/s', + default: [], + }), + remove: customFlags.integerArr({ + char: 'r', + description: 'Comma separated list of bucket IDs to remove from all bag/s', + default: [], + }), + bagIds: customFlags.bagId({ + char: 'i', + required: true, + multiple: true, + }), + updateStrategy: flags.enum<'atomic' | 'force'>({ + char: 's', + options: ['atomic', 'force'], + description: 'Update strategy to use. Either "atomic" or "force".', + default: 'atomic', + }), + + ...LeaderCommandBase.flags, + } + + async run(): Promise { + const { flags } = this.parse(LeaderUpdateBag) + + logger.info('Updating the bag...') + if (flags.dev) { + await this.ensureDevelopmentChain() + } + + const uniqueBagIds = _.uniqBy(flags.bagIds, (b) => b.toString()) + const uniqueAddBuckets = _.uniq(flags.add) + const uniqueRemoveBuckets = _.uniq(flags.remove) + + if (_.isEmpty(uniqueAddBuckets) && _.isEmpty(uniqueRemoveBuckets)) { + logger.error('No bucket ID provided.') + this.exit(ExitCodes.InvalidParameters) + } + + if (_.isEmpty(uniqueBagIds)) { + logger.error('No bag ID provided.') + this.exit(ExitCodes.InvalidParameters) + } + + const account = this.getAccount() + const api = await this.getApi() + + // Ensure that input bag ids exist + for (const bagId of uniqueBagIds) { + const bag = await api.query.storage.bags(bagId) + if (bag.isEmpty) { + logger.error(`Bag with ID ${bagId} does not exist`) + this.exit(ExitCodes.InvalidParameters) + } + } + + // Ensure that input add bucket ids exist + for (const b of uniqueAddBuckets) { + const bucket = await api.query.storage.storageBucketById(b) + if (bucket.isEmpty) { + logger.error(`Add Bucket input with ID ${b} does not exist`) + this.exit(ExitCodes.InvalidParameters) + } + } + + // Ensure that input remove bucket ids exist + for (const b of uniqueRemoveBuckets) { + const bucket = await api.query.storage.storageBucketById(b) + if (bucket.isEmpty) { + logger.error(`Remove Bucket input with ID ${b} does not exist`) + this.exit(ExitCodes.InvalidParameters) + } + } + + const [success, failedCalls] = await updateStorageBucketsForBags( + api, + uniqueBagIds, + account, + flags.add, + flags.remove, + flags.updateStrategy + ) + + if (!_.isEmpty(failedCalls)) { + logger.error(`Following extrinsic calls in the batch Tx failed:\n ${JSON.stringify(failedCalls, null, 2)}}`) + } else { + logger.info('All extrinsic calls in the batch Tx succeeded!') + } + + this.exitAfterRuntimeCall(success) + } +} diff --git a/storage-node/src/services/runtime/api.ts b/storage-node/src/services/runtime/api.ts index 9bb81576ed..b0e49d4297 100644 --- a/storage-node/src/services/runtime/api.ts +++ b/storage-node/src/services/runtime/api.ts @@ -1,17 +1,17 @@ -import { ApiPromise, WsProvider, SubmittableResult } from '@polkadot/api' -import type { Index } from '@polkadot/types/interfaces/runtime' -import { ISubmittableResult, IEvent } from '@polkadot/types/types' -import { TypeRegistry } from '@polkadot/types' +import { CLIError } from '@oclif/errors' +import { ApiPromise, SubmittableResult, WsProvider } from '@polkadot/api' +import { AugmentedEvent, SubmittableExtrinsic } from '@polkadot/api/types' import { KeyringPair } from '@polkadot/keyring/types' -import { SubmittableExtrinsic, AugmentedEvent } from '@polkadot/api/types' +import { TypeRegistry } from '@polkadot/types' +import type { Index } from '@polkadot/types/interfaces/runtime' import { DispatchError } from '@polkadot/types/interfaces/system' -import logger from '../../services/logger' -import ExitCodes from '../../command-base/ExitCodes' -import { CLIError } from '@oclif/errors' +import { IEvent, ISubmittableResult } from '@polkadot/types/types' import { formatBalance } from '@polkadot/util' +import AwaitLock from 'await-lock' import stringify from 'fast-safe-stringify' import sleep from 'sleep-promise' -import AwaitLock from 'await-lock' +import ExitCodes from '../../command-base/ExitCodes' +import logger from '../../services/logger' /** * Dedicated error for the failed extrinsics. @@ -171,7 +171,7 @@ async function lockAndGetNonce(api: ApiPromise, account: KeyringPair): Promise ? IEvent : never +>(result: SubmittableResult, section: S, eventNames: M[]): EventType[] { + return result.filterRecords(section, eventNames).map((e) => e.event as unknown as EventType) +} diff --git a/storage-node/src/services/runtime/extrinsics.ts b/storage-node/src/services/runtime/extrinsics.ts index 1adfe68a84..74f80f48b8 100644 --- a/storage-node/src/services/runtime/extrinsics.ts +++ b/storage-node/src/services/runtime/extrinsics.ts @@ -1,10 +1,11 @@ -import { sendAndFollowNamedTx, getEvent } from './api' -import { KeyringPair } from '@polkadot/keyring/types' import { ApiPromise } from '@polkadot/api' +import { KeyringPair } from '@polkadot/keyring/types' +import { DispatchError } from '@polkadot/types/interfaces/system' import { PalletStorageBagIdType as BagId, PalletStorageDynamicBagType as DynamicBagType } from '@polkadot/types/lookup' -import logger from '../../services/logger' -import { timeout } from 'promise-timeout' import BN from 'bn.js' +import { timeout } from 'promise-timeout' +import logger from '../../services/logger' +import { formatDispatchError, getEvent, getEvents, sendAndFollowNamedTx } from './api' /** * Creates storage bucket. @@ -72,33 +73,60 @@ export async function acceptStorageBucketInvitation( } /** - * Updates storage bucket to bags relationships. + * Updates storage buckets to bags relationships. * * @remarks * It sends an extrinsic to the runtime. * * @param api - runtime API promise - * @param bagId - BagId instance + * @param bagId - List of BagIds instance * @param account - KeyringPair instance * @param add - runtime storage bucket IDs to add * @param remove - runtime storage bucket IDs to remove * @returns promise with a success flag. */ -export async function updateStorageBucketsForBag( +export async function updateStorageBucketsForBags( api: ApiPromise, - bagId: BagId, + bagIds: BagId[], account: KeyringPair, add: number[], - remove: number[] -): Promise { - return await extrinsicWrapper(() => { + remove: number[], + txStrategy: 'atomic' | 'force' +): Promise<[boolean, { args: string; error: string }[] | void]> { + // List of failed batch calls + let failedCalls + + const success = await extrinsicWrapper(async () => { const removeBuckets = api.createType('BTreeSet', remove) const addBuckets = api.createType('BTreeSet', add) - const tx = api.tx.storage.updateStorageBucketsForBag(bagId, addBuckets, removeBuckets) - - return sendAndFollowNamedTx(api, account, tx) + const batchFn = txStrategy === 'atomic' ? api.tx.utility.batchAll : api.tx.utility.forceBatch + + const txs = bagIds.map((bagId) => api.tx.storage.updateStorageBucketsForBag(bagId, addBuckets, removeBuckets)) + const txBatch = batchFn(txs) + + failedCalls = await sendAndFollowNamedTx(api, account, txBatch, (result) => { + const [batchCompletedEvent] = getEvents(result, 'utility', ['BatchCompleted']) + if (batchCompletedEvent) { + return [] + } + + // find all the failed calls based on their index + const events = getEvents(result, 'utility', ['ItemCompleted', 'ItemFailed']) + return events + .map((e, i) => { + if (e.method === 'ItemFailed') { + return { + args: txs[i].args.toString(), + error: formatDispatchError(api, e.data[0] as DispatchError), + } + } + }) + .filter(Boolean) + }) }) + + return [success, failedCalls] } /** From c82a379cc73255be3096e2a273270964a27f6fe0 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 26 Jul 2023 20:13:47 +0500 Subject: [PATCH 2/4] bump package version & update CHANGELOG --- storage-node/CHANGELOG.md | 4 ++++ storage-node/package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md index 7b227c0fee..e1ba1a017a 100644 --- a/storage-node/CHANGELOG.md +++ b/storage-node/CHANGELOG.md @@ -1,3 +1,7 @@ +### 3.7.0 + +- Updates `leader:update-bag` CLI command to `leader:update-bags` to accept multiple bag ids as input. This allows the command to be used to update storage buckets of multiple bags in a single batched transaction. + ### 3.6.0 - Collosus can now store multiple keys in it's keyring. diff --git a/storage-node/package.json b/storage-node/package.json index 0d261e79f2..d33f501c02 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -1,7 +1,7 @@ { "name": "storage-node", "description": "Joystream storage subsystem.", - "version": "3.6.0", + "version": "3.7.0", "author": "Joystream contributors", "bin": { "storage-node": "./bin/run" From febcc686feed86e4be06c56c7a676b19153ce537 Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Mon, 31 Jul 2023 17:30:39 +0300 Subject: [PATCH 3/4] fix command name in tests --- tests/network-tests/src/flows/clis/initStorageBucket.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/network-tests/src/flows/clis/initStorageBucket.ts b/tests/network-tests/src/flows/clis/initStorageBucket.ts index f3a34aa81c..9507957ad6 100644 --- a/tests/network-tests/src/flows/clis/initStorageBucket.ts +++ b/tests/network-tests/src/flows/clis/initStorageBucket.ts @@ -38,7 +38,7 @@ export default async function initStorageBucket({ api }: FlowProps): Promise Date: Mon, 31 Jul 2023 19:52:31 +0300 Subject: [PATCH 4/4] fix arg name in update-bags command --- tests/network-tests/src/flows/clis/initStorageBucket.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/network-tests/src/flows/clis/initStorageBucket.ts b/tests/network-tests/src/flows/clis/initStorageBucket.ts index 9507957ad6..c5908e11da 100644 --- a/tests/network-tests/src/flows/clis/initStorageBucket.ts +++ b/tests/network-tests/src/flows/clis/initStorageBucket.ts @@ -38,7 +38,7 @@ export default async function initStorageBucket({ api }: FlowProps): Promise