From 6b638ad8c78419d68bb9a150d634fc42fdc2c7ea Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Tue, 21 Feb 2023 17:18:46 +0400 Subject: [PATCH 1/4] drop utils scripts that require sudo --- utils/api-scripts/dev-init-storage.sh | 1 - utils/api-scripts/package.json | 2 - utils/api-scripts/scripts/test-transfer.js | 4 +- utils/api-scripts/src/dev-set-runtime-code.ts | 108 - utils/api-scripts/src/helpers/extrinsics.ts | 27 - utils/api-scripts/src/sudo-set-multisig.ts | 44 - utils/api-scripts/test-sudo-multisig.sh | 31 - utils/migration-scripts/.eslintignore | 1 - utils/migration-scripts/.gitignore | 9 - utils/migration-scripts/.prettierignore | 3 - utils/migration-scripts/README.md | 237 - utils/migration-scripts/bin/run | 3 - utils/migration-scripts/bin/run.cmd | 3 - .../createContentDirectorySnapshot.ts | 32 - .../giza-olympia/createMembershipsSnapshot.ts | 32 - .../giza-olympia/fetchAllObjects.ts | 42 - .../giza-olympia/migrateContent.ts | 77 - .../giza-olympia/migrateMembers.ts | 51 - .../sumer-giza/migrateContent.ts | 79 - .../sumer-giza/retryFailedUploads.ts | 54 - .../giza-olympia/AssetsBase.ts | 94 - .../giza-olympia/BaseMigration.ts | 194 - .../giza-olympia/CategoryMigration.ts | 21 - .../ChannelCategoriesMigration.ts | 78 - .../giza-olympia/ChannelsMigration.ts | 199 - .../giza-olympia/ContentHash.ts | 22 - .../giza-olympia/ContentMigration.ts | 99 - .../giza-olympia/DownloadManager.ts | 158 - .../giza-olympia/MembershipMigration.ts | 102 - .../giza-olympia/SnapshotManager.ts | 106 - .../giza-olympia/UploadManager.ts | 209 - .../giza-olympia/UploadMigration.ts | 20 - .../giza-olympia/VideoCategoriesMigration.ts | 78 - .../giza-olympia/VideosMigration.ts | 225 - .../giza-olympia/giza-query-node/api.ts | 212 - .../giza-olympia/giza-query-node/codegen.yml | 33 - .../giza-query-node/generated/queries.ts | 386 - .../giza-query-node/generated/schema.ts | 3715 -- .../giza-query-node/queries/queries.graphql | 207 - .../sumer-giza/AssetsManager.ts | 322 - .../sumer-giza/AssetsMigration.ts | 20 - .../sumer-giza/BaseMigration.ts | 192 - .../sumer-giza/ChannelsMigration.ts | 178 - .../sumer-giza/ContentHash.ts | 22 - .../sumer-giza/ContentMigration.ts | 78 - .../sumer-giza/ImageResizer.ts | 30 - .../sumer-giza/VideosMigration.ts | 203 - .../sumer-giza/sumer-query-node/api.ts | 124 - .../sumer-giza/sumer-query-node/codegen.yml | 33 - .../sumer-query-node/generated/queries.ts | 229 - .../sumer-query-node/generated/schema.ts | 2565 -- .../sumer-query-node/queries/queries.graphql | 121 - utils/migration-scripts/package.json | 95 - utils/migration-scripts/src/RuntimeApi.ts | 141 - .../createMembershipsSnapshot.ts | 32 - .../olympia-carthage/migrateMembers.ts | 57 - utils/migration-scripts/src/index.ts | 1 - utils/migration-scripts/src/logging.ts | 27 - .../src/olympia-carthage/BaseMigration.ts | 189 - .../olympia-carthage/MembershipMigration.ts | 115 - .../src/olympia-carthage/SnapshotManager.ts | 50 - .../olympia-query-node/api.ts | 105 - .../olympia-query-node/codegen.yml | 34 - .../olympia-query-node/generated/queries.ts | 62 - .../olympia-query-node/generated/schema.ts | 32538 --------------- .../olympia-query-node/olympia.schema.graphql | 33228 ---------------- .../queries/queries.graphql | 35 - utils/migration-scripts/tsconfig.json | 17 - 68 files changed, 2 insertions(+), 77839 deletions(-) delete mode 100644 utils/api-scripts/src/dev-set-runtime-code.ts delete mode 100644 utils/api-scripts/src/sudo-set-multisig.ts delete mode 100755 utils/api-scripts/test-sudo-multisig.sh delete mode 100644 utils/migration-scripts/.eslintignore delete mode 100644 utils/migration-scripts/.gitignore delete mode 100644 utils/migration-scripts/.prettierignore delete mode 100644 utils/migration-scripts/README.md delete mode 100755 utils/migration-scripts/bin/run delete mode 100644 utils/migration-scripts/bin/run.cmd delete mode 100644 utils/migration-scripts/disabled-commands/giza-olympia/createContentDirectorySnapshot.ts delete mode 100644 utils/migration-scripts/disabled-commands/giza-olympia/createMembershipsSnapshot.ts delete mode 100644 utils/migration-scripts/disabled-commands/giza-olympia/fetchAllObjects.ts delete mode 100644 utils/migration-scripts/disabled-commands/giza-olympia/migrateContent.ts delete mode 100644 utils/migration-scripts/disabled-commands/giza-olympia/migrateMembers.ts delete mode 100644 utils/migration-scripts/disabled-commands/sumer-giza/migrateContent.ts delete mode 100644 utils/migration-scripts/disabled-commands/sumer-giza/retryFailedUploads.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/AssetsBase.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/BaseMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/CategoryMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/ChannelCategoriesMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/ChannelsMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/ContentHash.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/ContentMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/DownloadManager.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/MembershipMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/SnapshotManager.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/UploadManager.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/UploadMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/VideoCategoriesMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/VideosMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/api.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/codegen.yml delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/queries.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/schema.ts delete mode 100644 utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/queries/queries.graphql delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/AssetsManager.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/AssetsMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/BaseMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/ChannelsMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/ContentHash.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/ContentMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/ImageResizer.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/VideosMigration.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/sumer-query-node/api.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/sumer-query-node/codegen.yml delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/sumer-query-node/generated/queries.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/sumer-query-node/generated/schema.ts delete mode 100644 utils/migration-scripts/disabled-migrations/sumer-giza/sumer-query-node/queries/queries.graphql delete mode 100644 utils/migration-scripts/package.json delete mode 100644 utils/migration-scripts/src/RuntimeApi.ts delete mode 100644 utils/migration-scripts/src/commands/olympia-carthage/createMembershipsSnapshot.ts delete mode 100644 utils/migration-scripts/src/commands/olympia-carthage/migrateMembers.ts delete mode 100644 utils/migration-scripts/src/index.ts delete mode 100644 utils/migration-scripts/src/logging.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/BaseMigration.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/MembershipMigration.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/SnapshotManager.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/olympia-query-node/api.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/olympia-query-node/codegen.yml delete mode 100644 utils/migration-scripts/src/olympia-carthage/olympia-query-node/generated/queries.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/olympia-query-node/generated/schema.ts delete mode 100644 utils/migration-scripts/src/olympia-carthage/olympia-query-node/olympia.schema.graphql delete mode 100644 utils/migration-scripts/src/olympia-carthage/olympia-query-node/queries/queries.graphql delete mode 100644 utils/migration-scripts/tsconfig.json diff --git a/utils/api-scripts/dev-init-storage.sh b/utils/api-scripts/dev-init-storage.sh index 883266c839..b5b169b589 100755 --- a/utils/api-scripts/dev-init-storage.sh +++ b/utils/api-scripts/dev-init-storage.sh @@ -8,5 +8,4 @@ export GROUP="storageWorkingGroup" export WORKER_ROLE_URI="//Colossus" export INITIAL_WORKER_BALANCE_TOP_UP="10000" -yarn initialize-lead yarn initialize-worker diff --git a/utils/api-scripts/package.json b/utils/api-scripts/package.json index 45626cc508..ea4bcbdcf1 100644 --- a/utils/api-scripts/package.json +++ b/utils/api-scripts/package.json @@ -8,10 +8,8 @@ "status": "ts-node src/status", "script": "ts-node src/script", "tsnode-strict": "node -r ts-node/register --unhandled-rejections=strict", - "initialize-lead": "ts-node src/initialize-lead", "initialize-worker": "ts-node src/initialize-worker", "buy-membership": "ts-node src/buy-membership", - "sudo-set-multisig": "ts-node src/sudo-set-multisig", "storage-dev-init": "./dev-init-storage.sh" }, "dependencies": { diff --git a/utils/api-scripts/scripts/test-transfer.js b/utils/api-scripts/scripts/test-transfer.js index c5c58bae35..a109391474 100644 --- a/utils/api-scripts/scripts/test-transfer.js +++ b/utils/api-scripts/scripts/test-transfer.js @@ -8,8 +8,8 @@ // const script = async ({ api, keyring, userAddress }) => { - const sudoAddress = (await api.query.sudo.key()).toString() - const destination = userAddress || sudoAddress + const alice = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY' + const destination = userAddress || alice let sender if (typeof window === 'undefined') { // In node, get the keyPair if the keyring was provided diff --git a/utils/api-scripts/src/dev-set-runtime-code.ts b/utils/api-scripts/src/dev-set-runtime-code.ts deleted file mode 100644 index f4fccbfa8e..0000000000 --- a/utils/api-scripts/src/dev-set-runtime-code.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { ApiPromise, WsProvider } from '@polkadot/api' -import { Keyring } from '@polkadot/keyring' -import { ISubmittableResult } from '@polkadot/types/types/' -import { DispatchError, DispatchResult } from '@polkadot/types/interfaces/system' -import { TypeRegistry } from '@polkadot/types' -import fs from 'fs' -import { compactAddLength } from '@polkadot/util' - -function onApiDisconnected() { - process.exit(2) -} - -function onApiError() { - process.exit(3) -} - -async function main() { - const file = process.argv[2] - - if (!file) { - console.log('No wasm file argument provided.') - process.exit(1) - } - - const wasm = Uint8Array.from(fs.readFileSync(file)) - console.log('WASM bytes:', wasm.byteLength) - - const provider = new WsProvider('ws://127.0.0.1:9944') - - let api: ApiPromise - let retry = 6 - while (true) { - try { - api = new ApiPromise({ provider }) - await api.isReadyOrError - break - } catch (err) { - // failed to connect to node - // Exceptions are not being caught!? - } - - if (retry-- === 0) { - process.exit(-1) - } - - await new Promise((resolve) => { - setTimeout(resolve, 5000) - }) - } - - const keyring = new Keyring() - const sudo = keyring.addFromUri('//Alice', undefined, 'sr25519') - - // DO NOT SET UNCHECKED! - // const tx = api.tx.system.setCodeWithoutChecks(wasm) - - const setCodeTx = api.tx.system.setCode(compactAddLength(wasm)) - const sudoTx = api.tx.sudo.sudoUncheckedWeight(setCodeTx, 1) - const nonce = (await api.query.system.account(sudo.address)).nonce - const signedTx = sudoTx.sign(sudo, { nonce }) - - // console.log('Tx size:', signedTx.length) - // const wasmCodeInTxArg = (signedTx.method.args[0] as Call).args[0] - // console.log('WASM code arg byte length:', (wasmCodeInTxArg as Bytes).byteLength) - - await signedTx.send((result: ISubmittableResult) => { - if (result.status.isInBlock && result.events !== undefined) { - result.events.forEach((event) => { - if (event.event.method === 'ExtrinsicFailed') { - console.log('ExtrinsicFailed', (event.event.data[0] as DispatchError).toHuman()) - process.exit(4) - } - - if (event.event.method === 'Sudid') { - const result = event.event.data[0] as DispatchResult - if (result.isOk) { - process.exit(0) - } else if (result.isError) { - const err = result.asError - console.log('Error:', err.toHuman()) - if (err.isModule) { - const { name } = (api.registry as TypeRegistry).findMetaError(err.asModule) - console.log(`${name}\n`) - } - process.exit(5) - } else { - console.log('Sudid result:', result.toHuman()) - process.exit(-1) - } - } - }) - - // Wait a few seconds to display new runtime changes - setTimeout(() => { - process.exit(0) - }, 12000) - } - }) - - api.on('disconnected', onApiDisconnected) - api.on('error', onApiError) - - await new Promise(() => { - // wait until transaction finalizes - }) -} - -main().catch(console.error) diff --git a/utils/api-scripts/src/helpers/extrinsics.ts b/utils/api-scripts/src/helpers/extrinsics.ts index c568adc2db..c2fe557c63 100644 --- a/utils/api-scripts/src/helpers/extrinsics.ts +++ b/utils/api-scripts/src/helpers/extrinsics.ts @@ -41,33 +41,6 @@ export class ExtrinsicsHelper { return nonce } - async sendAndCheckSudo( - tx: SubmittableExtrinsic<'promise'>, - errorMsg?: string, - wrap = true - ): Promise { - const sudoUri = process.env.SUDO_URI || '//Alice' - const sudoMultiKeys = (process.env.SUDO_MULTISIG_KEYS || '').split(',').filter((v) => v) - const sudoMultiUris = (process.env.SUDO_MULTISIG_URIS || '').split(',').filter((v) => v) - tx = wrap ? this.api.tx.sudo.sudo(tx) : tx - - if (sudoMultiKeys.length && sudoMultiUris.length) { - const sudoSigners = sudoMultiUris.map((u) => getKeyFromSuri(u)) - console.log(`Using multisig sudo`) - console.log('Multisig keys:', sudoMultiKeys) - console.log('Multisig threshold', sudoSigners.length) - console.log( - 'Signers:', - sudoSigners.map((s) => s.address) - ) - return this.sendAndCheckMulisig(sudoSigners, sudoMultiKeys, tx, errorMsg) - } else { - const sudoKey = getKeyFromSuri(sudoUri) - console.log(`Using sudo key: ${sudoKey.address}`) - return (await this.sendAndCheck(sudoKey, [tx], errorMsg))[0] - } - } - async sendAndCheckMulisig( signers: KeyringPair[], keys: string[], diff --git a/utils/api-scripts/src/sudo-set-multisig.ts b/utils/api-scripts/src/sudo-set-multisig.ts deleted file mode 100644 index fb0ba01b4a..0000000000 --- a/utils/api-scripts/src/sudo-set-multisig.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { JOYSTREAM_ADDRESS_PREFIX } from '@joystream/types' -import { ApiPromise, WsProvider } from '@polkadot/api' -import { encodeMultiAddress } from '@polkadot/util-crypto' -import { ExtrinsicsHelper } from './helpers/extrinsics' - -async function main() { - const sudoMultiKeys = (process.env.NEW_SUDO_MULTISIG_KEYS || '').split(',').filter((v) => v) - const sudoMultisigThreshold = parseInt(process.env.SUDO_MULTISIG_THRESHOLD || '') - const wsUri = process.env.WS_URI || 'ws://127.0.0.1:9944' - - console.log('Multisig keys:', sudoMultiKeys) - console.log('Multisig threshold:', sudoMultisigThreshold) - - if (sudoMultiKeys.length < 2) { - throw new Error('Provide SUDO_MULTISIG_KEYS env with at least 2 keys in order to generate multisig address') - } - - if (sudoMultisigThreshold < 1) { - throw new Error('The SUDO_MULTISIG_THRESHOLD should be at least 1') - } - - const provider = new WsProvider(wsUri) - const api = await ApiPromise.create({ provider }) - - let multisigAddr: string - try { - multisigAddr = encodeMultiAddress(sudoMultiKeys, sudoMultisigThreshold, JOYSTREAM_ADDRESS_PREFIX) - } catch (e) { - throw new Error(`Failed to generate multisig address: ${(e as Error).message}`) - } - - console.log('Multisig address:', multisigAddr) - - const txHelper = new ExtrinsicsHelper(api) - - console.log('Updating the sudo key...') - await txHelper.sendAndCheckSudo(api.tx.sudo.setKey(multisigAddr), 'sudo.setKey failed', false) - - console.log(`Sudo key changed to: ${multisigAddr}`) -} - -main() - .then(() => process.exit(0)) - .catch(console.error) diff --git a/utils/api-scripts/test-sudo-multisig.sh b/utils/api-scripts/test-sudo-multisig.sh deleted file mode 100755 index 8999a37c73..0000000000 --- a/utils/api-scripts/test-sudo-multisig.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -e - -SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" -cd $SCRIPT_PATH - -# Setup 3-of-4 multisig sudo with //Alice, //Bob, //Charlie and //Dave -export NEW_SUDO_MULTISIG_KEYS=j4W7rVcUCxi2crhhjRq46fNDRbVHTjJrz6bKxZwehEMQxZeSf,j4UYhDYJ4pz2ihhDDzu69v2JTVeGaGmTebmBdWaX2ANVinXyE,j4UbMHiS79yvMLJctXggUugkkKmwxG5LW2YSy3ap8SmgF5qW9,j4SR5Mty5Mzy2dPTunA6TD4gBTwbSb8wRTabvu2gsLqC271d4 -export SUDO_MULTISIG_THRESHOLD=3 -yarn workspace api-scripts sudo-set-multisig - -# Use //Alice, //Bob and //Charlie as sudo multisig signers and execute initialize-lead script -export SUDO_MULTISIG_KEYS=$NEW_SUDO_MULTISIG_KEYS -export SUDO_MULTISIG_URIS=//Alice,//Bob,//Charlie -GROUP=operationsWorkingGroupAlpha yarn workspace api-scripts initialize-lead - -# Change multisig threshold to 2 -export SUDO_MULTISIG_THRESHOLD=2 -yarn workspace api-scripts sudo-set-multisig - -# Use //Charlie and //Dave as sudo multisig signers and execute initialize-lead script -export SUDO_MULTISIG_URIS=//Charlie,//Dave -GROUP=operationsWorkingGroupBeta yarn workspace api-scripts initialize-lead - -# Change multisig threshold to 1 -export SUDO_MULTISIG_THRESHOLD=1 -yarn workspace api-scripts sudo-set-multisig - -# Use //Bob as sudo multisig signer and execute initialize-lead script -export SUDO_MULTISIG_URIS=//Bob -GROUP=operationsWorkingGroupGamma yarn workspace api-scripts initialize-lead \ No newline at end of file diff --git a/utils/migration-scripts/.eslintignore b/utils/migration-scripts/.eslintignore deleted file mode 100644 index a29b344a11..0000000000 --- a/utils/migration-scripts/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -src/olympia-carthage/olympia-query-node/generated diff --git a/utils/migration-scripts/.gitignore b/utils/migration-scripts/.gitignore deleted file mode 100644 index 6773f1365b..0000000000 --- a/utils/migration-scripts/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*-debug.log -*-error.log -/.nyc_output -/dist -/lib -/package-lock.json -/tmp -node_modules -results diff --git a/utils/migration-scripts/.prettierignore b/utils/migration-scripts/.prettierignore deleted file mode 100644 index 6ae4dfa780..0000000000 --- a/utils/migration-scripts/.prettierignore +++ /dev/null @@ -1,3 +0,0 @@ -lib -results -src/sumer-giza/sumer-query-node/generated diff --git a/utils/migration-scripts/README.md b/utils/migration-scripts/README.md deleted file mode 100644 index 87c8f225ff..0000000000 --- a/utils/migration-scripts/README.md +++ /dev/null @@ -1,237 +0,0 @@ -migrations -========== - -Joystream migrations scripts - -[![oclif](https://img.shields.io/badge/cli-oclif-brightgreen.svg)](https://oclif.io) -[![Version](https://img.shields.io/npm/v/migrations.svg)](https://npmjs.org/package/migrations) -[![Downloads/week](https://img.shields.io/npm/dw/migrations.svg)](https://npmjs.org/package/migrations) -[![License](https://img.shields.io/npm/l/migrations.svg)](https://github.com/Joystream/joystream/blob/master/package.json) - - -* [Usage](#usage) -* [Commands](#commands) - -# Usage - -```sh-session -$ npm install -g migration-scripts -$ migration-scripts COMMAND -running command... -$ migration-scripts (-v|--version|version) -migration-scripts/0.1.0 linux-x64 node-v14.18.0 -$ migration-scripts --help [COMMAND] -USAGE - $ migration-scripts COMMAND -... -``` - -# Commands - -* [`migration-scripts giza-olympia:createContentDirectorySnapshot`](#migration-scripts-giza-olympiacreatecontentdirectorysnapshot) -* [`migration-scripts giza-olympia:createMembershipsSnapshot`](#migration-scripts-giza-olympiacreatemembershipssnapshot) -* [`migration-scripts giza-olympia:fetchAllObjects`](#migration-scripts-giza-olympiafetchallobjects) -* [`migration-scripts giza-olympia:migrateContent`](#migration-scripts-giza-olympiamigratecontent) -* [`migration-scripts giza-olympia:migrateMembers`](#migration-scripts-giza-olympiamigratemembers) -* [`migration-scripts help [COMMAND]`](#migration-scripts-help-command) -* [`migration-scripts sumer-giza:migrateContent`](#migration-scripts-sumer-gizamigratecontent) -* [`migration-scripts sumer-giza:retryFailedUploads`](#migration-scripts-sumer-gizaretryfaileduploads) - -## `migration-scripts giza-olympia:createContentDirectorySnapshot` - -``` -USAGE - $ migration-scripts giza-olympia:createContentDirectorySnapshot - -OPTIONS - -o, --output=output Output file path - --queryNodeUri=queryNodeUri [default: https://hydra.joystream.org/graphql] Giza query node uri -``` - -_See code: [src/commands/giza-olympia/createContentDirectorySnapshot.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/giza-olympia/createContentDirectorySnapshot.ts)_ - -## `migration-scripts giza-olympia:createMembershipsSnapshot` - -``` -USAGE - $ migration-scripts giza-olympia:createMembershipsSnapshot - -OPTIONS - -o, --output=output Output file path - --queryNodeUri=queryNodeUri [default: https://hydra.joystream.org/graphql] Giza query node uri -``` - -_See code: [src/commands/giza-olympia/createMembershipsSnapshot.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/giza-olympia/createMembershipsSnapshot.ts)_ - -## `migration-scripts giza-olympia:fetchAllObjects` - -``` -USAGE - $ migration-scripts giza-olympia:fetchAllObjects - -OPTIONS - --continously Whether the script should run continously - --dataDir=dataDir [default: /tmp/joystream/giza-olympia-migration] Directory for storing data objects - - --idleTime=idleTime [default: 300] Time (in seconds) to remain idle in case no new data objects were - found - - --objectsPerBatch=objectsPerBatch [default: 20] Max. number of storage objects to fetch simultaneously - - --queryNodeUri=queryNodeUri [default: https://hydra.joystream.org/graphql] Giza query node uri -``` - -_See code: [src/commands/giza-olympia/fetchAllObjects.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/giza-olympia/fetchAllObjects.ts)_ - -## `migration-scripts giza-olympia:migrateContent` - -``` -USAGE - $ migration-scripts giza-olympia:migrateContent - -OPTIONS - -c, --channelIds=channelIds (required) Channel ids to migrate - --channelBatchSize=channelBatchSize [default: 20] Channel batch size - - --dataDir=dataDir (required) Directory where data objects to upload are - stored - - --excludeVideoIds=excludeVideoIds [default: ] Video ids to exclude from migration - - --forceChannelOwnerMemberId=forceChannelOwnerMemberId Can be used to force a specific channel owner for all - channels, allowing to easily test the script in dev - environment - - --membershipsMigrationResultPath=membershipsMigrationResultPath (required) JSON artifact produced by membership - migration - - --migrationStatePath=migrationStatePath [default: - /home/leszek/projects/joystream/joystream-ws-2/utils/ - migration-scripts/results/giza-olympia] Path to - migration results directory - - --snapshotFilePath=snapshotFilePath (required) Path to giza content directory snapshot - (json) - - --sudoUri=sudoUri [default: //Alice] Sudo key Substrate uri - - --uploadSpBucketId=uploadSpBucketId [default: 0] Olympia storage bucket id - - --uploadSpEndpoint=uploadSpEndpoint [default: http://localhost:3333] Olympia storage node - endpoint to use for uploading - - --videoBatchSize=videoBatchSize [default: 20] Video batch size - - --wsProviderEndpointUri=wsProviderEndpointUri [default: ws://localhost:9944] WS provider endpoint - uri (Olympia) -``` - -_See code: [src/commands/giza-olympia/migrateContent.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/giza-olympia/migrateContent.ts)_ - -## `migration-scripts giza-olympia:migrateMembers` - -``` -USAGE - $ migration-scripts giza-olympia:migrateMembers - -OPTIONS - --batchSize=batchSize [default: 100] Members batch size - - --migrationStatePath=migrationStatePath [default: - /home/leszek/projects/joystream/joystream-ws-2/utils/migration-scripts/ - results/giza-olympia] Path to migration results directory - - --snapshotFilePath=snapshotFilePath (required) Path to giza memberships snapshot (json) - - --sudoUri=sudoUri [default: //Alice] Sudo key Substrate uri - - --wsProviderEndpointUri=wsProviderEndpointUri [default: ws://localhost:9944] WS provider endpoint uri (Olympia) -``` - -_See code: [src/commands/giza-olympia/migrateMembers.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/giza-olympia/migrateMembers.ts)_ - -## `migration-scripts help [COMMAND]` - -display help for migration-scripts - -``` -USAGE - $ migration-scripts help [COMMAND] - -ARGUMENTS - COMMAND command to show help for - -OPTIONS - --all see all commands in CLI -``` - -_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v3.2.3/src/commands/help.ts)_ - -## `migration-scripts sumer-giza:migrateContent` - -``` -USAGE - $ migration-scripts sumer-giza:migrateContent - -OPTIONS - -c, --channelIds=channelIds (required) Channel ids to migrate - --channelBatchSize=channelBatchSize [default: 20] Channel batch size - - --dataDir=dataDir [default: /tmp/joystream/sumer-giza-migration] Directory - for storing data objects to upload - - --excludeVideoIds=excludeVideoIds [default: ] Video ids to exclude from migration - - --forceChannelOwnerMemberId=forceChannelOwnerMemberId Can be used to force a specific channel owner for all - channels, allowing to easily test the script in dev - environment - - --migrationStatePath=migrationStatePath [default: - /home/leszek/projects/joystream/joystream-ws-2/utils/migr - ation-scripts/results/sumer-giza] Path to migration - results directory - - --preferredDownloadSpEndpoints=preferredDownloadSpEndpoints [default: https://storage-1.joystream.org/storage] - Preferred storage node endpoints when downloading data - objects - - --queryNodeUri=queryNodeUri [default: https://hydra.joystream.org/graphql] Query node - uri - - --sudoUri=sudoUri [default: //Alice] Sudo key Substrate uri - - --uploadSpBucketId=uploadSpBucketId [default: 0] Giza storage bucket id - - --uploadSpEndpoint=uploadSpEndpoint [default: http://localhost:3333] Giza storage node - endpoint to use for uploading - - --videoBatchSize=videoBatchSize [default: 20] Video batch size - - --wsProviderEndpointUri=wsProviderEndpointUri [default: ws://localhost:9944] WS provider endpoint uri - (Giza) -``` - -_See code: [src/commands/sumer-giza/migrateContent.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/sumer-giza/migrateContent.ts)_ - -## `migration-scripts sumer-giza:retryFailedUploads` - -``` -USAGE - $ migration-scripts sumer-giza:retryFailedUploads - -OPTIONS - -f, --failedUploadsPath=failedUploadsPath (required) Path to failed uploads file - - --dataDir=dataDir [default: /tmp/joystream/sumer-giza-migration] Directory where data - objects to upload are stored - - --uploadSpBucketId=uploadSpBucketId [default: 0] Giza storage bucket id - - --uploadSpEndpoint=uploadSpEndpoint [default: http://localhost:3333] Giza storage node endpoint to use for - uploading - - --wsProviderEndpointUri=wsProviderEndpointUri [default: ws://localhost:9944] WS provider endpoint uri (Giza) -``` - -_See code: [src/commands/sumer-giza/retryFailedUploads.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/sumer-giza/retryFailedUploads.ts)_ - diff --git a/utils/migration-scripts/bin/run b/utils/migration-scripts/bin/run deleted file mode 100755 index e332faed17..0000000000 --- a/utils/migration-scripts/bin/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env node - -require('@oclif/command').run().then(require('@oclif/command/flush')).catch(require('@oclif/errors/handle')) diff --git a/utils/migration-scripts/bin/run.cmd b/utils/migration-scripts/bin/run.cmd deleted file mode 100644 index 968fc30758..0000000000 --- a/utils/migration-scripts/bin/run.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off - -node "%~dp0\run" %* diff --git a/utils/migration-scripts/disabled-commands/giza-olympia/createContentDirectorySnapshot.ts b/utils/migration-scripts/disabled-commands/giza-olympia/createContentDirectorySnapshot.ts deleted file mode 100644 index e7105ffb16..0000000000 --- a/utils/migration-scripts/disabled-commands/giza-olympia/createContentDirectorySnapshot.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Command, flags } from '@oclif/command' -import { writeFileSync } from 'fs' -import { QueryNodeApi } from '../../giza-olympia/giza-query-node/api' -import { SnapshotManager } from '../../giza-olympia/SnapshotManager' - -export class CreateContentDirectorySnapshotCommand extends Command { - static flags = { - queryNodeUri: flags.string({ - description: 'Giza query node uri', - default: 'https://hydra.joystream.org/graphql', - }), - output: flags.string({ - char: 'o', - required: false, - description: 'Output file path', - }), - } - - async run(): Promise { - const { queryNodeUri, output } = this.parse(CreateContentDirectorySnapshotCommand).flags - const queryNodeApi = new QueryNodeApi(queryNodeUri) - - const snapshotManager = new SnapshotManager({ queryNodeApi }) - const snapshot = await snapshotManager.createContentDirectorySnapshot() - if (output) { - writeFileSync(output, JSON.stringify(snapshot, null, 2)) - } else { - this.log(JSON.stringify(snapshot, null, 2)) - } - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-commands/giza-olympia/createMembershipsSnapshot.ts b/utils/migration-scripts/disabled-commands/giza-olympia/createMembershipsSnapshot.ts deleted file mode 100644 index 57f6fa342f..0000000000 --- a/utils/migration-scripts/disabled-commands/giza-olympia/createMembershipsSnapshot.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Command, flags } from '@oclif/command' -import { writeFileSync } from 'fs' -import { QueryNodeApi } from '../../giza-olympia/giza-query-node/api' -import { SnapshotManager } from '../../giza-olympia/SnapshotManager' - -export class CreateMembershipsSnapshotCommand extends Command { - static flags = { - queryNodeUri: flags.string({ - description: 'Giza query node uri', - default: 'https://hydra.joystream.org/graphql', - }), - output: flags.string({ - char: 'o', - required: false, - description: 'Output file path', - }), - } - - async run(): Promise { - const { queryNodeUri, output } = this.parse(CreateMembershipsSnapshotCommand).flags - const queryNodeApi = new QueryNodeApi(queryNodeUri) - - const snapshotManager = new SnapshotManager({ queryNodeApi }) - const snapshot = await snapshotManager.createMembershipsSnapshot() - if (output) { - writeFileSync(output, JSON.stringify(snapshot, null, 2)) - } else { - this.log(JSON.stringify(snapshot, null, 2)) - } - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-commands/giza-olympia/fetchAllObjects.ts b/utils/migration-scripts/disabled-commands/giza-olympia/fetchAllObjects.ts deleted file mode 100644 index 475894f5cd..0000000000 --- a/utils/migration-scripts/disabled-commands/giza-olympia/fetchAllObjects.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Command, flags } from '@oclif/command' -import path from 'path' -import os from 'os' -import { QueryNodeApi } from '../../giza-olympia/giza-query-node/api' -import { DownloadManager } from '../../giza-olympia/DownloadManager' - -export class FetchAllObjectsCommand extends Command { - static flags = { - queryNodeUri: flags.string({ - description: 'Giza query node uri', - default: 'https://hydra.joystream.org/graphql', - }), - dataDir: flags.string({ - description: 'Directory for storing data objects', - default: path.join(os.tmpdir(), 'joystream/giza-olympia-migration'), - }), - continously: flags.boolean({ - description: 'Whether the script should run continously', - default: true, - }), - objectsPerBatch: flags.integer({ - required: false, - description: 'Max. number of storage objects to fetch simultaneously', - default: 20, - }), - idleTime: flags.integer({ - required: false, - description: 'Time (in seconds) to remain idle in case no new data objects were found', - default: 300, - dependsOn: ['continously'], - }), - } - - async run(): Promise { - const opts = this.parse(FetchAllObjectsCommand).flags - const queryNodeApi = new QueryNodeApi(opts.queryNodeUri) - - const downloadManager = new DownloadManager({ queryNodeApi, config: opts }) - await downloadManager.fetchAllDataObjects(undefined, opts.continously, opts.idleTime) - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-commands/giza-olympia/migrateContent.ts b/utils/migration-scripts/disabled-commands/giza-olympia/migrateContent.ts deleted file mode 100644 index 994fac747f..0000000000 --- a/utils/migration-scripts/disabled-commands/giza-olympia/migrateContent.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Command, flags } from '@oclif/command' -import path from 'path' -import { ContentMigration } from '../../giza-olympia/ContentMigration' - -export class MigrateContentCommand extends Command { - static flags = { - snapshotFilePath: flags.string({ - required: true, - description: 'Path to giza content directory snapshot (json)', - }), - membershipsMigrationResultPath: flags.string({ - required: true, - description: 'JSON artifact produced by membership migration', - }), - wsProviderEndpointUri: flags.string({ - description: 'WS provider endpoint uri (Olympia)', - default: 'ws://localhost:9944', - }), - sudoUri: flags.string({ - description: 'Sudo key Substrate uri', - default: '//Alice', - }), - channelIds: flags.integer({ - char: 'c', - multiple: true, - description: 'Channel ids to migrate', - required: true, - }), - dataDir: flags.string({ - required: true, - description: 'Directory where data objects to upload are stored', - }), - channelBatchSize: flags.integer({ - description: 'Channel batch size', - default: 20, - }), - videoBatchSize: flags.integer({ - description: 'Video batch size', - default: 20, - }), - forceChannelOwnerMemberId: flags.integer({ - description: - 'Can be used to force a specific channel owner for all channels, allowing to easily test the script in dev environment', - required: false, - }), - uploadSpEndpoint: flags.string({ - description: 'Olympia storage node endpoint to use for uploading', - default: 'http://localhost:3333', - }), - uploadSpBucketId: flags.integer({ - description: 'Olympia storage bucket id', - default: 0, - }), - migrationStatePath: flags.string({ - description: 'Path to migration results directory', - default: path.join(__dirname, '../../../results/giza-olympia'), - }), - excludeVideoIds: flags.integer({ - multiple: true, - description: 'Video ids to exclude from migration', - required: false, - default: [], - }), - } - - async run(): Promise { - const opts = this.parse(MigrateContentCommand).flags - try { - const migration = new ContentMigration(opts) - await migration.run() - } catch (e) { - console.error(e) - this.exit(-1) - } - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-commands/giza-olympia/migrateMembers.ts b/utils/migration-scripts/disabled-commands/giza-olympia/migrateMembers.ts deleted file mode 100644 index ca62419cef..0000000000 --- a/utils/migration-scripts/disabled-commands/giza-olympia/migrateMembers.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Command, flags } from '@oclif/command' -import { WsProvider } from '@polkadot/api' -import { readFileSync } from 'fs' -import path from 'path' -import { MembershipMigration } from '../../giza-olympia/MembershipMigration' -import { MembershipsSnapshot } from '../../giza-olympia/SnapshotManager' -import { RuntimeApi } from '../../RuntimeApi' - -export class MigrateMembersCommand extends Command { - static flags = { - snapshotFilePath: flags.string({ - required: true, - description: 'Path to giza memberships snapshot (json)', - }), - wsProviderEndpointUri: flags.string({ - description: 'WS provider endpoint uri (Olympia)', - default: 'ws://localhost:9944', - }), - sudoUri: flags.string({ - description: 'Sudo key Substrate uri', - default: '//Alice', - }), - batchSize: flags.integer({ - description: 'Members batch size', - default: 100, - }), - migrationStatePath: flags.string({ - description: 'Path to migration results directory', - default: path.join(__dirname, '../../../results/giza-olympia'), - }), - } - - async run(): Promise { - const opts = this.parse(MigrateMembersCommand).flags - try { - const api = new RuntimeApi({ provider: new WsProvider(opts.wsProviderEndpointUri) }) - await api.isReadyOrError - const snapshot = JSON.parse(readFileSync(opts.snapshotFilePath).toString()) as MembershipsSnapshot - const migration = new MembershipMigration({ - api, - snapshot, - config: opts, - }) - await migration.run() - } catch (e) { - console.error(e) - this.exit(-1) - } - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-commands/sumer-giza/migrateContent.ts b/utils/migration-scripts/disabled-commands/sumer-giza/migrateContent.ts deleted file mode 100644 index 5376c6bbed..0000000000 --- a/utils/migration-scripts/disabled-commands/sumer-giza/migrateContent.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Command, flags } from '@oclif/command' -import path from 'path' -import os from 'os' -import { ContentMigration } from '../../sumer-giza/ContentMigration' - -export class MigrateContentCommand extends Command { - static flags = { - queryNodeUri: flags.string({ - description: 'Query node uri', - default: 'https://hydra.joystream.org/graphql', - }), - wsProviderEndpointUri: flags.string({ - description: 'WS provider endpoint uri (Giza)', - default: 'ws://localhost:9944', - }), - sudoUri: flags.string({ - description: 'Sudo key Substrate uri', - default: '//Alice', - }), - channelIds: flags.integer({ - char: 'c', - multiple: true, - description: 'Channel ids to migrate', - required: true, - }), - dataDir: flags.string({ - description: 'Directory for storing data objects to upload', - default: path.join(os.tmpdir(), 'joystream/sumer-giza-migration'), - }), - channelBatchSize: flags.integer({ - description: 'Channel batch size', - default: 20, - }), - videoBatchSize: flags.integer({ - description: 'Video batch size', - default: 20, - }), - forceChannelOwnerMemberId: flags.integer({ - description: - 'Can be used to force a specific channel owner for all channels, allowing to easily test the script in dev environment', - required: false, - }), - preferredDownloadSpEndpoints: flags.string({ - multiple: true, - description: 'Preferred storage node endpoints when downloading data objects', - default: ['https://storage-1.joystream.org/storage'], - }), - uploadSpEndpoint: flags.string({ - description: 'Giza storage node endpoint to use for uploading', - default: 'http://localhost:3333', - }), - uploadSpBucketId: flags.integer({ - description: 'Giza storage bucket id', - default: 0, - }), - migrationStatePath: flags.string({ - description: 'Path to migration results directory', - default: path.join(__dirname, '../../../results/sumer-giza'), - }), - excludeVideoIds: flags.integer({ - multiple: true, - description: 'Video ids to exclude from migration', - required: false, - default: [], - }), - } - - async run(): Promise { - const opts = this.parse(MigrateContentCommand).flags - try { - const migration = new ContentMigration(opts) - await migration.run() - } catch (e) { - console.error(e) - this.exit(-1) - } - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-commands/sumer-giza/retryFailedUploads.ts b/utils/migration-scripts/disabled-commands/sumer-giza/retryFailedUploads.ts deleted file mode 100644 index 1825c95a44..0000000000 --- a/utils/migration-scripts/disabled-commands/sumer-giza/retryFailedUploads.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Command, flags } from '@oclif/command' -import path from 'path' -import os from 'os' -import { WsProvider } from '@polkadot/rpc-provider' -import { RuntimeApi } from '../../RuntimeApi' -import { AssetsManager } from '../../sumer-giza/AssetsManager' - -export class RetryFailedUploadsCommand extends Command { - static flags = { - wsProviderEndpointUri: flags.string({ - description: 'WS provider endpoint uri (Giza)', - default: 'ws://localhost:9944', - }), - dataDir: flags.string({ - description: 'Directory where data objects to upload are stored', - default: path.join(os.tmpdir(), 'joystream/sumer-giza-migration'), - }), - uploadSpEndpoint: flags.string({ - description: 'Giza storage node endpoint to use for uploading', - default: 'http://localhost:3333', - }), - uploadSpBucketId: flags.integer({ - description: 'Giza storage bucket id', - default: 0, - }), - failedUploadsPath: flags.string({ - char: 'f', - description: 'Path to failed uploads file', - required: true, - }), - } - - async run(): Promise { - const opts = this.parse(RetryFailedUploadsCommand).flags - try { - const provider = new WsProvider(opts.wsProviderEndpointUri) - const api = new RuntimeApi({ provider }) - await api.isReadyOrError - const assetsManager = await AssetsManager.create({ - api, - config: { - ...opts, - migrationStatePath: path.dirname(opts.failedUploadsPath), - }, - }) - assetsManager.loadQueue(opts.failedUploadsPath) - await assetsManager.processQueuedUploads() - } catch (e) { - console.error(e) - this.exit(-1) - } - this.exit(0) - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/AssetsBase.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/AssetsBase.ts deleted file mode 100644 index 330922f916..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/AssetsBase.ts +++ /dev/null @@ -1,94 +0,0 @@ -import axios from 'axios' -import stringify from 'fast-safe-stringify' -import { createReadStream, existsSync, statSync, mkdirSync } from 'fs' -import path from 'path' -import { Readable } from 'stream' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { ContentHash } from './ContentHash' -import { StorageDataObjectFieldsFragment } from './giza-query-node/generated/queries' - -export type AssetsBaseConfig = { - dataDir: string -} - -export type AssetsBaseParams = { - config: AssetsBaseConfig -} - -export abstract class AssetsBase { - protected config: AssetsBaseConfig - protected logger: Logger - - protected constructor(params: AssetsBaseParams) { - const { config } = params - this.config = config - this.logger = createLogger('Assets Base') - mkdirSync(this.tmpAssetPath(''), { recursive: true }) - mkdirSync(this.assetPath(''), { recursive: true }) - } - - protected tmpAssetPath(dataObjectId: string): string { - return path.join(this.config.dataDir, 'tmp', dataObjectId) - } - - protected assetPath(contentHash: string): string { - return path.join(this.config.dataDir, 'objects', contentHash) - } - - protected calcContentHash(assetPath: string): Promise { - return new Promise((resolve, reject) => { - const fReadStream = createReadStream(assetPath) - const hash = new ContentHash() - fReadStream.on('data', (chunk) => hash.update(chunk)) - fReadStream.on('end', () => resolve(hash.digest())) - fReadStream.on('error', (err) => reject(err)) - }) - } - - protected async isAssetMissing(dataObject: StorageDataObjectFieldsFragment): Promise { - const assetPath = this.assetPath(dataObject.ipfsHash) - if (!existsSync(assetPath)) { - this.logger.debug(`isAssetMissing: ${assetPath} not found`) - return true - } - const { size } = statSync(assetPath) - if (size.toString() !== dataObject.size) { - this.logger.debug(`isAssetMissing: Unexpected size (expected: ${dataObject.size}, got: ${size.toString()})`) - return true - } - const hash = await this.calcContentHash(assetPath) - if (hash !== dataObject.ipfsHash) { - this.logger.debug(`isAssetMissing: Unexpected hash (expected: ${dataObject.ipfsHash}, got: ${hash})`) - return true - } - return false - } - - private streamToString(stream: Readable) { - const chunks: Uint8Array[] = [] - return new Promise((resolve, reject) => { - stream.on('data', (chunk) => chunks.push(Buffer.from(chunk))) - stream.on('error', (err) => reject(err)) - stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) - }) - } - - protected async reqErrorMessage(e: unknown): Promise { - if (axios.isAxiosError(e)) { - let msg = e.message - if (e.response && typeof e.response.data === 'string') { - msg += `: ${e.response.data}` - } - if (e.response && e.response.data && e.response.data.message) { - msg += `: ${e.response.data.message}` - } - if (e.response && e.response.data && e.response.data instanceof Readable) { - msg += `: ${await this.streamToString(e.response.data)}` - } - - return msg - } - return e instanceof Error ? e.message : stringify(e) - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/BaseMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/BaseMigration.ts deleted file mode 100644 index 94120f8bbe..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/BaseMigration.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { SubmittableResult } from '@polkadot/api' -import { KeyringPair } from '@polkadot/keyring/types' -import { RuntimeApi } from '../RuntimeApi' -import { Keyring } from '@polkadot/keyring' -import { Logger } from 'winston' -import path from 'path' -import nodeCleanup from 'node-cleanup' -import _ from 'lodash' -import fs from 'fs' -import { SubmittableExtrinsic } from '@polkadot/api/types' - -export type MigrationResult = { - idsMap: Map - failedMigrations: number[] -} - -export type MigrationStateJson = { - idsMapEntries: [number, number][] - failedMigrations: number[] -} - -export type BaseMigrationConfig = { - migrationStatePath: string - sudoUri: string -} - -export type BaseMigrationParams = { - api: RuntimeApi - snapshot: T - config: BaseMigrationConfig -} - -export abstract class BaseMigration { - abstract readonly name: string - protected api: RuntimeApi - protected sudo!: KeyringPair - protected config: BaseMigrationConfig - protected snapshot: T - protected failedMigrations: Set - protected idsMap: Map - protected pendingMigrationIteration: Promise | undefined - protected abstract logger: Logger - - public constructor({ api, config, snapshot }: BaseMigrationParams) { - this.api = api - this.config = config - this.failedMigrations = new Set() - this.idsMap = new Map() - this.snapshot = snapshot - fs.mkdirSync(config.migrationStatePath, { recursive: true }) - } - - protected getMigrationStateFilePath(): string { - const { migrationStatePath } = this.config - return path.join(migrationStatePath, `${_.camelCase(this.name)}.json`) - } - - public async init(): Promise { - this.loadMigrationState() - nodeCleanup(this.onExit.bind(this)) - await this.loadSudoKey() - } - - public abstract run(): Promise - - protected abstract migrateBatch(batchTx: SubmittableExtrinsic<'promise'>, batch: { id: string }[]): Promise - - protected getMigrationStateJson(): MigrationStateJson { - return { - idsMapEntries: Array.from(this.idsMap.entries()), - failedMigrations: Array.from(this.failedMigrations), - } - } - - protected loadMigrationState(): void { - const stateFilePath = this.getMigrationStateFilePath() - if (fs.existsSync(stateFilePath)) { - const migrationStateJson = fs.readFileSync(stateFilePath).toString() - const migrationState: MigrationStateJson = JSON.parse(migrationStateJson) - this.idsMap = new Map(migrationState.idsMapEntries) - } - } - - protected onExit(exitCode: number | null, signal: string | null): void | false { - nodeCleanup.uninstall() // don't call cleanup handler again - this.logger.info('Exitting...') - if (signal && this.pendingMigrationIteration) { - this.logger.info('Waiting for currently pending migration iteration to finalize...') - this.pendingMigrationIteration.then(() => { - this.saveMigrationState(true) - this.logger.info('Done.') - process.kill(process.pid, signal) - }) - return false - } else { - this.saveMigrationState(true) - this.logger.info('Done.') - } - } - - protected saveMigrationState(isExitting: boolean): void { - this.logger.info(`Saving ${isExitting ? 'final' : 'intermediate'} migration state...`) - const stateFilePath = this.getMigrationStateFilePath() - const migrationState = this.getMigrationStateJson() - fs.writeFileSync(stateFilePath, JSON.stringify(migrationState, undefined, 2)) - } - - private async loadSudoKey() { - const { sudoUri } = this.config - const keyring = new Keyring({ type: 'sr25519' }) - this.sudo = keyring.createFromUri(sudoUri) - const sudoKey = await this.api.query.sudo.key() - if (sudoKey.toString() !== this.sudo.address) { - throw new Error(`Invalid sudo key! Expected: ${sudoKey.toString()}, Got: ${this.sudo.address}`) - } - } - - protected async executeBatchMigration( - batchTx: SubmittableExtrinsic<'promise'>, - batch: T[] - ): Promise { - this.pendingMigrationIteration = (async () => { - await this.migrateBatch(batchTx, batch) - this.saveMigrationState(false) - })() - await this.pendingMigrationIteration - this.pendingMigrationIteration = undefined - } - - /** - * Extract failed migrations (entity ids) from batch transaction result. - * Assumptions: - * - Each entity is migrated with a constant number of calls (2 by default: balnces.transferKeepAlive and sudo.sudoAs) - * - Ordering of the entities in the `batch` array matches the ordering of the batched calls through which they are migrated - * - If `usesSudoAs===true`: Last call for each entity is always sudo.sudoAs - * - If `usesSudoAs===true`: There is only one sudo.sudoAs call per entity - * - * Entity migration is considered failed if the last call (per entity) failed or was not executed at all, regardless of - * the result of any of the previous calls associated with that entity migration. - * (This means, for example, that regardless of whether balnces.transferKeepAlive failed and interrupted the batch or balnces.transferKeepAlive - * succeeded, but sudo.sudoAs failed - in both cases the migration is considered failed and should be fully re-executed on - * the next script run) - */ - protected extractFailedMigrations( - result: SubmittableResult, - batch: T[], - callsPerEntity = 2, - usesSudoAs = true - ): void { - const { api } = this - const batchInterruptedEvent = api.findEvent(result, 'utility', 'BatchInterrupted') - const numberOfSuccesfulCalls = batchInterruptedEvent - ? batchInterruptedEvent.data[0].toNumber() - : callsPerEntity * batch.length - const numberOfMigratedEntites = Math.floor(numberOfSuccesfulCalls / callsPerEntity) - - const sudoAsDoneEvents = api.findEvents(result, 'sudo', 'SudoAsDone') - if (usesSudoAs && sudoAsDoneEvents.length !== numberOfMigratedEntites) { - throw new Error( - `Unexpected number of SudoAsDone events (expected: ${numberOfMigratedEntites}, got: ${sudoAsDoneEvents.length})! ` + - `Could not extract failed migrations from: ${JSON.stringify(result.toHuman())}` - ) - } - - const failedIds: number[] = [] - batch.forEach((entity, i) => { - const entityId = parseInt(entity.id) - if (i >= numberOfMigratedEntites || (usesSudoAs && sudoAsDoneEvents[i].data[0].isFalse)) { - failedIds.push(entityId) - this.failedMigrations.add(entityId) - } - }) - - if (batchInterruptedEvent) { - this.logger.error( - `Batch interrupted at call ${numberOfSuccesfulCalls}: ${this.api.formatDispatchError( - batchInterruptedEvent.data[1] - )}` - ) - } - - if (failedIds.length) { - this.logger.error(`Failed to migrate:`, { failedIds }) - } - } - - public getResult(): MigrationResult { - const { idsMap, failedMigrations } = this - return { - idsMap: new Map(idsMap.entries()), - failedMigrations: Array.from(failedMigrations), - } - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/CategoryMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/CategoryMigration.ts deleted file mode 100644 index dcaa8c8128..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/CategoryMigration.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { BaseMigration } from './BaseMigration' -import { ContentDirectorySnapshot } from './SnapshotManager' - -export abstract class CategoryMigration extends BaseMigration { - protected contentLeadKey!: string - - public async init(): Promise { - await super.init() - await this.loadContentLeadKey() - } - - private async loadContentLeadKey(): Promise { - const { api } = this - const leadId = await api.query.contentWorkingGroup.currentLead() - if (!leadId.isSome) { - throw new Error('ContentWorkingGroup lead must be set!') - } - const leadWorker = await api.query.contentWorkingGroup.workerById(leadId.unwrap()) - this.contentLeadKey = leadWorker.role_account_id.toString() - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/ChannelCategoriesMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/ChannelCategoriesMigration.ts deleted file mode 100644 index 71fa90c6b1..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/ChannelCategoriesMigration.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { ChannelCategoryMetadata } from '@joystream/metadata-protobuf' -import { ChannelCategoryId } from '@joystream/types/content' -import { SubmittableExtrinsic } from '@polkadot/api/types' -import { ISubmittableResult } from '@polkadot/types/types' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { BaseMigrationParams, MigrationResult } from './BaseMigration' -import { CategoryMigration } from './CategoryMigration' -import { ContentDirectorySnapshot } from './SnapshotManager' - -export class ChannelCategoriesMigration extends CategoryMigration { - name = 'Channel categories migration' - protected logger: Logger - - public constructor(params: BaseMigrationParams) { - super(params) - this.logger = createLogger(this.name) - } - - protected async migrateBatch( - batchTx: SubmittableExtrinsic<'promise', ISubmittableResult>, - batch: { id: string }[] - ): Promise { - const { api } = this - const result = await api.sendExtrinsic(this.sudo, batchTx) - const categoryCreatedEvents = api.findEvents(result, 'content', 'ChannelCategoryCreated') - const createdCategoryIds: ChannelCategoryId[] = categoryCreatedEvents.map((e) => e.data[0]) - - if (createdCategoryIds.length !== batch.length) { - this.extractFailedMigrations(result, batch) - } - - let newCategoryIndex = 0 - batch.forEach((c) => { - if (this.failedMigrations.has(parseInt(c.id))) { - return - } - const newCategoryId = createdCategoryIds[newCategoryIndex++] - this.idsMap.set(parseInt(c.id), newCategoryId.toNumber()) - }) - this.logger.info(`Channel categories map created!`, this.idsMap.entries()) - if (this.failedMigrations.size) { - throw new Error(`Failed to create some channel categories: ${Array.from(this.failedMigrations).join(', ')}`) - } - this.logger.info(`All channel categories succesfully migrated!`) - } - - public async run(): Promise { - await this.init() - const { api } = this - const allCategories = this.snapshot.channelCategories - const categoriesToMigrate = allCategories.filter((c) => !this.idsMap.has(parseInt(c.id))) - - if (!categoriesToMigrate.length) { - this.logger.info('All channel categories already migrated, skipping...') - return this.getResult() - } - - this.logger.info(`Migrating ${categoriesToMigrate.length} channel categories...`) - const txs = categoriesToMigrate - .sort((a, b) => parseInt(a.id) - parseInt(b.id)) - .map((c) => { - const meta = new ChannelCategoryMetadata({ name: c.name }) - const metaBytes = '0x' + Buffer.from(ChannelCategoryMetadata.encode(meta).finish()).toString('hex') - return api.tx.sudo.sudoAs( - this.contentLeadKey, - api.tx.content.createChannelCategory('Lead', { - meta: metaBytes, - }) - ) - }) - - const batchTx = api.tx.utility.batch(txs) - await this.migrateBatch(batchTx, categoriesToMigrate) - - return this.getResult() - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/ChannelsMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/ChannelsMigration.ts deleted file mode 100644 index f408f2222b..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/ChannelsMigration.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { UploadMigration, UploadMigrationConfig, UploadMigrationParams } from './UploadMigration' -import { ChannelMetadata } from '@joystream/metadata-protobuf' -import { ChannelFieldsFragment } from './giza-query-node/generated/queries' -import { createType } from '@joystream/types' -import Long from 'long' -import { ChannelCreationParameters } from '@joystream/types/content' -import { ChannelId } from '@joystream/types/common' -import _ from 'lodash' -import { MigrationResult } from './BaseMigration' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { SubmittableExtrinsic } from '@polkadot/api/types' - -export type ChannelsMigrationConfig = UploadMigrationConfig & { - channelIds: number[] - channelBatchSize: number - forceChannelOwnerMemberId: number | undefined - excludeVideoIds: number[] -} - -export type ChannelsMigrationParams = UploadMigrationParams & { - config: ChannelsMigrationConfig - forcedChannelOwner: { id: string; controllerAccount: string } | undefined - categoriesMap: Map - membershipsMap: Map -} - -export type ChannelsMigrationResult = MigrationResult & { - videoIds: number[] -} - -export class ChannelMigration extends UploadMigration { - name = 'Channels migration' - protected config: ChannelsMigrationConfig - protected categoriesMap: Map - protected membershipsMap: Map - protected videoIds: number[] = [] - protected forcedChannelOwner: { id: string; controllerAccount: string } | undefined - protected logger: Logger - - public constructor(params: ChannelsMigrationParams) { - super(params) - this.config = params.config - this.forcedChannelOwner = params.forcedChannelOwner - this.categoriesMap = params.categoriesMap - this.membershipsMap = params.membershipsMap - this.logger = createLogger(this.name) - } - - private getNewCategoryId(oldCategoryId: string | null | undefined): Long | undefined { - if (typeof oldCategoryId !== 'string') { - return undefined - } - const newCategoryId = this.categoriesMap.get(parseInt(oldCategoryId)) - return newCategoryId ? Long.fromNumber(newCategoryId) : undefined - } - - private getChannelOwnerMember({ - id, - ownerMember, - }: ChannelFieldsFragment): Exclude { - if (!ownerMember) { - throw new Error(`Chanel ownerMember missing: ${id}. Only member-owned channels are supported!`) - } - - if (this.forcedChannelOwner) { - return this.forcedChannelOwner - } - - const newMemberId = this.membershipsMap.get(parseInt(ownerMember.id)) - if (newMemberId === undefined) { - throw new Error(`Missing member ${ownerMember.id} (owner of channel ${id}) in the memberships map!`) - } - - return { ...ownerMember, id: newMemberId.toString() } - } - - protected async migrateBatch(tx: SubmittableExtrinsic<'promise'>, channels: ChannelFieldsFragment[]): Promise { - const { api } = this - const result = await api.sendExtrinsic(this.sudo, tx) - const channelCreatedEvents = api.findEvents(result, 'content', 'ChannelCreated') - const newChannelIds: ChannelId[] = channelCreatedEvents.map((e) => e.data[1]) - if (channelCreatedEvents.length !== channels.length) { - this.extractFailedMigrations(result, channels) - } - const newChannelMapEntries: [number, number][] = [] - let newChannelIdIndex = 0 - channels.forEach(({ id }) => { - if (this.failedMigrations.has(parseInt(id))) { - return - } - const newChannelId = newChannelIds[newChannelIdIndex++].toNumber() - this.idsMap.set(parseInt(id), newChannelId) - newChannelMapEntries.push([parseInt(id), newChannelId]) - }) - if (newChannelMapEntries.length) { - this.logger.info('Channel map entries added!', { newChannelMapEntries }) - const dataObjectsUploadedEvents = this.api.findEvents(result, 'storage', 'DataObjectsUploaded') - this.uploadManager.queueUploadsFromEvents(dataObjectsUploadedEvents) - } - } - - public async run(): Promise { - await this.init() - const { - api, - config: { channelIds, channelBatchSize }, - } = this - const ids = channelIds.sort((a, b) => a - b) - while (ids.length) { - const idsBatch = ids.splice(0, channelBatchSize) - this.logger.info(`Preparing a batch of ${idsBatch.length} channels...`) - const channelsBatch = this.snapshot.channels.filter((c) => idsBatch.includes(parseInt(c.id))) - if (channelsBatch.length < idsBatch.length) { - this.logger.warn( - `Some channels were not be found: ${_.difference( - idsBatch, - channelsBatch.map((c) => parseInt(c.id)) - )}` - ) - } - const channelsToMigrate = channelsBatch.filter((c) => !this.idsMap.has(parseInt(c.id))) - if (channelsToMigrate.length < channelsBatch.length) { - this.logger.info( - `${channelsToMigrate.length ? 'Some' : 'All'} channels in this batch were already migrated ` + - `(${channelsBatch.length - channelsToMigrate.length}/${channelsBatch.length})` - ) - } - if (channelsToMigrate.length) { - const calls = _.flatten(await Promise.all(channelsToMigrate.map((c) => this.prepareChannel(c)))) - const batchTx = api.tx.utility.batch(calls) - await this.executeBatchMigration(batchTx, channelsToMigrate) - await this.uploadManager.processQueuedUploads() - } - const videoIdsToMigrate: number[] = channelsBatch.reduce( - (res, { id, videos }) => - this.idsMap.has(parseInt(id)) - ? res.concat(videos.map((v) => parseInt(v.id)).filter((id) => !this.config.excludeVideoIds.includes(id))) - : res, - [] - ) - this.videoIds = this.videoIds.concat(videoIdsToMigrate) - if (videoIdsToMigrate.length) { - this.logger.info(`Added ${videoIdsToMigrate.length} video ids to the list of videos to migrate`) - } - } - return { - ...this.getResult(), - videoIds: [...this.videoIds], - } - } - - private async prepareChannel(channel: ChannelFieldsFragment) { - const { api } = this - const { avatarPhoto, coverPhoto, title, description, categoryId, isPublic, language, collaborators } = channel - - const ownerMember = this.getChannelOwnerMember(channel) - - const assetsToPrepare = { - avatar: avatarPhoto || undefined, - coverPhoto: coverPhoto || undefined, - } - const preparedAssets = await this.uploadManager.prepareAssets(assetsToPrepare) - const meta = new ChannelMetadata({ - title, - description, - category: this.getNewCategoryId(categoryId), - avatarPhoto: preparedAssets.avatar?.index, - coverPhoto: preparedAssets.coverPhoto?.index, - isPublic, - language: language?.iso, - }) - const assetsParams = Object.values(preparedAssets) - .sort((a, b) => a.index - b.index) - .map((a) => a.params) - const channelCreationParams = createType( - 'ChannelCreationParameters', - { - assets: assetsParams.length - ? { - object_creation_list: assetsParams, - expected_data_size_fee: this.uploadManager.dataObjectFeePerMB, - } - : null, - meta: `0x${Buffer.from(ChannelMetadata.encode(meta).finish()).toString('hex')}`, - collaborators: collaborators.map(({ id }) => parseInt(id)), - reward_account: channel.rewardAccount, - } - ) - const feesToCover = this.uploadManager.calcDataObjectsFee(assetsParams) - return [ - api.tx.balances.transferKeepAlive(ownerMember.controllerAccount, feesToCover), - api.tx.sudo.sudoAs( - ownerMember.controllerAccount, - api.tx.content.createChannel({ Member: ownerMember.id }, channelCreationParams) - ), - ] - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/ContentHash.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/ContentHash.ts deleted file mode 100644 index e0441ce881..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/ContentHash.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createHash, HashInput, NodeHash } from 'blake3' -import { HashReader } from 'blake3/dist/wasm/nodejs' -import { toB58String, encode } from 'multihashes' - -// Based on distributor node's implementation -export class ContentHash { - private hash: NodeHash - public static readonly algorithm = 'blake3' - - constructor() { - this.hash = createHash() - } - - update(data: HashInput): this { - this.hash.update(data) - return this - } - - digest(): string { - return toB58String(encode(this.hash.digest(), ContentHash.algorithm)) - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/ContentMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/ContentMigration.ts deleted file mode 100644 index d4ece92623..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/ContentMigration.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { WsProvider } from '@polkadot/api' -import { RuntimeApi } from '../RuntimeApi' -import { VideosMigration } from './VideosMigration' -import { ChannelMigration } from './ChannelsMigration' -import { UploadManager } from './UploadManager' -import { ChannelCategoriesMigration } from './ChannelCategoriesMigration' -import { VideoCategoriesMigration } from './VideoCategoriesMigration' -import { ContentDirectorySnapshot } from './SnapshotManager' -import { readFileSync } from 'fs' -import { MigrationStateJson } from './BaseMigration' - -export type ContentMigrationConfig = { - wsProviderEndpointUri: string - sudoUri: string - channelIds: number[] - dataDir: string - channelBatchSize: number - videoBatchSize: number - forceChannelOwnerMemberId: number | undefined - preferredDownloadSpEndpoints?: string[] - uploadSpBucketId: number - uploadSpEndpoint: string - migrationStatePath: string - excludeVideoIds: number[] - snapshotFilePath: string - membershipsMigrationResultPath: string -} - -export class ContentMigration { - private api: RuntimeApi - private config: ContentMigrationConfig - - constructor(config: ContentMigrationConfig) { - const { wsProviderEndpointUri } = config - const provider = new WsProvider(wsProviderEndpointUri) - this.api = new RuntimeApi({ provider }) - this.config = config - } - - private async getForcedChannelOwner(): Promise<{ id: string; controllerAccount: string } | undefined> { - const { forceChannelOwnerMemberId } = this.config - if (forceChannelOwnerMemberId !== undefined) { - const ownerMember = await this.api.query.members.membershipById(forceChannelOwnerMemberId) - if (ownerMember.isEmpty) { - throw new Error(`Membership by id ${forceChannelOwnerMemberId} not found!`) - } - return { - id: forceChannelOwnerMemberId.toString(), - controllerAccount: ownerMember.controller_account.toString(), - } - } - return undefined - } - - private loadSnapshot(): ContentDirectorySnapshot { - const snapshotJson = readFileSync(this.config.snapshotFilePath).toString() - return JSON.parse(snapshotJson) as ContentDirectorySnapshot - } - - private loadMembershipsMap(): Map { - const resultJson = readFileSync(this.config.membershipsMigrationResultPath).toString() - const mapEntries = (JSON.parse(resultJson) as MigrationStateJson).idsMapEntries - return new Map(mapEntries) - } - - public async run(): Promise { - const { api, config } = this - await this.api.isReadyOrError - const snapshot = this.loadSnapshot() - const membershipsMap = this.loadMembershipsMap() - const { idsMap: channelCategoriesMap } = await new ChannelCategoriesMigration({ api, config, snapshot }).run() - const { idsMap: videoCategoriesMap } = await new VideoCategoriesMigration({ api, config, snapshot }).run() - const forcedChannelOwner = await this.getForcedChannelOwner() - const uploadManager = await UploadManager.create({ - api, - config, - }) - const { idsMap: channelsMap, videoIds } = await new ChannelMigration({ - api, - config, - snapshot, - forcedChannelOwner, - uploadManager, - categoriesMap: channelCategoriesMap, - membershipsMap, - }).run() - await new VideosMigration({ - api, - config, - snapshot, - channelsMap, - videoIds, - forcedChannelOwner, - uploadManager, - categoriesMap: videoCategoriesMap, - membershipsMap, - }).run() - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/DownloadManager.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/DownloadManager.ts deleted file mode 100644 index 63aa29e6b9..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/DownloadManager.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { MAX_RESULTS_PER_QUERY, QueryNodeApi } from './giza-query-node/api' -import { createWriteStream, renameSync, statSync } from 'fs' -import axios from 'axios' -import { - StorageDataObjectConnectionFieldsFragment, - StorageDataObjectFieldsFragment, -} from './giza-query-node/generated/queries' -import urljoin from 'url-join' -import { pipeline, Readable } from 'stream' -import { promisify } from 'util' -import { DistributionBucketOperatorStatus } from './giza-query-node/generated/schema' -import _ from 'lodash' -import { AssetsBase } from './AssetsBase' -import moment from 'moment' - -export type DownloadManagerConfig = { - objectsPerBatch: number - dataDir: string -} - -export type DownloadManagerParams = { - config: DownloadManagerConfig - queryNodeApi: QueryNodeApi -} - -export class DownloadManager extends AssetsBase { - name = 'Download Manager' - protected logger: Logger - protected config: DownloadManagerConfig - protected queryNodeApi: QueryNodeApi - - public constructor(params: DownloadManagerParams) { - super(params) - this.config = params.config - this.queryNodeApi = params.queryNodeApi - this.logger = createLogger(this.name) - } - - private async fetchAsset(dataObject: StorageDataObjectFieldsFragment, endpoint: string): Promise { - const assetEndpoint = urljoin(endpoint, `api/v1/assets/${dataObject.id}`) - const response = await axios.get(assetEndpoint, { responseType: 'stream', timeout: 5000 }) - const pipe = promisify(pipeline) - const destPath = this.tmpAssetPath(dataObject.id) - const fWriteStream = createWriteStream(destPath) - await pipe(response.data, fWriteStream) - const { size } = statSync(destPath) - if (size !== parseInt(dataObject.size)) { - throw new Error('Invalid file size') - } - const hash = await this.calcContentHash(destPath) - if (hash !== dataObject.ipfsHash) { - throw new Error('Invalid file hash') - } - renameSync(destPath, this.assetPath(dataObject.ipfsHash)) - } - - private async fetchAssetWithRetry( - dataObject: StorageDataObjectFieldsFragment, - distributors: Map - ): Promise { - const endpoints = distributors.get(dataObject.storageBagId) || [] - let lastError: Error | undefined - for (const endpoint of endpoints) { - try { - this.logger.debug(`Trying to fetch data object ${dataObject.id} (${dataObject.ipfsHash}) from ${endpoint}...`) - await this.fetchAsset(dataObject, endpoint) - return - } catch (e) { - this.logger.debug( - `Fetching object ${dataObject.id} (${dataObject.ipfsHash}) from ${endpoint} failed: ${(e as Error).message}` - ) - lastError = e as Error - continue - } - } - this.logger.error( - `Could not fetch data object ${dataObject.id} (${dataObject.ipfsHash}) from any distributor. Last error: ${ - lastError && (await this.reqErrorMessage(lastError)) - }` - ) - } - - private async getDistributors(dataObjects: StorageDataObjectFieldsFragment[]): Promise> { - this.logger.info(`Fetching the distributors for ${dataObjects.length} data objects...`) - const bagIds = _.uniq(dataObjects.map((o) => o.storageBagId)) - const buckets = await this.queryNodeApi.getDistributorsByBagIds(bagIds) - this.logger.info(`Fetched the data of ${buckets.length} unique distribution buckets`) - - const endpointsByBagId = new Map() - for (const bucket of buckets) { - for (const operator of bucket.operators) { - if (operator.status === DistributionBucketOperatorStatus.Active && operator.metadata?.nodeEndpoint) { - for (const bag of bucket.bags) { - const currEndpoints = endpointsByBagId.get(bag.id) || [] - endpointsByBagId.set(bag.id, [...currEndpoints, operator.metadata.nodeEndpoint]) - } - } - } - } - - return endpointsByBagId - } - - protected async fetchIfMissing( - dataObject: StorageDataObjectFieldsFragment, - knownDistributors: Map - ): Promise { - const missing = await this.isAssetMissing(dataObject) - if (missing) { - this.logger.debug(`Object ${dataObject.ipfsHash} missing, fetching...`) - await this.fetchAssetWithRetry(dataObject, knownDistributors) - return true - } - - this.logger.debug(`Object ${dataObject.ipfsHash} already exists, skipping...`) - return false - } - - public async fetchAllDataObjects(updatedAfter?: Date, continously = false, idleTimeSec?: number): Promise { - do { - let currentPage: StorageDataObjectConnectionFieldsFragment | undefined - let lastObjectUpdatedAt: Date | undefined - this.logger.info( - `Fetching all data objects${updatedAfter ? ` updated after ${updatedAfter.toISOString()}` : ''}...` - ) - do { - this.logger.info(`Fetching a page of up to ${MAX_RESULTS_PER_QUERY} data objects from the query node...`) - currentPage = await this.queryNodeApi.getStorageDataObjectsPage( - updatedAfter, - MAX_RESULTS_PER_QUERY, - currentPage?.pageInfo.endCursor || undefined - ) - const objects = currentPage.edges.map((e) => e.node) - const maxUpdatedAt = _.maxBy(objects, (o) => moment(o.updatedAt).unix())?.updatedAt - lastObjectUpdatedAt = maxUpdatedAt ? new Date(maxUpdatedAt) : undefined - this.logger.info(`Fetched ${objects.length} data object rows`) - if (objects.length) { - const distributors = await this.getDistributors(objects) - while (objects.length) { - const batch = objects.splice(0, this.config.objectsPerBatch) - this.logger.info(`Processing a batch of ${batch.length} data objects...`) - const results = await Promise.all(batch.map((o) => this.fetchIfMissing(o, distributors))) - const downloadedObjectsLength = results.reduce((a, b) => a + (b ? 1 : 0), 0) - this.logger.info(`Downloaded ${downloadedObjectsLength} new data objects...`) - } - } - } while (currentPage.pageInfo.hasNextPage) - if (continously && idleTimeSec) { - this.logger.info(`Waiting ${idleTimeSec} seconds...`) - await new Promise((resolve) => setTimeout(resolve, 1000 * idleTimeSec)) - } - updatedAfter = updatedAfter || lastObjectUpdatedAt - // eslint-disable-next-line no-unmodified-loop-condition - } while (continously) - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/MembershipMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/MembershipMigration.ts deleted file mode 100644 index ca0da3610c..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/MembershipMigration.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { MembershipMetadata } from '@joystream/metadata-protobuf' -import { MembershipFieldsFragment } from './giza-query-node/generated/queries' -import { createType } from '@joystream/types' -import { MemberId } from '@joystream/types/common' -import { BaseMigration, BaseMigrationConfig, BaseMigrationParams, MigrationResult } from './BaseMigration' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { SubmittableExtrinsic } from '@polkadot/api/types' -import { MembershipsSnapshot } from './SnapshotManager' -import { BuyMembershipParameters } from '@joystream/types/members' - -export type MembershipMigrationConfig = BaseMigrationConfig & { - batchSize: number -} - -export type MembershipMigrationParams = BaseMigrationParams & { - config: MembershipMigrationConfig -} - -export class MembershipMigration extends BaseMigration { - name = 'Membership migration' - protected config: MembershipMigrationConfig - protected logger: Logger - - public constructor(params: MembershipMigrationParams) { - super(params) - this.config = params.config - this.logger = createLogger(this.name) - } - - protected async migrateBatch( - tx: SubmittableExtrinsic<'promise'>, - members: MembershipFieldsFragment[] - ): Promise { - const { api } = this - const result = await api.sendExtrinsic(this.sudo, tx) - const membershipBoughtEvents = api.findEvents(result, 'members', 'MembershipBought') - const newMemberIds: MemberId[] = membershipBoughtEvents.map((e) => e.data[0]) - if (membershipBoughtEvents.length !== members.length) { - this.extractFailedMigrations(result, members, 1, false) - } - const newMembersMapEntries: [number, number][] = [] - let newMemberIdIndex = 0 - members.forEach(({ id }) => { - if (this.failedMigrations.has(parseInt(id))) { - return - } - const newMemberId = newMemberIds[newMemberIdIndex++].toNumber() - this.idsMap.set(parseInt(id), newMemberId) - newMembersMapEntries.push([parseInt(id), newMemberId]) - }) - if (newMembersMapEntries.length) { - this.logger.info('Members map entries added!', { newMembersMapEntries }) - } - } - - public async run(): Promise { - await this.init() - const { - api, - config: { batchSize }, - } = this - let membersBatch: MembershipFieldsFragment[] = [] - while ((membersBatch = this.snapshot.members.splice(0, batchSize)).length) { - this.logger.info(`Preparing a batch of ${membersBatch.length} members...`) - const membersToMigrate = membersBatch.filter((m) => !this.idsMap.has(parseInt(m.id))) - if (membersToMigrate.length < membersBatch.length) { - this.logger.info( - `${membersToMigrate.length ? 'Some' : 'All'} members in this batch were already migrated ` + - `(${membersBatch.length - membersToMigrate.length}/${membersBatch.length})` - ) - } - if (membersToMigrate.length) { - const calls = await Promise.all(membersToMigrate.map((m) => this.prepareMember(m))) - const batchTx = api.tx.utility.batch(calls) - await this.executeBatchMigration(batchTx, membersToMigrate) - } - } - return this.getResult() - } - - private async prepareMember(member: MembershipFieldsFragment) { - const { api } = this - const { handle, rootAccount, controllerAccount, about, avatarUri } = member - - const meta = new MembershipMetadata({ - about, - avatarUri, - }) - const buyMembershipParams = createType( - 'BuyMembershipParameters', - { - handle, - controller_account: controllerAccount, - root_account: rootAccount, - metadata: `0x${Buffer.from(MembershipMetadata.encode(meta).finish()).toString('hex')}`, - } - ) - - return api.tx.members.buyMembership(buyMembershipParams) - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/SnapshotManager.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/SnapshotManager.ts deleted file mode 100644 index 9d2bbdf76f..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/SnapshotManager.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { MAX_RESULTS_PER_QUERY, QueryNodeApi } from './giza-query-node/api' -import { - ChannelCategoryFieldsFragment, - ChannelConnectionFieldsFragment, - ChannelFieldsFragment, - MembershipConnectionFieldsFragment, - MembershipFieldsFragment, - VideoCategoryFieldsFragment, - VideoConnectionFieldsFragment, - VideoFieldsFragment, -} from './giza-query-node/generated/queries' - -export type SnapshotManagerParams = { - queryNodeApi: QueryNodeApi -} - -export type ContentDirectorySnapshot = { - channelCategories: ChannelCategoryFieldsFragment[] - videoCategories: VideoCategoryFieldsFragment[] - channels: ChannelFieldsFragment[] - videos: VideoFieldsFragment[] -} - -export type MembershipsSnapshot = { - members: MembershipFieldsFragment[] -} - -export class SnapshotManager { - name = 'Snapshot Manager' - protected logger: Logger - protected queryNodeApi: QueryNodeApi - - public constructor(params: SnapshotManagerParams) { - this.queryNodeApi = params.queryNodeApi - this.logger = createLogger(this.name) - } - - private sortEntitiesByIds(entities: T[]): T[] { - return entities.sort((a, b) => parseInt(a.id) - parseInt(b.id)) - } - - public async getChannels(): Promise { - let lastCursor: string | undefined - let currentPage: ChannelConnectionFieldsFragment - let channels: ChannelFieldsFragment[] = [] - do { - this.logger.info(`Fetching a page of up to ${MAX_RESULTS_PER_QUERY} channels...`) - currentPage = await this.queryNodeApi.getChannelsPage(lastCursor) - channels = channels.concat(currentPage.edges.map((e) => e.node)) - this.logger.info(`Total ${channels.length} channels fetched`) - lastCursor = currentPage.pageInfo.endCursor || undefined - } while (currentPage.pageInfo.hasNextPage) - this.logger.info('Finished channels fetching') - - return this.sortEntitiesByIds(channels) - } - - public async getVideos(): Promise { - let lastCursor: string | undefined - let currentPage: VideoConnectionFieldsFragment - let videos: VideoFieldsFragment[] = [] - do { - this.logger.info(`Fetching a page of up to ${MAX_RESULTS_PER_QUERY} videos...`) - currentPage = await this.queryNodeApi.getVideosPage(lastCursor) - videos = videos.concat(currentPage.edges.map((e) => e.node)) - this.logger.info(`Total ${videos.length} videos fetched`) - lastCursor = currentPage.pageInfo.endCursor || undefined - } while (currentPage.pageInfo.hasNextPage) - this.logger.info('Finished videos fetching') - - return this.sortEntitiesByIds(videos) - } - - public async getMemberships(): Promise { - let lastCursor: string | undefined - let currentPage: MembershipConnectionFieldsFragment - let members: MembershipFieldsFragment[] = [] - do { - this.logger.info(`Fetching a page of up to ${MAX_RESULTS_PER_QUERY} members...`) - currentPage = await this.queryNodeApi.getMembershipsPage(lastCursor) - members = members.concat(currentPage.edges.map((e) => e.node)) - this.logger.info(`Total ${members.length} members fetched`) - lastCursor = currentPage.pageInfo.endCursor || undefined - } while (currentPage.pageInfo.hasNextPage) - this.logger.info('Finished members fetching') - - return this.sortEntitiesByIds(members) - } - - public async createContentDirectorySnapshot(): Promise { - const channelCategories = this.sortEntitiesByIds(await this.queryNodeApi.getChannelCategories()) - const videoCategories = this.sortEntitiesByIds(await this.queryNodeApi.getVideoCategories()) - const channels = await this.getChannels() - const videos = await this.getVideos() - return { channelCategories, videoCategories, videos, channels } - } - - public async createMembershipsSnapshot(): Promise { - const members = await this.getMemberships() - return { - members, - } - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/UploadManager.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/UploadManager.ts deleted file mode 100644 index 97f3cc4bf9..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/UploadManager.ts +++ /dev/null @@ -1,209 +0,0 @@ -import BN from 'bn.js' -import urljoin from 'url-join' -import axios from 'axios' -import fs from 'fs' -import path from 'path' -import { BagId, DataObjectCreationParameters, DataObjectId, UploadParameters } from '@joystream/types/storage' -import { IEvent } from '@polkadot/types/types' -import { Bytes, Vec } from '@polkadot/types' -import { Balance } from '@polkadot/types/interfaces' -import FormData from 'form-data' -import { RuntimeApi } from '../RuntimeApi' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { AssetsBase } from './AssetsBase' -import { StorageDataObjectFieldsFragment } from './giza-query-node/generated/queries' -import { createType } from '@joystream/types' - -export type UploadManagerConfig = { - uploadSpBucketId: number - uploadSpEndpoint: string - dataDir: string - migrationStatePath: string -} - -export type UploadManagerParams = { - api: RuntimeApi - config: UploadManagerConfig -} - -export type UploadManagerLoadableParams = { - dataObjectFeePerMB: BN -} - -export type AssetsToPrepare = { - [name: string]: StorageDataObjectFieldsFragment | undefined -} - -export type PreparedAsset = { - params: DataObjectCreationParameters - index: number -} - -export class UploadManager extends AssetsBase { - private api: RuntimeApi - protected config: UploadManagerConfig - public readonly dataObjectFeePerMB: BN - private queuedUploads: Set - private isQueueProcessing = false - private queueFilePath: string - protected logger: Logger - - public get queueSize(): number { - return this.queuedUploads.size - } - - public static async create(params: UploadManagerParams): Promise { - const { api } = params - const dataObjectFeePerMB = await api.query.storage.dataObjectPerMegabyteFee() - return new UploadManager(params, { dataObjectFeePerMB }) - } - - private constructor(params: UploadManagerParams, loadableParams: UploadManagerLoadableParams) { - super(params) - const { api, config } = params - const { dataObjectFeePerMB } = loadableParams - this.dataObjectFeePerMB = dataObjectFeePerMB - this.api = api - this.config = config - this.queuedUploads = new Set() - this.logger = createLogger('Assets Manager') - this.queueFilePath = path.join(this.config.migrationStatePath, `unprocessedUploads_${Date.now()}.json`) - this.logger.info(`Failed/pending uploads will be saved to ${this.queueFilePath}`) - } - - public calcDataObjectsFee(params: DataObjectCreationParameters[]): BN { - const { dataObjectFeePerMB, api } = this - const deletionPrize = api.consts.storage.dataObjectDeletionPrize - const totalSize = params - .reduce((a, b) => { - return a.add(b.getField('size')) - }, new BN(0)) - .toNumber() - const totalStorageFee = dataObjectFeePerMB.muln(Math.ceil(totalSize / 1024 / 1024)) - const totalDeletionPrize = deletionPrize.muln(params.length) - return totalStorageFee.add(totalDeletionPrize) - } - - private async uploadDataObject(bagId: string, dataObjectId: number, contentHash: string): Promise { - const { - config: { uploadSpBucketId, uploadSpEndpoint }, - } = this - const dataPath = this.assetPath(contentHash) - if (!fs.existsSync(dataPath)) { - throw new Error(`Cannot upload object: ${dataObjectId} (${contentHash}): ${dataPath} not found`) - } - - const fileStream = fs.createReadStream(dataPath) - const formData = new FormData() - formData.append('dataObjectId', dataObjectId) - formData.append('storageBucketId', uploadSpBucketId) - formData.append('bagId', bagId) - formData.append('file', fileStream, { filename: path.basename(dataPath) }) - let uploadSuccesful: boolean - try { - await axios({ - method: 'POST', - url: urljoin(uploadSpEndpoint, 'api/v1/files'), - data: formData, - maxBodyLength: Infinity, - headers: { - 'content-type': 'multipart/form-data', - ...formData.getHeaders(), - }, - }) - uploadSuccesful = true - } catch (e) { - uploadSuccesful = false - const msg = await this.reqErrorMessage(e) - this.logger.error(`Upload of object ${dataObjectId} (${contentHash}) to ${uploadSpEndpoint} failed: ${msg}`) - } - - if (uploadSuccesful) { - this.finalizeUpload(bagId, dataObjectId, contentHash) - } - } - - public async processQueuedUploads(): Promise { - if (this.isQueueProcessing) { - throw new Error('Uploads queue is already beeing processed!') - } - this.isQueueProcessing = true - this.logger.info(`Uploading ${this.queueSize} data objects...`) - await Promise.all( - Array.from(this.queuedUploads).map((queuedUpload) => { - const [bagId, objectId, contentHash] = queuedUpload.split('|') - return this.uploadDataObject(bagId, parseInt(objectId), contentHash) - }) - ) - this.isQueueProcessing = false - } - - public loadQueue(queueFilePath: string): void { - const queue: string[] = JSON.parse(fs.readFileSync(queueFilePath).toString()) - this.queuedUploads = new Set(queue) - } - - public saveQueue(): void { - fs.writeFileSync(this.queueFilePath, JSON.stringify(Array.from(this.queuedUploads))) - this.logger.debug(`${this.queueFilePath} updated`, { queueSize: this.queuedUploads.size }) - } - - private queueUpload(bagId: BagId, objectId: DataObjectId, contentHash: Bytes): void { - const bagIdStr = `dynamic:channel:${bagId.asType('Dynamic').asType('Channel').toString()}` - const contentHashStr = Buffer.from(contentHash.toU8a(true)).toString() - this.queuedUploads.add(`${bagIdStr}|${objectId.toString()}|${contentHashStr}`) - this.saveQueue() - } - - private finalizeUpload(bagId: string, dataObjectId: number, contentHash: string) { - this.queuedUploads.delete(`${bagId}|${dataObjectId}|${contentHash}`) - this.saveQueue() - } - - public queueUploadsFromEvents(events: IEvent<[Vec, UploadParameters, Balance]>[]): void { - let queuedUploads = 0 - events.map(({ data: [objectIds, uploadParams] }) => { - objectIds.forEach((objectId, i) => { - this.queueUpload(uploadParams.bagId, objectId, uploadParams.objectCreationList[i].ipfsContentId) - ++queuedUploads - }) - }) - this.logger.info(`Added ${queuedUploads} new data object uploads to the upload queue`) - } - - private async prepareAsset( - dataObject: StorageDataObjectFieldsFragment - ): Promise { - if (await this.isAssetMissing(dataObject)) { - this.logger.warn( - `Data object ${dataObject.id} (${dataObject.ipfsHash}) missing in the data directory! Skipping...` - ) - return undefined - } - return createType('DataObjectCreationParameters', { - ipfsContentId: dataObject.ipfsHash, - size: dataObject.size, - }) - } - - public async prepareAssets( - assetsToPrepare: T - ): Promise<{ [K in keyof T]?: PreparedAsset }> { - const preparedAssets: { [K in keyof T]?: PreparedAsset } = {} - let assetIndex = 0 - await Promise.all( - Object.entries(assetsToPrepare).map(async ([assetName, dataObject]) => { - if (!dataObject) { - return - } - const params = await this.prepareAsset(dataObject) - if (!params) { - return - } - preparedAssets[assetName as keyof T] = { params, index: assetIndex++ } - }) - ) - return preparedAssets - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/UploadMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/UploadMigration.ts deleted file mode 100644 index e74ba5a154..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/UploadMigration.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { BaseMigration, BaseMigrationConfig, BaseMigrationParams } from './BaseMigration' -import { UploadManager } from './UploadManager' -import { ContentDirectorySnapshot } from './SnapshotManager' - -export type UploadMigrationConfig = BaseMigrationConfig - -export type UploadMigrationParams = BaseMigrationParams & { - uploadManager: UploadManager -} - -export abstract class UploadMigration extends BaseMigration { - protected config: UploadMigrationConfig - protected uploadManager: UploadManager - - public constructor({ api, snapshot, config, uploadManager }: UploadMigrationParams) { - super({ api, snapshot, config }) - this.config = config - this.uploadManager = uploadManager - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/VideoCategoriesMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/VideoCategoriesMigration.ts deleted file mode 100644 index 6f5efaf954..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/VideoCategoriesMigration.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { VideoCategoryMetadata } from '@joystream/metadata-protobuf' -import { VideoCategoryId } from '@joystream/types/content' -import { SubmittableExtrinsic } from '@polkadot/api/types' -import { ISubmittableResult } from '@polkadot/types/types' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { BaseMigrationParams, MigrationResult } from './BaseMigration' -import { CategoryMigration } from './CategoryMigration' -import { ContentDirectorySnapshot } from './SnapshotManager' - -export class VideoCategoriesMigration extends CategoryMigration { - name = 'Video categories migration' - protected logger: Logger - - public constructor(params: BaseMigrationParams) { - super(params) - this.logger = createLogger(this.name) - } - - protected async migrateBatch( - batchTx: SubmittableExtrinsic<'promise', ISubmittableResult>, - batch: { id: string }[] - ): Promise { - const { api } = this - const result = await api.sendExtrinsic(this.sudo, batchTx) - const categoryCreatedEvents = api.findEvents(result, 'content', 'VideoCategoryCreated') - const createdCategoryIds: VideoCategoryId[] = categoryCreatedEvents.map((e) => e.data[1]) - - if (createdCategoryIds.length !== batch.length) { - this.extractFailedMigrations(result, batch) - } - - let newCategoryIndex = 0 - batch.forEach((c) => { - if (this.failedMigrations.has(parseInt(c.id))) { - return - } - const newCategoryId = createdCategoryIds[newCategoryIndex++] - this.idsMap.set(parseInt(c.id), newCategoryId.toNumber()) - }) - this.logger.info(`Video categories map created!`, this.idsMap.entries()) - if (this.failedMigrations.size) { - throw new Error(`Failed to create some video categories: ${Array.from(this.failedMigrations).join(', ')}`) - } - this.logger.info(`All video categories succesfully migrated!`) - } - - public async run(): Promise { - await this.init() - const { api } = this - const allCategories = await this.snapshot.videoCategories - const categoriesToMigrate = allCategories.filter((c) => !this.idsMap.has(parseInt(c.id))) - - if (!categoriesToMigrate.length) { - this.logger.info('All video categories already migrated, skipping...') - return this.getResult() - } - - this.logger.info(`Migrating ${categoriesToMigrate.length} video categories...`) - const txs = categoriesToMigrate - .sort((a, b) => parseInt(a.id) - parseInt(b.id)) - .map((c) => { - const meta = new VideoCategoryMetadata({ name: c.name }) - const metaBytes = '0x' + Buffer.from(VideoCategoryMetadata.encode(meta).finish()).toString('hex') - return api.tx.sudo.sudoAs( - this.contentLeadKey, - api.tx.content.createVideoCategory('Lead', { - meta: metaBytes, - }) - ) - }) - - const batchTx = api.tx.utility.batch(txs) - await this.migrateBatch(batchTx, categoriesToMigrate) - - return this.getResult() - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/VideosMigration.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/VideosMigration.ts deleted file mode 100644 index cf4539b397..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/VideosMigration.ts +++ /dev/null @@ -1,225 +0,0 @@ -import { VideoMetadata } from '@joystream/metadata-protobuf' -import { VideoFieldsFragment } from './giza-query-node/generated/queries' -import _ from 'lodash' -import { createType } from '@joystream/types' -import Long from 'long' -import { VideoCreationParameters, VideoId } from '@joystream/types/content' -import moment from 'moment' -import { UploadMigration, UploadMigrationConfig, UploadMigrationParams } from './UploadMigration' -import { MigrationResult } from './BaseMigration' -import { Logger } from 'winston' -import { createLogger } from '../logging' -import { SubmittableExtrinsic } from '@polkadot/api/types' - -export type VideosMigrationConfig = UploadMigrationConfig & { - videoBatchSize: number -} - -export type VideosMigrationParams = UploadMigrationParams & { - config: VideosMigrationConfig - videoIds: number[] - channelsMap: Map - forcedChannelOwner: { id: string; controllerAccount: string } | undefined - categoriesMap: Map - membershipsMap: Map -} - -export class VideosMigration extends UploadMigration { - name = 'Videos migration' - protected config: VideosMigrationConfig - protected categoriesMap: Map - protected channelsMap: Map - protected membershipsMap: Map - protected videoIds: number[] - protected forcedChannelOwner: { id: string; controllerAccount: string } | undefined - protected logger: Logger - - public constructor(params: VideosMigrationParams) { - super(params) - this.config = params.config - this.channelsMap = params.channelsMap - this.videoIds = params.videoIds - this.forcedChannelOwner = params.forcedChannelOwner - this.categoriesMap = params.categoriesMap - this.membershipsMap = params.membershipsMap - this.logger = createLogger(this.name) - } - - private getNewCategoryId(oldCategoryId: string | null | undefined): Long | undefined { - if (typeof oldCategoryId !== 'string') { - return undefined - } - const newCategoryId = this.categoriesMap.get(parseInt(oldCategoryId)) - return newCategoryId ? Long.fromNumber(newCategoryId) : undefined - } - - private getNewChannelId(oldChannelId: number): number { - const newChannelId = this.channelsMap.get(oldChannelId) - if (!newChannelId) { - throw new Error(`Missing new channel id for channel ${oldChannelId} in the channelMap!`) - } - return newChannelId - } - - protected async migrateBatch(tx: SubmittableExtrinsic<'promise'>, videos: VideoFieldsFragment[]): Promise { - const { api } = this - const result = await api.sendExtrinsic(this.sudo, tx) - const videoCreatedEvents = api.findEvents(result, 'content', 'VideoCreated') - const newVideoIds: VideoId[] = videoCreatedEvents.map((e) => e.data[2]) - if (videoCreatedEvents.length !== videos.length) { - this.extractFailedMigrations(result, videos) - } - const newVideoMapEntries: [number, number][] = [] - let newVideoIdIndex = 0 - videos.forEach(({ id }) => { - if (this.failedMigrations.has(parseInt(id))) { - return - } - const newVideoId = newVideoIds[newVideoIdIndex++].toNumber() - this.idsMap.set(parseInt(id), newVideoId) - newVideoMapEntries.push([parseInt(id), newVideoId]) - }) - if (newVideoMapEntries.length) { - this.logger.info('Video map entries added!', { newVideoMapEntries }) - const dataObjectsUploadedEvents = api.findEvents(result, 'storage', 'DataObjectsUploaded') - this.uploadManager.queueUploadsFromEvents(dataObjectsUploadedEvents) - } - } - - public async run(): Promise { - await this.init() - const { - api, - videoIds, - config: { videoBatchSize }, - } = this - const idsToMigrate = videoIds.filter((id) => !this.idsMap.has(id)).sort((a, b) => a - b) - if (idsToMigrate.length < videoIds.length) { - const alreadyMigratedVideosNum = videoIds.length - idsToMigrate.length - this.logger.info( - (idsToMigrate.length ? `${alreadyMigratedVideosNum}/${videoIds.length}` : 'All') + - ' videos already migrated, skippping...' - ) - } - while (idsToMigrate.length) { - const idsBatch = idsToMigrate.splice(0, videoBatchSize) - this.logger.info(`Preparing a batch of ${idsBatch.length} videos...`) - const videosBatch = this.snapshot.videos.filter((v) => idsBatch.includes(parseInt(v.id))) - if (videosBatch.length < idsBatch.length) { - this.logger.warn( - `Some videos were not be found: ${_.difference( - idsBatch, - videosBatch.map((v) => parseInt(v.id)) - )}` - ) - } - const calls = _.flatten(await Promise.all(videosBatch.map((v) => this.prepareVideo(v)))) - const batchTx = api.tx.utility.batch(calls) - await this.executeBatchMigration(batchTx, videosBatch) - await this.uploadManager.processQueuedUploads() - } - return this.getResult() - } - - private getChannelOwner({ - id, - ownerMember, - }: VideoFieldsFragment['channel']): Exclude { - if (!ownerMember) { - throw new Error(`Channel ownerMember missing for channel ${id}`) - } - - if (this.forcedChannelOwner) { - return this.forcedChannelOwner - } - - const newMemberId = this.membershipsMap.get(parseInt(ownerMember.id)) - if (newMemberId === undefined) { - throw new Error(`Missing member ${ownerMember.id} (owner of channel ${id}) in the memberships map!`) - } - - return { ...ownerMember, id: newMemberId.toString() } - } - - private getVideoData(video: VideoFieldsFragment) { - const { id, channel } = video - - if (!channel) { - throw new Error(`Channel data missing for video: ${id}`) - } - - return { ...video, channel: { ...channel, ownerMember: this.getChannelOwner(channel) } } - } - - private async prepareVideo(video: VideoFieldsFragment) { - const { api } = this - - const { - categoryId, - description, - duration, - hasMarketing, - isExplicit, - isPublic, - language, - license, - media, - mediaMetadata, - publishedBeforeJoystream, - thumbnailPhoto, - title, - channel: { ownerMember, id: oldChannelId }, - } = this.getVideoData(video) - - const channelId = this.getNewChannelId(parseInt(oldChannelId)) - - const assetsToPrepare = { - thumbnail: thumbnailPhoto || undefined, - video: media || undefined, - } - const preparedAssets = await this.uploadManager.prepareAssets(assetsToPrepare) - const meta = new VideoMetadata({ - title, - description, - category: this.getNewCategoryId(categoryId), - duration, - hasMarketing, - isExplicit, - isPublic, - language: language?.iso, - license: license, - mediaPixelHeight: mediaMetadata?.pixelHeight, - mediaPixelWidth: mediaMetadata?.pixelWidth, - mediaType: mediaMetadata?.encoding, - publishedBeforeJoystream: { - isPublished: !!publishedBeforeJoystream, - date: moment(publishedBeforeJoystream).format('YYYY-MM-DD'), - }, - thumbnailPhoto: preparedAssets.thumbnail?.index, - video: preparedAssets.video?.index, - }) - const assetsParams = Object.values(preparedAssets) - .sort((a, b) => a.index - b.index) - .map((a) => a.params) - const videoCreationParams = createType( - 'VideoCreationParameters', - { - assets: assetsParams.length - ? { - object_creation_list: assetsParams, - expected_data_size_fee: this.uploadManager.dataObjectFeePerMB, - } - : null, - meta: `0x${Buffer.from(VideoMetadata.encode(meta).finish()).toString('hex')}`, - } - ) - const feesToCover = this.uploadManager.calcDataObjectsFee(assetsParams) - return [ - api.tx.balances.transferKeepAlive(ownerMember.controllerAccount, feesToCover), - api.tx.sudo.sudoAs( - ownerMember.controllerAccount, - api.tx.content.createVideo({ Member: ownerMember.id }, channelId, videoCreationParams) - ), - ] - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/api.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/api.ts deleted file mode 100644 index 3bf52cbc8b..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/api.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { - ApolloClient, - NormalizedCacheObject, - HttpLink, - InMemoryCache, - DocumentNode, - isApolloError, - ApolloQueryResult, -} from '@apollo/client/core' -import { disableFragmentWarnings } from 'graphql-tag' -import fetch from 'cross-fetch' -import { - ChannelCategoryFieldsFragment, - GetChannelsCategories, - GetChannelsCategoriesQuery, - GetChannelsCategoriesQueryVariables, - GetVideoCategories, - GetVideoCategoriesQuery, - GetVideoCategoriesQueryVariables, - VideoCategoryFieldsFragment, - GetDataObjectsPage, - GetDataObjectsPageQuery, - GetDataObjectsPageQueryVariables, - StorageDataObjectConnectionFieldsFragment, - DistributionBucketFieldsFragment, - GetDistributorsByBagIdsQuery, - GetDistributorsByBagIdsQueryVariables, - GetDistributorsByBagIds, - ChannelConnectionFieldsFragment, - GetChannelsPageQuery, - GetChannelsPageQueryVariables, - GetChannelsPage, - VideoConnectionFieldsFragment, - GetVideosPageQuery, - GetVideosPageQueryVariables, - GetVideosPage, - MembershipConnectionFieldsFragment, - GetMembershipsPageQuery, - GetMembershipsPageQueryVariables, - GetMembershipsPage, -} from './generated/queries' -import { Logger } from 'winston' -import { createLogger } from '../../logging' -import { Maybe } from '../../sumer-giza/sumer-query-node/generated/schema' - -disableFragmentWarnings() - -export const MAX_RESULTS_PER_QUERY = 1000 - -export class QueryNodeApi { - private endpoint: string - private apolloClient: ApolloClient - private retryAttempts: number - private retryIntervalMs: number - private logger: Logger - - public constructor(endpoint: string, retryAttempts = 5, retryIntervalMs = 5000) { - this.endpoint = endpoint - this.retryAttempts = retryAttempts - this.retryIntervalMs = retryIntervalMs - this.apolloClient = new ApolloClient({ - link: new HttpLink({ uri: endpoint, fetch }), - cache: new InMemoryCache({ addTypename: false }), - defaultOptions: { query: { fetchPolicy: 'no-cache', errorPolicy: 'all' } }, - }) - this.logger = createLogger('Query Node Api') - } - - private async query(queryFunc: () => Promise>): Promise> { - let attempts = 0 - while (true) { - try { - const result = await queryFunc() - return result - } catch (e) { - if (e instanceof Error && isApolloError(e) && e.networkError) { - this.logger.error(`${this.endpoint} network error: ${e.networkError.message}`) - if (attempts++ > this.retryAttempts) { - throw new Error(`Maximum number of query retry attempts reached for ${this.endpoint}`) - } - this.logger.info(`Retrying in ${this.retryIntervalMs}ms...`) - await new Promise((resolve) => setTimeout(resolve, this.retryIntervalMs)) - } else { - throw e - } - } - } - } - - // Get entity by unique input - protected async uniqueEntityQuery< - QueryT extends { [k: string]: Maybe> | undefined }, - VariablesT extends Record - >( - query: DocumentNode, - variables: VariablesT, - resultKey: keyof QueryT - ): Promise[keyof QueryT] | null> { - return (await this.apolloClient.query({ query, variables })).data[resultKey] || null - } - - // Query-node: get multiple entities - protected async multipleEntitiesQuery< - QueryT extends { [k: string]: unknown[] }, - VariablesT extends Record - >(query: DocumentNode, variables: VariablesT, resultKey: keyof QueryT): Promise { - const q = this.query(() => this.apolloClient.query({ query, variables })) - return (await q).data[resultKey] - } - - public getChannelCategories(): Promise { - return this.multipleEntitiesQuery( - GetChannelsCategories, - {}, - 'channelCategories' - ) - } - - public getVideoCategories(): Promise { - return this.multipleEntitiesQuery( - GetVideoCategories, - {}, - 'videoCategories' - ) - } - - public async getChannelsPage( - lastCursor?: string, - limit: number = MAX_RESULTS_PER_QUERY - ): Promise { - const conn = await this.uniqueEntityQuery( - GetChannelsPage, - { - limit, - lastCursor, - }, - 'channelsConnection' - ) - if (!conn) { - throw new Error('Cannot get channelsConnection!') - } - - return conn - } - - public async getVideosPage( - lastCursor?: string, - limit: number = MAX_RESULTS_PER_QUERY - ): Promise { - const conn = await this.uniqueEntityQuery( - GetVideosPage, - { - limit, - lastCursor, - }, - 'videosConnection' - ) - if (!conn) { - throw new Error('Cannot get videosConnection!') - } - - return conn - } - - public async getStorageDataObjectsPage( - updatedAfter?: Date, - limit: number = MAX_RESULTS_PER_QUERY, - lastCursor?: string - ): Promise { - const conn = await this.uniqueEntityQuery( - GetDataObjectsPage, - { - updatedAfter, - limit, - lastCursor, - }, - 'storageDataObjectsConnection' - ) - if (!conn) { - throw new Error('Cannot get storageDataObjectsConnection!') - } - - return conn - } - - public async getDistributorsByBagIds(bagIds: string[]): Promise { - return this.multipleEntitiesQuery( - GetDistributorsByBagIds, - { ids: bagIds }, - 'distributionBuckets' - ) - } - - public async getMembershipsPage( - lastCursor?: string, - limit: number = MAX_RESULTS_PER_QUERY - ): Promise { - const conn = await this.uniqueEntityQuery( - GetMembershipsPage, - { - limit, - lastCursor, - }, - 'membershipsConnection' - ) - if (!conn) { - throw new Error('Cannot get membershipsConnection!') - } - - return conn - } -} diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/codegen.yml b/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/codegen.yml deleted file mode 100644 index eec6585620..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/codegen.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Paths are relative to root distribution-node directory -overwrite: true - -schema: https://hydra.joystream.org/graphql - -documents: - - 'src/giza-olympia/giza-query-node/queries/*.graphql' - -config: - scalars: - Date: Date - preResolveTypes: true # avoid using Pick - skipTypename: true # skip __typename field in typings unless it's part of the query - -generates: - src/giza-olympia/giza-query-node/generated/schema.ts: - hooks: - afterOneFileWrite: - - prettier --write - - eslint --fix - plugins: - - typescript - src/giza-olympia/giza-query-node/generated/queries.ts: - preset: import-types - presetConfig: - typesPath: ./schema - hooks: - afterOneFileWrite: - - prettier --write - - eslint --fix - plugins: - - typescript-operations - - typescript-document-nodes diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/queries.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/queries.ts deleted file mode 100644 index 61f68ab75e..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/queries.ts +++ /dev/null @@ -1,386 +0,0 @@ -import * as Types from './schema' - -import gql from 'graphql-tag' -export type VideoCategoryFieldsFragment = { id: string; name?: Types.Maybe } - -export type ChannelCategoryFieldsFragment = { id: string; name?: Types.Maybe } - -export type StorageDataObjectFieldsFragment = { - id: string - updatedAt?: Types.Maybe - ipfsHash: string - isAccepted: boolean - size: any - storageBagId: string -} - -export type StorageDataObjectConnectionFieldsFragment = { - edges: Array<{ node: StorageDataObjectFieldsFragment }> - pageInfo: { hasNextPage: boolean; endCursor?: Types.Maybe } -} - -export type VideoFieldsFragment = { - id: string - categoryId?: Types.Maybe - title?: Types.Maybe - description?: Types.Maybe - duration?: Types.Maybe - hasMarketing?: Types.Maybe - publishedBeforeJoystream?: Types.Maybe - isPublic?: Types.Maybe - isCensored: boolean - isExplicit?: Types.Maybe - isFeatured: boolean - thumbnailPhoto?: Types.Maybe - language?: Types.Maybe<{ iso: string }> - license?: Types.Maybe<{ - code?: Types.Maybe - attribution?: Types.Maybe - customText?: Types.Maybe - }> - media?: Types.Maybe - mediaMetadata?: Types.Maybe<{ - pixelWidth?: Types.Maybe - pixelHeight?: Types.Maybe - size?: Types.Maybe - encoding?: Types.Maybe<{ - codecName?: Types.Maybe - container?: Types.Maybe - mimeMediaType?: Types.Maybe - }> - }> - channel: { id: string; ownerMember?: Types.Maybe<{ id: string; controllerAccount: string }> } -} - -export type VideoConnectionFieldsFragment = { - edges: Array<{ node: VideoFieldsFragment }> - pageInfo: { hasNextPage: boolean; endCursor?: Types.Maybe } -} - -export type ChannelFieldsFragment = { - id: string - categoryId?: Types.Maybe - rewardAccount?: Types.Maybe - title?: Types.Maybe - description?: Types.Maybe - isPublic?: Types.Maybe - isCensored: boolean - ownerMember?: Types.Maybe<{ id: string; controllerAccount: string }> - coverPhoto?: Types.Maybe - avatarPhoto?: Types.Maybe - language?: Types.Maybe<{ iso: string }> - videos: Array<{ id: string }> - collaborators: Array<{ id: string }> -} - -export type ChannelConnectionFieldsFragment = { - edges: Array<{ node: ChannelFieldsFragment }> - pageInfo: { hasNextPage: boolean; endCursor?: Types.Maybe } -} - -export type DistributionBucketFieldsFragment = { - distributing: boolean - bags: Array<{ id: string }> - operators: Array<{ - status: Types.DistributionBucketOperatorStatus - metadata?: Types.Maybe<{ nodeEndpoint?: Types.Maybe }> - }> -} - -export type MembershipFieldsFragment = { - id: string - handle: string - avatarUri?: Types.Maybe - about?: Types.Maybe - controllerAccount: string - rootAccount: string -} - -export type MembershipConnectionFieldsFragment = { - edges: Array<{ node: MembershipFieldsFragment }> - pageInfo: { hasNextPage: boolean; endCursor?: Types.Maybe } -} - -export type GetVideoCategoriesQueryVariables = Types.Exact<{ [key: string]: never }> - -export type GetVideoCategoriesQuery = { videoCategories: Array } - -export type GetChannelsCategoriesQueryVariables = Types.Exact<{ [key: string]: never }> - -export type GetChannelsCategoriesQuery = { channelCategories: Array } - -export type GetDistributorsByBagIdsQueryVariables = Types.Exact<{ - ids?: Types.Maybe | Types.Scalars['ID']> -}> - -export type GetDistributorsByBagIdsQuery = { distributionBuckets: Array } - -export type GetDataObjectsPageQueryVariables = Types.Exact<{ - updatedAfter?: Types.Maybe - limit: Types.Scalars['Int'] - lastCursor?: Types.Maybe -}> - -export type GetDataObjectsPageQuery = { storageDataObjectsConnection: StorageDataObjectConnectionFieldsFragment } - -export type GetChannelsPageQueryVariables = Types.Exact<{ - limit: Types.Scalars['Int'] - lastCursor?: Types.Maybe -}> - -export type GetChannelsPageQuery = { channelsConnection: ChannelConnectionFieldsFragment } - -export type GetVideosPageQueryVariables = Types.Exact<{ - limit: Types.Scalars['Int'] - lastCursor?: Types.Maybe -}> - -export type GetVideosPageQuery = { videosConnection: VideoConnectionFieldsFragment } - -export type GetMembershipsPageQueryVariables = Types.Exact<{ - limit: Types.Scalars['Int'] - lastCursor?: Types.Maybe -}> - -export type GetMembershipsPageQuery = { membershipsConnection: MembershipConnectionFieldsFragment } - -export const VideoCategoryFields = gql` - fragment VideoCategoryFields on VideoCategory { - id - name - } -` -export const ChannelCategoryFields = gql` - fragment ChannelCategoryFields on ChannelCategory { - id - name - } -` -export const StorageDataObjectFields = gql` - fragment StorageDataObjectFields on StorageDataObject { - id - updatedAt - ipfsHash - isAccepted - size - storageBagId - } -` -export const StorageDataObjectConnectionFields = gql` - fragment StorageDataObjectConnectionFields on StorageDataObjectConnection { - edges { - node { - ...StorageDataObjectFields - } - } - pageInfo { - hasNextPage - endCursor - } - } - ${StorageDataObjectFields} -` -export const VideoFields = gql` - fragment VideoFields on Video { - id - categoryId - title - description - duration - thumbnailPhoto { - ...StorageDataObjectFields - } - language { - iso - } - hasMarketing - publishedBeforeJoystream - isPublic - isCensored - isExplicit - license { - code - attribution - customText - } - media { - ...StorageDataObjectFields - } - mediaMetadata { - encoding { - codecName - container - mimeMediaType - } - pixelWidth - pixelHeight - size - } - isFeatured - channel { - id - ownerMember { - id - controllerAccount - } - } - } - ${StorageDataObjectFields} -` -export const VideoConnectionFields = gql` - fragment VideoConnectionFields on VideoConnection { - edges { - node { - ...VideoFields - } - } - pageInfo { - hasNextPage - endCursor - } - } - ${VideoFields} -` -export const ChannelFields = gql` - fragment ChannelFields on Channel { - id - ownerMember { - id - controllerAccount - } - categoryId - rewardAccount - title - description - coverPhoto { - ...StorageDataObjectFields - } - avatarPhoto { - ...StorageDataObjectFields - } - isPublic - isCensored - language { - iso - } - videos { - id - } - collaborators { - id - } - } - ${StorageDataObjectFields} -` -export const ChannelConnectionFields = gql` - fragment ChannelConnectionFields on ChannelConnection { - edges { - node { - ...ChannelFields - } - } - pageInfo { - hasNextPage - endCursor - } - } - ${ChannelFields} -` -export const DistributionBucketFields = gql` - fragment DistributionBucketFields on DistributionBucket { - distributing - bags { - id - } - operators { - status - metadata { - nodeEndpoint - } - } - } -` -export const MembershipFields = gql` - fragment MembershipFields on Membership { - id - handle - avatarUri - about - controllerAccount - rootAccount - } -` -export const MembershipConnectionFields = gql` - fragment MembershipConnectionFields on MembershipConnection { - edges { - node { - ...MembershipFields - } - } - pageInfo { - hasNextPage - endCursor - } - } - ${MembershipFields} -` -export const GetVideoCategories = gql` - query getVideoCategories { - videoCategories { - ...VideoCategoryFields - } - } - ${VideoCategoryFields} -` -export const GetChannelsCategories = gql` - query getChannelsCategories { - channelCategories { - ...ChannelCategoryFields - } - } - ${ChannelCategoryFields} -` -export const GetDistributorsByBagIds = gql` - query getDistributorsByBagIds($ids: [ID!]) { - distributionBuckets(where: { bags_some: { id_in: $ids }, distributing_eq: true }) { - ...DistributionBucketFields - } - } - ${DistributionBucketFields} -` -export const GetDataObjectsPage = gql` - query getDataObjectsPage($updatedAfter: DateTime, $limit: Int!, $lastCursor: String) { - storageDataObjectsConnection( - where: { updatedAt_gt: $updatedAfter, isAccepted_eq: true } - first: $limit - after: $lastCursor - ) { - ...StorageDataObjectConnectionFields - } - } - ${StorageDataObjectConnectionFields} -` -export const GetChannelsPage = gql` - query getChannelsPage($limit: Int!, $lastCursor: String) { - channelsConnection(first: $limit, after: $lastCursor) { - ...ChannelConnectionFields - } - } - ${ChannelConnectionFields} -` -export const GetVideosPage = gql` - query getVideosPage($limit: Int!, $lastCursor: String) { - videosConnection(first: $limit, after: $lastCursor) { - ...VideoConnectionFields - } - } - ${VideoConnectionFields} -` -export const GetMembershipsPage = gql` - query getMembershipsPage($limit: Int!, $lastCursor: String) { - membershipsConnection(first: $limit, after: $lastCursor) { - ...MembershipConnectionFields - } - } - ${MembershipConnectionFields} -` diff --git a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/schema.ts b/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/schema.ts deleted file mode 100644 index 089f5da5a7..0000000000 --- a/utils/migration-scripts/disabled-migrations/giza-olympia/giza-query-node/generated/schema.ts +++ /dev/null @@ -1,3715 +0,0 @@ -export type Maybe = T | null -export type Exact = { [K in keyof T]: T[K] } -export type MakeOptional = Omit & { [SubKey in K]?: Maybe } -export type MakeMaybe = Omit & { [SubKey in K]: Maybe } -/** All built-in and custom scalars, mapped to their actual values */ -export type Scalars = { - ID: string - String: string - Boolean: boolean - Int: number - Float: number - /** The javascript `Date` as string. Type represents date and time as the ISO Date string. */ - DateTime: any - /** GraphQL representation of BigInt */ - BigInt: any - /** The `JSONObject` scalar type represents JSON objects as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). */ - JSONObject: any -} - -export type BaseGraphQlObject = { - id: Scalars['ID'] - createdAt: Scalars['DateTime'] - createdById: Scalars['String'] - updatedAt?: Maybe - updatedById?: Maybe - deletedAt?: Maybe - deletedById?: Maybe - version: Scalars['Int'] -} - -export type BaseModel = BaseGraphQlObject & { - id: Scalars['ID'] - createdAt: Scalars['DateTime'] - createdById: Scalars['String'] - updatedAt?: Maybe - updatedById?: Maybe - deletedAt?: Maybe - deletedById?: Maybe - version: Scalars['Int'] -} - -export type BaseModelUuid = BaseGraphQlObject & { - id: Scalars['ID'] - createdAt: Scalars['DateTime'] - createdById: Scalars['String'] - updatedAt?: Maybe - updatedById?: Maybe - deletedAt?: Maybe - deletedById?: Maybe - version: Scalars['Int'] -} - -export type BaseWhereInput = { - id_eq?: Maybe - id_in?: Maybe> - createdAt_eq?: Maybe - createdAt_lt?: Maybe - createdAt_lte?: Maybe - createdAt_gt?: Maybe - createdAt_gte?: Maybe - createdById_eq?: Maybe - updatedAt_eq?: Maybe - updatedAt_lt?: Maybe - updatedAt_lte?: Maybe - updatedAt_gt?: Maybe - updatedAt_gte?: Maybe - updatedById_eq?: Maybe - deletedAt_all?: Maybe - deletedAt_eq?: Maybe - deletedAt_lt?: Maybe - deletedAt_lte?: Maybe - deletedAt_gt?: Maybe - deletedAt_gte?: Maybe - deletedById_eq?: Maybe -} - -export type Channel = BaseGraphQlObject & { - id: Scalars['ID'] - createdAt: Scalars['DateTime'] - createdById: Scalars['String'] - updatedAt?: Maybe - updatedById?: Maybe - deletedAt?: Maybe - deletedById?: Maybe - version: Scalars['Int'] - ownerMember?: Maybe - ownerMemberId?: Maybe - ownerCuratorGroup?: Maybe - ownerCuratorGroupId?: Maybe - category?: Maybe - categoryId?: Maybe - /** Reward account where revenue is sent if set. */ - rewardAccount?: Maybe - /** The title of the Channel */ - title?: Maybe - /** The description of a Channel */ - description?: Maybe - coverPhoto?: Maybe - coverPhotoId?: Maybe - avatarPhoto?: Maybe - avatarPhotoId?: Maybe - /** Flag signaling whether a channel is public. */ - isPublic?: Maybe - /** Flag signaling whether a channel is censored. */ - isCensored: Scalars['Boolean'] - language?: Maybe - languageId?: Maybe - videos: Array