From 8a074dcf8a56201dc3940b731cee8f266cdc5427 Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Tue, 10 Oct 2023 15:36:33 +0400 Subject: [PATCH 1/4] colossus: sync - longer default interval, add seprate interval for retry on failure --- storage-node/package.json | 2 +- storage-node/src/commands/server.ts | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/storage-node/package.json b/storage-node/package.json index d33f501c02..c83e036447 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -1,7 +1,7 @@ { "name": "storage-node", "description": "Joystream storage subsystem.", - "version": "3.7.0", + "version": "3.8.0", "author": "Joystream contributors", "bin": { "storage-node": "./bin/run" diff --git a/storage-node/src/commands/server.ts b/storage-node/src/commands/server.ts index 7a0aa2b628..f51532961b 100644 --- a/storage-node/src/commands/server.ts +++ b/storage-node/src/commands/server.ts @@ -60,7 +60,11 @@ export default class Server extends ApiCommandBase { syncInterval: flags.integer({ char: 'i', description: 'Interval between synchronizations (in minutes)', - default: 1, + default: 20, + }), + syncRetryInterval: flags.integer({ + description: 'Interval before retrying failed synchronization run (in minutes)', + default: 3, }), queryNodeEndpoint: flags.string({ char: 'q', @@ -221,7 +225,8 @@ Supported values: warn, error, debug, info. Default:debug`, TempDirName, flags.syncWorkersNumber, flags.syncWorkersTimeout, - flags.syncInterval + flags.syncInterval, + flags.syncRetryInterval ), 0 ) @@ -284,12 +289,12 @@ async function runSyncWithInterval( tempDirectory: string, syncWorkersNumber: number, syncWorkersTimeout: number, - syncIntervalMinutes: number + syncIntervalMinutes: number, + syncRetryIntervalMinutes: number ) { const sleepInteval = syncIntervalMinutes * 60 * 1000 + const retrySleepInterval = syncRetryIntervalMinutes * 60 * 1000 while (true) { - logger.info(`Sync paused for ${syncIntervalMinutes} minute(s).`) - await sleep(sleepInteval) try { logger.info(`Resume syncing....`) await performSync( @@ -302,8 +307,12 @@ async function runSyncWithInterval( uploadsDirectory, tempDirectory ) + logger.info(`Sync run complete. Next run in ${syncIntervalMinutes} minute(s).`) + await sleep(sleepInteval) } catch (err) { logger.error(`Critical sync error: ${err}`) + logger.info(`Will retry in ${syncRetryIntervalMinutes} minute(s)`) + await sleep(retrySleepInterval) } } } From bab30085b2338e76497db13fa143d19152a5c49d Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Tue, 10 Oct 2023 15:46:22 +0400 Subject: [PATCH 2/4] colossus: add function docs for runSyncWithInterval() --- storage-node/src/commands/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/storage-node/src/commands/server.ts b/storage-node/src/commands/server.ts index f51532961b..ea970c5b2b 100644 --- a/storage-node/src/commands/server.ts +++ b/storage-node/src/commands/server.ts @@ -277,6 +277,7 @@ Supported values: warn, error, debug, info. Default:debug`, * @param syncWorkersNumber - defines a number of the async processes for sync * @param syncWorkersTimeout - downloading asset timeout * @param syncIntervalMinutes - defines an interval between sync runs + * @param syncRetryIntervalMinutes - defines an interval before retrying sync run after critical error * * @returns void promise. */ From 5a30d80220aa4c076425bec45b56096dd5aeec42 Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Tue, 24 Oct 2023 10:25:43 +0400 Subject: [PATCH 3/4] storage-node: update changelog for v3.8.0 --- storage-node/CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md index 45da4a286e..ebf29f879e 100644 --- a/storage-node/CHANGELOG.md +++ b/storage-node/CHANGELOG.md @@ -1,6 +1,10 @@ +### 3.8.0 + +- Increase default interval between sync runs. Start sync run immediately do not wait initial interval on startup before starting sync. Adds additional argument to specify retry interval on failure. [#4924](https://github.com/Joystream/joystream/pull/4924) + ### 3.7.1 -- Disable open-api express response validation if NODE_ENV == 'production'. This should improve response times when serving assets. +- Disable open-api express response validation if NODE_ENV == 'production'. This should improve response times when serving assets. [#4810](https://github.com/Joystream/joystream/pull/4810) - Include `nodeEnv` in `/api/v1/status` response, to help detect mis-configured nodes. ### 3.7.0 From c25416db7ad3c2d37e5868c64a283f9f969d48ea Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Thu, 30 Nov 2023 23:32:50 +0400 Subject: [PATCH 4/4] fix typo --- storage-node/src/commands/server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-node/src/commands/server.ts b/storage-node/src/commands/server.ts index 1cda85853f..a358507f84 100644 --- a/storage-node/src/commands/server.ts +++ b/storage-node/src/commands/server.ts @@ -315,7 +315,7 @@ async function runSyncWithInterval( syncIntervalMinutes: number, syncRetryIntervalMinutes: number ) { - const sleepInteval = syncIntervalMinutes * 60 * 1000 + const sleepInterval = syncIntervalMinutes * 60 * 1000 const retrySleepInterval = syncRetryIntervalMinutes * 60 * 1000 while (true) { try { @@ -331,7 +331,7 @@ async function runSyncWithInterval( tempDirectory ) logger.info(`Sync run complete. Next run in ${syncIntervalMinutes} minute(s).`) - await sleep(sleepInteval) + await sleep(sleepInterval) } catch (err) { logger.error(`Critical sync error: ${err}`) logger.info(`Will retry in ${syncRetryIntervalMinutes} minute(s)`)