UBERF-6490: Rework backup tool

dev/tool/src/index.ts

@@ -25,7 +25,6 @@ import {
   getAccount,
   getWorkspaceById,
   listAccounts,
-  listWorkspaces,
   listWorkspacesPure,
   listWorkspacesRaw,
   replacePassword,
@@ -58,6 +57,7 @@ import core, {
   MeasureMetricsContext,
   metricsToString,
   RateLimiter,
+  versionToString,
   type AccountRole,
   type Data,
   type Tx,
@@ -68,6 +68,7 @@ import contact from '@hcengineering/model-contact'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { openAIConfigDefaults } from '@hcengineering/openai'
 import { type StorageAdapter } from '@hcengineering/server-core'
+import { deepEqual } from 'fast-equals'
 import path from 'path'
 import { benchmark } from './benchmark'
 import {
@@ -86,6 +87,18 @@ import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { openAIConfig } from './openai'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 
+const colorConstants = {
+  colorRed: '\u001b[31m',
+  colorBlue: '\u001b[34m',
+  colorWhiteCyan: '\u001b[37;46m',
+  colorRedYellow: '\u001b[31;43m',
+  colorPing: '\u001b[38;5;201m',
+  colorLavander: '\u001b[38;5;147m',
+  colorAqua: '\u001b[38;2;145;231;255m',
+  colorPencil: '\u001b[38;2;253;182;0m',
+  reset: '\u001b[0m'
+}
+
 /**
  * @public
  */
@@ -468,11 +481,42 @@ export function devTool (
   program
     .command('list-workspaces')
     .description('List workspaces')
-    .action(async () => {
+    .option('-e|--expired [expired]', 'Show only expired', false)
+    .action(async (cmd: { expired: boolean }) => {
       const { mongodbUri, version } = prepareTools()
       await withDatabase(mongodbUri, async (db) => {
-        const workspacesJSON = JSON.stringify(await listWorkspaces(toolCtx, db, productId), null, 2)
-        console.info(workspacesJSON)
+        const workspacesJSON = await listWorkspacesPure(db, productId)
+        for (const ws of workspacesJSON) {
+          let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
+          if (cmd.expired && lastVisit <= 7) {
+            continue
+          }
+          console.log(
+            colorConstants.colorBlue +
+              '####################################################################################################' +
+              colorConstants.reset
+          )
+          console.log('id:', colorConstants.colorWhiteCyan + ws.workspace + colorConstants.reset)
+          console.log('url:', ws.workspaceUrl, 'name:', ws.workspaceName)
+          console.log(
+            'version:',
+            ws.version !== undefined ? versionToString(ws.version) : 'not-set',
+            !deepEqual(ws.version, version) ? `upgrade to ${versionToString(version)} is required` : ''
+          )
+          console.log('disabled:', ws.disabled)
+          console.log('created by:', ws.createdBy)
+          console.log('members:', (ws.accounts ?? []).length)
+          if (Number.isNaN(lastVisit)) {
+            lastVisit = 365
+          }
+          if (lastVisit > 30) {
+            console.log(colorConstants.colorRed + `last visit: ${lastVisit} days ago` + colorConstants.reset)
+          } else if (lastVisit > 7) {
+            console.log(colorConstants.colorRedYellow + `last visit: ${lastVisit} days ago` + colorConstants.reset)
+          } else {
+            console.log('last visit:', lastVisit, 'days ago')
+          }
+        }
 
         console.log('latest model version:', JSON.stringify(version))
       })
@@ -481,7 +525,7 @@ export function devTool (
   program.command('fix-person-accounts').action(async () => {
     const { mongodbUri, version } = prepareTools()
     await withDatabase(mongodbUri, async (db, client) => {
-      const ws = await listWorkspaces(toolCtx, db, productId)
+      const ws = await listWorkspacesPure(db, productId)
       for (const w of ws) {
         const wsDb = getWorkspaceDB(client, { name: w.workspace, productId })
         await wsDb.collection('tx').updateMany(
@@ -534,6 +578,7 @@ export function devTool (
     .action(async (dirName: string, workspace: string, cmd: { skip: string, force: boolean }) => {
       const storage = await createFileBackupStorage(dirName)
       await backup(
+        toolCtx,
         transactorUrl,
         getWorkspaceId(workspace, productId),
         storage,
@@ -548,7 +593,7 @@ export function devTool (
     .option('-f, --force', 'Force compact.', false)
     .action(async (dirName: string, cmd: { force: boolean }) => {
       const storage = await createFileBackupStorage(dirName)
-      await compactBackup(storage, cmd.force)
+      await compactBackup(toolCtx, storage, cmd.force)
     })
 
   program
@@ -557,7 +602,14 @@ export function devTool (
     .description('dump workspace transactions and minio resources')
     .action(async (dirName: string, workspace: string, date, cmd: { merge: boolean }) => {
       const storage = await createFileBackupStorage(dirName)
-      await restore(transactorUrl, getWorkspaceId(workspace, productId), storage, parseInt(date ?? '-1'), cmd.merge)
+      await restore(
+        toolCtx,
+        transactorUrl,
+        getWorkspaceId(workspace, productId),
+        storage,
+        parseInt(date ?? '-1'),
+        cmd.merge
+      )
     })
 
   program
@@ -579,7 +631,7 @@ export function devTool (
         getWorkspaceId(bucketName, productId),
         dirName
       )
-      await backup(transactorUrl, getWorkspaceId(workspace, productId), storage)
+      await backup(toolCtx, transactorUrl, getWorkspaceId(workspace, productId), storage)
     })
 
   program
@@ -594,7 +646,7 @@ export function devTool (
         getWorkspaceId(bucketName, productId),
         dirName
       )
-      await compactBackup(storage, cmd.force)
+      await compactBackup(toolCtx, storage, cmd.force)
     })
 
   program
@@ -611,11 +663,11 @@ export function devTool (
           getWorkspaceId(bucketName, productId),
           dirName
         )
-        const workspaces = await listWorkspaces(toolCtx, db, productId)
+        const workspaces = await listWorkspacesPure(db, productId)
 
         for (const w of workspaces) {
           console.log(`clearing ${w.workspace} history:`)
-          await compactBackup(storage, cmd.force)
+          await compactBackup(toolCtx, storage, cmd.force)
         }
       })
     })
@@ -625,7 +677,7 @@ export function devTool (
     .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => {
       const { storageAdapter } = prepareTools()
       const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
-      await restore(transactorUrl, getWorkspaceId(workspace, productId), storage, parseInt(date ?? '-1'))
+      await restore(toolCtx, transactorUrl, getWorkspaceId(workspace, productId), storage, parseInt(date ?? '-1'))
     })
   program
     .command('backup-s3-list <bucketName> <dirName>')
@@ -695,7 +747,7 @@ export function devTool (
           process.exit(1)
         }
 
-        const workspaces = await listWorkspaces(toolCtx, db, productId)
+        const workspaces = await listWorkspacesPure(db, productId)
 
         for (const w of workspaces) {
           console.log(`clearing ${w.workspace} history:`)

packages/core/src/classes.ts

@@ -650,3 +650,20 @@ export interface DomainIndexConfiguration extends Doc {
 
   skip?: string[]
 }
+
+export interface BaseWorkspaceInfo {
+  workspace: string // An uniq workspace name, Database names
+  productId: string
+  disabled?: boolean
+  version?: Data<Version>
+
+  workspaceUrl?: string | null // An optional url to the workspace, if not set workspace will be used
+  workspaceName?: string // An displayed workspace name
+  createdOn: number
+  lastVisit: number
+
+  createdBy: string
+
+  creating?: boolean
+  createProgress?: number // Some progress
+}

plugins/client-resources/src/connection.ts

@@ -79,6 +79,8 @@ class Connection implements ClientConnection {
   private sessionId: string | undefined
   private closed = false
 
+  private upgrading: boolean = false
+
   private pingResponse: number = Date.now()
 
   constructor (
@@ -161,15 +163,21 @@ class Connection implements ClientConnection {
           throw new Error('connection closed')
         }
         this.pending = undefined
-        console.log('failed to connect', err)
+        if (!this.upgrading) {
+          console.log('connection: failed to connect', this.lastId)
+        } else {
+          console.log('connection: workspace during upgrade', this.lastId)
+        }
         if (err?.code === UNAUTHORIZED.code) {
           Analytics.handleError(err)
           this.onUnauthorized?.()
           throw err
         }
         await new Promise((resolve) => {
           setTimeout(() => {
-            console.log(`delay ${this.delay} second`)
+            if (!this.upgrading) {
+              console.log(`delay ${this.delay} second`)
+            }
             resolve(null)
             if (this.delay < 5) {
               this.delay++
@@ -220,7 +228,12 @@ class Connection implements ClientConnection {
 
       websocket.onmessage = (event: MessageEvent) => {
         const resp = readResponse<any>(event.data, binaryResponse)
+        if (resp.id === -1 && resp.result === 'upgrading') {
+          this.upgrading = true
+          return
+        }
         if (resp.id === -1 && resp.result === 'hello') {
+          this.upgrading = false
           if ((resp as HelloResponse).alreadyConnected === true) {
             this.sessionId = generateId()
             if (typeof sessionStorage !== 'undefined') {

plugins/guest-resources/src/components/GuestApp.svelte

@@ -26,11 +26,11 @@
   {#await connect(getMetadata(workbench.metadata.PlatformTitle) ?? 'Platform')}
     <Loading />
   {:then client}
-    {#if !client && versionError}
+    {#if $versionError}
       <div class="version-wrapper">
         <div class="antiPopup version-popup">
           <h1><Label label={workbench.string.ServerUnderMaintenance} /></h1>
-          {versionError}
+          {$versionError}
         </div>
       </div>
     {:else if client}