UBERF-7165: Storage + Backup improvements

dev/tool/src/index.ts

@@ -592,14 +592,28 @@ export function devTool (
     .description('dump workspace transactions and minio resources')
     .option('-i, --include <include>', 'A list of ; separated domain names to include during backup', '*')
     .option('-s, --skip <skip>', 'A list of ; separated domain names to skip during backup', '')
+    .option(
+      '-ct, --contentTypes <contentTypes>',
+      'A list of ; separated content types for blobs to skip download if size >= limit',
+      ''
+    )
+    .option('-bl, --blobLimit <blobLimit>', 'A blob size limit in megabytes (default 15mb)', '15')
     .option('-f, --force', 'Force backup', false)
     .option('-c, --recheck', 'Force hash recheck on server', false)
     .option('-t, --timeout <timeout>', 'Connect timeout in seconds', '30')
     .action(
       async (
         dirName: string,
         workspace: string,
-        cmd: { skip: string, force: boolean, recheck: boolean, timeout: string, include: string }
+        cmd: {
+          skip: string
+          force: boolean
+          recheck: boolean
+          timeout: string
+          include: string
+          blobLimit: string
+          contentTypes: string
+        }
       ) => {
         const storage = await createFileBackupStorage(dirName)
         await backup(toolCtx, transactorUrl, getWorkspaceId(workspace, productId), storage, {
@@ -608,7 +622,12 @@ export function devTool (
           include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())),
           skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()),
           timeout: 0,
-          connectTimeout: parseInt(cmd.timeout) * 1000
+          connectTimeout: parseInt(cmd.timeout) * 1000,
+          blobDownloadLimit: parseInt(cmd.blobLimit),
+          skipBlobContentTypes: cmd.contentTypes
+            .split(';')
+            .map((it) => it.trim())
+            .filter((it) => it.length > 0)
         })
       }
     )

models/core/src/core.ts

@@ -136,11 +136,11 @@ export class TAttachedDoc extends TDoc implements AttachedDoc {
 export class TBlob extends TDoc implements Blob {
   @Prop(TypeString(), core.string.Blob)
   @ReadOnly()
+  @Index(IndexKind.Indexed)
     provider!: string
 
   @Prop(TypeString(), core.string.BlobContentType)
   @ReadOnly()
-  @Index(IndexKind.Indexed)
     contentType!: string
 
   @Prop(TypeString(), core.string.BlobStorageId)

models/core/src/index.ts

@@ -87,10 +87,11 @@ import {
   TSpace,
   TSpaceType,
   TSpaceTypeDescriptor,
-  TTypedSpace,
-  TSystemSpace
+  TSystemSpace,
+  TTypedSpace
 } from './security'
-import { TStatus, TStatusCategory, TDomainStatusPlaceholder } from './status'
+import { defineSpaceType } from './spaceType'
+import { TDomainStatusPlaceholder, TStatus, TStatusCategory } from './status'
 import { TUserStatus } from './transient'
 import {
   TTx,
@@ -103,7 +104,6 @@ import {
   TTxUpdateDoc,
   TTxWorkspaceEvent
 } from './tx'
-import { defineSpaceType } from './spaceType'
 
 export { coreId } from '@hcengineering/core'
 export * from './core'

packages/presentation/src/plugin.ts

@@ -91,7 +91,7 @@ export default plugin(presentationId, {
     CollaboratorApiUrl: '' as Metadata<string>,
     Token: '' as Metadata<string>,
     FrontUrl: '' as Asset,
-    PreviewConfig: '' as Metadata<PreviewConfig>
+    PreviewConfig: '' as Metadata<PreviewConfig | undefined>
   },
   status: {
     FileTooLarge: '' as StatusCode

packages/presentation/src/preview.ts

@@ -44,22 +44,9 @@ const defaultPreview = (): ProviderPreviewConfig => ({
 - contentTypes - a ',' separated list of content type patterns.
 
  */
-export function parsePreviewConfig (config?: string): PreviewConfig {
+export function parsePreviewConfig (config?: string): PreviewConfig | undefined {
   if (config === undefined) {
-    // TODO: Remove after all migrated
-    return {
-      default: defaultPreview(),
-      previewers: {
-        '': [
-          {
-            providerId: '',
-            contentTypes: ['image/gif', 'image/apng', 'image/svg'], // Disable gif and apng format preview.
-            formats: [],
-            previewUrl: ''
-          }
-        ]
-      }
-    }
+    return
   }
   const result: PreviewConfig = { previewers: {} }
   const nolineData = config
@@ -99,7 +86,21 @@ export function parsePreviewConfig (config?: string): PreviewConfig {
 }
 
 export function getPreviewConfig (): PreviewConfig {
-  return getMetadata(presentation.metadata.PreviewConfig) as PreviewConfig
+  return (
+    (getMetadata(presentation.metadata.PreviewConfig) as PreviewConfig) ?? {
+      default: defaultPreview(),
+      previewers: {
+        '': [
+          {
+            providerId: '',
+            contentTypes: ['image/gif', 'image/apng', 'image/svg'], // Disable gif and apng format preview.
+            formats: [],
+            previewUrl: ''
+          }
+        ]
+      }
+    }
+  )
 }
 
 export async function getBlobRef (

packages/query/src/index.ts

@@ -1443,7 +1443,7 @@ export class LiveQuery implements WithTx, Client {
                 } else {
                   const d = await this.findOne(lookupClass, { _id: pops[pkey] }, { lookup: nestedLookup })
                   if (d !== undefined) {
-                    pp[pkey].push()
+                    pp[pkey].push(d)
                   }
                 }
               }

packages/storage/src/index.ts

@@ -15,12 +15,13 @@
 
 import {
   type Blob,
+  type Branding,
   type DocumentUpdate,
   type MeasureContext,
   type Ref,
+  type StorageIterator,
   type WorkspaceId,
-  type WorkspaceIdWithUrl,
-  type Branding
+  type WorkspaceIdWithUrl
 } from '@hcengineering/core'
 import type { BlobLookup } from '@hcengineering/core/src/classes'
 import { type Readable } from 'stream'
@@ -103,6 +104,8 @@ export interface StorageAdapterEx extends StorageAdapter {
     objectName: string,
     provider?: string
   ) => Promise<void>
+
+  find: (ctx: MeasureContext, workspaceId: WorkspaceId) => StorageIterator
 }
 
 /**
@@ -120,6 +123,13 @@ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx {
     return false
   }
 
+  find (ctx: MeasureContext, workspaceId: WorkspaceId): StorageIterator {
+    return {
+      next: async (ctx) => undefined,
+      close: async (ctx) => {}
+    }
+  }
+
   async listBuckets (ctx: MeasureContext, productId: string): Promise<BucketInfo[]> {
     return []
   }

packages/text/src/nodes/image.ts

@@ -141,7 +141,7 @@ export const ImageNode = Node.create<ImageOptions>({
       if (fileId != null) {
         const setBrokenImg = setTimeout(() => {
           imgElement.src = this.options.loadingImgSrc ?? `platform://platform/files/workspace/?file=${fileId}`
-        }, 200)
+        }, 500)
         if (fileId != null) {
           void this.options.getBlobRef(fileId).then((val) => {
             clearTimeout(setBrokenImg)