UBERF-6161: Storage configuration

dev/docker-compose.yaml

@@ -75,6 +75,7 @@ services:
     environment:
       - SERVER_PORT=8080
       - SERVER_SECRET=secret
+      - MONGO_URL=mongodb://mongodb:27017
       - ACCOUNTS_URL=http://localhost:3000
       - REKONI_URL=http://localhost:4004
       - CALENDAR_URL=http://localhost:8095

dev/storage/src/storage.ts

@@ -69,6 +69,7 @@ class InMemoryTxAdapter extends DummyDbAdapter implements TxAdapter {
  * @public
  */
 export async function createInMemoryTxAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspace: WorkspaceId

dev/tool/src/clean.ts

@@ -18,6 +18,7 @@ import chunter, { type ChatMessage } from '@hcengineering/chunter'
 import contact from '@hcengineering/contact'
 import core, {
   DOMAIN_TX,
+  type MeasureContext,
   SortingOrder,
   TxOperations,
   TxProcessor,
@@ -43,6 +44,7 @@ import { MongoClient } from 'mongodb'
 export const DOMAIN_ACTIVITY = 'activity' as Domain
 
 export async function cleanWorkspace (
+  ctx: MeasureContext,
   mongoUrl: string,
   workspaceId: WorkspaceId,
   storageAdapter: StorageAdapter,
@@ -67,14 +69,14 @@ export async function cleanWorkspace (
       attachments.map((it) => it.file).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[])
     )
 
-    const minioList = await storageAdapter.list(workspaceId)
+    const minioList = await storageAdapter.list(ctx, workspaceId)
     const toClean: string[] = []
     for (const mv of minioList) {
-      if (!files.has(mv.name)) {
-        toClean.push(mv.name)
+      if (!files.has(mv._id)) {
+        toClean.push(mv._id)
       }
     }
-    await storageAdapter.remove(workspaceId, toClean)
+    await storageAdapter.remove(ctx, workspaceId, toClean)
     // connection.loadChunk(DOMAIN_BLOB, idx = )
 
     if (opt.recruit) {
@@ -145,16 +147,20 @@ export async function cleanWorkspace (
   }
 }
 
-export async function fixMinioBW (workspaceId: WorkspaceId, storageService: StorageAdapter): Promise<void> {
+export async function fixMinioBW (
+  ctx: MeasureContext,
+  workspaceId: WorkspaceId,
+  storageService: StorageAdapter
+): Promise<void> {
   console.log('try clean bw miniature for ', workspaceId.name)
-  const from = new Date(new Date().setDate(new Date().getDate() - 7))
-  const list = await storageService.list(workspaceId)
+  const from = new Date(new Date().setDate(new Date().getDate() - 7)).getTime()
+  const list = await storageService.list(ctx, workspaceId)
   console.log('found', list.length)
   let removed = 0
   for (const obj of list) {
-    if (obj.lastModified < from) continue
-    if (obj.name.includes('%size%')) {
-      await storageService.remove(workspaceId, [obj.name])
+    if (obj.modifiedOn < from) continue
+    if ((obj._id as string).includes('%size%')) {
+      await storageService.remove(ctx, workspaceId, [obj._id])
       removed++
       if (removed % 100 === 0) {
         console.log('removed: ', removed)

dev/tool/src/cleanOrphan.ts

@@ -1,12 +1,13 @@
 import { dropWorkspace, setWorkspaceDisabled, type Workspace } from '@hcengineering/account'
-import core, { AccountRole, MeasureMetricsContext, SortingOrder } from '@hcengineering/core'
+import core, { AccountRole, type MeasureContext, MeasureMetricsContext, SortingOrder } from '@hcengineering/core'
 import contact from '@hcengineering/model-contact'
 import { getWorkspaceDB } from '@hcengineering/mongo'
 import { type StorageAdapter } from '@hcengineering/server-core'
 import { connect } from '@hcengineering/server-tool'
 import { type Db, type MongoClient } from 'mongodb'
 
 export async function checkOrphanWorkspaces (
+  ctx: MeasureContext,
   workspaces: Workspace[],
   transactorUrl: string,
   productId: string,
@@ -40,7 +41,7 @@ export async function checkOrphanWorkspaces (
 
       // Find last transaction index:
       const wspace = { name: ws.workspace, productId }
-      const hasBucket = await storageAdapter.exists(wspace)
+      const hasBucket = await storageAdapter.exists(ctx, wspace)
       const [lastTx] = await connection.findAll(
         core.class.Tx,
         {
@@ -69,12 +70,13 @@ export async function checkOrphanWorkspaces (
           const workspaceDb = getWorkspaceDB(client, { name: ws.workspace, productId })
           await workspaceDb.dropDatabase()
           if (storageAdapter !== undefined && hasBucket) {
-            const docs = await storageAdapter.list(wspace)
+            const docs = await storageAdapter.list(ctx, wspace)
             await storageAdapter.remove(
+              ctx,
               wspace,
-              docs.map((it) => it.name)
+              docs.map((it) => it._id)
             )
-            await storageAdapter?.delete(wspace)
+            await storageAdapter.delete(ctx, wspace)
           }
         }
       }

dev/tool/src/index.ts

@@ -410,6 +410,7 @@ export function devTool (
 
         // We need to update workspaces with missing workspaceUrl
         await checkOrphanWorkspaces(
+          toolCtx,
           workspaces,
           transactorUrl,
           productId,
@@ -482,7 +483,6 @@ export function devTool (
   program
     .command('backup <dirName> <workspace>')
     .description('dump workspace transactions and minio resources')
-    .requiredOption('-i --index <index>', 'Index name for elastic')
     .option('-s, --skip <skip>', 'A list of ; separated domain names to skip during backup', '')
     .option('-f, --force', 'Force backup', false)
     .action(async (dirName: string, workspace: string, cmd: { skip: string, force: boolean }) => {
@@ -518,15 +518,20 @@ export function devTool (
     .description('dump workspace transactions and minio resources')
     .action(async (bucketName: string, dirName: string, workspace: string, cmd) => {
       const { storageAdapter } = prepareTools()
-      const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName, productId), dirName)
+      const storage = await createStorageBackupStorage(
+        toolCtx,
+        storageAdapter,
+        getWorkspaceId(bucketName, productId),
+        dirName
+      )
       await backup(transactorUrl, getWorkspaceId(workspace, productId), storage)
     })
   program
     .command('backup-s3-restore <bucketName> <dirName> <workspace> [date]')
     .description('dump workspace transactions and minio resources')
     .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => {
       const { storageAdapter } = prepareTools()
-      const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName), dirName)
+      const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
       await restore(transactorUrl, getWorkspaceId(workspace, productId), storage, parseInt(date ?? '-1'))
     })
   program
@@ -535,7 +540,12 @@ export function devTool (
     .action(async (bucketName: string, dirName: string, cmd) => {
       const { storageAdapter } = prepareTools()
 
-      const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName, productId), dirName)
+      const storage = await createStorageBackupStorage(
+        toolCtx,
+        storageAdapter,
+        getWorkspaceId(bucketName, productId),
+        dirName
+      )
       await backupList(storage)
     })
 
@@ -576,7 +586,7 @@ export function devTool (
         }
 
         console.log(`clearing ${workspace} history:`)
-        await clearTelegramHistory(mongodbUri, getWorkspaceId(workspace, productId), telegramDB, minio)
+        await clearTelegramHistory(toolCtx, mongodbUri, getWorkspaceId(workspace, productId), telegramDB, minio)
       })
     })
 
@@ -596,7 +606,7 @@ export function devTool (
 
         for (const w of workspaces) {
           console.log(`clearing ${w.workspace} history:`)
-          await clearTelegramHistory(mongodbUri, getWorkspaceId(w.workspace, productId), telegramDB, minio)
+          await clearTelegramHistory(toolCtx, mongodbUri, getWorkspaceId(w.workspace, productId), telegramDB, minio)
         }
       })
     })
@@ -624,6 +634,7 @@ export function devTool (
       const { mongodbUri, storageAdapter: minio } = prepareTools()
       await withDatabase(mongodbUri, async (db) => {
         await cleanWorkspace(
+          toolCtx,
           mongodbUri,
           getWorkspaceId(workspace, productId),
           minio,
@@ -636,7 +647,7 @@ export function devTool (
 
   program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
     const { storageAdapter: minio } = prepareTools()
-    await fixMinioBW(getWorkspaceId(workspace, productId), minio)
+    await fixMinioBW(toolCtx, getWorkspaceId(workspace, productId), minio)
   })
 
   program

dev/tool/src/telegram.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 //
 
-import { DOMAIN_TX, type Ref, type WorkspaceId } from '@hcengineering/core'
+import { DOMAIN_TX, type MeasureContext, type Ref, type WorkspaceId } from '@hcengineering/core'
 import { type StorageAdapter } from '@hcengineering/server-core'
 import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
 import contact, { DOMAIN_CHANNEL } from '@hcengineering/model-contact'
@@ -29,6 +29,7 @@ const LastMessages = 'last-msgs'
  * @public
  */
 export async function clearTelegramHistory (
+  ctx: MeasureContext,
   mongoUrl: string,
   workspaceId: WorkspaceId,
   tgDb: string,
@@ -90,7 +91,7 @@ export async function clearTelegramHistory (
       workspaceDB.collection(DOMAIN_ATTACHMENT).deleteMany({
         attachedToClass: telegram.class.Message
       }),
-      storageAdapter.remove(workspaceId, Array.from(attachments))
+      storageAdapter.remove(ctx, workspaceId, Array.from(attachments))
     ])
 
     console.log('clearing telegram service data...')

models/core/src/component.ts

@@ -21,6 +21,12 @@ export default mergeIds(coreId, core, {
     Archived: '' as IntlString,
     ClassLabel: '' as IntlString,
     ClassPropertyLabel: '' as IntlString,
-    Members: '' as IntlString
+    Members: '' as IntlString,
+    Blob: '' as IntlString,
+    BlobContentType: '' as IntlString,
+    BlobEtag: '' as IntlString,
+    BlobVersion: '' as IntlString,
+    BlobStorageId: '' as IntlString,
+    BlobSize: '' as IntlString
   }
 })

models/core/src/core.ts

@@ -14,7 +14,9 @@
 //
 
 import {
+  type Blob,
   DOMAIN_BLOB,
+  DOMAIN_BLOB_DATA,
   DOMAIN_CONFIGURATION,
   DOMAIN_DOC_INDEX_STATE,
   DOMAIN_FULLTEXT_BLOB,
@@ -63,6 +65,7 @@ import {
   ReadOnly,
   TypeBoolean,
   TypeIntlString,
+  TypeNumber,
   TypeRecord,
   TypeRef,
   TypeString,
@@ -129,6 +132,34 @@ export class TAttachedDoc extends TDoc implements AttachedDoc {
     collection!: string
 }
 
+@Model(core.class.Blob, core.class.Doc, DOMAIN_BLOB_DATA)
+@UX(core.string.Object)
+export class TBlob extends TDoc implements Blob {
+  @Prop(TypeString(), core.string.Blob)
+  @ReadOnly()
+    provider!: string
+
+  @Prop(TypeString(), core.string.BlobContentType)
+  @ReadOnly()
+    contentType!: string
+
+  @Prop(TypeString(), core.string.BlobStorageId)
+  @ReadOnly()
+    storageId!: string
+
+  @Prop(TypeString(), core.string.BlobEtag)
+  @ReadOnly()
+    etag!: string
+
+  @Prop(TypeString(), core.string.BlobVersion)
+  @ReadOnly()
+    version!: string
+
+  @Prop(TypeNumber(), core.string.BlobSize)
+  @ReadOnly()
+    size!: number
+}
+
 @UX(core.string.ClassLabel)
 @Model(core.class.Class, core.class.Doc, DOMAIN_MODEL)
 export class TClass extends TDoc implements Class<Obj> {

models/core/src/index.ts

@@ -29,6 +29,7 @@ import {
   TArrOf,
   TAttachedDoc,
   TAttribute,
+  TBlob,
   TBlobData,
   TClass,
   TCollection,
@@ -151,7 +152,8 @@ export function createModel (builder: Builder): void {
     TIndexConfiguration,
     TStatus,
     TStatusCategory,
-    TMigrationState
+    TMigrationState,
+    TBlob
   )
 
   builder.createDoc(

packages/core/src/classes.ts

@@ -337,6 +337,12 @@ export const DOMAIN_TRANSIENT = 'transient' as Domain
  */
 export const DOMAIN_BLOB = 'blob' as Domain
 
+/**
+ * Special domain to access s3 blob data.
+ * @public
+ */
+export const DOMAIN_BLOB_DATA = 'blob-data' as Domain
+
 /**
  * Special domain to access s3 blob data.
  * @public
@@ -535,6 +541,29 @@ export interface IndexStageState extends Doc {
   attributes: Record<string, any>
 }
 
+/**
+ * @public
+ *
+ * A blob document to manage blob attached documents.
+ *
+ * _id: is a platform ID and it created using our regular generateId(),
+ * and storageId is a provider specified storage id.
+ */
+export interface Blob extends Doc {
+  // Provider
+  provider: string
+  // A provider specific id
+  storageId: string
+  // A content type for blob
+  contentType: string
+  // A etag for blob
+  etag: string
+  // Document version if supported by provider
+  version: string | null
+  // A document size
+  size: number
+}
+
 /**
  * @public
  *

packages/core/src/component.ts

@@ -20,6 +20,7 @@ import type {
   AnyAttribute,
   ArrOf,
   AttachedDoc,
+  Blob,
   BlobData,
   Class,
   Collection,
@@ -82,6 +83,7 @@ export default plugin(coreId, {
   class: {
     Obj: '' as Ref<Class<Obj>>,
     Doc: '' as Ref<Class<Doc>>,
+    Blob: '' as Ref<Class<Blob>>,
     AttachedDoc: '' as Ref<Class<AttachedDoc>>,
     Class: '' as Ref<Class<Class<Obj>>>,
     Mixin: '' as Ref<Class<Mixin<Doc>>>,

pods/backup/src/platform.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import { getWorkspaceId } from '@hcengineering/core'
+import { getWorkspaceId, MeasureMetricsContext } from '@hcengineering/core'
 import { MinioService } from '@hcengineering/minio'
 import { setMetadata } from '@hcengineering/platform'
 import { backup, createStorageBackupStorage } from '@hcengineering/server-backup'
@@ -92,10 +92,12 @@ export class PlatformWorker {
 
   async backup (): Promise<void> {
     const workspaces = await getWorkspaces()
+    const ctx = new MeasureMetricsContext('backup', {})
     for (const ws of workspaces) {
       console.log('\n\nBACKUP WORKSPACE ', ws.workspace, ws.productId)
       try {
         const storage = await createStorageBackupStorage(
+          ctx,
           this.storageAdapter,
           getWorkspaceId('backups', ws.productId),
           ws.workspace