UBERF-7796: Rework index creation logic

.vscode/launch.json

@@ -41,6 +41,7 @@
         "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds.,
         "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
         "SERVER_SECRET": "secret",
+        "ENABLE_CONSOLE": "true",
         "COLLABORATOR_URL": "ws://localhost:3078",
         "COLLABORATOR_API_URL": "http://localhost:3078",
         "REKONI_URL": "http://localhost:4004",

common/scripts/version.txt

@@ -1 +1 @@
-"0.6.271"
+"0.6.274"

models/core/src/core.ts

@@ -329,13 +329,13 @@ export class TDocIndexState extends TDoc implements DocIndexState {
   attributes!: Record<string, any>
 
   @Prop(TypeBoolean(), getEmbeddedLabel('Removed'))
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
   @Hidden()
     removed!: boolean
 
   // States for different stages
   @Prop(TypeRecord(), getEmbeddedLabel('Stages'))
-  @Index(IndexKind.Indexed)
+  // @Index(IndexKind.Indexed)
   @Hidden()
     stages!: Record<string, boolean | string>
 

models/core/src/index.ts

@@ -27,7 +27,6 @@ import {
   type AttachedDoc,
   type Class,
   type Doc,
-  type DocIndexState,
   type IndexingConfiguration,
   type TxCollectionCUD
 } from '@hcengineering/core'
@@ -284,8 +283,10 @@ export function createModel (builder: Builder): void {
 
   builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
     domain: DOMAIN_DOC_INDEX_STATE,
+    indexes: [{ keys: { removed: 1 }, filter: { removed: true } }],
     disabled: [
       { attachedToClass: 1 },
+      { objectClass: 1 },
       { stages: 1 },
       { generationId: 1 },
       { space: 1 },
@@ -298,24 +299,6 @@ export function createModel (builder: Builder): void {
     skip: ['stages.']
   })
 
-  builder.mixin<Class<DocIndexState>, IndexingConfiguration<TxCollectionCUD<Doc, AttachedDoc>>>(
-    core.class.DocIndexState,
-    core.class.Class,
-    core.mixin.IndexConfiguration,
-    {
-      indexes: [
-        {
-          keys: {
-            _class: 1,
-            stages: 1,
-            _id: 1,
-            modifiedOn: 1
-          }
-        }
-      ]
-    }
-  )
-
   builder.mixin(core.class.Space, core.class.Class, core.mixin.FullTextSearchContext, {
     childProcessingAllowed: false
   })

server/core/src/adapter.ts

@@ -23,7 +23,6 @@ import {
   type FindOptions,
   type FindResult,
   type Hierarchy,
-  type IndexingConfiguration,
   type MeasureContext,
   type ModelDb,
   type Ref,
@@ -38,19 +37,23 @@ import type { ServerFindOptions } from './types'
 export interface DomainHelperOperations {
   create: (domain: Domain) => Promise<void>
   exists: (domain: Domain) => boolean
+
+  listDomains: () => Promise<Set<Domain>>
   createIndex: (domain: Domain, value: string | FieldIndexConfig<Doc>, options?: { name: string }) => Promise<void>
   dropIndex: (domain: Domain, name: string) => Promise<void>
   listIndexes: (domain: Domain) => Promise<{ name: string }[]>
-  hasDocuments: (domain: Domain, count: number) => Promise<boolean>
+
+  // Could return 0 even if it has documents
+  estimatedCount: (domain: Domain) => Promise<number>
 }
 
 export interface DomainHelper {
   checkDomain: (
     ctx: MeasureContext,
     domain: Domain,
-    forceCreate: boolean,
+    documents: number,
     operations: DomainHelperOperations
-  ) => Promise<boolean>
+  ) => Promise<void>
 }
 
 export interface RawDBAdapterStream<T extends Doc> {
@@ -87,15 +90,20 @@ export interface RawDBAdapter {
   close: () => Promise<void>
 }
 
+export type DbAdapterHandler = (
+  domain: Domain,
+  event: 'add' | 'update' | 'delete' | 'read',
+  count: number,
+  time: number,
+  helper: DomainHelperOperations
+) => void
 /**
  * @public
  */
 export interface DbAdapter {
   init?: () => Promise<void>
 
-  helper?: () => DomainHelperOperations
-  createIndexes: (domain: Domain, config: Pick<IndexingConfiguration<Doc>, 'indexes'>) => Promise<void>
-  removeOldIndex: (domain: Domain, deletePattern: RegExp[], keepPattern: RegExp[]) => Promise<void>
+  helper: () => DomainHelperOperations
 
   close: () => Promise<void>
   findAll: <T extends Doc>(
@@ -116,6 +124,9 @@ export interface DbAdapter {
 
   // Bulk update operations
   update: (ctx: MeasureContext, domain: Domain, operations: Map<Ref<Doc>, DocumentUpdate<Doc>>) => Promise<void>
+
+  // Allow to register a handler to listen for domain operations
+  on?: (handler: DbAdapterHandler) => void
 }
 
 /**

server/core/src/indexer/indexer.ts

@@ -349,23 +349,36 @@ export class FullTextIndexPipeline implements FullTextPipeline {
           keepPattern.push(new RegExp(st.stageId))
         }
       }
+      const helper = this.storage.helper()
       if (deletePattern.length > 0) {
-        await this.storage.removeOldIndex(DOMAIN_DOC_INDEX_STATE, deletePattern, keepPattern)
+        try {
+          const existingIndexes = await helper.listIndexes(DOMAIN_DOC_INDEX_STATE)
+          for (const existingIndex of existingIndexes) {
+            if (existingIndex.name !== undefined) {
+              const name: string = existingIndex.name
+              if (deletePattern.some((it) => it.test(name)) && !keepPattern.some((it) => it.test(name))) {
+                await helper.dropIndex(DOMAIN_DOC_INDEX_STATE, name)
+              }
+            }
+          }
+        } catch (err: any) {
+          console.error(err)
+        }
       }
 
       for (const st of this.stages) {
         if (this.cancelling) {
           return
         }
-        await this.storage.createIndexes(DOMAIN_DOC_INDEX_STATE, {
-          indexes: [
-            {
-              keys: {
-                ['stages.' + st.stageId]: 1
-              }
+        await this.storage.helper().createIndex(
+          DOMAIN_DOC_INDEX_STATE,
+          {
+            keys: {
+              ['stages.' + st.stageId]: 1
             }
-          ]
-        })
+          },
+          { name: 'stages.' + st.stageId + '_1' }
+        )
       }
     }
 
@@ -481,7 +494,9 @@ export class FullTextIndexPipeline implements FullTextPipeline {
               async (ctx) =>
                 await this.storage.findAll(ctx, core.class.DocIndexState, q, {
                   sort: { modifiedOn: SortingOrder.Descending },
-                  limit: globalIndexer.processingSize
+                  limit: globalIndexer.processingSize,
+                  skipClass: true,
+                  skipSpace: true
                 })
             )
             const toRemove: DocIndexState[] = []
@@ -594,7 +609,9 @@ export class FullTextIndexPipeline implements FullTextPipeline {
             _id: 1,
             stages: 1,
             objectClass: 1
-          }
+          },
+          skipSpace: true,
+          skipClass: true
         }
       )
 

server/core/src/mem.ts

@@ -32,7 +32,7 @@ import core, {
   type TxResult,
   type WorkspaceId
 } from '@hcengineering/core'
-import { type DbAdapter } from './adapter'
+import { type DbAdapter, type DomainHelperOperations } from './adapter'
 
 /**
  * @public
@@ -49,6 +49,18 @@ export class DummyDbAdapter implements DbAdapter {
 
   async init (): Promise<void> {}
 
+  helper (): DomainHelperOperations {
+    return {
+      create: async () => {},
+      exists: () => true,
+      listDomains: async () => new Set(),
+      createIndex: async () => {},
+      dropIndex: async () => {},
+      listIndexes: async () => [],
+      estimatedCount: async () => 0
+    }
+  }
+
   async createIndexes (domain: Domain, config: Pick<IndexingConfiguration<Doc>, 'indexes'>): Promise<void> {}
   async removeOldIndex (domain: Domain, deletePattern: RegExp[], keepPattern: RegExp[]): Promise<void> {}
 

server/core/src/server/domainHelper.ts

@@ -74,41 +74,24 @@ export class DomainIndexHelperImpl implements DomainHelper {
   }
 
   /**
-   * return false if and only if domain underline structures are not required.
+   * Check if some indexes need to be created for domain.
    */
   async checkDomain (
     ctx: MeasureContext,
     domain: Domain,
-    forceCreate: boolean,
+    documents: number,
     operations: DomainHelperOperations
-  ): Promise<boolean> {
+  ): Promise<void> {
     const domainInfo = this.domains.get(domain)
     const cfg = this.domainConfigurations.find((it) => it.domain === domain)
 
-    let exists = operations.exists(domain)
-    const hasDocuments = exists && (await operations.hasDocuments(domain, 1))
-    // Drop collection if it exists and should not exists or doesn't have documents.
-    if (exists && (cfg?.disableCollection === true || (!hasDocuments && !forceCreate))) {
-      // We do not need this collection
-      return false
-    }
-
-    if (forceCreate && !exists) {
-      await operations.create(domain)
-      ctx.info('collection will be created', domain)
-      exists = true
-    }
-    if (!exists) {
-      // Do not need to create, since not force and no documents.
-      return false
-    }
     const bb: (string | FieldIndexConfig<Doc>)[] = []
     const added = new Set<string>()
 
     try {
-      const has50Documents = await operations.hasDocuments(domain, 50)
+      const has50Documents = documents > 50
       const allIndexes = (await operations.listIndexes(domain)).filter((it) => it.name !== '_id_')
-      ctx.info('check indexes', { domain, has50Documents })
+      ctx.info('check indexes', { domain, has50Documents, documents })
       if (has50Documents) {
         for (const vv of [...(domainInfo?.values() ?? []), ...(cfg?.indexes ?? [])]) {
           try {
@@ -188,7 +171,5 @@ export class DomainIndexHelperImpl implements DomainHelper {
     if (bb.length > 0) {
       ctx.info('created indexes', { domain, bb })
     }
-
-    return true
   }
 }

server/core/src/server/index.ts

@@ -167,7 +167,7 @@ export async function createServerStorage (
 
   const domainHelper = new DomainIndexHelperImpl(metrics, hierarchy, modelDb, conf.workspace)
 
-  return new TServerStorage(
+  const serverStorage = new TServerStorage(
     conf.domains,
     conf.defaultAdapter,
     adapters,
@@ -184,6 +184,10 @@ export async function createServerStorage (
     model,
     domainHelper
   )
+  await ctx.with('init-domain-info', {}, async () => {
+    await serverStorage.initDomainInfo()
+  })
+  return serverStorage
 }
 
 /**