fix(search-indexer): Only calculate folder hash on initial iteration (#…

…16084) * Only calculate folder hash on initial iteration * chore: charts update dirty files * Change conditional to be more clear * Only run post sync if there are postSyncOptions --------- Co-authored-by: andes-it <builders@andes.is> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
island-is · Sep 19, 2024 · e9ebcaf · e9ebcaf
1 parent bc0f27b
commit e9ebcaf
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 23 deletions.
diff --git a/apps/services/search-indexer/infra/search-indexer-service.ts b/apps/services/search-indexer/infra/search-indexer-service.ts
@@ -17,7 +17,7 @@ const envs = {
     prod: 'cdn.contentful.com',
   },
   CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: {
-    dev: '20',
+    dev: '40',
     staging: '40',
     prod: '40',
   },

diff --git a/charts/islandis/values.dev.yaml b/charts/islandis/values.dev.yaml
@@ -1954,7 +1954,7 @@ search-indexer-service:
   env:
     AIR_DISCOUNT_SCHEME_FRONTEND_HOSTNAME: 'loftbru.dev01.devland.is'
     APPLICATION_URL: 'http://search-indexer-service'
-    CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '20'
+    CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '40'
     CONTENTFUL_ENVIRONMENT: 'master'
     CONTENTFUL_HOST: 'preview.contentful.com'
     CONTENTFUL_SPACE: '8k0h54kbe6bj'
@@ -2036,7 +2036,7 @@ search-indexer-service:
     env:
       AIR_DISCOUNT_SCHEME_FRONTEND_HOSTNAME: 'loftbru.dev01.devland.is'
       APPLICATION_URL: 'http://search-indexer-service'
-      CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '20'
+      CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '40'
       CONTENTFUL_ENVIRONMENT: 'master'
       CONTENTFUL_HOST: 'preview.contentful.com'
       CONTENTFUL_SPACE: '8k0h54kbe6bj'

diff --git a/libs/cms/src/lib/search/cmsSync.service.ts b/libs/cms/src/lib/search/cmsSync.service.ts
@@ -184,35 +184,39 @@ export class CmsSyncService implements ContentSearchImporter<PostSyncOptions> {
     options: SyncOptions,
   ): Promise<SyncResponse<PostSyncOptions> | null> {
     logger.info('Doing cms sync', options)
-    let cmsSyncOptions: SyncOptions
+    let cmsSyncOptions: SyncOptions = options
 
     /**
      * We don't want full sync to run every time we start a new pod
      * We want full sync to run once when the first pod initializes the first container
      * and then never again until a new index is deployed
      */
-    let folderHash
+    let folderHash = options.folderHash
+
     if (options.syncType === 'initialize') {
       const { elasticIndex = getElasticsearchIndex(options.locale) } = options
-
-      folderHash = await this.getModelsFolderHash()
-      const lastFolderHash = await this.getLastFolderHash(elasticIndex)
-      if (folderHash !== lastFolderHash) {
-        logger.info(
-          'Folder and index folder hash do not match, running full sync',
-          { locale: options.locale },
-        )
-        cmsSyncOptions = { ...options, syncType: 'full' }
-      } else {
-        logger.info('Folder and index folder hash match, skipping sync', {
-          locale: options.locale,
-        })
-        // we skip import if it is not needed
-        return null
+      cmsSyncOptions = { ...options, syncType: 'full' }
+      if (folderHash === undefined) {
+        folderHash = await this.getModelsFolderHash()
+        const lastFolderHash = await this.getLastFolderHash(elasticIndex)
+        if (folderHash !== lastFolderHash) {
+          logger.info(
+            'Folder and index folder hash do not match, running full sync',
+            { locale: options.locale },
+          )
+        } else {
+          logger.info('Folder and index folder hash match, skipping sync', {
+            locale: options.locale,
+          })
+          // we skip import if it is not needed
+          return null
+        }
       }
     } else if (options.syncType === 'full') {
       cmsSyncOptions = options
-      folderHash = await this.getModelsFolderHash() // we know full will update all models so we can set the folder hash here
+      if (folderHash === undefined) {
+        folderHash = await this.getModelsFolderHash() // we know full will update all models so we can set the folder hash here
+      }
     } else {
       cmsSyncOptions = options
       folderHash = '' // this will always be a partial update so we don't want to update folder hash

diff --git a/libs/cms/src/lib/search/contentful.service.ts b/libs/cms/src/lib/search/contentful.service.ts
@@ -583,7 +583,7 @@ export class ContentfulService {
       nestedItems,
     } = await this.getPopulatedSyncEntries(typeOfSync, locale, chunkSize)
 
-    const isDeltaUpdate = syncType !== 'full'
+    const isDeltaUpdate = syncType === 'fromLast'
 
     let shouldResolveNestedEntries = false
     if (environment.runtimeEnvironment === 'local') {

diff --git a/libs/content-search-indexer/src/lib/indexing.service.ts b/libs/content-search-indexer/src/lib/indexing.service.ts
@@ -73,6 +73,7 @@ export class IndexingService {
             const importerResponse = await importer.doSync({
               ...options,
               nextPageToken,
+              folderHash: postSyncOptions?.folderHash,
             })
 
             // importers can skip import by returning null
@@ -111,7 +112,7 @@ export class IndexingService {
         postSyncOptions = { ...postSyncOptions, token: nextSyncToken }
       }
 
-      if (importer.postSync) {
+      if (postSyncOptions && importer.postSync) {
         logger.info('Importer started post sync', {
           importer: importer.constructor.name,
           index: elasticIndex,

diff --git a/libs/content-search-indexer/types/src/index.ts b/libs/content-search-indexer/types/src/index.ts
@@ -31,6 +31,7 @@ export interface SyncOptions {
   syncType: 'full' | 'fromLast' | 'initialize'
   elasticIndex?: string
   nextPageToken?: string
+  folderHash?: string
 }
 
 export interface SyncResponse<PostSyncOptionsType = any> {