From e9ebcaf6cb99513e691526bc564f160fdf899fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAnar=20Vestmann?= <43557895+RunarVestmann@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:05:31 +0000 Subject: [PATCH] fix(search-indexer): Only calculate folder hash on initial iteration (#16084) * Only calculate folder hash on initial iteration * chore: charts update dirty files * Change conditional to be more clear * Only run post sync if there are postSyncOptions --------- Co-authored-by: andes-it Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .../infra/search-indexer-service.ts | 2 +- charts/islandis/values.dev.yaml | 4 +- libs/cms/src/lib/search/cmsSync.service.ts | 40 ++++++++++--------- libs/cms/src/lib/search/contentful.service.ts | 2 +- .../src/lib/indexing.service.ts | 3 +- .../content-search-indexer/types/src/index.ts | 1 + 6 files changed, 29 insertions(+), 23 deletions(-) diff --git a/apps/services/search-indexer/infra/search-indexer-service.ts b/apps/services/search-indexer/infra/search-indexer-service.ts index d0c6bf7ba782..d1bc2fae6874 100644 --- a/apps/services/search-indexer/infra/search-indexer-service.ts +++ b/apps/services/search-indexer/infra/search-indexer-service.ts @@ -17,7 +17,7 @@ const envs = { prod: 'cdn.contentful.com', }, CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: { - dev: '20', + dev: '40', staging: '40', prod: '40', }, diff --git a/charts/islandis/values.dev.yaml b/charts/islandis/values.dev.yaml index c8fbd907f529..54fe3e284c71 100644 --- a/charts/islandis/values.dev.yaml +++ b/charts/islandis/values.dev.yaml @@ -1954,7 +1954,7 @@ search-indexer-service: env: AIR_DISCOUNT_SCHEME_FRONTEND_HOSTNAME: 'loftbru.dev01.devland.is' APPLICATION_URL: 'http://search-indexer-service' - CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '20' + CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '40' CONTENTFUL_ENVIRONMENT: 'master' CONTENTFUL_HOST: 'preview.contentful.com' CONTENTFUL_SPACE: '8k0h54kbe6bj' @@ -2036,7 +2036,7 @@ search-indexer-service: env: AIR_DISCOUNT_SCHEME_FRONTEND_HOSTNAME: 'loftbru.dev01.devland.is' APPLICATION_URL: 'http://search-indexer-service' - CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '20' + CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '40' CONTENTFUL_ENVIRONMENT: 'master' CONTENTFUL_HOST: 'preview.contentful.com' CONTENTFUL_SPACE: '8k0h54kbe6bj' diff --git a/libs/cms/src/lib/search/cmsSync.service.ts b/libs/cms/src/lib/search/cmsSync.service.ts index 29da35bcd872..b1593bf2c158 100644 --- a/libs/cms/src/lib/search/cmsSync.service.ts +++ b/libs/cms/src/lib/search/cmsSync.service.ts @@ -184,35 +184,39 @@ export class CmsSyncService implements ContentSearchImporter { options: SyncOptions, ): Promise | null> { logger.info('Doing cms sync', options) - let cmsSyncOptions: SyncOptions + let cmsSyncOptions: SyncOptions = options /** * We don't want full sync to run every time we start a new pod * We want full sync to run once when the first pod initializes the first container * and then never again until a new index is deployed */ - let folderHash + let folderHash = options.folderHash + if (options.syncType === 'initialize') { const { elasticIndex = getElasticsearchIndex(options.locale) } = options - - folderHash = await this.getModelsFolderHash() - const lastFolderHash = await this.getLastFolderHash(elasticIndex) - if (folderHash !== lastFolderHash) { - logger.info( - 'Folder and index folder hash do not match, running full sync', - { locale: options.locale }, - ) - cmsSyncOptions = { ...options, syncType: 'full' } - } else { - logger.info('Folder and index folder hash match, skipping sync', { - locale: options.locale, - }) - // we skip import if it is not needed - return null + cmsSyncOptions = { ...options, syncType: 'full' } + if (folderHash === undefined) { + folderHash = await this.getModelsFolderHash() + const lastFolderHash = await this.getLastFolderHash(elasticIndex) + if (folderHash !== lastFolderHash) { + logger.info( + 'Folder and index folder hash do not match, running full sync', + { locale: options.locale }, + ) + } else { + logger.info('Folder and index folder hash match, skipping sync', { + locale: options.locale, + }) + // we skip import if it is not needed + return null + } } } else if (options.syncType === 'full') { cmsSyncOptions = options - folderHash = await this.getModelsFolderHash() // we know full will update all models so we can set the folder hash here + if (folderHash === undefined) { + folderHash = await this.getModelsFolderHash() // we know full will update all models so we can set the folder hash here + } } else { cmsSyncOptions = options folderHash = '' // this will always be a partial update so we don't want to update folder hash diff --git a/libs/cms/src/lib/search/contentful.service.ts b/libs/cms/src/lib/search/contentful.service.ts index ac861908f89e..8b14f7ac5f4c 100644 --- a/libs/cms/src/lib/search/contentful.service.ts +++ b/libs/cms/src/lib/search/contentful.service.ts @@ -583,7 +583,7 @@ export class ContentfulService { nestedItems, } = await this.getPopulatedSyncEntries(typeOfSync, locale, chunkSize) - const isDeltaUpdate = syncType !== 'full' + const isDeltaUpdate = syncType === 'fromLast' let shouldResolveNestedEntries = false if (environment.runtimeEnvironment === 'local') { diff --git a/libs/content-search-indexer/src/lib/indexing.service.ts b/libs/content-search-indexer/src/lib/indexing.service.ts index 6d1b6b9925dd..ee38d07c8ea6 100644 --- a/libs/content-search-indexer/src/lib/indexing.service.ts +++ b/libs/content-search-indexer/src/lib/indexing.service.ts @@ -73,6 +73,7 @@ export class IndexingService { const importerResponse = await importer.doSync({ ...options, nextPageToken, + folderHash: postSyncOptions?.folderHash, }) // importers can skip import by returning null @@ -111,7 +112,7 @@ export class IndexingService { postSyncOptions = { ...postSyncOptions, token: nextSyncToken } } - if (importer.postSync) { + if (postSyncOptions && importer.postSync) { logger.info('Importer started post sync', { importer: importer.constructor.name, index: elasticIndex, diff --git a/libs/content-search-indexer/types/src/index.ts b/libs/content-search-indexer/types/src/index.ts index 3e2424223b27..f591d3d16076 100644 --- a/libs/content-search-indexer/types/src/index.ts +++ b/libs/content-search-indexer/types/src/index.ts @@ -31,6 +31,7 @@ export interface SyncOptions { syncType: 'full' | 'fromLast' | 'initialize' elasticIndex?: string nextPageToken?: string + folderHash?: string } export interface SyncResponse {