From 8ec466c46e63c997e44ee85060c8650893af1a35 Mon Sep 17 00:00:00 2001 From: runarvestmann Date: Thu, 19 Sep 2024 13:40:11 +0000 Subject: [PATCH 1/4] Only calculate folder hash on initial iteration --- .../infra/search-indexer-service.ts | 2 +- libs/cms/src/lib/search/cmsSync.service.ts | 39 +++++++++++-------- .../src/lib/indexing.service.ts | 1 + .../content-search-indexer/types/src/index.ts | 1 + 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/apps/services/search-indexer/infra/search-indexer-service.ts b/apps/services/search-indexer/infra/search-indexer-service.ts index d0c6bf7ba782..d1bc2fae6874 100644 --- a/apps/services/search-indexer/infra/search-indexer-service.ts +++ b/apps/services/search-indexer/infra/search-indexer-service.ts @@ -17,7 +17,7 @@ const envs = { prod: 'cdn.contentful.com', }, CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: { - dev: '20', + dev: '40', staging: '40', prod: '40', }, diff --git a/libs/cms/src/lib/search/cmsSync.service.ts b/libs/cms/src/lib/search/cmsSync.service.ts index 29da35bcd872..ba1f1a86a6cc 100644 --- a/libs/cms/src/lib/search/cmsSync.service.ts +++ b/libs/cms/src/lib/search/cmsSync.service.ts @@ -184,35 +184,40 @@ export class CmsSyncService implements ContentSearchImporter { options: SyncOptions, ): Promise | null> { logger.info('Doing cms sync', options) - let cmsSyncOptions: SyncOptions + let cmsSyncOptions: SyncOptions = options /** * We don't want full sync to run every time we start a new pod * We want full sync to run once when the first pod initializes the first container * and then never again until a new index is deployed */ - let folderHash + let folderHash = options.folderHash + if (options.syncType === 'initialize') { const { elasticIndex = getElasticsearchIndex(options.locale) } = options - folderHash = await this.getModelsFolderHash() - const lastFolderHash = await this.getLastFolderHash(elasticIndex) - if (folderHash !== lastFolderHash) { - logger.info( - 'Folder and index folder hash do not match, running full sync', - { locale: options.locale }, - ) - cmsSyncOptions = { ...options, syncType: 'full' } - } else { - logger.info('Folder and index folder hash match, skipping sync', { - locale: options.locale, - }) - // we skip import if it is not needed - return null + if (folderHash === undefined) { + folderHash = await this.getModelsFolderHash() + const lastFolderHash = await this.getLastFolderHash(elasticIndex) + if (folderHash !== lastFolderHash) { + logger.info( + 'Folder and index folder hash do not match, running full sync', + { locale: options.locale }, + ) + cmsSyncOptions = { ...options, syncType: 'full' } + } else { + logger.info('Folder and index folder hash match, skipping sync', { + locale: options.locale, + }) + // we skip import if it is not needed + return null + } } } else if (options.syncType === 'full') { cmsSyncOptions = options - folderHash = await this.getModelsFolderHash() // we know full will update all models so we can set the folder hash here + if (folderHash === undefined) { + folderHash = await this.getModelsFolderHash() // we know full will update all models so we can set the folder hash here + } } else { cmsSyncOptions = options folderHash = '' // this will always be a partial update so we don't want to update folder hash diff --git a/libs/content-search-indexer/src/lib/indexing.service.ts b/libs/content-search-indexer/src/lib/indexing.service.ts index 6d1b6b9925dd..8f4cc986168c 100644 --- a/libs/content-search-indexer/src/lib/indexing.service.ts +++ b/libs/content-search-indexer/src/lib/indexing.service.ts @@ -73,6 +73,7 @@ export class IndexingService { const importerResponse = await importer.doSync({ ...options, nextPageToken, + folderHash: postSyncOptions?.folderHash, }) // importers can skip import by returning null diff --git a/libs/content-search-indexer/types/src/index.ts b/libs/content-search-indexer/types/src/index.ts index 3e2424223b27..f591d3d16076 100644 --- a/libs/content-search-indexer/types/src/index.ts +++ b/libs/content-search-indexer/types/src/index.ts @@ -31,6 +31,7 @@ export interface SyncOptions { syncType: 'full' | 'fromLast' | 'initialize' elasticIndex?: string nextPageToken?: string + folderHash?: string } export interface SyncResponse { From 5ca99434af559d5e849db84131773458bbea641c Mon Sep 17 00:00:00 2001 From: andes-it Date: Thu, 19 Sep 2024 13:46:36 +0000 Subject: [PATCH 2/4] chore: charts update dirty files --- charts/islandis/values.dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/islandis/values.dev.yaml b/charts/islandis/values.dev.yaml index c8fbd907f529..54fe3e284c71 100644 --- a/charts/islandis/values.dev.yaml +++ b/charts/islandis/values.dev.yaml @@ -1954,7 +1954,7 @@ search-indexer-service: env: AIR_DISCOUNT_SCHEME_FRONTEND_HOSTNAME: 'loftbru.dev01.devland.is' APPLICATION_URL: 'http://search-indexer-service' - CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '20' + CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '40' CONTENTFUL_ENVIRONMENT: 'master' CONTENTFUL_HOST: 'preview.contentful.com' CONTENTFUL_SPACE: '8k0h54kbe6bj' @@ -2036,7 +2036,7 @@ search-indexer-service: env: AIR_DISCOUNT_SCHEME_FRONTEND_HOSTNAME: 'loftbru.dev01.devland.is' APPLICATION_URL: 'http://search-indexer-service' - CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '20' + CONTENTFUL_ENTRY_FETCH_CHUNK_SIZE: '40' CONTENTFUL_ENVIRONMENT: 'master' CONTENTFUL_HOST: 'preview.contentful.com' CONTENTFUL_SPACE: '8k0h54kbe6bj' From 2796cf8286bb7824056681d50ff19c3e4390fce2 Mon Sep 17 00:00:00 2001 From: runarvestmann Date: Thu, 19 Sep 2024 13:49:13 +0000 Subject: [PATCH 3/4] Change conditional to be more clear --- libs/cms/src/lib/search/cmsSync.service.ts | 3 +-- libs/cms/src/lib/search/contentful.service.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/cms/src/lib/search/cmsSync.service.ts b/libs/cms/src/lib/search/cmsSync.service.ts index ba1f1a86a6cc..b1593bf2c158 100644 --- a/libs/cms/src/lib/search/cmsSync.service.ts +++ b/libs/cms/src/lib/search/cmsSync.service.ts @@ -195,7 +195,7 @@ export class CmsSyncService implements ContentSearchImporter { if (options.syncType === 'initialize') { const { elasticIndex = getElasticsearchIndex(options.locale) } = options - + cmsSyncOptions = { ...options, syncType: 'full' } if (folderHash === undefined) { folderHash = await this.getModelsFolderHash() const lastFolderHash = await this.getLastFolderHash(elasticIndex) @@ -204,7 +204,6 @@ export class CmsSyncService implements ContentSearchImporter { 'Folder and index folder hash do not match, running full sync', { locale: options.locale }, ) - cmsSyncOptions = { ...options, syncType: 'full' } } else { logger.info('Folder and index folder hash match, skipping sync', { locale: options.locale, diff --git a/libs/cms/src/lib/search/contentful.service.ts b/libs/cms/src/lib/search/contentful.service.ts index ac861908f89e..8b14f7ac5f4c 100644 --- a/libs/cms/src/lib/search/contentful.service.ts +++ b/libs/cms/src/lib/search/contentful.service.ts @@ -583,7 +583,7 @@ export class ContentfulService { nestedItems, } = await this.getPopulatedSyncEntries(typeOfSync, locale, chunkSize) - const isDeltaUpdate = syncType !== 'full' + const isDeltaUpdate = syncType === 'fromLast' let shouldResolveNestedEntries = false if (environment.runtimeEnvironment === 'local') { From f091edb1f14b460d762b9f42f73acf7d545fe3ed Mon Sep 17 00:00:00 2001 From: runarvestmann Date: Thu, 19 Sep 2024 13:59:41 +0000 Subject: [PATCH 4/4] Only run post sync if there are postSyncOptions --- libs/content-search-indexer/src/lib/indexing.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/content-search-indexer/src/lib/indexing.service.ts b/libs/content-search-indexer/src/lib/indexing.service.ts index 8f4cc986168c..ee38d07c8ea6 100644 --- a/libs/content-search-indexer/src/lib/indexing.service.ts +++ b/libs/content-search-indexer/src/lib/indexing.service.ts @@ -112,7 +112,7 @@ export class IndexingService { postSyncOptions = { ...postSyncOptions, token: nextSyncToken } } - if (importer.postSync) { + if (postSyncOptions && importer.postSync) { logger.info('Importer started post sync', { importer: importer.constructor.name, index: elasticIndex,