From a059eef918898b3c855dadf2cb8593326374275f Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 23 Oct 2024 15:11:43 +0530 Subject: [PATCH] feat: added taxonomy mapper file support in import setup --- .../src/config/index.ts | 17 +- .../src/import/modules/taxonomies.ts | 155 ++++++++++++++++++ .../src/types/default-config.ts | 1 + .../src/types/index.ts | 7 + 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 packages/contentstack-import-setup/src/import/modules/taxonomies.ts diff --git a/packages/contentstack-import-setup/src/config/index.ts b/packages/contentstack-import-setup/src/config/index.ts index f211dbb22e..b8e371420d 100644 --- a/packages/contentstack-import-setup/src/config/index.ts +++ b/packages/contentstack-import-setup/src/config/index.ts @@ -35,7 +35,7 @@ const config: DefaultConfig = { 'content-types': { dirName: 'content_types', fileName: 'content_types.json', - dependencies: ['extensions'], + dependencies: ['extensions', 'taxonomies'], }, entries: { dirName: 'entries', @@ -54,6 +54,21 @@ const config: DefaultConfig = { taxonomies: { dirName: 'taxonomies', fileName: 'taxonomies.json', + invalidKeys: [ + 'updated_at', + 'created_by', + 'updated_by', + 'stackHeaders', + 'urlPath', + 'created_at', + 'ancestors', + 'update', + 'delete', + 'fetch', + 'descendants', + 'move', + 'search', + ], }, }, }; diff --git a/packages/contentstack-import-setup/src/import/modules/taxonomies.ts b/packages/contentstack-import-setup/src/import/modules/taxonomies.ts new file mode 100644 index 0000000000..cd74af03b7 --- /dev/null +++ b/packages/contentstack-import-setup/src/import/modules/taxonomies.ts @@ -0,0 +1,155 @@ +import { join } from 'path'; +import omit from 'lodash/omit'; +import isEmpty from 'lodash/isEmpty'; + +import { log, fsUtil } from '../../utils'; +import { ImportConfig, ModuleClassParams, TaxonomyQueryParams } from '../../types'; + +export default class TaxonomiesImportSetup { + private config: ImportConfig; + private taxonomiesFilePath: string; + private stackAPIClient: ModuleClassParams['stackAPIClient']; + private dependencies: ModuleClassParams['dependencies']; + private taxonomiesConfig: ImportConfig['modules']['taxonomies']; + private termsSuccessPath: string; + private taxSuccessPath: string; + private taxonomiesMapperDirPath: string; + private termsMapperDirPath: string; + public taxonomiesMapper: Record = {}; + public termsMapper: Record = {}; + + constructor({ config, stackAPIClient }: ModuleClassParams) { + this.config = config; + this.stackAPIClient = stackAPIClient; + this.taxonomiesFilePath = join(this.config.contentDir, 'taxonomies', 'taxonomies.json'); + this.taxonomiesConfig = config.modules.taxonomies; + this.taxonomiesMapperDirPath = join(this.config.backupDir, 'mapper', 'taxonomies'); + this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); + this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, 'terms'); + this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); + this.taxonomiesMapper = {}; + this.termsMapper = {}; + } + + /** + * Start the taxonomies import setup + * This method reads the taxonomies from the content folder and generates a mapper file + * @returns {Promise} + */ + async start(): Promise { + try { + const taxonomies: any = fsUtil.readFile(this.taxonomiesFilePath); + if (!isEmpty(taxonomies)) { + // 2. Create mapper directory + fsUtil.makeDirectory(this.taxonomiesMapperDirPath); + fsUtil.makeDirectory(this.termsMapperDirPath); + + for (const taxonomy of Object.values(taxonomies) as any) { + let targetTaxonomy: any = await this.getTaxonomies(taxonomy); + if (!targetTaxonomy) { + log(this.config, `Taxonomies with uid '${taxonomy.uid}' not found in the stack!`, 'error'); + continue; + } + targetTaxonomy = this.sanitizeTaxonomyAttribs(targetTaxonomy); + this.taxonomiesMapper[taxonomy.uid] = targetTaxonomy; + const terms = await this.getAllTermsOfTaxonomy(targetTaxonomy); + const sanitizedTerms = this.sanitizeTermsAttribs(terms); + this.termsMapper[taxonomy.uid] = sanitizedTerms; + } + + if (this.taxonomiesMapper !== undefined && !isEmpty(this.taxonomiesMapper)) { + fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesMapper); + } + if (this.termsMapper !== undefined && !isEmpty(this.termsMapper)) { + fsUtil.writeFile(this.termsSuccessPath, this.termsMapper); + } + + log(this.config, `Generated required setup files for taxonomies`, 'success'); + } else { + log(this.config, 'No taxonomies found in the content folder!', 'info'); + } + } catch (error) { + log(this.config, `Error generating taxonomies mapper: ${error.message}`, 'error'); + } + } + + /** + * Retrieves the taxonomies based on the provided taxonomy UID. + * + * @param taxonomy - The UID of the taxonomy to retrieve. + * @returns A promise that resolves to the retrieved taxonomies. + */ + async getTaxonomies(taxonomy: any): Promise { + return await this.stackAPIClient + .taxonomy(taxonomy.uid) + .fetch() + .then((data: any) => data) + .catch((err: any) => this.handleTaxonomyErrorMsg(err)); + } + + /** + * Sanitizes the attributes of a taxonomy object. + * + * @param taxonomy - The taxonomy object to sanitize. + * @returns The sanitized taxonomy object with invalid keys omitted. + */ + sanitizeTaxonomyAttribs(taxonomy: Record) { + return omit(taxonomy, this.taxonomiesConfig.invalidKeys); + } + + /** + * Retrieves all terms of a taxonomy. + * + * @param taxonomy - The taxonomy object. + * @param skip - The number of terms to skip (default: 0). + * @param terms - An array to store the retrieved terms (default: []). + * @returns A promise that resolves to an array of terms. + */ + async getAllTermsOfTaxonomy(taxonomy: any, skip = 0, terms: any[] = []): Promise { + const queryParams: TaxonomyQueryParams = { + include_count: true, + limit: 100, + skip, + }; + + if (skip >= 0) queryParams['skip'] = skip || 0; + queryParams['depth'] = 0; + + await this.stackAPIClient + .taxonomy(taxonomy.uid) + .terms() + .query(queryParams) + .find() + .then((data: any) => { + terms = terms.concat(data.items); + if (data.count >= skip + queryParams.limit) { + return this.getAllTermsOfTaxonomy(taxonomy, skip + 100, terms); + } + }) + .catch((err: any) => this.handleTaxonomyErrorMsg(err)); + return terms; + } + + /** + * Sanitizes the attributes of the given terms. + * + * @param terms - An array of terms to sanitize. + * @returns The sanitized terms. + */ + sanitizeTermsAttribs(terms: Record[]) { + for (let index = 0; index < terms?.length; index++) { + terms[index] = omit(terms[index], this.taxonomiesConfig.invalidKeys); + } + return terms; + } + + handleTaxonomyErrorMsg(err: any) { + if (err?.errorMessage || err?.message) { + const errorMsg = err?.errorMessage || err?.errors?.taxonomy || err?.errors?.term || err?.message; + log(this.config, errorMsg, 'error'); + } else { + log(this.config, 'Error fetching taxonomy data!', 'error'); + log(this.config, err, 'error'); + } + } +} diff --git a/packages/contentstack-import-setup/src/types/default-config.ts b/packages/contentstack-import-setup/src/types/default-config.ts index d0d1fa57f3..76e6ef995f 100644 --- a/packages/contentstack-import-setup/src/types/default-config.ts +++ b/packages/contentstack-import-setup/src/types/default-config.ts @@ -47,6 +47,7 @@ export default interface DefaultConfig { dirName: string; fileName: string; dependencies?: Modules[]; + invalidKeys: string[]; }; }; } diff --git a/packages/contentstack-import-setup/src/types/index.ts b/packages/contentstack-import-setup/src/types/index.ts index 984a661dbc..6dd22c883c 100644 --- a/packages/contentstack-import-setup/src/types/index.ts +++ b/packages/contentstack-import-setup/src/types/index.ts @@ -100,3 +100,10 @@ export type ExtensionType = { scope: Record; title: string; }; + +export type TaxonomyQueryParams = { + include_count: boolean; + limit: number; + skip: number; + depth?: number; +};