diff --git a/lib/metadataTypes/Automation.js b/lib/metadataTypes/Automation.js index 21160bdba..ddb6dfe3f 100644 --- a/lib/metadataTypes/Automation.js +++ b/lib/metadataTypes/Automation.js @@ -5,7 +5,6 @@ const TYPE = require('../../types/mcdev.d'); const Util = require('../util/util'); const File = require('../util/file'); const Definitions = require('../MetadataTypeDefinitions'); -const Verification = require('./Verification'); const cache = require('../util/cache'); const pLimit = require('p-limit'); @@ -26,65 +25,75 @@ class Automation extends MetadataType { * @returns {Promise.} Promise of metadata */ static async retrieve(retrieveDir, _, __, key) { - /** @type {TYPE.SoapRequestParams} */ - let requestParams = null; - if (key) { - requestParams = { - filter: { - leftOperand: 'CustomerKey', - operator: 'equals', - rightOperand: key, - }, - }; - } - const results = await this.client.soap.retrieveBulk('Program', ['ObjectID'], requestParams); - if (results.Results?.length && !key) { - // empty results will come back without "Results" defined - Util.logger.info( - Util.getGrayMsg( - ` - ${results.Results?.length} automation${ - results.Results?.length === 1 ? '' : 's' - } found. Retrieving details...` - ) + let metadataMap; + if (!key && this._cachedMetadataMap) { + metadataMap = this._cachedMetadataMap; + delete this._cachedMetadataMap; + } else { + /** @type {TYPE.SoapRequestParams} */ + let requestParams = null; + if (key) { + requestParams = { + filter: { + leftOperand: 'CustomerKey', + operator: 'equals', + rightOperand: key, + }, + }; + } + const results = await this.client.soap.retrieveBulk( + 'Program', + ['ObjectID'], + requestParams ); - } - // the API seems to handle 50 concurrent requests nicely - const rateLimit = pLimit(50); + if (results.Results?.length && !key) { + // empty results will come back without "Results" defined + Util.logger.info( + Util.getGrayMsg( + ` - ${results.Results?.length} automation${ + results.Results?.length === 1 ? '' : 's' + } found. Retrieving details...` + ) + ); + } + // the API seems to handle 50 concurrent requests nicely + const rateLimit = pLimit(50); - const details = results.Results - ? await Promise.all( - results.Results.map(async (item) => - rateLimit(async () => { - try { - return await this.client.rest.get( - '/automation/v1/automations/' + item.ObjectID - ); - } catch (ex) { + const details = results.Results + ? await Promise.all( + results.Results.map(async (item) => + rateLimit(async () => { try { - if (ex.message == 'socket hang up') { - // one more retry; it's a rare case but retrying again should solve the issue gracefully - return await this.client.rest.get( - '/automation/v1/automations/' + item.ObjectID - ); + return await this.client.rest.get( + '/automation/v1/automations/' + item.ObjectID + ); + } catch (ex) { + try { + if (ex.message == 'socket hang up') { + // one more retry; it's a rare case but retrying again should solve the issue gracefully + return await this.client.rest.get( + '/automation/v1/automations/' + item.ObjectID + ); + } + } catch { + // no extra action needed, handled below } - } catch { - // no extra action needed, handled below + // if we do get here, we should log the error and continue instead of failing to download all automations + Util.logger.error( + ` ☇ skipping Automation ${item.ObjectID}: ${ex.message} ${ex.code}` + ); + return null; } - // if we do get here, we should log the error and continue instead of failing to download all automations - Util.logger.error( - ` ☇ skipping Automation ${item.ObjectID}: ${ex.message} ${ex.code}` - ); - return null; - } - }) + }) + ) ) - ) - : []; + : []; - // * if retrieving some automations fails, a null element would remain in the details-array for each of them that needs to be filtered to prevent it from causing issues elsewhere - let metadataMap = this.parseResponseBody({ items: details.filter(Boolean) }); + // * if retrieving some automations fails, a null element would remain in the details-array for each of them that needs to be filtered to prevent it from causing issues elsewhere + metadataMap = this.parseResponseBody({ items: details.filter(Boolean) }); + } - if (Object.keys(metadataMap).length) { + if (!this._skipNotificationRetrieve && Object.keys(metadataMap).length) { // attach notification information to each automation that has any await this.#getAutomationNotificationsREST(metadataMap); } @@ -426,12 +435,7 @@ class Automation extends MetadataType { ); // empty if block continue; - } else if ( - !this.definition.dependencies.includes(activity.r__type) && - !this.definition.manuallyRetrievedDependencies.includes( - activity.r__type - ) - ) { + } else if (!this.definition.dependencies.includes(activity.r__type)) { Util.logger.debug( ` - skipping ${ metadata[this.definition.keyField] @@ -443,21 +447,6 @@ class Automation extends MetadataType { ); continue; } - if (activity.r__type === 'verification') { - // data verifications can only be retrieved 1-by-1. The get-API does not work without IDs it seems. - Verification.client = this.client; - Verification.buObject = this.buObject; - Verification.properties = this.properties; - const dvResult = await Verification.retrieve( - this.retrieveDir, - undefined, - undefined, - activity.activityObjectId - ); - if (dvResult?.metadata) { - cache.mergeMetadata('verification', dvResult.metadata); - } - } // / if managed by cache we can update references to support deployment if ( Definitions[activity.r__type]?.['idField'] && diff --git a/lib/metadataTypes/Verification.js b/lib/metadataTypes/Verification.js index 2345fc1b8..693f288bd 100644 --- a/lib/metadataTypes/Verification.js +++ b/lib/metadataTypes/Verification.js @@ -2,6 +2,7 @@ const TYPE = require('../../types/mcdev.d'); const MetadataType = require('./MetadataType'); +const Automation = require('./Automation'); const Util = require('../util/util'); const cache = require('../util/cache'); @@ -21,38 +22,97 @@ class Verification extends MetadataType { * @returns {Promise.} Promise of metadata */ static async retrieve(retrieveDir, _, __, key) { - let param = ''; + let paramArr = []; if (key?.startsWith('id:')) { - param = key.slice(3); + paramArr = [key.slice(3)]; } else if (key) { - param = key; + paramArr = [key]; } - if (param === '') { - throw new Error('Verification can only be retrieved if the ID is known'); + if (!paramArr.length) { + // there is no API endpoint to retrieve all dataVerification items, so we need to retrieve all automations and iterate over their activities + Util.logger.info(` - Caching dependent Metadata: automation`); + Automation.client = this.client; + Automation.buObject = this.buObject; + Automation.properties = this.properties; + Automation._skipNotificationRetrieve = true; + const automationsMapObj = await Automation.retrieve(); + delete Automation._skipNotificationRetrieve; + if (automationsMapObj?.metadata && Object.keys(automationsMapObj?.metadata).length) { + if (!key) { + // if we are not retrieving a single item, cache the automations for later use during retrieval of automations + Automation._cachedMetadataMap = automationsMapObj?.metadata; + } + // automations found, lets iterate over their activities to find the dataVerification items + const dataVerificationIds = []; + for (const automation of Object.values(automationsMapObj.metadata)) { + if (automation.steps) { + for (const step of automation.steps) { + for (const activity of step.activities) { + if ( + activity.objectTypeId === 1000 && + activity.activityObjectId && + activity.activityObjectId !== + '00000000-0000-0000-0000-000000000000' + ) { + dataVerificationIds.push(activity.activityObjectId); + } + } + } + } + } + if (dataVerificationIds.length) { + paramArr.push(...dataVerificationIds); + } + } } - try { - return await super.retrieveREST( - null, - '/automation/v1/dataverifications/' + param, - null, - key - ); - } catch (ex) { - if ( - ex.message === 'Not Found' || - ex.message === 'Request failed with status code 400' - ) { - if (retrieveDir) { - Util.logger.info( - `Downloaded: ${this.definition.type} (0)${Util.getKeysString(param)}` + const results = {}; + if (paramArr.length) { + if (key) { + try { + const verification = await super.retrieveREST( + null, + '/automation/v1/dataverifications/' + paramArr[0], + null, + key ); + const key = Object.values(verification?.metadata)[0]?.[ + this.definition.keyField + ]; + results[key] = verification?.metadata[key]; + } catch (ex) { + if ( + ex.message === 'Not Found' || + ex.message === 'Request failed with status code 400' + ) { + // if the ID is too short, the system will throw the 400 error + } else { + throw ex; + } } - // if the ID is too short, the system will throw the 400 error - return { metadata: {} }; } else { - throw ex; + const uri = '/automation/v1/dataverifications/'; + + const verificationArr = await this.client.rest.getCollection( + paramArr.map((id) => uri + id) + ); + for (const verification of verificationArr) { + const key = verification[this.definition.keyField]; + results[key] = verification; + } } } + if (retrieveDir) { + const savedMetadata = await this.saveResults(results, retrieveDir, null, null); + Util.logger.info( + `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + + Util.getKeysString(key) + ); + } + + return { + metadata: results, + type: this.definition.type, + }; } /** * Retrieves Metadata of Data Extract Activity for caching diff --git a/lib/metadataTypes/definitions/Automation.definition.js b/lib/metadataTypes/definitions/Automation.definition.js index 06597cd23..656bcf659 100644 --- a/lib/metadataTypes/definitions/Automation.definition.js +++ b/lib/metadataTypes/definitions/Automation.definition.js @@ -23,9 +23,7 @@ module.exports = { importMobileContact: 726, }, bodyIteratorField: 'items', - manuallyRetrievedDependencies: ['verification'], dependencies: [ - 'dataExtension', // for verification 'dataExtract', 'emailSend', 'fileTransfer', @@ -33,6 +31,7 @@ module.exports = { 'importFile', 'query', 'script', + 'verification', ], folderType: 'automation', hasExtended: false,