Skip to content

Commit

Permalink
#325: enable retrieve-all for type verification
Browse files Browse the repository at this point in the history 
via caching automations first
  • Loading branch information
@JoernBerkefeld
JoernBerkefeld committed Aug 22, 2023
1 parent c8ef433 commit 27504e1
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 98 deletions.
135 changes: 62 additions & 73 deletions lib/metadataTypes/Automation.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ const TYPE = require('../../types/mcdev.d');
const Util = require('../util/util');
const File = require('../util/file');
const Definitions = require('../MetadataTypeDefinitions');
const Verification = require('./Verification');
const cache = require('../util/cache');
const pLimit = require('p-limit');

Expand All @@ -26,65 +25,75 @@ class Automation extends MetadataType {
* @returns {Promise.<TYPE.AutomationMapObj>} Promise of metadata
*/
static async retrieve(retrieveDir, _, __, key) {
/** @type {TYPE.SoapRequestParams} */
let requestParams = null;
if (key) {
requestParams = {
filter: {
leftOperand: 'CustomerKey',
operator: 'equals',
rightOperand: key,
},
};
}
const results = await this.client.soap.retrieveBulk('Program', ['ObjectID'], requestParams);
if (results.Results?.length && !key) {
// empty results will come back without "Results" defined
Util.logger.info(
Util.getGrayMsg(
` - ${results.Results?.length} automation${
results.Results?.length === 1 ? '' : 's'
} found. Retrieving details...`
)
let metadataMap;
if (!key && this._cachedMetadataMap) {
metadataMap = this._cachedMetadataMap;
delete this._cachedMetadataMap;
} else {
/** @type {TYPE.SoapRequestParams} */
let requestParams = null;
if (key) {
requestParams = {
filter: {
leftOperand: 'CustomerKey',
operator: 'equals',
rightOperand: key,
},
};
}
const results = await this.client.soap.retrieveBulk(
'Program',
['ObjectID'],
requestParams
);
}
// the API seems to handle 50 concurrent requests nicely
const rateLimit = pLimit(50);
if (results.Results?.length && !key) {
// empty results will come back without "Results" defined
Util.logger.info(
Util.getGrayMsg(
` - ${results.Results?.length} automation${
results.Results?.length === 1 ? '' : 's'
} found. Retrieving details...`
)
);
}
// the API seems to handle 50 concurrent requests nicely
const rateLimit = pLimit(50);

const details = results.Results
? await Promise.all(
results.Results.map(async (item) =>
rateLimit(async () => {
try {
return await this.client.rest.get(
'/automation/v1/automations/' + item.ObjectID
);
} catch (ex) {
const details = results.Results
? await Promise.all(
results.Results.map(async (item) =>
rateLimit(async () => {
try {
if (ex.message == 'socket hang up') {
// one more retry; it's a rare case but retrying again should solve the issue gracefully
return await this.client.rest.get(
'/automation/v1/automations/' + item.ObjectID
);
return await this.client.rest.get(
'/automation/v1/automations/' + item.ObjectID
);
} catch (ex) {
try {
if (ex.message == 'socket hang up') {
// one more retry; it's a rare case but retrying again should solve the issue gracefully
return await this.client.rest.get(
'/automation/v1/automations/' + item.ObjectID
);
}
} catch {
// no extra action needed, handled below
}
} catch {
// no extra action needed, handled below
// if we do get here, we should log the error and continue instead of failing to download all automations
Util.logger.error(
` ☇ skipping Automation ${item.ObjectID}: ${ex.message} ${ex.code}`
);
return null;
}
// if we do get here, we should log the error and continue instead of failing to download all automations
Util.logger.error(
` ☇ skipping Automation ${item.ObjectID}: ${ex.message} ${ex.code}`
);
return null;
}
})
})
)
)
)
: [];
: [];

// * if retrieving some automations fails, a null element would remain in the details-array for each of them that needs to be filtered to prevent it from causing issues elsewhere
let metadataMap = this.parseResponseBody({ items: details.filter(Boolean) });
// * if retrieving some automations fails, a null element would remain in the details-array for each of them that needs to be filtered to prevent it from causing issues elsewhere
metadataMap = this.parseResponseBody({ items: details.filter(Boolean) });
}

if (Object.keys(metadataMap).length) {
if (!this._skipNotificationRetrieve && Object.keys(metadataMap).length) {
// attach notification information to each automation that has any
await this.#getAutomationNotificationsREST(metadataMap);
}
Expand Down Expand Up @@ -426,12 +435,7 @@ class Automation extends MetadataType {
);
// empty if block
continue;
} else if (
!this.definition.dependencies.includes(activity.r__type) &&
!this.definition.manuallyRetrievedDependencies.includes(
activity.r__type
)
) {
} else if (!this.definition.dependencies.includes(activity.r__type)) {
Util.logger.debug(
` - skipping ${
metadata[this.definition.keyField]
Expand All @@ -443,21 +447,6 @@ class Automation extends MetadataType {
);
continue;
}
if (activity.r__type === 'verification') {
// data verifications can only be retrieved 1-by-1. The get-API does not work without IDs it seems.
Verification.client = this.client;
Verification.buObject = this.buObject;
Verification.properties = this.properties;
const dvResult = await Verification.retrieve(
this.retrieveDir,
undefined,
undefined,
activity.activityObjectId
);
if (dvResult?.metadata) {
cache.mergeMetadata('verification', dvResult.metadata);
}
}
// / if managed by cache we can update references to support deployment
if (
Definitions[activity.r__type]?.['idField'] &&
Expand Down
106 changes: 83 additions & 23 deletions lib/metadataTypes/Verification.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

const TYPE = require('../../types/mcdev.d');
const MetadataType = require('./MetadataType');
const Automation = require('./Automation');
const Util = require('../util/util');
const cache = require('../util/cache');

Expand All @@ -21,38 +22,97 @@ class Verification extends MetadataType {
* @returns {Promise.<TYPE.MetadataTypeMapObj>} Promise of metadata
*/
static async retrieve(retrieveDir, _, __, key) {
let param = '';
let paramArr = [];
if (key?.startsWith('id:')) {
param = key.slice(3);
paramArr = [key.slice(3)];
} else if (key) {
param = key;
paramArr = [key];
}
if (param === '') {
throw new Error('Verification can only be retrieved if the ID is known');
if (!paramArr.length) {
// there is no API endpoint to retrieve all dataVerification items, so we need to retrieve all automations and iterate over their activities
Util.logger.info(` - Caching dependent Metadata: automation`);
Automation.client = this.client;
Automation.buObject = this.buObject;
Automation.properties = this.properties;
Automation._skipNotificationRetrieve = true;
const automationsMapObj = await Automation.retrieve();
delete Automation._skipNotificationRetrieve;
if (automationsMapObj?.metadata && Object.keys(automationsMapObj?.metadata).length) {
if (!key) {
// if we are not retrieving a single item, cache the automations for later use during retrieval of automations
Automation._cachedMetadataMap = automationsMapObj?.metadata;
}
// automations found, lets iterate over their activities to find the dataVerification items
const dataVerificationIds = [];
for (const automation of Object.values(automationsMapObj.metadata)) {
if (automation.steps) {
for (const step of automation.steps) {
for (const activity of step.activities) {
if (
activity.objectTypeId === 1000 &&
activity.activityObjectId &&
activity.activityObjectId !==
'00000000-0000-0000-0000-000000000000'
) {
dataVerificationIds.push(activity.activityObjectId);
}
}
}
}
}
if (dataVerificationIds.length) {
paramArr.push(...dataVerificationIds);
}
}
}
try {
return await super.retrieveREST(
null,
'/automation/v1/dataverifications/' + param,
null,
key
);
} catch (ex) {
if (
ex.message === 'Not Found' ||
ex.message === 'Request failed with status code 400'
) {
if (retrieveDir) {
Util.logger.info(
`Downloaded: ${this.definition.type} (0)${Util.getKeysString(param)}`
const results = {};
if (paramArr.length) {
if (key) {
try {
const verification = await super.retrieveREST(
null,
'/automation/v1/dataverifications/' + paramArr[0],
null,
key
);
const key = Object.values(verification?.metadata)[0]?.[
this.definition.keyField
];
results[key] = verification?.metadata[key];
} catch (ex) {
if (
ex.message === 'Not Found' ||
ex.message === 'Request failed with status code 400'
) {
// if the ID is too short, the system will throw the 400 error
} else {
throw ex;
}
}
// if the ID is too short, the system will throw the 400 error
return { metadata: {} };
} else {
throw ex;
const uri = '/automation/v1/dataverifications/';

const verificationArr = await this.client.rest.getCollection(
paramArr.map((id) => uri + id)
);
for (const verification of verificationArr) {
const key = verification[this.definition.keyField];
results[key] = verification;
}
}
}
if (retrieveDir) {
const savedMetadata = await this.saveResults(results, retrieveDir, null, null);
Util.logger.info(
`Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
Util.getKeysString(key)
);
}

return {
metadata: results,
type: this.definition.type,
};
}
/**
* Retrieves Metadata of Data Extract Activity for caching
Expand Down
3 changes: 1 addition & 2 deletions lib/metadataTypes/definitions/Automation.definition.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,15 @@ module.exports = {
importMobileContact: 726,
},
bodyIteratorField: 'items',
manuallyRetrievedDependencies: ['verification'],
dependencies: [
'dataExtension', // for verification
'dataExtract',
'emailSend',
'fileTransfer',
'folder-automations',
'importFile',
'query',
'script',
'verification',
],
folderType: 'automation',
hasExtended: false,
Expand Down

0 comments on commit 27504e1

Please sign in to comment.