diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 44cddb11c..82eedc210 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -495,8 +495,9 @@ main class * [.buildDefinition(businessUnit, selectedType, name, market)](#Mcdev.buildDefinition) ⇒ Promise.<void> * [.buildDefinitionBulk(listName, type, name)](#Mcdev.buildDefinitionBulk) ⇒ Promise.<void> * [.getFilesToCommit(businessUnit, selectedType, keyArr)](#Mcdev.getFilesToCommit) ⇒ Promise.<Array.<string>> - * [.execute(businessUnit, [selectedTypesArr], keys)](#Mcdev.execute) ⇒ Promise.<boolean> - * [._executeBU(cred, bu, [selectedTypesArr], keyArr)](#Mcdev._executeBU) ⇒ Promise.<boolean> + * [.execute(businessUnit, [selectedType], [keys])](#Mcdev.execute) ⇒ Promise.<boolean> + * [._executeBU(cred, bu, [type], keyArr)](#Mcdev._executeBU) ⇒ Promise.<boolean> + * [._retrieveKeysWithLike(selectedType, buObject)](#Mcdev._retrieveKeysWithLike) ⇒ Array.<string> @@ -751,7 +752,7 @@ Build a specific metadata file based on a template using a list of bu-market com -### Mcdev.execute(businessUnit, [selectedTypesArr], keys) ⇒ Promise.<boolean> +### Mcdev.execute(businessUnit, [selectedType], [keys]) ⇒ Promise.<boolean> Start an item (query) **Kind**: static method of [Mcdev](#Mcdev) @@ -760,13 +761,13 @@ Start an item (query) | Param | Type | Description | | --- | --- | --- | | businessUnit | string | name of BU | -| [selectedTypesArr] | Array.<TYPE.SupportedMetadataTypes> | limit to given metadata types | -| keys | Array.<string> | customerkey of the metadata | +| [selectedType] | TYPE.SupportedMetadataTypes | limit to given metadata types | +| [keys] | Array.<string> | customerkey of the metadata | -### Mcdev.\_executeBU(cred, bu, [selectedTypesArr], keyArr) ⇒ Promise.<boolean> -helper for [execute](execute) +### Mcdev.\_executeBU(cred, bu, [type], keyArr) ⇒ Promise.<boolean> +helper for [execute](#Mcdev.execute) **Kind**: static method of [Mcdev](#Mcdev) **Returns**: Promise.<boolean> - true if all items were executed, false otherwise @@ -775,9 +776,22 @@ helper for [execute](execute) | --- | --- | --- | | cred | string | name of Credential | | bu | string | name of BU | -| [selectedTypesArr] | Array.<TYPE.SupportedMetadataTypes> | limit execution to given metadata type | +| [type] | TYPE.SupportedMetadataTypes | limit execution to given metadata type | | keyArr | Array.<string> | customerkey of the metadata | + + +### Mcdev.\_retrieveKeysWithLike(selectedType, buObject) ⇒ Array.<string> +helper for [_executeBU](#Mcdev._executeBU) + +**Kind**: static method of [Mcdev](#Mcdev) +**Returns**: Array.<string> - keyArr + +| Param | Type | Description | +| --- | --- | --- | +| selectedType | TYPE.SupportedMetadataTypes | limit execution to given metadata type | +| buObject | TYPE.BuObject | properties for auth | + ## Asset ⇐ [MetadataType](#MetadataType) @@ -5892,6 +5906,8 @@ CLI entry for SFMC DevTools * [.getKeysString(keyArr, [isId])](#Util.getKeysString) ⇒ string * [.sleep(ms)](#Util.sleep) ⇒ Promise.<void> * [.getSsjs(code)](#Util.getSsjs) ⇒ string + * [.stringLike(testString, search)](#Util.stringLike) ⇒ boolean + * [.fieldsLike(metadata, [filters])](#Util.fieldsLike) ⇒ boolean @@ -6248,6 +6264,32 @@ the following is invalid: // 3 ``` + + +### Util.stringLike(testString, search) ⇒ boolean +allows us to filter just like with SQL's LIKE operator + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if testString matches search + +| Param | Type | Description | +| --- | --- | --- | +| testString | string | field value to test | +| search | string | search string in SQL LIKE format | + + + +### Util.fieldsLike(metadata, [filters]) ⇒ boolean +returns true if no LIKE filter is defined or if all filters match + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if no LIKE filter is defined or if all filters match + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.MetadataTypeItem | a single metadata item | +| [filters] | object | only used in recursive calls | + ## MetadataTypeDefinitions @@ -7782,6 +7824,8 @@ Util that contains logger and simple util methods * [.getKeysString(keyArr, [isId])](#Util.getKeysString) ⇒ string * [.sleep(ms)](#Util.sleep) ⇒ Promise.<void> * [.getSsjs(code)](#Util.getSsjs) ⇒ string + * [.stringLike(testString, search)](#Util.stringLike) ⇒ boolean + * [.fieldsLike(metadata, [filters])](#Util.fieldsLike) ⇒ boolean @@ -8138,6 +8182,32 @@ the following is invalid: // 3 ``` + + +### Util.stringLike(testString, search) ⇒ boolean +allows us to filter just like with SQL's LIKE operator + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if testString matches search + +| Param | Type | Description | +| --- | --- | --- | +| testString | string | field value to test | +| search | string | search string in SQL LIKE format | + + + +### Util.fieldsLike(metadata, [filters]) ⇒ boolean +returns true if no LIKE filter is defined or if all filters match + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if no LIKE filter is defined or if all filters match + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.MetadataTypeItem | a single metadata item | +| [filters] | object | only used in recursive calls | + ## csvToArray(csv) ⇒ Array.<string> diff --git a/lib/cli.js b/lib/cli.js index d60988992..7a7e5ea89 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -31,6 +31,12 @@ yargs .positional('KEY', { type: 'string', describe: 'metadata keys that shall be exclusively downloaded', + }) + .option('like', { + type: 'string', + group: 'Options for retrieve:', + describe: + 'filter metadata components (can include % as wildcard or _ for a single character)', }); }, handler: (argv) => { @@ -57,31 +63,32 @@ yargs type: 'string', describe: 'metadata key that shall be exclusively uploaded', }) - .group( - ['changeKeyField', 'changeKeyValue', 'fromRetrieve', 'refresh', 'execute'], - 'Options for deploy:' - ) .option('changeKeyField', { type: 'string', + group: 'Options for deploy:', describe: 'enables updating the key of the deployed metadata with the value in provided field (e.g. c__newKey). Can be used to sync name and key fields.', }) .option('changeKeyValue', { type: 'string', + group: 'Options for deploy:', describe: 'allows updating the key of the metadata to the provided value. Only available if a single type and key is deployed', }) .option('fromRetrieve', { type: 'boolean', + group: 'Options for deploy:', describe: 'optionally deploy from retrieve folder', }) .option('refresh', { type: 'boolean', + group: 'Options for deploy:', describe: 'optional for asset-message: runs refresh command for related triggeredSends after deploy', }) .option('execute', { type: 'boolean', + group: 'Options for deploy:', describe: 'optional for query: runs execute after deploy', }); }, @@ -313,8 +320,9 @@ yargs aliases: ['et'], desc: 'explains metadata types that can be retrieved', builder: (yargs) => { - yargs.group(['json'], 'Options for explainTypes:').option('json', { + yargs.option('json', { type: 'boolean', + group: 'Options for explainTypes:', describe: 'optionaly return info in json format', }); }, @@ -333,14 +341,15 @@ yargs type: 'string', describe: 'Pull Request target branch or git commit range', }) - .group(['filter', 'commitHistory'], 'Options for createDeltaPkg:') .option('filter', { type: 'string', + group: 'Options for createDeltaPkg:', describe: 'Disable templating & instead filter by the specified BU path (comma separated), can include subtype, will be prefixed with "retrieve/"', }) .option('commitHistory', { type: 'number', + group: 'Options for createDeltaPkg:', describe: 'Number of commits to look back for changes (supersedes config)', }); }, @@ -399,8 +408,8 @@ yargs }, }) .command({ - command: 'execute ', - aliases: ['exec'], + command: 'execute [KEY]', + aliases: ['exec', 'start'], desc: 'executes the entity (query/journey/automation etc.)', builder: (yargs) => { yargs @@ -415,11 +424,18 @@ yargs .positional('KEY', { type: 'string', describe: 'key(s) of the metadata component(s)', + }) + .option('like', { + type: 'string', + group: 'Options for execute:', + describe: + 'filter metadata components (can include % as wildcard or _ for a single character)', }); }, handler: (argv) => { Mcdev.setOptions(argv); - Mcdev.execute(argv.BU, csvToArray(argv.TYPE), csvToArray(argv.KEY)); + // ! do not allow multiple types to be passed in here via csvToArray + Mcdev.execute(argv.BU, argv.TYPE, csvToArray(argv.KEY)); }, }) .command({ diff --git a/lib/index.js b/lib/index.js index 968be4e7d..5928a854e 100644 --- a/lib/index.js +++ b/lib/index.js @@ -51,16 +51,17 @@ class Mcdev { static setOptions(argv) { const knownOptions = [ 'api', - 'commitHistory', 'changeKeyField', 'changeKeyValue', + 'commitHistory', 'filter', 'fromRetrieve', 'json', + 'like', + 'noLogFile', 'refresh', 'execute', 'skipInteraction', - 'noLogFile', ]; for (const option of knownOptions) { if (argv[option] !== undefined) { @@ -236,7 +237,7 @@ class Mcdev { } } /** - * helper for {@link retrieve} + * helper for {@link Mcdev.retrieve} * * @private * @param {string} cred name of Credential @@ -705,29 +706,43 @@ class Mcdev { * Start an item (query) * * @param {string} businessUnit name of BU - * @param {TYPE.SupportedMetadataTypes[]} [selectedTypesArr] limit to given metadata types - * @param {string[]} keys customerkey of the metadata + * @param {TYPE.SupportedMetadataTypes} [selectedType] limit to given metadata types + * @param {string[]} [keys] customerkey of the metadata * @returns {Promise.} true if all started successfully, false if not */ - static async execute(businessUnit, selectedTypesArr, keys) { + static async execute(businessUnit, selectedType, keys) { Util.startLogger(); - Util.logger.info('mcdev:: Execute'); + Util.logger.info('mcdev:: Executing ' + selectedType); const properties = await config.getProperties(); let counter_credBu = 0; let counter_failed = 0; if (!(await config.checkProperties(properties))) { // return null here to avoid seeing 2 error messages for the same issue - return null; + return false; } - if (Array.isArray(selectedTypesArr)) { - // types and keys can be provided but for each type all provided keys are applied as filter - for (const selectedType of Array.isArray(selectedTypesArr) - ? selectedTypesArr - : Object.keys(selectedTypesArr)) { - if (!Util._isValidType(selectedType)) { - return; - } - } + if (!Util._isValidType(selectedType)) { + return false; + } + if (!Object.prototype.hasOwnProperty.call(MetadataTypeInfo[selectedType], 'execute')) { + Util.logger.error( + ` ☇ skipping ${selectedType}: execute is not supported yet for ${selectedType}` + ); + return false; + } + if ( + (!Array.isArray(keys) || !keys.length) && + (!Util.OPTIONS.like || !Object.keys(Util.OPTIONS.like).length) + ) { + Util.logger.error('At least one key or a --like filter is required.'); + return false; + } else if ( + Array.isArray(keys) && + keys.length && + Util.OPTIONS.like && + Object.keys(Util.OPTIONS.like).length + ) { + Util.logger.error('You can either specify keys OR a --like filter.'); + return false; } if (businessUnit === '*') { Util.logger.info(':: Executing the entity on all BUs for all credentials'); @@ -736,7 +751,7 @@ class Mcdev { Util.logger.info(`:: Executing the entity on all BUs for ${cred}`); for (const bu in properties.credentials[cred].businessUnits) { - if (await this._executeBU(cred, bu, selectedTypesArr, keys)) { + if (await this._executeBU(cred, bu, selectedType, keys)) { counter_credBu++; } else { counter_failed++; @@ -762,7 +777,7 @@ class Mcdev { true ); if (buObject === null) { - return; + return false; } else { cred = buObject.credential; bu = buObject.businessUnit; @@ -772,7 +787,7 @@ class Mcdev { Util.logger.info(`\n :: Executing the entity on all BUs for ${cred}`); let counter_credBu = 0; for (const bu in properties.credentials[cred].businessUnits) { - if (await this._executeBU(cred, bu, selectedTypesArr, keys)) { + if (await this._executeBU(cred, bu, selectedType, keys)) { counter_credBu++; } else { counter_failed++; @@ -784,7 +799,7 @@ class Mcdev { ); } else { // execute the entity on one BU only - if (await this._executeBU(cred, bu, selectedTypesArr, keys)) { + if (await this._executeBU(cred, bu, selectedType, keys)) { counter_credBu++; } else { counter_failed++; @@ -798,15 +813,15 @@ class Mcdev { return counter_failed === 0 ? true : false; } /** - * helper for {@link execute} + * helper for {@link Mcdev.execute} * * @param {string} cred name of Credential * @param {string} bu name of BU - * @param {TYPE.SupportedMetadataTypes[]} [selectedTypesArr] limit execution to given metadata type + * @param {TYPE.SupportedMetadataTypes} [type] limit execution to given metadata type * @param {string[]} keyArr customerkey of the metadata * @returns {Promise.} true if all items were executed, false otherwise */ - static async _executeBU(cred, bu, selectedTypesArr, keyArr) { + static async _executeBU(cred, bu, type, keyArr) { const properties = await config.getProperties(); let counter_failed = 0; const buObject = await Cli.getCredentialObject( @@ -815,38 +830,98 @@ class Mcdev { null, true ); - if (!keyArr || (Array.isArray(keyArr) && !keyArr.length)) { - throw new Error('No keys were provided'); + try { + if (!type) { + throw new Error('No type was provided'); + } + if (buObject !== null) { + cache.initCache(buObject); + cred = buObject.credential; + bu = buObject.businessUnit; + } + Util.logger.info(`\n :: Executing ${type} on ${cred}/${bu}\n`); + MetadataTypeInfo[type].client = auth.getSDK(buObject); + if (Util.OPTIONS.like && Object.keys(Util.OPTIONS.like).length) { + keyArr = await this._retrieveKeysWithLike(type, buObject); + } + if (!keyArr || (Array.isArray(keyArr) && !keyArr.length)) { + throw new Error('No keys were provided'); + } + + // result will be undefined (false) if execute is not supported for the type + if (!(await MetadataTypeInfo[type].execute(keyArr))) { + counter_failed++; + } + } catch (ex) { + Util.logger.errorStack(ex, 'mcdev.execute failed'); } - if (!selectedTypesArr || (Array.isArray(selectedTypesArr) && !selectedTypesArr.length)) { - throw new Error('No type was provided'); + + return counter_failed === 0 ? true : false; + } + + /** + * helper for {@link Mcdev._executeBU} + * + * @param {TYPE.SupportedMetadataTypes} selectedType limit execution to given metadata type + * @param {TYPE.BuObject} buObject properties for auth + * @returns {string[]} keyArr + */ + static async _retrieveKeysWithLike(selectedType, buObject) { + const properties = await config.getProperties(); + + // cache depenencies + const deployOrder = Util.getMetadataHierachy([selectedType]); + for (const type in deployOrder) { + const subTypeArr = deployOrder[type]; + MetadataTypeInfo[type].client = auth.getSDK(buObject); + MetadataTypeInfo[type].properties = properties; + MetadataTypeInfo[type].buObject = buObject; + Util.logger.info(`Caching dependent Metadata: ${type}`); + Util.logSubtypes(subTypeArr); + const result = await MetadataTypeInfo[type].retrieveForCache(null, subTypeArr); + if (result) { + if (Array.isArray(result)) { + for (const result_i of result) { + if (result_i?.metadata && Object.keys(result_i.metadata).length) { + cache.mergeMetadata(type, result_i.metadata); + } + } + } else { + cache.setMetadata(type, result.metadata); + } + } } - if (buObject !== null) { - cache.initCache(buObject); - cred = buObject.credential; - bu = buObject.businessUnit; + + // find all keys in chosen type that match the like-filter + const keyArr = []; + const metadataMap = cache.getCache()[selectedType]; + if (!metadataMap) { + throw new Error(`Selected type ${selectedType} could not be cached`); } Util.logger.info( - `\n :: Executing ${selectedTypesArr.join(', ')} on ${cred}/${bu}\n` + Util.getGrayMsg(`Found ${Object.keys(metadataMap).length} ${selectedType}s`) ); - try { - // more than one type was provided, iterate types and execute items - for (const type of selectedTypesArr) { - try { - MetadataTypeInfo[type].client = auth.getSDK(buObject); - } catch (ex) { - Util.logger.error(ex.message); - return; - } - // result will be undefined (false) if execute is not supported for the type - if (!(await MetadataTypeInfo[type].execute(keyArr))) { - counter_failed++; - } + for (const originalKey in metadataMap) { + // hide postRetrieveOutput + Util.setLoggingLevel({ silent: true }); + metadataMap[originalKey] = MetadataTypeInfo[selectedType].postRetrieveTasks( + metadataMap[originalKey] + ); + // reactivate logging + Util.setLoggingLevel({}); + if (Util.fieldsLike(metadataMap[originalKey])) { + keyArr.push(originalKey); } - } catch (ex) { - Util.logger.errorStack(ex, 'mcdev.execute failed'); } - return counter_failed === 0 ? true : false; + Util.logger.info( + Util.getGrayMsg( + `Identified ${keyArr.length} ${selectedType}${ + keyArr.length === 1 ? '' : 's' + } that match${selectedType}${keyArr.length === 1 ? 'es' : ''} the like-filter` + ) + ); + + return keyArr; } } diff --git a/lib/metadataTypes/MetadataType.js b/lib/metadataTypes/MetadataType.js index a5e55bcfa..09533b907 100644 --- a/lib/metadataTypes/MetadataType.js +++ b/lib/metadataTypes/MetadataType.js @@ -1570,6 +1570,10 @@ class MetadataType { } } + if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey])) { + Util.logger.debug(`Filtered ${originalKey} because of --like option`); + continue; + } // we dont store Id on local disk, but we need it for caching logic, // so its in retrieve but not in save. Here we put into the clone so that the original // object used for caching doesnt have the Id removed. diff --git a/lib/util/util.js b/lib/util/util.js index 173e88cbd..e552fe382 100644 --- a/lib/util/util.js +++ b/lib/util/util.js @@ -780,6 +780,60 @@ const Util = { // no script found return null; }, + /** + * allows us to filter just like with SQL's LIKE operator + * + * @param {string} testString field value to test + * @param {string} search search string in SQL LIKE format + * @returns {boolean} true if testString matches search + */ + stringLike(testString, search) { + if (typeof search !== 'string' || this === null) { + return false; + } + // Remove special chars + search = search.replaceAll( + new RegExp('([\\.\\\\\\+\\*\\?\\[\\^\\]\\$\\(\\)\\{\\}\\=\\!\\<\\>\\|\\:\\-])', 'g'), + '\\$1' + ); + // Replace % and _ with equivalent regex + search = search.replaceAll('%', '.*').replaceAll('_', '.'); + // Check matches + return new RegExp('^' + search + '$', 'gi').test(testString); + }, + /** + * returns true if no LIKE filter is defined or if all filters match + * + * @param {TYPE.MetadataTypeItem} metadata a single metadata item + * @param {object} [filters] only used in recursive calls + * @returns {boolean} true if no LIKE filter is defined or if all filters match + */ + fieldsLike(metadata, filters) { + if (metadata.json && metadata.codeArr) { + // Compensate for CodeExtractItem format + metadata = metadata.json; + } + filters ||= Util.OPTIONS.like; + if (!filters) { + return true; + } + const fields = Object.keys(filters); + return fields.every((field) => { + const filter = filters[field]; + if (Array.isArray(metadata[field])) { + return metadata[field].some((f) => Util.fieldsLike(f, filter)); + } else { + if (typeof filter === 'string') { + return Util.stringLike(metadata[field], filter); + } else if (Array.isArray(filter)) { + return filter.some((f) => Util.stringLike(metadata[field], f)); + } else if (typeof filter === 'object') { + return Util.fieldsLike(metadata[field], filter); + } + } + return false; + }); + }, }; Util.startLogger(false, true); diff --git a/test/type.query.test.js b/test/type.query.test.js index e174cfdd1..b82df7cc4 100644 --- a/test/type.query.test.js +++ b/test/type.query.test.js @@ -53,7 +53,7 @@ describe('type: query', () => { ); return; }); - it('Should retrieve one specific query', async () => { + it('Should retrieve one specific query by key', async () => { // WHEN await handler.retrieve('testInstance/testBU', ['query'], ['testExisting_query']); // THEN @@ -80,6 +80,61 @@ describe('type: query', () => { ); return; }); + it('Should retrieve one specific query via --like', async () => { + // WHEN + handler.setOptions({ like: { key: '%Existing_query' } }); + await handler.retrieve('testInstance/testBU', ['query']); + // THEN + assert.equal(process.exitCode, false, 'retrieve should not have thrown an error'); + // get results from cache + const result = cache.getCache(); + assert.equal( + result.query ? Object.keys(result.query).length : 0, + 2, + 'two queries in cache expected' + ); + assert.deepEqual( + await testUtils.getActualJson('testExisting_query', 'query'), + await testUtils.getExpectedJson('9999999', 'query', 'get'), + 'returned metadata was not equal expected' + ); + expect(file(testUtils.getActualFile('testExisting_query', 'query', 'sql'))).to.equal( + file(testUtils.getExpectedFile('9999999', 'query', 'get', 'sql')) + ); + expect(file(testUtils.getActualFile('testExisting_query2', 'query', 'sql'))).to.not + .exist; + assert.equal( + testUtils.getAPIHistoryLength(), + 6, + 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' + ); + return; + }); + it('Should not retrieve any query via --like and key due to a mismatching filter', async () => { + // WHEN + handler.setOptions({ like: { key: 'NotExisting_query' } }); + await handler.retrieve('testInstance/testBU', ['query']); + // THEN + assert.equal(process.exitCode, false, 'retrieve should not have thrown an error'); + // get results from cache + const result = cache.getCache(); + assert.equal( + result.query ? Object.keys(result.query).length : 0, + 2, + 'two queries in cache expected' + ); + + expect(file(testUtils.getActualFile('testExisting_query', 'query', 'sql'))).to.not + .exist; + expect(file(testUtils.getActualFile('testExisting_query2', 'query', 'sql'))).to.not + .exist; + assert.equal( + testUtils.getAPIHistoryLength(), + 6, + 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' + ); + return; + }); }); describe('Deploy ================', () => { beforeEach(() => { @@ -306,15 +361,29 @@ describe('type: query', () => { }); }); describe('Execute ================', () => { - it('Should start executing a query', async () => { - const execute = await handler.execute( - 'testInstance/testBU', - ['query'], - ['testExisting_query'] - ); + it('Should start a query by key', async () => { + const execute = await handler.execute('testInstance/testBU', 'query', [ + 'testExisting_query', + ]); assert.equal(process.exitCode, false, 'execute should not have thrown an error'); assert.equal(execute, true, 'query was supposed to be executed'); return; }); + it('Should start a query selected via --like', async () => { + handler.setOptions({ like: { key: 'testExist%query' } }); + const execute = await handler.execute('testInstance/testBU', 'query'); + assert.equal(process.exitCode, false, 'execute should not have thrown an error'); + assert.equal(execute, true, 'query was supposed to be executed'); + return; + }); + it('Should not start executing a query because key and --like was specified', async () => { + handler.setOptions({ like: { key: 'testExisting%' } }); + const execute = await handler.execute('testInstance/testBU', 'query', [ + 'testExisting_query', + ]); + assert.equal(process.exitCode, true, 'execute should not have thrown an error'); + assert.equal(execute, false, 'query was not supposed to be executed'); + return; + }); }); });