From 7096dc548d3c386540d9db07025eb5cf6d491462 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 14:52:49 -0800 Subject: [PATCH 01/35] Make parallelism more visible --- apps/heft/src/plugins/DeleteGlobsPlugin.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/heft/src/plugins/DeleteGlobsPlugin.ts b/apps/heft/src/plugins/DeleteGlobsPlugin.ts index fdbfa6aa10d..941167de52f 100644 --- a/apps/heft/src/plugins/DeleteGlobsPlugin.ts +++ b/apps/heft/src/plugins/DeleteGlobsPlugin.ts @@ -29,6 +29,8 @@ const HEFT_STAGE_TAP: TapOptions<'promise'> = { stage: Number.MIN_SAFE_INTEGER }; +const MAX_PARALLELISM: number = 100; + export class DeleteGlobsPlugin implements IHeftPlugin { public readonly pluginName: string = PLUGIN_NAME; @@ -99,7 +101,7 @@ export class DeleteGlobsPlugin implements IHeftPlugin { } } - await Async.forEachLimitAsync(Array.from(pathsToDelete), 100, async (pathToDelete) => { + await Async.forEachLimitAsync(Array.from(pathsToDelete), MAX_PARALLELISM, async (pathToDelete) => { try { FileSystem.deleteFile(pathToDelete, { throwIfNotExists: true }); logger.terminal.writeVerboseLine(`Deleted "${pathToDelete}"`); From aa726ce2fa5d870609fbacd48870b8123e7dcaab Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 14:53:08 -0800 Subject: [PATCH 02/35] First implementation of CopyGlobs plugin --- .../heft/src/pluginFramework/PluginManager.ts | 2 + apps/heft/src/plugins/CopyGlobsPlugin.ts | 168 ++++++++++++++++++ apps/heft/src/schemas/heft.schema.json | 42 ++++- apps/heft/src/utilities/CoreConfigFiles.ts | 17 ++ 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 apps/heft/src/plugins/CopyGlobsPlugin.ts diff --git a/apps/heft/src/pluginFramework/PluginManager.ts b/apps/heft/src/pluginFramework/PluginManager.ts index 11ad4cedf18..83f26503b79 100644 --- a/apps/heft/src/pluginFramework/PluginManager.ts +++ b/apps/heft/src/pluginFramework/PluginManager.ts @@ -14,6 +14,7 @@ import { } from '../utilities/CoreConfigFiles'; // Default plugins +import { CopyGlobsPlugin } from '../plugins/CopyGlobsPlugin'; import { TypeScriptPlugin } from '../plugins/TypeScriptPlugin/TypeScriptPlugin'; import { DeleteGlobsPlugin } from '../plugins/DeleteGlobsPlugin'; import { CopyStaticAssetsPlugin } from '../plugins/CopyStaticAssetsPlugin'; @@ -46,6 +47,7 @@ export class PluginManager { public initializeDefaultPlugins(): void { this._applyPlugin(new TypeScriptPlugin()); this._applyPlugin(new CopyStaticAssetsPlugin()); + this._applyPlugin(new CopyGlobsPlugin()); this._applyPlugin(new DeleteGlobsPlugin()); this._applyPlugin(new ApiExtractorPlugin()); this._applyPlugin(new JestPlugin()); diff --git a/apps/heft/src/plugins/CopyGlobsPlugin.ts b/apps/heft/src/plugins/CopyGlobsPlugin.ts new file mode 100644 index 00000000000..8940dd890d9 --- /dev/null +++ b/apps/heft/src/plugins/CopyGlobsPlugin.ts @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'path'; +import glob from 'glob'; +import { AlreadyExistsBehavior, FileSystem, LegacyAdapters } from '@rushstack/node-core-library'; +import { TapOptions } from 'tapable'; + +import { IHeftPlugin } from '../pluginFramework/IHeftPlugin'; +import { HeftSession } from '../pluginFramework/HeftSession'; +import { HeftConfiguration } from '../configuration/HeftConfiguration'; +import { ScopedLogger } from '../pluginFramework/logging/ScopedLogger'; +import { IHeftEventActions, CoreConfigFiles, HeftEvent } from '../utilities/CoreConfigFiles'; +import { Async } from '../utilities/Async'; +import { + IBuildStageContext, + IBundleSubstage, + ICompileSubstage, + IPostBuildSubstage, + IPreCompileSubstage +} from '../stages/BuildStage'; + +const globEscape: (unescaped: string) => string = require('glob-escape'); // No @types/glob-escape package exists + +const PLUGIN_NAME: string = 'CopyFilesPlugin'; +const HEFT_STAGE_TAP: TapOptions<'promise'> = { + name: PLUGIN_NAME, + stage: Number.MAX_SAFE_INTEGER / 2 // This should give us some certainty that this will run after other plugins +}; + +const MAX_PARALLELISM: number = 100; + +export class CopyGlobsPlugin implements IHeftPlugin { + public readonly pluginName: string = PLUGIN_NAME; + + public apply(heftSession: HeftSession, heftConfiguration: HeftConfiguration): void { + const logger: ScopedLogger = heftSession.requestScopedLogger('copy-files'); + heftSession.hooks.build.tap(PLUGIN_NAME, (build: IBuildStageContext) => { + build.hooks.preCompile.tap(PLUGIN_NAME, (preCompile: IPreCompileSubstage) => { + preCompile.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { + await this._runCopyFilesForHeftEvent(HeftEvent.preCompile, logger, heftConfiguration); + }); + }); + + build.hooks.compile.tap(PLUGIN_NAME, (compile: ICompileSubstage) => { + compile.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { + await this._runCopyFilesForHeftEvent(HeftEvent.compile, logger, heftConfiguration); + }); + }); + + build.hooks.bundle.tap(PLUGIN_NAME, (bundle: IBundleSubstage) => { + bundle.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { + await this._runCopyFilesForHeftEvent(HeftEvent.bundle, logger, heftConfiguration); + }); + }); + + build.hooks.postBuild.tap(PLUGIN_NAME, (postBuild: IPostBuildSubstage) => { + postBuild.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { + await this._runCopyFilesForHeftEvent(HeftEvent.postBuild, logger, heftConfiguration); + }); + }); + }); + } + + private async _runCopyFilesForHeftEvent( + heftEvent: HeftEvent, + logger: ScopedLogger, + heftConfiguration: HeftConfiguration + ): Promise { + const eventActions: IHeftEventActions = await CoreConfigFiles.getConfigConfigFileEventActionsAsync( + logger.terminal, + heftConfiguration + ); + + // Build a map to dedupe copy operations + const fileOperationMap: Map> = new Map>(); + for (const copyFilesEventAction of eventActions.copyGlobs.get(heftEvent) || []) { + for (const globPattern of copyFilesEventAction.globsToCopy) { + const resolvedSourceFilePaths: string[] = await this._resolvePathAsync( + globPattern, + heftConfiguration.buildFolder + ); + for (const resolvedSourceFilePath of resolvedSourceFilePaths) { + let resolvedTargetPathsMap: Map | undefined = fileOperationMap.get( + resolvedSourceFilePath + ); + if (!resolvedTargetPathsMap) { + resolvedTargetPathsMap = new Map(); + fileOperationMap.set(resolvedSourceFilePath, resolvedTargetPathsMap); + } + + for (const targetFolder of copyFilesEventAction.targetFolders) { + resolvedTargetPathsMap.set( + path.resolve(heftConfiguration.buildFolder, targetFolder), + copyFilesEventAction.hardlink || false + ); + } + } + } + } + + // Flatten out the map to simplify processing + const flattenedOperationMap: [string, string, boolean][] = []; + for (const [sourceFilePath, destinationMap] of fileOperationMap.entries()) { + for (const [destinationFilePath, hardlink] of destinationMap.entries()) { + flattenedOperationMap.push([sourceFilePath, destinationFilePath, hardlink]); + } + } + + let linkedFiles: number = 0; + let copiedFiles: number = 0; + await Async.forEachLimitAsync( + flattenedOperationMap, + MAX_PARALLELISM, + async ([sourceFilePath, targetFilePath, hardlink]) => { + if (hardlink) { + // Hardlink doesn't allow passing in overwrite param, so delete ourselves + try { + await FileSystem.deleteFileAsync(targetFilePath); + } catch (e) { + if (!FileSystem.isFileDoesNotExistError(e)) { + throw e; + } + } + + await FileSystem.createHardLinkAsync({ + linkTargetPath: sourceFilePath, + newLinkPath: targetFilePath + }); + logger.terminal.writeVerboseLine(`Linked "${sourceFilePath}" to "${targetFilePath}"`); + linkedFiles++; + } else { + await FileSystem.copyFileAsync({ + sourcePath: sourceFilePath, + destinationPath: targetFilePath, + alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite + }); + logger.terminal.writeVerboseLine(`Copied "${sourceFilePath}" to "${targetFilePath}"`); + copiedFiles++; + } + } + ); + + if (linkedFiles > 0) { + logger.terminal.writeLine(`Linked ${linkedFiles} files`); + } + if (copiedFiles > 0) { + logger.terminal.writeLine(`Copied ${copiedFiles} files`); + } + } + + private async _resolvePathAsync(globPattern: string, buildFolder: string): Promise { + if (globEscape(globPattern) !== globPattern) { + const expandedGlob: string[] = await LegacyAdapters.convertCallbackToPromise(glob, globPattern, { + cwd: buildFolder + }); + + const result: string[] = []; + for (const pathFromGlob of expandedGlob) { + result.push(path.resolve(buildFolder, pathFromGlob)); + } + + return result; + } else { + return [path.resolve(buildFolder, globPattern)]; + } + } +} diff --git a/apps/heft/src/schemas/heft.schema.json b/apps/heft/src/schemas/heft.schema.json index 6f8cf9a089c..691898b6c12 100644 --- a/apps/heft/src/schemas/heft.schema.json +++ b/apps/heft/src/schemas/heft.schema.json @@ -30,7 +30,7 @@ "actionKind": { "type": "string", "description": "The kind of built-in operation that should be performed.", - "enum": ["deleteGlobs"] + "enum": ["deleteGlobs", "copyGlobs"] }, "heftEvent": { @@ -49,7 +49,7 @@ "oneOf": [ // Delete Globs { - "required": ["actionKind"], + "required": ["actionKind", "globsToDelete"], "properties": { "actionKind": { "type": "string", @@ -65,6 +65,44 @@ } } } + }, + // Copy Files + { + "required": ["actionKind", "globsToCopy", "targetFolders"], + "properties": { + "actionKind": { + "type": "string", + "enum": ["copyGlobs"] + }, + + "heftEvent": { + "type": "string", + "enum": ["pre-compile", "compile", "bundle", "post-build"] + }, + + "globsToCopy": { + "type": "array", + "description": "Glob patterns to be copied. The paths are resolved relative to the project folder.", + "items": { + "type": "string", + "pattern": "[^\\\\]" + } + }, + + "targetFolders": { + "type": "array", + "description": "Destination folders for the files to be copied. The paths are resolved relative to the project folder.", + "items": { + "type": "string", + "pattern": "[^\\\\]" + } + }, + + "hardlink": { + "type": "boolean", + "description": "Whether to copy or hardlink the files." + } + } } ] } diff --git a/apps/heft/src/utilities/CoreConfigFiles.ts b/apps/heft/src/utilities/CoreConfigFiles.ts index 2374a3727c5..975726f88d3 100644 --- a/apps/heft/src/utilities/CoreConfigFiles.ts +++ b/apps/heft/src/utilities/CoreConfigFiles.ts @@ -35,6 +35,13 @@ export interface IHeftConfigurationDeleteGlobsEventAction extends IHeftConfigura globsToDelete: string[]; } +export interface IHeftConfigurationCopyGlobsEventAction extends IHeftConfigurationJsonEventActionBase { + actionKind: 'copyGlobs'; + globsToCopy: string[]; + targetFolders: string[]; + hardlink?: boolean; +} + export interface IHeftConfigurationJsonPluginSpecifier { plugin: string; options?: object; @@ -46,6 +53,7 @@ export interface IHeftConfigurationJson { } export interface IHeftEventActions { + copyGlobs: Map; deleteGlobs: Map; } @@ -110,12 +118,21 @@ export class CoreConfigFiles { ); result = { + copyGlobs: new Map(), deleteGlobs: new Map() }; CoreConfigFiles._heftConfigFileEventActionsCache.set(heftConfiguration, result); for (const eventAction of heftConfigJson?.eventActions || []) { switch (eventAction.actionKind) { + case 'copyGlobs': { + CoreConfigFiles._addEventActionToMap( + eventAction as IHeftConfigurationCopyGlobsEventAction, + result.copyGlobs + ); + break; + } + case 'deleteGlobs': { CoreConfigFiles._addEventActionToMap( eventAction as IHeftConfigurationDeleteGlobsEventAction, From 995dcb1cfb296b779e4c882a88efc9110e6cf6b2 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 14:55:29 -0800 Subject: [PATCH 03/35] Add markdown copy from source to dist as a default member of the heft-node-rig --- .../profiles/default/config/heft.json | 35 +++++++++++++++++++ .../profiles/library/config/heft.json | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/rigs/heft-node-rig/profiles/default/config/heft.json b/rigs/heft-node-rig/profiles/default/config/heft.json index 99e058540fb..0abe7d6b9c6 100644 --- a/rigs/heft-node-rig/profiles/default/config/heft.json +++ b/rigs/heft-node-rig/profiles/default/config/heft.json @@ -29,6 +29,41 @@ * Glob patterns to be deleted. The paths are resolved relative to the project folder. */ "globsToDelete": ["dist", "lib", "temp"] + }, + { + /** + * The kind of built-in operation that should be performed. + * The "copyFiles" action copies files that match the specified glob patterns. It can be optionally + * configured to use hardlinks + */ + "actionKind": "copyGlobs", + + /** + * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + * occur at the end of the stage of the Heft run. + */ + "heftEvent": "pre-compile", + + /** + * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + * configs. + */ + "actionId": "defaultCopyGlobs", + + /** + * Glob patterns to be copied. The paths are resolved relative to the project folder. + */ + "globsToCopy": ["src/**/*.md"], + + /** + * Destination folders for the files to be copied. The paths are resolved relative to the project folder. + */ + "targetFolders": ["dist"], + + /** + * Whether to copy or hardlink the files. + */ + "hardlink": false } ], diff --git a/rigs/heft-web-rig/profiles/library/config/heft.json b/rigs/heft-web-rig/profiles/library/config/heft.json index 2633357c6d2..7c6f019378b 100644 --- a/rigs/heft-web-rig/profiles/library/config/heft.json +++ b/rigs/heft-web-rig/profiles/library/config/heft.json @@ -29,6 +29,41 @@ * Glob patterns to be deleted. The paths are resolved relative to the project folder. */ "globsToDelete": ["dist", "lib", "lib-amd", "lib-es6", "temp"] + }, + { + /** + * The kind of built-in operation that should be performed. + * The "copyFiles" action copies files that match the specified glob patterns. It can be optionally + * configured to use hardlinks + */ + "actionKind": "copyGlobs", + + /** + * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + * occur at the end of the stage of the Heft run. + */ + "heftEvent": "pre-compile", + + /** + * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + * configs. + */ + "actionId": "defaultCopyGlobs", + + /** + * Glob patterns to be copied. The paths are resolved relative to the project folder. + */ + "globsToCopy": ["src/**/*.md"], + + /** + * Destination folders for the files to be copied. The paths are resolved relative to the project folder. + */ + "targetFolders": ["dist"], + + /** + * Whether to copy or hardlink the files. + */ + "hardlink": false } ], From a7b78797dd88840f4aec7c3b58258d93ba5bb042 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 15:37:36 -0800 Subject: [PATCH 04/35] Undo rig change --- .../profiles/default/config/heft.json | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/rigs/heft-node-rig/profiles/default/config/heft.json b/rigs/heft-node-rig/profiles/default/config/heft.json index 0abe7d6b9c6..99e058540fb 100644 --- a/rigs/heft-node-rig/profiles/default/config/heft.json +++ b/rigs/heft-node-rig/profiles/default/config/heft.json @@ -29,41 +29,6 @@ * Glob patterns to be deleted. The paths are resolved relative to the project folder. */ "globsToDelete": ["dist", "lib", "temp"] - }, - { - /** - * The kind of built-in operation that should be performed. - * The "copyFiles" action copies files that match the specified glob patterns. It can be optionally - * configured to use hardlinks - */ - "actionKind": "copyGlobs", - - /** - * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json - * occur at the end of the stage of the Heft run. - */ - "heftEvent": "pre-compile", - - /** - * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other - * configs. - */ - "actionId": "defaultCopyGlobs", - - /** - * Glob patterns to be copied. The paths are resolved relative to the project folder. - */ - "globsToCopy": ["src/**/*.md"], - - /** - * Destination folders for the files to be copied. The paths are resolved relative to the project folder. - */ - "targetFolders": ["dist"], - - /** - * Whether to copy or hardlink the files. - */ - "hardlink": false } ], From ae709d2a8cac7431ec5670cb35acfc653c5f2c10 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 15:38:12 -0800 Subject: [PATCH 05/35] Undo rig changes --- .../profiles/library/config/heft.json | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/rigs/heft-web-rig/profiles/library/config/heft.json b/rigs/heft-web-rig/profiles/library/config/heft.json index 7c6f019378b..2633357c6d2 100644 --- a/rigs/heft-web-rig/profiles/library/config/heft.json +++ b/rigs/heft-web-rig/profiles/library/config/heft.json @@ -29,41 +29,6 @@ * Glob patterns to be deleted. The paths are resolved relative to the project folder. */ "globsToDelete": ["dist", "lib", "lib-amd", "lib-es6", "temp"] - }, - { - /** - * The kind of built-in operation that should be performed. - * The "copyFiles" action copies files that match the specified glob patterns. It can be optionally - * configured to use hardlinks - */ - "actionKind": "copyGlobs", - - /** - * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json - * occur at the end of the stage of the Heft run. - */ - "heftEvent": "pre-compile", - - /** - * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other - * configs. - */ - "actionId": "defaultCopyGlobs", - - /** - * Glob patterns to be copied. The paths are resolved relative to the project folder. - */ - "globsToCopy": ["src/**/*.md"], - - /** - * Destination folders for the files to be copied. The paths are resolved relative to the project folder. - */ - "targetFolders": ["dist"], - - /** - * Whether to copy or hardlink the files. - */ - "hardlink": false } ], From a4d05e73232c89a6e8d581e58ff25f34b98c1643 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 15:39:15 -0800 Subject: [PATCH 06/35] Ensure target folder when linking, and include filename in target path --- apps/heft/src/plugins/CopyGlobsPlugin.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/heft/src/plugins/CopyGlobsPlugin.ts b/apps/heft/src/plugins/CopyGlobsPlugin.ts index 8940dd890d9..77cf90c926d 100644 --- a/apps/heft/src/plugins/CopyGlobsPlugin.ts +++ b/apps/heft/src/plugins/CopyGlobsPlugin.ts @@ -34,7 +34,7 @@ export class CopyGlobsPlugin implements IHeftPlugin { public readonly pluginName: string = PLUGIN_NAME; public apply(heftSession: HeftSession, heftConfiguration: HeftConfiguration): void { - const logger: ScopedLogger = heftSession.requestScopedLogger('copy-files'); + const logger: ScopedLogger = heftSession.requestScopedLogger('copy-globs'); heftSession.hooks.build.tap(PLUGIN_NAME, (build: IBuildStageContext) => { build.hooks.preCompile.tap(PLUGIN_NAME, (preCompile: IPreCompileSubstage) => { preCompile.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { @@ -91,7 +91,11 @@ export class CopyGlobsPlugin implements IHeftPlugin { for (const targetFolder of copyFilesEventAction.targetFolders) { resolvedTargetPathsMap.set( - path.resolve(heftConfiguration.buildFolder, targetFolder), + path.resolve( + heftConfiguration.buildFolder, + targetFolder, + path.basename(resolvedSourceFilePath) + ), copyFilesEventAction.hardlink || false ); } @@ -123,6 +127,7 @@ export class CopyGlobsPlugin implements IHeftPlugin { } } + await FileSystem.ensureFolderAsync(path.dirname(targetFilePath)); await FileSystem.createHardLinkAsync({ linkTargetPath: sourceFilePath, newLinkPath: targetFilePath From 10081100757c1269edbdd328d2d235a16a28b4f8 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 15:46:20 -0800 Subject: [PATCH 07/35] Typo --- apps/heft/src/plugins/CopyGlobsPlugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/heft/src/plugins/CopyGlobsPlugin.ts b/apps/heft/src/plugins/CopyGlobsPlugin.ts index 77cf90c926d..68e33b07a61 100644 --- a/apps/heft/src/plugins/CopyGlobsPlugin.ts +++ b/apps/heft/src/plugins/CopyGlobsPlugin.ts @@ -22,7 +22,7 @@ import { const globEscape: (unescaped: string) => string = require('glob-escape'); // No @types/glob-escape package exists -const PLUGIN_NAME: string = 'CopyFilesPlugin'; +const PLUGIN_NAME: string = 'CopyGlobsPlugin'; const HEFT_STAGE_TAP: TapOptions<'promise'> = { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER / 2 // This should give us some certainty that this will run after other plugins From f0b0d2d5ceff4b887eb8abc4538ecf2297cd7333 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 15:48:06 -0800 Subject: [PATCH 08/35] Rush change --- .../heft/danade-copy-plugin_2020-11-02-23-47.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json diff --git a/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json b/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json new file mode 100644 index 00000000000..d9f35b4c476 --- /dev/null +++ b/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/heft", + "comment": "Add new default Heft action \"copyGlobs\" to copy or hardlink files during specified Heft events", + "type": "minor" + } + ], + "packageName": "@rushstack/heft", + "email": "3473356+D4N14L@users.noreply.github.com" +} \ No newline at end of file From ba86bafbcc625f4925bf9470103a97aed5dfe8e2 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 2 Nov 2020 15:52:03 -0800 Subject: [PATCH 09/35] Fix plural --- apps/heft/src/plugins/CopyGlobsPlugin.ts | 4 ++-- apps/heft/src/plugins/DeleteGlobsPlugin.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/heft/src/plugins/CopyGlobsPlugin.ts b/apps/heft/src/plugins/CopyGlobsPlugin.ts index 68e33b07a61..27b49ff3951 100644 --- a/apps/heft/src/plugins/CopyGlobsPlugin.ts +++ b/apps/heft/src/plugins/CopyGlobsPlugin.ts @@ -147,10 +147,10 @@ export class CopyGlobsPlugin implements IHeftPlugin { ); if (linkedFiles > 0) { - logger.terminal.writeLine(`Linked ${linkedFiles} files`); + logger.terminal.writeLine(`Linked ${linkedFiles} file${linkedFiles > 1 ? 's' : ''}`); } if (copiedFiles > 0) { - logger.terminal.writeLine(`Copied ${copiedFiles} files`); + logger.terminal.writeLine(`Copied ${copiedFiles} file${copiedFiles > 1 ? 's' : ''}`); } } diff --git a/apps/heft/src/plugins/DeleteGlobsPlugin.ts b/apps/heft/src/plugins/DeleteGlobsPlugin.ts index 941167de52f..8a15a199d06 100644 --- a/apps/heft/src/plugins/DeleteGlobsPlugin.ts +++ b/apps/heft/src/plugins/DeleteGlobsPlugin.ts @@ -116,7 +116,10 @@ export class DeleteGlobsPlugin implements IHeftPlugin { }); if (deletedFiles > 0 || deletedFolders > 0) { - logger.terminal.writeLine(`Deleted ${deletedFiles} files and ${deletedFolders} folders`); + logger.terminal.writeLine( + `Deleted ${deletedFiles} file${deletedFiles > 1 ? 's' : ''} ` + + `and ${deletedFolders} folder${deletedFolders > 1 ? 's' : ''}` + ); } } From 87bce91207176c391be7dd7c7445d4bba9ba7474 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Wed, 4 Nov 2020 14:12:58 -0800 Subject: [PATCH 10/35] PR feedback --- .../heft/src/pluginFramework/PluginManager.ts | 4 +- ...{CopyGlobsPlugin.ts => CopyFilesPlugin.ts} | 73 +++++++++++++------ apps/heft/src/schemas/heft.schema.json | 72 ++++++++++++------ apps/heft/src/utilities/CoreConfigFiles.ts | 25 ++++--- 4 files changed, 118 insertions(+), 56 deletions(-) rename apps/heft/src/plugins/{CopyGlobsPlugin.ts => CopyFilesPlugin.ts} (70%) diff --git a/apps/heft/src/pluginFramework/PluginManager.ts b/apps/heft/src/pluginFramework/PluginManager.ts index 83f26503b79..ec69a4838d5 100644 --- a/apps/heft/src/pluginFramework/PluginManager.ts +++ b/apps/heft/src/pluginFramework/PluginManager.ts @@ -14,7 +14,7 @@ import { } from '../utilities/CoreConfigFiles'; // Default plugins -import { CopyGlobsPlugin } from '../plugins/CopyGlobsPlugin'; +import { CopyFilesPlugin } from '../plugins/CopyFilesPlugin'; import { TypeScriptPlugin } from '../plugins/TypeScriptPlugin/TypeScriptPlugin'; import { DeleteGlobsPlugin } from '../plugins/DeleteGlobsPlugin'; import { CopyStaticAssetsPlugin } from '../plugins/CopyStaticAssetsPlugin'; @@ -47,7 +47,7 @@ export class PluginManager { public initializeDefaultPlugins(): void { this._applyPlugin(new TypeScriptPlugin()); this._applyPlugin(new CopyStaticAssetsPlugin()); - this._applyPlugin(new CopyGlobsPlugin()); + this._applyPlugin(new CopyFilesPlugin()); this._applyPlugin(new DeleteGlobsPlugin()); this._applyPlugin(new ApiExtractorPlugin()); this._applyPlugin(new JestPlugin()); diff --git a/apps/heft/src/plugins/CopyGlobsPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts similarity index 70% rename from apps/heft/src/plugins/CopyGlobsPlugin.ts rename to apps/heft/src/plugins/CopyFilesPlugin.ts index 27b49ff3951..dff71e10951 100644 --- a/apps/heft/src/plugins/CopyGlobsPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -22,7 +22,7 @@ import { const globEscape: (unescaped: string) => string = require('glob-escape'); // No @types/glob-escape package exists -const PLUGIN_NAME: string = 'CopyGlobsPlugin'; +const PLUGIN_NAME: string = 'CopyFilesPlugin'; const HEFT_STAGE_TAP: TapOptions<'promise'> = { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER / 2 // This should give us some certainty that this will run after other plugins @@ -30,11 +30,11 @@ const HEFT_STAGE_TAP: TapOptions<'promise'> = { const MAX_PARALLELISM: number = 100; -export class CopyGlobsPlugin implements IHeftPlugin { +export class CopyFilesPlugin implements IHeftPlugin { public readonly pluginName: string = PLUGIN_NAME; public apply(heftSession: HeftSession, heftConfiguration: HeftConfiguration): void { - const logger: ScopedLogger = heftSession.requestScopedLogger('copy-globs'); + const logger: ScopedLogger = heftSession.requestScopedLogger('copy-files'); heftSession.hooks.build.tap(PLUGIN_NAME, (build: IBuildStageContext) => { build.hooks.preCompile.tap(PLUGIN_NAME, (preCompile: IPreCompileSubstage) => { preCompile.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { @@ -74,30 +74,50 @@ export class CopyGlobsPlugin implements IHeftPlugin { // Build a map to dedupe copy operations const fileOperationMap: Map> = new Map>(); - for (const copyFilesEventAction of eventActions.copyGlobs.get(heftEvent) || []) { - for (const globPattern of copyFilesEventAction.globsToCopy) { - const resolvedSourceFilePaths: string[] = await this._resolvePathAsync( - globPattern, - heftConfiguration.buildFolder + for (const copyFilesEventAction of eventActions.copyFiles.get(heftEvent) || []) { + for (const copyOperation of copyFilesEventAction.copyOperations) { + // Default the resolved sourceFolder path to the current build folder + const resolvedSourceFolderPath: string = path.resolve( + heftConfiguration.buildFolder, + copyOperation.sourceFolder ?? '.' ); + // Run each glob against the resolved sourceFolder and flatten out the results + const resolvedSourceFilePaths: string[] = ( + await Promise.all( + copyOperation.includeGlobs.map((includeGlob) => { + return this._resolvePathAsync( + resolvedSourceFolderPath, + includeGlob, + copyOperation.excludeGlobs + ); + }) + ) + ).reduce((prev: string[], curr: string[]) => { + prev.push(...curr); + return prev; + }, []); + + // Determine the target path for each source file and append to the correct map for (const resolvedSourceFilePath of resolvedSourceFilePaths) { - let resolvedTargetPathsMap: Map | undefined = fileOperationMap.get( + let resolvedDestinationPathsMap: Map | undefined = fileOperationMap.get( resolvedSourceFilePath ); - if (!resolvedTargetPathsMap) { - resolvedTargetPathsMap = new Map(); - fileOperationMap.set(resolvedSourceFilePath, resolvedTargetPathsMap); + if (!resolvedDestinationPathsMap) { + resolvedDestinationPathsMap = new Map(); + fileOperationMap.set(resolvedSourceFilePath, resolvedDestinationPathsMap); } - for (const targetFolder of copyFilesEventAction.targetFolders) { - resolvedTargetPathsMap.set( - path.resolve( - heftConfiguration.buildFolder, - targetFolder, - path.basename(resolvedSourceFilePath) - ), - copyFilesEventAction.hardlink || false + for (const destinationFolder of copyOperation.destinationFolders) { + // Only include the relative path from the sourceFolder if flatten is false + const resolvedDestinationFilePath: string = path.resolve( + heftConfiguration.buildFolder, + destinationFolder, + copyOperation.flatten + ? '.' + : path.relative(resolvedSourceFolderPath, path.dirname(resolvedSourceFilePath)), + path.basename(resolvedSourceFilePath) ); + resolvedDestinationPathsMap.set(resolvedDestinationFilePath, copyOperation.hardlink || false); } } } @@ -154,20 +174,25 @@ export class CopyGlobsPlugin implements IHeftPlugin { } } - private async _resolvePathAsync(globPattern: string, buildFolder: string): Promise { + private async _resolvePathAsync( + sourceFolder: string, + globPattern: string, + excludeGlobPatterns?: string[] + ): Promise { if (globEscape(globPattern) !== globPattern) { const expandedGlob: string[] = await LegacyAdapters.convertCallbackToPromise(glob, globPattern, { - cwd: buildFolder + cwd: sourceFolder, + ignore: excludeGlobPatterns }); const result: string[] = []; for (const pathFromGlob of expandedGlob) { - result.push(path.resolve(buildFolder, pathFromGlob)); + result.push(path.resolve(sourceFolder, pathFromGlob)); } return result; } else { - return [path.resolve(buildFolder, globPattern)]; + return [path.resolve(sourceFolder, globPattern)]; } } } diff --git a/apps/heft/src/schemas/heft.schema.json b/apps/heft/src/schemas/heft.schema.json index 691898b6c12..a1c56bdbfd0 100644 --- a/apps/heft/src/schemas/heft.schema.json +++ b/apps/heft/src/schemas/heft.schema.json @@ -30,7 +30,7 @@ "actionKind": { "type": "string", "description": "The kind of built-in operation that should be performed.", - "enum": ["deleteGlobs", "copyGlobs"] + "enum": ["deleteGlobs", "copyFiles"] }, "heftEvent": { @@ -49,7 +49,7 @@ "oneOf": [ // Delete Globs { - "required": ["actionKind", "globsToDelete"], + "required": ["globsToDelete"], "properties": { "actionKind": { "type": "string", @@ -68,11 +68,11 @@ }, // Copy Files { - "required": ["actionKind", "globsToCopy", "targetFolders"], + "required": ["copyOperations"], "properties": { "actionKind": { "type": "string", - "enum": ["copyGlobs"] + "enum": ["copyFiles"] }, "heftEvent": { @@ -80,27 +80,57 @@ "enum": ["pre-compile", "compile", "bundle", "post-build"] }, - "globsToCopy": { + "copyOperations": { "type": "array", - "description": "Glob patterns to be copied. The paths are resolved relative to the project folder.", + "description": "An array of copy operations to run perform during the specified Heft event.", "items": { - "type": "string", - "pattern": "[^\\\\]" - } - }, + "type": "object", + "required": ["destinationFolders", "includeGlobs"], + "properties": { + "sourceFolder": { + "type": "string", + "description": "The source folder from which the specified globs will be run. The paths are resolved relative to the project root folder. Defaults to the project root folder.", + "pattern": "[^\\\\]" + }, - "targetFolders": { - "type": "array", - "description": "Destination folders for the files to be copied. The paths are resolved relative to the project folder.", - "items": { - "type": "string", - "pattern": "[^\\\\]" - } - }, + "destinationFolders": { + "type": "array", + "description": "The destination folders for the files to be copied. The paths are resolved relative to the project folder.", + "items": { + "type": "string", + "pattern": "[^\\\\]" + } + }, + + "includeGlobs": { + "type": "array", + "description": "Glob patterns to be copied. The paths are resolved relative to the \"sourceFolder\".", + "items": { + "type": "string", + "pattern": "[^\\\\]" + } + }, - "hardlink": { - "type": "boolean", - "description": "Whether to copy or hardlink the files." + "excludeGlobs": { + "type": "array", + "description": "Glob patterns to be excluded from copying. The paths are resolved relative to the \"sourceFolder\".", + "items": { + "type": "string", + "pattern": "[^\\\\]" + } + }, + + "flatten": { + "type": "boolean", + "description": "Whether to copy only the file and discard the relative path from the \"sourceFolder\". Defaults to false." + }, + + "hardlink": { + "type": "boolean", + "description": "Whether to copy or hardlink the files into the destination folder. Defaults to false." + } + } + } } } } diff --git a/apps/heft/src/utilities/CoreConfigFiles.ts b/apps/heft/src/utilities/CoreConfigFiles.ts index 975726f88d3..8d11c1c6c5f 100644 --- a/apps/heft/src/utilities/CoreConfigFiles.ts +++ b/apps/heft/src/utilities/CoreConfigFiles.ts @@ -35,13 +35,20 @@ export interface IHeftConfigurationDeleteGlobsEventAction extends IHeftConfigura globsToDelete: string[]; } -export interface IHeftConfigurationCopyGlobsEventAction extends IHeftConfigurationJsonEventActionBase { - actionKind: 'copyGlobs'; - globsToCopy: string[]; - targetFolders: string[]; +export interface ICopyFilesOperation { + sourceFolder?: string; + destinationFolders: string[]; + includeGlobs: string[]; + excludeGlobs?: string[]; + flatten?: boolean; hardlink?: boolean; } +export interface IHeftConfigurationCopyFilesEventAction extends IHeftConfigurationJsonEventActionBase { + actionKind: 'copyFiles'; + copyOperations: ICopyFilesOperation[]; +} + export interface IHeftConfigurationJsonPluginSpecifier { plugin: string; options?: object; @@ -53,7 +60,7 @@ export interface IHeftConfigurationJson { } export interface IHeftEventActions { - copyGlobs: Map; + copyFiles: Map; deleteGlobs: Map; } @@ -118,17 +125,17 @@ export class CoreConfigFiles { ); result = { - copyGlobs: new Map(), + copyFiles: new Map(), deleteGlobs: new Map() }; CoreConfigFiles._heftConfigFileEventActionsCache.set(heftConfiguration, result); for (const eventAction of heftConfigJson?.eventActions || []) { switch (eventAction.actionKind) { - case 'copyGlobs': { + case 'copyFiles': { CoreConfigFiles._addEventActionToMap( - eventAction as IHeftConfigurationCopyGlobsEventAction, - result.copyGlobs + eventAction as IHeftConfigurationCopyFilesEventAction, + result.copyFiles ); break; } From 9c8b5e20e550f1357c38843cfb02ec24197ce1f8 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Wed, 4 Nov 2020 15:36:12 -0800 Subject: [PATCH 11/35] Fix logged message --- apps/heft/src/plugins/CopyFilesPlugin.ts | 23 +++++++++++++--------- apps/heft/src/plugins/DeleteGlobsPlugin.ts | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index dff71e10951..aec8c4ac9d7 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -136,41 +136,42 @@ export class CopyFilesPlugin implements IHeftPlugin { await Async.forEachLimitAsync( flattenedOperationMap, MAX_PARALLELISM, - async ([sourceFilePath, targetFilePath, hardlink]) => { + async ([sourceFilePath, destinationFilePath, hardlink]) => { if (hardlink) { // Hardlink doesn't allow passing in overwrite param, so delete ourselves try { - await FileSystem.deleteFileAsync(targetFilePath); + await FileSystem.deleteFileAsync(destinationFilePath, { throwIfNotExists: true }); } catch (e) { if (!FileSystem.isFileDoesNotExistError(e)) { throw e; } + // Since the file doesn't exist, the parent folder may also not exist + await FileSystem.ensureFolderAsync(path.dirname(destinationFilePath)); } - await FileSystem.ensureFolderAsync(path.dirname(targetFilePath)); await FileSystem.createHardLinkAsync({ linkTargetPath: sourceFilePath, - newLinkPath: targetFilePath + newLinkPath: destinationFilePath }); - logger.terminal.writeVerboseLine(`Linked "${sourceFilePath}" to "${targetFilePath}"`); + logger.terminal.writeVerboseLine(`Linked "${sourceFilePath}" to "${destinationFilePath}"`); linkedFiles++; } else { await FileSystem.copyFileAsync({ sourcePath: sourceFilePath, - destinationPath: targetFilePath, + destinationPath: destinationFilePath, alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite }); - logger.terminal.writeVerboseLine(`Copied "${sourceFilePath}" to "${targetFilePath}"`); + logger.terminal.writeVerboseLine(`Copied "${sourceFilePath}" to "${destinationFilePath}"`); copiedFiles++; } } ); if (linkedFiles > 0) { - logger.terminal.writeLine(`Linked ${linkedFiles} file${linkedFiles > 1 ? 's' : ''}`); + logger.terminal.writeLine(`Linked ${linkedFiles} file${linkedFiles !== 1 ? 's' : ''}`); } if (copiedFiles > 0) { - logger.terminal.writeLine(`Copied ${copiedFiles} file${copiedFiles > 1 ? 's' : ''}`); + logger.terminal.writeLine(`Copied ${copiedFiles} file${copiedFiles !== 1 ? 's' : ''}`); } } @@ -192,6 +193,10 @@ export class CopyFilesPlugin implements IHeftPlugin { return result; } else { + // NOTE: Does not take excludeGlobPatterns into account as we cannot run glob + // against a path string. We could run the original globPattern through glob + // as well and solve this issue, however this carveout is done for performance + // and as such avoids glob return [path.resolve(sourceFolder, globPattern)]; } } diff --git a/apps/heft/src/plugins/DeleteGlobsPlugin.ts b/apps/heft/src/plugins/DeleteGlobsPlugin.ts index 8a15a199d06..d60173879be 100644 --- a/apps/heft/src/plugins/DeleteGlobsPlugin.ts +++ b/apps/heft/src/plugins/DeleteGlobsPlugin.ts @@ -117,8 +117,8 @@ export class DeleteGlobsPlugin implements IHeftPlugin { if (deletedFiles > 0 || deletedFolders > 0) { logger.terminal.writeLine( - `Deleted ${deletedFiles} file${deletedFiles > 1 ? 's' : ''} ` + - `and ${deletedFolders} folder${deletedFolders > 1 ? 's' : ''}` + `Deleted ${deletedFiles} file${deletedFiles !== 1 ? 's' : ''} ` + + `and ${deletedFolders} folder${deletedFolders !== 1 ? 's' : ''}` ); } } From 09157efb6c939c8d3c4f2c963581105fb6a5cf7d Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 6 Nov 2020 13:35:38 -0800 Subject: [PATCH 12/35] Refactor CopyStaticAssetsPlugin to be based off CopyFilesPlugin --- apps/heft/src/plugins/CopyFilesPlugin.ts | 254 +++++++++++------- .../src/plugins/CopyStaticAssetsPlugin.ts | 224 ++++----------- .../TypeScriptPlugin/TypeScriptPlugin.ts | 5 +- apps/heft/src/schemas/heft.schema.json | 27 +- apps/heft/src/schemas/typescript.schema.json | 2 +- apps/heft/src/utilities/CoreConfigFiles.ts | 50 +++- 6 files changed, 274 insertions(+), 288 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index aec8c4ac9d7..a4c40c74183 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -3,6 +3,7 @@ import * as path from 'path'; import glob from 'glob'; +import { performance } from 'perf_hooks'; import { AlreadyExistsBehavior, FileSystem, LegacyAdapters } from '@rushstack/node-core-library'; import { TapOptions } from 'tapable'; @@ -10,8 +11,13 @@ import { IHeftPlugin } from '../pluginFramework/IHeftPlugin'; import { HeftSession } from '../pluginFramework/HeftSession'; import { HeftConfiguration } from '../configuration/HeftConfiguration'; import { ScopedLogger } from '../pluginFramework/logging/ScopedLogger'; -import { IHeftEventActions, CoreConfigFiles, HeftEvent } from '../utilities/CoreConfigFiles'; import { Async } from '../utilities/Async'; +import { + IHeftEventActions, + CoreConfigFiles, + HeftEvent, + IExtendedSharedCopyConfiguration +} from '../utilities/CoreConfigFiles'; import { IBuildStageContext, IBundleSubstage, @@ -20,7 +26,7 @@ import { IPreCompileSubstage } from '../stages/BuildStage'; -const globEscape: (unescaped: string) => string = require('glob-escape'); // No @types/glob-escape package exists +const globEscape: (unescaped: string[]) => string[] = require('glob-escape'); // No @types/glob-escape package exists const PLUGIN_NAME: string = 'CopyFilesPlugin'; const HEFT_STAGE_TAP: TapOptions<'promise'> = { @@ -30,6 +36,18 @@ const HEFT_STAGE_TAP: TapOptions<'promise'> = { const MAX_PARALLELISM: number = 100; +export interface ICopyFileDescriptor { + sourceFilePath: string; + destinationFilePath: string; + hardlink: boolean; +} + +export interface ICopyFilesOptions { + buildFolder: string; + copyConfigurations: IExtendedSharedCopyConfiguration[]; + logger: ScopedLogger; +} + export class CopyFilesPlugin implements IHeftPlugin { public readonly pluginName: string = PLUGIN_NAME; @@ -72,132 +90,176 @@ export class CopyFilesPlugin implements IHeftPlugin { heftConfiguration ); - // Build a map to dedupe copy operations - const fileOperationMap: Map> = new Map>(); + const copyConfigurations: IExtendedSharedCopyConfiguration[] = []; for (const copyFilesEventAction of eventActions.copyFiles.get(heftEvent) || []) { - for (const copyOperation of copyFilesEventAction.copyOperations) { - // Default the resolved sourceFolder path to the current build folder - const resolvedSourceFolderPath: string = path.resolve( - heftConfiguration.buildFolder, - copyOperation.sourceFolder ?? '.' - ); - // Run each glob against the resolved sourceFolder and flatten out the results - const resolvedSourceFilePaths: string[] = ( - await Promise.all( - copyOperation.includeGlobs.map((includeGlob) => { - return this._resolvePathAsync( - resolvedSourceFolderPath, - includeGlob, - copyOperation.excludeGlobs - ); - }) - ) - ).reduce((prev: string[], curr: string[]) => { - prev.push(...curr); - return prev; - }, []); - - // Determine the target path for each source file and append to the correct map - for (const resolvedSourceFilePath of resolvedSourceFilePaths) { - let resolvedDestinationPathsMap: Map | undefined = fileOperationMap.get( - resolvedSourceFilePath - ); - if (!resolvedDestinationPathsMap) { - resolvedDestinationPathsMap = new Map(); - fileOperationMap.set(resolvedSourceFilePath, resolvedDestinationPathsMap); - } + copyConfigurations.push(...copyFilesEventAction.copyOperations); + } - for (const destinationFolder of copyOperation.destinationFolders) { - // Only include the relative path from the sourceFolder if flatten is false - const resolvedDestinationFilePath: string = path.resolve( - heftConfiguration.buildFolder, - destinationFolder, - copyOperation.flatten - ? '.' - : path.relative(resolvedSourceFolderPath, path.dirname(resolvedSourceFilePath)), - path.basename(resolvedSourceFilePath) - ); - resolvedDestinationPathsMap.set(resolvedDestinationFilePath, copyOperation.hardlink || false); - } - } - } + await this.runCopyAsync({ + buildFolder: heftConfiguration.buildFolder, + copyConfigurations, + logger + }); + } + + protected async runCopyAsync(options: ICopyFilesOptions): Promise { + const { logger, buildFolder, copyConfigurations } = options; + + const startTime: number = performance.now(); + const copyDescriptors: ICopyFileDescriptor[] = await this._getCopyFileDescriptorsAsync( + buildFolder, + copyConfigurations + ); + + if (copyDescriptors.length === 0) { + // No need to run copy and print to console + return; } - // Flatten out the map to simplify processing - const flattenedOperationMap: [string, string, boolean][] = []; - for (const [sourceFilePath, destinationMap] of fileOperationMap.entries()) { - for (const [destinationFilePath, hardlink] of destinationMap.entries()) { - flattenedOperationMap.push([sourceFilePath, destinationFilePath, hardlink]); - } + const [copyCount, hardlinkCount] = await this.copyFilesAsync(copyDescriptors); + const duration: number = performance.now() - startTime; + logger.terminal.writeLine( + `Copied ${copyCount} file${copyCount === 1 ? '' : 's'} and linked ${hardlinkCount} ` + + `file${hardlinkCount === 1 ? '' : 's'} in ${Math.round(duration)}ms` + ); + } + + protected async copyFilesAsync(copyDescriptors: ICopyFileDescriptor[]): Promise<[number, number]> { + if (copyDescriptors.length === 0) { + return [0, 0]; } - let linkedFiles: number = 0; - let copiedFiles: number = 0; + let copyCount: number = 0; + let hardlinkCount: number = 0; await Async.forEachLimitAsync( - flattenedOperationMap, + copyDescriptors, MAX_PARALLELISM, - async ([sourceFilePath, destinationFilePath, hardlink]) => { - if (hardlink) { + async (copyDescriptor: ICopyFileDescriptor) => { + if (copyDescriptor.hardlink) { // Hardlink doesn't allow passing in overwrite param, so delete ourselves try { - await FileSystem.deleteFileAsync(destinationFilePath, { throwIfNotExists: true }); + await FileSystem.deleteFileAsync(copyDescriptor.destinationFilePath, { throwIfNotExists: true }); } catch (e) { if (!FileSystem.isFileDoesNotExistError(e)) { throw e; } // Since the file doesn't exist, the parent folder may also not exist - await FileSystem.ensureFolderAsync(path.dirname(destinationFilePath)); + await FileSystem.ensureFolderAsync(path.dirname(copyDescriptor.destinationFilePath)); } await FileSystem.createHardLinkAsync({ - linkTargetPath: sourceFilePath, - newLinkPath: destinationFilePath + linkTargetPath: copyDescriptor.sourceFilePath, + newLinkPath: copyDescriptor.destinationFilePath }); - logger.terminal.writeVerboseLine(`Linked "${sourceFilePath}" to "${destinationFilePath}"`); - linkedFiles++; + hardlinkCount++; } else { + // If it's a copy, simply call the copy function await FileSystem.copyFileAsync({ - sourcePath: sourceFilePath, - destinationPath: destinationFilePath, + sourcePath: copyDescriptor.sourceFilePath, + destinationPath: copyDescriptor.destinationFilePath, alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite }); - logger.terminal.writeVerboseLine(`Copied "${sourceFilePath}" to "${destinationFilePath}"`); - copiedFiles++; + copyCount++; } } ); - if (linkedFiles > 0) { - logger.terminal.writeLine(`Linked ${linkedFiles} file${linkedFiles !== 1 ? 's' : ''}`); - } - if (copiedFiles > 0) { - logger.terminal.writeLine(`Copied ${copiedFiles} file${copiedFiles !== 1 ? 's' : ''}`); - } + return [copyCount, hardlinkCount]; } - private async _resolvePathAsync( - sourceFolder: string, - globPattern: string, - excludeGlobPatterns?: string[] - ): Promise { - if (globEscape(globPattern) !== globPattern) { - const expandedGlob: string[] = await LegacyAdapters.convertCallbackToPromise(glob, globPattern, { - cwd: sourceFolder, - ignore: excludeGlobPatterns - }); + private async _getCopyFileDescriptorsAsync( + buildFolder: string, + copyConfigurations: IExtendedSharedCopyConfiguration[] + ): Promise { + // Create a map to deduplicate and prevent double-writes + // resolvedDestinationFilePath -> [resolvedSourceFilePath, hardlink] + const destinationCopyDescriptors: Map = new Map(); + + for (const copyConfiguration of copyConfigurations) { + // Resolve the source folder path which is where the glob will be run from + const resolvedSourceFolderPath: string = path.resolve(buildFolder, copyConfiguration.sourceFolder); + + // Glob extensions with a specific glob to increase perf + let sourceFileRelativePaths: Set; + if (copyConfiguration.fileExtensions?.length) { + const escapedExtensions: string[] = globEscape(copyConfiguration.fileExtensions); + const pattern: string = `**/*+(${escapedExtensions.join('|')})`; + sourceFileRelativePaths = await this._expandGlobPatternAsync( + resolvedSourceFolderPath, + pattern, + copyConfiguration.excludeGlobs + ); + } else { + sourceFileRelativePaths = new Set(); + } - const result: string[] = []; - for (const pathFromGlob of expandedGlob) { - result.push(path.resolve(sourceFolder, pathFromGlob)); + // Now include the other glob as well + for (const include of copyConfiguration.includeGlobs || []) { + const explicitlyIncludedPaths: Set = await this._expandGlobPatternAsync( + resolvedSourceFolderPath, + include, + copyConfiguration.excludeGlobs + ); + + for (const explicitlyIncludedPath of explicitlyIncludedPaths) { + sourceFileRelativePaths.add(explicitlyIncludedPath); + } } - return result; - } else { - // NOTE: Does not take excludeGlobPatterns into account as we cannot run glob - // against a path string. We could run the original globPattern through glob - // as well and solve this issue, however this carveout is done for performance - // and as such avoids glob - return [path.resolve(sourceFolder, globPattern)]; + // Dedupe and throw if a double-write is detected + for (const destinationFolderRelativePath of copyConfiguration.destinationFolders) { + for (const sourceFileRelativePath of sourceFileRelativePaths) { + // Only include the relative path from the sourceFolder if flatten is false + const resolvedSourceFilePath: string = path.join(resolvedSourceFolderPath, sourceFileRelativePath); + const resolvedDestinationFilePath: string = path.resolve( + buildFolder, + destinationFolderRelativePath, + copyConfiguration.flatten ? '.' : path.dirname(sourceFileRelativePath), + path.basename(sourceFileRelativePath) + ); + + // Throw if a duplicate copy target with a different source or options is specified + const existingCopyDescriptor: ICopyFileDescriptor | undefined = destinationCopyDescriptors.get( + resolvedDestinationFilePath + ); + if (existingCopyDescriptor) { + if ( + existingCopyDescriptor.sourceFilePath === resolvedSourceFilePath && + existingCopyDescriptor.hardlink === !!copyConfiguration.hardlink + ) { + // Found a duplicate, avoid adding again + continue; + } + throw new Error( + `Cannot copy different files to the same destination "${resolvedDestinationFilePath}"` + ); + } + + // Finally, add to the map and default hardlink to false + destinationCopyDescriptors.set(resolvedDestinationFilePath, { + sourceFilePath: resolvedSourceFilePath, + destinationFilePath: resolvedDestinationFilePath, + hardlink: !!copyConfiguration.hardlink + }); + } + } } + + // We're done with the map, grab the values and return + return Array.from(destinationCopyDescriptors.values()); + } + + private async _expandGlobPatternAsync( + resolvedSourceFolderPath: string, + pattern: string, + exclude: string[] | undefined + ): Promise> { + const results: string[] = await LegacyAdapters.convertCallbackToPromise(glob, pattern, { + cwd: resolvedSourceFolderPath, + nodir: true, + ignore: exclude + }); + + return new Set(results); } } diff --git a/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts b/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts index 96ec7d1e481..fe49a23fa0f 100644 --- a/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts +++ b/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts @@ -1,88 +1,49 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { LegacyAdapters, FileSystem, Terminal } from '@rushstack/node-core-library'; -import glob from 'glob'; +import { FileSystem, Terminal } from '@rushstack/node-core-library'; import * as path from 'path'; import * as chokidar from 'chokidar'; -import { Async } from '../utilities/Async'; -import { performance } from 'perf_hooks'; -import { IHeftPlugin } from '../pluginFramework/IHeftPlugin'; import { HeftSession } from '../pluginFramework/HeftSession'; import { HeftConfiguration } from '../configuration/HeftConfiguration'; import { IBuildStageContext, ICompileSubstage } from '../stages/BuildStage'; import { ScopedLogger } from '../pluginFramework/logging/ScopedLogger'; -import { CoreConfigFiles } from '../utilities/CoreConfigFiles'; +import { CoreConfigFiles, IExtendedSharedCopyConfiguration } from '../utilities/CoreConfigFiles'; import { ITypeScriptConfigurationJson } from './TypeScriptPlugin/TypeScriptPlugin'; +import { CopyFilesPlugin, ICopyFilesOptions } from './CopyFilesPlugin'; const globEscape: (unescaped: string[]) => string[] = require('glob-escape'); // No @types/glob-escape package exists const PLUGIN_NAME: string = 'CopyStaticAssetsPlugin'; -export interface ISharedCopyStaticAssetsConfiguration { - /** - * File extensions that should be copied from the src folder to the destination folder(s) - */ - fileExtensions?: string[]; - - /** - * Globs that should be explicitly excluded. This takes precedence over globs listed in "includeGlobs" and - * files that match the file extensions provided in "fileExtensions". - */ - excludeGlobs?: string[]; - - /** - * Globs that should be explicitly included. - */ - includeGlobs?: string[]; +interface ICopyStaticAssetsOptions extends ICopyFilesOptions { + watchMode: boolean; } -interface ICopyStaticAssetsConfiguration extends ISharedCopyStaticAssetsConfiguration { +export class CopyStaticAssetsPlugin extends CopyFilesPlugin { /** - * The folder from which assets should be copied. For example, "src". This defaults to "src". - * - * This folder is directly under the folder containing the project's package.json file + * @override */ - sourceFolderName: string; + public readonly pluginName: string = PLUGIN_NAME; /** - * The folder(s) to which assets should be copied. For example ["lib", "lib-cjs"]. This defaults to ["lib"] - * - * These folders are directly under the folder containing the project's package.json file + * @override */ - destinationFolderNames: string[]; -} - -interface ICopyStaticAssetsOptions { - logger: ScopedLogger; - buildFolder: string; - copyStaticAssetsConfiguration: ICopyStaticAssetsConfiguration; - watchMode: boolean; -} - -interface IRunWatchOptions extends ICopyStaticAssetsOptions { - fileExtensionsGlobPattern: string | undefined; - resolvedSourceFolderPath: string; - resolvedDestinationFolderPaths: string[]; -} - -export class CopyStaticAssetsPlugin implements IHeftPlugin { - public readonly pluginName: string = PLUGIN_NAME; - public apply(heftSession: HeftSession, heftConfiguration: HeftConfiguration): void { heftSession.hooks.build.tap(PLUGIN_NAME, (build: IBuildStageContext) => { build.hooks.compile.tap(PLUGIN_NAME, (compile: ICompileSubstage) => { compile.hooks.run.tapPromise(PLUGIN_NAME, async () => { const logger: ScopedLogger = heftSession.requestScopedLogger('copy-static-assets'); - const copyStaticAssetsConfiguration: ICopyStaticAssetsConfiguration = await this._loadCopyStaticAssetsConfigurationAsync( + const copyStaticAssetsConfiguration: IExtendedSharedCopyConfiguration = await this._loadCopyStaticAssetsConfigurationAsync( logger.terminal, heftConfiguration ); - await this._runCopyAsync({ + + await this.runCopyAsync({ logger, - copyStaticAssetsConfiguration, + copyConfigurations: [copyStaticAssetsConfiguration], buildFolder: heftConfiguration.buildFolder, watchMode: build.properties.watchMode }); @@ -94,7 +55,7 @@ export class CopyStaticAssetsPlugin implements IHeftPlugin { private async _loadCopyStaticAssetsConfigurationAsync( terminal: Terminal, heftConfiguration: HeftConfiguration - ): Promise { + ): Promise { const typescriptConfiguration: | ITypeScriptConfigurationJson | undefined = await CoreConfigFiles.typeScriptConfigurationFileLoader.tryLoadConfigurationFileForProjectAsync( @@ -103,147 +64,72 @@ export class CopyStaticAssetsPlugin implements IHeftPlugin { heftConfiguration.rigConfig ); - const destinationFolderNames: string[] = ['lib']; + const destinationFolders: string[] = ['lib']; for (const emitModule of typescriptConfiguration?.additionalModuleKindsToEmit || []) { - destinationFolderNames.push(emitModule.outFolderName); + destinationFolders.push(emitModule.outFolderName); } return { ...typescriptConfiguration?.staticAssetsToCopy, // For now - these may need to be revised later - sourceFolderName: 'src', - destinationFolderNames + sourceFolder: 'src', + destinationFolders, + flatten: false, + hardlink: false }; } - private async _expandGlobPatternAsync( - resolvedSourceFolderPath: string, - pattern: string, - exclude: string[] | undefined - ): Promise> { - const results: string[] = await LegacyAdapters.convertCallbackToPromise(glob, pattern, { - cwd: resolvedSourceFolderPath, - nodir: true, - ignore: exclude - }); - - return new Set(results); - } - - private async _copyStaticAssetsAsync( - assetPathsToCopy: string[], - resolvedSourceFolderPath: string, - resolvedDestinationFolders: string[] - ): Promise { - if (assetPathsToCopy.length === 0) { - return 0; - } + /** + * @override + */ + protected async runCopyAsync(options: ICopyStaticAssetsOptions): Promise { + // First, run the actual copy + await super.runCopyAsync(options); - let copyCount: number = 0; - for (const resolvedDestinationFolder of resolvedDestinationFolders) { - await Async.forEachLimitAsync(assetPathsToCopy, 100, async (assetPath: string) => { - await FileSystem.copyFileAsync({ - sourcePath: path.join(resolvedSourceFolderPath, assetPath), - destinationPath: path.join(resolvedDestinationFolder, assetPath) - }); - copyCount++; - }); + // Then enter watch mode if requested + if (options.watchMode) { + await this._runWatchAsync(options); } - - return copyCount; } - private async _runCopyAsync(options: ICopyStaticAssetsOptions): Promise { - const { logger, buildFolder, copyStaticAssetsConfiguration, watchMode } = options; - - if (!copyStaticAssetsConfiguration.sourceFolderName) { - return; - } + private async _runWatchAsync(options: ICopyStaticAssetsOptions): Promise { + const { buildFolder, copyConfigurations, logger } = options; + const [copyStaticAssetsConfiguration] = copyConfigurations; - const startTime: number = performance.now(); - const resolvedSourceFolderPath: string = path.join( - buildFolder, - copyStaticAssetsConfiguration.sourceFolderName - ); - const resolvedDestinationFolderPaths: string[] = copyStaticAssetsConfiguration.destinationFolderNames.map( - (destinationFolder) => path.join(buildFolder, destinationFolder) - ); - - let fileExtensionsGlobPattern: string | undefined = undefined; + // Obtain the glob patterns to provide to the watcher + const globsToWatch: string[] = [...(copyStaticAssetsConfiguration.includeGlobs || [])]; if (copyStaticAssetsConfiguration.fileExtensions?.length) { const escapedExtensions: string[] = globEscape(copyStaticAssetsConfiguration.fileExtensions); - fileExtensionsGlobPattern = `**/*+(${escapedExtensions.join('|')})`; + globsToWatch.push(`**/*+(${escapedExtensions.join('|')})`); } - let assetsToCopy: Set; - if (copyStaticAssetsConfiguration.fileExtensions?.length) { - const escapedExtensions: string[] = globEscape(copyStaticAssetsConfiguration.fileExtensions); - const pattern: string = `**/*+(${escapedExtensions.join('|')})`; - assetsToCopy = await this._expandGlobPatternAsync( - resolvedSourceFolderPath, - pattern, - copyStaticAssetsConfiguration.excludeGlobs + if (globsToWatch.length) { + const resolvedSourceFolderPath: string = path.join( + buildFolder, + copyStaticAssetsConfiguration.sourceFolder ); - } else { - assetsToCopy = new Set(); - } - - for (const include of copyStaticAssetsConfiguration.includeGlobs || []) { - const explicitlyIncludedPaths: Set = await this._expandGlobPatternAsync( - resolvedSourceFolderPath, - include, - copyStaticAssetsConfiguration.excludeGlobs + const resolvedDestinationFolderPaths: string[] = copyStaticAssetsConfiguration.destinationFolders.map( + (destinationFolder) => { + return path.join(buildFolder, destinationFolder); + } ); - for (const explicitlyIncludedPath of explicitlyIncludedPaths) { - assetsToCopy.add(explicitlyIncludedPath); - } - } - - const copyCount: number = await this._copyStaticAssetsAsync( - Array.from(assetsToCopy), - resolvedSourceFolderPath, - resolvedDestinationFolderPaths - ); - const duration: number = performance.now() - startTime; - logger.terminal.writeLine( - `Copied ${copyCount} static asset${copyCount === 1 ? '' : 's'} in ${Math.round(duration)}ms` - ); - if (watchMode) { - await this._runWatchAsync({ - ...options, - resolvedSourceFolderPath, - resolvedDestinationFolderPaths, - fileExtensionsGlobPattern + const watcher: chokidar.FSWatcher = chokidar.watch(globsToWatch, { + cwd: resolvedSourceFolderPath, + ignoreInitial: true, + ignored: copyStaticAssetsConfiguration.excludeGlobs }); - } - } - - private async _runWatchAsync(options: IRunWatchOptions): Promise { - const { - logger, - fileExtensionsGlobPattern, - resolvedSourceFolderPath, - resolvedDestinationFolderPaths, - copyStaticAssetsConfiguration - } = options; - - if (fileExtensionsGlobPattern) { - const watcher: chokidar.FSWatcher = chokidar.watch( - [fileExtensionsGlobPattern, ...(copyStaticAssetsConfiguration.includeGlobs || [])], - { - cwd: resolvedSourceFolderPath, - ignoreInitial: true, - ignored: copyStaticAssetsConfiguration.excludeGlobs - } - ); const copyAsset: (assetPath: string) => Promise = async (assetPath: string) => { - const copyCount: number = await this._copyStaticAssetsAsync( - [assetPath], - resolvedSourceFolderPath, - resolvedDestinationFolderPaths + const [copyCount] = await this.copyFilesAsync( + resolvedDestinationFolderPaths.map((resolvedDestinationFolderPath) => { + return { + sourceFilePath: path.join(resolvedSourceFolderPath, assetPath), + destinationFilePath: path.join(resolvedDestinationFolderPath, assetPath), + hardlink: false + }; + }) ); logger.terminal.writeLine(`Copied ${copyCount} static asset${copyCount === 1 ? '' : 's'}`); }; diff --git a/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptPlugin.ts b/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptPlugin.ts index 3962165499c..2635f539eee 100644 --- a/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptPlugin.ts +++ b/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptPlugin.ts @@ -19,8 +19,7 @@ import { TaskPackageResolver, ITaskPackageResolution } from '../../utilities/Tas import { JestTypeScriptDataFile } from '../JestPlugin/JestTypeScriptDataFile'; import { ScopedLogger } from '../../pluginFramework/logging/ScopedLogger'; import { ICleanStageContext, ICleanStageProperties } from '../../stages/CleanStage'; -import { CoreConfigFiles } from '../../utilities/CoreConfigFiles'; -import { ISharedCopyStaticAssetsConfiguration } from '../CopyStaticAssetsPlugin'; +import { CoreConfigFiles, ISharedCopyConfiguration } from '../../utilities/CoreConfigFiles'; const PLUGIN_NAME: string = 'typescript'; @@ -79,7 +78,7 @@ export interface ISharedTypeScriptConfiguration { * Configures additional file types that should be copied into the TypeScript compiler's emit folders, for example * so that these files can be resolved by import statements. */ - staticAssetsToCopy?: ISharedCopyStaticAssetsConfiguration; + staticAssetsToCopy?: ISharedCopyConfiguration; } export interface ITypeScriptConfigurationJson extends ISharedTypeScriptConfiguration { diff --git a/apps/heft/src/schemas/heft.schema.json b/apps/heft/src/schemas/heft.schema.json index a1c56bdbfd0..1baa67da614 100644 --- a/apps/heft/src/schemas/heft.schema.json +++ b/apps/heft/src/schemas/heft.schema.json @@ -85,35 +85,44 @@ "description": "An array of copy operations to run perform during the specified Heft event.", "items": { "type": "object", - "required": ["destinationFolders", "includeGlobs"], + "required": ["sourceFolder", "destinationFolders"], "properties": { "sourceFolder": { "type": "string", - "description": "The source folder from which the specified globs will be run. The paths are resolved relative to the project root folder. Defaults to the project root folder.", + "description": "The folder from which files should be copied.", "pattern": "[^\\\\]" }, "destinationFolders": { "type": "array", - "description": "The destination folders for the files to be copied. The paths are resolved relative to the project folder.", + "description": "The folder(s) to which files should be copied.", "items": { "type": "string", "pattern": "[^\\\\]" } }, - "includeGlobs": { + "fileExtensions": { "type": "array", - "description": "Glob patterns to be copied. The paths are resolved relative to the \"sourceFolder\".", + "description": "File extensions that should be copied from the source folder to the destination folder(s)", "items": { "type": "string", - "pattern": "[^\\\\]" + "pattern": "^\\.[A-z0-9-_.]*[A-z0-9-_]+$" } }, "excludeGlobs": { "type": "array", - "description": "Glob patterns to be excluded from copying. The paths are resolved relative to the \"sourceFolder\".", + "description": "Globs that should be explicitly excluded. This takes precedence over globs listed in \"includeGlobs\" and files that match the file extensions provided in \"fileExtensions\".", + "items": { + "type": "string", + "pattern": "[^\\\\]" + } + }, + + "includeGlobs": { + "type": "array", + "description": "Globs that should be explicitly included.", "items": { "type": "string", "pattern": "[^\\\\]" @@ -122,12 +131,12 @@ "flatten": { "type": "boolean", - "description": "Whether to copy only the file and discard the relative path from the \"sourceFolder\". Defaults to false." + "description": "Copy only the file and discard the relative path from the source folder. This defaults to false." }, "hardlink": { "type": "boolean", - "description": "Whether to copy or hardlink the files into the destination folder. Defaults to false." + "description": "Hardlink files instead of copying. This defaults to false." } } } diff --git a/apps/heft/src/schemas/typescript.schema.json b/apps/heft/src/schemas/typescript.schema.json index 8acea6b5f9e..36fab474947 100644 --- a/apps/heft/src/schemas/typescript.schema.json +++ b/apps/heft/src/schemas/typescript.schema.json @@ -67,7 +67,7 @@ "properties": { "fileExtensions": { "type": "array", - "description": "File extensions that should be copied from the src folder to the destination folder(s)", + "description": "File extensions that should be copied from the source folder to the destination folder(s)", "items": { "type": "string", "pattern": "^\\.[A-z0-9-_.]*[A-z0-9-_]+$" diff --git a/apps/heft/src/utilities/CoreConfigFiles.ts b/apps/heft/src/utilities/CoreConfigFiles.ts index 8d11c1c6c5f..3e7c1e9ea0c 100644 --- a/apps/heft/src/utilities/CoreConfigFiles.ts +++ b/apps/heft/src/utilities/CoreConfigFiles.ts @@ -13,7 +13,6 @@ import { IApiExtractorPluginConfiguration } from '../plugins/ApiExtractorPlugin/ import { ITypeScriptConfigurationJson } from '../plugins/TypeScriptPlugin/TypeScriptPlugin'; import { HeftConfiguration } from '../configuration/HeftConfiguration'; import { Terminal } from '@rushstack/node-core-library'; -import { ISharedCopyStaticAssetsConfiguration } from '../plugins/CopyStaticAssetsPlugin'; import { ISassConfigurationJson } from '../plugins/SassTypingsPlugin/SassTypingsPlugin'; export enum HeftEvent { @@ -35,18 +34,49 @@ export interface IHeftConfigurationDeleteGlobsEventAction extends IHeftConfigura globsToDelete: string[]; } -export interface ICopyFilesOperation { - sourceFolder?: string; - destinationFolders: string[]; - includeGlobs: string[]; +export interface ISharedCopyConfiguration { + /** + * File extensions that should be copied from the source folder to the destination folder(s) + */ + fileExtensions?: string[]; + + /** + * Globs that should be explicitly excluded. This takes precedence over globs listed in "includeGlobs" and + * files that match the file extensions provided in "fileExtensions". + */ excludeGlobs?: string[]; + + /** + * Globs that should be explicitly included. + */ + includeGlobs?: string[]; + + /** + * Copy only the file and discard the relative path from the source folder. + */ flatten?: boolean; + + /** + * Hardlink files instead of copying. + */ hardlink?: boolean; } +export interface IExtendedSharedCopyConfiguration extends ISharedCopyConfiguration { + /** + * The folder from which files should be copied. For example, "src". + */ + sourceFolder: string; + + /** + * The folder(s) to which files should be copied. For example ["lib", "lib-cjs"]. + */ + destinationFolders: string[]; +} + export interface IHeftConfigurationCopyFilesEventAction extends IHeftConfigurationJsonEventActionBase { actionKind: 'copyFiles'; - copyOperations: ICopyFilesOperation[]; + copyOperations: IExtendedSharedCopyConfiguration[]; } export interface IHeftConfigurationJsonPluginSpecifier { @@ -195,10 +225,10 @@ export class CoreConfigFiles { staticAssetsToCopy: { inheritanceType: InheritanceType.custom, inheritanceFunction: ( - currentObject: ISharedCopyStaticAssetsConfiguration, - parentObject: ISharedCopyStaticAssetsConfiguration - ): ISharedCopyStaticAssetsConfiguration => { - const result: ISharedCopyStaticAssetsConfiguration = {}; + currentObject: ISharedCopyConfiguration, + parentObject: ISharedCopyConfiguration + ): ISharedCopyConfiguration => { + const result: ISharedCopyConfiguration = {}; CoreConfigFiles._inheritArray(result, 'fileExtensions', currentObject, parentObject); CoreConfigFiles._inheritArray(result, 'includeGlobs', currentObject, parentObject); From 44cfdbe78e1e455e763f08c641e47f4460ce09f2 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 17:54:10 -0800 Subject: [PATCH 13/35] Add ability to set AlreadyExistsBehavior when creating a hardlink --- .../TypeScriptPlugin/TypeScriptBuilder.ts | 11 ++-- .../fileSystem/TypeScriptCachedFileSystem.ts | 28 ---------- common/reviews/api/node-core-library.api.md | 1 + libraries/node-core-library/src/FileSystem.ts | 53 +++++++++++++++++-- 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptBuilder.ts b/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptBuilder.ts index b654c9e78cf..12710bdb483 100644 --- a/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptBuilder.ts +++ b/apps/heft/src/plugins/TypeScriptPlugin/TypeScriptBuilder.ts @@ -12,7 +12,8 @@ import { InternalError, ITerminalProvider, FileSystem, - Path + Path, + AlreadyExistsBehavior } from '@rushstack/node-core-library'; import * as crypto from 'crypto'; import type * as TTypescript from 'typescript'; @@ -510,11 +511,9 @@ export class TypeScriptBuilder extends SubprocessRunnerBase { linkPromises.push( this._cachedFileSystem - .createHardLinkExtendedAsync({ ...options, preserveExisting: true }) - .then((successful) => { - if (successful) { - linkCount++; - } + .createHardLinkAsync({ ...options, alreadyExistsBehavior: AlreadyExistsBehavior.Ignore }) + .then(() => { + linkCount++; }) .catch((error) => { if (!FileSystem.isNotExistError(error)) { diff --git a/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts b/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts index 3ae46b2eba3..4fcea855021 100644 --- a/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts +++ b/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts @@ -21,10 +21,6 @@ export interface IReadFolderFilesAndDirectoriesResult { directories: string[]; } -export interface ICreateHardLinkExtendedOptions extends IFileSystemCreateLinkOptions { - preserveExisting: boolean; -} - interface ICacheEntry { entry: TEntry | undefined; error?: NodeJS.ErrnoException; @@ -148,30 +144,6 @@ export class TypeScriptCachedFileSystem { ); }; - public createHardLinkExtendedAsync: (options: ICreateHardLinkExtendedOptions) => Promise = async ( - options: ICreateHardLinkExtendedOptions - ) => { - try { - await this.createHardLinkAsync(options); - return true; - } catch (error) { - if (error.code === 'EEXIST') { - if (options.preserveExisting) { - return false; - } - - this.deleteFile(options.newLinkPath); - } else if (FileSystem.isNotExistError(error)) { - await this.ensureFolderAsync(nodeJsPath.dirname(options.newLinkPath)); - } else { - throw error; - } - - await this.createHardLinkAsync(options); - return true; - } - }; - private _sortFolderEntries(folderEntries: fs.Dirent[]): IReadFolderFilesAndDirectoriesResult { // TypeScript expects entries sorted ordinally by name // In practice this might not matter diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md index 5068e9cf647..28c7313572f 100644 --- a/common/reviews/api/node-core-library.api.md +++ b/common/reviews/api/node-core-library.api.md @@ -300,6 +300,7 @@ export interface IFileSystemCopyFilesOptions extends IFileSystemCopyFilesAsyncOp // @public export interface IFileSystemCreateLinkOptions { + alreadyExistsBehavior?: AlreadyExistsBehavior; linkTargetPath: string; newLinkPath: string; } diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index 422f031b95e..74665b04dfe 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -260,6 +260,11 @@ export interface IFileSystemCreateLinkOptions { * The new path for the new symlink link to be created. */ newLinkPath: string; + + /** + * Specifies what to do if the target object already exists. Defaults to `AlreadyExistsBehavior.Error`. + */ + alreadyExistsBehavior?: AlreadyExistsBehavior; } const MOVE_DEFAULT_OPTIONS: Partial = { @@ -1119,7 +1124,28 @@ export class FileSystem { */ public static createHardLink(options: IFileSystemCreateLinkOptions): void { FileSystem._wrapException(() => { - fsx.linkSync(options.linkTargetPath, options.newLinkPath); + try { + fsx.linkSync(options.linkTargetPath, options.newLinkPath); + } catch (error) { + if (error.code === 'EEXIST') { + switch (options.alreadyExistsBehavior) { + case AlreadyExistsBehavior.Ignore: + return; + case AlreadyExistsBehavior.Overwrite: + this.deleteFile(options.newLinkPath); + break; + case AlreadyExistsBehavior.Error: + default: + throw error; + } + } else if (FileSystem.isNotExistError(error)) { + this.ensureFolder(nodeJsPath.dirname(options.newLinkPath)); + } else { + throw error; + } + + this.createHardLink(options); + } }); } @@ -1127,8 +1153,29 @@ export class FileSystem { * An async version of {@link FileSystem.createHardLink}. */ public static async createHardLinkAsync(options: IFileSystemCreateLinkOptions): Promise { - await FileSystem._wrapExceptionAsync(() => { - return fsx.link(options.linkTargetPath, options.newLinkPath); + await FileSystem._wrapExceptionAsync(async () => { + try { + await fsx.link(options.linkTargetPath, options.newLinkPath); + } catch (error) { + if (error.code === 'EEXIST') { + switch (options.alreadyExistsBehavior) { + case AlreadyExistsBehavior.Ignore: + return; + case AlreadyExistsBehavior.Overwrite: + await this.deleteFileAsync(options.newLinkPath); + break; + case AlreadyExistsBehavior.Error: + default: + throw error; + } + } else if (FileSystem.isNotExistError(error)) { + await this.ensureFolderAsync(nodeJsPath.dirname(options.newLinkPath)); + } else { + throw error; + } + + await this.createHardLinkAsync(options); + } }); } From 8d2d2733973822de755f1c1fd208eec3d7eba279 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:05:53 -0800 Subject: [PATCH 14/35] Add copyFileToManyAsync method --- common/reviews/api/node-core-library.api.md | 8 ++ libraries/node-core-library/src/FileSystem.ts | 101 ++++++++++++++++++ libraries/node-core-library/src/index.ts | 1 + 3 files changed, 110 insertions(+) diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md index 28c7313572f..54c25d82680 100644 --- a/common/reviews/api/node-core-library.api.md +++ b/common/reviews/api/node-core-library.api.md @@ -168,6 +168,7 @@ export class FileSystem { static copyFileAsync(options: IFileSystemCopyFileOptions): Promise; static copyFiles(options: IFileSystemCopyFilesOptions): void; static copyFilesAsync(options: IFileSystemCopyFilesOptions): Promise; + static copyFileToManyAsync(options: IFileSystemCopyFileToManyOptions): Promise; static createHardLink(options: IFileSystemCreateLinkOptions): void; static createHardLinkAsync(options: IFileSystemCreateLinkOptions): Promise; static createSymbolicLinkFile(options: IFileSystemCreateLinkOptions): void; @@ -298,6 +299,13 @@ export interface IFileSystemCopyFilesOptions extends IFileSystemCopyFilesAsyncOp filter?: FileSystemCopyFilesFilter; } +// @public +export interface IFileSystemCopyFileToManyOptions extends Omit { + alreadyExistsBehavior?: AlreadyExistsBehavior; + destinationPaths: string[]; + sourcePath: string; +} + // @public export interface IFileSystemCreateLinkOptions { alreadyExistsBehavior?: AlreadyExistsBehavior; diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index 74665b04dfe..32b99b4deee 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -128,6 +128,31 @@ export interface IFileSystemCopyFileOptions { alreadyExistsBehavior?: AlreadyExistsBehavior; } +/** + * The options for {@link FileSystem.copyFile} + * @public + */ +export interface IFileSystemCopyFileToManyOptions + extends Omit { + /** + * The path of the existing object to be copied. + * The path may be absolute or relative. + */ + sourcePath: string; + + /** + * The path that the object will be copied to. + * The path may be absolute or relative. + */ + destinationPaths: string[]; + + /** + * Specifies what to do if the target object already exists. + * @defaultValue {@link AlreadyExistsBehavior.Overwrite} + */ + alreadyExistsBehavior?: AlreadyExistsBehavior; +} + /** * Specifies the behavior of {@link FileSystem.copyFiles} in a situation where the target object * already exists. @@ -916,6 +941,82 @@ export class FileSystem { }); } + /** + * Copies a single file from one location to one or more other locations. + * By default, the file at the destination is overwritten if it already exists. + * + * @remarks + * The `copyFileToManyAsync()` API cannot be used to copy folders. It copies at most one file. + * + * The implementation is based on `createReadStream()` and `createWriteStream()` from the + * `fs-extra` package. + */ + public static async copyFileToManyAsync(options: IFileSystemCopyFileToManyOptions): Promise { + options = { + ...COPY_FILE_DEFAULT_OPTIONS, + ...options + }; + + if (FileSystem.getStatistics(options.sourcePath).isDirectory()) { + throw new Error( + 'The specified path refers to a folder; this operation expects a file object:\n' + options.sourcePath + ); + } + + await FileSystem._wrapExceptionAsync(async () => { + // See flags documentation: https://nodejs.org/api/fs.html#fs_file_system_flags + const writeFlags: string[] = []; + switch (options.alreadyExistsBehavior) { + case AlreadyExistsBehavior.Error: + case AlreadyExistsBehavior.Ignore: + writeFlags.push('wx'); + break; + case AlreadyExistsBehavior.Overwrite: + default: + writeFlags.push('w'); + } + const flags: string = writeFlags.join(); + + const createPipePromise: ( + sourceStream: fs.ReadStream, + destinationStream: fs.WriteStream + ) => Promise = (sourceStream: fs.ReadStream, destinationStream: fs.WriteStream) => { + return new Promise((resolve: () => void, reject: (error: Error) => void) => { + sourceStream.on('error', (e: Error) => { + if (destinationStream) { + destinationStream.destroy(); + } + reject(e); + }); + sourceStream + .pipe(destinationStream) + .on('close', () => { + resolve(); + }) + .on('error', (e: Error) => { + if ( + options.alreadyExistsBehavior === AlreadyExistsBehavior.Ignore && + FileSystem.isErrnoException(e) && + (e as NodeJS.ErrnoException).code === 'EEXIST' + ) { + resolve(); + } + reject(e); + }); + }); + }; + + const sourceStream: fs.ReadStream = fsx.createReadStream(options.sourcePath); + const uniqueDestinationPaths: Set = new Set(options.destinationPaths); + const pipePromises: Promise[] = []; + for (const destinationPath of uniqueDestinationPaths.values()) { + pipePromises.push(createPipePromise(sourceStream, fsx.createWriteStream(destinationPath, { flags }))); + } + + await Promise.all(pipePromises); + }); + } + /** * Copies a file or folder from one location to another, recursively copying any folder contents. * By default, destinationPath is overwritten if it already exists. diff --git a/libraries/node-core-library/src/index.ts b/libraries/node-core-library/src/index.ts index 5c31f2a147e..213581818b9 100644 --- a/libraries/node-core-library/src/index.ts +++ b/libraries/node-core-library/src/index.ts @@ -63,6 +63,7 @@ export { IFileSystemReadFileOptions, IFileSystemMoveOptions, IFileSystemCopyFileOptions, + IFileSystemCopyFileToManyOptions, IFileSystemDeleteFileOptions, IFileSystemUpdateTimeParameters, IFileSystemCreateLinkOptions, From 8684994045790195ff36ef2066f519e8dfaa6dac Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:14:03 -0800 Subject: [PATCH 15/35] Use new copy style in copy plugin --- apps/heft/src/plugins/CopyFilesPlugin.ts | 176 +++++++++--------- .../src/plugins/CopyStaticAssetsPlugin.ts | 20 +- 2 files changed, 100 insertions(+), 96 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index a4c40c74183..86b9836715b 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -2,9 +2,9 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import glob from 'glob'; +import glob from 'fast-glob'; import { performance } from 'perf_hooks'; -import { AlreadyExistsBehavior, FileSystem, LegacyAdapters } from '@rushstack/node-core-library'; +import { AlreadyExistsBehavior, FileSystem } from '@rushstack/node-core-library'; import { TapOptions } from 'tapable'; import { IHeftPlugin } from '../pluginFramework/IHeftPlugin'; @@ -25,8 +25,7 @@ import { IPostBuildSubstage, IPreCompileSubstage } from '../stages/BuildStage'; - -const globEscape: (unescaped: string[]) => string[] = require('glob-escape'); // No @types/glob-escape package exists +import { Constants } from '../utilities/Constants'; const PLUGIN_NAME: string = 'CopyFilesPlugin'; const HEFT_STAGE_TAP: TapOptions<'promise'> = { @@ -34,11 +33,9 @@ const HEFT_STAGE_TAP: TapOptions<'promise'> = { stage: Number.MAX_SAFE_INTEGER / 2 // This should give us some certainty that this will run after other plugins }; -const MAX_PARALLELISM: number = 100; - -export interface ICopyFileDescriptor { +interface ICopyFileDescriptor { sourceFilePath: string; - destinationFilePath: string; + destinationFilePaths: string[]; hardlink: boolean; } @@ -48,12 +45,17 @@ export interface ICopyFilesOptions { logger: ScopedLogger; } +export interface ICopyFilesResult { + copiedFileCount: number; + linkedFileCount: number; +} + export class CopyFilesPlugin implements IHeftPlugin { public readonly pluginName: string = PLUGIN_NAME; public apply(heftSession: HeftSession, heftConfiguration: HeftConfiguration): void { - const logger: ScopedLogger = heftSession.requestScopedLogger('copy-files'); heftSession.hooks.build.tap(PLUGIN_NAME, (build: IBuildStageContext) => { + const logger: ScopedLogger = heftSession.requestScopedLogger('copy-files'); build.hooks.preCompile.tap(PLUGIN_NAME, (preCompile: IPreCompileSubstage) => { preCompile.hooks.run.tapPromise(HEFT_STAGE_TAP, async () => { await this._runCopyFilesForHeftEvent(HeftEvent.preCompile, logger, heftConfiguration); @@ -116,55 +118,63 @@ export class CopyFilesPlugin implements IHeftPlugin { return; } - const [copyCount, hardlinkCount] = await this.copyFilesAsync(copyDescriptors); + const { copiedFileCount, linkedFileCount } = await this.copyFilesAsync(copyDescriptors); const duration: number = performance.now() - startTime; logger.terminal.writeLine( - `Copied ${copyCount} file${copyCount === 1 ? '' : 's'} and linked ${hardlinkCount} ` + - `file${hardlinkCount === 1 ? '' : 's'} in ${Math.round(duration)}ms` + `Copied ${copiedFileCount} file${copiedFileCount === 1 ? '' : 's'} and ` + + `linked ${linkedFileCount} file${linkedFileCount === 1 ? '' : 's'} in ${Math.round(duration)}ms` ); } - protected async copyFilesAsync(copyDescriptors: ICopyFileDescriptor[]): Promise<[number, number]> { + protected async copyFilesAsync(copyDescriptors: ICopyFileDescriptor[]): Promise { if (copyDescriptors.length === 0) { - return [0, 0]; + return { copiedFileCount: 0, linkedFileCount: 0 }; } - let copyCount: number = 0; - let hardlinkCount: number = 0; + let copiedFileCount: number = 0; + let linkedFileCount: number = 0; await Async.forEachLimitAsync( copyDescriptors, - MAX_PARALLELISM, + Constants.maxParallelism, async (copyDescriptor: ICopyFileDescriptor) => { if (copyDescriptor.hardlink) { - // Hardlink doesn't allow passing in overwrite param, so delete ourselves - try { - await FileSystem.deleteFileAsync(copyDescriptor.destinationFilePath, { throwIfNotExists: true }); - } catch (e) { - if (!FileSystem.isFileDoesNotExistError(e)) { - throw e; + const hardlinkPromises: Promise[] = copyDescriptor.destinationFilePaths.map( + (destinationFilePath) => { + return FileSystem.createHardLinkAsync({ + linkTargetPath: copyDescriptor.sourceFilePath, + newLinkPath: destinationFilePath, + alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite + }); } - // Since the file doesn't exist, the parent folder may also not exist - await FileSystem.ensureFolderAsync(path.dirname(copyDescriptor.destinationFilePath)); - } + ); + await Promise.all(hardlinkPromises); - await FileSystem.createHardLinkAsync({ - linkTargetPath: copyDescriptor.sourceFilePath, - newLinkPath: copyDescriptor.destinationFilePath - }); - hardlinkCount++; + linkedFileCount++; } else { - // If it's a copy, simply call the copy function - await FileSystem.copyFileAsync({ - sourcePath: copyDescriptor.sourceFilePath, - destinationPath: copyDescriptor.destinationFilePath, - alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite - }); - copyCount++; + // If it's a copy, we will call the copy function + if (copyDescriptor.destinationFilePaths.length === 1) { + await FileSystem.copyFileAsync({ + sourcePath: copyDescriptor.sourceFilePath, + destinationPath: copyDescriptor.destinationFilePaths[0], + alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite + }); + } else { + await FileSystem.copyFileToManyAsync({ + sourcePath: copyDescriptor.sourceFilePath, + destinationPaths: copyDescriptor.destinationFilePaths, + alreadyExistsBehavior: AlreadyExistsBehavior.Overwrite + }); + } + + copiedFileCount++; } } ); - return [copyCount, hardlinkCount]; + return { + copiedFileCount, + linkedFileCount + }; } private async _getCopyFileDescriptorsAsync( @@ -172,40 +182,35 @@ export class CopyFilesPlugin implements IHeftPlugin { copyConfigurations: IExtendedSharedCopyConfiguration[] ): Promise { // Create a map to deduplicate and prevent double-writes - // resolvedDestinationFilePath -> [resolvedSourceFilePath, hardlink] const destinationCopyDescriptors: Map = new Map(); + // And a map to contain the actual results + const sourceCopyDescriptors: Map = new Map(); for (const copyConfiguration of copyConfigurations) { // Resolve the source folder path which is where the glob will be run from const resolvedSourceFolderPath: string = path.resolve(buildFolder, copyConfiguration.sourceFolder); // Glob extensions with a specific glob to increase perf - let sourceFileRelativePaths: Set; - if (copyConfiguration.fileExtensions?.length) { - const escapedExtensions: string[] = globEscape(copyConfiguration.fileExtensions); - const pattern: string = `**/*+(${escapedExtensions.join('|')})`; - sourceFileRelativePaths = await this._expandGlobPatternAsync( - resolvedSourceFolderPath, - pattern, - copyConfiguration.excludeGlobs - ); - } else { - sourceFileRelativePaths = new Set(); + const patternsToGlob: Set = new Set(); + for (const fileExtension of copyConfiguration.fileExtensions || []) { + const escapedExtension: string = glob.escapePath(fileExtension); + patternsToGlob.add(`**/*${escapedExtension}`); } - // Now include the other glob as well + // Now include the other globs as well for (const include of copyConfiguration.includeGlobs || []) { - const explicitlyIncludedPaths: Set = await this._expandGlobPatternAsync( - resolvedSourceFolderPath, - include, - copyConfiguration.excludeGlobs - ); - - for (const explicitlyIncludedPath of explicitlyIncludedPaths) { - sourceFileRelativePaths.add(explicitlyIncludedPath); - } + patternsToGlob.add(include); } + const sourceFileRelativePaths: Set = new Set( + await glob(Array.from(patternsToGlob), { + cwd: resolvedSourceFolderPath, + ignore: copyConfiguration.excludeGlobs, + dot: true, + onlyFiles: true + }) + ); + // Dedupe and throw if a double-write is detected for (const destinationFolderRelativePath of copyConfiguration.destinationFolders) { for (const sourceFileRelativePath of sourceFileRelativePaths) { @@ -219,13 +224,13 @@ export class CopyFilesPlugin implements IHeftPlugin { ); // Throw if a duplicate copy target with a different source or options is specified - const existingCopyDescriptor: ICopyFileDescriptor | undefined = destinationCopyDescriptors.get( - resolvedDestinationFilePath - ); - if (existingCopyDescriptor) { + const existingDestinationCopyDescriptor: + | ICopyFileDescriptor + | undefined = destinationCopyDescriptors.get(resolvedDestinationFilePath); + if (existingDestinationCopyDescriptor) { if ( - existingCopyDescriptor.sourceFilePath === resolvedSourceFilePath && - existingCopyDescriptor.hardlink === !!copyConfiguration.hardlink + existingDestinationCopyDescriptor.sourceFilePath === resolvedSourceFilePath && + existingDestinationCopyDescriptor.hardlink === !!copyConfiguration.hardlink ) { // Found a duplicate, avoid adding again continue; @@ -236,30 +241,27 @@ export class CopyFilesPlugin implements IHeftPlugin { } // Finally, add to the map and default hardlink to false - destinationCopyDescriptors.set(resolvedDestinationFilePath, { - sourceFilePath: resolvedSourceFilePath, - destinationFilePath: resolvedDestinationFilePath, - hardlink: !!copyConfiguration.hardlink - }); + let sourceCopyDescriptor: ICopyFileDescriptor | undefined = sourceCopyDescriptors.get( + resolvedSourceFilePath + ); + if (!sourceCopyDescriptor) { + sourceCopyDescriptor = { + sourceFilePath: resolvedSourceFilePath, + destinationFilePaths: [resolvedDestinationFilePath], + hardlink: !!copyConfiguration.hardlink + }; + sourceCopyDescriptors.set(resolvedSourceFilePath, sourceCopyDescriptor); + } else { + sourceCopyDescriptor.destinationFilePaths.push(resolvedDestinationFilePath); + } + + // Add to other map to allow deduping + destinationCopyDescriptors.set(resolvedDestinationFilePath, sourceCopyDescriptor); } } } // We're done with the map, grab the values and return - return Array.from(destinationCopyDescriptors.values()); - } - - private async _expandGlobPatternAsync( - resolvedSourceFolderPath: string, - pattern: string, - exclude: string[] | undefined - ): Promise> { - const results: string[] = await LegacyAdapters.convertCallbackToPromise(glob, pattern, { - cwd: resolvedSourceFolderPath, - nodir: true, - ignore: exclude - }); - - return new Set(results); + return Array.from(sourceCopyDescriptors.values()); } } diff --git a/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts b/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts index fe49a23fa0f..bdf87b8b69b 100644 --- a/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts +++ b/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts @@ -122,16 +122,18 @@ export class CopyStaticAssetsPlugin extends CopyFilesPlugin { }); const copyAsset: (assetPath: string) => Promise = async (assetPath: string) => { - const [copyCount] = await this.copyFilesAsync( - resolvedDestinationFolderPaths.map((resolvedDestinationFolderPath) => { - return { - sourceFilePath: path.join(resolvedSourceFolderPath, assetPath), - destinationFilePath: path.join(resolvedDestinationFolderPath, assetPath), - hardlink: false - }; - }) + const { copiedFileCount } = await this.copyFilesAsync([ + { + sourceFilePath: path.join(resolvedSourceFolderPath, assetPath), + destinationFilePaths: resolvedDestinationFolderPaths.map((resolvedDestinationFolderPath) => { + return path.join(resolvedDestinationFolderPath, assetPath); + }), + hardlink: false + } + ]); + logger.terminal.writeLine( + `Copied ${copiedFileCount} static asset${copiedFileCount === 1 ? '' : 's'}` ); - logger.terminal.writeLine(`Copied ${copyCount} static asset${copyCount === 1 ? '' : 's'}`); }; watcher.on('add', copyAsset); From 931d1724a4bd81c033a684dd7f5f802e73b06c50 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:14:36 -0800 Subject: [PATCH 16/35] Add missing files for added fast-glob dep --- apps/heft/package.json | 1 + common/config/rush/pnpm-lock.yaml | 62 ++++++++++++++++++++++++++++++ common/config/rush/repo-state.json | 2 +- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/apps/heft/package.json b/apps/heft/package.json index a78e395f5e2..9f6945e8145 100644 --- a/apps/heft/package.json +++ b/apps/heft/package.json @@ -46,6 +46,7 @@ "chokidar": "~3.4.0", "glob-escape": "~0.0.2", "glob": "~7.0.5", + "fast-glob": "~3.2.4", "jest-snapshot": "~25.4.0", "node-sass": "4.14.1", "postcss-modules": "~1.5.0", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b2a8cb597f5..a4c79969d1b 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -109,6 +109,7 @@ importers: '@types/webpack': 4.41.24 argparse: 1.0.10 chokidar: 3.4.3 + fast-glob: 3.2.4 glob: 7.0.6 glob-escape: 0.0.2 jest-snapshot: 25.4.0 @@ -165,6 +166,7 @@ importers: argparse: ~1.0.9 chokidar: ~3.4.0 colors: ~1.2.1 + fast-glob: ~3.2.4 glob: ~7.0.5 glob-escape: ~0.0.2 jest-snapshot: ~25.4.0 @@ -3034,6 +3036,30 @@ packages: /@microsoft/tsdoc/0.12.21: resolution: integrity: sha512-j+9OJ0A0buZZaUn6NxeHUVpoa05tY2PgVs7kXJhJQiKRB0G1zQqbJxer3T7jWtzpqQWP89OBDluyIeyTsMk8Sg== + /@nodelib/fs.scandir/2.1.3: + dependencies: + '@nodelib/fs.stat': 2.0.3 + run-parallel: 1.1.10 + dev: false + engines: + node: '>= 8' + resolution: + integrity: sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw== + /@nodelib/fs.stat/2.0.3: + dev: false + engines: + node: '>= 8' + resolution: + integrity: sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA== + /@nodelib/fs.walk/1.2.4: + dependencies: + '@nodelib/fs.scandir': 2.1.3 + fastq: 1.9.0 + dev: false + engines: + node: '>= 8' + resolution: + integrity: sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ== /@pnpm/error/1.3.1: dev: false engines: @@ -6966,6 +6992,19 @@ packages: /fast-deep-equal/3.1.3: resolution: integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + /fast-glob/3.2.4: + dependencies: + '@nodelib/fs.stat': 2.0.3 + '@nodelib/fs.walk': 1.2.4 + glob-parent: 5.1.1 + merge2: 1.4.1 + micromatch: 4.0.2 + picomatch: 2.2.2 + dev: false + engines: + node: '>=8' + resolution: + integrity: sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ== /fast-json-stable-stringify/2.1.0: resolution: integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -6982,6 +7021,12 @@ packages: dev: false resolution: integrity: sha512-483XLLxTVIwWK3QTrMGRqUfUpoOs/0hbQrl2oz4J0pAcm3A3bu84wxTFqGqkJzewCLdME38xJLJAxBABfQT8sQ== + /fastq/1.9.0: + dependencies: + reusify: 1.0.4 + dev: false + resolution: + integrity: sha512-i7FVWL8HhVY+CTkwFxkN2mk3h+787ixS5S63eb78diVRc1MCssarHq3W5cj0av7YDSwmaV928RNag+U1etRQ7w== /faye-websocket/0.10.0: dependencies: websocket-driver: 0.6.5 @@ -9780,6 +9825,12 @@ packages: node: '>=0.10' resolution: integrity: sha1-+kT4siYmFaty8ICKQB1HinDjlNs= + /merge2/1.4.1: + dev: false + engines: + node: '>= 8' + resolution: + integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== /methods/1.1.2: engines: node: '>= 0.6' @@ -11735,6 +11786,13 @@ packages: node: '>= 4' resolution: integrity: sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs= + /reusify/1.0.4: + dev: false + engines: + iojs: '>=1.0.0' + node: '>=0.10.0' + resolution: + integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== /rimraf/2.6.3: dependencies: glob: 7.1.6 @@ -11769,6 +11827,10 @@ packages: node: '>=0.12.0' resolution: integrity: sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== + /run-parallel/1.1.10: + dev: false + resolution: + integrity: sha512-zb/1OuZ6flOlH6tQyMPUrE3x3Ulxjlo9WIVXR4yVYi4H9UXQaeIsPbLn2R3O3vQCnDKkAl2qHiuocKKX4Tz/Sw== /run-queue/1.0.3: dependencies: aproba: 1.2.0 diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 2069cf2cb51..b970ac245f4 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "2a5a228009553e6527a411e2c32f2dc4b04c298c", + "pnpmShrinkwrapHash": "04d9dbd5e3fb33326a517d3208d25de41a998dc5", "preferredVersionsHash": "2519e88d149a9cb84227de92c71a8d8063bdcfd4" } From 4eee88ebf7d4d9a7a1c88f72d432edc443d8f8e2 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:14:53 -0800 Subject: [PATCH 17/35] Use new constant for parallelization --- apps/heft/src/plugins/DeleteGlobsPlugin.ts | 29 ++++++++++++---------- apps/heft/src/utilities/Constants.ts | 2 ++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/apps/heft/src/plugins/DeleteGlobsPlugin.ts b/apps/heft/src/plugins/DeleteGlobsPlugin.ts index d60173879be..489c83a33d2 100644 --- a/apps/heft/src/plugins/DeleteGlobsPlugin.ts +++ b/apps/heft/src/plugins/DeleteGlobsPlugin.ts @@ -20,6 +20,7 @@ import { IPostBuildSubstage, IPreCompileSubstage } from '../stages/BuildStage'; +import { Constants } from '../utilities/Constants'; const globEscape: (unescaped: string) => string = require('glob-escape'); // No @types/glob-escape package exists @@ -29,8 +30,6 @@ const HEFT_STAGE_TAP: TapOptions<'promise'> = { stage: Number.MIN_SAFE_INTEGER }; -const MAX_PARALLELISM: number = 100; - export class DeleteGlobsPlugin implements IHeftPlugin { public readonly pluginName: string = PLUGIN_NAME; @@ -101,19 +100,23 @@ export class DeleteGlobsPlugin implements IHeftPlugin { } } - await Async.forEachLimitAsync(Array.from(pathsToDelete), MAX_PARALLELISM, async (pathToDelete) => { - try { - FileSystem.deleteFile(pathToDelete, { throwIfNotExists: true }); - logger.terminal.writeVerboseLine(`Deleted "${pathToDelete}"`); - deletedFiles++; - } catch (error) { - if (FileSystem.exists(pathToDelete)) { - FileSystem.deleteFolder(pathToDelete); - logger.terminal.writeVerboseLine(`Deleted folder "${pathToDelete}"`); - deletedFolders++; + await Async.forEachLimitAsync( + Array.from(pathsToDelete), + Constants.maxParallelism, + async (pathToDelete) => { + try { + FileSystem.deleteFile(pathToDelete, { throwIfNotExists: true }); + logger.terminal.writeVerboseLine(`Deleted "${pathToDelete}"`); + deletedFiles++; + } catch (error) { + if (FileSystem.exists(pathToDelete)) { + FileSystem.deleteFolder(pathToDelete); + logger.terminal.writeVerboseLine(`Deleted folder "${pathToDelete}"`); + deletedFolders++; + } } } - }); + ); if (deletedFiles > 0 || deletedFolders > 0) { logger.terminal.writeLine( diff --git a/apps/heft/src/utilities/Constants.ts b/apps/heft/src/utilities/Constants.ts index 8648cd7d02d..04520c85cde 100644 --- a/apps/heft/src/utilities/Constants.ts +++ b/apps/heft/src/utilities/Constants.ts @@ -11,4 +11,6 @@ export class Constants { public static pluginParameterLongName: string = '--plugin'; public static debugParameterLongName: string = '--debug'; + + public static maxParallelism: number = 100; } From 15efb638c13eda7d90960029c5adb5cc421613f6 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:15:05 -0800 Subject: [PATCH 18/35] Remove unused dep --- apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts b/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts index 4fcea855021..3905b6a53c7 100644 --- a/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts +++ b/apps/heft/src/utilities/fileSystem/TypeScriptCachedFileSystem.ts @@ -2,7 +2,6 @@ // See LICENSE in the project root for license information. import * as fs from 'fs'; -import * as nodeJsPath from 'path'; import { Encoding, Text, From 8d2c4ac9ecc99675d9cfba3268138addfd605b89 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:15:16 -0800 Subject: [PATCH 19/35] Fixed change file --- .../@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json b/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json index d9f35b4c476..17366b8f095 100644 --- a/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json +++ b/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/heft", - "comment": "Add new default Heft action \"copyGlobs\" to copy or hardlink files during specified Heft events", + "comment": "Add new built-in Heft action \"copyGlobs\" to copy or hardlink files during specified Heft events", "type": "minor" } ], From 37b738325f5001541c7c5e64c64456d9f14472df Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:15:26 -0800 Subject: [PATCH 20/35] Another bit for fast-glob --- common/config/rush/nonbrowser-approved-packages.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index cc1186ff6e7..ba757e7869f 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -378,6 +378,10 @@ "name": "express", "allowedCategories": [ "libraries" ] }, + { + "name": "fast-glob", + "allowedCategories": [ "libraries" ] + }, { "name": "file-loader", "allowedCategories": [ "tests" ] From 681fc819e07eb49c905690303cee7e3e68aed181 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 19:23:25 -0800 Subject: [PATCH 21/35] Ensure parent folder in multi-file copy --- libraries/node-core-library/src/FileSystem.ts | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index 32b99b4deee..bb4736c8439 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -977,14 +977,16 @@ export class FileSystem { } const flags: string = writeFlags.join(); - const createPipePromise: ( + const createPipePromise: (sourceStream: fs.ReadStream, destinationPath: string) => Promise = ( sourceStream: fs.ReadStream, - destinationStream: fs.WriteStream - ) => Promise = (sourceStream: fs.ReadStream, destinationStream: fs.WriteStream) => { + destinationPath: string + ) => { return new Promise((resolve: () => void, reject: (error: Error) => void) => { + const destinationStream: fs.WriteStream = fs.createWriteStream(destinationPath); + const streamsToDestroy: fs.WriteStream[] = [destinationStream]; sourceStream.on('error', (e: Error) => { - if (destinationStream) { - destinationStream.destroy(); + for (const streamToDestroy of streamsToDestroy) { + streamToDestroy.destroy(); } reject(e); }); @@ -993,15 +995,31 @@ export class FileSystem { .on('close', () => { resolve(); }) - .on('error', (e: Error) => { - if ( + .on('error', async (e: Error) => { + if (FileSystem.isNotExistError(e)) { + destinationStream.destroy(); + await FileSystem.ensureFolderAsync(nodeJsPath.dirname(destinationStream.path as string)); + const retryDestinationStream: fs.WriteStream = fsx.createWriteStream(destinationPath, { + flags + }); + streamsToDestroy.push(retryDestinationStream); + sourceStream + .pipe(retryDestinationStream) + .on('close', () => { + resolve(); + }) + .on('error', (e2: Error) => { + reject(e2); + }); + } else if ( options.alreadyExistsBehavior === AlreadyExistsBehavior.Ignore && FileSystem.isErrnoException(e) && - (e as NodeJS.ErrnoException).code === 'EEXIST' + e.code === 'EEXIST' ) { resolve(); + } else { + reject(e); } - reject(e); }); }); }; @@ -1010,7 +1028,7 @@ export class FileSystem { const uniqueDestinationPaths: Set = new Set(options.destinationPaths); const pipePromises: Promise[] = []; for (const destinationPath of uniqueDestinationPaths.values()) { - pipePromises.push(createPipePromise(sourceStream, fsx.createWriteStream(destinationPath, { flags }))); + pipePromises.push(createPipePromise(sourceStream, destinationPath)); } await Promise.all(pipePromises); From 801b81c1f8140b8d0f446728c9f889788a9bacae Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 19:24:33 -0800 Subject: [PATCH 22/35] Rush change --- .../danade-copy-plugin_2020-11-10-03-24.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-03-24.json diff --git a/common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-03-24.json b/common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-03-24.json new file mode 100644 index 00000000000..ef42fd99b6b --- /dev/null +++ b/common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-03-24.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/node-core-library", + "comment": "Add new \"copyFileToMany\" API to copy a single file to multiple locations", + "type": "minor" + } + ], + "packageName": "@rushstack/node-core-library", + "email": "3473356+D4N14L@users.noreply.github.com" +} \ No newline at end of file From 144cd487afaf9b4ec8c04ceed64fd0bfc7c1b7ee Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 19:42:08 -0800 Subject: [PATCH 23/35] Move watch functionality over to CopyFilesPlugin --- apps/heft/src/plugins/CopyFilesPlugin.ts | 99 ++++++++++++++++--- .../src/plugins/CopyStaticAssetsPlugin.ts | 85 +--------------- 2 files changed, 86 insertions(+), 98 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index 86b9836715b..cb590fa9c9c 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as chokidar from 'chokidar'; import * as path from 'path'; import glob from 'fast-glob'; import { performance } from 'perf_hooks'; @@ -43,6 +44,7 @@ export interface ICopyFilesOptions { buildFolder: string; copyConfigurations: IExtendedSharedCopyConfiguration[]; logger: ScopedLogger; + watchMode: boolean; } export interface ICopyFilesResult { @@ -100,7 +102,8 @@ export class CopyFilesPlugin implements IHeftPlugin { await this.runCopyAsync({ buildFolder: heftConfiguration.buildFolder, copyConfigurations, - logger + logger, + watchMode: false }); } @@ -124,6 +127,11 @@ export class CopyFilesPlugin implements IHeftPlugin { `Copied ${copiedFileCount} file${copiedFileCount === 1 ? '' : 's'} and ` + `linked ${linkedFileCount} file${linkedFileCount === 1 ? '' : 's'} in ${Math.round(duration)}ms` ); + + // Then enter watch mode if requested + if (options.watchMode) { + await this._runWatchAsync(options); + } } protected async copyFilesAsync(copyDescriptors: ICopyFileDescriptor[]): Promise { @@ -189,21 +197,8 @@ export class CopyFilesPlugin implements IHeftPlugin { for (const copyConfiguration of copyConfigurations) { // Resolve the source folder path which is where the glob will be run from const resolvedSourceFolderPath: string = path.resolve(buildFolder, copyConfiguration.sourceFolder); - - // Glob extensions with a specific glob to increase perf - const patternsToGlob: Set = new Set(); - for (const fileExtension of copyConfiguration.fileExtensions || []) { - const escapedExtension: string = glob.escapePath(fileExtension); - patternsToGlob.add(`**/*${escapedExtension}`); - } - - // Now include the other globs as well - for (const include of copyConfiguration.includeGlobs || []) { - patternsToGlob.add(include); - } - const sourceFileRelativePaths: Set = new Set( - await glob(Array.from(patternsToGlob), { + await glob(this._getIncludedGlobPatterns(copyConfiguration), { cwd: resolvedSourceFolderPath, ignore: copyConfiguration.excludeGlobs, dot: true, @@ -264,4 +259,78 @@ export class CopyFilesPlugin implements IHeftPlugin { // We're done with the map, grab the values and return return Array.from(sourceCopyDescriptors.values()); } + + private _getIncludedGlobPatterns(copyConfiguration: IExtendedSharedCopyConfiguration): string[] { + // Glob extensions with a specific glob to increase perf + const patternsToGlob: Set = new Set(); + for (const fileExtension of copyConfiguration.fileExtensions || []) { + const escapedExtension: string = glob.escapePath(fileExtension); + patternsToGlob.add(`**/*${escapedExtension}`); + } + + // Now include the other globs as well + for (const include of copyConfiguration.includeGlobs || []) { + patternsToGlob.add(include); + } + + return Array.from(patternsToGlob); + } + + private async _runWatchAsync(options: ICopyFilesOptions): Promise { + const { buildFolder, copyConfigurations, logger } = options; + + for (const copyConfiguration of copyConfigurations) { + // Obtain the glob patterns to provide to the watcher + const globsToWatch: string[] = this._getIncludedGlobPatterns(copyConfiguration); + if (globsToWatch.length) { + const resolvedSourceFolderPath: string = path.join(buildFolder, copyConfiguration.sourceFolder); + const resolvedDestinationFolderPaths: string[] = copyConfiguration.destinationFolders.map( + (destinationFolder) => { + return path.join(buildFolder, destinationFolder); + } + ); + + const watcher: chokidar.FSWatcher = chokidar.watch(globsToWatch, { + cwd: resolvedSourceFolderPath, + ignoreInitial: true, + ignored: copyConfiguration.excludeGlobs + }); + + const copyAsset: (assetPath: string) => Promise = async (assetPath: string) => { + const { copiedFileCount, linkedFileCount } = await this.copyFilesAsync([ + { + sourceFilePath: path.join(resolvedSourceFolderPath, assetPath), + destinationFilePaths: resolvedDestinationFolderPaths.map((resolvedDestinationFolderPath) => { + return path.join( + resolvedDestinationFolderPath, + !!copyConfiguration.flatten ? path.basename(assetPath) : assetPath + ); + }), + hardlink: !!copyConfiguration.hardlink + } + ]); + logger.terminal.writeLine( + !!copyConfiguration.hardlink + ? `Linked ${linkedFileCount} file${linkedFileCount === 1 ? '' : 's'}` + : `Copied ${copiedFileCount} file${copiedFileCount === 1 ? '' : 's'}` + ); + }; + + watcher.on('add', copyAsset); + watcher.on('change', copyAsset); + watcher.on('unlink', (assetPath) => { + let deleteCount: number = 0; + for (const resolvedDestinationFolder of resolvedDestinationFolderPaths) { + FileSystem.deleteFile(path.resolve(resolvedDestinationFolder, assetPath)); + deleteCount++; + } + logger.terminal.writeLine(`Deleted ${deleteCount} file${deleteCount === 1 ? '' : 's'}`); + }); + } + } + + return new Promise(() => { + /* never resolve */ + }); + } } diff --git a/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts b/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts index bdf87b8b69b..bbbc4804941 100644 --- a/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts +++ b/apps/heft/src/plugins/CopyStaticAssetsPlugin.ts @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { FileSystem, Terminal } from '@rushstack/node-core-library'; -import * as path from 'path'; -import * as chokidar from 'chokidar'; +import { Terminal } from '@rushstack/node-core-library'; import { HeftSession } from '../pluginFramework/HeftSession'; import { HeftConfiguration } from '../configuration/HeftConfiguration'; @@ -11,16 +9,10 @@ import { IBuildStageContext, ICompileSubstage } from '../stages/BuildStage'; import { ScopedLogger } from '../pluginFramework/logging/ScopedLogger'; import { CoreConfigFiles, IExtendedSharedCopyConfiguration } from '../utilities/CoreConfigFiles'; import { ITypeScriptConfigurationJson } from './TypeScriptPlugin/TypeScriptPlugin'; -import { CopyFilesPlugin, ICopyFilesOptions } from './CopyFilesPlugin'; - -const globEscape: (unescaped: string[]) => string[] = require('glob-escape'); // No @types/glob-escape package exists +import { CopyFilesPlugin } from './CopyFilesPlugin'; const PLUGIN_NAME: string = 'CopyStaticAssetsPlugin'; -interface ICopyStaticAssetsOptions extends ICopyFilesOptions { - watchMode: boolean; -} - export class CopyStaticAssetsPlugin extends CopyFilesPlugin { /** * @override @@ -79,77 +71,4 @@ export class CopyStaticAssetsPlugin extends CopyFilesPlugin { hardlink: false }; } - - /** - * @override - */ - protected async runCopyAsync(options: ICopyStaticAssetsOptions): Promise { - // First, run the actual copy - await super.runCopyAsync(options); - - // Then enter watch mode if requested - if (options.watchMode) { - await this._runWatchAsync(options); - } - } - - private async _runWatchAsync(options: ICopyStaticAssetsOptions): Promise { - const { buildFolder, copyConfigurations, logger } = options; - const [copyStaticAssetsConfiguration] = copyConfigurations; - - // Obtain the glob patterns to provide to the watcher - const globsToWatch: string[] = [...(copyStaticAssetsConfiguration.includeGlobs || [])]; - if (copyStaticAssetsConfiguration.fileExtensions?.length) { - const escapedExtensions: string[] = globEscape(copyStaticAssetsConfiguration.fileExtensions); - globsToWatch.push(`**/*+(${escapedExtensions.join('|')})`); - } - - if (globsToWatch.length) { - const resolvedSourceFolderPath: string = path.join( - buildFolder, - copyStaticAssetsConfiguration.sourceFolder - ); - const resolvedDestinationFolderPaths: string[] = copyStaticAssetsConfiguration.destinationFolders.map( - (destinationFolder) => { - return path.join(buildFolder, destinationFolder); - } - ); - - const watcher: chokidar.FSWatcher = chokidar.watch(globsToWatch, { - cwd: resolvedSourceFolderPath, - ignoreInitial: true, - ignored: copyStaticAssetsConfiguration.excludeGlobs - }); - - const copyAsset: (assetPath: string) => Promise = async (assetPath: string) => { - const { copiedFileCount } = await this.copyFilesAsync([ - { - sourceFilePath: path.join(resolvedSourceFolderPath, assetPath), - destinationFilePaths: resolvedDestinationFolderPaths.map((resolvedDestinationFolderPath) => { - return path.join(resolvedDestinationFolderPath, assetPath); - }), - hardlink: false - } - ]); - logger.terminal.writeLine( - `Copied ${copiedFileCount} static asset${copiedFileCount === 1 ? '' : 's'}` - ); - }; - - watcher.on('add', copyAsset); - watcher.on('change', copyAsset); - watcher.on('unlink', (assetPath) => { - let deleteCount: number = 0; - for (const resolvedDestinationFolder of resolvedDestinationFolderPaths) { - FileSystem.deleteFile(path.resolve(resolvedDestinationFolder, assetPath)); - deleteCount++; - } - logger.terminal.writeLine(`Deleted ${deleteCount} static asset${deleteCount === 1 ? '' : 's'}`); - }); - } - - return new Promise(() => { - /* never resolve */ - }); - } } From 9425c87fca7840411478e486fcf7278b8242e3d4 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 9 Nov 2020 19:58:21 -0800 Subject: [PATCH 24/35] Linting --- apps/heft/src/plugins/CopyFilesPlugin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index cb590fa9c9c..e01d2cf36c4 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -303,14 +303,14 @@ export class CopyFilesPlugin implements IHeftPlugin { destinationFilePaths: resolvedDestinationFolderPaths.map((resolvedDestinationFolderPath) => { return path.join( resolvedDestinationFolderPath, - !!copyConfiguration.flatten ? path.basename(assetPath) : assetPath + copyConfiguration.flatten ? path.basename(assetPath) : assetPath ); }), hardlink: !!copyConfiguration.hardlink } ]); logger.terminal.writeLine( - !!copyConfiguration.hardlink + copyConfiguration.hardlink ? `Linked ${linkedFileCount} file${linkedFileCount === 1 ? '' : 's'}` : `Copied ${copiedFileCount} file${copiedFileCount === 1 ? '' : 's'}` ); From 6c3f86e26ae1d98b81c5cf289ce9e68237ee5272 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 00:00:21 -0800 Subject: [PATCH 25/35] Update an error message. --- libraries/node-core-library/src/FileSystem.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index bb4736c8439..dbe8d36f95f 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -959,7 +959,7 @@ export class FileSystem { if (FileSystem.getStatistics(options.sourcePath).isDirectory()) { throw new Error( - 'The specified path refers to a folder; this operation expects a file object:\n' + options.sourcePath + 'The specified path refers to a folder; this operation expects a file path:\n' + options.sourcePath ); } From c6317b8738159a23d15521a5904db7b90bb49db6 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 00:00:38 -0800 Subject: [PATCH 26/35] Update changelog description. --- .../@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json b/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json index 17366b8f095..531e93a6fef 100644 --- a/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json +++ b/common/changes/@rushstack/heft/danade-copy-plugin_2020-11-02-23-47.json @@ -2,10 +2,10 @@ "changes": [ { "packageName": "@rushstack/heft", - "comment": "Add new built-in Heft action \"copyGlobs\" to copy or hardlink files during specified Heft events", + "comment": "Add new built-in Heft action \"copyFiles\" to copy or hardlink files during specified Heft events", "type": "minor" } ], "packageName": "@rushstack/heft", "email": "3473356+D4N14L@users.noreply.github.com" -} \ No newline at end of file +} From bd17301c347559544900141fb5545ec19d36815a Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 00:00:55 -0800 Subject: [PATCH 27/35] Simplify iteration over a Set. --- libraries/node-core-library/src/FileSystem.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index dbe8d36f95f..deabc1805cd 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -1027,7 +1027,7 @@ export class FileSystem { const sourceStream: fs.ReadStream = fsx.createReadStream(options.sourcePath); const uniqueDestinationPaths: Set = new Set(options.destinationPaths); const pipePromises: Promise[] = []; - for (const destinationPath of uniqueDestinationPaths.values()) { + for (const destinationPath of uniqueDestinationPaths) { pipePromises.push(createPipePromise(sourceStream, destinationPath)); } From 31d341e5b66a1adb0b680f23ee4a26d600edd0d4 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 00:19:26 -0800 Subject: [PATCH 28/35] Clean up IFileSystemCopyFileToManyOptions interface --- common/reviews/api/node-core-library.api.md | 14 ++++++---- libraries/node-core-library/src/FileSystem.ts | 28 +++++++------------ libraries/node-core-library/src/index.ts | 19 +++++++------ 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md index 54c25d82680..02d022829ba 100644 --- a/common/reviews/api/node-core-library.api.md +++ b/common/reviews/api/node-core-library.api.md @@ -277,13 +277,17 @@ export interface IExecutableSpawnSyncOptions extends IExecutableResolveOptions { timeoutMs?: number; } -// @public -export interface IFileSystemCopyFileOptions { +// @public (undocumented) +export interface IFileSystemCopyFileBaseOptions { alreadyExistsBehavior?: AlreadyExistsBehavior; - destinationPath: string; sourcePath: string; } +// @public +export interface IFileSystemCopyFileOptions extends IFileSystemCopyFileBaseOptions { + destinationPath: string; +} + // @public export interface IFileSystemCopyFilesAsyncOptions { alreadyExistsBehavior?: AlreadyExistsBehavior; @@ -300,10 +304,8 @@ export interface IFileSystemCopyFilesOptions extends IFileSystemCopyFilesAsyncOp } // @public -export interface IFileSystemCopyFileToManyOptions extends Omit { - alreadyExistsBehavior?: AlreadyExistsBehavior; +export interface IFileSystemCopyFileToManyOptions extends IFileSystemCopyFileBaseOptions { destinationPaths: string[]; - sourcePath: string; } // @public diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index deabc1805cd..6fc767a9405 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -105,22 +105,15 @@ export interface IFileSystemMoveOptions { } /** - * The options for {@link FileSystem.copyFile} * @public */ -export interface IFileSystemCopyFileOptions { +export interface IFileSystemCopyFileBaseOptions { /** * The path of the existing object to be copied. * The path may be absolute or relative. */ sourcePath: string; - /** - * The path that the object will be copied to. - * The path may be absolute or relative. - */ - destinationPath: string; - /** * Specifies what to do if the target object already exists. * @defaultValue {@link AlreadyExistsBehavior.Overwrite} @@ -132,25 +125,24 @@ export interface IFileSystemCopyFileOptions { * The options for {@link FileSystem.copyFile} * @public */ -export interface IFileSystemCopyFileToManyOptions - extends Omit { +export interface IFileSystemCopyFileOptions extends IFileSystemCopyFileBaseOptions { /** - * The path of the existing object to be copied. + * The path that the object will be copied to. * The path may be absolute or relative. */ - sourcePath: string; + destinationPath: string; +} +/** + * The options for {@link FileSystem.copyFile} + * @public + */ +export interface IFileSystemCopyFileToManyOptions extends IFileSystemCopyFileBaseOptions { /** * The path that the object will be copied to. * The path may be absolute or relative. */ destinationPaths: string[]; - - /** - * Specifies what to do if the target object already exists. - * @defaultValue {@link AlreadyExistsBehavior.Overwrite} - */ - alreadyExistsBehavior?: AlreadyExistsBehavior; } /** diff --git a/libraries/node-core-library/src/index.ts b/libraries/node-core-library/src/index.ts index 213581818b9..695972d024b 100644 --- a/libraries/node-core-library/src/index.ts +++ b/libraries/node-core-library/src/index.ts @@ -57,20 +57,21 @@ export { Sort } from './Sort'; export { AlreadyExistsBehavior, FileSystem, + FileSystemCopyFilesAsyncFilter, + FileSystemCopyFilesFilter, FileSystemStats, - IFileSystemReadFolderOptions, - IFileSystemWriteFileOptions, - IFileSystemReadFileOptions, - IFileSystemMoveOptions, + IFileSystemCopyFileBaseOptions, IFileSystemCopyFileOptions, + IFileSystemCopyFilesAsyncOptions, + IFileSystemCopyFilesOptions, IFileSystemCopyFileToManyOptions, + IFileSystemCreateLinkOptions, IFileSystemDeleteFileOptions, + IFileSystemMoveOptions, + IFileSystemReadFileOptions, + IFileSystemReadFolderOptions, IFileSystemUpdateTimeParameters, - IFileSystemCreateLinkOptions, - IFileSystemCopyFilesAsyncOptions, - IFileSystemCopyFilesOptions, - FileSystemCopyFilesAsyncFilter, - FileSystemCopyFilesFilter + IFileSystemWriteFileOptions } from './FileSystem'; export { FileWriter, IFileWriterFlags } from './FileWriter'; export { LegacyAdapters, LegacyCallback } from './LegacyAdapters'; From 0a36bcaf2b52da0a8f54eb0e930eb0d1a0400795 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 12:14:56 -0800 Subject: [PATCH 29/35] Fix an issue where a missing link target would be interpreted as a folder not found issue. --- libraries/node-core-library/src/FileSystem.ts | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index 6fc767a9405..640804e5c08 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -1249,13 +1249,15 @@ export class FileSystem { default: throw error; } - } else if (FileSystem.isNotExistError(error)) { - this.ensureFolder(nodeJsPath.dirname(options.newLinkPath)); } else { - throw error; + const linkTargetExists: boolean = FileSystem.exists(options.linkTargetPath); + if (FileSystem.isNotExistError(error) && linkTargetExists) { + this.ensureFolder(nodeJsPath.dirname(options.newLinkPath)); + this.createHardLink(options); + } else { + throw error; + } } - - this.createHardLink(options); } }); } @@ -1279,13 +1281,15 @@ export class FileSystem { default: throw error; } - } else if (FileSystem.isNotExistError(error)) { - await this.ensureFolderAsync(nodeJsPath.dirname(options.newLinkPath)); } else { - throw error; + const linkTargetExists: boolean = await FileSystem.exists(options.linkTargetPath); + if (FileSystem.isNotExistError(error) && linkTargetExists) { + await this.ensureFolderAsync(nodeJsPath.dirname(options.newLinkPath)); + await this.createHardLinkAsync(options); + } else { + throw error; + } } - - await this.createHardLinkAsync(options); } }); } From 2b8ff8c095803e8edb82cf7beb74356b12079381 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 12:16:11 -0800 Subject: [PATCH 30/35] Make the stream error handling in copyFileToManyAsync synchronous. --- libraries/node-core-library/src/FileSystem.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/node-core-library/src/FileSystem.ts b/libraries/node-core-library/src/FileSystem.ts index 640804e5c08..abd8d56ebb9 100644 --- a/libraries/node-core-library/src/FileSystem.ts +++ b/libraries/node-core-library/src/FileSystem.ts @@ -987,10 +987,10 @@ export class FileSystem { .on('close', () => { resolve(); }) - .on('error', async (e: Error) => { + .on('error', (e: Error) => { if (FileSystem.isNotExistError(e)) { destinationStream.destroy(); - await FileSystem.ensureFolderAsync(nodeJsPath.dirname(destinationStream.path as string)); + FileSystem.ensureFolder(nodeJsPath.dirname(destinationStream.path as string)); const retryDestinationStream: fs.WriteStream = fsx.createWriteStream(destinationPath, { flags }); From 4945b7976261741d2f6590c1a2f6926d30db94b2 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 12:19:28 -0800 Subject: [PATCH 31/35] Document what some maps are in CopyFilesPlugin --- apps/heft/src/plugins/CopyFilesPlugin.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index e01d2cf36c4..30913cb818f 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -189,9 +189,10 @@ export class CopyFilesPlugin implements IHeftPlugin { buildFolder: string, copyConfigurations: IExtendedSharedCopyConfiguration[] ): Promise { - // Create a map to deduplicate and prevent double-writes + // Create a map to deduplicate and prevent double-writes. The key in this map is the copy/link destination + // file path const destinationCopyDescriptors: Map = new Map(); - // And a map to contain the actual results + // And a map to contain the actual results. The key in this map is the copy/link source file path const sourceCopyDescriptors: Map = new Map(); for (const copyConfiguration of copyConfigurations) { From 113bbea810bfc6192d8289581f4cf255e9b9555d Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 14:06:03 -0800 Subject: [PATCH 32/35] Better optmize the file extensions glob --- apps/heft/src/plugins/CopyFilesPlugin.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/apps/heft/src/plugins/CopyFilesPlugin.ts b/apps/heft/src/plugins/CopyFilesPlugin.ts index 30913cb818f..d05705d2602 100644 --- a/apps/heft/src/plugins/CopyFilesPlugin.ts +++ b/apps/heft/src/plugins/CopyFilesPlugin.ts @@ -262,11 +262,26 @@ export class CopyFilesPlugin implements IHeftPlugin { } private _getIncludedGlobPatterns(copyConfiguration: IExtendedSharedCopyConfiguration): string[] { - // Glob extensions with a specific glob to increase perf const patternsToGlob: Set = new Set(); + + // Glob file extensions with a specific glob to increase perf + const escapedFileExtensions: Set = new Set(); for (const fileExtension of copyConfiguration.fileExtensions || []) { - const escapedExtension: string = glob.escapePath(fileExtension); - patternsToGlob.add(`**/*${escapedExtension}`); + let escapedFileExtension: string; + if (fileExtension.charAt(0) === '.') { + escapedFileExtension = fileExtension.substr(1); + } else { + escapedFileExtension = fileExtension; + } + + escapedFileExtension = glob.escapePath(escapedFileExtension); + escapedFileExtensions.add(escapedFileExtension); + } + + if (escapedFileExtensions.size > 1) { + patternsToGlob.add(`**/*.{${Array.from(escapedFileExtensions).join(',')}}`); + } else if (escapedFileExtensions.size === 1) { + patternsToGlob.add(`**/*.${Array.from(escapedFileExtensions)[0]}`); } // Now include the other globs as well From 88ddbcaf8bcca7d536c3d834bcb4ef4c0e0d4d3d Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 14:24:11 -0800 Subject: [PATCH 33/35] Include copyFiles in the heft.json template. --- apps/heft/src/schemas/heft.schema.json | 4 +- apps/heft/src/templates/heft.json | 63 ++++++++++++++++++++++ apps/heft/src/utilities/CoreConfigFiles.ts | 4 +- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/apps/heft/src/schemas/heft.schema.json b/apps/heft/src/schemas/heft.schema.json index 1baa67da614..315b2308249 100644 --- a/apps/heft/src/schemas/heft.schema.json +++ b/apps/heft/src/schemas/heft.schema.json @@ -89,13 +89,13 @@ "properties": { "sourceFolder": { "type": "string", - "description": "The folder from which files should be copied.", + "description": "", "pattern": "[^\\\\]" }, "destinationFolders": { "type": "array", - "description": "The folder(s) to which files should be copied.", + "description": "Folder(s) to which files should be copied, relative to the project root.", "items": { "type": "string", "pattern": "[^\\\\]" diff --git a/apps/heft/src/templates/heft.json b/apps/heft/src/templates/heft.json index 65ad672d093..f465d03c06d 100644 --- a/apps/heft/src/templates/heft.json +++ b/apps/heft/src/templates/heft.json @@ -40,6 +40,69 @@ // "lib-esnext", // "temp" // ] + // }, + // + // { + // /** + // * The kind of built-in operation that should be performed. + // * The "copyFiles" action copies files that match the specified patterns. + // */ + // "actionKind": "copyFiles", + // + // /** + // * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + // * occur at the end of the stage of the Heft run. + // */ + // "heftEvent": "pre-compile", + // + // /** + // * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + // * configs. + // */ + // "actionId": "defaultCopy", + // + // /** + // * An array of copy operations to run perform during the specified Heft event. + // */ + // "copyOperations": [ + // { + // /** + // * The folder from which files should be copied, relative to the project root. + // */ + // "sourceFolder": "src", + // + // /** + // * Folder(s) to which files should be copied, relative to the project root. + // */ + // "destinationFolders": ["dist/assets"], + // + // /** + // * File extensions that should be copied from the source folder to the destination folder(s) + // */ + // "fileExtensions": [".jpg", ".png"], + // + // /** + // * Globs that should be explicitly excluded. This takes precedence over globs listed in "includeGlobs" + // * and files that match the file extensions provided in "fileExtensions". + // */ + // "excludeGlobs": [], + // + // /** + // * Globs that should be explicitly included. + // */ + // "includeGlobs": ["assets/**/*"], + // + // /** + // * Copy only the file and discard the relative path from the source folder. This defaults to false. + // */ + // "flatten": false, + // + // /** + // * Hardlink files instead of copying. This defaults to false. + // */ + // "hardlink": false + // } + // ] // } ], diff --git a/apps/heft/src/utilities/CoreConfigFiles.ts b/apps/heft/src/utilities/CoreConfigFiles.ts index 3e7c1e9ea0c..99b1d34fde4 100644 --- a/apps/heft/src/utilities/CoreConfigFiles.ts +++ b/apps/heft/src/utilities/CoreConfigFiles.ts @@ -64,12 +64,12 @@ export interface ISharedCopyConfiguration { export interface IExtendedSharedCopyConfiguration extends ISharedCopyConfiguration { /** - * The folder from which files should be copied. For example, "src". + * The folder from which files should be copied, relative to the project root. For example, "src". */ sourceFolder: string; /** - * The folder(s) to which files should be copied. For example ["lib", "lib-cjs"]. + * Folder(s) to which files should be copied, relative to the project root. For example ["lib", "lib-cjs"]. */ destinationFolders: string[]; } From 23a7a514dc23ce3cdf9fd1f682b9a25fead76530 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 14:39:48 -0800 Subject: [PATCH 34/35] Add a test project for copy. --- build-tests/heft-copy-files-test/.gitignore | 1 + .../heft-copy-files-test/config/heft.json | 156 ++++++++++++++++++ build-tests/heft-copy-files-test/package.json | 13 ++ .../heft-copy-files-test/src/A/AA/aa1.txt | 0 build-tests/heft-copy-files-test/src/A/a1.txt | 0 build-tests/heft-copy-files-test/src/A/a2.txt | 0 build-tests/heft-copy-files-test/src/A/a3.png | 0 build-tests/heft-copy-files-test/src/A/a4.jpg | 0 .../heft-copy-files-test/src/B/BB/bb1.txt | 0 build-tests/heft-copy-files-test/src/B/b1.txt | 0 build-tests/heft-copy-files-test/src/B/b2.txt | 0 build-tests/heft-copy-files-test/src/B/b3.png | 0 build-tests/heft-copy-files-test/src/B/b4.jpg | 0 rush.json | 6 + 14 files changed, 176 insertions(+) create mode 100644 build-tests/heft-copy-files-test/.gitignore create mode 100644 build-tests/heft-copy-files-test/config/heft.json create mode 100644 build-tests/heft-copy-files-test/package.json create mode 100644 build-tests/heft-copy-files-test/src/A/AA/aa1.txt create mode 100644 build-tests/heft-copy-files-test/src/A/a1.txt create mode 100644 build-tests/heft-copy-files-test/src/A/a2.txt create mode 100644 build-tests/heft-copy-files-test/src/A/a3.png create mode 100644 build-tests/heft-copy-files-test/src/A/a4.jpg create mode 100644 build-tests/heft-copy-files-test/src/B/BB/bb1.txt create mode 100644 build-tests/heft-copy-files-test/src/B/b1.txt create mode 100644 build-tests/heft-copy-files-test/src/B/b2.txt create mode 100644 build-tests/heft-copy-files-test/src/B/b3.png create mode 100644 build-tests/heft-copy-files-test/src/B/b4.jpg diff --git a/build-tests/heft-copy-files-test/.gitignore b/build-tests/heft-copy-files-test/.gitignore new file mode 100644 index 00000000000..cadca90b0c7 --- /dev/null +++ b/build-tests/heft-copy-files-test/.gitignore @@ -0,0 +1 @@ +out-* diff --git a/build-tests/heft-copy-files-test/config/heft.json b/build-tests/heft-copy-files-test/config/heft.json new file mode 100644 index 00000000000..47858d4461e --- /dev/null +++ b/build-tests/heft-copy-files-test/config/heft.json @@ -0,0 +1,156 @@ +/** + * Defines configuration used by core Heft. + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/heft.schema.json", + + "eventActions": [ + { + /** + * The kind of built-in operation that should be performed. + * The "deleteGlobs" action deletes files or folders that match the + * specified glob patterns. + */ + "actionKind": "deleteGlobs", + + /** + * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + * occur at the end of the stage of the Heft run. + */ + "heftEvent": "clean", + + /** + * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + * configs. + */ + "actionId": "defaultClean", + + /** + * Glob patterns to be deleted. The paths are resolved relative to the project folder. + */ + "globsToDelete": ["out-*"] + }, + + { + /** + * The kind of built-in operation that should be performed. + * The "copyFiles" action copies files that match the specified patterns. + */ + "actionKind": "copyFiles", + + /** + * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + * occur at the end of the stage of the Heft run. + */ + "heftEvent": "pre-compile", + + /** + * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + * configs. + */ + "actionId": "preCompileCopy", + + /** + * An array of copy operations to run perform during the specified Heft event. + */ + "copyOperations": [ + { + /** + * The folder from which files should be copied, relative to the project root. + */ + "sourceFolder": "src", + + /** + * Folder(s) to which files should be copied, relative to the project root. + */ + "destinationFolders": ["out-all"], + + /** + * Globs that should be explicitly included. + */ + "includeGlobs": ["**/*"] + }, + { + /** + * The folder from which files should be copied, relative to the project root. + */ + "sourceFolder": "src", + + /** + * Folder(s) to which files should be copied, relative to the project root. + */ + "destinationFolders": ["out-all-linked"], + + /** + * Globs that should be explicitly included. + */ + "includeGlobs": ["**/*"], + + /** + * Hardlink files instead of copying. This defaults to false. + */ + "hardlink": true + }, + { + /** + * The folder from which files should be copied, relative to the project root. + */ + "sourceFolder": "src", + + /** + * Folder(s) to which files should be copied, relative to the project root. + */ + "destinationFolders": ["out-images-flattened"], + + /** + * File extensions that should be copied from the source folder to the destination folder(s) + */ + "fileExtensions": [".jpg", ".png"], + + /** + * Copy only the file and discard the relative path from the source folder. This defaults to false. + */ + "flatten": true + }, + { + /** + * The folder from which files should be copied, relative to the project root. + */ + "sourceFolder": "src", + + /** + * Folder(s) to which files should be copied, relative to the project root. + */ + "destinationFolders": ["out-all-except-for-images"], + + /** + * Globs that should be explicitly excluded. This takes precedence over globs listed in "includeGlobs" + * and files that match the file extensions provided in "fileExtensions". + */ + "excludeGlobs": ["**/*.png", "**/*.jpg"], + + /** + * Globs that should be explicitly included. + */ + "includeGlobs": ["**/*"] + }, + { + /** + * The folder from which files should be copied, relative to the project root. + */ + "sourceFolder": "src", + + /** + * Folder(s) to which files should be copied, relative to the project root. + */ + "destinationFolders": ["out-images1", "out-images2", "out-images3", "out-images4", "out-images5"], + + /** + * File extensions that should be copied from the source folder to the destination folder(s) + */ + "fileExtensions": [".jpg", ".png"] + } + ] + } + ] +} diff --git a/build-tests/heft-copy-files-test/package.json b/build-tests/heft-copy-files-test/package.json new file mode 100644 index 00000000000..71c5e7c5e30 --- /dev/null +++ b/build-tests/heft-copy-files-test/package.json @@ -0,0 +1,13 @@ +{ + "name": "heft-copy-files-test", + "description": "Building this project tests copying files with Heft", + "version": "1.0.0", + "private": true, + "license": "MIT", + "scripts": { + "build": "heft build --clean --verbose" + }, + "devDependencies": { + "@rushstack/heft": "workspace:*" + } +} diff --git a/build-tests/heft-copy-files-test/src/A/AA/aa1.txt b/build-tests/heft-copy-files-test/src/A/AA/aa1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/A/a1.txt b/build-tests/heft-copy-files-test/src/A/a1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/A/a2.txt b/build-tests/heft-copy-files-test/src/A/a2.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/A/a3.png b/build-tests/heft-copy-files-test/src/A/a3.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/A/a4.jpg b/build-tests/heft-copy-files-test/src/A/a4.jpg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/B/BB/bb1.txt b/build-tests/heft-copy-files-test/src/B/BB/bb1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/B/b1.txt b/build-tests/heft-copy-files-test/src/B/b1.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/B/b2.txt b/build-tests/heft-copy-files-test/src/B/b2.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/B/b3.png b/build-tests/heft-copy-files-test/src/B/b3.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build-tests/heft-copy-files-test/src/B/b4.jpg b/build-tests/heft-copy-files-test/src/B/b4.jpg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/rush.json b/rush.json index 7f1338d0fa0..6a407721e69 100644 --- a/rush.json +++ b/rush.json @@ -536,6 +536,12 @@ "reviewCategory": "tests", "shouldPublish": false }, + { + "packageName": "heft-copy-files-test", + "projectFolder": "build-tests/heft-copy-files-test", + "reviewCategory": "tests", + "shouldPublish": false + }, { "packageName": "heft-example-plugin-01", "projectFolder": "build-tests/heft-example-plugin-01", From c3984b1481950855da339b870b72bebe79c5fa65 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 10 Nov 2020 14:46:14 -0800 Subject: [PATCH 35/35] rush change --- .../danade-copy-plugin_2020-11-10-22-45.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-22-45.json diff --git a/common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-22-45.json b/common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-22-45.json new file mode 100644 index 00000000000..2a61efcab69 --- /dev/null +++ b/common/changes/@rushstack/node-core-library/danade-copy-plugin_2020-11-10-22-45.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/node-core-library", + "comment": "Add an alreadyExistsBehavior option to the options for creating links in FileSystem.", + "type": "minor" + } + ], + "packageName": "@rushstack/node-core-library", + "email": "iclanton@users.noreply.github.com" +} \ No newline at end of file