diff --git a/.changeset/real-eyes-vanish.md b/.changeset/real-eyes-vanish.md new file mode 100644 index 000000000..35a1e6ac5 --- /dev/null +++ b/.changeset/real-eyes-vanish.md @@ -0,0 +1,5 @@ +--- +"modular-scripts": minor +--- + +Selective build support diff --git a/__fixtures__/ghost-building/.editorconfig b/__fixtures__/ghost-building/.editorconfig new file mode 100644 index 000000000..7526bd7ab --- /dev/null +++ b/__fixtures__/ghost-building/.editorconfig @@ -0,0 +1,18 @@ +# https://editorconfig.org +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true +max_line_length = 80 +trim_trailing_whitespace = true + +[*.{md,mdx}] +max_line_length = 0 +trim_trailing_whitespace = false + +[COMMIT_EDITMSG] +max_line_length = 0 diff --git a/__fixtures__/ghost-building/.eslintignore b/__fixtures__/ghost-building/.eslintignore new file mode 100644 index 000000000..d529732cc --- /dev/null +++ b/__fixtures__/ghost-building/.eslintignore @@ -0,0 +1,23 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +packages/**/public +/dist diff --git a/__fixtures__/ghost-building/.gitignore b/__fixtures__/ghost-building/.gitignore new file mode 100644 index 000000000..d3f6241cb --- /dev/null +++ b/__fixtures__/ghost-building/.gitignore @@ -0,0 +1,24 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +/dist + +.vscode diff --git a/__fixtures__/ghost-building/.prettierignore b/__fixtures__/ghost-building/.prettierignore new file mode 100644 index 000000000..f60fee050 --- /dev/null +++ b/__fixtures__/ghost-building/.prettierignore @@ -0,0 +1,2 @@ +/dist +/packages/**/public diff --git a/__fixtures__/ghost-building/.yarnrc b/__fixtures__/ghost-building/.yarnrc new file mode 100644 index 000000000..67d13ac5f --- /dev/null +++ b/__fixtures__/ghost-building/.yarnrc @@ -0,0 +1 @@ +disable-self-update-check true \ No newline at end of file diff --git a/__fixtures__/ghost-building/README.md b/__fixtures__/ghost-building/README.md new file mode 100644 index 000000000..28ac919a4 --- /dev/null +++ b/__fixtures__/ghost-building/README.md @@ -0,0 +1 @@ +This is the `README.md` for the whole monorepo. diff --git a/__fixtures__/ghost-building/modular/setupEnvironment.ts b/__fixtures__/ghost-building/modular/setupEnvironment.ts new file mode 100644 index 000000000..5b785ac2b --- /dev/null +++ b/__fixtures__/ghost-building/modular/setupEnvironment.ts @@ -0,0 +1,2 @@ +// Allows for adding setup configuration to Jest +export {}; diff --git a/__fixtures__/ghost-building/modular/setupTests.ts b/__fixtures__/ghost-building/modular/setupTests.ts new file mode 100644 index 000000000..74b1a275a --- /dev/null +++ b/__fixtures__/ghost-building/modular/setupTests.ts @@ -0,0 +1,5 @@ +// jest-dom adds custom jest matchers for asserting on DOM nodes. +// allows you to do things like: +// expect(element).toHaveTextContent(/react/i) +// learn more: https://github.com/testing-library/jest-dom +import '@testing-library/jest-dom/extend-expect'; diff --git a/__fixtures__/ghost-building/package.json b/__fixtures__/ghost-building/package.json new file mode 100644 index 000000000..475e70a1b --- /dev/null +++ b/__fixtures__/ghost-building/package.json @@ -0,0 +1,60 @@ +{ + "name": "ghost-building", + "version": "1.0.0", + "main": "index.js", + "author": "Cristiano Belloni ", + "license": "MIT", + "private": true, + "workspaces": [ + "packages/**" + ], + "modular": { + "type": "root" + }, + "scripts": { + "start": "modular start", + "build": "modular build", + "test": "modular test", + "lint": "eslint . --ext .js,.ts,.tsx", + "prettier": "prettier --write ." + }, + "eslintConfig": { + "extends": "modular-app" + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + }, + "prettier": { + "singleQuote": true, + "trailingComma": "all", + "printWidth": 80, + "proseWrap": "always" + }, + "dependencies": { + "@testing-library/dom": "^8.19.0", + "@testing-library/jest-dom": "^5.16.5", + "@testing-library/react": "^13.4.0", + "@testing-library/user-event": "^7.2.1", + "@types/jest": "^29.2.3", + "@types/node": "^18.11.9", + "@types/react": "^18.0.25", + "@types/react-dom": "^18.0.9", + "eslint-config-modular-app": "^3.0.2", + "modular-scripts": "^3.5.0", + "modular-template-app": "^1.1.0", + "modular-template-package": "^1.1.0", + "prettier": "^2.7.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "typescript": ">=4.2.1 <4.5.0" + } +} diff --git a/__fixtures__/ghost-building/packages/a/package.json b/__fixtures__/ghost-building/packages/a/package.json new file mode 100644 index 000000000..b2426d1a2 --- /dev/null +++ b/__fixtures__/ghost-building/packages/a/package.json @@ -0,0 +1,13 @@ +{ + "name": "a", + "private": false, + "modular": { + "type": "package" + }, + "main": "./src/index.ts", + "version": "1.0.0", + "dependencies": { + "b": "1.0.0", + "c": "1.0.0" + } +} diff --git a/__fixtures__/ghost-building/packages/a/src/index.ts b/__fixtures__/ghost-building/packages/a/src/index.ts new file mode 100644 index 000000000..b92ce9fdf --- /dev/null +++ b/__fixtures__/ghost-building/packages/a/src/index.ts @@ -0,0 +1,3 @@ +export default function add(a: number, b: number): number { + return a + b; +} diff --git a/__fixtures__/ghost-building/packages/b/package.json b/__fixtures__/ghost-building/packages/b/package.json new file mode 100644 index 000000000..b32e433b5 --- /dev/null +++ b/__fixtures__/ghost-building/packages/b/package.json @@ -0,0 +1,12 @@ +{ + "name": "b", + "private": false, + "modular": { + "type": "package" + }, + "main": "./src/index.ts", + "version": "1.0.0", + "dependencies": { + "c": "1.0.0" + } +} diff --git a/__fixtures__/ghost-building/packages/b/src/index.ts b/__fixtures__/ghost-building/packages/b/src/index.ts new file mode 100644 index 000000000..b92ce9fdf --- /dev/null +++ b/__fixtures__/ghost-building/packages/b/src/index.ts @@ -0,0 +1,3 @@ +export default function add(a: number, b: number): number { + return a + b; +} diff --git a/__fixtures__/ghost-building/packages/c/package.json b/__fixtures__/ghost-building/packages/c/package.json new file mode 100644 index 000000000..c3e4cd1fc --- /dev/null +++ b/__fixtures__/ghost-building/packages/c/package.json @@ -0,0 +1,12 @@ +{ + "name": "c", + "private": false, + "modular": { + "type": "package" + }, + "main": "./src/index.ts", + "version": "1.0.0", + "dependencies": { + "d": "1.0.0" + } +} diff --git a/__fixtures__/ghost-building/packages/c/src/index.ts b/__fixtures__/ghost-building/packages/c/src/index.ts new file mode 100644 index 000000000..b92ce9fdf --- /dev/null +++ b/__fixtures__/ghost-building/packages/c/src/index.ts @@ -0,0 +1,3 @@ +export default function add(a: number, b: number): number { + return a + b; +} diff --git a/__fixtures__/ghost-building/packages/d/package.json b/__fixtures__/ghost-building/packages/d/package.json new file mode 100644 index 000000000..aadbc8a6a --- /dev/null +++ b/__fixtures__/ghost-building/packages/d/package.json @@ -0,0 +1,9 @@ +{ + "name": "d", + "private": false, + "modular": { + "type": "package" + }, + "main": "./src/index.ts", + "version": "1.0.0" +} diff --git a/__fixtures__/ghost-building/packages/d/src/index.ts b/__fixtures__/ghost-building/packages/d/src/index.ts new file mode 100644 index 000000000..b92ce9fdf --- /dev/null +++ b/__fixtures__/ghost-building/packages/d/src/index.ts @@ -0,0 +1,3 @@ +export default function add(a: number, b: number): number { + return a + b; +} diff --git a/__fixtures__/ghost-building/packages/e/package.json b/__fixtures__/ghost-building/packages/e/package.json new file mode 100644 index 000000000..552c61108 --- /dev/null +++ b/__fixtures__/ghost-building/packages/e/package.json @@ -0,0 +1,12 @@ +{ + "name": "e", + "private": false, + "modular": { + "type": "package" + }, + "main": "./src/index.ts", + "version": "1.0.0", + "dependencies": { + "a": "1.0.0" + } +} diff --git a/__fixtures__/ghost-building/packages/e/src/index.ts b/__fixtures__/ghost-building/packages/e/src/index.ts new file mode 100644 index 000000000..b92ce9fdf --- /dev/null +++ b/__fixtures__/ghost-building/packages/e/src/index.ts @@ -0,0 +1,3 @@ +export default function add(a: number, b: number): number { + return a + b; +} diff --git a/__fixtures__/ghost-building/tsconfig.json b/__fixtures__/ghost-building/tsconfig.json new file mode 100644 index 000000000..d2e2dbe0a --- /dev/null +++ b/__fixtures__/ghost-building/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "modular-scripts/tsconfig.json", + "include": ["modular", "packages/**/src"] +} diff --git a/docs/commands/build.md b/docs/commands/build.md index 6cb0cc4fc..174b59897 100644 --- a/docs/commands/build.md +++ b/docs/commands/build.md @@ -3,10 +3,11 @@ parent: Commands title: modular build --- -# `modular build ` +# `modular build [packages...]` Search workspaces based on their `name` field in the `package.json` and build -them according to their respective `modular.type`. +them according to their respective `modular.type`, in order of dependency (e.g. +if a package `a` depends on a package `b`, `b` is built first). The output directory for built artifacts is `dist/`, which has a flat structure of modular package names. Each built app/view/package is added to the `dist/` as @@ -20,6 +21,135 @@ this-is-param-case) in `dist/` ## Options: -`--private`: Allows the building of private packages +`--private`: Allows the building of private packages. -`--preserve-modules`: Preserve module structure in generated modules +`--preserve-modules`: Preserve module structure in generated modules. + +`--changed`: Build only packages whose workspaces contain files that have +changed. Files that have changed are calculated comparing the current state of +the repository with the branch specified by `compareBranch` or, if +`compareBranch` is not set, with the default git branch. + +`--compareBranch`: Specify the comparison branch used to determine which files +have changed when using the `changed` option. If this option is used without +`changed`, the command will fail. + +`--descendants`: Build the packages specified by the `[packages...]` argument +and/or the `--changed` option and additionally build all their descendants (i.e. +the packages they directly or indirectly depend on) in dependency order. + +`--ancestors`: Build the packages specified by the `[packages...]` argument +and/or the `--changed` option and additionally build all their ancestors (i.e. +the packages that have a direct or indirect dependency on them) in dependency +order. + +## Dependency selection and build order examples + +We'll be using this package manifests in our Modular monorepo for the following +examples: + +```js +{ + "name": "a", + "dependencies": { + "b": "*", + "c": "*", + "react": ">16.8.0", + // ... + } +} + +{ + "name": "b", + "dependencies": { + "c": "*", + "react": ">16.8.0", + // ... + } +} + +{ + "name": "c", + "dependencies": { + "d": "*", + // ... + } +} + +{ + "name": "d", + "dependencies": {} +} + +{ + "name": "e", + "dependencies": { + "a": "*", + // ... + } +} +``` + +Which internally are filtered into this set of `WorkspaceDependencyObject`s: + +```js +{ + a: { workspaceDependencies: ['b', 'c'] }, + b: { workspaceDependencies: ['c'] }, + c: { workspaceDependencies: ['d'] }, + d: { workspaceDependencies: undefined }, + e: { workspaceDependencies: ['a'] } +} +``` + +### Example: local workflow with descendants + +Let's say we just pulled an update to our monorepo and we want to work on +workspace `b`. To be able to work with the last modifications we pulled, we want +to build `b` and all the other workspaces that `b` depends on (descendants), +either directly or indirectly. We can tell Modular that we want to build `b` and +its all descendants by using this command: + +`modular build b --descendants` + +Modular will first select all the descendants of `b` (according to the previous +graph: `c` because it'a direct dependency and `d` because it's a dependency of +`c`), then build them in the correct build order, where workspaces depended on +are built before workspaces that depend on them, recursively. In this example: + +- `d` gets built first, because it has no dependencies +- `c` can now get built, because it only depends on `d`, that got built in the + previous step. +- `b` can now get built, because it only depends on `c`, that got built in the + previous step. + +### Example: incremental builds with ancestors + +Let's suppose we're building a PR of our monorepository on a CI pipeline, and we +want to incrementally build the workspaces that have code changes compared to +the base branch. Since those workspaces will generate new, different build +artefacts, we can't just build them and call it a day; we also need to re-build +all the workspaces that depend on the changed workspaces, and those who depend +on them, and so on. In other words, we need a way to tell Modular to build the +ancestors of the changed workspaces. This command: + +`modular build --changed --ancestors` + +will identify all the workspaces that have changed compared to the +`--compareBranch` (which is the repository's base branch by default), then +identify all the workspaces which directly or indirectly depend on them +(ancestors) and build the resulting set of packages in the correct build order, +where workspaces depended on are built before workspaces that depend on them, +recursively. If we suppose that workspaces `b` and `c` have changed, Modular +will: + +- Select all ancestors of `b` and `c`, which are `a` (because it depends on + both) and `e` (because it depends on `a`). +- Build `c` first, because it doesn't depend on any package that has changed (it + only depends on `d`, which is not in the changed set). +- Build `b`, because it only depends on `c`, that got built in the previous + step. +- Build `a`, because it depends on `b` and `c`, that got built in the previous + steps. +- Build `e`, because it only depends on `a`, that got built in the previous + step. diff --git a/packages/modular-scripts/src/__tests__/selectiveBuild.test.ts b/packages/modular-scripts/src/__tests__/selectiveBuild.test.ts new file mode 100644 index 000000000..b9fda2782 --- /dev/null +++ b/packages/modular-scripts/src/__tests__/selectiveBuild.test.ts @@ -0,0 +1,178 @@ +import execa from 'execa'; +import path from 'path'; +import fs from 'fs-extra'; + +import getModularRoot from '../utils/getModularRoot'; +import { createModularTestContext, runLocalModular } from '../test/utils'; + +// Temporary test context paths set by createTempModularRepoWithTemplate() +let tempModularRepo: string; + +const currentModularFolder = getModularRoot(); +const buildRegex = /building (\w)\.\.\./gm; + +describe('--changed builds all the changed packages in order', () => { + const fixturesFolder = path.join( + getModularRoot(), + '__fixtures__', + 'ghost-building', + ); + + beforeAll(() => { + tempModularRepo = createModularTestContext(); + fs.copySync(fixturesFolder, tempModularRepo); + + // Create git repo & commit + if (process.env.GIT_AUTHOR_NAME && process.env.GIT_AUTHOR_EMAIL) { + execa.sync('git', [ + 'config', + '--global', + 'user.email', + `"${process.env.GIT_AUTHOR_EMAIL}"`, + ]); + execa.sync('git', [ + 'config', + '--global', + 'user.name', + `"${process.env.GIT_AUTHOR_NAME}"`, + ]); + } + execa.sync('git', ['init'], { + cwd: tempModularRepo, + }); + + execa.sync('yarn', { + cwd: tempModularRepo, + }); + + execa.sync('git', ['add', '.'], { + cwd: tempModularRepo, + }); + + execa.sync('git', ['commit', '-am', '"First commit"'], { + cwd: tempModularRepo, + }); + }); + + it('builds nothing when everything committed', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + '--changed', + ]); + expect(result.stderr).toBeFalsy(); + expect(result.stdout).toContain('No changed workspaces found'); + }); + + it('builds multiple packages', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + 'e', + 'a', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['a', 'e']); + }); + + it('builds a single package and its descendants', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + 'b', + '--descendants', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['d', 'c', 'b']); + }); + + it('builds a single package and its ancestors', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + 'b', + '--ancestors', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['b', 'a', 'e']); + }); + + it('builds multiple packages and their descendants', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + 'd', + 'a', + '--descendants', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['d', 'c', 'b', 'a']); + }); + + it('builds multiple packages and their ancestors', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + 'd', + 'a', + '--ancestors', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['d', 'c', 'b', 'a', 'e']); + }); + + it('builds changed (uncommitted) packages', () => { + fs.appendFileSync( + path.join(tempModularRepo, '/packages/b/src/index.ts'), + "\n// Comment to package b's source", + ); + fs.appendFileSync( + path.join(tempModularRepo, '/packages/c/src/index.ts'), + "\n// Comment to package c's source", + ); + + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + '--changed', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['c', 'b']); + }); + + it('builds changed (uncommitted) packages + packages that are explicitly specified', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + 'e', + '--changed', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['c', 'b', 'e']); + }); + + it('builds changed (uncommitted) packages and their descendants', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + '--changed', + '--descendants', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['d', 'c', 'b']); + }); + + it('builds changed (uncommitted) packages and their ancestors', () => { + const result = runLocalModular(currentModularFolder, tempModularRepo, [ + 'build', + '--changed', + '--ancestors', + ]); + + expect(result.stderr).toBeFalsy(); + expect(getBuildOrder(result.stdout)).toEqual(['c', 'b', 'a', 'e']); + }); +}); + +function getBuildOrder(output: string) { + return [...output.matchAll(buildRegex)].map(([, group]) => group); +} diff --git a/packages/modular-scripts/src/__tests__/test.test.ts b/packages/modular-scripts/src/__tests__/test.test.ts index c21ed373c..9b517e8a3 100644 --- a/packages/modular-scripts/src/__tests__/test.test.ts +++ b/packages/modular-scripts/src/__tests__/test.test.ts @@ -3,6 +3,7 @@ import path from 'path'; import fs from 'fs-extra'; import tmp from 'tmp'; import getModularRoot from '../utils/getModularRoot'; +import { runLocalModular } from '../test/utils'; function setupTests(fixturesFolder: string) { const files = fs.readdirSync(path.join(fixturesFolder)); @@ -129,7 +130,7 @@ describe('Modular test command', () => { // These expects run in a single test, serially for performance reasons (the setup time is quite long) it('finds no unchanged using --changed / finds changed after modifying some workspaces / finds ancestors using --ancestors', () => { - const resultUnchanged = runRemoteModularTest( + const resultUnchanged = runLocalModular( currentModularFolder, randomOutputFolder, ['test', '--changed'], @@ -145,7 +146,7 @@ describe('Modular test command', () => { "\n// Comment to package c's source", ); - const resultChanged = runRemoteModularTest( + const resultChanged = runLocalModular( currentModularFolder, randomOutputFolder, ['test', '--changed'], @@ -163,7 +164,7 @@ describe('Modular test command', () => { 'packages/b/src/__tests__/b.test.ts', ); - const resultChangedWithAncestors = runRemoteModularTest( + const resultChangedWithAncestors = runLocalModular( currentModularFolder, randomOutputFolder, ['test', '--changed', '--ancestors'], @@ -220,7 +221,7 @@ describe('Modular test command', () => { // Run in a single test, serially for performance reasons (the setup time is quite long) it('finds --package after specifying a valid workspaces / finds ancestors using --ancestors', () => { - const resultPackages = runRemoteModularTest( + const resultPackages = runLocalModular( currentModularFolder, randomOutputFolder, ['test', '--package', 'b', '--package', 'c'], @@ -238,7 +239,7 @@ describe('Modular test command', () => { 'packages/b/src/__tests__/b.test.ts', ); - const resultPackagesWithAncestors = runRemoteModularTest( + const resultPackagesWithAncestors = runLocalModular( currentModularFolder, randomOutputFolder, ['test', '--ancestors', '--package', 'b', '--package', 'c'], @@ -398,24 +399,3 @@ describe('Modular test command', () => { }); }); }); - -function runRemoteModularTest( - modularFolder: string, - cwd: string, - modularArguments: string[], -) { - return execa.sync( - path.join(modularFolder, '/node_modules/.bin/ts-node'), - [ - path.join(modularFolder, '/packages/modular-scripts/src/cli.ts'), - ...modularArguments, - ], - { - cwd, - env: { - ...process.env, - CI: 'true', - }, - }, - ); -} diff --git a/packages/modular-scripts/src/build/index.ts b/packages/modular-scripts/src/build/index.ts index a265f55e3..e6b0bc72f 100644 --- a/packages/modular-scripts/src/build/index.ts +++ b/packages/modular-scripts/src/build/index.ts @@ -12,6 +12,7 @@ import actionPreflightCheck from '../utils/actionPreflightCheck'; import { getModularType } from '../utils/isModularType'; import execAsync from '../utils/execAsync'; import getLocation from '../utils/getLocation'; +import { selectWorkspaces } from '../utils/selectWorkspaces'; import { setupEnvForDirectory } from '../utils/setupEnv'; import createPaths from '../utils/createPaths'; import printHostingInstructions from './printHostingInstructions'; @@ -267,12 +268,43 @@ async function buildStandalone( ); } -async function build( - targets: string[], +async function build({ + packagePaths: targets, preserveModules = true, - includePrivate = false, -): Promise { - for (const target of targets) { + private: includePrivate, + ancestors, + descendants, + changed, + compareBranch, +}: { + packagePaths: string[]; + preserveModules: boolean; + private: boolean; + ancestors: boolean; + descendants: boolean; + changed: boolean; + compareBranch?: string; +}): Promise { + const selectedTargets = await selectWorkspaces({ + targets, + changed, + compareBranch, + descendants, + ancestors, + }); + + if (!selectedTargets.length) { + logger.log('No changed workspaces found'); + process.exit(0); + } + + logger.debug( + `Building the following workspaces in order: ${JSON.stringify( + selectedTargets, + )}`, + ); + + for (const target of selectedTargets) { try { const targetDirectory = await getLocation(target); diff --git a/packages/modular-scripts/src/program.ts b/packages/modular-scripts/src/program.ts index fdcd44668..a5f03a380 100755 --- a/packages/modular-scripts/src/program.ts +++ b/packages/modular-scripts/src/program.ts @@ -73,7 +73,7 @@ program }); program - .command('build ') + .command('build [packages...]') .description( 'Build a list of packages (multiple package names can be supplied separated by space)', ) @@ -84,22 +84,64 @@ program ) .option('--verbose', 'Run yarn commands with --verbose set') .option('--private', 'Enable the building of private packages', false) + .option( + '--changed', + 'Build only for workspaces that have changed compared to the branch specified in --compareBranch', + false, + ) + .option( + '--descendants', + 'Additionally build workspaces that the specified packages directly or indirectly depend on (can be combined with --changed)', + false, + ) + .option( + '--ancestors', + 'Additionally build workspaces that directly or indirectly depend on the specified packages (can be combined with --changed)', + false, + ) + .option( + '--compareBranch ', + "Specifies the branch to use with the --changed flag. If not specified, Modular will use the repo's default branch", + ) .action( async ( packagePaths: string[], options: { preserveModules: string; private: boolean; + changed: boolean; + compareBranch?: string; + ancestors: boolean; + descendants: boolean; }, ) => { const { default: build } = await import('./build'); - logger.log('building packages at:', packagePaths.join(', ')); - await build( + options.changed + ? logger.log('Building changed packages') + : logger.log('Building packages at:', packagePaths.join(', ')); + + if (!packagePaths.length && !options.changed) { + process.stderr.write("error: missing required argument 'packages'"); + process.exit(1); + } + + if (options.compareBranch && !options.changed) { + process.stderr.write( + "Option --compareBranch doesn't make sense without option --changed\n", + ); + process.exit(1); + } + + await build({ packagePaths, - JSON.parse(options.preserveModules) as boolean, - options.private, - ); + preserveModules: JSON.parse(options.preserveModules) as boolean, + private: options.private, + changed: options.changed, + compareBranch: options.compareBranch, + ancestors: options.ancestors, + descendants: options.descendants, + }); }, ); diff --git a/packages/modular-scripts/src/test/index.ts b/packages/modular-scripts/src/test/index.ts index 28e8063a8..aa4e783e8 100644 --- a/packages/modular-scripts/src/test/index.ts +++ b/packages/modular-scripts/src/test/index.ts @@ -6,7 +6,7 @@ import { ExecaError } from 'execa'; import execAsync from '../utils/execAsync'; import getModularRoot from '../utils/getModularRoot'; import { getAllWorkspaces } from '../utils/getAllWorkspaces'; -import { getChangedWorkspaces } from '../utils/getChangedWorkspaces'; +import { getChangedWorkspacesContent } from '../utils/getChangedWorkspaces'; import { resolveAsBin } from '../utils/resolveAsBin'; import * as logger from '../utils/logger'; import type { @@ -256,17 +256,6 @@ async function computeSelectiveRegexes({ return computeTestsRegexes(resultWorkspaceContent); } -// This function returns a WorkspaceContent containing all the changed workspaces, compared to targetBranch -async function getChangedWorkspacesContent(targetBranch: string | undefined) { - const allWorkspaces = await getAllWorkspaces(getModularRoot()); - // Get the changed workspaces compared to our target branch - const changedWorkspaces = await getChangedWorkspaces( - allWorkspaces, - targetBranch, - ); - return changedWorkspaces; -} - // This function returns a WorkspaceContent from an array of workspace names async function getSinglePackagesContent(singlePackages: string[]) { // Get all the workspaces diff --git a/packages/modular-scripts/src/test/utils.ts b/packages/modular-scripts/src/test/utils.ts index 55293532f..791496323 100644 --- a/packages/modular-scripts/src/test/utils.ts +++ b/packages/modular-scripts/src/test/utils.ts @@ -132,3 +132,34 @@ export function mockInstallTemplate( path.join(tempRepoNodeModules, templateName), ); } + +/** + * Run the Modular cli present at the root path `modularFolder` with a working directory of `cwd` + * passing an array of `modularArguments` + * + * Useful to run modular from source in a different directory (usually a temp directory created for tests) + * + * @param modularToRun The root modular repo folder containing the modular-scripts to run + * @param cwd The target working directory where we want to run Modular + * @param modularArguments A list of command-line arguments + */ +export function runLocalModular( + modularToRun: string, + cwd: string, + modularArguments: string[], +): execa.ExecaSyncReturnValue { + return execa.sync( + path.join(modularToRun, '/node_modules/.bin/ts-node'), + [ + path.join(modularToRun, '/packages/modular-scripts/src/cli.ts'), + ...modularArguments, + ], + { + cwd, + env: { + ...process.env, + CI: 'true', + }, + }, + ); +} diff --git a/packages/modular-scripts/src/utils/getChangedWorkspaces.ts b/packages/modular-scripts/src/utils/getChangedWorkspaces.ts index 37b5c3f4f..bde2d279c 100644 --- a/packages/modular-scripts/src/utils/getChangedWorkspaces.ts +++ b/packages/modular-scripts/src/utils/getChangedWorkspaces.ts @@ -2,8 +2,27 @@ import path from 'path'; import pkgUp from 'pkg-up'; import { getDiffedFiles } from './gitActions'; import getModularRoot from './getModularRoot'; +import { getAllWorkspaces } from './getAllWorkspaces'; import type { WorkspaceContent } from '@modular-scripts/modular-types'; +/** + * Return a WorkspaceContent containing all the changed workspaces, compared to a targetBranch + * + * @param targetBranch The branch to compare the current branch to select the changed workspaces + */ + +export async function getChangedWorkspacesContent( + targetBranch: string | undefined, +): Promise { + const allWorkspaces = await getAllWorkspaces(getModularRoot()); + // Get the changed workspaces compared to our target branch + const changedWorkspaces = await getChangedWorkspaces( + allWorkspaces, + targetBranch, + ); + return changedWorkspaces; +} + // Gets a list of changed files, then maps them to their workspace and returns a subset of WorkspaceContent export async function getChangedWorkspaces( workspaceContent: WorkspaceContent, diff --git a/packages/modular-scripts/src/utils/selectWorkspaces.ts b/packages/modular-scripts/src/utils/selectWorkspaces.ts new file mode 100644 index 000000000..7ad689809 --- /dev/null +++ b/packages/modular-scripts/src/utils/selectWorkspaces.ts @@ -0,0 +1,94 @@ +import { + computeDescendantSet, + computeAncestorSet, + traverseWorkspaceRelations, +} from '@modular-scripts/workspace-resolver'; + +import { getAllWorkspaces } from './getAllWorkspaces'; +import { getChangedWorkspacesContent } from './getChangedWorkspaces'; +import getModularRoot from './getModularRoot'; + +/** + * @typedef {Object} TargetOptions + * @property {string[]} targets - An array of package names to select + * @property {boolean} changed - Whether to additionally select packages that have changes, compared to "compareBranch" + * @property {boolean} ancestors - Whether to additionally select packages that are ancestors of (i.e.: [in]directly depend on) the selected packages + * @property {boolean} descendants - Whether to additionally select packages that descend from (i.e.: are [in]directly depended on by) the selected packages + * @property {string?} compareBranch - The git branch to compare with when "changed" is specified + */ + +/** + * Select target packages in workspaces, optionally including changed, ancestors and descendant packages + * @param {TargetOptions} name The target options to configure selection + * @return {Promise} A distinct list of selected package names + */ + +interface TargetOptions { + targets: string[]; + changed: boolean; + ancestors: boolean; + descendants: boolean; + compareBranch?: string; +} + +export async function selectWorkspaces({ + targets, + changed, + ancestors, + descendants, + compareBranch, +}: TargetOptions): Promise { + const [, allWorkspacesMap] = await getAllWorkspaces(getModularRoot()); + let changedTargets: string[] = []; + + if (changed) { + const [, buildTargetMap] = await getChangedWorkspacesContent(compareBranch); + changedTargets = Object.keys(buildTargetMap); + } + + const targetsToBuild = [...new Set(targets.concat(changedTargets))]; + + if (!targetsToBuild.length) { + return []; + } + + // Calculate the package scope, i.e. the deduped array of all the packages we need to generate an ordering graph for. + // We need to remember ancestor and descendant sets in order to select them later. + let ancestorsSet: Set = new Set(); + let descendantsSet: Set = new Set(); + let packageScope = targetsToBuild; + + if (descendants) { + descendantsSet = computeDescendantSet(targetsToBuild, allWorkspacesMap); + packageScope = packageScope.concat([...descendantsSet]); + } + + if (ancestors) { + ancestorsSet = computeAncestorSet(targetsToBuild, allWorkspacesMap); + packageScope = packageScope.concat([...ancestorsSet]); + } + + packageScope = [...new Set(packageScope)]; + + // traverseWorkspaceRelations will walk the graph and generate the whole ordering needed to build a package + // which means that it will possibly expand the scope (it just generates the dependency order starting from a dependency subset but taking into account the whole dependency graph) + // this means that we can build in order even manually selected subsets of packages (like: "the user wants to select only a and b, but execute tasks on them in order"), + // but it also means we need to filter out all the unwanted packages later. + const targetEntriesWithOrder = [ + ...traverseWorkspaceRelations(packageScope, allWorkspacesMap).entries(), + ]; + + return ( + targetEntriesWithOrder + .sort((a, b) => b[1] - a[1]) + .map(([packageName]) => packageName) + // Filter out descendants and ancestors if we don't explicitly need them. + // This allows us to get the correct dependency order even if we restrict the scope (for example, by explicit user input). + .filter( + (packageName) => + (descendants && descendantsSet.has(packageName)) || + (ancestors && ancestorsSet.has(packageName)) || + targetsToBuild.includes(packageName), + ) + ); +}