diff --git a/packages/ci/src/lib/monorepo/handlers/npm.ts b/packages/ci/src/lib/monorepo/handlers/npm.ts index 4657b4531..c232d8382 100644 --- a/packages/ci/src/lib/monorepo/handlers/npm.ts +++ b/packages/ci/src/lib/monorepo/handlers/npm.ts @@ -10,12 +10,14 @@ import type { MonorepoToolHandler } from '../tools.js'; export const npmHandler: MonorepoToolHandler = { tool: 'npm', + async isConfigured(options) { return ( (await fileExists(join(options.cwd, 'package-lock.json'))) && (await hasWorkspacesEnabled(options.cwd)) ); }, + async listProjects(options) { const { workspaces, rootPackageJson } = await listWorkspaces(options.cwd); return workspaces @@ -28,8 +30,13 @@ export const npmHandler: MonorepoToolHandler = { .map(({ name, packageJson }) => ({ name, bin: hasScript(packageJson, options.task) - ? `npm -w ${name} run ${options.task} --` - : `npm -w ${name} exec ${options.task} --`, + ? `npm --workspace=${name} run ${options.task} --` + : `npm --workspace=${name} exec ${options.task} --`, })); }, + + createRunManyCommand(options) { + // neither parallel execution nor projects filter are supported in NPM workspaces + return `npm run ${options.task} --workspaces --if-present --`; + }, }; diff --git a/packages/ci/src/lib/monorepo/handlers/nx.ts b/packages/ci/src/lib/monorepo/handlers/nx.ts index 7e7ad236e..19d26128f 100644 --- a/packages/ci/src/lib/monorepo/handlers/nx.ts +++ b/packages/ci/src/lib/monorepo/handlers/nx.ts @@ -9,6 +9,7 @@ import type { MonorepoToolHandler } from '../tools.js'; export const nxHandler: MonorepoToolHandler = { tool: 'nx', + async isConfigured(options) { return ( (await fileExists(join(options.cwd, 'nx.json'))) && @@ -18,10 +19,12 @@ export const nxHandler: MonorepoToolHandler = { args: ['nx', 'report'], cwd: options.cwd, observer: options.observer, + ignoreExitCode: true, }) ).code === 0 ); }, + async listProjects(options) { const { stdout } = await executeProcess({ command: 'npx', @@ -43,6 +46,19 @@ export const nxHandler: MonorepoToolHandler = { bin: `npx nx run ${project}:${options.task} --`, })); }, + + createRunManyCommand(options, onlyProjects) { + return [ + 'npx', + 'nx', + 'run-many', // TODO: allow affected instead of run-many? + `--targets=${options.task}`, + // TODO: add options.nxRunManyFilter? (e.g. --exclude=...) + ...(onlyProjects ? [`--projects=${onlyProjects.join(',')}`] : []), + `--parallel=${options.parallel}`, + '--', + ].join(' '); + }, }; function parseProjects(stdout: string): string[] { diff --git a/packages/ci/src/lib/monorepo/handlers/pnpm.ts b/packages/ci/src/lib/monorepo/handlers/pnpm.ts index 01bc368d5..3083ecc64 100644 --- a/packages/ci/src/lib/monorepo/handlers/pnpm.ts +++ b/packages/ci/src/lib/monorepo/handlers/pnpm.ts @@ -11,14 +11,19 @@ import type { MonorepoToolHandler } from '../tools.js'; const WORKSPACE_FILE = 'pnpm-workspace.yaml'; +// https://pnpm.io/cli/recursive#--workspace-concurrency +const DEFAULT_WORKSPACE_CONCURRENCY = 4; + export const pnpmHandler: MonorepoToolHandler = { tool: 'pnpm', + async isConfigured(options) { return ( (await fileExists(join(options.cwd, WORKSPACE_FILE))) && (await fileExists(join(options.cwd, 'package.json'))) ); }, + async listProjects(options) { const yaml = await readTextFile(join(options.cwd, WORKSPACE_FILE)); const workspace = YAML.parse(yaml) as { packages?: string[] }; @@ -34,8 +39,25 @@ export const pnpmHandler: MonorepoToolHandler = { .map(({ name, packageJson }) => ({ name, bin: hasScript(packageJson, options.task) - ? `pnpm -F ${name} run ${options.task}` - : `pnpm -F ${name} exec ${options.task}`, + ? `pnpm --filter=${name} run ${options.task}` + : `pnpm --filter=${name} exec ${options.task}`, })); }, + + createRunManyCommand(options, onlyProjects) { + const workspaceConcurrency: number = + options.parallel === true + ? DEFAULT_WORKSPACE_CONCURRENCY + : options.parallel === false + ? 1 + : options.parallel; + return [ + 'pnpm', + '--recursive', + `--workspace-concurrency=${workspaceConcurrency}`, + ...(onlyProjects?.map(project => `--filter=${project}`) ?? []), + 'run', + options.task, + ].join(' '); + }, }; diff --git a/packages/ci/src/lib/monorepo/handlers/turbo.ts b/packages/ci/src/lib/monorepo/handlers/turbo.ts index 0f8e3ff85..1c8e3c126 100644 --- a/packages/ci/src/lib/monorepo/handlers/turbo.ts +++ b/packages/ci/src/lib/monorepo/handlers/turbo.ts @@ -7,12 +7,16 @@ import { yarnHandler } from './yarn.js'; const WORKSPACE_HANDLERS = [pnpmHandler, yarnHandler, npmHandler]; +// https://turbo.build/repo/docs/reference/run#--concurrency-number--percentage +const DEFAULT_CONCURRENCY = 10; + type TurboConfig = { tasks: Record; }; export const turboHandler: MonorepoToolHandler = { tool: 'turbo', + async isConfigured(options) { const configPath = join(options.cwd, 'turbo.json'); return ( @@ -20,6 +24,7 @@ export const turboHandler: MonorepoToolHandler = { options.task in (await readJsonFile(configPath)).tasks ); }, + async listProjects(options) { // eslint-disable-next-line functional/no-loop-statements for (const handler of WORKSPACE_HANDLERS) { @@ -29,7 +34,7 @@ export const turboHandler: MonorepoToolHandler = { .filter(({ bin }) => bin.includes(`run ${options.task}`)) // must have package.json script .map(({ name }) => ({ name, - bin: `npx turbo run ${options.task} -F ${name} --`, + bin: `npx turbo run ${options.task} --filter=${name} --`, })); } } @@ -39,4 +44,22 @@ export const turboHandler: MonorepoToolHandler = { ).join('/')}`, ); }, + + createRunManyCommand(options, onlyProjects) { + const concurrency: number = + options.parallel === true + ? DEFAULT_CONCURRENCY + : options.parallel === false + ? 1 + : options.parallel; + return [ + 'npx', + 'turbo', + 'run', + options.task, + ...(onlyProjects?.map(project => `--filter=${project}`) ?? []), + `--concurrency=${concurrency}`, + '--', + ].join(' '); + }, }; diff --git a/packages/ci/src/lib/monorepo/handlers/yarn.ts b/packages/ci/src/lib/monorepo/handlers/yarn.ts index db5c3f632..e2022fb03 100644 --- a/packages/ci/src/lib/monorepo/handlers/yarn.ts +++ b/packages/ci/src/lib/monorepo/handlers/yarn.ts @@ -1,5 +1,5 @@ import { join } from 'node:path'; -import { fileExists } from '@code-pushup/utils'; +import { executeProcess, fileExists } from '@code-pushup/utils'; import { hasCodePushUpDependency, hasScript, @@ -10,12 +10,14 @@ import type { MonorepoToolHandler } from '../tools.js'; export const yarnHandler: MonorepoToolHandler = { tool: 'yarn', + async isConfigured(options) { return ( (await fileExists(join(options.cwd, 'yarn.lock'))) && (await hasWorkspacesEnabled(options.cwd)) ); }, + async listProjects(options) { const { workspaces, rootPackageJson } = await listWorkspaces(options.cwd); return workspaces @@ -32,4 +34,26 @@ export const yarnHandler: MonorepoToolHandler = { : `yarn workspace ${name} exec ${options.task}`, })); }, + + async createRunManyCommand(options, onlyProjects) { + const { stdout } = await executeProcess({ command: 'yarn', args: ['-v'] }); + const isV1 = stdout.startsWith('1.'); + + if (isV1) { + // neither parallel execution nor projects filter are supported in Yarn v1 + return `yarn workspaces run ${options.task}`; + } + + return [ + 'yarn', + 'workspaces', + 'foreach', + ...(options.parallel ? ['--parallel'] : []), + ...(typeof options.parallel === 'number' + ? [`--jobs=${options.parallel}`] + : []), + ...(onlyProjects?.map(project => `--include=${project}`) ?? ['--all']), + options.task, + ].join(' '); + }, }; diff --git a/packages/ci/src/lib/monorepo/list-projects.ts b/packages/ci/src/lib/monorepo/list-projects.ts index 961608af5..3caa64086 100644 --- a/packages/ci/src/lib/monorepo/list-projects.ts +++ b/packages/ci/src/lib/monorepo/list-projects.ts @@ -4,55 +4,80 @@ import type { Logger, Settings } from '../models.js'; import { detectMonorepoTool } from './detect-tool.js'; import { getToolHandler } from './handlers/index.js'; import { listPackages } from './packages.js'; -import type { MonorepoHandlerOptions, ProjectConfig } from './tools.js'; +import type { + MonorepoHandlerOptions, + MonorepoTool, + ProjectConfig, +} from './tools.js'; + +export type MonorepoProjects = { + tool: MonorepoTool | null; + projects: ProjectConfig[]; + runManyCommand?: (onlyProjects?: string[]) => string | Promise; +}; export async function listMonorepoProjects( settings: Settings, -): Promise { - if (!settings.monorepo) { - throw new Error('Monorepo mode not enabled'); - } - +): Promise { const logger = settings.logger; - const options = createMonorepoHandlerOptions(settings); - const tool = - settings.monorepo === true - ? await detectMonorepoTool(options) - : settings.monorepo; - if (settings.monorepo === true) { - if (tool) { - logger.info(`Auto-detected monorepo tool ${tool}`); - } else { - logger.info("Couldn't auto-detect any supported monorepo tool"); - } - } else { - logger.info(`Using monorepo tool "${tool}" from inputs`); - } + const tool = await resolveMonorepoTool(settings, options); if (tool) { const handler = getToolHandler(tool); const projects = await handler.listProjects(options); logger.info(`Found ${projects.length} projects in ${tool} monorepo`); logger.debug(`Projects: ${projects.map(({ name }) => name).join(', ')}`); - return projects; + return { + tool, + projects, + runManyCommand: onlyProjects => + handler.createRunManyCommand(options, onlyProjects), + }; } if (settings.projects) { - return listProjectsByGlobs({ + const projects = await listProjectsByGlobs({ patterns: settings.projects, cwd: options.cwd, bin: settings.bin, logger, }); + return { tool, projects }; } - return listProjectsByNpmPackages({ + const projects = await listProjectsByNpmPackages({ cwd: options.cwd, bin: settings.bin, logger, }); + return { tool, projects }; +} + +async function resolveMonorepoTool( + settings: Settings, + options: MonorepoHandlerOptions, +): Promise { + if (!settings.monorepo) { + // shouldn't happen, handled by caller + throw new Error('Monorepo mode not enabled'); + } + const logger = settings.logger; + + if (typeof settings.monorepo === 'string') { + logger.info(`Using monorepo tool "${settings.monorepo}" from inputs`); + return settings.monorepo; + } + + const tool = await detectMonorepoTool(options); + if (tool) { + logger.info(`Auto-detected monorepo tool ${tool}`); + } else { + logger.info("Couldn't auto-detect any supported monorepo tool"); + } + + return tool; } function createMonorepoHandlerOptions( @@ -61,6 +86,7 @@ function createMonorepoHandlerOptions( return { task: settings.task, cwd: settings.directory, + parallel: false, // TODO: add to settings nxProjectsFilter: settings.nxProjectsFilter, ...(!settings.silent && { observer: { diff --git a/packages/ci/src/lib/monorepo/list-projects.unit.test.ts b/packages/ci/src/lib/monorepo/list-projects.unit.test.ts index 439897473..d75a1b0d5 100644 --- a/packages/ci/src/lib/monorepo/list-projects.unit.test.ts +++ b/packages/ci/src/lib/monorepo/list-projects.unit.test.ts @@ -5,8 +5,10 @@ import { MEMFS_VOLUME } from '@code-pushup/test-utils'; import * as utils from '@code-pushup/utils'; import { DEFAULT_SETTINGS } from '../constants.js'; import type { Settings } from '../models.js'; -import { listMonorepoProjects } from './list-projects.js'; -import type { ProjectConfig } from './tools.js'; +import { + type MonorepoProjects, + listMonorepoProjects, +} from './list-projects.js'; describe('listMonorepoProjects', () => { const MONOREPO_SETTINGS: Settings = { @@ -53,10 +55,14 @@ describe('listMonorepoProjects', () => { MEMFS_VOLUME, ); - await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual([ - { name: 'backend', bin: 'npx nx run backend:code-pushup --' }, - { name: 'frontend', bin: 'npx nx run frontend:code-pushup --' }, - ] satisfies ProjectConfig[]); + await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual({ + tool: 'nx', + projects: [ + { name: 'backend', bin: 'npx nx run backend:code-pushup --' }, + { name: 'frontend', bin: 'npx nx run frontend:code-pushup --' }, + ], + runManyCommand: expect.any(Function), + } satisfies MonorepoProjects); expect(utils.executeProcess).toHaveBeenCalledWith< Parameters<(typeof utils)['executeProcess']> @@ -96,24 +102,28 @@ describe('listMonorepoProjects', () => { 'e2e/package.json': pkgJsonContent({ name: 'e2e', }), - 'frontend/backoffice/package.json': pkgJsonContent({ - name: 'backoffice', + 'frontend/cms/package.json': pkgJsonContent({ + name: 'cms', scripts: { 'code-pushup': 'code-pushup --no-progress' }, }), - 'frontend/website/package.json': pkgJsonContent({ - name: 'website', + 'frontend/web/package.json': pkgJsonContent({ + name: 'web', scripts: { 'code-pushup': 'code-pushup --no-progress' }, }), }, MEMFS_VOLUME, ); - await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual([ - { name: 'api', bin: 'npx turbo run code-pushup -F api --' }, - { name: 'auth', bin: 'npx turbo run code-pushup -F auth --' }, - { name: 'backoffice', bin: 'npx turbo run code-pushup -F backoffice --' }, - { name: 'website', bin: 'npx turbo run code-pushup -F website --' }, - ] satisfies ProjectConfig[]); + await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual({ + tool: 'turbo', + projects: [ + { name: 'api', bin: 'npx turbo run code-pushup --filter=api --' }, + { name: 'auth', bin: 'npx turbo run code-pushup --filter=auth --' }, + { name: 'cms', bin: 'npx turbo run code-pushup --filter=cms --' }, + { name: 'web', bin: 'npx turbo run code-pushup --filter=web --' }, + ], + runManyCommand: expect.any(Function), + } satisfies MonorepoProjects); }); it('should detect packages in PNPM workspace with code-pushup script', async () => { @@ -140,11 +150,24 @@ describe('listMonorepoProjects', () => { MEMFS_VOLUME, ); - await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual([ - { name: 'backend', bin: 'pnpm -F backend run code-pushup' }, - { name: 'frontend', bin: 'pnpm -F frontend run code-pushup' }, - { name: '@repo/utils', bin: 'pnpm -F @repo/utils run code-pushup' }, - ] satisfies ProjectConfig[]); + await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual({ + tool: 'pnpm', + projects: [ + { + name: 'backend', + bin: 'pnpm --filter=backend run code-pushup', + }, + { + name: 'frontend', + bin: 'pnpm --filter=frontend run code-pushup', + }, + { + name: '@repo/utils', + bin: 'pnpm --filter=@repo/utils run code-pushup', + }, + ], + runManyCommand: expect.any(Function), + } satisfies MonorepoProjects); }); it('should detect Yarn workspaces with code-pushup installed individually', async () => { @@ -170,10 +193,14 @@ describe('listMonorepoProjects', () => { MEMFS_VOLUME, ); - await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual([ - { name: 'cli', bin: 'yarn workspace cli exec code-pushup' }, - { name: 'core', bin: 'yarn workspace core exec code-pushup' }, - ] satisfies ProjectConfig[]); + await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual({ + tool: 'yarn', + projects: [ + { name: 'cli', bin: 'yarn workspace cli exec code-pushup' }, + { name: 'core', bin: 'yarn workspace core exec code-pushup' }, + ], + runManyCommand: expect.any(Function), + } satisfies MonorepoProjects); }); it('should detect NPM workspaces when code-pushup installed at root level', async () => { @@ -195,10 +222,20 @@ describe('listMonorepoProjects', () => { MEMFS_VOLUME, ); - await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual([ - { name: 'backend', bin: 'npm -w backend exec code-pushup --' }, - { name: 'frontend', bin: 'npm -w frontend exec code-pushup --' }, - ] satisfies ProjectConfig[]); + await expect(listMonorepoProjects(MONOREPO_SETTINGS)).resolves.toEqual({ + tool: 'npm', + projects: [ + { + name: 'backend', + bin: 'npm --workspace=backend exec code-pushup --', + }, + { + name: 'frontend', + bin: 'npm --workspace=frontend exec code-pushup --', + }, + ], + runManyCommand: expect.any(Function), + } satisfies MonorepoProjects); }); it('should list folders matching globs passed as input when no tool detected', async () => { @@ -226,23 +263,26 @@ describe('listMonorepoProjects', () => { monorepo: true, projects: ['backend/*', 'frontend'], }), - ).resolves.toEqual([ - { - name: join('backend', 'api'), - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME, 'backend', 'api'), - }, - { - name: join('backend', 'auth'), - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME, 'backend', 'auth'), - }, - { - name: 'frontend', - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME, 'frontend'), - }, - ] satisfies ProjectConfig[]); + ).resolves.toEqual({ + tool: null, + projects: [ + { + name: join('backend', 'api'), + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME, 'backend', 'api'), + }, + { + name: join('backend', 'auth'), + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME, 'backend', 'auth'), + }, + { + name: 'frontend', + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME, 'frontend'), + }, + ], + } satisfies MonorepoProjects); }); it('should list all folders with a package.json when no tool detected and no patterns provided', async () => { @@ -265,28 +305,31 @@ describe('listMonorepoProjects', () => { monorepo: true, projects: null, }), - ).resolves.toEqual([ - { - name: 'my-app', - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME), - }, - { - name: 'migrate', - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME, 'scripts', 'db', 'migrate'), - }, - { - name: 'seed', - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME, 'scripts', 'db', 'seed'), - }, - { - name: 'generate-token', - bin: 'npx --no-install code-pushup', - directory: join(MEMFS_VOLUME, 'scripts', 'generate-token'), - }, - ] satisfies ProjectConfig[]); + ).resolves.toEqual({ + tool: null, + projects: [ + { + name: 'my-app', + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME), + }, + { + name: 'migrate', + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME, 'scripts', 'db', 'migrate'), + }, + { + name: 'seed', + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME, 'scripts', 'db', 'seed'), + }, + { + name: 'generate-token', + bin: 'npx --no-install code-pushup', + directory: join(MEMFS_VOLUME, 'scripts', 'generate-token'), + }, + ], + } satisfies MonorepoProjects); }); it('should prefer tool provided as input (PNPM) over tool which would be auto-detected otherwise (Turborepo)', async () => { @@ -319,11 +362,27 @@ describe('listMonorepoProjects', () => { await expect( listMonorepoProjects({ ...MONOREPO_SETTINGS, monorepo: 'pnpm' }), - ).resolves.toEqual([ - { name: 'backoffice', bin: 'pnpm -F backoffice exec code-pushup' }, - { name: 'frontoffice', bin: 'pnpm -F frontoffice exec code-pushup' }, - { name: '@repo/models', bin: 'pnpm -F @repo/models exec code-pushup' }, - { name: '@repo/ui', bin: 'pnpm -F @repo/ui exec code-pushup' }, - ] satisfies ProjectConfig[]); + ).resolves.toEqual({ + tool: 'pnpm', + projects: [ + { + name: 'backoffice', + bin: 'pnpm --filter=backoffice exec code-pushup', + }, + { + name: 'frontoffice', + bin: 'pnpm --filter=frontoffice exec code-pushup', + }, + { + name: '@repo/models', + bin: 'pnpm --filter=@repo/models exec code-pushup', + }, + { + name: '@repo/ui', + bin: 'pnpm --filter=@repo/ui exec code-pushup', + }, + ], + runManyCommand: expect.any(Function), + } satisfies MonorepoProjects); }); }); diff --git a/packages/ci/src/lib/monorepo/tools.ts b/packages/ci/src/lib/monorepo/tools.ts index 435315eb0..2dc26b6ea 100644 --- a/packages/ci/src/lib/monorepo/tools.ts +++ b/packages/ci/src/lib/monorepo/tools.ts @@ -7,11 +7,16 @@ export type MonorepoToolHandler = { tool: MonorepoTool; isConfigured: (options: MonorepoHandlerOptions) => Promise; listProjects: (options: MonorepoHandlerOptions) => Promise; + createRunManyCommand: ( + options: MonorepoHandlerOptions, + onlyProjects?: string[], + ) => string | Promise; }; export type MonorepoHandlerOptions = { task: string; cwd: string; + parallel: boolean | number; observer?: ProcessObserver; nxProjectsFilter: string | string[]; }; diff --git a/packages/ci/src/lib/run.ts b/packages/ci/src/lib/run.ts index 1d5b865be..766e62fc7 100644 --- a/packages/ci/src/lib/run.ts +++ b/packages/ci/src/lib/run.ts @@ -50,7 +50,7 @@ export async function runInCI( if (settings.monorepo) { logger.info('Running Code PushUp in monorepo mode'); - const projects = await listMonorepoProjects(settings); + const { projects } = await listMonorepoProjects(settings); const projectResults = await projects.reduce>( async (acc, project) => [ ...(await acc),