WIP: Dependency finder for smarter builds

.teamcity/settings.kts

@@ -313,7 +313,7 @@ object SmartBuildLauncher : BuildType({
 		bashNodeScript {
 			name = "Launch relevant builds"
 			scriptContent = """
-				node ./packages/dependency-finder/dist/esm/index.js
+				node ./packages/dependency-finder/dist/esm/index.js --changedFiles %system.teamcity.build.changedFiles.file% --package apps/editing-toolkit
 			"""
 		}
 	}

apps/editing-toolkit/editing-toolkit-plugin/common/hide-plugin-buttons-mobile.js

@@ -1,5 +1,4 @@
 /**
  * Internal dependencies
  */
-
 import './hide-plugin-buttons-mobile.scss';

apps/editing-toolkit/jest.config.js

@@ -16,14 +16,10 @@
 const defaults = require( '@wordpress/scripts/config/jest-unit.config.js' );
 const path = require( 'path' );
 
-// Basically, CWD, so 'apps/editing-toolkit'.
-// Without this, it tries to use 'apps/editing-toolkit/bin'
-const pluginRoot = path.resolve( './' );
-
 const config = {
 	...defaults,
 	rootDir: path.normalize( '../../' ), // To detect wp-calypso root node_modules
-	testMatch: [ `${ pluginRoot }/**/?(*.)test.[jt]s?(x)` ],
+	testMatch: [ `${ __dirname }/**/?(*.)test.[jt]s?(x)` ],
 	transform: { '^.+\\.[jt]sx?$': path.join( __dirname, 'bin', 'babel-transform' ) },
 	setupFilesAfterEnv: [
 		...( defaults.setupFilesAfterEnv || [] ), // extend if present

apps/editing-toolkit/package.json

@@ -102,6 +102,7 @@
 		"@automattic/page-pattern-modal": "^1.0.0-alpha.0",
 		"@automattic/plans-grid": "^1.0.0-alpha.0",
 		"@automattic/typography": "^1.0.0",
+		"@automattic/whats-new": "^1.0.0",
 		"@babel/core": "^7.14.0",
 		"@wordpress/a11y": "^2.15.3",
 		"@wordpress/api-fetch": "^4.0.0",

client/lib/plans/features-list.js

@@ -42,7 +42,7 @@ import {
 	FEATURE_BLANK,
 	FEATURE_BLOG_DOMAIN,
 	FEATURE_BUSINESS_ONBOARDING,
-  FEATURE_CLOUDFLARE_ANALYTICS,
+	FEATURE_CLOUDFLARE_ANALYTICS,
 	FEATURE_COLLECT_PAYMENTS_V2,
 	FEATURE_COMMUNITY_SUPPORT,
 	FEATURE_CRM_LEADS_AND_FUNNEL,

package-map.json

@@ -0,0 +1,91 @@
+[
+	{ "path": "packages/accessible-focus" },
+	{ "path": "packages/accessible-focus" },
+	{ "path": "packages/babel-plugin-i18n-calypso" },
+	{ "path": "packages/babel-plugin-transform-wpcalypso-async" },
+	{ "path": "packages/browser-data-collector" },
+	{ "path": "packages/calypso-analytics" },
+	{ "path": "packages/calypso-build" },
+	{ "path": "packages/calypso-codemods" },
+	{ "path": "packages/calypso-color-schemes" },
+	{ "path": "packages/calypso-config" },
+	{ "path": "packages/calypso-doctor" },
+	{ "path": "packages/calypso-polyfills" },
+	{ "path": "packages/calypso-stripe" },
+	{ "path": "packages/components" },
+	{ "path": "packages/composite-checkout" },
+	{ "path": "packages/create-calypso-config" },
+	{ "path": "packages/data-stores" },
+	{ "path": "packages/dependency-finder" },
+	{ "path": "packages/domain-picker" },
+	{ "path": "packages/effective-module-tree" },
+	{ "path": "packages/eslint-plugin-wpcalypso" },
+	{ "path": "packages/explat-client-react-helpers" },
+	{ "path": "packages/explat-client" },
+	{ "path": "packages/format-currency" },
+	{ "path": "packages/i18n-calypso-cli" },
+	{ "path": "packages/i18n-calypso" },
+	{ "path": "packages/i18n-utils" },
+	{ "path": "packages/js-utils" },
+	{ "path": "packages/language-picker" },
+	{ "path": "packages/languages" },
+	{ "path": "packages/launch" },
+	{ "path": "packages/load-script" },
+	{ "path": "packages/material-design-icons" },
+	{ "path": "packages/onboarding" },
+	{ "path": "packages/page-pattern-modal" },
+	{ "path": "packages/photon" },
+	{ "path": "packages/plans-grid" },
+	{ "path": "packages/popup-monitor" },
+	{ "path": "packages/request-external-access" },
+	{ "path": "packages/retarget-open-prs" },
+	{ "path": "packages/search" },
+	{ "path": "packages/shopping-cart" },
+	{ "path": "packages/social-previews" },
+	{ "path": "packages/spec-junit-reporter" },
+	{ "path": "packages/spec-xunit-reporter" },
+	{ "path": "packages/state-utils" },
+	{ "path": "packages/tree-select" },
+	{ "path": "packages/typography" },
+	{ "path": "packages/viewport-react" },
+	{ "path": "packages/viewport" },
+	{ "path": "packages/webpack-config-flag-plugin" },
+	{ "path": "packages/webpack-extensive-lodash-replacement-plugin" },
+	{ "path": "packages/webpack-inline-constant-exports-plugin" },
+	{ "path": "packages/webpack-rtl-plugin" },
+	{ "path": "packages/whats-new" },
+	{ "path": "packages/wp-babel-makepot" },
+	{ "path": "packages/wpcom-checkout" },
+	{ "path": "packages/wpcom-proxy-request" },
+	{ "path": "packages/wpcom.js" },
+	{
+		"path": "apps/editing-toolkit",
+		"buildIds": [ "calypso_WPComPlugins_EditorToolKit" ],
+		"additionalEntryPoints": [
+			"apps/editing-toolkit/editing-toolkit-plugin/block-inserter-modifications/contextual-tips.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/block-patterns/index.ts",
+			"apps/editing-toolkit/editing-toolkit-plugin/common/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/common/data-stores/index.ts",
+			"apps/editing-toolkit/editing-toolkit-plugin/common/hide-plugin-buttons-mobile.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/dotcom-fse/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/editor-site-launch/focused-launch.ts",
+			"apps/editing-toolkit/editing-toolkit-plugin/editor-site-launch/gutenboarding-launch.ts",
+			"apps/editing-toolkit/editing-toolkit-plugin/editor-site-launch/launch-button.ts",
+			"apps/editing-toolkit/editing-toolkit-plugin/event-countdown-block/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/global-styles/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/jetpack-timeline/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/newspack-blocks/blog-posts-block-editor.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/newspack-blocks/blog-posts-block-view.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/newspack-blocks/carousel-block-editor.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/newspack-blocks/carousel-block-view.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/posts-list-block.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/starter-page-templates/index.tsx",
+			"apps/editing-toolkit/editing-toolkit-plugin/whats-new/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/wpcom-block-editor-nav-sidebar/index.ts",
+			"apps/editing-toolkit/editing-toolkit-plugin/wpcom-block-editor-nux/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/error-reporting/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/posts-list-block/index.js",
+			"apps/editing-toolkit/editing-toolkit-plugin/**/*.php"
+		]
+	}
+]

packages/dependency-finder/.eslintrc.cjs

@@ -0,0 +1,10 @@
+module.exports = {
+	env: {
+		node: true,
+	},
+	rules: {
+		// This is a node.js project, it is ok to import node modules
+		'import/no-nodejs-modules': 'off',
+		'no-console': 'off',
+	},
+};

packages/dependency-finder/package.json

@@ -0,0 +1,33 @@
+{
+	"name": "@automattic/dependency-finder",
+	"version": "0.1.0",
+	"description": "Generate the list of dependencies of a set of files, recursively",
+	"main": "index.ts",
+	"author": "Automattic Inc.",
+	"type": "module",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/Automattic/wp-calypso.git",
+		"directory": "packages/dependency-finder"
+	},
+	"keywords": [
+		"dependencies",
+		"node_modules"
+	],
+	"license": "GPL-2.0-or-later",
+	"dependencies": {
+		"dependency-tree": "^8.1.0",
+		"globby": "^11.0.3",
+		"jest-config": "^26.6.3",
+		"read-pkg": "^6.0.0",
+		"read-pkg-up": "^8.0.0",
+		"tslib": "^2.2.0",
+		"yargs": "^17.0.1"
+	},
+	"scripts": {
+		"clean": "tsc --build ./tsconfig.json --clean && npx rimraf dist",
+		"build": "tsc --build ./tsconfig.json",
+		"prepack": "yarn run clean && yarn run build",
+		"watch": "tsc --build ./tsconfig.json --watch"
+	}
+}

packages/dependency-finder/src/index.ts

@@ -0,0 +1,166 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+import { readFile, readdir } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import childProcess from 'child_process';
+import { promisify } from 'util';
+import yargs from 'yargs';
+
+const args = yargs( process.argv.slice( 2 ) )
+	.usage( 'Usage: $0' )
+	.option( 'changedFiles', {
+		describe: 'A path to the list of VCS changed files',
+		type: 'string',
+	} )
+	.option( 'package', {
+		describe: 'The package to process from package-map.json',
+		type: 'string',
+	} ).argv;
+
+const exec = promisify( childProcess.exec );
+
+/**
+ * Internal dependencies
+ */
+import { findDependencies } from './lib/find-dependencies.js';
+
+const packageMapPath = path.join(
+	path.dirname( fileURLToPath( import.meta.url ) ),
+	'../../../../package-map.json'
+);
+
+type PackageMapEntry = {
+	path: string;
+	additionalEntryPoints: Array< string > | undefined;
+	buildIds: Array< string > | undefined;
+};
+type PackageMap = Array< PackageMapEntry >;
+
+// TODO: import from build-tools
+async function getMonorepoPackages() {
+	const packages = await readdir( 'packages', { withFileTypes: true } );
+	return packages
+		.filter( ( entry ) => entry.isDirectory() )
+		.map( ( entry ) => path.resolve( 'packages', entry.name ) );
+}
+
+type Project = {
+	matchingFiles: Array< string >;
+	buildIds: Array< string > | undefined;
+};
+
+const findPackageDependencies = async ( entry: PackageMapEntry ): Promise< Project > => {
+	const { path: pkgPath, additionalEntryPoints } = entry;
+	const absolutePkgPath = path.resolve( pkgPath );
+
+	const { missing, packages, modules } = await findDependencies( {
+		pkg: absolutePkgPath,
+		additionalEntryPoints,
+		monorepoPackages: await getMonorepoPackages(),
+	} );
+	const { stdout } = await exec(
+		`find ${ absolutePkgPath } -type f -not \\( -path '*/node_modules/*' -o -path '*/.cache/*' -o -path '*/dist/*' \\)`
+	);
+	const allFiles = stdout.trim().split( '\n' );
+
+	// Files which exist in the filesystem, but the dep finder did not parse.
+	// We exclude files which do not impact builds, such as ".txt" or ".md" files.
+	const unknownFiles = allFiles.filter(
+		( file ) => ! modules.includes( file ) && ! [ '.md', '.txt' ].includes( path.extname( file ) )
+	);
+
+	console.log( 'Package:' );
+	console.log( '  ' + pkgPath );
+	console.log( 'Missing files:' );
+	console.log( missing.length ? missing.map( ( m ) => '  ' + m ).join( '\n' ) : '  -' );
+	console.log( 'Packages:' );
+	console.log( packages.length ? packages.map( ( m ) => '  ' + m ).join( '\n' ) : '  -' );
+	console.log( 'Found files:' );
+	console.log( modules.length ? modules.map( ( m ) => '  ' + m ).join( '\n' ) : '  -' );
+	console.log( 'Unkown files:' );
+	console.log( unknownFiles.length ? unknownFiles.map( ( m ) => '  ' + m ).join( '\n' ) : '  -' );
+	console.log();
+	console.log();
+	return {
+		...entry,
+		matchingFiles: modules,
+	};
+};
+
+/**
+ * Given a list of currently modified files, returns the CI jobs which need to be
+ * launched.
+ *
+ * TODO: improve algorithmic complexity. Currently O(modifiedFiles * projects * matchingFiles).
+ *
+ * @param projects The list of projects.
+ * @param modifiedFiles The list of currently modified files.
+ * @returns A Set of CI Job IDs to launch.
+ */
+// TODO: make sure project files are relative to repo root. Currently, modifiedFiles
+// are relative to the repo root, but project files are absolute.
+function findMatchingBuilds( projects: Project[], modifiedFiles: VCSFileChange[] ) {
+	return modifiedFiles.reduce< Set< string > >( ( acc, modifiedFile ) => {
+		const matchingProject = projects.find( ( proj ) =>
+			proj.matchingFiles.some( ( file ) => file.includes( modifiedFile.path ) )
+		);
+		if ( matchingProject?.buildIds ) {
+			matchingProject.buildIds.forEach( ( id ) => acc.add( id ) );
+		}
+		return acc;
+	}, new Set() );
+}
+
+// <relative file path>:<change type>:<revision>
+// see https://plugins.jetbrains.com/docs/teamcity/risk-tests-reordering-in-custom-test-runner.html
+type VCSFileChange = {
+	path: string;
+	changeType:
+		| 'CHANGED'
+		| 'ADDED'
+		| 'REMOVED'
+		| 'NOT_CHANGED'
+		| 'DIRECTORY_CHANGED'
+		| 'DIRECTORY_ADDED'
+		| 'DIRECTORY_REMOVED';
+	revision: string;
+};
+async function readTeamCityMatchedFiles( filePath: string ) {
+	const rawContents = await readFile( filePath, 'utf8' );
+	return rawContents.split( '\n' ).reduce< VCSFileChange[] >( ( acc, entry ) => {
+		const [ path, changeType, revision ] = entry.split( ':' );
+		if ( path && changeType && revision ) {
+			acc.push( { path, changeType, revision } as VCSFileChange );
+		}
+		return acc;
+	}, [] );
+}
+
+const main = async () => {
+	const packageMap: PackageMap = JSON.parse( await readFile( packageMapPath, 'utf8' ) );
+	const changedFiles = args.changedFiles
+		? await readTeamCityMatchedFiles( args.changedFiles )
+		: null;
+
+	if ( args.package ) {
+		const packageEntry = packageMap.find( ( { path } ) => path === args.package );
+		if ( packageEntry ) {
+			const project = await findPackageDependencies( packageEntry );
+			console.log( project );
+			console.log( changedFiles );
+			if ( changedFiles ) {
+				console.log( 'Finding builds...' );
+				const builds = await findMatchingBuilds( [ project ], changedFiles );
+				console.log( Array.from( builds ) );
+			}
+		}
+	} else {
+		for ( const packageEntry of packageMap ) {
+			await findPackageDependencies( packageEntry );
+		}
+	}
+};
+
+main().catch( console.error );

packages/dependency-finder/src/lib/entrypoints/additional.ts

@@ -0,0 +1,13 @@
+/**
+ * External dependencies
+ */
+import globby from 'globby';
+import { resolve } from 'path';
+
+export const findAdditionalEntryPoints = async (
+	additionalEntryPoints: string[]
+): Promise< string[] > => {
+	return ( await Promise.all( additionalEntryPoints.map( ( pattern ) => globby( pattern ) ) ) )
+		.flat()
+		.map( ( entry ) => resolve( entry ) );
+};

packages/dependency-finder/src/lib/entrypoints/jest.ts

@@ -0,0 +1,23 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+import jestConfig from 'jest-config';
+import globby from 'globby';
+
+export const findJestEntrypoints = async ( {
+	jestConfigPath,
+}: {
+	jestConfigPath: string;
+} ): Promise< string[] > => {
+	const config = await jestConfig.readConfig(
+		{
+			config: jestConfigPath,
+			_: [ '' ],
+			$0: '',
+		},
+		path.dirname( jestConfigPath )
+	);
+	const tests = await globby( config.projectConfig.testMatch );
+	return tests;
+};