feat: optimize runtime when build (#5082)

* chore: bump version

* feat: analyze runtime

* feat: support vite alias

* fix: vite config for hooks

* fix: add hook

* feat: optimize mpa build

* fix: mpa generator

* fix: compatible with disableRuntime

* fix: space size

* fix: max old space size

* chore: changelog and version

* chore: version

* chore: optimize code

* chore: optimize code

* fix: optimize code

* fix: optimize code

packages/build-app-helpers/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.5.0
+
+- [chore] bump version of `es-module-lexer`
+- [feat] support scan source code of import statement
+
 ## 2.4.2
 
 - [fix] import redirection with alias

packages/build-app-helpers/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@builder/app-helpers",
-  "version": "2.4.2",
+  "version": "2.5.0",
   "description": "build helpers for app framework",
   "author": "ice-admin@alibaba-inc.com",
   "homepage": "",
@@ -26,6 +26,8 @@
     "@babel/traverse": "^7.12.12",
     "fs-extra": "^8.1.0",
     "lodash": "^4.17.20",
-    "es-module-lexer": "^0.7.1"
+    "esbuild": "^0.13.12",
+    "es-module-lexer": "^0.9.0",
+    "fast-glob": "^3.2.7"
   }
 }

packages/build-app-helpers/src/analyzeRuntime.ts

@@ -0,0 +1,198 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import * as glob from 'fast-glob';
+import { init, parse } from 'es-module-lexer';
+import { transform } from 'esbuild';
+import type { Loader } from 'esbuild';
+
+type CheckMap = Record<string, boolean>;
+type WebpackAlias = Record<string, string>;
+type ViteAlias = {find: string | RegExp, replacement: string}[];
+interface Options {
+  rootDir: string;
+  parallel?: number;
+  analyzeRelativeImport?: boolean;
+  mode?: 'webpack' | 'vite';
+  // webpack mode
+  alias?: WebpackAlias | ViteAlias;
+  customRuntimeRules?: Record<string, string[]>
+}
+
+const defaultRuntimeRules = {
+  'build-plugin-ice-request': ['request', 'useRequest'],
+  'build-plugin-ice-auth': ['useAuth', 'withAuth'],
+};
+
+export async function globSourceFiles(sourceDir: string): Promise<string[]> {
+  return await glob('**/*.{js,ts,jsx,tsx}', {
+    cwd: sourceDir,
+    ignore: [
+      '**/node_modules/**',
+      'src/apis/**',
+      '**/__tests__/**',
+    ],
+    absolute: true,
+  });
+}
+
+function addLastSlash(filePath: string) {
+  return filePath.endsWith('/') ? filePath : `${filePath}/`;
+}
+
+function getWebpackAliasedPath(filePath: string, alias: WebpackAlias): string {
+  let aliasedPath = filePath;
+  // eslint-disable-next-line no-restricted-syntax
+  for (const aliasKey of Object.keys(alias || {})) {
+    const isStrict = aliasKey.endsWith('$');
+    const strictKey = isStrict ? aliasKey.slice(0, -1) : aliasKey;
+    const aliasValue = alias[aliasKey];
+
+    if (aliasValue.match(/.(j|t)s(x)?$/)) {
+      if (aliasedPath === strictKey) {
+        aliasedPath = aliasValue;
+        break;
+      }
+    } else {
+      // case: { xyz: '/some/dir' }, { xyz$: '/some/dir' }
+      // import xyz from 'xyz';   // resolved as '/some/dir'
+      if (aliasedPath === strictKey) {
+        aliasedPath = aliasValue;
+        break;
+      } else if (isStrict) {
+        // case: { xyz$: '/some/dir' }
+        // import xyz from 'xyz/file.js'; // resolved as /abc/node_modules/xyz/file.js
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+      // case: { xyz: '/some/dir' }
+      // import xyz from 'xyz/file.js'; // resolved as /some/dir/file.js
+      const slashedKey = addLastSlash(strictKey);
+      if (aliasedPath.startsWith(slashedKey)) {
+        aliasedPath = aliasedPath.replace(new RegExp(`^${slashedKey}`), addLastSlash(aliasValue));
+        break;
+      }
+    }
+  }
+  return aliasedPath;
+}
+
+function getViteAliasedPath(filePath: string, alias: ViteAlias): string {
+  // apply aliases
+  let aliasedPath = filePath;
+  // eslint-disable-next-line no-restricted-syntax
+  for (const { find, replacement } of (alias || [])) {
+    const matches =
+      typeof find === 'string' ? aliasedPath.startsWith(find) : find.test(aliasedPath);
+    if (matches) {
+      aliasedPath = aliasedPath.replace(find, replacement);
+      break;
+    }
+  }
+  return aliasedPath;
+}
+
+export function getImportPath(importSpecifier: string, importer: string, options: Pick<Options, 'alias'|'rootDir'|'mode'>) {
+  const { alias, rootDir, mode } = options;
+  let aliasedPath = mode === 'webpack'
+    ? getWebpackAliasedPath(importSpecifier, alias as WebpackAlias)
+    : getViteAliasedPath(importSpecifier, alias as ViteAlias);
+  if (!path.isAbsolute(aliasedPath)) {
+    try {
+      // 检测是否可以在 node_modules 下找到依赖，如果可以直接使用该依赖
+      aliasedPath = require.resolve(aliasedPath, { paths: [rootDir]});
+    } catch (e) {
+      // ignore errors
+      aliasedPath = path.resolve(path.dirname(importer), aliasedPath); 
+    }
+  }
+  // filter path with node_modules
+  if (aliasedPath.includes('node_modules')) {
+    return '';
+  } else if (!path.extname(aliasedPath)) {
+    // get index file of 
+    const basename = path.basename(aliasedPath);
+    const patterns = [`${basename}.{js,ts,jsx,tsx}`, `${basename}/index.{js,ts,jsx,tsx}`];
+
+    return glob.sync(patterns, {
+      cwd: path.dirname(aliasedPath),
+      absolute: true,
+    })[0];
+  }
+  return aliasedPath;
+}
+
+export default async function analyzeRuntime(files: string[], options: Options): Promise<CheckMap> {
+  const { analyzeRelativeImport, rootDir, alias, mode = 'webpack', parallel, customRuntimeRules = {} } = options;
+  const parallelNum = parallel ?? 10;
+  const sourceFiles = [...files];
+  const checkMap: CheckMap = {};
+  const runtimeRules = { ...defaultRuntimeRules, ...customRuntimeRules };
+  // init check map
+  const checkPlugins = Object.keys(runtimeRules);
+  checkPlugins.forEach((pluginName) => {
+    checkMap[pluginName] = false;
+  });
+
+  async function analyzeFile(filePath: string) {
+    let source = fs.readFileSync(filePath, 'utf-8');
+    const lang = path.extname(filePath).slice(1);
+    let loader: Loader;
+    if (lang === 'ts' || lang === 'tsx') {
+      loader = lang;
+    }
+    try {
+      if (loader) {
+        // transform content first since es-module-lexer can't handle ts file
+        source = (await transform(source, { loader })).code;
+      }
+      await init;
+      const imports = parse(source)[0];
+      await Promise.all(imports.map((importSpecifier) => {
+        return (async () => {
+          const importName = importSpecifier.n;
+          // filter source code
+          if (importName === 'ice') {
+            const importStr = source.substring(importSpecifier.ss, importSpecifier.se);
+            checkPlugins.forEach((pluginName) => {
+              const apiList: string[] = runtimeRules[pluginName];
+              if (apiList.some((apiName) => importStr.includes(apiName))) {
+                checkMap[pluginName] = true;
+              }
+            });
+          } else if (analyzeRelativeImport) {
+            let importPath = importName;
+            if (!path.isAbsolute(importPath)) {
+              importPath = getImportPath(importPath, filePath, { rootDir, alias, mode });
+            }
+            if (importPath && !sourceFiles.includes(importPath) && fs.existsSync(importPath)) {
+              await analyzeFile(importPath);
+            }
+          }
+        })();
+      }));
+    } catch (err) {
+      console.log('[ERROR]', `optimize runtime failed when analyze ${filePath}`);
+      throw err;
+    }
+  }
+
+  try {
+    for (
+      let i = 0;
+      i * parallelNum < sourceFiles.length && checkPlugins.some((pluginName) => checkMap[pluginName] === false);
+      i++) {
+      // eslint-disable-next-line no-await-in-loop
+      await Promise.all(sourceFiles.slice(i * parallelNum, parallelNum).map((filePath) => {
+        return analyzeFile(filePath);
+      }));
+    }
+  } catch(err) {
+    // 如果发生错误，兜底启用所有自动检测的运行时插件，防止错误地移除
+    checkPlugins.forEach((pluginName) => {
+      checkMap[pluginName] = true;
+    });
+    return checkMap;
+  }
+
+  return checkMap;
+}

packages/build-app-helpers/src/index.ts

@@ -9,3 +9,5 @@ export { default as checkExportDefaultDeclarationExists } from './checkExportDef
 export { default as getSassImplementation } from './getSassImplementation';
 export { default as getLessImplementation } from './getLessImplementation';
 export { default as redirectImport } from './redirectImport';
+export { default as analyzeRuntime } from './analyzeRuntime';
+export { globSourceFiles } from './analyzeRuntime';

packages/build-app-helpers/src/tests/analyzeRuntime.test.ts

@@ -0,0 +1,125 @@
+import path = require('path');
+import analyzeRuntime, { globSourceFiles, getImportPath } from '../analyzeRuntime';
+
+describe('get source file', () => {
+  const rootDir = path.join(__dirname, './fixtures/analyzeRuntime/');
+  it('glob js/ts files', async () => {
+    const files = await globSourceFiles(rootDir);
+    expect(files.length).toBe(4);
+  });
+});
+
+describe('get aliased path', () => {
+  const rootDir = path.join(__dirname, './fixtures/analyzeRuntime/');
+  it('webpack alias: { ice: \'./runApp\' }', () => {
+    const aliasedPath = getImportPath('ice', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: { ice: './runApp' },
+      mode: 'webpack',
+    });
+    // file not exists throw error
+    expect(aliasedPath).toBe(undefined);
+  });
+
+  it('get relative path with mode webpack', () => {
+    const aliasedPath = getImportPath('./store', path.join(rootDir, 'page/index.js'), {
+      rootDir,
+      alias: {},
+      mode: 'webpack',
+    });
+    // file not exists throw error
+    expect(aliasedPath).toBe(path.join(rootDir, 'page/store.js'));
+  });
+
+  it('get relative path with mode vite', () => {
+    const aliasedPath = getImportPath('./store', path.join(rootDir, 'page/index.js'), {
+      rootDir,
+      alias: {},
+      mode: 'vite',
+    });
+    // file not exists throw error
+    expect(aliasedPath).toBe(path.join(rootDir, 'page/store.js'));
+  });
+
+  it('webpack alias: { ice: \'react\' }', () => {
+    const aliasedPath = getImportPath('ice', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: { ice: 'react' },
+      mode: 'webpack',
+    });
+    expect(aliasedPath).toBe('');
+  });
+
+  it('webpack alias: { @: \'rootDir\' }', () => {
+    const aliasedPath = getImportPath('@/page', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: { '@': rootDir },
+      mode: 'webpack',
+    });
+    expect(aliasedPath).toBe(path.join(rootDir, 'page/index.js'));
+  });
+
+  it('webpack alias: { @$: \'rootDir\' }', () => {
+    const aliasedPath = getImportPath('@/page', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: { '@$': rootDir },
+      mode: 'webpack',
+    });
+    // without match any alias rule, throw error
+    expect(aliasedPath).toBe(undefined);
+  });
+
+  it('vite alias: [{find: \'ice\', replacement: \'react\'}]', () => {
+    const aliasedPath = getImportPath('ice', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: [{ find: 'ice', replacement: 'react' }],
+      mode: 'vite',
+    });
+    // filter node_modules dependencies
+    expect(aliasedPath).toBe('');
+  });
+
+  it('vite alias: [{find: \'@\', replacement: \'rootDir\'}]', () => {
+    const aliasedPath = getImportPath('@/page', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: [{ find: '@', replacement: rootDir }],
+      mode: 'vite',
+    });
+    expect(aliasedPath).toBe(path.join(rootDir, 'page/index.js'));
+  });
+
+  it('vite alias: [{find: \/@$\/, replacement: \'rootDir\'}]', () => {
+    const aliasedPath = getImportPath('@/page', path.join(rootDir, 'index.js'), {
+      rootDir,
+      alias: [{ find: /@$/, replacement: rootDir }],
+      mode: 'vite',
+    });
+    // without match any alias rule, throw error
+    expect(aliasedPath).toBe(undefined);
+  });
+});
+
+describe('Analyze Runtime', () => {
+  const rootDir = path.join(__dirname, './fixtures/analyzeRuntime/');
+  const entryFile = path.join(rootDir, 'index.js');
+
+  it('analyze entry file', async () => {
+    const checkMap = await analyzeRuntime([entryFile], { rootDir });
+    expect(checkMap).toStrictEqual({
+      'build-plugin-ice-request': true,
+      'build-plugin-ice-auth': false,
+    });
+  });
+
+  it('analyze relative import', async () => {
+    const checkMap = await analyzeRuntime(
+      [entryFile],
+      { rootDir, analyzeRelativeImport: true, alias: { '@': path.join(rootDir)}, customRuntimeRules: { 'build-plugin-ice-store': ['createStore']} }
+    );
+    expect(checkMap).toStrictEqual({
+      'build-plugin-ice-request': true,
+      'build-plugin-ice-auth': true,
+      'build-plugin-ice-store': true,
+    });
+  });
+});

packages/build-app-helpers/src/tests/fixtures/analyzeRuntime/common.ts

@@ -0,0 +1 @@
+export default function common() {};

packages/build-app-helpers/src/tests/fixtures/analyzeRuntime/index.js

@@ -0,0 +1,3 @@
+import { useRequest } from 'ice';
+import { page } from '@/page';
+import common from '@/common';

packages/build-app-helpers/src/tests/fixtures/analyzeRuntime/page/index.js

@@ -0,0 +1,4 @@
+import { useAuth } from 'ice';
+import store from './store';
+
+export const page = () => {};

packages/build-app-helpers/src/tests/fixtures/analyzeRuntime/page/store.js

@@ -0,0 +1,2 @@
+import { createStore } from 'ice';
+export default createStore;

packages/build-app-templates/src/templates/common/loadStaticModules.ts.ejs

@@ -1,10 +1,9 @@
-import { IAppConfig } from '../types';
-
 <% const moduleNames = []; %>
 <% if (runtimeModules.length) {%>
   <% runtimeModules.filter((moduleInfo) => moduleInfo.staticModule).forEach((runtimeModule, index) => { %>
     <% moduleNames.push('module' + index) %>import module<%= index %> from '<%= runtimeModule.path %>';<% }) %>
 <% } %>
+import type { IAppConfig } from '<%- typesPath %>';
 
 function loadStaticModules(appConfig: IAppConfig) {
   <% if (moduleNames.length) {%>