feat: remove need for babel

This updates the way we prescan the plugins so as to remove any need for CJS modules. We can thus remove the last remaining use of babel.

BREAKING CHANGE: this is a substantial change, and may introduce issues with plugins. It is unlikely, by it seems prudent to mark this as a major change.

bin/switch-client.sh

@@ -19,6 +19,8 @@
 set -e
 set -o pipefail
 
+rm -f dist/electron/kui.min.js
+
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 cd "$SCRIPTDIR"/..
 

package.json

@@ -33,15 +33,13 @@
     "watch:browser": "concurrently -n COMPILE,WEBPACK --kill-others 'npm run watch:source' 'npm run watch:webpack'",
     "watch:browser:offline": "export IS_OFFLINE_CLIENT=true; concurrently -n COMPILE,WEBPACK --kill-others 'npm run watch:source' 'npm run watch:webpack'",
     "watch:electron": "bash -c \"npm run pty:electron && TARGET=electron-renderer npm run _watch\"",
-    "watch": "bash -c \"npm run kill; npm run compile && npm run link && concurrently -n COMPILE,WEBPACK --kill-others 'npm run watch:source' 'npm run watch:electron'\"",
+    "watch": "bash -c \"npm run kill; rm -f dist/headless/kui.min.js && npm run compile && npm run link && concurrently -n COMPILE,WEBPACK --kill-others 'npm run watch:source' 'npm run watch:electron'\"",
     "proxy": "export PORT=8081; npm run kill:proxy; export KUI_USE_HTTP=true; cd packages/proxy/app && npm install && ../../../tools/travis/proxy.sh ../../..",
-    "watch:source": "concurrently -n TYPESCRIPT,BABEL --kill-others 'tsc --build tsconfig.json --watch' 'WATCH=--watch npx kui-babel'",
-    "compile:prep": "touch node_modules/@kui-shell/prescan.json",
+    "watch:source": "concurrently -n TYPESCRIPT --kill-others 'tsc --build tsconfig.json --watch'",
     "compile:source:es6": "tsc --build tsconfig.json",
-    "compile:source": "npm run compile:prep && npm run compile:source:es6 && npx kui-babel",
-    "compile:prescan": "npx kui-prescan",
+    "compile:source": "npm run compile:source:es6",
     "compile:clean": "tsc --build tsconfig.json --clean",
-    "compile": "npm run compile:source && npm run compile:prescan",
+    "compile": "npm run compile:source && npx kui-prescan",
     "rebuild": "npm run compile:clean; npm run compile",
     "clean": "npm run compile:clean; rm -rf node_modules; for i in plugins/*; do rm -rf $i/node_modules; done; for i in packages/*; do rm -rf $i/node_modules; done",
     "purge": "npm run clean; rm -f package-lock.json",

packages/builder/bin/compile.sh

@@ -31,7 +31,7 @@ set -o pipefail
 
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 
-# for compile.js below; give it an absolute path
+# these may no longer be needed, with the non-babel prescan?
 export CLIENT_HOME=${CLIENT_HOME-`pwd`}
 export PLUGIN_ROOT="$(cd "$TOPDIR" && pwd)/plugins"
 
@@ -47,12 +47,10 @@ else
   # (produced by builder/lib/configure.js, but which cannot be run
   # until after we've compiled the source)
   mkdir -p ./node_modules/@kui-shell
-  touch ./node_modules/@kui-shell/prescan.json
+  echo "{}" > ./node_modules/@kui-shell/prescan.json
 
   # compile the source
   npx tsc -b ${1-.}
-
-  npx kui-babel
 fi
 
 if [ ! -d node_modules/@kui-shell/build ]; then

packages/builder/bin/prescan.sh

@@ -23,7 +23,7 @@ SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 
 # for compile.js below; give it an absolute path
 export CLIENT_HOME=${CLIENT_HOME-`pwd`}
-export PLUGIN_ROOT="$(cd "$TOPDIR" && pwd)/plugins"
+export PLUGIN_ROOT="$(cd "$CLIENT_HOME" && pwd)/plugins"
 
 # webpack.config.js may set this in its watcher to ask us to generate
 # the prescan in a specific location
@@ -34,17 +34,27 @@ export KUI_PRESCAN
 # until after we've compiled the source)
 if [ -z "$KUI_PRESCAN" ]; then
     mkdir -p ./node_modules/@kui-shell
-    if [ ! -e ./node_modules/@kui-shell/prescan.json ]; then
-        touch ./node_modules/@kui-shell/prescan.json
+    if [ ! -s ./node_modules/@kui-shell/prescan.json ]; then
+        echo "{}" > ./node_modules/@kui-shell/prescan.json
     fi
 fi
 
-# pre-compile plugin registry
-if [ -f ./node_modules/@kui-shell/builder/dist/bin/compile.js ]; then
-    echo "compiling plugin registry $CLIENT_HOME to ${KUI_PRESCAN-default location}"
-    node ./node_modules/@kui-shell/builder/dist/bin/compile.js
+echo "compiling plugin registry $CLIENT_HOME to ${KUI_PRESCAN-default location}"
+MAIN=$(node -e "console.log(require(\"$CLIENT_HOME/package.json\").main)")
+MAIN_PROXY=$(echo "$MAIN" | sed -E 's/\.min\.js/-proxy.min.js/')
+if [ -f "$MAIN" ]
+then KUI_HEADLESS=true node "$MAIN" -- kui internal scan
+elif [ -f "$MAIN_PROXY" ]
+then KUI_HEADLESS=true node "$MAIN_PROXY" -- kui internal scan
+else echo "Deferring prescan"
 fi
 
+# pre-compile plugin registry
+#if [ -f ./node_modules/@kui-shell/builder/dist/bin/compile.js ]; then
+#    echo "compiling plugin registry $CLIENT_HOME to ${KUI_PRESCAN-default location}"
+#    node ./node_modules/@kui-shell/builder/dist/bin/compile.js
+#fi
+
 # generate the index.json in @kui-shell/client/notebooks/index.json
 if [ -d node_modules/@kui-shell/client/notebooks ]; then
     TGT=${KUI_PRESCAN_GUIDEBOOKS-node_modules/@kui-shell/build/client-guidebooks.json}

packages/builder/dist/electron/builders/electron.js

@@ -118,9 +118,9 @@ async function buildWebpack(buildPath, electronVersion, targetPlatform, targetAr
     KUI_BUILDER_HOME: `${CLIENT_HOME}/node_modules/@kui-shell/builder`
   })
 
-  console.log('Building headless bundles via webpack')
-  asyncs.push(
-    new Promise((resolve, reject) => {
+  const buildHeadless = () => {
+    console.log('Building headless bundles via webpack')
+    return new Promise((resolve, reject) => {
       exec(
         `npx --no-install webpack-cli --mode=production --config "${CLIENT_HOME}/node_modules/@kui-shell/webpack/headless-webpack.config.js"`,
         { env },
@@ -135,7 +135,24 @@ async function buildWebpack(buildPath, electronVersion, targetPlatform, targetAr
         }
       )
     })
-  )
+  }
+
+  // generate prescan model; to do so we need a headless build
+  await buildHeadless()
+  await new Promise((resolve, reject) => {
+    exec(`npx --no-install kui-prescan`, { env, cwd: env.CLIENT_HOME }, (err, stdout, stderr) => {
+      console.log('stdout', stdout)
+      if (err) {
+        console.error(err)
+        reject(stderr)
+      } else {
+        resolve()
+      }
+    })
+  })
+
+  // re-build headless now that we have the prescan
+  asyncs.push(buildHeadless())
 
   console.log('Building electron bundles via webpack')
   asyncs.push(

packages/builder/package.json

@@ -17,7 +17,6 @@
     "kui-build-electron": "./dist/electron/build.sh",
     "kui-set-icon": "./bin/seticon.js",
     "kui-init": "./build/kui-dist-init.js",
-    "kui-babel": "./bin/babel.sh",
     "kui-compile": "./bin/compile.sh",
     "kui-prescan": "./bin/prescan.sh"
   },
@@ -26,13 +25,8 @@
     "prepack": "chmod +x ./build/kui-dist-init.js"
   },
   "dependencies": {
-    "@babel/cli": "^7.20.7",
-    "@babel/core": "^7.20.12",
-    "@babel/plugin-transform-modules-commonjs": "^7.20.11",
     "@electron/rebuild": "^3.2.10",
     "@types/node": "^18.11.18",
-    "babel-plugin-dynamic-import-node-babel-7": "^2.0.7",
-    "babel-plugin-ignore-html-and-css-imports": "^0.0.2",
     "shelljs": "^0.8.5"
   },
   "kui": {

packages/core/src/core/command-tree.ts

@@ -582,6 +582,22 @@ export const read = async <T extends KResponse, O extends ParsedOptions>(
   execOptions: ExecOptions,
   tryCatchalls = true
 ): Promise<CommandTreeResolution<T, O>> => {
+  if (argv[0] === 'kui' && argv[1] === 'internal' && argv[2] === 'scan') {
+    // for bootstrapping the plugin registry
+    return {
+      eval: () => import('../plugins/assembler').then(_ => _.compile()).then(() => true as any as T),
+      subtree: undefined,
+      route: '/kui/internal/scan',
+      options: {},
+      success: () => {
+        /* noop */
+      },
+      error: (a, b, c, err) => {
+        throw err
+      }
+    }
+  }
+
   let cmd: false | CodedError | CommandHandlerWithEvents<T, O>
 
   if (!noRetry) {

packages/core/src/plugins/assembler.ts

@@ -33,7 +33,7 @@ const debug = Debug('core/plugins/assembler')
  * model that represents user-installed plugins)
  *
  */
-const prescanned = (): string => require.resolve('@kui-shell/prescan')
+const prescanned = (): string => join(process.env.CLIENT_HOME, 'node_modules/@kui-shell/prescan.json')
 
 /**
  * Serialize a `PrescanModel` to disk

packages/core/src/plugins/path.ts

@@ -22,7 +22,7 @@
  *
  */
 export function webpackPath(path: string) {
-  return path.replace(/^plugin-/, '').replace(/\/dist\/.*\.js$/, '')
+  return path.replace(/^plugin-/, '').replace(/\/mdist\/.*\.js$/, '')
 }
 
 /**

packages/core/src/plugins/plugins.ts

@@ -47,7 +47,7 @@ export const registrar: Record<string, KuiPlugin> = {}
 let basePrescan: PrescanModel // without any user-installed plugins
 let prescan: PrescanModel
 try {
-  prescan = require('@kui-shell/prescan.json') as PrescanModel // the result of unify(basePrescan, userPrescan)
+  prescan = (require('@kui-shell/prescan.json') || {}) as PrescanModel // the result of unify(basePrescan, userPrescan)
 
   // populate the typeahead trie from prescan (in command-tree, we
   // register again for dynamic command registrations); but we need
@@ -57,6 +57,17 @@ try {
     Object.keys(prescan.commandToPlugin).forEach(registerTypeahead)
   }
 } catch (err) {
+  prescan = {
+    docs: {},
+    topological: {},
+    flat: [],
+    overrides: {},
+    usage: {},
+    catchalls: [],
+    preloads: [],
+    themeSets: [],
+    commandToPlugin: {}
+  }
   debug(err)
 }
 export function prescanModel(): PrescanModel {

packages/core/src/plugins/preloader.ts

@@ -18,8 +18,6 @@ import Debug from 'debug'
 const debug = Debug('core/plugins/preloader')
 debug('loading')
 
-/* eslint-disable @typescript-eslint/camelcase */
-
 import { PrescanModel } from './prescan'
 
 import { Tab } from '../webapp/tab'
@@ -68,8 +66,9 @@ class PreloaderRegistrarImpl extends ImplForPlugins implements PreloadRegistrar
 export default async (prescan: PrescanModel) => {
   debug('init')
 
+  const preloads = (prescan && prescan.preloads) || []
   const jobs = Promise.all(
-    prescan.preloads.map(async module => {
+    preloads.map(async module => {
       // extends the capabilities of Kui
       try {
         debug('preloading capabilities.1 %s', module.path)
@@ -104,7 +103,7 @@ export default async (prescan: PrescanModel) => {
     })
     .then(() =>
       Promise.all(
-        prescan.preloads.map(async module => {
+        preloads.map(async module => {
           // FIXME to support field-installed plugin paths
           try {
             debug('preloading misc %s', module.path)

packages/core/src/plugins/resolver.ts

@@ -146,9 +146,9 @@ export const makeResolver = (prescan: PrescanModel, registrar: Record<string, Ku
 
   /** a plugin resolver impl */
   const resolver = {
-    isOverridden: (route: string): boolean => prescan.overrides[route] !== undefined,
+    isOverridden: (route: string): boolean => (prescan.overrides || {})[route] !== undefined,
     isAlpha: (route: string, plugin: string): boolean =>
-      prescan.overrides[route] === undefined || prescan.overrides[route] === plugin,
+      prescan.overrides[route] === undefined || (prescan.overrides || {})[route] === plugin,
 
     resolveOne,