Enhancement/issue 1088 refactor Workers out of SSR builds

packages/cli/src/commands/build.js

@@ -98,7 +98,7 @@ const runProductionBuild = async (compilation) => {
           return Promise.resolve(server);
         }));
 
-        if (prerenderPlugin.workerUrl) {
+        if (prerenderPlugin.executeModuleUrl) {
           await trackResourcesForRoutes(compilation);
           await preRenderCompilationWorker(compilation, prerenderPlugin);
         } else {

packages/cli/src/config/rollup.config.js

@@ -130,6 +130,31 @@ function greenwoodSyncPageResourceBundlesPlugin(compilation) {
   };
 }
 
+// TODO could we use this instead?
+// https://github.com/rollup/rollup/blob/v2.79.1/docs/05-plugin-development.md#resolveimportmeta
+// https://github.com/ProjectEvergreen/greenwood/issues/1087
+function greenwoodPatchSsrPagesEntryPointRuntimeImport() {
+  return {
+    name: 'greenwood-patch-ssr-pages-entry-point-runtime-import',
+    generateBundle(options, bundle) {
+      Object.keys(bundle).forEach((key) => {
+        if (key.startsWith('__')) {
+          console.log('this is a generated entry point', bundle[key]);
+          // ___GWD_ENTRY_FILE_URL=${filename}___
+          const needle = bundle[key].code.match(/___GWD_ENTRY_FILE_URL=(.*.)___/);
+          if (needle) {
+            const entryPathMatch = needle[1];
+
+            bundle[key].code = bundle[key].code.replace(/'___GWD_ENTRY_FILE_URL=(.*.)___'/, `new URL('./_${entryPathMatch}', import.meta.url)`);
+          } else {
+            console.warn(`Could not find entry path match for bundle => ${ley}`);
+          }
+        }
+      });
+    }
+  };
+}
+
 const getRollupConfigForScriptResources = async (compilation) => {
   const { outputDir } = compilation.context;
   const input = [...compilation.resources.values()]
@@ -193,7 +218,7 @@ const getRollupConfigForApis = async (compilation) => {
     .map(api => normalizePathnameForWindows(new URL(`.${api.path}`, userWorkspace)));
 
   // TODO should routes and APIs have chunks?
-  // https://github.com/ProjectEvergreen/greenwood/issues/1008
+  // https://github.com/ProjectEvergreen/greenwood/issues/1118
   return [{
     input,
     output: {
@@ -214,7 +239,7 @@ const getRollupConfigForSsr = async (compilation, input) => {
   const { outputDir } = compilation.context;
 
   // TODO should routes and APIs have chunks?
-  // https://github.com/ProjectEvergreen/greenwood/issues/1008
+  // https://github.com/ProjectEvergreen/greenwood/issues/1118
   return [{
     input,
     output: {
@@ -224,10 +249,34 @@ const getRollupConfigForSsr = async (compilation, input) => {
     },
     plugins: [
       greenwoodJsonLoader(),
-      nodeResolve(),
+      // TODO let this through for lit to enable nodeResolve({ preferBuiltins: true })
+      // https://github.com/lit/lit/issues/449
+      // https://github.com/ProjectEvergreen/greenwood/issues/1118
+      nodeResolve({
+        preferBuiltins: true
+      }),
       commonjs(),
-      importMetaAssets()
-    ]
+      importMetaAssets(),
+      greenwoodPatchSsrPagesEntryPointRuntimeImport() // TODO a little hacky but works for now
+    ],
+    onwarn: (errorObj) => {
+      const { code, message } = errorObj;
+
+      switch (code) {
+
+        case 'CIRCULAR_DEPENDENCY':
+          // TODO let this through for lit by suppressing it
+          // Error: the string "Circular dependency: ../../../../../node_modules/@lit-labs/ssr/lib/render-lit-html.js ->
+          // ../../../../../node_modules/@lit-labs/ssr/lib/lit-element-renderer.js -> ../../../../../node_modules/@lit-labs/ssr/lib/render-lit-html.js\n" was thrown, throw an Error :)
+          // https://github.com/lit/lit/issues/449
+          // https://github.com/ProjectEvergreen/greenwood/issues/1118
+          break;
+        default:
+          // otherwise, log all warnings from rollup
+          console.debug(message);
+
+      }
+    }
   }];
 };
 

packages/cli/src/lib/execute-route-module.js

@@ -0,0 +1,42 @@
+import { renderToString, renderFromHTML } from 'wc-compiler';
+
+async function executeRouteModule({ moduleUrl, compilation, page = {}, prerender = false, htmlContents = null, scripts = [] }) {
+  const data = {
+    template: null,
+    body: null,
+    frontmatter: null,
+    html: null
+  };
+
+  if (prerender) {
+    const scriptURLs = scripts.map(scriptFile => new URL(scriptFile));
+    const { html } = await renderFromHTML(htmlContents, scriptURLs);
+
+    data.html = html;
+  } else {
+    const module = await import(moduleUrl).then(module => module);
+    const { getTemplate = null, getBody = null, getFrontmatter = null } = module;
+
+    if (module.default) {
+      const { html } = await renderToString(new URL(moduleUrl), false);
+
+      data.body = html;
+    } else {
+      if (getBody) {
+        data.body = await getBody(compilation, page);
+      }
+    }
+
+    if (getTemplate) {
+      data.template = await getTemplate(compilation, page);
+    }
+
+    if (getFrontmatter) {
+      data.frontmatter = await getFrontmatter(compilation, page);
+    }
+  }
+
+  return data;
+}
+
+export { executeRouteModule };

packages/cli/src/lib/ssr-route-worker.js

@@ -1,47 +1,13 @@
 // https://github.com/nodejs/modules/issues/307#issuecomment-858729422
 import { parentPort } from 'worker_threads';
-import { renderToString, renderFromHTML } from 'wc-compiler';
 
-async function executeRouteModule({ moduleUrl, compilation, route, label, id, prerender, htmlContents, scripts }) {
-  const parsedCompilation = JSON.parse(compilation);
-  const data = {
-    template: null,
-    body: null,
-    frontmatter: null,
-    html: null
-  };
-
-  if (prerender) {
-    const scriptURLs = JSON.parse(scripts).map(scriptFile => new URL(scriptFile));
-    const { html } = await renderFromHTML(htmlContents, scriptURLs);
-
-    data.html = html;
-  } else {
-    const module = await import(moduleUrl).then(module => module);
-    const { getTemplate = null, getBody = null, getFrontmatter = null } = module;
-
-    if (module.default) {
-      const { html } = await renderToString(new URL(moduleUrl), false);
-
-      data.body = html;
-    } else {
-      if (getBody) {
-        data.body = await getBody(parsedCompilation, route);
-      }
-    }
-
-    if (getTemplate) {
-      data.template = await getTemplate(parsedCompilation, route);
-    }
-
-    if (getFrontmatter) {
-      data.frontmatter = await getFrontmatter(parsedCompilation, route, label, id);
-    }
-  }
+async function executeModule({ executeModuleUrl, moduleUrl, compilation, page, prerender = false, htmlContents = null, scripts = '[]' }) {
+  const { executeRouteModule } = await import(executeModuleUrl);
+  const data = await executeRouteModule({ moduleUrl, compilation: JSON.parse(compilation), page: JSON.parse(page), prerender, htmlContents, scripts: JSON.parse(scripts) });
 
   parentPort.postMessage(data);
 }
 
 parentPort.on('message', async (task) => {
-  await executeRouteModule(task);
+  await executeModule(task);
 });

packages/cli/src/lib/templating-utils.js

@@ -177,6 +177,8 @@ async function getAppTemplate(pageTemplateContents, context, customImports = [],
 }
 
 async function getUserScripts (contents, context) {
+  // TODO get rid of lit polyfills in core
+  // https://github.com/ProjectEvergreen/greenwood/issues/728
   // https://lit.dev/docs/tools/requirements/#polyfills
   if (process.env.__GWD_COMMAND__ === 'build') { // eslint-disable-line no-underscore-dangle
     const userPackageJson = await getPackageJson(context);

packages/cli/src/lifecycles/bundle.js

@@ -1,6 +1,7 @@
 /* eslint-disable max-depth, max-len */
 import fs from 'fs/promises';
 import { getRollupConfigForApis, getRollupConfigForScriptResources, getRollupConfigForSsr } from '../config/rollup.config.js';
+import { getAppTemplate, getPageTemplate, getUserScripts } from '../lib/templating-utils.js';
 import { hashString } from '../lib/hashing-utils.js';
 import { checkResourceExists, mergeResponse, normalizePathnameForWindows } from '../lib/resource-utils.js';
 import path from 'path';
@@ -174,108 +175,68 @@ async function bundleApiRoutes(compilation) {
 
 async function bundleSsrPages(compilation) {
   // https://rollupjs.org/guide/en/#differences-to-the-javascript-api
-  const { outputDir, pagesDir } = compilation.context;
   // TODO context plugins for SSR ?
   // https://github.com/ProjectEvergreen/greenwood/issues/1008
   // const contextPlugins = compilation.config.plugins.filter((plugin) => {
   //   return plugin.type === 'context';
   // }).map((plugin) => {
   //   return plugin.provider(compilation);
   // });
-
+  const hasSSRPages = compilation.graph.filter(page => page.isSSR).length > 0;
   const input = [];
 
-  if (!compilation.config.prerender) {
+  if (!compilation.config.prerender && hasSSRPages) {
+    const htmlOptimizer = compilation.config.plugins.find(plugin => plugin.name === 'plugin-standard-html').provider(compilation);
+    const { executeModuleUrl } = compilation.config.plugins.find(plugin => plugin.type === 'renderer').provider();
+    const { executeRouteModule } = await import(executeModuleUrl);
+    const { pagesDir, scratchDir } = compilation.context;
+
     for (const page of compilation.graph) {
       if (page.isSSR && !page.data.static) {
-        const { filename, path: pagePath } = page;
-        const scratchUrl = new URL(`./${filename}`, outputDir);
-
-        // better way to write out inline code like this?
-        await fs.writeFile(scratchUrl, `
-          import { Worker } from 'worker_threads';
-          import { getAppTemplate, getPageTemplate, getUserScripts } from '@greenwood/cli/src/lib/templating-utils.js';
-
-          export async function handler(request, compilation) {
-            const routeModuleLocationUrl = new URL('./_${filename}', '${outputDir}');
-            const routeWorkerUrl = '${compilation.config.plugins.find(plugin => plugin.type === 'renderer').provider().workerUrl}';
-            const htmlOptimizer = compilation.config.plugins.find(plugin => plugin.name === 'plugin-standard-html').provider(compilation);
-            let body = '';
-            let html = '';
-            let frontmatter;
-            let template;
-            let templateType = 'page';
-            let title = '';
-            let imports = [];
-
-            await new Promise((resolve, reject) => {
-              const worker = new Worker(new URL(routeWorkerUrl));
-
-              worker.on('message', (result) => {
-                if (result.body) {
-                  body = result.body;
-                }
-
-                if (result.template) {
-                  template = result.template;
-                }
-
-                if (result.frontmatter) {
-                  frontmatter = result.frontmatter;
-
-                  if (frontmatter.title) {
-                    title = frontmatter.title;
-                  }
-
-                  if (frontmatter.template) {
-                    templateType = frontmatter.template;
-                  }
-
-                  if (frontmatter.imports) {
-                    imports = imports.concat(frontmatter.imports);
-                  }
-                }
-
-                resolve();
-              });
-
-              worker.on('error', reject);
-              worker.on('exit', (code) => {
-                if (code !== 0) {
-                  reject(new Error(\`Worker stopped with exit code \${code}\`));
-                }
-              });
-
-              worker.postMessage({
-                moduleUrl: routeModuleLocationUrl.href,
-                compilation: \`${JSON.stringify(compilation)}\`,
-                route: '${pagePath}'
-              });
-            });
-
-            html = template ? template : await getPageTemplate('', compilation.context, templateType, []);
-            html = await getAppTemplate(html, compilation.context, imports, [], false, title);
-            html = await getUserScripts(html, compilation.context);
-            html = html.replace(\/\<content-outlet>(.*)<\\/content-outlet>\/s, body);
-            html = await (await htmlOptimizer.optimize(new URL(request.url), new Response(html))).text();
-
-            return new Response(html);
+        const { filename, imports, route, template, title } = page;
+        const entryFileUrl = new URL(`./_${filename}`, scratchDir);
+        const moduleUrl = new URL(`./${filename}`, pagesDir);
+        // TODO getTemplate has to be static (for now?)
+        // https://github.com/ProjectEvergreen/greenwood/issues/955
+        const data = await executeRouteModule({ moduleUrl, compilation, page, prerender: false, htmlContents: null, scripts: [] });
+        let staticHtml = '';
+
+        staticHtml = data.template ? data.template : await getPageTemplate(staticHtml, compilation.context, template, []);
+        staticHtml = await getAppTemplate(staticHtml, compilation.context, imports, [], false, title);
+        staticHtml = await getUserScripts(staticHtml, compilation.context);
+        staticHtml = await (await htmlOptimizer.optimize(new URL(`http://localhost:8080${route}`), new Response(staticHtml))).text();
+
+        // better way to write out this inline code?
+        await fs.writeFile(entryFileUrl, `
+          import { executeRouteModule } from '${normalizePathnameForWindows(executeModuleUrl)}';
+
+          export async function handler(request) {
+            const compilation = JSON.parse('${JSON.stringify(compilation)}');
+            const page = JSON.parse('${JSON.stringify(page)}');
+            const moduleUrl = '___GWD_ENTRY_FILE_URL=${filename}___';
+            const data = await executeRouteModule({ moduleUrl, compilation, page });
+            let staticHtml = \`${staticHtml}\`;
+
+            // console.log({ page })
+            // console.log({ staticHtml })
+            // console.log({ data });
+
+            if (data.body) {
+              staticHtml = staticHtml.replace(\/\<content-outlet>(.*)<\\/content-outlet>\/s, data.body);
+            }
+
+            return new Response(staticHtml);
           }
         `);
 
-        input.push(normalizePathnameForWindows(new URL(`./${filename}`, pagesDir)));
+        input.push(normalizePathnameForWindows(moduleUrl));
+        input.push(normalizePathnameForWindows(entryFileUrl));
       }
     }
 
     const [rollupConfig] = await getRollupConfigForSsr(compilation, input);
 
     if (rollupConfig.input.length > 0) {
-      const { userTemplatesDir, outputDir } = compilation.context;
-
-      if (await checkResourceExists(userTemplatesDir)) {
-        await fs.cp(userTemplatesDir, new URL('./_templates/', outputDir), { recursive: true });
-      }
-
       const bundle = await rollup(rollupConfig);
       await bundle.write(rollupConfig.output);
     }
@@ -309,13 +270,14 @@ const bundleCompilation = async (compilation) => {
 
       await Promise.all([
         await bundleApiRoutes(compilation),
-        await bundleSsrPages(compilation),
         await bundleScriptResources(compilation),
         await bundleStyleResources(compilation, optimizeResourcePlugins)
       ]);
 
-      console.info('optimizing static pages....');
+      // bundleSsrPages depends on bundleScriptResources having run first
+      await bundleSsrPages(compilation);
 
+      console.info('optimizing static pages....');
       await optimizeStaticPages(compilation, optimizeResourcePlugins);
       await cleanUpResources(compilation);
       await emitResources(compilation);