bug/issue 1151 custom imports not working for API routes and SSR pages (

#1152)

* custom imports for API routes and SSR pages and custom css import test cases

* add test cases for import json plugin

* all specs passing

* refactor rollup id cleaning

* import meta url refactoring

* normalize bundled import meta URL paths for Windows

* full bundling support for custom imports

* update final TODO comments

* leverage custom imports that can serve for meta import chunk vs asset bundling

* refactor custom import detection for found assetUrls

* add test case for bundling images with new URL

* document new URL and import.meta.url pattern

* clarify docs on new URL usage patterns and general content around assets

* fix windows specs

* callout support for isomorphic asset bundling in docs

---------

Co-authored-by: Owen Buckley <owenbuckley@Owens-Air-2.fios-router.home>

.eslintignore

@@ -3,4 +3,5 @@
 **/node_modules/**
 !.eslintrc.cjs
 !.mocharc.js
-packages/plugin-babel/test/cases/**/*main.js
+packages/plugin-babel/test/cases/**/*main.js
+TODO.md

packages/cli/package.json

@@ -34,7 +34,6 @@
     "@rollup/plugin-node-resolve": "^13.0.0",
     "@rollup/plugin-replace": "^2.3.4",
     "@rollup/plugin-terser": "^0.1.0",
-    "@web/rollup-plugin-import-meta-assets": "^1.0.0",
     "acorn": "^8.0.1",
     "acorn-walk": "^8.0.0",
     "commander": "^2.20.0",

packages/cli/src/config/rollup.config.js

@@ -1,20 +1,27 @@
-import fs from 'fs/promises';
+import fs from 'fs';
+import path from 'path';
 import { checkResourceExists, normalizePathnameForWindows } from '../lib/resource-utils.js';
 import { nodeResolve } from '@rollup/plugin-node-resolve';
 import commonjs from '@rollup/plugin-commonjs';
-import { importMetaAssets } from '@web/rollup-plugin-import-meta-assets';
+import * as walk from 'acorn-walk';
 
-// specifically to handle escodegen using require for package.json
+// https://github.com/rollup/rollup/issues/2121
+function cleanRollupId(id) {
+  return id.replace('\x00', '');
+}
+
+// specifically to handle escodegen and other node modules
+// using require for package.json or other json files
 // https://github.com/estools/escodegen/issues/455
 function greenwoodJsonLoader() {
   return {
     name: 'greenwood-json-loader',
     async load(id) {
-      const extension = id.split('.').pop();
+      const idUrl = new URL(`file://${cleanRollupId(id)}`);
+      const extension = idUrl.pathname.split('.').pop();
 
       if (extension === 'json') {
-        const url = new URL(`file://${id}`);
-        const json = JSON.parse(await fs.readFile(url, 'utf-8'));
+        const json = JSON.parse(await fs.promises.readFile(idUrl, 'utf-8'));
         const contents = `export default ${JSON.stringify(json)}`;
 
         return contents;
@@ -33,11 +40,11 @@ function greenwoodResourceLoader (compilation) {
   return {
     name: 'greenwood-resource-loader',
     async resolveId(id) {
-      const normalizedId = id.replace(/\?type=(.*)/, '');
+      const normalizedId = cleanRollupId(id); // idUrl.pathname;
       const { projectDirectory, userWorkspace } = compilation.context;
 
-      if (id.startsWith('.') && !id.startsWith(projectDirectory.pathname)) {
-        const prefix = id.startsWith('..') ? './' : '';
+      if (normalizedId.startsWith('.') && !normalizedId.startsWith(projectDirectory.pathname)) {
+        const prefix = normalizedId.startsWith('..') ? './' : '';
         const userWorkspaceUrl = new URL(`${prefix}${normalizedId.replace(/\.\.\//g, '')}`, userWorkspace);
 
         if (await checkResourceExists(userWorkspaceUrl)) {
@@ -46,11 +53,13 @@ function greenwoodResourceLoader (compilation) {
       }
     },
     async load(id) {
-      const pathname = id.indexOf('?') >= 0 ? id.slice(0, id.indexOf('?')) : id;
+      const idUrl = new URL(`file://${cleanRollupId(id)}`);
+      const { pathname } = idUrl;
       const extension = pathname.split('.').pop();
 
-      if (extension !== '' && extension !== 'js') {
-        const url = new URL(`file://${pathname}?type=${extension}`);
+      // filter first for any bare specifiers
+      if (await checkResourceExists(idUrl) && extension !== '' && extension !== 'js') {
+        const url = new URL(`${idUrl.href}?type=${extension}`);
         const request = new Request(url.href);
         let response = new Response('');
 
@@ -116,12 +125,12 @@ function greenwoodSyncPageResourceBundlesPlugin(compilation) {
             compilation.resources.set(resource.sourcePathURL.pathname, {
               ...compilation.resources.get(resource.sourcePathURL.pathname),
               optimizedFileName: fileName,
-              optimizedFileContents: await fs.readFile(outputPath, 'utf-8'),
+              optimizedFileContents: await fs.promises.readFile(outputPath, 'utf-8'),
               contents
             });
 
             if (noop) {
-              await fs.writeFile(outputPath, contents);
+              await fs.promises.writeFile(outputPath, contents);
             }
           }
         }
@@ -130,6 +139,138 @@ function greenwoodSyncPageResourceBundlesPlugin(compilation) {
   };
 }
 
+function getMetaImportPath(node) {
+  return node.arguments[0].value.split('/').join(path.sep);
+}
+
+function isNewUrlImportMetaUrl(node) {
+  return (
+    node.type === 'NewExpression' &&
+    node.callee.type === 'Identifier' &&
+    node.callee.name === 'URL' &&
+    node.arguments.length === 2 &&
+    node.arguments[0].type === 'Literal' &&
+    typeof getMetaImportPath(node) === 'string' &&
+    node.arguments[1].type === 'MemberExpression' &&
+    node.arguments[1].object.type === 'MetaProperty' &&
+    node.arguments[1].property.type === 'Identifier' &&
+    node.arguments[1].property.name === 'url'
+  );
+}
+
+// adapted from, and with credit to @web/rollup-plugin-import-meta-assets
+// https://modern-web.dev/docs/building/rollup-plugin-import-meta-assets/
+function greenwoodImportMetaUrl(compilation) {
+
+  return {
+    name: 'greenwood-import-meta-url',
+
+    async transform(code, id) {
+      const resourcePlugins = compilation.config.plugins.filter((plugin) => {
+        return plugin.type === 'resource';
+      }).map((plugin) => {
+        return plugin.provider(compilation);
+      });
+      const customResourcePlugins = compilation.config.plugins.filter((plugin) => {
+        return plugin.type === 'resource' && !plugin.isGreenwoodDefaultPlugin;
+      }).map((plugin) => {
+        return plugin.provider(compilation);
+      });
+      const idUrl = new URL(`file://${cleanRollupId(id)}`);
+      const { pathname } = idUrl;
+      const extension = pathname.split('.').pop();
+      const urlWithType = new URL(`${idUrl.href}?type=${extension}`);
+      const request = new Request(urlWithType.href);
+      let canTransform = false;
+      let response = new Response(code);
+
+      // handle any custom imports or pre-processing needed before passing to Rollup this.parse
+      if (await checkResourceExists(idUrl) && extension !== '' && extension !== 'json') {
+        for (const plugin of resourcePlugins) {
+          if (plugin.shouldServe && await plugin.shouldServe(urlWithType, request)) {
+            response = await plugin.serve(urlWithType, request);
+            canTransform = true;
+          }
+        }
+
+        for (const plugin of resourcePlugins) {
+          if (plugin.shouldIntercept && await plugin.shouldIntercept(urlWithType, request, response.clone())) {
+            response = await plugin.intercept(urlWithType, request, response.clone());
+            canTransform = true;
+          }
+        }
+      }
+
+      if (!canTransform) {
+        return null;
+      }
+
+      const ast = this.parse(await response.text());
+      const assetUrls = [];
+      let modifiedCode = false;
+
+      // aggregate all references of new URL + import.meta.url
+      walk.simple(ast, {
+        NewExpression(node) {
+          if (isNewUrlImportMetaUrl(node)) {
+            const absoluteScriptDir = path.dirname(id);
+            const relativeAssetPath = getMetaImportPath(node);
+            const absoluteAssetPath = path.resolve(absoluteScriptDir, relativeAssetPath);
+            const assetName = path.basename(absoluteAssetPath);
+            const assetExtension = assetName.split('.').pop();
+
+            assetUrls.push({
+              url: new URL(`file://${absoluteAssetPath}?type=${assetExtension}`),
+              relativeAssetPath
+            });
+          }
+        }
+      });
+
+      for (const assetUrl of assetUrls) {
+        const { url } = assetUrl;
+        const { pathname } = url;
+        const { relativeAssetPath } = assetUrl;
+        const assetName = path.basename(pathname);
+        const assetExtension = assetName.split('.').pop();
+        const assetContents = await fs.promises.readFile(url, 'utf-8');
+        const name = assetName.replace(`.${assetExtension}`, '');
+        let bundleExtensions = ['js'];
+
+        for (const plugin of customResourcePlugins) {
+          if (plugin.shouldServe && await plugin.shouldServe(url)) {
+            const response = await plugin.serve(url);
+
+            if (response?.headers?.get('content-type') || ''.indexOf('text/javascript') >= 0) {
+              bundleExtensions = [...bundleExtensions, ...plugin.extensions];
+            }
+          }
+        }
+
+        const type = bundleExtensions.indexOf(assetExtension) >= 0
+          ? 'chunk'
+          : 'asset';
+        const emitConfig = type === 'chunk'
+          ? { type, id: normalizePathnameForWindows(url), name }
+          : { type, name: assetName, source: assetContents };
+        const ref = this.emitFile(emitConfig);
+        // handle Windows style paths
+        const normalizedRelativeAssetPath = relativeAssetPath.replace(/\\/g, '/');
+        const importRef = `import.meta.ROLLUP_FILE_URL_${ref}`;
+
+        modifiedCode = code
+          .replace(`'${normalizedRelativeAssetPath}'`, importRef)
+          .replace(`"${normalizedRelativeAssetPath}"`, importRef);
+      }
+
+      return {
+        code: modifiedCode ? modifiedCode : code,
+        map: null
+      };
+    }
+  };
+}
+
 // TODO could we use this instead?
 // https://github.com/rollup/rollup/blob/v2.79.1/docs/05-plugin-development.md#resolveimportmeta
 // https://github.com/ProjectEvergreen/greenwood/issues/1087
@@ -177,6 +318,7 @@ const getRollupConfigForScriptResources = async (compilation) => {
     plugins: [
       greenwoodResourceLoader(compilation),
       greenwoodSyncPageResourceBundlesPlugin(compilation),
+      greenwoodImportMetaUrl(compilation),
       ...customRollupPlugins
     ],
     context: 'window',
@@ -216,6 +358,11 @@ const getRollupConfigForApis = async (compilation) => {
   const input = [...compilation.manifest.apis.values()]
     .map(api => normalizePathnameForWindows(new URL(`.${api.path}`, userWorkspace)));
 
+  // why is this needed?
+  await fs.promises.mkdir(new URL('./api/assets/', outputDir), {
+    recursive: true
+  });
+
   // TODO should routes and APIs have chunks?
   // https://github.com/ProjectEvergreen/greenwood/issues/1118
   return [{
@@ -227,9 +374,10 @@ const getRollupConfigForApis = async (compilation) => {
     },
     plugins: [
       greenwoodJsonLoader(),
+      greenwoodResourceLoader(compilation),
       nodeResolve(),
       commonjs(),
-      importMetaAssets()
+      greenwoodImportMetaUrl(compilation)
     ]
   }];
 };
@@ -248,14 +396,15 @@ const getRollupConfigForSsr = async (compilation, input) => {
     },
     plugins: [
       greenwoodJsonLoader(),
+      greenwoodResourceLoader(compilation),
       // TODO let this through for lit to enable nodeResolve({ preferBuiltins: true })
       // https://github.com/lit/lit/issues/449
       // https://github.com/ProjectEvergreen/greenwood/issues/1118
       nodeResolve({
         preferBuiltins: true
       }),
       commonjs(),
-      importMetaAssets(),
+      greenwoodImportMetaUrl(compilation),
       greenwoodPatchSsrPagesEntryPointRuntimeImport() // TODO a little hacky but works for now
     ],
     onwarn: (errorObj) => {

packages/cli/test/cases/serve.default.ssr/serve.default.ssr.spec.js

@@ -18,6 +18,8 @@
  *   components/
  *     card.js
  *     counter.js
+ *   images/
+ *     logo.svg
  *   pages/
  *     about.md
  *     artists.js
@@ -300,6 +302,60 @@ describe('Serve Greenwood With: ', function() {
       });
     });
 
+    describe('Bundled image using new URL and import.meta.url', function() {
+      const bundledName = 'assets/logo-abb2e884.svg';
+      let bundledImageResponse = {};
+      let usersResponse = {};
+
+      before(async function() {
+        await new Promise((resolve, reject) => {
+          request.get(`${hostname}/${bundledName}`, (err, res, body) => {
+            if (err) {
+              reject();
+            }
+
+            bundledImageResponse = res;
+            bundledImageResponse.body = body;
+
+            resolve();
+          });
+        });
+
+        await new Promise((resolve, reject) => {
+          request.get(`${hostname}/_users.js`, (err, res, body) => {
+            if (err) {
+              reject();
+            }
+
+            usersResponse = res;
+            usersResponse.body = body;
+
+            resolve();
+          });
+        });
+      });
+
+      it('should return a 200 status for the image', function(done) {
+        expect(bundledImageResponse.statusCode).to.equal(200);
+        done();
+      });
+
+      it('should return the expected content-type for the image', function(done) {
+        expect(bundledImageResponse.headers['content-type']).to.equal('image/svg+xml');
+        done();
+      });
+
+      it('should return the expected body for the image', function(done) {
+        expect(bundledImageResponse.body.startsWith('<svg')).to.equal(true);
+        done();
+      });
+
+      it('should return the expected bundled image name inside the bundled page route', function(done) {
+        expect(usersResponse.body.indexOf(bundledName) >= 0).to.equal(true);
+        done();
+      });
+    });
+
     describe('Serve command with 404 not found behavior', function() {
       let response = {};
 

packages/cli/test/cases/serve.default.ssr/src/components/card.js

@@ -1,3 +1,4 @@
+const logo = new URL('../images/logo.svg', import.meta.url);
 const template = document.createElement('template');
 
 template.innerHTML = `
@@ -23,6 +24,7 @@ template.innerHTML = `
   </style>
 
   <div class="card">
+    <img alt="logo" href="${logo.pathname}">
     <slot name="title">My default title</slot>
     <slot name="image"></slot>
   </div>