Compile within single slice only (#20)

Update our esbuild wrapper to compile a single assets directory at a time.

Previously, we crawled the entire app structure to find all assets directories (within the app and slices) and processed them all at once. This had some downsides:

- It made our esbuild wrapper and plugin complicated.
- It meant that we had to take extra steps to prevent naming conflicts for assets with the same name in different slices.
- The above also meant that every slice's assets had to be referred to with the slice name as a leading namespace (e.g. "admin/app.js" for an admin slice), which didn't fit well with the idea of slices being fully self-contained.
- In general, it meant we had redundant functionality in JS code (such as locating slices) that already existed in our Ruby code.

By processing a single assets directory at once, we have a much simpler interface with esbuild: we're not pushing it to do things outside its comfort zone. It also means we can eliminate that redundant logic and continue to leverage our Ruby code as the canonical source of knowledge around slices and the overall Hanami app structure.

To process a single directory and output the compiled files into the relevant location, we expect these to be passed as two new arguments when invoking `config/assets.js`:

- `--path`: the path for the app or slice containing the assets to process; e.g. `app` or `slices/admin`
- `--dest`: the directory to putout the compiled files; e.g. `public/assets` or `public/assets/admin` (in the case of the admin slice)

These args will not need to be supplied by the user. Instead, they'll be provided by the `hanami assets compile` and `hanami assets watch` commands, which will determine the slices with assets and then invoke one `config/assets.js` process per slice, along with the relevant CLI arguments. This means the user experience is still simple and streamlined.

dist/args.d.ts

@@ -1,4 +1,6 @@
 export interface Args {
+    path: string;
+    dest: string;
     watch: Boolean;
     sri: string[] | null;
 }

dist/args.js

@@ -5,6 +5,8 @@ export const parseArgs = (args) => {
         result[key] = value;
     });
     return {
+        path: result["path"],
+        dest: result["dest"],
         watch: result.hasOwnProperty("watch"),
         sri: result["sri"]?.split(","),
     };

dist/esbuild-plugin.d.ts

@@ -1,12 +1,11 @@
 import { Plugin } from "esbuild";
 export interface PluginOptions {
     root: string;
-    publicDir: string;
+    sourceDir: string;
     destDir: string;
-    manifestPath: string;
     sriAlgorithms: Array<string>;
     hash: boolean;
 }
-export declare const defaults: Pick<PluginOptions, "root" | "publicDir" | "destDir" | "manifestPath" | "sriAlgorithms" | "hash">;
-declare const hanamiEsbuild: (options?: PluginOptions) => Plugin;
+export declare const defaults: Pick<PluginOptions, "sriAlgorithms" | "hash">;
+declare const hanamiEsbuild: (options: PluginOptions) => Plugin;
 export default hanamiEsbuild;

dist/esbuild-plugin.js

@@ -3,29 +3,20 @@ import path from "path";
 import crypto from "node:crypto";
 const URL_SEPARATOR = "/";
 export const defaults = {
-    root: "",
-    publicDir: "public",
-    destDir: path.join("public", "assets"),
-    manifestPath: path.join("public", "assets.json"),
     sriAlgorithms: [],
     hash: true,
 };
-const hanamiEsbuild = (options = { ...defaults }) => {
+const assetsDirName = "assets";
+const hanamiEsbuild = (options) => {
     return {
         name: "hanami-esbuild",
         setup(build) {
             build.initialOptions.metafile = true;
-            options.root = options.root || process.cwd();
-            const manifest = path.join(options.root, options.manifestPath);
+            const manifestPath = path.join(options.root, options.destDir, "assets.json");
             const externalDirs = build.initialOptions.external || [];
             build.onEnd(async (result) => {
                 const outputs = result.metafile?.outputs;
                 const assetsManifest = {};
-                const calulateSourceUrl = (str) => {
-                    return normalizeUrl(str)
-                        .replace(/\/assets\//, "")
-                        .replace(/-[A-Z0-9]{8}/, "");
-                };
                 const calulateDestinationUrl = (str) => {
                     return normalizeUrl(str).replace(/public/, "");
                 };
@@ -45,17 +36,34 @@ const hanamiEsbuild = (options = { ...defaults }) => {
                     const result = crypto.createHash("sha256").update(hashBytes).digest("hex");
                     return result.slice(0, 8).toUpperCase();
                 };
-                function extractEsbuildInputs(inputData) {
-                    const inputs = {};
-                    for (const key in inputData) {
-                        const entry = inputData[key];
-                        if (entry.inputs) {
-                            for (const inputKey in entry.inputs) {
-                                inputs[inputKey] = true;
-                            }
+                // Transforms the esbuild metafile outputs into an object containing mappings of outputs
+                // generated from entryPoints only.
+                //
+                // Converts this:
+                //
+                // {
+                //   'public/assets/admin/app-ITGLRDE7.js': {
+                //     imports: [],
+                //     exports: [],
+                //     entryPoint: 'slices/admin/assets/js/app.js',
+                //     inputs: { 'slices/admin/assets/js/app.js': [Object] },
+                //     bytes: 95
+                //   }
+                // }
+                //
+                //  To this:
+                //
+                // {
+                //   'public/assets/admin/app-ITGLRDE7.js': true
+                // }
+                function extractEsbuildCompiledEntrypoints(esbuildOutputs) {
+                    const entryPoints = {};
+                    for (const key in esbuildOutputs) {
+                        if (!key.endsWith(".map")) {
+                            entryPoints[key] = true;
                         }
                     }
-                    return inputs;
+                    return entryPoints;
                 }
                 // TODO: profile the current implementation vs blindly copying the asset
                 const copyAsset = (srcPath, destPath) => {
@@ -73,62 +81,81 @@ const hanamiEsbuild = (options = { ...defaults }) => {
                     fs.copyFileSync(srcPath, destPath);
                     return;
                 };
-                const processAssetDirectory = (pattern, inputs, options) => {
+                const processAssetDirectory = (pattern, compiledEntryPoints, options) => {
                     const dirPath = path.dirname(pattern);
                     const files = fs.readdirSync(dirPath, { recursive: true });
                     const assets = [];
                     files.forEach((file) => {
-                        const srcPath = path.join(dirPath, file.toString());
+                        const sourcePath = path.join(dirPath, file.toString());
                         // Skip if the file is already processed by esbuild
-                        if (inputs.hasOwnProperty(srcPath)) {
+                        if (compiledEntryPoints.hasOwnProperty(sourcePath)) {
                             return;
                         }
                         // Skip directories and any other non-files
-                        if (!fs.statSync(srcPath).isFile()) {
+                        if (!fs.statSync(sourcePath).isFile()) {
                             return;
                         }
-                        const fileHash = calculateHash(fs.readFileSync(srcPath), options.hash);
-                        const fileExtension = path.extname(srcPath);
-                        const baseName = path.basename(srcPath, fileExtension);
+                        const fileHash = calculateHash(fs.readFileSync(sourcePath), options.hash);
+                        const fileExtension = path.extname(sourcePath);
+                        const baseName = path.basename(sourcePath, fileExtension);
                         const destFileName = [baseName, fileHash].filter((item) => item !== null).join("-") + fileExtension;
-                        const destPath = path.join(options.destDir, path.relative(dirPath, srcPath).replace(path.basename(file.toString()), destFileName));
-                        if (fs.lstatSync(srcPath).isDirectory()) {
-                            assets.push(...processAssetDirectory(destPath, inputs, options));
+                        const destPath = path.join(options.destDir, path
+                            .relative(dirPath, sourcePath)
+                            .replace(path.basename(file.toString()), destFileName));
+                        if (fs.lstatSync(sourcePath).isDirectory()) {
+                            assets.push(...processAssetDirectory(destPath, compiledEntryPoints, options));
                         }
                         else {
-                            copyAsset(srcPath, destPath);
-                            assets.push(destPath);
+                            copyAsset(sourcePath, destPath);
+                            assets.push({ sourcePath: sourcePath, destPath: destPath });
                         }
                     });
                     return assets;
                 };
                 if (typeof outputs === "undefined") {
                     return;
                 }
-                const inputs = extractEsbuildInputs(outputs);
+                const compiledEntryPoints = extractEsbuildCompiledEntrypoints(outputs);
                 const copiedAssets = [];
                 externalDirs.forEach((pattern) => {
-                    copiedAssets.push(...processAssetDirectory(pattern, inputs, options));
+                    copiedAssets.push(...processAssetDirectory(pattern, compiledEntryPoints, options));
                 });
-                const assetsToProcess = Object.keys(outputs).concat(copiedAssets);
-                for (const assetToProcess of assetsToProcess) {
-                    if (assetToProcess.endsWith(".map")) {
-                        continue;
-                    }
-                    const destinationUrl = calulateDestinationUrl(assetToProcess);
-                    const sourceUrl = calulateSourceUrl(destinationUrl);
+                function prepareAsset(assetPath, destinationUrl) {
                     var asset = { url: destinationUrl };
                     if (options.sriAlgorithms.length > 0) {
                         asset.sri = [];
                         for (const algorithm of options.sriAlgorithms) {
-                            const subresourceIntegrity = calculateSubresourceIntegrity(algorithm, assetToProcess);
+                            const subresourceIntegrity = calculateSubresourceIntegrity(algorithm, path.join(options.root, assetPath));
                             asset.sri.push(subresourceIntegrity);
                         }
                     }
-                    assetsManifest[sourceUrl] = asset;
+                    return asset;
+                }
+                // Process entrypoints
+                const fileHashRegexp = /(-[A-Z0-9]{8})(\.\S+)$/;
+                for (const compiledEntryPoint in compiledEntryPoints) {
+                    // Convert "public/assets/app-2TLUHCQ6.js" to "app.js"
+                    let sourceUrl = compiledEntryPoint
+                        .replace(options.destDir + "/", "")
+                        .replace(fileHashRegexp, "$2");
+                    const destinationUrl = calulateDestinationUrl(compiledEntryPoint);
+                    assetsManifest[sourceUrl] = prepareAsset(compiledEntryPoint, destinationUrl);
+                }
+                // Process copied assets
+                for (const copiedAsset of copiedAssets) {
+                    // TODO: I wonder if we can skip .map files earlier
+                    if (copiedAsset.sourcePath.endsWith(".map")) {
+                        continue;
+                    }
+                    const destinationUrl = calulateDestinationUrl(copiedAsset.destPath);
+                    // Take the full path of the copied asset and remove everything up to (and including) the "assets/" dir
+                    var sourceUrl = copiedAsset.sourcePath.replace(path.join(options.root, options.sourceDir, assetsDirName) + "/", "");
+                    // Then remove the first subdir (e.g. "images/"), since we do not include those in the asset paths
+                    sourceUrl = sourceUrl.substring(sourceUrl.indexOf("/") + 1);
+                    assetsManifest[sourceUrl] = prepareAsset(copiedAsset.destPath, destinationUrl);
                 }
                 // Write assets manifest to the destination directory
-                await fs.writeJson(manifest, assetsManifest, { spaces: 2 });
+                await fs.writeJson(manifestPath, assetsManifest, { spaces: 2 });
             });
         },
     };

dist/esbuild.js

@@ -18,36 +18,28 @@ const loader = {
     ".eot": "file",
     ".ttf": "file",
 };
+const assetsDirName = "assets";
 const entryPointExtensions = "app.{js,ts,mjs,mts,tsx,jsx}";
-// FIXME: make cross platform
-const entryPointsMatcher = /(app\/assets\/js\/|slices\/(.*\/)assets\/js\/)/;
-const findEntryPoints = (root) => {
+const findEntryPoints = (sliceRoot) => {
     const result = {};
-    // TODO: should this be done explicitly within the root?
     const entryPoints = globSync([
-        path.join("app", "assets", "js", "**", entryPointExtensions),
-        path.join("slices", "*", "assets", "js", "**", entryPointExtensions),
+        path.join(sliceRoot, assetsDirName, "js", "**", entryPointExtensions),
     ]);
     entryPoints.forEach((entryPoint) => {
-        let modifiedPath = entryPoint.replace(entryPointsMatcher, "$2");
-        const relativePath = path.relative(root, modifiedPath);
-        const { dir, name } = path.parse(relativePath);
+        let entryPointPath = entryPoint.replace(sliceRoot + "/assets/js/", "");
+        const { dir, name } = path.parse(entryPointPath);
         if (dir) {
-            modifiedPath = path.join(dir, name);
+            entryPointPath = path.join(dir, name);
         }
         else {
-            modifiedPath = name;
+            entryPointPath = name;
         }
-        result[modifiedPath] = entryPoint;
+        result[entryPointPath] = entryPoint;
     });
     return result;
 };
-// TODO: feels like this really should be passed a root too, to become the cwd for globSync
-const externalDirectories = () => {
-    const assetDirsPattern = [
-        path.join("app", "assets", "*"),
-        path.join("slices", "*", "assets", "*"),
-    ];
+const findExternalDirectories = (basePath) => {
+    const assetDirsPattern = [path.join(basePath, assetsDirName, "*")];
     const excludeDirs = ["js", "css"];
     try {
         const dirs = globSync(assetDirsPattern, { nodir: false });
@@ -66,41 +58,47 @@ const externalDirectories = () => {
 export const buildOptions = (root, args) => {
     const pluginOptions = {
         ...pluginDefaults,
+        root: root,
+        sourceDir: args.path,
+        destDir: args.dest,
         sriAlgorithms: args.sri || [],
     };
     const plugin = esbuildPlugin(pluginOptions);
     const options = {
         bundle: true,
-        outdir: path.join(root, "public", "assets"),
+        outdir: args.dest,
         absWorkingDir: root,
         loader: loader,
-        external: externalDirectories(),
+        external: findExternalDirectories(path.join(root, args.path)),
         logLevel: "info",
         minify: true,
         sourcemap: true,
         entryNames: "[dir]/[name]-[hash]",
-        entryPoints: findEntryPoints(root),
+        entryPoints: findEntryPoints(path.join(root, args.path)),
         plugins: [plugin],
     };
     return options;
 };
 export const watchOptions = (root, args) => {
     const pluginOptions = {
         ...pluginDefaults,
+        root: root,
+        sourceDir: args.path,
+        destDir: args.dest,
         hash: false,
     };
     const plugin = esbuildPlugin(pluginOptions);
     const options = {
         bundle: true,
-        outdir: path.join(root, "public", "assets"),
+        outdir: args.dest,
         absWorkingDir: root,
         loader: loader,
-        external: externalDirectories(),
+        external: findExternalDirectories(path.join(root, args.path)),
         logLevel: "info",
         minify: false,
         sourcemap: false,
         entryNames: "[dir]/[name]",
-        entryPoints: findEntryPoints(root),
+        entryPoints: findEntryPoints(path.join(root, args.path)),
         plugins: [plugin],
     };
     return options;

dist/index.d.ts

@@ -1,4 +1,3 @@
-#!/usr/bin/env node
 import { BuildContext } from "esbuild";
 import { Args } from "./args.js";
 import { EsbuildOptions } from "./esbuild.js";

dist/index.js

@@ -1,10 +1,10 @@
-#!/usr/bin/env node
 import fs from "fs-extra";
 import path from "path";
 import esbuild from "esbuild";
 import { parseArgs } from "./args.js";
 import { buildOptions, watchOptions } from "./esbuild.js";
 export const run = async function (options) {
+    // TODO: Allow root to be provided (optionally) as a --root arg
     const { root = process.cwd(), argv = process.argv, esbuildOptionsFn = null } = options || {};
     const args = parseArgs(argv);
     // TODO: make nicer

src/args.ts

@@ -1,4 +1,6 @@
 export interface Args {
+  path: string;
+  dest: string;
   watch: Boolean;
   sri: string[] | null;
 }
@@ -12,6 +14,8 @@ export const parseArgs = (args: string[]): Args => {
   });
 
   return {
+    path: result["path"],
+    dest: result["dest"],
     watch: result.hasOwnProperty("watch"),
     sri: result["sri"]?.split(","),
   };