perf: file system for transform api, not stdio

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Unreleased
+
+* Performance optimizations for large file transforms
+
+    There are two main JavaScript APIs: `build()` which operates on the file system and `transform()` which operates on in-memory data. Previously transforming large files using the JavaScript `transform()` API could be significantly slower than just writing the in-memory string to the file system, calling `build()`, and reading the result back from the file system. This is based on performance tests done on macOS 10.15.
+
+    Now esbuild will go through the file system when transforming large files (currently >1mb). This approach is only faster for large files, and can be significantly slower for small files, so small files still keep everything in memory.
+
 ## 0.6.8
 
 * Attempt to support the taobao.org registry ([#291](https://github.com/evanw/esbuild/issues/291))

cmd/esbuild/service.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"runtime/debug"
 	"sync"
@@ -36,7 +37,7 @@ func runService() {
 		callbacks:       make(map[uint32]responseCallback),
 		outgoingPackets: make(chan outgoingPacket),
 	}
-	buffer := make([]byte, 4096)
+	buffer := make([]byte, 16*1024)
 	stream := []byte{}
 
 	// Write messages on a single goroutine so they aren't interleaved
@@ -171,6 +172,15 @@ func (service *serviceType) handleIncomingMessage(bytes []byte) (result []byte)
 	return nil
 }
 
+func encodeErrorPacket(id uint32, err error) []byte {
+	return encodePacket(packet{
+		id: id,
+		value: map[string]interface{}{
+			"error": err.Error(),
+		},
+	})
+}
+
 func (service *serviceType) handleBuildRequest(id uint32, request map[string]interface{}) []byte {
 	write := request["write"].(bool)
 	flags := decodeStringArray(request["flags"].([]interface{}))
@@ -182,12 +192,7 @@ func (service *serviceType) handleBuildRequest(id uint32, request map[string]int
 		err = errors.New("Either provide \"outfile\" or set \"write\" to false")
 	}
 	if err != nil {
-		return encodePacket(packet{
-			id: id,
-			value: map[string]interface{}{
-				"error": err.Error(),
-			},
-		})
+		return encodeErrorPacket(id, err)
 	}
 
 	// Optionally allow input from the stdin channel
@@ -220,27 +225,58 @@ func (service *serviceType) handleBuildRequest(id uint32, request map[string]int
 }
 
 func (service *serviceType) handleTransformRequest(id uint32, request map[string]interface{}) []byte {
+	inputFS := request["inputFS"].(bool)
 	input := request["input"].(string)
 	flags := decodeStringArray(request["flags"].([]interface{}))
 
 	options, err := cli.ParseTransformOptions(flags)
 	if err != nil {
-		return encodePacket(packet{
-			id: id,
-			value: map[string]interface{}{
-				"error": err.Error(),
-			},
-		})
+		return encodeErrorPacket(id, err)
+	}
+
+	transformInput := input
+	if inputFS {
+		bytes, err := ioutil.ReadFile(input)
+		if err == nil {
+			err = os.Remove(input)
+		}
+		if err != nil {
+			return encodeErrorPacket(id, err)
+		}
+		transformInput = string(bytes)
+	}
+
+	result := api.Transform(transformInput, options)
+	jsFS := false
+	jsSourceMapFS := false
+
+	if inputFS && len(result.JS) > 0 {
+		file := input + ".js"
+		if err := ioutil.WriteFile(file, result.JS, 0644); err == nil {
+			result.JS = []byte(file)
+			jsFS = true
+		}
+	}
+
+	if inputFS && len(result.JSSourceMap) > 0 {
+		file := input + ".map"
+		if err := ioutil.WriteFile(file, result.JSSourceMap, 0644); err == nil {
+			result.JSSourceMap = []byte(file)
+			jsSourceMapFS = true
+		}
 	}
 
-	result := api.Transform(input, options)
 	return encodePacket(packet{
 		id: id,
 		value: map[string]interface{}{
-			"errors":      encodeMessages(result.Errors),
-			"warnings":    encodeMessages(result.Warnings),
-			"js":          string(result.JS),
-			"jsSourceMap": string(result.JSSourceMap),
+			"errors":   encodeMessages(result.Errors),
+			"warnings": encodeMessages(result.Warnings),
+
+			"jsFS": jsFS,
+			"js":   string(result.JS),
+
+			"jsSourceMapFS": jsSourceMapFS,
+			"jsSourceMap":   string(result.JSSourceMap),
 		},
 	})
 }

lib/browser.ts

@@ -67,8 +67,10 @@ let startService: typeof types.startService = options => {
       },
       transform: (input, options) =>
         new Promise((resolve, reject) =>
-          service.transform(input, options || {}, false, (err, res) =>
-            err ? reject(err) : resolve(res!))),
+          service.transform(input, options || {}, false, {
+            readFile(_, callback) { callback(new Error('Internal error'), null); },
+            writeFile(_, callback) { callback(null); },
+          }, (err, res) => err ? reject(err) : resolve(res!))),
       stop() {
         worker.terminate()
         afterClose()

lib/common.ts

@@ -106,9 +106,25 @@ export interface StreamOut {
   service: StreamService;
 }
 
+export interface StreamFS {
+  writeFile(contents: string, callback: (path: string | null) => void): void;
+  readFile(path: string, callback: (err: Error | null, contents: string | null) => void): void;
+}
+
 export interface StreamService {
-  build(options: types.BuildOptions, isTTY: boolean, callback: (err: Error | null, res: types.BuildResult | null) => void): void;
-  transform(input: string, options: types.TransformOptions, isTTY: boolean, callback: (err: Error | null, res: types.TransformResult | null) => void): void;
+  build(
+    options: types.BuildOptions,
+    isTTY: boolean,
+    callback: (err: Error | null, res: types.BuildResult | null) => void,
+  ): void;
+
+  transform(
+    input: string,
+    options: types.TransformOptions,
+    isTTY: boolean,
+    fs: StreamFS,
+    callback: (err: Error | null, res: types.TransformResult | null) => void,
+  ): void;
 }
 
 // This can't use any promises because it must work for both sync and async code
@@ -118,7 +134,7 @@ export function createChannel(options: StreamIn): StreamOut {
   let nextID = 0;
 
   // Use a long-lived buffer to store stdout data
-  let stdout = new Uint8Array(4096);
+  let stdout = new Uint8Array(16 * 1024);
   let stdoutUsed = 0;
   let readFromStdout = (chunk: Uint8Array) => {
     // Append the chunk to the stdout buffer, growing it as necessary
@@ -223,18 +239,75 @@ export function createChannel(options: StreamIn): StreamOut {
         );
       },
 
-      transform(input, options, isTTY, callback) {
+      transform(input, options, isTTY, fs, callback) {
         let flags = flagsForTransformOptions(options, isTTY);
-        sendRequest<protocol.TransformRequest, protocol.TransformResponse>(
-          ['transform', { flags, input: input + '' }],
-          (error, response) => {
-            if (error) return callback(new Error(error), null);
-            let errors = response!.errors;
-            let warnings = response!.warnings;
-            if (errors.length > 0) return callback(failureErrorWithLog('Transform failed', errors, warnings), null);
-            callback(null, { warnings, js: response!.js, jsSourceMap: response!.jsSourceMap });
-          },
-        );
+        input += '';
+
+        // Ideally the "transform()" API would be faster than calling "build()"
+        // since it doesn't need to touch the file system. However, performance
+        // measurements with large files on macOS indicate that sending the data
+        // over the stdio pipe can be 2x slower than just using a temporary file.
+        //
+        // This appears to be an OS limitation. Both the JavaScript and Go code
+        // are using large buffers but the pipe only writes data in 8kb chunks.
+        // An investigation seems to indicate that this number is hard-coded into
+        // the OS source code. Presumably files are faster because the OS uses
+        // a larger chunk size, or maybe even reads everything in one syscall.
+        //
+        // The cross-over size where this starts to be faster is around 1mb on
+        // my machine. In that case, this code tries to use a temporary file if
+        // possible but falls back to sending the data over the stdio pipe if
+        // that doesn't work.
+        let start = (inputPath: string | null) => {
+          sendRequest<protocol.TransformRequest, protocol.TransformResponse>(
+            ['transform', {
+              flags,
+              inputFS: inputPath !== null,
+              input: inputPath !== null ? inputPath : input,
+            }],
+            (error, response) => {
+              if (error) return callback(new Error(error), null);
+              let errors = response!.errors;
+              let warnings = response!.warnings;
+              let outstanding = 1;
+              let next = () => --outstanding === 0 && callback(null, { warnings, js: response!.js, jsSourceMap: response!.jsSourceMap });
+              if (errors.length > 0) return callback(failureErrorWithLog('Transform failed', errors, warnings), null);
+
+              // Read the JavaScript file from the file system
+              if (response!.jsFS) {
+                outstanding++;
+                fs.readFile(response!.js, (err, contents) => {
+                  if (err !== null) {
+                    callback(err, null);
+                  } else {
+                    response!.js = contents!;
+                    next();
+                  }
+                });
+              }
+
+              // Read the source map file from the file system
+              if (response!.jsSourceMapFS) {
+                outstanding++;
+                fs.readFile(response!.jsSourceMap, (err, contents) => {
+                  if (err !== null) {
+                    callback(err, null);
+                  } else {
+                    response!.jsSourceMap = contents!;
+                    next();
+                  }
+                });
+              }
+
+              next();
+            },
+          );
+        };
+        if (input.length > 1024 * 1024) {
+          let next = start;
+          start = () => fs.writeFile(input, next);
+        }
+        start(null);
       },
     },
   };

lib/node.ts

@@ -1,7 +1,11 @@
 import * as types from "./types";
 import * as common from "./common";
 import * as child_process from "child_process";
+import * as crypto from "crypto";
 import * as path from "path";
+import * as util from "util";
+import * as fs from "fs";
+import * as os from "os";
 import { isatty } from "tty";
 
 // This file is used for both the "esbuild" package and the "esbuild-wasm"
@@ -50,7 +54,29 @@ let buildSync: typeof types.buildSync = options => {
 
 let transformSync: typeof types.transformSync = (input, options) => {
   let result: types.TransformResult;
-  runServiceSync(service => service.transform(input, options || {}, isTTY(), (err, res) => {
+  runServiceSync(service => service.transform(input, options || {}, isTTY(), {
+    readFile(tempFile, callback) {
+      try {
+        let contents = fs.readFileSync(tempFile, 'utf8');
+        try {
+          fs.unlinkSync(tempFile);
+        } catch {
+        }
+        callback(null, contents);
+      } catch (err) {
+        callback(err, null);
+      }
+    },
+    writeFile(contents, callback) {
+      try {
+        let tempFile = randomFileName();
+        fs.writeFileSync(tempFile, contents);
+        callback(tempFile);
+      } catch {
+        callback(null);
+      }
+    },
+  }, (err, res) => {
     if (err) throw err;
     result = res!;
   }));
@@ -84,8 +110,30 @@ let startService: typeof types.startService = options => {
           err ? reject(err) : resolve(res!))),
     transform: (input, options) =>
       new Promise((resolve, reject) =>
-        service.transform(input, options || {}, isTTY(), (err, res) =>
-          err ? reject(err) : resolve(res!))),
+        service.transform(input, options || {}, isTTY(), {
+          readFile(tempFile, callback) {
+            try {
+              fs.readFile(tempFile, 'utf8', (err, contents) => {
+                try {
+                  fs.unlink(tempFile, () => callback(err, contents));
+                } catch {
+                  callback(err, contents);
+                }
+              });
+            } catch (err) {
+              callback(err, null);
+            }
+          },
+          writeFile(contents, callback) {
+            try {
+              let tempFile = randomFileName();
+              fs.writeFile(tempFile, contents, err =>
+                err !== null ? callback(null) : callback(tempFile));
+            } catch {
+              callback(null);
+            }
+          },
+        }, (err, res) => err ? reject(err) : resolve(res!))),
     stop() { child.kill(); },
   });
 };
@@ -115,6 +163,10 @@ let runServiceSync = (callback: (service: common.StreamService) => void): void =
   afterClose();
 };
 
+let randomFileName = () => {
+  return path.join(os.tmpdir(), `esbuild-${crypto.randomBytes(32).toString('hex')}`);
+};
+
 let api: typeof types = {
   build,
   buildSync,

lib/stdio_protocol.ts

@@ -22,13 +22,18 @@ export interface BuildResponse {
 export interface TransformRequest {
   flags: string[];
   input: string;
+  inputFS: boolean;
 }
 
 export interface TransformResponse {
   errors: types.Message[];
   warnings: types.Message[];
+
   js: string;
+  jsFS: boolean;
+
   jsSourceMap: string;
+  jsSourceMapFS: boolean;
 }
 
 ////////////////////////////////////////////////////////////////////////////////