limit simultaneously open file handles (#348)

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+* Avoid running out of file handles when ulimit is low ([#348](https://github.com/evanw/esbuild/issues/348))
+
+    When esbuild uses aggressive concurrency, it can sometimes simultaneously use more file handles than allowed by the system. This can be a problem when the limit is low (e.g. using `ulimit -n 32`). In this release, esbuild now limits itself to using a maximum of 32 file handles simultaneously. This limit was chosen to be low enough to not cause issues with normal ulimit values but high enough to not impact benchmark times.
+
 ## 0.6.27
 
 * Add parentheses when calling `require()` inside `new` ([#339](https://github.com/evanw/esbuild/issues/339))

internal/fs/fs.go

@@ -48,6 +48,8 @@ func (e *Entry) stat() {
 	entryPath := filepath.Join(e.dir, e.base)
 
 	// Use "lstat" since we want information about symbolic links
+	beforeFileOpen()
+	defer afterFileClose()
 	stat, err := os.Lstat(entryPath)
 	if err != nil {
 		return
@@ -227,13 +229,31 @@ type realFS struct {
 	cwd string
 }
 
+// Limit the number of files open simultaneously to avoid ulimit issues
+var fileOpenLimit = make(chan bool, 32)
+
+func beforeFileOpen() {
+	if fileOpenLimit != nil {
+		// This will block if the number of open files is already at the limit
+		fileOpenLimit <- false
+	}
+}
+
+func afterFileClose() {
+	if fileOpenLimit != nil {
+		<-fileOpenLimit
+	}
+}
+
 func realpath(path string) string {
 	dir := filepath.Dir(path)
 	if dir == path {
 		return path
 	}
 	dir = realpath(dir)
 	path = filepath.Join(dir, filepath.Base(path))
+	beforeFileOpen()
+	defer afterFileClose()
 	if link, err := os.Readlink(path); err == nil {
 		if filepath.IsAbs(link) {
 			return link
@@ -300,6 +320,8 @@ func (fs *realFS) ReadDirectory(dir string) map[string]*Entry {
 }
 
 func (fs *realFS) ReadFile(path string) (string, bool) {
+	beforeFileOpen()
+	defer afterFileClose()
 	buffer, err := ioutil.ReadFile(path)
 	return string(buffer), err == nil
 }
@@ -337,6 +359,8 @@ func (*realFS) Rel(base string, target string) (string, bool) {
 }
 
 func readdir(dirname string) ([]string, error) {
+	beforeFileOpen()
+	defer afterFileClose()
 	f, err := os.Open(dirname)
 	if err != nil {
 		return nil, err