From f88075af6879a589844a9516705f0633bce42e4f Mon Sep 17 00:00:00 2001 From: Jordan Pittman Date: Mon, 25 Sep 2023 10:55:20 -0400 Subject: [PATCH 1/5] Refactor --- src/lib/expandTailwindAtRules.js | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/lib/expandTailwindAtRules.js b/src/lib/expandTailwindAtRules.js index b9b54bd6219e..824ee0680e79 100644 --- a/src/lib/expandTailwindAtRules.js +++ b/src/lib/expandTailwindAtRules.js @@ -135,9 +135,11 @@ export default function expandTailwindAtRules(context) { env.DEBUG && console.time('Reading changed files') + /** @type {[item: {file?: string, content?: string}, meta: {transformer: any, extractor: any}][]} */ + let regexParserContent = [] + if (flagEnabled(context.tailwindConfig, 'oxideParser')) { let rustParserContent = [] - let regexParserContent = [] for (let item of context.changedContent) { let transformer = getTransformer(context.tailwindConfig, item.extension) @@ -158,26 +160,21 @@ export default function expandTailwindAtRules(context) { candidates.add(candidate) } } - - if (regexParserContent.length > 0) { - await Promise.all( - regexParserContent.map(async ([{ file, content }, { transformer, extractor }]) => { - content = file ? await fs.promises.readFile(file, 'utf8') : content - getClassCandidates(transformer(content), extractor, candidates, seen) - }) - ) - } } else { - await Promise.all( - context.changedContent.map(async ({ file, content, extension }) => { - let transformer = getTransformer(context.tailwindConfig, extension) - let extractor = getExtractor(context, extension) - content = file ? await fs.promises.readFile(file, 'utf8') : content - getClassCandidates(transformer(content), extractor, candidates, seen) - }) - ) + for (let item of context.changedContent) { + let transformer = getTransformer(context.tailwindConfig, item.extension) + let extractor = getExtractor(context, item.extension) + regexParserContent.push([item, { transformer, extractor }]) + } } + await Promise.all( + regexParserContent.map(async ([{ file, content }, { transformer, extractor }]) => { + content = file ? await fs.promises.readFile(file, 'utf8') : content + getClassCandidates(transformer(content), extractor, candidates, seen) + }) + ) + env.DEBUG && console.timeEnd('Reading changed files') // --- From 25de606b92332920b829bd9e2eef34b4b0fc9ea4 Mon Sep 17 00:00:00 2001 From: Jordan Pittman Date: Mon, 25 Sep 2023 10:57:26 -0400 Subject: [PATCH 2/5] Refactor --- src/lib/expandTailwindAtRules.js | 51 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/lib/expandTailwindAtRules.js b/src/lib/expandTailwindAtRules.js index 824ee0680e79..61763b4b01ec 100644 --- a/src/lib/expandTailwindAtRules.js +++ b/src/lib/expandTailwindAtRules.js @@ -138,36 +138,37 @@ export default function expandTailwindAtRules(context) { /** @type {[item: {file?: string, content?: string}, meta: {transformer: any, extractor: any}][]} */ let regexParserContent = [] - if (flagEnabled(context.tailwindConfig, 'oxideParser')) { - let rustParserContent = [] - - for (let item of context.changedContent) { - let transformer = getTransformer(context.tailwindConfig, item.extension) - let extractor = getExtractor(context, item.extension) - - if (transformer === builtInTransformers.DEFAULT && extractor?.DEFAULT_EXTRACTOR === true) { - rustParserContent.push(item) - } else { - regexParserContent.push([item, { transformer, extractor }]) - } + /** @type {{file?: string, content?: string}[]} */ + let rustParserContent = [] + + for (let item of context.changedContent) { + let transformer = getTransformer(context.tailwindConfig, item.extension) + let extractor = getExtractor(context, item.extension) + + if ( + flagEnabled(context.tailwindConfig, 'oxideParser') && + transformer === builtInTransformers.DEFAULT && + extractor?.DEFAULT_EXTRACTOR === true + ) { + rustParserContent.push(item) + } else { + regexParserContent.push([item, { transformer, extractor }]) } + } - if (rustParserContent.length > 0) { - for (let candidate of parseCandidateStrings( - rustParserContent, - IO.Parallel | Parsing.Parallel - )) { - candidates.add(candidate) - } - } - } else { - for (let item of context.changedContent) { - let transformer = getTransformer(context.tailwindConfig, item.extension) - let extractor = getExtractor(context, item.extension) - regexParserContent.push([item, { transformer, extractor }]) + // Read files using our newer, faster parser when: + // - Oxide is enabled; AND + // - The file is using default transfomers and extractors + if (rustParserContent.length > 0) { + for (let candidate of parseCandidateStrings( + rustParserContent, + IO.Parallel | Parsing.Parallel + )) { + candidates.add(candidate) } } + // Otherwise, read any files in node and parse with regexes await Promise.all( regexParserContent.map(async ([{ file, content }, { transformer, extractor }]) => { content = file ? await fs.promises.readFile(file, 'utf8') : content From ae2f9ea3e2761c4224aa5627d2a01343fb0528fd Mon Sep 17 00:00:00 2001 From: Jordan Pittman Date: Mon, 25 Sep 2023 11:08:58 -0400 Subject: [PATCH 3/5] Batch content file reads in Node into groups of 500 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We shouldn’t need to do this for our Rust code because it utilizes Rayon’s default thread pool for parallelism. This threadpool has roughly the number of cores as the number of available threads except when overridden. This generally is much, much lower than 500 and can be explicitly overridden via an env var to work around potential issues with open file descriptors if anyone ever runs into that. --- src/lib/expandTailwindAtRules.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib/expandTailwindAtRules.js b/src/lib/expandTailwindAtRules.js index 61763b4b01ec..7b4afaa8e1d8 100644 --- a/src/lib/expandTailwindAtRules.js +++ b/src/lib/expandTailwindAtRules.js @@ -169,12 +169,18 @@ export default function expandTailwindAtRules(context) { } // Otherwise, read any files in node and parse with regexes - await Promise.all( - regexParserContent.map(async ([{ file, content }, { transformer, extractor }]) => { - content = file ? await fs.promises.readFile(file, 'utf8') : content - getClassCandidates(transformer(content), extractor, candidates, seen) - }) - ) + const BATCH_SIZE = 500 + + for (let i = 0; i < regexParserContent.length; i += BATCH_SIZE) { + let batch = regexParserContent.slice(i, i + BATCH_SIZE) + + await Promise.all( + batch.map(async ([{ file, content }, { transformer, extractor }]) => { + content = file ? await fs.promises.readFile(file, 'utf8') : content + getClassCandidates(transformer(content), extractor, candidates, seen) + }) + ) + } env.DEBUG && console.timeEnd('Reading changed files') From c84722962eac3ac3fc6f359284206e9d8a0fe458 Mon Sep 17 00:00:00 2001 From: Jordan Pittman Date: Mon, 25 Sep 2023 11:17:57 -0400 Subject: [PATCH 4/5] Fix sequential/parallel flip --- oxide/crates/core/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oxide/crates/core/src/lib.rs b/oxide/crates/core/src/lib.rs index 9596c68556f7..c0bd47666baa 100644 --- a/oxide/crates/core/src/lib.rs +++ b/oxide/crates/core/src/lib.rs @@ -342,8 +342,8 @@ pub fn parse_candidate_strings(input: Vec, options: u8) -> Vec parse_all_blobs_sync(read_all_files_sync(input)), - (IO::Sequential, Parsing::Parallel) => parse_all_blobs_sync(read_all_files(input)), - (IO::Parallel, Parsing::Sequential) => parse_all_blobs(read_all_files_sync(input)), + (IO::Sequential, Parsing::Parallel) => parse_all_blobs(read_all_files_sync(input)), + (IO::Parallel, Parsing::Sequential) => parse_all_blobs_sync(read_all_files(input)), (IO::Parallel, Parsing::Parallel) => parse_all_blobs(read_all_files(input)), } } From c3ab1fc8a0cf64480924ed1c67cbd697872bf9d6 Mon Sep 17 00:00:00 2001 From: Jordan Pittman Date: Mon, 25 Sep 2023 11:50:10 -0400 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d253a01eddfe..72db61edbd74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix incorrectly generated CSS when using square brackets inside arbitrary properties ([#11709](https://github.com/tailwindlabs/tailwindcss/pull/11709)) - Make `content` optional for presets in TypeScript types ([#11730](https://github.com/tailwindlabs/tailwindcss/pull/11730)) - Handle variable colors that have variable fallback values ([#12049](https://github.com/tailwindlabs/tailwindcss/pull/12049)) +- Batch reading content files to prevent `too many open files` error ([#12079](https://github.com/tailwindlabs/tailwindcss/pull/12079)) ### Added