From 69c6d8c12b7c954294c06345a9e265302165def5 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 22 Feb 2018 16:24:33 +0000 Subject: [PATCH 1/3] fix(watchman): Parallelize Watchman calls in crawler again This is a follow up to #5615 where it made all async watchman commands serialized when converting them from promises to `async` functions. Existing Watchman crawler tests should pass, maybe slightly faster. --- .../jest-haste-map/src/crawlers/watchman.js | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.js b/packages/jest-haste-map/src/crawlers/watchman.js index 945f68edc9bc..834258603e0f 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.js +++ b/packages/jest-haste-map/src/crawlers/watchman.js @@ -51,48 +51,52 @@ module.exports = async function watchmanCrawl( try { const watchmanRoots = new Map(); - for (const root of roots) { - const response = await cmd('watch-project', root); - const existing = watchmanRoots.get(response.watch); - // A root can only be filtered if it was never seen with a relative_path before - const canBeFiltered = !existing || existing.length > 0; + await Promise.all( + roots.map(async root => { + const response = await cmd('watch-project', root); + const existing = watchmanRoots.get(response.watch); + // A root can only be filtered if it was never seen with a relative_path before + const canBeFiltered = !existing || existing.length > 0; - if (canBeFiltered) { - if (response.relative_path) { - watchmanRoots.set( - response.watch, - (existing || []).concat(response.relative_path), - ); - } else { - // Make the filter directories an empty array to signal that this root - // was already seen and needs to be watched for all files/directories - watchmanRoots.set(response.watch, []); + if (canBeFiltered) { + if (response.relative_path) { + watchmanRoots.set( + response.watch, + (existing || []).concat(response.relative_path), + ); + } else { + // Make the filter directories an empty array to signal that this root + // was already seen and needs to be watched for all files/directories + watchmanRoots.set(response.watch, []); + } } - } - } + }), + ); let shouldReset = false; const watchmanFileResults = new Map(); - for (const [root, directoryFilters] of watchmanRoots) { - const expression = Array.from(defaultWatchExpression); - if (directoryFilters.length > 0) { - expression.push([ - 'anyof', - ...directoryFilters.map(dir => ['dirname', dir]), - ]); - } - const fields = ['name', 'exists', 'mtime_ms']; + await Promise.all( + Array.from(watchmanRoots).map(async ([root, directoryFilters]) => { + const expression = Array.from(defaultWatchExpression); + if (directoryFilters.length > 0) { + expression.push([ + 'anyof', + ...directoryFilters.map(dir => ['dirname', dir]), + ]); + } + const fields = ['name', 'exists', 'mtime_ms']; - const query = clocks[root] - ? // Use the `since` generator if we have a clock available - {expression, fields, since: clocks[root]} - : // Otherwise use the `suffix` generator - {expression, fields, suffix: extensions}; + const query = clocks[root] + ? // Use the `since` generator if we have a clock available + {expression, fields, since: clocks[root]} + : // Otherwise use the `suffix` generator + {expression, fields, suffix: extensions}; - const response = await cmd('query', root, query); - shouldReset = shouldReset || response.is_fresh_instance; - watchmanFileResults.set(root, response); - } + const response = await cmd('query', root, query); + shouldReset = shouldReset || response.is_fresh_instance; + watchmanFileResults.set(root, response); + }), + ); // Reset the file map if watchman was restarted and sends us a list of files. if (shouldReset) { From ce4773f1fbdb98660aa504a44ffbbee967a07c40 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 22 Feb 2018 17:19:07 +0000 Subject: [PATCH 2/3] Refactor: pull things into functions --- .../jest-haste-map/src/crawlers/watchman.js | 129 ++++++++++-------- 1 file changed, 73 insertions(+), 56 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.js b/packages/jest-haste-map/src/crawlers/watchman.js index 834258603e0f..8a8c8d3568af 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.js +++ b/packages/jest-haste-map/src/crawlers/watchman.js @@ -34,22 +34,22 @@ module.exports = async function watchmanCrawl( ['type', 'f'], ['anyof'].concat(extensions.map(extension => ['suffix', extension])), ]; + const clocks = data.clocks; + const client = new watchman.Client(); let clientError; - client.on('error', error => (clientError = error)); + client.on('error', error => (clientError = WatchmanError(error))); const cmd = (...args) => new Promise((resolve, reject) => client.command( args, - (error, result) => (error ? reject(error) : resolve(result)), + (error, result) => + error ? reject(WatchmanError(error)) : resolve(result), ), ); - const clocks = data.clocks; - let files = data.files; - - try { + async function getWatcmanRoots(roots) { const watchmanRoots = new Map(); await Promise.all( roots.map(async root => { @@ -72,71 +72,88 @@ module.exports = async function watchmanCrawl( } }), ); + return watchmanRoots; + } - let shouldReset = false; - const watchmanFileResults = new Map(); + async function queryWatchmanForDirs(rootProjectDirMappings) { + const files = new Map(); + let isFresh = false; await Promise.all( - Array.from(watchmanRoots).map(async ([root, directoryFilters]) => { - const expression = Array.from(defaultWatchExpression); - if (directoryFilters.length > 0) { - expression.push([ - 'anyof', - ...directoryFilters.map(dir => ['dirname', dir]), - ]); - } - const fields = ['name', 'exists', 'mtime_ms']; + Array.from(rootProjectDirMappings).map( + async ([root, directoryFilters]) => { + const expression = Array.from(defaultWatchExpression); + if (directoryFilters.length > 0) { + expression.push([ + 'anyof', + ...directoryFilters.map(dir => ['dirname', dir]), + ]); + } + const fields = ['name', 'exists', 'mtime_ms']; - const query = clocks[root] - ? // Use the `since` generator if we have a clock available - {expression, fields, since: clocks[root]} - : // Otherwise use the `suffix` generator - {expression, fields, suffix: extensions}; + const query = clocks[root] + ? // Use the `since` generator if we have a clock available + {expression, fields, since: clocks[root]} + : // Otherwise use the `suffix` generator + {expression, fields, suffix: extensions}; - const response = await cmd('query', root, query); - shouldReset = shouldReset || response.is_fresh_instance; - watchmanFileResults.set(root, response); - }), + const response = await cmd('query', root, query); + if ('warning' in response) { + console.warn('watchman warning: ', response.warning); + } + isFresh = isFresh || response.is_fresh_instance; + files.set(root, response); + }, + ), ); + return { + files, + isFresh, + }; + } + + let files = data.files; + let watchmanFiles; + try { + const watchmanRoots = await getWatcmanRoots(roots); + const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); // Reset the file map if watchman was restarted and sends us a list of files. - if (shouldReset) { + if (watchmanFileResults.isFresh) { files = Object.create(null); } - - for (const [watchRoot, response] of watchmanFileResults) { - const fsRoot = normalizePathSep(watchRoot); - if ('warning' in response) { - console.warn('watchman warning: ', response.warning); - } - clocks[fsRoot] = response.clock; - for (const fileData of response.files) { - const name = fsRoot + path.sep + normalizePathSep(fileData.name); - if (!fileData.exists) { - delete files[name]; - } else if (!ignore(name)) { - const mtime = - typeof fileData.mtime_ms === 'number' - ? fileData.mtime_ms - : fileData.mtime_ms.toNumber(); - const isOld = data.files[name] && data.files[name][H.MTIME] === mtime; - if (isOld) { - files[name] = data.files[name]; - } else { - // See ../constants.js - files[name] = ['', mtime, 0, []]; - } - } - } - } - } catch (error) { - throw WatchmanError(error); + watchmanFiles = watchmanFileResults.files; } finally { client.end(); } if (clientError) { - throw WatchmanError(clientError); + throw clientError; } + + for (const [watchRoot, response] of watchmanFiles) { + const fsRoot = normalizePathSep(watchRoot); + clocks[fsRoot] = response.clock; + for (const fileData of response.files) { + const name = fsRoot + path.sep + normalizePathSep(fileData.name); + if (!fileData.exists) { + delete files[name]; + } else if (!ignore(name)) { + const mtime = + typeof fileData.mtime_ms === 'number' + ? fileData.mtime_ms + : fileData.mtime_ms.toNumber(); + const existingFileData = data.files[name]; + const isOld = existingFileData && existingFileData[H.MTIME] === mtime; + if (isOld) { + files[name] = existingFileData; + } else { + // See ../constants.js + files[name] = ['', mtime, 0, []]; + } + } + } + } + data.files = files; return data; }; From 2b062f4c73d002a953db0431986ed2227071600c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 22 Feb 2018 17:32:35 +0000 Subject: [PATCH 3/3] Fix: typo --- packages/jest-haste-map/src/crawlers/watchman.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.js b/packages/jest-haste-map/src/crawlers/watchman.js index 8a8c8d3568af..ed14bf213774 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.js +++ b/packages/jest-haste-map/src/crawlers/watchman.js @@ -49,7 +49,7 @@ module.exports = async function watchmanCrawl( ), ); - async function getWatcmanRoots(roots) { + async function getWatchmanRoots(roots) { const watchmanRoots = new Map(); await Promise.all( roots.map(async root => { @@ -115,7 +115,7 @@ module.exports = async function watchmanCrawl( let files = data.files; let watchmanFiles; try { - const watchmanRoots = await getWatcmanRoots(roots); + const watchmanRoots = await getWatchmanRoots(roots); const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); // Reset the file map if watchman was restarted and sends us a list of files. if (watchmanFileResults.isFresh) {