Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(watchman): Parallelize Watchman calls in crawler again #5640

Merged
merged 3 commits into from
Feb 22, 2018
Merged

fix(watchman): Parallelize Watchman calls in crawler again #5640

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
69c6d8c
fix(watchman): Parallelize Watchman calls in crawler again
BYK Feb 22, 2018
ce4773f
Refactor: pull things into functions
BYK Feb 22, 2018
2b062f4
Fix: typo
BYK Feb 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 95 additions & 74 deletions packages/jest-haste-map/src/crawlers/watchman.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,105 +34,126 @@ module.exports = async function watchmanCrawl(
['type', 'f'],
['anyof'].concat(extensions.map(extension => ['suffix', extension])),
];
const clocks = data.clocks;

const client = new watchman.Client();
let clientError;
client.on('error', error => (clientError = error));
client.on('error', error => (clientError = WatchmanError(error)));

const cmd = (...args) =>
new Promise((resolve, reject) =>
client.command(
args,
(error, result) => (error ? reject(error) : resolve(result)),
(error, result) =>
error ? reject(WatchmanError(error)) : resolve(result),
),
);

const clocks = data.clocks;
let files = data.files;

try {
async function getWatchmanRoots(roots) {
const watchmanRoots = new Map();
for (const root of roots) {
const response = await cmd('watch-project', root);
const existing = watchmanRoots.get(response.watch);
// A root can only be filtered if it was never seen with a relative_path before
const canBeFiltered = !existing || existing.length > 0;

if (canBeFiltered) {
if (response.relative_path) {
watchmanRoots.set(
response.watch,
(existing || []).concat(response.relative_path),
);
} else {
// Make the filter directories an empty array to signal that this root
// was already seen and needs to be watched for all files/directories
watchmanRoots.set(response.watch, []);
await Promise.all(
roots.map(async root => {
const response = await cmd('watch-project', root);
const existing = watchmanRoots.get(response.watch);
// A root can only be filtered if it was never seen with a relative_path before
const canBeFiltered = !existing || existing.length > 0;

if (canBeFiltered) {
if (response.relative_path) {
watchmanRoots.set(
response.watch,
(existing || []).concat(response.relative_path),
);
} else {
// Make the filter directories an empty array to signal that this root
// was already seen and needs to be watched for all files/directories
watchmanRoots.set(response.watch, []);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Slightly concerned about race conditions here but it should be fine since the main thread is memory-safe, right?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, this is completely fine

}
}
}
}
}),
);
return watchmanRoots;
}

let shouldReset = false;
const watchmanFileResults = new Map();
for (const [root, directoryFilters] of watchmanRoots) {
const expression = Array.from(defaultWatchExpression);
if (directoryFilters.length > 0) {
expression.push([
'anyof',
...directoryFilters.map(dir => ['dirname', dir]),
]);
}
const fields = ['name', 'exists', 'mtime_ms'];
async function queryWatchmanForDirs(rootProjectDirMappings) {
const files = new Map();
let isFresh = false;
await Promise.all(
Array.from(rootProjectDirMappings).map(
async ([root, directoryFilters]) => {
const expression = Array.from(defaultWatchExpression);
if (directoryFilters.length > 0) {
expression.push([
'anyof',
...directoryFilters.map(dir => ['dirname', dir]),
]);
}
const fields = ['name', 'exists', 'mtime_ms'];

const query = clocks[root]
? // Use the `since` generator if we have a clock available
{expression, fields, since: clocks[root]}
: // Otherwise use the `suffix` generator
{expression, fields, suffix: extensions};
const query = clocks[root]
? // Use the `since` generator if we have a clock available
{expression, fields, since: clocks[root]}
: // Otherwise use the `suffix` generator
{expression, fields, suffix: extensions};

const response = await cmd('query', root, query);
shouldReset = shouldReset || response.is_fresh_instance;
watchmanFileResults.set(root, response);
}
const response = await cmd('query', root, query);
if ('warning' in response) {
console.warn('watchman warning: ', response.warning);
}
isFresh = isFresh || response.is_fresh_instance;
files.set(root, response);
},
),
);

return {
files,
isFresh,
};
}

let files = data.files;
let watchmanFiles;
try {
const watchmanRoots = await getWatchmanRoots(roots);
const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots);
// Reset the file map if watchman was restarted and sends us a list of files.
if (shouldReset) {
if (watchmanFileResults.isFresh) {
files = Object.create(null);
}

for (const [watchRoot, response] of watchmanFileResults) {
const fsRoot = normalizePathSep(watchRoot);
if ('warning' in response) {
console.warn('watchman warning: ', response.warning);
}
clocks[fsRoot] = response.clock;
for (const fileData of response.files) {
const name = fsRoot + path.sep + normalizePathSep(fileData.name);
if (!fileData.exists) {
delete files[name];
} else if (!ignore(name)) {
const mtime =
typeof fileData.mtime_ms === 'number'
? fileData.mtime_ms
: fileData.mtime_ms.toNumber();
const isOld = data.files[name] && data.files[name][H.MTIME] === mtime;
if (isOld) {
files[name] = data.files[name];
} else {
// See ../constants.js
files[name] = ['', mtime, 0, []];
}
}
}
}
} catch (error) {
throw WatchmanError(error);
watchmanFiles = watchmanFileResults.files;
} finally {
client.end();
}

if (clientError) {
throw WatchmanError(clientError);
throw clientError;
}

for (const [watchRoot, response] of watchmanFiles) {
const fsRoot = normalizePathSep(watchRoot);
clocks[fsRoot] = response.clock;
for (const fileData of response.files) {
const name = fsRoot + path.sep + normalizePathSep(fileData.name);
if (!fileData.exists) {
delete files[name];
} else if (!ignore(name)) {
const mtime =
typeof fileData.mtime_ms === 'number'
? fileData.mtime_ms
: fileData.mtime_ms.toNumber();
const existingFileData = data.files[name];
const isOld = existingFileData && existingFileData[H.MTIME] === mtime;
if (isOld) {
files[name] = existingFileData;
} else {
// See ../constants.js
files[name] = ['', mtime, 0, []];
}
}
}
}

data.files = files;
return data;
};