Skip to content

Commit

Permalink
test_runner: add code coverage support to spec reporter
Browse files Browse the repository at this point in the history 
PR-URL: #46674
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
  • Loading branch information
@pulkit-30 @danielleadams
pulkit-30 authored and danielleadams committed Jul 6, 2023
1 parent 992833b commit 12fef4b
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 89 deletions.
5 changes: 4 additions & 1 deletion lib/internal/test_runner/reporter/spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const assert = require('assert');
const Transform = require('internal/streams/transform');
const { inspectWithNoCustomRetry } = require('internal/errors');
const { green, blue, red, white, gray } = require('internal/util/colors');

const { getCoverageReport } = require('internal/test_runner/utils');

const inspectOptions = { __proto__: null, colors: true, breakLength: Infinity };

Expand All @@ -30,6 +30,7 @@ const symbols = {
'test:fail': '\u2716 ',
'test:pass': '\u2714 ',
'test:diagnostic': '\u2139 ',
'test:coverage': '\u2139 ',
'arrow:right': '\u25B6 ',
'hyphen:minus': '\uFE63 ',
};
Expand Down Expand Up @@ -115,6 +116,8 @@ class SpecReporter extends Transform {
break;
case 'test:diagnostic':
return `${colors[type]}${this.#indent(data.nesting)}${symbols[type]}${data.message}${white}\n`;
case 'test:coverage':
return getCoverageReport(this.#indent(data.nesting), data.summary, symbols['test:coverage'], blue);
}
}
_transform({ type, data }, encoding, callback) {
Expand Down
39 changes: 2 additions & 37 deletions lib/internal/test_runner/reporter/tap.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ const {
ArrayPrototypeForEach,
ArrayPrototypeJoin,
ArrayPrototypePush,
NumberPrototypeToFixed,
ObjectEntries,
RegExpPrototypeSymbolReplace,
SafeMap,
Expand All @@ -13,7 +12,7 @@ const {
} = primordials;
const { inspectWithNoCustomRetry } = require('internal/errors');
const { isError, kEmptyObject } = require('internal/util');
const { relative } = require('path');
const { getCoverageReport } = require('internal/test_runner/utils');
const kDefaultIndent = ' '; // 4 spaces
const kFrameStartRegExp = /^ {4}at /;
const kLineBreakRegExp = /\n|\r\n/;
Expand Down Expand Up @@ -49,7 +48,7 @@ async function * tapReporter(source) {
yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`;
break;
case 'test:coverage':
yield reportCoverage(data.nesting, data.summary);
yield getCoverageReport(indent(data.nesting), data.summary, '# ', '');
break;
}
}
Expand All @@ -73,40 +72,6 @@ function reportTest(nesting, testNumber, status, name, skip, todo) {
return line;
}

function reportCoverage(nesting, summary) {
const pad = indent(nesting);
let report = `${pad}# start of coverage report\n`;

report += `${pad}# file | line % | branch % | funcs % | uncovered lines\n`;

for (let i = 0; i < summary.files.length; ++i) {
const {
path,
coveredLinePercent,
coveredBranchPercent,
coveredFunctionPercent,
uncoveredLineNumbers,
} = summary.files[i];
const relativePath = relative(summary.workingDirectory, path);
const lines = NumberPrototypeToFixed(coveredLinePercent, 2);
const branches = NumberPrototypeToFixed(coveredBranchPercent, 2);
const functions = NumberPrototypeToFixed(coveredFunctionPercent, 2);
const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', ');

report += `${pad}# ${relativePath} | ${lines} | ${branches} | ` +
`${functions} | ${uncovered}\n`;
}

const { totals } = summary;
report += `${pad}# all files | ` +
`${NumberPrototypeToFixed(totals.coveredLinePercent, 2)} | ` +
`${NumberPrototypeToFixed(totals.coveredBranchPercent, 2)} | ` +
`${NumberPrototypeToFixed(totals.coveredFunctionPercent, 2)} |\n`;

report += `${pad}# end of coverage report\n`;
return report;
}

function reportDetails(nesting, data = kEmptyObject) {
const { error, duration_ms } = data;
const _indent = indent(nesting);
Expand Down
52 changes: 51 additions & 1 deletion lib/internal/test_runner/utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
'use strict';
const {
ArrayPrototypeJoin,
ArrayPrototypeMap,
ArrayPrototypePush,
ObjectCreate,
ObjectGetOwnPropertyDescriptor,
NumberPrototypeToFixed,
SafePromiseAllReturnArrayLike,
RegExp,
RegExpPrototypeExec,
SafeMap,
} = primordials;

const { basename } = require('path');
const { basename, relative } = require('path');
const { createWriteStream } = require('fs');
const { pathToFileURL } = require('internal/url');
const { createDeferredPromise } = require('internal/util');
const { getOptionValue } = require('internal/options');
const { green, red, white } = require('internal/util/colors');

const {
codes: {
Expand Down Expand Up @@ -247,6 +250,52 @@ function countCompletedTest(test, harness = test.root.harness) {
harness.counters.all++;
}


function coverageThreshold(coverage, color) {
coverage = NumberPrototypeToFixed(coverage, 2);
if (color) {
if (coverage > 90) return `${green}${coverage}${color}`;
if (coverage < 50) return `${red}${coverage}${color}`;
}
return coverage;
}

function getCoverageReport(pad, summary, symbol, color) {
let report = `${color}${pad}${symbol}start of coverage report\n`;

report += `${pad}${symbol}file | line % | branch % | funcs % | uncovered lines\n`;

for (let i = 0; i < summary.files.length; ++i) {
const {
path,
coveredLinePercent,
coveredBranchPercent,
coveredFunctionPercent,
uncoveredLineNumbers,
} = summary.files[i];
const relativePath = relative(summary.workingDirectory, path);
const lines = coverageThreshold(coveredLinePercent, color);
const branches = coverageThreshold(coveredBranchPercent, color);
const functions = coverageThreshold(coveredFunctionPercent, color);
const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', ');

report += `${pad}${symbol}${relativePath} | ${lines} | ${branches} | ` +
`${functions} | ${uncovered}\n`;
}

const { totals } = summary;
report += `${pad}${symbol}all files | ` +
`${coverageThreshold(totals.coveredLinePercent, color)} | ` +
`${coverageThreshold(totals.coveredBranchPercent, color)} | ` +
`${coverageThreshold(totals.coveredFunctionPercent, color)} |\n`;

report += `${pad}${symbol}end of coverage report\n`;
if (color) {
report += white;
}
return report;
}

module.exports = {
convertStringToRegExp,
countCompletedTest,
Expand All @@ -256,4 +305,5 @@ module.exports = {
isTestFailureError,
parseCommandLine,
setupTestReporters,
getCoverageReport,
};
156 changes: 106 additions & 50 deletions test/parallel/test-runner-coverage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ function findCoverageFileForPid(pid) {
});
}

function getCoverageFixtureReport() {
function getTapCoverageFixtureReport() {
const report = [
'# start of coverage report',
'# file | line % | branch % | funcs % | uncovered lines',
Expand All @@ -37,64 +37,120 @@ function getCoverageFixtureReport() {
return report;
}

test('--experimental-test-coverage and --test cannot be combined', () => {
// TODO(cjihrig): This test can be removed once multi-process code coverage
// is supported.
const args = ['--test', '--experimental-test-coverage'];
const result = spawnSync(process.execPath, args);

// 9 is the documented exit code for an invalid CLI argument.
assert.strictEqual(result.status, 9);
assert.match(
result.stderr.toString(),
/--experimental-test-coverage cannot be used with --test/
);
});
function getSpecCoverageFixtureReport() {
const report = [
'\u2139 start of coverage report',
'\u2139 file | line % | branch % | funcs % | uncovered lines',
'\u2139 test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12, ' +
'13, 16, 17, 18, 19, 20, 21, 22, 27, 39, 43, 44, 61, 62, 66, 67, 71, 72',
'\u2139 test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ',
'\u2139 test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5, 6',
'\u2139 all files | 78.35 | 43.75 | 60.00 |',
'\u2139 end of coverage report',
].join('\n');

test('handles the inspector not being available', (t) => {
if (process.features.inspector) {
return;
if (common.isWindows) {
return report.replaceAll('/', '\\');
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const result = spawnSync(process.execPath, args);
return report;
}

assert(!result.stdout.toString().includes('# start of coverage report'));
assert(result.stderr.toString().includes('coverage could not be collected'));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
test('test coverage report', async (t) => {
await t.test('--experimental-test-coverage and --test cannot be combined', () => {
// TODO(cjihrig): This test can be removed once multi-process code coverage
// is supported.
const args = ['--test', '--experimental-test-coverage'];
const result = spawnSync(process.execPath, args);

// 9 is the documented exit code for an invalid CLI argument.
assert.strictEqual(result.status, 9);
assert.match(
result.stderr.toString(),
/--experimental-test-coverage cannot be used with --test/
);
});

test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
if (!process.features.inspector) {
return;
}
await t.test('handles the inspector not being available', (t) => {
if (process.features.inspector) {
return;
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
const result = spawnSync(process.execPath, args, options);
const report = getCoverageFixtureReport();
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const result = spawnSync(process.execPath, args);

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(findCoverageFileForPid(result.pid));
assert(!result.stdout.toString().includes('# start of coverage report'));
assert(result.stderr.toString().includes('coverage could not be collected'));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
});

test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
if (!process.features.inspector) {
return;
}
test('test tap coverage reporter', async (t) => {
await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
if (!process.features.inspector) {
return;
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture];
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
const result = spawnSync(process.execPath, args, options);
const report = getTapCoverageFixtureReport();

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(findCoverageFileForPid(result.pid));
});

await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
if (!process.features.inspector) {
return;
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture];
const result = spawnSync(process.execPath, args);
const report = getTapCoverageFixtureReport();

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const result = spawnSync(process.execPath, args);
const report = getCoverageFixtureReport();
assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
});

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
test('test spec coverage reporter', async (t) => {
await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
if (!process.features.inspector) {
return;
}
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture];
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
const result = spawnSync(process.execPath, args, options);
const report = getSpecCoverageFixtureReport();

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(findCoverageFileForPid(result.pid));
});

await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
if (!process.features.inspector) {
return;
}
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture];
const result = spawnSync(process.execPath, args);
const report = getSpecCoverageFixtureReport();

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
});

0 comments on commit 12fef4b

Please sign in to comment.