Add Beginning functionality for Daily Master (#379)

bin/ncu-ci

@@ -6,7 +6,7 @@ const {
   JobParser,
   parseJobFromURL,
   CI_TYPES_KEYS: {
-    PR, COMMIT, BENCHMARK, CITGM
+    PR, COMMIT, BENCHMARK, CITGM, DAILY_MASTER
   }
 } = require('../lib/ci/ci_type_parser');
 
@@ -15,6 +15,7 @@ const {
   BenchmarkRun,
   CommitBuild,
   CITGMBuild,
+  DailyBuild,
   HealthBuild,
   listBuilds,
   FailureAggregator,
@@ -137,6 +138,27 @@ const argv = yargs
     },
     handler
   })
+  .command({
+    command: 'daily',
+    desc: 'Show recent results of node-daily-master',
+    builder: (yargs) => {
+      yargs
+        .option('stats', {
+          default: false,
+          describe: 'Aggregate the results'
+        })
+        .option('cache', {
+          default: false,
+          describe: 'Cache the responses from Jenkins in .ncu/cache/ under' +
+                    ' the node-core-utils installation directory'
+        })
+        .option('limit', {
+          default: 15,
+          describe: 'Maximum number of CIs to get data from'
+        });
+    },
+    handler
+  })
   .demandCommand(1, 'must provide a valid command')
   .option('copy', {
     default: false,
@@ -214,6 +236,12 @@ class CICommand {
         case BENCHMARK:
           build = new BenchmarkRun(cli, request, job.jobid);
           break;
+        case DAILY_MASTER: {
+          const daily = new DailyBuild(cli, request, job.jobid);
+          const data = await daily.getBuildData();
+          build = new CommitBuild(cli, request, data.subBuilds[0].buildNumber);
+          break;
+        }
         default:
           throw new Error(`Unknown job type ${job.type}`);
       }
@@ -354,6 +382,38 @@ class URLCommand extends CICommand {
   }
 }
 
+class DailyCommand extends CICommand {
+  constructor(cli, request, argv) {
+    super(cli, request, argv);
+    if (argv.cache) {
+      jobCache.enable();
+    }
+  }
+
+  async initialize() {
+    const ciType = DAILY_MASTER;
+    const builds = await listBuilds(this.cli, this.request, ciType);
+    this.queue.push({ type: 'health', ciType, builds });
+    for (const build of builds.failed.slice(0, this.argv.limit)) {
+      this.queue.push(build);
+    }
+  }
+
+  async aggregate() {
+    const { argv, cli } = this;
+    const aggregator = new FailureAggregator(cli, this.json);
+    this.json = aggregator.aggregate();
+    cli.log('');
+    cli.separator('Stats');
+    cli.log('');
+    aggregator.display();
+
+    if (argv.markdown || argv.copy) {
+      this.markdown = aggregator.formatAsMarkdown();
+    }
+  }
+}
+
 async function main(command, argv) {
   const cli = new CLI();
   const credentials = await auth({
@@ -384,6 +444,10 @@ async function main(command, argv) {
       commandHandler = new JobCommand(cli, request, argv, command);
       break;
     }
+    case 'daily': {
+      commandHandler = new DailyCommand(cli, request, argv, command);
+      break;
+    }
     default:
       return yargs.showHelp();
   }

lib/ci/ci_result_parser.js

@@ -225,6 +225,13 @@ class TestBuild extends Job {
     this.builtOn = builtOn;
   }
 
+  setDailyBuildData({ result, changeSet, actions, timestamp, builtOn }) {
+    this.change = changeSet.items[0] || {};
+    this.date = new Date(timestamp);
+    this.result = result;
+    this.builtOn = builtOn;
+  }
+
   get sourceURL() {
     const { params } = this;
 
@@ -303,7 +310,9 @@ class TestBuild extends Job {
   displayBuilds() {
     const { cli, failures, builds } = this;
     for (const failure of failures) {
-      this.displayFailure(failure);
+      if (failure !== undefined) {
+        this.displayFailure(failure);
+      }
     }
     cli.separator('Other builds');
     for (const aborted of builds.aborted) {
@@ -329,6 +338,7 @@ class TestBuild extends Job {
     const { failures } = this;
     let output = `Failures in job ${this.jobUrl}\n\n`;
     for (const failure of failures) {
+      if (failure === undefined) continue;
       output += `#### [${getNodeName(failure.url)}](${failure.url})`;
       if (!failure.reason.includes('\n') && failure.reason.length < 20) {
         const builtOn = failure.builtOn ? `On ${failure.builtOn}: ` : '';
@@ -355,6 +365,10 @@ class TestBuild extends Job {
 }
 
 function getHighlight(f) {
+  if (!f.reason) {
+    f.reason = 'failure not found';
+    return f.reason;
+  }
   return f.reason.split('\n')[f.highlight]
     .replace(/not ok \d+ /, '')
     .replace(
@@ -893,6 +907,28 @@ class CITGMBuild extends TestBuild {
   }
 }
 
+class DailyBuild extends TestBuild {
+  constructor(cli, request, id) {
+    const path = `job/node-daily-master/${id}/`;
+    const tree = PR_TREE;
+    super(cli, request, path, tree);
+
+    this.commitBuild = null;
+  }
+
+  formatAsMarkdown() {
+    if (!this.commitBuild) {
+      let result = 'Failed to trigger node-daily-master';
+      if (this.builtOn) {
+        result += ` on ${this.builtOn}`;
+      }
+      result += `\n\nURL: ${this.jobUrl}`;
+      return result;
+    }
+    return super.formatAsMarkdown();
+  }
+}
+
 function filterBuild(builds, type) {
   return builds
     .filter(build => build.result === type)
@@ -969,7 +1005,9 @@ class FannedBuild extends Job {
     const { jobName, buildNumber } = failedPhase;
     const build = new NormalBuild(cli, request, jobName, buildNumber);
     let failures = await build.getResults();
-    failures = flatten(failures);
+    if (failures !== undefined) {
+      failures = flatten(failures);
+    }
 
     if (this.isResumed) {
       // XXX: if it's a resumed build, we replace the build/git failures
@@ -1045,6 +1083,8 @@ class NormalBuild extends Job {
       return this.failures;
     }
 
+    if (!runs) return [];
+
     if (!runs.length) {
       this.failures = [
         new BuildFailure(
@@ -1188,6 +1228,7 @@ module.exports = {
   BenchmarkRun,
   CommitBuild,
   CITGMBuild,
+  DailyBuild,
   HealthBuild,
   jobCache,
   parseJobFromURL,

lib/ci/ci_type_parser.js

@@ -19,6 +19,7 @@ const LINTER = 'LINTER';
 const LITE_PR = 'LITE_PR';
 const LITE_PR_PIPELINE = 'LITE_PR_PIPELINE';
 const LITE_COMMIT = 'LITE_COMMIT';
+const DAILY_MASTER = 'DAILY_MASTER';
 
 const CI_TYPE_ENUM = {
   FULL_CI: 1 << 0,
@@ -99,6 +100,12 @@ const CI_TYPES = new Map([
     jobName: 'node-test-commit-lite',
     pattern: /job\/node-test-commit-lite\/(\d+)/,
     type: JOB_CI | LITE_CI
+  }],
+  [DAILY_MASTER, {
+    name: 'Node Daily Master',
+    jobName: 'node-daily-master',
+    pattern: /job\/node-daily-master\/(\d+)/,
+    type: JOB_CI
   }]
 ]);
 
@@ -212,7 +219,7 @@ module.exports = {
   CI_TYPES,
   CI_TYPES_KEYS: {
     CITGM, PR, COMMIT, BENCHMARK, LIBUV, V8, NOINTL,
-    LINTER, LITE_PR, LITE_COMMIT
+    LINTER, LITE_PR, LITE_COMMIT, DAILY_MASTER
   },
   CI_PROVIDERS,
   isFullCI,