diff --git a/package.json b/package.json index 427b8a5f5ff1..302cdd0297bc 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@angular/platform-browser-dynamic": "^2.3.0", "@angular/platform-server": "^2.3.0", "@angular/router": "^3.3.0", + "@types/fs-extra": "0.0.37", "@types/glob": "^5.0.30", "@types/gulp": "^3.8.32", "@types/hammerjs": "^2.0.34", @@ -59,6 +60,7 @@ "firebase-tools": "^2.2.1", "fs-extra": "^2.0.0", "glob": "^7.1.1", + "google-cloud": "^0.45.1", "gulp": "^3.9.1", "gulp-autoprefixer": "^3.1.1", "gulp-better-rollup": "^1.0.2", @@ -77,6 +79,7 @@ "gulp-transform": "^1.1.0", "hammerjs": "^2.0.8", "highlight.js": "^9.9.0", + "image-diff": "^1.6.3", "jasmine-core": "^2.5.2", "karma": "^1.4.1", "karma-browserstack-launcher": "^1.2.0", diff --git a/tools/gulp/gulpfile.ts b/tools/gulp/gulpfile.ts index 02b2e1d0fbc4..540c73aacfcf 100644 --- a/tools/gulp/gulpfile.ts +++ b/tools/gulp/gulpfile.ts @@ -7,6 +7,7 @@ import './tasks/docs'; import './tasks/e2e'; import './tasks/lint'; import './tasks/release'; +import './tasks/screenshots'; import './tasks/serve'; import './tasks/unit-test'; import './tasks/docs'; diff --git a/tools/gulp/task_helpers.ts b/tools/gulp/task_helpers.ts index ffeffc59af19..68f9f8e24b79 100644 --- a/tools/gulp/task_helpers.ts +++ b/tools/gulp/task_helpers.ts @@ -2,7 +2,6 @@ import * as child_process from 'child_process'; import * as fs from 'fs'; import * as gulp from 'gulp'; import * as path from 'path'; - import {NPM_VENDOR_FILES, PROJECT_ROOT, DIST_ROOT, SASS_AUTOPREFIXER_OPTIONS} from './constants'; @@ -16,6 +15,7 @@ const gulpAutoprefixer = require('gulp-autoprefixer'); const gulpConnect = require('gulp-connect'); const resolveBin = require('resolve-bin'); const firebaseAdmin = require('firebase-admin'); +const gcloud = require('google-cloud'); /** If the string passed in is a glob, returns it, otherwise append '**\/*' to it. */ @@ -186,7 +186,7 @@ export function sequenceTask(...args: any[]) { } /** Opens a connection to the firebase realtime database. */ -export function openFirebaseDatabase() { +export function openFirebaseDashboardDatabase() { // Initialize the Firebase application with admin credentials. // Credentials need to be for a Service Account, which can be created in the Firebase console. firebaseAdmin.initializeApp({ @@ -207,3 +207,43 @@ export function openFirebaseDatabase() { export function isTravisPushBuild() { return process.env['TRAVIS_PULL_REQUEST'] === 'false'; } + +/** + * Open Google Cloud Storage for screenshots. + * The files uploaded to google cloud are also available to firebase storage. + */ +export function openScreenshotsBucket() { + let gcs = gcloud.storage({ + projectId: 'material2-screenshots', + credentials: { + client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com', + private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY']) + }, + }); + + // Reference an existing bucket. + return gcs.bucket('material2-screenshots.appspot.com'); +} + +/** Opens a connection to the firebase realtime database for screenshots. */ +export function openFirebaseScreenshotsDatabase() { + // Initialize the Firebase application with admin credentials. + // Credentials need to be for a Service Account, which can be created in the Firebase console. + let screenshotApp = firebaseAdmin.initializeApp({ + credential: firebaseAdmin.credential.cert({ + project_id: 'material2-screenshots', + client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com', + private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY']) + }), + databaseURL: 'https://material2-screenshots.firebaseio.com' + }, 'material2-screenshots'); + + return screenshotApp.database(); +} + +/** Decode the token for Travis to use. */ +function decode(str: string): string { + // In Travis CI the private key will be incorrect because the line-breaks are escaped. + // The line-breaks need to persist in the service account private key. + return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n'); +} diff --git a/tools/gulp/tasks/e2e.ts b/tools/gulp/tasks/e2e.ts index b2a2f7af4dd7..46ee68e8415e 100644 --- a/tools/gulp/tasks/e2e.ts +++ b/tools/gulp/tasks/e2e.ts @@ -64,6 +64,7 @@ task('e2e', (done: (err?: string) => void) => { 'serve:e2eapp', ':test:protractor', ':serve:e2eapp:stop', + 'screenshots', (err: any) => done(err) ); }); diff --git a/tools/gulp/tasks/payload.ts b/tools/gulp/tasks/payload.ts index ec83a652f259..45ec8bb25238 100644 --- a/tools/gulp/tasks/payload.ts +++ b/tools/gulp/tasks/payload.ts @@ -2,7 +2,7 @@ import {task} from 'gulp'; import {join} from 'path'; import {statSync, readFileSync} from 'fs'; import {DIST_COMPONENTS_ROOT} from '../constants'; -import {openFirebaseDatabase, isTravisPushBuild} from '../task_helpers'; +import {openFirebaseDashboardDatabase, isTravisPushBuild} from '../task_helpers'; import {spawnSync} from 'child_process'; // Those imports lack types. @@ -48,7 +48,7 @@ function getUglifiedSize(filePath: string) { /** Publishes the given results to the firebase database. */ function publishResults(results: any) { let latestSha = spawnSync('git', ['rev-parse', 'HEAD']).stdout.toString().trim(); - let database = openFirebaseDatabase(); + let database = openFirebaseDashboardDatabase(); // Write the results to the payloads object with the latest Git SHA as key. return database.ref('payloads').child(latestSha).set(results) diff --git a/tools/gulp/tasks/screenshots.ts b/tools/gulp/tasks/screenshots.ts new file mode 100644 index 000000000000..05143c823d8e --- /dev/null +++ b/tools/gulp/tasks/screenshots.ts @@ -0,0 +1,153 @@ +import {task} from 'gulp'; +import {readdirSync, statSync, existsSync, mkdirp} from 'fs-extra'; +import * as path from 'path'; +import * as admin from 'firebase-admin'; +import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../task_helpers'; +import {updateGithubStatus} from '../util-functions'; + +const request = require('request'); +const imageDiff = require('image-diff'); + +const SCREENSHOT_DIR = './screenshots'; +const FIREBASE_REPORT = 'screenshot/reports'; +const FIREBASE_FILELIST = 'screenshot/filenames'; + +/** Task which upload screenshots generated from e2e test. */ +task('screenshots', () => { + let prNumber = process.env['TRAVIS_PULL_REQUEST']; + if (prNumber) { + let database = openFirebaseScreenshotsDatabase(); + return getScreenshotFiles(database) + .then((files: any[]) => downloadAllGoldsAndCompare(files, database, prNumber)) + .then((results: boolean) => updateResult(database, prNumber, results)) + .then((result: boolean) => updateGithubStatus(result, prNumber)) + .then(() => uploadScreenshots(prNumber, 'diff')) + .then(() => uploadScreenshots(prNumber, 'test')) + .then(() => updateTravis(database, prNumber)) + .then(() => setScreenFilenames(database, prNumber)) + .then(() => database.goOffline(), () => database.goOffline()); + } +}); + +function updateFileResult(database: admin.database.Database, prNumber: string, + filenameKey: string, result: boolean) { + return database.ref(FIREBASE_REPORT).child(prNumber).child('results').child(filenameKey).set(result); +} + +function updateResult(database: admin.database.Database, prNumber: string, + result: boolean) { + return database.ref(FIREBASE_REPORT).child(prNumber).child('result').set(result).then(() => result); +} + +function updateTravis(database: admin.database.Database, + prNumber: string) { + return database.ref(FIREBASE_REPORT).child(prNumber).update({ + commit: process.env['TRAVIS_COMMIT'], + sha: process.env['TRAVIS_PULL_REQUEST_SHA'], + travis: process.env['TRAVIS_JOB_ID'], + }); +} + +/** Get a list of filenames from firebase database. */ +function getScreenshotFiles(database: admin.database.Database) { + let bucket = openScreenshotsBucket(); + return bucket.getFiles({ prefix: 'golds/' }).then(function(data: any) { + return data[0].filter((file:any) => file.name.endsWith('.screenshot.png')); + }); +} + +function extractScreenshotName(fileName: string) { + return path.basename(fileName, '.screenshot.png'); +} + +function getLocalScreenshotFiles(dir: string): string[] { + return readdirSync(dir) + .filter((fileName: string) => !statSync(path.join(SCREENSHOT_DIR, fileName)).isDirectory()) + .filter((fileName: string) => fileName.endsWith('.screenshot.png')); +} + +/** + * Upload screenshots to google cloud storage. + * @param prNumber - The key used in firebase. Here it is the PR number. + * If there's no prNumber, we will upload images to 'golds/' folder + * @param mode - Can be 'test' or 'diff' or null. + * If the images are the test results, mode should be 'test'. + * If the images are the diff images generated, mode should be 'diff'. + * For golds mode should be null. + */ +function uploadScreenshots(prNumber?: string, mode?: 'test' | 'diff') { + let bucket = openScreenshotsBucket(); + + let promises: any[] = []; + let localDir = mode == 'diff' ? path.join(SCREENSHOT_DIR, 'diff') : SCREENSHOT_DIR; + getLocalScreenshotFiles(localDir).forEach((file: string) => { + let fileName = path.join(localDir, file); + let destination = (mode == null || !prNumber) ? + `golds/${file}` : `screenshots/${prNumber}/${mode}/${file}`; + promises.push(bucket.upload(fileName, { destination: destination })); + }); + return Promise.all(promises); +} + +/** Download golds screenshots. */ +function downloadAllGoldsAndCompare( + files: any[], database: admin.database.Database, + prNumber: string) { + + mkdirp(path.join(SCREENSHOT_DIR, `golds`)); + mkdirp(path.join(SCREENSHOT_DIR, `diff`)); + + return Promise.all(files.map((file: any) => { + return downloadGold(file).then(() => diffScreenshot(file.name, database, prNumber)); + })).then((results: boolean[]) => results.every((value: boolean) => value == true)); +} + +/** Download one gold screenshot */ +function downloadGold(file: any) { + return file.download({ + destination: path.join(SCREENSHOT_DIR, file.name) + }); +} + +function diffScreenshot(filename: string, database: admin.database.Database, + prNumber: string) { + // TODO(tinayuangao): Run the downloads and diffs in parallel. + filename = path.basename(filename); + let goldUrl = path.join(SCREENSHOT_DIR, `golds`, filename); + let pullRequestUrl = path.join(SCREENSHOT_DIR, filename); + let diffUrl = path.join(SCREENSHOT_DIR, `diff`, filename); + let filenameKey = extractScreenshotName(filename); + + if (existsSync(goldUrl) && existsSync(pullRequestUrl)) { + return new Promise((resolve: any, reject: any) => { + imageDiff({ + actualImage: pullRequestUrl, + expectedImage: goldUrl, + diffImage: diffUrl, + }, (err: any, imagesAreSame: boolean) => { + if (err) { + console.log(err); + imagesAreSame = false; + reject(err); + } + resolve(imagesAreSame); + return updateFileResult(database, prNumber, filenameKey, imagesAreSame); + }); + }); + } else { + return updateFileResult(database, prNumber, filenameKey, false).then(() => false); + } +} + +/** + * Upload a list of filenames to firebase database as gold. + * This is necessary for control panel since google-cloud is not available to client side. + */ +function setScreenFilenames(database: admin.database.Database, + prNumber?: string) { + let filenames: string[] = getLocalScreenshotFiles(SCREENSHOT_DIR); + let filelistDatabase = prNumber ? + database.ref(FIREBASE_REPORT).child(prNumber).child('filenames') : + database.ref(FIREBASE_FILELIST); + return filelistDatabase.set(filenames); +} diff --git a/tools/gulp/util-functions.ts b/tools/gulp/util-functions.ts new file mode 100644 index 000000000000..e78f9b94b07d --- /dev/null +++ b/tools/gulp/util-functions.ts @@ -0,0 +1,37 @@ +const request = require('request'); + +/** Update github pr status to success/failure */ +export function updateGithubStatus(result: boolean, prNumber: string) { + let state = result ? 'success' : 'failure'; + let sha = process.env['TRAVIS_PULL_REQUEST_SHA']; + let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']); + + let data = JSON.stringify({ + state: state, + target_url: `http://material2-screenshots.firebaseapp.com/${prNumber}`, + context: "screenshot-diff", + description: `Screenshot test ${state}` + }); + + let headers = { + 'Authorization': `token ${token}`, + 'User-Agent': 'ScreenshotDiff/1.0.0', + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data) + }; + + return new Promise((resolve, reject) => { + request({ + url: `https://api.github.com/repos/angular/material2/statuses/${sha}`, + method: 'POST', + form: data, + headers: headers + }, function (error: any, response: any, body: any){ + resolve(response.statusCode); + }); + }); +} + +function decode(value: string): string { + return value.split('').reverse().join(''); +}