-
Notifications
You must be signed in to change notification settings - Fork 6.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(screenshot): Add a gulp to compare screenshot diffs, upload resu…
…lts to gcs & firebase (#2774) * Add google cloud storage * Add screenshots to e2e test * Renamed functions. Add types. Remove firebase, use firebase-admin * dependencies * Save pull request sha to firebase db * Save travis job id * Add function to update github commit status * remove update status code * Add update status * Address comments * Add fs-extra * try using ES6 Promise * . * Fix cannot find name promise * Add back function to save filenames to firebase * Update comments
- Loading branch information
1 parent
bb2392f
commit bd6feb3
Showing
7 changed files
with
239 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| import {task} from 'gulp'; | ||
| import {readdirSync, statSync, existsSync, mkdirp} from 'fs-extra'; | ||
| import * as path from 'path'; | ||
| import * as admin from 'firebase-admin'; | ||
| import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../task_helpers'; | ||
| import {updateGithubStatus} from '../util-functions'; | ||
|
|
||
| const request = require('request'); | ||
| const imageDiff = require('image-diff'); | ||
|
|
||
| const SCREENSHOT_DIR = './screenshots'; | ||
| const FIREBASE_REPORT = 'screenshot/reports'; | ||
| const FIREBASE_FILELIST = 'screenshot/filenames'; | ||
|
|
||
| /** Task which upload screenshots generated from e2e test. */ | ||
| task('screenshots', () => { | ||
| let prNumber = process.env['TRAVIS_PULL_REQUEST']; | ||
| if (prNumber) { | ||
| let database = openFirebaseScreenshotsDatabase(); | ||
| return getScreenshotFiles(database) | ||
| .then((files: any[]) => downloadAllGoldsAndCompare(files, database, prNumber)) | ||
| .then((results: boolean) => updateResult(database, prNumber, results)) | ||
| .then((result: boolean) => updateGithubStatus(result, prNumber)) | ||
| .then(() => uploadScreenshots(prNumber, 'diff')) | ||
| .then(() => uploadScreenshots(prNumber, 'test')) | ||
| .then(() => updateTravis(database, prNumber)) | ||
| .then(() => setScreenFilenames(database, prNumber)) | ||
| .then(() => database.goOffline(), () => database.goOffline()); | ||
| } | ||
| }); | ||
|
|
||
| function updateFileResult(database: admin.database.Database, prNumber: string, | ||
| filenameKey: string, result: boolean) { | ||
| return database.ref(FIREBASE_REPORT).child(prNumber).child('results').child(filenameKey).set(result); | ||
| } | ||
|
|
||
| function updateResult(database: admin.database.Database, prNumber: string, | ||
| result: boolean) { | ||
| return database.ref(FIREBASE_REPORT).child(prNumber).child('result').set(result).then(() => result); | ||
| } | ||
|
|
||
| function updateTravis(database: admin.database.Database, | ||
| prNumber: string) { | ||
| return database.ref(FIREBASE_REPORT).child(prNumber).update({ | ||
| commit: process.env['TRAVIS_COMMIT'], | ||
| sha: process.env['TRAVIS_PULL_REQUEST_SHA'], | ||
| travis: process.env['TRAVIS_JOB_ID'], | ||
| }); | ||
| } | ||
|
|
||
| /** Get a list of filenames from firebase database. */ | ||
| function getScreenshotFiles(database: admin.database.Database) { | ||
| let bucket = openScreenshotsBucket(); | ||
| return bucket.getFiles({ prefix: 'golds/' }).then(function(data: any) { | ||
| return data[0].filter((file:any) => file.name.endsWith('.screenshot.png')); | ||
| }); | ||
| } | ||
|
|
||
| function extractScreenshotName(fileName: string) { | ||
| return path.basename(fileName, '.screenshot.png'); | ||
| } | ||
|
|
||
| function getLocalScreenshotFiles(dir: string): string[] { | ||
| return readdirSync(dir) | ||
| .filter((fileName: string) => !statSync(path.join(SCREENSHOT_DIR, fileName)).isDirectory()) | ||
| .filter((fileName: string) => fileName.endsWith('.screenshot.png')); | ||
| } | ||
|
|
||
| /** | ||
| * Upload screenshots to google cloud storage. | ||
| * @param prNumber - The key used in firebase. Here it is the PR number. | ||
| * If there's no prNumber, we will upload images to 'golds/' folder | ||
| * @param mode - Can be 'test' or 'diff' or null. | ||
| * If the images are the test results, mode should be 'test'. | ||
| * If the images are the diff images generated, mode should be 'diff'. | ||
| * For golds mode should be null. | ||
| */ | ||
| function uploadScreenshots(prNumber?: string, mode?: 'test' | 'diff') { | ||
| let bucket = openScreenshotsBucket(); | ||
|
|
||
| let promises: any[] = []; | ||
| let localDir = mode == 'diff' ? path.join(SCREENSHOT_DIR, 'diff') : SCREENSHOT_DIR; | ||
| getLocalScreenshotFiles(localDir).forEach((file: string) => { | ||
| let fileName = path.join(localDir, file); | ||
| let destination = (mode == null || !prNumber) ? | ||
| `golds/${file}` : `screenshots/${prNumber}/${mode}/${file}`; | ||
| promises.push(bucket.upload(fileName, { destination: destination })); | ||
| }); | ||
| return Promise.all(promises); | ||
| } | ||
|
|
||
| /** Download golds screenshots. */ | ||
| function downloadAllGoldsAndCompare( | ||
| files: any[], database: admin.database.Database, | ||
| prNumber: string) { | ||
|
|
||
| mkdirp(path.join(SCREENSHOT_DIR, `golds`)); | ||
| mkdirp(path.join(SCREENSHOT_DIR, `diff`)); | ||
|
|
||
| return Promise.all(files.map((file: any) => { | ||
| return downloadGold(file).then(() => diffScreenshot(file.name, database, prNumber)); | ||
| })).then((results: boolean[]) => results.every((value: boolean) => value == true)); | ||
| } | ||
|
|
||
| /** Download one gold screenshot */ | ||
| function downloadGold(file: any) { | ||
| return file.download({ | ||
| destination: path.join(SCREENSHOT_DIR, file.name) | ||
| }); | ||
| } | ||
|
|
||
| function diffScreenshot(filename: string, database: admin.database.Database, | ||
| prNumber: string) { | ||
| // TODO(tinayuangao): Run the downloads and diffs in parallel. | ||
| filename = path.basename(filename); | ||
| let goldUrl = path.join(SCREENSHOT_DIR, `golds`, filename); | ||
| let pullRequestUrl = path.join(SCREENSHOT_DIR, filename); | ||
| let diffUrl = path.join(SCREENSHOT_DIR, `diff`, filename); | ||
| let filenameKey = extractScreenshotName(filename); | ||
|
|
||
| if (existsSync(goldUrl) && existsSync(pullRequestUrl)) { | ||
| return new Promise((resolve: any, reject: any) => { | ||
| imageDiff({ | ||
| actualImage: pullRequestUrl, | ||
| expectedImage: goldUrl, | ||
| diffImage: diffUrl, | ||
| }, (err: any, imagesAreSame: boolean) => { | ||
| if (err) { | ||
| console.log(err); | ||
| imagesAreSame = false; | ||
| reject(err); | ||
| } | ||
| resolve(imagesAreSame); | ||
| return updateFileResult(database, prNumber, filenameKey, imagesAreSame); | ||
| }); | ||
| }); | ||
| } else { | ||
| return updateFileResult(database, prNumber, filenameKey, false).then(() => false); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Upload a list of filenames to firebase database as gold. | ||
| * This is necessary for control panel since google-cloud is not available to client side. | ||
| */ | ||
| function setScreenFilenames(database: admin.database.Database, | ||
| prNumber?: string) { | ||
| let filenames: string[] = getLocalScreenshotFiles(SCREENSHOT_DIR); | ||
| let filelistDatabase = prNumber ? | ||
| database.ref(FIREBASE_REPORT).child(prNumber).child('filenames') : | ||
| database.ref(FIREBASE_FILELIST); | ||
| return filelistDatabase.set(filenames); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| const request = require('request'); | ||
|
|
||
| /** Update github pr status to success/failure */ | ||
| export function updateGithubStatus(result: boolean, prNumber: string) { | ||
| let state = result ? 'success' : 'failure'; | ||
| let sha = process.env['TRAVIS_PULL_REQUEST_SHA']; | ||
| let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']); | ||
|
|
||
| let data = JSON.stringify({ | ||
| state: state, | ||
| target_url: `http://material2-screenshots.firebaseapp.com/${prNumber}`, | ||
| context: "screenshot-diff", | ||
| description: `Screenshot test ${state}` | ||
| }); | ||
|
|
||
| let headers = { | ||
| 'Authorization': `token ${token}`, | ||
| 'User-Agent': 'ScreenshotDiff/1.0.0', | ||
| 'Content-Type': 'application/json', | ||
| 'Content-Length': Buffer.byteLength(data) | ||
| }; | ||
|
|
||
| return new Promise((resolve, reject) => { | ||
| request({ | ||
| url: `https://api.github.com/repos/angular/material2/statuses/${sha}`, | ||
| method: 'POST', | ||
| form: data, | ||
| headers: headers | ||
| }, function (error: any, response: any, body: any){ | ||
| resolve(response.statusCode); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| function decode(value: string): string { | ||
| return value.split('').reverse().join(''); | ||
| } |