From 5b42ac364a3b602d9aa3975503faa0e17fb1bcac Mon Sep 17 00:00:00 2001 From: Calixte Denizet Date: Mon, 30 Nov 2020 18:11:28 +0100 Subject: [PATCH] Add some integration tests using puppeteer and Jasmine * run with `gulp integrationtest` --- gulpfile.js | 85 +++++++++++++++++------ test/integration-boot.js | 53 +++++++++++++++ test/integration/annotation_spec.js | 63 +++++++++++++++++ test/integration/scripting_spec.js | 66 ++++++++++++++++++ test/test.js | 101 +++++++++++++++++++++------- web/app_options.js | 5 +- 6 files changed, 326 insertions(+), 47 deletions(-) create mode 100644 test/integration-boot.js create mode 100644 test/integration/annotation_spec.js create mode 100644 test/integration/scripting_spec.js diff --git a/gulpfile.js b/gulpfile.js index 056cf229302a0..291778226e8b3 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -98,6 +98,7 @@ const DEFINES = Object.freeze({ PRODUCTION: true, SKIP_BABEL: true, TESTING: false, + ENABLE_SCRIPTING: false, // The main build targets: GENERIC: false, MOZCENTRAL: false, @@ -303,14 +304,26 @@ function checkChromePreferencesFile(chromePrefsPath, webPrefsPath) { return false; } if (webPrefsKeys.length !== chromePrefsKeys.length) { + console.log("Warning: Prefs objects haven't the same length"); return false; } - return webPrefsKeys.every(function (value, index) { - return ( - chromePrefsKeys[index] === value && - chromePrefs.properties[value].default === webPrefs[value] - ); - }); + + let ret = true; + for (let i = 0, ii = webPrefsKeys.length; i < ii; i++) { + const value = webPrefsKeys[i]; + if (chromePrefsKeys[i] !== value) { + ret = false; + console.log( + `Warning: not the same keys: ${chromePrefsKeys[i]} !== ${value}` + ); + } else if (chromePrefs.properties[value].default !== webPrefs[value]) { + ret = false; + console.log( + `Warning: not the same values: ${chromePrefs.properties[value].default} !== ${webPrefs[value]}` + ); + } + } + return ret; } function replaceWebpackRequire() { @@ -515,6 +528,9 @@ function createTestSource(testsName, bot) { case "font": args.push("--fontTest"); break; + case "integration": + args.push("--integration"); + break; default: this.emit("error", new Error("Unknown name: " + testsName)); return null; @@ -646,6 +662,7 @@ gulp.task("default_preferences-pre", function () { LIB: true, BUNDLE_VERSION: 0, // Dummy version BUNDLE_BUILD: 0, // Dummy build + ENABLE_SCRIPTING: process.env.ENABLE_SCRIPTING === "true", }), map: { "pdfjs-lib": "../pdf", @@ -1507,27 +1524,46 @@ gulp.task("testing-pre", function (done) { done(); }); +gulp.task("enable-scripting", function (done) { + process.env.ENABLE_SCRIPTING = "true"; + done(); +}); + gulp.task( "test", - gulp.series("testing-pre", "generic", "components", function () { - return streamqueue( - { objectMode: true }, - createTestSource("unit"), - createTestSource("browser") - ); - }) + gulp.series( + "enable-scripting", + "testing-pre", + "generic", + "components", + function () { + return streamqueue( + { objectMode: true }, + createTestSource("unit"), + createTestSource("browser"), + createTestSource("integration") + ); + } + ) ); gulp.task( "bottest", - gulp.series("testing-pre", "generic", "components", function () { - return streamqueue( - { objectMode: true }, - createTestSource("unit", true), - createTestSource("font", true), - createTestSource("browser (no reftest)", true) - ); - }) + gulp.series( + "enable-scripting", + "testing-pre", + "generic", + "components", + function () { + return streamqueue( + { objectMode: true }, + createTestSource("unit", true), + createTestSource("font", true), + createTestSource("browser (no reftest)", true), + createTestSource("integration") + ); + } + ) ); gulp.task( @@ -1545,6 +1581,13 @@ gulp.task( }) ); +gulp.task( + "integrationtest", + gulp.series("enable-scripting", "testing-pre", "generic", function () { + return createTestSource("integration"); + }) +); + gulp.task( "fonttest", gulp.series("testing-pre", function () { diff --git a/test/integration-boot.js b/test/integration-boot.js new file mode 100644 index 0000000000000..e7559ee455d6e --- /dev/null +++ b/test/integration-boot.js @@ -0,0 +1,53 @@ +/* Copyright 2020 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +"use strict"; + +const Jasmine = require("jasmine"); + +async function runTests(results) { + const jasmine = new Jasmine(); + jasmine.jasmine.DEFAULT_TIMEOUT_INTERVAL = 30000; + + jasmine.loadConfig({ + random: false, + spec_dir: "integration", + spec_files: ["scripting_spec.js", "annotation_spec.js"], + }); + + jasmine.addReporter({ + jasmineDone(suiteInfo) {}, + jasmineStarted(suiteInfo) {}, + specDone(result) { + ++results.runs; + if (result.failedExpectations.length > 0) { + ++results.failures; + console.log(`TEST-UNEXPECTED-FAIL | ${result.description}`); + } else { + console.log(`TEST-PASSED | ${result.description}`); + } + }, + specStarted(result) {}, + suiteDone(result) {}, + suiteStarted(result) {}, + }); + + return new Promise(resolve => { + jasmine.onComplete(resolve); + jasmine.execute(); + }); +} + +exports.runTests = runTests; diff --git a/test/integration/annotation_spec.js b/test/integration/annotation_spec.js new file mode 100644 index 0000000000000..ba798461e763b --- /dev/null +++ b/test/integration/annotation_spec.js @@ -0,0 +1,63 @@ +/* Copyright 2020 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +describe("Annotation highlight", () => { + describe("annotation-highlight.pdf", () => { + let pages; + + beforeAll(async () => { + pages = await Promise.all( + global.integrationSessions.map(async session => { + const page = await session.browser.newPage(); + await page.goto( + `${global.integrationBaseUrl}?file=/test/pdfs/annotation-highlight.pdf` + ); + await page.bringToFront(); + await page.waitForSelector("[data-annotation-id='19R']", { + timeout: 0, + }); + return page; + }) + ); + }); + + afterAll(async () => { + await Promise.all( + pages.map(async page => { + await page.close(); + }) + ); + }); + + it("must show a popup on mouseover", async () => { + await Promise.all( + pages.map(async page => { + let hidden = await page.$eval( + "[data-annotation-id='21R']", + el => el.hidden + ); + expect(hidden).toEqual(true); + await page.hover("[data-annotation-id='19R']"); + await page.waitForTimeout(100); + hidden = await page.$eval( + "[data-annotation-id='21R']", + el => el.hidden + ); + expect(hidden).toEqual(false); + }) + ); + }); + }); +}); diff --git a/test/integration/scripting_spec.js b/test/integration/scripting_spec.js new file mode 100644 index 0000000000000..67835bfbb1d06 --- /dev/null +++ b/test/integration/scripting_spec.js @@ -0,0 +1,66 @@ +/* Copyright 2020 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +describe("Interaction", () => { + describe("in 160F-2019.pdf", () => { + let pages; + + beforeAll(async () => { + pages = await Promise.all( + global.integrationSessions.map(async session => { + const page = await session.browser.newPage(); + await page.goto( + `${global.integrationBaseUrl}?file=/test/pdfs/160F-2019.pdf` + ); + await page.bringToFront(); + await page.waitForSelector("#\\34 16R", { + timeout: 0, + }); + return [session.name, page]; + }) + ); + }); + + afterAll(async () => { + await Promise.all( + pages.map(async ([_, page]) => { + await page.close(); + }) + ); + }); + + it("must format the field with 2 digits and leave field with a click", async () => { + await Promise.all( + pages.map(async ([name, page]) => { + await page.type("#\\34 16R", "3.14159", { delay: 200 }); + await page.click("#\\34 19R"); + const text = await page.$eval("#\\34 16R", el => el.value); + expect(text).withContext(`In ${name}`).toEqual("3,14"); + }) + ); + }); + + it("must format the field with 2 digits and leave field with a TAB", async () => { + await Promise.all( + pages.map(async ([name, page]) => { + await page.type("#\\34 22R", "2.7182818", { delay: 200 }); + await page.keyboard.press("Tab"); + const text = await page.$eval("#\\34 22R", el => el.value); + expect(text).withContext(`In ${name}`).toEqual("2,72"); + }) + ); + }); + }); +}); diff --git a/test/test.js b/test/test.js index b05a7e70845ba..63e2a5be5d06f 100644 --- a/test/test.js +++ b/test/test.js @@ -254,7 +254,7 @@ function startRefTest(masterMode, showRefImages) { onAllSessionsClosed = finalize; const startUrl = `http://${host}:${server.port}/test/test_slave.html`; - startBrowsers(startUrl, function (session) { + startBrowsers(function (session) { session.masterMode = masterMode; session.taskResults = {}; session.tasks = {}; @@ -271,7 +271,7 @@ function startRefTest(masterMode, showRefImages) { session.numEqNoSnapshot = 0; session.numEqFailures = 0; monitorBrowserTimeout(session, handleSessionTimeout); - }); + }, makeTestUrl(startUrl)); } function checkRefsTmp() { if (masterMode && fs.existsSync(refsTmpDir)) { @@ -670,11 +670,9 @@ function refTestPostHandler(req, res) { return true; } -function startUnitTest(testUrl, name) { - var startTime = Date.now(); - startServer(); - server.hooks.POST.push(unitTestPostHandler); - onAllSessionsClosed = function () { +function onAllSessionsClosedAfterTests(name) { + const startTime = Date.now(); + return function () { stopServer(); var numRuns = 0, numErrors = 0; @@ -693,12 +691,53 @@ function startUnitTest(testUrl, name) { var runtime = (Date.now() - startTime) / 1000; console.log(name + " tests runtime was " + runtime.toFixed(1) + " seconds"); }; +} + +function makeTestUrl(startUrl) { + return function (browserName) { + const queryParameters = + `?browser=${encodeURIComponent(browserName)}` + + `&manifestFile=${encodeURIComponent("/test/" + options.manifestFile)}` + + `&testFilter=${JSON.stringify(options.testfilter)}` + + `&delay=${options.statsDelay}` + + `&masterMode=${options.masterMode}`; + return startUrl + queryParameters; + }; +} + +function startUnitTest(testUrl, name) { + onAllSessionsClosed = onAllSessionsClosedAfterTests(name); + startServer(); + server.hooks.POST.push(unitTestPostHandler); const startUrl = `http://${host}:${server.port}${testUrl}`; - startBrowsers(startUrl, function (session) { + startBrowsers(function (session) { + session.numRuns = 0; + session.numErrors = 0; + }, makeTestUrl(startUrl)); +} + +function startIntegrationTest() { + onAllSessionsClosed = onAllSessionsClosedAfterTests("integration"); + startServer(); + + const { runTests } = require("./integration-boot.js"); + startBrowsers(function (session) { session.numRuns = 0; session.numErrors = 0; }); + global.integrationBaseUrl = `http://${host}:${server.port}/build/generic/web/viewer.html`; + global.integrationSessions = sessions; + + Promise.all(sessions.map(session => session.browserPromise)).then( + async () => { + const results = { runs: 0, failures: 0 }; + await runTests(results); + sessions[0].numRuns = results.runs; + sessions[0].numErrors = results.failures; + await Promise.all(sessions.map(session => closeSession(session.name))); + } + ); } function unitTestPostHandler(req, res) { @@ -768,7 +807,7 @@ function unitTestPostHandler(req, res) { return true; } -async function startBrowser(browserName, startUrl) { +async function startBrowser(browserName, startUrl = "") { const revisions = require("puppeteer/lib/cjs/puppeteer/revisions.js") .PUPPETEER_REVISIONS; const wantedRevision = @@ -790,18 +829,37 @@ async function startBrowser(browserName, startUrl) { } } - const browser = await puppeteer.launch({ + const options = { product: browserName, headless: false, defaultViewport: null, - }); - const pages = await browser.pages(); - const page = pages[0]; - await page.goto(startUrl, { timeout: 0 }); + ignoreDefaultArgs: ["--disable-extensions"], + }; + + if (browserName === "chrome") { + // avoid crash + options.args = ["--no-sandbox", "--disable-setuid-sandbox"]; + } + + if (browserName === "firefox") { + options.extraPrefsFirefox = { + // avoid to have a prompt when leaving a page with a form + "dom.disable_beforeunload": true, + }; + } + + const browser = await puppeteer.launch(options); + + if (startUrl) { + const pages = await browser.pages(); + const page = pages[0]; + await page.goto(startUrl, { timeout: 0 }); + } + return browser; } -function startBrowsers(rootUrl, initSessionCallback) { +function startBrowsers(initSessionCallback, makeStartUrl = null) { const browserNames = options.noChrome ? ["firefox"] : ["firefox", "chrome"]; sessions = []; @@ -820,16 +878,9 @@ function startBrowsers(rootUrl, initSessionCallback) { closed: false, }; sessions.push(session); + const startUrl = makeStartUrl ? makeStartUrl(browserName) : ""; - const queryParameters = - `?browser=${encodeURIComponent(browserName)}` + - `&manifestFile=${encodeURIComponent("/test/" + options.manifestFile)}` + - `&testFilter=${JSON.stringify(options.testfilter)}` + - `&delay=${options.statsDelay}` + - `&masterMode=${options.masterMode}`; - const startUrl = rootUrl + queryParameters; - - startBrowser(browserName, startUrl) + session.browserPromise = startBrowser(browserName, startUrl) .then(function (browser) { session.browser = browser; if (initSessionCallback) { @@ -920,6 +971,8 @@ function main() { }); } else if (options.fontTest) { startUnitTest("/test/font/font_test.html", "font"); + } else if (options.integration) { + startIntegrationTest(); } else { startRefTest(options.masterMode, options.reftest); } diff --git a/web/app_options.js b/web/app_options.js index caf2455caed68..ac0fe1b9ebb14 100644 --- a/web/app_options.js +++ b/web/app_options.js @@ -67,7 +67,7 @@ const defaultOptions = { }, enableScripting: { /** @type {boolean} */ - value: false, + value: typeof PDFJSDev !== "undefined" && PDFJSDev.test("ENABLE_SCRIPTING"), kind: OptionKind.VIEWER + OptionKind.PREFERENCE, }, enableWebGL: { @@ -260,7 +260,8 @@ if ( defaultOptions.sandboxBundleSrc = { /** @type {string} */ value: - typeof PDFJSDev === "undefined" || !PDFJSDev.test("PRODUCTION") + typeof PDFJSDev === "undefined" || + PDFJSDev.test("!PRODUCTION && !ENABLE_SCRIPTING") ? "../build/dev-sandbox/pdf.sandbox.js" : "../build/pdf.sandbox.js", kind: OptionKind.VIEWER,