unskip dashboard_screenshot tests by using new setScreenshotSize (#46311

)

* use new setScreenshotSize

* change tolerance from 1% to 2% (originally 5%)

* retake baseline without notification dialog

* close add and save notifications

* wait for render complete when going in to FullScreen mode

* add 5 second sleep to see if it ever goes to full screen mode

* add debugging screenshots and sleeps

* try reloading the Kibana index before this test

* 30 iterations with only the first test enabled

* revert multi-run changes

* ran node scripts/build_renovate_config

* fix tslint errors

* fix tslint error

* update xpack dependency to match oss

* update yarn.lock

* revert notifications:lifetime:info change by reducing some other timeouts

* remove commented-out code

* fixed and Unskipped 2nd test

* re-arrange order of dashboard tests to pass :-(

* removed commented-out sleep

Hopefully this won't be an issue

* Add comment about the order of tests

```
      loadTestFile(require.resolve('./view_edit'));
      // Order of test suites *shouldn't* be important but there's a bug for the view_edit test above
      // #46752
      // The dashboard_snapshot test below requires the timestamped URL which breaks the view_edit test.
      // If we don't use the timestamp in the URL, the colors in the charts will be different.
      loadTestFile(require.resolve('./dashboard_snapshots'));
 ```

package.json

@@ -330,6 +330,7 @@
     "@types/mustache": "^0.8.31",
     "@types/node": "^10.12.27",
     "@types/opn": "^5.1.0",
+    "@types/pngjs": "^3.3.2",
     "@types/podium": "^1.0.0",
     "@types/prop-types": "^15.5.3",
     "@types/react": "^16.8.0",

renovate.json5

@@ -497,6 +497,14 @@
           '@types/opn',
         ],
       },
+      {
+        groupSlug: 'pngjs',
+        groupName: 'pngjs related packages',
+        packageNames: [
+          'pngjs',
+          '@types/pngjs',
+        ],
+      },
       {
         groupSlug: 'podium',
         groupName: 'podium related packages',
@@ -713,14 +721,6 @@
           '@types/papaparse',
         ],
       },
-      {
-        groupSlug: 'pngjs',
-        groupName: 'pngjs related packages',
-        packageNames: [
-          'pngjs',
-          '@types/pngjs',
-        ],
-      },
       {
         groupSlug: 'proper-lockfile',
         groupName: 'proper-lockfile related packages',

test/functional/apps/dashboard/dashboard_snapshots.js

@@ -26,21 +26,20 @@ export default function ({ getService, getPageObjects, updateBaselines }) {
   const dashboardPanelActions = getService('dashboardPanelActions');
   const dashboardAddPanel = getService('dashboardAddPanel');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/40173
-  describe.skip('dashboard snapshots', function describeIndexTests() {
+  describe('dashboard snapshots', function describeIndexTests() {
     before(async function () {
       // We use a really small window to minimize differences across os's and browsers.
-      await browser.setWindowSize(1000, 700);
+      await browser.setScreenshotSize(1000, 500);
+      // adding this navigate adds the timestamp hash to the url which invalidates previous
+      // session.  If we don't do this, the colors on the visualizations are different and the screenshots won't match.
+      await PageObjects.common.navigateToApp('dashboard');
     });
 
     after(async function () {
       await browser.setWindowSize(1300, 900);
-      const id = await PageObjects.dashboard.getDashboardIdFromCurrentUrl();
-      await PageObjects.dashboard.deleteDashboard('area', id);
     });
 
-    // Skip until https://github.com/elastic/kibana/issues/19471 is fixed
-    it.skip('compare TSVB snapshot', async () => {
+    it('compare TSVB snapshot', async () => {
       await PageObjects.dashboard.gotoDashboardLandingPage();
       await PageObjects.dashboard.clickNewDashboard();
       await PageObjects.dashboard.setTimepickerInLogstashDataRange();
@@ -49,7 +48,6 @@ export default function ({ getService, getPageObjects, updateBaselines }) {
 
       await PageObjects.dashboard.saveDashboard('tsvb');
       await PageObjects.common.closeToast();
-
       await PageObjects.dashboard.clickFullScreenMode();
       await dashboardPanelActions.openContextMenu();
       await dashboardPanelActions.clickExpandPanelToggle();
@@ -59,17 +57,18 @@ export default function ({ getService, getPageObjects, updateBaselines }) {
 
       await PageObjects.dashboard.clickExitFullScreenLogoButton();
 
-      expect(percentDifference).to.be.lessThan(0.05);
+      expect(percentDifference).to.be.lessThan(0.02);
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/40173
-    it.skip('compare area chart snapshot', async () => {
+    it('compare area chart snapshot', async () => {
       await PageObjects.dashboard.gotoDashboardLandingPage();
       await PageObjects.dashboard.clickNewDashboard();
       await PageObjects.dashboard.setTimepickerInLogstashDataRange();
       await dashboardAddPanel.addVisualization('Rendering Test: area with not filter');
-      await PageObjects.dashboard.saveDashboard('area');
+      await PageObjects.common.closeToast();
 
+      await PageObjects.dashboard.saveDashboard('area');
+      await PageObjects.common.closeToast();
       await PageObjects.dashboard.clickFullScreenMode();
       await dashboardPanelActions.openContextMenu();
       await dashboardPanelActions.clickExpandPanelToggle();
@@ -79,8 +78,7 @@ export default function ({ getService, getPageObjects, updateBaselines }) {
 
       await PageObjects.dashboard.clickExitFullScreenLogoButton();
 
-      // Testing some OS/browser differences were shown to cause .009 percent difference.
-      expect(percentDifference).to.be.lessThan(0.05);
+      expect(percentDifference).to.be.lessThan(0.02);
     });
   });
 }

test/functional/apps/dashboard/index.js

@@ -70,8 +70,12 @@ export default function ({ getService, loadTestFile, getPageObjects }) {
       loadTestFile(require.resolve('./dashboard_filtering'));
       loadTestFile(require.resolve('./panel_expand_toggle'));
       loadTestFile(require.resolve('./dashboard_grid'));
-      loadTestFile(require.resolve('./dashboard_snapshots'));
       loadTestFile(require.resolve('./view_edit'));
+      // Order of test suites *shouldn't* be important but there's a bug for the view_edit test above
+      // https://github.com/elastic/kibana/issues/46752
+      // The dashboard_snapshot test below requires the timestamped URL which breaks the view_edit test.
+      // If we don't use the timestamp in the URL, the colors in the charts will be different.
+      loadTestFile(require.resolve('./dashboard_snapshots'));
     });
 
     // Each of these tests call initTests themselves, the way it was originally written.  The above tests only load

test/functional/config.js

@@ -54,7 +54,7 @@ export default async function ({ readConfigFile }) {
       defaults: {
         'accessibility:disableAnimations': true,
         'dateFormat:tz': 'UTC',
-        'telemetry:optIn': false
+        'telemetry:optIn': false,
       },
     },
 

test/functional/page_objects/dashboard_page.js

@@ -72,6 +72,8 @@ export function DashboardPageProvider({ getService, getPageObjects }) {
     async clickFullScreenMode() {
       log.debug(`clickFullScreenMode`);
       await testSubjects.click('dashboardFullScreenMode');
+      await testSubjects.exists('exitFullScreenModeLogo');
+      await this.waitForRenderComplete();
     }
 
     async fullScreenModeMenuItemExists() {
@@ -96,10 +98,12 @@ export function DashboardPageProvider({ getService, getPageObjects }) {
 
     async clickExitFullScreenLogoButton() {
       await testSubjects.click('exitFullScreenModeLogo');
+      await this.waitForRenderComplete();
     }
 
     async clickExitFullScreenTextButton() {
       await testSubjects.click('exitFullScreenModeText');
+      await this.waitForRenderComplete();
     }
 
     async getDashboardIdFromCurrentUrl() {

test/functional/page_objects/header_page.js

@@ -79,7 +79,7 @@ export function HeaderPageProvider({ getService, getPageObjects }) {
 
     async isGlobalLoadingIndicatorVisible() {
       log.debug('isGlobalLoadingIndicatorVisible');
-      return await testSubjects.exists('globalLoadingIndicator');
+      return await testSubjects.exists('globalLoadingIndicator', { timeout: 1500 });
     }
 
     async awaitGlobalLoadingIndicatorHidden() {

test/functional/services/browser.ts

@@ -21,6 +21,7 @@ import { cloneDeep } from 'lodash';
 import { IKey, logging } from 'selenium-webdriver';
 import { takeUntil } from 'rxjs/operators';
 
+import Jimp, { Bitmap } from 'jimp';
 import { modifyUrl } from '../../../src/core/utils';
 import { WebElementWrapper } from './lib/web_element_wrapper';
 import { FtrProviderContext } from '../ftr_provider_context';
@@ -118,6 +119,60 @@ export async function BrowserProvider({ getService }: FtrProviderContext) {
       await (driver.manage().window() as any).setRect({ width: args[0], height: args[1] });
     }
 
+    /**
+     * Gets a screenshot of the focused window and returns it as a Bitmap object
+     */
+    public async getScreenshotAsBitmap(): Promise<Bitmap> {
+      const screenshot = await this.takeScreenshot();
+      const buffer = Buffer.from(screenshot.toString(), 'base64');
+      const session = (await Jimp.read(buffer)).clone();
+      return session.bitmap;
+    }
+
+    /**
+     * Sets the dimensions of a window to get the right size screenshot.
+     *
+     * @param {number} width
+     * @param {number} height
+     * @return {Promise<void>}
+     */
+    public async setScreenshotSize(width: number, height: number): Promise<void> {
+      log.debug(`======browser======== setWindowSize ${width} ${height}`);
+      // We really want to set the Kibana app to a specific size without regard to the browser chrome (borders)
+      // But that means we first need to figure out the display scaling factor.
+      // NOTE: None of this is required when running Chrome headless because there's no scaling and no borders.
+      await this.setWindowSize(1200, 800);
+      const bitmap1 = await this.getScreenshotAsBitmap();
+      log.debug(
+        `======browser======== actual initial screenshot size width=${bitmap1.width}, height=${bitmap1.height}`
+      );
+
+      // drasticly change the window size so we can calculate the scaling
+      await this.setWindowSize(600, 400);
+      const bitmap2 = await this.getScreenshotAsBitmap();
+      log.debug(
+        `======browser======== actual second screenshot size width= ${bitmap2.width}, height=${bitmap2.height}`
+      );
+
+      const xScaling = (bitmap1.width - bitmap2.width) / 600;
+      const yScaling = (bitmap1.height - bitmap2.height) / 400;
+      const xBorder = Math.round(600 - bitmap2.width / xScaling);
+      const yBorder = Math.round(400 - bitmap2.height / yScaling);
+      log.debug(
+        `======browser======== calculated values xBorder= ${xBorder}, yBorder=${yBorder}, xScaling=${xScaling}, yScaling=${yScaling}`
+      );
+      log.debug(
+        `======browser======== setting browser size to ${width + xBorder} x ${height + yBorder}`
+      );
+      await this.setWindowSize(width + xBorder, height + yBorder);
+
+      const bitmap3 = await this.getScreenshotAsBitmap();
+      // when there is display scaling this won't show the expected size.  It will show expected size * scaling factor
+      log.debug(
+        `======browser======== final screenshot size width=${bitmap3.width}, height=${bitmap3.height}`
+      );
+    }
+
     /**
      * Gets the URL that is loaded in the focused window/frame.
      * https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/lib/webdriver_exports_WebDriver.html#getCurrentUrl

test/functional/services/flyout.ts

@@ -31,11 +31,14 @@ export function FlyoutProvider({ getService }: FtrProviderContext) {
       const flyoutElement = await testSubjects.find(dataTestSubj);
       const closeBtn = await flyoutElement.findByCssSelector('[aria-label*="Close"]');
       await closeBtn.click();
-      await retry.waitFor('flyout closed', async () => !(await testSubjects.exists(dataTestSubj)));
+      await retry.waitFor(
+        'flyout closed',
+        async () => !(await testSubjects.exists(dataTestSubj, { timeout: 1000 }))
+      );
     }
 
     public async ensureClosed(dataTestSubj: string): Promise<void> {
-      if (await testSubjects.exists(dataTestSubj)) {
+      if (await testSubjects.exists(dataTestSubj, { timeout: 1000 })) {
         await this.close(dataTestSubj);
       }
     }

test/functional/services/lib/web_element_wrapper/web_element_wrapper.ts

@@ -19,7 +19,6 @@
 
 import { delay } from 'bluebird';
 import { WebElement, WebDriver, By, IKey, until } from 'selenium-webdriver';
-// @ts-ignore not supported yet
 import { PNG } from 'pngjs';
 // @ts-ignore not supported yet
 import cheerio from 'cheerio';
@@ -743,7 +742,7 @@ export class WebElementWrapper {
    *
    * @returns {Promise<void>}
    */
-  public async takeScreenshot(): Promise<void> {
+  public async takeScreenshot(): Promise<Buffer> {
     const screenshot = await this.driver.takeScreenshot();
     const buffer = Buffer.from(screenshot.toString(), 'base64');
     const { width, height, x, y } = await this.getPosition();
@@ -757,11 +756,12 @@ export class WebElementWrapper {
       src.height = src.height / 2;
       let h = false;
       let v = false;
-      src.data = src.data.filter((d: any, i: number) => {
+      const filteredData = src.data.filter((d: any, i: number) => {
         h = i % 4 ? h : !h;
         v = i % (src.width * 2 * 4) ? v : !v;
         return h && v;
       });
+      src.data = Buffer.from(filteredData);
     }
     const dst = new PNG({ width, height });
     PNG.bitblt(src, dst, x, y, width, height, 0, 0);

test/scripts/jenkins_build_kibana.sh

@@ -10,4 +10,3 @@ yarn run grunt functionalTests:ensureAllTestsInCiGroup;
 
 echo " -> building and extracting OSS Kibana distributable for use in functional tests"
 node scripts/build --debug --oss
-

x-pack/package.json

@@ -79,7 +79,7 @@
     "@types/nodemailer": "^6.2.1",
     "@types/object-hash": "^1.3.0",
     "@types/papaparse": "^4.5.11",
-    "@types/pngjs": "^3.3.1",
+    "@types/pngjs": "^3.3.2",
     "@types/prop-types": "^15.5.3",
     "@types/proper-lockfile": "^3.0.1",
     "@types/puppeteer": "^1.19.0",

yarn.lock

@@ -3670,10 +3670,10 @@
   dependencies:
     "@types/node" "*"
 
-"@types/pngjs@^3.3.1":
-  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-3.3.1.tgz#47d97bd29dd6372856050e9e5e366517dd1ba2d8"
-  integrity sha512-kcYYpvggAVtJmVp8ojGWkYGi1Q2lJS787f16IOfd7/fUJ0yUasn9XnxIRWQ/gY8eID8MV9h4IgpJU3j/UNJ6QQ==
+"@types/pngjs@^3.3.2":
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-3.3.2.tgz#8ed3bd655ab3a92ea32ada7a21f618e63b93b1d4"
+  integrity sha512-/SBsv93rVnjByzcau24rBwb+N7BHFp2LateaXz1e7m7M0Wzck/ymXTNdWVrCtkuMbwTHAnfdc3X/I/5szsTEAA==
   dependencies:
     "@types/node" "*"