Fix template scale in find element by image

lib/basedriver/commands/find.js

@@ -4,14 +4,18 @@ import _ from 'lodash';
 import { errors } from '../../..';
 import { MATCH_TEMPLATE_MODE } from './images';
 import { W3C_ELEMENT_KEY, MJSONWP_ELEMENT_KEY } from '../../protocol/protocol';
-import { ImageElement } from '../image-element';
+import { ImageElement, DEFAULT_TEMPLATE_IMAGE_SCALE } from '../image-element';
 
 
 const commands = {}, helpers = {}, extensions = {};
 
 const IMAGE_STRATEGY = '-image';
 const CUSTOM_STRATEGY = '-custom';
 
+// Used to compare ratio and screen width
+// Pixel is basically under 1080 for example. 100K is probably enough fo a while.
+const FLOAT_PRECISION = 100000;
+
 // Override the following function for your own driver, and the rest is taken
 // care of!
 
@@ -183,6 +187,9 @@ commands.findByCustom = async function (selector, multiple) {
  * image is merely to check staleness. If so we can bypass a lot of logic
  * @property {boolean} [multiple=false] - Whether we are finding one element or
  * multiple
+ * @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we
+ * ignore defaultImageTemplateScale. It can be used when you would like to
+ * scale b64Template with defaultImageTemplateScale setting.
  */
 
 /**
@@ -198,11 +205,13 @@ commands.findByCustom = async function (selector, multiple) {
 helpers.findByImage = async function (b64Template, {
  shouldCheckStaleness = false,
  multiple = false,
+ ignoreDefaultImageTemplateScale = false,
 }) {
  const {
  imageMatchThreshold: threshold,
  fixImageTemplateSize,
- fixImageTemplateScale
+ fixImageTemplateScale,
+ defaultImageTemplateScale
  } = this.settings.getSettings();
 
  log.info(`Finding image element with match threshold ${threshold}`);
@@ -225,9 +234,10 @@ helpers.findByImage = async function (b64Template, {
  try {
  const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);
 
- if (fixImageTemplateScale) {
- b64Template = await this.fixImageTemplateScale(b64Template, scale);
- }
+ b64Template = await this.fixImageTemplateScale(b64Template, {
+ defaultImageTemplateScale, ignoreDefaultImageTemplateScale,
+ fixImageTemplateScale, ...scale
+ });
 
  rect = (await this.compareImages(MATCH_TEMPLATE_MODE, b64Screenshot,
  b64Template, {threshold})).rect;
@@ -357,42 +367,126 @@ helpers.getScreenshotForImageFind = async function (screenWidth, screenHeight) {
  // of coordinates returned by the image match algorithm, since we match based
  // on the screenshot coordinates not the device coordinates themselves. There
  // are two potential types of mismatch: aspect ratio mismatch and scale
- // mismatch.
+ // mismatch. We need to detect and fix both
+
+ const scale = {xScale: 1.0, yScale: 1.0};
+
+ const screenAR = screenWidth / screenHeight;
+ const shotAR = shotWidth / shotHeight;
+ if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {
+ log.info(`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +
+ `screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`);
+ } else {
+ log.warn(`When trying to find an element, determined that the screen ` +
+ `aspect ratio and screenshot aspect ratio are different. Screen ` +
+ `is ${screenWidth}x${screenHeight} whereas screenshot is ` +
+ `${shotWidth}x${shotHeight}.`);
+
+ // In the case where the x-scale and y-scale are different, we need to decide
+ // which one to respect, otherwise the screenshot and template will end up
+ // being resized in a way that changes its aspect ratio (distorts it). For example, let's say:
+ // this.getScreenshot(shotWidth, shotHeight) is 540x397,
+ // this.getDeviceSize(screenWidth, screenHeight) is 1080x1920.
+ // The ratio would then be {xScale: 0.5, yScale: 0.2}.
+ // In this case, we must should `yScale: 0.2` as scaleFactor, because
+ // if we select the xScale, the height will be bigger than real screenshot size
+ // which is used to image comparison by OpenCV as a base image.
+ // All of this is primarily useful when the screenshot is a horizontal slice taken out of the
+ // screen (for example not including top/bottom nav bars)
+ const xScale = (1.0 * shotWidth) / screenWidth;
+ const yScale = (1.0 * shotHeight) / screenHeight;
+ const scaleFactor = xScale >= yScale ? yScale : xScale;
+
+ log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +
+ `screen aspect ratio so that image element coordinates have a ` +
+ `greater chance of being correct.`);
+ imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);
+
+ scale.xScale *= scaleFactor;
+ scale.yScale *= scaleFactor;
+
+ shotWidth = imgObj.bitmap.width;
+ shotHeight = imgObj.bitmap.height;
+ }
 
  // Resize based on the screen dimensions only if both width and height are mismatched
  // since except for that, it might be a situation which is different window rect and
  // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and
  // `"deviceScreenSize"=>"1080x1920"`
- let scale;
  if (screenWidth !== shotWidth && screenHeight !== shotHeight) {
  log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +
  `screen at ${screenWidth}x${screenHeight}`);
  imgObj = imgObj.resize(screenWidth, screenHeight);
 
- scale = {xScale: (1.0 * screenWidth) / shotWidth, yScale: (1.0 * screenHeight) / shotHeight};
+ scale.xScale *= (1.0 * screenWidth) / shotWidth;
+ scale.yScale *= (1.0 * screenHeight) / shotHeight;
  }
 
  b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
  return {b64Screenshot, scale};
 };
 
+/**
+ * @typedef {Object} ImageTemplateSettings
+ * @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings
+ * @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings
+ * @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.
+ * If b64Template has been scaled to defaultImageTemplateScale or should ignore the scale,
+ * this parameter should be true. e.g. click in image-element module
+ * @property {float} xScale - Scale ratio for width
+ * @property {float} yScale - Scale ratio for height
+
+ */
 /**
  * Get a image that will be used for template maching.
  * Returns scaled image if scale ratio is provided.
  *
+ *
  * @param {string} b64Template - base64-encoded image used as a template to be
  * matched in the screenshot
- * @param {ScreenshotScale} scale - scale of screen. Default to `{}`
+ * @param {ImageTemplateSettings} opts - Image template scale related options
  *
  * @returns {string} base64-encoded scaled template screenshot
  */
-helpers.fixImageTemplateScale = async function (b64Template, scale = {}) {
- if (!scale) {
+const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
+helpers.fixImageTemplateScale = async function (b64Template, opts = {}) {
+ if (!opts) {
+ return b64Template;
+ }
+
+ let {
+ fixImageTemplateScale = false,
+ defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE,
+ ignoreDefaultImageTemplateScale = false,
+ xScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
+ yScale = DEFAULT_FIX_IMAGE_TEMPLATE_SCALE
+ } = opts;
+
+ if (ignoreDefaultImageTemplateScale) {
+ defaultImageTemplateScale = DEFAULT_TEMPLATE_IMAGE_SCALE;
+ }
+
+ // Default
+ if (defaultImageTemplateScale === DEFAULT_TEMPLATE_IMAGE_SCALE && !fixImageTemplateScale) {
+ return b64Template;
+ }
+
+ // Calculate xScale and yScale Appium should scale
+ if (fixImageTemplateScale) {
+ xScale *= defaultImageTemplateScale;
+ yScale *= defaultImageTemplateScale;
+ } else {
+ xScale = yScale = 1 * defaultImageTemplateScale;
+ }
+
+ // xScale and yScale can be NaN if defaultImageTemplateScale is string, for example
+ if (!parseFloat(xScale) || !parseFloat(yScale)) {
  return b64Template;
  }
 
- const {xScale, yScale} = scale;
- if (!xScale || !yScale) {
+ // Return if the scale is default, 1, value
+ if (Math.round(xScale * FLOAT_PRECISION) === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION)
+ && Math.round(yScale * FLOAT_PRECISION === Math.round(DEFAULT_FIX_IMAGE_TEMPLATE_SCALE * FLOAT_PRECISION))) {
  return b64Template;
  }
 

lib/basedriver/device-settings.js

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 import log from './logger';
 import { DEFAULT_MATCH_THRESHOLD } from './commands/images';
-import { IMAGE_EL_TAP_STRATEGY_W3C } from './image-element';
+import { IMAGE_EL_TAP_STRATEGY_W3C, DEFAULT_TEMPLATE_IMAGE_SCALE } from './image-element';
 
 const GLOBAL_DEFAULT_SETTINGS = {
  // value between 0 and 1 representing match strength, below which an image
@@ -25,6 +25,14 @@ const GLOBAL_DEFAULT_SETTINGS = {
  // if a user use `width=750 × height=1334` pixels's base template image.
  fixImageTemplateScale: false,
 
+ // Users might have scaled template image to reduce their storage size.
+ // This setting allows users to scale a template image they send to Appium server
+ // so that the Appium server compares the actual scale users originally had.
+ // e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,
+ // the user can set {defaultImageTemplateScale: 4.0} to scale the small image
+ // to the original one so that Appium can compare it as the original one.
+ defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
+
  // whether Appium should re-check that an image element can be matched
  // against the current screenshot before clicking it
  checkForImageElementStaleness: true,
@@ -44,6 +52,8 @@ const BASEDRIVER_HANDLED_SETTINGS = [
  'imageMatchThreshold',
  'fixImageFindScreenshotDims',
  'fixImageTemplateSize',
+ 'fixImageTemplateScale',
+ 'defaultImageTemplateScale',
  'checkForImageElementStaleness',
  'autoUpdateImageElementPosition',
  'imageElementTapStrategy',

lib/basedriver/image-element.js

@@ -13,6 +13,7 @@ const IMAGE_TAP_STRATEGIES = [
  IMAGE_EL_TAP_STRATEGY_MJSONWP,
  IMAGE_EL_TAP_STRATEGY_W3C
 ];
+const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
 
 /**
  * @typedef {Object} Rect
@@ -127,7 +128,10 @@ class ImageElement {
  log.info('Checking image element for staleness before clicking');
  try {
  newImgEl = await driver.findByImage(this.template, {
- shouldCheckStaleness: true
+ shouldCheckStaleness: true,
+ // Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
+ // managed inside Appium after finidng image by template which managed by a user
+ ignoreDefaultImageTemplateScale: true
  });
  } catch (err) {
  throw new errors.StaleElementReferenceError();
@@ -246,4 +250,5 @@ function getImgElFromArgs (args) {
 export {
  ImageElement, getImgElFromArgs, makeImageElementCache,
  IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C,
+ DEFAULT_TEMPLATE_IMAGE_SCALE
 };