Skip to content
This repository has been archived by the owner on Apr 25, 2023. It is now read-only.

Commit

Permalink
Highlight elements that user searched for (#344)
Browse files Browse the repository at this point in the history 
When user selects an element in the 'Search Elements' modal it highlights it on the screenshot.
  • Loading branch information
@dpgraham
dpgraham authored Nov 21, 2017
1 parent 7216e1b commit f2c7ad4
Show file tree
Hide file tree
Showing 7 changed files with 114 additions and 60 deletions.
69 changes: 35 additions & 34 deletions app/main/appium-method-handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,42 @@ export default class AppiumMethodHandler {
return {variableName, variableType, strategy, selector, elements};
}

async executeElementCommand (elementId, methodName, args = []) {
const cachedEl = this.elementCache[elementId];
async _execute ({elementId, methodName, args, skipScreenshotAndSource}) {
let cachedEl;
let res = {};

// Give the cached element a variable name (el1, el2, el3,...) the first time it's used
if (!cachedEl.variableName && cachedEl.variableType === 'string') {
cachedEl.variableName = `el${this.elVariableCounter++}`;
if (elementId) {
// Give the cached element a variable name (el1, el2, el3,...) the first time it's used
cachedEl = this.elementCache[elementId];
if (!cachedEl.variableName && cachedEl.variableType === 'string') {
cachedEl.variableName = `el${this.elVariableCounter++}`;
}
res = await cachedEl.el[methodName].apply(cachedEl.el, args);
} else {
// Specially handle the tap and swipe method
if (methodName === 'tap') {
res = await (new wd.TouchAction(this.driver))
.tap({x: args[0], y: args[1]})
.perform();
} else if (methodName === 'swipe') {
const [startX, startY, endX, endY] = args;
res = await (new wd.TouchAction(this.driver))
.press({x: startX, y: startY})
.moveTo({x: endX, y: endY})
.release()
.perform();
} else if (methodName !== 'source' && methodName !== 'screenshot') {
res = await this.driver[methodName].apply(this.driver, args);
}
}
const res = await cachedEl.el[methodName].apply(cachedEl.el, args);

// Give the source/screenshot time to change before taking the screenshot
await Bluebird.delay(500);

let sourceAndScreenshot = await this._getSourceAndScreenshot();
let sourceAndScreenshot;
if (!skipScreenshotAndSource) {
sourceAndScreenshot = await this._getSourceAndScreenshot();
}

return {
...sourceAndScreenshot,
Expand All @@ -79,34 +102,12 @@ export default class AppiumMethodHandler {
};
}

async executeMethod (methodName, args = []) {
let res = {};

// Specially handle the tap and swipe method
if (methodName === 'tap') {
res = await (new wd.TouchAction(this.driver))
.tap({x: args[0], y: args[1]})
.perform();
} else if (methodName === 'swipe') {
const [startX, startY, endX, endY] = args;
res = await (new wd.TouchAction(this.driver))
.press({x: startX, y: startY})
.moveTo({x: endX, y: endY})
.release()
.perform();
} else if (methodName !== 'source' && methodName !== 'screenshot') {
res = await this.driver[methodName].apply(this.driver, args);
}

// Give the source/screenshot time to change before taking the screenshot
await Bluebird.delay(500);

let sourceAndScreenshot = await this._getSourceAndScreenshot();
async executeElementCommand (elementId, methodName, args = [], skipScreenshotAndSource = false) {
return await this._execute({elementId, methodName, args, skipScreenshotAndSource});
}

return {
...sourceAndScreenshot,
res,
};
async executeMethod (methodName, args = [], skipScreenshotAndSource = false) {
return await this._execute({methodName, args, skipScreenshotAndSource});
}

async _getSourceAndScreenshot () {
Expand Down
7 changes: 4 additions & 3 deletions app/main/appium.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ function connectClientMethodListener () {
fetchArray = false, // Optional. Are we fetching an array of elements or just one?
elementId, // Optional. Element being operated on
args = [], // Optional. Arguments passed to method
skipScreenshotAndSource = false, // Optional. Do we want the updated source and screenshot?
} = data;

let renderer = evt.sender;
Expand All @@ -322,15 +323,15 @@ function connectClientMethodListener () {
if (methodName) {
if (elementId) {
console.log(`Handling client method request with method '${methodName}', args ${JSON.stringify(args)} and elementId ${elementId}`);
res = await methodHandler.executeElementCommand(elementId, methodName, args);
res = await methodHandler.executeElementCommand(elementId, methodName, args, skipScreenshotAndSource);
} else {
console.log(`Handling client method request with method '${methodName}' and args ${JSON.stringify(args)}`);
res = await methodHandler.executeMethod(methodName, args);
res = await methodHandler.executeMethod(methodName, args, skipScreenshotAndSource);
}
} else if (strategy && selector) {
if (fetchArray) {
console.log(`Fetching elements with selector '${selector}' and strategy ${strategy}`);
res = await methodHandler.fetchElements(strategy, selector);
res = await methodHandler.fetchElements(strategy, selector, skipScreenshotAndSource);
} else {
console.log(`Fetching an element with selector '${selector}' and strategy ${strategy}`);
res = await methodHandler.fetchElement(strategy, selector);
Expand Down
19 changes: 16 additions & 3 deletions app/renderer/actions/Inspector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ipcRenderer } from 'electron';
import { notification } from 'antd';
import { push } from 'react-router-redux';
import _ from 'lodash';
import B from 'bluebird';
import { getLocators } from '../components/Inspector/shared';
import { showError } from './Session';
import { callClientMethod } from './shared';
Expand Down Expand Up @@ -46,11 +47,14 @@ export const CLEAR_SEARCH_RESULTS = 'CLEAR_SEARCH_RESULTS';
export const ADD_ASSIGNED_VAR_CACHE = 'ADD_ASSIGNED_VAR_CACHE';
export const CLEAR_ASSIGNED_VAR_CACHE = 'CLEAR_ASSIGNED_VAR_CACHE';
export const SET_SCREENSHOT_INTERACTION_MODE = 'SET_SCREENSHOT_INTERACTION_MODE';
export const SET_SEARCHED_FOR_ELEMENT_BOUNDS = 'SET_SEARCHED_FOR_ELEMENT_BOUNDS';
export const CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS = 'CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS';

export const SET_SWIPE_START = 'SET_SWIPE_START';
export const SET_SWIPE_END = 'SET_SWIPE_END';
export const CLEAR_SWIPE_ACTION = 'CLEAR_SWIPE_ACTION';


// Attributes on nodes that we know are unique to the node
const uniqueAttributes = [
'name',
Expand Down Expand Up @@ -335,6 +339,7 @@ export function showLocatorTestModal () {
export function hideLocatorTestModal () {
return (dispatch) => {
dispatch({type: HIDE_LOCATOR_TEST_MODAL});
dispatch({type: CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS});
};
}

Expand Down Expand Up @@ -377,11 +382,19 @@ export function findAndAssign (strategy, selector, variableName, isArray) {
};
}


// TODO: Is this obsolete? I don't think we use this.
export function setLocatorTestElement (elementId) {
return (dispatch) => {
return async (dispatch) => {
dispatch({type: SET_LOCATOR_TEST_ELEMENT, elementId});
dispatch({type: CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS});
if (elementId) {
try {
const [location, size] = await(B.all([
callClientMethod({methodName: 'getLocation', args: [elementId], skipScreenshotAndSource: true, skipRecord: true}),
callClientMethod({methodName: 'getSize', args: [elementId], skipScreenshotAndSource: true, skipRecord: true}),
]));
dispatch({type: SET_SEARCHED_FOR_ELEMENT_BOUNDS, location: location.res, size: size.res});
} catch (ign) { }
}
};
}

Expand Down
45 changes: 29 additions & 16 deletions app/renderer/components/Inspector/HighlighterRect.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,44 @@ import { parseCoordinates } from './shared';
export default class HighlighterRect extends Component {

render () {
const {selectedElement = {}, selectHoveredElement, unselectHoveredElement, hoveredElement = {}, selectElement, unselectElement, element, zIndex, scaleRatio, xOffset} = this.props;
const {selectedElement = {}, selectHoveredElement, unselectHoveredElement, hoveredElement = {}, selectElement, unselectElement, element,
zIndex, scaleRatio, xOffset, elLocation, elSize} = this.props;
const {path: hoveredPath} = hoveredElement;
const {path: selectedPath} = selectedElement;

// Calculate left, top, width and height coordinates
const {x1, y1, x2, y2} = parseCoordinates(element);
const left = x1 / scaleRatio + xOffset;
const top = y1 / scaleRatio;
const width = (x2 - x1) / scaleRatio;
const height = (y2 - y1) / scaleRatio;
let width, height, left, top, highlighterClasses, key;
highlighterClasses = [InspectorCSS['highlighter-box']];

// Add class + special classes to hovered and selected elements
const highlighterClasses = [InspectorCSS['highlighter-box']];
if (hoveredPath === element.path) {
highlighterClasses.push(InspectorCSS['hovered-element-box']);
}
if (selectedPath === element.path) {
if (element) {
// Calculate left, top, width and height coordinates
const {x1, y1, x2, y2} = parseCoordinates(element);
left = x1 / scaleRatio + xOffset;
top = y1 / scaleRatio;
width = (x2 - x1) / scaleRatio;
height = (y2 - y1) / scaleRatio;

// Add class + special classes to hovered and selected elements
if (hoveredPath === element.path) {
highlighterClasses.push(InspectorCSS['hovered-element-box']);
}
if (selectedPath === element.path) {
highlighterClasses.push(InspectorCSS['inspected-element-box']);
}
key = element.path;
} else if (elLocation && elSize) {
width = elSize.width / scaleRatio;
height = elSize.height / scaleRatio;
top = elLocation.y / scaleRatio;
left = elLocation.x / scaleRatio + xOffset;
key = 'searchedForElement';
highlighterClasses.push(InspectorCSS['inspected-element-box']);
}

return <div className={highlighterClasses.join(' ').trim()}
onMouseOver={() => selectHoveredElement(element.path)}
onMouseOver={() => selectHoveredElement(key)}
onMouseOut={unselectHoveredElement}
onClick={() => element.path === selectedPath ? unselectElement() : selectElement(element.path)}
key={element.path}
onClick={() => key === selectedPath ? unselectElement() : selectElement(key)}
key={key}
style={{zIndex, left: (left || 0), top: (top || 0), width: (width || 0), height: (height || 0)}}>
<div></div>
</div>;
Expand Down
14 changes: 11 additions & 3 deletions app/renderer/components/Inspector/HighlighterRects.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export default class HighlighterRects extends Component {
}

render () {
const {source, screenshotInteractionMode, containerEl} = this.props;
const {source, screenshotInteractionMode, containerEl, searchedForElementBounds, isLocatorTestModalVisible} = this.props;
const {scaleRatio} = this.state;

// Recurse through the 'source' JSON and render a highlighter rect for each element
Expand All @@ -111,7 +111,6 @@ export default class HighlighterRects extends Component {
containerEl.getBoundingClientRect().left;
}

// TODO: Refactor this into a separate component
let recursive = (element, zIndex = 0) => {
if (!element) {
return;
Expand All @@ -129,13 +128,22 @@ export default class HighlighterRects extends Component {
}
};

// If the use selected an element that they searched for, highlight that element
if (searchedForElementBounds && isLocatorTestModalVisible) {
const {location:elLocation, size} = searchedForElementBounds;
highlighterRects.push(<HighlighterRect elSize={size} elLocation={elLocation} scaleRatio={scaleRatio} xOffset={highlighterXOffset} />);
}

// If we're tapping or swiping, show the 'crosshair' cursor style
const screenshotStyle = {};
if (screenshotInteractionMode === 'tap' || screenshotInteractionMode === 'swipe') {
screenshotStyle.cursor = 'crosshair';
}

recursive(source);
// Don't show highlighter rects when Search Elements modal is open
if (!isLocatorTestModalVisible) {
recursive(source);
}

return <div>{ highlighterRects }</div>;
}
Expand Down
2 changes: 2 additions & 0 deletions app/renderer/components/Inspector/Screenshot.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ export default class Screenshot extends Component {
this.containerEl = null;
this.state = {
scaleRatio: 1,
x: null,
y: null,
};
this.updateScaleRatio = debounce(this.updateScaleRatio.bind(this), 1000);
}
Expand Down
18 changes: 17 additions & 1 deletion app/renderer/reducers/Inspector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { SET_SOURCE_AND_SCREENSHOT, QUIT_SESSION_REQUESTED, QUIT_SESSION_DONE,
SHOW_LOCATOR_TEST_MODAL, HIDE_LOCATOR_TEST_MODAL, SET_LOCATOR_TEST_STRATEGY, SET_LOCATOR_TEST_VALUE,
SEARCHING_FOR_ELEMENTS, SEARCHING_FOR_ELEMENTS_COMPLETED, SET_LOCATOR_TEST_ELEMENT, CLEAR_SEARCH_RESULTS,
ADD_ASSIGNED_VAR_CACHE, CLEAR_ASSIGNED_VAR_CACHE, SET_SCREENSHOT_INTERACTION_MODE,
SET_SWIPE_START, SET_SWIPE_END, CLEAR_SWIPE_ACTION
SET_SWIPE_START, SET_SWIPE_END, CLEAR_SWIPE_ACTION, SET_SEARCHED_FOR_ELEMENT_BOUNDS, CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS,
} from '../actions/Inspector';

const DEFAULT_FRAMEWORK = 'java';
Expand All @@ -28,6 +28,7 @@ const INITIAL_STATE = {
isSearchingForElements: false,
assignedVarCache: {},
screenshotInteractionMode: 'select',
searchedForElementBounds: null,
};

/**
Expand Down Expand Up @@ -287,6 +288,21 @@ export default function inspector (state=INITIAL_STATE, action) {
swipeEnd: null,
};

case SET_SEARCHED_FOR_ELEMENT_BOUNDS:
return {
...state,
searchedForElementBounds: {
location: action.location,
size: action.size,
}
};

case CLEAR_SEARCHED_FOR_ELEMENT_BOUNDS:
return {
...state,
searchedForElementBounds: null,
};

default:
return {...state};
}
Expand Down

0 comments on commit f2c7ad4

Please sign in to comment.