Skip to content

Commit

Permalink
#368 - New mouse step for low-level mouse down and mouse up (#369)
Browse files Browse the repository at this point in the history 
#368 - New mouse step for low-level mouse down and mouse up

Building on the series of enhancements around Sikuli integration #361 and #365 for greater control of mouse, publishing a PR that adds a new `mouse` step.

This step lets user send a low-level mouse down or up event to the user interface. This can be used for complex UI mouse actions. For eg, when dragging and dropping UI elements -

```
// below drags the UI element left by 200 pixels
hover some_element.png
mouse down
target_x = mouse_x() - 200
hover (`target_x`,`mouse_y()`)
mouse up
```
  • Loading branch information
@kensoh
kensoh authored Mar 28, 2019
1 parent b1a59ba commit 40f46c4
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 0 deletions.
19 changes: 19 additions & 0 deletions src/tagui.sikuli/tagui.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,21 @@ def snap_intent ( raw_intent ):
else:
return 0

# function for low-level mouse control
def mouse_intent ( raw_intent ):
params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip()
print '[tagui] ACTION - mouse ' + params
if params.lower() == 'down':
hover(Location(Env.getMouseLocation().getX(),Env.getMouseLocation().getY()))
mouseDown(Button.LEFT)
return 1
elif params.lower() == 'up':
hover(Location(Env.getMouseLocation().getX(),Env.getMouseLocation().getY()))
mouseUp(Button.LEFT)
return 1
else:
return 0

# function for sending custom commands
def vision_intent ( raw_intent ):
params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip()
Expand Down Expand Up @@ -225,6 +240,8 @@ def get_intent ( raw_intent ):
return 'save'
if raw_intent[:5].lower() == 'snap ':
return 'snap'
if raw_intent[:6].lower() == 'mouse ':
return 'mouse'
if raw_intent[:7].lower() == 'vision ':
return 'vision'
if raw_intent[:8].lower() == 'visible ' or raw_intent[:8].lower() == 'present ':
Expand Down Expand Up @@ -254,6 +271,8 @@ def parse_intent ( script_line ):
return save_intent(script_line)
elif intent_type == 'snap':
return snap_intent(script_line)
elif intent_type == 'mouse':
return mouse_intent(script_line)
elif intent_type == 'vision':
return vision_intent(script_line)
elif intent_type == 'visible':
Expand Down
10 changes: 10 additions & 0 deletions src/tagui_header.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,7 @@ case 'table': return table_intent(live_line); break;
case 'wait': return wait_intent(live_line); break;
case 'live': return live_intent(live_line); break;
case 'ask': return ask_intent(live_line); break;
case 'mouse': return mouse_intent(live_line); break;
case 'check': return check_intent(live_line); break;
case 'test': return test_intent(live_line); break;
case 'frame': return frame_intent(live_line); break;
Expand Down Expand Up @@ -758,6 +759,7 @@ if (lc_raw_intent.substr(0,6) == 'table ') return 'table';
if (lc_raw_intent.substr(0,5) == 'wait ') return 'wait';
if (lc_raw_intent.substr(0,5) == 'live ') return 'live';
if (lc_raw_intent.substr(0,4) == 'ask ') return 'ask';
if (lc_raw_intent.substr(0,6) == 'mouse ') return 'mouse';
if (lc_raw_intent.substr(0,6) == 'check ') return 'check';
if (lc_raw_intent.substr(0,5) == 'test ') return 'test';
if (lc_raw_intent.substr(0,6) == 'frame ') return 'frame';
Expand Down Expand Up @@ -793,6 +795,7 @@ if (lc_raw_intent == 'table') return 'table';
if (lc_raw_intent == 'wait') return 'wait';
if (lc_raw_intent == 'live') return 'live';
if (lc_raw_intent == 'ask') return 'ask';
if (lc_raw_intent == 'mouse') return 'mouse';
if (lc_raw_intent == 'check') return 'check';
if (lc_raw_intent == 'test') return 'test';
if (lc_raw_intent == 'frame') return 'frame';
Expand Down Expand Up @@ -1088,6 +1091,13 @@ return "this.echo('ERROR - you are already in live mode, type done to quit live
function ask_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
return "this.echo('ERROR - step is not relevant in live mode, set ask_result directly')";}

function mouse_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
if (params == '') return "this.echo('ERROR - up / down missing for " + raw_intent + "')";
else if (params.toLowerCase() == 'down') return call_sikuli(raw_intent,'down');
else if (params.toLowerCase() == 'up') return call_sikuli(raw_intent,'up');
else return "this.echo('ERROR - cannot understand step " + raw_intent + "')";}

function check_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
return "this.echo('ERROR - step not supported in live mode, there is no conditions language parser')";}

Expand Down
10 changes: 10 additions & 0 deletions src/tagui_parse.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,7 @@ function process_intent($intent_type, $script_line) {
case "wait": return wait_intent($script_line); break;
case "live": return live_intent($script_line); break;
case "ask": return ask_intent($script_line); break;
case "mouse": return mouse_intent($script_line); break;
case "check": return check_intent($script_line); break;
case "test": return test_intent($script_line); break;
case "frame": return frame_intent($script_line); break;
Expand Down Expand Up @@ -439,6 +440,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent);
if (substr($lc_raw_intent,0,5)=="wait ") return "wait";
if (substr($lc_raw_intent,0,5)=="live ") return "live";
if (substr($lc_raw_intent,0,4)=="ask ") return "ask";
if (substr($lc_raw_intent,0,6)=="mouse ") return "mouse";
if (substr($lc_raw_intent,0,6)=="check ") {$GLOBALS['test_automation']++; return "check";}
if (substr($lc_raw_intent,0,5)=="test ") return "test";
if (substr($lc_raw_intent,0,6)=="frame ") return "frame";
Expand Down Expand Up @@ -474,6 +476,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent);
if ($lc_raw_intent=="wait") return "wait";
if ($lc_raw_intent=="live") return "live";
if ($lc_raw_intent=="ask") return "ask";
if ($lc_raw_intent=="mouse") return "mouse";
if ($lc_raw_intent=="check") {$GLOBALS['test_automation']++; return "check";}
if ($lc_raw_intent=="test") return "test";
if ($lc_raw_intent=="frame") return "frame";
Expand Down Expand Up @@ -839,6 +842,13 @@ function ask_intent($raw_intent) { // ask user for input during automation and s
"{ask_result = ''; var sys = require('system');\nthis.echo('".$params." '); ".
"ask_result = sys.stdin.readLine();}".end_fi()."});"."\n\n";}

function mouse_intent($raw_intent) {
$params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," ")));
if ($params == "") echo "ERROR - " . current_line() . " up / down missing for " . $raw_intent . "\n";
else if (strtolower($params) == "down") return "casper.then(function() {".call_sikuli($raw_intent,"down");
else if (strtolower($params) == "up") return "casper.then(function() {".call_sikuli($raw_intent,"up");
else echo "ERROR - " . current_line() . " cannot understand step " . $raw_intent . "\n";}

function check_intent($raw_intent) {
$params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," ")));
$params = str_replace("||"," JAVASCRIPT_OR ",$params); // to handle conflict with "|" delimiter
Expand Down
4 changes: 4 additions & 0 deletions src/test/positive_test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,10 @@ wait 7.5 seconds
// test live
live

// test mouse
mouse down
mouse up

// test check
check eggs lesser than 10 | "eggs lesser than 10" | "eggs not lesser than 10"
check eggs lesser than 10|"eggs lesser than 10"|"eggs not lesser than 10"
Expand Down
21 changes: 21 additions & 0 deletions src/test/positive_test.signature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,6 +733,7 @@ case 'table': return table_intent(live_line); break;
case 'wait': return wait_intent(live_line); break;
case 'live': return live_intent(live_line); break;
case 'ask': return ask_intent(live_line); break;
case 'mouse': return mouse_intent(live_line); break;
case 'check': return check_intent(live_line); break;
case 'test': return test_intent(live_line); break;
case 'frame': return frame_intent(live_line); break;
Expand Down Expand Up @@ -785,6 +786,7 @@ if (lc_raw_intent.substr(0,6) == 'table ') return 'table';
if (lc_raw_intent.substr(0,5) == 'wait ') return 'wait';
if (lc_raw_intent.substr(0,5) == 'live ') return 'live';
if (lc_raw_intent.substr(0,4) == 'ask ') return 'ask';
if (lc_raw_intent.substr(0,6) == 'mouse ') return 'mouse';
if (lc_raw_intent.substr(0,6) == 'check ') return 'check';
if (lc_raw_intent.substr(0,5) == 'test ') return 'test';
if (lc_raw_intent.substr(0,6) == 'frame ') return 'frame';
Expand Down Expand Up @@ -820,6 +822,7 @@ if (lc_raw_intent == 'table') return 'table';
if (lc_raw_intent == 'wait') return 'wait';
if (lc_raw_intent == 'live') return 'live';
if (lc_raw_intent == 'ask') return 'ask';
if (lc_raw_intent == 'mouse') return 'mouse';
if (lc_raw_intent == 'check') return 'check';
if (lc_raw_intent == 'test') return 'test';
if (lc_raw_intent == 'frame') return 'frame';
Expand Down Expand Up @@ -1115,6 +1118,13 @@ return "this.echo('ERROR - you are already in live mode, type done to quit live
function ask_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
return "this.echo('ERROR - step is not relevant in live mode, set ask_result directly')";}

function mouse_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
if (params == '') return "this.echo('ERROR - up / down missing for " + raw_intent + "')";
else if (params.toLowerCase() == 'down') return call_sikuli(raw_intent,'down');
else if (params.toLowerCase() == 'up') return call_sikuli(raw_intent,'up');
else return "this.echo('ERROR - cannot understand step " + raw_intent + "')";}

function check_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
return "this.echo('ERROR - step not supported in live mode, there is no conditions language parser')";}

Expand Down Expand Up @@ -2611,6 +2621,17 @@ while (true) {live_input = sys.stdin.readLine(); // evaluate input in casperjs c
if (live_input.indexOf('done') == 0) break; try {eval(tagui_parse(live_input));}
catch(e) {this.echo('ERROR - ' + e.message.charAt(0).toLowerCase() + e.message.slice(1));}}}});

// test mouse
casper.then(function() {{techo('mouse down'); var fs = require('fs');
if (!sikuli_step('mouse down')) if (!fs.exists('down'))
this.echo('ERROR - cannot find image file down').exit(); else
this.echo('ERROR - cannot find down on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('mouse up'); var fs = require('fs');
if (!sikuli_step('mouse up')) if (!fs.exists('up'))
this.echo('ERROR - cannot find image file up').exit(); else
this.echo('ERROR - cannot find up on screen').exit(); this.wait(0);}});

// test check
casper.then(function() {{if ((eggs < 10))
this.echo("eggs lesser than 10");
Expand Down

0 comments on commit 40f46c4

Please sign in to comment.