Skip to content

Commit

Permalink
#370 - New keyboard step for advance keyboard entries and modifiers (#…
Browse files Browse the repository at this point in the history 
…371)

Building on the series of enhancements around Sikuli integration #368, #361 and #365 for greater control of mouse, creating and sending a PR that adds a new `keyboard` step. This step lets user send low-level keyboard keystrokes to the operating system user interface, including the special keys and modifier keys. Normal letter characters and numbers can also be entered.

Prior to this, users can only use 1. `type page.png as text` and limited to [enter] and [clear], or 2. use `vision` step to send custom commands to perform complex keyboard actions. Below are some examples.

**macOS**
```
keyboard [cmd][space]
keyboard safari[enter]
keyboard [cmd]c
keyboard [cmd]v
keyboard testing 123
```

**Windows**
```
keyboard [ctrl][home]
keyboard [printscreen]
keyboard [ctrl]c
keyboard v[ctrl]
keyboard testing 456
```

List of modifier keys - [shift] [ctrl] [alt] [cmd] [win] [meta]
List of special keys - [clear] [space] [enter] [backspace] [tab] [esc] [up] [down] [left] [right] [pageup] [pagedown] [delete] [home] [end] [insert] [f1] .. [f15] [printscreen] [scrolllock] [pause] [capslock] [numlock]
  • Loading branch information
@kensoh
kensoh authored Mar 29, 2019
1 parent 40f46c4 commit 8251ee9
Show file tree
Hide file tree
Showing 5 changed files with 200 additions and 5 deletions.
100 changes: 95 additions & 5 deletions src/tagui.sikuli/tagui.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,72 @@ def x_coordinate ( input_locator ):
def y_coordinate ( input_locator ):
return int(input_locator[input_locator.find(',')+1:-1])

# function to map modifier keys to unicode for use in type()
def modifiers_map ( input_keys ):
modifier_keys = 0
if '[shift]' in input_keys or '[SHIFT]' in input_keys: modifier_keys = modifier_keys + KeyModifier.SHIFT
if '[ctrl]' in input_keys or '[CTRL]' in input_keys: modifier_keys = modifier_keys + KeyModifier.CTRL
if '[alt]' in input_keys or '[ALT]' in input_keys: modifier_keys = modifier_keys + KeyModifier.ALT
if '[meta]' in input_keys or '[META]' in input_keys: modifier_keys = modifier_keys + KeyModifier.META
if '[cmd]' in input_keys or '[CMD]' in input_keys: modifier_keys = modifier_keys + KeyModifier.CMD
if '[win]' in input_keys or '[WIN]' in input_keys: modifier_keys = modifier_keys + KeyModifier.WIN
return modifier_keys

# function to map special keys to unicode for use in type()
def keyboard_map ( input_keys ):
input_keys = input_keys.replace('[clear]','\b').replace('[CLEAR]','\b')
input_keys = input_keys.replace('[space]',' ').replace('[SPACE]',' ')
input_keys = input_keys.replace('[enter]','\n').replace('[ENTER]','\n')
input_keys = input_keys.replace('[backspace]','\b').replace('[BACKSPACE]','\b')
input_keys = input_keys.replace('[tab]','\t').replace('[TAB]','\t')
input_keys = input_keys.replace('[esc]',u'\u001b').replace('[ESC]',u'\u001b')
input_keys = input_keys.replace('[up]',u'\ue000').replace('[UP]',u'\ue000')
input_keys = input_keys.replace('[right]',u'\ue001').replace('[RIGHT]',u'\ue001')
input_keys = input_keys.replace('[down]',u'\ue002').replace('[DOWN]',u'\ue002')
input_keys = input_keys.replace('[left]',u'\ue003').replace('[LEFT]',u'\ue003')
input_keys = input_keys.replace('[pageup]',u'\ue004').replace('[PAGEUP]',u'\ue004')
input_keys = input_keys.replace('[pagedown]',u'\ue005').replace('[PAGEDOWN]',u'\ue005')
input_keys = input_keys.replace('[delete]',u'\ue006').replace('[DELETE]',u'\ue006')
input_keys = input_keys.replace('[end]',u'\ue007').replace('[END]',u'\ue007')
input_keys = input_keys.replace('[home]',u'\ue008').replace('[HOME]',u'\ue008')
input_keys = input_keys.replace('[insert]',u'\ue009').replace('[INSERT]',u'\ue009')
input_keys = input_keys.replace('[f1]',u'\ue011').replace('[F1]',u'\ue011')
input_keys = input_keys.replace('[f2]',u'\ue012').replace('[F2]',u'\ue012')
input_keys = input_keys.replace('[f3]',u'\ue013').replace('[F3]',u'\ue013')
input_keys = input_keys.replace('[f4]',u'\ue014').replace('[F4]',u'\ue014')
input_keys = input_keys.replace('[f5]',u'\ue015').replace('[F5]',u'\ue015')
input_keys = input_keys.replace('[f6]',u'\ue016').replace('[F6]',u'\ue016')
input_keys = input_keys.replace('[f7]',u'\ue017').replace('[F7]',u'\ue017')
input_keys = input_keys.replace('[f8]',u'\ue018').replace('[F8]',u'\ue018')
input_keys = input_keys.replace('[f9]',u'\ue019').replace('[F9]',u'\ue019')
input_keys = input_keys.replace('[f10]',u'\ue01A').replace('[F10]',u'\ue01A')
input_keys = input_keys.replace('[f11]',u'\ue01B').replace('[F11]',u'\ue01B')
input_keys = input_keys.replace('[f12]',u'\ue01C').replace('[F12]',u'\ue01C')
input_keys = input_keys.replace('[f13]',u'\ue01D').replace('[F13]',u'\ue01D')
input_keys = input_keys.replace('[f14]',u'\ue01E').replace('[F14]',u'\ue01E')
input_keys = input_keys.replace('[f15]',u'\ue01F').replace('[F15]',u'\ue01F')
input_keys = input_keys.replace('[printscreen]',u'\ue024').replace('[PRINTSCREEN]',u'\ue024')
input_keys = input_keys.replace('[scrolllock]',u'\ue025').replace('[SCROLLLOCK]',u'\ue025')
input_keys = input_keys.replace('[pause]',u'\ue026').replace('[PAUSE]',u'\ue026')
input_keys = input_keys.replace('[capslock]',u'\ue027').replace('[CAPSLOCK]',u'\ue027')
input_keys = input_keys.replace('[numlock]',u'\ue03B').replace('[NUMLOCK]',u'\ue03B')

# if modifier key is the only input, treat as a keystroke instead of a modifier
if input_keys == '[shift]' or input_keys == '[SHIFT]': input_keys = u'\ue020'
elif input_keys == '[ctrl]' or input_keys == '[CTRL]': input_keys = u'\ue021'
elif input_keys == '[alt]' or input_keys == '[ALT]': input_keys = u'\ue022'
elif input_keys == '[meta]' or input_keys == '[META]': input_keys = u'\ue023'
elif input_keys == '[cmd]' or input_keys == '[CMD]': input_keys = u'\ue023'
elif input_keys == '[win]' or input_keys == '[WIN]': input_keys = u'\ue042'

input_keys = input_keys.replace('[shift]','').replace('[SHIFT]','')
input_keys = input_keys.replace('[ctrl]','').replace('[CTRL]','')
input_keys = input_keys.replace('[alt]','').replace('[ALT]','')
input_keys = input_keys.replace('[meta]','').replace('[META]','')
input_keys = input_keys.replace('[cmd]','').replace('[CMD]','')
input_keys = input_keys.replace('[win]','').replace('[WIN]','')
return input_keys

# function to output sikuli text to tagui
def output_sikuli_text ( output_text ):
import codecs
Expand Down Expand Up @@ -89,14 +155,23 @@ def type_intent ( raw_intent ):
param1 = params[:params.find(' as ')].strip()
param2 = params[4+params.find(' as '):].strip()
print '[tagui] ACTION - type ' + param1 + ' as ' + param2
param2 = param2.replace('[enter]','\n')
param2 = param2.replace('[clear]','\b')
modifier_keys = modifiers_map(param2)
param2 = keyboard_map(param2)
if param1.endswith('page.png') or param1.endswith('page.bmp'):
return type(param2)
if modifier_keys == 0:
return type(param2)
else:
return type(param2,modifier_keys)
elif is_coordinates(param1):
return type(Location(x_coordinate(param1),y_coordinate(param1)),param2)
if modifier_keys == 0:
return type(Location(x_coordinate(param1),y_coordinate(param1)),param2)
else:
return type(Location(x_coordinate(param1),y_coordinate(param1)),param2,modifier_keys)
elif exists(param1):
return type(param1,param2)
if modifier_keys == 0:
return type(param1,param2)
else:
return type(param1,param2,modifier_keys)
else:
return 0

Expand Down Expand Up @@ -185,6 +260,17 @@ def snap_intent ( raw_intent ):
else:
return 0

# function for low-level keyboard control
def keyboard_intent ( raw_intent ):
params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip()
print '[tagui] ACTION - keyboard ' + params
modifier_keys = modifiers_map(params)
params = keyboard_map(params)
if modifier_keys == 0:
return type(params)
else:
return type(params,modifier_keys)

# function for low-level mouse control
def mouse_intent ( raw_intent ):
params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip()
Expand Down Expand Up @@ -240,6 +326,8 @@ def get_intent ( raw_intent ):
return 'save'
if raw_intent[:5].lower() == 'snap ':
return 'snap'
if raw_intent[:9].lower() == 'keyboard ':
return 'keyboard'
if raw_intent[:6].lower() == 'mouse ':
return 'mouse'
if raw_intent[:7].lower() == 'vision ':
Expand Down Expand Up @@ -271,6 +359,8 @@ def parse_intent ( script_line ):
return save_intent(script_line)
elif intent_type == 'snap':
return snap_intent(script_line)
elif intent_type == 'keyboard':
return keyboard_intent(script_line)
elif intent_type == 'mouse':
return mouse_intent(script_line)
elif intent_type == 'vision':
Expand Down
8 changes: 8 additions & 0 deletions src/tagui_header.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,7 @@ case 'table': return table_intent(live_line); break;
case 'wait': return wait_intent(live_line); break;
case 'live': return live_intent(live_line); break;
case 'ask': return ask_intent(live_line); break;
case 'keyboard': return keyboard_intent(live_line); break;
case 'mouse': return mouse_intent(live_line); break;
case 'check': return check_intent(live_line); break;
case 'test': return test_intent(live_line); break;
Expand Down Expand Up @@ -759,6 +760,7 @@ if (lc_raw_intent.substr(0,6) == 'table ') return 'table';
if (lc_raw_intent.substr(0,5) == 'wait ') return 'wait';
if (lc_raw_intent.substr(0,5) == 'live ') return 'live';
if (lc_raw_intent.substr(0,4) == 'ask ') return 'ask';
if (lc_raw_intent.substr(0,9) == 'keyboard ') return 'keyboard';
if (lc_raw_intent.substr(0,6) == 'mouse ') return 'mouse';
if (lc_raw_intent.substr(0,6) == 'check ') return 'check';
if (lc_raw_intent.substr(0,5) == 'test ') return 'test';
Expand Down Expand Up @@ -795,6 +797,7 @@ if (lc_raw_intent == 'table') return 'table';
if (lc_raw_intent == 'wait') return 'wait';
if (lc_raw_intent == 'live') return 'live';
if (lc_raw_intent == 'ask') return 'ask';
if (lc_raw_intent == 'keyboard') return 'keyboard';
if (lc_raw_intent == 'mouse') return 'mouse';
if (lc_raw_intent == 'check') return 'check';
if (lc_raw_intent == 'test') return 'test';
Expand Down Expand Up @@ -1091,6 +1094,11 @@ return "this.echo('ERROR - you are already in live mode, type done to quit live
function ask_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
return "this.echo('ERROR - step is not relevant in live mode, set ask_result directly')";}

function keyboard_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
if (params == '') return "this.echo('ERROR - keys to type missing for " + raw_intent + "')";
else return call_sikuli(raw_intent,params);}

function mouse_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
if (params == '') return "this.echo('ERROR - up / down missing for " + raw_intent + "')";
Expand Down
8 changes: 8 additions & 0 deletions src/tagui_parse.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,7 @@ function process_intent($intent_type, $script_line) {
case "wait": return wait_intent($script_line); break;
case "live": return live_intent($script_line); break;
case "ask": return ask_intent($script_line); break;
case "keyboard": return keyboard_intent($script_line); break;
case "mouse": return mouse_intent($script_line); break;
case "check": return check_intent($script_line); break;
case "test": return test_intent($script_line); break;
Expand Down Expand Up @@ -440,6 +441,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent);
if (substr($lc_raw_intent,0,5)=="wait ") return "wait";
if (substr($lc_raw_intent,0,5)=="live ") return "live";
if (substr($lc_raw_intent,0,4)=="ask ") return "ask";
if (substr($lc_raw_intent,0,9)=="keyboard ") return "keyboard";
if (substr($lc_raw_intent,0,6)=="mouse ") return "mouse";
if (substr($lc_raw_intent,0,6)=="check ") {$GLOBALS['test_automation']++; return "check";}
if (substr($lc_raw_intent,0,5)=="test ") return "test";
Expand Down Expand Up @@ -476,6 +478,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent);
if ($lc_raw_intent=="wait") return "wait";
if ($lc_raw_intent=="live") return "live";
if ($lc_raw_intent=="ask") return "ask";
if ($lc_raw_intent=="keyboard") return "keyboard";
if ($lc_raw_intent=="mouse") return "mouse";
if ($lc_raw_intent=="check") {$GLOBALS['test_automation']++; return "check";}
if ($lc_raw_intent=="test") return "test";
Expand Down Expand Up @@ -842,6 +845,11 @@ function ask_intent($raw_intent) { // ask user for input during automation and s
"{ask_result = ''; var sys = require('system');\nthis.echo('".$params." '); ".
"ask_result = sys.stdin.readLine();}".end_fi()."});"."\n\n";}

function keyboard_intent($raw_intent) {
$params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," ")));
if ($params == "") echo "ERROR - " . current_line() . " keys to type missing for " . $raw_intent . "\n";
return "casper.then(function() {".call_sikuli($raw_intent,$params);}

function mouse_intent($raw_intent) {
$params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," ")));
if ($params == "") echo "ERROR - " . current_line() . " up / down missing for " . $raw_intent . "\n";
Expand Down
15 changes: 15 additions & 0 deletions src/test/positive_test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,21 @@ wait 7.5 seconds
// test live
live

// test keyboard
keyboard ls -lrt[enter]
keyboard ls -lrt[ENTER]
keyboard [pageup]
keyboard [PAGEDOWN]
keyboard 123[enter]456[ENTER]
keyboard [home]
keyboard [end]
keyboard [ctrl][home]
keyboard [ctrl][end]
keyboard [win]
keyboard e[win]
keyboard [win]e
keyboard [cmd][space]

// test mouse
mouse down
mouse up
Expand Down
74 changes: 74 additions & 0 deletions src/test/positive_test.signature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,6 +733,7 @@ case 'table': return table_intent(live_line); break;
case 'wait': return wait_intent(live_line); break;
case 'live': return live_intent(live_line); break;
case 'ask': return ask_intent(live_line); break;
case 'keyboard': return keyboard_intent(live_line); break;
case 'mouse': return mouse_intent(live_line); break;
case 'check': return check_intent(live_line); break;
case 'test': return test_intent(live_line); break;
Expand Down Expand Up @@ -786,6 +787,7 @@ if (lc_raw_intent.substr(0,6) == 'table ') return 'table';
if (lc_raw_intent.substr(0,5) == 'wait ') return 'wait';
if (lc_raw_intent.substr(0,5) == 'live ') return 'live';
if (lc_raw_intent.substr(0,4) == 'ask ') return 'ask';
if (lc_raw_intent.substr(0,9) == 'keyboard ') return 'keyboard';
if (lc_raw_intent.substr(0,6) == 'mouse ') return 'mouse';
if (lc_raw_intent.substr(0,6) == 'check ') return 'check';
if (lc_raw_intent.substr(0,5) == 'test ') return 'test';
Expand Down Expand Up @@ -822,6 +824,7 @@ if (lc_raw_intent == 'table') return 'table';
if (lc_raw_intent == 'wait') return 'wait';
if (lc_raw_intent == 'live') return 'live';
if (lc_raw_intent == 'ask') return 'ask';
if (lc_raw_intent == 'keyboard') return 'keyboard';
if (lc_raw_intent == 'mouse') return 'mouse';
if (lc_raw_intent == 'check') return 'check';
if (lc_raw_intent == 'test') return 'test';
Expand Down Expand Up @@ -1118,6 +1121,11 @@ return "this.echo('ERROR - you are already in live mode, type done to quit live
function ask_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
return "this.echo('ERROR - step is not relevant in live mode, set ask_result directly')";}

function keyboard_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
if (params == '') return "this.echo('ERROR - keys to type missing for " + raw_intent + "')";
else return call_sikuli(raw_intent,params);}

function mouse_intent(raw_intent) {raw_intent = eval("'" + raw_intent + "'"); // support dynamic variables
var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
if (params == '') return "this.echo('ERROR - up / down missing for " + raw_intent + "')";
Expand Down Expand Up @@ -2621,6 +2629,72 @@ while (true) {live_input = sys.stdin.readLine(); // evaluate input in casperjs c
if (live_input.indexOf('done') == 0) break; try {eval(tagui_parse(live_input));}
catch(e) {this.echo('ERROR - ' + e.message.charAt(0).toLowerCase() + e.message.slice(1));}}}});

// test keyboard
casper.then(function() {{techo('keyboard ls -lrt[enter]'); var fs = require('fs');
if (!sikuli_step('keyboard ls -lrt[enter]')) if (!fs.exists('ls -lrt[enter]'))
this.echo('ERROR - cannot find image file ls -lrt[enter]').exit(); else
this.echo('ERROR - cannot find ls -lrt[enter] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard ls -lrt[ENTER]'); var fs = require('fs');
if (!sikuli_step('keyboard ls -lrt[ENTER]')) if (!fs.exists('ls -lrt[ENTER]'))
this.echo('ERROR - cannot find image file ls -lrt[ENTER]').exit(); else
this.echo('ERROR - cannot find ls -lrt[ENTER] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [pageup]'); var fs = require('fs');
if (!sikuli_step('keyboard [pageup]')) if (!fs.exists('[pageup]'))
this.echo('ERROR - cannot find image file [pageup]').exit(); else
this.echo('ERROR - cannot find [pageup] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [PAGEDOWN]'); var fs = require('fs');
if (!sikuli_step('keyboard [PAGEDOWN]')) if (!fs.exists('[PAGEDOWN]'))
this.echo('ERROR - cannot find image file [PAGEDOWN]').exit(); else
this.echo('ERROR - cannot find [PAGEDOWN] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard 123[enter]456[ENTER]'); var fs = require('fs');
if (!sikuli_step('keyboard 123[enter]456[ENTER]')) if (!fs.exists('123[enter]456[ENTER]'))
this.echo('ERROR - cannot find image file 123[enter]456[ENTER]').exit(); else
this.echo('ERROR - cannot find 123[enter]456[ENTER] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [home]'); var fs = require('fs');
if (!sikuli_step('keyboard [home]')) if (!fs.exists('[home]'))
this.echo('ERROR - cannot find image file [home]').exit(); else
this.echo('ERROR - cannot find [home] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [end]'); var fs = require('fs');
if (!sikuli_step('keyboard [end]')) if (!fs.exists('[end]'))
this.echo('ERROR - cannot find image file [end]').exit(); else
this.echo('ERROR - cannot find [end] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [ctrl][home]'); var fs = require('fs');
if (!sikuli_step('keyboard [ctrl][home]')) if (!fs.exists('[ctrl][home]'))
this.echo('ERROR - cannot find image file [ctrl][home]').exit(); else
this.echo('ERROR - cannot find [ctrl][home] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [ctrl][end]'); var fs = require('fs');
if (!sikuli_step('keyboard [ctrl][end]')) if (!fs.exists('[ctrl][end]'))
this.echo('ERROR - cannot find image file [ctrl][end]').exit(); else
this.echo('ERROR - cannot find [ctrl][end] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [win]'); var fs = require('fs');
if (!sikuli_step('keyboard [win]')) if (!fs.exists('[win]'))
this.echo('ERROR - cannot find image file [win]').exit(); else
this.echo('ERROR - cannot find [win] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard e[win]'); var fs = require('fs');
if (!sikuli_step('keyboard e[win]')) if (!fs.exists('e[win]'))
this.echo('ERROR - cannot find image file e[win]').exit(); else
this.echo('ERROR - cannot find e[win] on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [win]e'); var fs = require('fs');
if (!sikuli_step('keyboard [win]e')) if (!fs.exists('[win]e'))
this.echo('ERROR - cannot find image file [win]e').exit(); else
this.echo('ERROR - cannot find [win]e on screen').exit(); this.wait(0);}});

casper.then(function() {{techo('keyboard [cmd][space]'); var fs = require('fs');
if (!sikuli_step('keyboard [cmd][space]')) if (!fs.exists('[cmd][space]'))
this.echo('ERROR - cannot find image file [cmd][space]').exit(); else
this.echo('ERROR - cannot find [cmd][space] on screen').exit(); this.wait(0);}});

// test mouse
casper.then(function() {{techo('mouse down'); var fs = require('fs');
if (!sikuli_step('mouse down')) if (!fs.exists('down'))
Expand Down

0 comments on commit 8251ee9

Please sign in to comment.