Merge pull request #8 from chaosparrot/feature/debugging

Feature/debugging

README.md

@@ -56,6 +56,22 @@ This widget is meant to guide users through a predefined workflow to familiarize
 You can use this to give an interactive experience for users to learn the ins and outs of various workflows.
 Included in the Talon HUD is a simple walkthrough for using the Talon HUD itself, but any package can make walkthroughs or workflows.
 
+### 7. Ability bar
+
+Much like the status bar, except it only allows you to show icons and can be displayed independently of the status bar.  
+By default, it does not have any visual indication, but I have used it in some of my personal scripts to show movement directions.  
+
+### 8. Cursor tracker
+
+This widget follows your given pointer around and can display an icon or a colour near it.  
+This can be used for example to show the status of a specific thing near your eyetracking gaze through your mouse cursor.  
+By default, there are no cursor indicators, they need to be programmed in using the content documentation.  
+
+### 9. Screen overlay
+
+This widget can annotate regions of the screen with text, icons and colours.
+You can use it to show different regions for virtual keyboard usage combined with noises for example.
+
 Voice commands
 ---
 
@@ -134,6 +150,29 @@ Theming
 If you want to add your own theme, simply copy and paste an existing theme folder over, give it a new name, define it in the commands.talon file and start changing values in the themes.csv file of your copied over directory.  
 In general, it is best to keep the images small for memory sake. But otherwise go nuts.
 
+There are some values that have special settings, like event_log_ttl_duration_seconds, which can be set to -1 to have the logs stay without disappearing.
+As there is not a lot of theming going around, it is best to ask me on the Talon slack if you have questions about them.
+
+Context aware Talon HUD environments
+---
+
+You can change your HUD layout entirely using the context management of Talon.  
+Let's say you want to change the placement and enabled widgets when you enter a browser.  
+This is entirely achievable using the 'user.talon_hud_environment' setting.  
+
+We implement this in the following talon_hud_browser.talon file example.  
+When you focus a browser after adding a talon file like this, it will automatically make a new set of preference files specifically for the 'browser_hud' Talon HUD environment.
+You can then change your HUD around as you see fit. When you switch out of your browser context, it will change the HUD back as you left it before opening the browser.
+And next time you open up the browser again, it will neatly place the widgets where they were left previously in the 'browser_hud' environment.
+
+```
+tag: user.talon_hud_visible
+and tag: browser
+-
+settings():
+    user.talon_hud_environment = "browser_hud"
+```
+
 Development guidelines
 ----
 
@@ -162,16 +201,14 @@ Roadmap
 
 These are ideas that I want to implement in no specific order and with no specific timeline in mind.
 
-- An indicator widget that follows the cursor around to show a single state that is important to the current task at hand
 - An image panel with a header and a close icon which displays image content
-- Splitting out topics from text boxes into separate text boxes
-- "Voice auto complete" content, which automatically opens documentation when a certain keyword is said, to help in the learning process.
+- Improved theming experience and more styling options
+- Splitting out or merging topics from widgets into separate widgets
 - Better default image, dimension and font scaling based on physical dimensions of the screen
 
 Known issues
 ---
-- Multiple page walkthrough panel does not work properly with text indecis
-- Walkthrough panel needs improving UX wise ( close button, skip button, read more button )
+- Multiple page walkthrough panel does not work properly with text indices
 
 If any of these ideas seem cool for you to work on, give me a message on the talon slack so we can coordinate stuff.
 

base_widget.py

@@ -49,7 +49,7 @@ def __init__(self, id, preferences_dict, theme, subscriptions = None):
             self.subscribed_logs = subscriptions['logs'] if 'logs' in subscriptions else self.subscribed_logs
 
     # Load the widgets preferences
-    def load(self, dict, initialize = True):
+    def load(self, dict, initialize = True, update_enabled = False):
         self.preferences.load(self.id, dict)
         self.sleep_enabled = self.preferences.sleep_enabled
         self.show_animations = self.preferences.show_animations
@@ -65,6 +65,11 @@ def load(self, dict, initialize = True):
         self.alignment = self.preferences.alignment
         self.expand_direction = self.preferences.expand_direction
         self.minimized = self.preferences.minimized
+
+        # For re-enabling or disabling widgets after a reload ( mostly for talon hud environment changes )
+        if update_enabled:
+            if self.enabled != self.preferences.enabled:
+                self.enable() if self.preferences.enabled else self.disable()
 
         if initialize:
             self.load_theme_values()        
@@ -91,7 +96,7 @@ def update_content(self, content):
         for key in content:
             self.content[key] = content[key]
 
-        if self.enabled:
+        if self.enabled and self.canvas:
             self.canvas.resume()
 
     def update_panel(self, panel_content) -> bool:
@@ -111,13 +116,12 @@ def enable(self, persisted=False):
         if not self.enabled:
             self.enabled = True
             self.canvas = canvas.Canvas(min(self.x, self.limit_x), min(self.y, self.limit_y), max(self.width, self.limit_width), max(self.height, self.limit_height))
-            self.canvas.register('draw', self.draw_cycle)
-            self.animation_tick = self.animation_max_duration if self.show_animations else 0
-            self.canvas.resume()
-
             if self.mouse_enabled:
                 self.canvas.blocks_mouse = True
                 self.canvas.register('mouse', self.on_mouse)
+            self.canvas.register('draw', self.draw_cycle)
+            self.animation_tick = self.animation_max_duration if self.show_animations else 0
+            self.canvas.resume()
 
             if persisted:
                 self.preferences.enabled = True
@@ -128,7 +132,6 @@ def enable(self, persisted=False):
     def disable(self, persisted=False):
         if self.enabled:
             if self.mouse_enabled:
-                self.canvas.blocks_mouse = False
                 self.canvas.unregister('mouse', self.on_mouse)
 
             self.enabled = False
@@ -305,10 +308,11 @@ def start_setup(self, setup_type, mouse_position = None):
         elif setup_type == "reload":
             self.drag_position = []
             self.setup_type = ""             
-            if self.canvas:
+            if self.canvas:            
                 rect = ui.Rect(self.x, self.y, self.width, self.height)                    
                 self.canvas.rect = rect
                 self.canvas.resume()
+
         # Start the setup state
         elif self.setup_type != setup_type:
             self.setup_type = setup_type

content/README.md

@@ -212,3 +212,44 @@ If you add this in a talon file, you can test it for yourself when saying **test
 ```
 test example poller: user.example_poller()
 ```
+
+# Screen regions and cursor icons
+
+The HUD also allows you to add icons next to your pointer or on given areas of the screen using screen regions.
+Screen regions are regions in which position related content is displayed. 
+
+For example, you might want to add an icon next to your mouse if it is in a specific screen region. 
+Or you might want to show positional information in various blocks on the screen as if you are using a virtual keyboard.
+
+Let's say we want to add the command icon to follow our mouse to indicate that we are in command mode.
+For this, we can use two actions:
+
+```
+regions = [actions.user.hud_create_screen_region('cursor', None, 'command_icon')]
+actions.user.hud_publish_screen_regions('cursor', regions, True)
+```
+
+`hud_create_screen_region` allows you to create a special region. The arguments are:
+- topic: The topic of the region  
+- colour: The background colour of our screen region icon / text if any is set.  
+- icon: The icon to display, if any is set.  
+- title: The text to display, if any is set. For cursor tracking, no text is rendered.  
+- hover_visibility: Whether or not the region should be visible only if the pointer is in the region - Default keeps the region visible at all times.  
+  If set to -1, the region will become visible only if the mouse isn't directly over the visible content
+- x: The X position of the region - Default 0  
+- y: The Y position of the region - Default 0  
+- width: The width of the region in pixels - Default 0  
+- height: The height of the region in pixels - Default 0
+- relative_x: The relative x position to place the content from its origin point - Default 0
+- relative_y: The relative y position to place the content from its origin point - Default 0
+
+For cursor regions, if the region surface is not larger than 0, like in our example above, the cursor will be visible on every available display.
+
+Then, we can publish the region using `hud_publish_screen_regions`. This takes three arguments:  
+- type: The widget type to publish the region to - Choice between cursor and overlay
+- regions: The regions to publish
+- clear: Whether or not to clear previous regions of the given topics in the type - Defaults to not clearing
+
+To entirely clear the screen regions, you can use `actions.user.hud_clear_screen_regions('cursor')`. The arguments are:
+- type: The widget type to clear the regions from  
+- topic: The screen region topics to remove - If not set, clears everything for the widget type  

content/focus_poller.py

@@ -0,0 +1,65 @@
+from talon import actions, cron, scope, ui, app, Module
+from user.talon_hud.content.poller import Poller
+
+# Polls the current focused applications to show an indicator of where it is
+class FocusPoller(Poller):
+    move_indicator_job = None
+    previous_window_x = 0
+    previous_window_y = 0
+
+    def enable(self):
+       if not self.enabled:
+            self.enabled = True
+            self.update_focus_indicator()
+            ui.register('win_focus', self.update_focus_indicator)
+            ui.register('win_resize', self.update_focus_indicator)
+            ui.register('win_move', self.move_focus_indicator)
+
+    def disable(self):
+        self.enabled = False
+        ui.unregister('win_focus', self.update_focus_indicator)
+        ui.unregister('win_resize', self.update_focus_indicator)
+        ui.unregister('win_move', self.move_focus_indicator)        
+        actions.user.hud_publish_screen_regions('overlay', [], True)
+        cron.cancel(self.move_indicator_job)
+
+    def update_focus_indicator(self, window = None):
+        if not window or window.rect.width * window.rect.height > 0:
+            active_window = ui.active_window()
+            if active_window:
+                app = ui.active_app()
+                theme = actions.user.hud_get_theme()
+                focus_colour = theme.get_colour('focus_indicator_background', 'DD4500')
+                focus_text_colour = theme.get_colour('focus_indicator_text_colour', 'FFFFFF')
+
+                self.previous_window_x = active_window.rect.x
+                self.previous_window_y = active_window.rect.y                
+                regions = [actions.user.hud_create_screen_region('focus', focus_colour, '', '<*' + app.name, -1, active_window.rect.x, active_window.rect.y, active_window.rect.width, active_window.rect.height )]
+                regions[0].text_colour = focus_text_colour
+                regions[0].vertical_centered = False
+                actions.user.hud_publish_screen_regions('overlay', regions, True)
+
+    def move_focus_indicator(self, window):
+        cron.cancel(self.move_indicator_job)
+
+        active_window = ui.active_window()
+        if active_window.rect.x != self.previous_window_x and active_window.rect.y != self.previous_window_y:
+            self.move_indicator_job = cron.after("30ms", self.update_focus_indicator)
+
+def append_poller():
+    actions.user.hud_add_poller('focus', FocusPoller())
+app.register('ready', append_poller)
+
+mod = Module()
+@mod.action_class
+class Actions:
+
+    def hud_add_focus_indicator():
+        """Start debugging the focus state in the Talon HUD"""
+        actions.user.hud_add_poller('focus', FocusPoller())
+        actions.user.hud_activate_poller('focus')
+
+    def hud_remove_focus_indicator():
+        """Stop debugging the focus state in the Talon HUD"""
+        actions.user.hud_deactivate_poller('focus')
+        actions.user.hud_clear_screen_regions('overlay', 'focus')

content/history_poller.py

@@ -1,11 +1,10 @@
 from talon import actions, cron, scope, speech_system, ui, app
 from user.talon_hud.content.poller import Poller
-from talon_plugins import menu
+import time
 
 # Handles state of phrases
 # Inspired by the command history from knausj
 class HistoryPoller(Poller):
-    commands = []
 
     def enable(self):
     	if not self.enabled:
@@ -21,5 +20,18 @@ def on_phrase(self, j):
             word_list = getattr(j["parsed"], "_unmapped", j["phrase"])
         except:
             word_list = j["phrase"]
-        command = " ".join(word.split("\\")[0] for word in word_list)        
-        actions.user.hud_add_log("command", command)
+        command = " ".join(word.split("\\")[0] for word in word_list)
+        actions.user.hud_add_log("command", command)
+
+        # Debugging data
+        time_ms = 0.0
+        timestamp = time.time()
+        model = "-"
+        mic = actions.sound.active_microphone()
+        if "_metadata" in j:
+            meta = j["_metadata"]
+            time_ms += meta["total_ms"] if "total_ms" in meta else 0
+            time_ms += meta["audio_ms"] if "audio_ms" in meta else 0
+            model = meta["desc"] if "desc" in meta else "-"
+
+        actions.user.hud_add_phrase(command, timestamp, float(time_ms), model, mic)        

content/list_poller.py

@@ -0,0 +1,75 @@
+from talon import actions, cron, registry, ui, app, Module
+from user.talon_hud.content.poller import Poller
+
+# Polls the current Talon registry lists for debugging purposes
+class ListPoller(Poller):
+    job = None
+    previous_list_state = ''
+    list = None
+
+    def enable(self):
+       if (self.job is None):
+            self.enabled = True
+            scope_state = self.get_list_in_text()        
+            actions.user.hud_publish_content(scope_state, 'list', 'List inspection')        
+
+            self.job = cron.interval('200ms', self.list_check)
+
+    def disable(self):
+        cron.cancel(self.job)
+        self.job = None
+        self.enabled = False
+
+    def list_check(self):        
+        list_state = self.get_list_in_text()
+        if (list_state != self.previous_list_state):
+            self.previous_list_state = list_state
+            actions.user.hud_publish_content(list_state, 'list', 'List inspection', False)
+
+    def get_list_in_text(self):
+        content = ""
+        if self.list in registry.lists:
+            list_contents = registry.lists[self.list][-1]
+
+            list_description = registry.decls.lists[self.list].desc if self.list in registry.decls.lists else ""
+            if len(list_contents) == 0:
+                content = "<*" + self.list + "/>\n" + list_description + "\n\n"
+            else:
+                content = "<*" + self.list + "(" + str(len(list_contents)) + ")/>\n" + list_description + "\n\n"                
+
+                # Bundle same values together so we have all the synonyms bundled
+                content_choices = {}
+                for option in list_contents:
+                    if list_contents[option] not in content_choices:
+                        content_choices[ list_contents[option] ] = []
+
+                    content_choices[list_contents[option]].append( option )\
+
+                for value in content_choices:
+                    content += "<*" + "/><!! or /><*".join(content_choices[value]) + "/>: " + value + "\n"
+        elif self.list is not None:
+            content = "<*" + self.list + "/>\nList no longer exists!"
+
+        return content
+
+def select_list(data):
+    list_poller = ListPoller()
+    list_poller.list = data["text"]
+    actions.user.hud_add_poller('list', list_poller)
+    actions.user.hud_activate_poller('list')
+
+mod = Module()
+@mod.action_class
+class Actions:
+
+    def hud_toolkit_lists():
+        """Show available  lists to view for the Talon HUD"""
+        lists = registry.lists
+        choices = []
+        for index, key in enumerate(lists):
+            if lists[key]:
+                choices.append({"text": key})
+
+        choices = actions.user.hud_create_choices(choices, select_list)
+        actions.user.hud_publish_choices(choices, "Toolkit lists", "Select a list to inspect by saying <*option <number>/> or saying the name of the list")
+