Wms tutorial (#1083)

* Made cursor visible and finished wms automated tutorial

* flake8 corrections

* Improved code for use in Linux systems.(It was working well in other systems)

* Changed header file for the python scripts in mss/tutorials/*

tutorials/cursor.py

@@ -0,0 +1,161 @@
+"""
+    mss.tutorials.cursor
+    ~~~~~~~~~~~~~~~~~~~
+
+    This python script generates the png image of the cursor for linux systems and store it in
+    'tutorials/cursor_image.png.' Then, this image is used inside screenrecorder.py file. This displays
+    the cursor movement in screen recordings while running automated tutorials.
+
+    This file is part of mss.
+
+    :copyright: Copyright 2021 Hrithik Kumar Verma
+    :copyright: Copyright 2021 by the mss team, see AUTHORS.
+    :license: APACHE-2.0, see LICENSE for details.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+
+import os
+import ctypes
+import ctypes.util
+import numpy as np
+
+# A helper function to convert data from Xlib to byte array.
+import struct
+import array
+
+# Define ctypes version of XFixesCursorImage structure.
+PIXEL_DATA_PTR = ctypes.POINTER(ctypes.c_ulong)
+Atom = ctypes.c_ulong
+
+
+class XFixesCursorImage(ctypes.Structure):
+    """
+    See /usr/include/X11/extensions/Xfixes.h
+    typedef struct {
+        short	    x, y;
+        unsigned short  width, height;
+        unsigned short  xhot, yhot;
+        unsigned long   cursor_serial;
+        unsigned long   *pixels;
+    if XFIXES_MAJOR >= 2
+        Atom	    atom;	/* Version >= 2 only */
+        const char	*name;	/* Version >= 2 only */
+    endif
+    } XFixesCursorImage;
+    """
+    _fields_ = [('x', ctypes.c_short),
+                ('y', ctypes.c_short),
+                ('width', ctypes.c_ushort),
+                ('height', ctypes.c_ushort),
+                ('xhot', ctypes.c_ushort),
+                ('yhot', ctypes.c_ushort),
+                ('cursor_serial', ctypes.c_ulong),
+                ('pixels', PIXEL_DATA_PTR),
+                ('atom', Atom),
+                ('name', ctypes.c_char_p)]
+
+
+class Display(ctypes.Structure):
+    pass
+
+
+class Xcursor:
+    display = None
+
+    def __init__(self, display=None):
+        if not display:
+            try:
+                display = os.environ["DISPLAY"].encode('utf-8')
+            except KeyError:
+                raise Exception("$DISPLAY not set.")
+
+        # XFixeslib = ctypes.CDLL('libXfixes.so')
+        XFixes = ctypes.util.find_library("Xfixes")
+        if not XFixes:
+            raise Exception("No XFixes library found.")
+        self.XFixeslib = ctypes.cdll.LoadLibrary(XFixes)
+
+        # xlib = ctypes.CDLL('libX11.so.6')
+        x11 = ctypes.util.find_library("X11")
+        if not x11:
+            raise Exception("No X11 library found.")
+        self.xlib = ctypes.cdll.LoadLibrary(x11)
+
+        # Define ctypes' version of XFixesGetCursorImage function
+        XFixesGetCursorImage = self.XFixeslib.XFixesGetCursorImage
+        XFixesGetCursorImage.restype = ctypes.POINTER(XFixesCursorImage)
+        XFixesGetCursorImage.argtypes = [ctypes.POINTER(Display)]
+        self.XFixesGetCursorImage = XFixesGetCursorImage
+
+        XOpenDisplay = self.xlib.XOpenDisplay
+        XOpenDisplay.restype = ctypes.POINTER(Display)
+        XOpenDisplay.argtypes = [ctypes.c_char_p]
+
+        if not self.display:
+            self.display = self.xlib.XOpenDisplay(display)  # (display) or (None)
+
+    def argbdata_to_pixdata(self, data, len):
+        if data is None or len < 1:
+            return None
+
+        # Create byte array
+        b = array.array('b', b'\x00' * 4 * len)
+
+        offset, i = 0, 0
+        while i < len:
+            argb = data[i] & 0xffffffff
+            rgba = (argb << 8) | (argb >> 24)
+            b1 = (rgba >> 24) & 0xff
+            b2 = (rgba >> 16) & 0xff
+            b3 = (rgba >> 8) & 0xff
+            b4 = rgba & 0xff
+
+            struct.pack_into("=BBBB", b, offset, b1, b2, b3, b4)
+            offset = offset + 4
+            i = i + 1
+
+        return b
+
+    def getCursorImageData(self):
+        # Call the function. Read data of cursor/mouse-pointer.
+        cursor_data = self.XFixesGetCursorImage(self.display)
+
+        if not (cursor_data and cursor_data[0]):
+            raise Exception("Cannot read XFixesGetCursorImage()")
+
+        # Note: cursor_data is a pointer, take cursor_data[0]
+        return cursor_data[0]
+
+    def getCursorImageArray(self):
+        data = self.getCursorImageData()
+        # x, y = data.x, data.y
+        height, width = data.height, data.width
+
+        bytearr = ctypes.cast(data.pixels, ctypes.POINTER(ctypes.c_ulong * height * width))[0]
+        imgarray = np.array(bytearray(bytearr))
+        imgarray = imgarray.reshape(height, width, 8)[:, :, (0, 1, 2, 3)]
+        del bytearr
+
+        return imgarray
+
+    def saveImage(self, imgarray, text):
+        from PIL import Image
+        img = Image.fromarray(imgarray)
+        img.save(text)
+
+
+if __name__ == "__main__":
+    cursor = Xcursor()
+    imgarray = cursor.getCursorImageArray()
+    cursor.saveImage(imgarray, 'cursor_image.png')

tutorials/screenrecorder.py

@@ -1,5 +1,5 @@
 """
-    mslib.msui.screenrecorder
+    mss.tutorials.screenrecorder
     ~~~~~~~~~~~~~~~~~~~
 
     This python script is meant for recording the screens while automated  tutorials are running.
@@ -29,7 +29,11 @@
 import sys
 import time
 import mss
+import pyautogui
 from PyQt5 import QtWidgets
+from cursor import Xcursor
+from sys import platform
+from PIL import Image
 
 
 class ScreenRecorder:
@@ -83,6 +87,59 @@ def get_fps(self):
         return fps
 
     def capture(self):
+        """
+        Captures the frames of the screen at the rate of fps frames/second and writes into the
+        video writer object with the defined fourcc, codec and colour format.
+        """
+        if platform == 'linux' or platform == 'linux2':
+            cursor = Xcursor()
+            cursor_imgarray = cursor.getCursorImageArray()
+            cursor.saveImage(cursor_imgarray, 'cursor_image.png')
+            mouse_pointer = Image.open('cursor_image.png')
+            width, height = mouse_pointer.size
+        elif platform == 'win32' or platform == 'darwin':
+            mouse_pointer = Image.open('pictures/cursor_image.png')
+            width, height = mouse_pointer.size
+        else:
+            print('Not supported in this platform')
+            mouse_pointer = None
+            width, height = None, None
+        with mss.mss() as sct:
+            bbox = {"top": 0, "left": 0, "width": self.width, "height": self.height}
+            self.start_rec_time = time.time()
+            frame_time_ms = 1000 / self.fps
+            frames = 0
+            print(f"Starting to record with FPS value {self.fps} ...")
+            while self.record:
+                if platform == 'win32' or platform == 'darwin':
+                    x, y = pyautogui.position()
+                elif platform == 'linux':
+                    x, y = 1000, 500  # Fixed mouse pointer in linux at this coordinate. Throwing error of some kind.
+                    # Hence, the mouse pointer is not visible in the recordings but is at a fixed place.
+                    # Alternatively if you just record anything running screenrecorder.py directly this
+                    # pyautogui.position() works perfectly in any operating system.(ofcourse you need to change the
+                    # code by removing this platform thing). But when tutorials are called, it
+                    # is throwing some untraceable error in Linux.
+                img = Image.frombytes("RGBA", sct.grab(bbox).size, sct.grab(bbox).bgra, "raw", "RGBA")
+                img.paste(mouse_pointer, (x, y, x + width, y + height), mask=mouse_pointer)
+                img_np = np.array(img)
+                img_final = cv2.cvtColor(img_np, cv2.COLOR_RGBA2RGB)
+                cv2.imshow(self.window_name, img_final)
+                self.recorded_video.write(img_final)
+                frames += 1
+                expected_frames = ((time.time() - self.start_rec_time) * 1000) / frame_time_ms
+                surplus = frames - expected_frames
+
+                # Duplicate previous frame to meet FPS quota. Consider lowering the FPS instead!
+                if int(surplus) <= -1:
+                    frames -= int(surplus)
+                    for i in range(int(-surplus)):
+                        self.recorded_video.write(img_final)
+                # Exits the screen capturing when user press 'q'
+                if cv2.waitKey(max(int(surplus * frame_time_ms), 1)) & 0xFF == ord('q'):
+                    break
+
+    def capture2(self):
         """
         Captures the frames of the screen at the rate of fps frames/second and writes into the
         video writer object with the defined fourcc, codec and colour format.

tutorials/tutorial_waypoints.py

@@ -1,5 +1,5 @@
 """
-    mslib.msui.tutorial_waypoints
+    mss.tutorials.tutorial_waypoints
     ~~~~~~~~~~~~~~~~~~~
 
     This python script generates an automatic demonstration of how to play with and use waypoints
@@ -220,7 +220,7 @@ def automate_waypoints():
         pag.write(fig_filename, interval=0.25)
         pag.press('enter', interval=1)
     if platform == 'linux' or platform == 'linux2':
-        # pag.hotkey('altleft', 'tab') if the save file system window is not in the forefront, use this statement.
+        pag.hotkey('altleft', 'tab')  # if the save file system window is not in the forefront, use this statement.
         # This can happen sometimes. At that time, you just need to uncomment it.
         pag.write(fig_filename, interval=0.25)
         pag.press('enter', interval=1)