Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Windows: capture cursor #272

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 107 additions & 5 deletions src/mss/windows.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
RECT,
UINT,
WORD,
POINT,
HICON
)
from threading import local
from typing import Any, Optional
Expand All @@ -38,7 +40,9 @@


class BITMAPINFOHEADER(Structure):
"""Information about the dimensions and color format of a DIB."""
"""
Information about the dimensions and color format of a DIB.
"""

_fields_ = [
("biSize", DWORD),
Expand All @@ -51,7 +55,7 @@ class BITMAPINFOHEADER(Structure):
("biXPelsPerMeter", LONG),
("biYPelsPerMeter", LONG),
("biClrUsed", DWORD),
("biClrImportant", DWORD),
("biClrImportant", DWORD)
]


Expand All @@ -63,6 +67,33 @@ class BITMAPINFO(Structure):
_fields_ = [("bmiHeader", BITMAPINFOHEADER), ("bmiColors", DWORD * 3)]


class CURSORINFO(Structure):
"""
Information about the cursor.
"""

_fields_ = [
("cbSize", DWORD),
("flags", DWORD),
("hCursor", HDC),
("ptScreenPos", POINT)
]


class ICONINFO(Structure):
"""
Information about an icon or cursor.
"""

_fields_ = [
("fIcon", BOOL),
("xHotspot", DWORD),
("yHotspot", DWORD),
("hbmMask", HBITMAP),
("hbmColor", HBITMAP)
]


MONITORNUMPROC = WINFUNCTYPE(INT, DWORD, DWORD, POINTER(RECT), DOUBLE)


Expand All @@ -80,9 +111,12 @@ class BITMAPINFO(Structure):
"CreateCompatibleDC": ("gdi32", [HDC], HDC),
"DeleteDC": ("gdi32", [HDC], HDC),
"DeleteObject": ("gdi32", [HGDIOBJ], INT),
"DrawIcon": ("user32", [HDC, INT, INT, HICON], BOOL),
"EnumDisplayMonitors": ("user32", [HDC, c_void_p, MONITORNUMPROC, LPARAM], BOOL),
"GetCursorInfo": ("user32", [POINTER(CURSORINFO)], BOOL),
"GetDeviceCaps": ("gdi32", [HWND, INT], INT),
"GetDIBits": ("gdi32", [HDC, HBITMAP, UINT, UINT, c_void_p, POINTER(BITMAPINFO), UINT], BOOL),
"GetIconInfo": ("user32", [HICON, POINTER(ICONINFO)], BOOL),
"GetSystemMetrics": ("user32", [INT], INT),
"GetWindowDC": ("user32", [HWND], HDC),
"ReleaseDC": ("user32", [HWND, HDC], c_int),
Expand Down Expand Up @@ -121,6 +155,13 @@ def __init__(self, /, **kwargs: Any) -> None:
bmi.bmiHeader.biClrImportant = 0 # See grab.__doc__ [3]
self._handles.bmi = bmi

cursor_info = CURSORINFO()
cursor_info.cbSize = ctypes.sizeof(CURSORINFO)
self._handles.cursor_info = cursor_info

icon_info = ICONINFO() # 'ii' felt uncomfortable
self._handles.icon_info = icon_info

def close(self) -> None:
# Clean-up
if self._handles.bmp:
Expand Down Expand Up @@ -200,7 +241,7 @@ def _callback(monitor: int, data: HDC, rect: LPRECT, dc_: LPARAM) -> int:
callback = MONITORNUMPROC(_callback)
user32.EnumDisplayMonitors(0, 0, callback, 0)

def _grab_impl(self, monitor: Monitor, /) -> ScreenShot:
def _grab_impl(self, monitor: Monitor, /) -> Optional[ScreenShot]:
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like to not change that part. If the process fails, then an exception will be raised.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it was changed for the linux implementation so i did it but i can see why you would like to avoid that

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"""
Retrieve all pixels from a monitor. Pixels have to be RGB.

Expand Down Expand Up @@ -254,5 +295,66 @@ def _grab_impl(self, monitor: Monitor, /) -> ScreenShot:
return self.cls_image(bytearray(self._handles.data), monitor)

def _cursor_impl(self) -> Optional[ScreenShot]:
"""Retrieve all cursor data. Pixels have to be RGB."""
return None
"""Retrieve all cursor data. Pixels have to be RGB.

[1] user32.DrawIcon(HDC(memdc), 0, 0, hcursor)
Sometimes the memdc value is greater than the 32 bit limit
and that results in
'ctypes.ArgumentError: argument 1: OverflowError: int too long to convert'
but casting it to HDC type seems to fix the issue.

[2] user32.GetIconInfo(hcursor, self._handles.icon_info)
GetIconInfo also returns the handle for mask bitmap and the handle for color bitmap
but the color bitmap handle is null in case of monochrome cursors.

[3] is_monochrome = self._handles.icon_info.hbmColor is None
The correct way to detect monochrome cursors seems to be a unique property of their
mask bitmap. The height of the mask bitmap of a monochrome cursor is twice its width
(https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-iconinfo)
But I cannot find the correct way of getting a bitmap's dimensions, therefore I
just eneded up checking if the color bitmap is null.

[4]
The data received using DrawIcon is in the format BGRA but in case of monochrome
cursors the alpha value of every pixel is 0 for some reason. Therefore, the alpha
value of every non black pixel has to be manually set to 255.
"""
srcdc, memdc = self._handles.srcdc, self._handles.memdc
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I love the documentation 💪🏻

For the implementation, do we need to copy all from the grab() method? Can it be simplified to only the cursor itself?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i did not care to look into it but now that you said it i will see if something can be done about simplifying it to just the cursor
i will update you once im done

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

UPDATE: im afraid it would not be possible since the cursor is a part of the screen's device context, it should not be a huge issue as im making a small 32x32 bitmap and capturing only the cursor's image
im sorry if im wrong, im not an expert at windows api and i would love to learn something new

gdi, user32 = self.gdi32, self.user32
width, height = 32, 32
user32.GetCursorInfo(self._handles.cursor_info)
hcursor = self._handles.cursor_info.hCursor
pos_screen = self._handles.cursor_info.ptScreenPos

if self._handles.region_width_height != (width, height):
self._handles.region_width_height = (width, height)
self._handles.bmi.bmiHeader.biWidth = width
self._handles.bmi.bmiHeader.biHeight = -height
self._handles.data = ctypes.create_string_buffer(width * height * 4)
if self._handles.bmp:
gdi.DeleteObject(self._handles.bmp)
self._handles.bmp = gdi.CreateCompatibleBitmap(srcdc, width, height)
gdi.SelectObject(memdc, self._handles.bmp)

user32.DrawIcon(HDC(memdc), 0, 0, hcursor) # Why HDC? [1]
bits = gdi.GetDIBits(memdc, self._handles.bmp, 0, height, self._handles.data, self._handles.bmi, DIB_RGB_COLORS)
if bits != height:
raise ScreenShotError("gdi32.GetDIBits() failed.")

user32.GetIconInfo(hcursor, self._handles.icon_info) # [2]
is_monochrome = self._handles.icon_info.hbmColor is None # [3]
ratio = ctypes.windll.shcore.GetScaleFactorForDevice(0) / 100
region = {
"left": round(pos_screen.x * ratio - self._handles.icon_info.xHotspot),
"top": round(pos_screen.y * ratio - self._handles.icon_info.yHotspot),
Comment on lines +348 to +349

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, I used this PR in my project and from my testing multiplying by ratio actually messes things up. When i removed it everything worked fine even on strange DPI scalings in multiple different positions of multi monitor setup. For testing I used https://github.com/pavlobu/deskreen with https://www.amyuni.com/forum/viewtopic.php?t=3030

"width": 32,
"height": 32
}
data = bytearray(self._handles.data)
if is_monochrome:
for i in range(3, len(data), 4): # [4]
if data[i-3:i] == b"\x00\x00\x00":
data[i] = 0
else:
data[i] = 255
return self.cls_image(data, region)