Add HID keyboard simulation via USB HID over AoAv2

BUILD.md

@@ -88,11 +88,12 @@ Install the required packages from your package manager.
 
 ```bash
 # runtime dependencies
-sudo apt install ffmpeg libsdl2-2.0-0 adb
+sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0
 
 # client build dependencies
 sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \
- libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
+ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev\
+ libusb-dev
 
 # server build dependencies
 sudo apt install openjdk-11-jdk
@@ -114,7 +115,7 @@ pip3 install meson
 sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm
 
 # client build dependencies
-sudo dnf install SDL2-devel ffms2-devel meson gcc make
+sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make
 
 # server build dependencies
 sudo dnf install java-devel
@@ -159,7 +160,8 @@ install the required packages:
 ```bash
 # runtime dependencies
 pacman -S mingw-w64-x86_64-SDL2 \
- mingw-w64-x86_64-ffmpeg
+ mingw-w64-x86_64-ffmpeg \
+ mingw-w64-x86_64-libusb
 
 # client build dependencies
 pacman -S mingw-w64-x86_64-make \
@@ -173,7 +175,8 @@ For a 32 bits version, replace `x86_64` by `i686`:
 ```bash
 # runtime dependencies
 pacman -S mingw-w64-i686-SDL2 \
- mingw-w64-i686-ffmpeg
+ mingw-w64-i686-ffmpeg \
+ mingw-w64-i686-libusb
 
 # client build dependencies
 pacman -S mingw-w64-i686-make \
@@ -197,7 +200,7 @@ Install the packages with [Homebrew]:
 
 ```bash
 # runtime dependencies
-brew install sdl2 ffmpeg
+brew install sdl2 ffmpeg libusb
 
 # client build dependencies
 brew install pkg-config meson

README.md

@@ -207,6 +207,29 @@ scrcpy --crop 1224:1440:0:0 # 1224x1440 at offset (0,0)
 
 If `--max-size` is also specified, resizing is applied after cropping.
 
+#### USB HID over AOAv2
+
+Scrcpy can simulate a USB physical keyboard on Android to provide better input
+experience, you need to connect your device via USB, not wireless.
+
+However, due to some limitation of libusb and WinUSB driver, you cannot use HID
+over AOAv2 on Windows.
+
+Currently a USB serial number is needed to use HID over AOAv2.
+
+```bash
+scrcpy --serial XXXXXXXXXXXXXXXX # don't use HID
+scrcpy --serial XXXXXXXXXXXXXXXX --input-mode inject # don't use HID
+scrcpy --serial XXXXXXXXXXXXXXXX --input-mode hid # try HID and exit if failed
+```
+
+Serial number can be found by `adb get-serialno`.
+
+If you are a non-QWERTY keyboard user and using HID mode, please remember to set
+correct physical keyboard layout manually in Android settings, because scrcpy
+just forwards scancodes to Android device and Android system is responsible for
+converting scancodes to correct keycode on Android device (your system does this
+on your PC).
 
 #### Lock video orientation
 

app/meson.build

@@ -1,6 +1,7 @@
 src = [
  'src/main.c',
  'src/adb.c',
+ 'src/aoa_hid.c',
  'src/cli.c',
  'src/clock.c',
  'src/compat.c',
@@ -12,6 +13,7 @@ src = [
  'src/file_handler.c',
  'src/fps_counter.c',
  'src/frame_buffer.c',
+ 'src/hid_keyboard.c',
  'src/input_manager.c',
  'src/opengl.c',
  'src/receiver.c',
@@ -54,6 +56,7 @@ if not get_option('crossbuild_windows')
  dependency('libavformat'),
  dependency('libavcodec'),
  dependency('libavutil'),
+ dependency('libusb-1.0'),
  dependency('sdl2'),
  ]
 
@@ -90,8 +93,20 @@ else
  include_directories: include_directories(ffmpeg_include_dir)
  )
 
+ prebuilt_libusb = meson.get_cross_property('prebuilt_libusb')
+ libusb_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_libusb + '/dll'
+ libusb_include_dir = '../prebuilt-deps/' + prebuilt_libusb + '/include'
+
+ libusb = declare_dependency(
+ dependencies: [
+ cc.find_library('libusb-1.0', dirs: libusb_bin_dir),
+ ],
+ include_directories: include_directories(libusb_include_dir)
+ )
+
  dependencies = [
  ffmpeg,
+ libusb,
  sdl2,
  cc.find_library('mingw32')
  ]

app/scrcpy.1

@@ -82,6 +82,16 @@ Start in fullscreen.
 .B \-h, \-\-help
 Print this help.
 
+.TP
+.B \-i, \-\-input\-mode mode
+Select input mode for keyboard events.
+
+Possible values are "hid" and "inject".
+
+"hid" uses Android's USB HID over AOAv2 feature to simulate physical keyboard's events, which provides better experience for IME users if supported by you device.
+
+"inject" is the legacy scrcpy way by injecting keycode events on Android, works on most devices and is the default.
+
 .TP
 .B \-\-legacy\-paste
 Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).