Implement mobile virtual keyboard support in web (#279)

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+- `prepare_render` step support for `EguiBevyPaintCallbackImpl` ([#306](https://github.com/mvlabat/bevy_egui/pull/306) by @PPakalns).
+- Mobile virtual keyboard support in web ([#279](https://github.com/mvlabat/bevy_egui/pull/279) by @v-kat).
+ - Requires `Window::prevent_default_event_handling` being set to `false`.
+- IME support (#[204](https://github.com/mvlabat/bevy_egui/pull/204) by @EReeves).
+
 ### Changed
 
 - Update Egui to 0.29 ([#313](https://github.com/mvlabat/bevy_egui/pull/313) by @PPakalns).

Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "bevy_egui"
 version = "0.29.0"
+rust-version = "1.80.0" # needed for LazyLock https://doc.rust-lang.org/stable/std/sync/struct.LazyLock.html
 authors = ["mvlabat <mvlabat@gmail.com>"]
 description = "A plugin for Egui integration into Bevy"
 license = "MIT"
@@ -21,6 +22,8 @@ open_url = ["webbrowser"]
 default_fonts = ["egui/default_fonts"]
 render = ["bevy/bevy_render"]
 serde = ["egui/serde"]
+# The enabled logs will print with the info log level, to make it less cumbersome to debug in browsers.
+log_input_events = []
 
 [[example]]
 name = "paint_callback"
@@ -76,18 +79,22 @@ egui = { version = "0.29", default-features = false, features = ["bytemuck"] }
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 winit = "0.30"
 web-sys = { version = "0.3.63", features = [
- "Clipboard",
- "ClipboardEvent",
- "DataTransfer",
- 'Document',
- 'EventTarget',
- "Window",
- "Navigator",
+ "Clipboard",
+ "ClipboardEvent",
+ "CompositionEvent",
+ "DataTransfer",
+ "Document",
+ "EventTarget",
+ "HtmlInputElement",
+ "InputEvent",
+ "KeyboardEvent",
+ "Navigator",
+ "TouchEvent",
+ "Window",
 ] }
 js-sys = "0.3.63"
 wasm-bindgen = "0.2.84"
 wasm-bindgen-futures = "0.4.36"
-console_log = "1.0.0"
 log = "0.4"
 crossbeam-channel = "0.5.8"
 

README.md

@@ -24,6 +24,7 @@ An example WASM project is live at [mvlabat.github.io/bevy_egui_web_showcase](ht
 - Opening URLs
 - Multiple windows support (see [./examples/two_windows.rs](https://github.com/mvlabat/bevy_egui/blob/v0.29.0/examples/two_windows.rs))
 - Paint callback support (see [./examples/paint_callback.rs](https://github.com/mvlabat/bevy_egui/blob/v0.29.0/examples/paint_callback.rs))
+- Mobile web virtual keyboard (still rough support and only works without prevent_default_event_handling set to false on the WindowPlugin primary_window)
 
 `bevy_egui` can be compiled with using only `bevy`, `egui` and `bytemuck` as dependencies: `manage_clipboard` and `open_url` features,
 that require additional crates, can be disabled.

examples/ui.rs

@@ -27,6 +27,7 @@ fn main() {
  .init_resource::<UiState>()
  .add_plugins(DefaultPlugins.set(WindowPlugin {
  primary_window: Some(Window {
+ // You may want this set to `true` if you need virtual keyboard work in mobile browsers.
  prevent_default_event_handling: false,
  ..default()
  }),

src/lib.rs

@@ -68,7 +68,10 @@ pub mod egui_render_to_texture_node;
 pub mod render_systems;
 /// Plugin systems.
 pub mod systems;
-/// Clipboard management for web.
+/// Mobile web keyboard hacky input support
+#[cfg(target_arch = "wasm32")]
+mod text_agent;
+/// Clipboard management for web
 #[cfg(all(
  feature = "manage_clipboard",
  target_arch = "wasm32",
@@ -129,6 +132,15 @@ use bevy::{
 ))]
 use std::cell::{RefCell, RefMut};
 
+#[cfg(target_arch = "wasm32")]
+use wasm_bindgen::prelude::*;
+
+#[cfg(target_arch = "wasm32")]
+use crate::text_agent::{
+ install_text_agent, is_mobile_safari, process_safari_virtual_keyboard, propagate_text,
+ SafariVirtualKeyboardHack, TextAgentChannel, VirtualTouchInfo,
+};
+
 /// Adds all Egui resources and render graph nodes.
 pub struct EguiPlugin;
 
@@ -673,6 +685,9 @@ impl Plugin for EguiPlugin {
  app.add_plugins(ExtractComponentPlugin::<EguiRenderToTextureHandle>::default());
  }
 
+ #[cfg(target_arch = "wasm32")]
+ app.init_non_send_resource::<SubscribedEvents>();
+
  #[cfg(all(feature = "manage_clipboard", not(target_os = "android")))]
  app.init_resource::<EguiClipboard>();
 
@@ -682,7 +697,6 @@ impl Plugin for EguiPlugin {
  web_sys_unstable_apis
  ))]
  {
- app.init_non_send_resource::<web_clipboard::SubscribedEvents>();
  app.add_systems(PreStartup, web_clipboard::startup_setup_web_events);
  }
 
@@ -716,6 +730,58 @@ impl Plugin for EguiPlugin {
  .after(InputSystem)
  .after(EguiSet::InitContexts),
  );
+ #[cfg(target_arch = "wasm32")]
+ {
+ use std::sync::{LazyLock, Mutex};
+
+ let maybe_window_plugin = app.get_added_plugins::<bevy::prelude::WindowPlugin>();
+
+ if !maybe_window_plugin.is_empty()
+ && maybe_window_plugin[0].primary_window.is_some()
+ && maybe_window_plugin[0]
+ .primary_window
+ .as_ref()
+ .unwrap()
+ .prevent_default_event_handling
+ {
+ app.init_resource::<TextAgentChannel>();
+
+ let (sender, receiver) = crossbeam_channel::unbounded();
+ static TOUCH_INFO: LazyLock<Mutex<VirtualTouchInfo>> =
+ LazyLock::new(|| Mutex::new(VirtualTouchInfo::default()));
+
+ app.insert_resource(SafariVirtualKeyboardHack {
+ sender,
+ receiver,
+ touch_info: &TOUCH_INFO,
+ });
+
+ app.add_systems(
+ PreStartup,
+ install_text_agent
+ .in_set(EguiSet::ProcessInput)
+ .after(process_input_system)
+ .after(InputSystem)
+ .after(EguiSet::InitContexts),
+ );
+
+ app.add_systems(
+ PreUpdate,
+ propagate_text
+ .in_set(EguiSet::ProcessInput)
+ .after(process_input_system)
+ .after(InputSystem)
+ .after(EguiSet::InitContexts),
+ );
+
+ if is_mobile_safari() {
+ app.add_systems(
+ PostUpdate,
+ process_safari_virtual_keyboard.after(process_output_system),
+ );
+ }
+ }
+ }
  app.add_systems(
  PreUpdate,
  begin_pass_system
@@ -978,6 +1044,63 @@ fn free_egui_textures_system(
  }
 }
 
+/// Helper function for outputting a String from a JsValue
+#[cfg(target_arch = "wasm32")]
+pub fn string_from_js_value(value: &JsValue) -> String {
+ value.as_string().unwrap_or_else(|| format!("{value:#?}"))
+}
+
+#[cfg(target_arch = "wasm32")]
+struct EventClosure<T> {
+ target: web_sys::EventTarget,
+ event_name: String,
+ closure: wasm_bindgen::closure::Closure<dyn FnMut(T)>,
+}
+
+/// Stores event listeners.
+#[cfg(target_arch = "wasm32")]
+#[derive(Default)]
+pub struct SubscribedEvents {
+ #[cfg(all(feature = "manage_clipboard", web_sys_unstable_apis))]
+ clipboard_event_closures: Vec<EventClosure<web_sys::ClipboardEvent>>,
+ composition_event_closures: Vec<EventClosure<web_sys::CompositionEvent>>,
+ keyboard_event_closures: Vec<EventClosure<web_sys::KeyboardEvent>>,
+ input_event_closures: Vec<EventClosure<web_sys::InputEvent>>,
+ touch_event_closures: Vec<EventClosure<web_sys::TouchEvent>>,
+}
+
+#[cfg(target_arch = "wasm32")]
+impl SubscribedEvents {
+ /// Use this method to unsubscribe from all stored events, this can be useful
+ /// for gracefully destroying a Bevy instance in a page.
+ pub fn unsubscribe_from_all_events(&mut self) {
+ #[cfg(all(feature = "manage_clipboard", web_sys_unstable_apis))]
+ Self::unsubscribe_from_events(&mut self.clipboard_event_closures);
+ Self::unsubscribe_from_events(&mut self.composition_event_closures);
+ Self::unsubscribe_from_events(&mut self.keyboard_event_closures);
+ Self::unsubscribe_from_events(&mut self.input_event_closures);
+ Self::unsubscribe_from_events(&mut self.touch_event_closures);
+ }
+
+ fn unsubscribe_from_events<T>(events: &mut Vec<EventClosure<T>>) {
+ let events_to_unsubscribe = std::mem::take(events);
+
+ if !events_to_unsubscribe.is_empty() {
+ for event in events_to_unsubscribe {
+ if let Err(err) = event.target.remove_event_listener_with_callback(
+ event.event_name.as_str(),
+ event.closure.as_ref().unchecked_ref(),
+ ) {
+ log::error!(
+ "Failed to unsubscribe from event: {}",
+ string_from_js_value(&err)
+ );
+ }
+ }
+ }
+ }
+}
+
 #[cfg(test)]
 mod tests {
  use super::*;

src/systems.rs

@@ -1,10 +1,11 @@
+#[cfg(target_arch = "wasm32")]
+use crate::text_agent::{is_mobile_safari, update_text_agent};
 #[cfg(feature = "render")]
 use crate::EguiRenderToTextureHandle;
 use crate::{
  EguiContext, EguiContextQuery, EguiContextQueryItem, EguiFullOutput, EguiInput, EguiSettings,
  RenderTargetSize,
 };
-
 #[cfg(feature = "render")]
 use bevy::{asset::Assets, render::texture::Image};
 use bevy::{
@@ -133,6 +134,8 @@ pub fn process_input_system(
  for event in input_events.ev_keyboard_input.read() {
  // Copy the events as we might want to pass them to an Egui context later.
  keyboard_input_events.push(event.clone());
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  let KeyboardInput {
  logical_key, state, ..
@@ -196,6 +199,8 @@ pub fn process_input_system(
  let Some(mut window_context) = context_params.window_context(event.window) else {
  continue;
  };
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  let button = match event.button {
  MouseButton::Left => Some(egui::PointerButton::Primary),
@@ -224,6 +229,8 @@ pub fn process_input_system(
  let Some(mut window_context) = context_params.window_context(event.window) else {
  continue;
  };
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  let delta = egui::vec2(event.x, event.y);
 
@@ -242,6 +249,17 @@ pub fn process_input_system(
  });
  }
 
+ #[cfg(target_arch = "wasm32")]
+ let mut editing_text = false;
+ #[cfg(target_arch = "wasm32")]
+ for context in context_params.contexts.iter() {
+ let platform_output = &context.egui_output.platform_output;
+ if platform_output.ime.is_some() || platform_output.mutable_text_under_cursor {
+ editing_text = true;
+ break;
+ }
+ }
+
  for event in input_events.ev_ime_input.read() {
  let window = match &event {
  Ime::Preedit { window, .. }
@@ -253,6 +271,8 @@ pub fn process_input_system(
  let Some(mut window_context) = context_params.window_context(window) else {
  continue;
  };
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  // Aligned with the egui-winit implementation: https://github.com/emilk/egui/blob/0f2b427ff4c0a8c68f6622ec7d0afb7ba7e71bba/crates/egui-winit/src/lib.rs#L348
  match event {
@@ -288,6 +308,8 @@ pub fn process_input_system(
  let Some(mut window_context) = context_params.window_context(event.window) else {
  continue;
  };
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  if text_event_allowed && event.state.is_pressed() {
  match &event.logical_key {
@@ -353,6 +375,8 @@ pub fn process_input_system(
  while let Some(event) = input_resources.egui_clipboard.try_receive_clipboard_event() {
  // In web, we assume that we have only 1 window per app.
  let mut window_context = context_params.contexts.single_mut();
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  match event {
  crate::web_clipboard::WebClipboardEvent::Copy => {
@@ -377,6 +401,8 @@ pub fn process_input_system(
  let Some(mut window_context) = context_params.window_context(event.window) else {
  continue;
  };
+ #[cfg(feature = "log_input_events")]
+ log::info!("{event:?}");
 
  let touch_id = egui::TouchId::from(event.id);
  let scale_factor = window_context.egui_settings.scale_factor;
@@ -456,6 +482,11 @@ pub fn process_input_system(
  .egui_input
  .events
  .push(egui::Event::PointerGone);
+
+ #[cfg(target_arch = "wasm32")]
+ if !is_mobile_safari() {
+ update_text_agent(editing_text);
+ }
  }
  bevy::input::touch::TouchPhase::Canceled => {
  window_context.ctx.pointer_touch_id = None;