desktop: Implement basic IME support

core/src/context.rs

@@ -21,6 +21,7 @@ use crate::backend::{
 };
 use crate::context_menu::ContextMenuState;
 use crate::display_object::{EditText, MovieClip, SoundTransform, Stage};
+use crate::events::PlayerNotification;
 use crate::external::ExternalInterface;
 use crate::focus_tracker::FocusTracker;
 use crate::frame_lifecycle::FramePhase;
@@ -41,6 +42,7 @@ use crate::system_properties::SystemProperties;
 use crate::tag_utils::{SwfMovie, SwfSlice};
 use crate::timer::Timers;
 use crate::vminterface::Instantiator;
+use async_channel::Sender;
 use core::fmt;
 use gc_arena::{Collect, Mutation};
 use rand::rngs::SmallRng;
@@ -226,6 +228,8 @@ pub struct UpdateContext<'gc> {
     /// Currently, this is just used for handling `Loader.loadBytes`
     #[allow(clippy::type_complexity)]
     pub post_frame_callbacks: &'gc mut Vec<PostFrameCallback<'gc>>,
+
+    pub notification_sender: Option<&'gc Sender<PlayerNotification>>,
 }
 
 impl<'gc> HasStringContext<'gc> for UpdateContext<'gc> {
@@ -445,6 +449,14 @@ impl<'gc> UpdateContext<'gc> {
 
         self.set_root_movie(movie);
     }
+
+    pub fn send_notification(&self, notification: PlayerNotification) {
+        if let Some(notification_sender) = self.notification_sender {
+            if let Err(e) = notification_sender.try_send(notification) {
+                tracing::error!("Failed to send player notification: {e}");
+            }
+        }
+    }
 }
 
 impl<'gc> UpdateContext<'gc> {

core/src/display_object/edit_text.rs

@@ -19,7 +19,10 @@ use crate::display_object::interactive::{
     InteractiveObject, InteractiveObjectBase, TInteractiveObject,
 };
 use crate::display_object::{DisplayObjectBase, DisplayObjectPtr};
-use crate::events::{ClipEvent, ClipEventResult, TextControlCode};
+use crate::events::{
+    ClipEvent, ClipEventResult, ImeCursorArea, ImeEvent, ImeNotification, ImePurpose,
+    PlayerNotification, TextControlCode,
+};
 use crate::font::{FontType, Glyph, TextRenderSettings};
 use crate::html;
 use crate::html::StyleSheet;
@@ -1848,6 +1851,15 @@ impl<'gc> EditText<'gc> {
         }
     }
 
+    pub fn ime(self, event: ImeEvent, context: &mut UpdateContext<'gc>) {
+        match event {
+            ImeEvent::Preedit(_, _) => {
+                // TODO Add support for IME preedit
+            }
+            ImeEvent::Commit(text) => self.text_input(text, context),
+        };
+    }
+
     /// Find the new position in the text for the given control code.
     ///
     /// * For selection codes it will represent the "to" part of the selection.
@@ -2880,6 +2892,17 @@ impl<'gc> EditText<'gc> {
             context.commands.draw_line_rect(border_color, text_box);
         }
     }
+
+    fn ime_cursor_area(self) -> ImeCursorArea {
+        // TODO We should be smarter here and return an area closer to the cursor.
+        let bounds = self.world_bounds();
+        ImeCursorArea {
+            x: bounds.x_min.to_pixels(),
+            y: bounds.y_min.to_pixels(),
+            width: bounds.width().to_pixels(),
+            height: bounds.height().to_pixels(),
+        }
+    }
 }
 
 impl<'gc> TInteractiveObject<'gc> for EditText<'gc> {
@@ -3050,6 +3073,20 @@ impl<'gc> TInteractiveObject<'gc> for EditText<'gc> {
         if !focused && is_avm1 {
             self.set_selection(None, context.gc());
         }
+
+        // Notify about IME
+        context.send_notification(PlayerNotification::ImeNotification(if focused {
+            ImeNotification::ImeReady {
+                purpose: if self.is_password() {
+                    ImePurpose::Password
+                } else {
+                    ImePurpose::Standard
+                },
+                cursor_area: self.ime_cursor_area(),
+            }
+        } else {
+            ImeNotification::ImeNotReady
+        }));
     }
 
     fn is_focusable_by_mouse(&self, _context: &mut UpdateContext<'gc>) -> bool {

core/src/events.rs

@@ -44,6 +44,7 @@ pub enum PlayerEvent {
     TextControl {
         code: TextControlCode,
     },
+    Ime(ImeEvent),
     FocusGained,
     FocusLost,
 }
@@ -467,6 +468,25 @@ impl TextControlCode {
     }
 }
 
+/// Input method allows inputting non-Latin characters on a Latin keyboard.
+///
+/// When IME is enabled, Ruffle will accept [`ImeEvent`]s instead of key events.
+/// It allows dynamically changing the inputted text and then committing it at
+/// the end.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ImeEvent {
+    /// A new composing text should be set.
+    ///
+    /// The second parameter is the position of the cursor. When it's `None`,
+    /// the cursor should be hidden.
+    ///
+    /// An empty text indicates that preedit was cleared.
+    Preedit(String, Option<(usize, usize)>),
+
+    /// Composition is finished, and the text should be inserted.
+    Commit(String),
+}
+
 /// Flash virtual keycode.
 ///
 /// See <https://docs.ruffle.rs/en_US/FlashPlatform/reference/actionscript/3/flash/ui/Keyboard.html#summaryTableConstant>
@@ -1130,3 +1150,33 @@ pub enum KeyLocation {
     Right = 2,
     Numpad = 3,
 }
+
+#[derive(Debug, Clone)]
+pub enum PlayerNotification {
+    ImeNotification(ImeNotification),
+}
+
+#[derive(Debug, Clone)]
+pub enum ImeNotification {
+    ImeReady {
+        purpose: ImePurpose,
+        cursor_area: ImeCursorArea,
+    },
+    ImePurposeUpdated(ImePurpose),
+    ImeCursorAreaUpdated(ImeCursorArea),
+    ImeNotReady,
+}
+
+#[derive(Debug, Clone)]
+pub enum ImePurpose {
+    Standard,
+    Password,
+}
+
+#[derive(Debug, Clone)]
+pub struct ImeCursorArea {
+    pub x: f64,
+    pub y: f64,
+    pub width: f64,
+    pub height: f64,
+}

core/src/input.rs

@@ -1,6 +1,6 @@
 use crate::events::{
-    GamepadButton, KeyCode, KeyDescriptor, KeyLocation, LogicalKey, MouseButton, MouseWheelDelta,
-    NamedKey, PhysicalKey, PlayerEvent, TextControlCode,
+    GamepadButton, ImeEvent, KeyCode, KeyDescriptor, KeyLocation, LogicalKey, MouseButton,
+    MouseWheelDelta, NamedKey, PhysicalKey, PlayerEvent, TextControlCode,
 };
 use chrono::{DateTime, TimeDelta, Utc};
 use std::collections::{HashMap, HashSet};
@@ -53,6 +53,7 @@ pub enum InputEvent {
     TextControl {
         code: TextControlCode,
     },
+    Ime(ImeEvent),
 }
 
 struct ClickEventData {
@@ -190,6 +191,7 @@ impl InputManager {
 
             PlayerEvent::TextInput { codepoint } => InputEvent::TextInput { codepoint },
             PlayerEvent::TextControl { code } => InputEvent::TextControl { code },
+            PlayerEvent::Ime(ime) => InputEvent::Ime(ime),
 
             // The following are not input events.
             PlayerEvent::FocusGained | PlayerEvent::FocusLost => return None,

core/src/player.rs

@@ -26,6 +26,7 @@ use crate::display_object::{
     TInteractiveObject, WindowMode,
 };
 use crate::events::GamepadButton;
+use crate::events::PlayerNotification;
 use crate::events::{ButtonKeyCode, ClipEvent, ClipEventResult, KeyCode, MouseButton, PlayerEvent};
 use crate::external::{ExternalInterface, ExternalInterfaceProvider, NullFsCommandProvider};
 use crate::external::{FsCommandProvider, Value as ExternalValue};
@@ -49,6 +50,7 @@ use crate::tag_utils::SwfMovie;
 use crate::timer::Timers;
 use crate::vminterface::Instantiator;
 use crate::DefaultFont;
+use async_channel::Sender;
 use gc_arena::lock::GcRefLock;
 use gc_arena::{Collect, DynamicRootSet, Mutation, Rootable};
 use rand::{rngs::SmallRng, SeedableRng};
@@ -382,6 +384,9 @@ pub struct Player {
     /// Any compatibility rules to apply for this movie.
     compatibility_rules: CompatibilityRules,
 
+    /// Sends notifications back from the core player to the frontend.
+    notification_sender: Option<Sender<PlayerNotification>>,
+
     /// Debug UI windows
     #[cfg(feature = "egui")]
     debug_ui: Rc<RefCell<crate::debug_ui::DebugUi>>,
@@ -1011,6 +1016,7 @@ impl Player {
             | PlayerEvent::MouseWheel { .. }
             | PlayerEvent::GamepadButtonDown { .. }
             | PlayerEvent::GamepadButtonUp { .. }
+            | PlayerEvent::Ime { .. }
             | PlayerEvent::TextInput { .. }
             | PlayerEvent::TextControl { .. } => self.handle_input_event(event),
         }
@@ -1311,6 +1317,9 @@ impl Player {
                     if let InputEvent::TextControl { code } = &event {
                         text.text_control_input(*code, context);
                     }
+                    if let InputEvent::Ime(ime) = &event {
+                        text.ime(ime.clone(), context);
+                    }
                 }
             }
 
@@ -2243,6 +2252,7 @@ impl Player {
                 local_connections,
                 dynamic_root,
                 post_frame_callbacks,
+                notification_sender: this.notification_sender.as_ref(),
             };
 
             let prev_frame_rate = *update_context.frame_rate;
@@ -2449,6 +2459,9 @@ pub struct PlayerBuilder {
     ui: Option<Ui>,
     video: Option<Video>,
 
+    // Notifications
+    notification_sender: Option<Sender<PlayerNotification>>,
+
     // Misc. player configuration
     autoplay: bool,
     align: StageAlign,
@@ -2496,6 +2509,8 @@ impl PlayerBuilder {
             ui: None,
             video: None,
 
+            notification_sender: None,
+
             autoplay: false,
             align: StageAlign::default(),
             forced_align: false,
@@ -2600,6 +2615,13 @@ impl PlayerBuilder {
         self
     }
 
+    /// Sets the channel for player notifications.
+    #[inline]
+    pub fn with_notification_sender(mut self, sender: Sender<PlayerNotification>) -> Self {
+        self.notification_sender = Some(sender);
+        self
+    }
+
     /// Sets the stage scale mode and optionally prevents movies from changing it.
     #[inline]
     pub fn with_align(mut self, align: StageAlign, force: bool) -> Self {
@@ -2885,6 +2907,7 @@ impl PlayerBuilder {
                 spoofed_url: self.spoofed_url.clone(),
                 compatibility_rules: self.compatibility_rules.clone(),
                 stub_tracker: StubCollection::new(),
+                notification_sender: self.notification_sender,
                 #[cfg(feature = "egui")]
                 debug_ui: Default::default(),
 

desktop/Cargo.toml

@@ -49,6 +49,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }
 tracing-tracy = { version = "0.11.3", optional = true, features = ["demangle"] }
 rand = "0.8.5"
 thiserror.workspace = true
+async-channel.workspace = true
 
 [target.'cfg(target_os = "linux")'.dependencies]
 ashpd = "0.10.2"

desktop/assets/texts/en-US/preferences_dialog.ftl

@@ -41,3 +41,11 @@ gamemode-tooltip =
     Ruffle requests GameMode only when a movie is being played.
 gamemode-default = Default
 gamemode-default-tooltip = GameMode will be enabled only when power preference is set to high.
+
+# See for context https://wiki.archlinux.org/title/Input_method
+ime-enabled = Input Method
+ime-enabled-experimental = (experimental)
+ime-enabled-tooltip = An input method allows inputting characters that are not available on the kaybord, for instance Chinese, Japanese, or Korean characters.
+ime-enabled-default = Default
+ime-enabled-on = On
+ime-enabled-off = Off

desktop/src/app.rs

@@ -8,6 +8,7 @@ use crate::util::{
 };
 use anyhow::Error;
 use gilrs::{Event, EventType, Gilrs};
+use ruffle_core::events::{ImeEvent, ImeNotification, PlayerNotification};
 use ruffle_core::swf::HeaderExt;
 use ruffle_core::PlayerEvent;
 use ruffle_render::backend::ViewportDimensions;
@@ -16,7 +17,7 @@ use std::time::Instant;
 use url::Url;
 use winit::application::ApplicationHandler;
 use winit::dpi::{LogicalSize, PhysicalPosition, PhysicalSize, Size};
-use winit::event::{ElementState, KeyEvent, Modifiers, StartCause, WindowEvent};
+use winit::event::{ElementState, Ime, KeyEvent, Modifiers, StartCause, WindowEvent};
 use winit::event_loop::{ActiveEventLoop, ControlFlow, EventLoop, EventLoopProxy};
 use winit::keyboard::{Key, NamedKey};
 use winit::window::{Fullscreen, Icon, WindowAttributes, WindowId};
@@ -227,6 +228,18 @@ impl MainWindow {
                 };
                 self.check_redraw();
             }
+            WindowEvent::Ime(ime) => match ime {
+                Ime::Enabled => {}
+                Ime::Preedit(text, cursor) => {
+                    self.player
+                        .handle_event(PlayerEvent::Ime(ImeEvent::Preedit(text, cursor)));
+                }
+                Ime::Commit(text) => {
+                    self.player
+                        .handle_event(PlayerEvent::Ime(ImeEvent::Commit(text)));
+                }
+                Ime::Disabled => {}
+            },
             _ => (),
         }
     }
@@ -580,6 +593,33 @@ impl ApplicationHandler<RuffleEvent> for App {
                 }
             }
 
+            (Some(main_window), RuffleEvent::PlayerNotification(notification)) => {
+                match notification {
+                    PlayerNotification::ImeNotification(ImeNotification::ImeReady {
+                        purpose,
+                        cursor_area,
+                    }) => {
+                        let ime_enabled = main_window.preferences.ime_enabled().unwrap_or(false);
+                        main_window.gui.set_ime_allowed(ime_enabled);
+                        main_window.gui.set_ime_purpose(purpose);
+                        main_window.gui.set_ime_cursor_area(cursor_area);
+                    }
+                    PlayerNotification::ImeNotification(ImeNotification::ImePurposeUpdated(
+                        purpose,
+                    )) => {
+                        main_window.gui.set_ime_purpose(purpose);
+                    }
+                    PlayerNotification::ImeNotification(ImeNotification::ImeCursorAreaUpdated(
+                        cursor_area,
+                    )) => {
+                        main_window.gui.set_ime_cursor_area(cursor_area);
+                    }
+                    PlayerNotification::ImeNotification(ImeNotification::ImeNotReady) => {
+                        main_window.gui.set_ime_allowed(false);
+                    }
+                }
+            }
+
             (_, RuffleEvent::ExitRequested) => {
                 event_loop.exit();
             }