Implement mobile virtual keyboard support in web (#279)

Co-authored-by: v-kat <[email protected]>

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+- `prepare_render` step support for `EguiBevyPaintCallbackImpl` ([#306](https://github.com/mvlabat/bevy_egui/pull/306) by @PPakalns).
+- Mobile virtual keyboard support in web ([#279](https://github.com/mvlabat/bevy_egui/pull/279) by @v-kat).
+  - Requires `Window::prevent_default_event_handling` being set to `false`.
+- IME support (#[204](https://github.com/mvlabat/bevy_egui/pull/204) by @EReeves).
+
 ### Changed
 
 - Update Egui to 0.29 ([#313](https://github.com/mvlabat/bevy_egui/pull/313) by @PPakalns).

Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "bevy_egui"
 version = "0.29.0"
+rust-version = "1.80.0" # needed for LazyLock https://doc.rust-lang.org/stable/std/sync/struct.LazyLock.html
 authors = ["mvlabat <[email protected]>"]
 description = "A plugin for Egui integration into Bevy"
 license = "MIT"
@@ -21,6 +22,8 @@ open_url = ["webbrowser"]
 default_fonts = ["egui/default_fonts"]
 render = ["bevy/bevy_render"]
 serde = ["egui/serde"]
+# The enabled logs will print with the info log level, to make it less cumbersome to debug in browsers.
+log_input_events = []
 
 [[example]]
 name = "paint_callback"
@@ -76,18 +79,22 @@ egui = { version = "0.29", default-features = false, features = ["bytemuck"] }
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 winit = "0.30"
 web-sys = { version = "0.3.63", features = [
-  "Clipboard",
-  "ClipboardEvent",
-  "DataTransfer",
-  'Document',
-  'EventTarget',
-  "Window",
-  "Navigator",
+    "Clipboard",
+    "ClipboardEvent",
+    "CompositionEvent",
+    "DataTransfer",
+    "Document",
+    "EventTarget",
+    "HtmlInputElement",
+    "InputEvent",
+    "KeyboardEvent",
+    "Navigator",
+    "TouchEvent",
+    "Window",
 ] }
 js-sys = "0.3.63"
 wasm-bindgen = "0.2.84"
 wasm-bindgen-futures = "0.4.36"
-console_log = "1.0.0"
 log = "0.4"
 crossbeam-channel = "0.5.8"
 

README.md

@@ -24,6 +24,7 @@ An example WASM project is live at [mvlabat.github.io/bevy_egui_web_showcase](ht
 - Opening URLs
 - Multiple windows support (see [./examples/two_windows.rs](https://github.com/mvlabat/bevy_egui/blob/v0.29.0/examples/two_windows.rs))
 - Paint callback support (see [./examples/paint_callback.rs](https://github.com/mvlabat/bevy_egui/blob/v0.29.0/examples/paint_callback.rs))
+- Mobile web virtual keyboard (still rough support and only works without prevent_default_event_handling set to false on the WindowPlugin primary_window)
 
 `bevy_egui` can be compiled with using only `bevy`, `egui` and `bytemuck` as dependencies: `manage_clipboard` and `open_url` features,
 that require additional crates, can be disabled.

examples/ui.rs

@@ -27,6 +27,7 @@ fn main() {
         .init_resource::<UiState>()
         .add_plugins(DefaultPlugins.set(WindowPlugin {
             primary_window: Some(Window {
+                // You may want this set to `true` if you need virtual keyboard work in mobile browsers.
                 prevent_default_event_handling: false,
                 ..default()
             }),

src/lib.rs

@@ -68,7 +68,10 @@ pub mod egui_render_to_texture_node;
 pub mod render_systems;
 /// Plugin systems.
 pub mod systems;
-/// Clipboard management for web.
+/// Mobile web keyboard hacky input support
+#[cfg(target_arch = "wasm32")]
+mod text_agent;
+/// Clipboard management for web
 #[cfg(all(
     feature = "manage_clipboard",
     target_arch = "wasm32",
@@ -129,6 +132,15 @@ use bevy::{
 ))]
 use std::cell::{RefCell, RefMut};
 
+#[cfg(target_arch = "wasm32")]
+use wasm_bindgen::prelude::*;
+
+#[cfg(target_arch = "wasm32")]
+use crate::text_agent::{
+    install_text_agent, is_mobile_safari, process_safari_virtual_keyboard, propagate_text,
+    SafariVirtualKeyboardHack, TextAgentChannel, VirtualTouchInfo,
+};
+
 /// Adds all Egui resources and render graph nodes.
 pub struct EguiPlugin;
 
@@ -673,6 +685,9 @@ impl Plugin for EguiPlugin {
             app.add_plugins(ExtractComponentPlugin::<EguiRenderToTextureHandle>::default());
         }
 
+        #[cfg(target_arch = "wasm32")]
+        app.init_non_send_resource::<SubscribedEvents>();
+
         #[cfg(all(feature = "manage_clipboard", not(target_os = "android")))]
         app.init_resource::<EguiClipboard>();
 
@@ -682,7 +697,6 @@ impl Plugin for EguiPlugin {
             web_sys_unstable_apis
         ))]
         {
-            app.init_non_send_resource::<web_clipboard::SubscribedEvents>();
             app.add_systems(PreStartup, web_clipboard::startup_setup_web_events);
         }
 
@@ -716,6 +730,58 @@ impl Plugin for EguiPlugin {
                 .after(InputSystem)
                 .after(EguiSet::InitContexts),
         );
+        #[cfg(target_arch = "wasm32")]
+        {
+            use std::sync::{LazyLock, Mutex};
+
+            let maybe_window_plugin = app.get_added_plugins::<bevy::prelude::WindowPlugin>();
+
+            if !maybe_window_plugin.is_empty()
+                && maybe_window_plugin[0].primary_window.is_some()
+                && maybe_window_plugin[0]
+                    .primary_window
+                    .as_ref()
+                    .unwrap()
+                    .prevent_default_event_handling
+            {
+                app.init_resource::<TextAgentChannel>();
+
+                let (sender, receiver) = crossbeam_channel::unbounded();
+                static TOUCH_INFO: LazyLock<Mutex<VirtualTouchInfo>> =
+                    LazyLock::new(|| Mutex::new(VirtualTouchInfo::default()));
+
+                app.insert_resource(SafariVirtualKeyboardHack {
+                    sender,
+                    receiver,
+                    touch_info: &TOUCH_INFO,
+                });
+
+                app.add_systems(
+                    PreStartup,
+                    install_text_agent
+                        .in_set(EguiSet::ProcessInput)
+                        .after(process_input_system)
+                        .after(InputSystem)
+                        .after(EguiSet::InitContexts),
+                );
+
+                app.add_systems(
+                    PreUpdate,
+                    propagate_text
+                        .in_set(EguiSet::ProcessInput)
+                        .after(process_input_system)
+                        .after(InputSystem)
+                        .after(EguiSet::InitContexts),
+                );
+
+                if is_mobile_safari() {
+                    app.add_systems(
+                        PostUpdate,
+                        process_safari_virtual_keyboard.after(process_output_system),
+                    );
+                }
+            }
+        }
         app.add_systems(
             PreUpdate,
             begin_pass_system
@@ -978,6 +1044,63 @@ fn free_egui_textures_system(
     }
 }
 
+/// Helper function for outputting a String from a JsValue
+#[cfg(target_arch = "wasm32")]
+pub fn string_from_js_value(value: &JsValue) -> String {
+    value.as_string().unwrap_or_else(|| format!("{value:#?}"))
+}
+
+#[cfg(target_arch = "wasm32")]
+struct EventClosure<T> {
+    target: web_sys::EventTarget,
+    event_name: String,
+    closure: wasm_bindgen::closure::Closure<dyn FnMut(T)>,
+}
+
+/// Stores event listeners.
+#[cfg(target_arch = "wasm32")]
+#[derive(Default)]
+pub struct SubscribedEvents {
+    #[cfg(all(feature = "manage_clipboard", web_sys_unstable_apis))]
+    clipboard_event_closures: Vec<EventClosure<web_sys::ClipboardEvent>>,
+    composition_event_closures: Vec<EventClosure<web_sys::CompositionEvent>>,
+    keyboard_event_closures: Vec<EventClosure<web_sys::KeyboardEvent>>,
+    input_event_closures: Vec<EventClosure<web_sys::InputEvent>>,
+    touch_event_closures: Vec<EventClosure<web_sys::TouchEvent>>,
+}
+
+#[cfg(target_arch = "wasm32")]
+impl SubscribedEvents {
+    /// Use this method to unsubscribe from all stored events, this can be useful
+    /// for gracefully destroying a Bevy instance in a page.
+    pub fn unsubscribe_from_all_events(&mut self) {
+        #[cfg(all(feature = "manage_clipboard", web_sys_unstable_apis))]
+        Self::unsubscribe_from_events(&mut self.clipboard_event_closures);
+        Self::unsubscribe_from_events(&mut self.composition_event_closures);
+        Self::unsubscribe_from_events(&mut self.keyboard_event_closures);
+        Self::unsubscribe_from_events(&mut self.input_event_closures);
+        Self::unsubscribe_from_events(&mut self.touch_event_closures);
+    }
+
+    fn unsubscribe_from_events<T>(events: &mut Vec<EventClosure<T>>) {
+        let events_to_unsubscribe = std::mem::take(events);
+
+        if !events_to_unsubscribe.is_empty() {
+            for event in events_to_unsubscribe {
+                if let Err(err) = event.target.remove_event_listener_with_callback(
+                    event.event_name.as_str(),
+                    event.closure.as_ref().unchecked_ref(),
+                ) {
+                    log::error!(
+                        "Failed to unsubscribe from event: {}",
+                        string_from_js_value(&err)
+                    );
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/systems.rs

@@ -1,10 +1,11 @@
+#[cfg(target_arch = "wasm32")]
+use crate::text_agent::{is_mobile_safari, update_text_agent};
 #[cfg(feature = "render")]
 use crate::EguiRenderToTextureHandle;
 use crate::{
     EguiContext, EguiContextQuery, EguiContextQueryItem, EguiFullOutput, EguiInput, EguiSettings,
     RenderTargetSize,
 };
-
 #[cfg(feature = "render")]
 use bevy::{asset::Assets, render::texture::Image};
 use bevy::{
@@ -133,6 +134,8 @@ pub fn process_input_system(
     for event in input_events.ev_keyboard_input.read() {
         // Copy the events as we might want to pass them to an Egui context later.
         keyboard_input_events.push(event.clone());
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         let KeyboardInput {
             logical_key, state, ..
@@ -196,6 +199,8 @@ pub fn process_input_system(
         let Some(mut window_context) = context_params.window_context(event.window) else {
             continue;
         };
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         let button = match event.button {
             MouseButton::Left => Some(egui::PointerButton::Primary),
@@ -224,6 +229,8 @@ pub fn process_input_system(
         let Some(mut window_context) = context_params.window_context(event.window) else {
             continue;
         };
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         let delta = egui::vec2(event.x, event.y);
 
@@ -242,6 +249,17 @@ pub fn process_input_system(
             });
     }
 
+    #[cfg(target_arch = "wasm32")]
+    let mut editing_text = false;
+    #[cfg(target_arch = "wasm32")]
+    for context in context_params.contexts.iter() {
+        let platform_output = &context.egui_output.platform_output;
+        if platform_output.ime.is_some() || platform_output.mutable_text_under_cursor {
+            editing_text = true;
+            break;
+        }
+    }
+
     for event in input_events.ev_ime_input.read() {
         let window = match &event {
             Ime::Preedit { window, .. }
@@ -253,6 +271,8 @@ pub fn process_input_system(
         let Some(mut window_context) = context_params.window_context(window) else {
             continue;
         };
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         // Aligned with the egui-winit implementation: https://github.com/emilk/egui/blob/0f2b427ff4c0a8c68f6622ec7d0afb7ba7e71bba/crates/egui-winit/src/lib.rs#L348
         match event {
@@ -288,6 +308,8 @@ pub fn process_input_system(
         let Some(mut window_context) = context_params.window_context(event.window) else {
             continue;
         };
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         if text_event_allowed && event.state.is_pressed() {
             match &event.logical_key {
@@ -353,6 +375,8 @@ pub fn process_input_system(
     while let Some(event) = input_resources.egui_clipboard.try_receive_clipboard_event() {
         // In web, we assume that we have only 1 window per app.
         let mut window_context = context_params.contexts.single_mut();
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         match event {
             crate::web_clipboard::WebClipboardEvent::Copy => {
@@ -377,6 +401,8 @@ pub fn process_input_system(
         let Some(mut window_context) = context_params.window_context(event.window) else {
             continue;
         };
+        #[cfg(feature = "log_input_events")]
+        log::info!("{event:?}");
 
         let touch_id = egui::TouchId::from(event.id);
         let scale_factor = window_context.egui_settings.scale_factor;
@@ -456,6 +482,11 @@ pub fn process_input_system(
                         .egui_input
                         .events
                         .push(egui::Event::PointerGone);
+
+                    #[cfg(target_arch = "wasm32")]
+                    if !is_mobile_safari() {
+                        update_text_agent(editing_text);
+                    }
                 }
                 bevy::input::touch::TouchPhase::Canceled => {
                     window_context.ctx.pointer_touch_id = None;