Basic iOS IME support (rust-windowing#3823)

This implements basic iOS IME support (typing, backspace, support for emojis
etc but no autocomplete or copy / paste menu).

Co-authored-by: Mads Marquart <[email protected]>

Cargo.toml

@@ -183,6 +183,8 @@ objc2-ui-kit = { version = "0.2.2", features = [
     "UIEvent",
     "UIGeometry",
     "UIGestureRecognizer",
+    "UITextInput",
+    "UITextInputTraits",
     "UIOrientation",
     "UIPanGestureRecognizer",
     "UIPinchGestureRecognizer",

src/changelog/unreleased.md

@@ -47,13 +47,13 @@ changelog entry.
   `DeviceEvent::MouseMotion` is returning raw data, not OS accelerated, when using
   `CursorGrabMode::Locked`.
 - On Web, implement `MonitorHandle` and `VideoModeHandle`.
-  
+
   Without prompting the user for permission, only the current monitor is returned. But when
   prompting and being granted permission through
   `ActiveEventLoop::request_detailed_monitor_permission()`, access to all monitors and their
   details is available. Handles created with "detailed monitor permissions" can be used in
   `Window::set_fullscreen()` as well.
-  
+
   Keep in mind that handles do not auto-upgrade after permissions are granted and have to be
   re-created to make full use of this feature.
 - Add `Touch::finger_id` with a new type `FingerId`.
@@ -62,6 +62,7 @@ changelog entry.
 - Implement `Clone`, `Copy`, `Debug`, `Deserialize`, `Eq`, `Hash`, `Ord`, `PartialEq`, `PartialOrd`
   and `Serialize` on many types.
 - Add `MonitorHandle::current_video_mode()`.
+- Add basic iOS IME support. The soft keyboard can now be shown using `Window::set_ime_allowed`.
 
 ### Changed
 

src/platform_impl/apple/uikit/view.rs

@@ -4,19 +4,23 @@ use std::cell::{Cell, RefCell};
 use objc2::rc::Retained;
 use objc2::runtime::{NSObjectProtocol, ProtocolObject};
 use objc2::{declare_class, msg_send, msg_send_id, mutability, sel, ClassType, DeclaredClass};
-use objc2_foundation::{CGFloat, CGPoint, CGRect, MainThreadMarker, NSObject, NSSet};
+use objc2_foundation::{CGFloat, CGPoint, CGRect, MainThreadMarker, NSObject, NSSet, NSString};
 use objc2_ui_kit::{
     UICoordinateSpace, UIEvent, UIForceTouchCapability, UIGestureRecognizer,
-    UIGestureRecognizerDelegate, UIGestureRecognizerState, UIPanGestureRecognizer,
+    UIGestureRecognizerDelegate, UIGestureRecognizerState, UIKeyInput, UIPanGestureRecognizer,
     UIPinchGestureRecognizer, UIResponder, UIRotationGestureRecognizer, UITapGestureRecognizer,
-    UITouch, UITouchPhase, UITouchType, UITraitEnvironment, UIView,
+    UITextInputTraits, UITouch, UITouchPhase, UITouchType, UITraitEnvironment, UIView,
 };
 
 use super::app_state::{self, EventWrapper};
 use super::window::WinitUIWindow;
 use super::{FingerId, DEVICE_ID};
 use crate::dpi::PhysicalPosition;
-use crate::event::{Event, FingerId as RootFingerId, Force, Touch, TouchPhase, WindowEvent};
+use crate::event::{
+    ElementState, Event, FingerId as RootFingerId, Force, KeyEvent, Touch, TouchPhase, WindowEvent,
+};
+use crate::keyboard::{Key, KeyCode, KeyLocation, NamedKey, NativeKeyCode, PhysicalKey};
+use crate::platform_impl::KeyEventExtra;
 use crate::window::{WindowAttributes, WindowId as RootWindowId};
 
 pub struct WinitViewState {
@@ -314,6 +318,11 @@ declare_class!(
             let mtm = MainThreadMarker::new().unwrap();
             app_state::handle_nonuser_event(mtm, gesture_event);
         }
+
+        #[method(canBecomeFirstResponder)]
+        fn can_become_first_responder(&self) -> bool {
+            true
+        }
     }
 
     unsafe impl NSObjectProtocol for WinitView {}
@@ -324,6 +333,26 @@ declare_class!(
             true
         }
     }
+
+    unsafe impl UITextInputTraits for WinitView {
+    }
+
+    unsafe impl UIKeyInput for WinitView {
+        #[method(hasText)]
+        fn has_text(&self) -> bool {
+            true
+        }
+
+        #[method(insertText:)]
+        fn insert_text(&self, text: &NSString) {
+            self.handle_insert_text(text)
+        }
+
+        #[method(deleteBackward)]
+        fn delete_backward(&self) {
+            self.handle_delete_backward()
+        }
+    }
 );
 
 impl WinitView {
@@ -512,4 +541,69 @@ impl WinitView {
         let mtm = MainThreadMarker::new().unwrap();
         app_state::handle_nonuser_events(mtm, touch_events);
     }
+
+    fn handle_insert_text(&self, text: &NSString) {
+        let window = self.window().unwrap();
+        let window_id = RootWindowId(window.id());
+        let mtm = MainThreadMarker::new().unwrap();
+        // send individual events for each character
+        app_state::handle_nonuser_events(
+            mtm,
+            text.to_string().chars().flat_map(|c| {
+                let text = smol_str::SmolStr::from_iter([c]);
+                // Emit both press and release events
+                [ElementState::Pressed, ElementState::Released].map(|state| {
+                    EventWrapper::StaticEvent(Event::WindowEvent {
+                        window_id,
+                        event: WindowEvent::KeyboardInput {
+                            event: KeyEvent {
+                                text: if state == ElementState::Pressed {
+                                    Some(text.clone())
+                                } else {
+                                    None
+                                },
+                                state,
+                                location: KeyLocation::Standard,
+                                repeat: false,
+                                logical_key: Key::Character(text.clone()),
+                                physical_key: PhysicalKey::Unidentified(
+                                    NativeKeyCode::Unidentified,
+                                ),
+                                platform_specific: KeyEventExtra {},
+                            },
+                            is_synthetic: false,
+                            device_id: DEVICE_ID,
+                        },
+                    })
+                })
+            }),
+        );
+    }
+
+    fn handle_delete_backward(&self) {
+        let window = self.window().unwrap();
+        let window_id = RootWindowId(window.id());
+        let mtm = MainThreadMarker::new().unwrap();
+        app_state::handle_nonuser_events(
+            mtm,
+            [ElementState::Pressed, ElementState::Released].map(|state| {
+                EventWrapper::StaticEvent(Event::WindowEvent {
+                    window_id,
+                    event: WindowEvent::KeyboardInput {
+                        device_id: DEVICE_ID,
+                        event: KeyEvent {
+                            state,
+                            logical_key: Key::Named(NamedKey::Backspace),
+                            physical_key: PhysicalKey::Code(KeyCode::Backspace),
+                            platform_specific: KeyEventExtra {},
+                            repeat: false,
+                            location: KeyLocation::Standard,
+                            text: None,
+                        },
+                        is_synthetic: false,
+                    },
+                })
+            }),
+        );
+    }
 }

src/platform_impl/apple/uikit/window.rs

@@ -370,12 +370,24 @@ impl Inner {
         warn!("`Window::set_ime_cursor_area` is ignored on iOS")
     }
 
-    pub fn set_ime_allowed(&self, _allowed: bool) {
-        warn!("`Window::set_ime_allowed` is ignored on iOS")
+    /// Show / hide the keyboard. To show the keyboard, we call `becomeFirstResponder`,
+    /// requesting focus for the [WinitView]. Since [WinitView] implements
+    /// [objc2_ui_kit::UIKeyInput], the keyboard will be shown.
+    /// <https://developer.apple.com/documentation/uikit/uiresponder/1621113-becomefirstresponder>
+    pub fn set_ime_allowed(&self, allowed: bool) {
+        if allowed {
+            unsafe {
+                self.view.becomeFirstResponder();
+            }
+        } else {
+            unsafe {
+                self.view.resignFirstResponder();
+            }
+        }
     }
 
     pub fn set_ime_purpose(&self, _purpose: ImePurpose) {
-        warn!("`Window::set_ime_allowed` is ignored on iOS")
+        warn!("`Window::set_ime_purpose` is ignored on iOS")
     }
 
     pub fn focus_window(&self) {

src/window.rs

@@ -1279,7 +1279,8 @@ impl Window {
     ///
     /// - **macOS:** IME must be enabled to receive text-input where dead-key sequences are
     ///   combined.
-    /// - **iOS / Android / Web / Orbital:** Unsupported.
+    /// - **iOS:** This will show / hide the soft keyboard.
+    /// - **Android / Web / Orbital:** Unsupported.
     /// - **X11**: Enabling IME will disable dead keys reporting during compose.
     ///
     /// [`Ime`]: crate::event::WindowEvent::Ime