From 2d478cc2750e233e0e6d9b2622dcf449eeeeb4e6 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 16:39:04 +0800 Subject: [PATCH 01/23] add com wrapper --- packages/desktop/src/common/windows/com.rs | 35 ++++++++++++++++++++++ packages/desktop/src/common/windows/mod.rs | 2 ++ 2 files changed, 37 insertions(+) create mode 100644 packages/desktop/src/common/windows/com.rs diff --git a/packages/desktop/src/common/windows/com.rs b/packages/desktop/src/common/windows/com.rs new file mode 100644 index 00000000..20f8e8b3 --- /dev/null +++ b/packages/desktop/src/common/windows/com.rs @@ -0,0 +1,35 @@ +use windows::Win32::System::Com::{ + CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, +}; + +thread_local! { + /// Manages per-thread COM initialization. COM must be initialized on each + /// thread that uses it, so we store this in thread-local storage to handle + /// the setup and cleanup automatically. + pub static COM_INIT: ComInit = ComInit::new(); +} + +pub struct ComInit(); + +impl ComInit { + /// Initializes COM on the current thread with apartment threading model. + /// `COINIT_APARTMENTTHREADED` is required for shell COM objects. + /// + /// # Panics + /// + /// Panics if COM initialization fails. This is typically only possible + /// if COM is already initialized with an incompatible threading model. + pub fn new() -> Self { + unsafe { CoInitializeEx(None, COINIT_APARTMENTTHREADED) } + .ok() + .expect("Unable to initialize COM."); + + Self() + } +} + +impl Drop for ComInit { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } +} diff --git a/packages/desktop/src/common/windows/mod.rs b/packages/desktop/src/common/windows/mod.rs index 26e2af99..6ea74834 100644 --- a/packages/desktop/src/common/windows/mod.rs +++ b/packages/desktop/src/common/windows/mod.rs @@ -1,5 +1,7 @@ mod app_bar; +mod com; mod window_ext_windows; pub use app_bar::*; +pub use com::*; pub use window_ext_windows::*; From 12f46f84bbfb09380177fae404c337e203d3bdcd Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 16:39:08 +0800 Subject: [PATCH 02/23] wip --- .../src/providers/audio/audio_provider.rs | 115 +++++++++++++----- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index d150b529..ca09866d 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -35,14 +35,17 @@ use crate::providers::{ CommonProviderState, Provider, ProviderEmitter, RuntimeType, }; -static PROVIDER_TX: OnceLock = OnceLock::new(); - -static AUDIO_STATE: OnceLock>> = OnceLock::new(); - #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] pub struct AudioProviderConfig {} +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AudioOutput { + pub playback_devices: Vec, + pub default_playback_device: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AudioDevice { @@ -52,20 +55,11 @@ pub struct AudioDevice { pub is_default: bool, } -#[derive(Debug, Clone, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AudioOutput { - pub playback_devices: Vec, - pub default_playback_device: Option, -} - -impl AudioOutput { - fn new() -> Self { - Self { - playback_devices: Vec::new(), - default_playback_device: None, - } - } +/// Events that can be emitted from audio state changes. +#[derive(Debug)] +enum AudioEvent { + DeviceStateChanged, + VolumeChanged(String, u32), } #[derive(Clone)] @@ -74,6 +68,75 @@ struct DeviceInfo { endpoint_volume: IAudioEndpointVolume, } +impl AudioProvider { + pub fn new( + _config: AudioProviderConfig, + common: CommonProviderState, + ) -> Self { + let (event_sender, event_receiver) = unbounded(); + + Self { + common, + event_sender, + event_receiver, + } + } + + fn create_audio_manager(&mut self) -> anyhow::Result<()> { + unsafe { + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + let enumerator: IMMDeviceEnumerator = + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + let mut device_manager = + DeviceManager::new(enumerator.clone(), self.event_sender.clone()); + + // Initial device enumeration and output emission + let output = device_manager.enumerate_devices()?; + self.common.emitter.emit_output(Ok(output)); + + // Register for device notifications + let notification_handler = + DeviceNotificationHandler::new(self.event_sender.clone()); + enumerator.RegisterEndpointNotificationCallback( + &IMMNotificationClient::from(notification_handler), + )?; + + // Process events + while let Ok(event) = self.event_receiver.recv() { + match event { + AudioEvent::DeviceStateChanged => { + if let Ok(output) = device_manager.enumerate_devices() { + self.common.emitter.emit_output(Ok(output)); + } + } + AudioEvent::VolumeChanged(device_id, volume) => { + if let Ok(mut output) = device_manager.enumerate_devices() { + if let Some(device) = output + .playback_devices + .iter_mut() + .find(|d| d.device_id == device_id) + { + device.volume = volume; + } + if let Some(default_device) = + &mut output.default_playback_device + { + if default_device.device_id == device_id { + default_device.volume = volume; + } + } + self.common.emitter.emit_output(Ok(output)); + } + } + } + } + + Ok(()) + } + } +} + #[derive(Clone)] #[windows::core::implement( IMMNotificationClient, @@ -377,21 +440,7 @@ impl Provider for AudioProvider { } fn start_sync(&mut self) { - PROVIDER_TX - .set(self.common.emitter.clone()) - .expect("Error setting provider tx in audio provider"); - - AUDIO_STATE - .set(Arc::new(Mutex::new(AudioOutput::new()))) - .expect("Error setting initial audio state"); - - // Create a channel for volume updates. - let (update_tx, update_rx) = mpsc::channel(100); - - // Spawn task for handling volume updates. - task::spawn_blocking(move || Self::handle_volume_updates(update_rx)); - - if let Err(err) = Self::create_audio_manager(update_tx) { + if let Err(err) = self.create_audio_manager() { self.common.emitter.emit_output::(Err(err)); } } From 8011a6d951081ac932d82a2088603ef6bf643a4c Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 17:07:07 +0800 Subject: [PATCH 03/23] wip --- .../src/providers/audio/audio_provider.rs | 560 +++++++++--------- 1 file changed, 265 insertions(+), 295 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index ca09866d..69b843e9 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -5,6 +5,8 @@ use std::{ time::Duration, }; +use anyhow::Context; +use crossbeam::channel; use serde::{Deserialize, Serialize}; use tokio::{ sync::mpsc::{self}, @@ -14,7 +16,7 @@ use tracing::debug; use windows::Win32::{ Devices::FunctionDiscovery::PKEY_Device_FriendlyName, Media::Audio::{ - eMultimedia, eRender, EDataFlow, ERole, + eCapture, eMultimedia, eRender, EDataFlow, ERole, Endpoints::{ IAudioEndpointVolume, IAudioEndpointVolumeCallback, IAudioEndpointVolumeCallback_Impl, @@ -43,7 +45,9 @@ pub struct AudioProviderConfig {} #[serde(rename_all = "camelCase")] pub struct AudioOutput { pub playback_devices: Vec, + pub recording_devices: Vec, pub default_playback_device: Option, + pub default_recording_device: Option, } #[derive(Debug, Clone, PartialEq, Serialize)] @@ -51,21 +55,48 @@ pub struct AudioOutput { pub struct AudioDevice { pub name: String, pub device_id: String, + pub device_type: DeviceType, pub volume: u32, pub is_default: bool, } +// TODO: Should there be handling for devices that can be both playback and +// recording? +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum DeviceType { + Playback, + Recording, +} + +impl From for DeviceType { + fn from(flow: EDataFlow) -> Self { + match flow { + e if e == eRender => Self::Playback, + e if e == eCapture => Self::Recording, + _ => Self::Playback, + } + } +} + /// Events that can be emitted from audio state changes. #[derive(Debug)] enum AudioEvent { - DeviceStateChanged, - VolumeChanged(String, u32), + DeviceAdded(String), + DeviceRemoved(String), + DeviceStateChanged(String, DEVICE_STATE), + DefaultDeviceChanged(String), + VolumeChanged(String, f32), } -#[derive(Clone)] -struct DeviceInfo { - name: String, - endpoint_volume: IAudioEndpointVolume, +// Main provider implementation +pub struct AudioProvider { + common: CommonProviderState, + enumerator: Option, + device_volumes: HashMap, + current_state: AudioOutput, + event_sender: channel::Sender, + event_receiver: channel::Receiver, } impl AudioProvider { @@ -73,375 +104,314 @@ impl AudioProvider { _config: AudioProviderConfig, common: CommonProviderState, ) -> Self { - let (event_sender, event_receiver) = unbounded(); + let (event_sender, event_receiver) = channel::unbounded(); Self { common, + enumerator: None, + device_volumes: HashMap::new(), + current_state: AudioOutput { + playback_devices: Vec::new(), + recording_devices: Vec::new(), + default_playback_device: None, + default_recording_device: None, + }, event_sender, event_receiver, } } - fn create_audio_manager(&mut self) -> anyhow::Result<()> { - unsafe { - let _ = CoInitializeEx(None, COINIT_MULTITHREADED); - let enumerator: IMMDeviceEnumerator = - CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; - - let mut device_manager = - DeviceManager::new(enumerator.clone(), self.event_sender.clone()); - - // Initial device enumeration and output emission - let output = device_manager.enumerate_devices()?; - self.common.emitter.emit_output(Ok(output)); - - // Register for device notifications - let notification_handler = - DeviceNotificationHandler::new(self.event_sender.clone()); - enumerator.RegisterEndpointNotificationCallback( - &IMMNotificationClient::from(notification_handler), - )?; - - // Process events - while let Ok(event) = self.event_receiver.recv() { - match event { - AudioEvent::DeviceStateChanged => { - if let Ok(output) = device_manager.enumerate_devices() { - self.common.emitter.emit_output(Ok(output)); - } - } - AudioEvent::VolumeChanged(device_id, volume) => { - if let Ok(mut output) = device_manager.enumerate_devices() { - if let Some(device) = output - .playback_devices - .iter_mut() - .find(|d| d.device_id == device_id) - { - device.volume = volume; - } - if let Some(default_device) = - &mut output.default_playback_device - { - if default_device.device_id == device_id { - default_device.volume = volume; - } - } - self.common.emitter.emit_output(Ok(output)); - } - } - } - } - - Ok(()) - } - } -} - -#[derive(Clone)] -#[windows::core::implement( - IMMNotificationClient, - IAudioEndpointVolumeCallback -)] -struct MediaDeviceEventHandler { - enumerator: IMMDeviceEnumerator, - device_state: Arc>>, - current_device: String, - update_tx: mpsc::Sender<(String, u32)>, -} - -impl MediaDeviceEventHandler { - fn new( - enumerator: IMMDeviceEnumerator, - update_tx: mpsc::Sender<(String, u32)>, - ) -> Self { - Self { - enumerator, - device_state: Arc::new(Mutex::new(HashMap::new())), - current_device: String::new(), - update_tx, - } - } - - fn get_device_name(device: &IMMDevice) -> windows::core::Result { - unsafe { - let store: IPropertyStore = device.OpenPropertyStore(STGM_READ)?; - let value = store.GetValue(&PKEY_Device_FriendlyName)?; - Ok(value.to_string()) - } - } - - fn get_device_info( + fn get_device_properties( &self, device: &IMMDevice, - ) -> windows::core::Result { + ) -> anyhow::Result<(String, String)> { unsafe { let device_id = device.GetId()?.to_string()?; - let mut device_state = self.device_state.lock().unwrap(); - - if !device_state.contains_key(&device_id) { - let new_device = self.register_new_device(device)?; - device_state.insert(device_id.clone(), new_device); - } - - let device_info = device_state.get(&device_id).unwrap(); - let is_default = self.is_default_device(&device_id)?; - let volume = device_info - .endpoint_volume - .GetMasterVolumeLevelScalar() - .unwrap_or(0.0) - .mul(100.0) - .round() as u32; - - Ok(AudioDevice { - device_id, - name: device_info.name.clone(), - volume, - is_default, - }) + let store: IPropertyStore = device.OpenPropertyStore(STGM_READ)?; + let friendly_name = + store.GetValue(&PKEY_Device_FriendlyName)?.to_string(); + Ok((device_id, friendly_name)) } } - fn register_new_device( + fn register_volume_callback( &self, device: &IMMDevice, - ) -> windows::core::Result { + device_id: String, + ) -> anyhow::Result { unsafe { - let device_name = Self::get_device_name(device)?; let endpoint_volume: IAudioEndpointVolume = device.Activate(CLSCTX_ALL, None)?; - let mut handler = self.clone(); - handler.current_device = device.GetId()?.to_string()?; + let callback = VolumeCallback { + device_id, + event_sender: self.event_sender.clone(), + }; + endpoint_volume.RegisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(handler), + &IAudioEndpointVolumeCallback::from(callback), )?; - Ok(DeviceInfo { - name: device_name, - endpoint_volume, - }) + Ok(endpoint_volume) } } - fn is_default_device( - &self, - device_id: &str, - ) -> windows::core::Result { - unsafe { - let default = self - .enumerator - .GetDefaultAudioEndpoint(eRender, eMultimedia)?; - let default_id = default.GetId()?.to_string()?; - Ok(default_id == device_id) - } - } + fn update_device_state(&mut self) -> anyhow::Result<()> { + let enumerator = self + .enumerator + .as_ref() + .context("Enumerator not initialized")?; - fn enumerate_devices(&self) -> windows::core::Result<()> { unsafe { - let collection = self - .enumerator - .EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE)?; - - let mut devices = Vec::new(); - let mut default_device = None; + let collection = + enumerator.EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE)?; + let default_device = enumerator + .GetDefaultAudioEndpoint(eRender, eMultimedia) + .ok(); + let default_id = default_device + .as_ref() + .and_then(|d| d.GetId().ok()) + .and_then(|id| id.to_string().ok()); + + let mut new_devices = Vec::new(); + let mut new_volumes = HashMap::new(); - // Get info for all active devices. for i in 0..collection.GetCount()? { if let Ok(device) = collection.Item(i) { - let info = self.get_device_info(&device)?; - if info.is_default { - default_device = Some(info.clone()); + let (device_id, name) = self.get_device_properties(&device)?; + + // Register/get volume interface + let endpoint_volume = + if let Some(existing) = self.device_volumes.get(&device_id) { + existing.clone() + } else { + self.register_volume_callback(&device, device_id.clone())? + }; + + let volume = + endpoint_volume.GetMasterVolumeLevelScalar()? * 100.0; + let is_default = + default_id.as_ref().map_or(false, |id| *id == device_id); + + new_volumes.insert(device_id.clone(), endpoint_volume); + + let device_info = AudioDevice { + name, + device_id, + volume: volume.round() as u32, + is_default, + }; + + if is_default { + self.current_state.default_playback_device = + Some(device_info.clone()); } - devices.push(info); + new_devices.push(device_info); } } - if let Some(state) = AUDIO_STATE.get() { - let mut audio_state = state.lock().unwrap(); - audio_state.playback_devices = devices; - audio_state.default_playback_device = default_device; + self.current_state.playback_devices = new_devices; + self.device_volumes = new_volumes; + + self + .common + .emitter + .emit_output(Ok(self.current_state.clone())); + } + + Ok(()) + } + + fn handle_event(&mut self, event: AudioEvent) -> anyhow::Result<()> { + match event { + AudioEvent::DeviceAdded(_) + | AudioEvent::DeviceRemoved(_) + | AudioEvent::DeviceStateChanged(_, _) + | AudioEvent::DefaultDeviceChanged(_) => { + self.update_device_state()?; + } + AudioEvent::VolumeChanged(device_id, new_volume) => { + let volume = (new_volume * 100.0).round() as u32; + + // Update volume in current state + for device in &mut self.current_state.playback_devices { + if device.device_id == device_id { + device.volume = volume; + if let Some(default_device) = + &mut self.current_state.default_playback_device + { + if default_device.device_id == device_id { + default_device.volume = volume; + } + } + break; + } + } + + self + .common + .emitter + .emit_output(Ok(self.current_state.clone())); + } + } + + Ok(()) + } + + fn create_audio_manager(&mut self) -> anyhow::Result<()> { + unsafe { + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + + let enumerator: IMMDeviceEnumerator = + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + // Register device callback + let device_callback = DeviceCallback { + event_sender: self.event_sender.clone(), + }; + enumerator.RegisterEndpointNotificationCallback( + &IMMNotificationClient::from(device_callback), + )?; + + self.enumerator = Some(enumerator); + + // Initial state update + self.update_device_state()?; + + // Event loop + while let Ok(event) = self.event_receiver.recv() { + if let Err(e) = self.handle_event(event) { + debug!("Error handling audio event: {}", e); + } } - AudioProvider::emit_volume(); Ok(()) } } } -impl Drop for MediaDeviceEventHandler { +impl Drop for AudioProvider { fn drop(&mut self) { - unsafe { - let device_state = self.device_state.lock().unwrap(); - for (_, device_info) in device_state.iter() { - let _ = device_info.endpoint_volume.UnregisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(self.clone()), + // Clean up volume callbacks + for volume in self.device_volumes.values() { + unsafe { + let _ = volume.UnregisterControlChangeNotify( + &IAudioEndpointVolumeCallback::null(), + ); + } + } + + // Clean up device notification callback + if let Some(enumerator) = &self.enumerator { + unsafe { + let _ = enumerator.UnregisterEndpointNotificationCallback( + &IMMNotificationClient::null(), ); } } } } -impl IAudioEndpointVolumeCallback_Impl for MediaDeviceEventHandler_Impl { +impl Provider for AudioProvider { + fn runtime_type(&self) -> RuntimeType { + RuntimeType::Sync + } + + fn start_sync(&mut self) { + if let Err(err) = self.create_audio_manager() { + self.common.emitter.emit_output::(Err(err)); + } + } +} + +/// Callback handler for volume notifications. +/// +/// Each device has a volume callback that is used to notify when the +/// volume changes. +#[windows::core::implement(IAudioEndpointVolumeCallback)] +struct VolumeCallback { + device_id: String, + event_sender: channel::Sender, +} + +impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { fn OnNotify( &self, data: *mut windows::Win32::Media::Audio::AUDIO_VOLUME_NOTIFICATION_DATA, ) -> windows::core::Result<()> { if let Some(data) = unsafe { data.as_ref() } { - let device_id = self.current_device.clone(); - let volume = data.fMasterVolume.mul(100.0).round() as u32; - - let _ = self.update_tx.blocking_send((device_id, volume)); + let _ = self.event_sender.send(AudioEvent::VolumeChanged( + self.device_id.clone(), + data.fMasterVolume, + )); } Ok(()) } } -impl IMMNotificationClient_Impl for MediaDeviceEventHandler_Impl { - fn OnDefaultDeviceChanged( +/// Callback handler for device notifications. +/// +/// This is used to detect when new devices are added or removed, and when +/// the default device changes. +#[windows::core::implement(IMMNotificationClient)] +struct DeviceCallback { + event_sender: channel::Sender, +} + +impl IMMNotificationClient_Impl for DeviceCallback_Impl { + fn OnDeviceAdded( &self, - data_flow: EDataFlow, - role: ERole, - _default_device_id: &PCWSTR, + device_id: &windows::core::PCWSTR, ) -> windows::core::Result<()> { - if data_flow == eRender && role == eMultimedia { - self.enumerate_devices()?; + if let Ok(id) = unsafe { device_id.to_string() } { + let _ = self.event_sender.send(AudioEvent::DeviceAdded(id)); } Ok(()) } - fn OnDeviceStateChanged( + fn OnDeviceRemoved( &self, - _device_id: &PCWSTR, - _new_state: DEVICE_STATE, + device_id: &windows::core::PCWSTR, ) -> windows::core::Result<()> { - self.enumerate_devices() + if let Ok(id) = unsafe { device_id.to_string() } { + let _ = self.event_sender.send(AudioEvent::DeviceRemoved(id)); + } + Ok(()) } - fn OnDeviceAdded( + fn OnDeviceStateChanged( &self, - _device_id: &PCWSTR, + device_id: &windows::core::PCWSTR, + new_state: DEVICE_STATE, ) -> windows::core::Result<()> { - self.enumerate_devices() + if let Ok(id) = unsafe { device_id.to_string() } { + let _ = self + .event_sender + .send(AudioEvent::DeviceStateChanged(id, new_state)); + } + Ok(()) } - fn OnDeviceRemoved( + fn OnDefaultDeviceChanged( &self, - _device_id: &PCWSTR, + flow: EDataFlow, + _role: ERole, + default_device_id: &windows::core::PCWSTR, ) -> windows::core::Result<()> { - self.enumerate_devices() + if flow == eRender { + if let Ok(id) = unsafe { default_device_id.to_string() } { + let _ = + self.event_sender.send(AudioEvent::DefaultDeviceChanged(id)); + } + } + Ok(()) } fn OnPropertyValueChanged( &self, - _device_id: &PCWSTR, - _key: &PROPERTYKEY, + _pwstrDeviceId: &windows::core::PCWSTR, + _key: &windows::Win32::UI::Shell::PropertiesSystem::PROPERTYKEY, ) -> windows::core::Result<()> { Ok(()) } } - -pub struct AudioProvider { - common: CommonProviderState, -} - -impl AudioProvider { - pub fn new( - _config: AudioProviderConfig, - common: CommonProviderState, - ) -> Self { - Self { common } - } - - fn emit_volume() { - if let Some(tx) = PROVIDER_TX.get() { - let output = AUDIO_STATE.get().unwrap().lock().unwrap().clone(); - tx.emit_output(Ok(output)); - } - } - - fn handle_volume_updates(mut rx: mpsc::Receiver<(String, u32)>) { - const PROCESS_DELAY: Duration = Duration::from_millis(50); - let mut latest_updates = HashMap::new(); - - while let Some((device_id, volume)) = rx.blocking_recv() { - latest_updates.insert(device_id, volume); - - // Collect any additional pending updates without waiting. - while let Ok((device_id, volume)) = rx.try_recv() { - latest_updates.insert(device_id, volume); - } - - // Brief delay to collect more potential updates. - std::thread::sleep(PROCESS_DELAY); - - // Process all collected updates. - if let Some(state) = AUDIO_STATE.get() { - { - let mut output = state.lock().unwrap(); - for (device_id, volume) in latest_updates.drain() { - debug!( - "Updating volume to {} for device: {}", - volume, device_id - ); - - // Update device in the devices list. - if let Some(device) = output - .playback_devices - .iter_mut() - .find(|d| d.device_id == device_id) - { - device.volume = volume; - } - - // Update default device if it matches. - if let Some(default_device) = - &mut output.default_playback_device - { - if default_device.device_id == device_id { - default_device.volume = volume; - } - } - } - } - - Self::emit_volume(); - } - } - } - - fn create_audio_manager( - update_tx: mpsc::Sender<(String, u32)>, - ) -> anyhow::Result<()> { - unsafe { - let _ = CoInitializeEx(None, COINIT_MULTITHREADED); - let enumerator: IMMDeviceEnumerator = - CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; - let handler = - MediaDeviceEventHandler::new(enumerator.clone(), update_tx); - - handler.enumerate_devices()?; - - let device_notification_callback = - IMMNotificationClient::from(handler.clone()); - enumerator.RegisterEndpointNotificationCallback( - &device_notification_callback, - )?; - - loop { - std::thread::sleep(std::time::Duration::from_secs(1)); - } - } - } -} - -impl Provider for AudioProvider { - fn runtime_type(&self) -> RuntimeType { - RuntimeType::Sync - } - - fn start_sync(&mut self) { - if let Err(err) = self.create_audio_manager() { - self.common.emitter.emit_output::(Err(err)); - } - } -} From 8fe63eeb78251d387f9dc9a69f62851bf466bfb1 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 18:03:56 +0800 Subject: [PATCH 04/23] wip --- .../src/providers/audio/audio_provider.rs | 271 ++++++++++-------- 1 file changed, 145 insertions(+), 126 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 69b843e9..b681650e 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, ops::Mul, sync::{Arc, Mutex, OnceLock}, time::Duration, @@ -89,12 +89,19 @@ enum AudioEvent { VolumeChanged(String, f32), } -// Main provider implementation +/// Holds the state of an audio device. +#[derive(Clone)] +struct DeviceState { + device: AudioDevice, + volume_callback: IAudioEndpointVolume, +} + pub struct AudioProvider { common: CommonProviderState, enumerator: Option, - device_volumes: HashMap, - current_state: AudioOutput, + default_playback_id: Option, + default_recording_id: Option, + devices: HashMap, event_sender: channel::Sender, event_receiver: channel::Receiver, } @@ -109,18 +116,45 @@ impl AudioProvider { Self { common, enumerator: None, - device_volumes: HashMap::new(), - current_state: AudioOutput { - playback_devices: Vec::new(), - recording_devices: Vec::new(), - default_playback_device: None, - default_recording_device: None, - }, + default_playback_id: None, + default_recording_id: None, + devices: HashMap::new(), event_sender, event_receiver, } } + fn create_audio_manager(&mut self) -> anyhow::Result<()> { + unsafe { + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + + let enumerator: IMMDeviceEnumerator = + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + // Register device callback + let device_callback = DeviceCallback { + event_sender: self.event_sender.clone(), + }; + enumerator.RegisterEndpointNotificationCallback( + &IMMNotificationClient::from(device_callback), + )?; + + self.enumerator = Some(enumerator); + + // Initial state update + self.update_device_state()?; + + // Event loop + while let Ok(event) = self.event_receiver.recv() { + if let Err(e) = self.handle_event(event) { + debug!("Error handling audio event: {}", e); + } + } + + Ok(()) + } + } + fn get_device_properties( &self, device: &IMMDevice, @@ -156,152 +190,137 @@ impl AudioProvider { } } + fn build_output(&self) -> AudioOutput { + let mut playback_devices = Vec::new(); + let mut recording_devices = Vec::new(); + let mut default_playback_device = None; + let mut default_recording_device = None; + + for (id, state) in &self.devices { + match &state.device.device_type { + DeviceType::Playback => { + if Some(id) == self.default_playback_id.as_ref() { + default_playback_device = Some(state.device.clone()); + } + playback_devices.push(state.device.clone()); + } + DeviceType::Recording => { + if Some(id) == self.default_recording_id.as_ref() { + default_recording_device = Some(state.device.clone()); + } + recording_devices.push(state.device.clone()); + } + } + } + + // Sort devices by name for consistent ordering. + playback_devices.sort_by(|a, b| a.name.cmp(&b.name)); + recording_devices.sort_by(|a, b| a.name.cmp(&b.name)); + + AudioOutput { + playback_devices, + recording_devices, + default_playback_device, + default_recording_device, + } + } + fn update_device_state(&mut self) -> anyhow::Result<()> { - let enumerator = self - .enumerator - .as_ref() - .context("Enumerator not initialized")?; + let mut active_devices = HashSet::new(); - unsafe { - let collection = - enumerator.EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE)?; - let default_device = enumerator - .GetDefaultAudioEndpoint(eRender, eMultimedia) - .ok(); + // Process both playback and recording devices + for flow in [eRender, eCapture] { + let devices = self.enumerate_devices(flow)?; + let default_device = self.get_default_device(flow).ok(); let default_id = default_device .as_ref() - .and_then(|d| d.GetId().ok()) - .and_then(|id| id.to_string().ok()); - - let mut new_devices = Vec::new(); - let mut new_volumes = HashMap::new(); - - for i in 0..collection.GetCount()? { - if let Ok(device) = collection.Item(i) { - let (device_id, name) = self.get_device_properties(&device)?; - - // Register/get volume interface - let endpoint_volume = - if let Some(existing) = self.device_volumes.get(&device_id) { - existing.clone() - } else { - self.register_volume_callback(&device, device_id.clone())? - }; - - let volume = - endpoint_volume.GetMasterVolumeLevelScalar()? * 100.0; - let is_default = - default_id.as_ref().map_or(false, |id| *id == device_id); - - new_volumes.insert(device_id.clone(), endpoint_volume); - - let device_info = AudioDevice { - name, - device_id, - volume: volume.round() as u32, - is_default, + .and_then(|d| unsafe { d.GetId().ok() }) + .and_then(|id| unsafe { id.to_string().ok() }); + + // Update default device IDs + match flow { + e if e == eRender => self.default_playback_id = default_id.clone(), + e if e == eCapture => self.default_recording_id = default_id, + _ => {} + } + + for device in devices { + let (device_id, _) = self.get_device_info(&device, flow)?; + active_devices.insert(device_id.clone()); + + let endpoint_volume = + if let Some(state) = self.devices.get(&device_id) { + state.volume_callback.clone() + } else { + self.register_volume_callback(&device, device_id.clone())? }; - if is_default { - self.current_state.default_playback_device = - Some(device_info.clone()); + let is_default = match flow { + e if e == eRender => { + Some(&device_id) == self.default_playback_id.as_ref() } - new_devices.push(device_info); - } + e if e == eCapture => { + Some(&device_id) == self.default_recording_id.as_ref() + } + _ => false, + }; + + let device_info = self.create_audio_device( + &device, + flow, + is_default, + &endpoint_volume, + )?; + + self.devices.insert( + device_id, + DeviceState { + device: device_info, + volume_callback: endpoint_volume, + }, + ); } - - self.current_state.playback_devices = new_devices; - self.device_volumes = new_volumes; - - self - .common - .emitter - .emit_output(Ok(self.current_state.clone())); } + // Remove devices that are no longer active + self.devices.retain(|id, _| active_devices.contains(id)); + + // Emit updated state + self.common.emitter.emit_output(Ok(self.build_output())); Ok(()) } fn handle_event(&mut self, event: AudioEvent) -> anyhow::Result<()> { match event { - AudioEvent::DeviceAdded(_) - | AudioEvent::DeviceRemoved(_) - | AudioEvent::DeviceStateChanged(_, _) - | AudioEvent::DefaultDeviceChanged(_) => { + AudioEvent::DeviceAdded(_, _) + | AudioEvent::DeviceRemoved(_, _) + | AudioEvent::DeviceStateChanged(_, _, _) + | AudioEvent::DefaultDeviceChanged(_, _) => { self.update_device_state()?; } AudioEvent::VolumeChanged(device_id, new_volume) => { - let volume = (new_volume * 100.0).round() as u32; - - // Update volume in current state - for device in &mut self.current_state.playback_devices { - if device.device_id == device_id { - device.volume = volume; - if let Some(default_device) = - &mut self.current_state.default_playback_device - { - if default_device.device_id == device_id { - default_device.volume = volume; - } - } - break; - } + if let Some(state) = self.devices.get_mut(&device_id) { + state.device.volume = (new_volume * 100.0).round() as u32; + self.common.emitter.emit_output(Ok(self.build_output())); } - - self - .common - .emitter - .emit_output(Ok(self.current_state.clone())); } } - Ok(()) } - - fn create_audio_manager(&mut self) -> anyhow::Result<()> { - unsafe { - let _ = CoInitializeEx(None, COINIT_MULTITHREADED); - - let enumerator: IMMDeviceEnumerator = - CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; - - // Register device callback - let device_callback = DeviceCallback { - event_sender: self.event_sender.clone(), - }; - enumerator.RegisterEndpointNotificationCallback( - &IMMNotificationClient::from(device_callback), - )?; - - self.enumerator = Some(enumerator); - - // Initial state update - self.update_device_state()?; - - // Event loop - while let Ok(event) = self.event_receiver.recv() { - if let Err(e) = self.handle_event(event) { - debug!("Error handling audio event: {}", e); - } - } - - Ok(()) - } - } } impl Drop for AudioProvider { fn drop(&mut self) { - // Clean up volume callbacks - for volume in self.device_volumes.values() { + // Deregister volume callbacks. + for state in self.devices.values() { unsafe { - let _ = volume.UnregisterControlChangeNotify( - &IAudioEndpointVolumeCallback::null(), + let _ = state.volume_callback.UnregisterControlChangeNotify( + &IAudioEndpointVolumeCallback::from(&state.volume_callback), ); } } - // Clean up device notification callback + // Deregister device notification callback. if let Some(enumerator) = &self.enumerator { unsafe { let _ = enumerator.UnregisterEndpointNotificationCallback( @@ -409,7 +428,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { fn OnPropertyValueChanged( &self, - _pwstrDeviceId: &windows::core::PCWSTR, + _device_id: &windows::core::PCWSTR, _key: &windows::Win32::UI::Shell::PropertiesSystem::PROPERTYKEY, ) -> windows::core::Result<()> { Ok(()) From 8e8bcc78c2936c91e86e34b54632d8747e7d9f11 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 21:55:09 +0800 Subject: [PATCH 05/23] update client-side types --- README.md | 9 ++++++--- .../src/providers/audio/audio-provider-types.ts | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 80a086ac..779dd651 100644 --- a/README.md +++ b/README.md @@ -92,10 +92,13 @@ No config options. | Variable | Description | Return type | | ------------------ | ----------------------------- | ----------------------- | -| `deviceId` | Device ID. | `string` | +| `id` | Device ID. | `string` | | `name` | Friendly display name of device. | `string` | -| `volume` | Volume as a % of maximum volume. Returned value is between `0` and `100`. | `number` | -| `isDefault` | `true` if the device is selected as the default playback device.| `boolean` | +| `volume` | Volume as a % of maximum volume. Returned value is between `0` and `100`. | `number \| null` | +| `roles` | Roles the device is assigned to. | `('multimedia' \| 'communications' \| 'console')[]` | +| `type` | Type of the device. | `'playback' \| 'recording' \| 'hybrid'` | +| `isDefaultPlayback` | `true` if the device is selected as the default playback device.| `boolean` | +| `isDefaultRecording` | `true` if the device is selected as the default recording device.| `boolean` | ### Battery diff --git a/packages/client-api/src/providers/audio/audio-provider-types.ts b/packages/client-api/src/providers/audio/audio-provider-types.ts index dab6d41d..06b4c359 100644 --- a/packages/client-api/src/providers/audio/audio-provider-types.ts +++ b/packages/client-api/src/providers/audio/audio-provider-types.ts @@ -7,13 +7,21 @@ export interface AudioProviderConfig { export type AudioProvider = Provider; export interface AudioOutput { - defaultPlaybackDevice: AudioDevice; + defaultPlaybackDevice: AudioDevice | null; + defaultRecordingDevice: AudioDevice | null; playbackDevices: AudioDevice[]; + recordingDevices: AudioDevice[]; } export interface AudioDevice { - deviceId: string; + id: string; name: string; - volume: number; - isDefault: boolean; + volume: number | null; + roles: AudioDeviceRole[]; + type: AudioDeviceType; + isDefaultPlayback: boolean; + isDefaultRecording: boolean; } + +export type AudioDeviceRole = 'multimedia' | 'communications' | 'console'; +export type AudioDeviceType = 'playback' | 'recording' | 'hybrid'; From 93eee4ee5f3fca4fccaee96007592782772aaa2f Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 21:55:13 +0800 Subject: [PATCH 06/23] wip --- .../src/providers/audio/audio_provider.rs | 144 +++++++++++------- 1 file changed, 89 insertions(+), 55 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index b681650e..76f7ad30 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -22,8 +22,8 @@ use windows::Win32::{ IAudioEndpointVolumeCallback_Impl, }, IMMDevice, IMMDeviceEnumerator, IMMNotificationClient, - IMMNotificationClient_Impl, MMDeviceEnumerator, DEVICE_STATE, - DEVICE_STATE_ACTIVE, + IMMNotificationClient_Impl, MMDeviceEnumerator, + AUDIO_VOLUME_NOTIFICATION_DATA, DEVICE_STATE, DEVICE_STATE_ACTIVE, }, System::Com::{ CoCreateInstance, CoInitializeEx, CLSCTX_ALL, COINIT_MULTITHREADED, @@ -33,8 +33,11 @@ use windows::Win32::{ }; use windows_core::PCWSTR; -use crate::providers::{ - CommonProviderState, Provider, ProviderEmitter, RuntimeType, +use crate::{ + common::windows::COM_INIT, + providers::{ + CommonProviderState, Provider, ProviderEmitter, RuntimeType, + }, }; #[derive(Deserialize, Debug)] @@ -85,14 +88,15 @@ enum AudioEvent { DeviceAdded(String), DeviceRemoved(String), DeviceStateChanged(String, DEVICE_STATE), - DefaultDeviceChanged(String), + DefaultDeviceChanged(String, EDataFlow), VolumeChanged(String, f32), } /// Holds the state of an audio device. #[derive(Clone)] struct DeviceState { - device: AudioDevice, + imm_device: IMMDevice, + output: AudioDevice, volume_callback: IAudioEndpointVolume, } @@ -124,34 +128,57 @@ impl AudioProvider { } } - fn create_audio_manager(&mut self) -> anyhow::Result<()> { - unsafe { - let _ = CoInitializeEx(None, COINIT_MULTITHREADED); - - let enumerator: IMMDeviceEnumerator = + /// Main entry point. + fn start_listening(&mut self) -> anyhow::Result<()> { + COM_INIT.with(|_| unsafe { + let device_enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; - // Register device callback - let device_callback = DeviceCallback { - event_sender: self.event_sender.clone(), - }; - enumerator.RegisterEndpointNotificationCallback( - &IMMNotificationClient::from(device_callback), + // Register device callback. + device_enumerator.RegisterEndpointNotificationCallback( + &IMMNotificationClient::from(DeviceCallback { + event_sender: self.event_sender.clone(), + }), )?; - self.enumerator = Some(enumerator); + self.enumerator = Some(device_enumerator); - // Initial state update + // Emit initial state. self.update_device_state()?; - // Event loop + // Listen to audio-related events. while let Ok(event) = self.event_receiver.recv() { - if let Err(e) = self.handle_event(event) { - debug!("Error handling audio event: {}", e); + if let Err(err) = self.handle_event(event) { + debug!("Error handling audio event: {}", err); } } Ok(()) + }) + } + + /// Enumerates active devices of a specific type + fn enumerate_devices( + &self, + flow: EDataFlow, + ) -> anyhow::Result> { + let enumerator = self + .enumerator + .as_ref() + .context("Enumerator not initialized.")?; + + unsafe { + let collection = + enumerator.EnumAudioEndpoints(flow, DEVICE_STATE_ACTIVE)?; + let count = collection.GetCount()?; + + let mut devices = Vec::with_capacity(count as usize); + for i in 0..count { + if let Ok(device) = collection.Item(i) { + device.devices.push(device); + } + } + Ok(devices) } } @@ -168,26 +195,27 @@ impl AudioProvider { } } + /// Registers volume callbacks for a device. fn register_volume_callback( &self, device: &IMMDevice, device_id: String, ) -> anyhow::Result { - unsafe { - let endpoint_volume: IAudioEndpointVolume = - device.Activate(CLSCTX_ALL, None)?; + let endpoint_volume: IAudioEndpointVolume = + unsafe { device.Activate(CLSCTX_ALL, None) }?; - let callback = VolumeCallback { - device_id, - event_sender: self.event_sender.clone(), - }; + let callback = VolumeCallback { + device_id, + event_sender: self.event_sender.clone(), + }; + unsafe { endpoint_volume.RegisterControlChangeNotify( &IAudioEndpointVolumeCallback::from(callback), - )?; + ) + }?; - Ok(endpoint_volume) - } + Ok(endpoint_volume) } fn build_output(&self) -> AudioOutput { @@ -197,18 +225,18 @@ impl AudioProvider { let mut default_recording_device = None; for (id, state) in &self.devices { - match &state.device.device_type { + match &state.output.device_type { DeviceType::Playback => { if Some(id) == self.default_playback_id.as_ref() { - default_playback_device = Some(state.device.clone()); + default_playback_device = Some(state.output.clone()); } - playback_devices.push(state.device.clone()); + playback_devices.push(state.output.clone()); } DeviceType::Recording => { if Some(id) == self.default_recording_id.as_ref() { - default_recording_device = Some(state.device.clone()); + default_recording_device = Some(state.output.clone()); } - recording_devices.push(state.device.clone()); + recording_devices.push(state.output.clone()); } } } @@ -275,7 +303,7 @@ impl AudioProvider { self.devices.insert( device_id, DeviceState { - device: device_info, + output: device_info, volume_callback: endpoint_volume, }, ); @@ -300,11 +328,12 @@ impl AudioProvider { } AudioEvent::VolumeChanged(device_id, new_volume) => { if let Some(state) = self.devices.get_mut(&device_id) { - state.device.volume = (new_volume * 100.0).round() as u32; + state.output.volume = (new_volume * 100.0).round() as u32; self.common.emitter.emit_output(Ok(self.build_output())); } } } + Ok(()) } } @@ -314,9 +343,9 @@ impl Drop for AudioProvider { // Deregister volume callbacks. for state in self.devices.values() { unsafe { - let _ = state.volume_callback.UnregisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(&state.volume_callback), - ); + let _ = state + .volume_callback + .UnregisterControlChangeNotify(&state.volume_callback); } } @@ -337,7 +366,7 @@ impl Provider for AudioProvider { } fn start_sync(&mut self) { - if let Err(err) = self.create_audio_manager() { + if let Err(err) = self.start_listening() { self.common.emitter.emit_output::(Err(err)); } } @@ -347,6 +376,7 @@ impl Provider for AudioProvider { /// /// Each device has a volume callback that is used to notify when the /// volume changes. +#[derive(Clone)] #[windows::core::implement(IAudioEndpointVolumeCallback)] struct VolumeCallback { device_id: String, @@ -356,7 +386,7 @@ struct VolumeCallback { impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { fn OnNotify( &self, - data: *mut windows::Win32::Media::Audio::AUDIO_VOLUME_NOTIFICATION_DATA, + data: *mut AUDIO_VOLUME_NOTIFICATION_DATA, ) -> windows::core::Result<()> { if let Some(data) = unsafe { data.as_ref() } { let _ = self.event_sender.send(AudioEvent::VolumeChanged( @@ -364,6 +394,7 @@ impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { data.fMasterVolume, )); } + Ok(()) } } @@ -380,27 +411,29 @@ struct DeviceCallback { impl IMMNotificationClient_Impl for DeviceCallback_Impl { fn OnDeviceAdded( &self, - device_id: &windows::core::PCWSTR, + device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { let _ = self.event_sender.send(AudioEvent::DeviceAdded(id)); } + Ok(()) } fn OnDeviceRemoved( &self, - device_id: &windows::core::PCWSTR, + device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { let _ = self.event_sender.send(AudioEvent::DeviceRemoved(id)); } + Ok(()) } fn OnDeviceStateChanged( &self, - device_id: &windows::core::PCWSTR, + device_id: &PCWSTR, new_state: DEVICE_STATE, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { @@ -408,6 +441,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { .event_sender .send(AudioEvent::DeviceStateChanged(id, new_state)); } + Ok(()) } @@ -415,21 +449,21 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { &self, flow: EDataFlow, _role: ERole, - default_device_id: &windows::core::PCWSTR, + default_device_id: &PCWSTR, ) -> windows::core::Result<()> { - if flow == eRender { - if let Ok(id) = unsafe { default_device_id.to_string() } { - let _ = - self.event_sender.send(AudioEvent::DefaultDeviceChanged(id)); - } + if let Ok(id) = unsafe { default_device_id.to_string() } { + let _ = self + .event_sender + .send(AudioEvent::DefaultDeviceChanged(id, flow)); } + Ok(()) } fn OnPropertyValueChanged( &self, - _device_id: &windows::core::PCWSTR, - _key: &windows::Win32::UI::Shell::PropertiesSystem::PROPERTYKEY, + _device_id: &PCWSTR, + _key: &PROPERTYKEY, ) -> windows::core::Result<()> { Ok(()) } From 4b1bdd595887ef76d2c5dcbb193dae6a4f15aa1c Mon Sep 17 00:00:00 2001 From: lars-berger Date: Thu, 28 Nov 2024 22:36:17 +0800 Subject: [PATCH 07/23] wip --- .../src/providers/audio/audio_provider.rs | 198 +++++++++--------- 1 file changed, 94 insertions(+), 104 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 76f7ad30..38324757 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -1,22 +1,13 @@ -use std::{ - collections::{HashMap, HashSet}, - ops::Mul, - sync::{Arc, Mutex, OnceLock}, - time::Duration, -}; +use std::collections::{HashMap, HashSet}; use anyhow::Context; use crossbeam::channel; use serde::{Deserialize, Serialize}; -use tokio::{ - sync::mpsc::{self}, - task, -}; use tracing::debug; use windows::Win32::{ Devices::FunctionDiscovery::PKEY_Device_FriendlyName, Media::Audio::{ - eCapture, eMultimedia, eRender, EDataFlow, ERole, + eCapture, eRender, EDataFlow, ERole, Endpoints::{ IAudioEndpointVolume, IAudioEndpointVolumeCallback, IAudioEndpointVolumeCallback_Impl, @@ -25,19 +16,14 @@ use windows::Win32::{ IMMNotificationClient_Impl, MMDeviceEnumerator, AUDIO_VOLUME_NOTIFICATION_DATA, DEVICE_STATE, DEVICE_STATE_ACTIVE, }, - System::Com::{ - CoCreateInstance, CoInitializeEx, CLSCTX_ALL, COINIT_MULTITHREADED, - STGM_READ, - }, + System::Com::{CoCreateInstance, CLSCTX_ALL, STGM_READ}, UI::Shell::PropertiesSystem::{IPropertyStore, PROPERTYKEY}, }; use windows_core::PCWSTR; use crate::{ common::windows::COM_INIT, - providers::{ - CommonProviderState, Provider, ProviderEmitter, RuntimeType, - }, + providers::{CommonProviderState, Provider, RuntimeType}, }; #[derive(Deserialize, Debug)] @@ -49,6 +35,7 @@ pub struct AudioProviderConfig {} pub struct AudioOutput { pub playback_devices: Vec, pub recording_devices: Vec, + pub all_devices: Vec, pub default_playback_device: Option, pub default_recording_device: Option, } @@ -63,13 +50,12 @@ pub struct AudioDevice { pub is_default: bool, } -// TODO: Should there be handling for devices that can be both playback and -// recording? #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum DeviceType { Playback, Recording, + Hybrid, } impl From for DeviceType { @@ -77,7 +63,7 @@ impl From for DeviceType { match flow { e if e == eRender => Self::Playback, e if e == eCapture => Self::Recording, - _ => Self::Playback, + _ => Self::Hybrid, } } } @@ -95,19 +81,19 @@ enum AudioEvent { /// Holds the state of an audio device. #[derive(Clone)] struct DeviceState { - imm_device: IMMDevice, + device: IMMDevice, output: AudioDevice, volume_callback: IAudioEndpointVolume, } pub struct AudioProvider { common: CommonProviderState, - enumerator: Option, + device_enumerator: Option, default_playback_id: Option, default_recording_id: Option, devices: HashMap, - event_sender: channel::Sender, - event_receiver: channel::Receiver, + event_tx: channel::Sender, + event_rx: channel::Receiver, } impl AudioProvider { @@ -115,39 +101,39 @@ impl AudioProvider { _config: AudioProviderConfig, common: CommonProviderState, ) -> Self { - let (event_sender, event_receiver) = channel::unbounded(); + let (event_tx, event_rx) = channel::unbounded(); Self { common, - enumerator: None, + device_enumerator: None, default_playback_id: None, default_recording_id: None, devices: HashMap::new(), - event_sender, - event_receiver, + event_tx, + event_rx, } } /// Main entry point. - fn start_listening(&mut self) -> anyhow::Result<()> { + fn start(&mut self) -> anyhow::Result<()> { COM_INIT.with(|_| unsafe { let device_enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; - // Register device callback. + // Register device add/remove callback. device_enumerator.RegisterEndpointNotificationCallback( &IMMNotificationClient::from(DeviceCallback { - event_sender: self.event_sender.clone(), + event_sender: self.event_tx.clone(), }), )?; - self.enumerator = Some(device_enumerator); + self.device_enumerator = Some(device_enumerator); // Emit initial state. self.update_device_state()?; // Listen to audio-related events. - while let Ok(event) = self.event_receiver.recv() { + while let Ok(event) = self.event_rx.recv() { if let Err(err) = self.handle_event(event) { debug!("Error handling audio event: {}", err); } @@ -157,29 +143,25 @@ impl AudioProvider { }) } - /// Enumerates active devices of a specific type + /// Enumerates active devices of a specific type. fn enumerate_devices( &self, flow: EDataFlow, ) -> anyhow::Result> { - let enumerator = self - .enumerator - .as_ref() - .context("Enumerator not initialized.")?; + let collection = unsafe { + self + .device_enumerator + .as_ref() + .context("Device enumerator not initialized.")? + .EnumAudioEndpoints(flow, DEVICE_STATE_ACTIVE) + }?; - unsafe { - let collection = - enumerator.EnumAudioEndpoints(flow, DEVICE_STATE_ACTIVE)?; - let count = collection.GetCount()?; - - let mut devices = Vec::with_capacity(count as usize); - for i in 0..count { - if let Ok(device) = collection.Item(i) { - device.devices.push(device); - } - } - Ok(devices) - } + let count = unsafe { collection.GetCount() }?; + let devices = (0..count) + .filter_map(|i| unsafe { collection.Item(i).ok() }) + .collect::>(); + + Ok(devices) } fn get_device_properties( @@ -206,7 +188,7 @@ impl AudioProvider { let callback = VolumeCallback { device_id, - event_sender: self.event_sender.clone(), + event_sender: self.event_tx.clone(), }; unsafe { @@ -218,39 +200,47 @@ impl AudioProvider { Ok(endpoint_volume) } - fn build_output(&self) -> AudioOutput { - let mut playback_devices = Vec::new(); - let mut recording_devices = Vec::new(); - let mut default_playback_device = None; - let mut default_recording_device = None; - - for (id, state) in &self.devices { - match &state.output.device_type { - DeviceType::Playback => { - if Some(id) == self.default_playback_id.as_ref() { - default_playback_device = Some(state.output.clone()); - } - playback_devices.push(state.output.clone()); - } - DeviceType::Recording => { - if Some(id) == self.default_recording_id.as_ref() { - default_recording_device = Some(state.output.clone()); - } - recording_devices.push(state.output.clone()); - } - } - } - - // Sort devices by name for consistent ordering. - playback_devices.sort_by(|a, b| a.name.cmp(&b.name)); - recording_devices.sort_by(|a, b| a.name.cmp(&b.name)); + /// Emits an `AudioOutput` update through the provider's emitter. + fn emit_output(&mut self) { + let default_playback_device = self + .default_playback_id + .as_ref() + .and_then(|id| self.devices.get(id)) + .map(|state| state.output.clone()); - AudioOutput { + let default_recording_device = self + .default_recording_id + .as_ref() + .and_then(|id| self.devices.get(id)) + .map(|state| state.output.clone()); + + let playback_devices = self + .devices + .values() + .filter(|state| state.output.device_type == DeviceType::Playback) + .map(|state| state.output.clone()) + .collect(); + + let recording_devices = self + .devices + .values() + .filter(|state| state.output.device_type == DeviceType::Recording) + .map(|state| state.output.clone()) + .collect(); + + let all_devices = self + .devices + .values() + .map(|state| state.output.clone()) + .collect(); + + self.common.emitter.emit_output_cached(Ok(AudioOutput { playback_devices, recording_devices, + all_devices, default_playback_device, default_recording_device, - } + })); } fn update_device_state(&mut self) -> anyhow::Result<()> { @@ -320,10 +310,10 @@ impl AudioProvider { fn handle_event(&mut self, event: AudioEvent) -> anyhow::Result<()> { match event { - AudioEvent::DeviceAdded(_, _) - | AudioEvent::DeviceRemoved(_, _) - | AudioEvent::DeviceStateChanged(_, _, _) - | AudioEvent::DefaultDeviceChanged(_, _) => { + AudioEvent::DeviceAdded(..) + | AudioEvent::DeviceRemoved(..) + | AudioEvent::DeviceStateChanged(..) + | AudioEvent::DefaultDeviceChanged(..) => { self.update_device_state()?; } AudioEvent::VolumeChanged(device_id, new_volume) => { @@ -340,23 +330,23 @@ impl AudioProvider { impl Drop for AudioProvider { fn drop(&mut self) { - // Deregister volume callbacks. - for state in self.devices.values() { - unsafe { - let _ = state - .volume_callback - .UnregisterControlChangeNotify(&state.volume_callback); - } - } - - // Deregister device notification callback. - if let Some(enumerator) = &self.enumerator { - unsafe { - let _ = enumerator.UnregisterEndpointNotificationCallback( - &IMMNotificationClient::null(), - ); - } - } + // // Deregister volume callbacks. + // for state in self.devices.values() { + // unsafe { + // let _ = state + // .volume_callback + // .UnregisterControlChangeNotify(&state.volume_callback); + // } + // } + + // // Deregister device notification callback. + // if let Some(enumerator) = &self.device_enumerator { + // unsafe { + // let _ = enumerator.UnregisterEndpointNotificationCallback( + // &IMMNotificationClient::null(), + // ); + // } + // } } } @@ -366,7 +356,7 @@ impl Provider for AudioProvider { } fn start_sync(&mut self) { - if let Err(err) = self.start_listening() { + if let Err(err) = self.start() { self.common.emitter.emit_output::(Err(err)); } } @@ -399,7 +389,7 @@ impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { } } -/// Callback handler for device notifications. +/// Callback handler for device change notifications. /// /// This is used to detect when new devices are added or removed, and when /// the default device changes. From b3f5af5c2f3dc74ca714e9e428ea7cf6f59d76ac Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 14:02:18 +0800 Subject: [PATCH 08/23] wip --- .../src/providers/audio/audio_provider.rs | 120 +++++++++--------- packages/desktop/src/providers/provider.rs | 2 +- .../desktop/src/providers/provider_manager.rs | 96 ++++++++------ 3 files changed, 116 insertions(+), 102 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 38324757..ba784b4e 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -164,17 +164,17 @@ impl AudioProvider { Ok(devices) } - fn get_device_properties( - &self, - device: &IMMDevice, - ) -> anyhow::Result<(String, String)> { - unsafe { - let device_id = device.GetId()?.to_string()?; - let store: IPropertyStore = device.OpenPropertyStore(STGM_READ)?; - let friendly_name = - store.GetValue(&PKEY_Device_FriendlyName)?.to_string(); - Ok((device_id, friendly_name)) - } + /// Gets the friendly name of a device. + /// + /// Returns a string. For example, "Headphones (WH-1000XM3 Stereo)". + fn device_name(&self, device: &IMMDevice) -> anyhow::Result { + let store: IPropertyStore = + unsafe { device.OpenPropertyStore(STGM_READ) }?; + + let friendly_name = + unsafe { store.GetValue(&PKEY_Device_FriendlyName)?.to_string() }; + + Ok(friendly_name) } /// Registers volume callbacks for a device. @@ -183,17 +183,16 @@ impl AudioProvider { device: &IMMDevice, device_id: String, ) -> anyhow::Result { - let endpoint_volume: IAudioEndpointVolume = - unsafe { device.Activate(CLSCTX_ALL, None) }?; - - let callback = VolumeCallback { - device_id, - event_sender: self.event_tx.clone(), - }; + let endpoint_volume = unsafe { + device.Activate::(CLSCTX_ALL, None) + }?; unsafe { endpoint_volume.RegisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(callback), + &IAudioEndpointVolumeCallback::from(VolumeCallback { + device_id, + event_sender: self.event_tx.clone(), + }), ) }?; @@ -202,45 +201,41 @@ impl AudioProvider { /// Emits an `AudioOutput` update through the provider's emitter. fn emit_output(&mut self) { - let default_playback_device = self - .default_playback_id - .as_ref() - .and_then(|id| self.devices.get(id)) - .map(|state| state.output.clone()); - - let default_recording_device = self - .default_recording_id - .as_ref() - .and_then(|id| self.devices.get(id)) - .map(|state| state.output.clone()); - - let playback_devices = self - .devices - .values() - .filter(|state| state.output.device_type == DeviceType::Playback) - .map(|state| state.output.clone()) - .collect(); - - let recording_devices = self - .devices - .values() - .filter(|state| state.output.device_type == DeviceType::Recording) - .map(|state| state.output.clone()) - .collect(); - - let all_devices = self - .devices - .values() - .map(|state| state.output.clone()) - .collect(); - - self.common.emitter.emit_output_cached(Ok(AudioOutput { - playback_devices, - recording_devices, - all_devices, - default_playback_device, - default_recording_device, - })); + let mut output = AudioOutput { + playback_devices: Vec::new(), + recording_devices: Vec::new(), + all_devices: Vec::new(), + default_playback_device: None, + default_recording_device: None, + }; + + for (id, state) in &self.devices { + let device = &state.output; + output.all_devices.push(device.clone()); + + match device.device_type { + DeviceType::Playback => { + output.playback_devices.push(device.clone()); + } + DeviceType::Recording => { + output.recording_devices.push(device.clone()); + } + _ => { + output.playback_devices.push(device.clone()); + output.recording_devices.push(device.clone()); + } + } + + if self.default_playback_id.as_ref() == Some(id) { + output.default_playback_device = Some(device.clone()); + } + + if self.default_recording_id.as_ref() == Some(id) { + output.default_recording_device = Some(device.clone()); + } + } + + self.common.emitter.emit_output(Ok(output)); } fn update_device_state(&mut self) -> anyhow::Result<()> { @@ -300,14 +295,13 @@ impl AudioProvider { } } - // Remove devices that are no longer active + // Remove devices that are no longer active. self.devices.retain(|id, _| active_devices.contains(id)); - // Emit updated state - self.common.emitter.emit_output(Ok(self.build_output())); Ok(()) } + /// Handles an audio event. fn handle_event(&mut self, event: AudioEvent) -> anyhow::Result<()> { match event { AudioEvent::DeviceAdded(..) @@ -319,11 +313,13 @@ impl AudioProvider { AudioEvent::VolumeChanged(device_id, new_volume) => { if let Some(state) = self.devices.get_mut(&device_id) { state.output.volume = (new_volume * 100.0).round() as u32; - self.common.emitter.emit_output(Ok(self.build_output())); } } } + // Emit new output after handling the event. + self.emit_output(); + Ok(()) } } diff --git a/packages/desktop/src/providers/provider.rs b/packages/desktop/src/providers/provider.rs index b0f26cd2..080c9b35 100644 --- a/packages/desktop/src/providers/provider.rs +++ b/packages/desktop/src/providers/provider.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use super::{ProviderFunction, ProviderFunctionResponse}; #[async_trait] -pub trait Provider: Send + Sync { +pub trait Provider { fn runtime_type(&self) -> RuntimeType; /// Callback for when the provider is started. diff --git a/packages/desktop/src/providers/provider_manager.rs b/packages/desktop/src/providers/provider_manager.rs index e33ea236..4477e7b1 100644 --- a/packages/desktop/src/providers/provider_manager.rs +++ b/packages/desktop/src/providers/provider_manager.rs @@ -244,59 +244,77 @@ impl ProviderManager { config_hash: String, common: CommonProviderState, ) -> anyhow::Result<(task::JoinHandle<()>, RuntimeType)> { - let mut provider: Box = match config { + type CreateProviderFn = ( + RuntimeType, + Box Box + Send + 'static>, + ); + + let (runtime_type, create_provider): CreateProviderFn = match config { #[cfg(windows)] - ProviderConfig::Audio(config) => { - Box::new(AudioProvider::new(config, common)) - } - ProviderConfig::Battery(config) => { - Box::new(BatteryProvider::new(config, common)) - } - ProviderConfig::Cpu(config) => { - Box::new(CpuProvider::new(config, common)) - } - ProviderConfig::Host(config) => { - Box::new(HostProvider::new(config, common)) - } - ProviderConfig::Ip(config) => { - Box::new(IpProvider::new(config, common)) - } + ProviderConfig::Audio(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(AudioProvider::new(config, common))), + ), + ProviderConfig::Battery(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(BatteryProvider::new(config, common))), + ), + ProviderConfig::Cpu(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(CpuProvider::new(config, common))), + ), + ProviderConfig::Host(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(HostProvider::new(config, common))), + ), + ProviderConfig::Ip(config) => ( + RuntimeType::Async, + Box::new(|| Box::new(IpProvider::new(config, common))), + ), #[cfg(windows)] - ProviderConfig::Komorebi(config) => { - Box::new(KomorebiProvider::new(config, common)) - } + ProviderConfig::Komorebi(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(KomorebiProvider::new(config, common))), + ), #[cfg(windows)] - ProviderConfig::Media(config) => { - Box::new(MediaProvider::new(config, common)) - } - ProviderConfig::Memory(config) => { - Box::new(MemoryProvider::new(config, common)) - } - ProviderConfig::Disk(config) => { - Box::new(DiskProvider::new(config, common)) - } - ProviderConfig::Network(config) => { - Box::new(NetworkProvider::new(config, common)) - } - ProviderConfig::Weather(config) => { - Box::new(WeatherProvider::new(config, common)) - } + ProviderConfig::Media(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(MediaProvider::new(config, common))), + ), + ProviderConfig::Memory(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(MemoryProvider::new(config, common))), + ), + ProviderConfig::Disk(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(DiskProvider::new(config, common))), + ), + ProviderConfig::Network(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(NetworkProvider::new(config, common))), + ), + ProviderConfig::Weather(config) => ( + RuntimeType::Async, + Box::new(|| Box::new(WeatherProvider::new(config, common))), + ), #[cfg(windows)] - ProviderConfig::Keyboard(config) => { - Box::new(KeyboardProvider::new(config, common)) - } + ProviderConfig::Keyboard(config) => ( + RuntimeType::Sync, + Box::new(|| Box::new(KeyboardProvider::new(config, common))), + ), #[allow(unreachable_patterns)] _ => bail!("Provider not supported on this operating system."), }; // Spawn the provider's task based on its runtime type. - let runtime_type = provider.runtime_type(); let task_handle = match &runtime_type { RuntimeType::Async => task::spawn(async move { - provider.start_async().await; + // let mut provider = create_provider(); + // provider.start_async().await; info!("Provider stopped: {}", config_hash); }), RuntimeType::Sync => task::spawn_blocking(move || { + let mut provider = create_provider(); provider.start_sync(); info!("Provider stopped: {}", config_hash); }), From 3fa2f7519ef2e5c8bd9fb0117e67244742da50e8 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 17:05:58 +0800 Subject: [PATCH 09/23] fix segfault --- .../src/providers/audio/audio_provider.rs | 216 +++++++++++------- 1 file changed, 130 insertions(+), 86 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index ba784b4e..6e2639e5 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -3,23 +3,28 @@ use std::collections::{HashMap, HashSet}; use anyhow::Context; use crossbeam::channel; use serde::{Deserialize, Serialize}; -use tracing::debug; +use tracing::{debug, info}; use windows::Win32::{ - Devices::FunctionDiscovery::PKEY_Device_FriendlyName, + Devices::FunctionDiscovery::{ + PKEY_Device_DeviceDesc, PKEY_Device_FriendlyName, + }, Media::Audio::{ - eCapture, eRender, EDataFlow, ERole, + eAll, eCapture, eRender, EDataFlow, ERole, Endpoints::{ IAudioEndpointVolume, IAudioEndpointVolumeCallback, IAudioEndpointVolumeCallback_Impl, }, - IMMDevice, IMMDeviceEnumerator, IMMNotificationClient, + IMMDevice, IMMDeviceEnumerator, IMMEndpoint, IMMNotificationClient, IMMNotificationClient_Impl, MMDeviceEnumerator, AUDIO_VOLUME_NOTIFICATION_DATA, DEVICE_STATE, DEVICE_STATE_ACTIVE, }, - System::Com::{CoCreateInstance, CLSCTX_ALL, STGM_READ}, + System::Com::{ + CoCreateInstance, CoInitializeEx, CLSCTX_ALL, COINIT_MULTITHREADED, + STGM_READ, + }, UI::Shell::PropertiesSystem::{IPropertyStore, PROPERTYKEY}, }; -use windows_core::PCWSTR; +use windows_core::{Interface, PCWSTR}; use crate::{ common::windows::COM_INIT, @@ -116,31 +121,66 @@ impl AudioProvider { /// Main entry point. fn start(&mut self) -> anyhow::Result<()> { - COM_INIT.with(|_| unsafe { - let device_enumerator: IMMDeviceEnumerator = - CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + + let device_enumerator: IMMDeviceEnumerator = + unsafe { CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) }?; + + // Note that this would sporadically segfault if we didn't keep a + // separate variable for `DeviceCallback` when registering the + // callback. Something funky with lifetimes and the COM API's. + let callback = DeviceCallback { + event_sender: self.event_tx.clone(), + }; - // Register device add/remove callback. + // Register device add/remove callback. + unsafe { device_enumerator.RegisterEndpointNotificationCallback( - &IMMNotificationClient::from(DeviceCallback { - event_sender: self.event_tx.clone(), - }), - )?; + &IMMNotificationClient::from(callback), + ) + }?; - self.device_enumerator = Some(device_enumerator); + self.device_enumerator = Some(device_enumerator); - // Emit initial state. - self.update_device_state()?; + // Emit initial state. - // Listen to audio-related events. - while let Ok(event) = self.event_rx.recv() { - if let Err(err) = self.handle_event(event) { - debug!("Error handling audio event: {}", err); - } + let devices = self.enumerate_devices(eRender)?; + + for (i, device) in devices.iter().enumerate() { + println!("Device: {:?}", device); + println!("Device ID: {:?}", unsafe { device.GetId()?.to_string() }); + let endpoint = unsafe { device.cast::() }?; + + println!("Device State: {:?}", unsafe { + device.GetState().unwrap() + }); + println!("Endpoint: {:?}", unsafe { endpoint.GetDataFlow() }); + // Get device properties for debugging + if let Ok(properties) = + unsafe { device.OpenPropertyStore(STGM_READ) } + { + let friendly_name = unsafe { + properties.GetValue(&PKEY_Device_FriendlyName)?.to_string() + }; + info!("Friendly Name: {}", friendly_name); + let device_desc = unsafe { + properties.GetValue(&PKEY_Device_DeviceDesc)?.to_string() + }; + info!("Description: {}", device_desc); + } else { + tracing::error!("Failed to get properties for device {}", i); } + } + // self.update_device_state()?; + + // Listen to audio-related events. + while let Ok(event) = self.event_rx.recv() { + if let Err(err) = self.handle_event(event) { + info!("Error handling audio event: {}", err); + } + } - Ok(()) - }) + Ok(()) } /// Enumerates active devices of a specific type. @@ -153,7 +193,7 @@ impl AudioProvider { .device_enumerator .as_ref() .context("Device enumerator not initialized.")? - .EnumAudioEndpoints(flow, DEVICE_STATE_ACTIVE) + .EnumAudioEndpoints(eAll, DEVICE_STATE_ACTIVE) }?; let count = unsafe { collection.GetCount() }?; @@ -187,12 +227,14 @@ impl AudioProvider { device.Activate::(CLSCTX_ALL, None) }?; + let callback = VolumeCallback { + device_id, + event_sender: self.event_tx.clone(), + }; + unsafe { endpoint_volume.RegisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(VolumeCallback { - device_id, - event_sender: self.event_tx.clone(), - }), + &IAudioEndpointVolumeCallback::from(callback), ) }?; @@ -239,66 +281,67 @@ impl AudioProvider { } fn update_device_state(&mut self) -> anyhow::Result<()> { - let mut active_devices = HashSet::new(); - - // Process both playback and recording devices - for flow in [eRender, eCapture] { - let devices = self.enumerate_devices(flow)?; - let default_device = self.get_default_device(flow).ok(); - let default_id = default_device - .as_ref() - .and_then(|d| unsafe { d.GetId().ok() }) - .and_then(|id| unsafe { id.to_string().ok() }); - - // Update default device IDs - match flow { - e if e == eRender => self.default_playback_id = default_id.clone(), - e if e == eCapture => self.default_recording_id = default_id, - _ => {} - } - - for device in devices { - let (device_id, _) = self.get_device_info(&device, flow)?; - active_devices.insert(device_id.clone()); - - let endpoint_volume = - if let Some(state) = self.devices.get(&device_id) { - state.volume_callback.clone() - } else { - self.register_volume_callback(&device, device_id.clone())? - }; - - let is_default = match flow { - e if e == eRender => { - Some(&device_id) == self.default_playback_id.as_ref() - } - e if e == eCapture => { - Some(&device_id) == self.default_recording_id.as_ref() - } - _ => false, - }; + Ok(()) + // let mut active_devices = HashSet::new(); + + // // Process both playback and recording devices + // for flow in [eRender, eCapture] { + // let devices = self.enumerate_devices(flow)?; + // let default_device = self.get_default_device(flow).ok(); + // let default_id = default_device + // .as_ref() + // .and_then(|d| unsafe { d.GetId().ok() }) + // .and_then(|id| unsafe { id.to_string().ok() }); + + // // Update default device IDs + // match flow { + // e if e == eRender => self.default_playback_id = + // default_id.clone(), e if e == eCapture => + // self.default_recording_id = default_id, _ => {} + // } - let device_info = self.create_audio_device( - &device, - flow, - is_default, - &endpoint_volume, - )?; - - self.devices.insert( - device_id, - DeviceState { - output: device_info, - volume_callback: endpoint_volume, - }, - ); - } - } + // for device in devices { + // let (device_id, _) = self.get_device_info(&device, flow)?; + // active_devices.insert(device_id.clone()); + + // let endpoint_volume = + // if let Some(state) = self.devices.get(&device_id) { + // state.volume_callback.clone() + // } else { + // self.register_volume_callback(&device, device_id.clone())? + // }; + + // let is_default = match flow { + // e if e == eRender => { + // Some(&device_id) == self.default_playback_id.as_ref() + // } + // e if e == eCapture => { + // Some(&device_id) == self.default_recording_id.as_ref() + // } + // _ => false, + // }; + + // let device_info = self.create_audio_device( + // &device, + // flow, + // is_default, + // &endpoint_volume, + // )?; + + // self.devices.insert( + // device_id, + // DeviceState { + // output: device_info, + // volume_callback: endpoint_volume, + // }, + // ); + // } + // } - // Remove devices that are no longer active. - self.devices.retain(|id, _| active_devices.contains(id)); + // // Remove devices that are no longer active. + // self.devices.retain(|id, _| active_devices.contains(id)); - Ok(()) + // Ok(()) } /// Handles an audio event. @@ -353,6 +396,7 @@ impl Provider for AudioProvider { fn start_sync(&mut self) { if let Err(err) = self.start() { + tracing::error!("Error starting audio provider: {}", err); self.common.emitter.emit_output::(Err(err)); } } From b559a32e701bd1c0ae01102e343607ad541bede2 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 17:30:29 +0800 Subject: [PATCH 10/23] wip update device states --- .../src/providers/audio/audio_provider.rs | 217 +++++++++--------- 1 file changed, 107 insertions(+), 110 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 6e2639e5..3bf1896e 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -9,7 +9,7 @@ use windows::Win32::{ PKEY_Device_DeviceDesc, PKEY_Device_FriendlyName, }, Media::Audio::{ - eAll, eCapture, eRender, EDataFlow, ERole, + eAll, eCapture, eMultimedia, eRender, EDataFlow, ERole, Endpoints::{ IAudioEndpointVolume, IAudioEndpointVolumeCallback, IAudioEndpointVolumeCallback_Impl, @@ -60,15 +60,22 @@ pub struct AudioDevice { pub enum DeviceType { Playback, Recording, - Hybrid, } impl From for DeviceType { fn from(flow: EDataFlow) -> Self { match flow { - e if e == eRender => Self::Playback, - e if e == eCapture => Self::Recording, - _ => Self::Hybrid, + eCapture => Self::Recording, + _ => Self::Playback, + } + } +} + +impl From for EDataFlow { + fn from(device_type: DeviceType) -> Self { + match device_type { + DeviceType::Playback => eRender, + DeviceType::Recording => eCapture, } } } @@ -86,17 +93,17 @@ enum AudioEvent { /// Holds the state of an audio device. #[derive(Clone)] struct DeviceState { - device: IMMDevice, + com_device: IMMDevice, + com_volume: IAudioEndpointVolume, output: AudioDevice, - volume_callback: IAudioEndpointVolume, } pub struct AudioProvider { common: CommonProviderState, - device_enumerator: Option, + com_enumerator: Option, default_playback_id: Option, default_recording_id: Option, - devices: HashMap, + device_states: HashMap, event_tx: channel::Sender, event_rx: channel::Receiver, } @@ -110,10 +117,10 @@ impl AudioProvider { Self { common, - device_enumerator: None, + com_enumerator: None, default_playback_id: None, default_recording_id: None, - devices: HashMap::new(), + device_states: HashMap::new(), event_tx, event_rx, } @@ -123,7 +130,7 @@ impl AudioProvider { fn start(&mut self) -> anyhow::Result<()> { let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; - let device_enumerator: IMMDeviceEnumerator = + let com_enumerator: IMMDeviceEnumerator = unsafe { CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) }?; // Note that this would sporadically segfault if we didn't keep a @@ -135,43 +142,14 @@ impl AudioProvider { // Register device add/remove callback. unsafe { - device_enumerator.RegisterEndpointNotificationCallback( + com_enumerator.RegisterEndpointNotificationCallback( &IMMNotificationClient::from(callback), ) }?; - self.device_enumerator = Some(device_enumerator); - - // Emit initial state. - - let devices = self.enumerate_devices(eRender)?; - - for (i, device) in devices.iter().enumerate() { - println!("Device: {:?}", device); - println!("Device ID: {:?}", unsafe { device.GetId()?.to_string() }); - let endpoint = unsafe { device.cast::() }?; - - println!("Device State: {:?}", unsafe { - device.GetState().unwrap() - }); - println!("Endpoint: {:?}", unsafe { endpoint.GetDataFlow() }); - // Get device properties for debugging - if let Ok(properties) = - unsafe { device.OpenPropertyStore(STGM_READ) } - { - let friendly_name = unsafe { - properties.GetValue(&PKEY_Device_FriendlyName)?.to_string() - }; - info!("Friendly Name: {}", friendly_name); - let device_desc = unsafe { - properties.GetValue(&PKEY_Device_DeviceDesc)?.to_string() - }; - info!("Description: {}", device_desc); - } else { - tracing::error!("Failed to get properties for device {}", i); - } - } - // self.update_device_state()?; + self.com_enumerator = Some(com_enumerator); + self.update_device_states()?; + self.emit_output(); // Listen to audio-related events. while let Ok(event) = self.event_rx.recv() { @@ -186,14 +164,17 @@ impl AudioProvider { /// Enumerates active devices of a specific type. fn enumerate_devices( &self, - flow: EDataFlow, + device_type: DeviceType, ) -> anyhow::Result> { let collection = unsafe { self - .device_enumerator + .com_enumerator .as_ref() .context("Device enumerator not initialized.")? - .EnumAudioEndpoints(eAll, DEVICE_STATE_ACTIVE) + .EnumAudioEndpoints( + EDataFlow::from(device_type), + DEVICE_STATE_ACTIVE, + ) }?; let count = unsafe { collection.GetCount() }?; @@ -251,7 +232,7 @@ impl AudioProvider { default_recording_device: None, }; - for (id, state) in &self.devices { + for (id, state) in &self.device_states { let device = &state.output; output.all_devices.push(device.clone()); @@ -280,68 +261,84 @@ impl AudioProvider { self.common.emitter.emit_output(Ok(output)); } - fn update_device_state(&mut self) -> anyhow::Result<()> { - Ok(()) - // let mut active_devices = HashSet::new(); - - // // Process both playback and recording devices - // for flow in [eRender, eCapture] { - // let devices = self.enumerate_devices(flow)?; - // let default_device = self.get_default_device(flow).ok(); - // let default_id = default_device - // .as_ref() - // .and_then(|d| unsafe { d.GetId().ok() }) - // .and_then(|id| unsafe { id.to_string().ok() }); - - // // Update default device IDs - // match flow { - // e if e == eRender => self.default_playback_id = - // default_id.clone(), e if e == eCapture => - // self.default_recording_id = default_id, _ => {} - // } + /// Gets the default device for the given device type. + fn default_device_for_type( + &self, + device_type: DeviceType, + ) -> anyhow::Result> { + let device = unsafe { + self + .com_enumerator + .as_ref() + .context("No enumerator")? + .GetDefaultAudioEndpoint(EDataFlow::from(device_type), eMultimedia) + } + .ok(); - // for device in devices { - // let (device_id, _) = self.get_device_info(&device, flow)?; - // active_devices.insert(device_id.clone()); - - // let endpoint_volume = - // if let Some(state) = self.devices.get(&device_id) { - // state.volume_callback.clone() - // } else { - // self.register_volume_callback(&device, device_id.clone())? - // }; - - // let is_default = match flow { - // e if e == eRender => { - // Some(&device_id) == self.default_playback_id.as_ref() - // } - // e if e == eCapture => { - // Some(&device_id) == self.default_recording_id.as_ref() - // } - // _ => false, - // }; - - // let device_info = self.create_audio_device( - // &device, - // flow, - // is_default, - // &endpoint_volume, - // )?; - - // self.devices.insert( - // device_id, - // DeviceState { - // output: device_info, - // volume_callback: endpoint_volume, - // }, - // ); - // } - // } + Ok(device) + } + + fn update_device_states(&mut self) -> anyhow::Result<()> { + let mut active_devices = HashSet::new(); + + // Process both playback and recording devices + for device_type in [DeviceType::Playback, DeviceType::Recording] { + let devices = self.enumerate_devices(device_type)?; + let default_device = self.default_device_for_type(device_type)?; + let default_id = default_device + .as_ref() + .and_then(|d| unsafe { d.GetId().ok() }) + .and_then(|id| unsafe { id.to_string().ok() }); - // // Remove devices that are no longer active. - // self.devices.retain(|id, _| active_devices.contains(id)); + // Update default device IDs. + if device_type == DeviceType::Recording { + self.default_recording_id = default_id; + } else { + self.default_playback_id = default_id.clone(); + } - // Ok(()) + for device in devices { + let device_id = unsafe { device.GetId()?.to_string() }?; + active_devices.insert(device_id); + + let endpoint_volume = + if let Some(state) = self.device_states.get(&device_id) { + state.com_volume.clone() + } else { + self.register_volume_callback(&device, device_id.clone())? + }; + + let is_default = match flow { + e if e == eRender => { + Some(&device_id) == self.default_playback_id.as_ref() + } + e if e == eCapture => { + Some(&device_id) == self.default_recording_id.as_ref() + } + _ => false, + }; + + let device_info = self.create_audio_device( + &device, + flow, + is_default, + &endpoint_volume, + )?; + + self.devices.insert( + device_id, + DeviceState { + output: device_info, + com_volume: endpoint_volume, + }, + ); + } + } + + // Remove devices that are no longer active. + self.devices.retain(|id, _| active_devices.contains(id)); + + Ok(()) } /// Handles an audio event. @@ -351,10 +348,10 @@ impl AudioProvider { | AudioEvent::DeviceRemoved(..) | AudioEvent::DeviceStateChanged(..) | AudioEvent::DefaultDeviceChanged(..) => { - self.update_device_state()?; + self.update_device_states()?; } AudioEvent::VolumeChanged(device_id, new_volume) => { - if let Some(state) = self.devices.get_mut(&device_id) { + if let Some(state) = self.device_states.get_mut(&device_id) { state.output.volume = (new_volume * 100.0).round() as u32; } } From 5ae949816cd064baf1c818af8798a735f84058d8 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 18:20:09 +0800 Subject: [PATCH 11/23] somewhat working but segfaults --- .../src/providers/audio/audio_provider.rs | 178 ++++++++++-------- 1 file changed, 101 insertions(+), 77 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 3bf1896e..3a822446 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -65,7 +65,7 @@ pub enum DeviceType { impl From for DeviceType { fn from(flow: EDataFlow) -> Self { match flow { - eCapture => Self::Recording, + flow if flow == eCapture => Self::Recording, _ => Self::Playback, } } @@ -86,7 +86,7 @@ enum AudioEvent { DeviceAdded(String), DeviceRemoved(String), DeviceStateChanged(String, DEVICE_STATE), - DefaultDeviceChanged(String, EDataFlow), + DefaultDeviceChanged(String, DeviceType), VolumeChanged(String, f32), } @@ -148,7 +148,15 @@ impl AudioProvider { }?; self.com_enumerator = Some(com_enumerator); + + // Update device list and default device IDs. + self.default_playback_id = + self.default_device_id(&DeviceType::Playback)?; + self.default_recording_id = + self.default_device_id(&DeviceType::Recording)?; self.update_device_states()?; + + // Emit initial output. self.emit_output(); // Listen to audio-related events. @@ -162,9 +170,9 @@ impl AudioProvider { } /// Enumerates active devices of a specific type. - fn enumerate_devices( + fn devices_of_type( &self, - device_type: DeviceType, + device_type: &DeviceType, ) -> anyhow::Result> { let collection = unsafe { self @@ -172,7 +180,7 @@ impl AudioProvider { .as_ref() .context("Device enumerator not initialized.")? .EnumAudioEndpoints( - EDataFlow::from(device_type), + EDataFlow::from(device_type.clone()), DEVICE_STATE_ACTIVE, ) }?; @@ -187,10 +195,10 @@ impl AudioProvider { /// Gets the friendly name of a device. /// - /// Returns a string. For example, "Headphones (WH-1000XM3 Stereo)". - fn device_name(&self, device: &IMMDevice) -> anyhow::Result { + /// Returns a string (e.g. `Headphones (WH-1000XM3 Stereo)`). + fn device_name(&self, com_device: &IMMDevice) -> anyhow::Result { let store: IPropertyStore = - unsafe { device.OpenPropertyStore(STGM_READ) }?; + unsafe { com_device.OpenPropertyStore(STGM_READ) }?; let friendly_name = unsafe { store.GetValue(&PKEY_Device_FriendlyName)?.to_string() }; @@ -201,11 +209,11 @@ impl AudioProvider { /// Registers volume callbacks for a device. fn register_volume_callback( &self, - device: &IMMDevice, + com_device: &IMMDevice, device_id: String, ) -> anyhow::Result { let endpoint_volume = unsafe { - device.Activate::(CLSCTX_ALL, None) + com_device.Activate::(CLSCTX_ALL, None) }?; let callback = VolumeCallback { @@ -243,10 +251,6 @@ impl AudioProvider { DeviceType::Recording => { output.recording_devices.push(device.clone()); } - _ => { - output.playback_devices.push(device.clone()); - output.recording_devices.push(device.clone()); - } } if self.default_playback_id.as_ref() == Some(id) { @@ -261,82 +265,90 @@ impl AudioProvider { self.common.emitter.emit_output(Ok(output)); } - /// Gets the default device for the given device type. - fn default_device_for_type( + /// Gets the default device ID for the given device type. + fn default_device_id( &self, - device_type: DeviceType, - ) -> anyhow::Result> { - let device = unsafe { + device_type: &DeviceType, + ) -> anyhow::Result> { + let default_device = unsafe { self .com_enumerator .as_ref() - .context("No enumerator")? - .GetDefaultAudioEndpoint(EDataFlow::from(device_type), eMultimedia) + .context("Device enumerator not initialized.")? + .GetDefaultAudioEndpoint( + EDataFlow::from(device_type.clone()), + eMultimedia, + ) } .ok(); - Ok(device) + let device_id = default_device + .and_then(|device| unsafe { device.GetId().ok() }) + .and_then(|id| unsafe { id.to_string().ok() }); + + Ok(device_id) + } + + /// Whether a device is the default for the given type. + fn is_default_device( + &self, + device_id: &str, + device_type: &DeviceType, + ) -> bool { + match device_type { + DeviceType::Playback => { + self.default_playback_id == Some(device_id.to_string()) + } + DeviceType::Recording => { + self.default_recording_id == Some(device_id.to_string()) + } + } } + /// Updates the list of active devices and default device IDs. fn update_device_states(&mut self) -> anyhow::Result<()> { - let mut active_devices = HashSet::new(); + let mut found_ids = HashSet::new(); - // Process both playback and recording devices + // Process both playback and recording devices. for device_type in [DeviceType::Playback, DeviceType::Recording] { - let devices = self.enumerate_devices(device_type)?; - let default_device = self.default_device_for_type(device_type)?; - let default_id = default_device - .as_ref() - .and_then(|d| unsafe { d.GetId().ok() }) - .and_then(|id| unsafe { id.to_string().ok() }); - - // Update default device IDs. - if device_type == DeviceType::Recording { - self.default_recording_id = default_id; - } else { - self.default_playback_id = default_id.clone(); - } + for com_device in self.devices_of_type(&device_type)? { + let device_id = unsafe { com_device.GetId()?.to_string() }?; + found_ids.insert(device_id.clone()); + + if !self.device_states.contains_key(&device_id) { + debug!("New audio device detected: {}", device_id); + + let is_default = + self.is_default_device(&device_id, &device_type); - for device in devices { - let device_id = unsafe { device.GetId()?.to_string() }?; - active_devices.insert(device_id); + let com_volume = self + .register_volume_callback(&com_device, device_id.clone())?; - let endpoint_volume = - if let Some(state) = self.device_states.get(&device_id) { - state.com_volume.clone() - } else { - self.register_volume_callback(&device, device_id.clone())? + let volume = unsafe { com_volume.GetMasterVolumeLevelScalar() }?; + + let output = AudioDevice { + name: self.device_name(&com_device)?, + device_id: device_id.clone(), + device_type: device_type.clone(), + volume: (volume * 100.0).round() as u32, + is_default, }; - let is_default = match flow { - e if e == eRender => { - Some(&device_id) == self.default_playback_id.as_ref() - } - e if e == eCapture => { - Some(&device_id) == self.default_recording_id.as_ref() - } - _ => false, - }; - - let device_info = self.create_audio_device( - &device, - flow, - is_default, - &endpoint_volume, - )?; - - self.devices.insert( - device_id, - DeviceState { - output: device_info, - com_volume: endpoint_volume, - }, - ); + self.device_states.insert( + device_id, + DeviceState { + com_device, + output, + com_volume, + }, + ); + } } } // Remove devices that are no longer active. - self.devices.retain(|id, _| active_devices.contains(id)); + // TODO: Remove volume callbacks. + self.device_states.retain(|id, _| found_ids.contains(id)); Ok(()) } @@ -346,10 +358,19 @@ impl AudioProvider { match event { AudioEvent::DeviceAdded(..) | AudioEvent::DeviceRemoved(..) - | AudioEvent::DeviceStateChanged(..) - | AudioEvent::DefaultDeviceChanged(..) => { + | AudioEvent::DeviceStateChanged(..) => { self.update_device_states()?; } + AudioEvent::DefaultDeviceChanged(device_id, device_type) => { + match device_type { + DeviceType::Playback => { + self.default_playback_id = Some(device_id); + } + DeviceType::Recording => { + self.default_recording_id = Some(device_id); + } + } + } AudioEvent::VolumeChanged(device_id, new_volume) => { if let Some(state) = self.device_states.get_mut(&device_id) { state.output.volume = (new_volume * 100.0).round() as u32; @@ -475,13 +496,16 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { fn OnDefaultDeviceChanged( &self, flow: EDataFlow, - _role: ERole, + role: ERole, default_device_id: &PCWSTR, ) -> windows::core::Result<()> { - if let Ok(id) = unsafe { default_device_id.to_string() } { - let _ = self - .event_sender - .send(AudioEvent::DefaultDeviceChanged(id, flow)); + if role == eMultimedia { + if let Ok(id) = unsafe { default_device_id.to_string() } { + let _ = self.event_sender.send(AudioEvent::DefaultDeviceChanged( + id, + DeviceType::from(flow), + )); + } } Ok(()) From 7dae5b0c9f350bf53fadc8abfb87a88574d33dea Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 19:16:26 +0800 Subject: [PATCH 12/23] feat: deregister volume callbacks on remove --- .../src/providers/audio/audio_provider.rs | 109 ++++++++++-------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 3a822446..a7b4d70f 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -95,6 +95,7 @@ enum AudioEvent { struct DeviceState { com_device: IMMDevice, com_volume: IAudioEndpointVolume, + com_volume_callback: VolumeCallback, output: AudioDevice, } @@ -169,20 +170,14 @@ impl AudioProvider { Ok(()) } - /// Enumerates active devices of a specific type. - fn devices_of_type( - &self, - device_type: &DeviceType, - ) -> anyhow::Result> { + /// Enumerates active devices of all types. + fn enumerate_devices(&self) -> anyhow::Result> { let collection = unsafe { self .com_enumerator .as_ref() .context("Device enumerator not initialized.")? - .EnumAudioEndpoints( - EDataFlow::from(device_type.clone()), - DEVICE_STATE_ACTIVE, - ) + .EnumAudioEndpoints(eAll, DEVICE_STATE_ACTIVE) }?; let count = unsafe { collection.GetCount() }?; @@ -211,7 +206,7 @@ impl AudioProvider { &self, com_device: &IMMDevice, device_id: String, - ) -> anyhow::Result { + ) -> anyhow::Result<(IAudioEndpointVolume, VolumeCallback)> { let endpoint_volume = unsafe { com_device.Activate::(CLSCTX_ALL, None) }?; @@ -223,11 +218,11 @@ impl AudioProvider { unsafe { endpoint_volume.RegisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(callback), + &IAudioEndpointVolumeCallback::from(callback.clone()), ) }?; - Ok(endpoint_volume) + Ok((endpoint_volume, callback)) } /// Emits an `AudioOutput` update through the provider's emitter. @@ -310,45 +305,65 @@ impl AudioProvider { let mut found_ids = HashSet::new(); // Process both playback and recording devices. - for device_type in [DeviceType::Playback, DeviceType::Recording] { - for com_device in self.devices_of_type(&device_type)? { - let device_id = unsafe { com_device.GetId()?.to_string() }?; - found_ids.insert(device_id.clone()); - - if !self.device_states.contains_key(&device_id) { - debug!("New audio device detected: {}", device_id); - - let is_default = - self.is_default_device(&device_id, &device_type); - - let com_volume = self - .register_volume_callback(&com_device, device_id.clone())?; - - let volume = unsafe { com_volume.GetMasterVolumeLevelScalar() }?; - - let output = AudioDevice { - name: self.device_name(&com_device)?, - device_id: device_id.clone(), - device_type: device_type.clone(), - volume: (volume * 100.0).round() as u32, - is_default, - }; - - self.device_states.insert( - device_id, - DeviceState { - com_device, - output, - com_volume, - }, - ); - } + for com_device in self.enumerate_devices()? { + let device_id = unsafe { com_device.GetId()?.to_string() }?; + found_ids.insert(device_id.clone()); + + if !self.device_states.contains_key(&device_id) { + debug!("New audio device detected: {}", device_id); + + let device_type = DeviceType::from(unsafe { + com_device.cast::()?.GetDataFlow() + }?); + + let is_default = self.is_default_device(&device_id, &device_type); + + let (com_volume, com_volume_callback) = + self.register_volume_callback(&com_device, device_id.clone())?; + + let volume = unsafe { com_volume.GetMasterVolumeLevelScalar() }?; + + let output = AudioDevice { + name: self.device_name(&com_device)?, + device_id: device_id.clone(), + device_type: device_type.clone(), + volume: (volume * 100.0).round() as u32, + is_default, + }; + + self.device_states.insert( + device_id, + DeviceState { + com_device, + output, + com_volume, + com_volume_callback, + }, + ); } } + let removed_ids = self + .device_states + .keys() + .filter(|id| !found_ids.contains(*id)) + .cloned() + .collect::>(); + // Remove devices that are no longer active. - // TODO: Remove volume callbacks. - self.device_states.retain(|id, _| found_ids.contains(id)); + for device_id in &removed_ids { + if let Some(state) = self.device_states.remove(device_id) { + debug!("Audio device removed: {}", device_id); + + unsafe { + state.com_volume.UnregisterControlChangeNotify( + &state + .com_volume_callback + .cast::()?, + ) + }?; + } + } Ok(()) } From b2aadd14bdec766617dcdddef6ed3f981d2f252f Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 20:41:23 +0800 Subject: [PATCH 13/23] feat: separate handling of add/remove devices --- .../src/providers/audio/audio_provider.rs | 169 +++++++++--------- 1 file changed, 87 insertions(+), 82 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index a7b4d70f..ef3b5b23 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -24,7 +24,7 @@ use windows::Win32::{ }, UI::Shell::PropertiesSystem::{IPropertyStore, PROPERTYKEY}, }; -use windows_core::{Interface, PCWSTR}; +use windows_core::{Interface, HSTRING, PCWSTR}; use crate::{ common::windows::COM_INIT, @@ -85,7 +85,6 @@ impl From for EDataFlow { enum AudioEvent { DeviceAdded(String), DeviceRemoved(String), - DeviceStateChanged(String, DEVICE_STATE), DefaultDeviceChanged(String, DeviceType), VolumeChanged(String, f32), } @@ -138,7 +137,7 @@ impl AudioProvider { // separate variable for `DeviceCallback` when registering the // callback. Something funky with lifetimes and the COM API's. let callback = DeviceCallback { - event_sender: self.event_tx.clone(), + event_tx: self.event_tx.clone(), }; // Register device add/remove callback. @@ -151,11 +150,14 @@ impl AudioProvider { self.com_enumerator = Some(com_enumerator); // Update device list and default device IDs. + for com_device in self.active_devices()? { + self.add_device(com_device)?; + } + self.default_playback_id = self.default_device_id(&DeviceType::Playback)?; self.default_recording_id = self.default_device_id(&DeviceType::Recording)?; - self.update_device_states()?; // Emit initial output. self.emit_output(); @@ -170,8 +172,8 @@ impl AudioProvider { Ok(()) } - /// Enumerates active devices of all types. - fn enumerate_devices(&self) -> anyhow::Result> { + /// Enumerates active devices of all device types. + fn active_devices(&self) -> anyhow::Result> { let collection = unsafe { self .com_enumerator @@ -213,7 +215,7 @@ impl AudioProvider { let callback = VolumeCallback { device_id, - event_sender: self.event_tx.clone(), + event_tx: self.event_tx.clone(), }; unsafe { @@ -300,69 +302,68 @@ impl AudioProvider { } } - /// Updates the list of active devices and default device IDs. - fn update_device_states(&mut self) -> anyhow::Result<()> { - let mut found_ids = HashSet::new(); - - // Process both playback and recording devices. - for com_device in self.enumerate_devices()? { - let device_id = unsafe { com_device.GetId()?.to_string() }?; - found_ids.insert(device_id.clone()); - - if !self.device_states.contains_key(&device_id) { - debug!("New audio device detected: {}", device_id); - - let device_type = DeviceType::from(unsafe { - com_device.cast::()?.GetDataFlow() - }?); - - let is_default = self.is_default_device(&device_id, &device_type); - - let (com_volume, com_volume_callback) = - self.register_volume_callback(&com_device, device_id.clone())?; - - let volume = unsafe { com_volume.GetMasterVolumeLevelScalar() }?; - - let output = AudioDevice { - name: self.device_name(&com_device)?, - device_id: device_id.clone(), - device_type: device_type.clone(), - volume: (volume * 100.0).round() as u32, - is_default, - }; - - self.device_states.insert( - device_id, - DeviceState { - com_device, - output, - com_volume, - com_volume_callback, - }, - ); - } - } + /// Adds a device by its ID. + fn add_device_by_id(&mut self, device_id: &str) -> anyhow::Result<()> { + let com_device = unsafe { + self + .com_enumerator + .as_ref() + .context("Device enumerator not initialized.")? + .GetDevice(&HSTRING::from(device_id)) + }?; - let removed_ids = self - .device_states - .keys() - .filter(|id| !found_ids.contains(*id)) - .cloned() - .collect::>(); - - // Remove devices that are no longer active. - for device_id in &removed_ids { - if let Some(state) = self.device_states.remove(device_id) { - debug!("Audio device removed: {}", device_id); - - unsafe { - state.com_volume.UnregisterControlChangeNotify( - &state - .com_volume_callback - .cast::()?, - ) - }?; - } + self.add_device(com_device) + } + + /// Adds a device by its COM object. + fn add_device(&mut self, com_device: IMMDevice) -> anyhow::Result<()> { + let device_id = unsafe { com_device.GetId()?.to_string() }?; + info!("Adding new audio device: {}", device_id); + + let device_type = DeviceType::from(unsafe { + com_device.cast::()?.GetDataFlow() + }?); + + let is_default = self.is_default_device(&device_id, &device_type); + + let (com_volume, com_volume_callback) = + self.register_volume_callback(&com_device, device_id.clone())?; + + let volume = unsafe { com_volume.GetMasterVolumeLevelScalar() }?; + + let output = AudioDevice { + name: self.device_name(&com_device)?, + device_id: device_id.clone(), + device_type: device_type.clone(), + volume: (volume * 100.0).round() as u32, + is_default, + }; + + self.device_states.insert( + device_id, + DeviceState { + com_device, + output, + com_volume, + com_volume_callback, + }, + ); + + Ok(()) + } + + /// Removes a device that is no longer active. + /// + /// Deregisters volume callback and removes device from state. + fn remove_device(&mut self, device_id: &str) -> anyhow::Result<()> { + if let Some(state) = self.device_states.remove(device_id) { + info!("Audio device removed: {}", device_id); + + unsafe { + state.com_volume.UnregisterControlChangeNotify( + &IAudioEndpointVolumeCallback::from(state.com_volume_callback), + ) + }?; } Ok(()) @@ -371,10 +372,11 @@ impl AudioProvider { /// Handles an audio event. fn handle_event(&mut self, event: AudioEvent) -> anyhow::Result<()> { match event { - AudioEvent::DeviceAdded(..) - | AudioEvent::DeviceRemoved(..) - | AudioEvent::DeviceStateChanged(..) => { - self.update_device_states()?; + AudioEvent::DeviceAdded(device_id) => { + self.add_device_by_id(&device_id)?; + } + AudioEvent::DeviceRemoved(device_id) => { + self.remove_device(&device_id)?; } AudioEvent::DefaultDeviceChanged(device_id, device_type) => { match device_type { @@ -443,7 +445,7 @@ impl Provider for AudioProvider { #[windows::core::implement(IAudioEndpointVolumeCallback)] struct VolumeCallback { device_id: String, - event_sender: channel::Sender, + event_tx: channel::Sender, } impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { @@ -452,7 +454,7 @@ impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { data: *mut AUDIO_VOLUME_NOTIFICATION_DATA, ) -> windows::core::Result<()> { if let Some(data) = unsafe { data.as_ref() } { - let _ = self.event_sender.send(AudioEvent::VolumeChanged( + let _ = self.event_tx.send(AudioEvent::VolumeChanged( self.device_id.clone(), data.fMasterVolume, )); @@ -468,7 +470,7 @@ impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { /// the default device changes. #[windows::core::implement(IMMNotificationClient)] struct DeviceCallback { - event_sender: channel::Sender, + event_tx: channel::Sender, } impl IMMNotificationClient_Impl for DeviceCallback_Impl { @@ -477,7 +479,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - let _ = self.event_sender.send(AudioEvent::DeviceAdded(id)); + let _ = self.event_tx.send(AudioEvent::DeviceAdded(id)); } Ok(()) @@ -488,7 +490,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - let _ = self.event_sender.send(AudioEvent::DeviceRemoved(id)); + let _ = self.event_tx.send(AudioEvent::DeviceRemoved(id)); } Ok(()) @@ -500,9 +502,12 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { new_state: DEVICE_STATE, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - let _ = self - .event_sender - .send(AudioEvent::DeviceStateChanged(id, new_state)); + let event = match new_state { + DEVICE_STATE_ACTIVE => AudioEvent::DeviceAdded(id), + _ => AudioEvent::DeviceRemoved(id), + }; + + let _ = self.event_tx.send(event); } Ok(()) @@ -516,7 +521,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { ) -> windows::core::Result<()> { if role == eMultimedia { if let Ok(id) = unsafe { default_device_id.to_string() } { - let _ = self.event_sender.send(AudioEvent::DefaultDeviceChanged( + let _ = self.event_tx.send(AudioEvent::DefaultDeviceChanged( id, DeviceType::from(flow), )); From 6faad66763a068519892056f57164d88716f8e9b Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 23:36:09 +0800 Subject: [PATCH 14/23] no more segfaults. state updates working except default device --- packages/desktop/src/common/windows/com.rs | 3 +- .../src/providers/audio/audio_provider.rs | 60 ++++++++++++------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/packages/desktop/src/common/windows/com.rs b/packages/desktop/src/common/windows/com.rs index 20f8e8b3..c8bd91bd 100644 --- a/packages/desktop/src/common/windows/com.rs +++ b/packages/desktop/src/common/windows/com.rs @@ -1,5 +1,6 @@ use windows::Win32::System::Com::{ CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, + COINIT_MULTITHREADED, }; thread_local! { @@ -20,7 +21,7 @@ impl ComInit { /// Panics if COM initialization fails. This is typically only possible /// if COM is already initialized with an incompatible threading model. pub fn new() -> Self { - unsafe { CoInitializeEx(None, COINIT_APARTMENTTHREADED) } + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } .ok() .expect("Unable to initialize COM."); diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index ef3b5b23..e16d736d 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -27,7 +27,7 @@ use windows::Win32::{ use windows_core::{Interface, HSTRING, PCWSTR}; use crate::{ - common::windows::COM_INIT, + common::windows::{ComInit, COM_INIT}, providers::{CommonProviderState, Provider, RuntimeType}, }; @@ -94,7 +94,7 @@ enum AudioEvent { struct DeviceState { com_device: IMMDevice, com_volume: IAudioEndpointVolume, - com_volume_callback: VolumeCallback, + com_volume_callback: IAudioEndpointVolumeCallback, output: AudioDevice, } @@ -128,23 +128,23 @@ impl AudioProvider { /// Main entry point. fn start(&mut self) -> anyhow::Result<()> { - let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + let _com = ComInit::new(); let com_enumerator: IMMDeviceEnumerator = unsafe { CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) }?; // Note that this would sporadically segfault if we didn't keep a - // separate variable for `DeviceCallback` when registering the + // separate variable for `IMMNotificationClient` when registering the // callback. Something funky with lifetimes and the COM API's. - let callback = DeviceCallback { + let com_device_callback: IMMNotificationClient = DeviceCallback { event_tx: self.event_tx.clone(), - }; + } + .into(); // Register device add/remove callback. unsafe { - com_enumerator.RegisterEndpointNotificationCallback( - &IMMNotificationClient::from(callback), - ) + com_enumerator + .RegisterEndpointNotificationCallback(&com_device_callback) }?; self.com_enumerator = Some(com_enumerator); @@ -208,23 +208,24 @@ impl AudioProvider { &self, com_device: &IMMDevice, device_id: String, - ) -> anyhow::Result<(IAudioEndpointVolume, VolumeCallback)> { - let endpoint_volume = unsafe { + ) -> anyhow::Result<(IAudioEndpointVolume, IAudioEndpointVolumeCallback)> + { + let com_volume = unsafe { com_device.Activate::(CLSCTX_ALL, None) }?; - let callback = VolumeCallback { - device_id, - event_tx: self.event_tx.clone(), - }; + let com_volume_callback: IAudioEndpointVolumeCallback = + VolumeCallback { + device_id, + event_tx: self.event_tx.clone(), + } + .into(); unsafe { - endpoint_volume.RegisterControlChangeNotify( - &IAudioEndpointVolumeCallback::from(callback.clone()), - ) + com_volume.RegisterControlChangeNotify(&com_volume_callback) }?; - Ok((endpoint_volume, callback)) + Ok((com_volume, com_volume_callback)) } /// Emits an `AudioOutput` update through the provider's emitter. @@ -454,10 +455,15 @@ impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { data: *mut AUDIO_VOLUME_NOTIFICATION_DATA, ) -> windows::core::Result<()> { if let Some(data) = unsafe { data.as_ref() } { + tracing::info!("Volume changed: {:?}", data.fMasterVolume); let _ = self.event_tx.send(AudioEvent::VolumeChanged( self.device_id.clone(), data.fMasterVolume, )); + tracing::info!( + "Sent volume changed event: {:?}", + data.fMasterVolume + ); } Ok(()) @@ -479,7 +485,9 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - let _ = self.event_tx.send(AudioEvent::DeviceAdded(id)); + tracing::info!("Device added: {:?}", id); + let _ = self.event_tx.send(AudioEvent::DeviceAdded(id.clone())); + tracing::info!("Sent device added event: {:?}", id); } Ok(()) @@ -490,7 +498,9 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - let _ = self.event_tx.send(AudioEvent::DeviceRemoved(id)); + tracing::info!("Device removed: {:?}", id); + let _ = self.event_tx.send(AudioEvent::DeviceRemoved(id.clone())); + tracing::info!("Sent device removed event: {:?}", id); } Ok(()) @@ -502,12 +512,14 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { new_state: DEVICE_STATE, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { + tracing::info!("Device state changed: {:?}", new_state); let event = match new_state { - DEVICE_STATE_ACTIVE => AudioEvent::DeviceAdded(id), + DEVICE_STATE_ACTIVE => AudioEvent::DeviceAdded(id.clone()), _ => AudioEvent::DeviceRemoved(id), }; let _ = self.event_tx.send(event); + tracing::info!("Sent device state changed event"); } Ok(()) @@ -519,12 +531,14 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { role: ERole, default_device_id: &PCWSTR, ) -> windows::core::Result<()> { + tracing::info!("Default device changed: {:?}", default_device_id); if role == eMultimedia { if let Ok(id) = unsafe { default_device_id.to_string() } { let _ = self.event_tx.send(AudioEvent::DefaultDeviceChanged( - id, + id.clone(), DeviceType::from(flow), )); + tracing::info!("Sent default device changed event: {:?}", id); } } From 2507c1724000ceadcfbfaba408b3e75828bfe572 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Fri, 29 Nov 2024 23:45:58 +0800 Subject: [PATCH 15/23] feat: use `COM_INIT` wrapper --- packages/desktop/src/common/windows/com.rs | 7 +- .../src/providers/audio/audio_provider.rs | 67 ++++++++++--------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/packages/desktop/src/common/windows/com.rs b/packages/desktop/src/common/windows/com.rs index c8bd91bd..77155d8a 100644 --- a/packages/desktop/src/common/windows/com.rs +++ b/packages/desktop/src/common/windows/com.rs @@ -1,6 +1,5 @@ use windows::Win32::System::Com::{ - CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, - COINIT_MULTITHREADED, + CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED, }; thread_local! { @@ -13,8 +12,8 @@ thread_local! { pub struct ComInit(); impl ComInit { - /// Initializes COM on the current thread with apartment threading model. - /// `COINIT_APARTMENTTHREADED` is required for shell COM objects. + /// Initializes COM on the current thread with multithreaded object + /// concurrency. /// /// # Panics /// diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index e16d736d..41d2ff54 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -128,48 +128,49 @@ impl AudioProvider { /// Main entry point. fn start(&mut self) -> anyhow::Result<()> { - let _com = ComInit::new(); - - let com_enumerator: IMMDeviceEnumerator = - unsafe { CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) }?; + COM_INIT.with(|_| { + let com_enumerator: IMMDeviceEnumerator = unsafe { + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) + }?; - // Note that this would sporadically segfault if we didn't keep a - // separate variable for `IMMNotificationClient` when registering the - // callback. Something funky with lifetimes and the COM API's. - let com_device_callback: IMMNotificationClient = DeviceCallback { - event_tx: self.event_tx.clone(), - } - .into(); + // Note that this would sporadically segfault if we didn't keep a + // separate variable for `IMMNotificationClient` when registering the + // callback. Something funky with lifetimes and the COM API's. + let com_device_callback: IMMNotificationClient = DeviceCallback { + event_tx: self.event_tx.clone(), + } + .into(); - // Register device add/remove callback. - unsafe { - com_enumerator - .RegisterEndpointNotificationCallback(&com_device_callback) - }?; + // Register device add/remove callback. + unsafe { + com_enumerator + .RegisterEndpointNotificationCallback(&com_device_callback) + }?; - self.com_enumerator = Some(com_enumerator); + self.com_enumerator = Some(com_enumerator); - // Update device list and default device IDs. - for com_device in self.active_devices()? { - self.add_device(com_device)?; - } + // Update device list and default device IDs. + for com_device in self.active_devices()? { + self.add_device(com_device)?; + } - self.default_playback_id = - self.default_device_id(&DeviceType::Playback)?; - self.default_recording_id = - self.default_device_id(&DeviceType::Recording)?; + self.default_playback_id = + self.default_device_id(&DeviceType::Playback)?; + self.default_recording_id = + self.default_device_id(&DeviceType::Recording)?; - // Emit initial output. - self.emit_output(); + // Emit initial output. + self.emit_output(); - // Listen to audio-related events. - while let Ok(event) = self.event_rx.recv() { - if let Err(err) = self.handle_event(event) { - info!("Error handling audio event: {}", err); + // Listen to audio-related events. + while let Ok(event) = self.event_rx.recv() { + if let Err(err) = self.handle_event(event) { + info!("Error handling audio event: {}", err); + } } - } - Ok(()) + Ok(()) + }) } /// Enumerates active devices of all device types. From 8b8eded6aa2eda0d2c576e8e8dffd30ab6788efb Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 15:33:51 +0800 Subject: [PATCH 16/23] feat: set volume --- examples/boilerplate-solid-ts/src/index.tsx | 18 +++-- .../src/desktop/desktop-commands.ts | 23 +++++- .../providers/audio/audio-provider-types.ts | 5 ++ .../providers/audio/create-audio-provider.ts | 31 +++++--- .../providers/media/create-media-provider.ts | 10 +-- .../src/providers/audio/audio_provider.rs | 70 +++++++++++++++++-- .../src/providers/provider_function.rs | 13 ++++ 7 files changed, 147 insertions(+), 23 deletions(-) diff --git a/examples/boilerplate-solid-ts/src/index.tsx b/examples/boilerplate-solid-ts/src/index.tsx index 03c0aa35..96faaed3 100644 --- a/examples/boilerplate-solid-ts/src/index.tsx +++ b/examples/boilerplate-solid-ts/src/index.tsx @@ -22,10 +22,20 @@ function App() { return (
-
- {output.audio?.defaultPlaybackDevice?.name} - - {output.audio?.defaultPlaybackDevice?.volume} -
+ {output.audio?.defaultPlaybackDevice && ( +
+ {output.audio.defaultPlaybackDevice.name}- + {output.audio.defaultPlaybackDevice.volume} + output.audio.setVolume(e.target.valueAsNumber)} + /> +
+ )}
Media: {output.media?.currentSession?.title} - {output.media?.currentSession?.artist} diff --git a/packages/client-api/src/desktop/desktop-commands.ts b/packages/client-api/src/desktop/desktop-commands.ts index 14474228..e31bdf55 100644 --- a/packages/client-api/src/desktop/desktop-commands.ts +++ b/packages/client-api/src/desktop/desktop-commands.ts @@ -19,7 +19,28 @@ export const desktopCommands = { setSkipTaskbar, }; -export type ProviderFunction = MediaFunction; +export type ProviderFunction = AudioFunction | MediaFunction; + +export interface AudioFunction { + type: 'audio'; + function: { + name: 'set_volume'; + args: SetVolumeArgs; + }; +} + +export interface SetVolumeArgs { + volume: number; + deviceId?: string; +} + +export interface MediaFunction { + type: 'media'; + function: { + name: 'play' | 'pause' | 'toggle_play_pause' | 'next' | 'previous'; + args: MediaControlArgs; + }; +} export interface MediaFunction { type: 'media'; diff --git a/packages/client-api/src/providers/audio/audio-provider-types.ts b/packages/client-api/src/providers/audio/audio-provider-types.ts index 06b4c359..55e063f3 100644 --- a/packages/client-api/src/providers/audio/audio-provider-types.ts +++ b/packages/client-api/src/providers/audio/audio-provider-types.ts @@ -11,6 +11,11 @@ export interface AudioOutput { defaultRecordingDevice: AudioDevice | null; playbackDevices: AudioDevice[]; recordingDevices: AudioDevice[]; + setVolume(volume: number, options?: SetVolumeOptions): Promise; +} + +export interface SetVolumeOptions { + deviceId?: string; } export interface AudioDevice { diff --git a/packages/client-api/src/providers/audio/create-audio-provider.ts b/packages/client-api/src/providers/audio/create-audio-provider.ts index a96af3c7..9b5894da 100644 --- a/packages/client-api/src/providers/audio/create-audio-provider.ts +++ b/packages/client-api/src/providers/audio/create-audio-provider.ts @@ -1,11 +1,12 @@ import { z } from 'zod'; import { createBaseProvider } from '../create-base-provider'; -import { onProviderEmit } from '~/desktop'; +import { desktopCommands, onProviderEmit } from '~/desktop'; import type { AudioOutput, AudioProvider, AudioProviderConfig, + SetVolumeOptions, } from './audio-provider-types'; const audioProviderConfigSchema = z.object({ @@ -18,12 +19,26 @@ export function createAudioProvider( const mergedConfig = audioProviderConfigSchema.parse(config); return createBaseProvider(mergedConfig, async queue => { - return onProviderEmit(mergedConfig, ({ result }) => { - if ('error' in result) { - queue.error(result.error); - } else { - queue.output(result.output); - } - }); + return onProviderEmit( + mergedConfig, + ({ configHash, result }) => { + if ('error' in result) { + queue.error(result.error); + } else { + queue.output({ + ...result.output, + setVolume: (volume: number, options?: SetVolumeOptions) => { + return desktopCommands.callProviderFunction(configHash, { + type: 'audio', + function: { + name: 'set_volume', + args: { volume, deviceId: options?.deviceId }, + }, + }); + }, + }); + } + }, + ); }); } diff --git a/packages/client-api/src/providers/media/create-media-provider.ts b/packages/client-api/src/providers/media/create-media-provider.ts index eb95f65d..1d1a899e 100644 --- a/packages/client-api/src/providers/media/create-media-provider.ts +++ b/packages/client-api/src/providers/media/create-media-provider.ts @@ -28,7 +28,7 @@ export function createMediaProvider( ...result.output, session: result.output.currentSession, play: (args?: MediaControlArgs) => { - desktopCommands.callProviderFunction(configHash, { + return desktopCommands.callProviderFunction(configHash, { type: 'media', function: { name: 'play', @@ -37,7 +37,7 @@ export function createMediaProvider( }); }, pause: (args?: MediaControlArgs) => { - desktopCommands.callProviderFunction(configHash, { + return desktopCommands.callProviderFunction(configHash, { type: 'media', function: { name: 'pause', @@ -46,7 +46,7 @@ export function createMediaProvider( }); }, togglePlayPause: (args?: MediaControlArgs) => { - desktopCommands.callProviderFunction(configHash, { + return desktopCommands.callProviderFunction(configHash, { type: 'media', function: { name: 'toggle_play_pause', @@ -55,7 +55,7 @@ export function createMediaProvider( }); }, next: (args?: MediaControlArgs) => { - desktopCommands.callProviderFunction(configHash, { + return desktopCommands.callProviderFunction(configHash, { type: 'media', function: { name: 'next', @@ -64,7 +64,7 @@ export function createMediaProvider( }); }, previous: (args?: MediaControlArgs) => { - desktopCommands.callProviderFunction(configHash, { + return desktopCommands.callProviderFunction(configHash, { type: 'media', function: { name: 'previous', diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 41d2ff54..15c7807a 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -24,11 +24,14 @@ use windows::Win32::{ }, UI::Shell::PropertiesSystem::{IPropertyStore, PROPERTYKEY}, }; -use windows_core::{Interface, HSTRING, PCWSTR}; +use windows_core::{Interface, GUID, HSTRING, PCWSTR}; use crate::{ common::windows::{ComInit, COM_INIT}, - providers::{CommonProviderState, Provider, RuntimeType}, + providers::{ + AudioFunction, CommonProviderState, Provider, ProviderFunction, + ProviderFunctionResponse, ProviderInputMsg, RuntimeType, + }, }; #[derive(Deserialize, Debug)] @@ -163,9 +166,32 @@ impl AudioProvider { self.emit_output(); // Listen to audio-related events. - while let Ok(event) = self.event_rx.recv() { - if let Err(err) = self.handle_event(event) { - info!("Error handling audio event: {}", err); + loop { + crossbeam::select! { + recv(self.event_rx) -> event => { + if let Ok(event) = event { + debug!("Got audio event: {:?}", event); + + if let Err(err) = self.handle_event(event) { + tracing::warn!("Error handling audio event: {}", err); + } + } + } + recv(self.common.input.sync_rx) -> input => { + match input { + Ok(ProviderInputMsg::Stop) => { + break; + } + Ok(ProviderInputMsg::Function( + ProviderFunction::Audio(audio_function), + sender, + )) => { + let res = self.handle_function(audio_function).map_err(|err| err.to_string()); + sender.send(res).unwrap(); + } + _ => {} + } + } } } @@ -402,6 +428,40 @@ impl AudioProvider { Ok(()) } + + /// Handles an incoming audio provider function call. + fn handle_function( + &mut self, + function: AudioFunction, + ) -> anyhow::Result { + match function { + AudioFunction::SetVolume(args) => { + // Get target device - use specified ID or default playback + // device. + let device_state = if let Some(id) = &args.device_id { + self + .device_states + .get(id) + .context("Specified device not found.")? + } else { + self + .default_playback_id + .as_ref() + .and_then(|id| self.device_states.get(id)) + .context("No active playback device.")? + }; + + unsafe { + device_state.com_volume.SetMasterVolumeLevelScalar( + args.volume / 100., + &GUID::zeroed(), + ) + }?; + + Ok(ProviderFunctionResponse::Null) + } + } + } } impl Drop for AudioProvider { diff --git a/packages/desktop/src/providers/provider_function.rs b/packages/desktop/src/providers/provider_function.rs index f95f88ad..234d4beb 100644 --- a/packages/desktop/src/providers/provider_function.rs +++ b/packages/desktop/src/providers/provider_function.rs @@ -3,9 +3,22 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type", content = "function", rename_all = "snake_case")] pub enum ProviderFunction { + Audio(AudioFunction), Media(MediaFunction), } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "name", content = "args", rename_all = "snake_case")] +pub enum AudioFunction { + SetVolume(SetVolumeArgs), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SetVolumeArgs { + pub volume: f32, + pub device_id: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "name", content = "args", rename_all = "snake_case")] pub enum MediaFunction { From 6163083682e39af217c3f4058d3b3d65e7b8bc2b Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 17:49:00 +0800 Subject: [PATCH 17/23] feat: add separate `is_default_playback` and `is_default_recording` --- .../src/providers/audio/audio_provider.rs | 81 ++++++------------- 1 file changed, 23 insertions(+), 58 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 15c7807a..af9c513f 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -1,13 +1,11 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use anyhow::Context; use crossbeam::channel; use serde::{Deserialize, Serialize}; use tracing::{debug, info}; use windows::Win32::{ - Devices::FunctionDiscovery::{ - PKEY_Device_DeviceDesc, PKEY_Device_FriendlyName, - }, + Devices::FunctionDiscovery::PKEY_Device_FriendlyName, Media::Audio::{ eAll, eCapture, eMultimedia, eRender, EDataFlow, ERole, Endpoints::{ @@ -18,16 +16,13 @@ use windows::Win32::{ IMMNotificationClient_Impl, MMDeviceEnumerator, AUDIO_VOLUME_NOTIFICATION_DATA, DEVICE_STATE, DEVICE_STATE_ACTIVE, }, - System::Com::{ - CoCreateInstance, CoInitializeEx, CLSCTX_ALL, COINIT_MULTITHREADED, - STGM_READ, - }, + System::Com::{CoCreateInstance, CLSCTX_ALL, STGM_READ}, UI::Shell::PropertiesSystem::{IPropertyStore, PROPERTYKEY}, }; use windows_core::{Interface, GUID, HSTRING, PCWSTR}; use crate::{ - common::windows::{ComInit, COM_INIT}, + common::windows::COM_INIT, providers::{ AudioFunction, CommonProviderState, Provider, ProviderFunction, ProviderFunctionResponse, ProviderInputMsg, RuntimeType, @@ -55,7 +50,8 @@ pub struct AudioDevice { pub device_id: String, pub device_type: DeviceType, pub volume: u32, - pub is_default: bool, + pub is_default_playback: bool, + pub is_default_recording: bool, } #[derive(Debug, Clone, PartialEq, Serialize)] @@ -291,6 +287,10 @@ impl AudioProvider { } /// Gets the default device ID for the given device type. + /// + /// Note that a device can have multiple roles (i.e. multimedia, + /// communications, and console). The device with the multimedia role is + /// typically seen as the default. fn default_device_id( &self, device_type: &DeviceType, @@ -314,22 +314,6 @@ impl AudioProvider { Ok(device_id) } - /// Whether a device is the default for the given type. - fn is_default_device( - &self, - device_id: &str, - device_type: &DeviceType, - ) -> bool { - match device_type { - DeviceType::Playback => { - self.default_playback_id == Some(device_id.to_string()) - } - DeviceType::Recording => { - self.default_recording_id == Some(device_id.to_string()) - } - } - } - /// Adds a device by its ID. fn add_device_by_id(&mut self, device_id: &str) -> anyhow::Result<()> { let com_device = unsafe { @@ -352,7 +336,10 @@ impl AudioProvider { com_device.cast::()?.GetDataFlow() }?); - let is_default = self.is_default_device(&device_id, &device_type); + let is_default_playback = + self.default_playback_id.as_ref() == Some(&device_id); + let is_default_recording = + self.default_recording_id.as_ref() == Some(&device_id); let (com_volume, com_volume_callback) = self.register_volume_callback(&com_device, device_id.clone())?; @@ -364,7 +351,8 @@ impl AudioProvider { device_id: device_id.clone(), device_type: device_type.clone(), volume: (volume * 100.0).round() as u32, - is_default, + is_default_playback, + is_default_recording, }; self.device_states.insert( @@ -466,23 +454,13 @@ impl AudioProvider { impl Drop for AudioProvider { fn drop(&mut self) { - // // Deregister volume callbacks. - // for state in self.devices.values() { - // unsafe { - // let _ = state - // .volume_callback - // .UnregisterControlChangeNotify(&state.volume_callback); - // } - // } - - // // Deregister device notification callback. - // if let Some(enumerator) = &self.device_enumerator { - // unsafe { - // let _ = enumerator.UnregisterEndpointNotificationCallback( - // &IMMNotificationClient::null(), - // ); - // } - // } + let device_ids = + self.device_states.keys().cloned().collect::>(); + + // Ensure volume callbacks are deregistered. + for device_id in device_ids { + let _ = self.remove_device(&device_id); + } } } @@ -516,15 +494,10 @@ impl IAudioEndpointVolumeCallback_Impl for VolumeCallback_Impl { data: *mut AUDIO_VOLUME_NOTIFICATION_DATA, ) -> windows::core::Result<()> { if let Some(data) = unsafe { data.as_ref() } { - tracing::info!("Volume changed: {:?}", data.fMasterVolume); let _ = self.event_tx.send(AudioEvent::VolumeChanged( self.device_id.clone(), data.fMasterVolume, )); - tracing::info!( - "Sent volume changed event: {:?}", - data.fMasterVolume - ); } Ok(()) @@ -546,9 +519,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - tracing::info!("Device added: {:?}", id); let _ = self.event_tx.send(AudioEvent::DeviceAdded(id.clone())); - tracing::info!("Sent device added event: {:?}", id); } Ok(()) @@ -559,9 +530,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - tracing::info!("Device removed: {:?}", id); let _ = self.event_tx.send(AudioEvent::DeviceRemoved(id.clone())); - tracing::info!("Sent device removed event: {:?}", id); } Ok(()) @@ -573,14 +542,12 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { new_state: DEVICE_STATE, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - tracing::info!("Device state changed: {:?}", new_state); let event = match new_state { DEVICE_STATE_ACTIVE => AudioEvent::DeviceAdded(id.clone()), _ => AudioEvent::DeviceRemoved(id), }; let _ = self.event_tx.send(event); - tracing::info!("Sent device state changed event"); } Ok(()) @@ -592,14 +559,12 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { role: ERole, default_device_id: &PCWSTR, ) -> windows::core::Result<()> { - tracing::info!("Default device changed: {:?}", default_device_id); if role == eMultimedia { if let Ok(id) = unsafe { default_device_id.to_string() } { let _ = self.event_tx.send(AudioEvent::DefaultDeviceChanged( id.clone(), DeviceType::from(flow), )); - tracing::info!("Sent default device changed event: {:?}", id); } } From b4a69d209e36847f4a036a3f686f050146160f7d Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 18:35:42 +0800 Subject: [PATCH 18/23] remove `output` from `DeviceState` --- .../src/providers/audio/audio_provider.rs | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index af9c513f..5d72c4a8 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -91,10 +91,13 @@ enum AudioEvent { /// Holds the state of an audio device. #[derive(Clone)] struct DeviceState { + name: String, + device_id: String, + device_type: DeviceType, + volume: u32, com_device: IMMDevice, com_volume: IAudioEndpointVolume, com_volume_callback: IAudioEndpointVolumeCallback, - output: AudioDevice, } pub struct AudioProvider { @@ -262,24 +265,33 @@ impl AudioProvider { }; for (id, state) in &self.device_states { - let device = &state.output; + let device = AudioDevice { + name: state.name.clone(), + device_id: state.device_id.clone(), + device_type: state.device_type.clone(), + volume: state.volume, + is_default_playback: self.default_playback_id.as_ref() == Some(id), + is_default_recording: self.default_recording_id.as_ref() + == Some(id), + }; + output.all_devices.push(device.clone()); match device.device_type { DeviceType::Playback => { output.playback_devices.push(device.clone()); + + if self.default_playback_id.as_ref() == Some(id) { + output.default_playback_device = Some(device.clone()); + } } DeviceType::Recording => { output.recording_devices.push(device.clone()); - } - } - - if self.default_playback_id.as_ref() == Some(id) { - output.default_playback_device = Some(device.clone()); - } - if self.default_recording_id.as_ref() == Some(id) { - output.default_recording_device = Some(device.clone()); + if self.default_recording_id.as_ref() == Some(id) { + output.default_recording_device = Some(device.clone()); + } + } } } @@ -336,34 +348,22 @@ impl AudioProvider { com_device.cast::()?.GetDataFlow() }?); - let is_default_playback = - self.default_playback_id.as_ref() == Some(&device_id); - let is_default_recording = - self.default_recording_id.as_ref() == Some(&device_id); - let (com_volume, com_volume_callback) = self.register_volume_callback(&com_device, device_id.clone())?; let volume = unsafe { com_volume.GetMasterVolumeLevelScalar() }?; - let output = AudioDevice { + let device_state = DeviceState { name: self.device_name(&com_device)?, device_id: device_id.clone(), device_type: device_type.clone(), volume: (volume * 100.0).round() as u32, - is_default_playback, - is_default_recording, + com_device, + com_volume, + com_volume_callback, }; - self.device_states.insert( - device_id, - DeviceState { - com_device, - output, - com_volume, - com_volume_callback, - }, - ); + self.device_states.insert(device_id, device_state); Ok(()) } @@ -406,7 +406,7 @@ impl AudioProvider { } AudioEvent::VolumeChanged(device_id, new_volume) => { if let Some(state) = self.device_states.get_mut(&device_id) { - state.output.volume = (new_volume * 100.0).round() as u32; + state.volume = (new_volume * 100.0).round() as u32; } } } @@ -530,7 +530,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { device_id: &PCWSTR, ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { - let _ = self.event_tx.send(AudioEvent::DeviceRemoved(id.clone())); + let _ = self.event_tx.send(AudioEvent::DeviceRemoved(id)); } Ok(()) @@ -543,7 +543,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { ) -> windows::core::Result<()> { if let Ok(id) = unsafe { device_id.to_string() } { let event = match new_state { - DEVICE_STATE_ACTIVE => AudioEvent::DeviceAdded(id.clone()), + DEVICE_STATE_ACTIVE => AudioEvent::DeviceAdded(id), _ => AudioEvent::DeviceRemoved(id), }; @@ -562,7 +562,7 @@ impl IMMNotificationClient_Impl for DeviceCallback_Impl { if role == eMultimedia { if let Ok(id) = unsafe { default_device_id.to_string() } { let _ = self.event_tx.send(AudioEvent::DefaultDeviceChanged( - id.clone(), + id, DeviceType::from(flow), )); } From f1a639ffb2ed2085df2948fe296c5ed155774f60 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 18:38:25 +0800 Subject: [PATCH 19/23] feat: create and start provider within its runtime --- packages/desktop/src/providers/provider.rs | 44 ------ .../desktop/src/providers/provider_manager.rs | 129 +++++++++--------- 2 files changed, 65 insertions(+), 108 deletions(-) diff --git a/packages/desktop/src/providers/provider.rs b/packages/desktop/src/providers/provider.rs index 080c9b35..6f077e03 100644 --- a/packages/desktop/src/providers/provider.rs +++ b/packages/desktop/src/providers/provider.rs @@ -1,7 +1,5 @@ use async_trait::async_trait; -use super::{ProviderFunction, ProviderFunctionResponse}; - #[async_trait] pub trait Provider { fn runtime_type(&self) -> RuntimeType; @@ -37,48 +35,6 @@ pub trait Provider { } } } - - /// Runs the given function. - /// - /// # Panics - /// - /// Panics if wrong runtime type is used. - fn call_function_sync( - &self, - function: ProviderFunction, - ) -> anyhow::Result { - let _function = function; - match self.runtime_type() { - RuntimeType::Sync => { - unreachable!("Sync providers must implement `call_function_sync`.") - } - RuntimeType::Async => { - panic!("Cannot call sync function on async provider.") - } - } - } - - /// Runs the given function. - /// - /// # Panics - /// - /// Panics if wrong runtime type is used. - async fn call_function_async( - &self, - function: ProviderFunction, - ) -> anyhow::Result { - let _function = function; - match self.runtime_type() { - RuntimeType::Async => { - unreachable!( - "Async providers must implement `call_function_async`." - ) - } - RuntimeType::Sync => { - panic!("Cannot call async function on sync provider.") - } - } - } } /// Determines whether `start_sync` or `start_async` is called. diff --git a/packages/desktop/src/providers/provider_manager.rs b/packages/desktop/src/providers/provider_manager.rs index 4477e7b1..3f547ebd 100644 --- a/packages/desktop/src/providers/provider_manager.rs +++ b/packages/desktop/src/providers/provider_manager.rs @@ -244,78 +244,79 @@ impl ProviderManager { config_hash: String, common: CommonProviderState, ) -> anyhow::Result<(task::JoinHandle<()>, RuntimeType)> { - type CreateProviderFn = ( - RuntimeType, - Box Box + Send + 'static>, - ); - - let (runtime_type, create_provider): CreateProviderFn = match config { - #[cfg(windows)] - ProviderConfig::Audio(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(AudioProvider::new(config, common))), - ), - ProviderConfig::Battery(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(BatteryProvider::new(config, common))), - ), - ProviderConfig::Cpu(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(CpuProvider::new(config, common))), - ), - ProviderConfig::Host(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(HostProvider::new(config, common))), - ), - ProviderConfig::Ip(config) => ( - RuntimeType::Async, - Box::new(|| Box::new(IpProvider::new(config, common))), - ), - #[cfg(windows)] - ProviderConfig::Komorebi(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(KomorebiProvider::new(config, common))), - ), - #[cfg(windows)] - ProviderConfig::Media(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(MediaProvider::new(config, common))), - ), - ProviderConfig::Memory(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(MemoryProvider::new(config, common))), - ), - ProviderConfig::Disk(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(DiskProvider::new(config, common))), - ), - ProviderConfig::Network(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(NetworkProvider::new(config, common))), - ), - ProviderConfig::Weather(config) => ( - RuntimeType::Async, - Box::new(|| Box::new(WeatherProvider::new(config, common))), - ), - #[cfg(windows)] - ProviderConfig::Keyboard(config) => ( - RuntimeType::Sync, - Box::new(|| Box::new(KeyboardProvider::new(config, common))), - ), - #[allow(unreachable_patterns)] - _ => bail!("Provider not supported on this operating system."), + let runtime_type = match config { + ProviderConfig::Ip(..) | ProviderConfig::Weather(..) => { + RuntimeType::Async + } + _ => RuntimeType::Sync, }; // Spawn the provider's task based on its runtime type. let task_handle = match &runtime_type { RuntimeType::Async => task::spawn(async move { - // let mut provider = create_provider(); - // provider.start_async().await; + match config { + ProviderConfig::Ip(config) => { + let mut provider = IpProvider::new(config, common); + provider.start_async().await; + } + ProviderConfig::Weather(config) => { + let mut provider = WeatherProvider::new(config, common); + provider.start_async().await; + } + _ => unreachable!(), + } + info!("Provider stopped: {}", config_hash); }), RuntimeType::Sync => task::spawn_blocking(move || { - let mut provider = create_provider(); - provider.start_sync(); + match config { + #[cfg(windows)] + ProviderConfig::Audio(config) => { + let mut provider = AudioProvider::new(config, common); + provider.start_sync(); + } + ProviderConfig::Battery(config) => { + let mut provider = BatteryProvider::new(config, common); + provider.start_sync(); + } + ProviderConfig::Cpu(config) => { + let mut provider = CpuProvider::new(config, common); + provider.start_sync(); + } + ProviderConfig::Host(config) => { + let mut provider = HostProvider::new(config, common); + provider.start_sync(); + } + #[cfg(windows)] + ProviderConfig::Komorebi(config) => { + let mut provider = KomorebiProvider::new(config, common); + provider.start_sync(); + } + #[cfg(windows)] + ProviderConfig::Media(config) => { + let mut provider = MediaProvider::new(config, common); + provider.start_sync(); + } + ProviderConfig::Memory(config) => { + let mut provider = MemoryProvider::new(config, common); + provider.start_sync(); + } + ProviderConfig::Disk(config) => { + let mut provider = DiskProvider::new(config, common); + provider.start_sync(); + } + ProviderConfig::Network(config) => { + let mut provider = NetworkProvider::new(config, common); + provider.start_sync(); + } + #[cfg(windows)] + ProviderConfig::Keyboard(config) => { + let mut provider = KeyboardProvider::new(config, common); + provider.start_sync(); + } + _ => unreachable!(), + } + info!("Provider stopped: {}", config_hash); }), }; From 741b64ba6989addaa6b22d496a5e62d2bd3c6ae7 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 20:35:13 +0800 Subject: [PATCH 20/23] wip --- .../src/providers/audio/audio_provider.rs | 20 ++++++++++++++++--- .../desktop/src/providers/provider_manager.rs | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 5d72c4a8..037c9c87 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -1,4 +1,7 @@ -use std::collections::HashMap; +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; use anyhow::Context; use crossbeam::channel; @@ -95,7 +98,6 @@ struct DeviceState { device_id: String, device_type: DeviceType, volume: u32, - com_device: IMMDevice, com_volume: IAudioEndpointVolume, com_volume_callback: IAudioEndpointVolumeCallback, } @@ -103,6 +105,8 @@ struct DeviceState { pub struct AudioProvider { common: CommonProviderState, com_enumerator: Option, + last_emit: Instant, + pending_emission: bool, default_playback_id: Option, default_recording_id: Option, device_states: HashMap, @@ -120,6 +124,8 @@ impl AudioProvider { Self { common, com_enumerator: None, + last_emit: Instant::now(), + pending_emission: false, default_playback_id: None, default_recording_id: None, device_states: HashMap::new(), @@ -191,6 +197,13 @@ impl AudioProvider { _ => {} } } + default(Duration::from_millis(20)) => { + // Batch emissions to reduce overhead. + if self.pending_emission { + self.emit_output(); + self.pending_emission = false; + } + } } } @@ -296,6 +309,8 @@ impl AudioProvider { } self.common.emitter.emit_output(Ok(output)); + self.last_emit = Instant::now(); + self.pending_emission = true; } /// Gets the default device ID for the given device type. @@ -358,7 +373,6 @@ impl AudioProvider { device_id: device_id.clone(), device_type: device_type.clone(), volume: (volume * 100.0).round() as u32, - com_device, com_volume, com_volume_callback, }; diff --git a/packages/desktop/src/providers/provider_manager.rs b/packages/desktop/src/providers/provider_manager.rs index 3f547ebd..186a5a4c 100644 --- a/packages/desktop/src/providers/provider_manager.rs +++ b/packages/desktop/src/providers/provider_manager.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, sync::Arc}; -use anyhow::{bail, Context}; +use anyhow::Context; use serde::{ser::SerializeStruct, Serialize}; use tauri::{AppHandle, Emitter}; use tokio::{ From bf0aa619d052c5fccca48da19e8c5d0bcf3cd450 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 21:46:45 +0800 Subject: [PATCH 21/23] batching --- .../src/providers/audio/audio_provider.rs | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/packages/desktop/src/providers/audio/audio_provider.rs b/packages/desktop/src/providers/audio/audio_provider.rs index 037c9c87..ab840199 100644 --- a/packages/desktop/src/providers/audio/audio_provider.rs +++ b/packages/desktop/src/providers/audio/audio_provider.rs @@ -4,7 +4,7 @@ use std::{ }; use anyhow::Context; -use crossbeam::channel; +use crossbeam::channel::{self, at, never}; use serde::{Deserialize, Serialize}; use tracing::{debug, info}; use windows::Win32::{ @@ -105,8 +105,6 @@ struct DeviceState { pub struct AudioProvider { common: CommonProviderState, com_enumerator: Option, - last_emit: Instant, - pending_emission: bool, default_playback_id: Option, default_recording_id: Option, device_states: HashMap, @@ -124,8 +122,6 @@ impl AudioProvider { Self { common, com_enumerator: None, - last_emit: Instant::now(), - pending_emission: false, default_playback_id: None, default_recording_id: None, device_states: HashMap::new(), @@ -170,8 +166,19 @@ impl AudioProvider { // Emit initial output. self.emit_output(); + // Audio events (especially volume changes) can be frequent, so we + // batch the emissions together. + let mut last_emit = Instant::now(); + let mut pending_emission = false; + const BATCH_DELAY: Duration = Duration::from_millis(25); + // Listen to audio-related events. loop { + let batch_timer = match pending_emission { + true => at(last_emit + BATCH_DELAY), + false => never(), + }; + crossbeam::select! { recv(self.event_rx) -> event => { if let Ok(event) = event { @@ -180,6 +187,14 @@ impl AudioProvider { if let Err(err) = self.handle_event(event) { tracing::warn!("Error handling audio event: {}", err); } + + // Check whether we should emit immediately or mark as pending. + if last_emit.elapsed() >= BATCH_DELAY { + self.emit_output(); + last_emit = Instant::now(); + } else { + pending_emission = true; + } } } recv(self.common.input.sync_rx) -> input => { @@ -197,11 +212,11 @@ impl AudioProvider { _ => {} } } - default(Duration::from_millis(20)) => { - // Batch emissions to reduce overhead. - if self.pending_emission { + recv(batch_timer) -> _ => { + if pending_emission { self.emit_output(); - self.pending_emission = false; + last_emit = Instant::now(); + pending_emission = false; } } } @@ -309,8 +324,6 @@ impl AudioProvider { } self.common.emitter.emit_output(Ok(output)); - self.last_emit = Instant::now(); - self.pending_emission = true; } /// Gets the default device ID for the given device type. @@ -425,9 +438,6 @@ impl AudioProvider { } } - // Emit new output after handling the event. - self.emit_output(); - Ok(()) } From 001951b6644fd195f4cd63469fa4da6745f1bee9 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 22:08:48 +0800 Subject: [PATCH 22/23] feat: update types + docs --- README.md | 24 ++++++++++++++----- .../providers/audio/audio-provider-types.ts | 8 +++---- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 779dd651..3201f404 100644 --- a/README.md +++ b/README.md @@ -84,22 +84,34 @@ No config options. | Variable | Description | Return type | Supported OS | | ------------------- | ----------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `defaultPlaybackDevice` | Default audio playback device. | `AudioDevice \| null` | microsoft icon | +| `defaultRecordingDevice` | Default audio recording device. | `AudioDevice \| null` | microsoft icon | | `playbackDevices` | All audio playback devices. | `AudioDevice[]` | microsoft icon | +| `recordingDevices` | All audio recording devices. | `AudioDevice[]` | microsoft icon | +| `allDevices` | All audio devices (both playback and recording). | `AudioDevice[]` | microsoft icon | -#### Return types +| Function | Description | Return type | Supported OS | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `setVolume` | Sets the volume of an audio device. Changes the volume of the default playback device, unless `SetVolumeOptions.deviceId` is specified.

**Parameters:**

- `volume`: _`number`_ Volume as a % of maximum volume. Returned value is between `0` and `100`.
- `options`: _`SetVolumeOptions \| undefined`_ Additional options.
| `Promise` | microsoft icon | + +#### Related types #### `AudioDevice` | Variable | Description | Return type | | ------------------ | ----------------------------- | ----------------------- | -| `id` | Device ID. | `string` | +| `deviceId` | Device ID. | `string` | | `name` | Friendly display name of device. | `string` | -| `volume` | Volume as a % of maximum volume. Returned value is between `0` and `100`. | `number \| null` | -| `roles` | Roles the device is assigned to. | `('multimedia' \| 'communications' \| 'console')[]` | -| `type` | Type of the device. | `'playback' \| 'recording' \| 'hybrid'` | +| `volume` | Volume as a % of maximum volume. Returned value is between `0` and `100`. | `number` | +| `type` | Type of the device. | `'playback' \| 'recording'` | | `isDefaultPlayback` | `true` if the device is selected as the default playback device.| `boolean` | | `isDefaultRecording` | `true` if the device is selected as the default recording device.| `boolean` | +#### `SetVolumeOptions` + +| Variable | Description | Return type | +| ------------------ | ----------------------------- | ----------------------- | +| `deviceId` | Device ID to set the volume of. | `string \| undefined` | + ### Battery #### Config @@ -174,7 +186,7 @@ No config options. | ------------------- | ----------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `disks` | Available disks on the system. | `Disk[]` | microsoft iconapple iconlinux icon | -#### Return types +#### Related types #### `Disk` diff --git a/packages/client-api/src/providers/audio/audio-provider-types.ts b/packages/client-api/src/providers/audio/audio-provider-types.ts index 55e063f3..7ccc8dde 100644 --- a/packages/client-api/src/providers/audio/audio-provider-types.ts +++ b/packages/client-api/src/providers/audio/audio-provider-types.ts @@ -19,14 +19,12 @@ export interface SetVolumeOptions { } export interface AudioDevice { - id: string; + deviceId: string; name: string; - volume: number | null; - roles: AudioDeviceRole[]; + volume: number; type: AudioDeviceType; isDefaultPlayback: boolean; isDefaultRecording: boolean; } -export type AudioDeviceRole = 'multimedia' | 'communications' | 'console'; -export type AudioDeviceType = 'playback' | 'recording' | 'hybrid'; +export type AudioDeviceType = 'playback' | 'recording'; From 6d377c775669bc22195b2152329e2e42cfddd379 Mon Sep 17 00:00:00 2001 From: lars-berger Date: Sat, 30 Nov 2024 22:11:53 +0800 Subject: [PATCH 23/23] feat: add react sample for audio --- .../boilerplate-react-buildless/index.html | 19 ++++++++++++++++++- examples/boilerplate-solid-ts/src/index.tsx | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/examples/boilerplate-react-buildless/index.html b/examples/boilerplate-react-buildless/index.html index f76e9e1a..3854db2a 100644 --- a/examples/boilerplate-react-buildless/index.html +++ b/examples/boilerplate-react-buildless/index.html @@ -37,6 +37,7 @@ memory: { type: 'memory' }, weather: { type: 'weather' }, media: { type: 'media' }, + audio: { type: 'audio' }, }); createRoot(document.getElementById('root')).render(); @@ -50,8 +51,24 @@ return (
+ {output.audio?.defaultPlaybackDevice && ( +
+ {output.audio.defaultPlaybackDevice.name}- + {output.audio.defaultPlaybackDevice.volume} + + output.audio.setVolume(e.target.valueAsNumber) + } + /> +
+ )}
- Media: {output.media?.currentSession?.title} - + Media: {output.media?.currentSession?.title}- {output.media?.currentSession?.artist}
)}
- Media: {output.media?.currentSession?.title} - + Media: {output.media?.currentSession?.title}- {output.media?.currentSession?.artist}