Use a failover pattern for finding device channel layout

VoiceProcessingIO cannot return the channel layout for their output
device. With this patch we try anyway, and if we are using vpio, we
failover to creating a dedicated AudioUnit for the output device and
querying that. If that fails, we failover to querying for the preferred
layout on the dedicated AudioUnit, if there is one, then for the
preferred layout on the main unit, in order to retain the old behavior.

src/backend/mod.rs

@@ -817,14 +817,14 @@ fn get_default_device_id(devtype: DeviceType) -> std::result::Result<AudioObject
     }
 }
 
-fn audiounit_convert_channel_layout(layout: &AudioChannelLayout) -> Vec<mixer::Channel> {
+fn audiounit_convert_channel_layout(layout: &AudioChannelLayout) -> Result<Vec<mixer::Channel>> {
     if layout.mChannelLayoutTag != kAudioChannelLayoutTag_UseChannelDescriptions {
         // kAudioChannelLayoutTag_UseChannelBitmap
         // kAudioChannelLayoutTag_Mono
         // kAudioChannelLayoutTag_Stereo
         // ....
         cubeb_log!("Only handling UseChannelDescriptions for now.\n");
-        return Vec::new();
+        return Err(Error::error());
     }
 
     let channel_descriptions = unsafe {
@@ -840,10 +840,10 @@ fn audiounit_convert_channel_layout(layout: &AudioChannelLayout) -> Vec<mixer::C
         channels.push(label.into());
     }
 
-    channels
+    Ok(channels)
 }
 
-fn audiounit_get_preferred_channel_layout(output_unit: AudioUnit) -> Vec<mixer::Channel> {
+fn audiounit_get_preferred_channel_layout(output_unit: AudioUnit) -> Result<Vec<mixer::Channel>> {
     let mut rv = NO_ERR;
     let mut size: usize = 0;
     rv = audio_unit_get_property_info(
@@ -859,7 +859,7 @@ fn audiounit_get_preferred_channel_layout(output_unit: AudioUnit) -> Vec<mixer::
             "AudioUnitGetPropertyInfo/kAudioDevicePropertyPreferredChannelLayout rv={}",
             rv
         );
-        return Vec::new();
+        return Err(Error::error());
     }
     debug_assert!(size > 0);
 
@@ -877,15 +877,15 @@ fn audiounit_get_preferred_channel_layout(output_unit: AudioUnit) -> Vec<mixer::
             "AudioUnitGetProperty/kAudioDevicePropertyPreferredChannelLayout rv={}",
             rv
         );
-        return Vec::new();
+        return Err(Error::error());
     }
 
     audiounit_convert_channel_layout(layout.as_ref())
 }
 
 // This is for output AudioUnit only. Calling this by input-only AudioUnit is prone
 // to crash intermittently.
-fn audiounit_get_current_channel_layout(output_unit: AudioUnit) -> Vec<mixer::Channel> {
+fn audiounit_get_current_channel_layout(output_unit: AudioUnit) -> Result<Vec<mixer::Channel>> {
     let mut rv = NO_ERR;
     let mut size: usize = 0;
     rv = audio_unit_get_property_info(
@@ -901,8 +901,7 @@ fn audiounit_get_current_channel_layout(output_unit: AudioUnit) -> Vec<mixer::Ch
             "AudioUnitGetPropertyInfo/kAudioUnitProperty_AudioChannelLayout rv={}",
             rv
         );
-        // This property isn't known before macOS 10.12, attempt another method.
-        return audiounit_get_preferred_channel_layout(output_unit);
+        return Err(Error::error());
     }
     debug_assert!(size > 0);
 
@@ -920,7 +919,7 @@ fn audiounit_get_current_channel_layout(output_unit: AudioUnit) -> Vec<mixer::Ch
             "AudioUnitGetProperty/kAudioUnitProperty_AudioChannelLayout rv={}",
             rv
         );
-        return Vec::new();
+        return Err(Error::error());
     }
 
     audiounit_convert_channel_layout(layout.as_ref())
@@ -2860,20 +2859,57 @@ impl<'ctx> CoreStreamData<'ctx> {
                 e
             })?;
 
-            // XXX
-            let device_layout = if using_voice_processing_unit {
-                let mut channels = Vec::with_capacity(output_hw_desc.mChannelsPerFrame as usize);
-                match output_hw_desc.mChannelsPerFrame {
-                    1 => channels.push(mixer::Channel::FrontCenter),
-                    _ => {
-                        channels.push(mixer::Channel::FrontLeft);
-                        channels.push(mixer::Channel::FrontRight);
+            let device_layout = audiounit_get_current_channel_layout(self.unit)
+                .or_else(|_| {
+                    // The VoiceProcessingIO unit is known to not support
+                    // kAudioUnitProperty_AudioChannelLayout queries. If we're using
+                    // VoiceProcessingIO, try standing up a regular AudioUnit and query that.
+                    if !using_voice_processing_unit {
+                        return Err(Error::error());
                     }
-                }
-                channels
-            } else {
-                audiounit_get_current_channel_layout(self.unit)
-            };
+                    cubeb_log!(
+                        "({:p}) Could not get current channel layout for vpio unit. \
+                         Retrying with a dedicated unit.",
+                        self.stm_ptr
+                    );
+                    let mut blank_in_dev_info = self.input_device.clone();
+                    blank_in_dev_info.flags = device_flags::DEV_UNKNOWN;
+                    let mut dedicated_unit = create_audiounit(&blank_in_dev_info, &self.output_device)?;
+                    let res = audiounit_get_current_channel_layout(dedicated_unit).or_else(|_| {
+                        cubeb_log!(
+                            "({:p}) Could not get current channel layout for dedicated unit. \
+                             Trying to get preferred channel layout with the dedicated unit.",
+                             self.stm_ptr
+                        );
+                        audiounit_get_preferred_channel_layout(dedicated_unit)
+                    });
+                    audio_unit_uninitialize(dedicated_unit);
+                    dispose_audio_unit(dedicated_unit);
+                    dedicated_unit = ptr::null_mut();
+                    res
+                })
+                .or_else(|_| {
+                    // The kAudioUnitProperty_AudioChannelLayout property isn't known before
+                    // macOS 10.12, attempt another method.
+                    cubeb_log!(
+                        "({:p}) Could not get current channel layout. Trying preferred channel layout.",
+                        self.stm_ptr
+                    );
+                    audiounit_get_preferred_channel_layout(self.unit)
+                })
+                .unwrap_or_else(|_| {
+                    cubeb_log!(
+                        "({:p}) Could not get any channel layout. Defaulting to no channels.",
+                        self.stm_ptr
+                    );
+                    Vec::new()
+                });
+
+            cubeb_log!(
+                "({:p} Using output device channel layout {:?}",
+                self.stm_ptr,
+                device_layout
+            );
 
             // The mixer will be set up when
             // 1. using aggregate device whose input device has output channels

src/backend/tests/api.rs

@@ -700,7 +700,7 @@ fn test_convert_channel_layout() {
         }
         let layout_ref = unsafe { &(*(&layout as *const TestLayout as *const AudioChannelLayout)) };
         assert_eq!(
-            &audiounit_convert_channel_layout(layout_ref),
+            &audiounit_convert_channel_layout(layout_ref).unwrap(),
             expected_layout
         );
     }
@@ -711,7 +711,9 @@ fn test_convert_channel_layout() {
 #[test]
 fn test_get_preferred_channel_layout_output() {
     match test_get_default_audiounit(Scope::Output) {
-        Some(unit) => assert!(!audiounit_get_preferred_channel_layout(unit.get_inner()).is_empty()),
+        Some(unit) => assert!(!audiounit_get_preferred_channel_layout(unit.get_inner())
+            .unwrap()
+            .is_empty()),
         None => println!("No output audiounit for test."),
     }
 }
@@ -721,7 +723,9 @@ fn test_get_preferred_channel_layout_output() {
 #[test]
 fn test_get_current_channel_layout_output() {
     match test_get_default_audiounit(Scope::Output) {
-        Some(unit) => assert!(!audiounit_get_current_channel_layout(unit.get_inner()).is_empty()),
+        Some(unit) => assert!(!audiounit_get_current_channel_layout(unit.get_inner())
+            .unwrap()
+            .is_empty()),
         None => println!("No output audiounit for test."),
     }
 }