From 7ae0132da973c85aeb0ec2e55fb452de308360a0 Mon Sep 17 00:00:00 2001 From: Brooooooklyn Date: Thu, 27 Mar 2025 12:55:09 +0000 Subject: [PATCH] fix(native): audio capture volume (#11240) --- .../media-capture-playground/server/main.ts | 11 ++-- .../media_capture/src/macos/audio_buffer.rs | 3 +- .../media_capture/src/macos/cf_types.rs | 12 ++++ .../src/macos/screen_capture_kit.rs | 11 ++-- .../media_capture/src/macos/tap_audio.rs | 56 +++++++++++-------- 5 files changed, 57 insertions(+), 36 deletions(-) diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.ts index 39d09ed62..2acc760f7 100644 --- a/packages/frontend/media-capture-playground/server/main.ts +++ b/packages/frontend/media-capture-playground/server/main.ts @@ -524,10 +524,13 @@ async function getAllApps(): Promise { } }); - const filteredApps = apps.filter( - (v): v is AppInfo => - v !== null && !v.bundleIdentifier.startsWith('com.apple') - ); + const filteredApps = apps.filter((v): v is AppInfo => { + return ( + v !== null && + (!v.bundleIdentifier.startsWith('com.apple') || + v.bundleIdentifier === 'com.apple.Music') + ); + }); for (const app of filteredApps) { if (filteredApps.some(a => a.processId === app.processGroupId)) { diff --git a/packages/frontend/native/media_capture/src/macos/audio_buffer.rs b/packages/frontend/native/media_capture/src/macos/audio_buffer.rs index 8b99d4ee2..e17117cb5 100644 --- a/packages/frontend/native/media_capture/src/macos/audio_buffer.rs +++ b/packages/frontend/native/media_capture/src/macos/audio_buffer.rs @@ -98,12 +98,11 @@ impl InputAndOutputAudioBufferList { .max(processed_samples_output.len()); let mut mixed_samples = vec![0.0; mixed_samples_length]; - for (sample_index, mixed_sample) in mixed_samples.iter_mut().enumerate() { let sample_in = processed_samples_input.get(sample_index).unwrap_or(&0.0); let sample_out = processed_samples_output.get(sample_index).unwrap_or(&0.0); - *mixed_sample = (sample_in + sample_out) / 2.0; + *mixed_sample = sample_in + sample_out; } Ok(mixed_samples) diff --git a/packages/frontend/native/media_capture/src/macos/cf_types.rs b/packages/frontend/native/media_capture/src/macos/cf_types.rs index 4dbc44ae4..f91da86da 100644 --- a/packages/frontend/native/media_capture/src/macos/cf_types.rs +++ b/packages/frontend/native/media_capture/src/macos/cf_types.rs @@ -18,12 +18,24 @@ impl ToCoreFoundation for CFString { } } +impl ToCoreFoundation for &CFString { + fn to_cf(&self) -> CFType { + self.as_CFType() + } +} + impl ToCoreFoundation for String { fn to_cf(&self) -> CFType { CFString::new(self).as_CFType() } } +impl ToCoreFoundation for &str { + fn to_cf(&self) -> CFType { + CFString::new(self).as_CFType() + } +} + impl ToCoreFoundation for bool { fn to_cf(&self) -> CFType { if *self { diff --git a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs index b26c5d554..7f6de9633 100644 --- a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs +++ b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs @@ -16,9 +16,8 @@ use core_foundation::{ use coreaudio::sys::{ kAudioHardwarePropertyProcessObjectList, kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal, kAudioObjectSystemObject, kAudioProcessPropertyBundleID, - kAudioProcessPropertyIsRunning, kAudioProcessPropertyIsRunningInput, kAudioProcessPropertyPID, - AudioObjectAddPropertyListenerBlock, AudioObjectID, AudioObjectPropertyAddress, - AudioObjectRemovePropertyListenerBlock, + kAudioProcessPropertyIsRunning, kAudioProcessPropertyPID, AudioObjectAddPropertyListenerBlock, + AudioObjectID, AudioObjectPropertyAddress, AudioObjectRemovePropertyListenerBlock, }; use libc; use napi::{ @@ -204,8 +203,8 @@ impl Application { } #[napi(getter)] - pub fn name(&self) -> String { - self.name.clone() + pub fn name(&self) -> &str { + &self.name } #[napi(getter)] @@ -423,7 +422,7 @@ impl TappableApplication { pub fn get_is_running(&self) -> Result { // Use catch_unwind to prevent any panics let result = std::panic::catch_unwind(|| { - match get_process_property(&self.object_id, kAudioProcessPropertyIsRunningInput) { + match get_process_property(&self.object_id, kAudioProcessPropertyIsRunning) { Ok(is_running) => Ok(is_running), Err(_) => Ok(false), } diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs index ff677ca70..63ac22780 100644 --- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs +++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs @@ -14,7 +14,7 @@ use coreaudio::sys::{ kAudioAggregateDeviceTapAutoStartKey, kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioDevicePropertyNominalSampleRate, kAudioHardwareBadDeviceError, kAudioHardwareBadStreamError, kAudioHardwareNoError, kAudioHardwarePropertyDefaultInputDevice, - kAudioHardwarePropertyDefaultSystemOutputDevice, kAudioSubDeviceUIDKey, kAudioSubTapUIDKey, + kAudioHardwarePropertyDefaultOutputDevice, kAudioSubDeviceUIDKey, kAudioSubTapUIDKey, AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID, AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice, AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus, @@ -31,7 +31,7 @@ use crate::{ audio_buffer::InputAndOutputAudioBufferList, ca_tap_description::CATapDescription, cf_types::CFDictionaryBuilder, - device::{get_device_audio_id, get_device_uid}, + device::get_device_uid, error::CoreAudioError, queue::create_audio_tap_queue, screen_capture_kit::TappableApplication, @@ -59,7 +59,7 @@ pub struct AggregateDevice { pub id: AudioObjectID, pub audio_stats: Option, pub input_device_id: AudioObjectID, - pub output_device_id: Option, + pub output_device_id: AudioObjectID, pub input_proc_id: Option, pub output_proc_id: Option, } @@ -80,8 +80,15 @@ impl AggregateDevice { let (input_device_id, default_input_uid) = get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?; - let description_dict = - Self::create_aggregate_description(tap_id, tap_description.get_uuid()?, default_input_uid)?; + // Get the default output device ID + let (output_device_id, output_device_uid) = + get_device_uid(kAudioHardwarePropertyDefaultOutputDevice)?; + let description_dict = Self::create_aggregate_description( + tap_id, + tap_description.get_uuid()?, + default_input_uid, + output_device_uid, + )?; let mut aggregate_device_id: AudioObjectID = 0; @@ -101,7 +108,7 @@ impl AggregateDevice { id: aggregate_device_id, audio_stats: None, input_device_id, - output_device_id: None, + output_device_id, input_proc_id: None, output_proc_id: None, }) @@ -122,10 +129,15 @@ impl AggregateDevice { get_device_uid(kAudioHardwarePropertyDefaultInputDevice)?; // Get the default output device ID - let output_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultSystemOutputDevice)?; + let (output_device_id, output_device_uid) = + get_device_uid(kAudioHardwarePropertyDefaultOutputDevice)?; - let description_dict = - Self::create_aggregate_description(tap_id, tap_description.get_uuid()?, default_input_uid)?; + let description_dict = Self::create_aggregate_description( + tap_id, + tap_description.get_uuid()?, + default_input_uid, + output_device_uid, + )?; let mut aggregate_device_id: AudioObjectID = 0; @@ -147,7 +159,7 @@ impl AggregateDevice { id: aggregate_device_id, audio_stats: None, input_device_id, - output_device_id: Some(output_device_id), + output_device_id, input_proc_id: None, output_proc_id: None, }; @@ -317,24 +329,21 @@ impl AggregateDevice { tap_id: AudioObjectID, tap_uuid_string: ItemRef, input_device_id: CFString, + output_device_id: CFString, ) -> Result> { let aggregate_device_name = CFString::new(&format!("Tap-{}", tap_id)); let aggregate_device_uid: uuid::Uuid = CFUUID::new().into(); let aggregate_device_uid_string = aggregate_device_uid.to_string(); - let (_, output_device_uid) = get_device_uid(kAudioHardwarePropertyDefaultSystemOutputDevice)?; - - let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[( - cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(), - input_device_id.as_CFType(), - )]); + let mut sub_device_input_dict = CFDictionaryBuilder::new(); + sub_device_input_dict.add(kAudioSubDeviceUIDKey.as_slice(), &input_device_id); let tap_device_dict = CFDictionary::from_CFType_pairs(&[( cfstring_from_bytes_with_nul(kAudioSubTapUIDKey).as_CFType(), tap_uuid_string.as_CFType(), )]); - let capture_device_list = vec![sub_device_input_dict]; + let capture_device_list = vec![sub_device_input_dict.build()]; // Create the aggregate device description dictionary with a balanced // configuration @@ -352,9 +361,10 @@ impl AggregateDevice { ) .add( kAudioAggregateDeviceMainSubDeviceKey.as_slice(), - output_device_uid, + &output_device_id, ) .add(kAudioAggregateDeviceIsPrivateKey.as_slice(), true) + // can't be stacked because we're using a tap .add(kAudioAggregateDeviceIsStackedKey.as_slice(), false) .add(kAudioAggregateDeviceTapAutoStartKey.as_slice(), true) .add( @@ -381,7 +391,7 @@ pub struct AudioTapStream { stop_called: bool, audio_stats: AudioStats, input_device_id: AudioObjectID, - output_device_id: Option, + output_device_id: AudioObjectID, input_proc_id: Option, output_proc_id: Option, } @@ -408,11 +418,9 @@ impl AudioTapStream { } // Stop the output device if it was activated - if let Some(output_id) = self.output_device_id { - if let Some(proc_id) = self.output_proc_id { - let _ = unsafe { AudioDeviceStop(output_id, proc_id) }; - let _ = unsafe { AudioDeviceDestroyIOProcID(output_id, proc_id) }; - } + if let Some(proc_id) = self.output_proc_id { + let _ = unsafe { AudioDeviceStop(self.output_device_id, proc_id) }; + let _ = unsafe { AudioDeviceDestroyIOProcID(self.output_device_id, proc_id) }; } // Destroy the main IO proc