From f8d6465c2ee44da8cd05f1810dae8813a0aa9a88 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Oct 2025 19:46:00 +0800 Subject: [PATCH 1/2] Fixes --- .../LiveKit/Audio/Manager/AudioManager.swift | 34 ++++------- .../AudioCustomProcessingDelegate.swift | 41 ++++++++++++- .../AudioProcessingLifecycle.swift | 61 ++++++++++++++++--- 3 files changed, 103 insertions(+), 33 deletions(-) diff --git a/Sources/LiveKit/Audio/Manager/AudioManager.swift b/Sources/LiveKit/Audio/Manager/AudioManager.swift index 01acd83bc..ee86fb4d5 100644 --- a/Sources/LiveKit/Audio/Manager/AudioManager.swift +++ b/Sources/LiveKit/Audio/Manager/AudioManager.swift @@ -117,9 +117,17 @@ public class AudioManager: Loggable { // MARK: - AudioProcessingModule - private lazy var capturePostProcessingDelegateAdapter = AudioCustomProcessingDelegateAdapter(label: "capturePost") - - private lazy var renderPreProcessingDelegateAdapter = AudioCustomProcessingDelegateAdapter(label: "renderPre") + private lazy var capturePostProcessingDelegateAdapter = AudioCustomProcessingDelegateAdapter( + label: "capturePost", + rtcDelegateGetter: { RTC.audioProcessingModule.capturePostProcessingDelegate }, + rtcDelegateSetter: { RTC.audioProcessingModule.capturePostProcessingDelegate = $0 } + ) + + private lazy var renderPreProcessingDelegateAdapter = AudioCustomProcessingDelegateAdapter( + label: "renderPre", + rtcDelegateGetter: { RTC.audioProcessingModule.renderPreProcessingDelegate }, + rtcDelegateSetter: { RTC.audioProcessingModule.renderPreProcessingDelegate = $0 } + ) let capturePostProcessingDelegateSubject = CurrentValueSubject(nil) @@ -128,15 +136,7 @@ public class AudioManager: Loggable { /// - Note: If you only need to observe the buffer (rather than modify it), use ``add(localAudioRenderer:)`` instead public var capturePostProcessingDelegate: AudioCustomProcessingDelegate? { didSet { - if let capturePostProcessingDelegate { - // Clear WebRTC delegate first - this triggers audioProcessingRelease() on the old target - RTC.audioProcessingModule.capturePostProcessingDelegate = nil - capturePostProcessingDelegateAdapter.set(target: capturePostProcessingDelegate) - RTC.audioProcessingModule.capturePostProcessingDelegate = capturePostProcessingDelegateAdapter - } else { - RTC.audioProcessingModule.capturePostProcessingDelegate = nil - capturePostProcessingDelegateAdapter.set(target: nil) - } + capturePostProcessingDelegateAdapter.set(target: capturePostProcessingDelegate, oldTarget: oldValue) capturePostProcessingDelegateSubject.send(capturePostProcessingDelegate) } } @@ -147,15 +147,7 @@ public class AudioManager: Loggable { /// - Note: If you need to observe the buffer for individual tracks, use ``RemoteAudioTrack/add(audioRenderer:)`` instead public var renderPreProcessingDelegate: AudioCustomProcessingDelegate? { didSet { - if let renderPreProcessingDelegate { - // Clear WebRTC delegate first - this triggers release() on the old target - RTC.audioProcessingModule.renderPreProcessingDelegate = nil - renderPreProcessingDelegateAdapter.set(target: renderPreProcessingDelegate) - RTC.audioProcessingModule.renderPreProcessingDelegate = renderPreProcessingDelegateAdapter - } else { - RTC.audioProcessingModule.renderPreProcessingDelegate = nil - renderPreProcessingDelegateAdapter.set(target: nil) - } + renderPreProcessingDelegateAdapter.set(target: renderPreProcessingDelegate, oldTarget: oldValue) } } diff --git a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift index d76bf467c..14284dfb1 100644 --- a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift +++ b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift @@ -57,16 +57,53 @@ class AudioCustomProcessingDelegateAdapter: MulticastDelegate, @u private var _state = StateSync(State()) - func set(target: AudioCustomProcessingDelegate?) { + private let rtcDelegateGetter: () -> LKRTCAudioCustomProcessingDelegate? + private let rtcDelegateSetter: (LKRTCAudioCustomProcessingDelegate?) -> Void + + func set(target: AudioCustomProcessingDelegate?, oldTarget: AudioCustomProcessingDelegate? = nil) { + // Clear WebRTC delegate first if there's an old target - this triggers audioProcessingRelease() on it + if oldTarget != nil { + rtcDelegateSetter(nil) + } _state.mutate { $0.target = target } + updateRTCConnection() } - init(label: String) { + init(label: String, + rtcDelegateGetter: @escaping () -> LKRTCAudioCustomProcessingDelegate?, + rtcDelegateSetter: @escaping (LKRTCAudioCustomProcessingDelegate?) -> Void) + { self.label = label + self.rtcDelegateGetter = rtcDelegateGetter + self.rtcDelegateSetter = rtcDelegateSetter super.init(label: "AudioCustomProcessingDelegateAdapter.\(label)") log("label: \(label)") } + // Override add/remove to manage RTC connection + override func add(delegate: AudioRenderer) { + super.add(delegate: delegate) + updateRTCConnection() + } + + override func remove(delegate: AudioRenderer) { + super.remove(delegate: delegate) + updateRTCConnection() + } + + private func updateRTCConnection() { + let shouldBeConnected = target != nil || isDelegatesNotEmpty + let isConnected = rtcDelegateGetter() === self + + if shouldBeConnected, !isConnected { + // Connect + rtcDelegateSetter(self) + } else if !shouldBeConnected, isConnected { + // Disconnect + rtcDelegateSetter(nil) + } + } + // MARK: - AudioCustomProcessingDelegate func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) { diff --git a/Tests/LiveKitAudioTests/AudioProcessingLifecycle.swift b/Tests/LiveKitAudioTests/AudioProcessingLifecycle.swift index 23384f588..5c18c46c0 100644 --- a/Tests/LiveKitAudioTests/AudioProcessingLifecycle.swift +++ b/Tests/LiveKitAudioTests/AudioProcessingLifecycle.swift @@ -66,11 +66,7 @@ class AudioProcessingLifecycle: LKTestCase { let room1 = rooms[0] // Publish mic try await room1.localParticipant.setMicrophone(enabled: true) - do { - // 1 secs... - let ns = UInt64(1 * 1_000_000_000) - try await Task.sleep(nanoseconds: ns) - } + await self.sleep(forSeconds: 1) // Verify processorA was initialized and received audio let stateA = processorA._state.copy() @@ -79,11 +75,7 @@ class AudioProcessingLifecycle: LKTestCase { // Switch to processorB AudioManager.shared.capturePostProcessingDelegate = processorB - do { - // 1 secs... - let ns = UInt64(1 * 1_000_000_000) - try await Task.sleep(nanoseconds: ns) - } + await self.sleep(forSeconds: 1) // Verify processorA was released let stateA2 = processorA._state.copy() @@ -102,4 +94,53 @@ class AudioProcessingLifecycle: LKTestCase { let stateB2 = processorB._state.copy() XCTAssertTrue(stateB2.entries.contains(.release), "Processor B should have been released") } + + func testLocalAudioTrackRendererAPI() async throws { + try await withRooms([RoomTestingOptions(canPublish: true)]) { rooms in + let room1 = rooms[0] + + // Create a test renderer + let renderer = TestAudioRenderer() + + // Publish microphone + try await room1.localParticipant.setMicrophone(enabled: true) + + // Get the local audio track + guard let localAudioTrack = room1.localParticipant.audioTracks.first?.track as? LocalAudioTrack else { + XCTFail("No local audio track found") + return + } + + // Add renderer via LocalAudioTrack extension method + localAudioTrack.add(audioRenderer: renderer) + + // Wait for audio to flow + await self.sleep(forSeconds: 1) + + // Verify renderer received audio + let count = renderer.renderCount.copy() + XCTAssertGreaterThan(count, 0, "Renderer should have received audio buffers via LocalAudioTrack.add()") + + // Remove renderer + localAudioTrack.remove(audioRenderer: renderer) + + // Reset count + renderer.renderCount.mutate { $0 = 0 } + + // Wait a bit + await self.sleep(forSeconds: 1) + + // Verify no more audio is received + let countAfterRemove = renderer.renderCount.copy() + XCTAssertEqual(countAfterRemove, 0, "Renderer should not receive audio after removal") + } + } +} + +private class TestAudioRenderer: AudioRenderer, @unchecked Sendable { + let renderCount = StateSync(0) + + func render(pcmBuffer _: AVAudioPCMBuffer) { + renderCount.mutate { $0 += 1 } + } } From dc238b6e9689578ff7a76a0067987a50888e7e35 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Oct 2025 22:11:42 +0800 Subject: [PATCH 2/2] wip 1 --- .../Audio/AudioSessionEngineObserver.swift | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioSessionEngineObserver.swift b/Sources/LiveKit/Audio/AudioSessionEngineObserver.swift index a01ecab59..b8b73f6df 100644 --- a/Sources/LiveKit/Audio/AudioSessionEngineObserver.swift +++ b/Sources/LiveKit/Audio/AudioSessionEngineObserver.swift @@ -64,28 +64,8 @@ public class AudioSessionEngineObserver: AudioEngineObserver, Loggable, @uncheck set { _state.mutate { $0.next = newValue } } } - public init() { - _state.onDidMutate = { new_, old_ in - if new_.isAutomaticConfigurationEnabled, new_.isPlayoutEnabled != old_.isPlayoutEnabled || - new_.isRecordingEnabled != old_.isRecordingEnabled || - new_.isSpeakerOutputPreferred != old_.isSpeakerOutputPreferred - { - // Legacy config func - if let config_func = AudioManager.shared._state.customConfigureFunc { - // Simulate state and invoke custom config func. - let old_state = AudioManager.State(localTracksCount: old_.isRecordingEnabled ? 1 : 0, remoteTracksCount: old_.isPlayoutEnabled ? 1 : 0) - let new_state = AudioManager.State(localTracksCount: new_.isRecordingEnabled ? 1 : 0, remoteTracksCount: new_.isPlayoutEnabled ? 1 : 0) - config_func(new_state, old_state) - } else { - self.configure(oldState: old_, newState: new_) - } - } - } - } - - @Sendable func configure(oldState: State, newState: State) { + @Sendable func tryConfigure(oldState: State, newState: State) throws { let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() defer { session.unlockForConfiguration() @@ -98,6 +78,7 @@ public class AudioSessionEngineObserver: AudioEngineObserver, Loggable, @uncheck try session.setActive(false) } catch { log("AudioSession failed to deactivate with error: \(error)", .error) + throw error } } else if newState.isRecordingEnabled || newState.isPlayoutEnabled { // Configure and activate the session with the appropriate category @@ -109,6 +90,7 @@ public class AudioSessionEngineObserver: AudioEngineObserver, Loggable, @uncheck try session.setConfiguration(config.toRTCType()) } catch { log("AudioSession failed to configure with error: \(error)", .error) + throw error } if !oldState.isPlayoutEnabled, !oldState.isRecordingEnabled { @@ -117,32 +99,63 @@ public class AudioSessionEngineObserver: AudioEngineObserver, Loggable, @uncheck try session.setActive(true) } catch { log("AudioSession failed to activate AudioSession with error: \(error)", .error) + throw error } } } } public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) -> Int { - _state.mutate { + // Copy current state + let oldState = _state.copy() + // Make a new state + let newState = oldState.copy { $0.isPlayoutEnabled = isPlayoutEnabled $0.isRecordingEnabled = isRecordingEnabled } - // Call next last - return _state.next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) ?? 0 + do { + try tryConfigure(oldState: oldState, newState: newState) + // Update state if configure succeeded + _state.mutate { $0 = newState } + // Call next last + return _state.next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) ?? 0 + } catch { + // Failed to configure + return -1 + } } public func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) -> Int { // Call next first - let nextResult = _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + let nextResult = _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) ?? 0 - _state.mutate { + // Copy current state + let oldState = _state.copy() + // Make a new state + let newState = oldState.copy { $0.isPlayoutEnabled = isPlayoutEnabled $0.isRecordingEnabled = isRecordingEnabled } - - return nextResult ?? 0 + do { + try tryConfigure(oldState: oldState, newState: newState) + // Update state if configure succeeded + _state.mutate { $0 = newState } + // Return result + return nextResult + } catch { + // Failed to configure + return -1 + } } } #endif + +extension AudioSessionEngineObserver.State { + func copy(_ block: (inout AudioSessionEngineObserver.State) -> Void) -> AudioSessionEngineObserver.State { + var stateCopy = self + block(&stateCopy) + return stateCopy + } +}