From 89e68f5cdb60ef3bbb8fa91065d4923155b723d3 Mon Sep 17 00:00:00 2001 From: Alex P Date: Mon, 8 Sep 2025 22:55:19 +0000 Subject: [PATCH] [WIP] Change playback latency spikes on Audio Output Quality changes --- audio_handlers.go | 7 ++ internal/audio/cgo_audio.go | 6 +- internal/audio/quality_presets.go | 94 ++++++++++---------------- ui/src/routes/devices.$id.tsx | 6 ++ ui/src/services/audioQualityService.ts | 27 +++++++- 5 files changed, 79 insertions(+), 61 deletions(-) diff --git a/audio_handlers.go b/audio_handlers.go index 36ba348b..7c29bc96 100644 --- a/audio_handlers.go +++ b/audio_handlers.go @@ -284,6 +284,13 @@ func handleSetAudioQuality(c *gin.Context) { return } + // Check if audio output is active before attempting quality change + // This prevents race conditions where quality changes are attempted before initialization + if !IsAudioOutputActive() { + c.JSON(503, gin.H{"error": "audio output not active - please wait for initialization to complete"}) + return + } + // Convert int to AudioQuality type quality := audio.AudioQuality(req.Quality) diff --git a/internal/audio/cgo_audio.go b/internal/audio/cgo_audio.go index 7ce55bd0..756b8e6e 100644 --- a/internal/audio/cgo_audio.go +++ b/internal/audio/cgo_audio.go @@ -87,8 +87,10 @@ static volatile int playback_initialized = 0; // Function to dynamically update Opus encoder parameters int update_opus_encoder_params(int bitrate, int complexity, int vbr, int vbr_constraint, int signal_type, int bandwidth, int dtx) { - if (!encoder || !capture_initialized) { - return -1; // Encoder not initialized + // This function is specifically for audio OUTPUT encoder parameters + // Only require playback initialization for audio output quality changes + if (!encoder || !playback_initialized) { + return -1; // Audio output encoder not initialized } // Update the static variables diff --git a/internal/audio/quality_presets.go b/internal/audio/quality_presets.go index fc4512b2..1888f872 100644 --- a/internal/audio/quality_presets.go +++ b/internal/audio/quality_presets.go @@ -204,69 +204,49 @@ func SetAudioQuality(quality AudioQuality) { dtx = Config.AudioQualityMediumOpusDTX } - // Restart audio output subprocess with new OPUS configuration + // Update audio output subprocess configuration dynamically without restart + logger := logging.GetDefaultLogger().With().Str("component", "audio").Logger() + logger.Info().Int("quality", int(quality)).Msg("updating audio output quality settings dynamically") + + // Immediately boost adaptive buffer sizes to handle quality change frame burst + // This prevents "Message channel full, dropping frame" warnings during transitions + adaptiveManager := GetAdaptiveBufferManager() + if adaptiveManager != nil { + // Immediately set buffers to maximum size for quality change + adaptiveManager.BoostBuffersForQualityChange() + logger.Debug().Msg("boosted adaptive buffers for quality change") + } + + // Set new OPUS configuration for future restarts if supervisor := GetAudioOutputSupervisor(); supervisor != nil { - logger := logging.GetDefaultLogger().With().Str("component", "audio").Logger() - logger.Info().Int("quality", int(quality)).Msg("restarting audio output subprocess with new quality settings") - - // Immediately boost adaptive buffer sizes to handle quality change frame burst - // This prevents "Message channel full, dropping frame" warnings during transitions - adaptiveManager := GetAdaptiveBufferManager() - if adaptiveManager != nil { - // Immediately set buffers to maximum size for quality change - adaptiveManager.BoostBuffersForQualityChange() - logger.Debug().Msg("boosted adaptive buffers for quality change") - } - - // Set new OPUS configuration supervisor.SetOpusConfig(config.Bitrate*1000, complexity, vbr, signalType, bandwidth, dtx) + } - // Stop current subprocess - supervisor.Stop() - - // Wait for supervisor to fully stop before starting again with timeout - // This prevents race conditions and audio breakage - stopTimeout := time.After(Config.QualityChangeSupervisorTimeout) - ticker := time.NewTicker(Config.QualityChangeTickerInterval) - defer ticker.Stop() - - for { - select { - case <-stopTimeout: - logger.Warn().Msg("supervisor did not stop within 5s timeout, proceeding anyway") - goto startSupervisor - case <-ticker.C: - if !supervisor.IsRunning() { - goto startSupervisor - } + // Send dynamic configuration update to running audio output + vbrConstraint := Config.CGOOpusVBRConstraint + if err := updateOpusEncoderParams(config.Bitrate*1000, complexity, vbr, vbrConstraint, signalType, bandwidth, dtx); err != nil { + logger.Warn().Err(err).Msg("failed to update OPUS encoder parameters dynamically") + // Fallback to subprocess restart if dynamic update fails + if supervisor := GetAudioOutputSupervisor(); supervisor != nil { + logger.Info().Msg("falling back to subprocess restart") + supervisor.Stop() + if err := supervisor.Start(); err != nil { + logger.Error().Err(err).Msg("failed to restart audio output subprocess after dynamic update failure") } } - - startSupervisor: - - // Start subprocess with new configuration - if err := supervisor.Start(); err != nil { - logger.Error().Err(err).Msg("failed to restart audio output subprocess") - } else { - logger.Info().Int("quality", int(quality)).Msg("audio output subprocess restarted successfully with new quality") - - // Reset audio input server stats after quality change - // Allow adaptive buffer manager to naturally adjust buffer sizes - go func() { - time.Sleep(Config.QualityChangeSettleDelay) // Wait for quality change to settle - // Reset audio input server stats to clear persistent warnings - ResetGlobalAudioInputServerStats() - // Attempt recovery if microphone is still having issues - time.Sleep(1 * time.Second) - RecoverGlobalAudioInputServer() - }() - } } else { - // Fallback to dynamic update if supervisor is not available - vbrConstraint := Config.CGOOpusVBRConstraint - if err := updateOpusEncoderParams(config.Bitrate*1000, complexity, vbr, vbrConstraint, signalType, bandwidth, dtx); err != nil { - logging.GetDefaultLogger().Error().Err(err).Msg("Failed to update OPUS encoder parameters") - } + logger.Info().Msg("audio output quality updated dynamically") + + // Reset audio output stats after config update + // Allow adaptive buffer manager to naturally adjust buffer sizes + go func() { + time.Sleep(Config.QualityChangeSettleDelay) // Wait for quality change to settle + // Reset audio input server stats to clear persistent warnings + ResetGlobalAudioInputServerStats() + // Attempt recovery if there are still issues + time.Sleep(1 * time.Second) + RecoverGlobalAudioInputServer() + }() } } } diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index af1e5e84..3eeb6d9d 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -54,6 +54,7 @@ import { useDeviceUiNavigation } from "@/hooks/useAppNavigation"; import { FeatureFlagProvider } from "@/providers/FeatureFlagProvider"; import { DeviceStatus } from "@routes/welcome-local"; import { SystemVersionInfo } from "@routes/devices.$id.settings.general.update"; +import audioQualityService from "@/services/audioQualityService"; interface LocalLoaderResp { authMode: "password" | "noPassword" | null; @@ -533,6 +534,11 @@ export default function KvmIdRoute() { }; }, [clearCandidatePairStats, clearInboundRtpStats, setPeerConnection, setSidebarView]); + // Register callback with audioQualityService + useEffect(() => { + audioQualityService.setReconnectionCallback(setupPeerConnection); + }, [setupPeerConnection]); + // TURN server usage detection useEffect(() => { if (peerConnectionState !== "connected") return; diff --git a/ui/src/services/audioQualityService.ts b/ui/src/services/audioQualityService.ts index 94cd4907..c722a456 100644 --- a/ui/src/services/audioQualityService.ts +++ b/ui/src/services/audioQualityService.ts @@ -24,6 +24,7 @@ class AudioQualityService { 2: 'High', 3: 'Ultra' }; + private reconnectionCallback: (() => Promise) | null = null; /** * Fetch audio quality presets from the backend @@ -96,12 +97,34 @@ class AudioQualityService { } /** - * Set audio quality + * Set reconnection callback for WebRTC reset + */ + setReconnectionCallback(callback: () => Promise): void { + this.reconnectionCallback = callback; + } + + /** + * Trigger audio track replacement using backend's track replacement mechanism + */ + private async replaceAudioTrack(): Promise { + if (this.reconnectionCallback) { + await this.reconnectionCallback(); + } + } + + /** + * Set audio quality with track replacement */ async setAudioQuality(quality: number): Promise { try { const response = await api.POST('/audio/quality', { quality }); - return response.ok; + + if (!response.ok) { + return false; + } + + await this.replaceAudioTrack(); + return true; } catch (error) { console.error('Failed to set audio quality:', error); return false;