From d68dc4eee8d50dca8ea6a3f41db430946f1b182a Mon Sep 17 00:00:00 2001 From: Siyuan Date: Fri, 14 Nov 2025 15:30:46 +0000 Subject: [PATCH] feat: show audio level metrics in connection stats sidebar --- ui/localization/messages/en.json | 5 + ui/src/components/sidebar/connectionStats.tsx | 257 ++++++++++-------- ui/src/hooks/stores.ts | 20 +- 3 files changed, 166 insertions(+), 116 deletions(-) diff --git a/ui/localization/messages/en.json b/ui/localization/messages/en.json index cdf541c2..f0db09cd 100644 --- a/ui/localization/messages/en.json +++ b/ui/localization/messages/en.json @@ -208,6 +208,8 @@ "connection_stats_connection_description": "The connection between the client and the JetKVM.", "connection_stats_frames_per_second": "Frames per second", "connection_stats_frames_per_second_description": "Number of inbound video frames displayed per second.", + "connection_stats_audio_level": "Audio Level", + "connection_stats_audio_level_description": "The level of the audio stream from the JetKVM to the client.", "connection_stats_network_stability": "Network Stability", "connection_stats_network_stability_description": "How steady the flow of inbound video packets is across the network.", "connection_stats_packets_lost": "Packets Lost", @@ -220,8 +222,11 @@ "connection_stats_unit_frames_per_second": " fps", "connection_stats_unit_milliseconds": " ms", "connection_stats_unit_packets": " packets", + "connection_stats_unit_decibels": " dB", "connection_stats_video": "Video", "connection_stats_video_description": "The video stream from the JetKVM to the client.", + "connection_stats_audio": "Audio", + "connection_stats_audio_description": "The audio stream from the JetKVM to the client.", "continue": "Continue", "creating_peer_connection": "Creating peer connection…", "dc_power_control_current": "Current", diff --git a/ui/src/components/sidebar/connectionStats.tsx b/ui/src/components/sidebar/connectionStats.tsx index e140ad5c..ffeba1e9 100644 --- a/ui/src/components/sidebar/connectionStats.tsx +++ b/ui/src/components/sidebar/connectionStats.tsx @@ -13,62 +13,15 @@ import { Button } from "@components/Button"; import { useCopyToClipboard } from "@components/useCopyToClipBoard"; import notifications from "@/notifications"; -export default function ConnectionStatsSidebar() { - const { sidebarView, setSidebarView } = useUiStore(); - const { - mediaStream, - peerConnection, - inboundRtpStats: inboundVideoRtpStats, - appendInboundRtpStats: appendInboundVideoRtpStats, - candidatePairStats: iceCandidatePairStats, - appendCandidatePairStats, - appendLocalCandidateStats, - appendRemoteCandidateStats, - appendDiskDataChannelStats, - } = useRTCStore(); - - const [remoteIPAddress, setRemoteIPAddress] = useState(null); - - useInterval(function collectWebRTCStats() { - (async () => { - if (!mediaStream) return; - - const videoTrack = mediaStream.getVideoTracks()[0]; - if (!videoTrack) return; - - const stats = await peerConnection?.getStats(); - let successfulLocalCandidateId: string | null = null; - let successfulRemoteCandidateId: string | null = null; - - stats?.forEach(report => { - if (report.type === "inbound-rtp" && report.kind === "video") { - appendInboundVideoRtpStats(report); - } else if (report.type === "candidate-pair" && report.nominated) { - if (report.state === "succeeded") { - successfulLocalCandidateId = report.localCandidateId; - successfulRemoteCandidateId = report.remoteCandidateId; - } - appendCandidatePairStats(report); - } else if (report.type === "local-candidate") { - // We only want to append the local candidate stats that were used in nominated candidate pair - if (successfulLocalCandidateId === report.id) { - appendLocalCandidateStats(report); - } - } else if (report.type === "remote-candidate") { - if (successfulRemoteCandidateId === report.id) { - appendRemoteCandidateStats(report); - setRemoteIPAddress(report.address); - } - } else if (report.type === "data-channel" && report.label === "disk") { - appendDiskDataChannelStats(report); - } - }); - })(); - }, 500); - - const jitterBufferDelay = createChartArray(inboundVideoRtpStats, "jitterBufferDelay"); +interface RtpStatChartsProps { + inboundRtpStats: Map; + showFramesPerSecond?: boolean; + showAudioLevel?: boolean; +} +function RtpStatCharts({ inboundRtpStats, showFramesPerSecond, showAudioLevel }: RtpStatChartsProps) { + const jitterBufferDelay = createChartArray(inboundRtpStats, "jitterBufferDelay"); const jitterBufferEmittedCount = createChartArray( - inboundVideoRtpStats, + inboundRtpStats, "jitterBufferEmittedCount", ); @@ -102,6 +55,137 @@ export default function ConnectionStatsSidebar() { return { date: d.date, metric: valueMs }; }); + return (<> + {/* RTP Jitter */} + ({ + date: x.date, + metric: x.metric != null ? Math.round(x.metric * 1000) : null, + })} + domain={[0, 10]} + unit={m.connection_stats_unit_milliseconds()} + /> + + {/* Playback Delay */} + x.jitterBufferDelay != null, + ) && + someIterable( + inboundRtpStats, + ([, x]) => x.jitterBufferEmittedCount != null, + ) + } + domain={[0, 30]} + unit={m.connection_stats_unit_milliseconds()} + /> + + {/* Packets Lost */} + + + {/* Frames Per Second */} + {showFramesPerSecond && } + + {showAudioLevel && } + ); +} + +export default function ConnectionStatsSidebar() { + const { sidebarView, setSidebarView } = useUiStore(); + const { + mediaStream, + peerConnection, + inboundVideoRtpStats, + appendInboundVideoRtpStats, + inboundAudioRtpStats, + appendInboundAudioRtpStats, + candidatePairStats: iceCandidatePairStats, + appendCandidatePairStats, + appendLocalCandidateStats, + appendRemoteCandidateStats, + appendDiskDataChannelStats, + } = useRTCStore(); + + const [remoteIPAddress, setRemoteIPAddress] = useState(null); + + useInterval(function collectWebRTCStats() { + (async () => { + if (!mediaStream) return; + + const videoTrack = mediaStream.getVideoTracks()[0]; + if (!videoTrack) return; + + const stats = await peerConnection?.getStats(); + let successfulLocalCandidateId: string | null = null; + let successfulRemoteCandidateId: string | null = null; + + stats?.forEach(report => { + if (report.type === "inbound-rtp") { + if (report.kind === "video") { + appendInboundVideoRtpStats(report); + } else if (report.kind === "audio") { + appendInboundAudioRtpStats(report); + } + } else if (report.type === "candidate-pair" && report.nominated) { + if (report.state === "succeeded") { + successfulLocalCandidateId = report.localCandidateId; + successfulRemoteCandidateId = report.remoteCandidateId; + } + appendCandidatePairStats(report); + } else if (report.type === "local-candidate") { + // We only want to append the local candidate stats that were used in nominated candidate pair + if (successfulLocalCandidateId === report.id) { + appendLocalCandidateStats(report); + } + } else if (report.type === "remote-candidate") { + if (successfulRemoteCandidateId === report.id) { + appendRemoteCandidateStats(report); + setRemoteIPAddress(report.address); + } + } else if (report.type === "data-channel" && report.label === "disk") { + appendDiskDataChannelStats(report); + } + }); + })(); + }, 500); + + + const { copy } = useCopyToClipboard(); return ( @@ -159,63 +243,16 @@ export default function ConnectionStatsSidebar() { description={m.connection_stats_video_description()} /> - {/* RTP Jitter */} - ({ - date: x.date, - metric: x.metric != null ? Math.round(x.metric * 1000) : null, - })} - domain={[0, 10]} - unit={m.connection_stats_unit_milliseconds()} - /> + + - {/* Playback Delay */} - x.jitterBufferDelay != null, - ) && - someIterable( - inboundVideoRtpStats, - ([, x]) => x.jitterBufferEmittedCount != null, - ) - } - domain={[0, 30]} - unit={m.connection_stats_unit_milliseconds()} - /> - - {/* Packets Lost */} - - - {/* Frames Per Second */} - + + )} diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 137eacc7..9fb194e5 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -153,8 +153,10 @@ export interface RTCState { isTurnServerInUse: boolean; setTurnServerInUse: (inUse: boolean) => void; - inboundRtpStats: Map; - appendInboundRtpStats: (stats: RTCInboundRtpStreamStats) => void; + inboundVideoRtpStats: Map; + appendInboundVideoRtpStats: (stats: RTCInboundRtpStreamStats) => void; + inboundAudioRtpStats: Map; + appendInboundAudioRtpStats: (stats: RTCInboundRtpStreamStats) => void; clearInboundRtpStats: () => void; candidatePairStats: Map; @@ -218,13 +220,19 @@ export const useRTCStore = create(set => ({ isTurnServerInUse: false, setTurnServerInUse: inUse => set({ isTurnServerInUse: inUse }), - inboundRtpStats: new Map(), - appendInboundRtpStats: stats => { + inboundVideoRtpStats: new Map(), + appendInboundVideoRtpStats: stats => { set(prevState => ({ - inboundRtpStats: appendStatToMap(stats, prevState.inboundRtpStats), + inboundVideoRtpStats: appendStatToMap(stats, prevState.inboundVideoRtpStats), })); }, - clearInboundRtpStats: () => set({ inboundRtpStats: new Map() }), + inboundAudioRtpStats: new Map(), + appendInboundAudioRtpStats: stats => { + set(prevState => ({ + inboundAudioRtpStats: appendStatToMap(stats, prevState.inboundAudioRtpStats), + })); + }, + clearInboundRtpStats: () => set({ inboundVideoRtpStats: new Map(), inboundAudioRtpStats: new Map() }), candidatePairStats: new Map(), appendCandidatePairStats: stats => {