[WIP] Improvements, Bugfixes: Improve audio experience when running in HTTP mode

jsonrpc.go

@@ -908,38 +908,7 @@ func updateUsbRelatedConfig() error {
 	return nil
 }
 
-// validateAudioConfiguration checks if audio functionality can be enabled
-func validateAudioConfiguration(enabled bool) error {
-	if !enabled {
-		return nil // Disabling audio is always allowed
-	}
-
-	// Check if audio supervisor is available
-	if audioSupervisor == nil {
-		return fmt.Errorf("audio supervisor not initialized - audio functionality not available")
-	}
-
-	// Check if ALSA devices are available by attempting to list them
-	// This is a basic check to ensure the system has audio capabilities
-	if _, err := os.Stat("/proc/asound/cards"); os.IsNotExist(err) {
-		return fmt.Errorf("no ALSA sound cards detected - audio hardware not available")
-	}
-
-	// Check if USB gadget audio function is supported
-	if _, err := os.Stat("/sys/kernel/config/usb_gadget"); os.IsNotExist(err) {
-		return fmt.Errorf("USB gadget configfs not available - cannot enable USB audio")
-	}
-
-	return nil
-}
-
 func rpcSetUsbDevices(usbDevices usbgadget.Devices) error {
-	// Validate audio configuration before proceeding
-	if err := validateAudioConfiguration(usbDevices.Audio); err != nil {
-		logger.Warn().Err(err).Msg("audio configuration validation failed")
-		return fmt.Errorf("audio validation failed: %w", err)
-	}
-
 	// Check if audio state is changing
 	previousAudioEnabled := config.UsbDevices != nil && config.UsbDevices.Audio
 	newAudioEnabled := usbDevices.Audio
@@ -1034,11 +1003,6 @@ func rpcSetUsbDeviceState(device string, enabled bool) error {
 	case "massStorage":
 		config.UsbDevices.MassStorage = enabled
 	case "audio":
-		// Validate audio configuration before proceeding
-		if err := validateAudioConfiguration(enabled); err != nil {
-			logger.Warn().Err(err).Msg("audio device state validation failed")
-			return fmt.Errorf("audio validation failed: %w", err)
-		}
 		// Handle audio process management
 		if !enabled {
 			// Stop audio processes when audio is disabled

ui/src/components/ActionBar.tsx

@@ -43,6 +43,8 @@ interface MicrophoneHookReturn {
   isStarting: boolean;
   isStopping: boolean;
   isToggling: boolean;
+  // HTTP/HTTPS detection
+  isHttpsRequired: boolean;
 }
 
 export default function Actionbar({
@@ -86,8 +88,9 @@ export default function Actionbar({
   const isMuted = audioMuted ?? false; // Default to false if WebSocket data not available yet
   
   // Get USB device configuration to check if audio is enabled
-  const { usbDeviceConfig } = useUsbDeviceConfig();
-  const isAudioEnabledInUsb = usbDeviceConfig?.audio ?? true; // Default to true while loading
+  const { usbDeviceConfig, loading: usbConfigLoading } = useUsbDeviceConfig();
+  // Default to false while loading to prevent premature access when audio hasn't been enabled yet
+  const isAudioEnabledInUsb = usbDeviceConfig?.audio ?? false;
 
   return (
     <Container className="border-b border-b-slate-800/20 bg-white dark:border-b-slate-300/20 dark:bg-slate-900">
@@ -320,27 +323,39 @@ export default function Actionbar({
             />
           </div>
           <Popover>
-            <PopoverButton as={Fragment} disabled={!isAudioEnabledInUsb}>
-              <div title={!isAudioEnabledInUsb ? "Audio needs to be enabled in USB device settings" : undefined}>
+            <PopoverButton as={Fragment} disabled={!isAudioEnabledInUsb || usbConfigLoading}>
+              <div title={
+                usbConfigLoading 
+                  ? "Loading audio configuration..." 
+                  : !isAudioEnabledInUsb 
+                    ? "Audio needs to be enabled in USB device settings" 
+                    : undefined
+              }>
                 <Button
                   size="XS"
                   theme="light"
                   text="Audio"
-                  disabled={!isAudioEnabledInUsb}
+                  disabled={!isAudioEnabledInUsb || usbConfigLoading}
                   LeadingIcon={({ className }) => (
                     <div className="flex items-center">
-                      {!isAudioEnabledInUsb ? (
+                      {usbConfigLoading ? (
+                        <div className={cx(className, "animate-spin rounded-full border border-gray-400 border-t-gray-600")} />
+                      ) : !isAudioEnabledInUsb ? (
                         <MdVolumeOff className={cx(className, "text-gray-400")} />
                       ) : isMuted ? (
                         <MdVolumeOff className={cx(className, "text-red-500")} />
                       ) : (
                         <MdVolumeUp className={cx(className, "text-green-500")} />
                       )}
-                      <MdGraphicEq className={cx(className, "ml-1", !isAudioEnabledInUsb ? "text-gray-400" : "text-blue-500")} />
+                      <MdGraphicEq className={cx(className, "ml-1", 
+                        usbConfigLoading ? "text-gray-400" : 
+                        !isAudioEnabledInUsb ? "text-gray-400" : 
+                        "text-blue-500"
+                      )} />
                     </div>
                   )}
                   onClick={() => {
-                    if (isAudioEnabledInUsb) {
+                    if (isAudioEnabledInUsb && !usbConfigLoading) {
                       setDisableVideoFocusTrap(true);
                     }
                   }}

ui/src/components/WebRTCVideo.tsx

@@ -43,6 +43,8 @@ interface MicrophoneHookReturn {
   isStarting: boolean;
   isStopping: boolean;
   isToggling: boolean;
+  // HTTP/HTTPS detection
+  isHttpsRequired: boolean;
 }
 
 interface WebRTCVideoProps {

ui/src/components/popovers/AudioControlPopover.tsx

@@ -29,6 +29,8 @@ interface MicrophoneHookReturn {
   isStarting: boolean;
   isStopping: boolean;
   isToggling: boolean;
+  // HTTP/HTTPS detection
+  isHttpsRequired: boolean;
 }
 
 interface AudioConfig {
@@ -85,6 +87,8 @@ export default function AudioControlPopover({ microphone }: AudioControlPopoverP
     isStarting,
     isStopping,
     isToggling,
+    // HTTP/HTTPS detection
+    isHttpsRequired,
   } = microphone;
   
   // Use WebSocket data exclusively - no polling fallback
@@ -217,44 +221,17 @@ export default function AudioControlPopover({ microphone }: AudioControlPopoverP
     setIsLoading(true);
     
     try {
-      // Use RPC for device communication - works for both local and cloud
-      if (rpcDataChannel?.readyState !== "open") {
-        throw new Error("Device connection not available");
-      }
-
       if (isMicrophoneActiveFromHook) {
-        // Disable: Stop microphone subprocess via RPC AND remove WebRTC tracks locally
-        await new Promise<void>((resolve, reject) => {
-          send("microphoneStop", {}, (resp: JsonRpcResponse) => {
-            if ("error" in resp) {
-              reject(new Error(resp.error.message));
-            } else {
-              resolve();
-            }
-          });
-        });
-        
-        // Also stop local WebRTC stream
+        // Disable: Use the hook's stopMicrophone which handles both RPC and local cleanup
         const result = await stopMicrophone();
         if (!result.success) {
-          console.warn("Local microphone stop failed:", result.error?.message);
+          throw new Error(result.error?.message || "Failed to stop microphone");
         }
       } else {
-        // Enable: Start microphone subprocess via RPC AND add WebRTC tracks locally
-        await new Promise<void>((resolve, reject) => {
-          send("microphoneStart", {}, (resp: JsonRpcResponse) => {
-            if ("error" in resp) {
-              reject(new Error(resp.error.message));
-            } else {
-              resolve();
-            }
-          });
-        });
-        
-        // Also start local WebRTC stream
+        // Enable: Use the hook's startMicrophone which handles both RPC and local setup
         const result = await startMicrophone();
         if (!result.success) {
-          throw new Error(result.error?.message || "Failed to start local microphone");
+          throw new Error(result.error?.message || "Failed to start microphone");
         }
       }
     } catch (error) {
@@ -267,6 +244,11 @@ export default function AudioControlPopover({ microphone }: AudioControlPopoverP
 
   // Handle microphone device change
   const handleMicrophoneDeviceChange = async (deviceId: string) => {
+    // Don't process device changes for HTTPS-required placeholder
+    if (deviceId === 'https-required') {
+      return;
+    }
+    
     setSelectedInputDevice(deviceId);
     
     // If microphone is currently active, restart it with the new device
@@ -369,10 +351,24 @@ export default function AudioControlPopover({ microphone }: AudioControlPopoverP
               theme={isMicrophoneActiveFromHook ? "danger" : "primary"}
               text={isMicrophoneActiveFromHook ? "Disable" : "Enable"}
               onClick={handleToggleMicrophoneEnable}
-              disabled={isLoading}
+              disabled={isLoading || isHttpsRequired}
             />
           </div>
           
+          {/* HTTPS requirement notice */}
+          {isHttpsRequired && (
+            <div className="text-xs text-amber-600 dark:text-amber-400 bg-amber-50 dark:bg-amber-900/20 p-2 rounded-md">
+              <p className="font-medium mb-1">HTTPS Required for Microphone Input</p>
+              <p>
+                Microphone access requires a secure connection due to browser security policies. Audio output works fine on HTTP, but microphone input needs HTTPS.
+              </p>
+              <p className="mt-1">
+                <span className="font-medium">Current:</span> {window.location.protocol + '//' + window.location.host}
+                <br />
+                <span className="font-medium">Secure:</span> {'https://' + window.location.host}
+              </p>
+            </div>
+          )}
 
         </div>
 
@@ -402,7 +398,7 @@ export default function AudioControlPopover({ microphone }: AudioControlPopoverP
             <select
                value={selectedInputDevice}
                onChange={(e) => handleMicrophoneDeviceChange(e.target.value)}
-               disabled={devicesLoading}
+               disabled={devicesLoading || isHttpsRequired}
               className="w-full rounded-md border border-slate-200 bg-white px-3 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500 disabled:bg-slate-50 disabled:text-slate-500 dark:border-slate-600 dark:bg-slate-700 dark:text-slate-300 dark:focus:border-blue-400 dark:disabled:bg-slate-800"
             >
               {audioInputDevices.map((device) => (
@@ -411,11 +407,15 @@ export default function AudioControlPopover({ microphone }: AudioControlPopoverP
                 </option>
               ))}
             </select>
-            {isMicrophoneActiveFromHook && (
+            {isHttpsRequired ? (
+              <p className="text-xs text-amber-600 dark:text-amber-400">
+                HTTPS connection required for microphone device selection
+              </p>
+            ) : isMicrophoneActiveFromHook ? (
                <p className="text-xs text-slate-500 dark:text-slate-400">
                  Changing device will restart the microphone
                </p>
-             )}
+             ) : null}
           </div>
           
           {/* Speaker Selection */}

ui/src/hooks/useAudioDevices.ts

@@ -33,6 +33,57 @@ export function useAudioDevices(): UseAudioDevicesReturn {
     setError(null);
     
     try {
+      // Check if we're on HTTP (microphone requires HTTPS, but speakers can work)
+      const isHttp = window.location.protocol === 'http:';
+      const hasMediaDevices = !!navigator.mediaDevices;
+      const hasGetUserMedia = !!navigator.mediaDevices?.getUserMedia;
+      const hasEnumerateDevices = !!navigator.mediaDevices?.enumerateDevices;
+      
+      if (isHttp || !hasMediaDevices || !hasGetUserMedia) {
+        // Set placeholder devices when HTTPS is required for microphone
+        setAudioInputDevices([
+          { deviceId: 'https-required', label: 'HTTPS Required for Microphone Access', kind: 'audioinput' }
+        ]);
+        
+        // Try to enumerate speakers if possible, otherwise provide defaults
+        if (hasMediaDevices && hasEnumerateDevices) {
+          try {
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            const outputDevices: AudioDevice[] = [
+              { deviceId: 'default', label: 'Default Speaker', kind: 'audiooutput' }
+            ];
+            
+            devices.forEach(device => {
+              if (device.kind === 'audiooutput' && device.deviceId !== 'default') {
+                outputDevices.push({
+                  deviceId: device.deviceId,
+                  label: device.label || `Speaker ${device.deviceId.slice(0, 8)}`,
+                  kind: 'audiooutput'
+                });
+              }
+            });
+            
+            setAudioOutputDevices(outputDevices);
+          } catch {
+            // Fallback to default speakers if enumeration fails
+            setAudioOutputDevices([
+              { deviceId: 'default', label: 'Default Speaker', kind: 'audiooutput' },
+              { deviceId: 'system-default', label: 'System Default Audio Output', kind: 'audiooutput' }
+            ]);
+          }
+        } else {
+          // No enumeration available, use defaults
+          setAudioOutputDevices([
+            { deviceId: 'default', label: 'Default Speaker', kind: 'audiooutput' },
+            { deviceId: 'system-default', label: 'System Default Audio Output', kind: 'audiooutput' }
+          ]);
+        }
+        
+        setSelectedInputDevice('https-required');
+        setSelectedOutputDevice('default');
+        return; // Exit gracefully without throwing error on HTTP
+      }
+      
       // Request permissions first to get device labels
       await navigator.mediaDevices.getUserMedia({ audio: true });
       
@@ -68,8 +119,32 @@ export function useAudioDevices(): UseAudioDevicesReturn {
       // Audio devices enumerated
       
     } catch (err) {
-      devError('Failed to enumerate audio devices:', err);
-      setError(err instanceof Error ? err.message : 'Failed to access audio devices');
+      // Only log errors on HTTPS where we expect full device access
+      const isHttp = window.location.protocol === 'http:';
+      if (!isHttp) {
+        devError('Failed to enumerate audio devices:', err);
+      }
+      
+      let errorMessage = 'Failed to access audio devices';
+      
+      if (err instanceof Error) {
+        if (err.message.includes('HTTPS')) {
+          errorMessage = err.message;
+        } else if (err.name === 'NotAllowedError' || err.name === 'PermissionDeniedError') {
+          errorMessage = 'Microphone permission denied. Please allow microphone access.';
+        } else if (err.name === 'NotFoundError' || err.name === 'DevicesNotFoundError') {
+          errorMessage = 'No microphone devices found.';
+        } else if (err.name === 'NotSupportedError') {
+          errorMessage = 'Audio devices are not supported on this connection. Please use HTTPS.';
+        } else {
+          errorMessage = err.message || errorMessage;
+        }
+      }
+      
+      // Only set error state on HTTPS where we expect device access to work
+      if (!isHttp) {
+        setError(errorMessage);
+      }
     } finally {
       setIsLoading(false);
     }
@@ -82,13 +157,19 @@ export function useAudioDevices(): UseAudioDevicesReturn {
       refreshDevices();
     };
 
-    navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange);
+    // Check if navigator.mediaDevices exists and supports addEventListener
+    if (navigator.mediaDevices && typeof navigator.mediaDevices.addEventListener === 'function') {
+      navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange);
+    }
     
     // Initial load
     refreshDevices();
 
     return () => {
-      navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange);
+      // Check if navigator.mediaDevices exists and supports removeEventListener
+      if (navigator.mediaDevices && typeof navigator.mediaDevices.removeEventListener === 'function') {
+        navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange);
+      }
     };
   }, [refreshDevices]);
 

ui/src/hooks/useMicrophone.ts

@@ -2,6 +2,8 @@ import { useCallback, useEffect, useRef, useState } from "react";
 
 import { useRTCStore, useSettingsStore } from "@/hooks/stores";
 import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
+import { useUsbDeviceConfig } from "@/hooks/useUsbDeviceConfig";
+import { useAudioEvents, AudioDeviceChangedData } from "@/hooks/useAudioEvents";
 import { devLog, devInfo, devWarn, devError, devOnly } from "@/utils/debug";
 import { AUDIO_CONFIG } from "@/config/constants";
 
@@ -10,6 +12,19 @@ export interface MicrophoneError {
   message: string;
 }
 
+// Helper function to check if HTTPS is required for microphone access
+export function isHttpsRequired(): boolean {
+  // Check if we're on HTTP (not HTTPS)
+  const isHttp = window.location.protocol === 'http:';
+  
+  // Check if media devices are available
+  const hasMediaDevices = !!navigator.mediaDevices;
+  const hasGetUserMedia = !!navigator.mediaDevices?.getUserMedia;
+  
+  // HTTPS is required if we're on HTTP OR if media devices aren't available
+  return isHttp || !hasMediaDevices || !hasGetUserMedia;
+}
+
 export function useMicrophone() {
   const {
     peerConnection,
@@ -27,6 +42,10 @@ export function useMicrophone() {
   const { microphoneWasEnabled, setMicrophoneWasEnabled } = useSettingsStore();
   const { send } = useJsonRpc();
 
+  // Check USB audio status and handle microphone restoration when USB audio is re-enabled
+  const { usbDeviceConfig } = useUsbDeviceConfig();
+  const isUsbAudioEnabled = usbDeviceConfig?.audio ?? true;
+
   // RPC helper functions to replace HTTP API calls
   const rpcMicrophoneStart = useCallback((): Promise<void> => {
     return new Promise((resolve, reject) => {
@@ -181,6 +200,20 @@ export function useMicrophone() {
     try {
       // Set flag to prevent sync during startup
       isStartingRef.current = true;
+      
+      // Check if getUserMedia is available (requires HTTPS in most browsers)
+      if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+        setIsStarting(false);
+        isStartingRef.current = false;
+        return {
+          success: false,
+          error: {
+            type: 'permission',
+            message: 'Microphone access requires HTTPS connection. Please use HTTPS to use audio input.'
+          }
+        };
+      }
+      
       // Request microphone permission and get stream
       const audioConstraints: MediaTrackConstraints = {
         echoCancellation: true,
@@ -261,9 +294,19 @@ export function useMicrophone() {
         });
       }
 
-      // Notify backend that microphone is started
+      // Notify backend that microphone is started - only if USB audio is enabled
+      if (!isUsbAudioEnabled) {
+        devInfo("USB audio is disabled, skipping backend microphone start");
+        // Still set frontend state as active since the stream was successfully created
+        setMicrophoneActive(true);
+        setMicrophoneMuted(false);
+        setMicrophoneWasEnabled(true);
+        isStartingRef.current = false;
+        setIsStarting(false);
+        return { success: true };
+      }
       
-            // Retry logic for backend failures
+      // Retry logic for backend failures
       let backendSuccess = false;
       let lastError: Error | string | null = null;
       
@@ -372,7 +415,7 @@ export function useMicrophone() {
       setIsStarting(false);
       return { success: false, error: micError };
     }
-  }, [peerConnection, setMicrophoneStream, setMicrophoneSender, setMicrophoneActive, setMicrophoneMuted, setMicrophoneWasEnabled, stopMicrophoneStream, isStarting, isStopping, isToggling, rpcMicrophoneStart, rpcDataChannel?.readyState, send]);
+  }, [peerConnection, setMicrophoneStream, setMicrophoneSender, setMicrophoneActive, setMicrophoneMuted, setMicrophoneWasEnabled, stopMicrophoneStream, isStarting, isStopping, isToggling, rpcMicrophoneStart, rpcDataChannel?.readyState, send, isUsbAudioEnabled]);
 
 
 
@@ -575,6 +618,51 @@ export function useMicrophone() {
     return () => clearTimeout(timer);
   }, [syncMicrophoneState, microphoneWasEnabled, isMicrophoneActive, peerConnection, startMicrophone, rpcDataChannel?.readyState]);
 
+  // Handle audio device changes (USB audio enable/disable) via WebSocket events
+  const handleAudioDeviceChanged = useCallback((data: AudioDeviceChangedData) => {
+    devInfo("Audio device changed:", data);
+    devInfo("Current microphone state:", { isMicrophoneActive, microphoneWasEnabled });
+    
+    // USB audio was just disabled
+    if (!data.enabled && data.reason === "usb_reconfiguration") {
+      devInfo(`USB audio disabled via device change event - microphone was ${isMicrophoneActive ? 'active' : 'inactive'}`);
+      
+      // The microphoneWasEnabled flag is already being managed by the microphone start/stop functions
+      // We don't need to do anything special here - it will be preserved for restoration
+      devInfo(`Current microphoneWasEnabled flag: ${microphoneWasEnabled}`);
+    }
+    
+    // USB audio was just re-enabled
+    else if (data.enabled && data.reason === "usb_reconfiguration") {
+      devInfo("USB audio re-enabled via device change event - checking if microphone should be restored");
+      devInfo(`microphoneWasEnabled: ${microphoneWasEnabled}`);
+      devInfo(`Current microphone active: ${isMicrophoneActive}`);
+      devInfo(`RPC ready: ${rpcDataChannel?.readyState === "open"}`);
+      
+      // If microphone was enabled before (using the same logic as page reload restore), restore it
+      if (microphoneWasEnabled && !isMicrophoneActive && rpcDataChannel?.readyState === "open") {
+        devInfo("Restoring microphone after USB audio re-enabled (using microphoneWasEnabled flag)");
+        setTimeout(async () => {
+          try {
+            const result = await startMicrophone();
+            if (result.success) {
+              devInfo("Microphone successfully restored after USB audio re-enable");
+            } else {
+              devWarn("Failed to restore microphone after USB audio re-enable:", result.error);
+            }
+          } catch (error) {
+            devWarn("Error restoring microphone after USB audio re-enable:", error);
+          }
+        }, 500); // Small delay to ensure USB device reconfiguration is complete
+      } else {
+        devInfo("Not restoring microphone - conditions not met or microphone was not previously enabled");
+      }
+    }
+  }, [isMicrophoneActive, microphoneWasEnabled, startMicrophone, rpcDataChannel?.readyState]);
+
+  // Subscribe to audio device change events
+  useAudioEvents(handleAudioDeviceChanged);
+
   // Cleanup on unmount - use ref to avoid dependency on stopMicrophoneStream
   useEffect(() => {
     return () => {
@@ -605,5 +693,8 @@ export function useMicrophone() {
     isStarting,
     isStopping,
     isToggling,
+    
+    // HTTP/HTTPS detection
+    isHttpsRequired: isHttpsRequired(),
   };
 }