Merge 28a8fa05cc into 353099001f

audio.go

@@ -0,0 +1,14 @@
+package kvm
+
+import (
+	"os/exec"
+)
+
+func runAudioClient() (cmd *exec.Cmd, err error) {
+	return startNativeBinary("/userdata/jetkvm/bin/jetkvm_audio")
+}
+
+func StartAudioServer() {
+	nativeAudioSocketListener = StartNativeSocketServer("/var/run/jetkvm_audio.sock", handleAudioClient, false)
+	nativeLogger.Debug().Msg("native app audio sock started")
+}

config.go

@@ -128,6 +128,7 @@ var defaultConfig = &Config{
 		RelativeMouse: true,
 		Keyboard:      true,
 		MassStorage:   true,
+		Audio:         true,
 	},
 	NetworkConfig:   &network.NetworkConfig{},
 	DefaultLogLevel: "INFO",

internal/usbgadget/config.go

@@ -59,6 +59,23 @@ var defaultGadgetConfig = map[string]gadgetConfigItem{
 	// mass storage
 	"mass_storage_base": massStorageBaseConfig,
 	"mass_storage_lun0": massStorageLun0Config,
+	// audio
+	"audio": {
+		order:      4000,
+		device:     "uac1.usb0",
+		path:       []string{"functions", "uac1.usb0"},
+		configPath: []string{"uac1.usb0"},
+		attrs: gadgetAttributes{
+			"p_chmask":         "3",
+			"p_srate":          "48000",
+			"p_ssize":          "2",
+			"p_volume_present": "0",
+			"c_chmask":         "3",
+			"c_srate":          "48000",
+			"c_ssize":          "2",
+			"c_volume_present": "0",
+		},
+	},
 }
 
 func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool {
@@ -73,6 +90,8 @@ func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool {
 		return u.enabledDevices.MassStorage
 	case "mass_storage_lun0":
 		return u.enabledDevices.MassStorage
+	case "audio":
+		return u.enabledDevices.Audio
 	default:
 		return true
 	}

internal/usbgadget/usbgadget.go

@@ -19,6 +19,7 @@ type Devices struct {
 	RelativeMouse bool `json:"relative_mouse"`
 	Keyboard      bool `json:"keyboard"`
 	MassStorage   bool `json:"mass_storage"`
+	Audio         bool `json:"audio"`
 }
 
 // Config is a struct that represents the customizations for a USB gadget.

main.go

@@ -77,6 +77,12 @@ func Main() {
 
 	// initialize usb gadget
 	initUsbGadget()
+
+	StartAudioServer()
+	if _, err := runAudioClient(); err != nil {
+		logger.Warn().Err(err).Msg("failed to run audio client")
+	}
+
 	if err := setInitialVirtualMediaState(); err != nil {
 		logger.Warn().Err(err).Msg("failed to set initial virtual media state")
 	}

native.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/jetkvm/kvm/resource"
-
 	"github.com/pion/webrtc/v4/pkg/media"
 )
 
@@ -106,6 +105,7 @@ func WriteCtrlMessage(message []byte) error {
 
 var nativeCtrlSocketListener net.Listener  //nolint:unused
 var nativeVideoSocketListener net.Listener //nolint:unused
+var nativeAudioSocketListener net.Listener //nolint:unused
 
 var ctrlClientConnected = make(chan struct{})
 
@@ -231,7 +231,7 @@ func handleVideoClient(conn net.Conn) {
 
 	scopedLogger.Info().Msg("native video socket client connected")
 
-	inboundPacket := make([]byte, maxFrameSize)
+	inboundPacket := make([]byte, maxVideoFrameSize)
 	lastFrame := time.Now()
 	for {
 		n, err := conn.Read(inboundPacket)
@@ -251,6 +251,32 @@ func handleVideoClient(conn net.Conn) {
 	}
 }
 
+func handleAudioClient(conn net.Conn) {
+	defer conn.Close()
+	scopedLogger := nativeLogger.With().
+		Str("type", "audio").
+		Logger()
+
+	scopedLogger.Info().Msg("native audio socket client connected")
+	inboundPacket := make([]byte, maxAudioFrameSize)
+	for {
+		n, err := conn.Read(inboundPacket)
+		if err != nil {
+			scopedLogger.Warn().Err(err).Msg("error during read")
+			return
+		}
+
+		if currentSession != nil {
+			if err := currentSession.AudioTrack.WriteSample(media.Sample{
+				Data:     inboundPacket[:n],
+				Duration: 20 * time.Millisecond,
+			}); err != nil {
+				scopedLogger.Warn().Err(err).Msg("error writing sample")
+			}
+		}
+	}
+}
+
 func startNativeBinaryWithLock(binaryPath string) (*exec.Cmd, error) {
 	nativeCmdLock.Lock()
 	defer nativeCmdLock.Unlock()

ui/src/components/WebRTCVideo.tsx

@@ -695,7 +695,7 @@ export default function WebRTCVideo() {
                           controls={false}
                           onPlaying={onVideoPlaying}
                           onPlay={onVideoPlaying}
-                          muted={true}
+                          muted={false}
                           playsInline
                           disablePictureInPicture
                           controlsList="nofullscreen"

ui/src/routes/devices.$id.tsx

@@ -479,6 +479,8 @@ export default function KvmIdRoute() {
     };
 
     setTransceiver(pc.addTransceiver("video", { direction: "recvonly" }));
+    // Add audio transceiver to receive audio from the server
+    pc.addTransceiver("audio", { direction: "recvonly" });
 
     const rpcDataChannel = pc.createDataChannel("rpc");
     rpcDataChannel.onopen = () => {

video.go

@@ -5,7 +5,8 @@ import (
 )
 
 // max frame size for 1080p video, specified in mpp venc setting
-const maxFrameSize = 1920 * 1080 / 2
+const maxVideoFrameSize = 1920 * 1080 / 2
+const maxAudioFrameSize = 1500
 
 func writeCtrlAction(action string) error {
 	actionMessage := map[string]string{

webrtc.go

@@ -18,6 +18,7 @@ import (
 type Session struct {
 	peerConnection           *webrtc.PeerConnection
 	VideoTrack               *webrtc.TrackLocalStaticSample
+	AudioTrack               *webrtc.TrackLocalStaticSample
 	ControlChannel           *webrtc.DataChannel
 	RPCChannel               *webrtc.DataChannel
 	HidChannel               *webrtc.DataChannel
@@ -136,7 +137,17 @@ func newSession(config SessionConfig) (*Session, error) {
 		return nil, err
 	}
 
-	rtpSender, err := peerConnection.AddTrack(session.VideoTrack)
+	session.AudioTrack, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "kvm")
+	if err != nil {
+		return nil, err
+	}
+
+	videoRtpSender, err := peerConnection.AddTrack(session.VideoTrack)
+	if err != nil {
+		return nil, err
+	}
+
+	audioRtpSender, err := peerConnection.AddTrack(session.AudioTrack)
 	if err != nil {
 		return nil, err
 	}
@@ -144,14 +155,9 @@ func newSession(config SessionConfig) (*Session, error) {
 	// Read incoming RTCP packets
 	// Before these packets are returned they are processed by interceptors. For things
 	// like NACK this needs to be called.
-	go func() {
+	go drainRtpSender(videoRtpSender)
-		rtcpBuf := make([]byte, 1500)
+	go drainRtpSender(audioRtpSender)
-		for {
+
-			if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
-				return
-			}
-		}
-	}()
 	var isConnected bool
 
 	peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
@@ -203,6 +209,15 @@ func newSession(config SessionConfig) (*Session, error) {
 	return session, nil
 }
 
+func drainRtpSender(rtpSender *webrtc.RTPSender) {
+	rtcpBuf := make([]byte, 1500)
+	for {
+		if _, _, err := rtpSender.Read(rtcpBuf); err != nil {
+			return
+		}
+	}
+}
+
 var actionSessions = 0
 
 func onActiveSessionsChanged() {