Merge c529c903d0 into fe127ed41c

audio.go

@@ -1,14 +1,81 @@
 package kvm
 
 import (
+	"fmt"
+	"net"
 	"os/exec"
+	"sync"
+	"syscall"
+	"time"
 )
 
-func runAudioClient() (cmd *exec.Cmd, err error) {
+func startFFmpeg() (cmd *exec.Cmd, err error) {
-	return startNativeBinary("/userdata/jetkvm/bin/jetkvm_audio")
+	binaryPath := "/userdata/jetkvm/bin/ffmpeg"
+	// Run the binary in the background
+	cmd = exec.Command(binaryPath,
+		"-f", "alsa",
+		"-channels", "2",
+		"-sample_rate", "48000",
+		"-i", "hw:1,0",
+		"-c:a", "libopus",
+		"-b:a", "64k", // ought to be enough for anybody
+		"-vbr", "off",
+		"-frame_duration", "20",
+		"-compression_level", "2",
+		"-f", "rtp",
+		"rtp://127.0.0.1:3333")
+
+	nativeOutputLock := sync.Mutex{}
+	nativeStdout := &nativeOutput{
+		mu:     &nativeOutputLock,
+		logger: nativeLogger.Info().Str("pipe", "stdout"),
+	}
+	nativeStderr := &nativeOutput{
+		mu:     &nativeOutputLock,
+		logger: nativeLogger.Info().Str("pipe", "stderr"),
+	}
+
+	// Redirect stdout and stderr to the current process
+	cmd.Stdout = nativeStdout
+	cmd.Stderr = nativeStderr
+
+	// Set the process group ID so we can kill the process and its children when this process exits
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid:   true,
+		Pdeathsig: syscall.SIGKILL,
+	}
+
+	// Start the command
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start binary: %w", err)
+	}
+
+	return
 }
 
-func StartAudioServer() {
+func StartRtpAudioServer(handleClient func(net.Conn)) {
-	nativeAudioSocketListener = StartNativeSocketServer("/var/run/jetkvm_audio.sock", handleAudioClient, false)
+	scopedLogger := nativeLogger.With().
-	nativeLogger.Debug().Msg("native app audio sock started")
+		Logger()
+
+	listener, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 3333})
+	if err != nil {
+		scopedLogger.Warn().Err(err).Msg("failed to start server")
+		return
+	}
+
+	scopedLogger.Info().Msg("server listening")
+
+	go func() {
+		for {
+			cmd, err := startFFmpeg()
+			if err != nil {
+				scopedLogger.Error().Err(err).Msg("failed to start ffmpeg")
+			}
+			err = cmd.Wait()
+			scopedLogger.Error().Err(err).Msg("ffmpeg exited, restarting")
+			time.Sleep(2 * time.Second)
+		}
+	}()
+
+	go handleClient(listener)
 }

main.go

@@ -77,11 +77,7 @@ func Main() {
 
 	// initialize usb gadget
 	initUsbGadget()
-
+	StartRtpAudioServer(handleAudioClient)
-	StartAudioServer()
-	if _, err := runAudioClient(); err != nil {
-		logger.Warn().Err(err).Msg("failed to run audio client")
-	}
 
 	if err := setInitialVirtualMediaState(); err != nil {
 		logger.Warn().Err(err).Msg("failed to set initial virtual media state")

native.go

@@ -13,6 +13,8 @@ import (
 	"time"
 
 	"github.com/jetkvm/kvm/resource"
+	"github.com/pion/rtp"
+
 	"github.com/pion/webrtc/v4/pkg/media"
 )
 
@@ -105,7 +107,6 @@ func WriteCtrlMessage(message []byte) error {
 
 var nativeCtrlSocketListener net.Listener  //nolint:unused
 var nativeVideoSocketListener net.Listener //nolint:unused
-var nativeAudioSocketListener net.Listener //nolint:unused
 
 var ctrlClientConnected = make(chan struct{})
 
@@ -259,6 +260,8 @@ func handleAudioClient(conn net.Conn) {
 
 	scopedLogger.Info().Msg("native audio socket client connected")
 	inboundPacket := make([]byte, maxAudioFrameSize)
+	var timestamp uint32
+	var packet rtp.Packet
 	for {
 		n, err := conn.Read(inboundPacket)
 		if err != nil {
@@ -267,10 +270,20 @@ func handleAudioClient(conn net.Conn) {
 		}
 
 		if currentSession != nil {
-			if err := currentSession.AudioTrack.WriteSample(media.Sample{
+			if err := packet.Unmarshal(inboundPacket[:n]); err != nil {
-				Data:     inboundPacket[:n],
+				scopedLogger.Warn().Err(err).Msg("error unmarshalling audio socket packet")
-				Duration: 20 * time.Millisecond,
+				continue
-			}); err != nil {
+			}
+
+			timestamp += 960
+			packet.Header.Timestamp = timestamp
+			buf, err := packet.Marshal()
+			if err != nil {
+				scopedLogger.Warn().Err(err).Msg("error marshalling packet")
+				continue
+			}
+
+			if _, err := currentSession.AudioTrack.Write(buf); err != nil {
 				scopedLogger.Warn().Err(err).Msg("error writing sample")
 			}
 		}

webrtc.go

@@ -18,7 +18,7 @@ import (
 type Session struct {
 	peerConnection           *webrtc.PeerConnection
 	VideoTrack               *webrtc.TrackLocalStaticSample
-	AudioTrack               *webrtc.TrackLocalStaticSample
+	AudioTrack               *webrtc.TrackLocalStaticRTP
 	ControlChannel           *webrtc.DataChannel
 	RPCChannel               *webrtc.DataChannel
 	HidChannel               *webrtc.DataChannel
@@ -137,7 +137,7 @@ func newSession(config SessionConfig) (*Session, error) {
 		return nil, err
 	}
 
-	session.AudioTrack, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "kvm")
+	session.AudioTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "kvm")
 	if err != nil {
 		return nil, err
 	}