diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index aa803f6..e5df18e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,10 +1,15 @@ { "name": "JetKVM", - "image": "mcr.microsoft.com/devcontainers/go:1-1.23-bookworm", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04", + "runArgs": ["--platform=linux/amd64" ], "features": { "ghcr.io/devcontainers/features/node:1": { // Should match what is defined in ui/package.json "version": "22.15.0" + }, + "ghcr.io/devcontainers/features/go:1": { + // Should match what is defined in go.mod + "version": "latest" } }, "mounts": [ diff --git a/.gitignore b/.gitignore index f37d922..21e27b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ bin/* static/* +.vscode/ +tmp/ +.devcontainer/devcontainer-lock.json .idea .DS_Store +*.log +*.tmp +*.code-workspace device-tests.tar.gz \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d95db77..7098c15 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -11,21 +11,39 @@ + # JetKVM Development Guide + Welcome to JetKVM development! This guide will help you get started quickly, whether you're fixing bugs, adding features, or just exploring the codebase. ## Get Started + ### Prerequisites - **A JetKVM device** (for full development) - **[Go 1.24.4+](https://go.dev/doc/install)** and **[Node.js 22.15.0](https://nodejs.org/en/download/)** - **[Git](https://git-scm.com/downloads)** for version control - **[SSH access](https://jetkvm.com/docs/advanced-usage/developing#developer-mode)** to your JetKVM device +- **Audio build dependencies:** + - **New in this release:** The audio pipeline is now fully in-process using CGO, ALSA, and Opus. You must run the provided scripts in `tools/` to set up the cross-compiler and build static ALSA/Opus libraries for ARM. See below. + ### Development Environment -**Recommended:** Development is best done on **Linux** or **macOS**. +**Recommended:** Development is best done on **Linux** or **macOS**. + +#### Apple Silicon (M1/M2/M3) Mac Users + +If you are developing on an Apple Silicon Mac, you should use a devcontainer to ensure compatibility with the JetKVM build environment (which targets linux/amd64 and ARM). There are two main options: + +- **VS Code Dev Containers**: Open the project in VS Code and use the built-in Dev Containers support. The configuration is in `.devcontainer/devcontainer.json`. +- **Devpod**: [Devpod](https://devpod.sh/) is a fast, open-source tool for running devcontainers anywhere. If you use Devpod, go to **Settings → Experimental → Additional Environmental Variables** and add: + - `DOCKER_DEFAULT_PLATFORM=linux/amd64` + This ensures all builds run in the correct architecture. +- **devcontainer CLI**: You can also use the [devcontainer CLI](https://github.com/devcontainers/cli) to launch the devcontainer from the terminal. + +This approach ensures compatibility with all shell scripts, build tools, and cross-compilation steps used in the project. If you're using Windows, we strongly recommend using **WSL (Windows Subsystem for Linux)** for the best development experience: - [Install WSL on Windows](https://docs.microsoft.com/en-us/windows/wsl/install) @@ -33,6 +51,7 @@ If you're using Windows, we strongly recommend using **WSL (Windows Subsystem fo This ensures compatibility with shell scripts and build tools used in the project. + ### Project Setup 1. **Clone the repository:** @@ -46,16 +65,25 @@ This ensures compatibility with shell scripts and build tools used in the projec go version && node --version ``` -3. **Find your JetKVM IP address** (check your router or device screen) +3. **Set up the cross-compiler and audio dependencies:** + ```bash + make dev_env + # This will run tools/setup_rv1106_toolchain.sh and tools/build_audio_deps.sh + # It will clone the cross-compiler and build ALSA/Opus static libs in $HOME/.jetkvm + # + # **Note:** This is required for the new in-process audio pipeline. If you skip this step, audio will not work. + ``` -4. **Deploy and test:** +4. **Find your JetKVM IP address** (check your router or device screen) + +5. **Deploy and test:** ```bash ./dev_deploy.sh -r 192.168.1.100 # Replace with your device IP ``` -5. **Open in browser:** `http://192.168.1.100` +6. **Open in browser:** `http://192.168.1.100` -That's it! You're now running your own development version of JetKVM. +That's it! You're now running your own development version of JetKVM, **with in-process audio streaming for the first time.** --- @@ -71,13 +99,15 @@ npm install Now edit files in `ui/src/` and see changes live in your browser! -### Modify the backend + +### Modify the backend (including audio) ```bash -# Edit Go files (config.go, web.go, etc.) +# Edit Go files (config.go, web.go, internal/audio, etc.) ./dev_deploy.sh -r 192.168.1.100 --skip-ui-build ``` + ### Run tests ```bash @@ -93,21 +123,26 @@ tail -f /var/log/jetkvm.log --- + ## Project Layout ``` /kvm/ ├── main.go # App entry point -├── config.go # Settings & configuration -├── web.go # API endpoints -├── ui/ # React frontend -│ ├── src/routes/ # Pages (login, settings, etc.) -│ └── src/components/ # UI components -└── internal/ # Internal Go packages +├── config.go # Settings & configuration +├── web.go # API endpoints +├── ui/ # React frontend +│ ├── src/routes/ # Pages (login, settings, etc.) +│ └── src/components/ # UI components +├── internal/ # Internal Go packages +│ └── audio/ # In-process audio pipeline (CGO, ALSA, Opus) [NEW] +├── tools/ # Toolchain and audio dependency setup scripts +└── Makefile # Build and dev automation (see audio targets) ``` **Key files for beginners:** +- `internal/audio/` - [NEW] In-process audio pipeline (CGO, ALSA, Opus) - `web.go` - Add new API endpoints here - `config.go` - Add new settings here - `ui/src/routes/` - Add new pages here @@ -136,9 +171,10 @@ npm install ./dev_device.sh ``` + ### Quick Backend Changes -*Best for: API or backend logic changes* +*Best for: API, backend, or audio logic changes (including audio pipeline)* ```bash # Skip frontend build for faster deployment @@ -206,7 +242,8 @@ curl -X POST http:///auth/password-local \ --- -## Common Issues & Solutions + +### Common Issues & Solutions ### "Build failed" or "Permission denied" @@ -218,6 +255,8 @@ ssh root@ chmod +x /userdata/jetkvm/bin/jetkvm_app_debug go clean -modcache go mod tidy make build_dev +# If you see errors about missing ALSA/Opus or toolchain, run: +make dev_env # Required for new audio support ``` ### "Can't connect to device" @@ -230,6 +269,15 @@ ping ssh root@ echo "Connection OK" ``` + +### "Audio not working" + +```bash +# Make sure you have run: +make dev_env +# If you see errors about ALSA/Opus, check logs and re-run the setup scripts in tools/. +``` + ### "Frontend not updating" ```bash @@ -244,18 +292,21 @@ npm install ## Next Steps + ### Adding a New Feature -1. **Backend:** Add API endpoint in `web.go` +1. **Backend:** Add API endpoint in `web.go` or extend audio in `internal/audio/` 2. **Config:** Add settings in `config.go` 3. **Frontend:** Add UI in `ui/src/routes/` 4. **Test:** Deploy and test with `./dev_deploy.sh` + ### Code Style - **Go:** Follow standard Go conventions - **TypeScript:** Use TypeScript for type safety - **React:** Keep components small and reusable +- **Audio/CGO:** Keep C/Go integration minimal, robust, and well-documented. Use zerolog for all logging. ### Environment Variables diff --git a/Makefile b/Makefile index c696dca..887add4 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,20 @@ +# --- JetKVM Audio/Toolchain Dev Environment Setup --- +.PHONY: setup_toolchain build_audio_deps dev_env + +# Clone the rv1106-system toolchain to $HOME/.jetkvm/rv1106-system +setup_toolchain: + bash tools/setup_rv1106_toolchain.sh + +# Build ALSA and Opus static libs for ARM in $HOME/.jetkvm/audio-libs +build_audio_deps: setup_toolchain + bash tools/build_audio_deps.sh + +# Prepare everything needed for local development (toolchain + audio deps) +dev_env: build_audio_deps + @echo "Development environment ready." +JETKVM_HOME ?= $(HOME)/.jetkvm +TOOLCHAIN_DIR ?= $(JETKVM_HOME)/rv1106-system +AUDIO_LIBS_DIR ?= $(JETKVM_HOME)/audio-libs BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) BUILDDATE ?= $(shell date -u +%FT%T%z) BUILDTS ?= $(shell date -u +%s) @@ -25,9 +42,14 @@ TEST_DIRS := $(shell find . -name "*_test.go" -type f -exec dirname {} \; | sort hash_resource: @shasum -a 256 resource/jetkvm_native | cut -d ' ' -f 1 > resource/jetkvm_native.sha256 -build_dev: hash_resource +build_dev: build_audio_deps hash_resource @echo "Building..." - $(GO_CMD) build \ + GOOS=linux GOARCH=arm GOARM=7 \ + CC=$(TOOLCHAIN_DIR)/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc \ + CGO_ENABLED=1 \ + CGO_CFLAGS="-I$(AUDIO_LIBS_DIR)/alsa-lib-1.2.14/include -I$(AUDIO_LIBS_DIR)/opus-1.5.2/include -I$(AUDIO_LIBS_DIR)/opus-1.5.2/celt" \ + CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-1.2.14/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-1.5.2/.libs -lopus -lm -ldl -static" \ + go build \ -ldflags="$(GO_LDFLAGS) -X $(KVM_PKG_NAME).builtAppVersion=$(VERSION_DEV)" \ $(GO_RELEASE_BUILD_ARGS) \ -o $(BIN_DIR)/jetkvm_app cmd/main.go @@ -70,9 +92,14 @@ dev_release: frontend build_dev rclone copyto bin/jetkvm_app r2://jetkvm-update/app/$(VERSION_DEV)/jetkvm_app rclone copyto bin/jetkvm_app.sha256 r2://jetkvm-update/app/$(VERSION_DEV)/jetkvm_app.sha256 -build_release: frontend hash_resource +build_release: frontend build_audio_deps hash_resource @echo "Building release..." - $(GO_CMD) build \ + GOOS=linux GOARCH=arm GOARM=7 \ + CC=$(TOOLCHAIN_DIR)/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc \ + CGO_ENABLED=1 \ + CGO_CFLAGS="-I$(AUDIO_LIBS_DIR)/alsa-lib-1.2.14/include -I$(AUDIO_LIBS_DIR)/opus-1.5.2/include -I$(AUDIO_LIBS_DIR)/opus-1.5.2/celt" \ + CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-1.2.14/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-1.5.2/.libs -lopus -lm -ldl -static" \ + go build \ -ldflags="$(GO_LDFLAGS) -X $(KVM_PKG_NAME).builtAppVersion=$(VERSION)" \ $(GO_RELEASE_BUILD_ARGS) \ -o bin/jetkvm_app cmd/main.go diff --git a/README.md b/README.md index 541578c..9a6d58f 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,20 @@ -JetKVM is a high-performance, open-source KVM over IP (Keyboard, Video, Mouse) solution designed for efficient remote management of computers, servers, and workstations. Whether you're dealing with boot failures, installing a new operating system, adjusting BIOS settings, or simply taking control of a machine from afar, JetKVM provides the tools to get it done effectively. + + +JetKVM is a high-performance, open-source KVM over IP (Keyboard, Video, Mouse, **Audio**) solution designed for efficient remote management of computers, servers, and workstations. Whether you're dealing with boot failures, installing a new operating system, adjusting BIOS settings, or simply taking control of a machine from afar, JetKVM provides the tools to get it done effectively. + + + + ## Features -- **Ultra-low Latency** - 1080p@60FPS video with 30-60ms latency using H.264 encoding. Smooth mouse and keyboard interaction for responsive remote control. +- **Ultra-low Latency** - 1080p@60FPS video with 30-60ms latency using H.264 encoding. Smooth mouse, keyboard, and audio for responsive remote control. +- **First-Class Audio Support** - JetKVM now supports in-process, low-latency audio streaming using ALSA and Opus, fully integrated via CGO. No external audio binaries or IPC required—audio is delivered directly from the device to your browser. - **Free & Optional Remote Access** - Remote management via JetKVM Cloud using WebRTC. -- **Open-source software** - Written in Golang on Linux. Easily customizable through SSH access to the JetKVM device. +- **Open-source software** - Written in Golang (with CGO for audio) on Linux. Easily customizable through SSH access to the JetKVM device. ## Contributing @@ -31,20 +38,23 @@ The best place to search for answers is our [Documentation](https://jetkvm.com/d If you've found an issue and want to report it, please check our [Issues](https://github.com/jetkvm/kvm/issues) page. Make sure the description contains information about the firmware version you're using, your platform, and a clear explanation of the steps to reproduce the issue. + + # Development -JetKVM is written in Go & TypeScript. with some bits and pieces written in C. An intermediate level of Go & TypeScript knowledge is recommended for comfortable programming. +JetKVM is written in Go & TypeScript, with some C for low-level integration. **Audio support is now fully in-process using CGO, ALSA, and Opus—no external audio binaries required.** -The project contains two main parts, the backend software that runs on the KVM device and the frontend software that is served by the KVM device, and also the cloud. +The project contains two main parts: the backend software (Go, CGO) that runs on the KVM device, and the frontend software (React/TypeScript) that is served by the KVM device and the cloud. For comprehensive development information, including setup, testing, debugging, and contribution guidelines, see **[DEVELOPMENT.md](DEVELOPMENT.md)**. For quick device development, use the `./dev_deploy.sh` script. It will build the frontend and backend and deploy them to the local KVM device. Run `./dev_deploy.sh --help` for more information. + ## Backend -The backend is written in Go and is responsible for the KVM device management, the cloud API and the cloud web. +The backend is written in Go and is responsible for KVM device management, audio/video streaming, the cloud API, and the cloud web. **Audio is now captured and encoded in-process using ALSA and Opus via CGO, with no external processes or IPC.** ## Frontend -The frontend is written in React and TypeScript and is served by the KVM device. It has three build targets: `device`, `development` and `production`. Development is used for development of the cloud version on your local machine, device is used for building the frontend for the KVM device and production is used for building the frontend for the cloud. +The frontend is written in React and TypeScript and is served by the KVM device. It has three build targets: `device`, `development`, and `production`. Development is used for the cloud version on your local machine, device is used for building the frontend for the KVM device, and production is used for building the frontend for the cloud. diff --git a/config.go b/config.go index d48e25b..6b539c8 100644 --- a/config.go +++ b/config.go @@ -130,6 +130,7 @@ var defaultConfig = &Config{ RelativeMouse: true, Keyboard: true, MassStorage: true, + Audio: true, }, NetworkConfig: &network.NetworkConfig{}, DefaultLogLevel: "INFO", diff --git a/internal/audio/api.go b/internal/audio/api.go new file mode 100644 index 0000000..2cb60b8 --- /dev/null +++ b/internal/audio/api.go @@ -0,0 +1,11 @@ +package audio + +// StartAudioStreaming launches the in-process audio stream and delivers Opus frames to the provided callback. +func StartAudioStreaming(send func([]byte)) error { + return StartCGOAudioStream(send) +} + +// StopAudioStreaming stops the in-process audio stream. +func StopAudioStreaming() { + StopCGOAudioStream() +} diff --git a/internal/audio/audio.go b/internal/audio/audio.go new file mode 100644 index 0000000..555e31f --- /dev/null +++ b/internal/audio/audio.go @@ -0,0 +1,126 @@ +package audio + +import ( + "sync/atomic" + "time" + // Explicit import for CGO audio stream glue +) + +const MaxAudioFrameSize = 1500 + +// AudioQuality represents different audio quality presets +type AudioQuality int + +const ( + AudioQualityLow AudioQuality = iota + AudioQualityMedium + AudioQualityHigh + AudioQualityUltra +) + +// AudioConfig holds configuration for audio processing +type AudioConfig struct { + Quality AudioQuality + Bitrate int // kbps + SampleRate int // Hz + Channels int + FrameSize time.Duration // ms +} + +// AudioMetrics tracks audio performance metrics +// Note: 64-bit fields must be first for proper alignment on 32-bit ARM +type AudioMetrics struct { + FramesReceived int64 + FramesDropped int64 + BytesProcessed int64 + ConnectionDrops int64 + LastFrameTime time.Time + AverageLatency time.Duration +} + +var ( + currentConfig = AudioConfig{ + Quality: AudioQualityMedium, + Bitrate: 64, + SampleRate: 48000, + Channels: 2, + FrameSize: 20 * time.Millisecond, + } + metrics AudioMetrics +) + +// GetAudioQualityPresets returns predefined quality configurations +func GetAudioQualityPresets() map[AudioQuality]AudioConfig { + return map[AudioQuality]AudioConfig{ + AudioQualityLow: { + Quality: AudioQualityLow, + Bitrate: 32, + SampleRate: 48000, + Channels: 2, + FrameSize: 20 * time.Millisecond, + }, + AudioQualityMedium: { + Quality: AudioQualityMedium, + Bitrate: 64, + SampleRate: 48000, + Channels: 2, + FrameSize: 20 * time.Millisecond, + }, + AudioQualityHigh: { + Quality: AudioQualityHigh, + Bitrate: 128, + SampleRate: 48000, + Channels: 2, + FrameSize: 20 * time.Millisecond, + }, + AudioQualityUltra: { + Quality: AudioQualityUltra, + Bitrate: 256, + SampleRate: 48000, + Channels: 2, + FrameSize: 10 * time.Millisecond, + }, + } +} + +// SetAudioQuality updates the current audio quality configuration +func SetAudioQuality(quality AudioQuality) { + presets := GetAudioQualityPresets() + if config, exists := presets[quality]; exists { + currentConfig = config + } +} + +// GetAudioConfig returns the current audio configuration +func GetAudioConfig() AudioConfig { + return currentConfig +} + +// GetAudioMetrics returns current audio metrics +func GetAudioMetrics() AudioMetrics { + return AudioMetrics{ + FramesReceived: atomic.LoadInt64(&metrics.FramesReceived), + FramesDropped: atomic.LoadInt64(&metrics.FramesDropped), + BytesProcessed: atomic.LoadInt64(&metrics.BytesProcessed), + LastFrameTime: metrics.LastFrameTime, + ConnectionDrops: atomic.LoadInt64(&metrics.ConnectionDrops), + AverageLatency: metrics.AverageLatency, + } +} + +// RecordFrameReceived increments the frames received counter +func RecordFrameReceived(bytes int) { + atomic.AddInt64(&metrics.FramesReceived, 1) + atomic.AddInt64(&metrics.BytesProcessed, int64(bytes)) + metrics.LastFrameTime = time.Now() +} + +// RecordFrameDropped increments the frames dropped counter +func RecordFrameDropped() { + atomic.AddInt64(&metrics.FramesDropped, 1) +} + +// RecordConnectionDrop increments the connection drops counter +func RecordConnectionDrop() { + atomic.AddInt64(&metrics.ConnectionDrops, 1) +} diff --git a/internal/audio/audio_mute.go b/internal/audio/audio_mute.go new file mode 100644 index 0000000..61d1811 --- /dev/null +++ b/internal/audio/audio_mute.go @@ -0,0 +1,26 @@ +package audio + +import ( + "sync" + + "github.com/jetkvm/kvm/internal/logging" +) + +var audioMuteState struct { + muted bool + mu sync.RWMutex +} + +func SetAudioMuted(muted bool) { + audioMuteState.mu.Lock() + prev := audioMuteState.muted + audioMuteState.muted = muted + logging.GetDefaultLogger().Info().Str("component", "audio").Msgf("SetAudioMuted: prev=%v, new=%v", prev, muted) + audioMuteState.mu.Unlock() +} + +func IsAudioMuted() bool { + audioMuteState.mu.RLock() + defer audioMuteState.mu.RUnlock() + return audioMuteState.muted +} diff --git a/internal/audio/cgo_audio.go b/internal/audio/cgo_audio.go new file mode 100644 index 0000000..ab5825e --- /dev/null +++ b/internal/audio/cgo_audio.go @@ -0,0 +1,157 @@ +//go:build linux && arm +// +build linux,arm + +package audio + +import ( + "errors" + "sync/atomic" + "time" + "unsafe" + + "github.com/jetkvm/kvm/internal/logging" +) + +/* +#cgo CFLAGS: -I${SRCDIR}/../../tools/alsa-opus-includes +#cgo LDFLAGS: -L$HOME/.jetkvm/audio-libs/alsa-lib-1.2.14/src/.libs -lasound -L$HOME/.jetkvm/audio-libs/opus-1.5.2/.libs -lopus -lm -ldl -static +#include +#include +#include + +// C state for ALSA/Opus +static snd_pcm_t *pcm_handle = NULL; +static OpusEncoder *encoder = NULL; +static int opus_bitrate = 64000; +static int opus_complexity = 5; +static int sample_rate = 48000; +static int channels = 2; +static int frame_size = 960; // 20ms for 48kHz +static int max_packet_size = 1500; + +// Initialize ALSA and Opus encoder +int jetkvm_audio_init() { + int err; + snd_pcm_hw_params_t *params; + if (pcm_handle) return 0; + if (snd_pcm_open(&pcm_handle, "hw:1,0", SND_PCM_STREAM_CAPTURE, 0) < 0) + return -1; + snd_pcm_hw_params_malloc(¶ms); + snd_pcm_hw_params_any(pcm_handle, params); + snd_pcm_hw_params_set_access(pcm_handle, params, SND_PCM_ACCESS_RW_INTERLEAVED); + snd_pcm_hw_params_set_format(pcm_handle, params, SND_PCM_FORMAT_S16_LE); + snd_pcm_hw_params_set_channels(pcm_handle, params, channels); + snd_pcm_hw_params_set_rate(pcm_handle, params, sample_rate, 0); + snd_pcm_hw_params_set_period_size(pcm_handle, params, frame_size, 0); + snd_pcm_hw_params(pcm_handle, params); + snd_pcm_hw_params_free(params); + snd_pcm_prepare(pcm_handle); + encoder = opus_encoder_create(sample_rate, channels, OPUS_APPLICATION_AUDIO, &err); + if (!encoder) return -2; + opus_encoder_ctl(encoder, OPUS_SET_BITRATE(opus_bitrate)); + opus_encoder_ctl(encoder, OPUS_SET_COMPLEXITY(opus_complexity)); + return 0; +} + +// Read and encode one frame, returns encoded size or <0 on error +int jetkvm_audio_read_encode(void *opus_buf) { + short pcm_buffer[1920]; // max 2ch*960 + unsigned char *out = (unsigned char*)opus_buf; + int pcm_rc = snd_pcm_readi(pcm_handle, pcm_buffer, frame_size); + if (pcm_rc < 0) return -1; + int nb_bytes = opus_encode(encoder, pcm_buffer, frame_size, out, max_packet_size); + return nb_bytes; +} + +void jetkvm_audio_close() { + if (encoder) { opus_encoder_destroy(encoder); encoder = NULL; } + if (pcm_handle) { snd_pcm_close(pcm_handle); pcm_handle = NULL; } +} +*/ +import "C" + +var ( + audioStreamRunning int32 +) + +// Go wrappers for initializing, starting, stopping, and controlling audio +func cgoAudioInit() error { + ret := C.jetkvm_audio_init() + if ret != 0 { + return errors.New("failed to init ALSA/Opus") + } + return nil +} + +func cgoAudioClose() { + C.jetkvm_audio_close() +} + +// Reads and encodes one frame, returns encoded bytes or error +func cgoAudioReadEncode(buf []byte) (int, error) { + if len(buf) < 1500 { + return 0, errors.New("buffer too small") + } + n := C.jetkvm_audio_read_encode(unsafe.Pointer(&buf[0])) + if n < 0 { + return 0, errors.New("audio read/encode error") + } + return int(n), nil +} + +func StartCGOAudioStream(send func([]byte)) error { + if !atomic.CompareAndSwapInt32(&audioStreamRunning, 0, 1) { + return errors.New("audio stream already running") + } + go func() { + defer atomic.StoreInt32(&audioStreamRunning, 0) + logger := logging.GetDefaultLogger().With().Str("component", "audio").Logger() + err := cgoAudioInit() + if err != nil { + logger.Error().Err(err).Msg("cgoAudioInit failed") + return + } + defer cgoAudioClose() + buf := make([]byte, 1500) + errorCount := 0 + for atomic.LoadInt32(&audioStreamRunning) == 1 { + m := IsAudioMuted() + // (debug) logger.Debug().Msgf("audio loop: IsAudioMuted=%v", m) + if m { + time.Sleep(20 * time.Millisecond) + continue + } + n, err := cgoAudioReadEncode(buf) + if err != nil { + logger.Warn().Err(err).Msg("cgoAudioReadEncode error") + RecordFrameDropped() + errorCount++ + if errorCount >= 10 { + logger.Warn().Msg("Too many audio read errors, reinitializing ALSA/Opus state") + cgoAudioClose() + time.Sleep(100 * time.Millisecond) + if err := cgoAudioInit(); err != nil { + logger.Error().Err(err).Msg("cgoAudioInit failed during recovery") + time.Sleep(500 * time.Millisecond) + continue + } + errorCount = 0 + } else { + time.Sleep(5 * time.Millisecond) + } + continue + } + errorCount = 0 + // (debug) logger.Debug().Msgf("frame encoded: %d bytes", n) + RecordFrameReceived(n) + send(buf[:n]) + } + logger.Info().Msg("audio loop exited") + }() + return nil +} + +// StopCGOAudioStream signals the audio stream goroutine to stop +func StopCGOAudioStream() { + atomic.StoreInt32(&audioStreamRunning, 0) +} diff --git a/internal/audio/cgo_audio_notlinux.go b/internal/audio/cgo_audio_notlinux.go new file mode 100644 index 0000000..209b7aa --- /dev/null +++ b/internal/audio/cgo_audio_notlinux.go @@ -0,0 +1,11 @@ +//go:build !linux || !arm +// +build !linux !arm + +package audio + +// Dummy implementations for non-linux/arm builds +func StartCGOAudioStream(send func([]byte)) error { + return nil +} + +func StopCGOAudioStream() {} diff --git a/internal/usbgadget/config.go b/internal/usbgadget/config.go index 6d1bd39..dad5b79 100644 --- a/internal/usbgadget/config.go +++ b/internal/usbgadget/config.go @@ -59,6 +59,23 @@ var defaultGadgetConfig = map[string]gadgetConfigItem{ // mass storage "mass_storage_base": massStorageBaseConfig, "mass_storage_lun0": massStorageLun0Config, + // audio + "audio": { + order: 4000, + device: "uac1.usb0", + path: []string{"functions", "uac1.usb0"}, + configPath: []string{"uac1.usb0"}, + attrs: gadgetAttributes{ + "p_chmask": "3", + "p_srate": "48000", + "p_ssize": "2", + "p_volume_present": "0", + "c_chmask": "3", + "c_srate": "48000", + "c_ssize": "2", + "c_volume_present": "0", + }, + }, } func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool { @@ -73,6 +90,8 @@ func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool { return u.enabledDevices.MassStorage case "mass_storage_lun0": return u.enabledDevices.MassStorage + case "audio": + return u.enabledDevices.Audio default: return true } diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index cb70655..f51050b 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -19,6 +19,7 @@ type Devices struct { RelativeMouse bool `json:"relative_mouse"` Keyboard bool `json:"keyboard"` MassStorage bool `json:"mass_storage"` + Audio bool `json:"audio"` } // Config is a struct that represents the customizations for a USB gadget. diff --git a/main.go b/main.go index c25d8b8..cccd5e6 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,8 @@ import ( "time" "github.com/gwatts/rootcerts" + "github.com/jetkvm/kvm/internal/audio" + "github.com/pion/webrtc/v4/pkg/media" ) var appCtx context.Context @@ -71,12 +73,41 @@ func Main() { err = ExtractAndRunNativeBin() if err != nil { logger.Warn().Err(err).Msg("failed to extract and run native bin") - //TODO: prepare an error message screen buffer to show on kvm screen + // (future) prepare an error message screen buffer to show on kvm screen } }() // initialize usb gadget initUsbGadget() + + // Start in-process audio streaming and deliver Opus frames to WebRTC + go func() { + err := audio.StartAudioStreaming(func(frame []byte) { + // Deliver Opus frame to WebRTC audio track if session is active + if currentSession != nil { + config := audio.GetAudioConfig() + var sampleData []byte + if audio.IsAudioMuted() { + sampleData = make([]byte, len(frame)) // silence + } else { + sampleData = frame + } + if err := currentSession.AudioTrack.WriteSample(media.Sample{ + Data: sampleData, + Duration: config.FrameSize, + }); err != nil { + logger.Warn().Err(err).Msg("error writing audio sample") + audio.RecordFrameDropped() + } + } else { + audio.RecordFrameDropped() + } + }) + if err != nil { + logger.Warn().Err(err).Msg("failed to start in-process audio streaming") + } + }() + if err := setInitialVirtualMediaState(); err != nil { logger.Warn().Err(err).Msg("failed to set initial virtual media state") } diff --git a/native.go b/native.go index 9807206..fc8cfcb 100644 --- a/native.go +++ b/native.go @@ -1,255 +1,46 @@ +//go:build linux + package kvm import ( - "bytes" - "encoding/json" - "errors" "fmt" - "io" - "net" "os" "os/exec" - "strings" "sync" + "syscall" "time" - "github.com/jetkvm/kvm/resource" - - "github.com/pion/webrtc/v4/pkg/media" + "github.com/rs/zerolog" ) -var ctrlSocketConn net.Conn - -type CtrlAction struct { - Action string `json:"action"` - Seq int32 `json:"seq,omitempty"` - Params map[string]interface{} `json:"params,omitempty"` +type nativeOutput struct { + logger *zerolog.Logger } -type CtrlResponse struct { - Seq int32 `json:"seq,omitempty"` - Error string `json:"error,omitempty"` - Errno int32 `json:"errno,omitempty"` - Result map[string]interface{} `json:"result,omitempty"` - Event string `json:"event,omitempty"` - Data json.RawMessage `json:"data,omitempty"` +func (n *nativeOutput) Write(p []byte) (int, error) { + n.logger.Debug().Str("output", string(p)).Msg("native binary output") + return len(p), nil } -type EventHandler func(event CtrlResponse) - -var seq int32 = 1 - -var ongoingRequests = make(map[int32]chan *CtrlResponse) - -var lock = &sync.Mutex{} - var ( nativeCmd *exec.Cmd nativeCmdLock = &sync.Mutex{} ) -func CallCtrlAction(action string, params map[string]interface{}) (*CtrlResponse, error) { - lock.Lock() - defer lock.Unlock() - ctrlAction := CtrlAction{ - Action: action, - Seq: seq, - Params: params, +func startNativeBinary(binaryPath string) (*exec.Cmd, error) { + cmd := exec.Command(binaryPath) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, } + cmd.Stdout = &nativeOutput{logger: nativeLogger} + cmd.Stderr = &nativeOutput{logger: nativeLogger} - responseChan := make(chan *CtrlResponse) - ongoingRequests[seq] = responseChan - seq++ - - jsonData, err := json.Marshal(ctrlAction) + err := cmd.Start() if err != nil { - delete(ongoingRequests, ctrlAction.Seq) - return nil, fmt.Errorf("error marshaling ctrl action: %w", err) + return nil, err } - scopedLogger := nativeLogger.With(). - Str("action", ctrlAction.Action). - Interface("params", ctrlAction.Params).Logger() - - scopedLogger.Debug().Msg("sending ctrl action") - - err = WriteCtrlMessage(jsonData) - if err != nil { - delete(ongoingRequests, ctrlAction.Seq) - return nil, ErrorfL(&scopedLogger, "error writing ctrl message", err) - } - - select { - case response := <-responseChan: - delete(ongoingRequests, seq) - if response.Error != "" { - return nil, ErrorfL( - &scopedLogger, - "error native response: %s", - errors.New(response.Error), - ) - } - return response, nil - case <-time.After(5 * time.Second): - close(responseChan) - delete(ongoingRequests, seq) - return nil, ErrorfL(&scopedLogger, "timeout waiting for response", nil) - } -} - -func WriteCtrlMessage(message []byte) error { - if ctrlSocketConn == nil { - return fmt.Errorf("ctrl socket not conn ected") - } - _, err := ctrlSocketConn.Write(message) - return err -} - -var nativeCtrlSocketListener net.Listener //nolint:unused -var nativeVideoSocketListener net.Listener //nolint:unused - -var ctrlClientConnected = make(chan struct{}) - -func waitCtrlClientConnected() { - <-ctrlClientConnected -} - -func StartNativeSocketServer(socketPath string, handleClient func(net.Conn), isCtrl bool) net.Listener { - scopedLogger := nativeLogger.With(). - Str("socket_path", socketPath). - Logger() - - // Remove the socket file if it already exists - if _, err := os.Stat(socketPath); err == nil { - if err := os.Remove(socketPath); err != nil { - scopedLogger.Warn().Err(err).Msg("failed to remove existing socket file") - os.Exit(1) - } - } - - listener, err := net.Listen("unixpacket", socketPath) - if err != nil { - scopedLogger.Warn().Err(err).Msg("failed to start server") - os.Exit(1) - } - - scopedLogger.Info().Msg("server listening") - - go func() { - for { - conn, err := listener.Accept() - - if err != nil { - scopedLogger.Warn().Err(err).Msg("failed to accept socket") - continue - } - if isCtrl { - // check if the channel is closed - select { - case <-ctrlClientConnected: - scopedLogger.Debug().Msg("ctrl client reconnected") - default: - close(ctrlClientConnected) - scopedLogger.Debug().Msg("first native ctrl socket client connected") - } - } - - go handleClient(conn) - } - }() - - return listener -} - -func StartNativeCtrlSocketServer() { - nativeCtrlSocketListener = StartNativeSocketServer("/var/run/jetkvm_ctrl.sock", handleCtrlClient, true) - nativeLogger.Debug().Msg("native app ctrl sock started") -} - -func StartNativeVideoSocketServer() { - nativeVideoSocketListener = StartNativeSocketServer("/var/run/jetkvm_video.sock", handleVideoClient, false) - nativeLogger.Debug().Msg("native app video sock started") -} - -func handleCtrlClient(conn net.Conn) { - defer conn.Close() - - scopedLogger := nativeLogger.With(). - Str("addr", conn.RemoteAddr().String()). - Str("type", "ctrl"). - Logger() - - scopedLogger.Info().Msg("native ctrl socket client connected") - if ctrlSocketConn != nil { - scopedLogger.Debug().Msg("closing existing native socket connection") - ctrlSocketConn.Close() - } - - ctrlSocketConn = conn - - // Restore HDMI EDID if applicable - go restoreHdmiEdid() - - readBuf := make([]byte, 4096) - for { - n, err := conn.Read(readBuf) - if err != nil { - scopedLogger.Warn().Err(err).Msg("error reading from ctrl sock") - break - } - readMsg := string(readBuf[:n]) - - ctrlResp := CtrlResponse{} - err = json.Unmarshal([]byte(readMsg), &ctrlResp) - if err != nil { - scopedLogger.Warn().Err(err).Str("data", readMsg).Msg("error parsing ctrl sock msg") - continue - } - scopedLogger.Trace().Interface("data", ctrlResp).Msg("ctrl sock msg") - - if ctrlResp.Seq != 0 { - responseChan, ok := ongoingRequests[ctrlResp.Seq] - if ok { - responseChan <- &ctrlResp - } - } - switch ctrlResp.Event { - case "video_input_state": - HandleVideoStateMessage(ctrlResp) - } - } - - scopedLogger.Debug().Msg("ctrl sock disconnected") -} - -func handleVideoClient(conn net.Conn) { - defer conn.Close() - - scopedLogger := nativeLogger.With(). - Str("addr", conn.RemoteAddr().String()). - Str("type", "video"). - Logger() - - scopedLogger.Info().Msg("native video socket client connected") - - inboundPacket := make([]byte, maxFrameSize) - lastFrame := time.Now() - for { - n, err := conn.Read(inboundPacket) - if err != nil { - scopedLogger.Warn().Err(err).Msg("error during read") - return - } - now := time.Now() - sinceLastFrame := now.Sub(lastFrame) - lastFrame = now - if currentSession != nil { - err := currentSession.VideoTrack.WriteSample(media.Sample{Data: inboundPacket[:n], Duration: sinceLastFrame}) - if err != nil { - scopedLogger.Warn().Err(err).Msg("error writing sample") - } - } - } + return cmd, nil } func startNativeBinaryWithLock(binaryPath string) (*exec.Cmd, error) { @@ -351,87 +142,3 @@ func ExtractAndRunNativeBin() error { return nil } - -func shouldOverwrite(destPath string, srcHash []byte) bool { - if srcHash == nil { - nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, doing overwriting") - return true - } - - dstHash, err := os.ReadFile(destPath + ".sha256") - if err != nil { - nativeLogger.Debug().Msg("error reading existing jetkvm_native.sha256, doing overwriting") - return true - } - - return !bytes.Equal(srcHash, dstHash) -} - -func getNativeSha256() ([]byte, error) { - version, err := resource.ResourceFS.ReadFile("jetkvm_native.sha256") - if err != nil { - return nil, err - } - return version, nil -} - -func GetNativeVersion() (string, error) { - version, err := getNativeSha256() - if err != nil { - return "", err - } - return strings.TrimSpace(string(version)), nil -} - -func ensureBinaryUpdated(destPath string) error { - srcFile, err := resource.ResourceFS.Open("jetkvm_native") - if err != nil { - return err - } - defer srcFile.Close() - - srcHash, err := getNativeSha256() - if err != nil { - nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, proceeding with update") - srcHash = nil - } - - _, err = os.Stat(destPath) - if shouldOverwrite(destPath, srcHash) || err != nil { - nativeLogger.Info(). - Interface("hash", srcHash). - Msg("writing jetkvm_native") - - _ = os.Remove(destPath) - destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_RDWR, 0755) - if err != nil { - return err - } - _, err = io.Copy(destFile, srcFile) - destFile.Close() - if err != nil { - return err - } - if srcHash != nil { - err = os.WriteFile(destPath+".sha256", srcHash, 0644) - if err != nil { - return err - } - } - nativeLogger.Info().Msg("jetkvm_native updated") - } - - return nil -} - -// Restore the HDMI EDID value from the config. -// Called after successful connection to jetkvm_native. -func restoreHdmiEdid() { - if config.EdidString != "" { - nativeLogger.Info().Str("edid", config.EdidString).Msg("Restoring HDMI EDID") - _, err := CallCtrlAction("set_edid", map[string]interface{}{"edid": config.EdidString}) - if err != nil { - nativeLogger.Warn().Err(err).Msg("Failed to restore HDMI EDID") - } - } -} diff --git a/native_linux.go b/native_linux.go deleted file mode 100644 index 54d2150..0000000 --- a/native_linux.go +++ /dev/null @@ -1,57 +0,0 @@ -//go:build linux - -package kvm - -import ( - "fmt" - "os/exec" - "sync" - "syscall" - - "github.com/rs/zerolog" -) - -type nativeOutput struct { - mu *sync.Mutex - logger *zerolog.Event -} - -func (w *nativeOutput) Write(p []byte) (n int, err error) { - w.mu.Lock() - defer w.mu.Unlock() - - w.logger.Msg(string(p)) - return len(p), nil -} - -func startNativeBinary(binaryPath string) (*exec.Cmd, error) { - // Run the binary in the background - cmd := exec.Command(binaryPath) - - nativeOutputLock := sync.Mutex{} - nativeStdout := &nativeOutput{ - mu: &nativeOutputLock, - logger: nativeLogger.Info().Str("pipe", "stdout"), - } - nativeStderr := &nativeOutput{ - mu: &nativeOutputLock, - logger: nativeLogger.Info().Str("pipe", "stderr"), - } - - // Redirect stdout and stderr to the current process - cmd.Stdout = nativeStdout - cmd.Stderr = nativeStderr - - // Set the process group ID so we can kill the process and its children when this process exits - cmd.SysProcAttr = &syscall.SysProcAttr{ - Setpgid: true, - Pdeathsig: syscall.SIGKILL, - } - - // Start the command - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("failed to start binary: %w", err) - } - - return cmd, nil -} diff --git a/native_notlinux.go b/native_notlinux.go index df6df74..baadf34 100644 --- a/native_notlinux.go +++ b/native_notlinux.go @@ -8,5 +8,9 @@ import ( ) func startNativeBinary(binaryPath string) (*exec.Cmd, error) { - return nil, fmt.Errorf("not supported") + return nil, fmt.Errorf("startNativeBinary is only supported on Linux") } + +func ExtractAndRunNativeBin() error { + return fmt.Errorf("ExtractAndRunNativeBin is only supported on Linux") +} \ No newline at end of file diff --git a/native_shared.go b/native_shared.go new file mode 100644 index 0000000..f7784f0 --- /dev/null +++ b/native_shared.go @@ -0,0 +1,330 @@ +package kvm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "sync" + "time" + + "github.com/jetkvm/kvm/resource" + "github.com/pion/webrtc/v4/pkg/media" +) + +type CtrlAction struct { + Action string `json:"action"` + Seq int32 `json:"seq,omitempty"` + Params map[string]interface{} `json:"params,omitempty"` +} + +type CtrlResponse struct { + Seq int32 `json:"seq,omitempty"` + Error string `json:"error,omitempty"` + Errno int32 `json:"errno,omitempty"` + Result map[string]interface{} `json:"result,omitempty"` + Event string `json:"event,omitempty"` + Data json.RawMessage `json:"data,omitempty"` +} + +type EventHandler func(event CtrlResponse) + +var seq int32 = 1 + +var ongoingRequests = make(map[int32]chan *CtrlResponse) + +var lock = &sync.Mutex{} + +var ctrlSocketConn net.Conn + +var nativeCtrlSocketListener net.Listener //nolint:unused +var nativeVideoSocketListener net.Listener //nolint:unused + +var ctrlClientConnected = make(chan struct{}) + +func waitCtrlClientConnected() { + <-ctrlClientConnected +} + +func CallCtrlAction(action string, params map[string]interface{}) (*CtrlResponse, error) { + lock.Lock() + defer lock.Unlock() + ctrlAction := CtrlAction{ + Action: action, + Seq: seq, + Params: params, + } + + responseChan := make(chan *CtrlResponse) + ongoingRequests[seq] = responseChan + seq++ + + jsonData, err := json.Marshal(ctrlAction) + if err != nil { + delete(ongoingRequests, ctrlAction.Seq) + return nil, fmt.Errorf("error marshaling ctrl action: %w", err) + } + + scopedLogger := nativeLogger.With(). + Str("action", ctrlAction.Action). + Interface("params", ctrlAction.Params).Logger() + + scopedLogger.Debug().Msg("sending ctrl action") + + err = WriteCtrlMessage(jsonData) + if err != nil { + delete(ongoingRequests, ctrlAction.Seq) + return nil, ErrorfL(&scopedLogger, "error writing ctrl message", err) + } + + select { + case response := <-responseChan: + delete(ongoingRequests, seq) + if response.Error != "" { + return nil, ErrorfL( + &scopedLogger, + "error native response: %s", + errors.New(response.Error), + ) + } + return response, nil + case <-time.After(5 * time.Second): + close(responseChan) + delete(ongoingRequests, seq) + return nil, ErrorfL(&scopedLogger, "timeout waiting for response", nil) + } +} + +func WriteCtrlMessage(message []byte) error { + if ctrlSocketConn == nil { + return fmt.Errorf("ctrl socket not connected") + } + _, err := ctrlSocketConn.Write(message) + return err +} + +func StartNativeSocketServer(socketPath string, handleClient func(net.Conn), isCtrl bool) net.Listener { + scopedLogger := nativeLogger.With(). + Str("socket_path", socketPath). + Logger() + + // Remove the socket file if it already exists + if _, err := os.Stat(socketPath); err == nil { + if err := os.Remove(socketPath); err != nil { + scopedLogger.Warn().Err(err).Msg("failed to remove existing socket file") + os.Exit(1) + } + } + + listener, err := net.Listen("unixpacket", socketPath) + if err != nil { + scopedLogger.Warn().Err(err).Msg("failed to start server") + os.Exit(1) + } + + scopedLogger.Info().Msg("server listening") + + go func() { + for { + conn, err := listener.Accept() + + if err != nil { + scopedLogger.Warn().Err(err).Msg("failed to accept socket") + continue + } + if isCtrl { + // check if the channel is closed + select { + case <-ctrlClientConnected: + scopedLogger.Debug().Msg("ctrl client reconnected") + default: + close(ctrlClientConnected) + scopedLogger.Debug().Msg("first native ctrl socket client connected") + } + } + + go handleClient(conn) + } + }() + + return listener +} + +func StartNativeCtrlSocketServer() { + nativeCtrlSocketListener = StartNativeSocketServer("/var/run/jetkvm_ctrl.sock", handleCtrlClient, true) + nativeLogger.Debug().Msg("native app ctrl sock started") +} + +func StartNativeVideoSocketServer() { + nativeVideoSocketListener = StartNativeSocketServer("/var/run/jetkvm_video.sock", handleVideoClient, false) + nativeLogger.Debug().Msg("native app video sock started") +} + +func handleCtrlClient(conn net.Conn) { + defer conn.Close() + + scopedLogger := nativeLogger.With(). + Str("addr", conn.RemoteAddr().String()). + Str("type", "ctrl"). + Logger() + + scopedLogger.Info().Msg("native ctrl socket client connected") + if ctrlSocketConn != nil { + scopedLogger.Debug().Msg("closing existing native socket connection") + ctrlSocketConn.Close() + } + + ctrlSocketConn = conn + + // Restore HDMI EDID if applicable + go restoreHdmiEdid() + + readBuf := make([]byte, 4096) + for { + n, err := conn.Read(readBuf) + if err != nil { + scopedLogger.Warn().Err(err).Msg("error reading from ctrl sock") + break + } + readMsg := string(readBuf[:n]) + + ctrlResp := CtrlResponse{} + err = json.Unmarshal([]byte(readMsg), &ctrlResp) + if err != nil { + scopedLogger.Warn().Err(err).Str("data", readMsg).Msg("error parsing ctrl sock msg") + continue + } + scopedLogger.Trace().Interface("data", ctrlResp).Msg("ctrl sock msg") + + if ctrlResp.Seq != 0 { + responseChan, ok := ongoingRequests[ctrlResp.Seq] + if ok { + responseChan <- &ctrlResp + } + } + switch ctrlResp.Event { + case "video_input_state": + HandleVideoStateMessage(ctrlResp) + } + } + + scopedLogger.Debug().Msg("ctrl sock disconnected") +} + +func handleVideoClient(conn net.Conn) { + defer conn.Close() + + scopedLogger := nativeLogger.With(). + Str("addr", conn.RemoteAddr().String()). + Str("type", "video"). + Logger() + + scopedLogger.Info().Msg("native video socket client connected") + + inboundPacket := make([]byte, maxVideoFrameSize) + lastFrame := time.Now() + for { + n, err := conn.Read(inboundPacket) + if err != nil { + scopedLogger.Warn().Err(err).Msg("error during read") + return + } + now := time.Now() + sinceLastFrame := now.Sub(lastFrame) + lastFrame = now + if currentSession != nil { + err := currentSession.VideoTrack.WriteSample(media.Sample{Data: inboundPacket[:n], Duration: sinceLastFrame}) + if err != nil { + scopedLogger.Warn().Err(err).Msg("error writing sample") + } + } + } +} + +func shouldOverwrite(destPath string, srcHash []byte) bool { + if srcHash == nil { + nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, doing overwriting") + return true + } + + dstHash, err := os.ReadFile(destPath + ".sha256") + if err != nil { + nativeLogger.Debug().Msg("error reading existing jetkvm_native.sha256, doing overwriting") + return true + } + + return !bytes.Equal(srcHash, dstHash) +} + +func getNativeSha256() ([]byte, error) { + version, err := resource.ResourceFS.ReadFile("jetkvm_native.sha256") + if err != nil { + return nil, err + } + return version, nil +} + +func GetNativeVersion() (string, error) { + version, err := getNativeSha256() + if err != nil { + return "", err + } + return strings.TrimSpace(string(version)), nil +} + +func ensureBinaryUpdated(destPath string) error { + srcFile, err := resource.ResourceFS.Open("jetkvm_native") + if err != nil { + return err + } + defer srcFile.Close() + + srcHash, err := getNativeSha256() + if err != nil { + nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, proceeding with update") + srcHash = nil + } + + _, err = os.Stat(destPath) + if shouldOverwrite(destPath, srcHash) || err != nil { + nativeLogger.Info(). + Interface("hash", srcHash). + Msg("writing jetkvm_native") + + _ = os.Remove(destPath) + destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_RDWR, 0755) + if err != nil { + return err + } + _, err = io.Copy(destFile, srcFile) + destFile.Close() + if err != nil { + return err + } + if srcHash != nil { + err = os.WriteFile(destPath+".sha256", srcHash, 0644) + if err != nil { + return err + } + } + nativeLogger.Info().Msg("jetkvm_native updated") + } + + return nil +} + +// Restore the HDMI EDID value from the config. +// Called after successful connection to jetkvm_native. +func restoreHdmiEdid() { + if config.EdidString != "" { + nativeLogger.Info().Str("edid", config.EdidString).Msg("Restoring HDMI EDID") + _, err := CallCtrlAction("set_edid", map[string]interface{}{"edid": config.EdidString}) + if err != nil { + nativeLogger.Warn().Err(err).Msg("Failed to restore HDMI EDID") + } + } +} diff --git a/tools/build_audio_deps.sh b/tools/build_audio_deps.sh new file mode 100644 index 0000000..e09cb6f --- /dev/null +++ b/tools/build_audio_deps.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# tools/build_audio_deps.sh +# Build ALSA and Opus static libs for ARM in $HOME/.jetkvm/audio-libs +set -e +JETKVM_HOME="$HOME/.jetkvm" +AUDIO_LIBS_DIR="$JETKVM_HOME/audio-libs" +TOOLCHAIN_DIR="$JETKVM_HOME/rv1106-system" +CROSS_PREFIX="$TOOLCHAIN_DIR/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf" + +mkdir -p "$AUDIO_LIBS_DIR" +cd "$AUDIO_LIBS_DIR" + +# Download sources +[ -f alsa-lib-1.2.14.tar.bz2 ] || wget -N https://www.alsa-project.org/files/pub/lib/alsa-lib-1.2.14.tar.bz2 +[ -f opus-1.5.2.tar.gz ] || wget -N https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz + +# Extract +[ -d alsa-lib-1.2.14 ] || tar xf alsa-lib-1.2.14.tar.bz2 +[ -d opus-1.5.2 ] || tar xf opus-1.5.2.tar.gz + +export CC="${CROSS_PREFIX}-gcc" + +# Build ALSA +cd alsa-lib-1.2.14 +if [ ! -f .built ]; then + ./configure --host arm-rockchip830-linux-uclibcgnueabihf --enable-static=yes --enable-shared=no --with-pcm-plugins=rate,linear --disable-seq --disable-rawmidi --disable-ucm + make -j$(nproc) + touch .built +fi +cd .. + +# Build Opus +cd opus-1.5.2 +if [ ! -f .built ]; then + ./configure --host arm-rockchip830-linux-uclibcgnueabihf --enable-static=yes --enable-shared=no --enable-fixed-point + make -j$(nproc) + touch .built +fi +cd .. + +echo "ALSA and Opus built in $AUDIO_LIBS_DIR" diff --git a/tools/setup_rv1106_toolchain.sh b/tools/setup_rv1106_toolchain.sh new file mode 100644 index 0000000..43e675b --- /dev/null +++ b/tools/setup_rv1106_toolchain.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# tools/setup_rv1106_toolchain.sh +# Clone the rv1106-system toolchain to $HOME/.jetkvm/rv1106-system if not already present +set -e +JETKVM_HOME="$HOME/.jetkvm" +TOOLCHAIN_DIR="$JETKVM_HOME/rv1106-system" +REPO_URL="https://github.com/jetkvm/rv1106-system.git" + +mkdir -p "$JETKVM_HOME" +if [ ! -d "$TOOLCHAIN_DIR" ]; then + echo "Cloning rv1106-system toolchain to $TOOLCHAIN_DIR ..." + git clone --depth 1 "$REPO_URL" "$TOOLCHAIN_DIR" +else + echo "Toolchain already present at $TOOLCHAIN_DIR" +fi diff --git a/ui/src/components/ActionBar.tsx b/ui/src/components/ActionBar.tsx index 801cc7a..409387e 100644 --- a/ui/src/components/ActionBar.tsx +++ b/ui/src/components/ActionBar.tsx @@ -1,8 +1,8 @@ -import { MdOutlineContentPasteGo } from "react-icons/md"; +import { MdOutlineContentPasteGo, MdVolumeOff, MdVolumeUp, MdGraphicEq } from "react-icons/md"; import { LuCable, LuHardDrive, LuMaximize, LuSettings, LuSignal } from "react-icons/lu"; import { FaKeyboard } from "react-icons/fa6"; import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react"; -import { Fragment, useCallback, useRef } from "react"; +import { Fragment, useCallback, useEffect, useRef, useState } from "react"; import { CommandLineIcon } from "@heroicons/react/20/solid"; import { Button } from "@components/Button"; @@ -18,7 +18,9 @@ import PasteModal from "@/components/popovers/PasteModal"; import WakeOnLanModal from "@/components/popovers/WakeOnLan/Index"; import MountPopopover from "@/components/popovers/MountPopover"; import ExtensionPopover from "@/components/popovers/ExtensionPopover"; +import AudioControlPopover from "@/components/popovers/AudioControlPopover"; import { useDeviceUiNavigation } from "@/hooks/useAppNavigation"; +import api from "@/api"; export default function Actionbar({ requestFullscreen, @@ -56,6 +58,28 @@ export default function Actionbar({ [setDisableFocusTrap], ); + // Mute/unmute state for button display + const [isMuted, setIsMuted] = useState(false); + useEffect(() => { + api.GET("/audio/mute").then(async resp => { + if (resp.ok) { + const data = await resp.json(); + setIsMuted(!!data.muted); + } + }); + + // Refresh mute state periodically for button display + const interval = setInterval(async () => { + const resp = await api.GET("/audio/mute"); + if (resp.ok) { + const data = await resp.json(); + setIsMuted(!!data.muted); + } + }, 1000); + + return () => clearInterval(interval); + }, []); + return (
+
+ + + + ))} + + + {currentConfig && ( +
+
+ Sample Rate: {currentConfig.SampleRate}Hz + Channels: {currentConfig.Channels} + Bitrate: {currentConfig.Bitrate}kbps + Frame: {currentConfig.FrameSize} +
+
+ )} + + + {/* Advanced Controls Toggle */} + + + {/* Advanced Metrics */} + {showAdvanced && ( +
+
+ + + Performance Metrics + +
+ + {metrics ? ( + <> +
+
+
Frames Received
+
+ {formatNumber(metrics.frames_received)} +
+
+ +
+
Frames Dropped
+
0 + ? "text-red-600 dark:text-red-400" + : "text-green-600 dark:text-green-400" + )}> + {formatNumber(metrics.frames_dropped)} +
+
+ +
+
Data Processed
+
+ {formatBytes(metrics.bytes_processed)} +
+
+ +
+
Connection Drops
+
0 + ? "text-red-600 dark:text-red-400" + : "text-green-600 dark:text-green-400" + )}> + {formatNumber(metrics.connection_drops)} +
+
+
+ + {metrics.frames_received > 0 && ( +
+
Drop Rate
+
5 + ? "text-red-600 dark:text-red-400" + : ((metrics.frames_dropped / metrics.frames_received) * 100) > 1 + ? "text-yellow-600 dark:text-yellow-400" + : "text-green-600 dark:text-green-400" + )}> + {((metrics.frames_dropped / metrics.frames_received) * 100).toFixed(2)}% +
+
+ )} + +
+ Last updated: {new Date().toLocaleTimeString()} +
+ + ) : ( +
+
+ Loading metrics... +
+
+ )} +
+ )} + + {/* Audio Metrics Dashboard Button */} +
+
+ +
+
+ + + ); +} \ No newline at end of file diff --git a/ui/src/components/sidebar/AudioMetricsSidebar.tsx b/ui/src/components/sidebar/AudioMetricsSidebar.tsx new file mode 100644 index 0000000..a07ad0f --- /dev/null +++ b/ui/src/components/sidebar/AudioMetricsSidebar.tsx @@ -0,0 +1,16 @@ +import SidebarHeader from "@/components/SidebarHeader"; +import { useUiStore } from "@/hooks/stores"; +import AudioMetricsDashboard from "@/components/AudioMetricsDashboard"; + +export default function AudioMetricsSidebar() { + const setSidebarView = useUiStore(state => state.setSidebarView); + + return ( + <> + +
+ +
+ + ); +} \ No newline at end of file diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index aa29528..1a1f6b6 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -38,7 +38,7 @@ const appendStatToMap = ( }; // Constants and types -export type AvailableSidebarViews = "connection-stats"; +export type AvailableSidebarViews = "connection-stats" | "audio-metrics"; export type AvailableTerminalTypes = "kvm" | "serial" | "none"; export interface User { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 1785bcd..3b90090 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -37,6 +37,7 @@ import WebRTCVideo from "@components/WebRTCVideo"; import { checkAuth, isInCloud, isOnDevice } from "@/main"; import DashboardNavbar from "@components/Header"; import ConnectionStatsSidebar from "@/components/sidebar/connectionStats"; +import AudioMetricsSidebar from "@/components/sidebar/AudioMetricsSidebar"; import { JsonRpcRequest, useJsonRpc } from "@/hooks/useJsonRpc"; import Terminal from "@components/Terminal"; import { CLOUD_API, DEVICE_API } from "@/ui.config"; @@ -479,6 +480,8 @@ export default function KvmIdRoute() { }; setTransceiver(pc.addTransceiver("video", { direction: "recvonly" })); + // Add audio transceiver to receive audio from the server + pc.addTransceiver("audio", { direction: "recvonly" }); const rpcDataChannel = pc.createDataChannel("rpc"); rpcDataChannel.onopen = () => { @@ -900,6 +903,22 @@ function SidebarContainer(props: SidebarContainerProps) { )} + {sidebarView === "audio-metrics" && ( + +
+ +
+ + )} diff --git a/video.go b/video.go index 6fa77b9..b8bf5e5 100644 --- a/video.go +++ b/video.go @@ -5,7 +5,8 @@ import ( ) // max frame size for 1080p video, specified in mpp venc setting -const maxFrameSize = 1920 * 1080 / 2 +const maxVideoFrameSize = 1920 * 1080 / 2 +const maxAudioFrameSize = 1500 func writeCtrlAction(action string) error { actionMessage := map[string]string{ diff --git a/web.go b/web.go index 21e17e7..5a0a4e9 100644 --- a/web.go +++ b/web.go @@ -14,8 +14,11 @@ import ( "strings" "time" + "github.com/jetkvm/kvm/internal/audio" + "github.com/coder/websocket" "github.com/coder/websocket/wsjson" + gin_logger "github.com/gin-contrib/logger" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -156,6 +159,67 @@ func setupRouter() *gin.Engine { protected.POST("/storage/upload", handleUploadHttp) } + protected.GET("/audio/mute", func(c *gin.Context) { + c.JSON(200, gin.H{"muted": audio.IsAudioMuted()}) + }) + + protected.POST("/audio/mute", func(c *gin.Context) { + type muteReq struct { + Muted bool `json:"muted"` + } + var req muteReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(400, gin.H{"error": "invalid request"}) + return + } + audio.SetAudioMuted(req.Muted) + c.JSON(200, gin.H{"muted": req.Muted}) + }) + + protected.GET("/audio/quality", func(c *gin.Context) { + config := audio.GetAudioConfig() + presets := audio.GetAudioQualityPresets() + c.JSON(200, gin.H{ + "current": config, + "presets": presets, + }) + }) + + protected.POST("/audio/quality", func(c *gin.Context) { + type qualityReq struct { + Quality int `json:"quality"` + } + var req qualityReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(400, gin.H{"error": "invalid request"}) + return + } + + // Validate quality level + if req.Quality < 0 || req.Quality > 3 { + c.JSON(400, gin.H{"error": "invalid quality level (0-3)"}) + return + } + + audio.SetAudioQuality(audio.AudioQuality(req.Quality)) + c.JSON(200, gin.H{ + "quality": req.Quality, + "config": audio.GetAudioConfig(), + }) + }) + + protected.GET("/audio/metrics", func(c *gin.Context) { + metrics := audio.GetAudioMetrics() + c.JSON(200, gin.H{ + "frames_received": metrics.FramesReceived, + "frames_dropped": metrics.FramesDropped, + "bytes_processed": metrics.BytesProcessed, + "last_frame_time": metrics.LastFrameTime, + "connection_drops": metrics.ConnectionDrops, + "average_latency": metrics.AverageLatency.String(), + }) + }) + // Catch-all route for SPA r.NoRoute(func(c *gin.Context) { if c.Request.Method == "GET" && c.NegotiateFormat(gin.MIMEHTML) == gin.MIMEHTML { diff --git a/webrtc.go b/webrtc.go index f6c8529..f14b72a 100644 --- a/webrtc.go +++ b/webrtc.go @@ -18,6 +18,7 @@ import ( type Session struct { peerConnection *webrtc.PeerConnection VideoTrack *webrtc.TrackLocalStaticSample + AudioTrack *webrtc.TrackLocalStaticSample ControlChannel *webrtc.DataChannel RPCChannel *webrtc.DataChannel HidChannel *webrtc.DataChannel @@ -136,7 +137,17 @@ func newSession(config SessionConfig) (*Session, error) { return nil, err } - rtpSender, err := peerConnection.AddTrack(session.VideoTrack) + session.AudioTrack, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "kvm") + if err != nil { + return nil, err + } + + videoRtpSender, err := peerConnection.AddTrack(session.VideoTrack) + if err != nil { + return nil, err + } + + audioRtpSender, err := peerConnection.AddTrack(session.AudioTrack) if err != nil { return nil, err } @@ -144,14 +155,9 @@ func newSession(config SessionConfig) (*Session, error) { // Read incoming RTCP packets // Before these packets are returned they are processed by interceptors. For things // like NACK this needs to be called. - go func() { - rtcpBuf := make([]byte, 1500) - for { - if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil { - return - } - } - }() + go drainRtpSender(videoRtpSender) + go drainRtpSender(audioRtpSender) + var isConnected bool peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { @@ -203,6 +209,15 @@ func newSession(config SessionConfig) (*Session, error) { return session, nil } +func drainRtpSender(rtpSender *webrtc.RTPSender) { + rtcpBuf := make([]byte, 1500) + for { + if _, _, err := rtpSender.Read(rtcpBuf); err != nil { + return + } + } +} + var actionSessions = 0 func onActiveSessionsChanged() {