This commit is contained in:
Alex 2025-08-24 23:58:53 +00:00 committed by GitHub
commit a7a9704fde
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
80 changed files with 13994 additions and 482 deletions

View File

@ -1,10 +1,15 @@
{
"name": "JetKVM",
"image": "mcr.microsoft.com/devcontainers/go:1-1.23-bookworm",
"image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04",
"runArgs": ["--platform=linux/amd64" ],
"features": {
"ghcr.io/devcontainers/features/node:1": {
// Should match what is defined in ui/package.json
"version": "22.15.0"
},
"ghcr.io/devcontainers/features/go:1": {
// Should match what is defined in go.mod
"version": "latest"
}
},
"mounts": [

View File

@ -27,11 +27,64 @@ jobs:
uses: actions/setup-go@fa96338abe5531f6e34c5cc0bbe28c1a533d5505 # v4.2.1
with:
go-version: 1.24.4
- name: Setup build environment variables
id: build-env
run: |
# Extract versions from Makefile
ALSA_VERSION=$(grep '^ALSA_VERSION' Makefile | cut -d'=' -f2 | tr -d ' ')
OPUS_VERSION=$(grep '^OPUS_VERSION' Makefile | cut -d'=' -f2 | tr -d ' ')
# Get rv1106-system latest commit
RV1106_COMMIT=$(git ls-remote https://github.com/jetkvm/rv1106-system.git HEAD | cut -f1)
# Set environment variables
echo "ALSA_VERSION=$ALSA_VERSION" >> $GITHUB_ENV
echo "OPUS_VERSION=$OPUS_VERSION" >> $GITHUB_ENV
echo "RV1106_COMMIT=$RV1106_COMMIT" >> $GITHUB_ENV
# Set outputs for use in other steps
echo "alsa_version=$ALSA_VERSION" >> $GITHUB_OUTPUT
echo "opus_version=$OPUS_VERSION" >> $GITHUB_OUTPUT
echo "rv1106_commit=$RV1106_COMMIT" >> $GITHUB_OUTPUT
# Set resolved cache path
CACHE_PATH="$HOME/.jetkvm/audio-libs"
echo "CACHE_PATH=$CACHE_PATH" >> $GITHUB_ENV
echo "cache_path=$CACHE_PATH" >> $GITHUB_OUTPUT
echo "Extracted ALSA version: $ALSA_VERSION"
echo "Extracted Opus version: $OPUS_VERSION"
echo "Latest rv1106-system commit: $RV1106_COMMIT"
echo "Cache path: $CACHE_PATH"
- name: Restore audio dependencies cache
id: cache-audio-deps
uses: actions/cache/restore@v4
with:
path: ${{ steps.build-env.outputs.cache_path }}
key: audio-deps-${{ runner.os }}-alsa-${{ steps.build-env.outputs.alsa_version }}-opus-${{ steps.build-env.outputs.opus_version }}-rv1106-${{ steps.build-env.outputs.rv1106_commit }}
- name: Setup development environment
if: steps.cache-audio-deps.outputs.cache-hit != 'true'
run: make dev_env
env:
ALSA_VERSION: ${{ env.ALSA_VERSION }}
OPUS_VERSION: ${{ env.OPUS_VERSION }}
- name: Create empty resource directory
run: |
mkdir -p static && touch static/.gitkeep
- name: Save audio dependencies cache
if: always() && steps.cache-audio-deps.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: ${{ steps.build-env.outputs.cache_path }}
key: ${{ steps.cache-audio-deps.outputs.cache-primary-key }}
- name: Lint
uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0
with:
args: --verbose
version: v2.0.2
env:
CGO_ENABLED: 1
ALSA_VERSION: ${{ env.ALSA_VERSION }}
OPUS_VERSION: ${{ env.OPUS_VERSION }}
CGO_CFLAGS: "-I${{ steps.build-env.outputs.cache_path }}/alsa-lib-${{ steps.build-env.outputs.alsa_version }}/include -I${{ steps.build-env.outputs.cache_path }}/opus-${{ steps.build-env.outputs.opus_version }}/include -I${{ steps.build-env.outputs.cache_path }}/opus-${{ steps.build-env.outputs.opus_version }}/celt"
CGO_LDFLAGS: "-L${{ steps.build-env.outputs.cache_path }}/alsa-lib-${{ steps.build-env.outputs.alsa_version }}/src/.libs -lasound -L${{ steps.build-env.outputs.cache_path }}/opus-${{ steps.build-env.outputs.opus_version }}/.libs -lopus -lm -ldl -static"

6
.gitignore vendored
View File

@ -1,6 +1,12 @@
bin/*
static/*
.vscode/
tmp/
.devcontainer/devcontainer-lock.json
.idea
.DS_Store
*.log
*.tmp
*.code-workspace
device-tests.tar.gz

View File

@ -1,4 +1,7 @@
version: "2"
run:
build-tags:
- nolint
linters:
enable:
- forbidigo

View File

@ -11,21 +11,39 @@
</div>
# JetKVM Development Guide
Welcome to JetKVM development! This guide will help you get started quickly, whether you're fixing bugs, adding features, or just exploring the codebase.
## Get Started
### Prerequisites
- **A JetKVM device** (for full development)
- **[Go 1.24.4+](https://go.dev/doc/install)** and **[Node.js 22.15.0](https://nodejs.org/en/download/)**
- **[Git](https://git-scm.com/downloads)** for version control
- **[SSH access](https://jetkvm.com/docs/advanced-usage/developing#developer-mode)** to your JetKVM device
- **Audio build dependencies:**
- **New:** The audio system uses a dual-subprocess architecture with CGO, ALSA, and Opus integration. You must run the provided scripts in `tools/` to set up the cross-compiler and build static ALSA/Opus libraries for ARM. See below.
### Development Environment
**Recommended:** Development is best done on **Linux** or **macOS**.
**Recommended:** Development is best done on **Linux** or **macOS**.
#### Apple Silicon (M1/M2/M3) Mac Users
If you are developing on an Apple Silicon Mac, you should use a devcontainer to ensure compatibility with the JetKVM build environment (which targets linux/amd64 and ARM). There are two main options:
- **VS Code Dev Containers**: Open the project in VS Code and use the built-in Dev Containers support. The configuration is in `.devcontainer/devcontainer.json`.
- **Devpod**: [Devpod](https://devpod.sh/) is a fast, open-source tool for running devcontainers anywhere. If you use Devpod, go to **Settings → Experimental → Additional Environmental Variables** and add:
- `DOCKER_DEFAULT_PLATFORM=linux/amd64`
This ensures all builds run in the correct architecture.
- **devcontainer CLI**: You can also use the [devcontainer CLI](https://github.com/devcontainers/cli) to launch the devcontainer from the terminal.
This approach ensures compatibility with all shell scripts, build tools, and cross-compilation steps used in the project.
If you're using Windows, we strongly recommend using **WSL (Windows Subsystem for Linux)** for the best development experience:
- [Install WSL on Windows](https://docs.microsoft.com/en-us/windows/wsl/install)
@ -33,6 +51,7 @@ If you're using Windows, we strongly recommend using **WSL (Windows Subsystem fo
This ensures compatibility with shell scripts and build tools used in the project.
### Project Setup
1. **Clone the repository:**
@ -46,16 +65,25 @@ This ensures compatibility with shell scripts and build tools used in the projec
go version && node --version
```
3. **Find your JetKVM IP address** (check your router or device screen)
3. **Set up the cross-compiler and audio dependencies:**
```bash
make dev_env
# This will run tools/setup_rv1106_toolchain.sh and tools/build_audio_deps.sh
# It will clone the cross-compiler and build ALSA/Opus static libs in $HOME/.jetkvm
#
# **Note:** This is required for the audio subprocess architecture. If you skip this step, builds will not succeed.
```
4. **Deploy and test:**
4. **Find your JetKVM IP address** (check your router or device screen)
5. **Deploy and test:**
```bash
./dev_deploy.sh -r 192.168.1.100 # Replace with your device IP
```
5. **Open in browser:** `http://192.168.1.100`
6. **Open in browser:** `http://192.168.1.100`
That's it! You're now running your own development version of JetKVM.
That's it! You're now running your own development version of JetKVM, **with bidirectional audio streaming using the dual-subprocess architecture.**
---
@ -71,13 +99,15 @@ npm install
Now edit files in `ui/src/` and see changes live in your browser!
### Modify the backend
### Modify the backend (including audio)
```bash
# Edit Go files (config.go, web.go, etc.)
# Edit Go files (config.go, web.go, internal/audio, etc.)
./dev_deploy.sh -r 192.168.1.100 --skip-ui-build
```
### Run tests
```bash
@ -93,21 +123,26 @@ tail -f /var/log/jetkvm.log
---
## Project Layout
```
/kvm/
├── main.go # App entry point
├── config.go # Settings & configuration
├── web.go # API endpoints
├── ui/ # React frontend
│ ├── src/routes/ # Pages (login, settings, etc.)
│ └── src/components/ # UI components
└── internal/ # Internal Go packages
├── config.go # Settings & configuration
├── web.go # API endpoints
├── ui/ # React frontend
│ ├── src/routes/ # Pages (login, settings, etc.)
│ └── src/components/ # UI components
├── internal/ # Internal Go packages
│ └── audio/ # Dual-subprocess audio architecture (CGO, ALSA, Opus) [NEW]
├── tools/ # Toolchain and audio dependency setup scripts
└── Makefile # Build and dev automation (see audio targets)
```
**Key files for beginners:**
- `internal/audio/` - [NEW] Dual-subprocess audio architecture (CGO, ALSA, Opus)
- `web.go` - Add new API endpoints here
- `config.go` - Add new settings here
- `ui/src/routes/` - Add new pages here
@ -136,9 +171,10 @@ npm install
./dev_device.sh <YOUR_DEVICE_IP>
```
### Quick Backend Changes
*Best for: API or backend logic changes*
*Best for: API, backend, or audio logic changes (including audio subprocess architecture)*
```bash
# Skip frontend build for faster deployment
@ -195,6 +231,103 @@ systemctl restart jetkvm
cd ui && npm run lint
```
### Essential Makefile Targets
The project includes several essential Makefile targets for development environment setup, building, and code quality:
#### Development Environment Setup
```bash
# Set up complete development environment (recommended first step)
make dev_env
# This runs setup_toolchain + build_audio_deps + installs Go tools
# - Clones rv1106-system toolchain to $HOME/.jetkvm/rv1106-system
# - Builds ALSA and Opus static libraries for ARM
# - Installs goimports and other Go development tools
# Set up only the cross-compiler toolchain
make setup_toolchain
# Build only the audio dependencies (requires setup_toolchain)
make build_audio_deps
```
#### Building
```bash
# Build development version with debug symbols
make build_dev
# Builds jetkvm_app with version like 0.4.7-dev20241222
# Requires: make dev_env (for toolchain and audio dependencies)
# Build release version (production)
make build_release
# Builds optimized release version
# Requires: make dev_env and frontend build
# Build test binaries for device testing
make build_dev_test
# Creates device-tests.tar.gz with all test binaries
```
#### Code Quality and Linting
```bash
# Run both Go and UI linting
make lint
# Run both Go and UI linting with auto-fix
make lint-fix
# Run only Go linting
make lint-go
# Run only Go linting with auto-fix
make lint-go-fix
# Run only UI linting
make lint-ui
# Run only UI linting with auto-fix
make lint-ui-fix
```
**Note:** The Go linting targets (`lint-go`, `lint-go-fix`, and the combined `lint`/`lint-fix` targets) require audio dependencies. Run `make dev_env` first if you haven't already.
### Development Deployment Script
The `dev_deploy.sh` script is the primary tool for deploying your development changes to a JetKVM device:
```bash
# Basic deployment (builds and deploys everything)
./dev_deploy.sh -r 192.168.1.100
# Skip UI build for faster backend-only deployment
./dev_deploy.sh -r 192.168.1.100 --skip-ui-build
# Run Go tests on the device after deployment
./dev_deploy.sh -r 192.168.1.100 --run-go-tests
# Deploy with release build and install
./dev_deploy.sh -r 192.168.1.100 -i
# View all available options
./dev_deploy.sh --help
```
**Key features:**
- Automatically builds the Go backend with proper cross-compilation
- Optionally builds the React frontend (unless `--skip-ui-build`)
- Deploys binaries to the device via SSH/SCP
- Restarts the JetKVM service
- Can run tests on the device
- Supports custom SSH user and various deployment options
**Requirements:**
- SSH access to your JetKVM device
- `make dev_env` must be run first (for toolchain and audio dependencies)
- Device IP address or hostname
### API Testing
```bash
@ -206,7 +339,8 @@ curl -X POST http://<IP>/auth/password-local \
---
## Common Issues & Solutions
### Common Issues & Solutions
### "Build failed" or "Permission denied"
@ -218,6 +352,8 @@ ssh root@<IP> chmod +x /userdata/jetkvm/bin/jetkvm_app_debug
go clean -modcache
go mod tidy
make build_dev
# If you see errors about missing ALSA/Opus or toolchain, run:
make dev_env # Required for audio subprocess architecture
```
### "Can't connect to device"
@ -230,6 +366,15 @@ ping <IP>
ssh root@<IP> echo "Connection OK"
```
### "Audio not working"
```bash
# Make sure you have run:
make dev_env
# If you see errors about ALSA/Opus, check logs and re-run the setup scripts in tools/.
```
### "Frontend not updating"
```bash
@ -244,18 +389,21 @@ npm install
## Next Steps
### Adding a New Feature
1. **Backend:** Add API endpoint in `web.go`
1. **Backend:** Add API endpoint in `web.go` or extend audio in `internal/audio/`
2. **Config:** Add settings in `config.go`
3. **Frontend:** Add UI in `ui/src/routes/`
4. **Test:** Deploy and test with `./dev_deploy.sh`
### Code Style
- **Go:** Follow standard Go conventions
- **TypeScript:** Use TypeScript for type safety
- **React:** Keep components small and reusable
- **Audio/CGO:** Keep C/Go integration minimal, robust, and well-documented. Use zerolog for all logging.
### Environment Variables

View File

@ -1,3 +1,22 @@
# --- JetKVM Audio/Toolchain Dev Environment Setup ---
.PHONY: setup_toolchain build_audio_deps dev_env lint lint-go lint-ui lint-fix lint-go-fix lint-ui-fix ui-lint
# Clone the rv1106-system toolchain to $HOME/.jetkvm/rv1106-system
setup_toolchain:
bash tools/setup_rv1106_toolchain.sh
# Build ALSA and Opus static libs for ARM in $HOME/.jetkvm/audio-libs
build_audio_deps: setup_toolchain
bash tools/build_audio_deps.sh $(ALSA_VERSION) $(OPUS_VERSION)
# Prepare everything needed for local development (toolchain + audio deps + Go tools)
dev_env: build_audio_deps
@echo "Installing Go development tools..."
go install golang.org/x/tools/cmd/goimports@latest
@echo "Development environment ready."
JETKVM_HOME ?= $(HOME)/.jetkvm
TOOLCHAIN_DIR ?= $(JETKVM_HOME)/rv1106-system
AUDIO_LIBS_DIR ?= $(JETKVM_HOME)/audio-libs
BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
BUILDDATE ?= $(shell date -u +%FT%T%z)
BUILDTS ?= $(shell date -u +%s)
@ -5,6 +24,13 @@ REVISION ?= $(shell git rev-parse HEAD)
VERSION_DEV ?= 0.4.7-dev$(shell date +%Y%m%d%H%M)
VERSION ?= 0.4.6
# Audio library versions
ALSA_VERSION ?= 1.2.14
OPUS_VERSION ?= 1.5.2
# Optimization flags for ARM Cortex-A7 with NEON
OPTIM_CFLAGS := -O3 -mcpu=cortex-a7 -mfpu=neon -mfloat-abi=hard -ftree-vectorize -ffast-math -funroll-loops
PROMETHEUS_TAG := github.com/prometheus/common/version
KVM_PKG_NAME := github.com/jetkvm/kvm
@ -25,9 +51,14 @@ TEST_DIRS := $(shell find . -name "*_test.go" -type f -exec dirname {} \; | sort
hash_resource:
@shasum -a 256 resource/jetkvm_native | cut -d ' ' -f 1 > resource/jetkvm_native.sha256
build_dev: hash_resource
build_dev: build_audio_deps hash_resource
@echo "Building..."
$(GO_CMD) build \
GOOS=linux GOARCH=arm GOARM=7 \
CC=$(TOOLCHAIN_DIR)/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc \
CGO_ENABLED=1 \
CGO_CFLAGS="$(OPTIM_CFLAGS) -I$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/celt" \
CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/.libs -lopus -lm -ldl -static" \
go build \
-ldflags="$(GO_LDFLAGS) -X $(KVM_PKG_NAME).builtAppVersion=$(VERSION_DEV)" \
$(GO_RELEASE_BUILD_ARGS) \
-o $(BIN_DIR)/jetkvm_app cmd/main.go
@ -40,7 +71,7 @@ build_gotestsum:
$(GO_CMD) install gotest.tools/gotestsum@latest
cp $(shell $(GO_CMD) env GOPATH)/bin/linux_arm/gotestsum $(BIN_DIR)/gotestsum
build_dev_test: build_test2json build_gotestsum
build_dev_test: build_audio_deps build_test2json build_gotestsum
# collect all directories that contain tests
@echo "Building tests for devices ..."
@rm -rf $(BIN_DIR)/tests && mkdir -p $(BIN_DIR)/tests
@ -50,7 +81,12 @@ build_dev_test: build_test2json build_gotestsum
test_pkg_name=$$(echo $$test | sed 's/^.\///g'); \
test_pkg_full_name=$(KVM_PKG_NAME)/$$(echo $$test | sed 's/^.\///g'); \
test_filename=$$(echo $$test_pkg_name | sed 's/\//__/g')_test; \
$(GO_CMD) test -v \
GOOS=linux GOARCH=arm GOARM=7 \
CC=$(TOOLCHAIN_DIR)/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc \
CGO_ENABLED=1 \
CGO_CFLAGS="$(OPTIM_CFLAGS) -I$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/celt" \
CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/.libs -lopus -lm -ldl -static" \
go test -v \
-ldflags="$(GO_LDFLAGS) -X $(KVM_PKG_NAME).builtAppVersion=$(VERSION_DEV)" \
$(GO_BUILD_ARGS) \
-c -o $(BIN_DIR)/tests/$$test_filename $$test; \
@ -70,9 +106,14 @@ dev_release: frontend build_dev
rclone copyto bin/jetkvm_app r2://jetkvm-update/app/$(VERSION_DEV)/jetkvm_app
rclone copyto bin/jetkvm_app.sha256 r2://jetkvm-update/app/$(VERSION_DEV)/jetkvm_app.sha256
build_release: frontend hash_resource
build_release: frontend build_audio_deps hash_resource
@echo "Building release..."
$(GO_CMD) build \
GOOS=linux GOARCH=arm GOARM=7 \
CC=$(TOOLCHAIN_DIR)/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc \
CGO_ENABLED=1 \
CGO_CFLAGS="$(OPTIM_CFLAGS) -I$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/celt" \
CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/.libs -lopus -lm -ldl -static" \
go build \
-ldflags="$(GO_LDFLAGS) -X $(KVM_PKG_NAME).builtAppVersion=$(VERSION)" \
$(GO_RELEASE_BUILD_ARGS) \
-o bin/jetkvm_app cmd/main.go
@ -87,3 +128,44 @@ release:
@shasum -a 256 bin/jetkvm_app | cut -d ' ' -f 1 > bin/jetkvm_app.sha256
rclone copyto bin/jetkvm_app r2://jetkvm-update/app/$(VERSION)/jetkvm_app
rclone copyto bin/jetkvm_app.sha256 r2://jetkvm-update/app/$(VERSION)/jetkvm_app.sha256
# Run both Go and UI linting
lint: lint-go lint-ui
@echo "All linting completed successfully!"
# Run golangci-lint locally with the same configuration as CI
lint-go: build_audio_deps
@echo "Running golangci-lint..."
@mkdir -p static && touch static/.gitkeep
CGO_ENABLED=1 \
CGO_CFLAGS="-I$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/celt" \
CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/.libs -lopus -lm -ldl -static" \
golangci-lint run --verbose
# Run both Go and UI linting with auto-fix
lint-fix: lint-go-fix lint-ui-fix
@echo "All linting with auto-fix completed successfully!"
# Run golangci-lint with auto-fix
lint-go-fix: build_audio_deps
@echo "Running golangci-lint with auto-fix..."
@mkdir -p static && touch static/.gitkeep
CGO_ENABLED=1 \
CGO_CFLAGS="-I$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/include -I$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/celt" \
CGO_LDFLAGS="-L$(AUDIO_LIBS_DIR)/alsa-lib-$(ALSA_VERSION)/src/.libs -lasound -L$(AUDIO_LIBS_DIR)/opus-$(OPUS_VERSION)/.libs -lopus -lm -ldl -static" \
golangci-lint run --fix --verbose
# Run UI linting locally (mirrors GitHub workflow ui-lint.yml)
lint-ui:
@echo "Running UI lint..."
@cd ui && npm ci
@cd ui && npm run lint
# Run UI linting with auto-fix
lint-ui-fix:
@echo "Running UI lint with auto-fix..."
@cd ui && npm ci
@cd ui && npm run lint:fix
# Legacy alias for UI linting (for backward compatibility)
ui-lint: lint-ui

View File

@ -11,13 +11,20 @@
</div>
JetKVM is a high-performance, open-source KVM over IP (Keyboard, Video, Mouse) solution designed for efficient remote management of computers, servers, and workstations. Whether you're dealing with boot failures, installing a new operating system, adjusting BIOS settings, or simply taking control of a machine from afar, JetKVM provides the tools to get it done effectively.
JetKVM is a high-performance, open-source KVM over IP (Keyboard, Video, Mouse, **Audio**) solution designed for efficient remote management of computers, servers, and workstations. Whether you're dealing with boot failures, installing a new operating system, adjusting BIOS settings, or simply taking control of a machine from afar, JetKVM provides the tools to get it done effectively.
## Features
- **Ultra-low Latency** - 1080p@60FPS video with 30-60ms latency using H.264 encoding. Smooth mouse and keyboard interaction for responsive remote control.
- **Ultra-low Latency** - 1080p@60FPS video with 30-60ms latency using H.264 encoding. Smooth mouse, keyboard, and audio for responsive remote control.
- **First-Class Audio Support** - JetKVM now supports bidirectional, low-latency audio streaming using a dual-subprocess architecture with ALSA and Opus integration via CGO. Features both audio output (PC→Browser) and audio input (Browser→PC) with dedicated subprocesses for optimal performance and isolation.
- **Free & Optional Remote Access** - Remote management via JetKVM Cloud using WebRTC.
- **Open-source software** - Written in Golang on Linux. Easily customizable through SSH access to the JetKVM device.
- **Open-source software** - Written in Golang (with CGO for audio) on Linux. Easily customizable through SSH access to the JetKVM device.
## Contributing
@ -31,20 +38,23 @@ The best place to search for answers is our [Documentation](https://jetkvm.com/d
If you've found an issue and want to report it, please check our [Issues](https://github.com/jetkvm/kvm/issues) page. Make sure the description contains information about the firmware version you're using, your platform, and a clear explanation of the steps to reproduce the issue.
# Development
JetKVM is written in Go & TypeScript. with some bits and pieces written in C. An intermediate level of Go & TypeScript knowledge is recommended for comfortable programming.
JetKVM is written in Go & TypeScript, with some C for low-level integration. **Audio support uses a sophisticated dual-subprocess architecture with CGO, ALSA, and Opus integration for bidirectional streaming with complete process isolation.**
The project contains two main parts, the backend software that runs on the KVM device and the frontend software that is served by the KVM device, and also the cloud.
The project contains two main parts: the backend software (Go, CGO) that runs on the KVM device, and the frontend software (React/TypeScript) that is served by the KVM device and the cloud.
For comprehensive development information, including setup, testing, debugging, and contribution guidelines, see **[DEVELOPMENT.md](DEVELOPMENT.md)**.
For quick device development, use the `./dev_deploy.sh` script. It will build the frontend and backend and deploy them to the local KVM device. Run `./dev_deploy.sh --help` for more information.
## Backend
The backend is written in Go and is responsible for the KVM device management, the cloud API and the cloud web.
The backend is written in Go and is responsible for KVM device management, audio/video streaming, the cloud API, and the cloud web. **Audio uses dedicated subprocesses for both output and input streams, with CGO-based ALSA and Opus processing, IPC communication via Unix sockets, and comprehensive process supervision for reliability.**
## Frontend
The frontend is written in React and TypeScript and is served by the KVM device. It has three build targets: `device`, `development` and `production`. Development is used for development of the cloud version on your local machine, device is used for building the frontend for the KVM device and production is used for building the frontend for the cloud.
The frontend is written in React and TypeScript and is served by the KVM device. It has three build targets: `device`, `development`, and `production`. Development is used for the cloud version on your local machine, device is used for building the frontend for the KVM device, and production is used for building the frontend for the cloud.

View File

@ -39,7 +39,8 @@ const (
// should be lower than the websocket response timeout set in cloud-api
CloudOidcRequestTimeout = 10 * time.Second
// WebsocketPingInterval is the interval at which the websocket client sends ping messages to the cloud
WebsocketPingInterval = 15 * time.Second
// Increased to 30 seconds for constrained environments to reduce overhead
WebsocketPingInterval = 30 * time.Second
)
var (
@ -447,35 +448,70 @@ func handleSessionRequest(
}
}
session, err := newSession(SessionConfig{
ws: c,
IsCloud: isCloudConnection,
LocalIP: req.IP,
ICEServers: req.ICEServers,
Logger: scopedLogger,
})
if err != nil {
_ = wsjson.Write(context.Background(), c, gin.H{"error": err})
return err
}
var session *Session
var err error
var sd string
sd, err := session.ExchangeOffer(req.Sd)
if err != nil {
_ = wsjson.Write(context.Background(), c, gin.H{"error": err})
return err
}
// Check if we have an existing session
if currentSession != nil {
scopedLogger.Info().Msg("existing session detected, creating new session and notifying old session")
// Always create a new session when there's an existing one
// This ensures the "otherSessionConnected" prompt is shown
session, err = newSession(SessionConfig{
ws: c,
IsCloud: isCloudConnection,
LocalIP: req.IP,
ICEServers: req.ICEServers,
Logger: scopedLogger,
})
if err != nil {
_ = wsjson.Write(context.Background(), c, gin.H{"error": err})
return err
}
sd, err = session.ExchangeOffer(req.Sd)
if err != nil {
_ = wsjson.Write(context.Background(), c, gin.H{"error": err})
return err
}
// Notify the old session about the takeover
writeJSONRPCEvent("otherSessionConnected", nil, currentSession)
peerConn := currentSession.peerConnection
go func() {
time.Sleep(1 * time.Second)
_ = peerConn.Close()
}()
currentSession = session
scopedLogger.Info().Interface("session", session).Msg("new session created, old session notified")
} else {
// No existing session, create a new one
scopedLogger.Info().Msg("creating new session")
session, err = newSession(SessionConfig{
ws: c,
IsCloud: isCloudConnection,
LocalIP: req.IP,
ICEServers: req.ICEServers,
Logger: scopedLogger,
})
if err != nil {
_ = wsjson.Write(context.Background(), c, gin.H{"error": err})
return err
}
sd, err = session.ExchangeOffer(req.Sd)
if err != nil {
_ = wsjson.Write(context.Background(), c, gin.H{"error": err})
return err
}
currentSession = session
cloudLogger.Info().Interface("session", session).Msg("new session accepted")
cloudLogger.Trace().Interface("session", session).Msg("new session accepted")
}
cloudLogger.Info().Interface("session", session).Msg("new session accepted")
cloudLogger.Trace().Interface("session", session).Msg("new session accepted")
currentSession = session
_ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd})
return nil
}

View File

@ -11,6 +11,8 @@ import (
func main() {
versionPtr := flag.Bool("version", false, "print version and exit")
versionJsonPtr := flag.Bool("version-json", false, "print version as json and exit")
audioServerPtr := flag.Bool("audio-output-server", false, "Run as audio server subprocess")
audioInputServerPtr := flag.Bool("audio-input-server", false, "Run as audio input server subprocess")
flag.Parse()
if *versionPtr || *versionJsonPtr {
@ -23,5 +25,5 @@ func main() {
return
}
kvm.Main()
kvm.Main(*audioServerPtr, *audioInputServerPtr)
}

View File

@ -138,6 +138,7 @@ var defaultConfig = &Config{
RelativeMouse: true,
Keyboard: true,
MassStorage: true,
Audio: true,
},
NetworkConfig: &network.NetworkConfig{},
DefaultLogLevel: "INFO",

View File

@ -159,8 +159,8 @@ else
msg_info "▶ Building development binary"
make build_dev
# Kill any existing instances of the application
ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app_debug || true"
# Kill any existing instances of the application (specific cleanup)
ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app || true; killall jetkvm_native || true; killall jetkvm_app_debug || true; sleep 2"
# Copy the binary to the remote host
ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat > ${REMOTE_PATH}/jetkvm_app_debug" < bin/jetkvm_app
@ -180,9 +180,18 @@ set -e
# Set the library path to include the directory where librockit.so is located
export LD_LIBRARY_PATH=/oem/usr/lib:\$LD_LIBRARY_PATH
# Kill any existing instances of the application
# Kill any existing instances of the application (specific cleanup)
killall jetkvm_app || true
killall jetkvm_native || true
killall jetkvm_app_debug || true
sleep 2
# Verify no processes are using port 80
if netstat -tlnp | grep :80 > /dev/null 2>&1; then
echo "Warning: Port 80 still in use, attempting to free it..."
fuser -k 80/tcp || true
sleep 1
fi
# Navigate to the directory where the binary will be stored
cd "${REMOTE_PATH}"

View File

@ -372,11 +372,8 @@ func startBacklightTickers() {
dimTicker = time.NewTicker(time.Duration(config.DisplayDimAfterSec) * time.Second)
go func() {
for { //nolint:staticcheck
select {
case <-dimTicker.C:
tick_displayDim()
}
for range dimTicker.C {
tick_displayDim()
}
}()
}
@ -386,11 +383,8 @@ func startBacklightTickers() {
offTicker = time.NewTicker(time.Duration(config.DisplayOffAfterSec) * time.Second)
go func() {
for { //nolint:staticcheck
select {
case <-offTicker.C:
tick_displayOff()
}
for range offTicker.C {
tick_displayOff()
}
}()
}

217
input_rpc.go Normal file
View File

@ -0,0 +1,217 @@
package kvm
import (
"fmt"
)
// Constants for input validation
const (
// MaxKeyboardKeys defines the maximum number of simultaneous key presses
// This matches the USB HID keyboard report specification
MaxKeyboardKeys = 6
)
// Input RPC Direct Handlers
// This module provides optimized direct handlers for high-frequency input events,
// bypassing the reflection-based RPC system for improved performance.
//
// Performance benefits:
// - Eliminates reflection overhead (~2-3ms per call)
// - Reduces memory allocations
// - Optimizes parameter parsing and validation
// - Provides faster code path for input methods
//
// The handlers maintain full compatibility with existing RPC interface
// while providing significant latency improvements for input events.
// Common validation helpers for parameter parsing
// These reduce code duplication and provide consistent error messages
// validateFloat64Param extracts and validates a float64 parameter from the params map
func validateFloat64Param(params map[string]interface{}, paramName, methodName string, min, max float64) (float64, error) {
value, ok := params[paramName].(float64)
if !ok {
return 0, fmt.Errorf("%s: %s parameter must be a number, got %T", methodName, paramName, params[paramName])
}
if value < min || value > max {
return 0, fmt.Errorf("%s: %s value %v out of range [%v to %v]", methodName, paramName, value, min, max)
}
return value, nil
}
// validateKeysArray extracts and validates a keys array parameter
func validateKeysArray(params map[string]interface{}, methodName string) ([]uint8, error) {
keysInterface, ok := params["keys"].([]interface{})
if !ok {
return nil, fmt.Errorf("%s: keys parameter must be an array, got %T", methodName, params["keys"])
}
if len(keysInterface) > MaxKeyboardKeys {
return nil, fmt.Errorf("%s: too many keys (%d), maximum is %d", methodName, len(keysInterface), MaxKeyboardKeys)
}
keys := make([]uint8, len(keysInterface))
for i, keyInterface := range keysInterface {
keyFloat, ok := keyInterface.(float64)
if !ok {
return nil, fmt.Errorf("%s: key at index %d must be a number, got %T", methodName, i, keyInterface)
}
if keyFloat < 0 || keyFloat > 255 {
return nil, fmt.Errorf("%s: key at index %d value %v out of range [0-255]", methodName, i, keyFloat)
}
keys[i] = uint8(keyFloat)
}
return keys, nil
}
// Input parameter structures for direct RPC handlers
// These mirror the original RPC method signatures but provide
// optimized parsing from JSON map parameters.
// KeyboardReportParams represents parameters for keyboard HID report
// Matches rpcKeyboardReport(modifier uint8, keys []uint8)
type KeyboardReportParams struct {
Modifier uint8 `json:"modifier"` // Keyboard modifier keys (Ctrl, Alt, Shift, etc.)
Keys []uint8 `json:"keys"` // Array of pressed key codes (up to 6 keys)
}
// AbsMouseReportParams represents parameters for absolute mouse positioning
// Matches rpcAbsMouseReport(x, y int, buttons uint8)
type AbsMouseReportParams struct {
X int `json:"x"` // Absolute X coordinate (0-32767)
Y int `json:"y"` // Absolute Y coordinate (0-32767)
Buttons uint8 `json:"buttons"` // Mouse button state bitmask
}
// RelMouseReportParams represents parameters for relative mouse movement
// Matches rpcRelMouseReport(dx, dy int8, buttons uint8)
type RelMouseReportParams struct {
Dx int8 `json:"dx"` // Relative X movement delta (-127 to +127)
Dy int8 `json:"dy"` // Relative Y movement delta (-127 to +127)
Buttons uint8 `json:"buttons"` // Mouse button state bitmask
}
// WheelReportParams represents parameters for mouse wheel events
// Matches rpcWheelReport(wheelY int8)
type WheelReportParams struct {
WheelY int8 `json:"wheelY"` // Wheel scroll delta (-127 to +127)
}
// Direct handler for keyboard reports
// Optimized path that bypasses reflection for keyboard input events
func handleKeyboardReportDirect(params map[string]interface{}) (interface{}, error) {
// Extract and validate modifier parameter
modifierFloat, err := validateFloat64Param(params, "modifier", "keyboardReport", 0, 255)
if err != nil {
return nil, err
}
modifier := uint8(modifierFloat)
// Extract and validate keys array
keys, err := validateKeysArray(params, "keyboardReport")
if err != nil {
return nil, err
}
return nil, rpcKeyboardReport(modifier, keys)
}
// Direct handler for absolute mouse reports
// Optimized path that bypasses reflection for absolute mouse positioning
func handleAbsMouseReportDirect(params map[string]interface{}) (interface{}, error) {
// Extract and validate x coordinate
xFloat, err := validateFloat64Param(params, "x", "absMouseReport", 0, 32767)
if err != nil {
return nil, err
}
x := int(xFloat)
// Extract and validate y coordinate
yFloat, err := validateFloat64Param(params, "y", "absMouseReport", 0, 32767)
if err != nil {
return nil, err
}
y := int(yFloat)
// Extract and validate buttons
buttonsFloat, err := validateFloat64Param(params, "buttons", "absMouseReport", 0, 255)
if err != nil {
return nil, err
}
buttons := uint8(buttonsFloat)
return nil, rpcAbsMouseReport(x, y, buttons)
}
// Direct handler for relative mouse reports
// Optimized path that bypasses reflection for relative mouse movement
func handleRelMouseReportDirect(params map[string]interface{}) (interface{}, error) {
// Extract and validate dx (relative X movement)
dxFloat, err := validateFloat64Param(params, "dx", "relMouseReport", -127, 127)
if err != nil {
return nil, err
}
dx := int8(dxFloat)
// Extract and validate dy (relative Y movement)
dyFloat, err := validateFloat64Param(params, "dy", "relMouseReport", -127, 127)
if err != nil {
return nil, err
}
dy := int8(dyFloat)
// Extract and validate buttons
buttonsFloat, err := validateFloat64Param(params, "buttons", "relMouseReport", 0, 255)
if err != nil {
return nil, err
}
buttons := uint8(buttonsFloat)
return nil, rpcRelMouseReport(dx, dy, buttons)
}
// Direct handler for wheel reports
// Optimized path that bypasses reflection for mouse wheel events
func handleWheelReportDirect(params map[string]interface{}) (interface{}, error) {
// Extract and validate wheelY (scroll delta)
wheelYFloat, err := validateFloat64Param(params, "wheelY", "wheelReport", -127, 127)
if err != nil {
return nil, err
}
wheelY := int8(wheelYFloat)
return nil, rpcWheelReport(wheelY)
}
// handleInputRPCDirect routes input method calls to their optimized direct handlers
// This is the main entry point for the fast path that bypasses reflection.
// It provides significant performance improvements for high-frequency input events.
//
// Performance monitoring: Consider adding metrics collection here to track
// latency improvements and call frequency for production monitoring.
func handleInputRPCDirect(method string, params map[string]interface{}) (interface{}, error) {
switch method {
case "keyboardReport":
return handleKeyboardReportDirect(params)
case "absMouseReport":
return handleAbsMouseReportDirect(params)
case "relMouseReport":
return handleRelMouseReportDirect(params)
case "wheelReport":
return handleWheelReportDirect(params)
default:
// This should never happen if isInputMethod is correctly implemented
return nil, fmt.Errorf("handleInputRPCDirect: unsupported method '%s'", method)
}
}
// isInputMethod determines if a given RPC method should use the optimized direct path
// Returns true for input-related methods that have direct handlers implemented.
// This function must be kept in sync with handleInputRPCDirect.
func isInputMethod(method string) bool {
switch method {
case "keyboardReport", "absMouseReport", "relMouseReport", "wheelReport":
return true
default:
return false
}
}

560
input_rpc_test.go Normal file
View File

@ -0,0 +1,560 @@
package kvm
import (
"testing"
"github.com/stretchr/testify/assert"
)
// Test validateFloat64Param function
func TestValidateFloat64Param(t *testing.T) {
tests := []struct {
name string
params map[string]interface{}
paramName string
methodName string
min float64
max float64
expected float64
expectError bool
}{
{
name: "valid parameter",
params: map[string]interface{}{"test": 50.0},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 50.0,
expectError: false,
},
{
name: "parameter at minimum boundary",
params: map[string]interface{}{"test": 0.0},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 0.0,
expectError: false,
},
{
name: "parameter at maximum boundary",
params: map[string]interface{}{"test": 100.0},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 100.0,
expectError: false,
},
{
name: "parameter below minimum",
params: map[string]interface{}{"test": -1.0},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 0,
expectError: true,
},
{
name: "parameter above maximum",
params: map[string]interface{}{"test": 101.0},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 0,
expectError: true,
},
{
name: "wrong parameter type",
params: map[string]interface{}{"test": "not a number"},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 0,
expectError: true,
},
{
name: "missing parameter",
params: map[string]interface{}{},
paramName: "test",
methodName: "testMethod",
min: 0,
max: 100,
expected: 0,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := validateFloat64Param(tt.params, tt.paramName, tt.methodName, tt.min, tt.max)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.expected, result)
}
})
}
}
// Test validateKeysArray function
func TestValidateKeysArray(t *testing.T) {
tests := []struct {
name string
params map[string]interface{}
methodName string
expected []uint8
expectError bool
}{
{
name: "valid keys array",
params: map[string]interface{}{"keys": []interface{}{65.0, 66.0, 67.0}},
methodName: "testMethod",
expected: []uint8{65, 66, 67},
expectError: false,
},
{
name: "empty keys array",
params: map[string]interface{}{"keys": []interface{}{}},
methodName: "testMethod",
expected: []uint8{},
expectError: false,
},
{
name: "maximum keys array",
params: map[string]interface{}{"keys": []interface{}{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}},
methodName: "testMethod",
expected: []uint8{1, 2, 3, 4, 5, 6},
expectError: false,
},
{
name: "too many keys",
params: map[string]interface{}{"keys": []interface{}{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}},
methodName: "testMethod",
expected: nil,
expectError: true,
},
{
name: "invalid key type",
params: map[string]interface{}{"keys": []interface{}{"not a number"}},
methodName: "testMethod",
expected: nil,
expectError: true,
},
{
name: "key value out of range (negative)",
params: map[string]interface{}{"keys": []interface{}{-1.0}},
methodName: "testMethod",
expected: nil,
expectError: true,
},
{
name: "key value out of range (too high)",
params: map[string]interface{}{"keys": []interface{}{256.0}},
methodName: "testMethod",
expected: nil,
expectError: true,
},
{
name: "wrong parameter type",
params: map[string]interface{}{"keys": "not an array"},
methodName: "testMethod",
expected: nil,
expectError: true,
},
{
name: "missing keys parameter",
params: map[string]interface{}{},
methodName: "testMethod",
expected: nil,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := validateKeysArray(tt.params, tt.methodName)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.expected, result)
}
})
}
}
// Test handleKeyboardReportDirect function
func TestHandleKeyboardReportDirect(t *testing.T) {
tests := []struct {
name string
params map[string]interface{}
expectError bool
}{
{
name: "valid keyboard report",
params: map[string]interface{}{
"modifier": 2.0, // Shift key
"keys": []interface{}{65.0, 66.0}, // A, B keys
},
expectError: false,
},
{
name: "empty keys array",
params: map[string]interface{}{
"modifier": 0.0,
"keys": []interface{}{},
},
expectError: false,
},
{
name: "invalid modifier",
params: map[string]interface{}{
"modifier": 256.0, // Out of range
"keys": []interface{}{65.0},
},
expectError: true,
},
{
name: "invalid keys",
params: map[string]interface{}{
"modifier": 0.0,
"keys": []interface{}{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}, // Too many keys
},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := handleKeyboardReportDirect(tt.params)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// Test handleAbsMouseReportDirect function
func TestHandleAbsMouseReportDirect(t *testing.T) {
tests := []struct {
name string
params map[string]interface{}
expectError bool
}{
{
name: "valid absolute mouse report",
params: map[string]interface{}{
"x": 1000.0,
"y": 500.0,
"buttons": 1.0, // Left button
},
expectError: false,
},
{
name: "boundary values",
params: map[string]interface{}{
"x": 0.0,
"y": 32767.0,
"buttons": 255.0,
},
expectError: false,
},
{
name: "invalid x coordinate",
params: map[string]interface{}{
"x": -1.0, // Out of range
"y": 500.0,
"buttons": 0.0,
},
expectError: true,
},
{
name: "invalid y coordinate",
params: map[string]interface{}{
"x": 1000.0,
"y": 32768.0, // Out of range
"buttons": 0.0,
},
expectError: true,
},
{
name: "invalid buttons",
params: map[string]interface{}{
"x": 1000.0,
"y": 500.0,
"buttons": 256.0, // Out of range
},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := handleAbsMouseReportDirect(tt.params)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// Test handleRelMouseReportDirect function
func TestHandleRelMouseReportDirect(t *testing.T) {
tests := []struct {
name string
params map[string]interface{}
expectError bool
}{
{
name: "valid relative mouse report",
params: map[string]interface{}{
"dx": 10.0,
"dy": -5.0,
"buttons": 2.0, // Right button
},
expectError: false,
},
{
name: "boundary values",
params: map[string]interface{}{
"dx": -127.0,
"dy": 127.0,
"buttons": 0.0,
},
expectError: false,
},
{
name: "invalid dx",
params: map[string]interface{}{
"dx": -128.0, // Out of range
"dy": 0.0,
"buttons": 0.0,
},
expectError: true,
},
{
name: "invalid dy",
params: map[string]interface{}{
"dx": 0.0,
"dy": 128.0, // Out of range
"buttons": 0.0,
},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := handleRelMouseReportDirect(tt.params)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// Test handleWheelReportDirect function
func TestHandleWheelReportDirect(t *testing.T) {
tests := []struct {
name string
params map[string]interface{}
expectError bool
}{
{
name: "valid wheel report",
params: map[string]interface{}{
"wheelY": 3.0,
},
expectError: false,
},
{
name: "boundary values",
params: map[string]interface{}{
"wheelY": -127.0,
},
expectError: false,
},
{
name: "invalid wheelY",
params: map[string]interface{}{
"wheelY": 128.0, // Out of range
},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := handleWheelReportDirect(tt.params)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// Test handleInputRPCDirect function
func TestHandleInputRPCDirect(t *testing.T) {
tests := []struct {
name string
method string
params map[string]interface{}
expectError bool
}{
{
name: "keyboard report",
method: "keyboardReport",
params: map[string]interface{}{
"modifier": 0.0,
"keys": []interface{}{65.0},
},
expectError: false,
},
{
name: "absolute mouse report",
method: "absMouseReport",
params: map[string]interface{}{
"x": 1000.0,
"y": 500.0,
"buttons": 1.0,
},
expectError: false,
},
{
name: "relative mouse report",
method: "relMouseReport",
params: map[string]interface{}{
"dx": 10.0,
"dy": -5.0,
"buttons": 2.0,
},
expectError: false,
},
{
name: "wheel report",
method: "wheelReport",
params: map[string]interface{}{
"wheelY": 3.0,
},
expectError: false,
},
{
name: "unknown method",
method: "unknownMethod",
params: map[string]interface{}{},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := handleInputRPCDirect(tt.method, tt.params)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
// Test isInputMethod function
func TestIsInputMethod(t *testing.T) {
tests := []struct {
name string
method string
expected bool
}{
{
name: "keyboard report method",
method: "keyboardReport",
expected: true,
},
{
name: "absolute mouse report method",
method: "absMouseReport",
expected: true,
},
{
name: "relative mouse report method",
method: "relMouseReport",
expected: true,
},
{
name: "wheel report method",
method: "wheelReport",
expected: true,
},
{
name: "non-input method",
method: "someOtherMethod",
expected: false,
},
{
name: "empty method",
method: "",
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isInputMethod(tt.method)
assert.Equal(t, tt.expected, result)
})
}
}
// Benchmark tests to verify performance improvements
func BenchmarkValidateFloat64Param(b *testing.B) {
params := map[string]interface{}{"test": 50.0}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = validateFloat64Param(params, "test", "benchmarkMethod", 0, 100)
}
}
func BenchmarkValidateKeysArray(b *testing.B) {
params := map[string]interface{}{"keys": []interface{}{65.0, 66.0, 67.0}}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = validateKeysArray(params, "benchmarkMethod")
}
}
func BenchmarkHandleKeyboardReportDirect(b *testing.B) {
params := map[string]interface{}{
"modifier": 2.0,
"keys": []interface{}{65.0, 66.0},
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = handleKeyboardReportDirect(params)
}
}
func BenchmarkHandleInputRPCDirect(b *testing.B) {
params := map[string]interface{}{
"modifier": 2.0,
"keys": []interface{}{65.0, 66.0},
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = handleInputRPCDirect("keyboardReport", params)
}
}

View File

@ -0,0 +1,338 @@
package audio
import (
"context"
"math"
"sync"
"sync/atomic"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// AdaptiveBufferConfig holds configuration for adaptive buffer sizing
type AdaptiveBufferConfig struct {
// Buffer size limits (in frames)
MinBufferSize int
MaxBufferSize int
DefaultBufferSize int
// System load thresholds
LowCPUThreshold float64 // Below this, increase buffer size
HighCPUThreshold float64 // Above this, decrease buffer size
LowMemoryThreshold float64 // Below this, increase buffer size
HighMemoryThreshold float64 // Above this, decrease buffer size
// Latency thresholds (in milliseconds)
TargetLatency time.Duration
MaxLatency time.Duration
// Adaptation parameters
AdaptationInterval time.Duration
SmoothingFactor float64 // 0.0-1.0, higher = more responsive
}
// DefaultAdaptiveBufferConfig returns optimized config for JetKVM hardware
func DefaultAdaptiveBufferConfig() AdaptiveBufferConfig {
return AdaptiveBufferConfig{
// Conservative buffer sizes for 256MB RAM constraint
MinBufferSize: 3, // Minimum 3 frames (slightly higher for stability)
MaxBufferSize: 20, // Maximum 20 frames (increased for high load scenarios)
DefaultBufferSize: 6, // Default 6 frames (increased for better stability)
// CPU thresholds optimized for single-core ARM Cortex A7 under load
LowCPUThreshold: 20.0, // Below 20% CPU
HighCPUThreshold: 60.0, // Above 60% CPU (lowered to be more responsive)
// Memory thresholds for 256MB total RAM
LowMemoryThreshold: 35.0, // Below 35% memory usage
HighMemoryThreshold: 75.0, // Above 75% memory usage (lowered for earlier response)
// Latency targets
TargetLatency: 20 * time.Millisecond, // Target 20ms latency
MaxLatency: 50 * time.Millisecond, // Max acceptable 50ms
// Adaptation settings
AdaptationInterval: 500 * time.Millisecond, // Check every 500ms
SmoothingFactor: 0.3, // Moderate responsiveness
}
}
// AdaptiveBufferManager manages dynamic buffer sizing based on system conditions
type AdaptiveBufferManager struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
currentInputBufferSize int64 // Current input buffer size (atomic)
currentOutputBufferSize int64 // Current output buffer size (atomic)
averageLatency int64 // Average latency in nanoseconds (atomic)
systemCPUPercent int64 // System CPU percentage * 100 (atomic)
systemMemoryPercent int64 // System memory percentage * 100 (atomic)
adaptationCount int64 // Metrics tracking (atomic)
config AdaptiveBufferConfig
logger zerolog.Logger
processMonitor *ProcessMonitor
// Control channels
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
// Metrics tracking
lastAdaptation time.Time
mutex sync.RWMutex
}
// NewAdaptiveBufferManager creates a new adaptive buffer manager
func NewAdaptiveBufferManager(config AdaptiveBufferConfig) *AdaptiveBufferManager {
ctx, cancel := context.WithCancel(context.Background())
return &AdaptiveBufferManager{
currentInputBufferSize: int64(config.DefaultBufferSize),
currentOutputBufferSize: int64(config.DefaultBufferSize),
config: config,
logger: logging.GetDefaultLogger().With().Str("component", "adaptive-buffer").Logger(),
processMonitor: GetProcessMonitor(),
ctx: ctx,
cancel: cancel,
lastAdaptation: time.Now(),
}
}
// Start begins the adaptive buffer management
func (abm *AdaptiveBufferManager) Start() {
abm.wg.Add(1)
go abm.adaptationLoop()
abm.logger.Info().Msg("Adaptive buffer manager started")
}
// Stop stops the adaptive buffer management
func (abm *AdaptiveBufferManager) Stop() {
abm.cancel()
abm.wg.Wait()
abm.logger.Info().Msg("Adaptive buffer manager stopped")
}
// GetInputBufferSize returns the current recommended input buffer size
func (abm *AdaptiveBufferManager) GetInputBufferSize() int {
return int(atomic.LoadInt64(&abm.currentInputBufferSize))
}
// GetOutputBufferSize returns the current recommended output buffer size
func (abm *AdaptiveBufferManager) GetOutputBufferSize() int {
return int(atomic.LoadInt64(&abm.currentOutputBufferSize))
}
// UpdateLatency updates the current latency measurement
func (abm *AdaptiveBufferManager) UpdateLatency(latency time.Duration) {
// Use exponential moving average for latency
currentAvg := atomic.LoadInt64(&abm.averageLatency)
newLatency := latency.Nanoseconds()
if currentAvg == 0 {
atomic.StoreInt64(&abm.averageLatency, newLatency)
} else {
// Exponential moving average: 70% historical, 30% current
newAvg := int64(float64(currentAvg)*0.7 + float64(newLatency)*0.3)
atomic.StoreInt64(&abm.averageLatency, newAvg)
}
}
// adaptationLoop is the main loop that adjusts buffer sizes
func (abm *AdaptiveBufferManager) adaptationLoop() {
defer abm.wg.Done()
ticker := time.NewTicker(abm.config.AdaptationInterval)
defer ticker.Stop()
for {
select {
case <-abm.ctx.Done():
return
case <-ticker.C:
abm.adaptBufferSizes()
}
}
}
// adaptBufferSizes analyzes system conditions and adjusts buffer sizes
func (abm *AdaptiveBufferManager) adaptBufferSizes() {
// Collect current system metrics
metrics := abm.processMonitor.GetCurrentMetrics()
if len(metrics) == 0 {
return // No metrics available
}
// Calculate system-wide CPU and memory usage
totalCPU := 0.0
totalMemory := 0.0
processCount := 0
for _, metric := range metrics {
totalCPU += metric.CPUPercent
totalMemory += metric.MemoryPercent
processCount++
}
if processCount == 0 {
return
}
// Store system metrics atomically
systemCPU := totalCPU // Total CPU across all monitored processes
systemMemory := totalMemory / float64(processCount) // Average memory usage
atomic.StoreInt64(&abm.systemCPUPercent, int64(systemCPU*100))
atomic.StoreInt64(&abm.systemMemoryPercent, int64(systemMemory*100))
// Get current latency
currentLatencyNs := atomic.LoadInt64(&abm.averageLatency)
currentLatency := time.Duration(currentLatencyNs)
// Calculate adaptation factors
cpuFactor := abm.calculateCPUFactor(systemCPU)
memoryFactor := abm.calculateMemoryFactor(systemMemory)
latencyFactor := abm.calculateLatencyFactor(currentLatency)
// Combine factors with weights (CPU has highest priority for KVM coexistence)
combinedFactor := 0.5*cpuFactor + 0.3*memoryFactor + 0.2*latencyFactor
// Apply adaptation with smoothing
currentInput := float64(atomic.LoadInt64(&abm.currentInputBufferSize))
currentOutput := float64(atomic.LoadInt64(&abm.currentOutputBufferSize))
// Calculate new buffer sizes
newInputSize := abm.applyAdaptation(currentInput, combinedFactor)
newOutputSize := abm.applyAdaptation(currentOutput, combinedFactor)
// Update buffer sizes if they changed significantly
adjustmentMade := false
if math.Abs(newInputSize-currentInput) >= 0.5 || math.Abs(newOutputSize-currentOutput) >= 0.5 {
atomic.StoreInt64(&abm.currentInputBufferSize, int64(math.Round(newInputSize)))
atomic.StoreInt64(&abm.currentOutputBufferSize, int64(math.Round(newOutputSize)))
atomic.AddInt64(&abm.adaptationCount, 1)
abm.mutex.Lock()
abm.lastAdaptation = time.Now()
abm.mutex.Unlock()
adjustmentMade = true
abm.logger.Debug().
Float64("cpu_percent", systemCPU).
Float64("memory_percent", systemMemory).
Dur("latency", currentLatency).
Float64("combined_factor", combinedFactor).
Int("new_input_size", int(newInputSize)).
Int("new_output_size", int(newOutputSize)).
Msg("Adapted buffer sizes")
}
// Update metrics with current state
currentInputSize := int(atomic.LoadInt64(&abm.currentInputBufferSize))
currentOutputSize := int(atomic.LoadInt64(&abm.currentOutputBufferSize))
UpdateAdaptiveBufferMetrics(currentInputSize, currentOutputSize, systemCPU, systemMemory, adjustmentMade)
}
// calculateCPUFactor returns adaptation factor based on CPU usage
// Returns: -1.0 (decrease buffers) to +1.0 (increase buffers)
func (abm *AdaptiveBufferManager) calculateCPUFactor(cpuPercent float64) float64 {
if cpuPercent > abm.config.HighCPUThreshold {
// High CPU: decrease buffers to reduce latency and give CPU to KVM
return -1.0
} else if cpuPercent < abm.config.LowCPUThreshold {
// Low CPU: increase buffers for better quality
return 1.0
}
// Medium CPU: linear interpolation
midpoint := (abm.config.HighCPUThreshold + abm.config.LowCPUThreshold) / 2
return (midpoint - cpuPercent) / (midpoint - abm.config.LowCPUThreshold)
}
// calculateMemoryFactor returns adaptation factor based on memory usage
func (abm *AdaptiveBufferManager) calculateMemoryFactor(memoryPercent float64) float64 {
if memoryPercent > abm.config.HighMemoryThreshold {
// High memory: decrease buffers to free memory
return -1.0
} else if memoryPercent < abm.config.LowMemoryThreshold {
// Low memory: increase buffers for better performance
return 1.0
}
// Medium memory: linear interpolation
midpoint := (abm.config.HighMemoryThreshold + abm.config.LowMemoryThreshold) / 2
return (midpoint - memoryPercent) / (midpoint - abm.config.LowMemoryThreshold)
}
// calculateLatencyFactor returns adaptation factor based on latency
func (abm *AdaptiveBufferManager) calculateLatencyFactor(latency time.Duration) float64 {
if latency > abm.config.MaxLatency {
// High latency: decrease buffers
return -1.0
} else if latency < abm.config.TargetLatency {
// Low latency: can increase buffers
return 1.0
}
// Medium latency: linear interpolation
midLatency := (abm.config.MaxLatency + abm.config.TargetLatency) / 2
return float64(midLatency-latency) / float64(midLatency-abm.config.TargetLatency)
}
// applyAdaptation applies the adaptation factor to current buffer size
func (abm *AdaptiveBufferManager) applyAdaptation(currentSize, factor float64) float64 {
// Calculate target size based on factor
var targetSize float64
if factor > 0 {
// Increase towards max
targetSize = currentSize + factor*(float64(abm.config.MaxBufferSize)-currentSize)
} else {
// Decrease towards min
targetSize = currentSize + factor*(currentSize-float64(abm.config.MinBufferSize))
}
// Apply smoothing
newSize := currentSize + abm.config.SmoothingFactor*(targetSize-currentSize)
// Clamp to valid range
return math.Max(float64(abm.config.MinBufferSize),
math.Min(float64(abm.config.MaxBufferSize), newSize))
}
// GetStats returns current adaptation statistics
func (abm *AdaptiveBufferManager) GetStats() map[string]interface{} {
abm.mutex.RLock()
lastAdaptation := abm.lastAdaptation
abm.mutex.RUnlock()
return map[string]interface{}{
"input_buffer_size": abm.GetInputBufferSize(),
"output_buffer_size": abm.GetOutputBufferSize(),
"average_latency_ms": float64(atomic.LoadInt64(&abm.averageLatency)) / 1e6,
"system_cpu_percent": float64(atomic.LoadInt64(&abm.systemCPUPercent)) / 100,
"system_memory_percent": float64(atomic.LoadInt64(&abm.systemMemoryPercent)) / 100,
"adaptation_count": atomic.LoadInt64(&abm.adaptationCount),
"last_adaptation": lastAdaptation,
}
}
// Global adaptive buffer manager instance
var globalAdaptiveBufferManager *AdaptiveBufferManager
var adaptiveBufferOnce sync.Once
// GetAdaptiveBufferManager returns the global adaptive buffer manager instance
func GetAdaptiveBufferManager() *AdaptiveBufferManager {
adaptiveBufferOnce.Do(func() {
globalAdaptiveBufferManager = NewAdaptiveBufferManager(DefaultAdaptiveBufferConfig())
})
return globalAdaptiveBufferManager
}
// StartAdaptiveBuffering starts the global adaptive buffer manager
func StartAdaptiveBuffering() {
GetAdaptiveBufferManager().Start()
}
// StopAdaptiveBuffering stops the global adaptive buffer manager
func StopAdaptiveBuffering() {
if globalAdaptiveBufferManager != nil {
globalAdaptiveBufferManager.Stop()
}
}

View File

@ -0,0 +1,198 @@
package audio
import (
"context"
"sync"
"sync/atomic"
"time"
"github.com/rs/zerolog"
)
// AdaptiveOptimizer automatically adjusts audio parameters based on latency metrics
type AdaptiveOptimizer struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
optimizationCount int64 // Number of optimizations performed (atomic)
lastOptimization int64 // Timestamp of last optimization (atomic)
optimizationLevel int64 // Current optimization level (0-10) (atomic)
latencyMonitor *LatencyMonitor
bufferManager *AdaptiveBufferManager
logger zerolog.Logger
// Control channels
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
// Configuration
config OptimizerConfig
}
// OptimizerConfig holds configuration for the adaptive optimizer
type OptimizerConfig struct {
MaxOptimizationLevel int // Maximum optimization level (0-10)
CooldownPeriod time.Duration // Minimum time between optimizations
Aggressiveness float64 // How aggressively to optimize (0.0-1.0)
RollbackThreshold time.Duration // Latency threshold to rollback optimizations
StabilityPeriod time.Duration // Time to wait for stability after optimization
}
// DefaultOptimizerConfig returns a sensible default configuration
func DefaultOptimizerConfig() OptimizerConfig {
return OptimizerConfig{
MaxOptimizationLevel: 8,
CooldownPeriod: 30 * time.Second,
Aggressiveness: 0.7,
RollbackThreshold: 300 * time.Millisecond,
StabilityPeriod: 10 * time.Second,
}
}
// NewAdaptiveOptimizer creates a new adaptive optimizer
func NewAdaptiveOptimizer(latencyMonitor *LatencyMonitor, bufferManager *AdaptiveBufferManager, config OptimizerConfig, logger zerolog.Logger) *AdaptiveOptimizer {
ctx, cancel := context.WithCancel(context.Background())
optimizer := &AdaptiveOptimizer{
latencyMonitor: latencyMonitor,
bufferManager: bufferManager,
config: config,
logger: logger.With().Str("component", "adaptive-optimizer").Logger(),
ctx: ctx,
cancel: cancel,
}
// Register as latency monitor callback
latencyMonitor.AddOptimizationCallback(optimizer.handleLatencyOptimization)
return optimizer
}
// Start begins the adaptive optimization process
func (ao *AdaptiveOptimizer) Start() {
ao.wg.Add(1)
go ao.optimizationLoop()
ao.logger.Info().Msg("Adaptive optimizer started")
}
// Stop stops the adaptive optimizer
func (ao *AdaptiveOptimizer) Stop() {
ao.cancel()
ao.wg.Wait()
ao.logger.Info().Msg("Adaptive optimizer stopped")
}
// initializeStrategies sets up the available optimization strategies
// handleLatencyOptimization is called when latency optimization is needed
func (ao *AdaptiveOptimizer) handleLatencyOptimization(metrics LatencyMetrics) error {
currentLevel := atomic.LoadInt64(&ao.optimizationLevel)
lastOpt := atomic.LoadInt64(&ao.lastOptimization)
// Check cooldown period
if time.Since(time.Unix(0, lastOpt)) < ao.config.CooldownPeriod {
return nil
}
// Determine if we need to increase or decrease optimization level
targetLevel := ao.calculateTargetOptimizationLevel(metrics)
if targetLevel > currentLevel {
return ao.increaseOptimization(int(targetLevel))
} else if targetLevel < currentLevel {
return ao.decreaseOptimization(int(targetLevel))
}
return nil
}
// calculateTargetOptimizationLevel determines the appropriate optimization level
func (ao *AdaptiveOptimizer) calculateTargetOptimizationLevel(metrics LatencyMetrics) int64 {
// Base calculation on current latency vs target
latencyRatio := float64(metrics.Current) / float64(50*time.Millisecond) // 50ms target
// Adjust based on trend
switch metrics.Trend {
case LatencyTrendIncreasing:
latencyRatio *= 1.2 // Be more aggressive
case LatencyTrendDecreasing:
latencyRatio *= 0.8 // Be less aggressive
case LatencyTrendVolatile:
latencyRatio *= 1.1 // Slightly more aggressive
}
// Apply aggressiveness factor
latencyRatio *= ao.config.Aggressiveness
// Convert to optimization level
targetLevel := int64(latencyRatio * 2) // Scale to 0-10 range
if targetLevel > int64(ao.config.MaxOptimizationLevel) {
targetLevel = int64(ao.config.MaxOptimizationLevel)
}
if targetLevel < 0 {
targetLevel = 0
}
return targetLevel
}
// increaseOptimization applies optimization strategies up to the target level
func (ao *AdaptiveOptimizer) increaseOptimization(targetLevel int) error {
atomic.StoreInt64(&ao.optimizationLevel, int64(targetLevel))
atomic.StoreInt64(&ao.lastOptimization, time.Now().UnixNano())
atomic.AddInt64(&ao.optimizationCount, 1)
return nil
}
// decreaseOptimization rolls back optimization strategies to the target level
func (ao *AdaptiveOptimizer) decreaseOptimization(targetLevel int) error {
atomic.StoreInt64(&ao.optimizationLevel, int64(targetLevel))
atomic.StoreInt64(&ao.lastOptimization, time.Now().UnixNano())
return nil
}
// optimizationLoop runs the main optimization monitoring loop
func (ao *AdaptiveOptimizer) optimizationLoop() {
defer ao.wg.Done()
ticker := time.NewTicker(ao.config.StabilityPeriod)
defer ticker.Stop()
for {
select {
case <-ao.ctx.Done():
return
case <-ticker.C:
ao.checkStability()
}
}
}
// checkStability monitors system stability and rolls back if needed
func (ao *AdaptiveOptimizer) checkStability() {
metrics := ao.latencyMonitor.GetMetrics()
// Check if we need to rollback due to excessive latency
if metrics.Current > ao.config.RollbackThreshold {
currentLevel := int(atomic.LoadInt64(&ao.optimizationLevel))
if currentLevel > 0 {
ao.logger.Warn().Dur("current_latency", metrics.Current).Dur("threshold", ao.config.RollbackThreshold).Msg("Rolling back optimizations due to excessive latency")
if err := ao.decreaseOptimization(currentLevel - 1); err != nil {
ao.logger.Error().Err(err).Msg("Failed to decrease optimization level")
}
}
}
}
// GetOptimizationStats returns current optimization statistics
func (ao *AdaptiveOptimizer) GetOptimizationStats() map[string]interface{} {
return map[string]interface{}{
"optimization_level": atomic.LoadInt64(&ao.optimizationLevel),
"optimization_count": atomic.LoadInt64(&ao.optimizationCount),
"last_optimization": time.Unix(0, atomic.LoadInt64(&ao.lastOptimization)),
}
}
// Strategy implementation methods (stubs for now)

72
internal/audio/api.go Normal file
View File

@ -0,0 +1,72 @@
package audio
import (
"os"
"strings"
"sync/atomic"
"unsafe"
)
var (
// Global audio output supervisor instance
globalOutputSupervisor unsafe.Pointer // *AudioServerSupervisor
)
// isAudioServerProcess detects if we're running as the audio server subprocess
func isAudioServerProcess() bool {
for _, arg := range os.Args {
if strings.Contains(arg, "--audio-output-server") {
return true
}
}
return false
}
// StartAudioStreaming launches the audio stream.
// In audio server subprocess: uses CGO-based audio streaming
// In main process: this should not be called (use StartAudioRelay instead)
func StartAudioStreaming(send func([]byte)) error {
if isAudioServerProcess() {
// Audio server subprocess: use CGO audio processing
return StartAudioOutputStreaming(send)
} else {
// Main process: should use relay system instead
// This is kept for backward compatibility but not recommended
return StartAudioOutputStreaming(send)
}
}
// StopAudioStreaming stops the audio stream.
func StopAudioStreaming() {
if isAudioServerProcess() {
// Audio server subprocess: stop CGO audio processing
StopAudioOutputStreaming()
} else {
// Main process: stop relay if running
StopAudioRelay()
}
}
// StartNonBlockingAudioStreaming is an alias for backward compatibility
func StartNonBlockingAudioStreaming(send func([]byte)) error {
return StartAudioOutputStreaming(send)
}
// StopNonBlockingAudioStreaming is an alias for backward compatibility
func StopNonBlockingAudioStreaming() {
StopAudioOutputStreaming()
}
// SetAudioOutputSupervisor sets the global audio output supervisor
func SetAudioOutputSupervisor(supervisor *AudioServerSupervisor) {
atomic.StorePointer(&globalOutputSupervisor, unsafe.Pointer(supervisor))
}
// GetAudioOutputSupervisor returns the global audio output supervisor
func GetAudioOutputSupervisor() *AudioServerSupervisor {
ptr := atomic.LoadPointer(&globalOutputSupervisor)
if ptr == nil {
return nil
}
return (*AudioServerSupervisor)(ptr)
}

189
internal/audio/audio.go Normal file
View File

@ -0,0 +1,189 @@
package audio
import (
"errors"
"sync/atomic"
"time"
)
var (
ErrAudioAlreadyRunning = errors.New("audio already running")
)
const MaxAudioFrameSize = 1500
// AudioQuality represents different audio quality presets
type AudioQuality int
const (
AudioQualityLow AudioQuality = iota
AudioQualityMedium
AudioQualityHigh
AudioQualityUltra
)
// AudioConfig holds configuration for audio processing
type AudioConfig struct {
Quality AudioQuality
Bitrate int // kbps
SampleRate int // Hz
Channels int
FrameSize time.Duration // ms
}
// AudioMetrics tracks audio performance metrics
type AudioMetrics struct {
FramesReceived int64
FramesDropped int64
BytesProcessed int64
ConnectionDrops int64
LastFrameTime time.Time
AverageLatency time.Duration
}
var (
currentConfig = AudioConfig{
Quality: AudioQualityMedium,
Bitrate: 64,
SampleRate: 48000,
Channels: 2,
FrameSize: 20 * time.Millisecond,
}
currentMicrophoneConfig = AudioConfig{
Quality: AudioQualityMedium,
Bitrate: 32,
SampleRate: 48000,
Channels: 1,
FrameSize: 20 * time.Millisecond,
}
metrics AudioMetrics
)
// qualityPresets defines the base quality configurations
var qualityPresets = map[AudioQuality]struct {
outputBitrate, inputBitrate int
sampleRate, channels int
frameSize time.Duration
}{
AudioQualityLow: {
outputBitrate: 32, inputBitrate: 16,
sampleRate: 22050, channels: 1,
frameSize: 40 * time.Millisecond,
},
AudioQualityMedium: {
outputBitrate: 64, inputBitrate: 32,
sampleRate: 44100, channels: 2,
frameSize: 20 * time.Millisecond,
},
AudioQualityHigh: {
outputBitrate: 128, inputBitrate: 64,
sampleRate: 48000, channels: 2,
frameSize: 20 * time.Millisecond,
},
AudioQualityUltra: {
outputBitrate: 192, inputBitrate: 96,
sampleRate: 48000, channels: 2,
frameSize: 10 * time.Millisecond,
},
}
// GetAudioQualityPresets returns predefined quality configurations for audio output
func GetAudioQualityPresets() map[AudioQuality]AudioConfig {
result := make(map[AudioQuality]AudioConfig)
for quality, preset := range qualityPresets {
result[quality] = AudioConfig{
Quality: quality,
Bitrate: preset.outputBitrate,
SampleRate: preset.sampleRate,
Channels: preset.channels,
FrameSize: preset.frameSize,
}
}
return result
}
// GetMicrophoneQualityPresets returns predefined quality configurations for microphone input
func GetMicrophoneQualityPresets() map[AudioQuality]AudioConfig {
result := make(map[AudioQuality]AudioConfig)
for quality, preset := range qualityPresets {
result[quality] = AudioConfig{
Quality: quality,
Bitrate: preset.inputBitrate,
SampleRate: func() int {
if quality == AudioQualityLow {
return 16000
}
return preset.sampleRate
}(),
Channels: 1, // Microphone is always mono
FrameSize: preset.frameSize,
}
}
return result
}
// SetAudioQuality updates the current audio quality configuration
func SetAudioQuality(quality AudioQuality) {
presets := GetAudioQualityPresets()
if config, exists := presets[quality]; exists {
currentConfig = config
}
}
// GetAudioConfig returns the current audio configuration
func GetAudioConfig() AudioConfig {
return currentConfig
}
// SetMicrophoneQuality updates the current microphone quality configuration
func SetMicrophoneQuality(quality AudioQuality) {
presets := GetMicrophoneQualityPresets()
if config, exists := presets[quality]; exists {
currentMicrophoneConfig = config
}
}
// GetMicrophoneConfig returns the current microphone configuration
func GetMicrophoneConfig() AudioConfig {
return currentMicrophoneConfig
}
// GetAudioMetrics returns current audio metrics
func GetAudioMetrics() AudioMetrics {
// Get base metrics
framesReceived := atomic.LoadInt64(&metrics.FramesReceived)
framesDropped := atomic.LoadInt64(&metrics.FramesDropped)
// If audio relay is running, use relay stats instead
if IsAudioRelayRunning() {
relayReceived, relayDropped := GetAudioRelayStats()
framesReceived = relayReceived
framesDropped = relayDropped
}
return AudioMetrics{
FramesReceived: framesReceived,
FramesDropped: framesDropped,
BytesProcessed: atomic.LoadInt64(&metrics.BytesProcessed),
LastFrameTime: metrics.LastFrameTime,
ConnectionDrops: atomic.LoadInt64(&metrics.ConnectionDrops),
AverageLatency: metrics.AverageLatency,
}
}
// RecordFrameReceived increments the frames received counter
func RecordFrameReceived(bytes int) {
atomic.AddInt64(&metrics.FramesReceived, 1)
atomic.AddInt64(&metrics.BytesProcessed, int64(bytes))
metrics.LastFrameTime = time.Now()
}
// RecordFrameDropped increments the frames dropped counter
func RecordFrameDropped() {
atomic.AddInt64(&metrics.FramesDropped, 1)
}
// RecordConnectionDrop increments the connection drops counter
func RecordConnectionDrop() {
atomic.AddInt64(&metrics.ConnectionDrops, 1)
}

View File

@ -0,0 +1,22 @@
package audio
import (
"sync"
)
var audioMuteState struct {
muted bool
mu sync.RWMutex
}
func SetAudioMuted(muted bool) {
audioMuteState.mu.Lock()
audioMuteState.muted = muted
audioMuteState.mu.Unlock()
}
func IsAudioMuted() bool {
audioMuteState.mu.RLock()
defer audioMuteState.mu.RUnlock()
return audioMuteState.muted
}

View File

@ -0,0 +1,314 @@
//go:build cgo
package audio
import (
"context"
"runtime"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// BatchAudioProcessor manages batched CGO operations to reduce syscall overhead
type BatchAudioProcessor struct {
// Statistics - MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
stats BatchAudioStats
// Control
ctx context.Context
cancel context.CancelFunc
logger *zerolog.Logger
batchSize int
batchDuration time.Duration
// Batch queues and state (atomic for lock-free access)
readQueue chan batchReadRequest
initialized int32
running int32
threadPinned int32
// Buffers (pre-allocated to avoid allocation overhead)
readBufPool *sync.Pool
}
type BatchAudioStats struct {
// int64 fields MUST be first for ARM32 alignment
BatchedReads int64
SingleReads int64
BatchedFrames int64
SingleFrames int64
CGOCallsReduced int64
OSThreadPinTime time.Duration // time.Duration is int64 internally
LastBatchTime time.Time
}
type batchReadRequest struct {
buffer []byte
resultChan chan batchReadResult
timestamp time.Time
}
type batchReadResult struct {
length int
err error
}
// NewBatchAudioProcessor creates a new batch audio processor
func NewBatchAudioProcessor(batchSize int, batchDuration time.Duration) *BatchAudioProcessor {
ctx, cancel := context.WithCancel(context.Background())
logger := logging.GetDefaultLogger().With().Str("component", "batch-audio").Logger()
processor := &BatchAudioProcessor{
ctx: ctx,
cancel: cancel,
logger: &logger,
batchSize: batchSize,
batchDuration: batchDuration,
readQueue: make(chan batchReadRequest, batchSize*2),
readBufPool: &sync.Pool{
New: func() interface{} {
return make([]byte, 1500) // Max audio frame size
},
},
}
return processor
}
// Start initializes and starts the batch processor
func (bap *BatchAudioProcessor) Start() error {
if !atomic.CompareAndSwapInt32(&bap.running, 0, 1) {
return nil // Already running
}
// Initialize CGO resources once per processor lifecycle
if !atomic.CompareAndSwapInt32(&bap.initialized, 0, 1) {
return nil // Already initialized
}
// Start batch processing goroutines
go bap.batchReadProcessor()
bap.logger.Info().Int("batch_size", bap.batchSize).
Dur("batch_duration", bap.batchDuration).
Msg("batch audio processor started")
return nil
}
// Stop cleanly shuts down the batch processor
func (bap *BatchAudioProcessor) Stop() {
if !atomic.CompareAndSwapInt32(&bap.running, 1, 0) {
return // Already stopped
}
bap.cancel()
// Wait for processing to complete
time.Sleep(bap.batchDuration + 10*time.Millisecond)
bap.logger.Info().Msg("batch audio processor stopped")
}
// BatchReadEncode performs batched audio read and encode operations
func (bap *BatchAudioProcessor) BatchReadEncode(buffer []byte) (int, error) {
if atomic.LoadInt32(&bap.running) == 0 {
// Fallback to single operation if batch processor is not running
atomic.AddInt64(&bap.stats.SingleReads, 1)
atomic.AddInt64(&bap.stats.SingleFrames, 1)
return CGOAudioReadEncode(buffer)
}
resultChan := make(chan batchReadResult, 1)
request := batchReadRequest{
buffer: buffer,
resultChan: resultChan,
timestamp: time.Now(),
}
select {
case bap.readQueue <- request:
// Successfully queued
case <-time.After(5 * time.Millisecond):
// Queue is full or blocked, fallback to single operation
atomic.AddInt64(&bap.stats.SingleReads, 1)
atomic.AddInt64(&bap.stats.SingleFrames, 1)
return CGOAudioReadEncode(buffer)
}
// Wait for result
select {
case result := <-resultChan:
return result.length, result.err
case <-time.After(50 * time.Millisecond):
// Timeout, fallback to single operation
atomic.AddInt64(&bap.stats.SingleReads, 1)
atomic.AddInt64(&bap.stats.SingleFrames, 1)
return CGOAudioReadEncode(buffer)
}
}
// batchReadProcessor processes batched read operations
func (bap *BatchAudioProcessor) batchReadProcessor() {
defer bap.logger.Debug().Msg("batch read processor stopped")
ticker := time.NewTicker(bap.batchDuration)
defer ticker.Stop()
var batch []batchReadRequest
batch = make([]batchReadRequest, 0, bap.batchSize)
for atomic.LoadInt32(&bap.running) == 1 {
select {
case <-bap.ctx.Done():
return
case req := <-bap.readQueue:
batch = append(batch, req)
if len(batch) >= bap.batchSize {
bap.processBatchRead(batch)
batch = batch[:0] // Clear slice but keep capacity
}
case <-ticker.C:
if len(batch) > 0 {
bap.processBatchRead(batch)
batch = batch[:0] // Clear slice but keep capacity
}
}
}
// Process any remaining requests
if len(batch) > 0 {
bap.processBatchRead(batch)
}
}
// processBatchRead processes a batch of read requests efficiently
func (bap *BatchAudioProcessor) processBatchRead(batch []batchReadRequest) {
if len(batch) == 0 {
return
}
// Pin to OS thread for the entire batch to minimize thread switching overhead
start := time.Now()
if atomic.CompareAndSwapInt32(&bap.threadPinned, 0, 1) {
runtime.LockOSThread()
// Set high priority for batch audio processing
if err := SetAudioThreadPriority(); err != nil {
bap.logger.Warn().Err(err).Msg("Failed to set batch audio processing priority")
}
defer func() {
if err := ResetThreadPriority(); err != nil {
bap.logger.Warn().Err(err).Msg("Failed to reset thread priority")
}
runtime.UnlockOSThread()
atomic.StoreInt32(&bap.threadPinned, 0)
bap.stats.OSThreadPinTime += time.Since(start)
}()
}
batchSize := len(batch)
atomic.AddInt64(&bap.stats.BatchedReads, 1)
atomic.AddInt64(&bap.stats.BatchedFrames, int64(batchSize))
if batchSize > 1 {
atomic.AddInt64(&bap.stats.CGOCallsReduced, int64(batchSize-1))
}
// Process each request in the batch
for _, req := range batch {
length, err := CGOAudioReadEncode(req.buffer)
result := batchReadResult{
length: length,
err: err,
}
// Send result back (non-blocking)
select {
case req.resultChan <- result:
default:
// Requestor timed out, drop result
}
}
bap.stats.LastBatchTime = time.Now()
}
// GetStats returns current batch processor statistics
func (bap *BatchAudioProcessor) GetStats() BatchAudioStats {
return BatchAudioStats{
BatchedReads: atomic.LoadInt64(&bap.stats.BatchedReads),
SingleReads: atomic.LoadInt64(&bap.stats.SingleReads),
BatchedFrames: atomic.LoadInt64(&bap.stats.BatchedFrames),
SingleFrames: atomic.LoadInt64(&bap.stats.SingleFrames),
CGOCallsReduced: atomic.LoadInt64(&bap.stats.CGOCallsReduced),
OSThreadPinTime: bap.stats.OSThreadPinTime,
LastBatchTime: bap.stats.LastBatchTime,
}
}
// IsRunning returns whether the batch processor is running
func (bap *BatchAudioProcessor) IsRunning() bool {
return atomic.LoadInt32(&bap.running) == 1
}
// Global batch processor instance
var (
globalBatchProcessor unsafe.Pointer // *BatchAudioProcessor
batchProcessorInitialized int32
)
// GetBatchAudioProcessor returns the global batch processor instance
func GetBatchAudioProcessor() *BatchAudioProcessor {
ptr := atomic.LoadPointer(&globalBatchProcessor)
if ptr != nil {
return (*BatchAudioProcessor)(ptr)
}
// Initialize on first use
if atomic.CompareAndSwapInt32(&batchProcessorInitialized, 0, 1) {
processor := NewBatchAudioProcessor(4, 5*time.Millisecond) // 4 frames per batch, 5ms timeout
atomic.StorePointer(&globalBatchProcessor, unsafe.Pointer(processor))
return processor
}
// Another goroutine initialized it, try again
ptr = atomic.LoadPointer(&globalBatchProcessor)
if ptr != nil {
return (*BatchAudioProcessor)(ptr)
}
// Fallback: create a new processor (should rarely happen)
return NewBatchAudioProcessor(4, 5*time.Millisecond)
}
// EnableBatchAudioProcessing enables the global batch processor
func EnableBatchAudioProcessing() error {
processor := GetBatchAudioProcessor()
return processor.Start()
}
// DisableBatchAudioProcessing disables the global batch processor
func DisableBatchAudioProcessing() {
ptr := atomic.LoadPointer(&globalBatchProcessor)
if ptr != nil {
processor := (*BatchAudioProcessor)(ptr)
processor.Stop()
}
}
// BatchCGOAudioReadEncode is a batched version of CGOAudioReadEncode
func BatchCGOAudioReadEncode(buffer []byte) (int, error) {
processor := GetBatchAudioProcessor()
if processor != nil && processor.IsRunning() {
return processor.BatchReadEncode(buffer)
}
return CGOAudioReadEncode(buffer)
}

View File

@ -0,0 +1,212 @@
package audio
import (
"sync"
"sync/atomic"
)
type AudioBufferPool struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
currentSize int64 // Current pool size (atomic)
hitCount int64 // Pool hit counter (atomic)
missCount int64 // Pool miss counter (atomic)
// Other fields
pool sync.Pool
bufferSize int
maxPoolSize int
mutex sync.RWMutex
// Memory optimization fields
preallocated []*[]byte // Pre-allocated buffers for immediate use
preallocSize int // Number of pre-allocated buffers
}
func NewAudioBufferPool(bufferSize int) *AudioBufferPool {
// Pre-allocate 20% of max pool size for immediate availability
preallocSize := 20
preallocated := make([]*[]byte, 0, preallocSize)
// Pre-allocate buffers to reduce initial allocation overhead
for i := 0; i < preallocSize; i++ {
buf := make([]byte, 0, bufferSize)
preallocated = append(preallocated, &buf)
}
return &AudioBufferPool{
bufferSize: bufferSize,
maxPoolSize: 100, // Limit pool size to prevent excessive memory usage
preallocated: preallocated,
preallocSize: preallocSize,
pool: sync.Pool{
New: func() interface{} {
return make([]byte, 0, bufferSize)
},
},
}
}
func (p *AudioBufferPool) Get() []byte {
// First try pre-allocated buffers for fastest access
p.mutex.Lock()
if len(p.preallocated) > 0 {
buf := p.preallocated[len(p.preallocated)-1]
p.preallocated = p.preallocated[:len(p.preallocated)-1]
p.mutex.Unlock()
atomic.AddInt64(&p.hitCount, 1)
return (*buf)[:0] // Reset length but keep capacity
}
p.mutex.Unlock()
// Try sync.Pool next
if buf := p.pool.Get(); buf != nil {
bufPtr := buf.(*[]byte)
// Update pool size counter when retrieving from pool
p.mutex.Lock()
if p.currentSize > 0 {
p.currentSize--
}
p.mutex.Unlock()
atomic.AddInt64(&p.hitCount, 1)
return (*bufPtr)[:0] // Reset length but keep capacity
}
// Last resort: allocate new buffer
atomic.AddInt64(&p.missCount, 1)
return make([]byte, 0, p.bufferSize)
}
func (p *AudioBufferPool) Put(buf []byte) {
if cap(buf) < p.bufferSize {
return // Buffer too small, don't pool it
}
// Reset buffer for reuse
resetBuf := buf[:0]
// First try to return to pre-allocated pool for fastest reuse
p.mutex.Lock()
if len(p.preallocated) < p.preallocSize {
p.preallocated = append(p.preallocated, &resetBuf)
p.mutex.Unlock()
return
}
p.mutex.Unlock()
// Check sync.Pool size limit to prevent excessive memory usage
p.mutex.RLock()
currentSize := p.currentSize
p.mutex.RUnlock()
if currentSize >= int64(p.maxPoolSize) {
return // Pool is full, let GC handle this buffer
}
// Return to sync.Pool
p.pool.Put(&resetBuf)
// Update pool size counter
p.mutex.Lock()
p.currentSize++
p.mutex.Unlock()
}
var (
audioFramePool = NewAudioBufferPool(1500)
audioControlPool = NewAudioBufferPool(64)
)
func GetAudioFrameBuffer() []byte {
return audioFramePool.Get()
}
func PutAudioFrameBuffer(buf []byte) {
audioFramePool.Put(buf)
}
func GetAudioControlBuffer() []byte {
return audioControlPool.Get()
}
func PutAudioControlBuffer(buf []byte) {
audioControlPool.Put(buf)
}
// GetPoolStats returns detailed statistics about this buffer pool
func (p *AudioBufferPool) GetPoolStats() AudioBufferPoolDetailedStats {
p.mutex.RLock()
preallocatedCount := len(p.preallocated)
currentSize := p.currentSize
p.mutex.RUnlock()
hitCount := atomic.LoadInt64(&p.hitCount)
missCount := atomic.LoadInt64(&p.missCount)
totalRequests := hitCount + missCount
var hitRate float64
if totalRequests > 0 {
hitRate = float64(hitCount) / float64(totalRequests) * 100
}
return AudioBufferPoolDetailedStats{
BufferSize: p.bufferSize,
MaxPoolSize: p.maxPoolSize,
CurrentPoolSize: currentSize,
PreallocatedCount: int64(preallocatedCount),
PreallocatedMax: int64(p.preallocSize),
HitCount: hitCount,
MissCount: missCount,
HitRate: hitRate,
}
}
// AudioBufferPoolDetailedStats provides detailed pool statistics
type AudioBufferPoolDetailedStats struct {
BufferSize int
MaxPoolSize int
CurrentPoolSize int64
PreallocatedCount int64
PreallocatedMax int64
HitCount int64
MissCount int64
HitRate float64 // Percentage
}
// GetAudioBufferPoolStats returns statistics about the audio buffer pools
type AudioBufferPoolStats struct {
FramePoolSize int64
FramePoolMax int
ControlPoolSize int64
ControlPoolMax int
// Enhanced statistics
FramePoolHitRate float64
ControlPoolHitRate float64
FramePoolDetails AudioBufferPoolDetailedStats
ControlPoolDetails AudioBufferPoolDetailedStats
}
func GetAudioBufferPoolStats() AudioBufferPoolStats {
audioFramePool.mutex.RLock()
frameSize := audioFramePool.currentSize
frameMax := audioFramePool.maxPoolSize
audioFramePool.mutex.RUnlock()
audioControlPool.mutex.RLock()
controlSize := audioControlPool.currentSize
controlMax := audioControlPool.maxPoolSize
audioControlPool.mutex.RUnlock()
// Get detailed statistics
frameDetails := audioFramePool.GetPoolStats()
controlDetails := audioControlPool.GetPoolStats()
return AudioBufferPoolStats{
FramePoolSize: frameSize,
FramePoolMax: frameMax,
ControlPoolSize: controlSize,
ControlPoolMax: controlMax,
FramePoolHitRate: frameDetails.HitRate,
ControlPoolHitRate: controlDetails.HitRate,
FramePoolDetails: frameDetails,
ControlPoolDetails: controlDetails,
}
}

494
internal/audio/cgo_audio.go Normal file
View File

@ -0,0 +1,494 @@
//go:build cgo
package audio
import (
"errors"
"unsafe"
)
/*
#cgo CFLAGS: -I$HOME/.jetkvm/audio-libs/alsa-lib-$ALSA_VERSION/include -I$HOME/.jetkvm/audio-libs/opus-$OPUS_VERSION/include -I$HOME/.jetkvm/audio-libs/opus-$OPUS_VERSION/celt
#cgo LDFLAGS: -L$HOME/.jetkvm/audio-libs/alsa-lib-$ALSA_VERSION/src/.libs -lasound -L$HOME/.jetkvm/audio-libs/opus-$OPUS_VERSION/.libs -lopus -lm -ldl -static
#include <alsa/asoundlib.h>
#include <opus.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
// C state for ALSA/Opus with safety flags
static snd_pcm_t *pcm_handle = NULL;
static snd_pcm_t *pcm_playback_handle = NULL;
static OpusEncoder *encoder = NULL;
static OpusDecoder *decoder = NULL;
// Optimized Opus encoder settings for ARM Cortex-A7
static int opus_bitrate = 96000; // Increased for better quality
static int opus_complexity = 3; // Reduced for ARM performance
static int opus_vbr = 1; // Variable bitrate enabled
static int opus_vbr_constraint = 1; // Constrained VBR for consistent latency
static int opus_signal_type = OPUS_SIGNAL_MUSIC; // Optimized for general audio
static int opus_bandwidth = OPUS_BANDWIDTH_FULLBAND; // Full bandwidth
static int opus_dtx = 0; // Disable DTX for real-time audio
static int sample_rate = 48000;
static int channels = 2;
static int frame_size = 960; // 20ms for 48kHz
static int max_packet_size = 1500;
// State tracking to prevent race conditions during rapid start/stop
static volatile int capture_initializing = 0;
static volatile int capture_initialized = 0;
static volatile int playback_initializing = 0;
static volatile int playback_initialized = 0;
// Safe ALSA device opening with retry logic
static int safe_alsa_open(snd_pcm_t **handle, const char *device, snd_pcm_stream_t stream) {
int attempts = 3;
int err;
while (attempts-- > 0) {
err = snd_pcm_open(handle, device, stream, SND_PCM_NONBLOCK);
if (err >= 0) {
// Switch to blocking mode after successful open
snd_pcm_nonblock(*handle, 0);
return 0;
}
if (err == -EBUSY && attempts > 0) {
// Device busy, wait and retry
usleep(50000); // 50ms
continue;
}
break;
}
return err;
}
// Optimized ALSA configuration with stack allocation and performance tuning
static int configure_alsa_device(snd_pcm_t *handle, const char *device_name) {
snd_pcm_hw_params_t *params;
snd_pcm_sw_params_t *sw_params;
int err;
if (!handle) return -1;
// Use stack allocation for better performance
snd_pcm_hw_params_alloca(&params);
snd_pcm_sw_params_alloca(&sw_params);
// Hardware parameters
err = snd_pcm_hw_params_any(handle, params);
if (err < 0) return err;
err = snd_pcm_hw_params_set_access(handle, params, SND_PCM_ACCESS_RW_INTERLEAVED);
if (err < 0) return err;
err = snd_pcm_hw_params_set_format(handle, params, SND_PCM_FORMAT_S16_LE);
if (err < 0) return err;
err = snd_pcm_hw_params_set_channels(handle, params, channels);
if (err < 0) return err;
// Set exact rate for better performance
err = snd_pcm_hw_params_set_rate(handle, params, sample_rate, 0);
if (err < 0) {
// Fallback to near rate if exact fails
unsigned int rate = sample_rate;
err = snd_pcm_hw_params_set_rate_near(handle, params, &rate, 0);
if (err < 0) return err;
}
// Optimize buffer sizes for low latency
snd_pcm_uframes_t period_size = frame_size;
err = snd_pcm_hw_params_set_period_size_near(handle, params, &period_size, 0);
if (err < 0) return err;
// Set buffer size to 4 periods for good latency/stability balance
snd_pcm_uframes_t buffer_size = period_size * 4;
err = snd_pcm_hw_params_set_buffer_size_near(handle, params, &buffer_size);
if (err < 0) return err;
err = snd_pcm_hw_params(handle, params);
if (err < 0) return err;
// Software parameters for optimal performance
err = snd_pcm_sw_params_current(handle, sw_params);
if (err < 0) return err;
// Start playback/capture when buffer is period_size frames
err = snd_pcm_sw_params_set_start_threshold(handle, sw_params, period_size);
if (err < 0) return err;
// Allow transfers when at least period_size frames are available
err = snd_pcm_sw_params_set_avail_min(handle, sw_params, period_size);
if (err < 0) return err;
err = snd_pcm_sw_params(handle, sw_params);
if (err < 0) return err;
return snd_pcm_prepare(handle);
}
// Initialize ALSA and Opus encoder with improved safety
int jetkvm_audio_init() {
int err;
// Prevent concurrent initialization
if (__sync_bool_compare_and_swap(&capture_initializing, 0, 1) == 0) {
return -EBUSY; // Already initializing
}
// Check if already initialized
if (capture_initialized) {
capture_initializing = 0;
return 0;
}
// Clean up any existing resources first
if (encoder) {
opus_encoder_destroy(encoder);
encoder = NULL;
}
if (pcm_handle) {
snd_pcm_close(pcm_handle);
pcm_handle = NULL;
}
// Try to open ALSA capture device
err = safe_alsa_open(&pcm_handle, "hw:1,0", SND_PCM_STREAM_CAPTURE);
if (err < 0) {
capture_initializing = 0;
return -1;
}
// Configure the device
err = configure_alsa_device(pcm_handle, "capture");
if (err < 0) {
snd_pcm_close(pcm_handle);
pcm_handle = NULL;
capture_initializing = 0;
return -1;
}
// Initialize Opus encoder with optimized settings
int opus_err = 0;
encoder = opus_encoder_create(sample_rate, channels, OPUS_APPLICATION_AUDIO, &opus_err);
if (!encoder || opus_err != OPUS_OK) {
if (pcm_handle) { snd_pcm_close(pcm_handle); pcm_handle = NULL; }
capture_initializing = 0;
return -2;
}
// Apply optimized Opus encoder settings
opus_encoder_ctl(encoder, OPUS_SET_BITRATE(opus_bitrate));
opus_encoder_ctl(encoder, OPUS_SET_COMPLEXITY(opus_complexity));
opus_encoder_ctl(encoder, OPUS_SET_VBR(opus_vbr));
opus_encoder_ctl(encoder, OPUS_SET_VBR_CONSTRAINT(opus_vbr_constraint));
opus_encoder_ctl(encoder, OPUS_SET_SIGNAL(opus_signal_type));
opus_encoder_ctl(encoder, OPUS_SET_BANDWIDTH(opus_bandwidth));
opus_encoder_ctl(encoder, OPUS_SET_DTX(opus_dtx));
// Enable packet loss concealment for better resilience
opus_encoder_ctl(encoder, OPUS_SET_PACKET_LOSS_PERC(5));
// Set prediction disabled for lower latency
opus_encoder_ctl(encoder, OPUS_SET_PREDICTION_DISABLED(1));
capture_initialized = 1;
capture_initializing = 0;
return 0;
}
// Read and encode one frame with enhanced error handling
int jetkvm_audio_read_encode(void *opus_buf) {
short pcm_buffer[1920]; // max 2ch*960
unsigned char *out = (unsigned char*)opus_buf;
int err = 0;
// Safety checks
if (!capture_initialized || !pcm_handle || !encoder || !opus_buf) {
return -1;
}
int pcm_rc = snd_pcm_readi(pcm_handle, pcm_buffer, frame_size);
// Handle ALSA errors with enhanced recovery
if (pcm_rc < 0) {
if (pcm_rc == -EPIPE) {
// Buffer underrun - try to recover
err = snd_pcm_prepare(pcm_handle);
if (err < 0) return -1;
pcm_rc = snd_pcm_readi(pcm_handle, pcm_buffer, frame_size);
if (pcm_rc < 0) return -1;
} else if (pcm_rc == -EAGAIN) {
// No data available - return 0 to indicate no frame
return 0;
} else if (pcm_rc == -ESTRPIPE) {
// Device suspended, try to resume
while ((err = snd_pcm_resume(pcm_handle)) == -EAGAIN) {
usleep(1000); // 1ms
}
if (err < 0) {
err = snd_pcm_prepare(pcm_handle);
if (err < 0) return -1;
}
return 0; // Skip this frame
} else {
// Other error - return error code
return -1;
}
}
// If we got fewer frames than expected, pad with silence
if (pcm_rc < frame_size) {
memset(&pcm_buffer[pcm_rc * channels], 0, (frame_size - pcm_rc) * channels * sizeof(short));
}
int nb_bytes = opus_encode(encoder, pcm_buffer, frame_size, out, max_packet_size);
return nb_bytes;
}
// Initialize ALSA playback with improved safety
int jetkvm_audio_playback_init() {
int err;
// Prevent concurrent initialization
if (__sync_bool_compare_and_swap(&playback_initializing, 0, 1) == 0) {
return -EBUSY; // Already initializing
}
// Check if already initialized
if (playback_initialized) {
playback_initializing = 0;
return 0;
}
// Clean up any existing resources first
if (decoder) {
opus_decoder_destroy(decoder);
decoder = NULL;
}
if (pcm_playback_handle) {
snd_pcm_close(pcm_playback_handle);
pcm_playback_handle = NULL;
}
// Try to open the USB gadget audio device for playback
err = safe_alsa_open(&pcm_playback_handle, "hw:1,0", SND_PCM_STREAM_PLAYBACK);
if (err < 0) {
// Fallback to default device
err = safe_alsa_open(&pcm_playback_handle, "default", SND_PCM_STREAM_PLAYBACK);
if (err < 0) {
playback_initializing = 0;
return -1;
}
}
// Configure the device
err = configure_alsa_device(pcm_playback_handle, "playback");
if (err < 0) {
snd_pcm_close(pcm_playback_handle);
pcm_playback_handle = NULL;
playback_initializing = 0;
return -1;
}
// Initialize Opus decoder
int opus_err = 0;
decoder = opus_decoder_create(sample_rate, channels, &opus_err);
if (!decoder || opus_err != OPUS_OK) {
snd_pcm_close(pcm_playback_handle);
pcm_playback_handle = NULL;
playback_initializing = 0;
return -2;
}
playback_initialized = 1;
playback_initializing = 0;
return 0;
}
// Decode Opus and write PCM with enhanced error handling
int jetkvm_audio_decode_write(void *opus_buf, int opus_size) {
short pcm_buffer[1920]; // max 2ch*960
unsigned char *in = (unsigned char*)opus_buf;
int err = 0;
// Safety checks
if (!playback_initialized || !pcm_playback_handle || !decoder || !opus_buf || opus_size <= 0) {
return -1;
}
// Additional bounds checking
if (opus_size > max_packet_size) {
return -1;
}
// Decode Opus to PCM
int pcm_frames = opus_decode(decoder, in, opus_size, pcm_buffer, frame_size, 0);
if (pcm_frames < 0) return -1;
// Write PCM to playback device with enhanced recovery
int pcm_rc = snd_pcm_writei(pcm_playback_handle, pcm_buffer, pcm_frames);
if (pcm_rc < 0) {
if (pcm_rc == -EPIPE) {
// Buffer underrun - try to recover
err = snd_pcm_prepare(pcm_playback_handle);
if (err < 0) return -2;
pcm_rc = snd_pcm_writei(pcm_playback_handle, pcm_buffer, pcm_frames);
} else if (pcm_rc == -ESTRPIPE) {
// Device suspended, try to resume
while ((err = snd_pcm_resume(pcm_playback_handle)) == -EAGAIN) {
usleep(1000); // 1ms
}
if (err < 0) {
err = snd_pcm_prepare(pcm_playback_handle);
if (err < 0) return -2;
}
return 0; // Skip this frame
}
if (pcm_rc < 0) return -2;
}
return pcm_frames;
}
// Safe playback cleanup with double-close protection
void jetkvm_audio_playback_close() {
// Wait for any ongoing operations to complete
while (playback_initializing) {
usleep(1000); // 1ms
}
// Atomic check and set to prevent double cleanup
if (__sync_bool_compare_and_swap(&playback_initialized, 1, 0) == 0) {
return; // Already cleaned up
}
if (decoder) {
opus_decoder_destroy(decoder);
decoder = NULL;
}
if (pcm_playback_handle) {
snd_pcm_drain(pcm_playback_handle);
snd_pcm_close(pcm_playback_handle);
pcm_playback_handle = NULL;
}
}
// Safe capture cleanup
void jetkvm_audio_close() {
// Wait for any ongoing operations to complete
while (capture_initializing) {
usleep(1000); // 1ms
}
capture_initialized = 0;
if (encoder) {
opus_encoder_destroy(encoder);
encoder = NULL;
}
if (pcm_handle) {
snd_pcm_drop(pcm_handle); // Drop pending samples
snd_pcm_close(pcm_handle);
pcm_handle = NULL;
}
// Also clean up playback
jetkvm_audio_playback_close();
}
*/
import "C"
// Optimized Go wrappers with reduced overhead
var (
errAudioInitFailed = errors.New("failed to init ALSA/Opus")
errBufferTooSmall = errors.New("buffer too small")
errAudioReadEncode = errors.New("audio read/encode error")
errAudioDecodeWrite = errors.New("audio decode/write error")
errAudioPlaybackInit = errors.New("failed to init ALSA playback/Opus decoder")
errEmptyBuffer = errors.New("empty buffer")
errNilBuffer = errors.New("nil buffer")
errBufferTooLarge = errors.New("buffer too large")
errInvalidBufferPtr = errors.New("invalid buffer pointer")
)
func cgoAudioInit() error {
ret := C.jetkvm_audio_init()
if ret != 0 {
return errAudioInitFailed
}
return nil
}
func cgoAudioClose() {
C.jetkvm_audio_close()
}
func cgoAudioReadEncode(buf []byte) (int, error) {
if len(buf) < 1276 {
return 0, errBufferTooSmall
}
n := C.jetkvm_audio_read_encode(unsafe.Pointer(&buf[0]))
if n < 0 {
return 0, errAudioReadEncode
}
if n == 0 {
return 0, nil // No data available
}
return int(n), nil
}
// Audio playback functions
func cgoAudioPlaybackInit() error {
ret := C.jetkvm_audio_playback_init()
if ret != 0 {
return errAudioPlaybackInit
}
return nil
}
func cgoAudioPlaybackClose() {
C.jetkvm_audio_playback_close()
}
func cgoAudioDecodeWrite(buf []byte) (int, error) {
if len(buf) == 0 {
return 0, errEmptyBuffer
}
if buf == nil {
return 0, errNilBuffer
}
if len(buf) > 4096 {
return 0, errBufferTooLarge
}
bufPtr := unsafe.Pointer(&buf[0])
if bufPtr == nil {
return 0, errInvalidBufferPtr
}
defer func() {
if r := recover(); r != nil {
_ = r
}
}()
n := C.jetkvm_audio_decode_write(bufPtr, C.int(len(buf)))
if n < 0 {
return 0, errAudioDecodeWrite
}
return int(n), nil
}
// CGO function aliases
var (
CGOAudioInit = cgoAudioInit
CGOAudioClose = cgoAudioClose
CGOAudioReadEncode = cgoAudioReadEncode
CGOAudioPlaybackInit = cgoAudioPlaybackInit
CGOAudioPlaybackClose = cgoAudioPlaybackClose
CGOAudioDecodeWrite = cgoAudioDecodeWrite
)

View File

@ -0,0 +1,42 @@
//go:build !cgo
package audio
import "errors"
// Stub implementations for linting (no CGO dependencies)
func cgoAudioInit() error {
return errors.New("audio not available in lint mode")
}
func cgoAudioClose() {
// No-op
}
func cgoAudioReadEncode(buf []byte) (int, error) {
return 0, errors.New("audio not available in lint mode")
}
func cgoAudioPlaybackInit() error {
return errors.New("audio not available in lint mode")
}
func cgoAudioPlaybackClose() {
// No-op
}
func cgoAudioDecodeWrite(buf []byte) (int, error) {
return 0, errors.New("audio not available in lint mode")
}
// Uppercase aliases for external API compatibility
var (
CGOAudioInit = cgoAudioInit
CGOAudioClose = cgoAudioClose
CGOAudioReadEncode = cgoAudioReadEncode
CGOAudioPlaybackInit = cgoAudioPlaybackInit
CGOAudioPlaybackClose = cgoAudioPlaybackClose
CGOAudioDecodeWrite = cgoAudioDecodeWrite
)

29
internal/audio/config.go Normal file
View File

@ -0,0 +1,29 @@
package audio
import "time"
// MonitoringConfig contains configuration constants for audio monitoring
type MonitoringConfig struct {
// MetricsUpdateInterval defines how often metrics are collected and broadcast
MetricsUpdateInterval time.Duration
}
// DefaultMonitoringConfig returns the default monitoring configuration
func DefaultMonitoringConfig() MonitoringConfig {
return MonitoringConfig{
MetricsUpdateInterval: 1000 * time.Millisecond, // 1 second interval
}
}
// Global monitoring configuration instance
var monitoringConfig = DefaultMonitoringConfig()
// GetMetricsUpdateInterval returns the current metrics update interval
func GetMetricsUpdateInterval() time.Duration {
return monitoringConfig.MetricsUpdateInterval
}
// SetMetricsUpdateInterval sets the metrics update interval
func SetMetricsUpdateInterval(interval time.Duration) {
monitoringConfig.MetricsUpdateInterval = interval
}

491
internal/audio/events.go Normal file
View File

@ -0,0 +1,491 @@
package audio
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// AudioEventType represents different types of audio events
type AudioEventType string
const (
AudioEventMuteChanged AudioEventType = "audio-mute-changed"
AudioEventMetricsUpdate AudioEventType = "audio-metrics-update"
AudioEventMicrophoneState AudioEventType = "microphone-state-changed"
AudioEventMicrophoneMetrics AudioEventType = "microphone-metrics-update"
AudioEventProcessMetrics AudioEventType = "audio-process-metrics"
AudioEventMicProcessMetrics AudioEventType = "microphone-process-metrics"
)
// AudioEvent represents a WebSocket audio event
type AudioEvent struct {
Type AudioEventType `json:"type"`
Data interface{} `json:"data"`
}
// AudioMuteData represents audio mute state change data
type AudioMuteData struct {
Muted bool `json:"muted"`
}
// AudioMetricsData represents audio metrics data
type AudioMetricsData struct {
FramesReceived int64 `json:"frames_received"`
FramesDropped int64 `json:"frames_dropped"`
BytesProcessed int64 `json:"bytes_processed"`
LastFrameTime string `json:"last_frame_time"`
ConnectionDrops int64 `json:"connection_drops"`
AverageLatency string `json:"average_latency"`
}
// MicrophoneStateData represents microphone state data
type MicrophoneStateData struct {
Running bool `json:"running"`
SessionActive bool `json:"session_active"`
}
// MicrophoneMetricsData represents microphone metrics data
type MicrophoneMetricsData struct {
FramesSent int64 `json:"frames_sent"`
FramesDropped int64 `json:"frames_dropped"`
BytesProcessed int64 `json:"bytes_processed"`
LastFrameTime string `json:"last_frame_time"`
ConnectionDrops int64 `json:"connection_drops"`
AverageLatency string `json:"average_latency"`
}
// ProcessMetricsData represents process metrics data for WebSocket events
type ProcessMetricsData struct {
PID int `json:"pid"`
CPUPercent float64 `json:"cpu_percent"`
MemoryRSS int64 `json:"memory_rss"`
MemoryVMS int64 `json:"memory_vms"`
MemoryPercent float64 `json:"memory_percent"`
Running bool `json:"running"`
ProcessName string `json:"process_name"`
}
// AudioEventSubscriber represents a WebSocket connection subscribed to audio events
type AudioEventSubscriber struct {
conn *websocket.Conn
ctx context.Context
logger *zerolog.Logger
}
// AudioEventBroadcaster manages audio event subscriptions and broadcasting
type AudioEventBroadcaster struct {
subscribers map[string]*AudioEventSubscriber
mutex sync.RWMutex
logger *zerolog.Logger
}
var (
audioEventBroadcaster *AudioEventBroadcaster
audioEventOnce sync.Once
)
// initializeBroadcaster creates and initializes the audio event broadcaster
func initializeBroadcaster() {
l := logging.GetDefaultLogger().With().Str("component", "audio-events").Logger()
audioEventBroadcaster = &AudioEventBroadcaster{
subscribers: make(map[string]*AudioEventSubscriber),
logger: &l,
}
// Start metrics broadcasting goroutine
go audioEventBroadcaster.startMetricsBroadcasting()
}
// InitializeAudioEventBroadcaster initializes the global audio event broadcaster
func InitializeAudioEventBroadcaster() {
audioEventOnce.Do(initializeBroadcaster)
}
// GetAudioEventBroadcaster returns the singleton audio event broadcaster
func GetAudioEventBroadcaster() *AudioEventBroadcaster {
audioEventOnce.Do(initializeBroadcaster)
return audioEventBroadcaster
}
// Subscribe adds a WebSocket connection to receive audio events
func (aeb *AudioEventBroadcaster) Subscribe(connectionID string, conn *websocket.Conn, ctx context.Context, logger *zerolog.Logger) {
aeb.mutex.Lock()
defer aeb.mutex.Unlock()
// Check if there's already a subscription for this connectionID
if _, exists := aeb.subscribers[connectionID]; exists {
aeb.logger.Debug().Str("connectionID", connectionID).Msg("duplicate audio events subscription detected; replacing existing entry")
// Do NOT close the existing WebSocket connection here because it's shared
// with the signaling channel. Just replace the subscriber map entry.
delete(aeb.subscribers, connectionID)
}
aeb.subscribers[connectionID] = &AudioEventSubscriber{
conn: conn,
ctx: ctx,
logger: logger,
}
aeb.logger.Info().Str("connectionID", connectionID).Msg("audio events subscription added")
// Send initial state to new subscriber
go aeb.sendInitialState(connectionID)
}
// Unsubscribe removes a WebSocket connection from audio events
func (aeb *AudioEventBroadcaster) Unsubscribe(connectionID string) {
aeb.mutex.Lock()
defer aeb.mutex.Unlock()
delete(aeb.subscribers, connectionID)
aeb.logger.Info().Str("connectionID", connectionID).Msg("audio events subscription removed")
}
// BroadcastAudioMuteChanged broadcasts audio mute state changes
func (aeb *AudioEventBroadcaster) BroadcastAudioMuteChanged(muted bool) {
event := createAudioEvent(AudioEventMuteChanged, AudioMuteData{Muted: muted})
aeb.broadcast(event)
}
// BroadcastMicrophoneStateChanged broadcasts microphone state changes
func (aeb *AudioEventBroadcaster) BroadcastMicrophoneStateChanged(running, sessionActive bool) {
event := createAudioEvent(AudioEventMicrophoneState, MicrophoneStateData{
Running: running,
SessionActive: sessionActive,
})
aeb.broadcast(event)
}
// sendInitialState sends current audio state to a new subscriber
func (aeb *AudioEventBroadcaster) sendInitialState(connectionID string) {
aeb.mutex.RLock()
subscriber, exists := aeb.subscribers[connectionID]
aeb.mutex.RUnlock()
if !exists {
return
}
// Send current audio mute state
muteEvent := AudioEvent{
Type: AudioEventMuteChanged,
Data: AudioMuteData{Muted: IsAudioMuted()},
}
aeb.sendToSubscriber(subscriber, muteEvent)
// Send current microphone state using session provider
sessionProvider := GetSessionProvider()
sessionActive := sessionProvider.IsSessionActive()
var running bool
if sessionActive {
if inputManager := sessionProvider.GetAudioInputManager(); inputManager != nil {
running = inputManager.IsRunning()
}
}
micStateEvent := AudioEvent{
Type: AudioEventMicrophoneState,
Data: MicrophoneStateData{
Running: running,
SessionActive: sessionActive,
},
}
aeb.sendToSubscriber(subscriber, micStateEvent)
// Send current metrics
aeb.sendCurrentMetrics(subscriber)
}
// convertAudioMetricsToEventData converts internal audio metrics to AudioMetricsData for events
func convertAudioMetricsToEventData(metrics AudioMetrics) AudioMetricsData {
return AudioMetricsData{
FramesReceived: metrics.FramesReceived,
FramesDropped: metrics.FramesDropped,
BytesProcessed: metrics.BytesProcessed,
LastFrameTime: metrics.LastFrameTime.Format("2006-01-02T15:04:05.000Z"),
ConnectionDrops: metrics.ConnectionDrops,
AverageLatency: metrics.AverageLatency.String(),
}
}
// convertAudioMetricsToEventDataWithLatencyMs converts internal audio metrics to AudioMetricsData with millisecond latency formatting
func convertAudioMetricsToEventDataWithLatencyMs(metrics AudioMetrics) AudioMetricsData {
return AudioMetricsData{
FramesReceived: metrics.FramesReceived,
FramesDropped: metrics.FramesDropped,
BytesProcessed: metrics.BytesProcessed,
LastFrameTime: metrics.LastFrameTime.Format("2006-01-02T15:04:05.000Z"),
ConnectionDrops: metrics.ConnectionDrops,
AverageLatency: fmt.Sprintf("%.1fms", float64(metrics.AverageLatency.Nanoseconds())/1e6),
}
}
// convertAudioInputMetricsToEventData converts internal audio input metrics to MicrophoneMetricsData for events
func convertAudioInputMetricsToEventData(metrics AudioInputMetrics) MicrophoneMetricsData {
return MicrophoneMetricsData{
FramesSent: metrics.FramesSent,
FramesDropped: metrics.FramesDropped,
BytesProcessed: metrics.BytesProcessed,
LastFrameTime: metrics.LastFrameTime.Format("2006-01-02T15:04:05.000Z"),
ConnectionDrops: metrics.ConnectionDrops,
AverageLatency: metrics.AverageLatency.String(),
}
}
// convertAudioInputMetricsToEventDataWithLatencyMs converts internal audio input metrics to MicrophoneMetricsData with millisecond latency formatting
func convertAudioInputMetricsToEventDataWithLatencyMs(metrics AudioInputMetrics) MicrophoneMetricsData {
return MicrophoneMetricsData{
FramesSent: metrics.FramesSent,
FramesDropped: metrics.FramesDropped,
BytesProcessed: metrics.BytesProcessed,
LastFrameTime: metrics.LastFrameTime.Format("2006-01-02T15:04:05.000Z"),
ConnectionDrops: metrics.ConnectionDrops,
AverageLatency: fmt.Sprintf("%.1fms", float64(metrics.AverageLatency.Nanoseconds())/1e6),
}
}
// convertProcessMetricsToEventData converts internal process metrics to ProcessMetricsData for events
func convertProcessMetricsToEventData(metrics ProcessMetrics, running bool) ProcessMetricsData {
return ProcessMetricsData{
PID: metrics.PID,
CPUPercent: metrics.CPUPercent,
MemoryRSS: metrics.MemoryRSS,
MemoryVMS: metrics.MemoryVMS,
MemoryPercent: metrics.MemoryPercent,
Running: running,
ProcessName: metrics.ProcessName,
}
}
// createProcessMetricsData creates ProcessMetricsData from ProcessMetrics with running status
func createProcessMetricsData(metrics *ProcessMetrics, running bool, processName string) ProcessMetricsData {
if metrics == nil {
return ProcessMetricsData{
PID: 0,
CPUPercent: 0.0,
MemoryRSS: 0,
MemoryVMS: 0,
MemoryPercent: 0.0,
Running: false,
ProcessName: processName,
}
}
return ProcessMetricsData{
PID: metrics.PID,
CPUPercent: metrics.CPUPercent,
MemoryRSS: metrics.MemoryRSS,
MemoryVMS: metrics.MemoryVMS,
MemoryPercent: metrics.MemoryPercent,
Running: running,
ProcessName: metrics.ProcessName,
}
}
// getInactiveProcessMetrics returns ProcessMetricsData for an inactive audio input process
func getInactiveProcessMetrics() ProcessMetricsData {
return createProcessMetricsData(nil, false, "audio-input-server")
}
// getActiveAudioInputSupervisor safely retrieves the audio input supervisor if session is active
func getActiveAudioInputSupervisor() *AudioInputSupervisor {
sessionProvider := GetSessionProvider()
if !sessionProvider.IsSessionActive() {
return nil
}
inputManager := sessionProvider.GetAudioInputManager()
if inputManager == nil {
return nil
}
return inputManager.GetSupervisor()
}
// createAudioEvent creates an AudioEvent
func createAudioEvent(eventType AudioEventType, data interface{}) AudioEvent {
return AudioEvent{
Type: eventType,
Data: data,
}
}
func (aeb *AudioEventBroadcaster) getMicrophoneProcessMetrics() ProcessMetricsData {
inputSupervisor := getActiveAudioInputSupervisor()
if inputSupervisor == nil {
return getInactiveProcessMetrics()
}
processMetrics := inputSupervisor.GetProcessMetrics()
if processMetrics == nil {
return getInactiveProcessMetrics()
}
// If process is running but CPU is 0%, it means we're waiting for the second sample
// to calculate CPU percentage. Return metrics with correct running status.
if inputSupervisor.IsRunning() && processMetrics.CPUPercent == 0.0 {
return createProcessMetricsData(processMetrics, true, processMetrics.ProcessName)
}
// Subprocess is running, return actual metrics
return createProcessMetricsData(processMetrics, inputSupervisor.IsRunning(), processMetrics.ProcessName)
}
// sendCurrentMetrics sends current audio and microphone metrics to a subscriber
func (aeb *AudioEventBroadcaster) sendCurrentMetrics(subscriber *AudioEventSubscriber) {
// Send audio metrics
audioMetrics := GetAudioMetrics()
audioMetricsEvent := createAudioEvent(AudioEventMetricsUpdate, convertAudioMetricsToEventData(audioMetrics))
aeb.sendToSubscriber(subscriber, audioMetricsEvent)
// Send audio process metrics
if outputSupervisor := GetAudioOutputSupervisor(); outputSupervisor != nil {
if processMetrics := outputSupervisor.GetProcessMetrics(); processMetrics != nil {
audioProcessEvent := createAudioEvent(AudioEventProcessMetrics, convertProcessMetricsToEventData(*processMetrics, outputSupervisor.IsRunning()))
aeb.sendToSubscriber(subscriber, audioProcessEvent)
}
}
// Send microphone metrics using session provider
sessionProvider := GetSessionProvider()
if sessionProvider.IsSessionActive() {
if inputManager := sessionProvider.GetAudioInputManager(); inputManager != nil {
micMetrics := inputManager.GetMetrics()
micMetricsEvent := createAudioEvent(AudioEventMicrophoneMetrics, convertAudioInputMetricsToEventData(micMetrics))
aeb.sendToSubscriber(subscriber, micMetricsEvent)
}
}
// Send microphone process metrics (always send, even when subprocess is not running)
micProcessEvent := createAudioEvent(AudioEventMicProcessMetrics, aeb.getMicrophoneProcessMetrics())
aeb.sendToSubscriber(subscriber, micProcessEvent)
}
// startMetricsBroadcasting starts a goroutine that periodically broadcasts metrics
func (aeb *AudioEventBroadcaster) startMetricsBroadcasting() {
// Use centralized interval to match process monitor frequency for synchronized metrics
ticker := time.NewTicker(GetMetricsUpdateInterval())
defer ticker.Stop()
for range ticker.C {
aeb.mutex.RLock()
subscriberCount := len(aeb.subscribers)
// Early exit if no subscribers to save CPU
if subscriberCount == 0 {
aeb.mutex.RUnlock()
continue
}
// Create a copy for safe iteration
subscribersCopy := make([]*AudioEventSubscriber, 0, subscriberCount)
for _, sub := range aeb.subscribers {
subscribersCopy = append(subscribersCopy, sub)
}
aeb.mutex.RUnlock()
// Pre-check for cancelled contexts to avoid unnecessary work
activeSubscribers := 0
for _, sub := range subscribersCopy {
if sub.ctx.Err() == nil {
activeSubscribers++
}
}
// Skip metrics gathering if no active subscribers
if activeSubscribers == 0 {
continue
}
// Broadcast audio metrics
audioMetrics := GetAudioMetrics()
audioMetricsEvent := createAudioEvent(AudioEventMetricsUpdate, convertAudioMetricsToEventDataWithLatencyMs(audioMetrics))
aeb.broadcast(audioMetricsEvent)
// Broadcast microphone metrics if available using session provider
sessionProvider := GetSessionProvider()
if sessionProvider.IsSessionActive() {
if inputManager := sessionProvider.GetAudioInputManager(); inputManager != nil {
micMetrics := inputManager.GetMetrics()
micMetricsEvent := createAudioEvent(AudioEventMicrophoneMetrics, convertAudioInputMetricsToEventDataWithLatencyMs(micMetrics))
aeb.broadcast(micMetricsEvent)
}
}
// Broadcast audio process metrics
if outputSupervisor := GetAudioOutputSupervisor(); outputSupervisor != nil {
if processMetrics := outputSupervisor.GetProcessMetrics(); processMetrics != nil {
audioProcessEvent := createAudioEvent(AudioEventProcessMetrics, convertProcessMetricsToEventData(*processMetrics, outputSupervisor.IsRunning()))
aeb.broadcast(audioProcessEvent)
}
}
// Broadcast microphone process metrics (always broadcast, even when subprocess is not running)
micProcessEvent := createAudioEvent(AudioEventMicProcessMetrics, aeb.getMicrophoneProcessMetrics())
aeb.broadcast(micProcessEvent)
}
}
// broadcast sends an event to all subscribers
func (aeb *AudioEventBroadcaster) broadcast(event AudioEvent) {
aeb.mutex.RLock()
// Create a copy of subscribers to avoid holding the lock during sending
subscribersCopy := make(map[string]*AudioEventSubscriber)
for id, sub := range aeb.subscribers {
subscribersCopy[id] = sub
}
aeb.mutex.RUnlock()
// Track failed subscribers to remove them after sending
var failedSubscribers []string
// Send to all subscribers without holding the lock
for connectionID, subscriber := range subscribersCopy {
if !aeb.sendToSubscriber(subscriber, event) {
failedSubscribers = append(failedSubscribers, connectionID)
}
}
// Remove failed subscribers if any
if len(failedSubscribers) > 0 {
aeb.mutex.Lock()
for _, connectionID := range failedSubscribers {
delete(aeb.subscribers, connectionID)
aeb.logger.Warn().Str("connectionID", connectionID).Msg("removed failed audio events subscriber")
}
aeb.mutex.Unlock()
}
}
// sendToSubscriber sends an event to a specific subscriber
func (aeb *AudioEventBroadcaster) sendToSubscriber(subscriber *AudioEventSubscriber, event AudioEvent) bool {
// Check if subscriber context is already cancelled
if subscriber.ctx.Err() != nil {
return false
}
ctx, cancel := context.WithTimeout(subscriber.ctx, 2*time.Second)
defer cancel()
err := wsjson.Write(ctx, subscriber.conn, event)
if err != nil {
// Don't log network errors for closed connections as warnings, they're expected
if strings.Contains(err.Error(), "use of closed network connection") ||
strings.Contains(err.Error(), "connection reset by peer") ||
strings.Contains(err.Error(), "context canceled") {
subscriber.logger.Debug().Err(err).Msg("websocket connection closed during audio event send")
} else {
subscriber.logger.Warn().Err(err).Msg("failed to send audio event to subscriber")
}
return false
}
return true
}

211
internal/audio/input.go Normal file
View File

@ -0,0 +1,211 @@
package audio
import (
"sync/atomic"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// AudioInputMetrics holds metrics for microphone input
type AudioInputMetrics struct {
FramesSent int64
FramesDropped int64
BytesProcessed int64
ConnectionDrops int64
AverageLatency time.Duration // time.Duration is int64
LastFrameTime time.Time
}
// AudioInputManager manages microphone input stream using IPC mode only
type AudioInputManager struct {
metrics AudioInputMetrics
ipcManager *AudioInputIPCManager
logger zerolog.Logger
running int32
}
// NewAudioInputManager creates a new audio input manager (IPC mode only)
func NewAudioInputManager() *AudioInputManager {
return &AudioInputManager{
ipcManager: NewAudioInputIPCManager(),
logger: logging.GetDefaultLogger().With().Str("component", "audio-input").Logger(),
}
}
// Start begins processing microphone input
func (aim *AudioInputManager) Start() error {
if !atomic.CompareAndSwapInt32(&aim.running, 0, 1) {
return nil // Already running
}
aim.logger.Info().Msg("Starting audio input manager")
// Start the IPC-based audio input
err := aim.ipcManager.Start()
if err != nil {
aim.logger.Error().Err(err).Msg("Failed to start IPC audio input")
atomic.StoreInt32(&aim.running, 0)
return err
}
return nil
}
// Stop stops processing microphone input
func (aim *AudioInputManager) Stop() {
if !atomic.CompareAndSwapInt32(&aim.running, 1, 0) {
return // Already stopped
}
aim.logger.Info().Msg("Stopping audio input manager")
// Stop the IPC-based audio input
aim.ipcManager.Stop()
aim.logger.Info().Msg("Audio input manager stopped")
}
// WriteOpusFrame writes an Opus frame to the audio input system with latency tracking
func (aim *AudioInputManager) WriteOpusFrame(frame []byte) error {
if !aim.IsRunning() {
return nil // Not running, silently drop
}
// Track end-to-end latency from WebRTC to IPC
startTime := time.Now()
err := aim.ipcManager.WriteOpusFrame(frame)
processingTime := time.Since(startTime)
// Log high latency warnings
if processingTime > 10*time.Millisecond {
aim.logger.Warn().
Dur("latency_ms", processingTime).
Msg("High audio processing latency detected")
}
if err != nil {
atomic.AddInt64(&aim.metrics.FramesDropped, 1)
return err
}
// Update metrics
atomic.AddInt64(&aim.metrics.FramesSent, 1)
atomic.AddInt64(&aim.metrics.BytesProcessed, int64(len(frame)))
aim.metrics.LastFrameTime = time.Now()
aim.metrics.AverageLatency = processingTime
return nil
}
// WriteOpusFrameZeroCopy writes an Opus frame using zero-copy optimization
func (aim *AudioInputManager) WriteOpusFrameZeroCopy(frame *ZeroCopyAudioFrame) error {
if !aim.IsRunning() {
return nil // Not running, silently drop
}
if frame == nil {
atomic.AddInt64(&aim.metrics.FramesDropped, 1)
return nil
}
// Track end-to-end latency from WebRTC to IPC
startTime := time.Now()
err := aim.ipcManager.WriteOpusFrameZeroCopy(frame)
processingTime := time.Since(startTime)
// Log high latency warnings
if processingTime > 10*time.Millisecond {
aim.logger.Warn().
Dur("latency_ms", processingTime).
Msg("High audio processing latency detected")
}
if err != nil {
atomic.AddInt64(&aim.metrics.FramesDropped, 1)
return err
}
// Update metrics
atomic.AddInt64(&aim.metrics.FramesSent, 1)
atomic.AddInt64(&aim.metrics.BytesProcessed, int64(frame.Length()))
aim.metrics.LastFrameTime = time.Now()
aim.metrics.AverageLatency = processingTime
return nil
}
// GetMetrics returns current audio input metrics
func (aim *AudioInputManager) GetMetrics() AudioInputMetrics {
return AudioInputMetrics{
FramesSent: atomic.LoadInt64(&aim.metrics.FramesSent),
FramesDropped: atomic.LoadInt64(&aim.metrics.FramesDropped),
BytesProcessed: atomic.LoadInt64(&aim.metrics.BytesProcessed),
AverageLatency: aim.metrics.AverageLatency,
LastFrameTime: aim.metrics.LastFrameTime,
}
}
// GetComprehensiveMetrics returns detailed performance metrics across all components
func (aim *AudioInputManager) GetComprehensiveMetrics() map[string]interface{} {
// Get base metrics
baseMetrics := aim.GetMetrics()
// Get detailed IPC metrics
ipcMetrics, detailedStats := aim.ipcManager.GetDetailedMetrics()
comprehensiveMetrics := map[string]interface{}{
"manager": map[string]interface{}{
"frames_sent": baseMetrics.FramesSent,
"frames_dropped": baseMetrics.FramesDropped,
"bytes_processed": baseMetrics.BytesProcessed,
"average_latency_ms": float64(baseMetrics.AverageLatency.Nanoseconds()) / 1e6,
"last_frame_time": baseMetrics.LastFrameTime,
"running": aim.IsRunning(),
},
"ipc": map[string]interface{}{
"frames_sent": ipcMetrics.FramesSent,
"frames_dropped": ipcMetrics.FramesDropped,
"bytes_processed": ipcMetrics.BytesProcessed,
"average_latency_ms": float64(ipcMetrics.AverageLatency.Nanoseconds()) / 1e6,
"last_frame_time": ipcMetrics.LastFrameTime,
},
"detailed": detailedStats,
}
return comprehensiveMetrics
}
// LogPerformanceStats logs current performance statistics
func (aim *AudioInputManager) LogPerformanceStats() {
metrics := aim.GetComprehensiveMetrics()
managerStats := metrics["manager"].(map[string]interface{})
ipcStats := metrics["ipc"].(map[string]interface{})
detailedStats := metrics["detailed"].(map[string]interface{})
aim.logger.Info().
Int64("manager_frames_sent", managerStats["frames_sent"].(int64)).
Int64("manager_frames_dropped", managerStats["frames_dropped"].(int64)).
Float64("manager_latency_ms", managerStats["average_latency_ms"].(float64)).
Int64("ipc_frames_sent", ipcStats["frames_sent"].(int64)).
Int64("ipc_frames_dropped", ipcStats["frames_dropped"].(int64)).
Float64("ipc_latency_ms", ipcStats["average_latency_ms"].(float64)).
Float64("client_drop_rate", detailedStats["client_drop_rate"].(float64)).
Float64("frames_per_second", detailedStats["frames_per_second"].(float64)).
Msg("Audio input performance metrics")
}
// IsRunning returns whether the audio input manager is running
func (aim *AudioInputManager) IsRunning() bool {
return atomic.LoadInt32(&aim.running) == 1
}
// IsReady returns whether the audio input manager is ready to receive frames
// This checks both that it's running and that the IPC connection is established
func (aim *AudioInputManager) IsReady() bool {
if !aim.IsRunning() {
return false
}
return aim.ipcManager.IsReady()
}

View File

@ -0,0 +1,94 @@
package audio
import (
"sync/atomic"
"unsafe"
)
var (
// Global audio input manager instance
globalInputManager unsafe.Pointer // *AudioInputManager
)
// AudioInputInterface defines the common interface for audio input managers
type AudioInputInterface interface {
Start() error
Stop()
WriteOpusFrame(frame []byte) error
IsRunning() bool
GetMetrics() AudioInputMetrics
}
// GetSupervisor returns the audio input supervisor for advanced management
func (m *AudioInputManager) GetSupervisor() *AudioInputSupervisor {
return m.ipcManager.GetSupervisor()
}
// getAudioInputManager returns the audio input manager
func getAudioInputManager() AudioInputInterface {
ptr := atomic.LoadPointer(&globalInputManager)
if ptr == nil {
// Create new manager
newManager := NewAudioInputManager()
if atomic.CompareAndSwapPointer(&globalInputManager, nil, unsafe.Pointer(newManager)) {
return newManager
}
// Another goroutine created it, use that one
ptr = atomic.LoadPointer(&globalInputManager)
}
return (*AudioInputManager)(ptr)
}
// StartAudioInput starts the audio input system using the appropriate manager
func StartAudioInput() error {
manager := getAudioInputManager()
return manager.Start()
}
// StopAudioInput stops the audio input system
func StopAudioInput() {
manager := getAudioInputManager()
manager.Stop()
}
// WriteAudioInputFrame writes an Opus frame to the audio input system
func WriteAudioInputFrame(frame []byte) error {
manager := getAudioInputManager()
return manager.WriteOpusFrame(frame)
}
// IsAudioInputRunning returns whether the audio input system is running
func IsAudioInputRunning() bool {
manager := getAudioInputManager()
return manager.IsRunning()
}
// GetAudioInputMetrics returns current audio input metrics
func GetAudioInputMetrics() AudioInputMetrics {
manager := getAudioInputManager()
return manager.GetMetrics()
}
// GetAudioInputIPCSupervisor returns the IPC supervisor
func GetAudioInputIPCSupervisor() *AudioInputSupervisor {
ptr := atomic.LoadPointer(&globalInputManager)
if ptr == nil {
return nil
}
manager := (*AudioInputManager)(ptr)
return manager.GetSupervisor()
}
// Helper functions
// ResetAudioInputManagers resets the global manager (for testing)
func ResetAudioInputManagers() {
// Stop existing manager first
if ptr := atomic.LoadPointer(&globalInputManager); ptr != nil {
(*AudioInputManager)(ptr).Stop()
}
// Reset pointer
atomic.StorePointer(&globalInputManager, nil)
}

961
internal/audio/input_ipc.go Normal file
View File

@ -0,0 +1,961 @@
package audio
import (
"context"
"encoding/binary"
"fmt"
"io"
"net"
"os"
"path/filepath"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/jetkvm/kvm/internal/logging"
)
const (
inputMagicNumber uint32 = 0x4A4B4D49 // "JKMI" (JetKVM Microphone Input)
inputSocketName = "audio_input.sock"
maxFrameSize = 4096 // Maximum Opus frame size
writeTimeout = 15 * time.Millisecond // Non-blocking write timeout (increased for high load)
maxDroppedFrames = 100 // Maximum consecutive dropped frames before reconnect
headerSize = 17 // Fixed header size: 4+1+4+8 bytes
messagePoolSize = 256 // Pre-allocated message pool size
)
// InputMessageType represents the type of IPC message
type InputMessageType uint8
const (
InputMessageTypeOpusFrame InputMessageType = iota
InputMessageTypeConfig
InputMessageTypeStop
InputMessageTypeHeartbeat
InputMessageTypeAck
)
// InputIPCMessage represents a message sent over IPC
type InputIPCMessage struct {
Magic uint32
Type InputMessageType
Length uint32
Timestamp int64
Data []byte
}
// OptimizedIPCMessage represents an optimized message with pre-allocated buffers
type OptimizedIPCMessage struct {
header [headerSize]byte // Pre-allocated header buffer
data []byte // Reusable data buffer
msg InputIPCMessage // Embedded message
}
// MessagePool manages a pool of reusable messages to reduce allocations
type MessagePool struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
hitCount int64 // Pool hit counter (atomic)
missCount int64 // Pool miss counter (atomic)
// Other fields
pool chan *OptimizedIPCMessage
// Memory optimization fields
preallocated []*OptimizedIPCMessage // Pre-allocated messages for immediate use
preallocSize int // Number of pre-allocated messages
maxPoolSize int // Maximum pool size to prevent memory bloat
mutex sync.RWMutex // Protects preallocated slice
}
// Global message pool instance
var globalMessagePool = &MessagePool{
pool: make(chan *OptimizedIPCMessage, messagePoolSize),
}
var messagePoolInitOnce sync.Once
// initializeMessagePool initializes the message pool with pre-allocated messages
func initializeMessagePool() {
messagePoolInitOnce.Do(func() {
// Pre-allocate 30% of pool size for immediate availability
preallocSize := messagePoolSize * 30 / 100
globalMessagePool.preallocSize = preallocSize
globalMessagePool.maxPoolSize = messagePoolSize * 2 // Allow growth up to 2x
globalMessagePool.preallocated = make([]*OptimizedIPCMessage, 0, preallocSize)
// Pre-allocate messages to reduce initial allocation overhead
for i := 0; i < preallocSize; i++ {
msg := &OptimizedIPCMessage{
data: make([]byte, 0, maxFrameSize),
}
globalMessagePool.preallocated = append(globalMessagePool.preallocated, msg)
}
// Fill the channel pool with remaining messages
for i := preallocSize; i < messagePoolSize; i++ {
globalMessagePool.pool <- &OptimizedIPCMessage{
data: make([]byte, 0, maxFrameSize),
}
}
})
}
// Get retrieves a message from the pool
func (mp *MessagePool) Get() *OptimizedIPCMessage {
initializeMessagePool()
// First try pre-allocated messages for fastest access
mp.mutex.Lock()
if len(mp.preallocated) > 0 {
msg := mp.preallocated[len(mp.preallocated)-1]
mp.preallocated = mp.preallocated[:len(mp.preallocated)-1]
mp.mutex.Unlock()
atomic.AddInt64(&mp.hitCount, 1)
return msg
}
mp.mutex.Unlock()
// Try channel pool next
select {
case msg := <-mp.pool:
atomic.AddInt64(&mp.hitCount, 1)
return msg
default:
// Pool exhausted, create new message
atomic.AddInt64(&mp.missCount, 1)
return &OptimizedIPCMessage{
data: make([]byte, 0, maxFrameSize),
}
}
}
// Put returns a message to the pool
func (mp *MessagePool) Put(msg *OptimizedIPCMessage) {
// Reset the message for reuse
msg.data = msg.data[:0]
msg.msg = InputIPCMessage{}
// First try to return to pre-allocated pool for fastest reuse
mp.mutex.Lock()
if len(mp.preallocated) < mp.preallocSize {
mp.preallocated = append(mp.preallocated, msg)
mp.mutex.Unlock()
return
}
mp.mutex.Unlock()
// Try channel pool next
select {
case mp.pool <- msg:
// Successfully returned to pool
default:
// Pool full, let GC handle it
}
}
// InputIPCConfig represents configuration for audio input
type InputIPCConfig struct {
SampleRate int
Channels int
FrameSize int
}
// AudioInputServer handles IPC communication for audio input processing
type AudioInputServer struct {
// Atomic fields must be first for proper alignment on ARM
bufferSize int64 // Current buffer size (atomic)
processingTime int64 // Average processing time in nanoseconds (atomic)
droppedFrames int64 // Dropped frames counter (atomic)
totalFrames int64 // Total frames counter (atomic)
listener net.Listener
conn net.Conn
mtx sync.Mutex
running bool
// Triple-goroutine architecture
messageChan chan *InputIPCMessage // Buffered channel for incoming messages
processChan chan *InputIPCMessage // Buffered channel for processing queue
stopChan chan struct{} // Stop signal for all goroutines
wg sync.WaitGroup // Wait group for goroutine coordination
// Socket buffer configuration
socketBufferConfig SocketBufferConfig
}
// NewAudioInputServer creates a new audio input server
func NewAudioInputServer() (*AudioInputServer, error) {
socketPath := getInputSocketPath()
// Remove existing socket if any
os.Remove(socketPath)
listener, err := net.Listen("unix", socketPath)
if err != nil {
return nil, fmt.Errorf("failed to create unix socket: %w", err)
}
// Get initial buffer size from adaptive buffer manager
adaptiveManager := GetAdaptiveBufferManager()
initialBufferSize := int64(adaptiveManager.GetInputBufferSize())
// Initialize socket buffer configuration
socketBufferConfig := DefaultSocketBufferConfig()
return &AudioInputServer{
listener: listener,
messageChan: make(chan *InputIPCMessage, initialBufferSize),
processChan: make(chan *InputIPCMessage, initialBufferSize),
stopChan: make(chan struct{}),
bufferSize: initialBufferSize,
socketBufferConfig: socketBufferConfig,
}, nil
}
// Start starts the audio input server
func (ais *AudioInputServer) Start() error {
ais.mtx.Lock()
defer ais.mtx.Unlock()
if ais.running {
return fmt.Errorf("server already running")
}
ais.running = true
// Start triple-goroutine architecture
ais.startReaderGoroutine()
ais.startProcessorGoroutine()
ais.startMonitorGoroutine()
// Accept connections in a goroutine
go ais.acceptConnections()
return nil
}
// Stop stops the audio input server
func (ais *AudioInputServer) Stop() {
ais.mtx.Lock()
defer ais.mtx.Unlock()
if !ais.running {
return
}
ais.running = false
// Signal all goroutines to stop
close(ais.stopChan)
ais.wg.Wait()
if ais.conn != nil {
ais.conn.Close()
ais.conn = nil
}
if ais.listener != nil {
ais.listener.Close()
}
}
// Close closes the server and cleans up resources
func (ais *AudioInputServer) Close() {
ais.Stop()
// Remove socket file
os.Remove(getInputSocketPath())
}
// acceptConnections accepts incoming connections
func (ais *AudioInputServer) acceptConnections() {
for ais.running {
conn, err := ais.listener.Accept()
if err != nil {
if ais.running {
// Only log error if we're still supposed to be running
continue
}
return
}
// Configure socket buffers for optimal performance
if err := ConfigureSocketBuffers(conn, ais.socketBufferConfig); err != nil {
// Log warning but don't fail - socket buffer optimization is not critical
logger := logging.GetDefaultLogger().With().Str("component", "audio-input-server").Logger()
logger.Warn().Err(err).Msg("Failed to configure socket buffers, continuing with defaults")
} else {
// Record socket buffer metrics for monitoring
RecordSocketBufferMetrics(conn, "audio-input")
}
ais.mtx.Lock()
// Close existing connection if any
if ais.conn != nil {
ais.conn.Close()
}
ais.conn = conn
ais.mtx.Unlock()
// Handle this connection
go ais.handleConnection(conn)
}
}
// handleConnection handles a single client connection
func (ais *AudioInputServer) handleConnection(conn net.Conn) {
defer conn.Close()
// Connection is now handled by the reader goroutine
// Just wait for connection to close or stop signal
for {
select {
case <-ais.stopChan:
return
default:
// Check if connection is still alive
if ais.conn == nil {
return
}
time.Sleep(100 * time.Millisecond)
}
}
}
// readMessage reads a complete message from the connection
func (ais *AudioInputServer) readMessage(conn net.Conn) (*InputIPCMessage, error) {
// Get optimized message from pool
optMsg := globalMessagePool.Get()
defer globalMessagePool.Put(optMsg)
// Read header directly into pre-allocated buffer
_, err := io.ReadFull(conn, optMsg.header[:])
if err != nil {
return nil, err
}
// Parse header using optimized access
msg := &optMsg.msg
msg.Magic = binary.LittleEndian.Uint32(optMsg.header[0:4])
msg.Type = InputMessageType(optMsg.header[4])
msg.Length = binary.LittleEndian.Uint32(optMsg.header[5:9])
msg.Timestamp = int64(binary.LittleEndian.Uint64(optMsg.header[9:17]))
// Validate magic number
if msg.Magic != inputMagicNumber {
return nil, fmt.Errorf("invalid magic number: %x", msg.Magic)
}
// Validate message length
if msg.Length > maxFrameSize {
return nil, fmt.Errorf("message too large: %d bytes", msg.Length)
}
// Read data if present using pooled buffer
if msg.Length > 0 {
// Ensure buffer capacity
if cap(optMsg.data) < int(msg.Length) {
optMsg.data = make([]byte, msg.Length)
} else {
optMsg.data = optMsg.data[:msg.Length]
}
_, err = io.ReadFull(conn, optMsg.data)
if err != nil {
return nil, err
}
msg.Data = optMsg.data
}
// Return a copy of the message (data will be copied by caller if needed)
result := &InputIPCMessage{
Magic: msg.Magic,
Type: msg.Type,
Length: msg.Length,
Timestamp: msg.Timestamp,
}
if msg.Length > 0 {
// Copy data to ensure it's not affected by buffer reuse
result.Data = make([]byte, msg.Length)
copy(result.Data, msg.Data)
}
return result, nil
}
// processMessage processes a received message
func (ais *AudioInputServer) processMessage(msg *InputIPCMessage) error {
switch msg.Type {
case InputMessageTypeOpusFrame:
return ais.processOpusFrame(msg.Data)
case InputMessageTypeConfig:
return ais.processConfig(msg.Data)
case InputMessageTypeStop:
return fmt.Errorf("stop message received")
case InputMessageTypeHeartbeat:
return ais.sendAck()
default:
return fmt.Errorf("unknown message type: %d", msg.Type)
}
}
// processOpusFrame processes an Opus audio frame
func (ais *AudioInputServer) processOpusFrame(data []byte) error {
if len(data) == 0 {
return nil // Empty frame, ignore
}
// Process the Opus frame using CGO
_, err := CGOAudioDecodeWrite(data)
return err
}
// processConfig processes a configuration update
func (ais *AudioInputServer) processConfig(data []byte) error {
// Acknowledge configuration receipt
return ais.sendAck()
}
// sendAck sends an acknowledgment message
func (ais *AudioInputServer) sendAck() error {
ais.mtx.Lock()
defer ais.mtx.Unlock()
if ais.conn == nil {
return fmt.Errorf("no connection")
}
msg := &InputIPCMessage{
Magic: inputMagicNumber,
Type: InputMessageTypeAck,
Length: 0,
Timestamp: time.Now().UnixNano(),
}
return ais.writeMessage(ais.conn, msg)
}
// writeMessage writes a message to the connection using optimized buffers
func (ais *AudioInputServer) writeMessage(conn net.Conn, msg *InputIPCMessage) error {
// Get optimized message from pool for header preparation
optMsg := globalMessagePool.Get()
defer globalMessagePool.Put(optMsg)
// Prepare header in pre-allocated buffer
binary.LittleEndian.PutUint32(optMsg.header[0:4], msg.Magic)
optMsg.header[4] = byte(msg.Type)
binary.LittleEndian.PutUint32(optMsg.header[5:9], msg.Length)
binary.LittleEndian.PutUint64(optMsg.header[9:17], uint64(msg.Timestamp))
// Write header
_, err := conn.Write(optMsg.header[:])
if err != nil {
return err
}
// Write data if present
if msg.Length > 0 && msg.Data != nil {
_, err = conn.Write(msg.Data)
if err != nil {
return err
}
}
return nil
}
// AudioInputClient handles IPC communication from the main process
type AudioInputClient struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
droppedFrames int64 // Atomic counter for dropped frames
totalFrames int64 // Atomic counter for total frames
conn net.Conn
mtx sync.Mutex
running bool
}
// NewAudioInputClient creates a new audio input client
func NewAudioInputClient() *AudioInputClient {
return &AudioInputClient{}
}
// Connect connects to the audio input server
func (aic *AudioInputClient) Connect() error {
aic.mtx.Lock()
defer aic.mtx.Unlock()
if aic.running {
return nil // Already connected
}
socketPath := getInputSocketPath()
// Try connecting multiple times as the server might not be ready
// Reduced retry count and delay for faster startup
for i := 0; i < 10; i++ {
conn, err := net.Dial("unix", socketPath)
if err == nil {
aic.conn = conn
aic.running = true
return nil
}
// Exponential backoff starting at 50ms
delay := time.Duration(50*(1<<uint(i/3))) * time.Millisecond
if delay > 500*time.Millisecond {
delay = 500 * time.Millisecond
}
time.Sleep(delay)
}
return fmt.Errorf("failed to connect to audio input server")
}
// Disconnect disconnects from the audio input server
func (aic *AudioInputClient) Disconnect() {
aic.mtx.Lock()
defer aic.mtx.Unlock()
if !aic.running {
return
}
aic.running = false
if aic.conn != nil {
// Send stop message
msg := &InputIPCMessage{
Magic: inputMagicNumber,
Type: InputMessageTypeStop,
Length: 0,
Timestamp: time.Now().UnixNano(),
}
_ = aic.writeMessage(msg) // Ignore errors during shutdown
aic.conn.Close()
aic.conn = nil
}
}
// SendFrame sends an Opus frame to the audio input server
func (aic *AudioInputClient) SendFrame(frame []byte) error {
aic.mtx.Lock()
defer aic.mtx.Unlock()
if !aic.running || aic.conn == nil {
return fmt.Errorf("not connected")
}
if len(frame) == 0 {
return nil // Empty frame, ignore
}
if len(frame) > maxFrameSize {
return fmt.Errorf("frame too large: %d bytes", len(frame))
}
msg := &InputIPCMessage{
Magic: inputMagicNumber,
Type: InputMessageTypeOpusFrame,
Length: uint32(len(frame)),
Timestamp: time.Now().UnixNano(),
Data: frame,
}
return aic.writeMessage(msg)
}
// SendFrameZeroCopy sends a zero-copy Opus frame to the audio input server
func (aic *AudioInputClient) SendFrameZeroCopy(frame *ZeroCopyAudioFrame) error {
aic.mtx.Lock()
defer aic.mtx.Unlock()
if !aic.running || aic.conn == nil {
return fmt.Errorf("not connected")
}
if frame == nil || frame.Length() == 0 {
return nil // Empty frame, ignore
}
if frame.Length() > maxFrameSize {
return fmt.Errorf("frame too large: %d bytes", frame.Length())
}
// Use zero-copy data directly
msg := &InputIPCMessage{
Magic: inputMagicNumber,
Type: InputMessageTypeOpusFrame,
Length: uint32(frame.Length()),
Timestamp: time.Now().UnixNano(),
Data: frame.Data(), // Zero-copy data access
}
return aic.writeMessage(msg)
}
// SendConfig sends a configuration update to the audio input server
func (aic *AudioInputClient) SendConfig(config InputIPCConfig) error {
aic.mtx.Lock()
defer aic.mtx.Unlock()
if !aic.running || aic.conn == nil {
return fmt.Errorf("not connected")
}
// Serialize config (simple binary format)
data := make([]byte, 12) // 3 * int32
binary.LittleEndian.PutUint32(data[0:4], uint32(config.SampleRate))
binary.LittleEndian.PutUint32(data[4:8], uint32(config.Channels))
binary.LittleEndian.PutUint32(data[8:12], uint32(config.FrameSize))
msg := &InputIPCMessage{
Magic: inputMagicNumber,
Type: InputMessageTypeConfig,
Length: uint32(len(data)),
Timestamp: time.Now().UnixNano(),
Data: data,
}
return aic.writeMessage(msg)
}
// SendHeartbeat sends a heartbeat message
func (aic *AudioInputClient) SendHeartbeat() error {
aic.mtx.Lock()
defer aic.mtx.Unlock()
if !aic.running || aic.conn == nil {
return fmt.Errorf("not connected")
}
msg := &InputIPCMessage{
Magic: inputMagicNumber,
Type: InputMessageTypeHeartbeat,
Length: 0,
Timestamp: time.Now().UnixNano(),
}
return aic.writeMessage(msg)
}
// writeMessage writes a message to the server
func (aic *AudioInputClient) writeMessage(msg *InputIPCMessage) error {
// Increment total frames counter
atomic.AddInt64(&aic.totalFrames, 1)
// Get optimized message from pool for header preparation
optMsg := globalMessagePool.Get()
defer globalMessagePool.Put(optMsg)
// Prepare header in pre-allocated buffer
binary.LittleEndian.PutUint32(optMsg.header[0:4], msg.Magic)
optMsg.header[4] = byte(msg.Type)
binary.LittleEndian.PutUint32(optMsg.header[5:9], msg.Length)
binary.LittleEndian.PutUint64(optMsg.header[9:17], uint64(msg.Timestamp))
// Use non-blocking write with timeout
ctx, cancel := context.WithTimeout(context.Background(), writeTimeout)
defer cancel()
// Create a channel to signal write completion
done := make(chan error, 1)
go func() {
// Write header using pre-allocated buffer
_, err := aic.conn.Write(optMsg.header[:])
if err != nil {
done <- err
return
}
// Write data if present
if msg.Length > 0 && msg.Data != nil {
_, err = aic.conn.Write(msg.Data)
if err != nil {
done <- err
return
}
}
done <- nil
}()
// Wait for completion or timeout
select {
case err := <-done:
if err != nil {
atomic.AddInt64(&aic.droppedFrames, 1)
return err
}
return nil
case <-ctx.Done():
// Timeout occurred - drop frame to prevent blocking
atomic.AddInt64(&aic.droppedFrames, 1)
return fmt.Errorf("write timeout - frame dropped")
}
}
// IsConnected returns whether the client is connected
func (aic *AudioInputClient) IsConnected() bool {
aic.mtx.Lock()
defer aic.mtx.Unlock()
return aic.running && aic.conn != nil
}
// GetFrameStats returns frame statistics
func (aic *AudioInputClient) GetFrameStats() (total, dropped int64) {
return atomic.LoadInt64(&aic.totalFrames), atomic.LoadInt64(&aic.droppedFrames)
}
// GetDropRate returns the current frame drop rate as a percentage
func (aic *AudioInputClient) GetDropRate() float64 {
total := atomic.LoadInt64(&aic.totalFrames)
dropped := atomic.LoadInt64(&aic.droppedFrames)
if total == 0 {
return 0.0
}
return float64(dropped) / float64(total) * 100.0
}
// ResetStats resets frame statistics
func (aic *AudioInputClient) ResetStats() {
atomic.StoreInt64(&aic.totalFrames, 0)
atomic.StoreInt64(&aic.droppedFrames, 0)
}
// startReaderGoroutine starts the message reader goroutine
func (ais *AudioInputServer) startReaderGoroutine() {
ais.wg.Add(1)
go func() {
defer ais.wg.Done()
for {
select {
case <-ais.stopChan:
return
default:
if ais.conn != nil {
msg, err := ais.readMessage(ais.conn)
if err != nil {
continue // Connection error, retry
}
// Send to message channel with non-blocking write
select {
case ais.messageChan <- msg:
atomic.AddInt64(&ais.totalFrames, 1)
default:
// Channel full, drop message
atomic.AddInt64(&ais.droppedFrames, 1)
}
}
}
}
}()
}
// startProcessorGoroutine starts the message processor goroutine
func (ais *AudioInputServer) startProcessorGoroutine() {
ais.wg.Add(1)
go func() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// Set high priority for audio processing
logger := logging.GetDefaultLogger().With().Str("component", "audio-input-processor").Logger()
if err := SetAudioThreadPriority(); err != nil {
logger.Warn().Err(err).Msg("Failed to set audio processing priority")
}
defer func() {
if err := ResetThreadPriority(); err != nil {
logger.Warn().Err(err).Msg("Failed to reset thread priority")
}
}()
defer ais.wg.Done()
for {
select {
case <-ais.stopChan:
return
case msg := <-ais.messageChan:
// Intelligent frame dropping: prioritize recent frames
if msg.Type == InputMessageTypeOpusFrame {
// Check if processing queue is getting full
queueLen := len(ais.processChan)
bufferSize := int(atomic.LoadInt64(&ais.bufferSize))
if queueLen > bufferSize*3/4 {
// Drop oldest frames, keep newest
select {
case <-ais.processChan: // Remove oldest
atomic.AddInt64(&ais.droppedFrames, 1)
default:
}
}
}
// Send to processing queue
select {
case ais.processChan <- msg:
default:
// Processing queue full, drop frame
atomic.AddInt64(&ais.droppedFrames, 1)
}
}
}
}()
}
// startMonitorGoroutine starts the performance monitoring goroutine
func (ais *AudioInputServer) startMonitorGoroutine() {
ais.wg.Add(1)
go func() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// Set I/O priority for monitoring
logger := logging.GetDefaultLogger().With().Str("component", "audio-input-monitor").Logger()
if err := SetAudioIOThreadPriority(); err != nil {
logger.Warn().Err(err).Msg("Failed to set audio I/O priority")
}
defer func() {
if err := ResetThreadPriority(); err != nil {
logger.Warn().Err(err).Msg("Failed to reset thread priority")
}
}()
defer ais.wg.Done()
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
// Buffer size update ticker (less frequent)
bufferUpdateTicker := time.NewTicker(500 * time.Millisecond)
defer bufferUpdateTicker.Stop()
for {
select {
case <-ais.stopChan:
return
case <-ticker.C:
// Process frames from processing queue
for {
select {
case msg := <-ais.processChan:
start := time.Now()
err := ais.processMessage(msg)
processingTime := time.Since(start)
// Calculate end-to-end latency using message timestamp
var latency time.Duration
if msg.Type == InputMessageTypeOpusFrame && msg.Timestamp > 0 {
msgTime := time.Unix(0, msg.Timestamp)
latency = time.Since(msgTime)
// Use exponential moving average for end-to-end latency tracking
currentAvg := atomic.LoadInt64(&ais.processingTime)
// Weight: 90% historical, 10% current (for smoother averaging)
newAvg := (currentAvg*9 + latency.Nanoseconds()) / 10
atomic.StoreInt64(&ais.processingTime, newAvg)
} else {
// Fallback to processing time only
latency = processingTime
currentAvg := atomic.LoadInt64(&ais.processingTime)
newAvg := (currentAvg + processingTime.Nanoseconds()) / 2
atomic.StoreInt64(&ais.processingTime, newAvg)
}
// Report latency to adaptive buffer manager
ais.ReportLatency(latency)
if err != nil {
atomic.AddInt64(&ais.droppedFrames, 1)
}
default:
// No more messages to process
goto checkBufferUpdate
}
}
checkBufferUpdate:
// Check if we need to update buffer size
select {
case <-bufferUpdateTicker.C:
// Update buffer size from adaptive buffer manager
ais.UpdateBufferSize()
default:
// No buffer update needed
}
}
}
}()
}
// GetServerStats returns server performance statistics
func (ais *AudioInputServer) GetServerStats() (total, dropped int64, avgProcessingTime time.Duration, bufferSize int64) {
return atomic.LoadInt64(&ais.totalFrames),
atomic.LoadInt64(&ais.droppedFrames),
time.Duration(atomic.LoadInt64(&ais.processingTime)),
atomic.LoadInt64(&ais.bufferSize)
}
// UpdateBufferSize updates the buffer size from adaptive buffer manager
func (ais *AudioInputServer) UpdateBufferSize() {
adaptiveManager := GetAdaptiveBufferManager()
newSize := int64(adaptiveManager.GetInputBufferSize())
atomic.StoreInt64(&ais.bufferSize, newSize)
}
// ReportLatency reports processing latency to adaptive buffer manager
func (ais *AudioInputServer) ReportLatency(latency time.Duration) {
adaptiveManager := GetAdaptiveBufferManager()
adaptiveManager.UpdateLatency(latency)
}
// GetMessagePoolStats returns detailed statistics about the message pool
func (mp *MessagePool) GetMessagePoolStats() MessagePoolStats {
mp.mutex.RLock()
preallocatedCount := len(mp.preallocated)
mp.mutex.RUnlock()
hitCount := atomic.LoadInt64(&mp.hitCount)
missCount := atomic.LoadInt64(&mp.missCount)
totalRequests := hitCount + missCount
var hitRate float64
if totalRequests > 0 {
hitRate = float64(hitCount) / float64(totalRequests) * 100
}
// Calculate channel pool size
channelPoolSize := len(mp.pool)
return MessagePoolStats{
MaxPoolSize: mp.maxPoolSize,
ChannelPoolSize: channelPoolSize,
PreallocatedCount: int64(preallocatedCount),
PreallocatedMax: int64(mp.preallocSize),
HitCount: hitCount,
MissCount: missCount,
HitRate: hitRate,
}
}
// MessagePoolStats provides detailed message pool statistics
type MessagePoolStats struct {
MaxPoolSize int
ChannelPoolSize int
PreallocatedCount int64
PreallocatedMax int64
HitCount int64
MissCount int64
HitRate float64 // Percentage
}
// GetGlobalMessagePoolStats returns statistics for the global message pool
func GetGlobalMessagePoolStats() MessagePoolStats {
return globalMessagePool.GetMessagePoolStats()
}
// Helper functions
// getInputSocketPath returns the path to the input socket
func getInputSocketPath() string {
if path := os.Getenv("JETKVM_AUDIO_INPUT_SOCKET"); path != "" {
return path
}
return filepath.Join("/var/run", inputSocketName)
}

View File

@ -0,0 +1,238 @@
package audio
import (
"context"
"sync/atomic"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// AudioInputIPCManager manages microphone input using IPC when enabled
type AudioInputIPCManager struct {
metrics AudioInputMetrics
supervisor *AudioInputSupervisor
logger zerolog.Logger
running int32
ctx context.Context
cancel context.CancelFunc
}
// NewAudioInputIPCManager creates a new IPC-based audio input manager
func NewAudioInputIPCManager() *AudioInputIPCManager {
ctx, cancel := context.WithCancel(context.Background())
return &AudioInputIPCManager{
supervisor: NewAudioInputSupervisor(),
logger: logging.GetDefaultLogger().With().Str("component", "audio-input-ipc").Logger(),
ctx: ctx,
cancel: cancel,
}
}
// Start starts the IPC-based audio input system
func (aim *AudioInputIPCManager) Start() error {
if !atomic.CompareAndSwapInt32(&aim.running, 0, 1) {
return nil
}
aim.logger.Info().Msg("Starting IPC-based audio input system")
err := aim.supervisor.Start()
if err != nil {
atomic.StoreInt32(&aim.running, 0)
aim.logger.Error().Err(err).Msg("Failed to start audio input supervisor")
return err
}
config := InputIPCConfig{
SampleRate: 48000,
Channels: 2,
FrameSize: 960,
}
// Wait with timeout for subprocess readiness
select {
case <-time.After(200 * time.Millisecond):
case <-aim.ctx.Done():
aim.supervisor.Stop()
atomic.StoreInt32(&aim.running, 0)
return aim.ctx.Err()
}
err = aim.supervisor.SendConfig(config)
if err != nil {
aim.logger.Warn().Err(err).Msg("Failed to send initial config, will retry later")
}
aim.logger.Info().Msg("IPC-based audio input system started")
return nil
}
// Stop stops the IPC-based audio input system
func (aim *AudioInputIPCManager) Stop() {
if !atomic.CompareAndSwapInt32(&aim.running, 1, 0) {
return
}
aim.logger.Info().Msg("Stopping IPC-based audio input system")
aim.cancel()
aim.supervisor.Stop()
aim.logger.Info().Msg("IPC-based audio input system stopped")
}
// WriteOpusFrame sends an Opus frame to the audio input server via IPC
func (aim *AudioInputIPCManager) WriteOpusFrame(frame []byte) error {
if atomic.LoadInt32(&aim.running) == 0 {
return nil // Not running, silently ignore
}
if len(frame) == 0 {
return nil // Empty frame, ignore
}
// Start latency measurement
startTime := time.Now()
// Update metrics
atomic.AddInt64(&aim.metrics.FramesSent, 1)
atomic.AddInt64(&aim.metrics.BytesProcessed, int64(len(frame)))
aim.metrics.LastFrameTime = startTime
// Send frame via IPC
err := aim.supervisor.SendFrame(frame)
if err != nil {
// Count as dropped frame
atomic.AddInt64(&aim.metrics.FramesDropped, 1)
aim.logger.Debug().Err(err).Msg("Failed to send frame via IPC")
return err
}
// Calculate and update latency (end-to-end IPC transmission time)
latency := time.Since(startTime)
aim.updateLatencyMetrics(latency)
return nil
}
// WriteOpusFrameZeroCopy sends an Opus frame via IPC using zero-copy optimization
func (aim *AudioInputIPCManager) WriteOpusFrameZeroCopy(frame *ZeroCopyAudioFrame) error {
if atomic.LoadInt32(&aim.running) == 0 {
return nil // Not running, silently ignore
}
if frame == nil || frame.Length() == 0 {
return nil // Empty frame, ignore
}
// Start latency measurement
startTime := time.Now()
// Update metrics
atomic.AddInt64(&aim.metrics.FramesSent, 1)
atomic.AddInt64(&aim.metrics.BytesProcessed, int64(frame.Length()))
aim.metrics.LastFrameTime = startTime
// Send frame via IPC using zero-copy data
err := aim.supervisor.SendFrameZeroCopy(frame)
if err != nil {
// Count as dropped frame
atomic.AddInt64(&aim.metrics.FramesDropped, 1)
aim.logger.Debug().Err(err).Msg("Failed to send zero-copy frame via IPC")
return err
}
// Calculate and update latency (end-to-end IPC transmission time)
latency := time.Since(startTime)
aim.updateLatencyMetrics(latency)
return nil
}
// IsRunning returns whether the IPC manager is running
func (aim *AudioInputIPCManager) IsRunning() bool {
return atomic.LoadInt32(&aim.running) == 1
}
// IsReady returns whether the IPC manager is ready to receive frames
// This checks that the supervisor is connected to the audio input server
func (aim *AudioInputIPCManager) IsReady() bool {
if !aim.IsRunning() {
return false
}
return aim.supervisor.IsConnected()
}
// GetMetrics returns current metrics
func (aim *AudioInputIPCManager) GetMetrics() AudioInputMetrics {
return AudioInputMetrics{
FramesSent: atomic.LoadInt64(&aim.metrics.FramesSent),
FramesDropped: atomic.LoadInt64(&aim.metrics.FramesDropped),
BytesProcessed: atomic.LoadInt64(&aim.metrics.BytesProcessed),
ConnectionDrops: atomic.LoadInt64(&aim.metrics.ConnectionDrops),
AverageLatency: aim.metrics.AverageLatency,
LastFrameTime: aim.metrics.LastFrameTime,
}
}
// updateLatencyMetrics updates the latency metrics with exponential moving average
func (aim *AudioInputIPCManager) updateLatencyMetrics(latency time.Duration) {
// Use exponential moving average for smooth latency calculation
currentAvg := aim.metrics.AverageLatency
if currentAvg == 0 {
aim.metrics.AverageLatency = latency
} else {
// EMA with alpha = 0.1 for smooth averaging
aim.metrics.AverageLatency = time.Duration(float64(currentAvg)*0.9 + float64(latency)*0.1)
}
}
// GetDetailedMetrics returns comprehensive performance metrics
func (aim *AudioInputIPCManager) GetDetailedMetrics() (AudioInputMetrics, map[string]interface{}) {
metrics := aim.GetMetrics()
// Get client frame statistics
client := aim.supervisor.GetClient()
totalFrames, droppedFrames := int64(0), int64(0)
dropRate := 0.0
if client != nil {
totalFrames, droppedFrames = client.GetFrameStats()
dropRate = client.GetDropRate()
}
// Get server statistics if available
serverStats := make(map[string]interface{})
if aim.supervisor.IsRunning() {
serverStats["status"] = "running"
} else {
serverStats["status"] = "stopped"
}
detailedStats := map[string]interface{}{
"client_total_frames": totalFrames,
"client_dropped_frames": droppedFrames,
"client_drop_rate": dropRate,
"server_stats": serverStats,
"ipc_latency_ms": float64(metrics.AverageLatency.Nanoseconds()) / 1e6,
"frames_per_second": aim.calculateFrameRate(),
}
return metrics, detailedStats
}
// calculateFrameRate calculates the current frame rate
func (aim *AudioInputIPCManager) calculateFrameRate() float64 {
framesSent := atomic.LoadInt64(&aim.metrics.FramesSent)
if framesSent == 0 {
return 0.0
}
// Return typical Opus frame rate
return 50.0
}
// GetSupervisor returns the supervisor for advanced operations
func (aim *AudioInputIPCManager) GetSupervisor() *AudioInputSupervisor {
return aim.supervisor
}

View File

@ -0,0 +1,71 @@
package audio
import (
"context"
"os"
"os/signal"
"syscall"
"time"
"github.com/jetkvm/kvm/internal/logging"
)
// RunAudioInputServer runs the audio input server subprocess
// This should be called from main() when the subprocess is detected
func RunAudioInputServer() error {
logger := logging.GetDefaultLogger().With().Str("component", "audio-input-server").Logger()
logger.Info().Msg("Starting audio input server subprocess")
// Start adaptive buffer management for optimal performance
StartAdaptiveBuffering()
defer StopAdaptiveBuffering()
// Initialize CGO audio system
err := CGOAudioPlaybackInit()
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize CGO audio playback")
return err
}
defer CGOAudioPlaybackClose()
// Create and start the IPC server
server, err := NewAudioInputServer()
if err != nil {
logger.Error().Err(err).Msg("Failed to create audio input server")
return err
}
defer server.Close()
err = server.Start()
if err != nil {
logger.Error().Err(err).Msg("Failed to start audio input server")
return err
}
logger.Info().Msg("Audio input server started, waiting for connections")
// Set up signal handling for graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
// Wait for shutdown signal
select {
case sig := <-sigChan:
logger.Info().Str("signal", sig.String()).Msg("Received shutdown signal")
case <-ctx.Done():
logger.Info().Msg("Context cancelled")
}
// Graceful shutdown
logger.Info().Msg("Shutting down audio input server")
server.Stop()
// Give some time for cleanup
time.Sleep(100 * time.Millisecond)
logger.Info().Msg("Audio input server subprocess stopped")
return nil
}

View File

@ -0,0 +1,271 @@
package audio
import (
"context"
"fmt"
"os"
"os/exec"
"sync"
"syscall"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// AudioInputSupervisor manages the audio input server subprocess
type AudioInputSupervisor struct {
cmd *exec.Cmd
cancel context.CancelFunc
mtx sync.Mutex
running bool
logger zerolog.Logger
client *AudioInputClient
processMonitor *ProcessMonitor
}
// NewAudioInputSupervisor creates a new audio input supervisor
func NewAudioInputSupervisor() *AudioInputSupervisor {
return &AudioInputSupervisor{
logger: logging.GetDefaultLogger().With().Str("component", "audio-input-supervisor").Logger(),
client: NewAudioInputClient(),
processMonitor: GetProcessMonitor(),
}
}
// Start starts the audio input server subprocess
func (ais *AudioInputSupervisor) Start() error {
ais.mtx.Lock()
defer ais.mtx.Unlock()
if ais.running {
return fmt.Errorf("audio input supervisor already running")
}
// Create context for subprocess management
ctx, cancel := context.WithCancel(context.Background())
ais.cancel = cancel
// Get current executable path
execPath, err := os.Executable()
if err != nil {
return fmt.Errorf("failed to get executable path: %w", err)
}
// Create command for audio input server subprocess
cmd := exec.CommandContext(ctx, execPath, "--audio-input-server")
cmd.Env = append(os.Environ(),
"JETKVM_AUDIO_INPUT_IPC=true", // Enable IPC mode
)
// Set process group to allow clean termination
cmd.SysProcAttr = &syscall.SysProcAttr{
Setpgid: true,
}
ais.cmd = cmd
ais.running = true
// Start the subprocess
err = cmd.Start()
if err != nil {
ais.running = false
cancel()
return fmt.Errorf("failed to start audio input server: %w", err)
}
ais.logger.Info().Int("pid", cmd.Process.Pid).Msg("Audio input server subprocess started")
// Add process to monitoring
ais.processMonitor.AddProcess(cmd.Process.Pid, "audio-input-server")
// Monitor the subprocess in a goroutine
go ais.monitorSubprocess()
// Connect client to the server
go ais.connectClient()
return nil
}
// Stop stops the audio input server subprocess
func (ais *AudioInputSupervisor) Stop() {
ais.mtx.Lock()
defer ais.mtx.Unlock()
if !ais.running {
return
}
ais.running = false
// Disconnect client first
if ais.client != nil {
ais.client.Disconnect()
}
// Cancel context to signal subprocess to stop
if ais.cancel != nil {
ais.cancel()
}
// Try graceful termination first
if ais.cmd != nil && ais.cmd.Process != nil {
ais.logger.Info().Int("pid", ais.cmd.Process.Pid).Msg("Stopping audio input server subprocess")
// Send SIGTERM
err := ais.cmd.Process.Signal(syscall.SIGTERM)
if err != nil {
ais.logger.Warn().Err(err).Msg("Failed to send SIGTERM to audio input server")
}
// Wait for graceful shutdown with timeout
done := make(chan error, 1)
go func() {
done <- ais.cmd.Wait()
}()
select {
case <-done:
ais.logger.Info().Msg("Audio input server subprocess stopped gracefully")
case <-time.After(5 * time.Second):
// Force kill if graceful shutdown failed
ais.logger.Warn().Msg("Audio input server subprocess did not stop gracefully, force killing")
err := ais.cmd.Process.Kill()
if err != nil {
ais.logger.Error().Err(err).Msg("Failed to kill audio input server subprocess")
}
}
}
ais.cmd = nil
ais.cancel = nil
}
// IsRunning returns whether the supervisor is running
func (ais *AudioInputSupervisor) IsRunning() bool {
ais.mtx.Lock()
defer ais.mtx.Unlock()
return ais.running
}
// IsConnected returns whether the client is connected to the audio input server
func (ais *AudioInputSupervisor) IsConnected() bool {
if !ais.IsRunning() {
return false
}
return ais.client.IsConnected()
}
// GetClient returns the IPC client for sending audio frames
func (ais *AudioInputSupervisor) GetClient() *AudioInputClient {
return ais.client
}
// GetProcessMetrics returns current process metrics if the process is running
func (ais *AudioInputSupervisor) GetProcessMetrics() *ProcessMetrics {
ais.mtx.Lock()
defer ais.mtx.Unlock()
if ais.cmd == nil || ais.cmd.Process == nil {
return nil
}
pid := ais.cmd.Process.Pid
metrics := ais.processMonitor.GetCurrentMetrics()
for _, metric := range metrics {
if metric.PID == pid {
return &metric
}
}
return nil
}
// monitorSubprocess monitors the subprocess and handles unexpected exits
func (ais *AudioInputSupervisor) monitorSubprocess() {
if ais.cmd == nil {
return
}
pid := ais.cmd.Process.Pid
err := ais.cmd.Wait()
// Remove process from monitoring
ais.processMonitor.RemoveProcess(pid)
ais.mtx.Lock()
defer ais.mtx.Unlock()
if ais.running {
// Unexpected exit
if err != nil {
ais.logger.Error().Err(err).Msg("Audio input server subprocess exited unexpectedly")
} else {
ais.logger.Warn().Msg("Audio input server subprocess exited unexpectedly")
}
// Disconnect client
if ais.client != nil {
ais.client.Disconnect()
}
// Mark as not running
ais.running = false
ais.cmd = nil
ais.logger.Info().Msg("Audio input server subprocess monitoring stopped")
}
}
// connectClient attempts to connect the client to the server
func (ais *AudioInputSupervisor) connectClient() {
// Wait briefly for the server to start (reduced from 500ms)
time.Sleep(100 * time.Millisecond)
err := ais.client.Connect()
if err != nil {
ais.logger.Error().Err(err).Msg("Failed to connect to audio input server")
return
}
ais.logger.Info().Msg("Connected to audio input server")
}
// SendFrame sends an audio frame to the subprocess (convenience method)
func (ais *AudioInputSupervisor) SendFrame(frame []byte) error {
if ais.client == nil {
return fmt.Errorf("client not initialized")
}
if !ais.client.IsConnected() {
return fmt.Errorf("client not connected")
}
return ais.client.SendFrame(frame)
}
// SendFrameZeroCopy sends a zero-copy frame to the subprocess
func (ais *AudioInputSupervisor) SendFrameZeroCopy(frame *ZeroCopyAudioFrame) error {
if ais.client == nil {
return fmt.Errorf("client not initialized")
}
if !ais.client.IsConnected() {
return fmt.Errorf("client not connected")
}
return ais.client.SendFrameZeroCopy(frame)
}
// SendConfig sends a configuration update to the subprocess (convenience method)
func (ais *AudioInputSupervisor) SendConfig(config InputIPCConfig) error {
if ais.client == nil {
return fmt.Errorf("client not initialized")
}
if !ais.client.IsConnected() {
return fmt.Errorf("client not connected")
}
return ais.client.SendConfig(config)
}

525
internal/audio/ipc.go Normal file
View File

@ -0,0 +1,525 @@
package audio
import (
"context"
"encoding/binary"
"fmt"
"io"
"net"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
const (
outputMagicNumber uint32 = 0x4A4B4F55 // "JKOU" (JetKVM Output)
outputSocketName = "audio_output.sock"
outputMaxFrameSize = 4096 // Maximum Opus frame size
outputWriteTimeout = 10 * time.Millisecond // Non-blocking write timeout (increased for high load)
outputMaxDroppedFrames = 50 // Maximum consecutive dropped frames
outputHeaderSize = 17 // Fixed header size: 4+1+4+8 bytes
outputMessagePoolSize = 128 // Pre-allocated message pool size
)
// OutputMessageType represents the type of IPC message
type OutputMessageType uint8
const (
OutputMessageTypeOpusFrame OutputMessageType = iota
OutputMessageTypeConfig
OutputMessageTypeStop
OutputMessageTypeHeartbeat
OutputMessageTypeAck
)
// OutputIPCMessage represents an IPC message for audio output
type OutputIPCMessage struct {
Magic uint32
Type OutputMessageType
Length uint32
Timestamp int64
Data []byte
}
// OutputOptimizedMessage represents a pre-allocated message for zero-allocation operations
type OutputOptimizedMessage struct {
header [outputHeaderSize]byte // Pre-allocated header buffer
data []byte // Reusable data buffer
}
// OutputMessagePool manages pre-allocated messages for zero-allocation IPC
type OutputMessagePool struct {
pool chan *OutputOptimizedMessage
}
// NewOutputMessagePool creates a new message pool
func NewOutputMessagePool(size int) *OutputMessagePool {
pool := &OutputMessagePool{
pool: make(chan *OutputOptimizedMessage, size),
}
// Pre-allocate messages
for i := 0; i < size; i++ {
msg := &OutputOptimizedMessage{
data: make([]byte, outputMaxFrameSize),
}
pool.pool <- msg
}
return pool
}
// Get retrieves a message from the pool
func (p *OutputMessagePool) Get() *OutputOptimizedMessage {
select {
case msg := <-p.pool:
return msg
default:
// Pool exhausted, create new message
return &OutputOptimizedMessage{
data: make([]byte, outputMaxFrameSize),
}
}
}
// Put returns a message to the pool
func (p *OutputMessagePool) Put(msg *OutputOptimizedMessage) {
select {
case p.pool <- msg:
// Successfully returned to pool
default:
// Pool full, let GC handle it
}
}
// Global message pool for output IPC
var globalOutputMessagePool = NewOutputMessagePool(outputMessagePoolSize)
type AudioServer struct {
// Atomic fields must be first for proper alignment on ARM
bufferSize int64 // Current buffer size (atomic)
droppedFrames int64 // Dropped frames counter (atomic)
totalFrames int64 // Total frames counter (atomic)
listener net.Listener
conn net.Conn
mtx sync.Mutex
running bool
// Advanced message handling
messageChan chan *OutputIPCMessage // Buffered channel for incoming messages
stopChan chan struct{} // Stop signal
wg sync.WaitGroup // Wait group for goroutine coordination
// Latency monitoring
latencyMonitor *LatencyMonitor
adaptiveOptimizer *AdaptiveOptimizer
// Socket buffer configuration
socketBufferConfig SocketBufferConfig
}
func NewAudioServer() (*AudioServer, error) {
socketPath := getOutputSocketPath()
// Remove existing socket if any
os.Remove(socketPath)
listener, err := net.Listen("unix", socketPath)
if err != nil {
return nil, fmt.Errorf("failed to create unix socket: %w", err)
}
// Initialize with adaptive buffer size (start with 500 frames)
initialBufferSize := int64(500)
// Initialize latency monitoring
latencyConfig := DefaultLatencyConfig()
logger := zerolog.New(os.Stderr).With().Timestamp().Str("component", "audio-server").Logger()
latencyMonitor := NewLatencyMonitor(latencyConfig, logger)
// Initialize adaptive buffer manager with default config
bufferConfig := DefaultAdaptiveBufferConfig()
bufferManager := NewAdaptiveBufferManager(bufferConfig)
// Initialize adaptive optimizer
optimizerConfig := DefaultOptimizerConfig()
adaptiveOptimizer := NewAdaptiveOptimizer(latencyMonitor, bufferManager, optimizerConfig, logger)
// Initialize socket buffer configuration
socketBufferConfig := DefaultSocketBufferConfig()
return &AudioServer{
listener: listener,
messageChan: make(chan *OutputIPCMessage, initialBufferSize),
stopChan: make(chan struct{}),
bufferSize: initialBufferSize,
latencyMonitor: latencyMonitor,
adaptiveOptimizer: adaptiveOptimizer,
socketBufferConfig: socketBufferConfig,
}, nil
}
func (s *AudioServer) Start() error {
s.mtx.Lock()
defer s.mtx.Unlock()
if s.running {
return fmt.Errorf("server already running")
}
s.running = true
// Start latency monitoring and adaptive optimization
if s.latencyMonitor != nil {
s.latencyMonitor.Start()
}
if s.adaptiveOptimizer != nil {
s.adaptiveOptimizer.Start()
}
// Start message processor goroutine
s.startProcessorGoroutine()
// Accept connections in a goroutine
go s.acceptConnections()
return nil
}
// acceptConnections accepts incoming connections
func (s *AudioServer) acceptConnections() {
for s.running {
conn, err := s.listener.Accept()
if err != nil {
if s.running {
// Only log error if we're still supposed to be running
continue
}
return
}
// Configure socket buffers for optimal performance
if err := ConfigureSocketBuffers(conn, s.socketBufferConfig); err != nil {
// Log warning but don't fail - socket buffer optimization is not critical
logger := logging.GetDefaultLogger().With().Str("component", "audio-server").Logger()
logger.Warn().Err(err).Msg("Failed to configure socket buffers, continuing with defaults")
} else {
// Record socket buffer metrics for monitoring
RecordSocketBufferMetrics(conn, "audio-output")
}
s.mtx.Lock()
// Close existing connection if any
if s.conn != nil {
s.conn.Close()
}
s.conn = conn
s.mtx.Unlock()
}
}
// startProcessorGoroutine starts the message processor
func (s *AudioServer) startProcessorGoroutine() {
s.wg.Add(1)
go func() {
defer s.wg.Done()
for {
select {
case msg := <-s.messageChan:
// Process message (currently just frame sending)
if msg.Type == OutputMessageTypeOpusFrame {
if err := s.sendFrameToClient(msg.Data); err != nil {
// Log error but continue processing
atomic.AddInt64(&s.droppedFrames, 1)
}
}
case <-s.stopChan:
return
}
}
}()
}
func (s *AudioServer) Stop() {
s.mtx.Lock()
defer s.mtx.Unlock()
if !s.running {
return
}
s.running = false
// Stop latency monitoring and adaptive optimization
if s.adaptiveOptimizer != nil {
s.adaptiveOptimizer.Stop()
}
if s.latencyMonitor != nil {
s.latencyMonitor.Stop()
}
// Signal processor to stop
close(s.stopChan)
s.wg.Wait()
if s.conn != nil {
s.conn.Close()
s.conn = nil
}
}
func (s *AudioServer) Close() error {
s.Stop()
if s.listener != nil {
s.listener.Close()
}
// Remove socket file
os.Remove(getOutputSocketPath())
return nil
}
func (s *AudioServer) SendFrame(frame []byte) error {
if len(frame) > outputMaxFrameSize {
return fmt.Errorf("frame size %d exceeds maximum %d", len(frame), outputMaxFrameSize)
}
start := time.Now()
// Create IPC message
msg := &OutputIPCMessage{
Magic: outputMagicNumber,
Type: OutputMessageTypeOpusFrame,
Length: uint32(len(frame)),
Timestamp: start.UnixNano(),
Data: frame,
}
// Try to send via message channel (non-blocking)
select {
case s.messageChan <- msg:
atomic.AddInt64(&s.totalFrames, 1)
// Record latency for monitoring
if s.latencyMonitor != nil {
processingTime := time.Since(start)
s.latencyMonitor.RecordLatency(processingTime, "ipc_send")
}
return nil
default:
// Channel full, drop frame to prevent blocking
atomic.AddInt64(&s.droppedFrames, 1)
return fmt.Errorf("message channel full - frame dropped")
}
}
// sendFrameToClient sends frame data directly to the connected client
func (s *AudioServer) sendFrameToClient(frame []byte) error {
s.mtx.Lock()
defer s.mtx.Unlock()
if s.conn == nil {
return fmt.Errorf("no client connected")
}
start := time.Now()
// Get optimized message from pool
optMsg := globalOutputMessagePool.Get()
defer globalOutputMessagePool.Put(optMsg)
// Prepare header in pre-allocated buffer
binary.LittleEndian.PutUint32(optMsg.header[0:4], outputMagicNumber)
optMsg.header[4] = byte(OutputMessageTypeOpusFrame)
binary.LittleEndian.PutUint32(optMsg.header[5:9], uint32(len(frame)))
binary.LittleEndian.PutUint64(optMsg.header[9:17], uint64(start.UnixNano()))
// Use non-blocking write with timeout
ctx, cancel := context.WithTimeout(context.Background(), outputWriteTimeout)
defer cancel()
// Create a channel to signal write completion
done := make(chan error, 1)
go func() {
// Write header using pre-allocated buffer
_, err := s.conn.Write(optMsg.header[:])
if err != nil {
done <- err
return
}
// Write frame data
if len(frame) > 0 {
_, err = s.conn.Write(frame)
if err != nil {
done <- err
return
}
}
done <- nil
}()
// Wait for completion or timeout
select {
case err := <-done:
if err != nil {
atomic.AddInt64(&s.droppedFrames, 1)
return err
}
// Record latency for monitoring
if s.latencyMonitor != nil {
writeLatency := time.Since(start)
s.latencyMonitor.RecordLatency(writeLatency, "ipc_write")
}
return nil
case <-ctx.Done():
// Timeout occurred - drop frame to prevent blocking
atomic.AddInt64(&s.droppedFrames, 1)
return fmt.Errorf("write timeout - frame dropped")
}
}
// GetServerStats returns server performance statistics
func (s *AudioServer) GetServerStats() (total, dropped int64, bufferSize int64) {
return atomic.LoadInt64(&s.totalFrames),
atomic.LoadInt64(&s.droppedFrames),
atomic.LoadInt64(&s.bufferSize)
}
type AudioClient struct {
// Atomic fields must be first for proper alignment on ARM
droppedFrames int64 // Atomic counter for dropped frames
totalFrames int64 // Atomic counter for total frames
conn net.Conn
mtx sync.Mutex
running bool
}
func NewAudioClient() *AudioClient {
return &AudioClient{}
}
// Connect connects to the audio output server
func (c *AudioClient) Connect() error {
c.mtx.Lock()
defer c.mtx.Unlock()
if c.running {
return nil // Already connected
}
socketPath := getOutputSocketPath()
// Try connecting multiple times as the server might not be ready
// Reduced retry count and delay for faster startup
for i := 0; i < 8; i++ {
conn, err := net.Dial("unix", socketPath)
if err == nil {
c.conn = conn
c.running = true
return nil
}
// Exponential backoff starting at 50ms
delay := time.Duration(50*(1<<uint(i/3))) * time.Millisecond
if delay > 400*time.Millisecond {
delay = 400 * time.Millisecond
}
time.Sleep(delay)
}
return fmt.Errorf("failed to connect to audio output server")
}
// Disconnect disconnects from the audio output server
func (c *AudioClient) Disconnect() {
c.mtx.Lock()
defer c.mtx.Unlock()
if !c.running {
return
}
c.running = false
if c.conn != nil {
c.conn.Close()
c.conn = nil
}
}
// IsConnected returns whether the client is connected
func (c *AudioClient) IsConnected() bool {
c.mtx.Lock()
defer c.mtx.Unlock()
return c.running && c.conn != nil
}
func (c *AudioClient) Close() error {
c.Disconnect()
return nil
}
func (c *AudioClient) ReceiveFrame() ([]byte, error) {
c.mtx.Lock()
defer c.mtx.Unlock()
if !c.running || c.conn == nil {
return nil, fmt.Errorf("not connected")
}
// Get optimized message from pool for header reading
optMsg := globalOutputMessagePool.Get()
defer globalOutputMessagePool.Put(optMsg)
// Read header
if _, err := io.ReadFull(c.conn, optMsg.header[:]); err != nil {
return nil, fmt.Errorf("failed to read header: %w", err)
}
// Parse header
magic := binary.LittleEndian.Uint32(optMsg.header[0:4])
if magic != outputMagicNumber {
return nil, fmt.Errorf("invalid magic number: %x", magic)
}
msgType := OutputMessageType(optMsg.header[4])
if msgType != OutputMessageTypeOpusFrame {
return nil, fmt.Errorf("unexpected message type: %d", msgType)
}
size := binary.LittleEndian.Uint32(optMsg.header[5:9])
if size > outputMaxFrameSize {
return nil, fmt.Errorf("frame size %d exceeds maximum %d", size, outputMaxFrameSize)
}
// Read frame data
frame := make([]byte, size)
if size > 0 {
if _, err := io.ReadFull(c.conn, frame); err != nil {
return nil, fmt.Errorf("failed to read frame data: %w", err)
}
}
atomic.AddInt64(&c.totalFrames, 1)
return frame, nil
}
// GetClientStats returns client performance statistics
func (c *AudioClient) GetClientStats() (total, dropped int64) {
return atomic.LoadInt64(&c.totalFrames),
atomic.LoadInt64(&c.droppedFrames)
}
// Helper functions
// getOutputSocketPath returns the path to the output socket
func getOutputSocketPath() string {
if path := os.Getenv("JETKVM_AUDIO_OUTPUT_SOCKET"); path != "" {
return path
}
return filepath.Join("/var/run", outputSocketName)
}

View File

@ -0,0 +1,312 @@
package audio
import (
"context"
"sync"
"sync/atomic"
"time"
"github.com/rs/zerolog"
)
// LatencyMonitor tracks and optimizes audio latency in real-time
type LatencyMonitor struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
currentLatency int64 // Current latency in nanoseconds (atomic)
averageLatency int64 // Rolling average latency in nanoseconds (atomic)
minLatency int64 // Minimum observed latency in nanoseconds (atomic)
maxLatency int64 // Maximum observed latency in nanoseconds (atomic)
latencySamples int64 // Number of latency samples collected (atomic)
jitterAccumulator int64 // Accumulated jitter for variance calculation (atomic)
lastOptimization int64 // Timestamp of last optimization in nanoseconds (atomic)
config LatencyConfig
logger zerolog.Logger
// Control channels
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
// Optimization callbacks
optimizationCallbacks []OptimizationCallback
mutex sync.RWMutex
// Performance tracking
latencyHistory []LatencyMeasurement
historyMutex sync.RWMutex
}
// LatencyConfig holds configuration for latency monitoring
type LatencyConfig struct {
TargetLatency time.Duration // Target latency to maintain
MaxLatency time.Duration // Maximum acceptable latency
OptimizationInterval time.Duration // How often to run optimization
HistorySize int // Number of latency measurements to keep
JitterThreshold time.Duration // Jitter threshold for optimization
AdaptiveThreshold float64 // Threshold for adaptive adjustments (0.0-1.0)
}
// LatencyMeasurement represents a single latency measurement
type LatencyMeasurement struct {
Timestamp time.Time
Latency time.Duration
Jitter time.Duration
Source string // Source of the measurement (e.g., "input", "output", "processing")
}
// OptimizationCallback is called when latency optimization is triggered
type OptimizationCallback func(metrics LatencyMetrics) error
// LatencyMetrics provides comprehensive latency statistics
type LatencyMetrics struct {
Current time.Duration
Average time.Duration
Min time.Duration
Max time.Duration
Jitter time.Duration
SampleCount int64
Trend LatencyTrend
}
// LatencyTrend indicates the direction of latency changes
type LatencyTrend int
const (
LatencyTrendStable LatencyTrend = iota
LatencyTrendIncreasing
LatencyTrendDecreasing
LatencyTrendVolatile
)
// DefaultLatencyConfig returns a sensible default configuration
func DefaultLatencyConfig() LatencyConfig {
return LatencyConfig{
TargetLatency: 50 * time.Millisecond,
MaxLatency: 200 * time.Millisecond,
OptimizationInterval: 5 * time.Second,
HistorySize: 100,
JitterThreshold: 20 * time.Millisecond,
AdaptiveThreshold: 0.8, // Trigger optimization when 80% above target
}
}
// NewLatencyMonitor creates a new latency monitoring system
func NewLatencyMonitor(config LatencyConfig, logger zerolog.Logger) *LatencyMonitor {
ctx, cancel := context.WithCancel(context.Background())
return &LatencyMonitor{
config: config,
logger: logger.With().Str("component", "latency-monitor").Logger(),
ctx: ctx,
cancel: cancel,
latencyHistory: make([]LatencyMeasurement, 0, config.HistorySize),
minLatency: int64(time.Hour), // Initialize to high value
}
}
// Start begins latency monitoring and optimization
func (lm *LatencyMonitor) Start() {
lm.wg.Add(1)
go lm.monitoringLoop()
lm.logger.Info().Msg("Latency monitor started")
}
// Stop stops the latency monitor
func (lm *LatencyMonitor) Stop() {
lm.cancel()
lm.wg.Wait()
lm.logger.Info().Msg("Latency monitor stopped")
}
// RecordLatency records a new latency measurement
func (lm *LatencyMonitor) RecordLatency(latency time.Duration, source string) {
now := time.Now()
latencyNanos := latency.Nanoseconds()
// Update atomic counters
atomic.StoreInt64(&lm.currentLatency, latencyNanos)
atomic.AddInt64(&lm.latencySamples, 1)
// Update min/max
for {
oldMin := atomic.LoadInt64(&lm.minLatency)
if latencyNanos >= oldMin || atomic.CompareAndSwapInt64(&lm.minLatency, oldMin, latencyNanos) {
break
}
}
for {
oldMax := atomic.LoadInt64(&lm.maxLatency)
if latencyNanos <= oldMax || atomic.CompareAndSwapInt64(&lm.maxLatency, oldMax, latencyNanos) {
break
}
}
// Update rolling average using exponential moving average
oldAvg := atomic.LoadInt64(&lm.averageLatency)
newAvg := oldAvg + (latencyNanos-oldAvg)/10 // Alpha = 0.1
atomic.StoreInt64(&lm.averageLatency, newAvg)
// Calculate jitter (difference from average)
jitter := latencyNanos - newAvg
if jitter < 0 {
jitter = -jitter
}
atomic.AddInt64(&lm.jitterAccumulator, jitter)
// Store in history
lm.historyMutex.Lock()
measurement := LatencyMeasurement{
Timestamp: now,
Latency: latency,
Jitter: time.Duration(jitter),
Source: source,
}
if len(lm.latencyHistory) >= lm.config.HistorySize {
// Remove oldest measurement
copy(lm.latencyHistory, lm.latencyHistory[1:])
lm.latencyHistory[len(lm.latencyHistory)-1] = measurement
} else {
lm.latencyHistory = append(lm.latencyHistory, measurement)
}
lm.historyMutex.Unlock()
}
// GetMetrics returns current latency metrics
func (lm *LatencyMonitor) GetMetrics() LatencyMetrics {
current := atomic.LoadInt64(&lm.currentLatency)
average := atomic.LoadInt64(&lm.averageLatency)
min := atomic.LoadInt64(&lm.minLatency)
max := atomic.LoadInt64(&lm.maxLatency)
samples := atomic.LoadInt64(&lm.latencySamples)
jitterSum := atomic.LoadInt64(&lm.jitterAccumulator)
var jitter time.Duration
if samples > 0 {
jitter = time.Duration(jitterSum / samples)
}
return LatencyMetrics{
Current: time.Duration(current),
Average: time.Duration(average),
Min: time.Duration(min),
Max: time.Duration(max),
Jitter: jitter,
SampleCount: samples,
Trend: lm.calculateTrend(),
}
}
// AddOptimizationCallback adds a callback for latency optimization
func (lm *LatencyMonitor) AddOptimizationCallback(callback OptimizationCallback) {
lm.mutex.Lock()
lm.optimizationCallbacks = append(lm.optimizationCallbacks, callback)
lm.mutex.Unlock()
}
// monitoringLoop runs the main monitoring and optimization loop
func (lm *LatencyMonitor) monitoringLoop() {
defer lm.wg.Done()
ticker := time.NewTicker(lm.config.OptimizationInterval)
defer ticker.Stop()
for {
select {
case <-lm.ctx.Done():
return
case <-ticker.C:
lm.runOptimization()
}
}
}
// runOptimization checks if optimization is needed and triggers callbacks
func (lm *LatencyMonitor) runOptimization() {
metrics := lm.GetMetrics()
// Check if optimization is needed
needsOptimization := false
// Check if current latency exceeds threshold
if metrics.Current > lm.config.MaxLatency {
needsOptimization = true
lm.logger.Warn().Dur("current_latency", metrics.Current).Dur("max_latency", lm.config.MaxLatency).Msg("Latency exceeds maximum threshold")
}
// Check if average latency is above adaptive threshold
adaptiveThreshold := time.Duration(float64(lm.config.TargetLatency.Nanoseconds()) * (1.0 + lm.config.AdaptiveThreshold))
if metrics.Average > adaptiveThreshold {
needsOptimization = true
lm.logger.Info().Dur("average_latency", metrics.Average).Dur("threshold", adaptiveThreshold).Msg("Average latency above adaptive threshold")
}
// Check if jitter is too high
if metrics.Jitter > lm.config.JitterThreshold {
needsOptimization = true
lm.logger.Info().Dur("jitter", metrics.Jitter).Dur("threshold", lm.config.JitterThreshold).Msg("Jitter above threshold")
}
if needsOptimization {
atomic.StoreInt64(&lm.lastOptimization, time.Now().UnixNano())
// Run optimization callbacks
lm.mutex.RLock()
callbacks := make([]OptimizationCallback, len(lm.optimizationCallbacks))
copy(callbacks, lm.optimizationCallbacks)
lm.mutex.RUnlock()
for _, callback := range callbacks {
if err := callback(metrics); err != nil {
lm.logger.Error().Err(err).Msg("Optimization callback failed")
}
}
lm.logger.Info().Interface("metrics", metrics).Msg("Latency optimization triggered")
}
}
// calculateTrend analyzes recent latency measurements to determine trend
func (lm *LatencyMonitor) calculateTrend() LatencyTrend {
lm.historyMutex.RLock()
defer lm.historyMutex.RUnlock()
if len(lm.latencyHistory) < 10 {
return LatencyTrendStable
}
// Analyze last 10 measurements
recentMeasurements := lm.latencyHistory[len(lm.latencyHistory)-10:]
var increasing, decreasing int
for i := 1; i < len(recentMeasurements); i++ {
if recentMeasurements[i].Latency > recentMeasurements[i-1].Latency {
increasing++
} else if recentMeasurements[i].Latency < recentMeasurements[i-1].Latency {
decreasing++
}
}
// Determine trend based on direction changes
if increasing > 6 {
return LatencyTrendIncreasing
} else if decreasing > 6 {
return LatencyTrendDecreasing
} else if increasing+decreasing > 7 {
return LatencyTrendVolatile
}
return LatencyTrendStable
}
// GetLatencyHistory returns a copy of recent latency measurements
func (lm *LatencyMonitor) GetLatencyHistory() []LatencyMeasurement {
lm.historyMutex.RLock()
defer lm.historyMutex.RUnlock()
history := make([]LatencyMeasurement, len(lm.latencyHistory))
copy(history, lm.latencyHistory)
return history
}

View File

@ -0,0 +1,198 @@
package audio
import (
"encoding/json"
"net/http"
"runtime"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// MemoryMetrics provides comprehensive memory allocation statistics
type MemoryMetrics struct {
// Runtime memory statistics
RuntimeStats RuntimeMemoryStats `json:"runtime_stats"`
// Audio buffer pool statistics
BufferPools AudioBufferPoolStats `json:"buffer_pools"`
// Zero-copy frame pool statistics
ZeroCopyPool ZeroCopyFramePoolStats `json:"zero_copy_pool"`
// Message pool statistics
MessagePool MessagePoolStats `json:"message_pool"`
// Batch processor statistics
BatchProcessor BatchProcessorMemoryStats `json:"batch_processor,omitempty"`
// Collection timestamp
Timestamp time.Time `json:"timestamp"`
}
// RuntimeMemoryStats provides Go runtime memory statistics
type RuntimeMemoryStats struct {
Alloc uint64 `json:"alloc"` // Bytes allocated and not yet freed
TotalAlloc uint64 `json:"total_alloc"` // Total bytes allocated (cumulative)
Sys uint64 `json:"sys"` // Total bytes obtained from OS
Lookups uint64 `json:"lookups"` // Number of pointer lookups
Mallocs uint64 `json:"mallocs"` // Number of mallocs
Frees uint64 `json:"frees"` // Number of frees
HeapAlloc uint64 `json:"heap_alloc"` // Bytes allocated and not yet freed (heap)
HeapSys uint64 `json:"heap_sys"` // Bytes obtained from OS for heap
HeapIdle uint64 `json:"heap_idle"` // Bytes in idle spans
HeapInuse uint64 `json:"heap_inuse"` // Bytes in non-idle spans
HeapReleased uint64 `json:"heap_released"` // Bytes released to OS
HeapObjects uint64 `json:"heap_objects"` // Total number of allocated objects
StackInuse uint64 `json:"stack_inuse"` // Bytes used by stack spans
StackSys uint64 `json:"stack_sys"` // Bytes obtained from OS for stack
MSpanInuse uint64 `json:"mspan_inuse"` // Bytes used by mspan structures
MSpanSys uint64 `json:"mspan_sys"` // Bytes obtained from OS for mspan
MCacheInuse uint64 `json:"mcache_inuse"` // Bytes used by mcache structures
MCacheSys uint64 `json:"mcache_sys"` // Bytes obtained from OS for mcache
BuckHashSys uint64 `json:"buck_hash_sys"` // Bytes used by profiling bucket hash table
GCSys uint64 `json:"gc_sys"` // Bytes used for garbage collection metadata
OtherSys uint64 `json:"other_sys"` // Bytes used for other system allocations
NextGC uint64 `json:"next_gc"` // Target heap size for next GC
LastGC uint64 `json:"last_gc"` // Time of last GC (nanoseconds since epoch)
PauseTotalNs uint64 `json:"pause_total_ns"` // Total GC pause time
NumGC uint32 `json:"num_gc"` // Number of completed GC cycles
NumForcedGC uint32 `json:"num_forced_gc"` // Number of forced GC cycles
GCCPUFraction float64 `json:"gc_cpu_fraction"` // Fraction of CPU time used by GC
}
// BatchProcessorMemoryStats provides batch processor memory statistics
type BatchProcessorMemoryStats struct {
Initialized bool `json:"initialized"`
Running bool `json:"running"`
Stats BatchAudioStats `json:"stats"`
BufferPool AudioBufferPoolDetailedStats `json:"buffer_pool,omitempty"`
}
// GetBatchAudioProcessor is defined in batch_audio.go
// BatchAudioStats is defined in batch_audio.go
var memoryMetricsLogger *zerolog.Logger
func getMemoryMetricsLogger() *zerolog.Logger {
if memoryMetricsLogger == nil {
logger := logging.GetDefaultLogger().With().Str("component", "memory-metrics").Logger()
memoryMetricsLogger = &logger
}
return memoryMetricsLogger
}
// CollectMemoryMetrics gathers comprehensive memory allocation statistics
func CollectMemoryMetrics() MemoryMetrics {
// Collect runtime memory statistics
var m runtime.MemStats
runtime.ReadMemStats(&m)
runtimeStats := RuntimeMemoryStats{
Alloc: m.Alloc,
TotalAlloc: m.TotalAlloc,
Sys: m.Sys,
Lookups: m.Lookups,
Mallocs: m.Mallocs,
Frees: m.Frees,
HeapAlloc: m.HeapAlloc,
HeapSys: m.HeapSys,
HeapIdle: m.HeapIdle,
HeapInuse: m.HeapInuse,
HeapReleased: m.HeapReleased,
HeapObjects: m.HeapObjects,
StackInuse: m.StackInuse,
StackSys: m.StackSys,
MSpanInuse: m.MSpanInuse,
MSpanSys: m.MSpanSys,
MCacheInuse: m.MCacheInuse,
MCacheSys: m.MCacheSys,
BuckHashSys: m.BuckHashSys,
GCSys: m.GCSys,
OtherSys: m.OtherSys,
NextGC: m.NextGC,
LastGC: m.LastGC,
PauseTotalNs: m.PauseTotalNs,
NumGC: m.NumGC,
NumForcedGC: m.NumForcedGC,
GCCPUFraction: m.GCCPUFraction,
}
// Collect audio buffer pool statistics
bufferPoolStats := GetAudioBufferPoolStats()
// Collect zero-copy frame pool statistics
zeroCopyStats := GetGlobalZeroCopyPoolStats()
// Collect message pool statistics
messagePoolStats := GetGlobalMessagePoolStats()
// Collect batch processor statistics if available
var batchStats BatchProcessorMemoryStats
if processor := GetBatchAudioProcessor(); processor != nil {
batchStats.Initialized = true
batchStats.Running = processor.IsRunning()
batchStats.Stats = processor.GetStats()
// Note: BatchAudioProcessor uses sync.Pool, detailed stats not available
}
return MemoryMetrics{
RuntimeStats: runtimeStats,
BufferPools: bufferPoolStats,
ZeroCopyPool: zeroCopyStats,
MessagePool: messagePoolStats,
BatchProcessor: batchStats,
Timestamp: time.Now(),
}
}
// HandleMemoryMetrics provides an HTTP handler for memory metrics
func HandleMemoryMetrics(w http.ResponseWriter, r *http.Request) {
logger := getMemoryMetricsLogger()
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
metrics := CollectMemoryMetrics()
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-cache")
if err := json.NewEncoder(w).Encode(metrics); err != nil {
logger.Error().Err(err).Msg("failed to encode memory metrics")
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
logger.Debug().Msg("memory metrics served")
}
// LogMemoryMetrics logs current memory metrics for debugging
func LogMemoryMetrics() {
logger := getMemoryMetricsLogger()
metrics := CollectMemoryMetrics()
logger.Info().
Uint64("heap_alloc_mb", metrics.RuntimeStats.HeapAlloc/1024/1024).
Uint64("heap_sys_mb", metrics.RuntimeStats.HeapSys/1024/1024).
Uint64("heap_objects", metrics.RuntimeStats.HeapObjects).
Uint32("num_gc", metrics.RuntimeStats.NumGC).
Float64("gc_cpu_fraction", metrics.RuntimeStats.GCCPUFraction).
Float64("buffer_pool_hit_rate", metrics.BufferPools.FramePoolHitRate).
Float64("zero_copy_hit_rate", metrics.ZeroCopyPool.HitRate).
Float64("message_pool_hit_rate", metrics.MessagePool.HitRate).
Msg("memory metrics snapshot")
}
// StartMemoryMetricsLogging starts periodic memory metrics logging
func StartMemoryMetricsLogging(interval time.Duration) {
logger := getMemoryMetricsLogger()
logger.Info().Dur("interval", interval).Msg("starting memory metrics logging")
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
LogMemoryMetrics()
}
}()
}

480
internal/audio/metrics.go Normal file
View File

@ -0,0 +1,480 @@
package audio
import (
"sync"
"sync/atomic"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
// Adaptive buffer metrics
adaptiveInputBufferSize = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_adaptive_input_buffer_size_bytes",
Help: "Current adaptive input buffer size in bytes",
},
)
adaptiveOutputBufferSize = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_adaptive_output_buffer_size_bytes",
Help: "Current adaptive output buffer size in bytes",
},
)
adaptiveBufferAdjustmentsTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_adaptive_buffer_adjustments_total",
Help: "Total number of adaptive buffer size adjustments",
},
)
adaptiveSystemCpuPercent = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_adaptive_system_cpu_percent",
Help: "System CPU usage percentage used by adaptive buffer manager",
},
)
adaptiveSystemMemoryPercent = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_adaptive_system_memory_percent",
Help: "System memory usage percentage used by adaptive buffer manager",
},
)
// Socket buffer metrics
socketBufferSizeGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "jetkvm_audio_socket_buffer_size_bytes",
Help: "Current socket buffer size in bytes",
},
[]string{"component", "buffer_type"}, // buffer_type: send, receive
)
socketBufferUtilizationGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "jetkvm_audio_socket_buffer_utilization_percent",
Help: "Socket buffer utilization percentage",
},
[]string{"component", "buffer_type"}, // buffer_type: send, receive
)
socketBufferOverflowCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "jetkvm_audio_socket_buffer_overflow_total",
Help: "Total number of socket buffer overflows",
},
[]string{"component", "buffer_type"}, // buffer_type: send, receive
)
// Audio output metrics
audioFramesReceivedTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_audio_frames_received_total",
Help: "Total number of audio frames received",
},
)
audioFramesDroppedTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_audio_frames_dropped_total",
Help: "Total number of audio frames dropped",
},
)
audioBytesProcessedTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_audio_bytes_processed_total",
Help: "Total number of audio bytes processed",
},
)
audioConnectionDropsTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_audio_connection_drops_total",
Help: "Total number of audio connection drops",
},
)
audioAverageLatencySeconds = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_average_latency_seconds",
Help: "Average audio latency in seconds",
},
)
audioLastFrameTimestamp = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_last_frame_timestamp_seconds",
Help: "Timestamp of the last audio frame received",
},
)
// Microphone input metrics
microphoneFramesSentTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_microphone_frames_sent_total",
Help: "Total number of microphone frames sent",
},
)
microphoneFramesDroppedTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_microphone_frames_dropped_total",
Help: "Total number of microphone frames dropped",
},
)
microphoneBytesProcessedTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_microphone_bytes_processed_total",
Help: "Total number of microphone bytes processed",
},
)
microphoneConnectionDropsTotal = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_microphone_connection_drops_total",
Help: "Total number of microphone connection drops",
},
)
microphoneAverageLatencySeconds = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_average_latency_seconds",
Help: "Average microphone latency in seconds",
},
)
microphoneLastFrameTimestamp = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_last_frame_timestamp_seconds",
Help: "Timestamp of the last microphone frame sent",
},
)
// Audio subprocess process metrics
audioProcessCpuPercent = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_process_cpu_percent",
Help: "CPU usage percentage of audio output subprocess",
},
)
audioProcessMemoryPercent = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_process_memory_percent",
Help: "Memory usage percentage of audio output subprocess",
},
)
audioProcessMemoryRssBytes = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_process_memory_rss_bytes",
Help: "RSS memory usage in bytes of audio output subprocess",
},
)
audioProcessMemoryVmsBytes = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_process_memory_vms_bytes",
Help: "VMS memory usage in bytes of audio output subprocess",
},
)
audioProcessRunning = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_process_running",
Help: "Whether audio output subprocess is running (1=running, 0=stopped)",
},
)
// Microphone subprocess process metrics
microphoneProcessCpuPercent = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_process_cpu_percent",
Help: "CPU usage percentage of microphone input subprocess",
},
)
microphoneProcessMemoryPercent = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_process_memory_percent",
Help: "Memory usage percentage of microphone input subprocess",
},
)
microphoneProcessMemoryRssBytes = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_process_memory_rss_bytes",
Help: "RSS memory usage in bytes of microphone input subprocess",
},
)
microphoneProcessMemoryVmsBytes = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_process_memory_vms_bytes",
Help: "VMS memory usage in bytes of microphone input subprocess",
},
)
microphoneProcessRunning = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_process_running",
Help: "Whether microphone input subprocess is running (1=running, 0=stopped)",
},
)
// Audio configuration metrics
audioConfigQuality = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_config_quality",
Help: "Current audio quality setting (0=Low, 1=Medium, 2=High, 3=Ultra)",
},
)
audioConfigBitrate = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_config_bitrate_kbps",
Help: "Current audio bitrate in kbps",
},
)
audioConfigSampleRate = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_config_sample_rate_hz",
Help: "Current audio sample rate in Hz",
},
)
audioConfigChannels = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_audio_config_channels",
Help: "Current audio channel count",
},
)
microphoneConfigQuality = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_config_quality",
Help: "Current microphone quality setting (0=Low, 1=Medium, 2=High, 3=Ultra)",
},
)
microphoneConfigBitrate = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_config_bitrate_kbps",
Help: "Current microphone bitrate in kbps",
},
)
microphoneConfigSampleRate = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_config_sample_rate_hz",
Help: "Current microphone sample rate in Hz",
},
)
microphoneConfigChannels = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_microphone_config_channels",
Help: "Current microphone channel count",
},
)
// Metrics update tracking
metricsUpdateMutex sync.RWMutex
lastMetricsUpdate int64
// Counter value tracking (since prometheus counters don't have Get() method)
audioFramesReceivedValue int64
audioFramesDroppedValue int64
audioBytesProcessedValue int64
audioConnectionDropsValue int64
micFramesSentValue int64
micFramesDroppedValue int64
micBytesProcessedValue int64
micConnectionDropsValue int64
)
// UpdateAudioMetrics updates Prometheus metrics with current audio data
func UpdateAudioMetrics(metrics AudioMetrics) {
oldReceived := atomic.SwapInt64(&audioFramesReceivedValue, metrics.FramesReceived)
if metrics.FramesReceived > oldReceived {
audioFramesReceivedTotal.Add(float64(metrics.FramesReceived - oldReceived))
}
oldDropped := atomic.SwapInt64(&audioFramesDroppedValue, metrics.FramesDropped)
if metrics.FramesDropped > oldDropped {
audioFramesDroppedTotal.Add(float64(metrics.FramesDropped - oldDropped))
}
oldBytes := atomic.SwapInt64(&audioBytesProcessedValue, metrics.BytesProcessed)
if metrics.BytesProcessed > oldBytes {
audioBytesProcessedTotal.Add(float64(metrics.BytesProcessed - oldBytes))
}
oldDrops := atomic.SwapInt64(&audioConnectionDropsValue, metrics.ConnectionDrops)
if metrics.ConnectionDrops > oldDrops {
audioConnectionDropsTotal.Add(float64(metrics.ConnectionDrops - oldDrops))
}
// Update gauges
audioAverageLatencySeconds.Set(float64(metrics.AverageLatency.Nanoseconds()) / 1e9)
if !metrics.LastFrameTime.IsZero() {
audioLastFrameTimestamp.Set(float64(metrics.LastFrameTime.Unix()))
}
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// UpdateMicrophoneMetrics updates Prometheus metrics with current microphone data
func UpdateMicrophoneMetrics(metrics AudioInputMetrics) {
oldSent := atomic.SwapInt64(&micFramesSentValue, metrics.FramesSent)
if metrics.FramesSent > oldSent {
microphoneFramesSentTotal.Add(float64(metrics.FramesSent - oldSent))
}
oldDropped := atomic.SwapInt64(&micFramesDroppedValue, metrics.FramesDropped)
if metrics.FramesDropped > oldDropped {
microphoneFramesDroppedTotal.Add(float64(metrics.FramesDropped - oldDropped))
}
oldBytes := atomic.SwapInt64(&micBytesProcessedValue, metrics.BytesProcessed)
if metrics.BytesProcessed > oldBytes {
microphoneBytesProcessedTotal.Add(float64(metrics.BytesProcessed - oldBytes))
}
oldDrops := atomic.SwapInt64(&micConnectionDropsValue, metrics.ConnectionDrops)
if metrics.ConnectionDrops > oldDrops {
microphoneConnectionDropsTotal.Add(float64(metrics.ConnectionDrops - oldDrops))
}
// Update gauges
microphoneAverageLatencySeconds.Set(float64(metrics.AverageLatency.Nanoseconds()) / 1e9)
if !metrics.LastFrameTime.IsZero() {
microphoneLastFrameTimestamp.Set(float64(metrics.LastFrameTime.Unix()))
}
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// UpdateAudioProcessMetrics updates Prometheus metrics with audio subprocess data
func UpdateAudioProcessMetrics(metrics ProcessMetrics, isRunning bool) {
metricsUpdateMutex.Lock()
defer metricsUpdateMutex.Unlock()
audioProcessCpuPercent.Set(metrics.CPUPercent)
audioProcessMemoryPercent.Set(metrics.MemoryPercent)
audioProcessMemoryRssBytes.Set(float64(metrics.MemoryRSS))
audioProcessMemoryVmsBytes.Set(float64(metrics.MemoryVMS))
if isRunning {
audioProcessRunning.Set(1)
} else {
audioProcessRunning.Set(0)
}
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// UpdateMicrophoneProcessMetrics updates Prometheus metrics with microphone subprocess data
func UpdateMicrophoneProcessMetrics(metrics ProcessMetrics, isRunning bool) {
metricsUpdateMutex.Lock()
defer metricsUpdateMutex.Unlock()
microphoneProcessCpuPercent.Set(metrics.CPUPercent)
microphoneProcessMemoryPercent.Set(metrics.MemoryPercent)
microphoneProcessMemoryRssBytes.Set(float64(metrics.MemoryRSS))
microphoneProcessMemoryVmsBytes.Set(float64(metrics.MemoryVMS))
if isRunning {
microphoneProcessRunning.Set(1)
} else {
microphoneProcessRunning.Set(0)
}
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// UpdateAudioConfigMetrics updates Prometheus metrics with audio configuration
func UpdateAudioConfigMetrics(config AudioConfig) {
metricsUpdateMutex.Lock()
defer metricsUpdateMutex.Unlock()
audioConfigQuality.Set(float64(config.Quality))
audioConfigBitrate.Set(float64(config.Bitrate))
audioConfigSampleRate.Set(float64(config.SampleRate))
audioConfigChannels.Set(float64(config.Channels))
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// UpdateMicrophoneConfigMetrics updates Prometheus metrics with microphone configuration
func UpdateMicrophoneConfigMetrics(config AudioConfig) {
metricsUpdateMutex.Lock()
defer metricsUpdateMutex.Unlock()
microphoneConfigQuality.Set(float64(config.Quality))
microphoneConfigBitrate.Set(float64(config.Bitrate))
microphoneConfigSampleRate.Set(float64(config.SampleRate))
microphoneConfigChannels.Set(float64(config.Channels))
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// UpdateAdaptiveBufferMetrics updates Prometheus metrics with adaptive buffer information
func UpdateAdaptiveBufferMetrics(inputBufferSize, outputBufferSize int, cpuPercent, memoryPercent float64, adjustmentMade bool) {
metricsUpdateMutex.Lock()
defer metricsUpdateMutex.Unlock()
adaptiveInputBufferSize.Set(float64(inputBufferSize))
adaptiveOutputBufferSize.Set(float64(outputBufferSize))
adaptiveSystemCpuPercent.Set(cpuPercent)
adaptiveSystemMemoryPercent.Set(memoryPercent)
if adjustmentMade {
adaptiveBufferAdjustmentsTotal.Inc()
}
atomic.StoreInt64(&lastMetricsUpdate, time.Now().Unix())
}
// GetLastMetricsUpdate returns the timestamp of the last metrics update
func GetLastMetricsUpdate() time.Time {
timestamp := atomic.LoadInt64(&lastMetricsUpdate)
return time.Unix(timestamp, 0)
}
// StartMetricsUpdater starts a goroutine that periodically updates Prometheus metrics
func StartMetricsUpdater() {
go func() {
ticker := time.NewTicker(5 * time.Second) // Update every 5 seconds
defer ticker.Stop()
for range ticker.C {
// Update audio output metrics
audioMetrics := GetAudioMetrics()
UpdateAudioMetrics(audioMetrics)
// Update microphone input metrics
micMetrics := GetAudioInputMetrics()
UpdateMicrophoneMetrics(micMetrics)
// Update microphone subprocess process metrics
if inputSupervisor := GetAudioInputIPCSupervisor(); inputSupervisor != nil {
if processMetrics := inputSupervisor.GetProcessMetrics(); processMetrics != nil {
UpdateMicrophoneProcessMetrics(*processMetrics, inputSupervisor.IsRunning())
}
}
// Update audio configuration metrics
audioConfig := GetAudioConfig()
UpdateAudioConfigMetrics(audioConfig)
micConfig := GetMicrophoneConfig()
UpdateMicrophoneConfigMetrics(micConfig)
}
}()
}

View File

@ -0,0 +1,127 @@
package audio
import (
"sync/atomic"
"time"
"unsafe"
)
// MicrophoneContentionManager manages microphone access with cooldown periods
type MicrophoneContentionManager struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
lastOpNano int64
cooldownNanos int64
operationID int64
lockPtr unsafe.Pointer
}
func NewMicrophoneContentionManager(cooldown time.Duration) *MicrophoneContentionManager {
return &MicrophoneContentionManager{
cooldownNanos: int64(cooldown),
}
}
type OperationResult struct {
Allowed bool
RemainingCooldown time.Duration
OperationID int64
}
func (mcm *MicrophoneContentionManager) TryOperation() OperationResult {
now := time.Now().UnixNano()
cooldown := atomic.LoadInt64(&mcm.cooldownNanos)
lastOp := atomic.LoadInt64(&mcm.lastOpNano)
elapsed := now - lastOp
if elapsed >= cooldown {
if atomic.CompareAndSwapInt64(&mcm.lastOpNano, lastOp, now) {
opID := atomic.AddInt64(&mcm.operationID, 1)
return OperationResult{
Allowed: true,
RemainingCooldown: 0,
OperationID: opID,
}
}
// Retry once if CAS failed
lastOp = atomic.LoadInt64(&mcm.lastOpNano)
elapsed = now - lastOp
if elapsed >= cooldown && atomic.CompareAndSwapInt64(&mcm.lastOpNano, lastOp, now) {
opID := atomic.AddInt64(&mcm.operationID, 1)
return OperationResult{
Allowed: true,
RemainingCooldown: 0,
OperationID: opID,
}
}
}
remaining := time.Duration(cooldown - elapsed)
if remaining < 0 {
remaining = 0
}
return OperationResult{
Allowed: false,
RemainingCooldown: remaining,
OperationID: atomic.LoadInt64(&mcm.operationID),
}
}
func (mcm *MicrophoneContentionManager) SetCooldown(cooldown time.Duration) {
atomic.StoreInt64(&mcm.cooldownNanos, int64(cooldown))
}
func (mcm *MicrophoneContentionManager) GetCooldown() time.Duration {
return time.Duration(atomic.LoadInt64(&mcm.cooldownNanos))
}
func (mcm *MicrophoneContentionManager) GetLastOperationTime() time.Time {
nanos := atomic.LoadInt64(&mcm.lastOpNano)
if nanos == 0 {
return time.Time{}
}
return time.Unix(0, nanos)
}
func (mcm *MicrophoneContentionManager) GetOperationCount() int64 {
return atomic.LoadInt64(&mcm.operationID)
}
func (mcm *MicrophoneContentionManager) Reset() {
atomic.StoreInt64(&mcm.lastOpNano, 0)
atomic.StoreInt64(&mcm.operationID, 0)
}
var (
globalMicContentionManager unsafe.Pointer
micContentionInitialized int32
)
func GetMicrophoneContentionManager() *MicrophoneContentionManager {
ptr := atomic.LoadPointer(&globalMicContentionManager)
if ptr != nil {
return (*MicrophoneContentionManager)(ptr)
}
if atomic.CompareAndSwapInt32(&micContentionInitialized, 0, 1) {
manager := NewMicrophoneContentionManager(200 * time.Millisecond)
atomic.StorePointer(&globalMicContentionManager, unsafe.Pointer(manager))
return manager
}
ptr = atomic.LoadPointer(&globalMicContentionManager)
if ptr != nil {
return (*MicrophoneContentionManager)(ptr)
}
return NewMicrophoneContentionManager(200 * time.Millisecond)
}
func TryMicrophoneOperation() OperationResult {
return GetMicrophoneContentionManager().TryOperation()
}
func SetMicrophoneCooldown(cooldown time.Duration) {
GetMicrophoneContentionManager().SetCooldown(cooldown)
}

View File

@ -0,0 +1,71 @@
package audio
import (
"context"
"os"
"os/signal"
"syscall"
"time"
"github.com/jetkvm/kvm/internal/logging"
)
// RunAudioOutputServer runs the audio output server subprocess
// This should be called from main() when the subprocess is detected
func RunAudioOutputServer() error {
logger := logging.GetDefaultLogger().With().Str("component", "audio-output-server").Logger()
logger.Info().Msg("Starting audio output server subprocess")
// Create audio server
server, err := NewAudioServer()
if err != nil {
logger.Error().Err(err).Msg("failed to create audio server")
return err
}
defer server.Close()
// Start accepting connections
if err := server.Start(); err != nil {
logger.Error().Err(err).Msg("failed to start audio server")
return err
}
// Initialize audio processing
err = StartNonBlockingAudioStreaming(func(frame []byte) {
if err := server.SendFrame(frame); err != nil {
logger.Warn().Err(err).Msg("failed to send audio frame")
RecordFrameDropped()
}
})
if err != nil {
logger.Error().Err(err).Msg("failed to start audio processing")
return err
}
logger.Info().Msg("Audio output server started, waiting for connections")
// Set up signal handling for graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
// Wait for shutdown signal
select {
case sig := <-sigChan:
logger.Info().Str("signal", sig.String()).Msg("Received shutdown signal")
case <-ctx.Done():
logger.Info().Msg("Context cancelled")
}
// Graceful shutdown
logger.Info().Msg("Shutting down audio output server")
StopNonBlockingAudioStreaming()
// Give some time for cleanup
time.Sleep(100 * time.Millisecond)
logger.Info().Msg("Audio output server subprocess stopped")
return nil
}

View File

@ -0,0 +1,369 @@
package audio
import (
"context"
"fmt"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// OutputStreamer manages high-performance audio output streaming
type OutputStreamer struct {
// Atomic fields must be first for proper alignment on ARM
processedFrames int64 // Total processed frames counter (atomic)
droppedFrames int64 // Dropped frames counter (atomic)
processingTime int64 // Average processing time in nanoseconds (atomic)
lastStatsTime int64 // Last statistics update time (atomic)
client *AudioClient
bufferPool *AudioBufferPool
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
running bool
mtx sync.Mutex
// Performance optimization fields
batchSize int // Adaptive batch size for frame processing
processingChan chan []byte // Buffered channel for frame processing
statsInterval time.Duration // Statistics reporting interval
}
var (
outputStreamingRunning int32
outputStreamingCancel context.CancelFunc
outputStreamingLogger *zerolog.Logger
)
func getOutputStreamingLogger() *zerolog.Logger {
if outputStreamingLogger == nil {
logger := logging.GetDefaultLogger().With().Str("component", "audio-output").Logger()
outputStreamingLogger = &logger
}
return outputStreamingLogger
}
func NewOutputStreamer() (*OutputStreamer, error) {
client := NewAudioClient()
// Get initial batch size from adaptive buffer manager
adaptiveManager := GetAdaptiveBufferManager()
initialBatchSize := adaptiveManager.GetOutputBufferSize()
ctx, cancel := context.WithCancel(context.Background())
return &OutputStreamer{
client: client,
bufferPool: NewAudioBufferPool(MaxAudioFrameSize), // Use existing buffer pool
ctx: ctx,
cancel: cancel,
batchSize: initialBatchSize, // Use adaptive batch size
processingChan: make(chan []byte, 500), // Large buffer for smooth processing
statsInterval: 5 * time.Second, // Statistics every 5 seconds
lastStatsTime: time.Now().UnixNano(),
}, nil
}
func (s *OutputStreamer) Start() error {
s.mtx.Lock()
defer s.mtx.Unlock()
if s.running {
return fmt.Errorf("output streamer already running")
}
// Connect to audio output server
if err := s.client.Connect(); err != nil {
return fmt.Errorf("failed to connect to audio output server: %w", err)
}
s.running = true
// Start multiple goroutines for optimal performance
s.wg.Add(3)
go s.streamLoop() // Main streaming loop
go s.processingLoop() // Frame processing loop
go s.statisticsLoop() // Performance monitoring loop
return nil
}
func (s *OutputStreamer) Stop() {
s.mtx.Lock()
defer s.mtx.Unlock()
if !s.running {
return
}
s.running = false
s.cancel()
// Close processing channel to signal goroutines
close(s.processingChan)
// Wait for all goroutines to finish
s.wg.Wait()
if s.client != nil {
s.client.Close()
}
}
func (s *OutputStreamer) streamLoop() {
defer s.wg.Done()
// Pin goroutine to OS thread for consistent performance
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// Adaptive timing for frame reading
frameInterval := time.Duration(20) * time.Millisecond // 50 FPS base rate
ticker := time.NewTicker(frameInterval)
defer ticker.Stop()
// Batch size update ticker
batchUpdateTicker := time.NewTicker(500 * time.Millisecond)
defer batchUpdateTicker.Stop()
for {
select {
case <-s.ctx.Done():
return
case <-batchUpdateTicker.C:
// Update batch size from adaptive buffer manager
s.UpdateBatchSize()
case <-ticker.C:
// Read audio data from CGO with timing measurement
startTime := time.Now()
frameBuf := s.bufferPool.Get()
n, err := CGOAudioReadEncode(frameBuf)
processingDuration := time.Since(startTime)
if err != nil {
getOutputStreamingLogger().Warn().Err(err).Msg("Failed to read audio data")
s.bufferPool.Put(frameBuf)
atomic.AddInt64(&s.droppedFrames, 1)
continue
}
if n > 0 {
// Send frame for processing (non-blocking)
frameData := make([]byte, n)
copy(frameData, frameBuf[:n])
select {
case s.processingChan <- frameData:
atomic.AddInt64(&s.processedFrames, 1)
// Update processing time statistics
atomic.StoreInt64(&s.processingTime, int64(processingDuration))
// Report latency to adaptive buffer manager
s.ReportLatency(processingDuration)
default:
// Processing channel full, drop frame
atomic.AddInt64(&s.droppedFrames, 1)
}
}
s.bufferPool.Put(frameBuf)
}
}
}
// processingLoop handles frame processing in a separate goroutine
func (s *OutputStreamer) processingLoop() {
defer s.wg.Done()
// Pin goroutine to OS thread for consistent performance
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// Set high priority for audio output processing
if err := SetAudioThreadPriority(); err != nil {
getOutputStreamingLogger().Warn().Err(err).Msg("Failed to set audio output processing priority")
}
defer func() {
if err := ResetThreadPriority(); err != nil {
getOutputStreamingLogger().Warn().Err(err).Msg("Failed to reset thread priority")
}
}()
for range s.processingChan {
// Process frame (currently just receiving, but can be extended)
if _, err := s.client.ReceiveFrame(); err != nil {
if s.client.IsConnected() {
getOutputStreamingLogger().Warn().Err(err).Msg("Failed to receive frame")
atomic.AddInt64(&s.droppedFrames, 1)
}
// Try to reconnect if disconnected
if !s.client.IsConnected() {
if err := s.client.Connect(); err != nil {
getOutputStreamingLogger().Warn().Err(err).Msg("Failed to reconnect")
}
}
}
}
}
// statisticsLoop monitors and reports performance statistics
func (s *OutputStreamer) statisticsLoop() {
defer s.wg.Done()
ticker := time.NewTicker(s.statsInterval)
defer ticker.Stop()
for {
select {
case <-s.ctx.Done():
return
case <-ticker.C:
s.reportStatistics()
}
}
}
// reportStatistics logs current performance statistics
func (s *OutputStreamer) reportStatistics() {
processed := atomic.LoadInt64(&s.processedFrames)
dropped := atomic.LoadInt64(&s.droppedFrames)
processingTime := atomic.LoadInt64(&s.processingTime)
if processed > 0 {
dropRate := float64(dropped) / float64(processed+dropped) * 100
avgProcessingTime := time.Duration(processingTime)
getOutputStreamingLogger().Info().Int64("processed", processed).Int64("dropped", dropped).Float64("drop_rate", dropRate).Dur("avg_processing", avgProcessingTime).Msg("Output Audio Stats")
// Get client statistics
clientTotal, clientDropped := s.client.GetClientStats()
getOutputStreamingLogger().Info().Int64("total", clientTotal).Int64("dropped", clientDropped).Msg("Client Stats")
}
}
// GetStats returns streaming statistics
func (s *OutputStreamer) GetStats() (processed, dropped int64, avgProcessingTime time.Duration) {
processed = atomic.LoadInt64(&s.processedFrames)
dropped = atomic.LoadInt64(&s.droppedFrames)
processingTimeNs := atomic.LoadInt64(&s.processingTime)
avgProcessingTime = time.Duration(processingTimeNs)
return
}
// GetDetailedStats returns comprehensive streaming statistics
func (s *OutputStreamer) GetDetailedStats() map[string]interface{} {
processed := atomic.LoadInt64(&s.processedFrames)
dropped := atomic.LoadInt64(&s.droppedFrames)
processingTime := atomic.LoadInt64(&s.processingTime)
stats := map[string]interface{}{
"processed_frames": processed,
"dropped_frames": dropped,
"avg_processing_time_ns": processingTime,
"batch_size": s.batchSize,
"channel_buffer_size": cap(s.processingChan),
"channel_current_size": len(s.processingChan),
"connected": s.client.IsConnected(),
}
if processed+dropped > 0 {
stats["drop_rate_percent"] = float64(dropped) / float64(processed+dropped) * 100
}
// Add client statistics
clientTotal, clientDropped := s.client.GetClientStats()
stats["client_total_frames"] = clientTotal
stats["client_dropped_frames"] = clientDropped
return stats
}
// UpdateBatchSize updates the batch size from adaptive buffer manager
func (s *OutputStreamer) UpdateBatchSize() {
s.mtx.Lock()
adaptiveManager := GetAdaptiveBufferManager()
s.batchSize = adaptiveManager.GetOutputBufferSize()
s.mtx.Unlock()
}
// ReportLatency reports processing latency to adaptive buffer manager
func (s *OutputStreamer) ReportLatency(latency time.Duration) {
adaptiveManager := GetAdaptiveBufferManager()
adaptiveManager.UpdateLatency(latency)
}
// StartAudioOutputStreaming starts audio output streaming (capturing system audio)
func StartAudioOutputStreaming(send func([]byte)) error {
if !atomic.CompareAndSwapInt32(&outputStreamingRunning, 0, 1) {
return ErrAudioAlreadyRunning
}
// Initialize CGO audio capture
if err := CGOAudioInit(); err != nil {
atomic.StoreInt32(&outputStreamingRunning, 0)
return err
}
ctx, cancel := context.WithCancel(context.Background())
outputStreamingCancel = cancel
// Start audio capture loop
go func() {
defer func() {
CGOAudioClose()
atomic.StoreInt32(&outputStreamingRunning, 0)
getOutputStreamingLogger().Info().Msg("Audio output streaming stopped")
}()
getOutputStreamingLogger().Info().Msg("Audio output streaming started")
buffer := make([]byte, MaxAudioFrameSize)
for {
select {
case <-ctx.Done():
return
default:
// Capture audio frame
n, err := CGOAudioReadEncode(buffer)
if err != nil {
getOutputStreamingLogger().Warn().Err(err).Msg("Failed to read/encode audio")
continue
}
if n > 0 {
// Get frame buffer from pool to reduce allocations
frame := GetAudioFrameBuffer()
frame = frame[:n] // Resize to actual frame size
copy(frame, buffer[:n])
send(frame)
// Return buffer to pool after sending
PutAudioFrameBuffer(frame)
RecordFrameReceived(n)
}
// Small delay to prevent busy waiting
time.Sleep(10 * time.Millisecond)
}
}
}()
return nil
}
// StopAudioOutputStreaming stops audio output streaming
func StopAudioOutputStreaming() {
if atomic.LoadInt32(&outputStreamingRunning) == 0 {
return
}
if outputStreamingCancel != nil {
outputStreamingCancel()
outputStreamingCancel = nil
}
// Wait for streaming to stop
for atomic.LoadInt32(&outputStreamingRunning) == 1 {
time.Sleep(10 * time.Millisecond)
}
}

View File

@ -0,0 +1,165 @@
//go:build linux
package audio
import (
"runtime"
"syscall"
"unsafe"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// SchedParam represents scheduling parameters for Linux
type SchedParam struct {
Priority int32
}
// Priority levels for audio processing
const (
// SCHED_FIFO priorities (1-99, higher = more priority)
AudioHighPriority = 80 // High priority for critical audio processing
AudioMediumPriority = 60 // Medium priority for regular audio processing
AudioLowPriority = 40 // Low priority for background audio tasks
// SCHED_NORMAL is the default (priority 0)
NormalPriority = 0
)
// Scheduling policies
const (
SCHED_NORMAL = 0
SCHED_FIFO = 1
SCHED_RR = 2
)
// PriorityScheduler manages thread priorities for audio processing
type PriorityScheduler struct {
logger zerolog.Logger
enabled bool
}
// NewPriorityScheduler creates a new priority scheduler
func NewPriorityScheduler() *PriorityScheduler {
return &PriorityScheduler{
logger: logging.GetDefaultLogger().With().Str("component", "priority-scheduler").Logger(),
enabled: true,
}
}
// SetThreadPriority sets the priority of the current thread
func (ps *PriorityScheduler) SetThreadPriority(priority int, policy int) error {
if !ps.enabled {
return nil
}
// Lock to OS thread to ensure we're setting priority for the right thread
runtime.LockOSThread()
// Get current thread ID
tid := syscall.Gettid()
// Set scheduling parameters
param := &SchedParam{
Priority: int32(priority),
}
// Use syscall to set scheduler
_, _, errno := syscall.Syscall(syscall.SYS_SCHED_SETSCHEDULER,
uintptr(tid),
uintptr(policy),
uintptr(unsafe.Pointer(param)))
if errno != 0 {
// If we can't set real-time priority, try nice value instead
if policy != SCHED_NORMAL {
ps.logger.Warn().Int("errno", int(errno)).Msg("Failed to set real-time priority, falling back to nice")
return ps.setNicePriority(priority)
}
return errno
}
ps.logger.Debug().Int("tid", tid).Int("priority", priority).Int("policy", policy).Msg("Thread priority set")
return nil
}
// setNicePriority sets nice value as fallback when real-time scheduling is not available
func (ps *PriorityScheduler) setNicePriority(rtPriority int) error {
// Convert real-time priority to nice value (inverse relationship)
// RT priority 80 -> nice -10, RT priority 40 -> nice 0
niceValue := (40 - rtPriority) / 4
if niceValue < -20 {
niceValue = -20
}
if niceValue > 19 {
niceValue = 19
}
err := syscall.Setpriority(syscall.PRIO_PROCESS, 0, niceValue)
if err != nil {
ps.logger.Warn().Err(err).Int("nice", niceValue).Msg("Failed to set nice priority")
return err
}
ps.logger.Debug().Int("nice", niceValue).Msg("Nice priority set as fallback")
return nil
}
// SetAudioProcessingPriority sets high priority for audio processing threads
func (ps *PriorityScheduler) SetAudioProcessingPriority() error {
return ps.SetThreadPriority(AudioHighPriority, SCHED_FIFO)
}
// SetAudioIOPriority sets medium priority for audio I/O threads
func (ps *PriorityScheduler) SetAudioIOPriority() error {
return ps.SetThreadPriority(AudioMediumPriority, SCHED_FIFO)
}
// SetAudioBackgroundPriority sets low priority for background audio tasks
func (ps *PriorityScheduler) SetAudioBackgroundPriority() error {
return ps.SetThreadPriority(AudioLowPriority, SCHED_FIFO)
}
// ResetPriority resets thread to normal scheduling
func (ps *PriorityScheduler) ResetPriority() error {
return ps.SetThreadPriority(NormalPriority, SCHED_NORMAL)
}
// Disable disables priority scheduling (useful for testing or fallback)
func (ps *PriorityScheduler) Disable() {
ps.enabled = false
ps.logger.Info().Msg("Priority scheduling disabled")
}
// Enable enables priority scheduling
func (ps *PriorityScheduler) Enable() {
ps.enabled = true
ps.logger.Info().Msg("Priority scheduling enabled")
}
// Global priority scheduler instance
var globalPriorityScheduler *PriorityScheduler
// GetPriorityScheduler returns the global priority scheduler instance
func GetPriorityScheduler() *PriorityScheduler {
if globalPriorityScheduler == nil {
globalPriorityScheduler = NewPriorityScheduler()
}
return globalPriorityScheduler
}
// SetAudioThreadPriority is a convenience function to set audio processing priority
func SetAudioThreadPriority() error {
return GetPriorityScheduler().SetAudioProcessingPriority()
}
// SetAudioIOThreadPriority is a convenience function to set audio I/O priority
func SetAudioIOThreadPriority() error {
return GetPriorityScheduler().SetAudioIOPriority()
}
// ResetThreadPriority is a convenience function to reset thread priority
func ResetThreadPriority() error {
return GetPriorityScheduler().ResetPriority()
}

View File

@ -0,0 +1,384 @@
package audio
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
// Constants for process monitoring
const (
// System constants
pageSize = 4096
maxCPUPercent = 100.0
minCPUPercent = 0.01
defaultClockTicks = 250.0 // Common for embedded ARM systems
defaultMemoryGB = 8
// Monitoring thresholds
maxWarmupSamples = 3
warmupCPUSamples = 2
logThrottleInterval = 10
// Channel buffer size
metricsChannelBuffer = 100
// Clock tick detection ranges
minValidClockTicks = 50
maxValidClockTicks = 1000
)
// ProcessMetrics represents CPU and memory usage metrics for a process
type ProcessMetrics struct {
PID int `json:"pid"`
CPUPercent float64 `json:"cpu_percent"`
MemoryRSS int64 `json:"memory_rss_bytes"`
MemoryVMS int64 `json:"memory_vms_bytes"`
MemoryPercent float64 `json:"memory_percent"`
Timestamp time.Time `json:"timestamp"`
ProcessName string `json:"process_name"`
}
type ProcessMonitor struct {
logger zerolog.Logger
mutex sync.RWMutex
monitoredPIDs map[int]*processState
running bool
stopChan chan struct{}
metricsChan chan ProcessMetrics
updateInterval time.Duration
totalMemory int64
memoryOnce sync.Once
clockTicks float64
clockTicksOnce sync.Once
}
// processState tracks the state needed for CPU calculation
type processState struct {
name string
lastCPUTime int64
lastSysTime int64
lastUserTime int64
lastSample time.Time
warmupSamples int
}
// NewProcessMonitor creates a new process monitor
func NewProcessMonitor() *ProcessMonitor {
return &ProcessMonitor{
logger: logging.GetDefaultLogger().With().Str("component", "process-monitor").Logger(),
monitoredPIDs: make(map[int]*processState),
stopChan: make(chan struct{}),
metricsChan: make(chan ProcessMetrics, metricsChannelBuffer),
updateInterval: GetMetricsUpdateInterval(),
}
}
// Start begins monitoring processes
func (pm *ProcessMonitor) Start() {
pm.mutex.Lock()
defer pm.mutex.Unlock()
if pm.running {
return
}
pm.running = true
go pm.monitorLoop()
pm.logger.Info().Msg("Process monitor started")
}
// Stop stops monitoring processes
func (pm *ProcessMonitor) Stop() {
pm.mutex.Lock()
defer pm.mutex.Unlock()
if !pm.running {
return
}
pm.running = false
close(pm.stopChan)
pm.logger.Info().Msg("Process monitor stopped")
}
// AddProcess adds a process to monitor
func (pm *ProcessMonitor) AddProcess(pid int, name string) {
pm.mutex.Lock()
defer pm.mutex.Unlock()
pm.monitoredPIDs[pid] = &processState{
name: name,
lastSample: time.Now(),
}
pm.logger.Info().Int("pid", pid).Str("name", name).Msg("Added process to monitor")
}
// RemoveProcess removes a process from monitoring
func (pm *ProcessMonitor) RemoveProcess(pid int) {
pm.mutex.Lock()
defer pm.mutex.Unlock()
delete(pm.monitoredPIDs, pid)
pm.logger.Info().Int("pid", pid).Msg("Removed process from monitor")
}
// GetMetricsChan returns the channel for receiving metrics
func (pm *ProcessMonitor) GetMetricsChan() <-chan ProcessMetrics {
return pm.metricsChan
}
// GetCurrentMetrics returns current metrics for all monitored processes
func (pm *ProcessMonitor) GetCurrentMetrics() []ProcessMetrics {
pm.mutex.RLock()
defer pm.mutex.RUnlock()
var metrics []ProcessMetrics
for pid, state := range pm.monitoredPIDs {
if metric, err := pm.collectMetrics(pid, state); err == nil {
metrics = append(metrics, metric)
}
}
return metrics
}
// monitorLoop is the main monitoring loop
func (pm *ProcessMonitor) monitorLoop() {
ticker := time.NewTicker(pm.updateInterval)
defer ticker.Stop()
for {
select {
case <-pm.stopChan:
return
case <-ticker.C:
pm.collectAllMetrics()
}
}
}
func (pm *ProcessMonitor) collectAllMetrics() {
pm.mutex.RLock()
pidsToCheck := make([]int, 0, len(pm.monitoredPIDs))
states := make([]*processState, 0, len(pm.monitoredPIDs))
for pid, state := range pm.monitoredPIDs {
pidsToCheck = append(pidsToCheck, pid)
states = append(states, state)
}
pm.mutex.RUnlock()
deadPIDs := make([]int, 0)
for i, pid := range pidsToCheck {
if metric, err := pm.collectMetrics(pid, states[i]); err == nil {
select {
case pm.metricsChan <- metric:
default:
}
} else {
deadPIDs = append(deadPIDs, pid)
}
}
for _, pid := range deadPIDs {
pm.RemoveProcess(pid)
}
}
func (pm *ProcessMonitor) collectMetrics(pid int, state *processState) (ProcessMetrics, error) {
now := time.Now()
metric := ProcessMetrics{
PID: pid,
Timestamp: now,
ProcessName: state.name,
}
statPath := fmt.Sprintf("/proc/%d/stat", pid)
statData, err := os.ReadFile(statPath)
if err != nil {
return metric, err
}
fields := strings.Fields(string(statData))
if len(fields) < 24 {
return metric, fmt.Errorf("invalid stat format")
}
utime, _ := strconv.ParseInt(fields[13], 10, 64)
stime, _ := strconv.ParseInt(fields[14], 10, 64)
totalCPUTime := utime + stime
vsize, _ := strconv.ParseInt(fields[22], 10, 64)
rss, _ := strconv.ParseInt(fields[23], 10, 64)
metric.MemoryRSS = rss * pageSize
metric.MemoryVMS = vsize
// Calculate CPU percentage
metric.CPUPercent = pm.calculateCPUPercent(totalCPUTime, state, now)
// Increment warmup counter
if state.warmupSamples < maxWarmupSamples {
state.warmupSamples++
}
// Calculate memory percentage (RSS / total system memory)
if totalMem := pm.getTotalMemory(); totalMem > 0 {
metric.MemoryPercent = float64(metric.MemoryRSS) / float64(totalMem) * 100.0
}
// Update state for next calculation
state.lastCPUTime = totalCPUTime
state.lastUserTime = utime
state.lastSysTime = stime
state.lastSample = now
return metric, nil
}
// calculateCPUPercent calculates CPU percentage for a process
func (pm *ProcessMonitor) calculateCPUPercent(totalCPUTime int64, state *processState, now time.Time) float64 {
if state.lastSample.IsZero() {
// First sample - initialize baseline
state.warmupSamples = 0
return 0.0
}
timeDelta := now.Sub(state.lastSample).Seconds()
cpuDelta := float64(totalCPUTime - state.lastCPUTime)
if timeDelta <= 0 {
return 0.0
}
if cpuDelta > 0 {
// Convert from clock ticks to seconds using actual system clock ticks
clockTicks := pm.getClockTicks()
cpuSeconds := cpuDelta / clockTicks
cpuPercent := (cpuSeconds / timeDelta) * 100.0
// Apply bounds
if cpuPercent > maxCPUPercent {
cpuPercent = maxCPUPercent
}
if cpuPercent < minCPUPercent {
cpuPercent = minCPUPercent
}
return cpuPercent
}
// No CPU delta - process was idle
if state.warmupSamples < warmupCPUSamples {
// During warmup, provide a small non-zero value to indicate process is alive
return minCPUPercent
}
return 0.0
}
func (pm *ProcessMonitor) getClockTicks() float64 {
pm.clockTicksOnce.Do(func() {
// Try to detect actual clock ticks from kernel boot parameters or /proc/stat
if data, err := os.ReadFile("/proc/cmdline"); err == nil {
// Look for HZ parameter in kernel command line
cmdline := string(data)
if strings.Contains(cmdline, "HZ=") {
fields := strings.Fields(cmdline)
for _, field := range fields {
if strings.HasPrefix(field, "HZ=") {
if hz, err := strconv.ParseFloat(field[3:], 64); err == nil && hz > 0 {
pm.clockTicks = hz
return
}
}
}
}
}
// Try reading from /proc/timer_list for more accurate detection
if data, err := os.ReadFile("/proc/timer_list"); err == nil {
timer := string(data)
// Look for tick device frequency
lines := strings.Split(timer, "\n")
for _, line := range lines {
if strings.Contains(line, "tick_period:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
if period, err := strconv.ParseInt(fields[1], 10, 64); err == nil && period > 0 {
// Convert nanoseconds to Hz
hz := 1000000000.0 / float64(period)
if hz >= minValidClockTicks && hz <= maxValidClockTicks {
pm.clockTicks = hz
return
}
}
}
}
}
}
// Fallback: Most embedded ARM systems (like jetKVM) use 250 Hz or 1000 Hz
// rather than the traditional 100 Hz
pm.clockTicks = defaultClockTicks
pm.logger.Warn().Float64("clock_ticks", pm.clockTicks).Msg("Using fallback clock ticks value")
// Log successful detection for non-fallback values
if pm.clockTicks != defaultClockTicks {
pm.logger.Info().Float64("clock_ticks", pm.clockTicks).Msg("Detected system clock ticks")
}
})
return pm.clockTicks
}
func (pm *ProcessMonitor) getTotalMemory() int64 {
pm.memoryOnce.Do(func() {
file, err := os.Open("/proc/meminfo")
if err != nil {
pm.totalMemory = defaultMemoryGB * 1024 * 1024 * 1024
return
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "MemTotal:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
if kb, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
pm.totalMemory = kb * 1024
return
}
}
break
}
}
pm.totalMemory = defaultMemoryGB * 1024 * 1024 * 1024 // Fallback
})
return pm.totalMemory
}
// GetTotalMemory returns total system memory in bytes (public method)
func (pm *ProcessMonitor) GetTotalMemory() int64 {
return pm.getTotalMemory()
}
// Global process monitor instance
var globalProcessMonitor *ProcessMonitor
var processMonitorOnce sync.Once
// GetProcessMonitor returns the global process monitor instance
func GetProcessMonitor() *ProcessMonitor {
processMonitorOnce.Do(func() {
globalProcessMonitor = NewProcessMonitor()
globalProcessMonitor.Start()
})
return globalProcessMonitor
}

208
internal/audio/relay.go Normal file
View File

@ -0,0 +1,208 @@
package audio
import (
"context"
"fmt"
"sync"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/pion/webrtc/v4/pkg/media"
"github.com/rs/zerolog"
)
// AudioRelay handles forwarding audio frames from the audio server subprocess
// to WebRTC without any CGO audio processing. This runs in the main process.
type AudioRelay struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
framesRelayed int64
framesDropped int64
client *AudioClient
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
logger *zerolog.Logger
running bool
mutex sync.RWMutex
// WebRTC integration
audioTrack AudioTrackWriter
config AudioConfig
muted bool
}
// AudioTrackWriter interface for WebRTC audio track
type AudioTrackWriter interface {
WriteSample(sample media.Sample) error
}
// NewAudioRelay creates a new audio relay for the main process
func NewAudioRelay() *AudioRelay {
ctx, cancel := context.WithCancel(context.Background())
logger := logging.GetDefaultLogger().With().Str("component", "audio-relay").Logger()
return &AudioRelay{
ctx: ctx,
cancel: cancel,
logger: &logger,
}
}
// Start begins the audio relay process
func (r *AudioRelay) Start(audioTrack AudioTrackWriter, config AudioConfig) error {
r.mutex.Lock()
defer r.mutex.Unlock()
if r.running {
return nil // Already running
}
// Create audio client to connect to subprocess
client := NewAudioClient()
r.client = client
r.audioTrack = audioTrack
r.config = config
// Connect to the audio output server
if err := client.Connect(); err != nil {
return fmt.Errorf("failed to connect to audio output server: %w", err)
}
// Start relay goroutine
r.wg.Add(1)
go r.relayLoop()
r.running = true
r.logger.Info().Msg("Audio relay started")
return nil
}
// Stop stops the audio relay
func (r *AudioRelay) Stop() {
r.mutex.Lock()
defer r.mutex.Unlock()
if !r.running {
return
}
r.cancel()
r.wg.Wait()
if r.client != nil {
r.client.Disconnect()
r.client = nil
}
r.running = false
r.logger.Info().Msg("Audio relay stopped")
}
// SetMuted sets the mute state
func (r *AudioRelay) SetMuted(muted bool) {
r.mutex.Lock()
defer r.mutex.Unlock()
r.muted = muted
}
// IsMuted returns the current mute state (checks both relay and global mute)
func (r *AudioRelay) IsMuted() bool {
r.mutex.RLock()
defer r.mutex.RUnlock()
return r.muted || IsAudioMuted()
}
// GetStats returns relay statistics
func (r *AudioRelay) GetStats() (framesRelayed, framesDropped int64) {
r.mutex.RLock()
defer r.mutex.RUnlock()
return r.framesRelayed, r.framesDropped
}
// UpdateTrack updates the WebRTC audio track for the relay
func (r *AudioRelay) UpdateTrack(audioTrack AudioTrackWriter) {
r.mutex.Lock()
defer r.mutex.Unlock()
r.audioTrack = audioTrack
}
func (r *AudioRelay) relayLoop() {
defer r.wg.Done()
r.logger.Debug().Msg("Audio relay loop started")
const maxConsecutiveErrors = 10
consecutiveErrors := 0
for {
select {
case <-r.ctx.Done():
r.logger.Debug().Msg("Audio relay loop stopping")
return
default:
frame, err := r.client.ReceiveFrame()
if err != nil {
consecutiveErrors++
r.logger.Error().Err(err).Int("consecutive_errors", consecutiveErrors).Msg("Failed to receive audio frame")
r.incrementDropped()
if consecutiveErrors >= maxConsecutiveErrors {
r.logger.Error().Msg("Too many consecutive errors, stopping relay")
return
}
time.Sleep(10 * time.Millisecond)
continue
}
consecutiveErrors = 0
if err := r.forwardToWebRTC(frame); err != nil {
r.logger.Warn().Err(err).Msg("Failed to forward frame to WebRTC")
r.incrementDropped()
} else {
r.incrementRelayed()
}
}
}
}
// forwardToWebRTC forwards a frame to the WebRTC audio track
func (r *AudioRelay) forwardToWebRTC(frame []byte) error {
r.mutex.RLock()
audioTrack := r.audioTrack
config := r.config
muted := r.muted
r.mutex.RUnlock()
if audioTrack == nil {
return nil // No audio track available
}
// Prepare sample data
var sampleData []byte
if muted {
// Send silence when muted
sampleData = make([]byte, len(frame))
} else {
sampleData = frame
}
// Write sample to WebRTC track
return audioTrack.WriteSample(media.Sample{
Data: sampleData,
Duration: config.FrameSize,
})
}
// incrementRelayed atomically increments the relayed frames counter
func (r *AudioRelay) incrementRelayed() {
r.mutex.Lock()
r.framesRelayed++
r.mutex.Unlock()
}
// incrementDropped atomically increments the dropped frames counter
func (r *AudioRelay) incrementDropped() {
r.mutex.Lock()
r.framesDropped++
r.mutex.Unlock()
}

109
internal/audio/relay_api.go Normal file
View File

@ -0,0 +1,109 @@
package audio
import (
"sync"
)
// Global relay instance for the main process
var (
globalRelay *AudioRelay
relayMutex sync.RWMutex
)
// StartAudioRelay starts the audio relay system for the main process
// This replaces the CGO-based audio system when running in main process mode
// audioTrack can be nil initially and updated later via UpdateAudioRelayTrack
func StartAudioRelay(audioTrack AudioTrackWriter) error {
relayMutex.Lock()
defer relayMutex.Unlock()
if globalRelay != nil {
return nil // Already running
}
// Create new relay
relay := NewAudioRelay()
// Get current audio config
config := GetAudioConfig()
// Start the relay (audioTrack can be nil initially)
if err := relay.Start(audioTrack, config); err != nil {
return err
}
globalRelay = relay
return nil
}
// StopAudioRelay stops the audio relay system
func StopAudioRelay() {
relayMutex.Lock()
defer relayMutex.Unlock()
if globalRelay != nil {
globalRelay.Stop()
globalRelay = nil
}
}
// SetAudioRelayMuted sets the mute state for the audio relay
func SetAudioRelayMuted(muted bool) {
relayMutex.RLock()
defer relayMutex.RUnlock()
if globalRelay != nil {
globalRelay.SetMuted(muted)
}
}
// IsAudioRelayMuted returns the current mute state of the audio relay
func IsAudioRelayMuted() bool {
relayMutex.RLock()
defer relayMutex.RUnlock()
if globalRelay != nil {
return globalRelay.IsMuted()
}
return false
}
// GetAudioRelayStats returns statistics from the audio relay
func GetAudioRelayStats() (framesRelayed, framesDropped int64) {
relayMutex.RLock()
defer relayMutex.RUnlock()
if globalRelay != nil {
return globalRelay.GetStats()
}
return 0, 0
}
// IsAudioRelayRunning returns whether the audio relay is currently running
func IsAudioRelayRunning() bool {
relayMutex.RLock()
defer relayMutex.RUnlock()
return globalRelay != nil
}
// UpdateAudioRelayTrack updates the WebRTC audio track for the relay
func UpdateAudioRelayTrack(audioTrack AudioTrackWriter) error {
relayMutex.Lock()
defer relayMutex.Unlock()
if globalRelay == nil {
// No relay running, start one with the provided track
relay := NewAudioRelay()
config := GetAudioConfig()
if err := relay.Start(audioTrack, config); err != nil {
return err
}
globalRelay = relay
return nil
}
// Update the track in the existing relay
globalRelay.UpdateTrack(audioTrack)
return nil
}

30
internal/audio/session.go Normal file
View File

@ -0,0 +1,30 @@
package audio
// SessionProvider interface abstracts session management for audio events
type SessionProvider interface {
IsSessionActive() bool
GetAudioInputManager() *AudioInputManager
}
// DefaultSessionProvider is a no-op implementation
type DefaultSessionProvider struct{}
func (d *DefaultSessionProvider) IsSessionActive() bool {
return false
}
func (d *DefaultSessionProvider) GetAudioInputManager() *AudioInputManager {
return nil
}
var sessionProvider SessionProvider = &DefaultSessionProvider{}
// SetSessionProvider allows the main package to inject session management
func SetSessionProvider(provider SessionProvider) {
sessionProvider = provider
}
// GetSessionProvider returns the current session provider
func GetSessionProvider() SessionProvider {
return sessionProvider
}

View File

@ -0,0 +1,160 @@
package audio
import (
"fmt"
"net"
"syscall"
)
const (
// Socket buffer sizes optimized for JetKVM's audio workload
OptimalSocketBuffer = 128 * 1024 // 128KB (32 frames @ 4KB each)
MaxSocketBuffer = 256 * 1024 // 256KB for high-load scenarios
MinSocketBuffer = 32 * 1024 // 32KB minimum for basic functionality
)
// SocketBufferConfig holds socket buffer configuration
type SocketBufferConfig struct {
SendBufferSize int
RecvBufferSize int
Enabled bool
}
// DefaultSocketBufferConfig returns the default socket buffer configuration
func DefaultSocketBufferConfig() SocketBufferConfig {
return SocketBufferConfig{
SendBufferSize: OptimalSocketBuffer,
RecvBufferSize: OptimalSocketBuffer,
Enabled: true,
}
}
// HighLoadSocketBufferConfig returns configuration for high-load scenarios
func HighLoadSocketBufferConfig() SocketBufferConfig {
return SocketBufferConfig{
SendBufferSize: MaxSocketBuffer,
RecvBufferSize: MaxSocketBuffer,
Enabled: true,
}
}
// ConfigureSocketBuffers applies socket buffer configuration to a Unix socket connection
func ConfigureSocketBuffers(conn net.Conn, config SocketBufferConfig) error {
if !config.Enabled {
return nil
}
if err := ValidateSocketBufferConfig(config); err != nil {
return fmt.Errorf("invalid socket buffer config: %w", err)
}
unixConn, ok := conn.(*net.UnixConn)
if !ok {
return fmt.Errorf("connection is not a Unix socket")
}
file, err := unixConn.File()
if err != nil {
return fmt.Errorf("failed to get socket file descriptor: %w", err)
}
defer file.Close()
fd := int(file.Fd())
if config.SendBufferSize > 0 {
if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, config.SendBufferSize); err != nil {
return fmt.Errorf("failed to set SO_SNDBUF to %d: %w", config.SendBufferSize, err)
}
}
if config.RecvBufferSize > 0 {
if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, config.RecvBufferSize); err != nil {
return fmt.Errorf("failed to set SO_RCVBUF to %d: %w", config.RecvBufferSize, err)
}
}
return nil
}
// GetSocketBufferSizes retrieves current socket buffer sizes
func GetSocketBufferSizes(conn net.Conn) (sendSize, recvSize int, err error) {
unixConn, ok := conn.(*net.UnixConn)
if !ok {
return 0, 0, fmt.Errorf("socket buffer query only supported for Unix sockets")
}
file, err := unixConn.File()
if err != nil {
return 0, 0, fmt.Errorf("failed to get socket file descriptor: %w", err)
}
defer file.Close()
fd := int(file.Fd())
// Get send buffer size
sendSize, err = syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF)
if err != nil {
return 0, 0, fmt.Errorf("failed to get SO_SNDBUF: %w", err)
}
// Get receive buffer size
recvSize, err = syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF)
if err != nil {
return 0, 0, fmt.Errorf("failed to get SO_RCVBUF: %w", err)
}
return sendSize, recvSize, nil
}
// ValidateSocketBufferConfig validates socket buffer configuration
func ValidateSocketBufferConfig(config SocketBufferConfig) error {
if !config.Enabled {
return nil
}
if config.SendBufferSize < MinSocketBuffer {
return fmt.Errorf("send buffer size %d is below minimum %d", config.SendBufferSize, MinSocketBuffer)
}
if config.RecvBufferSize < MinSocketBuffer {
return fmt.Errorf("receive buffer size %d is below minimum %d", config.RecvBufferSize, MinSocketBuffer)
}
if config.SendBufferSize > MaxSocketBuffer {
return fmt.Errorf("send buffer size %d exceeds maximum %d", config.SendBufferSize, MaxSocketBuffer)
}
if config.RecvBufferSize > MaxSocketBuffer {
return fmt.Errorf("receive buffer size %d exceeds maximum %d", config.RecvBufferSize, MaxSocketBuffer)
}
return nil
}
// RecordSocketBufferMetrics records socket buffer metrics for monitoring
func RecordSocketBufferMetrics(conn net.Conn, component string) {
if conn == nil {
return
}
// Get current socket buffer sizes
sendSize, recvSize, err := GetSocketBufferSizes(conn)
if err != nil {
// Log error but don't fail
return
}
// Record buffer sizes
socketBufferSizeGauge.WithLabelValues(component, "send").Set(float64(sendSize))
socketBufferSizeGauge.WithLabelValues(component, "receive").Set(float64(recvSize))
}
// RecordSocketBufferOverflow records a socket buffer overflow event
func RecordSocketBufferOverflow(component, bufferType string) {
socketBufferOverflowCounter.WithLabelValues(component, bufferType).Inc()
}
// UpdateSocketBufferUtilization updates socket buffer utilization metrics
func UpdateSocketBufferUtilization(component, bufferType string, utilizationPercent float64) {
socketBufferUtilizationGauge.WithLabelValues(component, bufferType).Set(utilizationPercent)
}

View File

@ -0,0 +1,429 @@
//go:build cgo
// +build cgo
package audio
import (
"context"
"fmt"
"os"
"os/exec"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/jetkvm/kvm/internal/logging"
"github.com/rs/zerolog"
)
const (
// Maximum number of restart attempts within the restart window
maxRestartAttempts = 5
// Time window for counting restart attempts
restartWindow = 5 * time.Minute
// Delay between restart attempts
restartDelay = 2 * time.Second
// Maximum restart delay (exponential backoff)
maxRestartDelay = 30 * time.Second
)
// AudioServerSupervisor manages the audio server subprocess lifecycle
type AudioServerSupervisor struct {
ctx context.Context
cancel context.CancelFunc
logger *zerolog.Logger
mutex sync.RWMutex
running int32
// Process management
cmd *exec.Cmd
processPID int
// Restart management
restartAttempts []time.Time
lastExitCode int
lastExitTime time.Time
// Channels for coordination
processDone chan struct{}
stopChan chan struct{}
// Process monitoring
processMonitor *ProcessMonitor
// Callbacks
onProcessStart func(pid int)
onProcessExit func(pid int, exitCode int, crashed bool)
onRestart func(attempt int, delay time.Duration)
}
// NewAudioServerSupervisor creates a new audio server supervisor
func NewAudioServerSupervisor() *AudioServerSupervisor {
ctx, cancel := context.WithCancel(context.Background())
logger := logging.GetDefaultLogger().With().Str("component", "audio-supervisor").Logger()
return &AudioServerSupervisor{
ctx: ctx,
cancel: cancel,
logger: &logger,
processDone: make(chan struct{}),
stopChan: make(chan struct{}),
processMonitor: GetProcessMonitor(),
}
}
// SetCallbacks sets optional callbacks for process lifecycle events
func (s *AudioServerSupervisor) SetCallbacks(
onStart func(pid int),
onExit func(pid int, exitCode int, crashed bool),
onRestart func(attempt int, delay time.Duration),
) {
s.mutex.Lock()
defer s.mutex.Unlock()
s.onProcessStart = onStart
s.onProcessExit = onExit
s.onRestart = onRestart
}
// Start begins supervising the audio server process
func (s *AudioServerSupervisor) Start() error {
if !atomic.CompareAndSwapInt32(&s.running, 0, 1) {
return fmt.Errorf("supervisor already running")
}
s.logger.Info().Msg("starting audio server supervisor")
// Start the supervision loop
go s.supervisionLoop()
return nil
}
// Stop gracefully stops the audio server and supervisor
func (s *AudioServerSupervisor) Stop() error {
if !atomic.CompareAndSwapInt32(&s.running, 1, 0) {
return nil // Already stopped
}
s.logger.Info().Msg("stopping audio server supervisor")
// Signal stop and wait for cleanup
close(s.stopChan)
s.cancel()
// Wait for process to exit
select {
case <-s.processDone:
s.logger.Info().Msg("audio server process stopped gracefully")
case <-time.After(10 * time.Second):
s.logger.Warn().Msg("audio server process did not stop gracefully, forcing termination")
s.forceKillProcess()
}
return nil
}
// IsRunning returns true if the supervisor is running
func (s *AudioServerSupervisor) IsRunning() bool {
return atomic.LoadInt32(&s.running) == 1
}
// GetProcessPID returns the current process PID (0 if not running)
func (s *AudioServerSupervisor) GetProcessPID() int {
s.mutex.RLock()
defer s.mutex.RUnlock()
return s.processPID
}
// GetLastExitInfo returns information about the last process exit
func (s *AudioServerSupervisor) GetLastExitInfo() (exitCode int, exitTime time.Time) {
s.mutex.RLock()
defer s.mutex.RUnlock()
return s.lastExitCode, s.lastExitTime
}
// GetProcessMetrics returns current process metrics if the process is running
func (s *AudioServerSupervisor) GetProcessMetrics() *ProcessMetrics {
s.mutex.RLock()
pid := s.processPID
s.mutex.RUnlock()
if pid == 0 {
return nil
}
metrics := s.processMonitor.GetCurrentMetrics()
for _, metric := range metrics {
if metric.PID == pid {
return &metric
}
}
return nil
}
// supervisionLoop is the main supervision loop
func (s *AudioServerSupervisor) supervisionLoop() {
defer func() {
close(s.processDone)
s.logger.Info().Msg("audio server supervision ended")
}()
for atomic.LoadInt32(&s.running) == 1 {
select {
case <-s.stopChan:
s.logger.Info().Msg("received stop signal")
s.terminateProcess()
return
case <-s.ctx.Done():
s.logger.Info().Msg("context cancelled")
s.terminateProcess()
return
default:
// Start or restart the process
if err := s.startProcess(); err != nil {
s.logger.Error().Err(err).Msg("failed to start audio server process")
// Check if we should attempt restart
if !s.shouldRestart() {
s.logger.Error().Msg("maximum restart attempts exceeded, stopping supervisor")
return
}
delay := s.calculateRestartDelay()
s.logger.Warn().Dur("delay", delay).Msg("retrying process start after delay")
if s.onRestart != nil {
s.onRestart(len(s.restartAttempts), delay)
}
select {
case <-time.After(delay):
case <-s.stopChan:
return
case <-s.ctx.Done():
return
}
continue
}
// Wait for process to exit
s.waitForProcessExit()
// Check if we should restart
if !s.shouldRestart() {
s.logger.Error().Msg("maximum restart attempts exceeded, stopping supervisor")
return
}
// Calculate restart delay
delay := s.calculateRestartDelay()
s.logger.Info().Dur("delay", delay).Msg("restarting audio server process after delay")
if s.onRestart != nil {
s.onRestart(len(s.restartAttempts), delay)
}
// Wait for restart delay
select {
case <-time.After(delay):
case <-s.stopChan:
return
case <-s.ctx.Done():
return
}
}
}
}
// startProcess starts the audio server process
func (s *AudioServerSupervisor) startProcess() error {
execPath, err := os.Executable()
if err != nil {
return fmt.Errorf("failed to get executable path: %w", err)
}
s.mutex.Lock()
defer s.mutex.Unlock()
// Create new command
s.cmd = exec.CommandContext(s.ctx, execPath, "--audio-output-server")
s.cmd.Stdout = os.Stdout
s.cmd.Stderr = os.Stderr
// Start the process
if err := s.cmd.Start(); err != nil {
return fmt.Errorf("failed to start process: %w", err)
}
s.processPID = s.cmd.Process.Pid
s.logger.Info().Int("pid", s.processPID).Msg("audio server process started")
// Add process to monitoring
s.processMonitor.AddProcess(s.processPID, "audio-output-server")
if s.onProcessStart != nil {
s.onProcessStart(s.processPID)
}
return nil
}
// waitForProcessExit waits for the current process to exit and logs the result
func (s *AudioServerSupervisor) waitForProcessExit() {
s.mutex.RLock()
cmd := s.cmd
pid := s.processPID
s.mutex.RUnlock()
if cmd == nil {
return
}
// Wait for process to exit
err := cmd.Wait()
s.mutex.Lock()
s.lastExitTime = time.Now()
s.processPID = 0
var exitCode int
var crashed bool
if err != nil {
if exitError, ok := err.(*exec.ExitError); ok {
exitCode = exitError.ExitCode()
crashed = exitCode != 0
} else {
// Process was killed or other error
exitCode = -1
crashed = true
}
} else {
exitCode = 0
crashed = false
}
s.lastExitCode = exitCode
s.mutex.Unlock()
// Remove process from monitoring
s.processMonitor.RemoveProcess(pid)
if crashed {
s.logger.Error().Int("pid", pid).Int("exit_code", exitCode).Msg("audio server process crashed")
s.recordRestartAttempt()
} else {
s.logger.Info().Int("pid", pid).Msg("audio server process exited gracefully")
}
if s.onProcessExit != nil {
s.onProcessExit(pid, exitCode, crashed)
}
}
// terminateProcess gracefully terminates the current process
func (s *AudioServerSupervisor) terminateProcess() {
s.mutex.RLock()
cmd := s.cmd
pid := s.processPID
s.mutex.RUnlock()
if cmd == nil || cmd.Process == nil {
return
}
s.logger.Info().Int("pid", pid).Msg("terminating audio server process")
// Send SIGTERM first
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
s.logger.Warn().Err(err).Int("pid", pid).Msg("failed to send SIGTERM")
}
// Wait for graceful shutdown
done := make(chan struct{})
go func() {
_ = cmd.Wait()
close(done)
}()
select {
case <-done:
s.logger.Info().Int("pid", pid).Msg("audio server process terminated gracefully")
case <-time.After(5 * time.Second):
s.logger.Warn().Int("pid", pid).Msg("process did not terminate gracefully, sending SIGKILL")
s.forceKillProcess()
}
}
// forceKillProcess forcefully kills the current process
func (s *AudioServerSupervisor) forceKillProcess() {
s.mutex.RLock()
cmd := s.cmd
pid := s.processPID
s.mutex.RUnlock()
if cmd == nil || cmd.Process == nil {
return
}
s.logger.Warn().Int("pid", pid).Msg("force killing audio server process")
if err := cmd.Process.Kill(); err != nil {
s.logger.Error().Err(err).Int("pid", pid).Msg("failed to kill process")
}
}
// shouldRestart determines if the process should be restarted
func (s *AudioServerSupervisor) shouldRestart() bool {
if atomic.LoadInt32(&s.running) == 0 {
return false // Supervisor is stopping
}
s.mutex.RLock()
defer s.mutex.RUnlock()
// Clean up old restart attempts outside the window
now := time.Now()
var recentAttempts []time.Time
for _, attempt := range s.restartAttempts {
if now.Sub(attempt) < restartWindow {
recentAttempts = append(recentAttempts, attempt)
}
}
s.restartAttempts = recentAttempts
return len(s.restartAttempts) < maxRestartAttempts
}
// recordRestartAttempt records a restart attempt
func (s *AudioServerSupervisor) recordRestartAttempt() {
s.mutex.Lock()
defer s.mutex.Unlock()
s.restartAttempts = append(s.restartAttempts, time.Now())
}
// calculateRestartDelay calculates the delay before next restart attempt
func (s *AudioServerSupervisor) calculateRestartDelay() time.Duration {
s.mutex.RLock()
defer s.mutex.RUnlock()
// Exponential backoff based on recent restart attempts
attempts := len(s.restartAttempts)
if attempts == 0 {
return restartDelay
}
// Calculate exponential backoff: 2^attempts * base delay
delay := restartDelay
for i := 0; i < attempts && delay < maxRestartDelay; i++ {
delay *= 2
}
if delay > maxRestartDelay {
delay = maxRestartDelay
}
return delay
}

314
internal/audio/zero_copy.go Normal file
View File

@ -0,0 +1,314 @@
package audio
import (
"sync"
"sync/atomic"
"unsafe"
)
// ZeroCopyAudioFrame represents an audio frame that can be passed between
// components without copying the underlying data
type ZeroCopyAudioFrame struct {
data []byte
length int
capacity int
refCount int32
mutex sync.RWMutex
pooled bool
}
// ZeroCopyFramePool manages reusable zero-copy audio frames
type ZeroCopyFramePool struct {
// Atomic fields MUST be first for ARM32 alignment (int64 fields need 8-byte alignment)
counter int64 // Frame counter (atomic)
hitCount int64 // Pool hit counter (atomic)
missCount int64 // Pool miss counter (atomic)
// Other fields
pool sync.Pool
maxSize int
mutex sync.RWMutex
// Memory optimization fields
preallocated []*ZeroCopyAudioFrame // Pre-allocated frames for immediate use
preallocSize int // Number of pre-allocated frames
maxPoolSize int // Maximum pool size to prevent memory bloat
}
// NewZeroCopyFramePool creates a new zero-copy frame pool
func NewZeroCopyFramePool(maxFrameSize int) *ZeroCopyFramePool {
// Pre-allocate 15 frames for immediate availability
preallocSize := 15
maxPoolSize := 50 // Limit total pool size
preallocated := make([]*ZeroCopyAudioFrame, 0, preallocSize)
// Pre-allocate frames to reduce initial allocation overhead
for i := 0; i < preallocSize; i++ {
frame := &ZeroCopyAudioFrame{
data: make([]byte, 0, maxFrameSize),
capacity: maxFrameSize,
pooled: true,
}
preallocated = append(preallocated, frame)
}
return &ZeroCopyFramePool{
maxSize: maxFrameSize,
preallocated: preallocated,
preallocSize: preallocSize,
maxPoolSize: maxPoolSize,
pool: sync.Pool{
New: func() interface{} {
return &ZeroCopyAudioFrame{
data: make([]byte, 0, maxFrameSize),
capacity: maxFrameSize,
pooled: true,
}
},
},
}
}
// Get retrieves a zero-copy frame from the pool
func (p *ZeroCopyFramePool) Get() *ZeroCopyAudioFrame {
// First try pre-allocated frames for fastest access
p.mutex.Lock()
if len(p.preallocated) > 0 {
frame := p.preallocated[len(p.preallocated)-1]
p.preallocated = p.preallocated[:len(p.preallocated)-1]
p.mutex.Unlock()
frame.mutex.Lock()
frame.refCount = 1
frame.length = 0
frame.data = frame.data[:0]
frame.mutex.Unlock()
atomic.AddInt64(&p.hitCount, 1)
return frame
}
p.mutex.Unlock()
// Try sync.Pool next
frame := p.pool.Get().(*ZeroCopyAudioFrame)
frame.mutex.Lock()
frame.refCount = 1
frame.length = 0
frame.data = frame.data[:0]
frame.mutex.Unlock()
atomic.AddInt64(&p.hitCount, 1)
return frame
}
// Put returns a zero-copy frame to the pool
func (p *ZeroCopyFramePool) Put(frame *ZeroCopyAudioFrame) {
if frame == nil || !frame.pooled {
return
}
frame.mutex.Lock()
frame.refCount--
if frame.refCount <= 0 {
frame.refCount = 0
frame.length = 0
frame.data = frame.data[:0]
frame.mutex.Unlock()
// First try to return to pre-allocated pool for fastest reuse
p.mutex.Lock()
if len(p.preallocated) < p.preallocSize {
p.preallocated = append(p.preallocated, frame)
p.mutex.Unlock()
return
}
p.mutex.Unlock()
// Check pool size limit to prevent excessive memory usage
p.mutex.RLock()
currentCount := atomic.LoadInt64(&p.counter)
p.mutex.RUnlock()
if currentCount >= int64(p.maxPoolSize) {
return // Pool is full, let GC handle this frame
}
// Return to sync.Pool
p.pool.Put(frame)
atomic.AddInt64(&p.counter, 1)
} else {
frame.mutex.Unlock()
}
}
// Data returns the frame data as a slice (zero-copy view)
func (f *ZeroCopyAudioFrame) Data() []byte {
f.mutex.RLock()
defer f.mutex.RUnlock()
return f.data[:f.length]
}
// SetData sets the frame data (zero-copy if possible)
func (f *ZeroCopyAudioFrame) SetData(data []byte) error {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(data) > f.capacity {
// Need to reallocate - not zero-copy but necessary
f.data = make([]byte, len(data))
f.capacity = len(data)
f.pooled = false // Can't return to pool anymore
}
// Zero-copy assignment when data fits in existing buffer
if cap(f.data) >= len(data) {
f.data = f.data[:len(data)]
copy(f.data, data)
} else {
f.data = append(f.data[:0], data...)
}
f.length = len(data)
return nil
}
// SetDataDirect sets frame data using direct buffer assignment (true zero-copy)
// WARNING: The caller must ensure the buffer remains valid for the frame's lifetime
func (f *ZeroCopyAudioFrame) SetDataDirect(data []byte) {
f.mutex.Lock()
defer f.mutex.Unlock()
f.data = data
f.length = len(data)
f.capacity = cap(data)
f.pooled = false // Direct assignment means we can't pool this frame
}
// AddRef increments the reference count for shared access
func (f *ZeroCopyAudioFrame) AddRef() {
f.mutex.Lock()
f.refCount++
f.mutex.Unlock()
}
// Release decrements the reference count
func (f *ZeroCopyAudioFrame) Release() {
f.mutex.Lock()
f.refCount--
f.mutex.Unlock()
}
// Length returns the current data length
func (f *ZeroCopyAudioFrame) Length() int {
f.mutex.RLock()
defer f.mutex.RUnlock()
return f.length
}
// Capacity returns the buffer capacity
func (f *ZeroCopyAudioFrame) Capacity() int {
f.mutex.RLock()
defer f.mutex.RUnlock()
return f.capacity
}
// UnsafePointer returns an unsafe pointer to the data for CGO calls
// WARNING: Only use this for CGO interop, ensure frame lifetime
func (f *ZeroCopyAudioFrame) UnsafePointer() unsafe.Pointer {
f.mutex.RLock()
defer f.mutex.RUnlock()
if len(f.data) == 0 {
return nil
}
return unsafe.Pointer(&f.data[0])
}
// Global zero-copy frame pool
// GetZeroCopyPoolStats returns detailed statistics about the zero-copy frame pool
func (p *ZeroCopyFramePool) GetZeroCopyPoolStats() ZeroCopyFramePoolStats {
p.mutex.RLock()
preallocatedCount := len(p.preallocated)
currentCount := atomic.LoadInt64(&p.counter)
p.mutex.RUnlock()
hitCount := atomic.LoadInt64(&p.hitCount)
missCount := atomic.LoadInt64(&p.missCount)
totalRequests := hitCount + missCount
var hitRate float64
if totalRequests > 0 {
hitRate = float64(hitCount) / float64(totalRequests) * 100
}
return ZeroCopyFramePoolStats{
MaxFrameSize: p.maxSize,
MaxPoolSize: p.maxPoolSize,
CurrentPoolSize: currentCount,
PreallocatedCount: int64(preallocatedCount),
PreallocatedMax: int64(p.preallocSize),
HitCount: hitCount,
MissCount: missCount,
HitRate: hitRate,
}
}
// ZeroCopyFramePoolStats provides detailed zero-copy pool statistics
type ZeroCopyFramePoolStats struct {
MaxFrameSize int
MaxPoolSize int
CurrentPoolSize int64
PreallocatedCount int64
PreallocatedMax int64
HitCount int64
MissCount int64
HitRate float64 // Percentage
}
var (
globalZeroCopyPool = NewZeroCopyFramePool(MaxAudioFrameSize)
)
// GetZeroCopyFrame gets a frame from the global pool
func GetZeroCopyFrame() *ZeroCopyAudioFrame {
return globalZeroCopyPool.Get()
}
// GetGlobalZeroCopyPoolStats returns statistics for the global zero-copy pool
func GetGlobalZeroCopyPoolStats() ZeroCopyFramePoolStats {
return globalZeroCopyPool.GetZeroCopyPoolStats()
}
// PutZeroCopyFrame returns a frame to the global pool
func PutZeroCopyFrame(frame *ZeroCopyAudioFrame) {
globalZeroCopyPool.Put(frame)
}
// ZeroCopyAudioReadEncode performs audio read and encode with zero-copy optimization
func ZeroCopyAudioReadEncode() (*ZeroCopyAudioFrame, error) {
frame := GetZeroCopyFrame()
// Ensure frame has enough capacity
if frame.Capacity() < MaxAudioFrameSize {
// Reallocate if needed
frame.data = make([]byte, MaxAudioFrameSize)
frame.capacity = MaxAudioFrameSize
frame.pooled = false
}
// Use unsafe pointer for direct CGO call
n, err := CGOAudioReadEncode(frame.data[:MaxAudioFrameSize])
if err != nil {
PutZeroCopyFrame(frame)
return nil, err
}
if n == 0 {
PutZeroCopyFrame(frame)
return nil, nil
}
// Set the actual data length
frame.mutex.Lock()
frame.length = n
frame.data = frame.data[:n]
frame.mutex.Unlock()
return frame, nil
}

View File

@ -59,6 +59,23 @@ var defaultGadgetConfig = map[string]gadgetConfigItem{
// mass storage
"mass_storage_base": massStorageBaseConfig,
"mass_storage_lun0": massStorageLun0Config,
// audio
"audio": {
order: 4000,
device: "uac1.usb0",
path: []string{"functions", "uac1.usb0"},
configPath: []string{"uac1.usb0"},
attrs: gadgetAttributes{
"p_chmask": "3",
"p_srate": "48000",
"p_ssize": "2",
"p_volume_present": "0",
"c_chmask": "3",
"c_srate": "48000",
"c_ssize": "2",
"c_volume_present": "0",
},
},
}
func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool {
@ -73,6 +90,8 @@ func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool {
return u.enabledDevices.MassStorage
case "mass_storage_lun0":
return u.enabledDevices.MassStorage
case "audio":
return u.enabledDevices.Audio
default:
return true
}
@ -182,6 +201,9 @@ func (u *UsbGadget) Init() error {
return u.logError("unable to initialize USB stack", err)
}
// Pre-open HID files to reduce input latency
u.PreOpenHidFiles()
return nil
}

View File

@ -203,8 +203,7 @@ func (u *UsbGadget) keyboardWriteHidFile(data []byte) error {
_, err := u.keyboardHidFile.Write(data)
if err != nil {
u.logWithSuppression("keyboardWriteHidFile", 100, u.log, err, "failed to write to hidg0")
u.keyboardHidFile.Close()
u.keyboardHidFile = nil
// Keep file open on write errors to reduce I/O overhead
return err
}
u.resetLogSuppressionCounter("keyboardWriteHidFile")

View File

@ -77,8 +77,7 @@ func (u *UsbGadget) absMouseWriteHidFile(data []byte) error {
_, err := u.absMouseHidFile.Write(data)
if err != nil {
u.logWithSuppression("absMouseWriteHidFile", 100, u.log, err, "failed to write to hidg1")
u.absMouseHidFile.Close()
u.absMouseHidFile = nil
// Keep file open on write errors to reduce I/O overhead
return err
}
u.resetLogSuppressionCounter("absMouseWriteHidFile")

View File

@ -60,15 +60,14 @@ func (u *UsbGadget) relMouseWriteHidFile(data []byte) error {
var err error
u.relMouseHidFile, err = os.OpenFile("/dev/hidg2", os.O_RDWR, 0666)
if err != nil {
return fmt.Errorf("failed to open hidg1: %w", err)
return fmt.Errorf("failed to open hidg2: %w", err)
}
}
_, err := u.relMouseHidFile.Write(data)
if err != nil {
u.logWithSuppression("relMouseWriteHidFile", 100, u.log, err, "failed to write to hidg2")
u.relMouseHidFile.Close()
u.relMouseHidFile = nil
// Keep file open on write errors to reduce I/O overhead
return err
}
u.resetLogSuppressionCounter("relMouseWriteHidFile")

View File

@ -19,6 +19,7 @@ type Devices struct {
RelativeMouse bool `json:"relative_mouse"`
Keyboard bool `json:"keyboard"`
MassStorage bool `json:"mass_storage"`
Audio bool `json:"audio"`
}
// Config is a struct that represents the customizations for a USB gadget.
@ -94,6 +95,33 @@ func NewUsbGadget(name string, enabledDevices *Devices, config *Config, logger *
return newUsbGadget(name, defaultGadgetConfig, enabledDevices, config, logger)
}
// PreOpenHidFiles opens all HID files to reduce input latency
func (u *UsbGadget) PreOpenHidFiles() {
if u.enabledDevices.Keyboard {
if err := u.openKeyboardHidFile(); err != nil {
u.log.Debug().Err(err).Msg("failed to pre-open keyboard HID file")
}
}
if u.enabledDevices.AbsoluteMouse {
if u.absMouseHidFile == nil {
var err error
u.absMouseHidFile, err = os.OpenFile("/dev/hidg1", os.O_RDWR, 0666)
if err != nil {
u.log.Debug().Err(err).Msg("failed to pre-open absolute mouse HID file")
}
}
}
if u.enabledDevices.RelativeMouse {
if u.relMouseHidFile == nil {
var err error
u.relMouseHidFile, err = os.OpenFile("/dev/hidg2", os.O_RDWR, 0666)
if err != nil {
u.log.Debug().Err(err).Msg("failed to pre-open relative mouse HID file")
}
}
}
}
func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDevices *Devices, config *Config, logger *zerolog.Logger) *UsbGadget {
if logger == nil {
logger = defaultLogger

View File

@ -18,6 +18,8 @@ import (
"github.com/jetkvm/kvm/internal/usbgadget"
)
// Direct RPC message handling for optimal input responsiveness
type JSONRPCRequest struct {
JSONRPC string `json:"jsonrpc"`
Method string `json:"method"`
@ -119,6 +121,39 @@ func onRPCMessage(message webrtc.DataChannelMessage, session *Session) {
scopedLogger.Trace().Msg("Received RPC request")
// Fast path for input methods - bypass reflection for performance
// This optimization reduces latency by 3-6ms per input event by:
// - Eliminating reflection overhead
// - Reducing memory allocations
// - Optimizing parameter parsing and validation
// See input_rpc.go for implementation details
if isInputMethod(request.Method) {
result, err := handleInputRPCDirect(request.Method, request.Params)
if err != nil {
scopedLogger.Error().Err(err).Msg("Error calling direct input handler")
errorResponse := JSONRPCResponse{
JSONRPC: "2.0",
Error: map[string]interface{}{
"code": -32603,
"message": "Internal error",
"data": err.Error(),
},
ID: request.ID,
}
writeJSONRPCResponse(errorResponse, session)
return
}
response := JSONRPCResponse{
JSONRPC: "2.0",
Result: result,
ID: request.ID,
}
writeJSONRPCResponse(response, session)
return
}
// Fallback to reflection-based handler for non-input methods
handler, ok := rpcHandlers[request.Method]
if !ok {
errorResponse := JSONRPCResponse{

130
main.go
View File

@ -2,6 +2,7 @@ package kvm
import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
@ -9,11 +10,107 @@ import (
"time"
"github.com/gwatts/rootcerts"
"github.com/jetkvm/kvm/internal/audio"
)
var appCtx context.Context
var (
appCtx context.Context
isAudioServer bool
audioProcessDone chan struct{}
audioSupervisor *audio.AudioServerSupervisor
)
func Main() {
// runAudioServer is now handled by audio.RunAudioOutputServer
// This function is kept for backward compatibility but delegates to the audio package
func runAudioServer() {
err := audio.RunAudioOutputServer()
if err != nil {
logger.Error().Err(err).Msg("audio output server failed")
os.Exit(1)
}
}
func startAudioSubprocess() error {
// Start adaptive buffer management for optimal performance
audio.StartAdaptiveBuffering()
// Create audio server supervisor
audioSupervisor = audio.NewAudioServerSupervisor()
// Set the global supervisor for access from audio package
audio.SetAudioOutputSupervisor(audioSupervisor)
// Set up callbacks for process lifecycle events
audioSupervisor.SetCallbacks(
// onProcessStart
func(pid int) {
logger.Info().Int("pid", pid).Msg("audio server process started")
// Start audio relay system for main process without a track initially
// The track will be updated when a WebRTC session is created
if err := audio.StartAudioRelay(nil); err != nil {
logger.Error().Err(err).Msg("failed to start audio relay")
}
},
// onProcessExit
func(pid int, exitCode int, crashed bool) {
if crashed {
logger.Error().Int("pid", pid).Int("exit_code", exitCode).Msg("audio server process crashed")
} else {
logger.Info().Int("pid", pid).Msg("audio server process exited gracefully")
}
// Stop audio relay when process exits
audio.StopAudioRelay()
// Stop adaptive buffering
audio.StopAdaptiveBuffering()
},
// onRestart
func(attempt int, delay time.Duration) {
logger.Warn().Int("attempt", attempt).Dur("delay", delay).Msg("restarting audio server process")
},
)
// Start the supervisor
if err := audioSupervisor.Start(); err != nil {
return fmt.Errorf("failed to start audio supervisor: %w", err)
}
// Monitor supervisor and handle cleanup
go func() {
defer close(audioProcessDone)
// Wait for supervisor to stop
for audioSupervisor.IsRunning() {
time.Sleep(100 * time.Millisecond)
}
logger.Info().Msg("audio supervisor stopped")
}()
return nil
}
func Main(audioServer bool, audioInputServer bool) {
// Initialize channel and set audio server flag
isAudioServer = audioServer
audioProcessDone = make(chan struct{})
// If running as audio server, only initialize audio processing
if isAudioServer {
runAudioServer()
return
}
// If running as audio input server, only initialize audio input processing
if audioInputServer {
err := audio.RunAudioInputServer()
if err != nil {
logger.Error().Err(err).Msg("audio input server failed")
os.Exit(1)
}
return
}
LoadConfig()
var cancel context.CancelFunc
@ -71,12 +168,26 @@ func Main() {
err = ExtractAndRunNativeBin()
if err != nil {
logger.Warn().Err(err).Msg("failed to extract and run native bin")
//TODO: prepare an error message screen buffer to show on kvm screen
// (future) prepare an error message screen buffer to show on kvm screen
}
}()
// initialize usb gadget
initUsbGadget()
// Start audio subprocess
err = startAudioSubprocess()
if err != nil {
logger.Warn().Err(err).Msg("failed to start audio subprocess")
}
// Initialize session provider for audio events
initializeAudioSessionProvider()
// Initialize audio event broadcaster for WebSocket-based real-time updates
audio.InitializeAudioEventBroadcaster()
logger.Info().Msg("audio event broadcaster initialized")
if err := setInitialVirtualMediaState(); err != nil {
logger.Warn().Err(err).Msg("failed to set initial virtual media state")
}
@ -126,6 +237,19 @@ func Main() {
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
<-sigs
logger.Info().Msg("JetKVM Shutting Down")
// Stop audio subprocess and wait for cleanup
if !isAudioServer {
if audioSupervisor != nil {
logger.Info().Msg("stopping audio supervisor")
if err := audioSupervisor.Stop(); err != nil {
logger.Error().Err(err).Msg("failed to stop audio supervisor")
}
}
<-audioProcessDone
} else {
audio.StopNonBlockingAudioStreaming()
}
//if fuseServer != nil {
// err := setMassStorageImage(" ")
// if err != nil {

329
native.go
View File

@ -1,255 +1,46 @@
//go:build linux
package kvm
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"os/exec"
"strings"
"sync"
"syscall"
"time"
"github.com/jetkvm/kvm/resource"
"github.com/pion/webrtc/v4/pkg/media"
"github.com/rs/zerolog"
)
var ctrlSocketConn net.Conn
type CtrlAction struct {
Action string `json:"action"`
Seq int32 `json:"seq,omitempty"`
Params map[string]interface{} `json:"params,omitempty"`
type nativeOutput struct {
logger *zerolog.Logger
}
type CtrlResponse struct {
Seq int32 `json:"seq,omitempty"`
Error string `json:"error,omitempty"`
Errno int32 `json:"errno,omitempty"`
Result map[string]interface{} `json:"result,omitempty"`
Event string `json:"event,omitempty"`
Data json.RawMessage `json:"data,omitempty"`
func (n *nativeOutput) Write(p []byte) (int, error) {
n.logger.Debug().Str("output", string(p)).Msg("native binary output")
return len(p), nil
}
type EventHandler func(event CtrlResponse)
var seq int32 = 1
var ongoingRequests = make(map[int32]chan *CtrlResponse)
var lock = &sync.Mutex{}
var (
nativeCmd *exec.Cmd
nativeCmdLock = &sync.Mutex{}
)
func CallCtrlAction(action string, params map[string]interface{}) (*CtrlResponse, error) {
lock.Lock()
defer lock.Unlock()
ctrlAction := CtrlAction{
Action: action,
Seq: seq,
Params: params,
func startNativeBinary(binaryPath string) (*exec.Cmd, error) {
cmd := exec.Command(binaryPath)
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGTERM,
}
cmd.Stdout = &nativeOutput{logger: nativeLogger}
cmd.Stderr = &nativeOutput{logger: nativeLogger}
responseChan := make(chan *CtrlResponse)
ongoingRequests[seq] = responseChan
seq++
jsonData, err := json.Marshal(ctrlAction)
err := cmd.Start()
if err != nil {
delete(ongoingRequests, ctrlAction.Seq)
return nil, fmt.Errorf("error marshaling ctrl action: %w", err)
return nil, err
}
scopedLogger := nativeLogger.With().
Str("action", ctrlAction.Action).
Interface("params", ctrlAction.Params).Logger()
scopedLogger.Debug().Msg("sending ctrl action")
err = WriteCtrlMessage(jsonData)
if err != nil {
delete(ongoingRequests, ctrlAction.Seq)
return nil, ErrorfL(&scopedLogger, "error writing ctrl message", err)
}
select {
case response := <-responseChan:
delete(ongoingRequests, seq)
if response.Error != "" {
return nil, ErrorfL(
&scopedLogger,
"error native response: %s",
errors.New(response.Error),
)
}
return response, nil
case <-time.After(5 * time.Second):
close(responseChan)
delete(ongoingRequests, seq)
return nil, ErrorfL(&scopedLogger, "timeout waiting for response", nil)
}
}
func WriteCtrlMessage(message []byte) error {
if ctrlSocketConn == nil {
return fmt.Errorf("ctrl socket not conn ected")
}
_, err := ctrlSocketConn.Write(message)
return err
}
var nativeCtrlSocketListener net.Listener //nolint:unused
var nativeVideoSocketListener net.Listener //nolint:unused
var ctrlClientConnected = make(chan struct{})
func waitCtrlClientConnected() {
<-ctrlClientConnected
}
func StartNativeSocketServer(socketPath string, handleClient func(net.Conn), isCtrl bool) net.Listener {
scopedLogger := nativeLogger.With().
Str("socket_path", socketPath).
Logger()
// Remove the socket file if it already exists
if _, err := os.Stat(socketPath); err == nil {
if err := os.Remove(socketPath); err != nil {
scopedLogger.Warn().Err(err).Msg("failed to remove existing socket file")
os.Exit(1)
}
}
listener, err := net.Listen("unixpacket", socketPath)
if err != nil {
scopedLogger.Warn().Err(err).Msg("failed to start server")
os.Exit(1)
}
scopedLogger.Info().Msg("server listening")
go func() {
for {
conn, err := listener.Accept()
if err != nil {
scopedLogger.Warn().Err(err).Msg("failed to accept socket")
continue
}
if isCtrl {
// check if the channel is closed
select {
case <-ctrlClientConnected:
scopedLogger.Debug().Msg("ctrl client reconnected")
default:
close(ctrlClientConnected)
scopedLogger.Debug().Msg("first native ctrl socket client connected")
}
}
go handleClient(conn)
}
}()
return listener
}
func StartNativeCtrlSocketServer() {
nativeCtrlSocketListener = StartNativeSocketServer("/var/run/jetkvm_ctrl.sock", handleCtrlClient, true)
nativeLogger.Debug().Msg("native app ctrl sock started")
}
func StartNativeVideoSocketServer() {
nativeVideoSocketListener = StartNativeSocketServer("/var/run/jetkvm_video.sock", handleVideoClient, false)
nativeLogger.Debug().Msg("native app video sock started")
}
func handleCtrlClient(conn net.Conn) {
defer conn.Close()
scopedLogger := nativeLogger.With().
Str("addr", conn.RemoteAddr().String()).
Str("type", "ctrl").
Logger()
scopedLogger.Info().Msg("native ctrl socket client connected")
if ctrlSocketConn != nil {
scopedLogger.Debug().Msg("closing existing native socket connection")
ctrlSocketConn.Close()
}
ctrlSocketConn = conn
// Restore HDMI EDID if applicable
go restoreHdmiEdid()
readBuf := make([]byte, 4096)
for {
n, err := conn.Read(readBuf)
if err != nil {
scopedLogger.Warn().Err(err).Msg("error reading from ctrl sock")
break
}
readMsg := string(readBuf[:n])
ctrlResp := CtrlResponse{}
err = json.Unmarshal([]byte(readMsg), &ctrlResp)
if err != nil {
scopedLogger.Warn().Err(err).Str("data", readMsg).Msg("error parsing ctrl sock msg")
continue
}
scopedLogger.Trace().Interface("data", ctrlResp).Msg("ctrl sock msg")
if ctrlResp.Seq != 0 {
responseChan, ok := ongoingRequests[ctrlResp.Seq]
if ok {
responseChan <- &ctrlResp
}
}
switch ctrlResp.Event {
case "video_input_state":
HandleVideoStateMessage(ctrlResp)
}
}
scopedLogger.Debug().Msg("ctrl sock disconnected")
}
func handleVideoClient(conn net.Conn) {
defer conn.Close()
scopedLogger := nativeLogger.With().
Str("addr", conn.RemoteAddr().String()).
Str("type", "video").
Logger()
scopedLogger.Info().Msg("native video socket client connected")
inboundPacket := make([]byte, maxFrameSize)
lastFrame := time.Now()
for {
n, err := conn.Read(inboundPacket)
if err != nil {
scopedLogger.Warn().Err(err).Msg("error during read")
return
}
now := time.Now()
sinceLastFrame := now.Sub(lastFrame)
lastFrame = now
if currentSession != nil {
err := currentSession.VideoTrack.WriteSample(media.Sample{Data: inboundPacket[:n], Duration: sinceLastFrame})
if err != nil {
scopedLogger.Warn().Err(err).Msg("error writing sample")
}
}
}
return cmd, nil
}
func startNativeBinaryWithLock(binaryPath string) (*exec.Cmd, error) {
@ -351,87 +142,3 @@ func ExtractAndRunNativeBin() error {
return nil
}
func shouldOverwrite(destPath string, srcHash []byte) bool {
if srcHash == nil {
nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, doing overwriting")
return true
}
dstHash, err := os.ReadFile(destPath + ".sha256")
if err != nil {
nativeLogger.Debug().Msg("error reading existing jetkvm_native.sha256, doing overwriting")
return true
}
return !bytes.Equal(srcHash, dstHash)
}
func getNativeSha256() ([]byte, error) {
version, err := resource.ResourceFS.ReadFile("jetkvm_native.sha256")
if err != nil {
return nil, err
}
return version, nil
}
func GetNativeVersion() (string, error) {
version, err := getNativeSha256()
if err != nil {
return "", err
}
return strings.TrimSpace(string(version)), nil
}
func ensureBinaryUpdated(destPath string) error {
srcFile, err := resource.ResourceFS.Open("jetkvm_native")
if err != nil {
return err
}
defer srcFile.Close()
srcHash, err := getNativeSha256()
if err != nil {
nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, proceeding with update")
srcHash = nil
}
_, err = os.Stat(destPath)
if shouldOverwrite(destPath, srcHash) || err != nil {
nativeLogger.Info().
Interface("hash", srcHash).
Msg("writing jetkvm_native")
_ = os.Remove(destPath)
destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_RDWR, 0755)
if err != nil {
return err
}
_, err = io.Copy(destFile, srcFile)
destFile.Close()
if err != nil {
return err
}
if srcHash != nil {
err = os.WriteFile(destPath+".sha256", srcHash, 0644)
if err != nil {
return err
}
}
nativeLogger.Info().Msg("jetkvm_native updated")
}
return nil
}
// Restore the HDMI EDID value from the config.
// Called after successful connection to jetkvm_native.
func restoreHdmiEdid() {
if config.EdidString != "" {
nativeLogger.Info().Str("edid", config.EdidString).Msg("Restoring HDMI EDID")
_, err := CallCtrlAction("set_edid", map[string]interface{}{"edid": config.EdidString})
if err != nil {
nativeLogger.Warn().Err(err).Msg("Failed to restore HDMI EDID")
}
}
}

View File

@ -1,57 +0,0 @@
//go:build linux
package kvm
import (
"fmt"
"os/exec"
"sync"
"syscall"
"github.com/rs/zerolog"
)
type nativeOutput struct {
mu *sync.Mutex
logger *zerolog.Event
}
func (w *nativeOutput) Write(p []byte) (n int, err error) {
w.mu.Lock()
defer w.mu.Unlock()
w.logger.Msg(string(p))
return len(p), nil
}
func startNativeBinary(binaryPath string) (*exec.Cmd, error) {
// Run the binary in the background
cmd := exec.Command(binaryPath)
nativeOutputLock := sync.Mutex{}
nativeStdout := &nativeOutput{
mu: &nativeOutputLock,
logger: nativeLogger.Info().Str("pipe", "stdout"),
}
nativeStderr := &nativeOutput{
mu: &nativeOutputLock,
logger: nativeLogger.Info().Str("pipe", "stderr"),
}
// Redirect stdout and stderr to the current process
cmd.Stdout = nativeStdout
cmd.Stderr = nativeStderr
// Set the process group ID so we can kill the process and its children when this process exits
cmd.SysProcAttr = &syscall.SysProcAttr{
Setpgid: true,
Pdeathsig: syscall.SIGKILL,
}
// Start the command
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("failed to start binary: %w", err)
}
return cmd, nil
}

View File

@ -8,5 +8,9 @@ import (
)
func startNativeBinary(binaryPath string) (*exec.Cmd, error) {
return nil, fmt.Errorf("not supported")
return nil, fmt.Errorf("startNativeBinary is only supported on Linux")
}
func ExtractAndRunNativeBin() error {
return fmt.Errorf("ExtractAndRunNativeBin is only supported on Linux")
}

343
native_shared.go Normal file
View File

@ -0,0 +1,343 @@
package kvm
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"runtime"
"strings"
"sync"
"time"
"github.com/jetkvm/kvm/resource"
"github.com/pion/webrtc/v4/pkg/media"
)
type CtrlAction struct {
Action string `json:"action"`
Seq int32 `json:"seq,omitempty"`
Params map[string]interface{} `json:"params,omitempty"`
}
type CtrlResponse struct {
Seq int32 `json:"seq,omitempty"`
Error string `json:"error,omitempty"`
Errno int32 `json:"errno,omitempty"`
Result map[string]interface{} `json:"result,omitempty"`
Event string `json:"event,omitempty"`
Data json.RawMessage `json:"data,omitempty"`
}
type EventHandler func(event CtrlResponse)
var seq int32 = 1
var ongoingRequests = make(map[int32]chan *CtrlResponse)
var lock = &sync.Mutex{}
var ctrlSocketConn net.Conn
var nativeCtrlSocketListener net.Listener //nolint:unused
var nativeVideoSocketListener net.Listener //nolint:unused
var ctrlClientConnected = make(chan struct{})
func waitCtrlClientConnected() {
<-ctrlClientConnected
}
func CallCtrlAction(action string, params map[string]interface{}) (*CtrlResponse, error) {
lock.Lock()
defer lock.Unlock()
ctrlAction := CtrlAction{
Action: action,
Seq: seq,
Params: params,
}
responseChan := make(chan *CtrlResponse)
ongoingRequests[seq] = responseChan
seq++
jsonData, err := json.Marshal(ctrlAction)
if err != nil {
delete(ongoingRequests, ctrlAction.Seq)
return nil, fmt.Errorf("error marshaling ctrl action: %w", err)
}
scopedLogger := nativeLogger.With().
Str("action", ctrlAction.Action).
Interface("params", ctrlAction.Params).Logger()
scopedLogger.Debug().Msg("sending ctrl action")
err = WriteCtrlMessage(jsonData)
if err != nil {
delete(ongoingRequests, ctrlAction.Seq)
return nil, ErrorfL(&scopedLogger, "error writing ctrl message", err)
}
select {
case response := <-responseChan:
delete(ongoingRequests, seq)
if response.Error != "" {
return nil, ErrorfL(
&scopedLogger,
"error native response: %s",
errors.New(response.Error),
)
}
return response, nil
case <-time.After(5 * time.Second):
close(responseChan)
delete(ongoingRequests, seq)
return nil, ErrorfL(&scopedLogger, "timeout waiting for response", nil)
}
}
func WriteCtrlMessage(message []byte) error {
if ctrlSocketConn == nil {
return fmt.Errorf("ctrl socket not connected")
}
_, err := ctrlSocketConn.Write(message)
return err
}
func StartNativeSocketServer(socketPath string, handleClient func(net.Conn), isCtrl bool) net.Listener {
scopedLogger := nativeLogger.With().
Str("socket_path", socketPath).
Logger()
// Remove the socket file if it already exists
if _, err := os.Stat(socketPath); err == nil {
if err := os.Remove(socketPath); err != nil {
scopedLogger.Warn().Err(err).Msg("failed to remove existing socket file")
os.Exit(1)
}
}
listener, err := net.Listen("unixpacket", socketPath)
if err != nil {
scopedLogger.Warn().Err(err).Msg("failed to start server")
os.Exit(1)
}
scopedLogger.Info().Msg("server listening")
go func() {
for {
conn, err := listener.Accept()
if err != nil {
scopedLogger.Warn().Err(err).Msg("failed to accept socket")
continue
}
if isCtrl {
// check if the channel is closed
select {
case <-ctrlClientConnected:
scopedLogger.Debug().Msg("ctrl client reconnected")
default:
close(ctrlClientConnected)
scopedLogger.Debug().Msg("first native ctrl socket client connected")
}
}
go handleClient(conn)
}
}()
return listener
}
func StartNativeCtrlSocketServer() {
nativeCtrlSocketListener = StartNativeSocketServer("/var/run/jetkvm_ctrl.sock", handleCtrlClient, true)
nativeLogger.Debug().Msg("native app ctrl sock started")
}
func StartNativeVideoSocketServer() {
nativeVideoSocketListener = StartNativeSocketServer("/var/run/jetkvm_video.sock", handleVideoClient, false)
nativeLogger.Debug().Msg("native app video sock started")
}
func handleCtrlClient(conn net.Conn) {
// Lock to OS thread to isolate blocking socket I/O
runtime.LockOSThread()
defer runtime.UnlockOSThread()
defer conn.Close()
scopedLogger := nativeLogger.With().
Str("addr", conn.RemoteAddr().String()).
Str("type", "ctrl").
Logger()
scopedLogger.Info().Msg("native ctrl socket client connected (OS thread locked)")
if ctrlSocketConn != nil {
scopedLogger.Debug().Msg("closing existing native socket connection")
ctrlSocketConn.Close()
}
ctrlSocketConn = conn
// Restore HDMI EDID if applicable
go restoreHdmiEdid()
readBuf := make([]byte, 4096)
for {
n, err := conn.Read(readBuf)
if err != nil {
scopedLogger.Warn().Err(err).Msg("error reading from ctrl sock")
break
}
readMsg := string(readBuf[:n])
ctrlResp := CtrlResponse{}
err = json.Unmarshal([]byte(readMsg), &ctrlResp)
if err != nil {
scopedLogger.Warn().Err(err).Str("data", readMsg).Msg("error parsing ctrl sock msg")
continue
}
scopedLogger.Trace().Interface("data", ctrlResp).Msg("ctrl sock msg")
if ctrlResp.Seq != 0 {
responseChan, ok := ongoingRequests[ctrlResp.Seq]
if ok {
responseChan <- &ctrlResp
}
}
switch ctrlResp.Event {
case "video_input_state":
HandleVideoStateMessage(ctrlResp)
}
}
scopedLogger.Debug().Msg("ctrl sock disconnected")
}
func handleVideoClient(conn net.Conn) {
// Lock to OS thread to isolate blocking video I/O
runtime.LockOSThread()
defer runtime.UnlockOSThread()
defer conn.Close()
scopedLogger := nativeLogger.With().
Str("addr", conn.RemoteAddr().String()).
Str("type", "video").
Logger()
scopedLogger.Info().Msg("native video socket client connected (OS thread locked)")
inboundPacket := make([]byte, maxVideoFrameSize)
lastFrame := time.Now()
for {
n, err := conn.Read(inboundPacket)
if err != nil {
scopedLogger.Warn().Err(err).Msg("error during read")
return
}
now := time.Now()
sinceLastFrame := now.Sub(lastFrame)
lastFrame = now
if currentSession != nil {
err := currentSession.VideoTrack.WriteSample(media.Sample{Data: inboundPacket[:n], Duration: sinceLastFrame})
if err != nil {
scopedLogger.Warn().Err(err).Msg("error writing sample")
}
}
}
}
func shouldOverwrite(destPath string, srcHash []byte) bool {
if srcHash == nil {
nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, doing overwriting")
return true
}
dstHash, err := os.ReadFile(destPath + ".sha256")
if err != nil {
nativeLogger.Debug().Msg("error reading existing jetkvm_native.sha256, doing overwriting")
return true
}
return !bytes.Equal(srcHash, dstHash)
}
func getNativeSha256() ([]byte, error) {
version, err := resource.ResourceFS.ReadFile("jetkvm_native.sha256")
if err != nil {
return nil, err
}
return version, nil
}
func GetNativeVersion() (string, error) {
version, err := getNativeSha256()
if err != nil {
return "", err
}
return strings.TrimSpace(string(version)), nil
}
func ensureBinaryUpdated(destPath string) error {
// Lock to OS thread for file I/O operations
runtime.LockOSThread()
defer runtime.UnlockOSThread()
srcFile, err := resource.ResourceFS.Open("jetkvm_native")
if err != nil {
return err
}
defer srcFile.Close()
srcHash, err := getNativeSha256()
if err != nil {
nativeLogger.Debug().Msg("error reading embedded jetkvm_native.sha256, proceeding with update")
srcHash = nil
}
_, err = os.Stat(destPath)
if shouldOverwrite(destPath, srcHash) || err != nil {
nativeLogger.Info().
Interface("hash", srcHash).
Msg("writing jetkvm_native")
_ = os.Remove(destPath)
destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_RDWR, 0755)
if err != nil {
return err
}
_, err = io.Copy(destFile, srcFile)
destFile.Close()
if err != nil {
return err
}
if srcHash != nil {
err = os.WriteFile(destPath+".sha256", srcHash, 0644)
if err != nil {
return err
}
}
nativeLogger.Info().Msg("jetkvm_native updated")
}
return nil
}
// Restore the HDMI EDID value from the config.
// Called after successful connection to jetkvm_native.
func restoreHdmiEdid() {
if config.EdidString != "" {
nativeLogger.Info().Str("edid", config.EdidString).Msg("Restoring HDMI EDID")
_, err := CallCtrlAction("set_edid", map[string]interface{}{"edid": config.EdidString})
if err != nil {
nativeLogger.Warn().Err(err).Msg("Failed to restore HDMI EDID")
}
}
}

View File

@ -1,6 +1,7 @@
package kvm
import (
"github.com/jetkvm/kvm/internal/audio"
"github.com/prometheus/client_golang/prometheus"
versioncollector "github.com/prometheus/client_golang/prometheus/collectors/version"
"github.com/prometheus/common/version"
@ -10,4 +11,7 @@ func initPrometheus() {
// A Prometheus metrics endpoint.
version.Version = builtAppVersion
prometheus.MustRegister(versioncollector.NewCollector("jetkvm"))
// Start audio metrics collection
audio.StartMetricsUpdater()
}

2
resource/dev_test.sh Normal file → Executable file
View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
JSON_OUTPUT=false
GET_COMMANDS=false
if [ "$1" = "-json" ]; then

View File

@ -3,6 +3,7 @@ package kvm
import (
"bufio"
"io"
"runtime"
"strconv"
"strings"
"time"
@ -141,6 +142,10 @@ func unmountDCControl() error {
var dcState DCPowerState
func runDCControl() {
// Lock to OS thread to isolate DC control serial I/O
runtime.LockOSThread()
defer runtime.UnlockOSThread()
scopedLogger := serialLogger.With().Str("service", "dc_control").Logger()
reader := bufio.NewReader(port)
hasRestoreFeature := false
@ -290,6 +295,10 @@ func handleSerialChannel(d *webrtc.DataChannel) {
d.OnOpen(func() {
go func() {
// Lock to OS thread to isolate serial I/O
runtime.LockOSThread()
defer runtime.UnlockOSThread()
buf := make([]byte, 1024)
for {
n, err := port.Read(buf)

24
session_provider.go Normal file
View File

@ -0,0 +1,24 @@
package kvm
import "github.com/jetkvm/kvm/internal/audio"
// KVMSessionProvider implements the audio.SessionProvider interface
type KVMSessionProvider struct{}
// IsSessionActive returns whether there's an active session
func (k *KVMSessionProvider) IsSessionActive() bool {
return currentSession != nil
}
// GetAudioInputManager returns the current session's audio input manager
func (k *KVMSessionProvider) GetAudioInputManager() *audio.AudioInputManager {
if currentSession == nil {
return nil
}
return currentSession.AudioInputManager
}
// initializeAudioSessionProvider sets up the session provider for the audio package
func initializeAudioSessionProvider() {
audio.SetSessionProvider(&KVMSessionProvider{})
}

View File

@ -6,6 +6,7 @@ import (
"io"
"os"
"os/exec"
"runtime"
"github.com/creack/pty"
"github.com/pion/webrtc/v4"
@ -33,6 +34,10 @@ func handleTerminalChannel(d *webrtc.DataChannel) {
}
go func() {
// Lock to OS thread to isolate PTY I/O
runtime.LockOSThread()
defer runtime.UnlockOSThread()
buf := make([]byte, 1024)
for {
n, err := ptmx.Read(buf)

51
tools/build_audio_deps.sh Executable file
View File

@ -0,0 +1,51 @@
#!/bin/bash
# tools/build_audio_deps.sh
# Build ALSA and Opus static libs for ARM in $HOME/.jetkvm/audio-libs
set -e
# Accept version parameters or use defaults
ALSA_VERSION="${1:-1.2.14}"
OPUS_VERSION="${2:-1.5.2}"
JETKVM_HOME="$HOME/.jetkvm"
AUDIO_LIBS_DIR="$JETKVM_HOME/audio-libs"
TOOLCHAIN_DIR="$JETKVM_HOME/rv1106-system"
CROSS_PREFIX="$TOOLCHAIN_DIR/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf"
mkdir -p "$AUDIO_LIBS_DIR"
cd "$AUDIO_LIBS_DIR"
# Download sources
[ -f alsa-lib-${ALSA_VERSION}.tar.bz2 ] || wget -N https://www.alsa-project.org/files/pub/lib/alsa-lib-${ALSA_VERSION}.tar.bz2
[ -f opus-${OPUS_VERSION}.tar.gz ] || wget -N https://downloads.xiph.org/releases/opus/opus-${OPUS_VERSION}.tar.gz
# Extract
[ -d alsa-lib-${ALSA_VERSION} ] || tar xf alsa-lib-${ALSA_VERSION}.tar.bz2
[ -d opus-${OPUS_VERSION} ] || tar xf opus-${OPUS_VERSION}.tar.gz
# Optimization flags for ARM Cortex-A7 with NEON
OPTIM_CFLAGS="-O3 -mcpu=cortex-a7 -mfpu=neon -mfloat-abi=hard -ftree-vectorize -ffast-math -funroll-loops"
export CC="${CROSS_PREFIX}-gcc"
export CFLAGS="$OPTIM_CFLAGS"
export CXXFLAGS="$OPTIM_CFLAGS"
# Build ALSA
cd alsa-lib-${ALSA_VERSION}
if [ ! -f .built ]; then
CFLAGS="$OPTIM_CFLAGS" ./configure --host arm-rockchip830-linux-uclibcgnueabihf --enable-static=yes --enable-shared=no --with-pcm-plugins=rate,linear --disable-seq --disable-rawmidi --disable-ucm
make -j$(nproc)
touch .built
fi
cd ..
# Build Opus
cd opus-${OPUS_VERSION}
if [ ! -f .built ]; then
CFLAGS="$OPTIM_CFLAGS" ./configure --host arm-rockchip830-linux-uclibcgnueabihf --enable-static=yes --enable-shared=no --enable-fixed-point
make -j$(nproc)
touch .built
fi
cd ..
echo "ALSA and Opus built in $AUDIO_LIBS_DIR"

15
tools/setup_rv1106_toolchain.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
# tools/setup_rv1106_toolchain.sh
# Clone the rv1106-system toolchain to $HOME/.jetkvm/rv1106-system if not already present
set -e
JETKVM_HOME="$HOME/.jetkvm"
TOOLCHAIN_DIR="$JETKVM_HOME/rv1106-system"
REPO_URL="https://github.com/jetkvm/rv1106-system.git"
mkdir -p "$JETKVM_HOME"
if [ ! -d "$TOOLCHAIN_DIR" ]; then
echo "Cloning rv1106-system toolchain to $TOOLCHAIN_DIR ..."
git clone --depth 1 "$REPO_URL" "$TOOLCHAIN_DIR"
else
echo "Toolchain already present at $TOOLCHAIN_DIR"
fi

View File

@ -1,4 +1,4 @@
import { MdOutlineContentPasteGo } from "react-icons/md";
import { MdOutlineContentPasteGo, MdVolumeOff, MdVolumeUp, MdGraphicEq } from "react-icons/md";
import { LuCable, LuHardDrive, LuMaximize, LuSettings, LuSignal } from "react-icons/lu";
import { FaKeyboard } from "react-icons/fa6";
import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
@ -18,12 +18,38 @@ import PasteModal from "@/components/popovers/PasteModal";
import WakeOnLanModal from "@/components/popovers/WakeOnLan/Index";
import MountPopopover from "@/components/popovers/MountPopover";
import ExtensionPopover from "@/components/popovers/ExtensionPopover";
import AudioControlPopover from "@/components/popovers/AudioControlPopover";
import { useDeviceUiNavigation } from "@/hooks/useAppNavigation";
import { useAudioEvents } from "@/hooks/useAudioEvents";
// Type for microphone error
interface MicrophoneError {
type: 'permission' | 'device' | 'network' | 'unknown';
message: string;
}
// Type for microphone hook return value
interface MicrophoneHookReturn {
isMicrophoneActive: boolean;
isMicrophoneMuted: boolean;
microphoneStream: MediaStream | null;
startMicrophone: (deviceId?: string) => Promise<{ success: boolean; error?: MicrophoneError }>;
stopMicrophone: () => Promise<{ success: boolean; error?: MicrophoneError }>;
toggleMicrophoneMute: () => Promise<{ success: boolean; error?: MicrophoneError }>;
syncMicrophoneState: () => Promise<void>;
// Loading states
isStarting: boolean;
isStopping: boolean;
isToggling: boolean;
}
export default function Actionbar({
requestFullscreen,
microphone,
}: {
requestFullscreen: () => Promise<void>;
microphone: MicrophoneHookReturn;
}) {
const { navigateTo } = useDeviceUiNavigation();
const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
@ -56,6 +82,12 @@ export default function Actionbar({
[setDisableFocusTrap],
);
// Use WebSocket-based audio events for real-time updates
const { audioMuted } = useAudioEvents();
// Use WebSocket data exclusively - no polling fallback
const isMuted = audioMuted ?? false; // Default to false if WebSocket data not available yet
return (
<Container className="border-b border-b-slate-800/20 bg-white dark:border-b-slate-300/20 dark:bg-slate-900">
<div
@ -93,7 +125,7 @@ export default function Actionbar({
"flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
)}
>
{({ open }) => {
{({ open }: { open: boolean }) => {
checkIfStateChanged(open);
return (
<div className="mx-auto w-full max-w-xl">
@ -135,7 +167,7 @@ export default function Actionbar({
"flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
)}
>
{({ open }) => {
{({ open }: { open: boolean }) => {
checkIfStateChanged(open);
return (
<div className="mx-auto w-full max-w-xl">
@ -187,7 +219,7 @@ export default function Actionbar({
"flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
)}
>
{({ open }) => {
{({ open }: { open: boolean }) => {
checkIfStateChanged(open);
return (
<div className="mx-auto w-full max-w-xl">
@ -230,7 +262,7 @@ export default function Actionbar({
"flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
)}
>
{({ open }) => {
{({ open }: { open: boolean }) => {
checkIfStateChanged(open);
return <ExtensionPopover />;
}}
@ -262,6 +294,7 @@ export default function Actionbar({
}}
/>
</div>
<div>
<Button
size="XS"
@ -282,6 +315,45 @@ export default function Actionbar({
onClick={() => requestFullscreen()}
/>
</div>
<Popover>
<PopoverButton as={Fragment}>
<Button
size="XS"
theme="light"
text="Audio"
LeadingIcon={({ className }) => (
<div className="flex items-center">
{isMuted ? (
<MdVolumeOff className={cx(className, "text-red-500")} />
) : (
<MdVolumeUp className={cx(className, "text-green-500")} />
)}
<MdGraphicEq className={cx(className, "ml-1 text-blue-500")} />
</div>
)}
onClick={() => {
setDisableFocusTrap(true);
}}
/>
</PopoverButton>
<PopoverPanel
anchor="bottom end"
transition
className={cx(
"z-10 flex origin-top flex-col overflow-visible!",
"flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
)}
>
{({ open }: { open: boolean }) => {
checkIfStateChanged(open);
return (
<div className="mx-auto">
<AudioControlPopover microphone={microphone} open={open} />
</div>
);
}}
</PopoverPanel>
</Popover>
</div>
</div>
</Container>

View File

@ -0,0 +1,77 @@
import React from 'react';
import clsx from 'clsx';
interface AudioLevelMeterProps {
level: number; // 0-100 percentage
isActive: boolean;
className?: string;
size?: 'sm' | 'md' | 'lg';
showLabel?: boolean;
}
export const AudioLevelMeter: React.FC<AudioLevelMeterProps> = ({
level,
isActive,
className,
size = 'md',
showLabel = true
}) => {
const sizeClasses = {
sm: 'h-1',
md: 'h-2',
lg: 'h-3'
};
const getLevelColor = (level: number) => {
if (level < 20) return 'bg-green-500';
if (level < 60) return 'bg-yellow-500';
return 'bg-red-500';
};
const getTextColor = (level: number) => {
if (level < 20) return 'text-green-600 dark:text-green-400';
if (level < 60) return 'text-yellow-600 dark:text-yellow-400';
return 'text-red-600 dark:text-red-400';
};
return (
<div className={clsx('space-y-1', className)}>
{showLabel && (
<div className="flex justify-between text-xs">
<span className="text-slate-500 dark:text-slate-400">
Microphone Level
</span>
<span className={clsx(
'font-mono',
isActive ? getTextColor(level) : 'text-slate-400 dark:text-slate-500'
)}>
{isActive ? `${Math.round(level)}%` : 'No Signal'}
</span>
</div>
)}
<div className={clsx(
'w-full rounded-full bg-slate-200 dark:bg-slate-700',
sizeClasses[size]
)}>
<div
className={clsx(
'rounded-full transition-all duration-150 ease-out',
sizeClasses[size],
isActive ? getLevelColor(level) : 'bg-slate-300 dark:bg-slate-600'
)}
style={{
width: isActive ? `${Math.min(100, Math.max(2, level))}%` : '0%'
}}
/>
</div>
{/* Peak indicators */}
<div className="flex justify-between text-xs text-slate-400 dark:text-slate-500">
<span>0%</span>
<span>50%</span>
<span>100%</span>
</div>
</div>
);
};

View File

@ -0,0 +1,887 @@
import { useEffect, useState } from "react";
import { MdGraphicEq, MdSignalWifi4Bar, MdError, MdMic } from "react-icons/md";
import { LuActivity, LuClock, LuHardDrive, LuSettings, LuCpu, LuMemoryStick } from "react-icons/lu";
import { AudioLevelMeter } from "@components/AudioLevelMeter";
import StatChart from "@components/StatChart";
import { cx } from "@/cva.config";
import { useMicrophone } from "@/hooks/useMicrophone";
import { useAudioLevel } from "@/hooks/useAudioLevel";
import { useAudioEvents } from "@/hooks/useAudioEvents";
import api from "@/api";
interface AudioMetrics {
frames_received: number;
frames_dropped: number;
bytes_processed: number;
last_frame_time: string;
connection_drops: number;
average_latency: string;
}
interface MicrophoneMetrics {
frames_sent: number;
frames_dropped: number;
bytes_processed: number;
last_frame_time: string;
connection_drops: number;
average_latency: string;
}
interface ProcessMetrics {
cpu_percent: number;
memory_percent: number;
memory_rss: number;
memory_vms: number;
running: boolean;
}
interface AudioConfig {
Quality: number;
Bitrate: number;
SampleRate: number;
Channels: number;
FrameSize: string;
}
const qualityLabels = {
0: "Low",
1: "Medium",
2: "High",
3: "Ultra"
};
// Format percentage values to 2 decimal places
function formatPercentage(value: number | null | undefined): string {
if (value === null || value === undefined || isNaN(value)) {
return "0.00%";
}
return `${value.toFixed(2)}%`;
}
function formatMemoryMB(rssBytes: number | null | undefined): string {
if (rssBytes === null || rssBytes === undefined || isNaN(rssBytes)) {
return "0.00 MB";
}
const mb = rssBytes / (1024 * 1024);
return `${mb.toFixed(2)} MB`;
}
// Default system memory estimate in MB (will be replaced by actual value from backend)
const DEFAULT_SYSTEM_MEMORY_MB = 4096; // 4GB default
// Create chart array similar to connectionStats.tsx
function createChartArray<T, K extends keyof T>(
stream: Map<number, T>,
metric: K,
): { date: number; stat: T[K] | null }[] {
const stat = Array.from(stream).map(([key, stats]) => {
return { date: key, stat: stats[metric] };
});
// Sort the dates to ensure they are in chronological order
const sortedStat = stat.map(x => x.date).sort((a, b) => a - b);
// Determine the earliest statistic date
const earliestStat = sortedStat[0];
// Current time in seconds since the Unix epoch
const now = Math.floor(Date.now() / 1000);
// Determine the starting point for the chart data
const firstChartDate = earliestStat ? Math.min(earliestStat, now - 120) : now - 120;
// Generate the chart array for the range between 'firstChartDate' and 'now'
return Array.from({ length: now - firstChartDate }, (_, i) => {
const currentDate = firstChartDate + i;
return {
date: currentDate,
// Find the statistic for 'currentDate', or use the last known statistic if none exists for that date
stat: stat.find(x => x.date === currentDate)?.stat ?? null,
};
});
}
export default function AudioMetricsDashboard() {
// System memory state
const [systemMemoryMB, setSystemMemoryMB] = useState(DEFAULT_SYSTEM_MEMORY_MB);
// Use WebSocket-based audio events for real-time updates
const {
audioMetrics,
microphoneMetrics: wsMicrophoneMetrics,
audioProcessMetrics: wsAudioProcessMetrics,
microphoneProcessMetrics: wsMicrophoneProcessMetrics,
isConnected: wsConnected
} = useAudioEvents();
// Fetch system memory information on component mount
useEffect(() => {
const fetchSystemMemory = async () => {
try {
const response = await api.GET('/system/memory');
const data = await response.json();
setSystemMemoryMB(data.total_memory_mb);
} catch (error) {
console.warn('Failed to fetch system memory, using default:', error);
}
};
fetchSystemMemory();
}, []);
// Update historical data when WebSocket process metrics are received
useEffect(() => {
if (wsConnected && wsAudioProcessMetrics && wsAudioProcessMetrics.running) {
const now = Math.floor(Date.now() / 1000); // Convert to seconds for StatChart
// Validate that now is a valid number
if (isNaN(now)) return;
const cpuStat = isNaN(wsAudioProcessMetrics.cpu_percent) ? null : wsAudioProcessMetrics.cpu_percent;
setAudioCpuStats(prev => {
const newMap = new Map(prev);
newMap.set(now, { cpu_percent: cpuStat });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
setAudioMemoryStats(prev => {
const newMap = new Map(prev);
const memoryRss = isNaN(wsAudioProcessMetrics.memory_rss) ? null : wsAudioProcessMetrics.memory_rss;
newMap.set(now, { memory_rss: memoryRss });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
}
}, [wsConnected, wsAudioProcessMetrics]);
useEffect(() => {
if (wsConnected && wsMicrophoneProcessMetrics) {
const now = Math.floor(Date.now() / 1000); // Convert to seconds for StatChart
// Validate that now is a valid number
if (isNaN(now)) return;
const cpuStat = isNaN(wsMicrophoneProcessMetrics.cpu_percent) ? null : wsMicrophoneProcessMetrics.cpu_percent;
setMicCpuStats(prev => {
const newMap = new Map(prev);
newMap.set(now, { cpu_percent: cpuStat });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
setMicMemoryStats(prev => {
const newMap = new Map(prev);
const memoryRss = isNaN(wsMicrophoneProcessMetrics.memory_rss) ? null : wsMicrophoneProcessMetrics.memory_rss;
newMap.set(now, { memory_rss: memoryRss });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
}
}, [wsConnected, wsMicrophoneProcessMetrics]);
// Fallback state for when WebSocket is not connected
const [fallbackMetrics, setFallbackMetrics] = useState<AudioMetrics | null>(null);
const [fallbackMicrophoneMetrics, setFallbackMicrophoneMetrics] = useState<MicrophoneMetrics | null>(null);
const [fallbackConnected, setFallbackConnected] = useState(false);
// Process metrics state (fallback for when WebSocket is not connected)
const [fallbackAudioProcessMetrics, setFallbackAudioProcessMetrics] = useState<ProcessMetrics | null>(null);
const [fallbackMicrophoneProcessMetrics, setFallbackMicrophoneProcessMetrics] = useState<ProcessMetrics | null>(null);
// Historical data for charts using Maps for better memory management
const [audioCpuStats, setAudioCpuStats] = useState<Map<number, { cpu_percent: number | null }>>(new Map());
const [audioMemoryStats, setAudioMemoryStats] = useState<Map<number, { memory_rss: number | null }>>(new Map());
const [micCpuStats, setMicCpuStats] = useState<Map<number, { cpu_percent: number | null }>>(new Map());
const [micMemoryStats, setMicMemoryStats] = useState<Map<number, { memory_rss: number | null }>>(new Map());
// Configuration state (these don't change frequently, so we can load them once)
const [config, setConfig] = useState<AudioConfig | null>(null);
const [microphoneConfig, setMicrophoneConfig] = useState<AudioConfig | null>(null);
const [lastUpdate, setLastUpdate] = useState<Date>(new Date());
// Use WebSocket data when available, fallback to polling data otherwise
const metrics = wsConnected && audioMetrics !== null ? audioMetrics : fallbackMetrics;
const microphoneMetrics = wsConnected && wsMicrophoneMetrics !== null ? wsMicrophoneMetrics : fallbackMicrophoneMetrics;
const audioProcessMetrics = wsConnected && wsAudioProcessMetrics !== null ? wsAudioProcessMetrics : fallbackAudioProcessMetrics;
const microphoneProcessMetrics = wsConnected && wsMicrophoneProcessMetrics !== null ? wsMicrophoneProcessMetrics : fallbackMicrophoneProcessMetrics;
const isConnected = wsConnected ? wsConnected : fallbackConnected;
// Microphone state for audio level monitoring
const { isMicrophoneActive, isMicrophoneMuted, microphoneStream } = useMicrophone();
const { audioLevel, isAnalyzing } = useAudioLevel(
isMicrophoneActive ? microphoneStream : null,
{
enabled: isMicrophoneActive,
updateInterval: 120,
});
useEffect(() => {
// Load initial configuration (only once)
loadAudioConfig();
// Set up fallback polling only when WebSocket is not connected
if (!wsConnected) {
loadAudioData();
const interval = setInterval(loadAudioData, 1000);
return () => clearInterval(interval);
}
}, [wsConnected]);
const loadAudioConfig = async () => {
try {
// Load config
const configResp = await api.GET("/audio/quality");
if (configResp.ok) {
const configData = await configResp.json();
setConfig(configData.current);
}
// Load microphone config
try {
const micConfigResp = await api.GET("/microphone/quality");
if (micConfigResp.ok) {
const micConfigData = await micConfigResp.json();
setMicrophoneConfig(micConfigData.current);
}
} catch (micConfigError) {
console.debug("Microphone config not available:", micConfigError);
}
} catch (error) {
console.error("Failed to load audio config:", error);
}
};
const loadAudioData = async () => {
try {
// Load metrics
const metricsResp = await api.GET("/audio/metrics");
if (metricsResp.ok) {
const metricsData = await metricsResp.json();
setFallbackMetrics(metricsData);
// Consider connected if API call succeeds, regardless of frame count
setFallbackConnected(true);
setLastUpdate(new Date());
} else {
setFallbackConnected(false);
}
// Load audio process metrics
try {
const audioProcessResp = await api.GET("/audio/process-metrics");
if (audioProcessResp.ok) {
const audioProcessData = await audioProcessResp.json();
setFallbackAudioProcessMetrics(audioProcessData);
// Update historical data for charts (keep last 120 seconds)
if (audioProcessData.running) {
const now = Math.floor(Date.now() / 1000); // Convert to seconds for StatChart
// Validate that now is a valid number
if (isNaN(now)) return;
const cpuStat = isNaN(audioProcessData.cpu_percent) ? null : audioProcessData.cpu_percent;
const memoryRss = isNaN(audioProcessData.memory_rss) ? null : audioProcessData.memory_rss;
setAudioCpuStats(prev => {
const newMap = new Map(prev);
newMap.set(now, { cpu_percent: cpuStat });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
setAudioMemoryStats(prev => {
const newMap = new Map(prev);
newMap.set(now, { memory_rss: memoryRss });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
}
}
} catch (audioProcessError) {
console.debug("Audio process metrics not available:", audioProcessError);
}
// Load microphone metrics
try {
const micResp = await api.GET("/microphone/metrics");
if (micResp.ok) {
const micData = await micResp.json();
setFallbackMicrophoneMetrics(micData);
}
} catch (micError) {
// Microphone metrics might not be available, that's okay
console.debug("Microphone metrics not available:", micError);
}
// Load microphone process metrics
try {
const micProcessResp = await api.GET("/microphone/process-metrics");
if (micProcessResp.ok) {
const micProcessData = await micProcessResp.json();
setFallbackMicrophoneProcessMetrics(micProcessData);
// Update historical data for charts (keep last 120 seconds)
const now = Math.floor(Date.now() / 1000); // Convert to seconds for StatChart
// Validate that now is a valid number
if (isNaN(now)) return;
const cpuStat = isNaN(micProcessData.cpu_percent) ? null : micProcessData.cpu_percent;
const memoryRss = isNaN(micProcessData.memory_rss) ? null : micProcessData.memory_rss;
setMicCpuStats(prev => {
const newMap = new Map(prev);
newMap.set(now, { cpu_percent: cpuStat });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
setMicMemoryStats(prev => {
const newMap = new Map(prev);
newMap.set(now, { memory_rss: memoryRss });
// Keep only last 120 seconds of data for memory management
const cutoff = now - 120;
for (const [key] of newMap) {
if (key < cutoff) newMap.delete(key);
}
return newMap;
});
}
} catch (micProcessError) {
console.debug("Microphone process metrics not available:", micProcessError);
}
} catch (error) {
console.error("Failed to load audio data:", error);
setFallbackConnected(false);
}
};
const formatBytes = (bytes: number) => {
if (bytes === 0) return "0 B";
const k = 1024;
const sizes = ["B", "KB", "MB", "GB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
};
const formatNumber = (num: number) => {
return new Intl.NumberFormat().format(num);
};
const getDropRate = () => {
if (!metrics || metrics.frames_received === 0) return 0;
return ((metrics.frames_dropped / metrics.frames_received) * 100);
};
const getQualityColor = (quality: number) => {
switch (quality) {
case 0: return "text-yellow-600 dark:text-yellow-400";
case 1: return "text-blue-600 dark:text-blue-400";
case 2: return "text-green-600 dark:text-green-400";
case 3: return "text-purple-600 dark:text-purple-400";
default: return "text-slate-600 dark:text-slate-400";
}
};
return (
<div className="space-y-4">
{/* Header */}
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<MdGraphicEq className="h-5 w-5 text-blue-600 dark:text-blue-400" />
<h3 className="text-lg font-semibold text-slate-900 dark:text-slate-100">
Audio Metrics
</h3>
</div>
<div className="flex items-center gap-2">
<div className={cx(
"h-2 w-2 rounded-full",
isConnected ? "bg-green-500" : "bg-red-500"
)} />
<span className="text-xs text-slate-500 dark:text-slate-400">
{isConnected ? "Active" : "Inactive"}
</span>
</div>
</div>
{/* Current Configuration */}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
{config && (
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-2 flex items-center gap-2">
<LuSettings className="h-4 w-4 text-blue-600 dark:text-blue-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Audio Output Config
</span>
</div>
<div className="space-y-2 text-sm">
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Quality:</span>
<span className={cx("font-medium", getQualityColor(config.Quality))}>
{qualityLabels[config.Quality as keyof typeof qualityLabels]}
</span>
</div>
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Bitrate:</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{config.Bitrate}kbps
</span>
</div>
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Sample Rate:</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{config.SampleRate}Hz
</span>
</div>
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Channels:</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{config.Channels}
</span>
</div>
</div>
</div>
)}
{microphoneConfig && (
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-2 flex items-center gap-2">
<MdMic className="h-4 w-4 text-green-600 dark:text-green-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Microphone Input Config
</span>
</div>
<div className="space-y-2 text-sm">
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Quality:</span>
<span className={cx("font-medium", getQualityColor(microphoneConfig.Quality))}>
{qualityLabels[microphoneConfig.Quality as keyof typeof qualityLabels]}
</span>
</div>
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Bitrate:</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{microphoneConfig.Bitrate}kbps
</span>
</div>
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Sample Rate:</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{microphoneConfig.SampleRate}Hz
</span>
</div>
<div className="flex justify-between">
<span className="text-slate-500 dark:text-slate-400">Channels:</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{microphoneConfig.Channels}
</span>
</div>
</div>
</div>
)}
</div>
{/* Subprocess Resource Usage - Histogram View */}
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
{/* Audio Output Subprocess */}
{audioProcessMetrics && (
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-3 flex items-center gap-2">
<LuCpu className="h-4 w-4 text-blue-600 dark:text-blue-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Audio Output Process
</span>
<div className={cx(
"h-2 w-2 rounded-full ml-auto",
audioProcessMetrics.running ? "bg-green-500" : "bg-red-500"
)} />
</div>
<div className="space-y-4">
<div>
<h4 className="text-sm font-medium text-slate-900 dark:text-slate-100 mb-2">CPU Usage</h4>
<div className="h-24">
<StatChart
data={createChartArray(audioCpuStats, 'cpu_percent')}
unit="%"
domain={[0, 100]}
/>
</div>
</div>
<div>
<h4 className="text-sm font-medium text-slate-900 dark:text-slate-100 mb-2">Memory Usage</h4>
<div className="h-24">
<StatChart
data={createChartArray(audioMemoryStats, 'memory_rss').map(item => ({
date: item.date,
stat: item.stat ? item.stat / (1024 * 1024) : null // Convert bytes to MB
}))}
unit="MB"
domain={[0, systemMemoryMB]}
/>
</div>
</div>
<div className="grid grid-cols-2 gap-2 text-xs">
<div className="text-center p-2 bg-slate-50 dark:bg-slate-800 rounded">
<div className="font-medium text-slate-900 dark:text-slate-100">
{formatPercentage(audioProcessMetrics.cpu_percent)}
</div>
<div className="text-slate-500 dark:text-slate-400">CPU</div>
</div>
<div className="text-center p-2 bg-slate-50 dark:bg-slate-800 rounded">
<div className="font-medium text-slate-900 dark:text-slate-100">
{formatMemoryMB(audioProcessMetrics.memory_rss)}
</div>
<div className="text-slate-500 dark:text-slate-400">Memory</div>
</div>
</div>
</div>
</div>
)}
{/* Microphone Input Subprocess */}
{microphoneProcessMetrics && (
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-3 flex items-center gap-2">
<LuMemoryStick className="h-4 w-4 text-green-600 dark:text-green-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Microphone Input Process
</span>
<div className={cx(
"h-2 w-2 rounded-full ml-auto",
microphoneProcessMetrics.running ? "bg-green-500" : "bg-red-500"
)} />
</div>
<div className="space-y-4">
<div>
<h4 className="text-sm font-medium text-slate-900 dark:text-slate-100 mb-2">CPU Usage</h4>
<div className="h-24">
<StatChart
data={createChartArray(micCpuStats, 'cpu_percent')}
unit="%"
domain={[0, 100]}
/>
</div>
</div>
<div>
<h4 className="text-sm font-medium text-slate-900 dark:text-slate-100 mb-2">Memory Usage</h4>
<div className="h-24">
<StatChart
data={createChartArray(micMemoryStats, 'memory_rss').map(item => ({
date: item.date,
stat: item.stat ? item.stat / (1024 * 1024) : null // Convert bytes to MB
}))}
unit="MB"
domain={[0, systemMemoryMB]}
/>
</div>
</div>
<div className="grid grid-cols-2 gap-2 text-xs">
<div className="text-center p-2 bg-slate-50 dark:bg-slate-800 rounded">
<div className="font-medium text-slate-900 dark:text-slate-100">
{formatPercentage(microphoneProcessMetrics.cpu_percent)}
</div>
<div className="text-slate-500 dark:text-slate-400">CPU</div>
</div>
<div className="text-center p-2 bg-slate-50 dark:bg-slate-800 rounded">
<div className="font-medium text-slate-900 dark:text-slate-100">
{formatMemoryMB(microphoneProcessMetrics.memory_rss)}
</div>
<div className="text-slate-500 dark:text-slate-400">Memory</div>
</div>
</div>
</div>
</div>
)}
</div>
{/* Performance Metrics */}
{metrics && (
<div className="space-y-3">
{/* Audio Output Frames */}
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-2 flex items-center gap-2">
<LuActivity className="h-4 w-4 text-green-600 dark:text-green-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Audio Output
</span>
</div>
<div className="grid grid-cols-2 gap-3">
<div className="text-center">
<div className="text-2xl font-bold text-green-600 dark:text-green-400">
{formatNumber(metrics.frames_received)}
</div>
<div className="text-xs text-slate-500 dark:text-slate-400">
Frames Received
</div>
</div>
<div className="text-center">
<div className={cx(
"text-2xl font-bold",
metrics.frames_dropped > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(metrics.frames_dropped)}
</div>
<div className="text-xs text-slate-500 dark:text-slate-400">
Frames Dropped
</div>
</div>
</div>
{/* Drop Rate */}
<div className="mt-3 rounded-md bg-slate-50 p-2 dark:bg-slate-700">
<div className="flex items-center justify-between">
<span className="text-sm text-slate-600 dark:text-slate-400">
Drop Rate
</span>
<span className={cx(
"font-bold",
getDropRate() > 5
? "text-red-600 dark:text-red-400"
: getDropRate() > 1
? "text-yellow-600 dark:text-yellow-400"
: "text-green-600 dark:text-green-400"
)}>
{getDropRate().toFixed(2)}%
</span>
</div>
<div className="mt-1 h-2 w-full rounded-full bg-slate-200 dark:bg-slate-600">
<div
className={cx(
"h-2 rounded-full transition-all duration-300",
getDropRate() > 5
? "bg-red-500"
: getDropRate() > 1
? "bg-yellow-500"
: "bg-green-500"
)}
style={{ width: `${Math.min(getDropRate(), 100)}%` }}
/>
</div>
</div>
</div>
{/* Microphone Input Metrics */}
{microphoneMetrics && (
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-2 flex items-center gap-2">
<MdMic className="h-4 w-4 text-orange-600 dark:text-orange-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Microphone Input
</span>
</div>
<div className="grid grid-cols-2 gap-3">
<div className="text-center">
<div className="text-2xl font-bold text-orange-600 dark:text-orange-400">
{formatNumber(microphoneMetrics.frames_sent)}
</div>
<div className="text-xs text-slate-500 dark:text-slate-400">
Frames Sent
</div>
</div>
<div className="text-center">
<div className={cx(
"text-2xl font-bold",
microphoneMetrics.frames_dropped > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(microphoneMetrics.frames_dropped)}
</div>
<div className="text-xs text-slate-500 dark:text-slate-400">
Frames Dropped
</div>
</div>
</div>
{/* Microphone Drop Rate */}
<div className="mt-3 rounded-md bg-slate-50 p-2 dark:bg-slate-700">
<div className="flex items-center justify-between">
<span className="text-sm text-slate-600 dark:text-slate-400">
Drop Rate
</span>
<span className={cx(
"font-bold",
(microphoneMetrics.frames_sent > 0 ? (microphoneMetrics.frames_dropped / microphoneMetrics.frames_sent) * 100 : 0) > 5
? "text-red-600 dark:text-red-400"
: (microphoneMetrics.frames_sent > 0 ? (microphoneMetrics.frames_dropped / microphoneMetrics.frames_sent) * 100 : 0) > 1
? "text-yellow-600 dark:text-yellow-400"
: "text-green-600 dark:text-green-400"
)}>
{microphoneMetrics.frames_sent > 0 ? ((microphoneMetrics.frames_dropped / microphoneMetrics.frames_sent) * 100).toFixed(2) : "0.00"}%
</span>
</div>
<div className="mt-1 h-2 w-full rounded-full bg-slate-200 dark:bg-slate-600">
<div
className={cx(
"h-2 rounded-full transition-all duration-300",
(microphoneMetrics.frames_sent > 0 ? (microphoneMetrics.frames_dropped / microphoneMetrics.frames_sent) * 100 : 0) > 5
? "bg-red-500"
: (microphoneMetrics.frames_sent > 0 ? (microphoneMetrics.frames_dropped / microphoneMetrics.frames_sent) * 100 : 0) > 1
? "bg-yellow-500"
: "bg-green-500"
)}
style={{
width: `${Math.min(microphoneMetrics.frames_sent > 0 ? (microphoneMetrics.frames_dropped / microphoneMetrics.frames_sent) * 100 : 0, 100)}%`
}}
/>
</div>
</div>
{/* Microphone Audio Level */}
{isMicrophoneActive && (
<div className="mt-3 rounded-md bg-slate-50 p-2 dark:bg-slate-700">
<AudioLevelMeter
level={audioLevel}
isActive={isMicrophoneActive && !isMicrophoneMuted && isAnalyzing}
size="sm"
showLabel={true}
/>
</div>
)}
{/* Microphone Connection Health */}
<div className="mt-3 rounded-md bg-slate-50 p-2 dark:bg-slate-700">
<div className="mb-2 flex items-center gap-2">
<MdSignalWifi4Bar className="h-3 w-3 text-purple-600 dark:text-purple-400" />
<span className="text-sm font-medium text-slate-900 dark:text-slate-100">
Connection Health
</span>
</div>
<div className="space-y-2">
<div className="flex justify-between">
<span className="text-xs text-slate-500 dark:text-slate-400">
Connection Drops:
</span>
<span className={cx(
"text-xs font-medium",
microphoneMetrics.connection_drops > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(microphoneMetrics.connection_drops)}
</span>
</div>
{microphoneMetrics.average_latency && (
<div className="flex justify-between">
<span className="text-xs text-slate-500 dark:text-slate-400">
Avg Latency:
</span>
<span className="text-xs font-medium text-slate-900 dark:text-slate-100">
{microphoneMetrics.average_latency}
</span>
</div>
)}
</div>
</div>
</div>
)}
{/* Data Transfer */}
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-2 flex items-center gap-2">
<LuHardDrive className="h-4 w-4 text-blue-600 dark:text-blue-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Data Transfer
</span>
</div>
<div className="text-center">
<div className="text-2xl font-bold text-blue-600 dark:text-blue-400">
{formatBytes(metrics.bytes_processed)}
</div>
<div className="text-xs text-slate-500 dark:text-slate-400">
Total Processed
</div>
</div>
</div>
{/* Connection Health */}
<div className="rounded-lg border border-slate-200 p-3 dark:border-slate-700">
<div className="mb-2 flex items-center gap-2">
<MdSignalWifi4Bar className="h-4 w-4 text-purple-600 dark:text-purple-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Connection Health
</span>
</div>
<div className="space-y-2">
<div className="flex justify-between">
<span className="text-sm text-slate-500 dark:text-slate-400">
Connection Drops:
</span>
<span className={cx(
"font-medium",
metrics.connection_drops > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(metrics.connection_drops)}
</span>
</div>
{metrics.average_latency && (
<div className="flex justify-between">
<span className="text-sm text-slate-500 dark:text-slate-400">
Avg Latency:
</span>
<span className="font-medium text-slate-900 dark:text-slate-100">
{metrics.average_latency}
</span>
</div>
)}
</div>
</div>
</div>
)}
{/* Last Update */}
<div className="flex items-center justify-center gap-2 text-xs text-slate-500 dark:text-slate-400">
<LuClock className="h-3 w-3" />
<span>Last updated: {lastUpdate.toLocaleTimeString()}</span>
</div>
{/* No Data State */}
{!metrics && (
<div className="flex flex-col items-center justify-center py-8 text-center">
<MdError className="h-12 w-12 text-slate-400 dark:text-slate-600" />
<h3 className="mt-2 text-sm font-medium text-slate-900 dark:text-slate-100">
No Audio Data
</h3>
<p className="mt-1 text-sm text-slate-500 dark:text-slate-400">
Audio metrics will appear when audio streaming is active.
</p>
</div>
)}
</div>
);
}

View File

@ -25,7 +25,32 @@ import {
PointerLockBar,
} from "./VideoOverlay";
export default function WebRTCVideo() {
// Type for microphone error
interface MicrophoneError {
type: 'permission' | 'device' | 'network' | 'unknown';
message: string;
}
// Interface for microphone hook return type
interface MicrophoneHookReturn {
isMicrophoneActive: boolean;
isMicrophoneMuted: boolean;
microphoneStream: MediaStream | null;
startMicrophone: (deviceId?: string) => Promise<{ success: boolean; error?: MicrophoneError }>;
stopMicrophone: () => Promise<{ success: boolean; error?: MicrophoneError }>;
toggleMicrophoneMute: () => Promise<{ success: boolean; error?: MicrophoneError }>;
syncMicrophoneState: () => Promise<void>;
// Loading states
isStarting: boolean;
isStopping: boolean;
isToggling: boolean;
}
interface WebRTCVideoProps {
microphone: MicrophoneHookReturn;
}
export default function WebRTCVideo({ microphone }: WebRTCVideoProps) {
// Video and stream related refs and states
const videoElm = useRef<HTMLVideoElement>(null);
const mediaStream = useRTCStore(state => state.mediaStream);
@ -675,7 +700,7 @@ export default function WebRTCVideo() {
disabled={peerConnection?.connectionState !== "connected"}
className="contents"
>
<Actionbar requestFullscreen={requestFullscreen} />
<Actionbar requestFullscreen={requestFullscreen} microphone={microphone} />
<MacroBar />
</fieldset>
</div>
@ -705,7 +730,7 @@ export default function WebRTCVideo() {
controls={false}
onPlaying={onVideoPlaying}
onPlay={onVideoPlaying}
muted
muted={false}
playsInline
disablePictureInPicture
controlsList="nofullscreen"

View File

@ -0,0 +1,749 @@
import { useEffect, useState } from "react";
import { MdVolumeOff, MdVolumeUp, MdGraphicEq, MdMic, MdMicOff, MdRefresh } from "react-icons/md";
import { LuActivity, LuSettings, LuSignal } from "react-icons/lu";
import { Button } from "@components/Button";
import { AudioLevelMeter } from "@components/AudioLevelMeter";
import { cx } from "@/cva.config";
import { useUiStore } from "@/hooks/stores";
import { useAudioDevices } from "@/hooks/useAudioDevices";
import { useAudioLevel } from "@/hooks/useAudioLevel";
import { useAudioEvents } from "@/hooks/useAudioEvents";
import api from "@/api";
import notifications from "@/notifications";
// Type for microphone error
interface MicrophoneError {
type: 'permission' | 'device' | 'network' | 'unknown';
message: string;
}
// Type for microphone hook return value
interface MicrophoneHookReturn {
isMicrophoneActive: boolean;
isMicrophoneMuted: boolean;
microphoneStream: MediaStream | null;
startMicrophone: (deviceId?: string) => Promise<{ success: boolean; error?: MicrophoneError }>;
stopMicrophone: () => Promise<{ success: boolean; error?: MicrophoneError }>;
toggleMicrophoneMute: () => Promise<{ success: boolean; error?: MicrophoneError }>;
syncMicrophoneState: () => Promise<void>;
// Loading states
isStarting: boolean;
isStopping: boolean;
isToggling: boolean;
}
interface AudioConfig {
Quality: number;
Bitrate: number;
SampleRate: number;
Channels: number;
FrameSize: string;
}
const qualityLabels = {
0: "Low (32kbps)",
1: "Medium (64kbps)",
2: "High (128kbps)",
3: "Ultra (256kbps)"
};
interface AudioControlPopoverProps {
microphone: MicrophoneHookReturn;
open?: boolean; // whether the popover is open (controls analysis)
}
export default function AudioControlPopover({ microphone, open }: AudioControlPopoverProps) {
const [currentConfig, setCurrentConfig] = useState<AudioConfig | null>(null);
const [currentMicrophoneConfig, setCurrentMicrophoneConfig] = useState<AudioConfig | null>(null);
const [showAdvanced, setShowAdvanced] = useState(false);
const [isLoading, setIsLoading] = useState(false);
// Add cache flags to prevent unnecessary API calls
const [configsLoaded, setConfigsLoaded] = useState(false);
// Add cooldown to prevent rapid clicking
const [lastClickTime, setLastClickTime] = useState(0);
const CLICK_COOLDOWN = 500; // 500ms cooldown between clicks
// Use WebSocket-based audio events for real-time updates
const {
audioMuted,
audioMetrics,
microphoneMetrics,
isConnected: wsConnected
} = useAudioEvents();
// WebSocket-only implementation - no fallback polling
// Microphone state from props
const {
isMicrophoneActive,
isMicrophoneMuted,
microphoneStream,
startMicrophone,
stopMicrophone,
toggleMicrophoneMute,
syncMicrophoneState,
// Loading states
isStarting,
isStopping,
isToggling,
} = microphone;
// Use WebSocket data exclusively - no polling fallback
const isMuted = audioMuted ?? false;
const metrics = audioMetrics;
const micMetrics = microphoneMetrics;
const isConnected = wsConnected;
// Audio level monitoring - enable only when popover is open and microphone is active to save resources
const analysisEnabled = (open ?? true) && isMicrophoneActive;
const { audioLevel, isAnalyzing } = useAudioLevel(analysisEnabled ? microphoneStream : null, {
enabled: analysisEnabled,
updateInterval: 120, // 8-10 fps to reduce CPU without losing UX quality
});
// Audio devices
const {
audioInputDevices,
audioOutputDevices,
selectedInputDevice,
selectedOutputDevice,
setSelectedInputDevice,
setSelectedOutputDevice,
isLoading: devicesLoading,
error: devicesError,
refreshDevices
} = useAudioDevices();
const { toggleSidebarView } = useUiStore();
// Load initial configurations once - cache to prevent repeated calls
useEffect(() => {
if (!configsLoaded) {
loadAudioConfigurations();
}
}, [configsLoaded]);
// WebSocket-only implementation - sync microphone state when needed
useEffect(() => {
// Always sync microphone state, but debounce it
const syncTimeout = setTimeout(() => {
syncMicrophoneState();
}, 500);
return () => clearTimeout(syncTimeout);
}, [syncMicrophoneState]);
const loadAudioConfigurations = async () => {
try {
// Parallel loading for better performance
const [qualityResp, micQualityResp] = await Promise.all([
api.GET("/audio/quality"),
api.GET("/microphone/quality")
]);
if (qualityResp.ok) {
const qualityData = await qualityResp.json();
setCurrentConfig(qualityData.current);
}
if (micQualityResp.ok) {
const micQualityData = await micQualityResp.json();
setCurrentMicrophoneConfig(micQualityData.current);
}
setConfigsLoaded(true);
} catch (error) {
console.error("Failed to load audio configurations:", error);
}
};
const handleToggleMute = async () => {
setIsLoading(true);
try {
const resp = await api.POST("/audio/mute", { muted: !isMuted });
if (!resp.ok) {
console.error("Failed to toggle mute:", resp.statusText);
}
// WebSocket will handle the state update automatically
} catch (error) {
console.error("Failed to toggle mute:", error);
} finally {
setIsLoading(false);
}
};
const handleQualityChange = async (quality: number) => {
setIsLoading(true);
try {
const resp = await api.POST("/audio/quality", { quality });
if (resp.ok) {
const data = await resp.json();
setCurrentConfig(data.config);
}
} catch (error) {
console.error("Failed to change audio quality:", error);
} finally {
setIsLoading(false);
}
};
const handleMicrophoneQualityChange = async (quality: number) => {
try {
const resp = await api.POST("/microphone/quality", { quality });
if (resp.ok) {
const data = await resp.json();
setCurrentMicrophoneConfig(data.config);
}
} catch (error) {
console.error("Failed to change microphone quality:", error);
}
};
const handleToggleMicrophone = async () => {
const now = Date.now();
// Prevent rapid clicking - if any operation is in progress or within cooldown, ignore the click
if (isStarting || isStopping || isToggling || (now - lastClickTime < CLICK_COOLDOWN)) {
return;
}
setLastClickTime(now);
try {
const result = isMicrophoneActive ? await stopMicrophone() : await startMicrophone(selectedInputDevice);
if (!result.success && result.error) {
notifications.error(result.error.message);
}
} catch (error) {
console.error("Failed to toggle microphone:", error);
notifications.error("An unexpected error occurred");
}
};
const handleToggleMicrophoneMute = async () => {
const now = Date.now();
// Prevent rapid clicking - if any operation is in progress or within cooldown, ignore the click
if (isStarting || isStopping || isToggling || (now - lastClickTime < CLICK_COOLDOWN)) {
return;
}
setLastClickTime(now);
try {
const result = await toggleMicrophoneMute();
if (!result.success && result.error) {
notifications.error(result.error.message);
}
} catch (error) {
console.error("Failed to toggle microphone mute:", error);
notifications.error("Failed to toggle microphone mute");
}
};
// Handle microphone device change
const handleMicrophoneDeviceChange = async (deviceId: string) => {
setSelectedInputDevice(deviceId);
// If microphone is currently active, restart it with the new device
if (isMicrophoneActive) {
try {
// Stop current microphone
await stopMicrophone();
// Start with new device
const result = await startMicrophone(deviceId);
if (!result.success && result.error) {
notifications.error(result.error.message);
}
} catch (error) {
console.error("Failed to change microphone device:", error);
notifications.error("Failed to change microphone device");
}
}
};
const handleAudioOutputDeviceChange = async (deviceId: string) => {
setSelectedOutputDevice(deviceId);
// Find the video element and set the audio output device
const videoElement = document.querySelector('video');
if (videoElement && 'setSinkId' in videoElement) {
try {
await (videoElement as HTMLVideoElement & { setSinkId: (deviceId: string) => Promise<void> }).setSinkId(deviceId);
} catch (error: unknown) {
console.error('Failed to change audio output device:', error);
}
} else {
console.warn('setSinkId not supported or video element not found');
}
};
const formatBytes = (bytes: number) => {
if (bytes === 0) return "0 B";
const k = 1024;
const sizes = ["B", "KB", "MB", "GB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
};
const formatNumber = (num: number) => {
return new Intl.NumberFormat().format(num);
};
return (
<div className="w-full max-w-md rounded-lg border border-slate-200 bg-white p-4 shadow-lg dark:border-slate-700 dark:bg-slate-800">
<div className="space-y-4">
{/* Header */}
<div className="flex items-center justify-between">
<h3 className="text-lg font-semibold text-slate-900 dark:text-slate-100">
Audio Controls
</h3>
<div className="flex items-center gap-2">
<div className={cx(
"h-2 w-2 rounded-full",
isConnected ? "bg-green-500" : "bg-red-500"
)} />
<span className="text-xs text-slate-500 dark:text-slate-400">
{isConnected ? "Connected" : "Disconnected"}
</span>
</div>
</div>
{/* Mute Control */}
<div className="flex items-center justify-between rounded-lg bg-slate-50 p-3 dark:bg-slate-700">
<div className="flex items-center gap-3">
{isMuted ? (
<MdVolumeOff className="h-5 w-5 text-red-500" />
) : (
<MdVolumeUp className="h-5 w-5 text-green-500" />
)}
<span className="font-medium text-slate-900 dark:text-slate-100">
{isMuted ? "Muted" : "Unmuted"}
</span>
</div>
<Button
size="SM"
theme={isMuted ? "danger" : "primary"}
text={isMuted ? "Unmute" : "Mute"}
onClick={handleToggleMute}
disabled={isLoading}
/>
</div>
{/* Microphone Control */}
<div className="space-y-3">
<div className="flex items-center gap-2">
<MdMic className="h-4 w-4 text-slate-600 dark:text-slate-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Microphone Input
</span>
</div>
<div className="flex items-center justify-between rounded-lg bg-slate-50 p-3 dark:bg-slate-700">
<div className="flex items-center gap-3">
{isMicrophoneActive ? (
isMicrophoneMuted ? (
<MdMicOff className="h-5 w-5 text-yellow-500" />
) : (
<MdMic className="h-5 w-5 text-green-500" />
)
) : (
<MdMicOff className="h-5 w-5 text-red-500" />
)}
<span className="font-medium text-slate-900 dark:text-slate-100">
{!isMicrophoneActive
? "Inactive"
: isMicrophoneMuted
? "Muted"
: "Active"
}
</span>
</div>
<div className="flex gap-2">
<Button
size="SM"
theme={isMicrophoneActive ? "danger" : "primary"}
text={
isStarting ? "Starting..." :
isStopping ? "Stopping..." :
isMicrophoneActive ? "Stop" : "Start"
}
onClick={handleToggleMicrophone}
disabled={isStarting || isStopping || isToggling}
loading={isStarting || isStopping}
/>
{isMicrophoneActive && (
<Button
size="SM"
theme={isMicrophoneMuted ? "danger" : "light"}
text={
isToggling ? (isMicrophoneMuted ? "Unmuting..." : "Muting...") :
isMicrophoneMuted ? "Unmute" : "Mute"
}
onClick={handleToggleMicrophoneMute}
disabled={isStarting || isStopping || isToggling}
loading={isToggling}
/>
)}
</div>
</div>
{/* Audio Level Meter */}
{isMicrophoneActive && (
<div className="rounded-lg bg-slate-50 p-3 dark:bg-slate-700">
<AudioLevelMeter
level={audioLevel}
isActive={isMicrophoneActive && !isMicrophoneMuted && isAnalyzing}
size="md"
showLabel={true}
/>
{/* Debug information */}
<div className="mt-2 text-xs text-slate-500 dark:text-slate-400">
<div className="grid grid-cols-2 gap-1">
<span>Stream: {microphoneStream ? '✓' : '✗'}</span>
<span>Analyzing: {isAnalyzing ? '✓' : '✗'}</span>
<span>Active: {isMicrophoneActive ? '✓' : '✗'}</span>
<span>Muted: {isMicrophoneMuted ? '✓' : '✗'}</span>
</div>
{microphoneStream && (
<div className="mt-1">
Tracks: {microphoneStream.getAudioTracks().length}
{microphoneStream.getAudioTracks().length > 0 && (
<span className="ml-2">
(Enabled: {microphoneStream.getAudioTracks().filter((t: MediaStreamTrack) => t.enabled).length})
</span>
)}
</div>
)}
<button
onClick={syncMicrophoneState}
className="mt-1 text-blue-500 hover:text-blue-600 dark:text-blue-400 dark:hover:text-blue-300"
>
Sync State
</button>
</div>
</div>
)}
</div>
{/* Device Selection */}
<div className="space-y-3">
<div className="flex items-center gap-2">
<MdMic className="h-4 w-4 text-slate-600 dark:text-slate-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Audio Devices
</span>
{devicesLoading && (
<div className="h-3 w-3 animate-spin rounded-full border border-slate-300 border-t-slate-600 dark:border-slate-600 dark:border-t-slate-300" />
)}
</div>
{devicesError && (
<div className="rounded-md bg-red-50 p-2 text-xs text-red-600 dark:bg-red-900/20 dark:text-red-400">
{devicesError}
</div>
)}
{/* Microphone Selection */}
<div className="space-y-2">
<label className="text-sm font-medium text-slate-700 dark:text-slate-300">
Microphone
</label>
<select
value={selectedInputDevice}
onChange={(e) => handleMicrophoneDeviceChange(e.target.value)}
disabled={devicesLoading}
className="w-full rounded-md border border-slate-200 bg-white px-3 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500 disabled:bg-slate-50 disabled:text-slate-500 dark:border-slate-600 dark:bg-slate-700 dark:text-slate-300 dark:focus:border-blue-400 dark:disabled:bg-slate-800"
>
{audioInputDevices.map((device) => (
<option key={device.deviceId} value={device.deviceId}>
{device.label}
</option>
))}
</select>
{isMicrophoneActive && (
<p className="text-xs text-slate-500 dark:text-slate-400">
Changing device will restart the microphone
</p>
)}
</div>
{/* Speaker Selection */}
<div className="space-y-2">
<label className="text-sm font-medium text-slate-700 dark:text-slate-300">
Speaker
</label>
<select
value={selectedOutputDevice}
onChange={(e) => handleAudioOutputDeviceChange(e.target.value)}
disabled={devicesLoading}
className="w-full rounded-md border border-slate-200 bg-white px-3 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500 disabled:bg-slate-50 disabled:text-slate-500 dark:border-slate-600 dark:bg-slate-700 dark:text-slate-300 dark:focus:border-blue-400 dark:disabled:bg-slate-800"
>
{audioOutputDevices.map((device) => (
<option key={device.deviceId} value={device.deviceId}>
{device.label}
</option>
))}
</select>
</div>
<button
onClick={refreshDevices}
disabled={devicesLoading}
className="flex w-full items-center justify-center gap-2 rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50 disabled:opacity-50 dark:border-slate-600 dark:text-slate-300 dark:hover:bg-slate-700"
>
<MdRefresh className={cx("h-4 w-4", devicesLoading && "animate-spin")} />
Refresh Devices
</button>
</div>
{/* Microphone Quality Settings */}
{isMicrophoneActive && (
<div className="space-y-3">
<div className="flex items-center gap-2">
<MdMic className="h-4 w-4 text-slate-600 dark:text-slate-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Microphone Quality
</span>
</div>
<div className="grid grid-cols-2 gap-2">
{Object.entries(qualityLabels).map(([quality, label]) => (
<button
key={`mic-${quality}`}
onClick={() => handleMicrophoneQualityChange(parseInt(quality))}
disabled={isStarting || isStopping || isToggling}
className={cx(
"rounded-md border px-3 py-2 text-sm font-medium transition-colors",
currentMicrophoneConfig?.Quality === parseInt(quality)
? "border-green-500 bg-green-50 text-green-700 dark:bg-green-900/20 dark:text-green-300"
: "border-slate-200 bg-white text-slate-700 hover:bg-slate-50 dark:border-slate-600 dark:bg-slate-700 dark:text-slate-300 dark:hover:bg-slate-600",
(isStarting || isStopping || isToggling) && "opacity-50 cursor-not-allowed"
)}
>
{label}
</button>
))}
</div>
{currentMicrophoneConfig && (
<div className="rounded-md bg-green-50 p-2 text-xs text-green-600 dark:bg-green-900/20 dark:text-green-400">
<div className="grid grid-cols-2 gap-1">
<span>Sample Rate: {currentMicrophoneConfig.SampleRate}Hz</span>
<span>Channels: {currentMicrophoneConfig.Channels}</span>
<span>Bitrate: {currentMicrophoneConfig.Bitrate}kbps</span>
<span>Frame: {currentMicrophoneConfig.FrameSize}</span>
</div>
</div>
)}
</div>
)}
{/* Quality Settings */}
<div className="space-y-3">
<div className="flex items-center gap-2">
<MdGraphicEq className="h-4 w-4 text-slate-600 dark:text-slate-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Audio Output Quality
</span>
</div>
<div className="grid grid-cols-2 gap-2">
{Object.entries(qualityLabels).map(([quality, label]) => (
<button
key={quality}
onClick={() => handleQualityChange(parseInt(quality))}
disabled={isLoading}
className={cx(
"rounded-md border px-3 py-2 text-sm font-medium transition-colors",
currentConfig?.Quality === parseInt(quality)
? "border-blue-500 bg-blue-50 text-blue-700 dark:bg-blue-900/20 dark:text-blue-300"
: "border-slate-200 bg-white text-slate-700 hover:bg-slate-50 dark:border-slate-600 dark:bg-slate-700 dark:text-slate-300 dark:hover:bg-slate-600",
isLoading && "opacity-50 cursor-not-allowed"
)}
>
{label}
</button>
))}
</div>
{currentConfig && (
<div className="rounded-md bg-slate-50 p-2 text-xs text-slate-600 dark:bg-slate-700 dark:text-slate-400">
<div className="grid grid-cols-2 gap-1">
<span>Sample Rate: {currentConfig.SampleRate}Hz</span>
<span>Channels: {currentConfig.Channels}</span>
<span>Bitrate: {currentConfig.Bitrate}kbps</span>
<span>Frame: {currentConfig.FrameSize}</span>
</div>
</div>
)}
</div>
{/* Advanced Controls Toggle */}
<button
onClick={() => setShowAdvanced(!showAdvanced)}
className="flex w-full items-center justify-between rounded-md border border-slate-200 p-2 text-sm font-medium text-slate-700 hover:bg-slate-50 dark:border-slate-600 dark:text-slate-300 dark:hover:bg-slate-700"
>
<div className="flex items-center gap-2">
<LuSettings className="h-4 w-4" />
<span>Advanced Metrics</span>
</div>
<span className={cx(
"transition-transform",
showAdvanced ? "rotate-180" : "rotate-0"
)}>
</span>
</button>
{/* Advanced Metrics */}
{showAdvanced && (
<div className="space-y-3 rounded-lg border border-slate-200 p-3 dark:border-slate-600">
<div className="flex items-center gap-2">
<LuActivity className="h-4 w-4 text-slate-600 dark:text-slate-400" />
<span className="font-medium text-slate-900 dark:text-slate-100">
Performance Metrics
</span>
</div>
{metrics ? (
<>
<div className="mb-4">
<h4 className="text-sm font-medium text-slate-700 dark:text-slate-300 mb-2">Audio Output</h4>
<div className="grid grid-cols-2 gap-3 text-xs">
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Frames Received</div>
<div className="font-mono text-green-600 dark:text-green-400">
{formatNumber(metrics.frames_received)}
</div>
</div>
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Frames Dropped</div>
<div className={cx(
"font-mono",
metrics.frames_dropped > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(metrics.frames_dropped)}
</div>
</div>
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Data Processed</div>
<div className="font-mono text-blue-600 dark:text-blue-400">
{formatBytes(metrics.bytes_processed)}
</div>
</div>
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Connection Drops</div>
<div className={cx(
"font-mono",
metrics.connection_drops > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(metrics.connection_drops)}
</div>
</div>
</div>
</div>
{micMetrics && (
<div className="mb-4">
<h4 className="text-sm font-medium text-slate-700 dark:text-slate-300 mb-2">Microphone Input</h4>
<div className="grid grid-cols-2 gap-3 text-xs">
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Frames Sent</div>
<div className="font-mono text-green-600 dark:text-green-400">
{formatNumber(micMetrics.frames_sent)}
</div>
</div>
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Frames Dropped</div>
<div className={cx(
"font-mono",
micMetrics.frames_dropped > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(micMetrics.frames_dropped)}
</div>
</div>
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Data Processed</div>
<div className="font-mono text-blue-600 dark:text-blue-400">
{formatBytes(micMetrics.bytes_processed)}
</div>
</div>
<div className="space-y-1">
<div className="text-slate-500 dark:text-slate-400">Connection Drops</div>
<div className={cx(
"font-mono",
micMetrics.connection_drops > 0
? "text-red-600 dark:text-red-400"
: "text-green-600 dark:text-green-400"
)}>
{formatNumber(micMetrics.connection_drops)}
</div>
</div>
</div>
</div>
)}
{metrics.frames_received > 0 && (
<div className="mt-3 rounded-md bg-slate-50 p-2 dark:bg-slate-700">
<div className="text-xs text-slate-500 dark:text-slate-400">Drop Rate</div>
<div className={cx(
"font-mono text-sm",
((metrics.frames_dropped / metrics.frames_received) * 100) > 5
? "text-red-600 dark:text-red-400"
: ((metrics.frames_dropped / metrics.frames_received) * 100) > 1
? "text-yellow-600 dark:text-yellow-400"
: "text-green-600 dark:text-green-400"
)}>
{((metrics.frames_dropped / metrics.frames_received) * 100).toFixed(2)}%
</div>
</div>
)}
<div className="text-xs text-slate-500 dark:text-slate-400">
Last updated: {new Date().toLocaleTimeString()}
</div>
</>
) : (
<div className="text-center py-4">
<div className="text-sm text-slate-500 dark:text-slate-400">
Loading metrics...
</div>
</div>
)}
</div>
)}
{/* Audio Metrics Dashboard Button */}
<div className="pt-2 border-t border-slate-200 dark:border-slate-600">
<div className="flex justify-center">
<button
onClick={() => {
toggleSidebarView("audio-metrics");
}}
className="flex items-center gap-2 rounded-md border border-slate-200 bg-white px-4 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50 dark:border-slate-600 dark:bg-slate-700 dark:text-slate-300 dark:hover:bg-slate-600 transition-colors"
>
<LuSignal className="h-4 w-4 text-blue-500" />
<span>View Full Audio Metrics</span>
</button>
</div>
</div>
</div>
</div>
);
}

View File

@ -0,0 +1,16 @@
import SidebarHeader from "@/components/SidebarHeader";
import { useUiStore } from "@/hooks/stores";
import AudioMetricsDashboard from "@/components/AudioMetricsDashboard";
export default function AudioMetricsSidebar() {
const setSidebarView = useUiStore(state => state.setSidebarView);
return (
<>
<SidebarHeader title="Audio Metrics" setSidebarView={setSidebarView} />
<div className="h-full overflow-y-scroll bg-white px-4 py-2 pb-8 dark:bg-slate-900">
<AudioMetricsDashboard />
</div>
</>
);
}

View File

@ -38,7 +38,7 @@ const appendStatToMap = <T extends { timestamp: number }>(
};
// Constants and types
export type AvailableSidebarViews = "connection-stats";
export type AvailableSidebarViews = "connection-stats" | "audio-metrics";
export type AvailableTerminalTypes = "kvm" | "serial" | "none";
export interface User {
@ -117,6 +117,16 @@ interface RTCState {
mediaStream: MediaStream | null;
setMediaStream: (stream: MediaStream) => void;
// Microphone stream management
microphoneStream: MediaStream | null;
setMicrophoneStream: (stream: MediaStream | null) => void;
microphoneSender: RTCRtpSender | null;
setMicrophoneSender: (sender: RTCRtpSender | null) => void;
isMicrophoneActive: boolean;
setMicrophoneActive: (active: boolean) => void;
isMicrophoneMuted: boolean;
setMicrophoneMuted: (muted: boolean) => void;
videoStreamStats: RTCInboundRtpStreamStats | null;
appendVideoStreamStats: (state: RTCInboundRtpStreamStats) => void;
videoStreamStatsHistory: Map<number, RTCInboundRtpStreamStats>;
@ -166,6 +176,16 @@ export const useRTCStore = create<RTCState>(set => ({
mediaStream: null,
setMediaStream: stream => set({ mediaStream: stream }),
// Microphone stream management
microphoneStream: null,
setMicrophoneStream: stream => set({ microphoneStream: stream }),
microphoneSender: null,
setMicrophoneSender: sender => set({ microphoneSender: sender }),
isMicrophoneActive: false,
setMicrophoneActive: active => set({ isMicrophoneActive: active }),
isMicrophoneMuted: false,
setMicrophoneMuted: muted => set({ isMicrophoneMuted: muted }),
videoStreamStats: null,
appendVideoStreamStats: stats => set({ videoStreamStats: stats }),
videoStreamStatsHistory: new Map(),

View File

@ -0,0 +1,107 @@
import { useState, useEffect, useCallback } from 'react';
export interface AudioDevice {
deviceId: string;
label: string;
kind: 'audioinput' | 'audiooutput';
}
export interface UseAudioDevicesReturn {
audioInputDevices: AudioDevice[];
audioOutputDevices: AudioDevice[];
selectedInputDevice: string;
selectedOutputDevice: string;
isLoading: boolean;
error: string | null;
refreshDevices: () => Promise<void>;
setSelectedInputDevice: (deviceId: string) => void;
setSelectedOutputDevice: (deviceId: string) => void;
}
export function useAudioDevices(): UseAudioDevicesReturn {
const [audioInputDevices, setAudioInputDevices] = useState<AudioDevice[]>([]);
const [audioOutputDevices, setAudioOutputDevices] = useState<AudioDevice[]>([]);
const [selectedInputDevice, setSelectedInputDevice] = useState<string>('default');
const [selectedOutputDevice, setSelectedOutputDevice] = useState<string>('default');
const [isLoading, setIsLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const refreshDevices = useCallback(async () => {
setIsLoading(true);
setError(null);
try {
// Request permissions first to get device labels
await navigator.mediaDevices.getUserMedia({ audio: true });
const devices = await navigator.mediaDevices.enumerateDevices();
const inputDevices: AudioDevice[] = [
{ deviceId: 'default', label: 'Default Microphone', kind: 'audioinput' }
];
const outputDevices: AudioDevice[] = [
{ deviceId: 'default', label: 'Default Speaker', kind: 'audiooutput' }
];
devices.forEach(device => {
if (device.kind === 'audioinput' && device.deviceId !== 'default') {
inputDevices.push({
deviceId: device.deviceId,
label: device.label || `Microphone ${device.deviceId.slice(0, 8)}`,
kind: 'audioinput'
});
} else if (device.kind === 'audiooutput' && device.deviceId !== 'default') {
outputDevices.push({
deviceId: device.deviceId,
label: device.label || `Speaker ${device.deviceId.slice(0, 8)}`,
kind: 'audiooutput'
});
}
});
setAudioInputDevices(inputDevices);
setAudioOutputDevices(outputDevices);
console.log('Audio devices enumerated:', {
inputs: inputDevices.length,
outputs: outputDevices.length
});
} catch (err) {
console.error('Failed to enumerate audio devices:', err);
setError(err instanceof Error ? err.message : 'Failed to access audio devices');
} finally {
setIsLoading(false);
}
}, []);
// Listen for device changes
useEffect(() => {
const handleDeviceChange = () => {
console.log('Audio devices changed, refreshing...');
refreshDevices();
};
navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange);
// Initial load
refreshDevices();
return () => {
navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange);
};
}, [refreshDevices]);
return {
audioInputDevices,
audioOutputDevices,
selectedInputDevice,
selectedOutputDevice,
isLoading,
error,
refreshDevices,
setSelectedInputDevice,
setSelectedOutputDevice,
};
}

View File

@ -0,0 +1,316 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import useWebSocket, { ReadyState } from 'react-use-websocket';
// Audio event types matching the backend
export type AudioEventType =
| 'audio-mute-changed'
| 'audio-metrics-update'
| 'microphone-state-changed'
| 'microphone-metrics-update'
| 'audio-process-metrics'
| 'microphone-process-metrics';
// Audio event data interfaces
export interface AudioMuteData {
muted: boolean;
}
export interface AudioMetricsData {
frames_received: number;
frames_dropped: number;
bytes_processed: number;
last_frame_time: string;
connection_drops: number;
average_latency: string;
}
export interface MicrophoneStateData {
running: boolean;
session_active: boolean;
}
export interface MicrophoneMetricsData {
frames_sent: number;
frames_dropped: number;
bytes_processed: number;
last_frame_time: string;
connection_drops: number;
average_latency: string;
}
export interface ProcessMetricsData {
pid: number;
cpu_percent: number;
memory_rss: number;
memory_vms: number;
memory_percent: number;
running: boolean;
process_name: string;
}
// Audio event structure
export interface AudioEvent {
type: AudioEventType;
data: AudioMuteData | AudioMetricsData | MicrophoneStateData | MicrophoneMetricsData | ProcessMetricsData;
}
// Hook return type
export interface UseAudioEventsReturn {
// Connection state
connectionState: ReadyState;
isConnected: boolean;
// Audio state
audioMuted: boolean | null;
audioMetrics: AudioMetricsData | null;
// Microphone state
microphoneState: MicrophoneStateData | null;
microphoneMetrics: MicrophoneMetricsData | null;
// Process metrics
audioProcessMetrics: ProcessMetricsData | null;
microphoneProcessMetrics: ProcessMetricsData | null;
// Manual subscription control
subscribe: () => void;
unsubscribe: () => void;
}
// Global subscription management to prevent multiple subscriptions per WebSocket connection
const globalSubscriptionState = {
isSubscribed: false,
subscriberCount: 0,
connectionId: null as string | null
};
export function useAudioEvents(): UseAudioEventsReturn {
// State for audio data
const [audioMuted, setAudioMuted] = useState<boolean | null>(null);
const [audioMetrics, setAudioMetrics] = useState<AudioMetricsData | null>(null);
const [microphoneState, setMicrophoneState] = useState<MicrophoneStateData | null>(null);
const [microphoneMetrics, setMicrophoneMetricsData] = useState<MicrophoneMetricsData | null>(null);
const [audioProcessMetrics, setAudioProcessMetrics] = useState<ProcessMetricsData | null>(null);
const [microphoneProcessMetrics, setMicrophoneProcessMetrics] = useState<ProcessMetricsData | null>(null);
// Local subscription state
const [isLocallySubscribed, setIsLocallySubscribed] = useState(false);
const subscriptionTimeoutRef = useRef<number | null>(null);
// Get WebSocket URL
const getWebSocketUrl = () => {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const host = window.location.host;
return `${protocol}//${host}/webrtc/signaling/client`;
};
// Shared WebSocket connection using the `share` option for better resource management
const {
sendMessage,
lastMessage,
readyState,
} = useWebSocket(getWebSocketUrl(), {
shouldReconnect: () => true,
reconnectAttempts: 10,
reconnectInterval: 3000,
share: true, // Share the WebSocket connection across multiple hooks
onOpen: () => {
console.log('[AudioEvents] WebSocket connected');
// Reset global state on new connection
globalSubscriptionState.isSubscribed = false;
globalSubscriptionState.connectionId = Math.random().toString(36);
},
onClose: () => {
console.log('[AudioEvents] WebSocket disconnected');
// Reset global state on disconnect
globalSubscriptionState.isSubscribed = false;
globalSubscriptionState.subscriberCount = 0;
globalSubscriptionState.connectionId = null;
},
onError: (event) => {
console.error('[AudioEvents] WebSocket error:', event);
},
});
// Subscribe to audio events
const subscribe = useCallback(() => {
if (readyState === ReadyState.OPEN && !globalSubscriptionState.isSubscribed) {
// Clear any pending subscription timeout
if (subscriptionTimeoutRef.current) {
clearTimeout(subscriptionTimeoutRef.current);
subscriptionTimeoutRef.current = null;
}
// Add a small delay to prevent rapid subscription attempts
subscriptionTimeoutRef.current = setTimeout(() => {
if (readyState === ReadyState.OPEN && !globalSubscriptionState.isSubscribed) {
const subscribeMessage = {
type: 'subscribe-audio-events',
data: {}
};
sendMessage(JSON.stringify(subscribeMessage));
globalSubscriptionState.isSubscribed = true;
console.log('[AudioEvents] Subscribed to audio events');
}
}, 100); // 100ms delay to debounce subscription attempts
}
// Track local subscription regardless of global state
if (!isLocallySubscribed) {
globalSubscriptionState.subscriberCount++;
setIsLocallySubscribed(true);
}
}, [readyState, sendMessage, isLocallySubscribed]);
// Unsubscribe from audio events
const unsubscribe = useCallback(() => {
// Clear any pending subscription timeout
if (subscriptionTimeoutRef.current) {
clearTimeout(subscriptionTimeoutRef.current);
subscriptionTimeoutRef.current = null;
}
if (isLocallySubscribed) {
globalSubscriptionState.subscriberCount--;
setIsLocallySubscribed(false);
// Only send unsubscribe message if this is the last subscriber and connection is still open
if (globalSubscriptionState.subscriberCount <= 0 &&
readyState === ReadyState.OPEN &&
globalSubscriptionState.isSubscribed) {
const unsubscribeMessage = {
type: 'unsubscribe-audio-events',
data: {}
};
sendMessage(JSON.stringify(unsubscribeMessage));
globalSubscriptionState.isSubscribed = false;
globalSubscriptionState.subscriberCount = 0;
console.log('[AudioEvents] Sent unsubscribe message to backend');
}
}
console.log('[AudioEvents] Component unsubscribed from audio events');
}, [readyState, isLocallySubscribed, sendMessage]);
// Handle incoming messages
useEffect(() => {
if (lastMessage !== null) {
try {
const message = JSON.parse(lastMessage.data);
// Handle audio events
if (message.type && message.data) {
const audioEvent = message as AudioEvent;
switch (audioEvent.type) {
case 'audio-mute-changed': {
const muteData = audioEvent.data as AudioMuteData;
setAudioMuted(muteData.muted);
console.log('[AudioEvents] Audio mute changed:', muteData.muted);
break;
}
case 'audio-metrics-update': {
const audioMetricsData = audioEvent.data as AudioMetricsData;
setAudioMetrics(audioMetricsData);
break;
}
case 'microphone-state-changed': {
const micStateData = audioEvent.data as MicrophoneStateData;
setMicrophoneState(micStateData);
console.log('[AudioEvents] Microphone state changed:', micStateData);
break;
}
case 'microphone-metrics-update': {
const micMetricsData = audioEvent.data as MicrophoneMetricsData;
setMicrophoneMetricsData(micMetricsData);
break;
}
case 'audio-process-metrics': {
const audioProcessData = audioEvent.data as ProcessMetricsData;
setAudioProcessMetrics(audioProcessData);
break;
}
case 'microphone-process-metrics': {
const micProcessData = audioEvent.data as ProcessMetricsData;
setMicrophoneProcessMetrics(micProcessData);
break;
}
default:
// Ignore other message types (WebRTC signaling, etc.)
break;
}
}
} catch (error) {
// Ignore parsing errors for non-JSON messages (like "pong")
if (lastMessage.data !== 'pong') {
console.warn('[AudioEvents] Failed to parse WebSocket message:', error);
}
}
}
}, [lastMessage]);
// Auto-subscribe when connected
useEffect(() => {
if (readyState === ReadyState.OPEN) {
subscribe();
}
// Cleanup subscription on component unmount or connection change
return () => {
if (subscriptionTimeoutRef.current) {
clearTimeout(subscriptionTimeoutRef.current);
subscriptionTimeoutRef.current = null;
}
unsubscribe();
};
}, [readyState, subscribe, unsubscribe]);
// Reset local subscription state on disconnect
useEffect(() => {
if (readyState === ReadyState.CLOSED || readyState === ReadyState.CLOSING) {
setIsLocallySubscribed(false);
if (subscriptionTimeoutRef.current) {
clearTimeout(subscriptionTimeoutRef.current);
subscriptionTimeoutRef.current = null;
}
}
}, [readyState]);
// Cleanup on component unmount
useEffect(() => {
return () => {
unsubscribe();
};
}, [unsubscribe]);
return {
// Connection state
connectionState: readyState,
isConnected: readyState === ReadyState.OPEN && globalSubscriptionState.isSubscribed,
// Audio state
audioMuted,
audioMetrics,
// Microphone state
microphoneState,
microphoneMetrics: microphoneMetrics,
// Process metrics
audioProcessMetrics,
microphoneProcessMetrics,
// Manual subscription control
subscribe,
unsubscribe,
};
}

View File

@ -0,0 +1,134 @@
import { useEffect, useRef, useState } from 'react';
interface AudioLevelHookResult {
audioLevel: number; // 0-100 percentage
isAnalyzing: boolean;
}
interface AudioLevelOptions {
enabled?: boolean; // Allow external control of analysis
updateInterval?: number; // Throttle updates (default: 100ms for 10fps instead of 60fps)
}
export const useAudioLevel = (
stream: MediaStream | null,
options: AudioLevelOptions = {}
): AudioLevelHookResult => {
const { enabled = true, updateInterval = 100 } = options;
const [audioLevel, setAudioLevel] = useState(0);
const [isAnalyzing, setIsAnalyzing] = useState(false);
const audioContextRef = useRef<AudioContext | null>(null);
const analyserRef = useRef<AnalyserNode | null>(null);
const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
const intervalRef = useRef<number | null>(null);
const lastUpdateTimeRef = useRef<number>(0);
useEffect(() => {
if (!stream || !enabled) {
// Clean up when stream is null or disabled
if (intervalRef.current !== null) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
if (sourceRef.current) {
sourceRef.current.disconnect();
sourceRef.current = null;
}
if (audioContextRef.current) {
audioContextRef.current.close();
audioContextRef.current = null;
}
analyserRef.current = null;
setIsAnalyzing(false);
setAudioLevel(0);
return;
}
const audioTracks = stream.getAudioTracks();
if (audioTracks.length === 0) {
setIsAnalyzing(false);
setAudioLevel(0);
return;
}
try {
// Create audio context and analyser
const audioContext = new (window.AudioContext || (window as Window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();
const analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(stream);
// Configure analyser - use smaller FFT for better performance
analyser.fftSize = 128; // Reduced from 256 for better performance
analyser.smoothingTimeConstant = 0.8;
// Connect nodes
source.connect(analyser);
// Store references
audioContextRef.current = audioContext;
analyserRef.current = analyser;
sourceRef.current = source;
const dataArray = new Uint8Array(analyser.frequencyBinCount);
const updateLevel = () => {
if (!analyserRef.current) return;
const now = performance.now();
// Throttle updates to reduce CPU usage
if (now - lastUpdateTimeRef.current < updateInterval) {
return;
}
lastUpdateTimeRef.current = now;
analyserRef.current.getByteFrequencyData(dataArray);
// Optimized RMS calculation - process only relevant frequency bands
let sum = 0;
const relevantBins = Math.min(dataArray.length, 32); // Focus on lower frequencies for voice
for (let i = 0; i < relevantBins; i++) {
const value = dataArray[i];
sum += value * value;
}
const rms = Math.sqrt(sum / relevantBins);
// Convert to percentage (0-100) with better scaling
const level = Math.min(100, Math.max(0, (rms / 180) * 100)); // Adjusted scaling for better sensitivity
setAudioLevel(Math.round(level));
};
setIsAnalyzing(true);
// Use setInterval instead of requestAnimationFrame for more predictable timing
intervalRef.current = window.setInterval(updateLevel, updateInterval);
} catch (error) {
console.error('Failed to create audio level analyzer:', error);
setIsAnalyzing(false);
setAudioLevel(0);
}
// Cleanup function
return () => {
if (intervalRef.current !== null) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
if (sourceRef.current) {
sourceRef.current.disconnect();
sourceRef.current = null;
}
if (audioContextRef.current) {
audioContextRef.current.close();
audioContextRef.current = null;
}
analyserRef.current = null;
setIsAnalyzing(false);
setAudioLevel(0);
};
}, [stream, enabled, updateInterval]);
return { audioLevel, isAnalyzing };
};

View File

@ -0,0 +1,964 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useRTCStore } from "@/hooks/stores";
import api from "@/api";
export interface MicrophoneError {
type: 'permission' | 'device' | 'network' | 'unknown';
message: string;
}
export function useMicrophone() {
const {
peerConnection,
microphoneStream,
setMicrophoneStream,
microphoneSender,
setMicrophoneSender,
isMicrophoneActive,
setMicrophoneActive,
isMicrophoneMuted,
setMicrophoneMuted,
} = useRTCStore();
const microphoneStreamRef = useRef<MediaStream | null>(null);
// Loading states
const [isStarting, setIsStarting] = useState(false);
const [isStopping, setIsStopping] = useState(false);
const [isToggling, setIsToggling] = useState(false);
// Add debouncing refs to prevent rapid operations
const lastOperationRef = useRef<number>(0);
const operationTimeoutRef = useRef<number | null>(null);
const OPERATION_DEBOUNCE_MS = 1000; // 1 second debounce
// Debounced operation wrapper
const debouncedOperation = useCallback((operation: () => Promise<void>, operationType: string) => {
const now = Date.now();
const timeSinceLastOp = now - lastOperationRef.current;
if (timeSinceLastOp < OPERATION_DEBOUNCE_MS) {
console.log(`Debouncing ${operationType} operation - too soon (${timeSinceLastOp}ms since last)`);
return;
}
// Clear any pending operation
if (operationTimeoutRef.current) {
clearTimeout(operationTimeoutRef.current);
operationTimeoutRef.current = null;
}
lastOperationRef.current = now;
operation().catch(error => {
console.error(`Debounced ${operationType} operation failed:`, error);
});
}, []);
// Cleanup function to stop microphone stream
const stopMicrophoneStream = useCallback(async () => {
console.log("stopMicrophoneStream called - cleaning up stream");
console.trace("stopMicrophoneStream call stack");
if (microphoneStreamRef.current) {
console.log("Stopping microphone stream:", microphoneStreamRef.current.id);
microphoneStreamRef.current.getTracks().forEach(track => {
track.stop();
});
microphoneStreamRef.current = null;
setMicrophoneStream(null);
console.log("Microphone stream cleared from ref and store");
} else {
console.log("No microphone stream to stop");
}
if (microphoneSender && peerConnection) {
// Instead of removing the track, replace it with null to keep the transceiver
try {
await microphoneSender.replaceTrack(null);
} catch (error) {
console.warn("Failed to replace track with null:", error);
// Fallback to removing the track
peerConnection.removeTrack(microphoneSender);
}
setMicrophoneSender(null);
}
setMicrophoneActive(false);
setMicrophoneMuted(false);
}, [microphoneSender, peerConnection, setMicrophoneStream, setMicrophoneSender, setMicrophoneActive, setMicrophoneMuted]);
// Debug function to check current state (can be called from browser console)
const debugMicrophoneState = useCallback(() => {
const refStream = microphoneStreamRef.current;
const state = {
isMicrophoneActive,
isMicrophoneMuted,
streamInRef: !!refStream,
streamInStore: !!microphoneStream,
senderInStore: !!microphoneSender,
streamId: refStream?.id,
storeStreamId: microphoneStream?.id,
audioTracks: refStream?.getAudioTracks().length || 0,
storeAudioTracks: microphoneStream?.getAudioTracks().length || 0,
audioTrackDetails: refStream?.getAudioTracks().map(track => ({
id: track.id,
label: track.label,
enabled: track.enabled,
readyState: track.readyState,
muted: track.muted
})) || [],
peerConnectionState: peerConnection ? {
connectionState: peerConnection.connectionState,
iceConnectionState: peerConnection.iceConnectionState,
signalingState: peerConnection.signalingState
} : "No peer connection",
streamMatch: refStream === microphoneStream
};
console.log("Microphone Debug State:", state);
// Also check if streams are active
if (refStream) {
console.log("Ref stream active tracks:", refStream.getAudioTracks().filter(t => t.readyState === 'live').length);
}
if (microphoneStream && microphoneStream !== refStream) {
console.log("Store stream active tracks:", microphoneStream.getAudioTracks().filter(t => t.readyState === 'live').length);
}
return state;
}, [isMicrophoneActive, isMicrophoneMuted, microphoneStream, microphoneSender, peerConnection]);
// Make debug function available globally for console access
useEffect(() => {
(window as Window & { debugMicrophoneState?: () => unknown }).debugMicrophoneState = debugMicrophoneState;
return () => {
delete (window as Window & { debugMicrophoneState?: () => unknown }).debugMicrophoneState;
};
}, [debugMicrophoneState]);
const lastSyncRef = useRef<number>(0);
const isStartingRef = useRef<boolean>(false); // Track if we're in the middle of starting
const syncMicrophoneState = useCallback(async () => {
// Debounce sync calls to prevent race conditions
const now = Date.now();
if (now - lastSyncRef.current < 1000) { // Increased debounce time
console.log("Skipping sync - too frequent");
return;
}
lastSyncRef.current = now;
// Don't sync if we're in the middle of starting the microphone
if (isStartingRef.current) {
console.log("Skipping sync - microphone is starting");
return;
}
try {
const response = await api.GET("/microphone/status", {});
if (response.ok) {
const data = await response.json();
const backendRunning = data.running;
// Only sync if there's a significant state difference and we're not in a transition
if (backendRunning !== isMicrophoneActive) {
console.info(`Syncing microphone state: backend=${backendRunning}, frontend=${isMicrophoneActive}`);
// If backend is running but frontend thinks it's not, just update frontend state
if (backendRunning && !isMicrophoneActive) {
console.log("Backend running, updating frontend state to active");
setMicrophoneActive(true);
}
// If backend is not running but frontend thinks it is, clean up and update state
else if (!backendRunning && isMicrophoneActive) {
console.log("Backend not running, cleaning up frontend state");
setMicrophoneActive(false);
// Only clean up stream if we actually have one
if (microphoneStreamRef.current) {
console.log("Cleaning up orphaned stream");
await stopMicrophoneStream();
}
}
}
}
} catch (error) {
console.warn("Failed to sync microphone state:", error);
}
}, [isMicrophoneActive, setMicrophoneActive, stopMicrophoneStream]);
// Start microphone stream
const startMicrophone = useCallback(async (deviceId?: string): Promise<{ success: boolean; error?: MicrophoneError }> => {
// Prevent multiple simultaneous start operations
if (isStarting || isStopping || isToggling) {
console.log("Microphone operation already in progress, skipping start");
return { success: false, error: { type: 'unknown', message: 'Operation already in progress' } };
}
setIsStarting(true);
try {
// Set flag to prevent sync during startup
isStartingRef.current = true;
// Request microphone permission and get stream
const audioConstraints: MediaTrackConstraints = {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
sampleRate: 48000,
channelCount: 1,
};
// Add device ID if specified
if (deviceId && deviceId !== 'default') {
audioConstraints.deviceId = { exact: deviceId };
}
console.log("Requesting microphone with constraints:", audioConstraints);
const stream = await navigator.mediaDevices.getUserMedia({
audio: audioConstraints
});
console.log("Microphone stream created successfully:", {
streamId: stream.id,
audioTracks: stream.getAudioTracks().length,
videoTracks: stream.getVideoTracks().length,
audioTrackDetails: stream.getAudioTracks().map(track => ({
id: track.id,
label: track.label,
enabled: track.enabled,
readyState: track.readyState
}))
});
// Store the stream in both ref and store
microphoneStreamRef.current = stream;
setMicrophoneStream(stream);
// Verify the stream was stored correctly
console.log("Stream storage verification:", {
refSet: !!microphoneStreamRef.current,
refId: microphoneStreamRef.current?.id,
storeWillBeSet: true // Store update is async
});
// Add audio track to peer connection if available
console.log("Peer connection state:", peerConnection ? {
connectionState: peerConnection.connectionState,
iceConnectionState: peerConnection.iceConnectionState,
signalingState: peerConnection.signalingState
} : "No peer connection");
if (peerConnection && stream.getAudioTracks().length > 0) {
const audioTrack = stream.getAudioTracks()[0];
console.log("Starting microphone with audio track:", audioTrack.id, "kind:", audioTrack.kind);
// Find the audio transceiver (should already exist with sendrecv direction)
const transceivers = peerConnection.getTransceivers();
console.log("Available transceivers:", transceivers.map(t => ({
direction: t.direction,
mid: t.mid,
senderTrack: t.sender.track?.kind,
receiverTrack: t.receiver.track?.kind
})));
// Look for an audio transceiver that can send (has sendrecv or sendonly direction)
const audioTransceiver = transceivers.find(transceiver => {
// Check if this transceiver is for audio and can send
const canSend = transceiver.direction === 'sendrecv' || transceiver.direction === 'sendonly';
// For newly created transceivers, we need to check if they're for audio
// We can do this by checking if the sender doesn't have a track yet and direction allows sending
if (canSend && !transceiver.sender.track) {
return true;
}
// For existing transceivers, check if they already have an audio track
if (transceiver.sender.track?.kind === 'audio' || transceiver.receiver.track?.kind === 'audio') {
return canSend;
}
return false;
});
console.log("Found audio transceiver:", audioTransceiver ? {
direction: audioTransceiver.direction,
mid: audioTransceiver.mid,
senderTrack: audioTransceiver.sender.track?.kind,
receiverTrack: audioTransceiver.receiver.track?.kind
} : null);
let sender: RTCRtpSender;
if (audioTransceiver && audioTransceiver.sender) {
// Use the existing audio transceiver's sender
await audioTransceiver.sender.replaceTrack(audioTrack);
sender = audioTransceiver.sender;
console.log("Replaced audio track on existing transceiver");
// Verify the track was set correctly
console.log("Transceiver after track replacement:", {
direction: audioTransceiver.direction,
senderTrack: audioTransceiver.sender.track?.id,
senderTrackKind: audioTransceiver.sender.track?.kind,
senderTrackEnabled: audioTransceiver.sender.track?.enabled,
senderTrackReadyState: audioTransceiver.sender.track?.readyState
});
} else {
// Fallback: add new track if no transceiver found
sender = peerConnection.addTrack(audioTrack, stream);
console.log("Added new audio track to peer connection");
// Find the transceiver that was created for this track
const newTransceiver = peerConnection.getTransceivers().find(t => t.sender === sender);
console.log("New transceiver created:", newTransceiver ? {
direction: newTransceiver.direction,
senderTrack: newTransceiver.sender.track?.id,
senderTrackKind: newTransceiver.sender.track?.kind
} : "Not found");
}
setMicrophoneSender(sender);
console.log("Microphone sender set:", {
senderId: sender,
track: sender.track?.id,
trackKind: sender.track?.kind,
trackEnabled: sender.track?.enabled,
trackReadyState: sender.track?.readyState
});
// Check sender stats to verify audio is being transmitted
setTimeout(async () => {
try {
const stats = await sender.getStats();
console.log("Sender stats after 2 seconds:");
stats.forEach((report, id) => {
if (report.type === 'outbound-rtp' && report.kind === 'audio') {
console.log("Outbound audio RTP stats:", {
id,
packetsSent: report.packetsSent,
bytesSent: report.bytesSent,
timestamp: report.timestamp
});
}
});
} catch (error) {
console.error("Failed to get sender stats:", error);
}
}, 2000);
}
// Notify backend that microphone is started
console.log("Notifying backend about microphone start...");
// Retry logic for backend failures
let backendSuccess = false;
let lastError: Error | string | null = null;
for (let attempt = 1; attempt <= 3; attempt++) {
try {
// If this is a retry, first try to reset the backend microphone state
if (attempt > 1) {
console.log(`Backend start attempt ${attempt}, first trying to reset backend state...`);
try {
// Try the new reset endpoint first
const resetResp = await api.POST("/microphone/reset", {});
if (resetResp.ok) {
console.log("Backend reset successful");
} else {
// Fallback to stop
await api.POST("/microphone/stop", {});
}
// Wait a bit for the backend to reset
await new Promise(resolve => setTimeout(resolve, 200));
} catch (resetError) {
console.warn("Failed to reset backend state:", resetError);
}
}
const backendResp = await api.POST("/microphone/start", {});
console.log(`Backend response status (attempt ${attempt}):`, backendResp.status, "ok:", backendResp.ok);
if (!backendResp.ok) {
lastError = `Backend returned status ${backendResp.status}`;
console.error(`Backend microphone start failed with status: ${backendResp.status} (attempt ${attempt})`);
// For 500 errors, try again after a short delay
if (backendResp.status === 500 && attempt < 3) {
console.log(`Retrying backend start in 500ms (attempt ${attempt + 1}/3)...`);
await new Promise(resolve => setTimeout(resolve, 500));
continue;
}
} else {
// Success!
const responseData = await backendResp.json();
console.log("Backend response data:", responseData);
if (responseData.status === "already running") {
console.info("Backend microphone was already running");
// If we're on the first attempt and backend says "already running",
// but frontend thinks it's not active, this might be a stuck state
if (attempt === 1 && !isMicrophoneActive) {
console.warn("Backend reports 'already running' but frontend is not active - possible stuck state");
console.log("Attempting to reset backend state and retry...");
try {
const resetResp = await api.POST("/microphone/reset", {});
if (resetResp.ok) {
console.log("Backend reset successful, retrying start...");
await new Promise(resolve => setTimeout(resolve, 200));
continue; // Retry the start
}
} catch (resetError) {
console.warn("Failed to reset stuck backend state:", resetError);
}
}
}
console.log("Backend microphone start successful");
backendSuccess = true;
break;
}
} catch (error) {
lastError = error instanceof Error ? error : String(error);
console.error(`Backend microphone start threw error (attempt ${attempt}):`, error);
// For network errors, try again after a short delay
if (attempt < 3) {
console.log(`Retrying backend start in 500ms (attempt ${attempt + 1}/3)...`);
await new Promise(resolve => setTimeout(resolve, 500));
continue;
}
}
}
// If all backend attempts failed, cleanup and return error
if (!backendSuccess) {
console.error("All backend start attempts failed, cleaning up stream");
await stopMicrophoneStream();
isStartingRef.current = false;
setIsStarting(false);
return {
success: false,
error: {
type: 'network',
message: `Failed to start microphone on backend after 3 attempts. Last error: ${lastError}`
}
};
}
// Only set active state after backend confirms success
setMicrophoneActive(true);
setMicrophoneMuted(false);
console.log("Microphone state set to active. Verifying state:", {
streamInRef: !!microphoneStreamRef.current,
streamInStore: !!microphoneStream,
isActive: true,
isMuted: false
});
// Don't sync immediately after starting - it causes race conditions
// The sync will happen naturally through other triggers
setTimeout(() => {
// Just verify state after a delay for debugging
console.log("State check after delay:", {
streamInRef: !!microphoneStreamRef.current,
streamInStore: !!microphoneStream,
isActive: isMicrophoneActive,
isMuted: isMicrophoneMuted
});
}, 100);
// Clear the starting flag
isStartingRef.current = false;
setIsStarting(false);
return { success: true };
} catch (error) {
console.error("Failed to start microphone:", error);
let micError: MicrophoneError;
if (error instanceof Error) {
if (error.name === 'NotAllowedError' || error.name === 'PermissionDeniedError') {
micError = {
type: 'permission',
message: 'Microphone permission denied. Please allow microphone access and try again.'
};
} else if (error.name === 'NotFoundError' || error.name === 'DevicesNotFoundError') {
micError = {
type: 'device',
message: 'No microphone device found. Please check your microphone connection.'
};
} else {
micError = {
type: 'unknown',
message: error.message || 'Failed to access microphone'
};
}
} else {
micError = {
type: 'unknown',
message: 'Unknown error occurred while accessing microphone'
};
}
// Clear the starting flag on error
isStartingRef.current = false;
setIsStarting(false);
return { success: false, error: micError };
}
}, [peerConnection, setMicrophoneStream, setMicrophoneSender, setMicrophoneActive, setMicrophoneMuted, stopMicrophoneStream, isMicrophoneActive, isMicrophoneMuted, microphoneStream, isStarting, isStopping, isToggling]);
// Reset backend microphone state
const resetBackendMicrophoneState = useCallback(async (): Promise<boolean> => {
try {
console.log("Resetting backend microphone state...");
const response = await api.POST("/microphone/reset", {});
if (response.ok) {
const data = await response.json();
console.log("Backend microphone reset successful:", data);
// Update frontend state to match backend
setMicrophoneActive(false);
setMicrophoneMuted(false);
// Clean up any orphaned streams
if (microphoneStreamRef.current) {
console.log("Cleaning up orphaned stream after reset");
await stopMicrophoneStream();
}
// Wait a bit for everything to settle
await new Promise(resolve => setTimeout(resolve, 200));
// Sync state to ensure consistency
await syncMicrophoneState();
return true;
} else {
console.error("Backend microphone reset failed:", response.status);
return false;
}
} catch (error) {
console.warn("Failed to reset backend microphone state:", error);
// Fallback to old method
try {
console.log("Trying fallback reset method...");
await api.POST("/microphone/stop", {});
await new Promise(resolve => setTimeout(resolve, 300));
return true;
} catch (fallbackError) {
console.error("Fallback reset also failed:", fallbackError);
return false;
}
}
}, [setMicrophoneActive, setMicrophoneMuted, stopMicrophoneStream, syncMicrophoneState]);
// Stop microphone
const stopMicrophone = useCallback(async (): Promise<{ success: boolean; error?: MicrophoneError }> => {
// Prevent multiple simultaneous stop operations
if (isStarting || isStopping || isToggling) {
console.log("Microphone operation already in progress, skipping stop");
return { success: false, error: { type: 'unknown', message: 'Operation already in progress' } };
}
setIsStopping(true);
try {
// First stop the stream
await stopMicrophoneStream();
// Then notify backend that microphone is stopped
try {
await api.POST("/microphone/stop", {});
console.log("Backend notified about microphone stop");
} catch (error) {
console.warn("Failed to notify backend about microphone stop:", error);
}
// Update frontend state immediately
setMicrophoneActive(false);
setMicrophoneMuted(false);
// Sync state after stopping to ensure consistency (with longer delay)
setTimeout(() => syncMicrophoneState(), 500);
setIsStopping(false);
return { success: true };
} catch (error) {
console.error("Failed to stop microphone:", error);
setIsStopping(false);
return {
success: false,
error: {
type: 'unknown',
message: error instanceof Error ? error.message : 'Failed to stop microphone'
}
};
}
}, [stopMicrophoneStream, syncMicrophoneState, setMicrophoneActive, setMicrophoneMuted, isStarting, isStopping, isToggling]);
// Toggle microphone mute
const toggleMicrophoneMute = useCallback(async (): Promise<{ success: boolean; error?: MicrophoneError }> => {
// Prevent multiple simultaneous toggle operations
if (isStarting || isStopping || isToggling) {
console.log("Microphone operation already in progress, skipping toggle");
return { success: false, error: { type: 'unknown', message: 'Operation already in progress' } };
}
setIsToggling(true);
try {
// Use the ref instead of store value to avoid race conditions
const currentStream = microphoneStreamRef.current || microphoneStream;
console.log("Toggle microphone mute - current state:", {
hasRefStream: !!microphoneStreamRef.current,
hasStoreStream: !!microphoneStream,
isActive: isMicrophoneActive,
isMuted: isMicrophoneMuted,
streamId: currentStream?.id,
audioTracks: currentStream?.getAudioTracks().length || 0
});
if (!currentStream || !isMicrophoneActive) {
const errorDetails = {
hasStream: !!currentStream,
isActive: isMicrophoneActive,
storeStream: !!microphoneStream,
refStream: !!microphoneStreamRef.current,
streamId: currentStream?.id,
audioTracks: currentStream?.getAudioTracks().length || 0
};
console.warn("Microphone mute failed: stream or active state missing", errorDetails);
// Provide more specific error message
let errorMessage = 'Microphone is not active';
if (!currentStream) {
errorMessage = 'No microphone stream found. Please restart the microphone.';
} else if (!isMicrophoneActive) {
errorMessage = 'Microphone is not marked as active. Please restart the microphone.';
}
setIsToggling(false);
return {
success: false,
error: {
type: 'device',
message: errorMessage
}
};
}
const audioTracks = currentStream.getAudioTracks();
if (audioTracks.length === 0) {
setIsToggling(false);
return {
success: false,
error: {
type: 'device',
message: 'No audio tracks found in microphone stream'
}
};
}
const newMutedState = !isMicrophoneMuted;
// Mute/unmute the audio track
audioTracks.forEach(track => {
track.enabled = !newMutedState;
console.log(`Audio track ${track.id} enabled: ${track.enabled}`);
});
setMicrophoneMuted(newMutedState);
// Notify backend about mute state
try {
await api.POST("/microphone/mute", { muted: newMutedState });
} catch (error) {
console.warn("Failed to notify backend about microphone mute:", error);
}
setIsToggling(false);
return { success: true };
} catch (error) {
console.error("Failed to toggle microphone mute:", error);
setIsToggling(false);
return {
success: false,
error: {
type: 'unknown',
message: error instanceof Error ? error.message : 'Failed to toggle microphone mute'
}
};
}
}, [microphoneStream, isMicrophoneActive, isMicrophoneMuted, setMicrophoneMuted, isStarting, isStopping, isToggling]);
// Function to check WebRTC audio transmission stats
const checkAudioTransmissionStats = useCallback(async () => {
if (!microphoneSender) {
console.log("No microphone sender available");
return null;
}
try {
const stats = await microphoneSender.getStats();
const audioStats: {
id: string;
type: string;
kind: string;
packetsSent?: number;
bytesSent?: number;
timestamp?: number;
ssrc?: number;
}[] = [];
stats.forEach((report, id) => {
if (report.type === 'outbound-rtp' && report.kind === 'audio') {
audioStats.push({
id,
type: report.type,
kind: report.kind,
packetsSent: report.packetsSent,
bytesSent: report.bytesSent,
timestamp: report.timestamp,
ssrc: report.ssrc
});
}
});
console.log("Audio transmission stats:", audioStats);
return audioStats;
} catch (error) {
console.error("Failed to get audio transmission stats:", error);
return null;
}
}, [microphoneSender]);
// Comprehensive test function to diagnose microphone issues
const testMicrophoneAudio = useCallback(async () => {
console.log("=== MICROPHONE AUDIO TEST ===");
// 1. Check if we have a stream
const stream = microphoneStreamRef.current;
if (!stream) {
console.log("❌ No microphone stream available");
return;
}
console.log("✅ Microphone stream exists:", stream.id);
// 2. Check audio tracks
const audioTracks = stream.getAudioTracks();
console.log("Audio tracks:", audioTracks.length);
if (audioTracks.length === 0) {
console.log("❌ No audio tracks in stream");
return;
}
const track = audioTracks[0];
console.log("✅ Audio track details:", {
id: track.id,
label: track.label,
enabled: track.enabled,
readyState: track.readyState,
muted: track.muted
});
// 3. Test audio level detection manually
try {
const audioContext = new (window.AudioContext || (window as Window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();
const analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(stream);
analyser.fftSize = 256;
source.connect(analyser);
const dataArray = new Uint8Array(analyser.frequencyBinCount);
console.log("🎤 Testing audio level detection for 5 seconds...");
console.log("Please speak into your microphone now!");
let maxLevel = 0;
let sampleCount = 0;
const testInterval = setInterval(() => {
analyser.getByteFrequencyData(dataArray);
let sum = 0;
for (const value of dataArray) {
sum += value * value;
}
const rms = Math.sqrt(sum / dataArray.length);
const level = Math.min(100, (rms / 255) * 100);
maxLevel = Math.max(maxLevel, level);
sampleCount++;
if (sampleCount % 10 === 0) { // Log every 10th sample
console.log(`Audio level: ${level.toFixed(1)}% (max so far: ${maxLevel.toFixed(1)}%)`);
}
}, 100);
setTimeout(() => {
clearInterval(testInterval);
source.disconnect();
audioContext.close();
console.log("🎤 Audio test completed!");
console.log(`Maximum audio level detected: ${maxLevel.toFixed(1)}%`);
if (maxLevel > 5) {
console.log("✅ Microphone is detecting audio!");
} else {
console.log("❌ No significant audio detected. Check microphone permissions and hardware.");
}
}, 5000);
} catch (error) {
console.error("❌ Failed to test audio level:", error);
}
// 4. Check WebRTC sender
if (microphoneSender) {
console.log("✅ WebRTC sender exists");
console.log("Sender track:", {
id: microphoneSender.track?.id,
kind: microphoneSender.track?.kind,
enabled: microphoneSender.track?.enabled,
readyState: microphoneSender.track?.readyState
});
// Check if sender track matches stream track
if (microphoneSender.track === track) {
console.log("✅ Sender track matches stream track");
} else {
console.log("❌ Sender track does NOT match stream track");
}
} else {
console.log("❌ No WebRTC sender available");
}
// 5. Check peer connection
if (peerConnection) {
console.log("✅ Peer connection exists");
console.log("Connection state:", peerConnection.connectionState);
console.log("ICE connection state:", peerConnection.iceConnectionState);
const transceivers = peerConnection.getTransceivers();
const audioTransceivers = transceivers.filter(t =>
t.sender.track?.kind === 'audio' || t.receiver.track?.kind === 'audio'
);
console.log("Audio transceivers:", audioTransceivers.map(t => ({
direction: t.direction,
senderTrack: t.sender.track?.id,
receiverTrack: t.receiver.track?.id
})));
} else {
console.log("❌ No peer connection available");
}
}, [microphoneSender, peerConnection]);
const startMicrophoneDebounced = useCallback((deviceId?: string) => {
debouncedOperation(async () => {
await startMicrophone(deviceId).catch(console.error);
}, "start");
}, [startMicrophone, debouncedOperation]);
const stopMicrophoneDebounced = useCallback(() => {
debouncedOperation(async () => {
await stopMicrophone().catch(console.error);
}, "stop");
}, [stopMicrophone, debouncedOperation]);
// Make debug functions available globally for console access
useEffect(() => {
(window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).debugMicrophone = debugMicrophoneState;
(window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).checkAudioStats = checkAudioTransmissionStats;
(window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).testMicrophoneAudio = testMicrophoneAudio;
(window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).resetBackendMicrophone = resetBackendMicrophoneState;
return () => {
delete (window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).debugMicrophone;
delete (window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).checkAudioStats;
delete (window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).testMicrophoneAudio;
delete (window as Window & {
debugMicrophone?: () => unknown;
checkAudioStats?: () => unknown;
testMicrophoneAudio?: () => unknown;
resetBackendMicrophone?: () => unknown;
}).resetBackendMicrophone;
};
}, [debugMicrophoneState, checkAudioTransmissionStats, testMicrophoneAudio, resetBackendMicrophoneState]);
// Sync state on mount
useEffect(() => {
syncMicrophoneState();
}, [syncMicrophoneState]);
// Cleanup on unmount - use ref to avoid dependency on stopMicrophoneStream
useEffect(() => {
return () => {
// Clean up stream directly without depending on the callback
const stream = microphoneStreamRef.current;
if (stream) {
console.log("Cleanup: stopping microphone stream on unmount");
stream.getAudioTracks().forEach(track => {
track.stop();
console.log(`Cleanup: stopped audio track ${track.id}`);
});
microphoneStreamRef.current = null;
}
};
}, []); // No dependencies to prevent re-running
return {
isMicrophoneActive,
isMicrophoneMuted,
microphoneStream,
startMicrophone,
stopMicrophone,
toggleMicrophoneMute,
debugMicrophoneState,
// Expose debounced variants for UI handlers
startMicrophoneDebounced,
stopMicrophoneDebounced,
// Expose sync and loading flags for consumers that expect them
syncMicrophoneState,
isStarting,
isStopping,
isToggling,
};
}

View File

@ -33,10 +33,12 @@ import {
useVideoStore,
VideoState,
} from "@/hooks/stores";
import { useMicrophone } from "@/hooks/useMicrophone";
import WebRTCVideo from "@components/WebRTCVideo";
import { checkAuth, isInCloud, isOnDevice } from "@/main";
import DashboardNavbar from "@components/Header";
import ConnectionStatsSidebar from "@/components/sidebar/connectionStats";
import AudioMetricsSidebar from "@/components/sidebar/AudioMetricsSidebar";
import { JsonRpcRequest, JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import Terminal from "@components/Terminal";
import { CLOUD_API, DEVICE_API } from "@/ui.config";
@ -141,6 +143,9 @@ export default function KvmIdRoute() {
const setTransceiver = useRTCStore(state => state.setTransceiver);
const location = useLocation();
// Microphone hook - moved here to prevent unmounting when popover closes
const microphoneHook = useMicrophone();
const isLegacySignalingEnabled = useRef(false);
const [connectionFailed, setConnectionFailed] = useState(false);
@ -479,6 +484,8 @@ export default function KvmIdRoute() {
};
setTransceiver(pc.addTransceiver("video", { direction: "recvonly" }));
// Add audio transceiver to receive audio from the server and send microphone audio
pc.addTransceiver("audio", { direction: "sendrecv" });
const rpcDataChannel = pc.createDataChannel("rpc");
rpcDataChannel.onopen = () => {
@ -828,7 +835,7 @@ export default function KvmIdRoute() {
/>
<div className="relative flex h-full w-full overflow-hidden">
<WebRTCVideo />
<WebRTCVideo microphone={microphoneHook} />
<div
style={{ animationDuration: "500ms" }}
className="animate-slideUpFade pointer-events-none absolute inset-0 flex items-center justify-center p-4"
@ -900,6 +907,22 @@ function SidebarContainer(props: SidebarContainerProps) {
<ConnectionStatsSidebar />
</motion.div>
)}
{sidebarView === "audio-metrics" && (
<motion.div
className="absolute inset-0"
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
exit={{ opacity: 0 }}
transition={{
duration: 0.5,
ease: "easeInOut",
}}
>
<div className="grid h-full grid-rows-(--grid-headerBody) shadow-xs">
<AudioMetricsSidebar />
</div>
</motion.div>
)}
</AnimatePresence>
</div>
</div>

View File

@ -17,11 +17,7 @@ export default defineConfig(({ mode, command }) => {
const { JETKVM_PROXY_URL, USE_SSL } = process.env;
const useSSL = USE_SSL === "true";
const plugins = [
tailwindcss(),
tsconfigPaths(),
react()
];
const plugins = [tailwindcss(), tsconfigPaths(), react()];
if (useSSL) {
plugins.push(basicSsl());
}
@ -41,6 +37,8 @@ export default defineConfig(({ mode, command }) => {
"/storage": JETKVM_PROXY_URL,
"/cloud": JETKVM_PROXY_URL,
"/developer": JETKVM_PROXY_URL,
"/microphone": JETKVM_PROXY_URL,
"/audio": JETKVM_PROXY_URL,
}
: undefined,
},

View File

@ -5,7 +5,7 @@ import (
)
// max frame size for 1080p video, specified in mpp venc setting
const maxFrameSize = 1920 * 1080 / 2
const maxVideoFrameSize = 1920 * 1080 / 2
func writeCtrlAction(action string) error {
actionMessage := map[string]string{

453
web.go
View File

@ -14,8 +14,11 @@ import (
"strings"
"time"
"github.com/jetkvm/kvm/internal/audio"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
gin_logger "github.com/gin-contrib/logger"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
@ -156,6 +159,392 @@ func setupRouter() *gin.Engine {
protected.POST("/storage/upload", handleUploadHttp)
}
protected.POST("/audio/mute", func(c *gin.Context) {
type muteReq struct {
Muted bool `json:"muted"`
}
var req muteReq
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"error": "invalid request"})
return
}
audio.SetAudioMuted(req.Muted)
// Also set relay mute state if in main process
audio.SetAudioRelayMuted(req.Muted)
// Broadcast audio mute state change via WebSocket
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.BroadcastAudioMuteChanged(req.Muted)
c.JSON(200, gin.H{"muted": req.Muted})
})
protected.GET("/audio/quality", func(c *gin.Context) {
config := audio.GetAudioConfig()
presets := audio.GetAudioQualityPresets()
c.JSON(200, gin.H{
"current": config,
"presets": presets,
})
})
protected.POST("/audio/quality", func(c *gin.Context) {
type qualityReq struct {
Quality int `json:"quality"`
}
var req qualityReq
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"error": "invalid request"})
return
}
// Validate quality level
if req.Quality < 0 || req.Quality > 3 {
c.JSON(400, gin.H{"error": "invalid quality level (0-3)"})
return
}
audio.SetAudioQuality(audio.AudioQuality(req.Quality))
c.JSON(200, gin.H{
"quality": req.Quality,
"config": audio.GetAudioConfig(),
})
})
protected.GET("/audio/metrics", func(c *gin.Context) {
metrics := audio.GetAudioMetrics()
c.JSON(200, gin.H{
"frames_received": metrics.FramesReceived,
"frames_dropped": metrics.FramesDropped,
"bytes_processed": metrics.BytesProcessed,
"last_frame_time": metrics.LastFrameTime,
"connection_drops": metrics.ConnectionDrops,
"average_latency": fmt.Sprintf("%.1fms", float64(metrics.AverageLatency.Nanoseconds())/1e6),
})
})
protected.GET("/microphone/quality", func(c *gin.Context) {
config := audio.GetMicrophoneConfig()
presets := audio.GetMicrophoneQualityPresets()
c.JSON(200, gin.H{
"current": config,
"presets": presets,
})
})
protected.POST("/microphone/quality", func(c *gin.Context) {
type qualityReq struct {
Quality int `json:"quality"`
}
var req qualityReq
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"error": "invalid request"})
return
}
// Validate quality level
if req.Quality < 0 || req.Quality > 3 {
c.JSON(400, gin.H{"error": "invalid quality level (0-3)"})
return
}
audio.SetMicrophoneQuality(audio.AudioQuality(req.Quality))
c.JSON(200, gin.H{
"quality": req.Quality,
"config": audio.GetMicrophoneConfig(),
})
})
// Microphone API endpoints
protected.GET("/microphone/status", func(c *gin.Context) {
sessionActive := currentSession != nil
var running bool
if sessionActive && currentSession.AudioInputManager != nil {
running = currentSession.AudioInputManager.IsRunning()
}
c.JSON(200, gin.H{
"running": running,
"session_active": sessionActive,
})
})
protected.POST("/microphone/start", func(c *gin.Context) {
if currentSession == nil {
c.JSON(400, gin.H{"error": "no active session"})
return
}
if currentSession.AudioInputManager == nil {
c.JSON(500, gin.H{"error": "audio input manager not available"})
return
}
// Optimized server-side cooldown using atomic operations
opResult := audio.TryMicrophoneOperation()
if !opResult.Allowed {
running := currentSession.AudioInputManager.IsRunning()
c.JSON(200, gin.H{
"status": "cooldown",
"running": running,
"cooldown_ms_remaining": opResult.RemainingCooldown.Milliseconds(),
"operation_id": opResult.OperationID,
})
return
}
// Check if already running before attempting to start
if currentSession.AudioInputManager.IsRunning() {
c.JSON(200, gin.H{
"status": "already running",
"running": true,
})
return
}
err := currentSession.AudioInputManager.Start()
if err != nil {
// Log the error for debugging but don't expose internal details
logger.Warn().Err(err).Msg("failed to start microphone")
// Check if it's already running after the failed start attempt
// This handles race conditions where another request started it
if currentSession.AudioInputManager.IsRunning() {
c.JSON(200, gin.H{
"status": "started by concurrent request",
"running": true,
})
return
}
c.JSON(500, gin.H{"error": "failed to start microphone"})
return
}
// Broadcast microphone state change via WebSocket
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.BroadcastMicrophoneStateChanged(true, true)
c.JSON(200, gin.H{
"status": "started",
"running": currentSession.AudioInputManager.IsRunning(),
})
})
protected.POST("/microphone/stop", func(c *gin.Context) {
if currentSession == nil {
c.JSON(400, gin.H{"error": "no active session"})
return
}
if currentSession.AudioInputManager == nil {
c.JSON(500, gin.H{"error": "audio input manager not available"})
return
}
// Optimized server-side cooldown using atomic operations
opResult := audio.TryMicrophoneOperation()
if !opResult.Allowed {
running := currentSession.AudioInputManager.IsRunning()
c.JSON(200, gin.H{
"status": "cooldown",
"running": running,
"cooldown_ms_remaining": opResult.RemainingCooldown.Milliseconds(),
"operation_id": opResult.OperationID,
})
return
}
// Check if already stopped before attempting to stop
if !currentSession.AudioInputManager.IsRunning() {
c.JSON(200, gin.H{
"status": "already stopped",
"running": false,
})
return
}
currentSession.AudioInputManager.Stop()
// AudioInputManager.Stop() already coordinates a clean stop via IPC audio input system
// so we don't need to call it again here
// Broadcast microphone state change via WebSocket
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.BroadcastMicrophoneStateChanged(false, true)
c.JSON(200, gin.H{
"status": "stopped",
"running": currentSession.AudioInputManager.IsRunning(),
})
})
protected.POST("/microphone/mute", func(c *gin.Context) {
var req struct {
Muted bool `json:"muted"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"error": "invalid request body"})
return
}
// Note: Microphone muting is typically handled at the frontend level
// This endpoint is provided for consistency but doesn't affect backend processing
c.JSON(200, gin.H{
"status": "mute state updated",
"muted": req.Muted,
})
})
protected.GET("/microphone/metrics", func(c *gin.Context) {
if currentSession == nil || currentSession.AudioInputManager == nil {
c.JSON(200, gin.H{
"frames_sent": 0,
"frames_dropped": 0,
"bytes_processed": 0,
"last_frame_time": "",
"connection_drops": 0,
"average_latency": "0.0ms",
})
return
}
metrics := currentSession.AudioInputManager.GetMetrics()
c.JSON(200, gin.H{
"frames_sent": metrics.FramesSent,
"frames_dropped": metrics.FramesDropped,
"bytes_processed": metrics.BytesProcessed,
"last_frame_time": metrics.LastFrameTime.Format("2006-01-02T15:04:05.000Z"),
"connection_drops": metrics.ConnectionDrops,
"average_latency": fmt.Sprintf("%.1fms", float64(metrics.AverageLatency.Nanoseconds())/1e6),
})
})
// Audio subprocess process metrics endpoints
protected.GET("/audio/process-metrics", func(c *gin.Context) {
// Access the global audio supervisor from main.go
if audioSupervisor == nil {
c.JSON(200, gin.H{
"cpu_percent": 0.0,
"memory_percent": 0.0,
"memory_rss": 0,
"memory_vms": 0,
"running": false,
})
return
}
metrics := audioSupervisor.GetProcessMetrics()
if metrics == nil {
c.JSON(200, gin.H{
"cpu_percent": 0.0,
"memory_percent": 0.0,
"memory_rss": 0,
"memory_vms": 0,
"running": false,
})
return
}
c.JSON(200, gin.H{
"cpu_percent": metrics.CPUPercent,
"memory_percent": metrics.MemoryPercent,
"memory_rss": metrics.MemoryRSS,
"memory_vms": metrics.MemoryVMS,
"running": true,
})
})
// Audio memory allocation metrics endpoint
protected.GET("/audio/memory-metrics", gin.WrapF(audio.HandleMemoryMetrics))
protected.GET("/microphone/process-metrics", func(c *gin.Context) {
if currentSession == nil || currentSession.AudioInputManager == nil {
c.JSON(200, gin.H{
"cpu_percent": 0.0,
"memory_percent": 0.0,
"memory_rss": 0,
"memory_vms": 0,
"running": false,
})
return
}
// Get the supervisor from the audio input manager
supervisor := currentSession.AudioInputManager.GetSupervisor()
if supervisor == nil {
c.JSON(200, gin.H{
"cpu_percent": 0.0,
"memory_percent": 0.0,
"memory_rss": 0,
"memory_vms": 0,
"running": false,
})
return
}
metrics := supervisor.GetProcessMetrics()
if metrics == nil {
c.JSON(200, gin.H{
"cpu_percent": 0.0,
"memory_percent": 0.0,
"memory_rss": 0,
"memory_vms": 0,
"running": false,
})
return
}
c.JSON(200, gin.H{
"cpu_percent": metrics.CPUPercent,
"memory_percent": metrics.MemoryPercent,
"memory_rss": metrics.MemoryRSS,
"memory_vms": metrics.MemoryVMS,
"running": true,
})
})
// System memory information endpoint
protected.GET("/system/memory", func(c *gin.Context) {
processMonitor := audio.GetProcessMonitor()
totalMemory := processMonitor.GetTotalMemory()
c.JSON(200, gin.H{
"total_memory_bytes": totalMemory,
"total_memory_mb": totalMemory / (1024 * 1024),
})
})
protected.POST("/microphone/reset", func(c *gin.Context) {
if currentSession == nil {
c.JSON(400, gin.H{"error": "no active session"})
return
}
if currentSession.AudioInputManager == nil {
c.JSON(500, gin.H{"error": "audio input manager not available"})
return
}
logger.Info().Msg("forcing microphone state reset")
// Force stop the AudioInputManager
currentSession.AudioInputManager.Stop()
// Wait a bit to ensure everything is stopped
time.Sleep(100 * time.Millisecond)
// Broadcast microphone state change via WebSocket
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.BroadcastMicrophoneStateChanged(false, true)
c.JSON(200, gin.H{
"status": "reset",
"audio_input_running": currentSession.AudioInputManager.IsRunning(),
})
})
// Catch-all route for SPA
r.NoRoute(func(c *gin.Context) {
if c.Request.Method == "GET" && c.NegotiateFormat(gin.MIMEHTML) == gin.MIMEHTML {
@ -179,26 +568,57 @@ func handleWebRTCSession(c *gin.Context) {
return
}
session, err := newSession(SessionConfig{})
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err})
return
}
var session *Session
var err error
var sd string
sd, err := session.ExchangeOffer(req.Sd)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err})
return
}
// Check if we have an existing session
if currentSession != nil {
logger.Info().Msg("existing session detected, creating new session and notifying old session")
// Always create a new session when there's an existing one
// This ensures the "otherSessionConnected" prompt is shown
session, err = newSession(SessionConfig{})
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err})
return
}
sd, err = session.ExchangeOffer(req.Sd)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err})
return
}
// Notify the old session about the takeover
writeJSONRPCEvent("otherSessionConnected", nil, currentSession)
peerConn := currentSession.peerConnection
go func() {
time.Sleep(1 * time.Second)
_ = peerConn.Close()
}()
currentSession = session
logger.Info().Interface("session", session).Msg("new session created, old session notified")
} else {
// No existing session, create a new one
logger.Info().Msg("creating new session")
session, err = newSession(SessionConfig{})
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err})
return
}
sd, err = session.ExchangeOffer(req.Sd)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err})
return
}
currentSession = session
logger.Info().Interface("session", session).Msg("new session accepted")
}
currentSession = session
c.JSON(http.StatusOK, gin.H{"sd": sd})
}
@ -267,6 +687,9 @@ func handleWebRTCSignalWsMessages(
if isCloudConnection {
setCloudConnectionState(CloudConnectionStateDisconnected)
}
// Clean up audio event subscription
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.Unsubscribe(connectionID)
cancelRun()
}()
@ -424,6 +847,14 @@ func handleWebRTCSignalWsMessages(
if err = currentSession.peerConnection.AddICECandidate(candidate); err != nil {
l.Warn().Str("error", err.Error()).Msg("failed to add incoming ICE candidate to our peer connection")
}
} else if message.Type == "subscribe-audio-events" {
l.Info().Msg("client subscribing to audio events")
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.Subscribe(connectionID, wsCon, runCtx, &l)
} else if message.Type == "unsubscribe-audio-events" {
l.Info().Msg("client unsubscribing from audio events")
broadcaster := audio.GetAudioEventBroadcaster()
broadcaster.Unsubscribe(connectionID)
}
}
}

150
webrtc.go
View File

@ -5,11 +5,15 @@ import (
"encoding/base64"
"encoding/json"
"net"
"runtime"
"strings"
"sync"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"github.com/gin-gonic/gin"
"github.com/jetkvm/kvm/internal/audio"
"github.com/jetkvm/kvm/internal/logging"
"github.com/pion/webrtc/v4"
"github.com/rs/zerolog"
@ -18,12 +22,20 @@ import (
type Session struct {
peerConnection *webrtc.PeerConnection
VideoTrack *webrtc.TrackLocalStaticSample
AudioTrack *webrtc.TrackLocalStaticSample
ControlChannel *webrtc.DataChannel
RPCChannel *webrtc.DataChannel
HidChannel *webrtc.DataChannel
DiskChannel *webrtc.DataChannel
AudioInputManager *audio.AudioInputManager
shouldUmountVirtualMedia bool
rpcQueue chan webrtc.DataChannelMessage
// Microphone operation throttling
micCooldown time.Duration
// Audio frame processing
audioFrameChan chan []byte
audioStopChan chan struct{}
audioWg sync.WaitGroup
rpcQueue chan webrtc.DataChannelMessage
}
type SessionConfig struct {
@ -105,7 +117,18 @@ func newSession(config SessionConfig) (*Session, error) {
if err != nil {
return nil, err
}
session := &Session{peerConnection: peerConnection}
session := &Session{
peerConnection: peerConnection,
AudioInputManager: audio.NewAudioInputManager(),
micCooldown: 100 * time.Millisecond,
audioFrameChan: make(chan []byte, 1000),
audioStopChan: make(chan struct{}),
}
// Start audio processing goroutine
session.startAudioProcessor(*logger)
session.rpcQueue = make(chan webrtc.DataChannelMessage, 256)
go func() {
for msg := range session.rpcQueue {
@ -144,22 +167,72 @@ func newSession(config SessionConfig) (*Session, error) {
return nil, err
}
rtpSender, err := peerConnection.AddTrack(session.VideoTrack)
session.AudioTrack, err = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "kvm")
if err != nil {
return nil, err
}
// Update the audio relay with the new WebRTC audio track
if err := audio.UpdateAudioRelayTrack(session.AudioTrack); err != nil {
scopedLogger.Warn().Err(err).Msg("Failed to update audio relay track")
}
videoRtpSender, err := peerConnection.AddTrack(session.VideoTrack)
if err != nil {
return nil, err
}
// Add bidirectional audio transceiver for microphone input
audioTransceiver, err := peerConnection.AddTransceiverFromTrack(session.AudioTrack, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendrecv,
})
if err != nil {
return nil, err
}
audioRtpSender := audioTransceiver.Sender()
// Handle incoming audio track (microphone from browser)
peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
scopedLogger.Info().Str("codec", track.Codec().MimeType).Str("id", track.ID()).Msg("Got remote track")
if track.Kind() == webrtc.RTPCodecTypeAudio && track.Codec().MimeType == webrtc.MimeTypeOpus {
scopedLogger.Info().Msg("Processing incoming audio track for microphone input")
go func() {
// Lock to OS thread to isolate RTP processing
runtime.LockOSThread()
defer runtime.UnlockOSThread()
for {
rtpPacket, _, err := track.ReadRTP()
if err != nil {
scopedLogger.Debug().Err(err).Msg("Error reading RTP packet from audio track")
return
}
// Extract Opus payload from RTP packet
opusPayload := rtpPacket.Payload
if len(opusPayload) > 0 {
// Send to buffered channel for processing
select {
case session.audioFrameChan <- opusPayload:
// Frame sent successfully
default:
// Channel is full, drop the frame
scopedLogger.Warn().Msg("Audio frame channel full, dropping frame")
}
}
}
}()
}
})
// Read incoming RTCP packets
// Before these packets are returned they are processed by interceptors. For things
// like NACK this needs to be called.
go func() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
return
}
}
}()
go drainRtpSender(videoRtpSender)
go drainRtpSender(audioRtpSender)
var isConnected bool
peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
@ -203,6 +276,11 @@ func newSession(config SessionConfig) (*Session, error) {
err := rpcUnmountImage()
scopedLogger.Warn().Err(err).Msg("unmount image failed on connection close")
}
// Stop audio processing and input manager
session.stopAudioProcessor()
if session.AudioInputManager != nil {
session.AudioInputManager.Stop()
}
if isConnected {
isConnected = false
actionSessions--
@ -216,6 +294,56 @@ func newSession(config SessionConfig) (*Session, error) {
return session, nil
}
// startAudioProcessor starts the dedicated audio processing goroutine
func (s *Session) startAudioProcessor(logger zerolog.Logger) {
s.audioWg.Add(1)
go func() {
defer s.audioWg.Done()
logger.Debug().Msg("Audio processor goroutine started")
for {
select {
case frame := <-s.audioFrameChan:
if s.AudioInputManager != nil {
// Check if audio input manager is ready before processing frames
if s.AudioInputManager.IsReady() {
err := s.AudioInputManager.WriteOpusFrame(frame)
if err != nil {
logger.Warn().Err(err).Msg("Failed to write Opus frame to audio input manager")
}
} else {
// Audio input manager not ready, drop frame silently
// This prevents the "client not connected" errors during startup
logger.Debug().Msg("Audio input manager not ready, dropping frame")
}
}
case <-s.audioStopChan:
logger.Debug().Msg("Audio processor goroutine stopping")
return
}
}
}()
}
// stopAudioProcessor stops the audio processing goroutine
func (s *Session) stopAudioProcessor() {
close(s.audioStopChan)
s.audioWg.Wait()
}
func drainRtpSender(rtpSender *webrtc.RTPSender) {
// Lock to OS thread to isolate RTCP processing
runtime.LockOSThread()
defer runtime.UnlockOSThread()
rtcpBuf := make([]byte, 1500)
for {
if _, _, err := rtpSender.Read(rtcpBuf); err != nil {
return
}
}
}
var actionSessions = 0
func onActiveSessionsChanged() {