feat: add failsafe mode to recover from infinite restarts caused by cgo panics

failsafe.go

@@ -0,0 +1,105 @@
+package kvm
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+)
+
+const (
+	failsafeDefaultLastCrashPath = "/userdata/jetkvm/crashdump/last-crash.log"
+	failsafeFile                 = "/userdata/jetkvm/.enablefailsafe"
+	failsafeLastCrashEnv         = "JETKVM_LAST_ERROR_PATH"
+	failsafeEnv                  = "JETKVM_FORCE_FAILSAFE"
+)
+
+var (
+	failsafeOnce       sync.Once
+	failsafeCrashLog   = ""
+	failsafeModeActive = false
+	failsafeModeReason = ""
+)
+
+type FailsafeModeNotification struct {
+	Active bool   `json:"active"`
+	Reason string `json:"reason"`
+}
+
+// this function has side effects and can be only executed once
+func checkFailsafeReason() {
+	failsafeOnce.Do(func() {
+		// check if the failsafe environment variable is set
+		if os.Getenv(failsafeEnv) == "1" {
+			failsafeModeActive = true
+			failsafeModeReason = "failsafe_env_set"
+			return
+		}
+
+		// check if the failsafe file exists
+		if _, err := os.Stat(failsafeFile); err == nil {
+			failsafeModeActive = true
+			failsafeModeReason = "failsafe_file_exists"
+			_ = os.Remove(failsafeFile)
+			return
+		}
+
+		// get the last crash log path from the environment variable
+		lastCrashPath := os.Getenv(failsafeLastCrashEnv)
+		if lastCrashPath == "" {
+			lastCrashPath = failsafeDefaultLastCrashPath
+		}
+
+		// check if the last crash log file exists
+		l := failsafeLogger.With().Str("path", lastCrashPath).Logger()
+		fi, err := os.Lstat(lastCrashPath)
+		if err != nil {
+			l.Warn().Err(err).Msg("failed to stat last crash log")
+			return
+		}
+
+		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
+			l.Warn().Msg("last crash log is not a symlink, ignoring")
+			return
+		}
+
+		// open the last crash log file and find if it contains the string "panic"
+		content, err := os.ReadFile(lastCrashPath)
+		if err != nil {
+			l.Warn().Err(err).Msg("failed to read last crash log")
+			return
+		}
+
+		// unlink the last crash log file
+		failsafeCrashLog = string(content)
+		_ = os.Remove(lastCrashPath)
+
+		// TODO: read the goroutine stack trace and check which goroutine is panicking
+		if strings.Contains(failsafeCrashLog, "runtime.cgocall") {
+			failsafeModeActive = true
+			failsafeModeReason = "video"
+			return
+		}
+	})
+}
+
+func notifyFailsafeMode(session *Session) {
+	if !failsafeModeActive || session == nil {
+		return
+	}
+
+	jsonRpcLogger.Info().Str("reason", failsafeModeReason).Msg("sending failsafe mode notification")
+
+	writeJSONRPCEvent("failsafeMode", FailsafeModeNotification{
+		Active: true,
+		Reason: failsafeModeReason,
+	}, session)
+}
+
+func rpcGetFailsafeLogs() (string, error) {
+	if !failsafeModeActive {
+		return "", fmt.Errorf("failsafe mode is not active")
+	}
+
+	return failsafeCrashLog, nil
+}

jsonrpc.go

@@ -1268,4 +1268,5 @@ var rpcHandlers = map[string]RPCHandler{
 	"setKeyboardMacros":      {Func: setKeyboardMacros, Params: []string{"params"}},
 	"getLocalLoopbackOnly":   {Func: rpcGetLocalLoopbackOnly},
 	"setLocalLoopbackOnly":   {Func: rpcSetLocalLoopbackOnly, Params: []string{"enabled"}},
+	"getFailSafeLogs":        {Func: rpcGetFailsafeLogs},
 }

main.go

@@ -15,8 +15,8 @@ var appCtx context.Context
 
 func Main() {
 	checkFailsafeReason()
-	if shouldActivateFailsafe {
+	if failsafeModeActive {
-		logger.Warn().Str("reason", shouldActivateFailsafeReason).Msg("failsafe mode activated")
+		logger.Warn().Str("reason", failsafeModeReason).Msg("failsafe mode activated")
 	}
 
 	LoadConfig()

native.go

@@ -17,7 +17,7 @@ var (
 
 func initNative(systemVersion *semver.Version, appVersion *semver.Version) {
 	nativeInstance = native.NewNative(native.NativeOptions{
-		Disable:              shouldActivateFailsafe,
+		Disable:              failsafeModeActive,
 		SystemVersion:        systemVersion,
 		AppVersion:           appVersion,
 		DisplayRotation:      config.GetDisplayRotation(),

ui/src/components/FaileSafeModeOverlay.tsx

@@ -108,7 +108,7 @@ Please attach the recovery logs file that was downloaded to your computer:
   };
 
   const handleDowngrade = () => {
-    navigateTo("/settings/general/update?appVersion=0.4.8");
+    navigateTo("/settings/general/update?app=0.4.8");
   };
 
   return (

ui/src/hooks/stores.ts

@@ -930,12 +930,12 @@ export const useMacrosStore = create<MacrosState>((set, get) => ({
 export interface FailsafeModeState {
   isFailsafeMode: boolean;
   reason: string | null; // "video", "network", etc.
-  setFailsafeMode: (enabled: boolean, reason: string | null) => void;
+  setFailsafeMode: (active: boolean, reason: string | null) => void;
 }
 
 export const useFailsafeModeStore = create<FailsafeModeState>(set => ({
   isFailsafeMode: false,
   reason: null,
-  setFailsafeMode: (enabled: boolean, reason: string | null) =>
+  setFailsafeMode: (active: boolean, reason: string | null) =>
-    set({ isFailsafeMode: enabled, reason }),
+    set({ isFailsafeMode: active, reason }),
 }));

ui/src/routes/devices.$id.tsx

@@ -676,9 +676,9 @@ export default function KvmIdRoute() {
     }
 
     if (resp.method === "failsafeMode") {
-      const { enabled, reason } = resp.params as { enabled: boolean; reason: string };
+      const { active, reason } = resp.params as { active: boolean; reason: string };
-      console.debug("Setting failsafe mode", { enabled, reason });
+      console.debug("Setting failsafe mode", { active, reason });
-      setFailsafeMode(enabled, reason);
+      setFailsafeMode(active, reason);
     }
   }
 

video.go

@@ -27,6 +27,7 @@ func triggerVideoStateUpdate() {
 }
 
 func rpcGetVideoState() (native.VideoState, error) {
+	notifyFailsafeMode(currentSession)
 	return lastVideoState, nil
 }
 

webrtc.go

@@ -289,6 +289,7 @@ func newSession(config SessionConfig) (*Session, error) {
 			triggerOTAStateUpdate()
 			triggerVideoStateUpdate()
 			triggerUSBStateUpdate()
+			notifyFailsafeMode(session)
 		case "terminal":
 			handleTerminalChannel(d)
 		case "serial":
@@ -391,10 +392,12 @@ func newSession(config SessionConfig) (*Session, error) {
 }
 
 func onActiveSessionsChanged() {
+	notifyFailsafeMode(currentSession)
 	requestDisplayUpdate(true, "active_sessions_changed")
 }
 
 func onFirstSessionConnected() {
+	notifyFailsafeMode(currentSession)
 	_ = nativeInstance.VideoStart()
 	stopVideoSleepModeTicker()
 }