diff --git a/failsafe.go b/failsafe.go new file mode 100644 index 00000000..1168b9e0 --- /dev/null +++ b/failsafe.go @@ -0,0 +1,105 @@ +package kvm + +import ( + "fmt" + "os" + "strings" + "sync" +) + +const ( + failsafeDefaultLastCrashPath = "/userdata/jetkvm/crashdump/last-crash.log" + failsafeFile = "/userdata/jetkvm/.enablefailsafe" + failsafeLastCrashEnv = "JETKVM_LAST_ERROR_PATH" + failsafeEnv = "JETKVM_FORCE_FAILSAFE" +) + +var ( + failsafeOnce sync.Once + failsafeCrashLog = "" + failsafeModeActive = false + failsafeModeReason = "" +) + +type FailsafeModeNotification struct { + Active bool `json:"active"` + Reason string `json:"reason"` +} + +// this function has side effects and can be only executed once +func checkFailsafeReason() { + failsafeOnce.Do(func() { + // check if the failsafe environment variable is set + if os.Getenv(failsafeEnv) == "1" { + failsafeModeActive = true + failsafeModeReason = "failsafe_env_set" + return + } + + // check if the failsafe file exists + if _, err := os.Stat(failsafeFile); err == nil { + failsafeModeActive = true + failsafeModeReason = "failsafe_file_exists" + _ = os.Remove(failsafeFile) + return + } + + // get the last crash log path from the environment variable + lastCrashPath := os.Getenv(failsafeLastCrashEnv) + if lastCrashPath == "" { + lastCrashPath = failsafeDefaultLastCrashPath + } + + // check if the last crash log file exists + l := failsafeLogger.With().Str("path", lastCrashPath).Logger() + fi, err := os.Lstat(lastCrashPath) + if err != nil { + l.Warn().Err(err).Msg("failed to stat last crash log") + return + } + + if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + l.Warn().Msg("last crash log is not a symlink, ignoring") + return + } + + // open the last crash log file and find if it contains the string "panic" + content, err := os.ReadFile(lastCrashPath) + if err != nil { + l.Warn().Err(err).Msg("failed to read last crash log") + return + } + + // unlink the last crash log file + failsafeCrashLog = string(content) + _ = os.Remove(lastCrashPath) + + // TODO: read the goroutine stack trace and check which goroutine is panicking + if strings.Contains(failsafeCrashLog, "runtime.cgocall") { + failsafeModeActive = true + failsafeModeReason = "video" + return + } + }) +} + +func notifyFailsafeMode(session *Session) { + if !failsafeModeActive || session == nil { + return + } + + jsonRpcLogger.Info().Str("reason", failsafeModeReason).Msg("sending failsafe mode notification") + + writeJSONRPCEvent("failsafeMode", FailsafeModeNotification{ + Active: true, + Reason: failsafeModeReason, + }, session) +} + +func rpcGetFailsafeLogs() (string, error) { + if !failsafeModeActive { + return "", fmt.Errorf("failsafe mode is not active") + } + + return failsafeCrashLog, nil +} diff --git a/jsonrpc.go b/jsonrpc.go index 99e7bdcf..ea6c6fba 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1268,4 +1268,5 @@ var rpcHandlers = map[string]RPCHandler{ "setKeyboardMacros": {Func: setKeyboardMacros, Params: []string{"params"}}, "getLocalLoopbackOnly": {Func: rpcGetLocalLoopbackOnly}, "setLocalLoopbackOnly": {Func: rpcSetLocalLoopbackOnly, Params: []string{"enabled"}}, + "getFailSafeLogs": {Func: rpcGetFailsafeLogs}, } diff --git a/main.go b/main.go index d0971713..29a742ed 100644 --- a/main.go +++ b/main.go @@ -15,8 +15,8 @@ var appCtx context.Context func Main() { checkFailsafeReason() - if shouldActivateFailsafe { - logger.Warn().Str("reason", shouldActivateFailsafeReason).Msg("failsafe mode activated") + if failsafeModeActive { + logger.Warn().Str("reason", failsafeModeReason).Msg("failsafe mode activated") } LoadConfig() diff --git a/native.go b/native.go index 8cbd181e..c42cad3c 100644 --- a/native.go +++ b/native.go @@ -17,7 +17,7 @@ var ( func initNative(systemVersion *semver.Version, appVersion *semver.Version) { nativeInstance = native.NewNative(native.NativeOptions{ - Disable: shouldActivateFailsafe, + Disable: failsafeModeActive, SystemVersion: systemVersion, AppVersion: appVersion, DisplayRotation: config.GetDisplayRotation(), diff --git a/ui/src/components/FaileSafeModeOverlay.tsx b/ui/src/components/FaileSafeModeOverlay.tsx index e7d37c52..c0163747 100644 --- a/ui/src/components/FaileSafeModeOverlay.tsx +++ b/ui/src/components/FaileSafeModeOverlay.tsx @@ -108,7 +108,7 @@ Please attach the recovery logs file that was downloaded to your computer: }; const handleDowngrade = () => { - navigateTo("/settings/general/update?appVersion=0.4.8"); + navigateTo("/settings/general/update?app=0.4.8"); }; return ( diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index b1270b1c..047bdaba 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -930,12 +930,12 @@ export const useMacrosStore = create((set, get) => ({ export interface FailsafeModeState { isFailsafeMode: boolean; reason: string | null; // "video", "network", etc. - setFailsafeMode: (enabled: boolean, reason: string | null) => void; + setFailsafeMode: (active: boolean, reason: string | null) => void; } export const useFailsafeModeStore = create(set => ({ isFailsafeMode: false, reason: null, - setFailsafeMode: (enabled: boolean, reason: string | null) => - set({ isFailsafeMode: enabled, reason }), + setFailsafeMode: (active: boolean, reason: string | null) => + set({ isFailsafeMode: active, reason }), })); diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index b85a3f44..e0c6ec2b 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -676,9 +676,9 @@ export default function KvmIdRoute() { } if (resp.method === "failsafeMode") { - const { enabled, reason } = resp.params as { enabled: boolean; reason: string }; - console.debug("Setting failsafe mode", { enabled, reason }); - setFailsafeMode(enabled, reason); + const { active, reason } = resp.params as { active: boolean; reason: string }; + console.debug("Setting failsafe mode", { active, reason }); + setFailsafeMode(active, reason); } } diff --git a/video.go b/video.go index cd74e680..7ce342a7 100644 --- a/video.go +++ b/video.go @@ -27,6 +27,7 @@ func triggerVideoStateUpdate() { } func rpcGetVideoState() (native.VideoState, error) { + notifyFailsafeMode(currentSession) return lastVideoState, nil } diff --git a/webrtc.go b/webrtc.go index 37488f77..abe1aba7 100644 --- a/webrtc.go +++ b/webrtc.go @@ -289,6 +289,7 @@ func newSession(config SessionConfig) (*Session, error) { triggerOTAStateUpdate() triggerVideoStateUpdate() triggerUSBStateUpdate() + notifyFailsafeMode(session) case "terminal": handleTerminalChannel(d) case "serial": @@ -391,10 +392,12 @@ func newSession(config SessionConfig) (*Session, error) { } func onActiveSessionsChanged() { + notifyFailsafeMode(currentSession) requestDisplayUpdate(true, "active_sessions_changed") } func onFirstSessionConnected() { + notifyFailsafeMode(currentSession) _ = nativeInstance.VideoStart() stopVideoSleepModeTicker() }