From 437b415fa8d4d304601b81a8fd65706b202350ee Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 19 Aug 2026 23:56:38 +0200 Subject: [PATCH] server: add --sleep-mode rst --- common/arg.cpp | 16 ++++ common/common.h | 6 ++ tools/server/README.md | 7 +- tools/server/server-common.cpp | 144 ++++++++++++++++++++++++++++++++ tools/server/server-common.h | 38 ++++++++- tools/server/server-context.cpp | 127 ++++++++++++++++++++++------ tools/server/server-context.h | 9 +- tools/server/server-queue.cpp | 22 +++++ tools/server/server-queue.h | 7 ++ tools/server/server.cpp | 9 +- 10 files changed, 350 insertions(+), 35 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 6f5fe377d5..cd4f0aa486 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3757,6 +3757,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sleep_idle_seconds = value; } ).set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--sleep-mode"}, "MODE", + "what to release when the server sleeps:\n" + "- 'free' frees context and model memory\n" + "- 'rst' restarts the whole process, may help reset memory to zero on certain backend\n" + "(default: free)", + [](common_params & params, const std::string & value) { + if (value == "free") { + params.sleep_mode = COMMON_SLEEP_MODE_FREE; + } else if (value == "rst") { + params.sleep_mode = COMMON_SLEEP_MODE_RST; + } else { + throw std::invalid_argument("invalid value: " + value); + } + } + ).set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"--simple-io"}, "use basic IO for better compatibility in subprocesses and limited consoles", diff --git a/common/common.h b/common/common.h index d8a16897b8..83d0a85c3f 100644 --- a/common/common.h +++ b/common/common.h @@ -408,6 +408,11 @@ struct common_params_diffusion { // reasoning API response format (not to be confused as chat template's reasoning format) // only used by server +enum common_sleep_mode { + COMMON_SLEEP_MODE_FREE, // free context and model memory + COMMON_SLEEP_MODE_RST, // also restart the process, releasing all backend resources +}; + enum common_reasoning_format { COMMON_REASONING_FORMAT_NONE, COMMON_REASONING_FORMAT_AUTO, // Same as deepseek, using `message.reasoning_content` @@ -633,6 +638,7 @@ struct common_params { int enable_reasoning = -1; // -1 = auto, 0 = disable, 1 = enable bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response int sleep_idle_seconds = -1; // if >0, server will sleep after this many seconds of idle time + common_sleep_mode sleep_mode = COMMON_SLEEP_MODE_FREE; std::vector api_keys; diff --git a/tools/server/README.md b/tools/server/README.md index b63a0e6dac..3296a59847 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -196,11 +196,11 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | -| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | -| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | +| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | | `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)
(env: LLAMA_ARG_RERANKING) | @@ -237,6 +237,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-sps, --slot-prompt-similarity SIMILARITY` | how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled) | | `--lora-init-without-apply` | load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled) | | `--sleep-idle-seconds SECONDS` | number of seconds of idleness after which the server will sleep (default: -1; -1 = disabled) | +| `--sleep-mode MODE` | what to release when the server sleeps:
- 'free' frees context and model memory
- 'rst' restarts the whole process, may help reset memory to zero on certain backend
(default: free) | | `--log-prompts-dir PATH` | Log prompts to directory (auto-created if not present; only used for debugging, default: disabled) | | `--spec-draft-hf, -hfd, -hfrd, --hf-repo-draft /[:quant]` | Same as --hf-repo, but for the draft model (default: unused)
(env: LLAMA_ARG_SPEC_DRAFT_HF_REPO) | | `--spec-draft-threads, -td, --threads-draft N` | number of threads to use during generation (default: same as --threads) | @@ -2073,6 +2074,8 @@ Note that the following endpoints are exempt from being considered as incoming t - `GET /models` - `GET /metrics` +Some backends keep memory allocated even after the model is unloaded, for example a CUDA context stays on the GPU. To also release that memory, use `--sleep-mode rst`, which restarts the server process upon sleeping. The process keeps the same PID and port, and the responses of the endpoints listed above are preserved across the restart. This mode is not supported on Windows. + ## More examples ### Interactive mode diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index e587b884d4..f5a88747cb 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -16,6 +16,18 @@ #include #include +#if !defined(_WIN32) +#include +#include +#include +#include +#include +#endif + +#if defined(__APPLE__) +#include +#endif + json format_error_response(const std::string & message, const enum error_type type) { std::string type_str; int code = 500; @@ -1877,3 +1889,135 @@ server_tokens format_prompt_rerank( return result; } + + +// +// server_sleep_rst +// + +#if !defined(_WIN32) +static std::string server_proc_exe_path(char ** argv) { + char buf[PATH_MAX]; +#if defined(__linux__) + const ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf) - 1); + if (len > 0) { + buf[len] = '\0'; + return buf; + } +#elif defined(__APPLE__) + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) { + return buf; + } +#endif + return argv[0]; +} + +// exec() keeps the file descriptors open, so mark them all to be closed instead +// this releases the listening port and the backend devices, and makes child processes see EOF +static void server_proc_close_fds_on_exec() { + int n_fd = 4096; + + struct rlimit lim; + if (getrlimit(RLIMIT_NOFILE, &lim) == 0 && lim.rlim_cur != RLIM_INFINITY) { + n_fd = std::min(lim.rlim_cur, 65536); + } + + // skip stdin/stdout/stderr, they are used to communicate with the router + for (int fd = 3; fd < n_fd; fd++) { + const int flags = fcntl(fd, F_GETFD); + if (flags != -1) { + fcntl(fd, F_SETFD, flags | FD_CLOEXEC); + } + } +} +#endif + +static void server_proc_restart(char ** argv, const char * env_name, const std::string & env_value) { +#if defined(_WIN32) || defined(__EMSCRIPTEN__) + GGML_UNUSED(argv); + GGML_UNUSED(env_name); + GGML_UNUSED(env_value); + SRV_ERR("%s", "restarting the process is not supported on this platform\n"); +#else + GGML_ASSERT(argv != nullptr); + + // exec() rejects an env var larger than MAX_ARG_STRLEN (128 kB on linux) + if (env_value.size() > 64*1024) { + SRV_ERR("cannot restart the process, '%s' is too large (%zu bytes)\n", env_name, env_value.size()); + return; + } + + setenv(env_name, env_value.c_str(), 1); + + const std::string exe = server_proc_exe_path(argv); + SRV_INF("restarting the process, exe = '%s'\n", exe.c_str()); + + server_proc_close_fds_on_exec(); + + // the log worker thread does not survive exec(), flush it while we still can + common_log_pause(common_log_main()); + fflush(stdout); + fflush(stderr); + + execv(exe.c_str(), argv); + + // exec() only returns on error, the server can no longer serve requests at this point + GGML_ABORT("execv() failed: %s", strerror(errno)); +#endif +} + +static const char * SLEEP_STATE_ENV = "LLAMA_SERVER_SLEEP_STATE"; + +void server_sleep_rst::init(int argc, char ** argv) { + GGML_ASSERT(argv == nullptr || argc > 0); + + this->argv = argv; + + const char * state = std::getenv(SLEEP_STATE_ENV); + if (state == nullptr) { + return; + } + + try { + boot_state = json::parse(state); + } catch (const std::exception & e) { + SRV_ERR("failed to read the state left by the previous process: %s\n", e.what()); + } + +#if defined(_WIN32) + _putenv_s(SLEEP_STATE_ENV, ""); +#else + // clear it now, so that child processes do not inherit it + unsetenv(SLEEP_STATE_ENV); +#endif +} + +void server_sleep_rst::enable(common_params & params) { + if (params.sleep_mode != COMMON_SLEEP_MODE_RST) { + boot_state = json(); + return; + } + + if (argv == nullptr) { + // exec() can only restart a standalone process + SRV_WRN("%s", "--sleep-mode rst is not supported in this mode, using --sleep-mode free\n"); + params.sleep_mode = COMMON_SLEEP_MODE_FREE; + boot_state = json(); + return; + } + + if (params.sleep_idle_seconds < 0) { + SRV_WRN("%s", "--sleep-mode has no effect without --sleep-idle-seconds\n"); + } + + enabled = true; +} + +void server_sleep_rst::restart() const { + if (!enabled) { + return; + } + + server_proc_restart(argv, SLEEP_STATE_ENV, safe_json_to_str(state_provider ? state_provider() : json())); +} diff --git a/tools/server/server-common.h b/tools/server/server-common.h index c69c462c60..e9c823daca 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -494,7 +494,7 @@ struct server_metrics { n_prompt_cached += n_tokens; } - // used to keep the metrics across a process restart, see --sleep-mode all + // used to keep the metrics across a process restart, see --sleep-mode rst json to_json() const; void from_json(const json & data); }; @@ -611,3 +611,39 @@ struct server_pipe { return true; } }; + +// +// server_sleep_rst +// this allow --sleep-mode rst to reset the whole process, but still preserve metrics and props data +// see README-dev.md for more info +// + +struct server_sleep_rst { + // remember argv and read the state left by the previous process + // must be called once at startup, before spawning any thread or child process + void init(int argc, char ** argv); + + // enable the restart upon sleeping, warns and falls back to --sleep-mode free if not possible + void enable(common_params & params); + + // true if the process was restarted by a previous instance + // in this case, the model is only loaded upon the first request + bool is_boot_to_sleep() const { return !boot_state.is_null(); } + + // state left by the previous process, only valid if is_boot_to_sleep() + const json & get_boot_state() const { return boot_state; } + + // set the state to be preserved across the restart + void set_state_provider(std::function provider) { state_provider = std::move(provider); } + + // restart the process, does nothing if not enabled, does not return on success + void restart() const; + +private: + bool enabled = false; + char ** argv = nullptr; + + json boot_state; + + std::function state_provider; +}; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 0d17fab23c..08eb042352 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -798,6 +798,15 @@ public: mtmd_context * mctx = nullptr; const llama_vocab * vocab = nullptr; + server_sleep_rst sleep_rst; + + server_metrics metrics; + + // called each time the model is loaded, used by server_routes to refresh its metadata + void on_model_loaded(std::function callback) { + callback_model_loaded = std::move(callback); + } + server_queue queue_tasks; server_response queue_results; @@ -818,14 +827,6 @@ public: } } - server_metrics get_metrics() const { - return metrics; - } - - void reset_metrics_bucket() { - metrics.reset_bucket(); - } - private: // note: accessing these fields outside of this class is not thread-safe // use server_context methods instead @@ -866,8 +867,6 @@ private: std::unique_ptr prompt_cache; - server_metrics metrics; - // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync // note: kept out of server_metrics, which is copied as-is into the task result int64_t t_decode_start = 0; // start of the last submitted decode @@ -885,6 +884,13 @@ private: bool sleeping = false; + // set once init() has run, which requires a loaded model + bool initialized = false; + + bool queue_initialized = false; + + std::function callback_model_loaded; + int64_t t_last_load_progress_ms = 0; void destroy() { @@ -912,11 +918,15 @@ private: } SRV_INF("%s", "server is entering sleeping state\n"); destroy(); - } else { - SRV_INF("%s", "server is exiting sleeping state\n"); - if (!load_model(params_base)) { - GGML_ABORT("failed to reload model after sleeping"); - } + sleeping = new_state; + // everything is released, the process can now restart itself + sleep_rst.restart(); + return; + } + + SRV_INF("%s", "server is exiting sleeping state\n"); + if (!load_model(params_base)) { + GGML_ABORT("failed to reload model after sleeping"); } sleeping = new_state; } @@ -956,6 +966,15 @@ private: // load the model and initialize llama_context // this may also be called to resume from sleeping state bool load_model(common_params & params) { + if (!initialized && !sleeping) { + sleep_rst.enable(params); + + if (sleep_rst.is_boot_to_sleep()) { + init_sleeping(params); + return true; + } + } + load_progress_data load_progress_text (this, "text_model"); load_progress_data load_progress_mmproj(this, "mmproj_model"); load_progress_data load_progress_spec (this, "spec_model"); @@ -1349,25 +1368,34 @@ private: // propagate new defaults back to caller params = params_base; - if (!is_resume) { - return init(); + // a process restarted into sleeping state loads the model here for the first time + if (!initialized) { + initialized = true; + if (!init()) { + return false; + } } - if (callback_state) { + SRV_INF("%s", "model loaded\n"); + + if (callback_model_loaded) { + callback_model_loaded(); + } + + if (is_resume && callback_state) { callback_state(SERVER_STATE_READY, {}); } return true; } - // unlike load_model(), this is only called once during initialization - bool init() { - GGML_ASSERT(ctx_tgt != nullptr); - GGML_ASSERT(model_tgt != nullptr); + // wiring up server queues, must be done once before start_loop() + void init_queue() { + if (queue_initialized) { + return; // already done by init_sleeping() + } + queue_initialized = true; - GGML_ASSERT(!sleeping); - - // wiring up server queues queue_tasks.on_new_task([this](server_task && task, bool is_yielding) { return process_single_task(std::move(task), is_yielding); }); @@ -1379,6 +1407,28 @@ private: }); metrics.init(); + } + + // enter sleeping state without a model, the model is loaded upon leaving that state + void init_sleeping(common_params & params) { + GGML_ASSERT(!initialized); + + params_base = params; + + init_queue(); + + sleeping = true; + queue_tasks.init_sleeping(); + + SRV_INF("%s", "restarted in sleeping state, the model will be loaded upon the first request\n"); + } + + // unlike load_model(), this is only called once during initialization + bool init() { + GGML_ASSERT(ctx_tgt != nullptr); + GGML_ASSERT(model_tgt != nullptr); + + init_queue(); if (params_base.cache_idle_slots) { if (params_base.cache_ram_mib == 0) { @@ -4074,6 +4124,10 @@ private: server_context::server_context() : impl(new server_context_impl()) {} server_context::~server_context() = default; +void server_context::init(int argc, char ** argv) { + impl->sleep_rst.init(argc, argv); +} + bool server_context::load_model(common_params & params) { return impl->load_model(params); } @@ -4454,6 +4508,21 @@ server_routes::server_routes(const common_params & params, server_context & ctx_ queue_tasks.on_sleeping_state([this](bool is_sleeping) { update_cached_responses(is_sleeping); }); + + // meta is only available once the model is loaded, which may happen upon leaving sleeping state + this->ctx_server.on_model_loaded([this, &ctx_server]() { + update_meta(ctx_server); + }); + + // set the hook to allow sleep_rst to capture the state BEFORE resetting the process + this->ctx_server.sleep_rst.set_state_provider([this]() { + return cache_to_json(); + }); + + // reverse of above: if we just booted AFTER sleep_rst reset the process, we restore it + if (this->ctx_server.sleep_rst.is_boot_to_sleep()) { + cache_from_json(this->ctx_server.sleep_rst.get_boot_state()); + } } static json get_res_model_info(const server_context_meta & meta) { @@ -5459,7 +5528,7 @@ void server_routes::update_cached_responses(bool is_sleeping) { if (is_sleeping) { cached_models = get_res_models(*meta); cached_props = get_res_props(*meta, params, true); - cached_metrics = ctx_server.get_metrics(); + cached_metrics = ctx_server.metrics; should_reset_buckets = false; @@ -5467,7 +5536,7 @@ void server_routes::update_cached_responses(bool is_sleeping) { } else if (should_reset_buckets) { // a scrape during sleep already reported these buckets - ctx_server.reset_metrics_bucket(); + ctx_server.metrics.reset_bucket(); should_reset_buckets = false; } @@ -5495,5 +5564,9 @@ bool server_routes::cache_from_json(const json & data) { return false; } + // keep counting from where the previous process stopped + ctx_server.metrics = cached_metrics; + return true; } + diff --git a/tools/server/server-context.h b/tools/server/server-context.h index afc2d2bb09..f394afed0a 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -87,8 +87,14 @@ struct server_context { server_context(); ~server_context(); + // remember the command line, needed to restart the process, see --sleep-mode rst + // must be called once at startup, before spawning any thread or child process + void init(int argc, char ** argv); + // load the model and initialize llama_context // returns true on success + // note: if the process was restarted into sleeping state, no model is loaded and it + // returns true right away, the model is then loaded upon the first request bool load_model(common_params & params); // this function will block main thread until termination @@ -158,7 +164,7 @@ struct server_routes { // to be used in router mode json get_model_info() const; - // save / restore the cached responses across a process restart, see --sleep-mode all + // save / restore the cached responses across a process restart, see --sleep-mode rst // only valid while sleeping, as the cache is only updated upon entering that state json cache_to_json(); bool cache_from_json(const json & data); @@ -196,3 +202,4 @@ private: // call right before sleep to update the cached responses void update_cached_responses(bool is_sleeping); }; + diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index 78169e9a5d..b3f1c63a66 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -286,6 +286,28 @@ void server_queue::start_loop(int64_t idle_sleep_ms) { worker.yielding = false; worker.thread = std::thread([this]() { worker_loop(); }); + // the process may start already sleeping, see init_sleeping() + { + std::unique_lock lock(mutex_tasks); + if (sleeping) { + QUE_INF("%s", "starting in sleeping state\n"); + condition_tasks.wait(lock, [&]{ + return (!running || req_stop_sleeping); + }); + if (running) { + QUE_INF("%s", "exiting sleeping state\n"); + req_stop_sleeping = false; + // Call order cb{N} -> cb1 -> cb0 + for (size_t i = callback_sleeping_state.size(); i > 0; i--) { + callback_sleeping_state[i - 1](false); + } + sleeping = false; + condition_tasks.notify_all(); // notify wait_until_no_sleep() + } + time_last_task = ggml_time_ms(); + } + } + constexpr auto max_wait_time = std::chrono::seconds(1); auto should_sleep = [&]() -> bool { // caller must hold mutex_tasks diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index e17733a743..2073e38753 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -74,6 +74,13 @@ public: return sleeping; } + // enter sleeping state before start_loop(), for a process restarted into that state + // note: callback_sleeping_state(true) is not called, the state is already known + void init_sleeping() { + std::unique_lock lock(mutex_tasks); + sleeping = true; + } + // end the start_loop routine void terminate(); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 01cc6633a3..9ff5faefd9 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -167,6 +167,7 @@ int llama_server(common_params & params, int argc, char ** argv) { // struct that contains llama context and inference server_context ctx_server; + ctx_server.init(argc, argv); server_http_context ctx_http; if (!ctx_http.init(params)) { @@ -458,11 +459,8 @@ int llama_server(common_params & params, int argc, char ** argv) { return 1; } - routes.update_meta(ctx_server); ctx_http.is_ready.store(true); - SRV_INF("%s", "model loaded\n"); - shutdown_handler = [&](int) { mcp_mgr.shutdown(); // this will unblock start_loop() @@ -513,7 +511,10 @@ int llama_server(common_params & params, int argc, char ** argv) { std::thread monitor_thread; if (child.is_child()) { monitor_thread = child.setup(shutdown_handler); - child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info()); + // if no model is loaded, the process restarted into sleeping state and the router knows it + if (ctx_server.get_llama_context() != nullptr) { + child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info()); + } } // this call blocks the main thread until queue_tasks.terminate() is called