diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c2bacbc214..98aec0e5e9 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -806,8 +806,6 @@ public: server_state_callback_t callback_state = [](server_state, json) -> void {}; - server_metrics metrics; - server_context_impl() { mtmd_helper_log_set(common_log_default_callback, nullptr); } @@ -820,6 +818,10 @@ public: } } + server_metrics get_metrics() const { + return metrics; + } + private: // note: accessing these fields outside of this class is not thread-safe // use server_context methods instead @@ -860,6 +862,8 @@ private: std::unique_ptr prompt_cache; + server_metrics metrics; + // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync // note: kept out of server_metrics, which is copied as-is into the task result int64_t t_decode_start = 0; // start of the last submitted decode @@ -4568,10 +4572,12 @@ void server_routes::init_routes() { res->status = 200; if (queue_tasks.is_sleeping()) { + std::unique_lock lock(mutex_cache); res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start); // render response using cached_metrics server_task_result_metrics tmp; tmp.metrics = cached_metrics; + tmp.n_idle_slots = params.n_parallel; res->data = tmp.to_metrics(); } else { @@ -4685,9 +4691,12 @@ void server_routes::init_routes() { this->get_props = [this](const server_http_req &) { auto res = create_response(true); // note: do NOT use ctx_server here, this endpoint must be accessible during sleep - res->ok(queue_tasks.is_sleeping() - ? cached_props - : get_res_props(*meta, params, false)); + if (queue_tasks.is_sleeping()) { + std::unique_lock lock(mutex_cache); + res->ok(cached_props); + } else { + res->ok(get_res_props(*meta, params, false)); + } return res; }; @@ -4950,9 +4959,12 @@ void server_routes::init_routes() { this->get_models = [this](const server_http_req &) { auto res = create_response(true); // note: do NOT use ctx_server here, this endpoint must be accessible during sleep - res->ok(queue_tasks.is_sleeping() - ? cached_models - : get_res_models(*meta)); + if (queue_tasks.is_sleeping()) { + std::unique_lock lock(mutex_cache); + res->ok(cached_models); + } else { + res->ok(get_res_models(*meta)); + } return res; }; @@ -5411,13 +5423,15 @@ std::unique_ptr server_routes::handle_count_tokens(const l return res; } -void server_routes::update_cached_responses(bool enabled) { - if (enabled) { +void server_routes::update_cached_responses(bool is_sleeping) { + if (is_sleeping) { + std::unique_lock lock(mutex_cache); + cached_models = get_res_models(*meta); cached_props = get_res_props(*meta, params, true); // caller is task_queue, so we don't need to hold locks here - cached_metrics = ctx_server.metrics; + cached_metrics = ctx_server.get_metrics(); SRV_DBG("%s\n", "cached responses updated"); } diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 8cc61646ef..9bca1cc4f9 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -8,6 +8,7 @@ #include #include +#include #include struct server_context_impl; // private implementation @@ -181,9 +182,10 @@ private: std::unique_ptr create_response(bool bypass_sleep = false); // cached responses, to be used during sleep + std::mutex mutex_cache; json cached_models = nullptr; json cached_props = nullptr; server_metrics cached_metrics; // call right before sleep to update the cached responses - void update_cached_responses(bool enabled); + void update_cached_responses(bool is_sleeping); }; diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index 9db6b52f40..e17733a743 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -131,6 +131,7 @@ public: // for example: register order cb0, cb1, cb2 // entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true) // leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false + // note: caller will hold mutex_tasks while calling the callbacks void on_sleeping_state(std::function callback) { callback_sleeping_state.push_back(std::move(callback)); }