fix

tools/server/server-context.cpp

@@ -806,8 +806,6 @@ public:
 
     server_state_callback_t callback_state = [](server_state, json) -> void {};
 
-    server_metrics metrics;
-
     server_context_impl() {
         mtmd_helper_log_set(common_log_default_callback, nullptr);
     }
@@ -820,6 +818,10 @@ public:
         }
     }
 
+    server_metrics get_metrics() const {
+        return metrics;
+    }
+
 private:
     // note: accessing these fields outside of this class is not thread-safe
     // use server_context methods instead
@@ -860,6 +862,8 @@ private:
 
     std::unique_ptr<server_prompt_cache> prompt_cache;
 
+    server_metrics metrics;
+
     // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
     // note: kept out of server_metrics, which is copied as-is into the task result
     int64_t  t_decode_start  = 0; // start of the last submitted decode
@@ -4568,10 +4572,12 @@ void server_routes::init_routes() {
         res->status = 200;
 
         if (queue_tasks.is_sleeping()) {
+            std::unique_lock<std::mutex> lock(mutex_cache);
             res->headers["Process-Start-Time-Unix"] = std::to_string(cached_metrics.t_start);
             // render response using cached_metrics
             server_task_result_metrics tmp;
             tmp.metrics = cached_metrics;
+            tmp.n_idle_slots = params.n_parallel;
             res->data = tmp.to_metrics();
 
         } else {
@@ -4685,9 +4691,12 @@ void server_routes::init_routes() {
     this->get_props = [this](const server_http_req &) {
         auto res = create_response(true);
         // note: do NOT use ctx_server here, this endpoint must be accessible during sleep
-        res->ok(queue_tasks.is_sleeping()
-            ? cached_props
-            : get_res_props(*meta, params, false));
+        if (queue_tasks.is_sleeping()) {
+            std::unique_lock<std::mutex> lock(mutex_cache);
+            res->ok(cached_props);
+        } else {
+            res->ok(get_res_props(*meta, params, false));
+        }
         return res;
     };
 
@@ -4950,9 +4959,12 @@ void server_routes::init_routes() {
     this->get_models = [this](const server_http_req &) {
         auto res = create_response(true);
         // note: do NOT use ctx_server here, this endpoint must be accessible during sleep
-        res->ok(queue_tasks.is_sleeping()
-            ? cached_models
-            : get_res_models(*meta));
+        if (queue_tasks.is_sleeping()) {
+            std::unique_lock<std::mutex> lock(mutex_cache);
+            res->ok(cached_models);
+        } else {
+            res->ok(get_res_models(*meta));
+        }
         return res;
     };
 
@@ -5411,13 +5423,15 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
     return res;
 }
 
-void server_routes::update_cached_responses(bool enabled) {
-    if (enabled) {
+void server_routes::update_cached_responses(bool is_sleeping) {
+    if (is_sleeping) {
+        std::unique_lock<std::mutex> lock(mutex_cache);
+
         cached_models  = get_res_models(*meta);
         cached_props   = get_res_props(*meta, params, true);
 
         // caller is task_queue, so we don't need to hold locks here
-        cached_metrics = ctx_server.metrics;
+        cached_metrics = ctx_server.get_metrics();
 
         SRV_DBG("%s\n", "cached responses updated");
     }

tools/server/server-context.h

@@ -8,6 +8,7 @@
 
 #include <cstddef>
 #include <memory>
+#include <mutex>
 #include <set>
 
 struct server_context_impl; // private implementation
@@ -181,9 +182,10 @@ private:
     std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
 
     // cached responses, to be used during sleep
+    std::mutex     mutex_cache;
     json           cached_models  = nullptr;
     json           cached_props   = nullptr;
     server_metrics cached_metrics;
     // call right before sleep to update the cached responses
-    void update_cached_responses(bool enabled);
+    void update_cached_responses(bool is_sleeping);
 };

tools/server/server-queue.h

@@ -131,6 +131,7 @@ public:
     // for example: register order cb0, cb1, cb2
     // entering sleep: queue.sleeping = true --> cb0(true) --> cb1(true) --> cb2(true)
     // leaving sleep: cb2(false) --> cb1(false) --> cb0(false) --> queue.sleeping = false
+    // note: caller will hold mutex_tasks while calling the callbacks
     void on_sleeping_state(std::function<void(bool)> callback) {
         callback_sleeping_state.push_back(std::move(callback));
     }