Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-07 20:47:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

metrics task should not reset timer

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-19 13:04:33 +02:00
parent abdb18dd0c
commit f5aef6ab5c
2 changed files with 20 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-queue.cpp
+19 -4
View File
@@ -3,6 +3,7 @@
#include "log.h"
#include <algorithm>
#include <chrono>
#include <thread>
@@ -20,6 +21,10 @@
// server_queue
//
static bool task_resets_idle_timer(server_task_type type) {
return type != SERVER_TASK_TYPE_METRICS;
}
int server_queue::post(server_task && task, bool front) {
std::unique_lock<std::mutex> lock(mutex_tasks);
GGML_ASSERT(task.id != -1);
@@ -27,20 +32,24 @@ int server_queue::post(server_task && task, bool front) {
if (task.type == SERVER_TASK_TYPE_CANCEL) {
cleanup_pending_task(task.id_target);
}
const int task_id = task.id;
const int task_id = task.id;
const bool reset_timer = task_resets_idle_timer(task.type);
QUE_DBG("new task, id = %d, front = %d\n", task_id, front);
if (front) {
queue_tasks.push_front(std::move(task));
} else {
queue_tasks.push_back(std::move(task));
}
time_last_task = ggml_time_ms();
if (reset_timer) {
time_last_task = ggml_time_ms();
}
condition_tasks.notify_one();
return task_id;
}
int server_queue::post(std::vector<server_task> && tasks, bool front) {
std::unique_lock<std::mutex> lock(mutex_tasks);
bool reset_timer = false;
for (auto & task : tasks) {
if (task.id == -1) {
task.id = id++;
@@ -49,6 +58,7 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
if (task.type == SERVER_TASK_TYPE_CANCEL) {
cleanup_pending_task(task.id_target);
}
reset_timer |= task_resets_idle_timer(task.type);
QUE_DBG("new task, id = %d/%d, front = %d\n", task.id, (int) tasks.size(), front);
if (front) {
queue_tasks.push_front(std::move(task));
@@ -56,7 +66,9 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
queue_tasks.push_back(std::move(task));
}
}
time_last_task = ggml_time_ms();
if (reset_timer) {
time_last_task = ggml_time_ms();
}
condition_tasks.notify_one();
return 0;
}
@@ -294,11 +306,14 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
QUE_DBG("%s", "update slots\n");
// this will run the main inference process for all slots
const int64_t t_update_slots = ggml_time_ms();
callback_update_slots();
{
// update_slots() may take a while to finish, we need to make sure it's not counted as idle
// shift instead of reset, so that non-task_resets_idle_timer tasks do not delay the sleep
std::unique_lock<std::mutex> lock(mutex_tasks);
time_last_task = ggml_time_ms();
const int64_t now = ggml_time_ms();
time_last_task = std::min(now, time_last_task + (now - t_update_slots));
}
QUE_DBG("%s", "waiting for new tasks\n");