Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 02:37:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

llama: add lazy mode auto, fix iGPU regression

Browse Source
This commit is contained in:
Ruben Ortlam
2026-09-03 15:25:24 +02:00
parent de8656bd94
commit 38d20da312
5 changed files with 38 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+8 -6
View File
@@ -2736,13 +2736,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
add_opt(common_arg(
{"-lzm", "--lazy-mode"}, "MODE",
"on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
"- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
"- auto: on, but only for tensors larger than 4 GiB\n"
"- off: always keep them resident",
"- auto: large where mmap is supported, off otherwise\n"
"- off: always keep them resident\n"
"- large: read only large tensors (>4GiB) from disk on demand (requires mmap)\n"
"- all: read the all such tensors from disk on demand instead of keeping them resident (requires mmap)",
[](common_params & params, const std::string & value) {
/**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; }
else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }
/**/ if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }
else if (value == "large") { params.lazy_mode = LLAMA_LAZY_MODE_LARGE; }
else if (value == "all") { params.lazy_mode = LLAMA_LAZY_MODE_ALL; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LAZY_MODE"));
include/llama.h
+4 -3
View File
@@ -215,9 +215,10 @@ extern "C" {
LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
enum llama_lazy_mode {
LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front
LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
LLAMA_LAZY_MODE_AUTO = 0, // pick a good default for the system: LARGE where mmap is supported, else OFF
LLAMA_LAZY_MODE_OFF = 1, // always read the whole tensor up front
LLAMA_LAZY_MODE_LARGE = 2, // read the rows on demand, but only for marked tensors larger than 4 GiB (requires mmap)
LLAMA_LAZY_MODE_ALL = 3, // read the rows of all tensors marked by the arch on demand (requires mmap)
};
enum llama_context_type {
src/llama-model-loader.cpp
+1 -1
View File
@@ -1085,7 +1085,7 @@ bool llama_model_loader::lazy_read::add(const std::string & name, const ggml_ten
// do not lazy-read small tensors, it has significant overhead and is not worth it
constexpr size_t auto_min_size = 4ull * 1024 * 1024 * 1024;
if (mode != LLAMA_LAZY_MODE_ON && ggml_nbytes(t) <= auto_min_size) {
if (mode != LLAMA_LAZY_MODE_ALL && ggml_nbytes(t) <= auto_min_size) {
return false;
}
src/llama-model.cpp
+13
View File
@@ -1421,6 +1421,19 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
// resolve AUTO: LARGE where mmap is supported, else OFF (e.g. iGPUs); see #28160
if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) {
ml.lazy.mode = LLAMA_LAZY_MODE_LARGE;
for (const auto & dev : devices) {
ggml_backend_dev_props props;
ggml_backend_dev_get_props(dev.dev, &props);
if (!props.caps.mmap_support) {
ml.lazy.mode = LLAMA_LAZY_MODE_OFF;
break;
}
}
}
const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO
? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE)
: llama_load_mode_name(params.load_mode);
tools/llama-bench/llama-bench.cpp
+12 -8
View File
@@ -273,12 +273,14 @@ static const char * split_mode_str(llama_split_mode mode) {
static const char * lazy_mode_str(llama_lazy_mode mode) {
switch (mode) {
case LLAMA_LAZY_MODE_OFF:
return "off";
case LLAMA_LAZY_MODE_AUTO:
return "auto";
case LLAMA_LAZY_MODE_ON:
return "on";
case LLAMA_LAZY_MODE_OFF:
return "off";
case LLAMA_LAZY_MODE_LARGE:
return "large";
case LLAMA_LAZY_MODE_ALL:
return "all";
default:
GGML_ABORT("invalid lazy mode");
}
@@ -475,7 +477,7 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" -lzm, --lazy-mode <on|auto|off> (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str());
printf(" -lzm, --lazy-mode <auto|off|large|all> (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str());
printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -812,12 +814,14 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
std::vector<llama_lazy_mode> modes;
for (const auto & m : p) {
llama_lazy_mode mode;
if (m == "on") {
mode = LLAMA_LAZY_MODE_ON;
} else if (m == "auto") {
if (m == "auto") {
mode = LLAMA_LAZY_MODE_AUTO;
} else if (m == "off") {
mode = LLAMA_LAZY_MODE_OFF;
} else if (m == "large") {
mode = LLAMA_LAZY_MODE_LARGE;
} else if (m == "all") {
mode = LLAMA_LAZY_MODE_ALL;
} else {
invalid_param = true;
break;