diff --git a/common/arg.cpp b/common/arg.cpp index 2669cacd6c..bc87070457 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2736,13 +2736,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex add_opt(common_arg( {"-lzm", "--lazy-mode"}, "MODE", "on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n" - "- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n" - "- auto: on, but only for tensors larger than 4 GiB\n" - "- off: always keep them resident", + "- auto: large where mmap is supported, off otherwise\n" + "- off: always keep them resident\n" + "- large: read only large tensors (>4GiB) from disk on demand (requires mmap)\n" + "- all: read the all such tensors from disk on demand instead of keeping them resident (requires mmap)", [](common_params & params, const std::string & value) { - /**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; } - else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; } - else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; } + /**/ if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; } + else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; } + else if (value == "large") { params.lazy_mode = LLAMA_LAZY_MODE_LARGE; } + else if (value == "all") { params.lazy_mode = LLAMA_LAZY_MODE_ALL; } else { throw std::invalid_argument("invalid value"); } } ).set_env("LLAMA_ARG_LAZY_MODE")); diff --git a/include/llama.h b/include/llama.h index ef7a012c43..0fe8e8116f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -215,9 +215,10 @@ extern "C" { LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); enum llama_lazy_mode { - LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front - LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap) - LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap) + LLAMA_LAZY_MODE_AUTO = 0, // pick a good default for the system: LARGE where mmap is supported, else OFF + LLAMA_LAZY_MODE_OFF = 1, // always read the whole tensor up front + LLAMA_LAZY_MODE_LARGE = 2, // read the rows on demand, but only for marked tensors larger than 4 GiB (requires mmap) + LLAMA_LAZY_MODE_ALL = 3, // read the rows of all tensors marked by the arch on demand (requires mmap) }; enum llama_context_type { diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 49f3c4f8ea..ee17c2b2b4 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1085,7 +1085,7 @@ bool llama_model_loader::lazy_read::add(const std::string & name, const ggml_ten // do not lazy-read small tensors, it has significant overhead and is not worth it constexpr size_t auto_min_size = 4ull * 1024 * 1024 * 1024; - if (mode != LLAMA_LAZY_MODE_ON && ggml_nbytes(t) <= auto_min_size) { + if (mode != LLAMA_LAZY_MODE_ALL && ggml_nbytes(t) <= auto_min_size) { return false; } diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6344f2d8ae..e72d01768c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1421,6 +1421,19 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } + // resolve AUTO: LARGE where mmap is supported, else OFF (e.g. iGPUs); see #28160 + if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) { + ml.lazy.mode = LLAMA_LAZY_MODE_LARGE; + for (const auto & dev : devices) { + ggml_backend_dev_props props; + ggml_backend_dev_get_props(dev.dev, &props); + if (!props.caps.mmap_support) { + ml.lazy.mode = LLAMA_LAZY_MODE_OFF; + break; + } + } + } + const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO ? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE) : llama_load_mode_name(params.load_mode); diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 1fff21f701..99f421fe09 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -273,12 +273,14 @@ static const char * split_mode_str(llama_split_mode mode) { static const char * lazy_mode_str(llama_lazy_mode mode) { switch (mode) { - case LLAMA_LAZY_MODE_OFF: - return "off"; case LLAMA_LAZY_MODE_AUTO: return "auto"; - case LLAMA_LAZY_MODE_ON: - return "on"; + case LLAMA_LAZY_MODE_OFF: + return "off"; + case LLAMA_LAZY_MODE_LARGE: + return "large"; + case LLAMA_LAZY_MODE_ALL: + return "all"; default: GGML_ABORT("invalid lazy mode"); } @@ -475,7 +477,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -fa, --flash-attn (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str()); printf(" -dev, --device (default: auto)\n"); printf(" -lm, --load-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str()); - printf(" -lzm, --lazy-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str()); + printf(" -lzm, --lazy-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str()); printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); @@ -812,12 +814,14 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { std::vector modes; for (const auto & m : p) { llama_lazy_mode mode; - if (m == "on") { - mode = LLAMA_LAZY_MODE_ON; - } else if (m == "auto") { + if (m == "auto") { mode = LLAMA_LAZY_MODE_AUTO; } else if (m == "off") { mode = LLAMA_LAZY_MODE_OFF; + } else if (m == "large") { + mode = LLAMA_LAZY_MODE_LARGE; + } else if (m == "all") { + mode = LLAMA_LAZY_MODE_ALL; } else { invalid_param = true; break;