From a56cfe1bcf1ad0ee7cc6d2101c08d3068f0ee855 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 2 Sep 2026 12:39:06 +0200 Subject: [PATCH] common : resolve - download tags and list cached sidecars A Q4_0-mtp style tag now resolves the sidecar file when no model file matches it, so a solo draft or mmproj download actually pulls the file. Cached sidecar files list as their own entries so the state survives a restart, and removing such a tag deletes only the sidecar. Assisted-by: pi:zai-org/GLM-5.3 --- common/download.cpp | 279 +++++++++++++++++++++++++------- tests/test-model-resolution.cpp | 141 +++++++++++++++- 2 files changed, 358 insertions(+), 62 deletions(-) diff --git a/common/download.cpp b/common/download.cpp index 4b28a708c8..9459c3b29e 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -8,6 +8,7 @@ #include "json.h" #include +#include #include #include #include @@ -534,15 +535,18 @@ static gguf_split_info get_gguf_split_info(const std::string & path) { } // Q4_0 -> 4, F16 -> 16, NVFP4 -> 4, Q8_K_M -> 8, etc -static int extract_quant_bits(const std::string & filename) { - auto split = get_gguf_split_info(filename); +static int quant_bits_from_tag(const std::string & tag) { + auto pos = tag.find_first_of("0123456789"); - auto pos = split.tag.find_first_of("0123456789"); if (pos == std::string::npos) { return 0; } - return std::stoi(split.tag.substr(pos)); + return std::stoi(tag.substr(pos)); +} + +static int extract_quant_bits(const std::string & filename) { + return quant_bits_from_tag(get_gguf_split_info(filename).tag); } static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, @@ -563,12 +567,127 @@ static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, return result; } -// pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp"), +// sidecar filename tokens, as used in `-` download tags, +// e.g. `Q4_0-mtp` for `mtp-Model-Q4_0.gguf`, `BF16-mmproj` for `mmproj-BF16.gguf` +static const std::vector sidecar_tokens = { + "mtp", "eagle3", "dflash", "dspark", "mmproj", "imatrix", +}; + +static bool iequals(const std::string & a, const std::string & b) { + return a.size() == b.size() && + std::equal(a.begin(), a.end(), b.begin(), [](char x, char y) { + return std::tolower((unsigned char) x) == std::tolower((unsigned char) y); + }); +} + +// split a `-` tag into its parts, e.g. `Q4_0-mtp` -> {`Q4_0`, `mtp`}; +// a bare sidecar tag (`mtp`) yields an empty quant; no sidecar yields an empty token +static std::pair split_sidecar_tag(const std::string & tag) { + for (const auto & t : sidecar_tokens) { + if (tag.size() > t.size() + 1 && iequals(tag.substr(tag.size() - t.size() - 1), "-" + t)) { + return { tag.substr(0, tag.size() - t.size() - 1), t }; + } + if (iequals(tag, t)) { + return { "", t }; + } + } + return { tag, "" }; +} + +// filename with directory and extension removed, e.g. `sub/mtp-Model-Q4_0.gguf` -> `mtp-Model-Q4_0` +static std::string stem_of(const std::string & path) { + std::string base = path; + if (auto pos = base.rfind('/'); pos != std::string::npos) { + base = base.substr(pos + 1); + } + string_remove_suffix(base, ".gguf"); + return base; +} + +// a sidecar file carries its token as a name segment in any position and case, +// e.g. `mmproj-Model-F16.gguf`, `Model-mtp-Q4_0.gguf`, `Model-Q4_0-mtp.gguf` +// or the short `mmproj-F16.gguf`; the token must be a whole segment, so an +// unrelated name that merely contains it (`smtp-Model.gguf`) is a plain model +static std::string sidecar_token_of(const std::string & path) { + std::string base = stem_of(path); + + for (char & c : base) { + c = (char) std::tolower((unsigned char) c); + } + if (base.empty()) { + return {}; + } + + for (const auto & t : sidecar_tokens) { + if (base == t) { + return t; // the sidecar file itself, e.g. `imatrix.gguf` + } + if (base.rfind(t + "-", 0) == 0) { + return t; // `mtp-Model-Q4_0.gguf` + } + if (base.find("-" + t + "-") != std::string::npos) { + return t; // `Model-mtp-Q4_0.gguf` + } + // `Model-Q4_0-mtp.gguf`, optionally with a `-draft` tail + if (string_ends_with(base, "-" + t) || string_ends_with(base, "-" + t + "-draft")) { + return t; + } + } + return {}; +} + +// name with the sidecar token segment removed, lowercased for tag parsing, +// e.g. `Model-MTP-Q4_0` -> `model-q4_0` +static std::string strip_sidecar_token(const std::string & base, const std::string & token) { + std::string lower = base; + + for (char & c : lower) { + c = (char) std::tolower((unsigned char) c); + } + + if (lower == token) { + return {}; + } + if (lower.rfind(token + "-", 0) == 0) { + return lower.substr(token.size() + 1); + } + const std::string seg = "-" + token + "-"; + if (auto pos = lower.find(seg); pos != std::string::npos) { + return lower.substr(0, pos) + "-" + lower.substr(pos + seg.size()); + } + if (string_ends_with(lower, "-" + token + "-draft")) { + return lower.substr(0, lower.size() - token.size() - 7); + } + if (string_ends_with(lower, "-" + token)) { + return lower.substr(0, lower.size() - token.size() - 1); + } + return lower; +} + +// the quant a sidecar file belongs to, from its name with the token stripped; +// a short-form name (`mmproj-F16.gguf`) leaves the bare quant, which has no +// `-` separator for the tag regex, so it becomes the tag directly +static std::string sidecar_quant(const std::string & path, const std::string & token) { + std::string name = strip_sidecar_token(stem_of(path), token); + std::string tag = get_gguf_split_info(name + ".gguf").tag; + + if (tag.empty() && name.find('-') == std::string::npos) { + for (char & c : name) { + c = (char) std::toupper((unsigned char) c); + } + tag = name; + } + + return tag; +} + +// pick the best sibling GGUF carrying the sidecar `token` (e.g. "mmproj" / "mtp"), // preferring deeper shared directory prefix with the model, then exact `tag` match, // then closest quantization to the tag when given, or to the model otherwise +// an empty `model` skips the directory constraint: the sidecar is matched by tag alone static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, const std::string & model, - const std::string & keyword, + const std::string & token, const std::string & tag = "") { hf_cache::hf_file best; size_t best_depth = 0; @@ -589,32 +708,32 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, model_bits = extract_quant_bits(model); } auto model_parts = string_split(model, '/'); - auto model_dir = model_parts.end() - 1; for (const auto & f : files) { - if (!string_ends_with(f.path, ".gguf") || - f.path.find(keyword) == std::string::npos) { + if (sidecar_token_of(f.path) != token) { continue; } auto sib_parts = string_split(f.path, '/'); auto sib_dir = sib_parts.end() - 1; - auto [_, dir] = std::mismatch(model_parts.begin(), model_dir, - sib_parts.begin(), sib_dir); - if (dir != sib_dir) { - continue; + size_t depth = 0; + if (!model.empty()) { + auto model_dir = model_parts.end() - 1; + auto [_, dir] = std::mismatch(model_parts.begin(), model_dir, + sib_parts.begin(), sib_dir); + if (dir != sib_dir) { + continue; + } + depth = dir - sib_parts.begin(); } - size_t depth = dir - sib_parts.begin(); - auto bits = extract_quant_bits(f.path); - auto diff = std::abs(bits - model_bits); - - std::string path_upper = f.path; - for (char & c : path_upper) { - c = (char) std::toupper((unsigned char) c); - } - bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos; + // rank by the quant the sidecar belongs to, with the token segment + // stripped from its name + auto tag = sidecar_quant(f.path, token); + auto bits = quant_bits_from_tag(tag); + auto diff = std::abs(bits - model_bits); + bool exact = !tag_upper.empty() && tag == tag_upper; if (!found || depth > best_depth || (depth == best_depth && exact && !best_exact) || @@ -637,43 +756,31 @@ static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, const std::string & model, const std::string & tag = "") { - return find_best_sibling(files, model, "mtp-", tag); + return find_best_sibling(files, model, "mtp", tag); } static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, const std::string & model, const std::string & tag = "") { - return find_best_sibling(files, model, "eagle3-", tag); + return find_best_sibling(files, model, "eagle3", tag); } static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, const std::string & model, const std::string & tag = "") { - return find_best_sibling(files, model, "dflash-", tag); + return find_best_sibling(files, model, "dflash", tag); } static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, const std::string & model, const std::string & tag = "") { - return find_best_sibling(files, model, "dspark-", tag); + return find_best_sibling(files, model, "dspark", tag); } +// a plain model file: a GGUF whose name carries no sidecar token segment, +// so `smtp-Model.gguf` counts and every sidecar form does not static bool gguf_filename_is_model(const std::string & filepath) { - if (!string_ends_with(filepath, ".gguf")) { - return false; - } - - std::string filename = filepath; - if (auto pos = filename.rfind('/'); pos != std::string::npos) { - filename = filename.substr(pos + 1); - } - - return filename.find("mmproj") == std::string::npos && - filename.find("imatrix") == std::string::npos && - filename.find("mtp-") == std::string::npos && - filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos && - filename.find("dspark-") == std::string::npos; + return string_ends_with(filepath, ".gguf") && sidecar_token_of(filepath).empty(); } static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, @@ -765,8 +872,27 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } } else { primary = find_best_model(all, tag); + + // a `-` tag (e.g. `Q4_0-mtp`) requests that sidecar alone; + // every token-bearing file is a sidecar (find_best_model skips them), + // so the sidecar resolves here whenever the tag carries one + auto [base_tag, sidecar] = split_sidecar_tag(tag); + + if (primary.path.empty() && !sidecar.empty()) { + auto found = find_best_sibling(all, "", sidecar, base_tag); + + if (!found.path.empty()) { + if (sidecar == "mtp") plan.mtp = found; + else if (sidecar == "eagle3") plan.eagle3 = found; + else if (sidecar == "dflash") plan.dflash = found; + else if (sidecar == "dspark") plan.dspark = found; + else plan.mmproj = found; + } + } + // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { + if (primary.path.empty() && sidecar.empty() && + !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; @@ -794,7 +920,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & plan.dspark = find_best_dspark(all, primary.path, tag); } - if (primary.path.empty() && + if (primary.path.empty() && plan.mmproj.local_path.empty() && plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); @@ -968,17 +1094,34 @@ std::vector common_list_cached_models() { auto files = hf_cache::get_cached_files(); for (const auto & f : files) { - auto split = get_gguf_split_info(f.path); - if (split.index != 1 || split.tag.empty() || - split.prefix.find("mmproj") != std::string::npos || - split.prefix.find("mtp-") != std::string::npos || - split.prefix.find("eagle3-") != std::string::npos || - split.prefix.find("dflash-") != std::string::npos || - split.prefix.find("dspark-") != std::string::npos) { - continue; + // a sidecar file is listed under its own `-` tag, so a + // cached `mtp-Model-Q4_0.gguf`, `Model-mtp-Q4_0.gguf`, `Model-Q4_0-mtp.gguf` + // or short `mmproj-F16.gguf` shows up as `:Q4_0-mtp` / `:F16-mmproj`; + // files whose name carries no token stay loadable models + auto token = sidecar_token_of(f.path); + + std::string tag; + if (token.empty()) { + auto split = get_gguf_split_info(f.path); + + if (split.index != 1 || split.tag.empty()) { + continue; + } + + tag = split.tag; + } else { + tag = sidecar_quant(f.path, token); + + // a bare sidecar file has no tag to request it by, stay hidden + if (tag.empty()) { + continue; + } + + tag += "-" + token; } - if (seen.insert(f.repo_id + ":" + split.tag).second) { - result.push_back({f.repo_id, split.tag}); + + if (seen.insert(f.repo_id + ":" + tag).second) { + result.push_back({f.repo_id, tag}); } } @@ -1014,7 +1157,15 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return hf_cache::remove_cached_repo(repo_id); } - std::string tag_upper = tag; + // a `-` tag (`Q4_0-mtp`) targets that sidecar alone; a bare + // sidecar tag (`mtp`) is ambiguous across quants and is rejected + auto [base_tag, sidecar] = split_sidecar_tag(tag); + if (!sidecar.empty() && base_tag.empty()) { + LOG_ERR("%s: bare sidecar tag '%s': use `-`\n", __func__, tag.c_str()); + return false; + } + + std::string tag_upper = sidecar.empty() ? tag : base_tag; for (char & c : tag_upper) { c = (char) std::toupper((unsigned char) c); } @@ -1024,13 +1175,25 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return false; } - // collect snapshot entries whose tag matches + // collect the snapshot entries the tag selects; sidecar files keep their + // own tags, so a plain quant tag never removes them std::vector to_remove; for (const auto & f : files) { - auto split = get_gguf_split_info(f.path); - if (split.tag == tag_upper) { - to_remove.emplace_back(f.local_path); + auto token = sidecar_token_of(f.path); + + if (sidecar.empty()) { + if (!token.empty()) { + continue; + } + if (get_gguf_split_info(f.path).tag != tag_upper) { + continue; + } + } else { + if (token != sidecar || sidecar_quant(f.path, sidecar) != tag_upper) { + continue; + } } + to_remove.emplace_back(f.local_path); } if (to_remove.empty()) { diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 5191e77514..18e6907003 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -139,7 +140,7 @@ static const std::vector hole = { "dflash-model-Q8_0.gguf", }; -// unsloth-style naming with UD quants and a suffix MTP file +// unsloth-style naming with UD quants and an uppercase infix MTP head static const std::vector unsloth = { "model-UD-Q8_K_XL.gguf", "mmproj-BF16.gguf", @@ -180,6 +181,13 @@ static const std::vector spark = { "dspark-model-MXFP4.gguf", }; +// main weights plus a trailing-form mtp head, as in unsloth +// gemma-4-E2B-it-GGUF (Model-Q4_0-mtp.gguf) +static const std::vector trailing = { + "model-BF16.gguf", + "model-BF16-mtp.gguf", +}; + // dspark outranks dflash in the type auto-selection static const std::vector dspark_dflash = { "model-Q8_0.gguf", @@ -223,7 +231,7 @@ static const plan_case plan_cases[] = { // no tag and no default match falls back to the first model in the listing {"unsloth fallback", unsloth, "test/repo", "", true, true, "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"}, - "mmproj-BF16.gguf", "", "", "", ""}, + "mmproj-BF16.gguf", "model-MTP-BF16.gguf", "", "", ""}, // explicit hf_file picks that exact file {"flat hf_file", flat, "test/repo", "model-BF16.gguf", false, false, @@ -258,10 +266,11 @@ static const plan_case plan_cases[] = { "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"}, "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""}, - // the mtp- keyword is case sensitive, a suffix -MTP file is not discovered + // the uppercase infix MTP head is a sidecar too; it resolves at the + // nearest quant to the tag when the exact one does not exist {"unsloth suffix mtp", unsloth, "test/repo:Q8_K_XL", "", true, false, "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"}, - "mmproj-BF16.gguf", "", "", "", ""}, + "mmproj-BF16.gguf", "model-MTP-BF16.gguf", "", "", ""}, // vendor prefixes and the dot quant convention both match the tag, // first match wins between two files at the same quant @@ -283,6 +292,45 @@ static const plan_case plan_cases[] = { {"spark tag sidecar", spark, "test/repo:BF16", "", true, false, "", {}, "", "", "", "", "dspark-model-BF16.gguf"}, + + // a `-` tag resolves that sidecar alone, without a primary + {"hole quant-sidecar tag", hole, "test/repo:Q4_0-mtp", "", false, false, + "", {}, + "", "mtp-model-Q4_0.gguf", "", "", ""}, + + {"spark quant-sidecar tag", spark, "test/repo:BF16-dspark", "", false, false, + "", {}, + "", "", "", "", "dspark-model-BF16.gguf"}, + + // a bare sidecar tag resolves the sidecar at any quant + {"hole bare sidecar tag", hole, "test/repo:mtp", "", false, false, + "", {}, + "", "mtp-model-Q4_0.gguf", "", "", ""}, + + // a trailing-form sidecar resolves as the sidecar alone, never as a primary + {"trailing quant-sidecar tag", trailing, "test/repo:BF16-mtp", "", false, false, + "", {}, + "", "model-BF16-mtp.gguf", "", "", ""}, + + // the plain tag resolves the plain model; the trailing-token file is skipped + {"trailing plain tag", trailing, "test/repo:BF16", "", false, false, + "model-BF16.gguf", {"model-BF16.gguf"}, + "", "", "", "", ""}, + + // a short-form sidecar (`mmproj-F16.gguf`) resolves by its bare quant tag + {"unsloth quant-sidecar tag", unsloth, "test/repo:BF16-mmproj", "", false, false, + "", {}, + "mmproj-BF16.gguf", "", "", "", ""}, + + // an uppercase infix sidecar resolves through the lowercase tag + {"unsloth uppercase infix tag", unsloth, "test/repo:BF16-mtp", "", false, false, + "", {}, + "", "model-MTP-BF16.gguf", "", "", ""}, + + // a sidecar token in the middle of the name resolves as the sidecar alone + {"subdir quant-sidecar tag", subdir, "test/repo:Q8_0-mtp", "", false, false, + "", {}, + "", "model-mtp-Q8_0.gguf", "", "", ""}, }; static void check_plan(const plan_case & c) { @@ -468,6 +516,90 @@ static void test_task_assembly() { g_repos.clear(); } +// +// cache listing and removal against the isolated LLAMA_CACHE, using the +// same filename grammar the plan tests exercise above +// + +static void cache_put(const std::string & repo, const std::string & path) { + namespace fs = std::filesystem; + auto local = fs::path(cached(repo, path)); + fs::create_directories(local.parent_path()); + { std::ofstream(local) << "gguf"; } + auto repo_dir = local.parent_path().parent_path().parent_path(); + auto refs = repo_dir / "refs"; + fs::create_directories(refs); + auto ref = refs / "main"; + if (!fs::exists(ref)) { + std::ofstream(ref) << COMMIT << "\n"; + } +} + +static bool cache_lists(const std::string & repo_tag) { + for (const auto & e : common_list_cached_models()) { + if (e.to_string() == repo_tag) { + return true; + } + } + return false; +} + +static void test_cache_listing_and_remove() { + namespace fs = std::filesystem; + const std::string repo = "test/eh"; + + printf("test-model-resolution: cache listing and removal\n"); + + g_context = "cache fixture"; + cache_put(repo, "gemma-4-E2B-it-BF16.gguf"); // main weights + cache_put(repo, "gemma-4-E2B-it-BF16-mtp.gguf"); // trailing-form draft head + cache_put(repo, "gemma-4-31B-it-MTP-BF16.gguf"); // uppercase infix draft head + cache_put(repo, "mmproj-gemma-4-E2B-it-BF16.gguf"); // mmproj sidecar + cache_put(repo, "mmproj-F16.gguf"); // short-form mmproj + cache_put(repo, "model-mtp-Q8_0.gguf"); // mid-name mtp sidecar + + // a sidecar is listed under `-` in every form and case it + // can be named; only the token-less main weights stay a loadable model + REQUIRE(cache_lists("test/eh:BF16")); + REQUIRE(cache_lists("test/eh:BF16-mtp")); + REQUIRE(cache_lists("test/eh:BF16-mmproj")); + REQUIRE(cache_lists("test/eh:F16-mmproj")); + REQUIRE(cache_lists("test/eh:Q8_0-mtp")); + REQUIRE(!cache_lists("test/eh:MTP")); + + // a plain quant tag removes the model files and leaves every sidecar, + // whatever form or case its name carries + g_context = "remove plain quant"; + REQUIRE(common_download_remove("test/eh:BF16")); + REQUIRE(!fs::exists(cached(repo, "gemma-4-E2B-it-BF16.gguf"))); + REQUIRE(fs::exists(cached(repo, "gemma-4-E2B-it-BF16-mtp.gguf"))); + REQUIRE(fs::exists(cached(repo, "gemma-4-31B-it-MTP-BF16.gguf"))); + REQUIRE(fs::exists(cached(repo, "mmproj-gemma-4-E2B-it-BF16.gguf"))); + REQUIRE(fs::exists(cached(repo, "mmproj-F16.gguf"))); + REQUIRE(fs::exists(cached(repo, "model-mtp-Q8_0.gguf"))); + + // a `-` tag removes every sidecar listed under it, + // across forms and cases + g_context = "remove quant-sidecar"; + REQUIRE(common_download_remove("test/eh:BF16-mtp")); + REQUIRE(!fs::exists(cached(repo, "gemma-4-E2B-it-BF16-mtp.gguf"))); + REQUIRE(!fs::exists(cached(repo, "gemma-4-31B-it-MTP-BF16.gguf"))); + REQUIRE(fs::exists(cached(repo, "mmproj-gemma-4-E2B-it-BF16.gguf"))); + + // the short form removes by its bare quant tag + REQUIRE(common_download_remove("test/eh:F16-mmproj")); + REQUIRE(!fs::exists(cached(repo, "mmproj-F16.gguf"))); + + REQUIRE(common_download_remove("test/eh:Q8_0-mtp")); + REQUIRE(!fs::exists(cached(repo, "model-mtp-Q8_0.gguf"))); + + // a bare sidecar tag is ambiguous across quants and is rejected + g_context = "remove bare sidecar"; + cache_put(repo, "mtp-Model-Q4_0.gguf"); + REQUIRE(!common_download_remove("test/eh:mtp")); + REQUIRE(fs::exists(cached(repo, "mtp-Model-Q4_0.gguf"))); +} + int main(void) { // unbuffered, so a crash cannot swallow the reports already printed setvbuf(stdout, nullptr, _IONBF, 0); @@ -496,6 +628,7 @@ int main(void) { test_plan_resolution(); test_task_assembly(); + test_cache_listing_and_remove(); server.stop(); server_thread.join();