From f44d2ee27cb992cf92f8b4fca7754160b19f24ff Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Fri, 31 Jul 2026 16:33:06 +0800 Subject: [PATCH] tools : add FA-vec (Q,NE) sweep, compression and table emit --- tools/tuning/fa-vec.cpp | 382 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 377 insertions(+), 5 deletions(-) diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index 38060576ab..ac235920b1 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -187,12 +187,384 @@ static std::vector fa_legal_ne(int dk, int dv) { return r; } -bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts) { - fprintf(stderr, "fa-vec tuner: sweep not implemented yet\n"); +using set_override_t = void (*)(int, int); +using clear_override_t = void (*)(void); +using bucket_t = int (*)(int64_t); +using baseline_ne_t = int (*)(int, int); +using device_token_t = const char * (*)(ggml_backend_dev_t); - (void) backend; - (void) dev; - (void) opts; +struct fa_procs { + set_override_t set_ov = nullptr; + clear_override_t clr_ov = nullptr; + bucket_t ne11_bucket = nullptr; + bucket_t ne01_bucket = nullptr; + baseline_ne_t baseline_ne = nullptr; + device_token_t dev_token = nullptr; + + bool ok() const { + return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; + } +}; + +static fa_procs fa_resolve_procs(ggml_backend_dev_t dev) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + + fa_procs p; + p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); + p.clr_ov = (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); + p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); + p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); + p.baseline_ne = (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); + p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token"); + + return p; +} + +static const char * fa_type_token(ggml_type t) { + switch (t) { + case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; + case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; + case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; + case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; + case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; + default: return "GGML_TYPE_F16"; + } +} + +// "f16,q4_0" -> does it contain ggml_type_name(t)? null filter accepts everything +static bool fa_filter_has(const char * filter, const char * name) { + if (!filter) { + return true; + } + + const std::string f = std::string(",") + filter + ","; + + return f.find(std::string(",") + name + ",") != std::string::npos; +} + +struct fa_cand { int Q, NE; }; + +struct fa_point { // one swept grid point with its candidate times + int dk, dv, ne11, ne01; + std::vector t; // indexed like the shape's candidate list +}; + +// candidate list for one shape, identical for every grid point of it. base_i is the index of +// the (Q=1, baseline NE) candidate: the anchor config, and what the tuning gates compare to. +static std::vector fa_build_cands(const fa_procs & procs, int dk, int dv, int & base_i) { + const int base_ne = procs.baseline_ne(dk, dv); + + std::vector cands; + base_i = -1; + for (int ne : fa_legal_ne(dk, dv)) { + for (int Q : { 1, 2, 4 }) { + if (Q == 1 && ne == base_ne) { + base_i = (int) cands.size(); + } + cands.push_back({ Q, ne }); + } + } + GGML_ASSERT(base_i >= 0); + + return cands; +} + +bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts) { + const fa_procs procs = fa_resolve_procs(dev); + if (!procs.ok()) { + fprintf(stderr, "error: metal fa_vec tuning procs unavailable\n"); + return false; + } + + const char * dev_token = procs.dev_token(dev); + + struct shape_t { int dk, dv; }; + const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, + { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; + const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives + const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // point buckets (1-4) + tail mod-4 cycle + anchor + const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, + GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; + + const double TUNE_TAU = 0.05; // max POINTWISE regret to ride a domain default + const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune at all + + cooldown_opts cool; + cool.enabled = opts.cooldown; + cool.drift = opts.cool_drift; + cool.eps = opts.cool_eps; + cool.max_wait = opts.cool_max_wait; + cool.max_retry = opts.cool_max_retry; + + fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", + opts.seed, opts.reps, cool.enabled ? "on" : "off", + cool.drift, cool.eps, cool.max_wait, cool.max_retry); + fprintf(stderr, "device token: %s\n", dev_token); + + int n_untrusted = 0; + + printf("// ==== BEGIN fa_vec_tuned_table rows (%s) ====\n", dev_token); + + for (ggml_type type_kv : types) { + if (!fa_filter_has(opts.dtype_filter, ggml_type_name(type_kv))) { + continue; + } + + fprintf(stderr, "\n### dtype=%s\n", ggml_type_name(type_kv)); + + std::vector pts; + + for (auto s : shapes) { + if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) { + continue; + } + + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + + for (int ne11 : ne11_rep) { + for (int ne01 : ne01_rep) { + const fa_shape sh = { s.dk, s.dv, ne01, ne11, type_kv }; + + perf_cell cell = build_perf_cell(backend, + [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); }, + [&](ggml_context * ctx) { fa_init_tensors(ctx, sh, opts.seed); }, + [&](ggml_tensor *) { return fa_op_flops(sh); }); + + if (!cell.ok) { + continue; + } + + // randomize candidate order to decorrelate thermal drift across the cell + std::vector order((size_t) cands.size()); + for (size_t i = 0; i < order.size(); ++i) { + order[i] = (int) i; + } + std::shuffle(order.begin(), order.end(), std::mt19937(opts.seed)); + + char label[128]; + snprintf(label, sizeof(label), "dk=%d ne11=%d", s.dk, ne11); + + cell_result r = measure_cell(backend, cell, opts.reps, + (int) cands.size(), order, + [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); }, + [&]() { procs.clr_ov(); }, + base_i, cool, label); + + // per-cell noise floor: spread of the repeated same-config anchor + if (r.anchor_min > 0.0) { + fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", + s.dk, s.dv, ne11, ne01, 100.0*(r.anchor_max - r.anchor_min)/r.anchor_min); + } + + if (!r.trusted) { + n_untrusted++; + fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", + s.dk, s.dv, ne11, ne01); + continue; + } + + int best_i = -1; + for (size_t i = 0; i < cands.size(); ++i) { + if (r.t[i] > 0.0 && (best_i < 0 || r.t[i] < r.t[best_i])) { + best_i = (int) i; + } + } + const double base_t = r.t[base_i]; + const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t*0.98; + + fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", + ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); + for (size_t i = 0; i < cands.size(); ++i) { + fprintf(stderr, " Q%dNE%d=%.1f%s", cands[i].Q, cands[i].NE, r.t[i], + (int) i == best_i ? "*" : ""); + } + if (keep) { + fprintf(stderr, " => Q%d,NE%d %.2fx\n", + cands[best_i].Q, cands[best_i].NE, base_t/r.t[best_i]); + } else { + fprintf(stderr, " => baseline\n"); + } + + pts.push_back({ s.dk, s.dv, ne11, ne01, r.t }); + } + } + } + + // compress into pasteable rows. per (dk,dv) and ne01 domain {decode==1, batch>=2}, + // emit one ne11-collapsed default cfg (ne11_b=-1) plus a per-bucket exception wherever + // the default's pointwise regret vs the bucket target, or its aggregate slowdown vs + // baseline, exceeds TUNE_TAU. + std::vector rows_out; + char rbuf[192]; + + for (auto s : shapes) { + if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) { + continue; + } + + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + + struct bkt_t { + int b11, b01, Ti; + std::vector agg; + double base_agg; + std::vector bp; + }; + + // bucket the grid points with the runtime's bucketers, so keys match fa_vec_pick. + // short-KV points (ne11 bucket 0) are dropped: the runtime serves those from baseline. + std::set> seen; + for (const auto & p : pts) { + if (p.dk != s.dk || p.dv != s.dv) { + continue; + } + const int b11 = procs.ne11_bucket(p.ne11); + if (b11 == 0) { + continue; + } + seen.insert({ b11, procs.ne01_bucket(p.ne01) }); + } + + std::vector bks; + for (const auto & bb : seen) { + const int b11 = bb.first, b01 = bb.second; + + std::vector bp; + for (const auto & p : pts) { + if (p.dk == s.dk && p.dv == s.dv && + procs.ne11_bucket(p.ne11) == b11 && procs.ne01_bucket(p.ne01) == b01) { + bp.push_back(&p); + } + } + + std::vector agg(cands.size(), 0.0), worst(cands.size(), 0.0); + for (const auto * p : bp) { + double bestt = 0.0; + for (size_t i = 0; i < cands.size(); ++i) { + if (p->t[i] > 0.0 && (bestt == 0.0 || p->t[i] < bestt)) { + bestt = p->t[i]; + } + } + for (size_t i = 0; i < cands.size(); ++i) { + agg[i] += p->t[i]; + if (p->t[i] > 0.0 && bestt > 0.0) { + worst[i] = std::max(worst[i], p->t[i]/bestt); + } + } + } + + int robust = 0; + for (size_t i = 1; i < cands.size(); ++i) { + if (worst[i] < worst[robust] || + (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || + (cands[i].Q == cands[robust].Q && cands[i].NE < cands[robust].NE)))) { + robust = (int) i; + } + } + + const bool tune = robust != base_i && agg[base_i] > 0.0 && agg[robust] > 0.0 && + agg[base_i]/agg[robust] >= TUNE_THETA; + + bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); + } + + // bucket coverage: a hardcoded sampling grid can't produce a wrong key, only miss + // a bucket, so report what each bucket actually got + for (const auto & b : bks) { + fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", + s.dk, s.dv, b.b11, b.b01, b.bp.size()); + if (b.bp.empty()) { + fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", + s.dk, s.dv, b.b11, b.b01); + } + } + + // pointwise regret of default cfg d vs the bucket target: a ratio-of-sums lets a + // default that wins on aligned ne01 hide a large penalty on a misaligned point + auto reg_pointwise = [&](const bkt_t * b, int d) { + double r = 0.0; + for (const auto * p : b->bp) { + const double td = p->t[d], tT = p->t[b->Ti]; + if (td > 0.0 && tT > 0.0) { + r = std::max(r, td/tT - 1.0); + } + } + return r; + }; + + for (int dom = 0; dom <= 1; ++dom) { // 0 = decode (ne01==1), 1 = batch (ne01>=2) + std::vector db; + for (const auto & b : bks) { + if ((dom == 0) == (b.b01 == 0)) { + db.push_back(&b); + } + } + if (db.empty()) { + continue; + } + + // default cfg = the one minimizing (#rows, total achieved time, Q, NE) + int bestD = -1, bestRows = 1 << 30; + double bestTot = 0.0; + for (size_t d = 0; d < cands.size(); ++d) { + int rows = ((int) d != base_i) ? 1 : 0; + double tot = 0.0; + for (const auto * b : db) { + const double reg = reg_pointwise(b, (int) d); + const double slow = b->base_agg > 0.0 ? b->agg[d]/b->base_agg - 1.0 : 0.0; + if (reg > TUNE_TAU || slow > TUNE_TAU) { + rows++; + tot += b->agg[b->Ti]; + } else { + tot += b->agg[d]; + } + } + const bool better = bestD < 0 || rows < bestRows || + (rows == bestRows && (tot < bestTot || + (tot == bestTot && (cands[d].Q < cands[bestD].Q || + (cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE))))); + if (better) { + bestD = (int) d; + bestRows = rows; + bestTot = tot; + } + } + + const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH + if (bestD != base_i) { + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", + dev_token, fa_type_token(type_kv), s.dk, s.dv, dom_id, + cands[bestD].Q, cands[bestD].NE); + rows_out.emplace_back(rbuf); + } + for (const auto * b : db) { + const double reg = reg_pointwise(b, bestD); + const double slow = b->base_agg > 0.0 ? b->agg[bestD]/b->base_agg - 1.0 : 0.0; + if (reg <= TUNE_TAU && slow <= TUNE_TAU) { + continue; // rides the default / baseline + } + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", + dev_token, fa_type_token(type_kv), s.dk, s.dv, b->b11, b->b01, + cands[b->Ti].Q, cands[b->Ti].NE); + rows_out.emplace_back(rbuf); + } + } + } + + printf("\n // ---- %s: %zu rows ----\n", ggml_type_name(type_kv), rows_out.size()); + for (const auto & r : rows_out) { + printf("%s\n", r.c_str()); + } + fflush(stdout); + } + + printf("// ==== END fa_vec_tuned_table rows (%s) ====\n", dev_token); + + if (n_untrusted > 0) { + fprintf(stderr, "\n%d cells excluded as untrusted (see DROP lines above)\n", n_untrusted); + } return true; }