mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-04 02:37:27 +02:00
Georgi Gerganov
4af6790656
The fusion test is invoked with a specific device name (e.g. MTL0), but its output - the recorded baseline and the header it writes - should be named after the backend base name (e.g. MTL, via ggml_backend_reg_name), since the counters depend on the backend, not on the specific device index. Rename the committed baseline to MTL.tsv. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731
491 lines
19 KiB
C++
491 lines
19 KiB
C++
// test-fusion: verify the Metal fusion logic against a per-device baseline.
|
|
//
|
|
// for every dummy model generated by test-llama-archs, the tool runs the model on a single
|
|
// device (single-threaded encoding, n_cb = 0) with fusion enabled and disabled, and reports:
|
|
// - the per-fusion-type counters for each mode (prefill / decode)
|
|
// - the NMSE between the fused and unfused logits
|
|
// - the NMSE between the device and a CPU reference
|
|
//
|
|
// the per-fusion-type counters are compared against a per-device baseline file (TSV) so a
|
|
// fusion pattern that silently stops matching (or fires when it should not) is caught as a
|
|
// regression.
|
|
//
|
|
// usage:
|
|
// test-fusion --models DIR --device MTL0 --record baseline.tsv # generate a baseline
|
|
// test-fusion --models DIR --device MTL0 --check baseline.tsv # validate against it
|
|
// test-fusion --model FILE --device MTL0 --check baseline.tsv # validate a single model
|
|
|
|
#include "common.h"
|
|
#include "log.h"
|
|
#include "llama-cpp.h"
|
|
|
|
#include "ggml.h"
|
|
#include "gguf.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <random>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism
|
|
// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging
|
|
// exports these exact names.
|
|
typedef void ( * fusion_stats_init_t) (ggml_backend_dev_t);
|
|
typedef void ( * fusion_stats_reset_t)(ggml_backend_dev_t);
|
|
typedef int ( * fusion_stats_get_t) (ggml_backend_dev_t, const char **, uint64_t *, int);
|
|
typedef void ( * fusion_set_enabled_t)(ggml_backend_dev_t, bool);
|
|
|
|
static bool silent_model_load_progress(float, void *) {
|
|
return true;
|
|
}
|
|
|
|
struct gguf_context_ptr {
|
|
gguf_context * ctx;
|
|
gguf_context_ptr(gguf_context * c) : ctx(c) {}
|
|
~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } }
|
|
gguf_context * get() const { return ctx; }
|
|
gguf_context_ptr(const gguf_context_ptr &) = delete;
|
|
gguf_context_ptr & operator=(const gguf_context_ptr &) = delete;
|
|
};
|
|
|
|
// NMSE between two vectors (same as tests/test-llama-archs.cpp)
|
|
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
|
|
GGML_ASSERT(a.size() == b.size());
|
|
double mse_a_b = 0.0;
|
|
double mse_a_0 = 0.0;
|
|
|
|
for (size_t i = 0; i < a.size(); i++) {
|
|
const float a_i = a[i];
|
|
const float b_i = b[i];
|
|
|
|
mse_a_b += (a_i - b_i) * (a_i - b_i);
|
|
mse_a_0 += a_i * a_i;
|
|
}
|
|
|
|
return mse_a_b / mse_a_0;
|
|
}
|
|
|
|
// deterministic token sequence
|
|
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {
|
|
std::mt19937 gen(seed);
|
|
std::uniform_int_distribution<> dis(0, n_vocab - 1);
|
|
std::vector<llama_token> ret;
|
|
ret.reserve(n_tokens);
|
|
for (uint32_t i = 0; i < n_tokens; i++) {
|
|
ret.push_back(dis(gen));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static std::string get_arch(const std::string & path) {
|
|
gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr };
|
|
gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params));
|
|
if (!ctx.get()) {
|
|
throw std::runtime_error("failed to read gguf: " + path);
|
|
}
|
|
const int idx = gguf_find_key(ctx.get(), "general.architecture");
|
|
if (idx < 0) {
|
|
return "unknown";
|
|
}
|
|
const char * val = gguf_get_val_str(ctx.get(), idx);
|
|
return val ? val : "unknown";
|
|
}
|
|
|
|
static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) {
|
|
llama_model_params model_params = llama_model_default_params();
|
|
model_params.progress_callback = silent_model_load_progress;
|
|
std::vector<ggml_backend_dev_t> devs = { dev, nullptr };
|
|
model_params.devices = devs.data();
|
|
model_params.split_mode = LLAMA_SPLIT_MODE_LAYER;
|
|
|
|
llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params));
|
|
if (!model) {
|
|
throw std::runtime_error("failed to load model: " + path);
|
|
}
|
|
return model;
|
|
}
|
|
|
|
// a fresh context (fresh state) from an already-loaded model
|
|
static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) {
|
|
llama_context_params ctx_params = llama_context_default_params();
|
|
ctx_params.n_ctx = 0;
|
|
ctx_params.n_threads = 4;
|
|
ctx_params.n_threads_batch = 4;
|
|
ctx_params.n_ubatch = n_ubatch;
|
|
ctx_params.n_batch = n_ubatch;
|
|
|
|
llama_context_ptr lctx(llama_init_from_model(model, ctx_params));
|
|
if (!lctx) {
|
|
throw std::runtime_error("failed to init context");
|
|
}
|
|
return lctx;
|
|
}
|
|
|
|
// decode all tokens in one batch; returns the logits of every token
|
|
static std::vector<float> decode_prefill(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
|
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
|
llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
common_batch_add(batch, tokens[i], i, { 0 }, true);
|
|
}
|
|
batch.n_tokens = tokens.size();
|
|
if (llama_decode(lctx, batch)) {
|
|
llama_batch_free(batch);
|
|
throw std::runtime_error("prefill decode failed");
|
|
}
|
|
|
|
std::vector<float> ret;
|
|
ret.reserve(tokens.size() * n_vocab);
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
const float * logits_ith = llama_get_logits_ith(lctx, i);
|
|
for (uint32_t j = 0; j < n_vocab; j++) {
|
|
ret.push_back(logits_ith[j]);
|
|
}
|
|
}
|
|
llama_batch_free(batch);
|
|
return ret;
|
|
}
|
|
|
|
// decode one token at a time; returns the logits of the last token of each step
|
|
static std::vector<float> decode_token_by_token(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
|
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
|
llama_batch batch = llama_batch_init(1, 0, 1);
|
|
std::vector<float> ret;
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
common_batch_clear(batch);
|
|
common_batch_add(batch, tokens[i], i, { 0 }, true);
|
|
if (llama_decode(lctx, batch)) {
|
|
llama_batch_free(batch);
|
|
throw std::runtime_error("decode failed");
|
|
}
|
|
const float * logits = llama_get_logits_ith(lctx, 0);
|
|
for (uint32_t j = 0; j < n_vocab; j++) {
|
|
ret.push_back(logits[j]);
|
|
}
|
|
}
|
|
llama_batch_free(batch);
|
|
return ret;
|
|
}
|
|
|
|
static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_dev_t dev,
|
|
std::vector<const char *> & labels, std::vector<uint64_t> & counts) {
|
|
const int n = api_stats_get(dev, nullptr, nullptr, 0);
|
|
labels.assign(n, nullptr);
|
|
counts.assign(n, 0);
|
|
api_stats_get(dev, labels.data(), counts.data(), n);
|
|
}
|
|
|
|
// one row of the per-label report
|
|
struct fusion_row {
|
|
std::string arch;
|
|
bool moe;
|
|
std::string mode;
|
|
const char * label;
|
|
uint64_t count_fused;
|
|
uint64_t count_unfused;
|
|
uint64_t expected;
|
|
double nmse_fus;
|
|
double nmse_dev;
|
|
bool ok_count; // counts match the baseline
|
|
bool ok_nmse; // nmse within epsilon
|
|
bool skip_nmse; // nmse unreliable (NaN logits or arch already broken on the device)
|
|
};
|
|
|
|
static void usage(const char * argv0) {
|
|
printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0);
|
|
printf("usage: %s [options]\n\n", argv0);
|
|
printf("options:\n");
|
|
printf(" --models DIR run over all .gguf models in a directory\n");
|
|
printf(" --model FILE run over a single model file (mutually exclusive with --models)\n");
|
|
printf(" --device NAME device to run on (e.g. MTL0, CPU)\n");
|
|
printf(" --record TSV write the golden baseline\n");
|
|
printf(" --check TSV validate the counters against a baseline (default)\n");
|
|
printf(" -h, --help show this message and exit\n");
|
|
}
|
|
|
|
int main(int argc, char ** argv) {
|
|
std::string models_dir;
|
|
std::string model_file;
|
|
std::string device_name;
|
|
std::string record_path;
|
|
std::string check_path;
|
|
|
|
for (int i = 1; i < argc; i++) {
|
|
const std::string arg = argv[i];
|
|
const auto next = [&](const char * name) -> std::string {
|
|
if (i + 1 >= argc) {
|
|
LOG_ERR("%s: %s requires an argument\n", __func__, name);
|
|
exit(1);
|
|
}
|
|
return argv[++i];
|
|
};
|
|
if (arg == "-h" || arg == "--help") {
|
|
usage(argv[0]);
|
|
exit(0);
|
|
}
|
|
if (arg == "--models") { models_dir = next("--models"); }
|
|
else if (arg == "--model") { model_file = next("--model"); }
|
|
else if (arg == "--device"){ device_name = next("--device"); }
|
|
else if (arg == "--record"){ record_path = next("--record"); }
|
|
else if (arg == "--check") { check_path = next("--check"); }
|
|
else {
|
|
LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
if (device_name.empty()) {
|
|
LOG_ERR("%s: --device NAME is required\n", __func__);
|
|
return 1;
|
|
}
|
|
if (models_dir.empty() && model_file.empty()) {
|
|
LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__);
|
|
return 1;
|
|
}
|
|
if (!models_dir.empty() && !model_file.empty()) {
|
|
LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__);
|
|
return 1;
|
|
}
|
|
if (!record_path.empty() && !check_path.empty()) {
|
|
LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__);
|
|
return 1;
|
|
}
|
|
|
|
std::vector<std::string> models;
|
|
if (!model_file.empty()) {
|
|
if (!std::filesystem::is_regular_file(model_file)) {
|
|
LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str());
|
|
return 1;
|
|
}
|
|
models.push_back(model_file);
|
|
} else {
|
|
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
|
|
LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
|
|
return 1;
|
|
}
|
|
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
|
|
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
|
|
models.push_back(entry.path().string());
|
|
}
|
|
}
|
|
std::sort(models.begin(), models.end());
|
|
|
|
if (models.empty()) {
|
|
LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
common_init();
|
|
ggml_backend_load_all();
|
|
|
|
ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str());
|
|
if (!dev) {
|
|
LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n",
|
|
__func__, device_name.c_str());
|
|
return 0;
|
|
}
|
|
|
|
// resolve the generic fusion debugging functions through the ad-hoc get_proc_address
|
|
// mechanism; a backend that does not adopt fusion debugging exports none of them
|
|
auto * reg = ggml_backend_dev_backend_reg(dev);
|
|
|
|
// output naming uses the backend base name (e.g. "MTL") rather than the specific device
|
|
// name (e.g. "MTL0") the test was invoked with
|
|
const std::string base_name = ggml_backend_reg_name(reg);
|
|
|
|
auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init");
|
|
auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset");
|
|
auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get");
|
|
auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled");
|
|
|
|
if (!api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) {
|
|
LOG_ERR("%s: device '%s' does not export the generic fusion debugging API "
|
|
"(ggml_backend_fusion_*) - cannot run the fusion regression test\n",
|
|
__func__, device_name.c_str());
|
|
return 1;
|
|
}
|
|
|
|
// enable fusions stats
|
|
api_stats_init(dev);
|
|
|
|
const bool has_counts = true;
|
|
|
|
// load the baseline (if any): key arch|moe|mode|label -> expected count
|
|
std::map<std::string, uint64_t> baseline;
|
|
if (!check_path.empty()) {
|
|
std::ifstream in(check_path);
|
|
if (!in) {
|
|
LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str());
|
|
return 1;
|
|
}
|
|
std::string line;
|
|
while (std::getline(in, line)) {
|
|
if (line.empty() || line[0] == '#') {
|
|
continue;
|
|
}
|
|
std::vector<std::string> cols;
|
|
size_t pos = 0;
|
|
while ((pos = line.find('\t')) != std::string::npos) {
|
|
cols.push_back(line.substr(0, pos));
|
|
line.erase(0, pos + 1);
|
|
}
|
|
cols.push_back(line);
|
|
if (cols.size() != 5) {
|
|
continue;
|
|
}
|
|
baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]);
|
|
}
|
|
}
|
|
|
|
std::vector<fusion_row> rows;
|
|
|
|
LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());
|
|
|
|
const size_t seed = 1;
|
|
|
|
for (const auto & model_path : models) {
|
|
const std::string arch = get_arch(model_path);
|
|
const bool moe = arch.find("moe") != std::string::npos;
|
|
|
|
llama_model_ptr model;
|
|
llama_model_ptr model_cpu;
|
|
uint32_t n_vocab = 0;
|
|
try {
|
|
model = load_model(model_path, dev);
|
|
model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU"));
|
|
n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get()));
|
|
} catch (const std::exception & e) {
|
|
LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what());
|
|
continue;
|
|
}
|
|
|
|
struct mode_cfg {
|
|
std::string name;
|
|
std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);
|
|
int n_tokens;
|
|
};
|
|
const mode_cfg modes[] = {
|
|
{ "prefill", decode_prefill, 32 },
|
|
{ "decode", decode_token_by_token, 16 },
|
|
};
|
|
|
|
for (const auto & mode : modes) {
|
|
const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed);
|
|
|
|
// CPU reference for this mode (fresh context, fresh state)
|
|
std::vector<float> logits_cpu;
|
|
try {
|
|
llama_context_ptr ctx = create_ctx(model_cpu.get(), 32);
|
|
logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens);
|
|
} catch (const std::exception & e) {
|
|
LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what());
|
|
}
|
|
|
|
// fused run on a fresh context (fresh state)
|
|
std::vector<float> logits_fused;
|
|
std::vector<const char *> labels;
|
|
std::vector<uint64_t> counts_fused;
|
|
{
|
|
llama_context_ptr ctx = create_ctx(model.get(), 32);
|
|
if (has_counts) {
|
|
api_set_enabled(dev, true);
|
|
api_stats_reset(dev);
|
|
}
|
|
logits_fused = mode.decode(model.get(), ctx.get(), tokens);
|
|
if (has_counts) {
|
|
read_counts(api_stats_get, dev, labels, counts_fused);
|
|
}
|
|
}
|
|
|
|
// unfused run on another fresh context (fresh state)
|
|
std::vector<float> logits_unfused;
|
|
std::vector<uint64_t> counts_unfused;
|
|
{
|
|
llama_context_ptr ctx = create_ctx(model.get(), 32);
|
|
if (has_counts) {
|
|
api_set_enabled(dev, false);
|
|
api_stats_reset(dev);
|
|
}
|
|
logits_unfused = mode.decode(model.get(), ctx.get(), tokens);
|
|
if (has_counts) {
|
|
read_counts(api_stats_get, dev, labels, counts_unfused);
|
|
}
|
|
}
|
|
|
|
const double nmse_fus = nmse(logits_fused, logits_unfused);
|
|
const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu);
|
|
|
|
// an arch that is already broken on the device (huge device-vs-CPU NMSE, e.g. plamo2
|
|
// on Metal) produces garbage regardless of fusion, so the fused-vs-unfused NMSE is not
|
|
// a meaningful signal - skip it and rely on the count regression check only
|
|
// TODO: run in Debug build to get an assert in `ggml-alloc.c` and fix it
|
|
const bool dev_broken = !std::isnan(nmse_dev) && nmse_dev > 1e-4;
|
|
|
|
// build the per-label rows
|
|
if (has_counts) {
|
|
for (int i = 0; i < (int) labels.size(); i++) {
|
|
if (counts_fused[i] == 0 && counts_unfused[i] == 0) {
|
|
continue;
|
|
}
|
|
const std::string key = arch + "|" + (moe ? "1" : "0") + "|" + mode.name + "|" + labels[i];
|
|
const uint64_t expected = baseline.count(key) ? baseline.at(key) : 0;
|
|
const bool skip_nmse = dev_broken || std::isnan(nmse_fus);
|
|
rows.push_back({ arch, moe, mode.name, labels[i], counts_fused[i], counts_unfused[i],
|
|
expected, nmse_fus, nmse_dev,
|
|
!check_path.empty() ? (counts_fused[i] == expected) : true,
|
|
!skip_nmse && nmse_fus <= 1e-4,
|
|
skip_nmse });
|
|
}
|
|
} else {
|
|
const bool skip_nmse = dev_broken || std::isnan(nmse_fus);
|
|
rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true,
|
|
!skip_nmse && nmse_fus <= 1e-4, skip_nmse });
|
|
}
|
|
}
|
|
|
|
LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());
|
|
}
|
|
|
|
// print the report
|
|
{
|
|
std::ofstream out(record_path);
|
|
std::ostream & os = record_path.empty() ? std::cout : out;
|
|
if (!record_path.empty()) {
|
|
os << "# test-fusion baseline for device " << base_name << "\n";
|
|
os << "# arch\tmoe\tmode\tlabel\tcount\n";
|
|
}
|
|
|
|
LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n",
|
|
"arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status");
|
|
int n_ok = 0;
|
|
int n_bad = 0;
|
|
int n_skip = 0;
|
|
for (const auto & r : rows) {
|
|
const bool ok = r.ok_count && (r.skip_nmse || r.ok_nmse);
|
|
const char * status = ok ? (r.skip_nmse ? "skip" : "ok") : "FAIL";
|
|
if (r.skip_nmse) { n_skip++; }
|
|
if (ok) { n_ok++; } else { n_bad++; }
|
|
LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n",
|
|
r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label,
|
|
(unsigned long long) r.count_fused, (unsigned long long) r.count_unfused,
|
|
(unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status);
|
|
if (!record_path.empty()) {
|
|
os << r.arch << '\t' << (r.moe ? "1" : "0") << '\t' << r.mode << '\t' << r.label << '\t'
|
|
<< r.count_fused << '\n';
|
|
}
|
|
}
|
|
LOG_INF("summary: %d ok, %d failed, %d skipped (of %d rows)\n", n_ok, n_bad, n_skip, n_ok + n_bad);
|
|
if (!record_path.empty()) {
|
|
LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str());
|
|
}
|
|
return n_bad;
|
|
}
|
|
}
|