mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-27 21:46:57 +02:00
Georgi Gerganov
259faf94ec
Use comma-separated values like the rest of the project, keeping the padded, aligned columns. Split on ',' and trim on parse. Rename the committed baseline to MTL.csv (data unchanged, verified identical modulo padding/separator). Update the ci/run.sh check path accordingly. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731
511 lines
20 KiB
C++
511 lines
20 KiB
C++
// test-fusion: verify the Metal fusion logic against a per-device baseline.
|
|
//
|
|
// for every dummy model generated by test-llama-archs, the tool runs the model on a single
|
|
// device (single-threaded encoding, n_cb = 0) with fusion enabled and disabled, and reports:
|
|
// - the per-fusion-type counters for each mode (prefill / decode)
|
|
// - the NMSE between the fused and unfused logits
|
|
// - the NMSE between the device and a CPU reference
|
|
//
|
|
// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a
|
|
// fusion pattern that silently stops matching (or fires when it should not) is caught as a
|
|
// regression.
|
|
//
|
|
// usage:
|
|
// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline
|
|
// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it
|
|
// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model
|
|
|
|
#include "common.h"
|
|
#include "log.h"
|
|
#include "llama-cpp.h"
|
|
|
|
#include "ggml.h"
|
|
#include "gguf.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iomanip>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <random>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism
|
|
// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging
|
|
// exports these exact names.
|
|
typedef void ( * fusion_stats_init_t) (ggml_backend_dev_t);
|
|
typedef void ( * fusion_stats_reset_t)(ggml_backend_dev_t);
|
|
typedef int ( * fusion_stats_get_t) (ggml_backend_dev_t, const char **, uint64_t *, int);
|
|
typedef void ( * fusion_set_enabled_t)(ggml_backend_dev_t, bool);
|
|
|
|
static bool silent_model_load_progress(float, void *) {
|
|
return true;
|
|
}
|
|
|
|
struct gguf_context_ptr {
|
|
gguf_context * ctx;
|
|
gguf_context_ptr(gguf_context * c) : ctx(c) {}
|
|
~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } }
|
|
gguf_context * get() const { return ctx; }
|
|
gguf_context_ptr(const gguf_context_ptr &) = delete;
|
|
gguf_context_ptr & operator=(const gguf_context_ptr &) = delete;
|
|
};
|
|
|
|
// NMSE between two vectors (same as tests/test-llama-archs.cpp)
|
|
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
|
|
GGML_ASSERT(a.size() == b.size());
|
|
double mse_a_b = 0.0;
|
|
double mse_a_0 = 0.0;
|
|
|
|
for (size_t i = 0; i < a.size(); i++) {
|
|
const float a_i = a[i];
|
|
const float b_i = b[i];
|
|
|
|
mse_a_b += (a_i - b_i) * (a_i - b_i);
|
|
mse_a_0 += a_i * a_i;
|
|
}
|
|
|
|
return mse_a_b / mse_a_0;
|
|
}
|
|
|
|
// deterministic token sequence
|
|
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {
|
|
std::mt19937 gen(seed);
|
|
std::uniform_int_distribution<> dis(0, n_vocab - 1);
|
|
std::vector<llama_token> ret;
|
|
ret.reserve(n_tokens);
|
|
for (uint32_t i = 0; i < n_tokens; i++) {
|
|
ret.push_back(dis(gen));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// trim leading/trailing whitespace (used when parsing padded CSV columns)
|
|
static std::string trim(const std::string & s) {
|
|
const size_t b = s.find_first_not_of(" \t\r\n");
|
|
if (b == std::string::npos) {
|
|
return "";
|
|
}
|
|
const size_t e = s.find_last_not_of(" \t\r\n");
|
|
return s.substr(b, e - b + 1);
|
|
}
|
|
|
|
static std::string get_arch(const std::string & path) {
|
|
gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr };
|
|
gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params));
|
|
if (!ctx.get()) {
|
|
throw std::runtime_error("failed to read gguf: " + path);
|
|
}
|
|
const int idx = gguf_find_key(ctx.get(), "general.architecture");
|
|
if (idx < 0) {
|
|
return "unknown";
|
|
}
|
|
const char * val = gguf_get_val_str(ctx.get(), idx);
|
|
return val ? val : "unknown";
|
|
}
|
|
|
|
static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) {
|
|
llama_model_params model_params = llama_model_default_params();
|
|
model_params.progress_callback = silent_model_load_progress;
|
|
std::vector<ggml_backend_dev_t> devs = { dev, nullptr };
|
|
model_params.devices = devs.data();
|
|
model_params.split_mode = LLAMA_SPLIT_MODE_LAYER;
|
|
|
|
llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params));
|
|
if (!model) {
|
|
throw std::runtime_error("failed to load model: " + path);
|
|
}
|
|
return model;
|
|
}
|
|
|
|
// a fresh context (fresh state) from an already-loaded model
|
|
static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) {
|
|
llama_context_params ctx_params = llama_context_default_params();
|
|
ctx_params.n_ctx = 0;
|
|
ctx_params.n_threads = 4;
|
|
ctx_params.n_threads_batch = 4;
|
|
ctx_params.n_ubatch = n_ubatch;
|
|
ctx_params.n_batch = n_ubatch;
|
|
|
|
llama_context_ptr lctx(llama_init_from_model(model, ctx_params));
|
|
if (!lctx) {
|
|
throw std::runtime_error("failed to init context");
|
|
}
|
|
return lctx;
|
|
}
|
|
|
|
// decode all tokens in one batch; returns the logits of every token
|
|
static std::vector<float> decode_prefill(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
|
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
|
llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
common_batch_add(batch, tokens[i], i, { 0 }, true);
|
|
}
|
|
batch.n_tokens = tokens.size();
|
|
if (llama_decode(lctx, batch)) {
|
|
llama_batch_free(batch);
|
|
throw std::runtime_error("prefill decode failed");
|
|
}
|
|
|
|
std::vector<float> ret;
|
|
ret.reserve(tokens.size() * n_vocab);
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
const float * logits_ith = llama_get_logits_ith(lctx, i);
|
|
for (uint32_t j = 0; j < n_vocab; j++) {
|
|
ret.push_back(logits_ith[j]);
|
|
}
|
|
}
|
|
llama_batch_free(batch);
|
|
return ret;
|
|
}
|
|
|
|
// decode one token at a time; returns the logits of the last token of each step
|
|
static std::vector<float> decode_token_by_token(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
|
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
|
llama_batch batch = llama_batch_init(1, 0, 1);
|
|
std::vector<float> ret;
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
common_batch_clear(batch);
|
|
common_batch_add(batch, tokens[i], i, { 0 }, true);
|
|
if (llama_decode(lctx, batch)) {
|
|
llama_batch_free(batch);
|
|
throw std::runtime_error("decode failed");
|
|
}
|
|
const float * logits = llama_get_logits_ith(lctx, 0);
|
|
for (uint32_t j = 0; j < n_vocab; j++) {
|
|
ret.push_back(logits[j]);
|
|
}
|
|
}
|
|
llama_batch_free(batch);
|
|
return ret;
|
|
}
|
|
|
|
static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_dev_t dev,
|
|
std::vector<const char *> & labels, std::vector<uint64_t> & counts) {
|
|
const int n = api_stats_get(dev, nullptr, nullptr, 0);
|
|
labels.assign(n, nullptr);
|
|
counts.assign(n, 0);
|
|
api_stats_get(dev, labels.data(), counts.data(), n);
|
|
}
|
|
|
|
// one row of the per-label report
|
|
struct fusion_row {
|
|
std::string arch;
|
|
bool moe;
|
|
std::string mode;
|
|
const char * label;
|
|
uint64_t count_fused;
|
|
uint64_t count_unfused;
|
|
uint64_t expected;
|
|
double nmse_fus;
|
|
double nmse_dev;
|
|
bool ok_count; // counts match the baseline
|
|
bool ok_nmse; // nmse within epsilon
|
|
bool skip_nmse; // nmse unreliable (NaN logits or arch already broken on the device)
|
|
};
|
|
|
|
static void usage(const char * argv0) {
|
|
printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0);
|
|
printf("usage: %s [options]\n\n", argv0);
|
|
printf("options:\n");
|
|
printf(" --models DIR run over all .gguf models in a directory\n");
|
|
printf(" --model FILE run over a single model file (mutually exclusive with --models)\n");
|
|
printf(" --device NAME device to run on (e.g. MTL0, CPU)\n");
|
|
printf(" --record CSV write the golden baseline\n");
|
|
printf(" --check CSV validate the counters against a baseline (default)\n");
|
|
printf(" -h, --help show this message and exit\n");
|
|
}
|
|
|
|
int main(int argc, char ** argv) {
|
|
std::string models_dir;
|
|
std::string model_file;
|
|
std::string device_name;
|
|
std::string record_path;
|
|
std::string check_path;
|
|
|
|
for (int i = 1; i < argc; i++) {
|
|
const std::string arg = argv[i];
|
|
const auto next = [&](const char * name) -> std::string {
|
|
if (i + 1 >= argc) {
|
|
LOG_ERR("%s: %s requires an argument\n", __func__, name);
|
|
exit(1);
|
|
}
|
|
return argv[++i];
|
|
};
|
|
if (arg == "-h" || arg == "--help") {
|
|
usage(argv[0]);
|
|
exit(0);
|
|
}
|
|
if (arg == "--models") { models_dir = next("--models"); }
|
|
else if (arg == "--model") { model_file = next("--model"); }
|
|
else if (arg == "--device"){ device_name = next("--device"); }
|
|
else if (arg == "--record"){ record_path = next("--record"); }
|
|
else if (arg == "--check") { check_path = next("--check"); }
|
|
else {
|
|
LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
if (device_name.empty()) {
|
|
LOG_ERR("%s: --device NAME is required\n", __func__);
|
|
return 1;
|
|
}
|
|
if (models_dir.empty() && model_file.empty()) {
|
|
LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__);
|
|
return 1;
|
|
}
|
|
if (!models_dir.empty() && !model_file.empty()) {
|
|
LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__);
|
|
return 1;
|
|
}
|
|
if (!record_path.empty() && !check_path.empty()) {
|
|
LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__);
|
|
return 1;
|
|
}
|
|
|
|
std::vector<std::string> models;
|
|
if (!model_file.empty()) {
|
|
if (!std::filesystem::is_regular_file(model_file)) {
|
|
LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str());
|
|
return 1;
|
|
}
|
|
models.push_back(model_file);
|
|
} else {
|
|
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
|
|
LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
|
|
return 1;
|
|
}
|
|
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
|
|
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
|
|
models.push_back(entry.path().string());
|
|
}
|
|
}
|
|
std::sort(models.begin(), models.end());
|
|
|
|
if (models.empty()) {
|
|
LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
common_init();
|
|
ggml_backend_load_all();
|
|
|
|
ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str());
|
|
if (!dev) {
|
|
LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n",
|
|
__func__, device_name.c_str());
|
|
return 0;
|
|
}
|
|
|
|
// resolve the generic fusion debugging functions through the ad-hoc get_proc_address
|
|
// mechanism; a backend that does not adopt fusion debugging exports none of them
|
|
auto * reg = ggml_backend_dev_backend_reg(dev);
|
|
|
|
// output naming uses the backend base name (e.g. "MTL") rather than the specific device
|
|
// name (e.g. "MTL0") the test was invoked with
|
|
const std::string base_name = ggml_backend_reg_name(reg);
|
|
|
|
auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init");
|
|
auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset");
|
|
auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get");
|
|
auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled");
|
|
|
|
if (!api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) {
|
|
LOG_ERR("%s: device '%s' does not export the generic fusion debugging API "
|
|
"(ggml_backend_fusion_*) - cannot run the fusion regression test\n",
|
|
__func__, device_name.c_str());
|
|
return 1;
|
|
}
|
|
|
|
// enable fusions stats
|
|
api_stats_init(dev);
|
|
|
|
const bool has_counts = true;
|
|
|
|
// load the baseline (if any): key arch|moe|mode|label -> expected count
|
|
std::map<std::string, uint64_t> baseline;
|
|
if (!check_path.empty()) {
|
|
std::ifstream in(check_path);
|
|
if (!in) {
|
|
LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str());
|
|
return 1;
|
|
}
|
|
std::string line;
|
|
while (std::getline(in, line)) {
|
|
if (line.empty() || line[0] == '#') {
|
|
continue;
|
|
}
|
|
std::vector<std::string> cols;
|
|
size_t pos = 0;
|
|
while ((pos = line.find(',')) != std::string::npos) {
|
|
cols.push_back(trim(line.substr(0, pos)));
|
|
line.erase(0, pos + 1);
|
|
}
|
|
cols.push_back(trim(line));
|
|
if (cols.size() != 5) {
|
|
continue;
|
|
}
|
|
baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]);
|
|
}
|
|
}
|
|
|
|
std::vector<fusion_row> rows;
|
|
|
|
LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());
|
|
|
|
const size_t seed = 1;
|
|
|
|
for (const auto & model_path : models) {
|
|
const std::string arch = get_arch(model_path);
|
|
const bool moe = arch.find("moe") != std::string::npos;
|
|
|
|
llama_model_ptr model;
|
|
llama_model_ptr model_cpu;
|
|
uint32_t n_vocab = 0;
|
|
try {
|
|
model = load_model(model_path, dev);
|
|
model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU"));
|
|
n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get()));
|
|
} catch (const std::exception & e) {
|
|
LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what());
|
|
continue;
|
|
}
|
|
|
|
struct mode_cfg {
|
|
std::string name;
|
|
std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);
|
|
int n_tokens;
|
|
};
|
|
const mode_cfg modes[] = {
|
|
{ "prefill", decode_prefill, 32 },
|
|
{ "decode", decode_token_by_token, 16 },
|
|
};
|
|
|
|
for (const auto & mode : modes) {
|
|
const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed);
|
|
|
|
// CPU reference for this mode (fresh context, fresh state)
|
|
std::vector<float> logits_cpu;
|
|
try {
|
|
llama_context_ptr ctx = create_ctx(model_cpu.get(), 32);
|
|
logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens);
|
|
} catch (const std::exception & e) {
|
|
LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what());
|
|
}
|
|
|
|
// fused run on a fresh context (fresh state)
|
|
std::vector<float> logits_fused;
|
|
std::vector<const char *> labels;
|
|
std::vector<uint64_t> counts_fused;
|
|
{
|
|
llama_context_ptr ctx = create_ctx(model.get(), 32);
|
|
if (has_counts) {
|
|
api_set_enabled(dev, true);
|
|
api_stats_reset(dev);
|
|
}
|
|
logits_fused = mode.decode(model.get(), ctx.get(), tokens);
|
|
if (has_counts) {
|
|
read_counts(api_stats_get, dev, labels, counts_fused);
|
|
}
|
|
}
|
|
|
|
// unfused run on another fresh context (fresh state)
|
|
std::vector<float> logits_unfused;
|
|
std::vector<uint64_t> counts_unfused;
|
|
{
|
|
llama_context_ptr ctx = create_ctx(model.get(), 32);
|
|
if (has_counts) {
|
|
api_set_enabled(dev, false);
|
|
api_stats_reset(dev);
|
|
}
|
|
logits_unfused = mode.decode(model.get(), ctx.get(), tokens);
|
|
if (has_counts) {
|
|
read_counts(api_stats_get, dev, labels, counts_unfused);
|
|
}
|
|
}
|
|
|
|
const double nmse_fus = nmse(logits_fused, logits_unfused);
|
|
const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu);
|
|
|
|
// an arch that is already broken on the device (huge device-vs-CPU NMSE, e.g. plamo2
|
|
// on Metal) produces garbage regardless of fusion, so the fused-vs-unfused NMSE is not
|
|
// a meaningful signal - skip it and rely on the count regression check only
|
|
// TODO: run in Debug build to get an assert in `ggml-alloc.c` and fix it
|
|
const bool dev_broken = !std::isnan(nmse_dev) && nmse_dev > 1e-4;
|
|
|
|
// build the per-label rows
|
|
if (has_counts) {
|
|
for (int i = 0; i < (int) labels.size(); i++) {
|
|
if (counts_fused[i] == 0 && counts_unfused[i] == 0) {
|
|
continue;
|
|
}
|
|
const std::string key = arch + "|" + (moe ? "1" : "0") + "|" + mode.name + "|" + labels[i];
|
|
const uint64_t expected = baseline.count(key) ? baseline.at(key) : 0;
|
|
const bool skip_nmse = dev_broken || std::isnan(nmse_fus);
|
|
rows.push_back({ arch, moe, mode.name, labels[i], counts_fused[i], counts_unfused[i],
|
|
expected, nmse_fus, nmse_dev,
|
|
!check_path.empty() ? (counts_fused[i] == expected) : true,
|
|
!skip_nmse && nmse_fus <= 1e-4,
|
|
skip_nmse });
|
|
}
|
|
} else {
|
|
const bool skip_nmse = dev_broken || std::isnan(nmse_fus);
|
|
rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true,
|
|
!skip_nmse && nmse_fus <= 1e-4, skip_nmse });
|
|
}
|
|
}
|
|
|
|
LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());
|
|
}
|
|
|
|
// print the report
|
|
{
|
|
std::ofstream out(record_path);
|
|
std::ostream & os = record_path.empty() ? std::cout : out;
|
|
if (!record_path.empty()) {
|
|
os << "# test-fusion baseline for device " << base_name << "\n";
|
|
os << "# " << std::left
|
|
<< std::setw(18) << "arch" << ','
|
|
<< std::setw(4) << "moe" << ','
|
|
<< std::setw(8) << "mode" << ','
|
|
<< std::setw(28) << "label" << ','
|
|
<< std::right << std::setw(7) << "count" << '\n';
|
|
}
|
|
|
|
LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n",
|
|
"arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status");
|
|
int n_ok = 0;
|
|
int n_bad = 0;
|
|
int n_skip = 0;
|
|
for (const auto & r : rows) {
|
|
const bool ok = r.ok_count && (r.skip_nmse || r.ok_nmse);
|
|
const char * status = ok ? (r.skip_nmse ? "skip" : "ok") : "FAIL";
|
|
if (r.skip_nmse) { n_skip++; }
|
|
if (ok) { n_ok++; } else { n_bad++; }
|
|
LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n",
|
|
r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label,
|
|
(unsigned long long) r.count_fused, (unsigned long long) r.count_unfused,
|
|
(unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status);
|
|
if (!record_path.empty()) {
|
|
os << std::left
|
|
<< std::setw(20) << r.arch << ','
|
|
<< std::setw(4) << (r.moe ? "1" : "0") << ','
|
|
<< std::setw(8) << r.mode << ','
|
|
<< std::setw(28) << r.label << ','
|
|
<< std::right << std::setw(7) << r.count_fused << '\n';
|
|
}
|
|
}
|
|
LOG_INF("summary: %d ok, %d failed, %d skipped (of %d rows)\n", n_ok, n_bad, n_skip, n_ok + n_bad);
|
|
if (!record_path.empty()) {
|
|
LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str());
|
|
}
|
|
return n_bad;
|
|
}
|
|
}
|