Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-27 21:46:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

test-llama-archs : make tensor data stdev configurable and improve help (#29133)

Browse Source 
* test-llama-archs : make tensor data stdev configurable

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* test-llama-archs : expand usage and add examples

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* test-llama-archs : fail on unknown args and log usage

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* test-llama-archs : add test run summary

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* test-llama-archs : initialize Mamba ssm_a negative

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* test-recurrent-state-rollback : report NMSE for logits mismatches

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* test-recurrent-state-rollback : use NMSE for rollback logits checks

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* tests : disable invalid test

* cont : adjust nmse_eps

* tests : zero DSA indexer score projection in synthetic fixtures

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* cont : add support for `--arch` regex

* cont : alternative top-k stability

* cont : indentation

* cont : fix top-k value

* cont : consistent logs
This commit is contained in:
Georgi Gerganov
2026-09-21 13:57:36 +03:00
committed by GitHub
parent 1d72b05d38
commit 161755f29e
3 changed files with 232 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/minimax-m3.cpp
+6 -1
View File
@@ -23,7 +23,12 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size);
ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);
msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks };
msa_p = {
/*.blk =*/ (int) hparams.indexer_block_size,
/*.topk_blocks =*/ (int) hparams.indexer_top_k,
/*.local =*/ (int) hparams.indexer_local_blocks,
};
GGML_ASSERT(hparams.indexer_block_size > 0); // avoid div by zero
tests/test-llama-archs.cpp
+138 -60
View File
@@ -12,16 +12,25 @@
#include "../src/llama-model-saver.h"
#include <cinttypes>
#include <cmath>
#include <cstddef>
#include <cstdio>
#include <cstring>
#include <cstdint>
#include <random>
#include <regex>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
static bool arch_matches(const std::string & filter, llm_arch arch) {
if (filter.empty()) {
return true;
}
return std::regex_search(llm_arch_name(arch), std::regex(filter));
}
// normalized mean squared error = mse(a, b) / mse(a, 0)
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
GGML_ASSERT(a.size() == b.size());
@@ -39,24 +48,36 @@ static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
return mse_a_b / mse_a_0;
}
struct tensor_data_params {
size_t seed;
float stdev;
};
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
size_t seed = *(const size_t *) userdata;
const tensor_data_params & params = *(const tensor_data_params *) userdata;
size_t seed = params.seed;
std::hash<std::string> hasher;
seed ^= hasher(tensor->name);
std::mt19937 gen(seed);
std::normal_distribution<float> dis(0.0f, 1.0e-2f);
std::normal_distribution<float> dis(0.0f, params.stdev);
// TODO: refactor per-tensor initialization logic in a cleaner way
// note: Mamba A must be negative (state decay)
const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr;
const int64_t ne = ggml_nelements(tensor);
if (tensor->type == GGML_TYPE_F32) {
std::vector<float> tmp(ne);
for (int64_t i = 0; i < ne; i++) {
tmp[i] = dis(gen);
float val = dis(gen);
tmp[i] = is_ssm_a ? -fabsf(val) : val;
}
ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
} else if (tensor->type == GGML_TYPE_F16) {
std::vector<ggml_fp16_t> tmp(ne);
for (int64_t i = 0; i < ne; i++) {
tmp[i] = ggml_fp32_to_fp16(dis(gen));
float val = dis(gen);
tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val);
}
ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
} else {
@@ -65,7 +86,19 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
}
static void usage(char ** argv) {
printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help]\n", argv[0]);
LOG("Usage: %s [options]\n\n", argv[0]);
LOG("Options:\n");
LOG(" -a, --arch <arch|regex> Run only matching LLM architectures (default: all supported)\n");
LOG(" -s, --seed <seed> Set the random seed for tensor initialization and token generation\n");
LOG(" -d, --stdev <stdev> Set the standard deviation of the tensor initialization distribution (default: 0.1f)\n");
LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n");
LOG(" -v <N> Set log verbosity level\n");
LOG(" -h, --help Show this help message\n\n");
LOG("Examples:\n");
LOG(" %s\n", argv[0]);
LOG(" %s -a qwen35moe\n", argv[0]);
LOG(" %s -a deepseek4 -o tests/test-models/\n", argv[0]);
LOG(" %s -a cohere2moe -v 5\n", argv[0]);
}
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){
@@ -292,7 +325,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
// note: using a realistic top-k here makes the results unstable and hard to match between CPU and GPU
// a large value makes things deterministic since all data is selected by the indexer
//ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(131072));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));
@@ -314,13 +351,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
}
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
@@ -381,19 +418,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));
ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));
}
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
for (uint32_t il = 0; il < n_layer; il++) {
ggml_tensor t;
@@ -416,7 +453,8 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/)
}
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const std::vector<ggml_backend_dev_t> & devs,
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
const std::vector<ggml_backend_dev_t> & devs,
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false) {
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
llama_model_params model_params = llama_model_default_params();
@@ -434,9 +472,9 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
ctx_params.n_ubatch = 64;
}
size_t tmp = seed;
tensor_data_params tensor_params = { seed, stdev };
llama_model_ptr model(gguf_ctx != nullptr ?
llama_model_init_from_user(gguf_ctx, set_tensor_data, &tmp, model_params) :
llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
llama_model_load_from_file_ptr(file, model_params));
if (!model) {
throw std::runtime_error("failed to create llama model");
@@ -608,7 +646,7 @@ static bool arch_supported(const llm_arch arch) {
return true;
}
static int save_models(const llm_arch target_arch, const size_t seed, const int verbosity, const std::string & dir) {
static int save_models(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const std::string & dir) {
struct user_data_t {
struct {
ggml_log_callback callback;
@@ -635,7 +673,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {
if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
@@ -656,7 +694,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int
continue;
}
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {});
auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {});
const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
@@ -666,7 +704,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int
return 0;
}
static int test_backends(const llm_arch target_arch, const size_t seed, const int verbosity) {
static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity) {
struct user_data_t {
struct {
ggml_log_callback callback;
@@ -731,22 +769,24 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
const std::string template_row_res = "%15s %10s|%20s|\n";
bool all_ok = true;
size_t n_tests = 0;
size_t n_failed = 0;
common_log_flush(common_log_main());
printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");
printf("|");
LOG(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");
LOG("|");
for (size_t i = 0; i < max_arch_name_length; i++) {
printf("-");
LOG("-");
}
printf("|");
LOG("|");
for (size_t i = 0; i < max_device_label_length; i++) {
printf("-");
LOG("-");
}
printf("|------|---------------|---------|\n");
LOG("|------|---------------|---------|\n");
for (const llm_arch & arch : llm_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {
if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
@@ -773,8 +813,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
std::vector<float> logits_cpu;
for (device_config & dc : dev_configs) {
// print test config first; should anything fail during model loading or inference, at least we know which test case caused it
printf(template_row_cfg.c_str(),
llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
LOG(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
fflush(stdout);
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;
@@ -784,19 +823,22 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
char nmse_str[12] = {0};
bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());
bool test_executed = false;
bool test_ok = true;
if (!skip) {
if (logits_cpu.empty()) {
model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode);
model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}, LLAMA_SPLIT_MODE_LAYER, encode);
logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);
}
if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {
model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, dc.devs, dc.split_mode, encode);
test_executed = true;
model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode);
logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
const double nmse_val = nmse(logits_cpu, logits_dev);
snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
status_nmse = "\033[1;32mOK\033[0m";
if (nmse_val > 1e-4) {
all_ok = false;
test_ok = false;
status_nmse = "\033[1;31mFAIL\033[0m";
}
}
@@ -805,6 +847,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
// FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend
// in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible
if (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
test_executed = true;
GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);
llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get());
ms.add_kv_from_model();
@@ -812,14 +855,14 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
ms.save(file);
rewind(file);
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode);
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode);
const std::vector<float> logits_roundtrip = get_logits(
model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);
status_roundtrip = "\033[1;32mOK\033[0m";
GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());
for (size_t i = 0; i < logits_roundtrip.size(); i++) {
if (logits_roundtrip[i] != logits_dev[i]) {
all_ok = false;
test_ok = false;
status_roundtrip = "\033[1;31mFAIL\033[0m";
break;
}
@@ -827,12 +870,28 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
}
}
if (test_executed) {
n_tests++;
if (!test_ok) {
n_failed++;
all_ok = false;
}
}
// log the results for this test case
printf(template_row_res.c_str(),
status_nmse.c_str(), nmse_str, status_roundtrip.c_str());
LOG(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str());
}
}
}
if (n_tests == 0) {
LOG("Summary: no tests executed\n");
} else if (n_failed == 0) {
LOG("Summary: all %zu test(s) passed\n", n_tests);
} else {
LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed);
}
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
return all_ok ? 0 : 1;
}
@@ -844,8 +903,9 @@ int main(int argc, char ** argv) {
std::random_device rd;
llm_arch arch = LLM_ARCH_UNKNOWN;
std::string arch_filter;
size_t seed = rd();
float stdev = 0.1f;
std::string out;
int verbosity = LOG_LEVEL_ERROR;
@@ -854,52 +914,70 @@ int main(int argc, char ** argv) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
usage(argv);
return 0;
}
if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
} else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
if (i + 1 < argc) {
const std::string arch_name = argv[++i];
arch = llm_arch_from_string(arch_name);
if (arch == LLM_ARCH_UNKNOWN) {
LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str());
return 1;
if (llm_arch_from_string(arch_name) != LLM_ARCH_UNKNOWN) {
// exact architecture name
arch_filter = "^" + arch_name + "$";
} else {
try {
std::regex re(arch_name);
arch_filter = arch_name;
} catch (const std::regex_error & err) {
LOG_ERR("%s: invalid architecture regex: %s (%s)\n", __func__, arch_name.c_str(), err.what());
return 1;
}
}
} else {
usage(argv);
return 1;
}
}
if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
} else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
if (i + 1 < argc) {
seed = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
}
if (strcmp(argv[i], "-v") == 0) {
} else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--stdev") == 0) {
if (i + 1 < argc) {
stdev = std::stof(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-v") == 0) {
if (i + 1 < argc) {
verbosity = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
}
if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
if (i + 1 < argc) {
out = argv[++i];
} else {
usage(argv);
return 1;
}
} else {
LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]);
usage(argv);
return 1;
}
}
printf("%s: using seed %zu\n", __func__, seed);
if (stdev <= 0.0f) {
LOG_ERR("%s: stdev must be > 0\n", __func__);
return 1;
}
LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);
try {
if (!out.empty()) {
return save_models(arch, seed, verbosity, out);
return save_models(arch_filter, seed, stdev, verbosity, out);
}
return test_backends(arch, seed, verbosity);
return test_backends(arch_filter, seed, stdev, verbosity);
} catch (const std::exception & err) {
fprintf(stderr, "encountered runtime error: %s\n", err.what());
return -1;
tests/test-recurrent-state-rollback.cpp
+88 -35
View File
@@ -90,6 +90,20 @@ static float logit_diff(float a, float b) {
return std::isfinite(a) && std::isfinite(b) ? std::fabs(a - b) : std::numeric_limits<float>::infinity();
}
static double nmse(const float * a, const float * b, int n) {
double mse_ab = 0.0;
double mse_a0 = 0.0;
for (int i = 0; i < n; i++) {
if (!std::isfinite(a[i]) || !std::isfinite(b[i])) {
return std::numeric_limits<double>::infinity();
}
const double diff = (double) a[i] - b[i];
mse_ab += diff*diff;
mse_a0 += (double) a[i]*a[i];
}
return mse_a0 == 0.0 ? (mse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : mse_ab/mse_a0;
}
// Roll back multiple sequences, then replay them in a single batch whose
// per-seq token count exceeds n_ubatch: each seq's replay spans several
// ubatches while its rollback restore is still pending. Compared against a
@@ -182,13 +196,15 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
return false;
}
// identical ubatch shapes from bit-exact states: a correct implementation
// matches bitwise, so eps only allows backend scheduling noise
constexpr float eps = 1e-7f;
// identical ubatch shapes should produce identical states, but the larger
// stdev makes the model sensitive to backend scheduling/rounding noise
constexpr float nmse_eps = 1e-5f;
float diff_max = 0.0f;
uint32_t seq_first = 0;
int32_t pos_first = -1;
double nmse_ab = 0.0;
double nmse_a0 = 0.0;
for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {
const float * l_roll = llama_get_logits_ith(ctx_roll, i);
const float * l_ref = llama_get_logits_ith(ctx_ref, i);
@@ -198,23 +214,34 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
return false;
}
for (int t = 0; t < n_vocab; ++t) {
const float diff = logit_diff(l_roll[t], l_ref[t]);
if (diff > eps && pos_first < 0) {
const float r = l_roll[t];
const float f = l_ref[t];
const float diff = logit_diff(r, f);
if (diff > 0.0f && pos_first < 0) {
seq_first = i/n_replay;
pos_first = p0 + (int32_t) (i%n_replay);
}
diff_max = std::max(diff_max, diff);
if (std::isfinite(r) && std::isfinite(f)) {
const double d = (double) r - f;
nmse_ab += d*d;
nmse_a0 += (double) r*r;
} else {
nmse_ab = std::numeric_limits<double>::infinity();
nmse_a0 = 1.0;
}
}
}
const double nmse_val = nmse_a0 == 0.0 ? (nmse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_ab/nmse_a0;
if (diff_max > eps) {
fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n",
__func__, (double) diff_max, seq_first, pos_first);
if (nmse_val > nmse_eps) {
fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, nmse %g, first at seq %u pos %d)\n",
__func__, (double) diff_max, nmse_val, seq_first, pos_first);
cleanup();
return false;
}
fprintf(stderr, "%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max);
fprintf(stderr, "%s : multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff_max, nmse_val);
// seq-1-only decodes must be independent of seq 0's content: diverge seq 0
// in ctx_ref only, then compare identical seq-1-only continuations bitwise
@@ -231,6 +258,8 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
}
float diff_tail = 0.0f;
double nmse_tail_ab = 0.0;
double nmse_tail_a0 = 0.0;
for (uint32_t i = 0; i < n_tail && ok; ++i) {
const llama_pos pos = p0 + (llama_pos) (n_replay + i);
llama_batch batch_one = llama_batch_init(1, 0, 1);
@@ -246,18 +275,29 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
const float * l_ref = llama_get_logits_ith(ctx_ref, 0);
ok = l_roll != nullptr && l_ref != nullptr;
for (int t = 0; ok && t < n_vocab; ++t) {
diff_tail = std::max(diff_tail, logit_diff(l_roll[t], l_ref[t]));
const float r = l_roll[t];
const float f = l_ref[t];
diff_tail = std::max(diff_tail, logit_diff(r, f));
if (std::isfinite(r) && std::isfinite(f)) {
const double d = (double) r - f;
nmse_tail_ab += d*d;
nmse_tail_a0 += (double) r*r;
} else {
nmse_tail_ab = std::numeric_limits<double>::infinity();
nmse_tail_a0 = 1.0;
}
}
}
const double nmse_tail = nmse_tail_a0 == 0.0 ? (nmse_tail_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_tail_ab/nmse_tail_a0;
if (!ok || diff_tail > eps) {
fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n",
__func__, ok ? 1 : 0, (double) diff_tail);
if (!ok || nmse_tail > nmse_eps) {
fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g, nmse %g)\n",
__func__, ok ? 1 : 0, (double) diff_tail, nmse_tail);
cleanup();
return false;
}
fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail);
fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g, nmse %g)\n", __func__, (double) diff_tail, nmse_tail);
cleanup();
return true;
}
@@ -266,6 +306,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint
const llama_vocab * vocab = llama_model_get_vocab(model);
const int n_vocab = llama_vocab_n_tokens(vocab);
// TODO: use smart pointers
llama_context * ctx_src = make_ctx(params, model, fill);
llama_context * ctx_dst = make_ctx(params, model, fill);
if (ctx_src == nullptr || ctx_dst == nullptr) {
@@ -320,7 +361,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint
ckpt.update_tgt(ctx_src, 0, 0);
ckpt.load_tgt(ctx_dst, 0, 0);
constexpr float eps = 1e-5f;
constexpr float nmse_eps = 0.0;
std::vector<std::vector<float>> logits_src_replay(n_rollback);
const auto replay_and_compare = [&](const char * mode) {
for (uint32_t i = 0; i < n_rollback; ++i) {
@@ -339,13 +380,18 @@ static int test_rollback(const common_params & params, llama_model * model, uint
}
logits_src_replay[i].assign(logits_src, logits_src + n_vocab);
const double nmse_val = nmse(logits_src, logits_dst, n_vocab);
int token_first = -1;
for (int token = 0; token < n_vocab; ++token) {
if (logit_diff(logits_src[token], logits_dst[token]) > eps) {
fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n",
__func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]);
return false;
if (logit_diff(logits_src[token], logits_dst[token]) > 0.0f && token_first < 0) {
token_first = token;
}
}
if (nmse_val > nmse_eps) {
fprintf(stderr, "%s : %s logits mismatch at position %d, first token %d, nmse %g\n",
__func__, mode, pos, token_first, nmse_val);
return false;
}
}
return true;
};
@@ -353,20 +399,22 @@ static int test_rollback(const common_params & params, llama_model * model, uint
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||
!llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {
fprintf(stderr, "%s : partial rollback failed\n", __func__);
return 1;
}
// TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens
// this is not the case here. add asserts and guardrails to prevent such attempts
//if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||
// !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {
// fprintf(stderr, "%s : partial rollback failed\n", __func__);
// return 1;
//}
constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
common_prompt_checkpoint ckpt_partial;
ckpt_partial.update_tgt(ctx_src, 0, partial_flags);
ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);
//constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
//common_prompt_checkpoint ckpt_partial;
//ckpt_partial.update_tgt(ctx_src, 0, partial_flags);
//ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);
if (!replay_and_compare("partial")) {
return 1;
}
//if (!replay_and_compare("partial")) {
// return 1;
//}
// Repeat the load into a context that already has its own rollback state:
// groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is
@@ -408,13 +456,18 @@ static int test_rollback(const common_params & params, llama_model * model, uint
return 1;
}
const double nmse_dirty = nmse(logits_src_replay[i].data(), logits_dirty, n_vocab);
int token_first = -1;
for (int token = 0; token < n_vocab; ++token) {
if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > eps) {
fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n",
__func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]);
return 1;
if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > 0.0f && token_first < 0) {
token_first = token;
}
}
if (nmse_dirty > nmse_eps) {
fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, first token %d, nmse %g\n",
__func__, pos, token_first, nmse_dirty);
return 1;
}
}
fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__);