diff --git a/ci/run.sh b/ci/run.sh index 0851e20aab..dd249e223d 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -346,7 +346,7 @@ function gg_run_test_fusion { # generate the dummy models and run the fusion regression test ./build-ci-release/bin/test-llama-archs -o build-ci-fusion-models 2>&1 - ./build-ci-release/bin/test-fusion --models build-ci-fusion-models --device MTL0 --check tests/fusion/MTL.tsv 2>&1 + ./build-ci-release/bin/test-fusion --models build-ci-fusion-models --device MTL0 --check tests/fusion/MTL.csv 2>&1 fi set +e diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv new file mode 100644 index 0000000000..3edf43fe9c --- /dev/null +++ b/tests/fusion/MTL.csv @@ -0,0 +1,300 @@ +# test-fusion baseline for device MTL +# arch ,moe ,mode ,label , count +arcee ,0 ,prefill ,RMS_NORM+MUL , 5 +arcee ,0 ,decode ,RMS_NORM+MUL , 80 +arctic ,0 ,prefill ,RMS_NORM+MUL , 7 +arctic ,0 ,decode ,RMS_NORM+MUL , 112 +baichuan ,0 ,prefill ,RMS_NORM+MUL , 5 +baichuan ,0 ,decode ,RMS_NORM+MUL , 80 +bailingmoe ,1 ,prefill ,RMS_NORM+MUL , 5 +bailingmoe ,1 ,prefill ,ADD+ADD , 2 +bailingmoe ,1 ,decode ,RMS_NORM+MUL , 80 +bailingmoe ,1 ,decode ,ADD+ADD , 32 +bailingmoe2 ,1 ,prefill ,RMS_NORM+MUL , 9 +bailingmoe2 ,1 ,prefill ,ADD+ADD , 1 +bailingmoe2 ,1 ,decode ,RMS_NORM+MUL , 144 +bailingmoe2 ,1 ,decode ,ADD+ADD , 16 +bailingmoe3 ,1 ,prefill ,RMS_NORM+MUL , 8 +bailingmoe3 ,1 ,prefill ,ADD+ADD , 1 +bailingmoe3 ,1 ,prefill ,GATED_DELTA_NET+CPY , 1 +bailingmoe3 ,1 ,decode ,RMS_NORM+MUL , 128 +bailingmoe3 ,1 ,decode ,ADD+ADD , 16 +bailingmoe3 ,1 ,decode ,GATED_DELTA_NET+CPY , 16 +bloom ,0 ,prefill ,NORM+MUL+ADD , 6 +bloom ,0 ,decode ,NORM+MUL+ADD , 96 +chatglm ,0 ,prefill ,RMS_NORM+MUL , 5 +chatglm ,0 ,decode ,RMS_NORM+MUL , 80 +codeshell ,0 ,prefill ,NORM+MUL+ADD , 5 +codeshell ,0 ,decode ,NORM+MUL+ADD , 80 +cogvlm ,0 ,prefill ,RMS_NORM+MUL , 5 +cogvlm ,0 ,decode ,RMS_NORM+MUL , 80 +command-r ,0 ,prefill ,NORM+MUL , 3 +command-r ,0 ,decode ,NORM+MUL , 48 +dbrx ,0 ,prefill ,NORM+MUL , 5 +dbrx ,0 ,decode ,NORM+MUL , 80 +deci ,0 ,prefill ,RMS_NORM+MUL , 5 +deci ,0 ,decode ,RMS_NORM+MUL , 80 +deepseek ,0 ,prefill ,RMS_NORM+MUL , 5 +deepseek ,0 ,prefill ,ADD+ADD , 1 +deepseek ,0 ,decode ,RMS_NORM+MUL , 80 +deepseek ,0 ,decode ,ADD+ADD , 16 +deepseek2 ,0 ,prefill ,RMS_NORM+MUL , 9 +deepseek2 ,0 ,prefill ,ADD+ADD , 1 +deepseek2 ,0 ,decode ,RMS_NORM+MUL , 144 +deepseek2 ,0 ,decode ,ADD+ADD , 16 +deepseek32 ,0 ,prefill ,NORM+MUL+ADD , 2 +deepseek32 ,0 ,prefill ,RMS_NORM+MUL , 9 +deepseek32 ,0 ,prefill ,ADD+ADD , 1 +deepseek32 ,0 ,decode ,NORM+MUL+ADD , 32 +deepseek32 ,0 ,decode ,RMS_NORM+MUL , 144 +deepseek32 ,0 ,decode ,ADD+ADD , 16 +deepseek4 ,0 ,prefill ,RMS_NORM+MUL , 20 +deepseek4 ,0 ,decode ,RMS_NORM+MUL , 320 +dots1 ,0 ,prefill ,RMS_NORM+MUL , 9 +dots1 ,0 ,prefill ,ADD+ADD , 1 +dots1 ,0 ,decode ,RMS_NORM+MUL , 144 +dots1 ,0 ,decode ,ADD+ADD , 16 +dream ,0 ,prefill ,RMS_NORM+MUL , 5 +dream ,0 ,decode ,RMS_NORM+MUL , 80 +ernie4_5-moe ,1 ,prefill ,RMS_NORM+MUL , 5 +ernie4_5-moe ,1 ,prefill ,ADD+ADD , 1 +ernie4_5-moe ,1 ,decode ,RMS_NORM+MUL , 80 +ernie4_5-moe ,1 ,decode ,ADD+ADD , 16 +ernie4_5 ,0 ,prefill ,RMS_NORM+MUL , 5 +ernie4_5 ,0 ,decode ,RMS_NORM+MUL , 80 +exaone ,0 ,prefill ,RMS_NORM+MUL , 5 +exaone ,0 ,decode ,RMS_NORM+MUL , 80 +exaone4 ,0 ,prefill ,RMS_NORM+MUL , 5 +exaone4 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 +exaone4 ,0 ,decode ,RMS_NORM+MUL , 80 +exaone4 ,0 ,decode ,RMS_NORM+MUL+ADD , 64 +falcon ,0 ,prefill ,NORM+MUL+ADD , 5 +falcon ,0 ,prefill ,ADD+ADD , 2 +falcon ,0 ,decode ,NORM+MUL+ADD , 80 +falcon ,0 ,decode ,ADD+ADD , 32 +falcon-h1 ,0 ,prefill ,RMS_NORM+MUL , 9 +falcon-h1 ,0 ,prefill ,ADD+ADD , 2 +falcon-h1 ,0 ,decode ,RMS_NORM+MUL , 144 +falcon-h1 ,0 ,decode ,ADD+ADD , 32 +gemma ,0 ,prefill ,RMS_NORM+MUL , 5 +gemma ,0 ,decode ,RMS_NORM+MUL , 80 +gemma2 ,0 ,prefill ,RMS_NORM+MUL , 5 +gemma2 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 +gemma2 ,0 ,decode ,RMS_NORM+MUL , 80 +gemma2 ,0 ,decode ,RMS_NORM+MUL+ADD , 64 +glm-dsa ,0 ,prefill ,NORM+MUL+ADD , 2 +glm-dsa ,0 ,prefill ,RMS_NORM+MUL , 9 +glm-dsa ,0 ,prefill ,ADD+ADD , 1 +glm-dsa ,0 ,decode ,NORM+MUL+ADD , 32 +glm-dsa ,0 ,decode ,RMS_NORM+MUL , 144 +glm-dsa ,0 ,decode ,ADD+ADD , 16 +glm4 ,0 ,prefill ,RMS_NORM+MUL , 5 +glm4 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 +glm4 ,0 ,decode ,RMS_NORM+MUL , 80 +glm4 ,0 ,decode ,RMS_NORM+MUL+ADD , 64 +glm4moe ,1 ,prefill ,RMS_NORM+MUL , 9 +glm4moe ,1 ,prefill ,ADD+ADD , 1 +glm4moe ,1 ,decode ,RMS_NORM+MUL , 144 +glm4moe ,1 ,decode ,ADD+ADD , 16 +gpt-oss ,0 ,prefill ,RMS_NORM+MUL , 5 +gpt-oss ,0 ,decode ,RMS_NORM+MUL , 80 +gpt2 ,0 ,prefill ,NORM+MUL+ADD , 5 +gpt2 ,0 ,decode ,NORM+MUL+ADD , 80 +gptneox ,0 ,prefill ,NORM+MUL+ADD , 5 +gptneox ,0 ,decode ,NORM+MUL+ADD , 80 +granite ,0 ,prefill ,RMS_NORM+MUL , 5 +granite ,0 ,decode ,RMS_NORM+MUL , 80 +granite ,0 ,prefill ,RMS_NORM+MUL , 5 +granite ,0 ,decode ,RMS_NORM+MUL , 80 +granitehybrid ,0 ,prefill ,RMS_NORM+MUL , 6 +granitehybrid ,0 ,decode ,RMS_NORM+MUL , 96 +granitemoe ,1 ,prefill ,RMS_NORM+MUL , 5 +granitemoe ,1 ,decode ,RMS_NORM+MUL , 80 +granitemoe ,1 ,prefill ,RMS_NORM+MUL , 5 +granitemoe ,1 ,decode ,RMS_NORM+MUL , 80 +grok ,0 ,prefill ,RMS_NORM+MUL , 5 +grok ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 +grok ,0 ,decode ,RMS_NORM+MUL , 80 +grok ,0 ,decode ,RMS_NORM+MUL+ADD , 64 +grovemoe ,1 ,prefill ,RMS_NORM+MUL , 9 +grovemoe ,1 ,prefill ,ADD+ADD , 2 +grovemoe ,1 ,decode ,RMS_NORM+MUL , 144 +grovemoe ,1 ,decode ,ADD+ADD , 32 +hunyuan-dense ,0 ,prefill ,RMS_NORM+MUL , 9 +hunyuan-dense ,0 ,decode ,RMS_NORM+MUL , 144 +hunyuan-moe ,1 ,prefill ,RMS_NORM+MUL , 9 +hunyuan-moe ,1 ,prefill ,ADD+ADD , 2 +hunyuan-moe ,1 ,decode ,RMS_NORM+MUL , 144 +hunyuan-moe ,1 ,decode ,ADD+ADD , 32 +hunyuan_vl ,0 ,prefill ,RMS_NORM+MUL , 9 +hunyuan_vl ,0 ,decode ,RMS_NORM+MUL , 144 +hy_v3 ,0 ,prefill ,RMS_NORM+MUL , 9 +hy_v3 ,0 ,prefill ,ADD+ADD , 2 +hy_v3 ,0 ,decode ,RMS_NORM+MUL , 144 +hy_v3 ,0 ,decode ,ADD+ADD , 32 +internlm2 ,0 ,prefill ,RMS_NORM+MUL , 5 +internlm2 ,0 ,decode ,RMS_NORM+MUL , 80 +jais ,0 ,prefill ,NORM+MUL+ADD , 5 +jais ,0 ,decode ,NORM+MUL+ADD , 80 +jais2 ,0 ,prefill ,NORM+MUL+ADD , 5 +jais2 ,0 ,decode ,NORM+MUL+ADD , 80 +jamba ,0 ,prefill ,RMS_NORM+MUL , 8 +jamba ,0 ,decode ,RMS_NORM+MUL , 128 +kimi-k3 ,0 ,prefill ,RMS_NORM+MUL , 17 +kimi-k3 ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 +kimi-k3 ,0 ,decode ,RMS_NORM+MUL , 272 +kimi-k3 ,0 ,decode ,GATED_DELTA_NET+CPY , 16 +kimi-linear ,0 ,prefill ,RMS_NORM+MUL , 7 +kimi-linear ,0 ,prefill ,ADD+ADD , 1 +kimi-linear ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 +kimi-linear ,0 ,decode ,RMS_NORM+MUL , 112 +kimi-linear ,0 ,decode ,ADD+ADD , 16 +kimi-linear ,0 ,decode ,GATED_DELTA_NET+CPY , 16 +lfm2 ,0 ,prefill ,RMS_NORM+MUL , 7 +lfm2 ,0 ,decode ,RMS_NORM+MUL , 112 +lfm2moe ,1 ,prefill ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,decode ,RMS_NORM+MUL , 112 +llada ,0 ,prefill ,RMS_NORM+MUL , 5 +llada ,0 ,decode ,RMS_NORM+MUL , 80 +llada-moe ,1 ,prefill ,RMS_NORM+MUL , 9 +llada-moe ,1 ,decode ,RMS_NORM+MUL , 144 +llama ,0 ,prefill ,RMS_NORM+MUL , 5 +llama ,0 ,decode ,RMS_NORM+MUL , 80 +llama ,0 ,prefill ,RMS_NORM+MUL , 5 +llama ,0 ,decode ,RMS_NORM+MUL , 80 +llama4 ,0 ,prefill ,RMS_NORM+MUL , 9 +llama4 ,0 ,prefill ,ADD+ADD , 2 +llama4 ,0 ,decode ,RMS_NORM+MUL , 144 +llama4 ,0 ,decode ,ADD+ADD , 32 +maincoder ,0 ,prefill ,RMS_NORM+MUL , 9 +maincoder ,0 ,decode ,RMS_NORM+MUL , 144 +mamba ,0 ,prefill ,RMS_NORM+MUL , 3 +mamba ,0 ,decode ,RMS_NORM+MUL , 48 +mamba2 ,0 ,prefill ,RMS_NORM+MUL , 5 +mamba2 ,0 ,decode ,RMS_NORM+MUL , 80 +minicpm ,0 ,prefill ,RMS_NORM+MUL , 5 +minicpm ,0 ,decode ,RMS_NORM+MUL , 80 +minicpm ,0 ,prefill ,RMS_NORM+MUL , 5 +minicpm ,0 ,decode ,RMS_NORM+MUL , 80 +minicpm3 ,0 ,prefill ,RMS_NORM+MUL , 9 +minicpm3 ,0 ,decode ,RMS_NORM+MUL , 144 +minimax-01 ,0 ,prefill ,RMS_NORM+MUL , 6 +minimax-01 ,0 ,decode ,RMS_NORM+MUL , 96 +minimax-m2 ,0 ,prefill ,RMS_NORM+MUL , 9 +minimax-m2 ,0 ,decode ,RMS_NORM+MUL , 144 +minimax-m3 ,0 ,prefill ,RMS_NORM+MUL , 11 +minimax-m3 ,0 ,prefill ,ADD+ADD , 1 +minimax-m3 ,0 ,decode ,RMS_NORM+MUL , 176 +minimax-m3 ,0 ,decode ,ADD+ADD , 16 +mistral3 ,0 ,prefill ,RMS_NORM+MUL , 5 +mistral3 ,0 ,decode ,RMS_NORM+MUL , 80 +mistral3 ,0 ,prefill ,RMS_NORM+MUL , 5 +mistral3 ,0 ,decode ,RMS_NORM+MUL , 80 +mistral4 ,0 ,prefill ,RMS_NORM+MUL , 9 +mistral4 ,0 ,prefill ,ADD+ADD , 1 +mistral4 ,0 ,decode ,RMS_NORM+MUL , 144 +mistral4 ,0 ,decode ,ADD+ADD , 16 +mpt ,0 ,prefill ,NORM+MUL+ADD , 5 +mpt ,0 ,decode ,NORM+MUL+ADD , 80 +nanbeige ,0 ,prefill ,RMS_NORM+MUL , 5 +nanbeige ,0 ,decode ,RMS_NORM+MUL , 80 +nemotron ,0 ,prefill ,NORM+MUL+ADD , 5 +nemotron ,0 ,decode ,NORM+MUL+ADD , 80 +nemotron_h ,0 ,prefill ,RMS_NORM+MUL , 5 +nemotron_h ,0 ,decode ,RMS_NORM+MUL , 80 +nemotron_h_moe ,1 ,prefill ,RMS_NORM+MUL , 5 +nemotron_h_moe ,1 ,decode ,RMS_NORM+MUL , 80 +olmoe ,1 ,prefill ,RMS_NORM+MUL , 9 +olmoe ,1 ,decode ,RMS_NORM+MUL , 144 +openelm ,0 ,prefill ,RMS_NORM+MUL , 9 +openelm ,0 ,decode ,RMS_NORM+MUL , 144 +orion ,0 ,prefill ,NORM+MUL+ADD , 5 +orion ,0 ,decode ,NORM+MUL+ADD , 80 +paddleocr ,0 ,prefill ,RMS_NORM+MUL , 5 +paddleocr ,0 ,decode ,RMS_NORM+MUL , 80 +pangu-embedded ,0 ,prefill ,RMS_NORM+MUL , 5 +pangu-embedded ,0 ,decode ,RMS_NORM+MUL , 80 +phi2 ,0 ,prefill ,NORM+MUL+ADD , 3 +phi2 ,0 ,prefill ,ADD+ADD , 2 +phi2 ,0 ,decode ,NORM+MUL+ADD , 48 +phi2 ,0 ,decode ,ADD+ADD , 32 +phi3 ,0 ,prefill ,RMS_NORM+MUL , 5 +phi3 ,0 ,decode ,RMS_NORM+MUL , 80 +phimoe ,1 ,prefill ,RMS_NORM+MUL+ADD , 5 +phimoe ,1 ,decode ,RMS_NORM+MUL+ADD , 80 +plamo ,0 ,prefill ,RMS_NORM+MUL , 3 +plamo ,0 ,prefill ,ADD+ADD , 2 +plamo ,0 ,decode ,RMS_NORM+MUL , 48 +plamo ,0 ,decode ,ADD+ADD , 32 +plamo2 ,0 ,prefill ,RMS_NORM+MUL , 11 +plamo2 ,0 ,prefill ,RMS_NORM+MUL+ADD , 3 +plamo2 ,0 ,decode ,RMS_NORM+MUL , 176 +plamo2 ,0 ,decode ,RMS_NORM+MUL+ADD , 48 +pockettts ,0 ,prefill ,NORM+MUL+ADD , 5 +pockettts ,0 ,decode ,NORM+MUL+ADD , 80 +qwen ,0 ,prefill ,RMS_NORM+MUL , 5 +qwen ,0 ,decode ,RMS_NORM+MUL , 80 +qwen2 ,0 ,prefill ,RMS_NORM+MUL , 5 +qwen2 ,0 ,decode ,RMS_NORM+MUL , 80 +qwen2moe ,1 ,prefill ,RMS_NORM+MUL , 5 +qwen2moe ,1 ,prefill ,ADD+ADD , 2 +qwen2moe ,1 ,decode ,RMS_NORM+MUL , 80 +qwen2moe ,1 ,decode ,ADD+ADD , 32 +qwen2vl ,0 ,prefill ,RMS_NORM+MUL , 5 +qwen2vl ,0 ,decode ,RMS_NORM+MUL , 80 +qwen3 ,0 ,prefill ,RMS_NORM+MUL , 9 +qwen3 ,0 ,decode ,RMS_NORM+MUL , 144 +qwen35 ,0 ,prefill ,RMS_NORM+MUL , 8 +qwen35 ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 +qwen35 ,0 ,decode ,RMS_NORM+MUL , 128 +qwen35 ,0 ,decode ,GATED_DELTA_NET+CPY , 16 +qwen35moe ,1 ,prefill ,RMS_NORM+MUL , 8 +qwen35moe ,1 ,prefill ,ADD+ADD , 2 +qwen35moe ,1 ,prefill ,GATED_DELTA_NET+CPY , 1 +qwen35moe ,1 ,decode ,RMS_NORM+MUL , 128 +qwen35moe ,1 ,decode ,ADD+ADD , 32 +qwen35moe ,1 ,decode ,GATED_DELTA_NET+CPY , 16 +qwen3moe ,1 ,prefill ,RMS_NORM+MUL , 9 +qwen3moe ,1 ,decode ,RMS_NORM+MUL , 144 +qwen3next ,0 ,prefill ,RMS_NORM+MUL , 8 +qwen3next ,0 ,prefill ,ADD+ADD , 2 +qwen3next ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 +qwen3next ,0 ,decode ,RMS_NORM+MUL , 128 +qwen3next ,0 ,decode ,ADD+ADD , 32 +qwen3next ,0 ,decode ,GATED_DELTA_NET+CPY , 16 +qwen3tts ,0 ,prefill ,RMS_NORM+MUL , 9 +qwen3tts ,0 ,decode ,RMS_NORM+MUL , 144 +qwen3vl ,0 ,prefill ,RMS_NORM+MUL , 9 +qwen3vl ,0 ,decode ,RMS_NORM+MUL , 144 +qwen3vlmoe ,1 ,prefill ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,decode ,RMS_NORM+MUL , 144 +qwen4exp ,0 ,prefill ,RMS_NORM+MUL , 5 +qwen4exp ,0 ,prefill ,ADD+ADD+ADD , 5 +qwen4exp ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 +qwen4exp ,0 ,decode ,RMS_NORM+MUL , 80 +qwen4exp ,0 ,decode ,ADD+ADD+ADD , 80 +qwen4exp ,0 ,decode ,GATED_DELTA_NET+CPY , 16 +refact ,0 ,prefill ,RMS_NORM+MUL , 5 +refact ,0 ,decode ,RMS_NORM+MUL , 80 +refact ,0 ,prefill ,RMS_NORM+MUL , 5 +refact ,0 ,decode ,RMS_NORM+MUL , 80 +rnd1 ,0 ,prefill ,RMS_NORM+MUL , 9 +rnd1 ,0 ,decode ,RMS_NORM+MUL , 144 +seed_oss ,0 ,prefill ,RMS_NORM+MUL , 5 +seed_oss ,0 ,decode ,RMS_NORM+MUL , 80 +smallthinker ,0 ,prefill ,RMS_NORM+MUL , 5 +smallthinker ,0 ,decode ,RMS_NORM+MUL , 80 +smollm3 ,0 ,prefill ,RMS_NORM+MUL , 5 +smollm3 ,0 ,decode ,RMS_NORM+MUL , 80 +stablelm ,0 ,prefill ,NORM+MUL , 4 +stablelm ,0 ,prefill ,NORM+MUL+ADD , 5 +stablelm ,0 ,decode ,NORM+MUL , 64 +stablelm ,0 ,decode ,NORM+MUL+ADD , 80 +starcoder ,0 ,prefill ,NORM+MUL+ADD , 5 +starcoder ,0 ,decode ,NORM+MUL+ADD , 80 +starcoder2 ,0 ,prefill ,NORM+MUL+ADD , 5 +starcoder2 ,0 ,decode ,NORM+MUL+ADD , 80 +talkie ,0 ,prefill ,ADD+ADD , 2 +talkie ,0 ,decode ,ADD+ADD , 32 +xverse ,0 ,prefill ,RMS_NORM+MUL , 5 +xverse ,0 ,decode ,RMS_NORM+MUL , 80 diff --git a/tests/fusion/MTL.tsv b/tests/fusion/MTL.tsv deleted file mode 100644 index f2e8ab3729..0000000000 --- a/tests/fusion/MTL.tsv +++ /dev/null @@ -1,300 +0,0 @@ -# test-fusion baseline for device MTL -# arch moe mode label count -arcee 0 prefill RMS_NORM+MUL 5 -arcee 0 decode RMS_NORM+MUL 80 -arctic 0 prefill RMS_NORM+MUL 7 -arctic 0 decode RMS_NORM+MUL 112 -baichuan 0 prefill RMS_NORM+MUL 5 -baichuan 0 decode RMS_NORM+MUL 80 -bailingmoe 1 prefill RMS_NORM+MUL 5 -bailingmoe 1 prefill ADD+ADD 2 -bailingmoe 1 decode RMS_NORM+MUL 80 -bailingmoe 1 decode ADD+ADD 32 -bailingmoe2 1 prefill RMS_NORM+MUL 9 -bailingmoe2 1 prefill ADD+ADD 1 -bailingmoe2 1 decode RMS_NORM+MUL 144 -bailingmoe2 1 decode ADD+ADD 16 -bailingmoe3 1 prefill RMS_NORM+MUL 8 -bailingmoe3 1 prefill ADD+ADD 1 -bailingmoe3 1 prefill GATED_DELTA_NET+CPY 1 -bailingmoe3 1 decode RMS_NORM+MUL 128 -bailingmoe3 1 decode ADD+ADD 16 -bailingmoe3 1 decode GATED_DELTA_NET+CPY 16 -bloom 0 prefill NORM+MUL+ADD 6 -bloom 0 decode NORM+MUL+ADD 96 -chatglm 0 prefill RMS_NORM+MUL 5 -chatglm 0 decode RMS_NORM+MUL 80 -codeshell 0 prefill NORM+MUL+ADD 5 -codeshell 0 decode NORM+MUL+ADD 80 -cogvlm 0 prefill RMS_NORM+MUL 5 -cogvlm 0 decode RMS_NORM+MUL 80 -command-r 0 prefill NORM+MUL 3 -command-r 0 decode NORM+MUL 48 -dbrx 0 prefill NORM+MUL 5 -dbrx 0 decode NORM+MUL 80 -deci 0 prefill RMS_NORM+MUL 5 -deci 0 decode RMS_NORM+MUL 80 -deepseek 0 prefill RMS_NORM+MUL 5 -deepseek 0 prefill ADD+ADD 1 -deepseek 0 decode RMS_NORM+MUL 80 -deepseek 0 decode ADD+ADD 16 -deepseek2 0 prefill RMS_NORM+MUL 9 -deepseek2 0 prefill ADD+ADD 1 -deepseek2 0 decode RMS_NORM+MUL 144 -deepseek2 0 decode ADD+ADD 16 -deepseek32 0 prefill NORM+MUL+ADD 2 -deepseek32 0 prefill RMS_NORM+MUL 9 -deepseek32 0 prefill ADD+ADD 1 -deepseek32 0 decode NORM+MUL+ADD 32 -deepseek32 0 decode RMS_NORM+MUL 144 -deepseek32 0 decode ADD+ADD 16 -deepseek4 0 prefill RMS_NORM+MUL 20 -deepseek4 0 decode RMS_NORM+MUL 320 -dots1 0 prefill RMS_NORM+MUL 9 -dots1 0 prefill ADD+ADD 1 -dots1 0 decode RMS_NORM+MUL 144 -dots1 0 decode ADD+ADD 16 -dream 0 prefill RMS_NORM+MUL 5 -dream 0 decode RMS_NORM+MUL 80 -ernie4_5-moe 1 prefill RMS_NORM+MUL 5 -ernie4_5-moe 1 prefill ADD+ADD 1 -ernie4_5-moe 1 decode RMS_NORM+MUL 80 -ernie4_5-moe 1 decode ADD+ADD 16 -ernie4_5 0 prefill RMS_NORM+MUL 5 -ernie4_5 0 decode RMS_NORM+MUL 80 -exaone 0 prefill RMS_NORM+MUL 5 -exaone 0 decode RMS_NORM+MUL 80 -exaone4 0 prefill RMS_NORM+MUL 5 -exaone4 0 prefill RMS_NORM+MUL+ADD 4 -exaone4 0 decode RMS_NORM+MUL 80 -exaone4 0 decode RMS_NORM+MUL+ADD 64 -falcon 0 prefill NORM+MUL+ADD 5 -falcon 0 prefill ADD+ADD 2 -falcon 0 decode NORM+MUL+ADD 80 -falcon 0 decode ADD+ADD 32 -falcon-h1 0 prefill RMS_NORM+MUL 9 -falcon-h1 0 prefill ADD+ADD 2 -falcon-h1 0 decode RMS_NORM+MUL 144 -falcon-h1 0 decode ADD+ADD 32 -gemma 0 prefill RMS_NORM+MUL 5 -gemma 0 decode RMS_NORM+MUL 80 -gemma2 0 prefill RMS_NORM+MUL 5 -gemma2 0 prefill RMS_NORM+MUL+ADD 4 -gemma2 0 decode RMS_NORM+MUL 80 -gemma2 0 decode RMS_NORM+MUL+ADD 64 -glm-dsa 0 prefill NORM+MUL+ADD 2 -glm-dsa 0 prefill RMS_NORM+MUL 9 -glm-dsa 0 prefill ADD+ADD 1 -glm-dsa 0 decode NORM+MUL+ADD 32 -glm-dsa 0 decode RMS_NORM+MUL 144 -glm-dsa 0 decode ADD+ADD 16 -glm4 0 prefill RMS_NORM+MUL 5 -glm4 0 prefill RMS_NORM+MUL+ADD 4 -glm4 0 decode RMS_NORM+MUL 80 -glm4 0 decode RMS_NORM+MUL+ADD 64 -glm4moe 1 prefill RMS_NORM+MUL 9 -glm4moe 1 prefill ADD+ADD 1 -glm4moe 1 decode RMS_NORM+MUL 144 -glm4moe 1 decode ADD+ADD 16 -gpt-oss 0 prefill RMS_NORM+MUL 5 -gpt-oss 0 decode RMS_NORM+MUL 80 -gpt2 0 prefill NORM+MUL+ADD 5 -gpt2 0 decode NORM+MUL+ADD 80 -gptneox 0 prefill NORM+MUL+ADD 5 -gptneox 0 decode NORM+MUL+ADD 80 -granite 0 prefill RMS_NORM+MUL 5 -granite 0 decode RMS_NORM+MUL 80 -granite 0 prefill RMS_NORM+MUL 5 -granite 0 decode RMS_NORM+MUL 80 -granitehybrid 0 prefill RMS_NORM+MUL 6 -granitehybrid 0 decode RMS_NORM+MUL 96 -granitemoe 1 prefill RMS_NORM+MUL 5 -granitemoe 1 decode RMS_NORM+MUL 80 -granitemoe 1 prefill RMS_NORM+MUL 5 -granitemoe 1 decode RMS_NORM+MUL 80 -grok 0 prefill RMS_NORM+MUL 5 -grok 0 prefill RMS_NORM+MUL+ADD 4 -grok 0 decode RMS_NORM+MUL 80 -grok 0 decode RMS_NORM+MUL+ADD 64 -grovemoe 1 prefill RMS_NORM+MUL 9 -grovemoe 1 prefill ADD+ADD 2 -grovemoe 1 decode RMS_NORM+MUL 144 -grovemoe 1 decode ADD+ADD 32 -hunyuan-dense 0 prefill RMS_NORM+MUL 9 -hunyuan-dense 0 decode RMS_NORM+MUL 144 -hunyuan-moe 1 prefill RMS_NORM+MUL 9 -hunyuan-moe 1 prefill ADD+ADD 2 -hunyuan-moe 1 decode RMS_NORM+MUL 144 -hunyuan-moe 1 decode ADD+ADD 32 -hunyuan_vl 0 prefill RMS_NORM+MUL 9 -hunyuan_vl 0 decode RMS_NORM+MUL 144 -hy_v3 0 prefill RMS_NORM+MUL 9 -hy_v3 0 prefill ADD+ADD 2 -hy_v3 0 decode RMS_NORM+MUL 144 -hy_v3 0 decode ADD+ADD 32 -internlm2 0 prefill RMS_NORM+MUL 5 -internlm2 0 decode RMS_NORM+MUL 80 -jais 0 prefill NORM+MUL+ADD 5 -jais 0 decode NORM+MUL+ADD 80 -jais2 0 prefill NORM+MUL+ADD 5 -jais2 0 decode NORM+MUL+ADD 80 -jamba 0 prefill RMS_NORM+MUL 8 -jamba 0 decode RMS_NORM+MUL 128 -kimi-k3 0 prefill RMS_NORM+MUL 17 -kimi-k3 0 prefill GATED_DELTA_NET+CPY 1 -kimi-k3 0 decode RMS_NORM+MUL 272 -kimi-k3 0 decode GATED_DELTA_NET+CPY 16 -kimi-linear 0 prefill RMS_NORM+MUL 7 -kimi-linear 0 prefill ADD+ADD 1 -kimi-linear 0 prefill GATED_DELTA_NET+CPY 1 -kimi-linear 0 decode RMS_NORM+MUL 112 -kimi-linear 0 decode ADD+ADD 16 -kimi-linear 0 decode GATED_DELTA_NET+CPY 16 -lfm2 0 prefill RMS_NORM+MUL 7 -lfm2 0 decode RMS_NORM+MUL 112 -lfm2moe 1 prefill RMS_NORM+MUL 7 -lfm2moe 1 decode RMS_NORM+MUL 112 -llada 0 prefill RMS_NORM+MUL 5 -llada 0 decode RMS_NORM+MUL 80 -llada-moe 1 prefill RMS_NORM+MUL 9 -llada-moe 1 decode RMS_NORM+MUL 144 -llama 0 prefill RMS_NORM+MUL 5 -llama 0 decode RMS_NORM+MUL 80 -llama 0 prefill RMS_NORM+MUL 5 -llama 0 decode RMS_NORM+MUL 80 -llama4 0 prefill RMS_NORM+MUL 9 -llama4 0 prefill ADD+ADD 2 -llama4 0 decode RMS_NORM+MUL 144 -llama4 0 decode ADD+ADD 32 -maincoder 0 prefill RMS_NORM+MUL 9 -maincoder 0 decode RMS_NORM+MUL 144 -mamba 0 prefill RMS_NORM+MUL 3 -mamba 0 decode RMS_NORM+MUL 48 -mamba2 0 prefill RMS_NORM+MUL 5 -mamba2 0 decode RMS_NORM+MUL 80 -minicpm 0 prefill RMS_NORM+MUL 5 -minicpm 0 decode RMS_NORM+MUL 80 -minicpm 0 prefill RMS_NORM+MUL 5 -minicpm 0 decode RMS_NORM+MUL 80 -minicpm3 0 prefill RMS_NORM+MUL 9 -minicpm3 0 decode RMS_NORM+MUL 144 -minimax-01 0 prefill RMS_NORM+MUL 6 -minimax-01 0 decode RMS_NORM+MUL 96 -minimax-m2 0 prefill RMS_NORM+MUL 9 -minimax-m2 0 decode RMS_NORM+MUL 144 -minimax-m3 0 prefill RMS_NORM+MUL 11 -minimax-m3 0 prefill ADD+ADD 1 -minimax-m3 0 decode RMS_NORM+MUL 176 -minimax-m3 0 decode ADD+ADD 16 -mistral3 0 prefill RMS_NORM+MUL 5 -mistral3 0 decode RMS_NORM+MUL 80 -mistral3 0 prefill RMS_NORM+MUL 5 -mistral3 0 decode RMS_NORM+MUL 80 -mistral4 0 prefill RMS_NORM+MUL 9 -mistral4 0 prefill ADD+ADD 1 -mistral4 0 decode RMS_NORM+MUL 144 -mistral4 0 decode ADD+ADD 16 -mpt 0 prefill NORM+MUL+ADD 5 -mpt 0 decode NORM+MUL+ADD 80 -nanbeige 0 prefill RMS_NORM+MUL 5 -nanbeige 0 decode RMS_NORM+MUL 80 -nemotron 0 prefill NORM+MUL+ADD 5 -nemotron 0 decode NORM+MUL+ADD 80 -nemotron_h 0 prefill RMS_NORM+MUL 5 -nemotron_h 0 decode RMS_NORM+MUL 80 -nemotron_h_moe 1 prefill RMS_NORM+MUL 5 -nemotron_h_moe 1 decode RMS_NORM+MUL 80 -olmoe 1 prefill RMS_NORM+MUL 9 -olmoe 1 decode RMS_NORM+MUL 144 -openelm 0 prefill RMS_NORM+MUL 9 -openelm 0 decode RMS_NORM+MUL 144 -orion 0 prefill NORM+MUL+ADD 5 -orion 0 decode NORM+MUL+ADD 80 -paddleocr 0 prefill RMS_NORM+MUL 5 -paddleocr 0 decode RMS_NORM+MUL 80 -pangu-embedded 0 prefill RMS_NORM+MUL 5 -pangu-embedded 0 decode RMS_NORM+MUL 80 -phi2 0 prefill NORM+MUL+ADD 3 -phi2 0 prefill ADD+ADD 2 -phi2 0 decode NORM+MUL+ADD 48 -phi2 0 decode ADD+ADD 32 -phi3 0 prefill RMS_NORM+MUL 5 -phi3 0 decode RMS_NORM+MUL 80 -phimoe 1 prefill RMS_NORM+MUL+ADD 5 -phimoe 1 decode RMS_NORM+MUL+ADD 80 -plamo 0 prefill RMS_NORM+MUL 3 -plamo 0 prefill ADD+ADD 2 -plamo 0 decode RMS_NORM+MUL 48 -plamo 0 decode ADD+ADD 32 -plamo2 0 prefill RMS_NORM+MUL 11 -plamo2 0 prefill RMS_NORM+MUL+ADD 3 -plamo2 0 decode RMS_NORM+MUL 176 -plamo2 0 decode RMS_NORM+MUL+ADD 48 -pockettts 0 prefill NORM+MUL+ADD 5 -pockettts 0 decode NORM+MUL+ADD 80 -qwen 0 prefill RMS_NORM+MUL 5 -qwen 0 decode RMS_NORM+MUL 80 -qwen2 0 prefill RMS_NORM+MUL 5 -qwen2 0 decode RMS_NORM+MUL 80 -qwen2moe 1 prefill RMS_NORM+MUL 5 -qwen2moe 1 prefill ADD+ADD 2 -qwen2moe 1 decode RMS_NORM+MUL 80 -qwen2moe 1 decode ADD+ADD 32 -qwen2vl 0 prefill RMS_NORM+MUL 5 -qwen2vl 0 decode RMS_NORM+MUL 80 -qwen3 0 prefill RMS_NORM+MUL 9 -qwen3 0 decode RMS_NORM+MUL 144 -qwen35 0 prefill RMS_NORM+MUL 8 -qwen35 0 prefill GATED_DELTA_NET+CPY 1 -qwen35 0 decode RMS_NORM+MUL 128 -qwen35 0 decode GATED_DELTA_NET+CPY 16 -qwen35moe 1 prefill RMS_NORM+MUL 8 -qwen35moe 1 prefill ADD+ADD 2 -qwen35moe 1 prefill GATED_DELTA_NET+CPY 1 -qwen35moe 1 decode RMS_NORM+MUL 128 -qwen35moe 1 decode ADD+ADD 32 -qwen35moe 1 decode GATED_DELTA_NET+CPY 16 -qwen3moe 1 prefill RMS_NORM+MUL 9 -qwen3moe 1 decode RMS_NORM+MUL 144 -qwen3next 0 prefill RMS_NORM+MUL 8 -qwen3next 0 prefill ADD+ADD 2 -qwen3next 0 prefill GATED_DELTA_NET+CPY 1 -qwen3next 0 decode RMS_NORM+MUL 128 -qwen3next 0 decode ADD+ADD 32 -qwen3next 0 decode GATED_DELTA_NET+CPY 16 -qwen3tts 0 prefill RMS_NORM+MUL 9 -qwen3tts 0 decode RMS_NORM+MUL 144 -qwen3vl 0 prefill RMS_NORM+MUL 9 -qwen3vl 0 decode RMS_NORM+MUL 144 -qwen3vlmoe 1 prefill RMS_NORM+MUL 9 -qwen3vlmoe 1 decode RMS_NORM+MUL 144 -qwen4exp 0 prefill RMS_NORM+MUL 5 -qwen4exp 0 prefill ADD+ADD+ADD 5 -qwen4exp 0 prefill GATED_DELTA_NET+CPY 1 -qwen4exp 0 decode RMS_NORM+MUL 80 -qwen4exp 0 decode ADD+ADD+ADD 80 -qwen4exp 0 decode GATED_DELTA_NET+CPY 16 -refact 0 prefill RMS_NORM+MUL 5 -refact 0 decode RMS_NORM+MUL 80 -refact 0 prefill RMS_NORM+MUL 5 -refact 0 decode RMS_NORM+MUL 80 -rnd1 0 prefill RMS_NORM+MUL 9 -rnd1 0 decode RMS_NORM+MUL 144 -seed_oss 0 prefill RMS_NORM+MUL 5 -seed_oss 0 decode RMS_NORM+MUL 80 -smallthinker 0 prefill RMS_NORM+MUL 5 -smallthinker 0 decode RMS_NORM+MUL 80 -smollm3 0 prefill RMS_NORM+MUL 5 -smollm3 0 decode RMS_NORM+MUL 80 -stablelm 0 prefill NORM+MUL 4 -stablelm 0 prefill NORM+MUL+ADD 5 -stablelm 0 decode NORM+MUL 64 -stablelm 0 decode NORM+MUL+ADD 80 -starcoder 0 prefill NORM+MUL+ADD 5 -starcoder 0 decode NORM+MUL+ADD 80 -starcoder2 0 prefill NORM+MUL+ADD 5 -starcoder2 0 decode NORM+MUL+ADD 80 -talkie 0 prefill ADD+ADD 2 -talkie 0 decode ADD+ADD 32 -xverse 0 prefill RMS_NORM+MUL 5 -xverse 0 decode RMS_NORM+MUL 80 diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp index 10919090a7..e8771e7440 100644 --- a/tests/test-fusion.cpp +++ b/tests/test-fusion.cpp @@ -6,14 +6,14 @@ // - the NMSE between the fused and unfused logits // - the NMSE between the device and a CPU reference // -// the per-fusion-type counters are compared against a per-device baseline file (TSV) so a +// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a // fusion pattern that silently stops matching (or fires when it should not) is caught as a // regression. // // usage: -// test-fusion --models DIR --device MTL0 --record baseline.tsv # generate a baseline -// test-fusion --models DIR --device MTL0 --check baseline.tsv # validate against it -// test-fusion --model FILE --device MTL0 --check baseline.tsv # validate a single model +// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline +// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it +// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model #include "common.h" #include "log.h" @@ -84,7 +84,7 @@ static std::vector get_tokens(const uint32_t n_tokens, const uint32 return ret; } -// trim leading/trailing whitespace (used when parsing padded TSV columns) +// trim leading/trailing whitespace (used when parsing padded CSV columns) static std::string trim(const std::string & s) { const size_t b = s.find_first_not_of(" \t\r\n"); if (b == std::string::npos) { @@ -215,8 +215,8 @@ static void usage(const char * argv0) { printf(" --models DIR run over all .gguf models in a directory\n"); printf(" --model FILE run over a single model file (mutually exclusive with --models)\n"); printf(" --device NAME device to run on (e.g. MTL0, CPU)\n"); - printf(" --record TSV write the golden baseline\n"); - printf(" --check TSV validate the counters against a baseline (default)\n"); + printf(" --record CSV write the golden baseline\n"); + printf(" --check CSV validate the counters against a baseline (default)\n"); printf(" -h, --help show this message and exit\n"); } @@ -343,7 +343,7 @@ int main(int argc, char ** argv) { } std::vector cols; size_t pos = 0; - while ((pos = line.find('\t')) != std::string::npos) { + while ((pos = line.find(',')) != std::string::npos) { cols.push_back(trim(line.substr(0, pos))); line.erase(0, pos + 1); } @@ -471,10 +471,10 @@ int main(int argc, char ** argv) { if (!record_path.empty()) { os << "# test-fusion baseline for device " << base_name << "\n"; os << "# " << std::left - << std::setw(18) << "arch" << '\t' - << std::setw(4) << "moe" << '\t' - << std::setw(8) << "mode" << '\t' - << std::setw(28) << "label" << '\t' + << std::setw(18) << "arch" << ',' + << std::setw(4) << "moe" << ',' + << std::setw(8) << "mode" << ',' + << std::setw(28) << "label" << ',' << std::right << std::setw(7) << "count" << '\n'; } @@ -494,10 +494,10 @@ int main(int argc, char ** argv) { (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status); if (!record_path.empty()) { os << std::left - << std::setw(20) << r.arch << '\t' - << std::setw(4) << (r.moe ? "1" : "0") << '\t' - << std::setw(8) << r.mode << '\t' - << std::setw(28) << r.label << '\t' + << std::setw(20) << r.arch << ',' + << std::setw(4) << (r.moe ? "1" : "0") << ',' + << std::setw(8) << r.mode << ',' + << std::setw(28) << r.label << ',' << std::right << std::setw(7) << r.count_fused << '\n'; } }