mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-09-04 02:37:27 +02:00
tests : dedup fusion baseline into any mode
prefill and decode always produce the same per-graph fusion count, so store a single row per label with mode = "any" and the per-graph count instead of two rows. this halves the baseline size and keeps the check stable. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731
This commit is contained in:
Georgi Gerganov
+150
-300
@@ -1,302 +1,152 @@
|
||||
# test-fusion baseline for device MTL
|
||||
# arch ,moe ,mode ,label , count
|
||||
arcee ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
arcee ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
arctic ,0 ,prefill ,RMS_NORM+MUL , 7
|
||||
arctic ,0 ,decode ,RMS_NORM+MUL , 112
|
||||
baichuan ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
baichuan ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
bailingmoe ,1 ,prefill ,RMS_NORM+MUL , 5
|
||||
bailingmoe ,1 ,prefill ,ADD+ADD , 2
|
||||
bailingmoe ,1 ,decode ,RMS_NORM+MUL , 80
|
||||
bailingmoe ,1 ,decode ,ADD+ADD , 32
|
||||
bailingmoe2 ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
bailingmoe2 ,1 ,prefill ,ADD+ADD , 1
|
||||
bailingmoe2 ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
bailingmoe2 ,1 ,decode ,ADD+ADD , 16
|
||||
bailingmoe3 ,1 ,prefill ,RMS_NORM+MUL , 8
|
||||
bailingmoe3 ,1 ,prefill ,ADD+ADD , 1
|
||||
bailingmoe3 ,1 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
bailingmoe3 ,1 ,decode ,RMS_NORM+MUL , 128
|
||||
bailingmoe3 ,1 ,decode ,ADD+ADD , 16
|
||||
bailingmoe3 ,1 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
bloom ,0 ,prefill ,NORM+MUL+ADD , 6
|
||||
bloom ,0 ,decode ,NORM+MUL+ADD , 96
|
||||
chatglm ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
chatglm ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
codeshell ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
codeshell ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
cogvlm ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
cogvlm ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
command-r ,0 ,prefill ,NORM+MUL , 3
|
||||
command-r ,0 ,decode ,NORM+MUL , 48
|
||||
dbrx ,0 ,prefill ,NORM+MUL , 5
|
||||
dbrx ,0 ,decode ,NORM+MUL , 80
|
||||
deci ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
deci ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
deepseek ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
deepseek ,0 ,prefill ,ADD+ADD , 1
|
||||
deepseek ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
deepseek ,0 ,decode ,ADD+ADD , 16
|
||||
deepseek2 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
deepseek2 ,0 ,prefill ,ADD+ADD , 1
|
||||
deepseek2 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
deepseek2 ,0 ,decode ,ADD+ADD , 16
|
||||
deepseek32 ,0 ,prefill ,NORM+MUL+ADD , 2
|
||||
deepseek32 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
deepseek32 ,0 ,prefill ,ADD+ADD , 1
|
||||
deepseek32 ,0 ,decode ,NORM+MUL+ADD , 32
|
||||
deepseek32 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
deepseek32 ,0 ,decode ,ADD+ADD , 16
|
||||
deepseek4 ,0 ,prefill ,RMS_NORM+MUL , 20
|
||||
deepseek4 ,0 ,decode ,RMS_NORM+MUL , 320
|
||||
dots1 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
dots1 ,0 ,prefill ,ADD+ADD , 1
|
||||
dots1 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
dots1 ,0 ,decode ,ADD+ADD , 16
|
||||
dream ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
dream ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
ernie4_5-moe ,1 ,prefill ,RMS_NORM+MUL , 5
|
||||
ernie4_5-moe ,1 ,prefill ,ADD+ADD , 1
|
||||
ernie4_5-moe ,1 ,decode ,RMS_NORM+MUL , 80
|
||||
ernie4_5-moe ,1 ,decode ,ADD+ADD , 16
|
||||
ernie4_5 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
ernie4_5 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
exaone ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
exaone ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
exaone4 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
exaone4 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4
|
||||
exaone4 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
exaone4 ,0 ,decode ,RMS_NORM+MUL+ADD , 64
|
||||
falcon ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
falcon ,0 ,prefill ,ADD+ADD , 2
|
||||
falcon ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
falcon ,0 ,decode ,ADD+ADD , 32
|
||||
falcon-h1 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
falcon-h1 ,0 ,prefill ,ADD+ADD , 2
|
||||
falcon-h1 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
falcon-h1 ,0 ,decode ,ADD+ADD , 32
|
||||
gemma ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
gemma ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
gemma2 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
gemma2 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4
|
||||
gemma2 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
gemma2 ,0 ,decode ,RMS_NORM+MUL+ADD , 64
|
||||
glm-dsa ,0 ,prefill ,NORM+MUL+ADD , 2
|
||||
glm-dsa ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
glm-dsa ,0 ,prefill ,ADD+ADD , 1
|
||||
glm-dsa ,0 ,decode ,NORM+MUL+ADD , 32
|
||||
glm-dsa ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
glm-dsa ,0 ,decode ,ADD+ADD , 16
|
||||
glm4 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
glm4 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4
|
||||
glm4 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
glm4 ,0 ,decode ,RMS_NORM+MUL+ADD , 64
|
||||
glm4moe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
glm4moe ,1 ,prefill ,ADD+ADD , 1
|
||||
glm4moe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
glm4moe ,1 ,decode ,ADD+ADD , 16
|
||||
gpt-oss ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
gpt-oss ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
gpt2 ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
gpt2 ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
gptneox ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
gptneox ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
granite ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
granite ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
granite ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
granite ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
granitehybrid ,0 ,prefill ,RMS_NORM+MUL , 6
|
||||
granitehybrid ,0 ,decode ,RMS_NORM+MUL , 96
|
||||
granitemoe ,1 ,prefill ,RMS_NORM+MUL , 5
|
||||
granitemoe ,1 ,decode ,RMS_NORM+MUL , 80
|
||||
granitemoe ,1 ,prefill ,RMS_NORM+MUL , 5
|
||||
granitemoe ,1 ,decode ,RMS_NORM+MUL , 80
|
||||
grok ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
grok ,0 ,prefill ,RMS_NORM+MUL+ADD , 4
|
||||
grok ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
grok ,0 ,decode ,RMS_NORM+MUL+ADD , 64
|
||||
grovemoe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
grovemoe ,1 ,prefill ,ADD+ADD , 2
|
||||
grovemoe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
grovemoe ,1 ,decode ,ADD+ADD , 32
|
||||
hunyuan-dense ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
hunyuan-dense ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
hunyuan-moe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
hunyuan-moe ,1 ,prefill ,ADD+ADD , 2
|
||||
hunyuan-moe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
hunyuan-moe ,1 ,decode ,ADD+ADD , 32
|
||||
hunyuan_vl ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
hunyuan_vl ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
hy_v3 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
hy_v3 ,0 ,prefill ,ADD+ADD , 2
|
||||
hy_v3 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
hy_v3 ,0 ,decode ,ADD+ADD , 32
|
||||
internlm2 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
internlm2 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
jais ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
jais ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
jais2 ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
jais2 ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
jamba ,0 ,prefill ,RMS_NORM+MUL , 8
|
||||
jamba ,0 ,decode ,RMS_NORM+MUL , 128
|
||||
kimi-k3 ,0 ,prefill ,RMS_NORM+MUL , 17
|
||||
kimi-k3 ,0 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
kimi-k3 ,0 ,decode ,RMS_NORM+MUL , 272
|
||||
kimi-k3 ,0 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
kimi-linear ,0 ,prefill ,RMS_NORM+MUL , 7
|
||||
kimi-linear ,0 ,prefill ,ADD+ADD , 1
|
||||
kimi-linear ,0 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
kimi-linear ,0 ,decode ,RMS_NORM+MUL , 112
|
||||
kimi-linear ,0 ,decode ,ADD+ADD , 16
|
||||
kimi-linear ,0 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
lfm2 ,0 ,prefill ,RMS_NORM+MUL , 7
|
||||
lfm2 ,0 ,decode ,RMS_NORM+MUL , 112
|
||||
lfm2moe ,1 ,prefill ,RMS_NORM+MUL , 7
|
||||
lfm2moe ,1 ,decode ,RMS_NORM+MUL , 112
|
||||
llada ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
llada ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
llada-moe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
llada-moe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
llama ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
llama ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
llama ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
llama ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
llama4 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
llama4 ,0 ,prefill ,ADD+ADD , 2
|
||||
llama4 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
llama4 ,0 ,decode ,ADD+ADD , 32
|
||||
maincoder ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
maincoder ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
mamba ,0 ,prefill ,RMS_NORM+MUL , 3
|
||||
mamba ,0 ,decode ,RMS_NORM+MUL , 48
|
||||
mamba2 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
mamba2 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
minicpm ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
minicpm ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
minicpm3 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
minicpm3 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
minimax-01 ,0 ,prefill ,RMS_NORM+MUL , 6
|
||||
minimax-01 ,0 ,decode ,RMS_NORM+MUL , 96
|
||||
minimax-m2 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
minimax-m2 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
minimax-m3 ,0 ,prefill ,RMS_NORM+MUL , 11
|
||||
minimax-m3 ,0 ,prefill ,ADD+ADD , 1
|
||||
minimax-m3 ,0 ,decode ,RMS_NORM+MUL , 176
|
||||
minimax-m3 ,0 ,decode ,ADD+ADD , 16
|
||||
mistral3 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
mistral3 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
mistral3 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
mistral3 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
mistral4 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
mistral4 ,0 ,prefill ,ADD+ADD , 1
|
||||
mistral4 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
mistral4 ,0 ,decode ,ADD+ADD , 16
|
||||
mpt ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
mpt ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
nanbeige ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
nanbeige ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
nemotron ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
nemotron ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
nemotron_h ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
nemotron_h ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
nemotron_h_moe ,1 ,prefill ,RMS_NORM+MUL , 5
|
||||
nemotron_h_moe ,1 ,decode ,RMS_NORM+MUL , 80
|
||||
olmoe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
olmoe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
openelm ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
openelm ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
orion ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
orion ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
paddleocr ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
paddleocr ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
pangu-embedded ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
pangu-embedded ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
phi2 ,0 ,prefill ,NORM+MUL+ADD , 3
|
||||
phi2 ,0 ,prefill ,ADD+ADD , 2
|
||||
phi2 ,0 ,decode ,NORM+MUL+ADD , 48
|
||||
phi2 ,0 ,decode ,ADD+ADD , 32
|
||||
phi3 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
phi3 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
phimoe ,1 ,prefill ,RMS_NORM+MUL+ADD , 5
|
||||
phimoe ,1 ,decode ,RMS_NORM+MUL+ADD , 80
|
||||
plamo ,0 ,prefill ,RMS_NORM+MUL , 3
|
||||
plamo ,0 ,prefill ,ADD+ADD , 2
|
||||
plamo ,0 ,decode ,RMS_NORM+MUL , 48
|
||||
plamo ,0 ,decode ,ADD+ADD , 32
|
||||
plamo2 ,0 ,prefill ,RMS_NORM+MUL , 11
|
||||
plamo2 ,0 ,prefill ,RMS_NORM+MUL+ADD , 3
|
||||
plamo2 ,0 ,decode ,RMS_NORM+MUL , 176
|
||||
plamo2 ,0 ,decode ,RMS_NORM+MUL+ADD , 48
|
||||
pockettts ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
pockettts ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
qwen ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
qwen ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
qwen2 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
qwen2 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
qwen2moe ,1 ,prefill ,RMS_NORM+MUL , 5
|
||||
qwen2moe ,1 ,prefill ,ADD+ADD , 2
|
||||
qwen2moe ,1 ,decode ,RMS_NORM+MUL , 80
|
||||
qwen2moe ,1 ,decode ,ADD+ADD , 32
|
||||
qwen2vl ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
qwen2vl ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
qwen3 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
qwen3 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
qwen35 ,0 ,prefill ,RMS_NORM+MUL , 8
|
||||
qwen35 ,0 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
qwen35 ,0 ,decode ,RMS_NORM+MUL , 128
|
||||
qwen35 ,0 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
qwen35moe ,1 ,prefill ,RMS_NORM+MUL , 8
|
||||
qwen35moe ,1 ,prefill ,ADD+ADD , 2
|
||||
qwen35moe ,1 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
qwen35moe ,1 ,decode ,RMS_NORM+MUL , 128
|
||||
qwen35moe ,1 ,decode ,ADD+ADD , 32
|
||||
qwen35moe ,1 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
qwen3moe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
qwen3moe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
qwen3next ,0 ,prefill ,RMS_NORM+MUL , 8
|
||||
qwen3next ,0 ,prefill ,ADD+ADD , 2
|
||||
qwen3next ,0 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
qwen3next ,0 ,decode ,RMS_NORM+MUL , 128
|
||||
qwen3next ,0 ,decode ,ADD+ADD , 32
|
||||
qwen3next ,0 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
qwen3tts ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
qwen3tts ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
qwen3vl ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
qwen3vl ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
qwen3vlmoe ,1 ,prefill ,RMS_NORM+MUL , 9
|
||||
qwen3vlmoe ,1 ,decode ,RMS_NORM+MUL , 144
|
||||
qwen4exp ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
qwen4exp ,0 ,prefill ,ADD+ADD+ADD , 5
|
||||
qwen4exp ,0 ,prefill ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9
|
||||
qwen4exp ,0 ,prefill ,GATED_DELTA_NET+CPY , 1
|
||||
qwen4exp ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
qwen4exp ,0 ,decode ,ADD+ADD+ADD , 80
|
||||
qwen4exp ,0 ,decode ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 144
|
||||
qwen4exp ,0 ,decode ,GATED_DELTA_NET+CPY , 16
|
||||
refact ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
refact ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
refact ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
refact ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
rnd1 ,0 ,prefill ,RMS_NORM+MUL , 9
|
||||
rnd1 ,0 ,decode ,RMS_NORM+MUL , 144
|
||||
seed_oss ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
seed_oss ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
smallthinker ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
smallthinker ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
smollm3 ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
smollm3 ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
stablelm ,0 ,prefill ,NORM+MUL , 4
|
||||
stablelm ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
stablelm ,0 ,decode ,NORM+MUL , 64
|
||||
stablelm ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
starcoder ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
starcoder ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
starcoder2 ,0 ,prefill ,NORM+MUL+ADD , 5
|
||||
starcoder2 ,0 ,decode ,NORM+MUL+ADD , 80
|
||||
talkie ,0 ,prefill ,ADD+ADD , 2
|
||||
talkie ,0 ,decode ,ADD+ADD , 32
|
||||
xverse ,0 ,prefill ,RMS_NORM+MUL , 5
|
||||
xverse ,0 ,decode ,RMS_NORM+MUL , 80
|
||||
arcee ,0 ,any ,RMS_NORM+MUL , 5
|
||||
arctic ,0 ,any ,RMS_NORM+MUL , 7
|
||||
baichuan ,0 ,any ,RMS_NORM+MUL , 5
|
||||
bailingmoe ,1 ,any ,ADD+ADD , 2
|
||||
bailingmoe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
bailingmoe2 ,1 ,any ,ADD+ADD , 1
|
||||
bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9
|
||||
bailingmoe3 ,1 ,any ,ADD+ADD , 1
|
||||
bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8
|
||||
bloom ,0 ,any ,NORM+MUL+ADD , 6
|
||||
chatglm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
codeshell ,0 ,any ,NORM+MUL+ADD , 5
|
||||
cogvlm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
command-r ,0 ,any ,NORM+MUL , 3
|
||||
dbrx ,0 ,any ,NORM+MUL , 5
|
||||
deci ,0 ,any ,RMS_NORM+MUL , 5
|
||||
deepseek ,0 ,any ,ADD+ADD , 1
|
||||
deepseek ,0 ,any ,RMS_NORM+MUL , 5
|
||||
deepseek2 ,0 ,any ,ADD+ADD , 1
|
||||
deepseek2 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
deepseek32 ,0 ,any ,ADD+ADD , 1
|
||||
deepseek32 ,0 ,any ,NORM+MUL+ADD , 2
|
||||
deepseek32 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
deepseek4 ,0 ,any ,RMS_NORM+MUL , 20
|
||||
dots1 ,0 ,any ,ADD+ADD , 1
|
||||
dots1 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
dream ,0 ,any ,RMS_NORM+MUL , 5
|
||||
ernie4_5-moe ,1 ,any ,ADD+ADD , 1
|
||||
ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
exaone ,0 ,any ,RMS_NORM+MUL , 5
|
||||
exaone4 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
falcon ,0 ,any ,ADD+ADD , 2
|
||||
falcon ,0 ,any ,NORM+MUL+ADD , 5
|
||||
falcon-h1 ,0 ,any ,ADD+ADD , 2
|
||||
falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
gemma ,0 ,any ,RMS_NORM+MUL , 5
|
||||
gemma2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
glm-dsa ,0 ,any ,ADD+ADD , 1
|
||||
glm-dsa ,0 ,any ,NORM+MUL+ADD , 2
|
||||
glm-dsa ,0 ,any ,RMS_NORM+MUL , 9
|
||||
glm4 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
glm4moe ,1 ,any ,ADD+ADD , 1
|
||||
glm4moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
gpt-oss ,0 ,any ,RMS_NORM+MUL , 5
|
||||
gpt2 ,0 ,any ,NORM+MUL+ADD , 5
|
||||
gptneox ,0 ,any ,NORM+MUL+ADD , 5
|
||||
granite ,0 ,any ,RMS_NORM+MUL , 5
|
||||
granite ,0 ,any ,RMS_NORM+MUL , 5
|
||||
granitehybrid ,0 ,any ,RMS_NORM+MUL , 6
|
||||
granitemoe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
granitemoe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
grok ,0 ,any ,RMS_NORM+MUL , 5
|
||||
grok ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
grovemoe ,1 ,any ,ADD+ADD , 2
|
||||
grovemoe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9
|
||||
hunyuan-moe ,1 ,any ,ADD+ADD , 2
|
||||
hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9
|
||||
hy_v3 ,0 ,any ,ADD+ADD , 2
|
||||
hy_v3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
internlm2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
jais ,0 ,any ,NORM+MUL+ADD , 5
|
||||
jais2 ,0 ,any ,NORM+MUL+ADD , 5
|
||||
jamba ,0 ,any ,RMS_NORM+MUL , 8
|
||||
kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17
|
||||
kimi-linear ,0 ,any ,ADD+ADD , 1
|
||||
kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
kimi-linear ,0 ,any ,RMS_NORM+MUL , 7
|
||||
lfm2 ,0 ,any ,RMS_NORM+MUL , 7
|
||||
lfm2moe ,1 ,any ,RMS_NORM+MUL , 7
|
||||
llada ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llada-moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
llama ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llama ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llama4 ,0 ,any ,ADD+ADD , 2
|
||||
llama4 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
maincoder ,0 ,any ,RMS_NORM+MUL , 9
|
||||
mamba ,0 ,any ,RMS_NORM+MUL , 3
|
||||
mamba2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
minimax-01 ,0 ,any ,RMS_NORM+MUL , 6
|
||||
minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
minimax-m3 ,0 ,any ,ADD+ADD , 1
|
||||
minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11
|
||||
mistral3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
mistral3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
mistral4 ,0 ,any ,ADD+ADD , 1
|
||||
mistral4 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
mpt ,0 ,any ,NORM+MUL+ADD , 5
|
||||
nanbeige ,0 ,any ,RMS_NORM+MUL , 5
|
||||
nemotron ,0 ,any ,NORM+MUL+ADD , 5
|
||||
nemotron_h ,0 ,any ,RMS_NORM+MUL , 5
|
||||
nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
olmoe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
openelm ,0 ,any ,RMS_NORM+MUL , 9
|
||||
orion ,0 ,any ,NORM+MUL+ADD , 5
|
||||
paddleocr ,0 ,any ,RMS_NORM+MUL , 5
|
||||
pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5
|
||||
phi2 ,0 ,any ,ADD+ADD , 2
|
||||
phi2 ,0 ,any ,NORM+MUL+ADD , 3
|
||||
phi3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5
|
||||
plamo ,0 ,any ,ADD+ADD , 2
|
||||
plamo ,0 ,any ,RMS_NORM+MUL , 3
|
||||
plamo2 ,0 ,any ,RMS_NORM+MUL , 11
|
||||
plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 3
|
||||
pockettts ,0 ,any ,NORM+MUL+ADD , 5
|
||||
qwen ,0 ,any ,RMS_NORM+MUL , 5
|
||||
qwen2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
qwen2moe ,1 ,any ,ADD+ADD , 2
|
||||
qwen2moe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
qwen2vl ,0 ,any ,RMS_NORM+MUL , 5
|
||||
qwen3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen35 ,0 ,any ,RMS_NORM+MUL , 8
|
||||
qwen35moe ,1 ,any ,ADD+ADD , 2
|
||||
qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen35moe ,1 ,any ,RMS_NORM+MUL , 8
|
||||
qwen3moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3next ,0 ,any ,ADD+ADD , 2
|
||||
qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen3next ,0 ,any ,RMS_NORM+MUL , 8
|
||||
qwen3tts ,0 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3vl ,0 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
qwen4exp ,0 ,any ,ADD+ADD+ADD , 5
|
||||
qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9
|
||||
qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen4exp ,0 ,any ,RMS_NORM+MUL , 5
|
||||
refact ,0 ,any ,RMS_NORM+MUL , 5
|
||||
refact ,0 ,any ,RMS_NORM+MUL , 5
|
||||
rnd1 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
seed_oss ,0 ,any ,RMS_NORM+MUL , 5
|
||||
smallthinker ,0 ,any ,RMS_NORM+MUL , 5
|
||||
smollm3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
stablelm ,0 ,any ,NORM+MUL , 4
|
||||
stablelm ,0 ,any ,NORM+MUL+ADD , 5
|
||||
starcoder ,0 ,any ,NORM+MUL+ADD , 5
|
||||
starcoder2 ,0 ,any ,NORM+MUL+ADD , 5
|
||||
talkie ,0 ,any ,ADD+ADD , 2
|
||||
xverse ,0 ,any ,RMS_NORM+MUL , 5
|
||||
|
||||
|
+73
-16
@@ -2,7 +2,8 @@
|
||||
//
|
||||
// for every dummy model generated by test-llama-archs, the tool runs the model on a single
|
||||
// device (single-threaded encoding, n_cb = 0) with fusion enabled and disabled, and reports:
|
||||
// - the per-fusion-type counters for each mode (prefill / decode)
|
||||
// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the
|
||||
// per-graph counts match)
|
||||
// - the NMSE between the fused and unfused logits
|
||||
// - the NMSE between the device and a CPU reference
|
||||
//
|
||||
@@ -23,6 +24,7 @@
|
||||
#include "gguf.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
@@ -164,7 +166,7 @@ static std::vector<float> decode_prefill(llama_model * model, llama_context * lc
|
||||
}
|
||||
|
||||
// decode one token at a time; returns the logits of the last token of each step
|
||||
static std::vector<float> decode_token_by_token(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
||||
static std::vector<float> decode_gen(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
||||
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
||||
llama_batch batch = llama_batch_init(1, 0, 1);
|
||||
std::vector<float> ret;
|
||||
@@ -197,7 +199,7 @@ struct fusion_row {
|
||||
std::string arch;
|
||||
bool moe;
|
||||
std::string mode;
|
||||
const char * label;
|
||||
std::string label;
|
||||
uint64_t count_fused;
|
||||
uint64_t count_unfused;
|
||||
uint64_t expected;
|
||||
@@ -381,13 +383,28 @@ int main(int argc, char ** argv) {
|
||||
std::string name;
|
||||
std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);
|
||||
int n_tokens;
|
||||
int n_graphs; // graph runs per mode (prefill=1, decode=16)
|
||||
};
|
||||
const mode_cfg modes[] = {
|
||||
{ "prefill", decode_prefill, 32 },
|
||||
{ "decode", decode_token_by_token, 16 },
|
||||
{ "prefill", decode_prefill, 32, 1 },
|
||||
{ "decode", decode_gen, 16, 16 },
|
||||
};
|
||||
|
||||
for (const auto & mode : modes) {
|
||||
// per-label, per-mode data for this model; prefill and decode are merged into a single
|
||||
// "any" row when their per-graph counts match
|
||||
struct mode_data {
|
||||
bool present;
|
||||
uint64_t count_fused; // per graph
|
||||
uint64_t count_unfused; // per graph
|
||||
double nmse_fus;
|
||||
double nmse_dev;
|
||||
bool skip_nmse;
|
||||
bool ok_nmse;
|
||||
};
|
||||
std::map<std::string, std::array<mode_data, 2>> mdata;
|
||||
|
||||
for (int mi = 0; mi < 2; mi++) {
|
||||
const mode_cfg & mode = modes[mi];
|
||||
const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed);
|
||||
|
||||
// CPU reference for this mode (fresh context, fresh state)
|
||||
@@ -439,20 +456,22 @@ int main(int argc, char ** argv) {
|
||||
// TODO: run in Debug build to get an assert in `ggml-alloc.c` and fix it
|
||||
const bool dev_broken = !std::isnan(nmse_dev) && nmse_dev > 1e-4;
|
||||
|
||||
// build the per-label rows
|
||||
if (has_counts) {
|
||||
for (int i = 0; i < (int) labels.size(); i++) {
|
||||
if (counts_fused[i] == 0 && counts_unfused[i] == 0) {
|
||||
const uint64_t fused = counts_fused[i] / mode.n_graphs;
|
||||
const uint64_t unfused = counts_unfused[i] / mode.n_graphs;
|
||||
if (fused == 0 && unfused == 0) {
|
||||
continue;
|
||||
}
|
||||
const std::string key = arch + "|" + (moe ? "1" : "0") + "|" + mode.name + "|" + labels[i];
|
||||
const uint64_t expected = baseline.count(key) ? baseline.at(key) : 0;
|
||||
const bool skip_nmse = dev_broken || std::isnan(nmse_fus);
|
||||
rows.push_back({ arch, moe, mode.name, labels[i], counts_fused[i], counts_unfused[i],
|
||||
expected, nmse_fus, nmse_dev,
|
||||
!check_path.empty() ? (counts_fused[i] == expected) : true,
|
||||
!skip_nmse && nmse_fus <= 1e-4,
|
||||
skip_nmse });
|
||||
auto & d = mdata[labels[i]][mi];
|
||||
d.present = true;
|
||||
d.count_fused = fused;
|
||||
d.count_unfused = unfused;
|
||||
d.nmse_fus = nmse_fus;
|
||||
d.nmse_dev = nmse_dev;
|
||||
d.skip_nmse = skip_nmse;
|
||||
d.ok_nmse = !skip_nmse && nmse_fus <= 1e-4;
|
||||
}
|
||||
} else {
|
||||
const bool skip_nmse = dev_broken || std::isnan(nmse_fus);
|
||||
@@ -461,6 +480,44 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
}
|
||||
|
||||
// build the per-label rows, merging prefill and decode into "any" when the per-graph
|
||||
// counts match (they always do for the deterministic fusion table)
|
||||
if (has_counts) {
|
||||
for (auto & kv : mdata) {
|
||||
const std::string & label = kv.first;
|
||||
const auto & d = kv.second;
|
||||
const bool both = d[0].present && d[1].present;
|
||||
const bool match = both && d[0].count_fused == d[1].count_fused;
|
||||
|
||||
if (match) {
|
||||
// one "any" row; use the worst NMSE across the two modes
|
||||
const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label;
|
||||
const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0;
|
||||
const bool ok_count = check_path.empty() || d[0].count_fused == expected;
|
||||
const bool skip_nmse = d[0].skip_nmse || d[1].skip_nmse;
|
||||
const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse;
|
||||
const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus);
|
||||
const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev);
|
||||
rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused,
|
||||
expected, nmse_fus, nmse_dev, ok_count, ok_nmse, skip_nmse });
|
||||
} else {
|
||||
// counts differ - keep a separate row per mode
|
||||
for (int mi = 0; mi < 2; mi++) {
|
||||
if (!d[mi].present) {
|
||||
continue;
|
||||
}
|
||||
const mode_data & a = d[mi];
|
||||
const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" +
|
||||
modes[mi].name + "|" + label;
|
||||
const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0;
|
||||
const bool ok_count = check_path.empty() || a.count_fused == expected;
|
||||
rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused,
|
||||
expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse, a.skip_nmse });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());
|
||||
}
|
||||
|
||||
@@ -489,7 +546,7 @@ int main(int argc, char ** argv) {
|
||||
if (r.skip_nmse) { n_skip++; }
|
||||
if (ok) { n_ok++; } else { n_bad++; }
|
||||
LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n",
|
||||
r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label,
|
||||
r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(),
|
||||
(unsigned long long) r.count_fused, (unsigned long long) r.count_unfused,
|
||||
(unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status);
|
||||
if (!record_path.empty()) {
|
||||
|
||||
Reference in New Issue
Block a user