diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 4c692eca0a..a33c3772ff 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -1,302 +1,152 @@ # test-fusion baseline for device MTL # arch ,moe ,mode ,label , count -arcee ,0 ,prefill ,RMS_NORM+MUL , 5 -arcee ,0 ,decode ,RMS_NORM+MUL , 80 -arctic ,0 ,prefill ,RMS_NORM+MUL , 7 -arctic ,0 ,decode ,RMS_NORM+MUL , 112 -baichuan ,0 ,prefill ,RMS_NORM+MUL , 5 -baichuan ,0 ,decode ,RMS_NORM+MUL , 80 -bailingmoe ,1 ,prefill ,RMS_NORM+MUL , 5 -bailingmoe ,1 ,prefill ,ADD+ADD , 2 -bailingmoe ,1 ,decode ,RMS_NORM+MUL , 80 -bailingmoe ,1 ,decode ,ADD+ADD , 32 -bailingmoe2 ,1 ,prefill ,RMS_NORM+MUL , 9 -bailingmoe2 ,1 ,prefill ,ADD+ADD , 1 -bailingmoe2 ,1 ,decode ,RMS_NORM+MUL , 144 -bailingmoe2 ,1 ,decode ,ADD+ADD , 16 -bailingmoe3 ,1 ,prefill ,RMS_NORM+MUL , 8 -bailingmoe3 ,1 ,prefill ,ADD+ADD , 1 -bailingmoe3 ,1 ,prefill ,GATED_DELTA_NET+CPY , 1 -bailingmoe3 ,1 ,decode ,RMS_NORM+MUL , 128 -bailingmoe3 ,1 ,decode ,ADD+ADD , 16 -bailingmoe3 ,1 ,decode ,GATED_DELTA_NET+CPY , 16 -bloom ,0 ,prefill ,NORM+MUL+ADD , 6 -bloom ,0 ,decode ,NORM+MUL+ADD , 96 -chatglm ,0 ,prefill ,RMS_NORM+MUL , 5 -chatglm ,0 ,decode ,RMS_NORM+MUL , 80 -codeshell ,0 ,prefill ,NORM+MUL+ADD , 5 -codeshell ,0 ,decode ,NORM+MUL+ADD , 80 -cogvlm ,0 ,prefill ,RMS_NORM+MUL , 5 -cogvlm ,0 ,decode ,RMS_NORM+MUL , 80 -command-r ,0 ,prefill ,NORM+MUL , 3 -command-r ,0 ,decode ,NORM+MUL , 48 -dbrx ,0 ,prefill ,NORM+MUL , 5 -dbrx ,0 ,decode ,NORM+MUL , 80 -deci ,0 ,prefill ,RMS_NORM+MUL , 5 -deci ,0 ,decode ,RMS_NORM+MUL , 80 -deepseek ,0 ,prefill ,RMS_NORM+MUL , 5 -deepseek ,0 ,prefill ,ADD+ADD , 1 -deepseek ,0 ,decode ,RMS_NORM+MUL , 80 -deepseek ,0 ,decode ,ADD+ADD , 16 -deepseek2 ,0 ,prefill ,RMS_NORM+MUL , 9 -deepseek2 ,0 ,prefill ,ADD+ADD , 1 -deepseek2 ,0 ,decode ,RMS_NORM+MUL , 144 -deepseek2 ,0 ,decode ,ADD+ADD , 16 -deepseek32 ,0 ,prefill ,NORM+MUL+ADD , 2 -deepseek32 ,0 ,prefill ,RMS_NORM+MUL , 9 -deepseek32 ,0 ,prefill ,ADD+ADD , 1 -deepseek32 ,0 ,decode ,NORM+MUL+ADD , 32 -deepseek32 ,0 ,decode ,RMS_NORM+MUL , 144 -deepseek32 ,0 ,decode ,ADD+ADD , 16 -deepseek4 ,0 ,prefill ,RMS_NORM+MUL , 20 -deepseek4 ,0 ,decode ,RMS_NORM+MUL , 320 -dots1 ,0 ,prefill ,RMS_NORM+MUL , 9 -dots1 ,0 ,prefill ,ADD+ADD , 1 -dots1 ,0 ,decode ,RMS_NORM+MUL , 144 -dots1 ,0 ,decode ,ADD+ADD , 16 -dream ,0 ,prefill ,RMS_NORM+MUL , 5 -dream ,0 ,decode ,RMS_NORM+MUL , 80 -ernie4_5-moe ,1 ,prefill ,RMS_NORM+MUL , 5 -ernie4_5-moe ,1 ,prefill ,ADD+ADD , 1 -ernie4_5-moe ,1 ,decode ,RMS_NORM+MUL , 80 -ernie4_5-moe ,1 ,decode ,ADD+ADD , 16 -ernie4_5 ,0 ,prefill ,RMS_NORM+MUL , 5 -ernie4_5 ,0 ,decode ,RMS_NORM+MUL , 80 -exaone ,0 ,prefill ,RMS_NORM+MUL , 5 -exaone ,0 ,decode ,RMS_NORM+MUL , 80 -exaone4 ,0 ,prefill ,RMS_NORM+MUL , 5 -exaone4 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 -exaone4 ,0 ,decode ,RMS_NORM+MUL , 80 -exaone4 ,0 ,decode ,RMS_NORM+MUL+ADD , 64 -falcon ,0 ,prefill ,NORM+MUL+ADD , 5 -falcon ,0 ,prefill ,ADD+ADD , 2 -falcon ,0 ,decode ,NORM+MUL+ADD , 80 -falcon ,0 ,decode ,ADD+ADD , 32 -falcon-h1 ,0 ,prefill ,RMS_NORM+MUL , 9 -falcon-h1 ,0 ,prefill ,ADD+ADD , 2 -falcon-h1 ,0 ,decode ,RMS_NORM+MUL , 144 -falcon-h1 ,0 ,decode ,ADD+ADD , 32 -gemma ,0 ,prefill ,RMS_NORM+MUL , 5 -gemma ,0 ,decode ,RMS_NORM+MUL , 80 -gemma2 ,0 ,prefill ,RMS_NORM+MUL , 5 -gemma2 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 -gemma2 ,0 ,decode ,RMS_NORM+MUL , 80 -gemma2 ,0 ,decode ,RMS_NORM+MUL+ADD , 64 -glm-dsa ,0 ,prefill ,NORM+MUL+ADD , 2 -glm-dsa ,0 ,prefill ,RMS_NORM+MUL , 9 -glm-dsa ,0 ,prefill ,ADD+ADD , 1 -glm-dsa ,0 ,decode ,NORM+MUL+ADD , 32 -glm-dsa ,0 ,decode ,RMS_NORM+MUL , 144 -glm-dsa ,0 ,decode ,ADD+ADD , 16 -glm4 ,0 ,prefill ,RMS_NORM+MUL , 5 -glm4 ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 -glm4 ,0 ,decode ,RMS_NORM+MUL , 80 -glm4 ,0 ,decode ,RMS_NORM+MUL+ADD , 64 -glm4moe ,1 ,prefill ,RMS_NORM+MUL , 9 -glm4moe ,1 ,prefill ,ADD+ADD , 1 -glm4moe ,1 ,decode ,RMS_NORM+MUL , 144 -glm4moe ,1 ,decode ,ADD+ADD , 16 -gpt-oss ,0 ,prefill ,RMS_NORM+MUL , 5 -gpt-oss ,0 ,decode ,RMS_NORM+MUL , 80 -gpt2 ,0 ,prefill ,NORM+MUL+ADD , 5 -gpt2 ,0 ,decode ,NORM+MUL+ADD , 80 -gptneox ,0 ,prefill ,NORM+MUL+ADD , 5 -gptneox ,0 ,decode ,NORM+MUL+ADD , 80 -granite ,0 ,prefill ,RMS_NORM+MUL , 5 -granite ,0 ,decode ,RMS_NORM+MUL , 80 -granite ,0 ,prefill ,RMS_NORM+MUL , 5 -granite ,0 ,decode ,RMS_NORM+MUL , 80 -granitehybrid ,0 ,prefill ,RMS_NORM+MUL , 6 -granitehybrid ,0 ,decode ,RMS_NORM+MUL , 96 -granitemoe ,1 ,prefill ,RMS_NORM+MUL , 5 -granitemoe ,1 ,decode ,RMS_NORM+MUL , 80 -granitemoe ,1 ,prefill ,RMS_NORM+MUL , 5 -granitemoe ,1 ,decode ,RMS_NORM+MUL , 80 -grok ,0 ,prefill ,RMS_NORM+MUL , 5 -grok ,0 ,prefill ,RMS_NORM+MUL+ADD , 4 -grok ,0 ,decode ,RMS_NORM+MUL , 80 -grok ,0 ,decode ,RMS_NORM+MUL+ADD , 64 -grovemoe ,1 ,prefill ,RMS_NORM+MUL , 9 -grovemoe ,1 ,prefill ,ADD+ADD , 2 -grovemoe ,1 ,decode ,RMS_NORM+MUL , 144 -grovemoe ,1 ,decode ,ADD+ADD , 32 -hunyuan-dense ,0 ,prefill ,RMS_NORM+MUL , 9 -hunyuan-dense ,0 ,decode ,RMS_NORM+MUL , 144 -hunyuan-moe ,1 ,prefill ,RMS_NORM+MUL , 9 -hunyuan-moe ,1 ,prefill ,ADD+ADD , 2 -hunyuan-moe ,1 ,decode ,RMS_NORM+MUL , 144 -hunyuan-moe ,1 ,decode ,ADD+ADD , 32 -hunyuan_vl ,0 ,prefill ,RMS_NORM+MUL , 9 -hunyuan_vl ,0 ,decode ,RMS_NORM+MUL , 144 -hy_v3 ,0 ,prefill ,RMS_NORM+MUL , 9 -hy_v3 ,0 ,prefill ,ADD+ADD , 2 -hy_v3 ,0 ,decode ,RMS_NORM+MUL , 144 -hy_v3 ,0 ,decode ,ADD+ADD , 32 -internlm2 ,0 ,prefill ,RMS_NORM+MUL , 5 -internlm2 ,0 ,decode ,RMS_NORM+MUL , 80 -jais ,0 ,prefill ,NORM+MUL+ADD , 5 -jais ,0 ,decode ,NORM+MUL+ADD , 80 -jais2 ,0 ,prefill ,NORM+MUL+ADD , 5 -jais2 ,0 ,decode ,NORM+MUL+ADD , 80 -jamba ,0 ,prefill ,RMS_NORM+MUL , 8 -jamba ,0 ,decode ,RMS_NORM+MUL , 128 -kimi-k3 ,0 ,prefill ,RMS_NORM+MUL , 17 -kimi-k3 ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 -kimi-k3 ,0 ,decode ,RMS_NORM+MUL , 272 -kimi-k3 ,0 ,decode ,GATED_DELTA_NET+CPY , 16 -kimi-linear ,0 ,prefill ,RMS_NORM+MUL , 7 -kimi-linear ,0 ,prefill ,ADD+ADD , 1 -kimi-linear ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 -kimi-linear ,0 ,decode ,RMS_NORM+MUL , 112 -kimi-linear ,0 ,decode ,ADD+ADD , 16 -kimi-linear ,0 ,decode ,GATED_DELTA_NET+CPY , 16 -lfm2 ,0 ,prefill ,RMS_NORM+MUL , 7 -lfm2 ,0 ,decode ,RMS_NORM+MUL , 112 -lfm2moe ,1 ,prefill ,RMS_NORM+MUL , 7 -lfm2moe ,1 ,decode ,RMS_NORM+MUL , 112 -llada ,0 ,prefill ,RMS_NORM+MUL , 5 -llada ,0 ,decode ,RMS_NORM+MUL , 80 -llada-moe ,1 ,prefill ,RMS_NORM+MUL , 9 -llada-moe ,1 ,decode ,RMS_NORM+MUL , 144 -llama ,0 ,prefill ,RMS_NORM+MUL , 5 -llama ,0 ,decode ,RMS_NORM+MUL , 80 -llama ,0 ,prefill ,RMS_NORM+MUL , 5 -llama ,0 ,decode ,RMS_NORM+MUL , 80 -llama4 ,0 ,prefill ,RMS_NORM+MUL , 9 -llama4 ,0 ,prefill ,ADD+ADD , 2 -llama4 ,0 ,decode ,RMS_NORM+MUL , 144 -llama4 ,0 ,decode ,ADD+ADD , 32 -maincoder ,0 ,prefill ,RMS_NORM+MUL , 9 -maincoder ,0 ,decode ,RMS_NORM+MUL , 144 -mamba ,0 ,prefill ,RMS_NORM+MUL , 3 -mamba ,0 ,decode ,RMS_NORM+MUL , 48 -mamba2 ,0 ,prefill ,RMS_NORM+MUL , 5 -mamba2 ,0 ,decode ,RMS_NORM+MUL , 80 -minicpm ,0 ,prefill ,RMS_NORM+MUL , 5 -minicpm ,0 ,decode ,RMS_NORM+MUL , 80 -minicpm ,0 ,prefill ,RMS_NORM+MUL , 5 -minicpm ,0 ,decode ,RMS_NORM+MUL , 80 -minicpm3 ,0 ,prefill ,RMS_NORM+MUL , 9 -minicpm3 ,0 ,decode ,RMS_NORM+MUL , 144 -minimax-01 ,0 ,prefill ,RMS_NORM+MUL , 6 -minimax-01 ,0 ,decode ,RMS_NORM+MUL , 96 -minimax-m2 ,0 ,prefill ,RMS_NORM+MUL , 9 -minimax-m2 ,0 ,decode ,RMS_NORM+MUL , 144 -minimax-m3 ,0 ,prefill ,RMS_NORM+MUL , 11 -minimax-m3 ,0 ,prefill ,ADD+ADD , 1 -minimax-m3 ,0 ,decode ,RMS_NORM+MUL , 176 -minimax-m3 ,0 ,decode ,ADD+ADD , 16 -mistral3 ,0 ,prefill ,RMS_NORM+MUL , 5 -mistral3 ,0 ,decode ,RMS_NORM+MUL , 80 -mistral3 ,0 ,prefill ,RMS_NORM+MUL , 5 -mistral3 ,0 ,decode ,RMS_NORM+MUL , 80 -mistral4 ,0 ,prefill ,RMS_NORM+MUL , 9 -mistral4 ,0 ,prefill ,ADD+ADD , 1 -mistral4 ,0 ,decode ,RMS_NORM+MUL , 144 -mistral4 ,0 ,decode ,ADD+ADD , 16 -mpt ,0 ,prefill ,NORM+MUL+ADD , 5 -mpt ,0 ,decode ,NORM+MUL+ADD , 80 -nanbeige ,0 ,prefill ,RMS_NORM+MUL , 5 -nanbeige ,0 ,decode ,RMS_NORM+MUL , 80 -nemotron ,0 ,prefill ,NORM+MUL+ADD , 5 -nemotron ,0 ,decode ,NORM+MUL+ADD , 80 -nemotron_h ,0 ,prefill ,RMS_NORM+MUL , 5 -nemotron_h ,0 ,decode ,RMS_NORM+MUL , 80 -nemotron_h_moe ,1 ,prefill ,RMS_NORM+MUL , 5 -nemotron_h_moe ,1 ,decode ,RMS_NORM+MUL , 80 -olmoe ,1 ,prefill ,RMS_NORM+MUL , 9 -olmoe ,1 ,decode ,RMS_NORM+MUL , 144 -openelm ,0 ,prefill ,RMS_NORM+MUL , 9 -openelm ,0 ,decode ,RMS_NORM+MUL , 144 -orion ,0 ,prefill ,NORM+MUL+ADD , 5 -orion ,0 ,decode ,NORM+MUL+ADD , 80 -paddleocr ,0 ,prefill ,RMS_NORM+MUL , 5 -paddleocr ,0 ,decode ,RMS_NORM+MUL , 80 -pangu-embedded ,0 ,prefill ,RMS_NORM+MUL , 5 -pangu-embedded ,0 ,decode ,RMS_NORM+MUL , 80 -phi2 ,0 ,prefill ,NORM+MUL+ADD , 3 -phi2 ,0 ,prefill ,ADD+ADD , 2 -phi2 ,0 ,decode ,NORM+MUL+ADD , 48 -phi2 ,0 ,decode ,ADD+ADD , 32 -phi3 ,0 ,prefill ,RMS_NORM+MUL , 5 -phi3 ,0 ,decode ,RMS_NORM+MUL , 80 -phimoe ,1 ,prefill ,RMS_NORM+MUL+ADD , 5 -phimoe ,1 ,decode ,RMS_NORM+MUL+ADD , 80 -plamo ,0 ,prefill ,RMS_NORM+MUL , 3 -plamo ,0 ,prefill ,ADD+ADD , 2 -plamo ,0 ,decode ,RMS_NORM+MUL , 48 -plamo ,0 ,decode ,ADD+ADD , 32 -plamo2 ,0 ,prefill ,RMS_NORM+MUL , 11 -plamo2 ,0 ,prefill ,RMS_NORM+MUL+ADD , 3 -plamo2 ,0 ,decode ,RMS_NORM+MUL , 176 -plamo2 ,0 ,decode ,RMS_NORM+MUL+ADD , 48 -pockettts ,0 ,prefill ,NORM+MUL+ADD , 5 -pockettts ,0 ,decode ,NORM+MUL+ADD , 80 -qwen ,0 ,prefill ,RMS_NORM+MUL , 5 -qwen ,0 ,decode ,RMS_NORM+MUL , 80 -qwen2 ,0 ,prefill ,RMS_NORM+MUL , 5 -qwen2 ,0 ,decode ,RMS_NORM+MUL , 80 -qwen2moe ,1 ,prefill ,RMS_NORM+MUL , 5 -qwen2moe ,1 ,prefill ,ADD+ADD , 2 -qwen2moe ,1 ,decode ,RMS_NORM+MUL , 80 -qwen2moe ,1 ,decode ,ADD+ADD , 32 -qwen2vl ,0 ,prefill ,RMS_NORM+MUL , 5 -qwen2vl ,0 ,decode ,RMS_NORM+MUL , 80 -qwen3 ,0 ,prefill ,RMS_NORM+MUL , 9 -qwen3 ,0 ,decode ,RMS_NORM+MUL , 144 -qwen35 ,0 ,prefill ,RMS_NORM+MUL , 8 -qwen35 ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 -qwen35 ,0 ,decode ,RMS_NORM+MUL , 128 -qwen35 ,0 ,decode ,GATED_DELTA_NET+CPY , 16 -qwen35moe ,1 ,prefill ,RMS_NORM+MUL , 8 -qwen35moe ,1 ,prefill ,ADD+ADD , 2 -qwen35moe ,1 ,prefill ,GATED_DELTA_NET+CPY , 1 -qwen35moe ,1 ,decode ,RMS_NORM+MUL , 128 -qwen35moe ,1 ,decode ,ADD+ADD , 32 -qwen35moe ,1 ,decode ,GATED_DELTA_NET+CPY , 16 -qwen3moe ,1 ,prefill ,RMS_NORM+MUL , 9 -qwen3moe ,1 ,decode ,RMS_NORM+MUL , 144 -qwen3next ,0 ,prefill ,RMS_NORM+MUL , 8 -qwen3next ,0 ,prefill ,ADD+ADD , 2 -qwen3next ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 -qwen3next ,0 ,decode ,RMS_NORM+MUL , 128 -qwen3next ,0 ,decode ,ADD+ADD , 32 -qwen3next ,0 ,decode ,GATED_DELTA_NET+CPY , 16 -qwen3tts ,0 ,prefill ,RMS_NORM+MUL , 9 -qwen3tts ,0 ,decode ,RMS_NORM+MUL , 144 -qwen3vl ,0 ,prefill ,RMS_NORM+MUL , 9 -qwen3vl ,0 ,decode ,RMS_NORM+MUL , 144 -qwen3vlmoe ,1 ,prefill ,RMS_NORM+MUL , 9 -qwen3vlmoe ,1 ,decode ,RMS_NORM+MUL , 144 -qwen4exp ,0 ,prefill ,RMS_NORM+MUL , 5 -qwen4exp ,0 ,prefill ,ADD+ADD+ADD , 5 -qwen4exp ,0 ,prefill ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9 -qwen4exp ,0 ,prefill ,GATED_DELTA_NET+CPY , 1 -qwen4exp ,0 ,decode ,RMS_NORM+MUL , 80 -qwen4exp ,0 ,decode ,ADD+ADD+ADD , 80 -qwen4exp ,0 ,decode ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 144 -qwen4exp ,0 ,decode ,GATED_DELTA_NET+CPY , 16 -refact ,0 ,prefill ,RMS_NORM+MUL , 5 -refact ,0 ,decode ,RMS_NORM+MUL , 80 -refact ,0 ,prefill ,RMS_NORM+MUL , 5 -refact ,0 ,decode ,RMS_NORM+MUL , 80 -rnd1 ,0 ,prefill ,RMS_NORM+MUL , 9 -rnd1 ,0 ,decode ,RMS_NORM+MUL , 144 -seed_oss ,0 ,prefill ,RMS_NORM+MUL , 5 -seed_oss ,0 ,decode ,RMS_NORM+MUL , 80 -smallthinker ,0 ,prefill ,RMS_NORM+MUL , 5 -smallthinker ,0 ,decode ,RMS_NORM+MUL , 80 -smollm3 ,0 ,prefill ,RMS_NORM+MUL , 5 -smollm3 ,0 ,decode ,RMS_NORM+MUL , 80 -stablelm ,0 ,prefill ,NORM+MUL , 4 -stablelm ,0 ,prefill ,NORM+MUL+ADD , 5 -stablelm ,0 ,decode ,NORM+MUL , 64 -stablelm ,0 ,decode ,NORM+MUL+ADD , 80 -starcoder ,0 ,prefill ,NORM+MUL+ADD , 5 -starcoder ,0 ,decode ,NORM+MUL+ADD , 80 -starcoder2 ,0 ,prefill ,NORM+MUL+ADD , 5 -starcoder2 ,0 ,decode ,NORM+MUL+ADD , 80 -talkie ,0 ,prefill ,ADD+ADD , 2 -talkie ,0 ,decode ,ADD+ADD , 32 -xverse ,0 ,prefill ,RMS_NORM+MUL , 5 -xverse ,0 ,decode ,RMS_NORM+MUL , 80 +arcee ,0 ,any ,RMS_NORM+MUL , 5 +arctic ,0 ,any ,RMS_NORM+MUL , 7 +baichuan ,0 ,any ,RMS_NORM+MUL , 5 +bailingmoe ,1 ,any ,ADD+ADD , 2 +bailingmoe ,1 ,any ,RMS_NORM+MUL , 5 +bailingmoe2 ,1 ,any ,ADD+ADD , 1 +bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9 +bailingmoe3 ,1 ,any ,ADD+ADD , 1 +bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1 +bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8 +bloom ,0 ,any ,NORM+MUL+ADD , 6 +chatglm ,0 ,any ,RMS_NORM+MUL , 5 +codeshell ,0 ,any ,NORM+MUL+ADD , 5 +cogvlm ,0 ,any ,RMS_NORM+MUL , 5 +command-r ,0 ,any ,NORM+MUL , 3 +dbrx ,0 ,any ,NORM+MUL , 5 +deci ,0 ,any ,RMS_NORM+MUL , 5 +deepseek ,0 ,any ,ADD+ADD , 1 +deepseek ,0 ,any ,RMS_NORM+MUL , 5 +deepseek2 ,0 ,any ,ADD+ADD , 1 +deepseek2 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek32 ,0 ,any ,ADD+ADD , 1 +deepseek32 ,0 ,any ,NORM+MUL+ADD , 2 +deepseek32 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek4 ,0 ,any ,RMS_NORM+MUL , 20 +dots1 ,0 ,any ,ADD+ADD , 1 +dots1 ,0 ,any ,RMS_NORM+MUL , 9 +dream ,0 ,any ,RMS_NORM+MUL , 5 +ernie4_5-moe ,1 ,any ,ADD+ADD , 1 +ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5 +ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5 +exaone ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +falcon ,0 ,any ,ADD+ADD , 2 +falcon ,0 ,any ,NORM+MUL+ADD , 5 +falcon-h1 ,0 ,any ,ADD+ADD , 2 +falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 +gemma ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm-dsa ,0 ,any ,ADD+ADD , 1 +glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 +glm-dsa ,0 ,any ,RMS_NORM+MUL , 9 +glm4 ,0 ,any ,RMS_NORM+MUL , 5 +glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm4moe ,1 ,any ,ADD+ADD , 1 +glm4moe ,1 ,any ,RMS_NORM+MUL , 9 +gpt-oss ,0 ,any ,RMS_NORM+MUL , 5 +gpt2 ,0 ,any ,NORM+MUL+ADD , 5 +gptneox ,0 ,any ,NORM+MUL+ADD , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granitehybrid ,0 ,any ,RMS_NORM+MUL , 6 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL+ADD , 4 +grovemoe ,1 ,any ,ADD+ADD , 2 +grovemoe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9 +hunyuan-moe ,1 ,any ,ADD+ADD , 2 +hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9 +hy_v3 ,0 ,any ,ADD+ADD , 2 +hy_v3 ,0 ,any ,RMS_NORM+MUL , 9 +internlm2 ,0 ,any ,RMS_NORM+MUL , 5 +jais ,0 ,any ,NORM+MUL+ADD , 5 +jais2 ,0 ,any ,NORM+MUL+ADD , 5 +jamba ,0 ,any ,RMS_NORM+MUL , 8 +kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17 +kimi-linear ,0 ,any ,ADD+ADD , 1 +kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 +lfm2 ,0 ,any ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,any ,RMS_NORM+MUL , 7 +llada ,0 ,any ,RMS_NORM+MUL , 5 +llada-moe ,1 ,any ,RMS_NORM+MUL , 9 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama4 ,0 ,any ,ADD+ADD , 2 +llama4 ,0 ,any ,RMS_NORM+MUL , 9 +maincoder ,0 ,any ,RMS_NORM+MUL , 9 +mamba ,0 ,any ,RMS_NORM+MUL , 3 +mamba2 ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm3 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-01 ,0 ,any ,RMS_NORM+MUL , 6 +minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-m3 ,0 ,any ,ADD+ADD , 1 +minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral4 ,0 ,any ,ADD+ADD , 1 +mistral4 ,0 ,any ,RMS_NORM+MUL , 9 +mpt ,0 ,any ,NORM+MUL+ADD , 5 +nanbeige ,0 ,any ,RMS_NORM+MUL , 5 +nemotron ,0 ,any ,NORM+MUL+ADD , 5 +nemotron_h ,0 ,any ,RMS_NORM+MUL , 5 +nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5 +olmoe ,1 ,any ,RMS_NORM+MUL , 9 +openelm ,0 ,any ,RMS_NORM+MUL , 9 +orion ,0 ,any ,NORM+MUL+ADD , 5 +paddleocr ,0 ,any ,RMS_NORM+MUL , 5 +pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5 +phi2 ,0 ,any ,ADD+ADD , 2 +phi2 ,0 ,any ,NORM+MUL+ADD , 3 +phi3 ,0 ,any ,RMS_NORM+MUL , 5 +phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5 +plamo ,0 ,any ,ADD+ADD , 2 +plamo ,0 ,any ,RMS_NORM+MUL , 3 +plamo2 ,0 ,any ,RMS_NORM+MUL , 11 +plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 3 +pockettts ,0 ,any ,NORM+MUL+ADD , 5 +qwen ,0 ,any ,RMS_NORM+MUL , 5 +qwen2 ,0 ,any ,RMS_NORM+MUL , 5 +qwen2moe ,1 ,any ,ADD+ADD , 2 +qwen2moe ,1 ,any ,RMS_NORM+MUL , 5 +qwen2vl ,0 ,any ,RMS_NORM+MUL , 5 +qwen3 ,0 ,any ,RMS_NORM+MUL , 9 +qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35 ,0 ,any ,RMS_NORM+MUL , 8 +qwen35moe ,1 ,any ,ADD+ADD , 2 +qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35moe ,1 ,any ,RMS_NORM+MUL , 8 +qwen3moe ,1 ,any ,RMS_NORM+MUL , 9 +qwen3next ,0 ,any ,ADD+ADD , 2 +qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen3next ,0 ,any ,RMS_NORM+MUL , 8 +qwen3tts ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vl ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9 +qwen4exp ,0 ,any ,ADD+ADD+ADD , 5 +qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9 +qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen4exp ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +rnd1 ,0 ,any ,RMS_NORM+MUL , 9 +seed_oss ,0 ,any ,RMS_NORM+MUL , 5 +smallthinker ,0 ,any ,RMS_NORM+MUL , 5 +smollm3 ,0 ,any ,RMS_NORM+MUL , 5 +stablelm ,0 ,any ,NORM+MUL , 4 +stablelm ,0 ,any ,NORM+MUL+ADD , 5 +starcoder ,0 ,any ,NORM+MUL+ADD , 5 +starcoder2 ,0 ,any ,NORM+MUL+ADD , 5 +talkie ,0 ,any ,ADD+ADD , 2 +xverse ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp index e8771e7440..ed6236e28c 100644 --- a/tests/test-fusion.cpp +++ b/tests/test-fusion.cpp @@ -2,7 +2,8 @@ // // for every dummy model generated by test-llama-archs, the tool runs the model on a single // device (single-threaded encoding, n_cb = 0) with fusion enabled and disabled, and reports: -// - the per-fusion-type counters for each mode (prefill / decode) +// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the +// per-graph counts match) // - the NMSE between the fused and unfused logits // - the NMSE between the device and a CPU reference // @@ -23,6 +24,7 @@ #include "gguf.h" #include +#include #include #include #include @@ -164,7 +166,7 @@ static std::vector decode_prefill(llama_model * model, llama_context * lc } // decode one token at a time; returns the logits of the last token of each step -static std::vector decode_token_by_token(llama_model * model, llama_context * lctx, const std::vector & tokens) { +static std::vector decode_gen(llama_model * model, llama_context * lctx, const std::vector & tokens) { const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); llama_batch batch = llama_batch_init(1, 0, 1); std::vector ret; @@ -197,7 +199,7 @@ struct fusion_row { std::string arch; bool moe; std::string mode; - const char * label; + std::string label; uint64_t count_fused; uint64_t count_unfused; uint64_t expected; @@ -381,13 +383,28 @@ int main(int argc, char ** argv) { std::string name; std::vector (*decode)(llama_model *, llama_context *, const std::vector &); int n_tokens; + int n_graphs; // graph runs per mode (prefill=1, decode=16) }; const mode_cfg modes[] = { - { "prefill", decode_prefill, 32 }, - { "decode", decode_token_by_token, 16 }, + { "prefill", decode_prefill, 32, 1 }, + { "decode", decode_gen, 16, 16 }, }; - for (const auto & mode : modes) { + // per-label, per-mode data for this model; prefill and decode are merged into a single + // "any" row when their per-graph counts match + struct mode_data { + bool present; + uint64_t count_fused; // per graph + uint64_t count_unfused; // per graph + double nmse_fus; + double nmse_dev; + bool skip_nmse; + bool ok_nmse; + }; + std::map> mdata; + + for (int mi = 0; mi < 2; mi++) { + const mode_cfg & mode = modes[mi]; const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed); // CPU reference for this mode (fresh context, fresh state) @@ -439,20 +456,22 @@ int main(int argc, char ** argv) { // TODO: run in Debug build to get an assert in `ggml-alloc.c` and fix it const bool dev_broken = !std::isnan(nmse_dev) && nmse_dev > 1e-4; - // build the per-label rows if (has_counts) { for (int i = 0; i < (int) labels.size(); i++) { - if (counts_fused[i] == 0 && counts_unfused[i] == 0) { + const uint64_t fused = counts_fused[i] / mode.n_graphs; + const uint64_t unfused = counts_unfused[i] / mode.n_graphs; + if (fused == 0 && unfused == 0) { continue; } - const std::string key = arch + "|" + (moe ? "1" : "0") + "|" + mode.name + "|" + labels[i]; - const uint64_t expected = baseline.count(key) ? baseline.at(key) : 0; const bool skip_nmse = dev_broken || std::isnan(nmse_fus); - rows.push_back({ arch, moe, mode.name, labels[i], counts_fused[i], counts_unfused[i], - expected, nmse_fus, nmse_dev, - !check_path.empty() ? (counts_fused[i] == expected) : true, - !skip_nmse && nmse_fus <= 1e-4, - skip_nmse }); + auto & d = mdata[labels[i]][mi]; + d.present = true; + d.count_fused = fused; + d.count_unfused = unfused; + d.nmse_fus = nmse_fus; + d.nmse_dev = nmse_dev; + d.skip_nmse = skip_nmse; + d.ok_nmse = !skip_nmse && nmse_fus <= 1e-4; } } else { const bool skip_nmse = dev_broken || std::isnan(nmse_fus); @@ -461,6 +480,44 @@ int main(int argc, char ** argv) { } } + // build the per-label rows, merging prefill and decode into "any" when the per-graph + // counts match (they always do for the deterministic fusion table) + if (has_counts) { + for (auto & kv : mdata) { + const std::string & label = kv.first; + const auto & d = kv.second; + const bool both = d[0].present && d[1].present; + const bool match = both && d[0].count_fused == d[1].count_fused; + + if (match) { + // one "any" row; use the worst NMSE across the two modes + const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label; + const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0; + const bool ok_count = check_path.empty() || d[0].count_fused == expected; + const bool skip_nmse = d[0].skip_nmse || d[1].skip_nmse; + const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse; + const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus); + const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev); + rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused, + expected, nmse_fus, nmse_dev, ok_count, ok_nmse, skip_nmse }); + } else { + // counts differ - keep a separate row per mode + for (int mi = 0; mi < 2; mi++) { + if (!d[mi].present) { + continue; + } + const mode_data & a = d[mi]; + const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + + modes[mi].name + "|" + label; + const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0; + const bool ok_count = check_path.empty() || a.count_fused == expected; + rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused, + expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse, a.skip_nmse }); + } + } + } + } + LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str()); } @@ -489,7 +546,7 @@ int main(int argc, char ** argv) { if (r.skip_nmse) { n_skip++; } if (ok) { n_ok++; } else { n_bad++; } LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n", - r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label, + r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(), (unsigned long long) r.count_fused, (unsigned long long) r.count_unfused, (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status); if (!record_path.empty()) {