Star22
29
代码
代码
Issues2
Pull Requests6
项目讨论
Wiki
分析
Star22
29
已开启
feat: 补充昇腾 950 低精度量化用例 #107
wesseI创建于 24 天前
feat: 补充昇腾 950 低精度量化用例 #107
已开启
wesseI创建于 24 天前
从wesseI/ascendc-kernelgen-datafeat/950-low-precision-workloads合入到Ascend/ascendc-kernelgen-datamaster
共 21 个文件变更+134-78
Mnpu-kernelbench/data/kernel_generator/level1/Add/workload.jsonl+1-0
@@ -48,3 +48,4 @@
48{"uuid": "Add-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [384, 784, 1552], "dtype": "float16"}, "y": {"type": "random", "shape": [384, 1, 1552], "dtype": "float16", "distribution": "randn"}, "alpha": {"type": "scalar", "value": 1.0}}}48{"uuid": "Add-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [384, 784, 1552], "dtype": "float16"}, "y": {"type": "random", "shape": [384, 1, 1552], "dtype": "float16", "distribution": "randn"}, "alpha": {"type": "scalar", "value": 1.0}}}
49{"uuid": "Add-049", "axes": null, "inputs": {"x": {"type": "random", "shape": [784, 784, 784], "dtype": "float16"}, "y": {"type": "random", "shape": [784, 1, 784], "dtype": "float16", "distribution": "randn"}, "alpha": {"type": "scalar", "value": 1.0}}}49{"uuid": "Add-049", "axes": null, "inputs": {"x": {"type": "random", "shape": [784, 784, 784], "dtype": "float16"}, "y": {"type": "random", "shape": [784, 1, 784], "dtype": "float16", "distribution": "randn"}, "alpha": {"type": "scalar", "value": 1.0}}}
50{"uuid": "Add-050", "axes": null, "inputs": {"x": {"type": "random", "shape": [784, 784, 784], "dtype": "float16"}, "y": {"type": "random", "shape": [1, 784, 784], "dtype": "float16", "distribution": "randn"}, "alpha": {"type": "scalar", "value": 1.0}}}50{"uuid": "Add-050", "axes": null, "inputs": {"x": {"type": "random", "shape": [784, 784, 784], "dtype": "float16"}, "y": {"type": "random", "shape": [1, 784, 784], "dtype": "float16", "distribution": "randn"}, "alpha": {"type": "scalar", "value": 1.0}}}
51+{"uuid": "Add-051", "axes": null, "inputs": {"x": {"type": "random", "shape": [96, 96, 96], "dtype": "int8", "range": [-8, 8]}, "y": {"type": "random", "shape": [96, 96, 96], "dtype": "int8", "distribution": "randn", "range": [-8, 8]}, "alpha": {"type": "scalar", "value": 1}}}
Mnpu-kernelbench/data/kernel_generator/level1/Cat/workload.jsonl+2-0
@@ -49,3 +49,5 @@
49{"uuid": "Cat-049", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "float32", "shapes": [[14016, 14016], [14016, 14016]]}, "dim": {"type": "scalar", "value": 0}}}49{"uuid": "Cat-049", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "float32", "shapes": [[14016, 14016], [14016, 14016]]}, "dim": {"type": "scalar", "value": 0}}}
50{"uuid": "Cat-050", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "bfloat16", "shapes": [[5760, 34544], [5760, 34544]]}, "dim": {"type": "scalar", "value": 0}}}50{"uuid": "Cat-050", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "bfloat16", "shapes": [[5760, 34544], [5760, 34544]]}, "dim": {"type": "scalar", "value": 0}}}
51{"uuid": "Cat-051", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "float16", "shapes": [[11280, 22560], [11280, 45120], [11280, 11280], [11280, 5648]]}, "dim": {"type": "scalar", "value": 1}}}51{"uuid": "Cat-051", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "float16", "shapes": [[11280, 22560], [11280, 45120], [11280, 11280], [11280, 5648]]}, "dim": {"type": "scalar", "value": 1}}}
52+{"uuid": "Cat-052", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "float8_e4m3fn", "shapes": [[1, 23, 512, 512], [1, 23, 512, 512]]}, "dim": {"type": "scalar", "value": 0}}}
53+{"uuid": "Cat-053", "axes": null, "inputs": {"tensors": {"type": "tensor_list", "dtype": "float8_e5m2", "shapes": [[1, 23, 512, 512], [1, 23, 512, 512]]}, "dim": {"type": "scalar", "value": 0}}}
Mnpu-kernelbench/data/kernel_generator/level1/Cumsum/workload.jsonl+52-51
@@ -1,51 +1,52 @@
1-{"uuid":"Cumsum-001","axes":null,"inputs":{"x":{"type":"random","shape":[1,1,1],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}1+{"uuid": "Cumsum-001", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 1, 1], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
2-{"uuid":"Cumsum-002","axes":null,"inputs":{"x":{"type":"random","shape":[1,1,1],"dtype":"float32"},"dim":{"type":"scalar","value":2}}}2+{"uuid": "Cumsum-002", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 1, 1], "dtype": "float32"}, "dim": {"type": "scalar", "value": 2}}}
3-{"uuid":"Cumsum-003","axes":null,"inputs":{"x":{"type":"random","shape":[1,24,96,96],"dtype":"float16"},"dim":{"type":"scalar","value":2}}}3+{"uuid": "Cumsum-003", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 24, 96, 96], "dtype": "float16"}, "dim": {"type": "scalar", "value": 2}}}
4-{"uuid":"Cumsum-004","axes":null,"inputs":{"x":{"type":"random","shape":[23,23,23,23],"dtype":"float32"},"dim":{"type":"scalar","value":3}}}4+{"uuid": "Cumsum-004", "axes": null, "inputs": {"x": {"type": "random", "shape": [23, 23, 23, 23], "dtype": "float32"}, "dim": {"type": "scalar", "value": 3}}}
5-{"uuid":"Cumsum-005","axes":null,"inputs":{"x":{"type":"random","shape":[1,192,160,160],"dtype":"bfloat16"},"dim":{"type":"scalar","value":-2}}}5+{"uuid": "Cumsum-005", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 192, 160, 160], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": -2}}}
6-{"uuid":"Cumsum-006","axes":null,"inputs":{"x":{"type":"random","shape":[1,4,336,336],"dtype":"float32"},"dim":{"type":"scalar","value":2}}}6+{"uuid": "Cumsum-006", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 4, 336, 336], "dtype": "float32"}, "dim": {"type": "scalar", "value": 2}}}
7-{"uuid":"Cumsum-007","axes":null,"inputs":{"x":{"type":"random","shape":[195,98,49],"dtype":"bfloat16"},"dim":{"type":"scalar","value":1}}}7+{"uuid": "Cumsum-007", "axes": null, "inputs": {"x": {"type": "random", "shape": [195, 98, 49], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 1}}}
8-{"uuid":"Cumsum-008","axes":null,"inputs":{"x":{"type":"random","shape":[951,1901],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}8+{"uuid": "Cumsum-008", "axes": null, "inputs": {"x": {"type": "random", "shape": [951, 1901], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
9-{"uuid":"Cumsum-009","axes":null,"inputs":{"x":{"type":"random","shape":[640,2878],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}9+{"uuid": "Cumsum-009", "axes": null, "inputs": {"x": {"type": "random", "shape": [640, 2878], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
10-{"uuid":"Cumsum-010","axes":null,"inputs":{"x":{"type":"random","shape":[131,131,131],"dtype":"float32"},"dim":{"type":"scalar","value":1}}}10+{"uuid": "Cumsum-010", "axes": null, "inputs": {"x": {"type": "random", "shape": [131, 131, 131], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}}}
11-{"uuid":"Cumsum-011","axes":null,"inputs":{"x":{"type":"random","shape":[1844,1844],"dtype":"bfloat16"},"dim":{"type":"scalar","value":-1}}}11+{"uuid": "Cumsum-011", "axes": null, "inputs": {"x": {"type": "random", "shape": [1844, 1844], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": -1}}}
12-{"uuid":"Cumsum-012","axes":null,"inputs":{"x":{"type":"random","shape":[37,37,37,37],"dtype":"float32"},"dim":{"type":"scalar","value":2}}}12+{"uuid": "Cumsum-012", "axes": null, "inputs": {"x": {"type": "random", "shape": [37, 37, 37, 37], "dtype": "float32"}, "dim": {"type": "scalar", "value": 2}}}
13-{"uuid":"Cumsum-013","axes":null,"inputs":{"x":{"type":"random","shape":[107,228,350],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}13+{"uuid": "Cumsum-013", "axes": null, "inputs": {"x": {"type": "random", "shape": [107, 228, 350], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
14-{"uuid":"Cumsum-014","axes":null,"inputs":{"x":{"type":"random","shape":[3,480,112,112],"dtype":"bfloat16"},"dim":{"type":"scalar","value":-3}}}14+{"uuid": "Cumsum-014", "axes": null, "inputs": {"x": {"type": "random", "shape": [3, 480, 112, 112], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": -3}}}
15-{"uuid":"Cumsum-015","axes":null,"inputs":{"x":{"type":"random","shape":[211,211,211],"dtype":"float16"},"dim":{"type":"scalar","value":-3}}}15+{"uuid": "Cumsum-015", "axes": null, "inputs": {"x": {"type": "random", "shape": [211, 211, 211], "dtype": "float16"}, "dim": {"type": "scalar", "value": -3}}}
16-{"uuid":"Cumsum-016","axes":null,"inputs":{"x":{"type":"random","shape":[43,43,43,43],"dtype":"float32"},"dim":{"type":"scalar","value":-1}}}16+{"uuid": "Cumsum-016", "axes": null, "inputs": {"x": {"type": "random", "shape": [43, 43, 43, 43], "dtype": "float32"}, "dim": {"type": "scalar", "value": -1}}}
17-{"uuid":"Cumsum-017","axes":null,"inputs":{"x":{"type":"random","shape":[858,17230],"dtype":"float16"},"dim":{"type":"scalar","value":0}}}17+{"uuid": "Cumsum-017", "axes": null, "inputs": {"x": {"type": "random", "shape": [858, 17230], "dtype": "float16"}, "dim": {"type": "scalar", "value": 0}}}
18-{"uuid":"Cumsum-018","axes":null,"inputs":{"x":{"type":"random","shape":[373400],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}18+{"uuid": "Cumsum-018", "axes": null, "inputs": {"x": {"type": "random", "shape": [373400], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
19-{"uuid":"Cumsum-019","axes":null,"inputs":{"x":{"type":"random","shape":[114,245,376],"dtype":"float32"},"dim":{"type":"scalar","value":1}}}19+{"uuid": "Cumsum-019", "axes": null, "inputs": {"x": {"type": "random", "shape": [114, 245, 376], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}}}
20-{"uuid":"Cumsum-020","axes":null,"inputs":{"x":{"type":"random","shape":[439,220,110],"dtype":"bfloat16"},"dim":{"type":"scalar","value":0}}}20+{"uuid": "Cumsum-020", "axes": null, "inputs": {"x": {"type": "random", "shape": [439, 220, 110], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 0}}}
21-{"uuid":"Cumsum-021","axes":null,"inputs":{"x":{"type":"random","shape":[1200,1200],"dtype":"bfloat16"},"dim":{"type":"scalar","value":0}}}21+{"uuid": "Cumsum-021", "axes": null, "inputs": {"x": {"type": "random", "shape": [1200, 1200], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 0}}}
22-{"uuid":"Cumsum-022","axes":null,"inputs":{"x":{"type":"random","shape":[4330,4330],"dtype":"float16"},"dim":{"type":"scalar","value":0}}}22+{"uuid": "Cumsum-022", "axes": null, "inputs": {"x": {"type": "random", "shape": [4330, 4330], "dtype": "float16"}, "dim": {"type": "scalar", "value": 0}}}
23-{"uuid":"Cumsum-023","axes":null,"inputs":{"x":{"type":"random","shape":[409709],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}23+{"uuid": "Cumsum-023", "axes": null, "inputs": {"x": {"type": "random", "shape": [409709], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
24-{"uuid":"Cumsum-024","axes":null,"inputs":{"x":{"type":"random","shape":[345,56143],"dtype":"bfloat16"},"dim":{"type":"scalar","value":-2}}}24+{"uuid": "Cumsum-024", "axes": null, "inputs": {"x": {"type": "random", "shape": [345, 56143], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": -2}}}
25-{"uuid":"Cumsum-025","axes":null,"inputs":{"x":{"type":"random","shape":[416372],"dtype":"float32"},"dim":{"type":"scalar","value":-1}}}25+{"uuid": "Cumsum-025", "axes": null, "inputs": {"x": {"type": "random", "shape": [416372], "dtype": "float32"}, "dim": {"type": "scalar", "value": -1}}}
26-{"uuid":"Cumsum-026","axes":null,"inputs":{"x":{"type":"random","shape":[4440,4440],"dtype":"float16"},"dim":{"type":"scalar","value":-2}}}26+{"uuid": "Cumsum-026", "axes": null, "inputs": {"x": {"type": "random", "shape": [4440, 4440], "dtype": "float16"}, "dim": {"type": "scalar", "value": -2}}}
27-{"uuid":"Cumsum-027","axes":null,"inputs":{"x":{"type":"random","shape":[3142,6285],"dtype":"float16"},"dim":{"type":"scalar","value":-1}}}27+{"uuid": "Cumsum-027", "axes": null, "inputs": {"x": {"type": "random", "shape": [3142, 6285], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}}}
28-{"uuid":"Cumsum-028","axes":null,"inputs":{"x":{"type":"random","shape":[1006,20196],"dtype":"float16"},"dim":{"type":"scalar","value":1}}}28+{"uuid": "Cumsum-028", "axes": null, "inputs": {"x": {"type": "random", "shape": [1006, 20196], "dtype": "float16"}, "dim": {"type": "scalar", "value": 1}}}
29-{"uuid":"Cumsum-029","axes":null,"inputs":{"x":{"type":"random","shape":[434531],"dtype":"float32"},"dim":{"type":"scalar","value":-1}}}29+{"uuid": "Cumsum-029", "axes": null, "inputs": {"x": {"type": "random", "shape": [434531], "dtype": "float32"}, "dim": {"type": "scalar", "value": -1}}}
30-{"uuid":"Cumsum-030","axes":null,"inputs":{"x":{"type":"random","shape":[3,48,194,194],"dtype":"float16"},"dim":{"type":"scalar","value":1}}}30+{"uuid": "Cumsum-030", "axes": null, "inputs": {"x": {"type": "random", "shape": [3, 48, 194, 194], "dtype": "float16"}, "dim": {"type": "scalar", "value": 1}}}
31-{"uuid":"Cumsum-031","axes":null,"inputs":{"x":{"type":"random","shape":[3334,6667],"dtype":"float32"},"dim":{"type":"scalar","value":-1}}}31+{"uuid": "Cumsum-031", "axes": null, "inputs": {"x": {"type": "random", "shape": [3334, 6667], "dtype": "float32"}, "dim": {"type": "scalar", "value": -1}}}
32-{"uuid":"Cumsum-032","axes":null,"inputs":{"x":{"type":"random","shape":[462222],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}32+{"uuid": "Cumsum-032", "axes": null, "inputs": {"x": {"type": "random", "shape": [462222], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
33-{"uuid":"Cumsum-033","axes":null,"inputs":{"x":{"type":"random","shape":[3,10,711,711],"dtype":"float32"},"dim":{"type":"scalar","value":3}}}33+{"uuid": "Cumsum-033", "axes": null, "inputs": {"x": {"type": "random", "shape": [3, 10, 711, 711], "dtype": "float32"}, "dim": {"type": "scalar", "value": 3}}}
34-{"uuid":"Cumsum-034","axes":null,"inputs":{"x":{"type":"random","shape":[4,63,254,254],"dtype":"float16"},"dim":{"type":"scalar","value":-1}}}34+{"uuid": "Cumsum-034", "axes": null, "inputs": {"x": {"type": "random", "shape": [4, 63, 254, 254], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}}}
35-{"uuid":"Cumsum-035","axes":null,"inputs":{"x":{"type":"random","shape":[302,302,302],"dtype":"float32"},"dim":{"type":"scalar","value":2}}}35+{"uuid": "Cumsum-035", "axes": null, "inputs": {"x": {"type": "random", "shape": [302, 302, 302], "dtype": "float32"}, "dim": {"type": "scalar", "value": 2}}}
36-{"uuid":"Cumsum-036","axes":null,"inputs":{"x":{"type":"random","shape":[71,71,71,71],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}36+{"uuid": "Cumsum-036", "axes": null, "inputs": {"x": {"type": "random", "shape": [71, 71, 71, 71], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
37-{"uuid":"Cumsum-037","axes":null,"inputs":{"x":{"type":"random","shape":[42,71,99,155],"dtype":"float16"},"dim":{"type":"scalar","value":-1}}}37+{"uuid": "Cumsum-037", "axes": null, "inputs": {"x": {"type": "random", "shape": [42, 71, 99, 155], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}}}
38-{"uuid":"Cumsum-038","axes":null,"inputs":{"x":{"type":"random","shape":[6,28,611,611],"dtype":"bfloat16"},"dim":{"type":"scalar","value":2}}}38+{"uuid": "Cumsum-038", "axes": null, "inputs": {"x": {"type": "random", "shape": [6, 28, 611, 611], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 2}}}
39-{"uuid":"Cumsum-039","axes":null,"inputs":{"x":{"type":"random","shape":[11372,22743],"dtype":"float32"},"dim":{"type":"scalar","value":1}}}39+{"uuid": "Cumsum-039", "axes": null, "inputs": {"x": {"type": "random", "shape": [11372, 22743], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}}}
40-{"uuid":"Cumsum-040","axes":null,"inputs":{"x":{"type":"random","shape":[7723,34756],"dtype":"float32"},"dim":{"type":"scalar","value":1}}}40+{"uuid": "Cumsum-040", "axes": null, "inputs": {"x": {"type": "random", "shape": [7723, 34756], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}}}
41-{"uuid":"Cumsum-041","axes":null,"inputs":{"x":{"type":"random","shape":[11585,23170],"dtype":"float32"},"dim":{"type":"scalar","value":-2}}}41+{"uuid": "Cumsum-041", "axes": null, "inputs": {"x": {"type": "random", "shape": [11585, 23170], "dtype": "float32"}, "dim": {"type": "scalar", "value": -2}}}
42-{"uuid":"Cumsum-042","axes":null,"inputs":{"x":{"type":"random","shape":[50,84,117,184],"dtype":"float16"},"dim":{"type":"scalar","value":0}}}42+{"uuid": "Cumsum-042", "axes": null, "inputs": {"x": {"type": "random", "shape": [50, 84, 117, 184], "dtype": "float16"}, "dim": {"type": "scalar", "value": 0}}}
43-{"uuid":"Cumsum-043","axes":null,"inputs":{"x":{"type":"random","shape":[4192,74226],"dtype":"float16"},"dim":{"type":"scalar","value":-1}}}43+{"uuid": "Cumsum-043", "axes": null, "inputs": {"x": {"type": "random", "shape": [4192, 74226], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}}}
44-{"uuid":"Cumsum-044","axes":null,"inputs":{"x":{"type":"random","shape":[100,100,100,100],"dtype":"float32"},"dim":{"type":"scalar","value":1}}}44+{"uuid": "Cumsum-044", "axes": null, "inputs": {"x": {"type": "random", "shape": [100, 100, 100, 100], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}}}
45-{"uuid":"Cumsum-045","axes":null,"inputs":{"x":{"type":"random","shape":[7,34,758,758],"dtype":"bfloat16"},"dim":{"type":"scalar","value":3}}}45+{"uuid": "Cumsum-045", "axes": null, "inputs": {"x": {"type": "random", "shape": [7, 34, 758, 758], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 3}}}
46-{"uuid":"Cumsum-046","axes":null,"inputs":{"x":{"type":"random","shape":[23170,23170],"dtype":"float16"},"dim":{"type":"scalar","value":1}}}46+{"uuid": "Cumsum-046", "axes": null, "inputs": {"x": {"type": "random", "shape": [23170, 23170], "dtype": "float16"}, "dim": {"type": "scalar", "value": 1}}}
47-{"uuid":"Cumsum-047","axes":null,"inputs":{"x":{"type":"random","shape":[23170,23170],"dtype":"float16"},"dim":{"type":"scalar","value":-1}}}47+{"uuid": "Cumsum-047", "axes": null, "inputs": {"x": {"type": "random", "shape": [23170, 23170], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}}}
48-{"uuid":"Cumsum-048","axes":null,"inputs":{"x":{"type":"random","shape":[16384,32768],"dtype":"float16"},"dim":{"type":"scalar","value":-2}}}48+{"uuid": "Cumsum-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [16384, 32768], "dtype": "float16"}, "dim": {"type": "scalar", "value": -2}}}
49-{"uuid":"Cumsum-049","axes":null,"inputs":{"x":{"type":"random","shape":[468,468,468],"dtype":"float16"},"dim":{"type":"scalar","value":-2}}}49+{"uuid": "Cumsum-049", "axes": null, "inputs": {"x": {"type": "random", "shape": [468, 468, 468], "dtype": "float16"}, "dim": {"type": "scalar", "value": -2}}}
50-{"uuid":"Cumsum-050","axes":null,"inputs":{"x":{"type":"random","shape":[3563755],"dtype":"float32"},"dim":{"type":"scalar","value":0}}}50+{"uuid": "Cumsum-050", "axes": null, "inputs": {"x": {"type": "random", "shape": [3563755], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}}}
51-{"uuid":"Cumsum-051","axes":null,"inputs":{"x":{"type":"random","shape":[479,479,479],"dtype":"float32"},"dim":{"type":"scalar","value":-1}}}51+{"uuid": "Cumsum-051", "axes": null, "inputs": {"x": {"type": "random", "shape": [479, 479, 479], "dtype": "float32"}, "dim": {"type": "scalar", "value": -1}}}
52+{"uuid": "Cumsum-052", "axes": null, "inputs": {"x": {"type": "random", "shape": [128, 512], "dtype": "int8", "range": [-8, 8]}, "dim": {"type": "scalar", "value": 0}}}
Mnpu-kernelbench/data/kernel_generator/level1/DynamicQuant/definition.json+19-5
@@ -2,13 +2,27 @@
2 "name": "DynamicQuant",2 "name": "DynamicQuant",
3 "axes": {},3 "axes": {},
4 "inputs": {4 "inputs": {
5- "x": {"description": "Input tensor to quantize. Supports float16, bfloat16."},5+ "x": {
6- "smooth_scales": {"description": "Optional tensor, smooth scale factors for per-channel scaling. Same dtype as x. Omit if absent."}6+ "description": "Input tensor to quantize. Supports float16, bfloat16."
7+ },
8+ "smooth_scales": {
9+ "description": "Optional tensor, smooth scale factors for per-channel scaling. Same dtype as x. Omit if absent."
10+ },
11+ "group_index": {
12+ "description": "optional; group index tensor for grouped quant"
13+ },
14+ "dst_type": {
15+ "description": "optional; output quantized dtype, 'int8' or 'int4' (int4 packed, output last dim halved)"
16+ }
7 },17 },
8 "outputs": {18 "outputs": {
9- "quantized": {"description": "Quantized tensor, int8."},19+ "quantized": {
10- "scale": {"description": "Scale tensor used for quantization, float32."}20+ "description": "Quantized tensor, int8; or int32 packed int4 when dst_type='int4' (8 int4 per int32, output last dim is 1/8 of x)"
21+ },
22+ "scale": {
23+ "description": "Scale tensor used for quantization, float32."
24+ }
11 },25 },
12- "reference": "import torch\nimport torch_npu\n\ndef run(x, smooth_scales=None):\n return torch_npu.npu_dynamic_quant(x, smooth_scales=smooth_scales)\n",26+ "reference": "import torch\nimport torch_npu\n\ndef run(x, smooth_scales=None, group_index=None, dst_type=None):\n _DST_TYPE_MAP = {\"int8\": torch.int8, \"int4\": torch.quint4x2, \"quint4x2\": torch.quint4x2}\n if isinstance(dst_type, str):\n dst_type = _DST_TYPE_MAP.get(dst_type, dst_type)\n quant_out, quant_scale = torch_npu.npu_dynamic_quant(\n x, smooth_scales=smooth_scales, group_index=group_index, dst_type=dst_type)\n return quant_out, quant_scale\n",
13 "description": "Model that performs dynamic quantization on NPU.\nPyTorch native implementation\ndef forward(self, x: torch.Tensor, smooth_scales: torch.Tensor = None,\n group_index: torch.Tensor = None, dst_type=None):\n if dst_type is None:\n dst_type = torch.int8\n\n x_float = x.float()\n\n if smooth_scales is not None:\n smooth_scales_float = smooth_scales.float()\n x_float = x_float * smooth_scales_float\n\n if group_index is not None:\n return self._quant_with_groups(x_float, group_index, dst_type)\n\n return self._quant_per_token(x_float, dst_type)\n\ndef _quant_per_token(self, x: torch.Tensor, dst_type):\n if x.dim() == 2:\n max_abs = x.abs().max(dim=1, keepdim=True)[0]\n scale = max_abs / 127.0\n scale = scale.clamp(min=1e-10)\n quantized = torch.round(x / scale)\n quantized = quantized.clamp(-128, 127).to(dst_type)\n scale = scale.squeeze(1)\n return quantized, scale\n elif x.dim() == 3:\n max_abs = x.abs().max(dim=2, keepdim=True)[0]\n scale = max_abs / 127.0\n scale = scale.clamp(min=1e-10)\n quantized = torch.round(x / scale)\n quantized = quantized.clamp(-128, 127).to(dst_type)\n scale = scale.squeeze(2)\n return quantized, scale\n else:\n max_abs = x.abs().max()\n scale = max_abs / 127.0\n scale = torch.tensor(scale, device=x.device)\n quantized = torch.round(x / scale)\n quantized = quantized.clamp(-128, 127).to(dst_type)\n return quantized, scale\n\ndef _quant_with_groups(self, x: torch.Tensor, group_index: torch.Tensor, dst_type):\n if x.dim() != 2:\n raise ValueError(\"Group quantization only supports 2D tensors\")\n\n num_tokens = x.shape[0]\n quantized = torch.zeros_like(x, dtype=dst_type)\n scales = torch.zeros(num_tokens, device=x.device)\n\n num_groups = group_index.max().item() + 1 if group_index.numel() > 0 else 1\n\n for g in range(num_groups):\n mask = (group_index == g)\n if mask.sum() == 0:\n continue\n\n group_x = x[mask]\n max_abs = group_x.abs().max()\n scale = max_abs / 127.0\n scale = max(scale, 1e-10)\n\n group_quantized = torch.round(group_x / scale)\n group_quantized = group_quantized.clamp(-128, 127).to(dst_type)\n\n quantized[mask] = group_quantized\n scales[mask] = scale\n\n return quantized, scales"27 "description": "Model that performs dynamic quantization on NPU.\nPyTorch native implementation\ndef forward(self, x: torch.Tensor, smooth_scales: torch.Tensor = None,\n group_index: torch.Tensor = None, dst_type=None):\n if dst_type is None:\n dst_type = torch.int8\n\n x_float = x.float()\n\n if smooth_scales is not None:\n smooth_scales_float = smooth_scales.float()\n x_float = x_float * smooth_scales_float\n\n if group_index is not None:\n return self._quant_with_groups(x_float, group_index, dst_type)\n\n return self._quant_per_token(x_float, dst_type)\n\ndef _quant_per_token(self, x: torch.Tensor, dst_type):\n if x.dim() == 2:\n max_abs = x.abs().max(dim=1, keepdim=True)[0]\n scale = max_abs / 127.0\n scale = scale.clamp(min=1e-10)\n quantized = torch.round(x / scale)\n quantized = quantized.clamp(-128, 127).to(dst_type)\n scale = scale.squeeze(1)\n return quantized, scale\n elif x.dim() == 3:\n max_abs = x.abs().max(dim=2, keepdim=True)[0]\n scale = max_abs / 127.0\n scale = scale.clamp(min=1e-10)\n quantized = torch.round(x / scale)\n quantized = quantized.clamp(-128, 127).to(dst_type)\n scale = scale.squeeze(2)\n return quantized, scale\n else:\n max_abs = x.abs().max()\n scale = max_abs / 127.0\n scale = torch.tensor(scale, device=x.device)\n quantized = torch.round(x / scale)\n quantized = quantized.clamp(-128, 127).to(dst_type)\n return quantized, scale\n\ndef _quant_with_groups(self, x: torch.Tensor, group_index: torch.Tensor, dst_type):\n if x.dim() != 2:\n raise ValueError(\"Group quantization only supports 2D tensors\")\n\n num_tokens = x.shape[0]\n quantized = torch.zeros_like(x, dtype=dst_type)\n scales = torch.zeros(num_tokens, device=x.device)\n\n num_groups = group_index.max().item() + 1 if group_index.numel() > 0 else 1\n\n for g in range(num_groups):\n mask = (group_index == g)\n if mask.sum() == 0:\n continue\n\n group_x = x[mask]\n max_abs = group_x.abs().max()\n scale = max_abs / 127.0\n scale = max(scale, 1e-10)\n\n group_quantized = torch.round(group_x / scale)\n group_quantized = group_quantized.clamp(-128, 127).to(dst_type)\n\n quantized[mask] = group_quantized\n scales[mask] = scale\n\n return quantized, scales"
14}28}
Mnpu-kernelbench/data/kernel_generator/level1/DynamicQuant/workload.jsonl+2-0
@@ -40,3 +40,5 @@
40{"uuid":"DynamicQuant-040","axes":null,"inputs":{"x":{"type":"random","shape":[22592,22592],"dtype":"bfloat16"}}}40{"uuid":"DynamicQuant-040","axes":null,"inputs":{"x":{"type":"random","shape":[22592,22592],"dtype":"bfloat16"}}}
41{"uuid":"DynamicQuant-041","axes":null,"inputs":{"x":{"type":"random","shape":[12496,41632],"dtype":"bfloat16"}}}41{"uuid":"DynamicQuant-041","axes":null,"inputs":{"x":{"type":"random","shape":[12496,41632],"dtype":"bfloat16"}}}
42{"uuid":"DynamicQuant-042","axes":null,"inputs":{"x":{"type":"random","shape":[9520,55488],"dtype":"float16"}}}42{"uuid":"DynamicQuant-042","axes":null,"inputs":{"x":{"type":"random","shape":[9520,55488],"dtype":"float16"}}}
43+{"uuid": "DynamicQuant-043", "axes": null, "inputs": {"x": {"type": "random", "shape": [1328, 4496], "dtype": "bfloat16"}, "dst_type": {"type": "scalar", "value": "int4"}}}
44+{"uuid": "DynamicQuant-044", "axes": null, "inputs": {"x": {"type": "random", "shape": [368, 11520], "dtype": "bfloat16"}, "smooth_scales": {"type": "random", "shape": [11520], "dtype": "bfloat16"}, "dst_type": {"type": "scalar", "value": "int4"}}}
Mnpu-kernelbench/data/kernel_generator/level1/Gather/workload.jsonl+1-0
@@ -45,3 +45,4 @@
45{"uuid":"Gather-045","axes":null,"inputs":{"x":{"type":"random","shape":[14,912,18080],"dtype":"float16"},"dim":{"type":"scalar","value":-1},"index":{"type":"random","shape":[14,912,9040],"dtype":"int64","range":[0,5120]},"sparse_grad":{"type":"scalar","value":false}}}45{"uuid":"Gather-045","axes":null,"inputs":{"x":{"type":"random","shape":[14,912,18080],"dtype":"float16"},"dim":{"type":"scalar","value":-1},"index":{"type":"random","shape":[14,912,9040],"dtype":"int64","range":[0,5120]},"sparse_grad":{"type":"scalar","value":false}}}
46{"uuid":"Gather-046","axes":null,"inputs":{"x":{"type":"random","shape":[1,8912,26720],"dtype":"float32"},"dim":{"type":"scalar","value":1},"index":{"type":"random","shape":[1,4448,26720],"dtype":"int64","range":[0,1024]},"sparse_grad":{"type":"scalar","value":false}}}46{"uuid":"Gather-046","axes":null,"inputs":{"x":{"type":"random","shape":[1,8912,26720],"dtype":"float32"},"dim":{"type":"scalar","value":1},"index":{"type":"random","shape":[1,4448,26720],"dtype":"int64","range":[0,1024]},"sparse_grad":{"type":"scalar","value":false}}}
47{"uuid":"Gather-047","axes":null,"inputs":{"x":{"type":"random","shape":[334,715,1088],"dtype":"float32"},"dim":{"type":"scalar","value":1},"index":{"type":"random","shape":[334,334,1088],"dtype":"int64","range":[0,15]},"sparse_grad":{"type":"scalar","value":false}}}47{"uuid":"Gather-047","axes":null,"inputs":{"x":{"type":"random","shape":[334,715,1088],"dtype":"float32"},"dim":{"type":"scalar","value":1},"index":{"type":"random","shape":[334,334,1088],"dtype":"int64","range":[0,15]},"sparse_grad":{"type":"scalar","value":false}}}
48+{"uuid": "Gather-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [48, 224], "dtype": "int8", "range": [-8, 8]}, "dim": {"type": "scalar", "value": 0}, "index": {"type": "random", "shape": [16, 224], "dtype": "int64", "range": [0, 48]}, "sparse_grad": {"type": "scalar", "value": true}}}
Mnpu-kernelbench/data/kernel_generator/level1/Index/workload.jsonl+1-0
@@ -39,3 +39,4 @@
39{"uuid": "Index-039", "axes": null, "inputs": {"x": {"type": "random", "shape": [58, 112, 192, 400], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 3}, "index": {"type": "random", "shape": [96], "dtype": "int64", "range": [0, 63]}}}39{"uuid": "Index-039", "axes": null, "inputs": {"x": {"type": "random", "shape": [58, 112, 192, 400], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 3}, "index": {"type": "random", "shape": [96], "dtype": "int64", "range": [0, 63]}}}
40{"uuid": "Index-040", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 26, 1696, 11872], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": -1}, "index": {"type": "random", "shape": [1696], "dtype": "int64", "range": [0, 3584]}}}40{"uuid": "Index-040", "axes": null, "inputs": {"x": {"type": "random", "shape": [1, 26, 1696, 11872], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": -1}, "index": {"type": "random", "shape": [1696], "dtype": "int64", "range": [0, 3584]}}}
41{"uuid": "Index-041", "axes": null, "inputs": {"x": {"type": "random", "shape": [6, 11, 704, 11392], "dtype": "float16"}, "dim": {"type": "scalar", "value": 2}, "index": {"type": "random", "shape": [176], "dtype": "int64", "range": [0, 256]}}}41{"uuid": "Index-041", "axes": null, "inputs": {"x": {"type": "random", "shape": [6, 11, 704, 11392], "dtype": "float16"}, "dim": {"type": "scalar", "value": 2}, "index": {"type": "random", "shape": [176], "dtype": "int64", "range": [0, 256]}}}
42+{"uuid": "Index-042", "axes": null, "inputs": {"x": {"type": "random", "shape": [624, 624, 624], "dtype": "int8", "range": [-8, 8]}, "dim": {"type": "scalar", "value": 0}, "index": {"type": "random", "shape": [160], "dtype": "int64", "range": [0, 64]}}}
Mnpu-kernelbench/data/kernel_generator/level1/Scatter/workload.jsonl+1-0
@@ -39,3 +39,4 @@
39{"uuid": "Scatter-039", "axes": null, "inputs": {"x": {"type": "random", "shape": [8192, 8192], "dtype": "float32"}, "dim": {"type": "scalar", "value": -2}, "index": {"type": "random", "shape": [4096, 8192], "dtype": "int64", "range": [0, 8192]}, "src": {"type": "random", "shape": [4096, 8192], "dtype": "float32"}}}39{"uuid": "Scatter-039", "axes": null, "inputs": {"x": {"type": "random", "shape": [8192, 8192], "dtype": "float32"}, "dim": {"type": "scalar", "value": -2}, "index": {"type": "random", "shape": [4096, 8192], "dtype": "int64", "range": [0, 8192]}, "src": {"type": "random", "shape": [4096, 8192], "dtype": "float32"}}}
40{"uuid": "Scatter-040", "axes": null, "inputs": {"x": {"type": "random", "shape": [3584, 18944], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}, "index": {"type": "random", "shape": [3584, 9472], "dtype": "int64", "range": [0, 18944]}, "src": {"type": "random", "shape": [3584, 9472], "dtype": "float16"}}}40{"uuid": "Scatter-040", "axes": null, "inputs": {"x": {"type": "random", "shape": [3584, 18944], "dtype": "float16"}, "dim": {"type": "scalar", "value": -1}, "index": {"type": "random", "shape": [3584, 9472], "dtype": "int64", "range": [0, 18944]}, "src": {"type": "random", "shape": [3584, 9472], "dtype": "float16"}}}
41{"uuid": "Scatter-041", "axes": null, "inputs": {"x": {"type": "random", "shape": [5120, 27648], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 0}, "index": {"type": "random", "shape": [2560, 27648], "dtype": "int64", "range": [0, 5120]}, "src": {"type": "random", "shape": [2560, 27648], "dtype": "bfloat16"}}}41{"uuid": "Scatter-041", "axes": null, "inputs": {"x": {"type": "random", "shape": [5120, 27648], "dtype": "bfloat16"}, "dim": {"type": "scalar", "value": 0}, "index": {"type": "random", "shape": [2560, 27648], "dtype": "int64", "range": [0, 5120]}, "src": {"type": "random", "shape": [2560, 27648], "dtype": "bfloat16"}}}
42+{"uuid": "Scatter-042", "axes": null, "inputs": {"x": {"type": "random", "shape": [2, 16, 64, 64], "dtype": "int8", "range": [-8, 8]}, "dim": {"type": "scalar", "value": 0}, "index": {"type": "random", "shape": [1, 16, 64, 64], "dtype": "int64", "range": [0, 2]}, "src": {"type": "random", "shape": [1, 16, 64, 64], "dtype": "int8", "range": [-8, 8]}}}
Mnpu-kernelbench/data/kernel_generator/level1/Sort/workload.jsonl+1-0
@@ -29,3 +29,4 @@
29{"uuid": "Sort-029", "axes": null, "inputs": {"x": {"type": "random", "shape": [224, 224, 224], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}, "descending": {"type": "scalar", "value": false}}}29{"uuid": "Sort-029", "axes": null, "inputs": {"x": {"type": "random", "shape": [224, 224, 224], "dtype": "float32"}, "dim": {"type": "scalar", "value": 0}, "descending": {"type": "scalar", "value": false}}}
30{"uuid": "Sort-030", "axes": null, "inputs": {"x": {"type": "random", "shape": [144, 144, 144], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}, "descending": {"type": "scalar", "value": true}}}30{"uuid": "Sort-030", "axes": null, "inputs": {"x": {"type": "random", "shape": [144, 144, 144], "dtype": "float32"}, "dim": {"type": "scalar", "value": 1}, "descending": {"type": "scalar", "value": true}}}
31{"uuid": "Sort-031", "axes": null, "inputs": {"x": {"type": "random", "shape": [176, 176, 176], "dtype": "float32"}, "dim": {"type": "scalar", "value": 2}, "descending": {"type": "scalar", "value": false}}}31{"uuid": "Sort-031", "axes": null, "inputs": {"x": {"type": "random", "shape": [176, 176, 176], "dtype": "float32"}, "dim": {"type": "scalar", "value": 2}, "descending": {"type": "scalar", "value": false}}}
32+{"uuid": "Sort-032", "axes": null, "inputs": {"x": {"type": "random", "shape": [256, 256], "dtype": "int8", "range": [-8, 8]}, "dim": {"type": "scalar", "value": 0}, "descending": {"type": "scalar", "value": false}}}
Mnpu-kernelbench/data/kernel_generator/level1/TopK/workload.jsonl+1-0
@@ -27,3 +27,4 @@
27{"uuid":"TopK-027","axes":null,"inputs":{"x":{"type":"random","shape":[13368,13368],"dtype":"float16"},"k":{"type":"scalar","value":16},"dim":{"type":"scalar","value":-2},"largest":{"type":"scalar","value":false}}}27{"uuid":"TopK-027","axes":null,"inputs":{"x":{"type":"random","shape":[13368,13368],"dtype":"float16"},"k":{"type":"scalar","value":16},"dim":{"type":"scalar","value":-2},"largest":{"type":"scalar","value":false}}}
28{"uuid":"TopK-028","axes":null,"inputs":{"x":{"type":"random","shape":[574,574,574],"dtype":"float16"},"k":{"type":"scalar","value":8},"dim":{"type":"scalar","value":-2},"largest":{"type":"scalar","value":true}}}28{"uuid":"TopK-028","axes":null,"inputs":{"x":{"type":"random","shape":[574,574,574],"dtype":"float16"},"k":{"type":"scalar","value":8},"dim":{"type":"scalar","value":-2},"largest":{"type":"scalar","value":true}}}
29{"uuid":"TopK-029","axes":null,"inputs":{"x":{"type":"random","shape":[7723,34756],"dtype":"float32"},"k":{"type":"scalar","value":64},"dim":{"type":"scalar","value":0},"largest":{"type":"scalar","value":true}}}29{"uuid":"TopK-029","axes":null,"inputs":{"x":{"type":"random","shape":[7723,34756],"dtype":"float32"},"k":{"type":"scalar","value":64},"dim":{"type":"scalar","value":0},"largest":{"type":"scalar","value":true}}}
30+{"uuid": "TopK-030", "axes": null, "inputs": {"x": {"type": "random", "shape": [64, 1024], "dtype": "int8", "range": [-8, 8]}, "k": {"type": "scalar", "value": 10}, "dim": {"type": "scalar", "value": -1}, "largest": {"type": "scalar", "value": true}}}
Mnpu-kernelbench/data/kernel_generator/level2/KvRmsnormRopeCache/workload.jsonl+2-0
@@ -48,3 +48,5 @@
48{"uuid":"KvRmsnormRopeCache-048","axes":null,"inputs":{"kv":{"type":"random","shape":[32,1,256,576],"dtype":"bfloat16"},"gamma":{"type":"random","shape":[512],"dtype":"bfloat16"},"cos":{"type":"random","shape":[32,1,256,64],"dtype":"bfloat16"},"sin":{"type":"random","shape":[32,1,256,64],"dtype":"bfloat16"},"index":{"type":"random","shape":[8192],"dtype":"int64","range":[0,576]},"k_cache":{"type":"random","shape":[64,128,1,64],"dtype":"bfloat16"},"ckv_cache":{"type":"random","shape":[64,128,1,512],"dtype":"bfloat16"},"epsilon":{"type":"scalar","value":1e-05},"cache_mode":{"type":"scalar","value":"PA_NZ"},"is_output_kv":{"type":"scalar","value":false}}}48{"uuid":"KvRmsnormRopeCache-048","axes":null,"inputs":{"kv":{"type":"random","shape":[32,1,256,576],"dtype":"bfloat16"},"gamma":{"type":"random","shape":[512],"dtype":"bfloat16"},"cos":{"type":"random","shape":[32,1,256,64],"dtype":"bfloat16"},"sin":{"type":"random","shape":[32,1,256,64],"dtype":"bfloat16"},"index":{"type":"random","shape":[8192],"dtype":"int64","range":[0,576]},"k_cache":{"type":"random","shape":[64,128,1,64],"dtype":"bfloat16"},"ckv_cache":{"type":"random","shape":[64,128,1,512],"dtype":"bfloat16"},"epsilon":{"type":"scalar","value":1e-05},"cache_mode":{"type":"scalar","value":"PA_NZ"},"is_output_kv":{"type":"scalar","value":false}}}
49{"uuid":"KvRmsnormRopeCache-049","axes":null,"inputs":{"kv":{"type":"random","shape":[32,1,512,576],"dtype":"float16"},"gamma":{"type":"random","shape":[512],"dtype":"float16"},"cos":{"type":"random","shape":[32,1,512,64],"dtype":"float16"},"sin":{"type":"random","shape":[32,1,512,64],"dtype":"float16"},"index":{"type":"random","shape":[128],"dtype":"int64","range":[0,576]},"k_cache":{"type":"random","shape":[128,128,1,64],"dtype":"float16"},"ckv_cache":{"type":"random","shape":[128,128,1,512],"dtype":"float16"},"epsilon":{"type":"scalar","value":1e-06},"cache_mode":{"type":"scalar","value":"PA_BLK_BNSD"},"is_output_kv":{"type":"scalar","value":true}}}49{"uuid":"KvRmsnormRopeCache-049","axes":null,"inputs":{"kv":{"type":"random","shape":[32,1,512,576],"dtype":"float16"},"gamma":{"type":"random","shape":[512],"dtype":"float16"},"cos":{"type":"random","shape":[32,1,512,64],"dtype":"float16"},"sin":{"type":"random","shape":[32,1,512,64],"dtype":"float16"},"index":{"type":"random","shape":[128],"dtype":"int64","range":[0,576]},"k_cache":{"type":"random","shape":[128,128,1,64],"dtype":"float16"},"ckv_cache":{"type":"random","shape":[128,128,1,512],"dtype":"float16"},"epsilon":{"type":"scalar","value":1e-06},"cache_mode":{"type":"scalar","value":"PA_BLK_BNSD"},"is_output_kv":{"type":"scalar","value":true}}}
50{"uuid":"KvRmsnormRopeCache-050","axes":null,"inputs":{"kv":{"type":"random","shape":[32,1,1024,576],"dtype":"bfloat16"},"gamma":{"type":"random","shape":[512],"dtype":"bfloat16"},"cos":{"type":"random","shape":[32,1,1024,64],"dtype":"bfloat16"},"sin":{"type":"random","shape":[32,1,1024,64],"dtype":"bfloat16"},"index":{"type":"random","shape":[256],"dtype":"int64","range":[0,576]},"k_cache":{"type":"random","shape":[256,128,1,64],"dtype":"bfloat16"},"ckv_cache":{"type":"random","shape":[256,128,1,512],"dtype":"bfloat16"},"epsilon":{"type":"scalar","value":1e-05},"cache_mode":{"type":"scalar","value":"PA_BLK_NZ"},"is_output_kv":{"type":"scalar","value":false}}}50{"uuid":"KvRmsnormRopeCache-050","axes":null,"inputs":{"kv":{"type":"random","shape":[32,1,1024,576],"dtype":"bfloat16"},"gamma":{"type":"random","shape":[512],"dtype":"bfloat16"},"cos":{"type":"random","shape":[32,1,1024,64],"dtype":"bfloat16"},"sin":{"type":"random","shape":[32,1,1024,64],"dtype":"bfloat16"},"index":{"type":"random","shape":[256],"dtype":"int64","range":[0,576]},"k_cache":{"type":"random","shape":[256,128,1,64],"dtype":"bfloat16"},"ckv_cache":{"type":"random","shape":[256,128,1,512],"dtype":"bfloat16"},"epsilon":{"type":"scalar","value":1e-05},"cache_mode":{"type":"scalar","value":"PA_BLK_NZ"},"is_output_kv":{"type":"scalar","value":false}}}
51+{"uuid": "KvRmsnormRopeCache-051", "axes": null, "inputs": {"kv": {"type": "random", "shape": [1, 1, 128, 576], "dtype": "float16"}, "gamma": {"type": "random", "shape": [512], "dtype": "float16"}, "cos": {"type": "random", "shape": [1, 1, 128, 64], "dtype": "float16"}, "sin": {"type": "random", "shape": [1, 1, 128, 64], "dtype": "float16"}, "index": {"type": "random", "shape": [1, 128], "dtype": "int64", "range": [0, 576]}, "k_cache": {"type": "random", "shape": [1, 1, 2048, 64], "dtype": "int8", "range": [-8, 8]}, "ckv_cache": {"type": "random", "shape": [1, 1, 2048, 512], "dtype": "int8", "range": [-8, 8]}, "epsilon": {"type": "scalar", "value": 1e-05}, "cache_mode": {"type": "scalar", "value": "Norm"}, "is_output_kv": {"type": "scalar", "value": false}, "k_rope_scale": {"type": "random", "shape": [64], "dtype": "float32"}, "c_kv_scale": {"type": "random", "shape": [512], "dtype": "float32"}, "k_rope_offset": {"type": "random", "shape": [64], "dtype": "float32"}, "c_kv_offset": {"type": "random", "shape": [512], "dtype": "float32"}}}
52+{"uuid": "KvRmsnormRopeCache-052", "axes": null, "inputs": {"kv": {"type": "random", "shape": [16, 1, 64, 576], "dtype": "float16"}, "gamma": {"type": "random", "shape": [512], "dtype": "float16"}, "cos": {"type": "random", "shape": [16, 1, 64, 64], "dtype": "float16"}, "sin": {"type": "random", "shape": [16, 1, 64, 64], "dtype": "float16"}, "index": {"type": "random", "shape": [16, 64], "dtype": "int64", "range": [0, 576]}, "k_cache": {"type": "random", "shape": [16, 1, 2048, 64], "dtype": "int8", "range": [-8, 8]}, "ckv_cache": {"type": "random", "shape": [16, 1, 2048, 512], "dtype": "int8", "range": [-8, 8]}, "epsilon": {"type": "scalar", "value": 1e-05}, "cache_mode": {"type": "scalar", "value": "Norm"}, "is_output_kv": {"type": "scalar", "value": false}, "k_rope_scale": {"type": "random", "shape": [64], "dtype": "float32"}, "c_kv_scale": {"type": "random", "shape": [512], "dtype": "float32"}, "k_rope_offset": {"type": "random", "shape": [64], "dtype": "float32"}, "c_kv_offset": {"type": "random", "shape": [512], "dtype": "float32"}}}
Mnpu-kernelbench/data/kernel_generator/level2/SwigluQuant/definition.json+2-2
@@ -30,7 +30,7 @@
30 },30 },
31 "outputs": {31 "outputs": {
32 "0": {32 "0": {
33- "description": "Quantized output, int8"33+ "description": "Quantized output, int8; int4 when dst_type='int4' (packed, last dim halved)"
34 },34 },
35 "1": {35 "1": {
36 "description": "Quantization scales"36 "description": "Quantization scales"
@@ -38,4 +38,4 @@
38 },38 },
39 "reference": "import torch\nimport torch_npu\n\n_DST_TYPE_MAP = {\"int4\": torch.quint4x2, \"int8\": torch.int8}\n\ndef run(x, smooth_scales=None, offsets=None, group_index=None, activate_left=False, quant_mode=0, group_list_type=0, dst_type=None):\n if isinstance(dst_type, str):\n dst_type = _DST_TYPE_MAP.get(dst_type, dst_type)\n return torch_npu.npu_swiglu_quant(x, smooth_scales=smooth_scales, offsets=offsets, group_index=group_index, activate_left=activate_left, quant_mode=quant_mode, group_list_type=group_list_type, dst_type=dst_type)\n",39 "reference": "import torch\nimport torch_npu\n\n_DST_TYPE_MAP = {\"int4\": torch.quint4x2, \"int8\": torch.int8}\n\ndef run(x, smooth_scales=None, offsets=None, group_index=None, activate_left=False, quant_mode=0, group_list_type=0, dst_type=None):\n if isinstance(dst_type, str):\n dst_type = _DST_TYPE_MAP.get(dst_type, dst_type)\n return torch_npu.npu_swiglu_quant(x, smooth_scales=smooth_scales, offsets=offsets, group_index=group_index, activate_left=activate_left, quant_mode=quant_mode, group_list_type=group_list_type, dst_type=dst_type)\n",
40 "description": "Simple model that performs SwiGLU with quantization.\ntorch_npu.npu_swiglu_quant(x, *, smooth_scales=None, offsets=None, group_index=None, activate_left=False, quant_mode=0, group_list_type=0, dst_type=None) -> (Tensor, Tensor)\nPyTorch native implementation of forward function\nimport torch\n\n_DST_TYPE_MAP = {\"int4\": torch.quint4x2, \"int8\": torch.int8}\n\ndef run(x, smooth_scales=None, offsets=None, group_index=None, activate_left=False, quant_mode=0, group_list_type=0, dst_type=None):\n if isinstance(dst_type, str):\n dst_type = _DST_TYPE_MAP.get(dst_type, dst_type)\n if dst_type is None:\n dst_type = torch.int8\n\n is_int4 = (dst_type == torch.quint4x2)\n dst_scale = 7.0 if is_int4 else 127.0\n\n x1, x2 = torch.chunk(x, 2, dim=-1)\n x1f = x1.float()\n x2f = x2.float()\n if activate_left:\n y = torch.nn.functional.silu(x1f) * x2f\n else:\n y = x1f * torch.nn.functional.silu(x2f)\n\n y_last_dim = y.shape[-1]\n batch_dim = x.shape[0]\n\n if group_index is not None:\n begin_index = 0\n for i in range(group_index.shape[0]):\n end_index = group_index[i].item() if group_list_type == 0 else begin_index + group_index[i].item()\n end_index = min(end_index, batch_dim)\n if end_index <= begin_index:\n begin_index = end_index\n continue\n y_slice = y[begin_index:end_index]\n scale_slice = smooth_scales[i].float()\n if offsets is not None and quant_mode == 0:\n offset_slice = offsets[i].float()\n else:\n offset_slice = None\n\n if scale_slice.shape[-1] == y_last_dim:\n y[begin_index:end_index] = y_slice * scale_slice + (offset_slice if offset_slice is not None else 0)\n elif scale_slice.shape[-1] == batch_dim:\n token_scale = scale_slice[begin_index:end_index].unsqueeze(-1)\n if offset_slice is not None:\n token_offset = offset_slice[begin_index:end_index].unsqueeze(-1)\n else:\n token_offset = 0\n y[begin_index:end_index] = y_slice * token_scale + token_offset\n else:\n y[begin_index:end_index] = y_slice * scale_slice + (offset_slice if offset_slice is not None else 0)\n\n begin_index = end_index\n else:\n if smooth_scales is not None:\n if smooth_scales.shape[-1] == y_last_dim:\n y = y * smooth_scales.float()\n elif smooth_scales.shape[-1] == batch_dim:\n y = y * smooth_scales.float().unsqueeze(-1)\n if offsets is not None and quant_mode == 0:\n if offsets.shape[-1] == y_last_dim:\n y = y + offsets.float()\n elif offsets.shape[-1] == batch_dim:\n y = y + offsets.float().unsqueeze(-1)\n\n scale = None\n if quant_mode == 1:\n scale = dst_scale / torch.max(torch.abs(y), dim=1)[0]\n y = y * scale.unsqueeze(1)\n\n y = torch.round(y)\n y = torch.clamp(y, -1 - dst_scale, dst_scale)\n\n if is_int4:\n y_reshaped = y.reshape(*y.shape[:-1], -1, 2)\n low = (y_reshaped[..., 0].to(torch.int32) & 0xF).to(torch.uint8)\n high = (y_reshaped[..., 1].to(torch.int32) & 0xF).to(torch.uint8)\n y = ((high << 4) | low).to(torch.int8)\n else:\n y = y.to(torch.int8)\n\n if scale is None:\n scale = torch.zeros(x.shape[0], dtype=torch.float32, device=x.device)\n\n return y, scale\n"40 "description": "Simple model that performs SwiGLU with quantization.\ntorch_npu.npu_swiglu_quant(x, *, smooth_scales=None, offsets=None, group_index=None, activate_left=False, quant_mode=0, group_list_type=0, dst_type=None) -> (Tensor, Tensor)\nPyTorch native implementation of forward function\nimport torch\n\n_DST_TYPE_MAP = {\"int4\": torch.quint4x2, \"int8\": torch.int8}\n\ndef run(x, smooth_scales=None, offsets=None, group_index=None, activate_left=False, quant_mode=0, group_list_type=0, dst_type=None):\n if isinstance(dst_type, str):\n dst_type = _DST_TYPE_MAP.get(dst_type, dst_type)\n if dst_type is None:\n dst_type = torch.int8\n\n is_int4 = (dst_type == torch.quint4x2)\n dst_scale = 7.0 if is_int4 else 127.0\n\n x1, x2 = torch.chunk(x, 2, dim=-1)\n x1f = x1.float()\n x2f = x2.float()\n if activate_left:\n y = torch.nn.functional.silu(x1f) * x2f\n else:\n y = x1f * torch.nn.functional.silu(x2f)\n\n y_last_dim = y.shape[-1]\n batch_dim = x.shape[0]\n\n if group_index is not None:\n begin_index = 0\n for i in range(group_index.shape[0]):\n end_index = group_index[i].item() if group_list_type == 0 else begin_index + group_index[i].item()\n end_index = min(end_index, batch_dim)\n if end_index <= begin_index:\n begin_index = end_index\n continue\n y_slice = y[begin_index:end_index]\n scale_slice = smooth_scales[i].float()\n if offsets is not None and quant_mode == 0:\n offset_slice = offsets[i].float()\n else:\n offset_slice = None\n\n if scale_slice.shape[-1] == y_last_dim:\n y[begin_index:end_index] = y_slice * scale_slice + (offset_slice if offset_slice is not None else 0)\n elif scale_slice.shape[-1] == batch_dim:\n token_scale = scale_slice[begin_index:end_index].unsqueeze(-1)\n if offset_slice is not None:\n token_offset = offset_slice[begin_index:end_index].unsqueeze(-1)\n else:\n token_offset = 0\n y[begin_index:end_index] = y_slice * token_scale + token_offset\n else:\n y[begin_index:end_index] = y_slice * scale_slice + (offset_slice if offset_slice is not None else 0)\n\n begin_index = end_index\n else:\n if smooth_scales is not None:\n if smooth_scales.shape[-1] == y_last_dim:\n y = y * smooth_scales.float()\n elif smooth_scales.shape[-1] == batch_dim:\n y = y * smooth_scales.float().unsqueeze(-1)\n if offsets is not None and quant_mode == 0:\n if offsets.shape[-1] == y_last_dim:\n y = y + offsets.float()\n elif offsets.shape[-1] == batch_dim:\n y = y + offsets.float().unsqueeze(-1)\n\n scale = None\n if quant_mode == 1:\n scale = dst_scale / torch.max(torch.abs(y), dim=1)[0]\n y = y * scale.unsqueeze(1)\n\n y = torch.round(y)\n y = torch.clamp(y, -1 - dst_scale, dst_scale)\n\n if is_int4:\n y_reshaped = y.reshape(*y.shape[:-1], -1, 2)\n low = (y_reshaped[..., 0].to(torch.int32) & 0xF).to(torch.uint8)\n high = (y_reshaped[..., 1].to(torch.int32) & 0xF).to(torch.uint8)\n y = ((high << 4) | low).to(torch.int8)\n else:\n y = y.to(torch.int8)\n\n if scale is None:\n scale = torch.zeros(x.shape[0], dtype=torch.float32, device=x.device)\n\n return y, scale\n"
41-}41+}
Mnpu-kernelbench/data/kernel_generator/level2/SwigluQuant/workload.jsonl+2-0
@@ -50,3 +50,5 @@
50{"uuid":"SwigluQuant-050","axes":null,"inputs":{"x":{"type":"random","shape":[1600,3200],"dtype":"bfloat16"},"smooth_scales":{"type":"random","shape":[64,1600],"dtype":"float32"},"offsets":{"type":"random","shape":[64,1600],"dtype":"float32"},"group_index":{"type":"random","shape":[64],"dtype":"int32","range":[0,3200]},"activate_left":{"type":"scalar","value":false},"quant_mode":{"type":"scalar","value":0},"group_list_type":{"type":"scalar","value":1},"dst_type":{"type":"scalar","value":"int4"}}}50{"uuid":"SwigluQuant-050","axes":null,"inputs":{"x":{"type":"random","shape":[1600,3200],"dtype":"bfloat16"},"smooth_scales":{"type":"random","shape":[64,1600],"dtype":"float32"},"offsets":{"type":"random","shape":[64,1600],"dtype":"float32"},"group_index":{"type":"random","shape":[64],"dtype":"int32","range":[0,3200]},"activate_left":{"type":"scalar","value":false},"quant_mode":{"type":"scalar","value":0},"group_list_type":{"type":"scalar","value":1},"dst_type":{"type":"scalar","value":"int4"}}}
51{"uuid":"SwigluQuant-051","axes":null,"inputs":{"x":{"type":"random","shape":[3200,6400],"dtype":"float32"},"smooth_scales":{"type":"random","shape":[128,3200],"dtype":"float32"},"offsets":{"type":"random","shape":[128,3200],"dtype":"float32"},"group_index":{"type":"random","shape":[128],"dtype":"int32","range":[0,6400]},"activate_left":{"type":"scalar","value":true},"quant_mode":{"type":"scalar","value":1},"group_list_type":{"type":"scalar","value":0},"dst_type":{"type":"scalar","value":"int8"}}}51{"uuid":"SwigluQuant-051","axes":null,"inputs":{"x":{"type":"random","shape":[3200,6400],"dtype":"float32"},"smooth_scales":{"type":"random","shape":[128,3200],"dtype":"float32"},"offsets":{"type":"random","shape":[128,3200],"dtype":"float32"},"group_index":{"type":"random","shape":[128],"dtype":"int32","range":[0,6400]},"activate_left":{"type":"scalar","value":true},"quant_mode":{"type":"scalar","value":1},"group_list_type":{"type":"scalar","value":0},"dst_type":{"type":"scalar","value":"int8"}}}
52{"uuid":"SwigluQuant-052","axes":null,"inputs":{"x":{"type":"random","shape":[6400,8192],"dtype":"float16"},"smooth_scales":{"type":"random","shape":[256,6400],"dtype":"float32"},"offsets":{"type":"random","shape":[256,6400],"dtype":"float32"},"group_index":{"type":"random","shape":[256],"dtype":"int32","range":[0,8192]},"activate_left":{"type":"scalar","value":false},"quant_mode":{"type":"scalar","value":0},"group_list_type":{"type":"scalar","value":1},"dst_type":{"type":"scalar","value":"int8"}}}52{"uuid":"SwigluQuant-052","axes":null,"inputs":{"x":{"type":"random","shape":[6400,8192],"dtype":"float16"},"smooth_scales":{"type":"random","shape":[256,6400],"dtype":"float32"},"offsets":{"type":"random","shape":[256,6400],"dtype":"float32"},"group_index":{"type":"random","shape":[256],"dtype":"int32","range":[0,8192]},"activate_left":{"type":"scalar","value":false},"quant_mode":{"type":"scalar","value":0},"group_list_type":{"type":"scalar","value":1},"dst_type":{"type":"scalar","value":"int8"}}}
53+{"uuid": "SwigluQuant-053", "axes": null, "inputs": {"x": {"type": "random", "shape": [1108, 2216], "dtype": "float32"}, "smooth_scales": {"type": "random", "shape": [9, 1108], "dtype": "float32"}, "offsets": {"type": "random", "shape": [9, 1108], "dtype": "float32"}, "group_index": {"type": "random", "shape": [9], "dtype": "int32", "range": [0, 4096]}, "activate_left": {"type": "scalar", "value": true}, "quant_mode": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 0}, "dst_type": {"type": "scalar", "value": "int4"}}}
54+{"uuid": "SwigluQuant-054", "axes": null, "inputs": {"x": {"type": "random", "shape": [192, 384], "dtype": "float32"}, "smooth_scales": {"type": "random", "shape": [2, 192], "dtype": "float32"}, "offsets": {"type": "random", "shape": [2, 192], "dtype": "float32"}, "group_index": {"type": "random", "shape": [2], "dtype": "int32", "range": [0, 384]}, "activate_left": {"type": "scalar", "value": false}, "quant_mode": {"type": "scalar", "value": 0}, "group_list_type": {"type": "scalar", "value": 1}, "dst_type": {"type": "scalar", "value": "int4"}}}
Mnpu-kernelbench/data/kernel_generator/level4/FusedInferAttentionScore/definition.json+5-5
@@ -4,13 +4,13 @@
4 "axes": {},4 "axes": {},
5 "inputs": {5 "inputs": {
6 "query": {6 "query": {
7- "description": "query tensor for inference attention, dtype float16"7+ "description": "query tensor for inference attention, dtype float16/bfloat16"
8 },8 },
9 "key": {9 "key": {
10- "description": "key tensor for inference attention, dtype float16"10+ "description": "key tensor for inference attention, dtype float16/bfloat16; or int8 with key_antiquant_scale, or int32 packed int4 (H dim is 1/8 of actual), or float8_e4m3fn — for KV-cache quantization (decode scenario)"
11 },11 },
12 "value": {12 "value": {
13- "description": "value tensor for inference attention, dtype float16"13+ "description": "value tensor for inference attention, dtype float16/bfloat16; or int8 with value_antiquant_scale, or int32 packed int4 (H dim is 1/8 of actual), or float8_e4m3fn — for KV-cache quantization (decode scenario)"
14 },14 },
15 "pse_shift": {15 "pse_shift": {
16 "description": "Tensor, optional; position encoding parameter"16 "description": "Tensor, optional; position encoding parameter"
@@ -55,13 +55,13 @@
55 "description": "Tensor, optional; key/value right-alignment padding info, dtype int64"55 "description": "Tensor, optional; key/value right-alignment padding info, dtype int64"
56 },56 },
57 "key_antiquant_scale": {57 "key_antiquant_scale": {
58- "description": "Tensor, optional; key anti-quantization scale"58+ "description": "Tensor, optional; key anti-quantization scale, perchannel shape [H] under BSH layout, dtype same as query"
59 },59 },
60 "key_antiquant_offset": {60 "key_antiquant_offset": {
61 "description": "Tensor, optional; key anti-quantization offset"61 "description": "Tensor, optional; key anti-quantization offset"
62 },62 },
63 "value_antiquant_scale": {63 "value_antiquant_scale": {
64- "description": "Tensor, optional; value anti-quantization scale"64+ "description": "Tensor, optional; value anti-quantization scale, perchannel shape [H] under BSH layout, dtype same as query"
65 },65 },
66 "value_antiquant_offset": {66 "value_antiquant_offset": {
67 "description": "Tensor, optional; value anti-quantization offset"67 "description": "Tensor, optional; value anti-quantization offset"
Mnpu-kernelbench/data/kernel_generator/level4/FusedInferAttentionScore/workload.jsonl+12-0
@@ -48,3 +48,15 @@
48{"uuid":"FusedInferAttentionScore-048","axes":null,"inputs":{"query":{"type":"random","shape":[1,22480,256],"dtype":"float16"},"key":{"type":"random","shape":[1,22480,256],"dtype":"float16"},"value":{"type":"random","shape":[1,22480,256],"dtype":"float16"},"num_heads":{"type":"scalar","value":2},"input_layout":{"type":"scalar","value":"BSH"},"scale":{"type":"scalar","value":0.088388},"num_key_value_heads":{"type":"scalar","value":0},"sparse_mode":{"type":"scalar","value":0}}}48{"uuid":"FusedInferAttentionScore-048","axes":null,"inputs":{"query":{"type":"random","shape":[1,22480,256],"dtype":"float16"},"key":{"type":"random","shape":[1,22480,256],"dtype":"float16"},"value":{"type":"random","shape":[1,22480,256],"dtype":"float16"},"num_heads":{"type":"scalar","value":2},"input_layout":{"type":"scalar","value":"BSH"},"scale":{"type":"scalar","value":0.088388},"num_key_value_heads":{"type":"scalar","value":0},"sparse_mode":{"type":"scalar","value":0}}}
49{"uuid":"FusedInferAttentionScore-049","axes":null,"inputs":{"query":{"type":"random","shape":[1,4,9664,192],"dtype":"bfloat16"},"key":{"type":"random","shape":[1,4,9664,192],"dtype":"bfloat16"},"value":{"type":"random","shape":[1,4,9664,192],"dtype":"bfloat16"},"num_heads":{"type":"scalar","value":4},"input_layout":{"type":"scalar","value":"BNSD"},"scale":{"type":"scalar","value":0.072169},"num_key_value_heads":{"type":"scalar","value":4},"atten_mask":{"type":"random","shape":[1,1,9664,9664],"dtype":"bool"},"sparse_mode":{"type":"scalar","value":1}}}49{"uuid":"FusedInferAttentionScore-049","axes":null,"inputs":{"query":{"type":"random","shape":[1,4,9664,192],"dtype":"bfloat16"},"key":{"type":"random","shape":[1,4,9664,192],"dtype":"bfloat16"},"value":{"type":"random","shape":[1,4,9664,192],"dtype":"bfloat16"},"num_heads":{"type":"scalar","value":4},"input_layout":{"type":"scalar","value":"BNSD"},"scale":{"type":"scalar","value":0.072169},"num_key_value_heads":{"type":"scalar","value":4},"atten_mask":{"type":"random","shape":[1,1,9664,9664],"dtype":"bool"},"sparse_mode":{"type":"scalar","value":1}}}
50{"uuid":"FusedInferAttentionScore-050","axes":null,"inputs":{"query":{"type":"random","shape":[1,4,9088,512],"dtype":"bfloat16"},"key":{"type":"random","shape":[1,2,9088,512],"dtype":"bfloat16"},"value":{"type":"random","shape":[1,2,9088,512],"dtype":"bfloat16"},"num_heads":{"type":"scalar","value":4},"input_layout":{"type":"scalar","value":"BNSD"},"scale":{"type":"scalar","value":0.039528},"num_key_value_heads":{"type":"scalar","value":2},"atten_mask":{"type":"random","shape":[1,1,9088,9088],"dtype":"bool"},"sparse_mode":{"type":"scalar","value":1}}}50{"uuid":"FusedInferAttentionScore-050","axes":null,"inputs":{"query":{"type":"random","shape":[1,4,9088,512],"dtype":"bfloat16"},"key":{"type":"random","shape":[1,2,9088,512],"dtype":"bfloat16"},"value":{"type":"random","shape":[1,2,9088,512],"dtype":"bfloat16"},"num_heads":{"type":"scalar","value":4},"input_layout":{"type":"scalar","value":"BNSD"},"scale":{"type":"scalar","value":0.039528},"num_key_value_heads":{"type":"scalar","value":2},"atten_mask":{"type":"random","shape":[1,1,9088,9088],"dtype":"bool"},"sparse_mode":{"type":"scalar","value":1}}}
51+{"uuid": "FusedInferAttentionScore-051", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 64, 2048], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 64, 2048], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 0}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
52+{"uuid": "FusedInferAttentionScore-052", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 64, 512], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 64, 512], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [512], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [512], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 4}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
53+{"uuid": "FusedInferAttentionScore-053", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 4096], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 256, 1024], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 256, 1024], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [1024], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [1024], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 32}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 8}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
54+{"uuid": "FusedInferAttentionScore-054", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 512], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 1024, 128], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 1024, 128], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [128], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [128], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 8}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.125}, "num_key_value_heads": {"type": "scalar", "value": 2}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
55+{"uuid": "FusedInferAttentionScore-055", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "bfloat16"}, "key": {"type": "random", "shape": [1, 64, 2048], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 64, 2048], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "bfloat16"}, "value_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "bfloat16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 0}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
56+{"uuid": "FusedInferAttentionScore-056", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "bfloat16"}, "key": {"type": "random", "shape": [1, 128, 512], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 128, 512], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [512], "dtype": "bfloat16"}, "value_antiquant_scale": {"type": "random", "shape": [512], "dtype": "bfloat16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 4}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
57+{"uuid": "FusedInferAttentionScore-057", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 1024], "dtype": "bfloat16"}, "key": {"type": "random", "shape": [1, 512, 1024], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 512, 1024], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [1024], "dtype": "bfloat16"}, "value_antiquant_scale": {"type": "random", "shape": [1024], "dtype": "bfloat16"}, "num_heads": {"type": "scalar", "value": 8}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 0}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
58+{"uuid": "FusedInferAttentionScore-058", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 4096], "dtype": "bfloat16"}, "key": {"type": "random", "shape": [1, 2048, 512], "dtype": "int8", "range": [-5, 5]}, "value": {"type": "random", "shape": [1, 2048, 512], "dtype": "int8", "range": [-5, 5]}, "key_antiquant_scale": {"type": "random", "shape": [512], "dtype": "bfloat16"}, "value_antiquant_scale": {"type": "random", "shape": [512], "dtype": "bfloat16"}, "num_heads": {"type": "scalar", "value": 32}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 4}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
59+{"uuid": "FusedInferAttentionScore-059", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 64, 2048], "dtype": "float8_e4m3fn"}, "value": {"type": "random", "shape": [1, 64, 2048], "dtype": "float8_e4m3fn"}, "key_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 0}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
60+{"uuid": "FusedInferAttentionScore-060", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 128, 512], "dtype": "float8_e4m3fn"}, "value": {"type": "random", "shape": [1, 128, 512], "dtype": "float8_e4m3fn"}, "key_antiquant_scale": {"type": "random", "shape": [512], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [512], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 4}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
61+{"uuid": "FusedInferAttentionScore-061", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 64, 256], "dtype": "int32"}, "value": {"type": "random", "shape": [1, 64, 256], "dtype": "int32"}, "key_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [2048], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 0}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
62+{"uuid": "FusedInferAttentionScore-062", "axes": null, "inputs": {"query": {"type": "random", "shape": [1, 1, 2048], "dtype": "float16"}, "key": {"type": "random", "shape": [1, 128, 64], "dtype": "int32"}, "value": {"type": "random", "shape": [1, 128, 64], "dtype": "int32"}, "key_antiquant_scale": {"type": "random", "shape": [512], "dtype": "float16"}, "value_antiquant_scale": {"type": "random", "shape": [512], "dtype": "float16"}, "num_heads": {"type": "scalar", "value": 16}, "input_layout": {"type": "scalar", "value": "BSH"}, "scale": {"type": "scalar", "value": 0.088388}, "num_key_value_heads": {"type": "scalar", "value": 4}, "sparse_mode": {"type": "scalar", "value": 0}, "antiquant_mode": {"type": "scalar", "value": 0}}}
Mnpu-kernelbench/data/kernel_generator/level4/GroupedMatmulSwigluQuantV2/definition.json+21-12
@@ -4,16 +4,16 @@
4 "axes": {},4 "axes": {},
5 "inputs": {5 "inputs": {
6 "x": {6 "x": {
7- "description": "input matrix (m, k), dtype int8"7+ "description": "input matrix (m, k), dtype int8; Ascend 950 MX quant: float8_e4m3fn/float8_e5m2, or uint8 packed fp4 (m, k/2)"
8 },8 },
9 "weight": {9 "weight": {
10- "description": "weight list, each element (k, n), dtype int8"10+ "description": "weight list, each element (k, n), dtype int8; Ascend 950 MX quant: float8 or uint8 packed fp4"
11 },11 },
12 "weight_scale": {12 "weight_scale": {
13- "description": "per-channel weight dequantization scale, dtype float32"13+ "description": "weight dequantization scale; float32 for int8 cases; int8 bit-pattern of float8_e8m0fnu for Ascend 950 MX quant"
14 },14 },
15 "x_scale": {15 "x_scale": {
16- "description": "per-token input dequantization scale, dtype float32"16+ "description": "input dequantization scale; float32 for int8 cases; int8 bit-pattern of float8_e8m0fnu for Ascend 950 MX quant"
17 },17 },
18 "group_list": {18 "group_list": {
19 "description": "int64 tensor of group boundaries (cumsum or count per group_list_type)"19 "description": "int64 tensor of group boundaries (cumsum or count per group_list_type)"
@@ -28,7 +28,7 @@
28 "description": "optional bias term, dtype int32"28 "description": "optional bias term, dtype int32"
29 },29 },
30 "dequant_mode": {30 "dequant_mode": {
31- "description": "int, default=0; optional"31+ "description": "int, default=0; Ascend 950 MX quant: 2"
32 },32 },
33 "dequant_dtype": {33 "dequant_dtype": {
34 "description": "int, default=0; optional"34 "description": "int, default=0; optional"
@@ -37,23 +37,32 @@
37 "description": "int, default=0; optional"37 "description": "int, default=0; optional"
38 },38 },
39 "quant_dtype": {39 "quant_dtype": {
40- "description": "int, default=0; optional"40+ "description": "int or dtype string; Ascend 950 MX quant: 'float8_e4m3fn'"
41 },41 },
42 "group_list_type": {42 "group_list_type": {
43 "description": "int, default=0; optional"43 "description": "int, default=0; optional"
44 },44 },
45 "tuning_config": {45 "tuning_config": {
46 "description": "optional list of int for kernel tuning configuration"46 "description": "optional list of int for kernel tuning configuration"
47+ },
48+ "x_dtype": {
49+ "description": "optional; dtype string for mx-quant x, e.g. 'float4_e2m1fn_x2' (Ascend 950)"
50+ },
51+ "weight_dtype": {
52+ "description": "optional; dtype string for mx-quant weight, e.g. 'float4_e2m1fn_x2' (Ascend 950)"
53+ },
54+ "weight_scale_dtype": {
55+ "description": "optional; dtype string for weight scale, 'float8_e8m0fnu' for Ascend 950 MX quant"
56+ },
57+ "x_scale_dtype": {
58+ "description": "optional; dtype string for x scale, 'float8_e8m0fnu' for Ascend 950 MX quant"
47 }59 }
48 },60 },
49 "outputs": {61 "outputs": {
50 "0": {62 "0": {
51- "description": "Quantized output"63+ "description": "Quantized output (int8 for legacy A8W8 cases; float8 for Ascend 950 MX quant cases)"
52- },
53- "1": {
54- "description": "Output quantization scale"
55 }64 }
56 },65 },
57- "reference": "import torch\nimport torch_npu\n\ntorch.npu.config.allow_internal_format = True\n\ndef _cast_list_to_nz(tensor_list):\n if tensor_list is None:\n return None\n result = []\n for t in tensor_list:\n if not isinstance(t, torch.Tensor):\n result.append(t)\n continue\n if t.device.type != 'npu':\n t = t.npu()\n fmt = torch_npu.get_npu_format(t)\n if fmt != 29:\n t = torch_npu.npu_format_cast(t, 29)\n result.append(t)\n return result\n\ndef run(x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, x_scale: torch.Tensor, group_list: torch.Tensor, smooth_scale=None, weight_assist_matrix=None, bias=None, dequant_mode=0, dequant_dtype=0, quant_mode=0, quant_dtype=0, group_list_type=0, tuning_config=None):\n if group_list is None:\n group_list = torch.tensor([x.shape[0]], dtype=torch.int64, device=x.device)\n if isinstance(group_list, list):\n group_list = torch.tensor(group_list, dtype=torch.int64, device=x.device)\n w_list = [weight]\n ws_list = [weight_scale]\n w_list = _cast_list_to_nz(w_list)\n if weight_assist_matrix is not None:\n weight_assist_matrix = _cast_list_to_nz(weight_assist_matrix)\n quant_out, quant_scale = torch_npu.npu_grouped_matmul_swiglu_quant_v2(\n x, w_list, ws_list, x_scale, group_list,\n smooth_scale=smooth_scale,\n weight_assist_matrix=weight_assist_matrix,\n bias=bias,\n dequant_mode=dequant_mode,\n dequant_dtype=dequant_dtype,\n quant_mode=quant_mode,\n quant_dtype=quant_dtype,\n group_list_type=group_list_type,\n tuning_config=tuning_config\n )\n return quant_out, quant_scale\n",66+ "reference": "import torch\nimport torch_npu\n\ntorch.npu.config.allow_internal_format = True\n\n_TORCH_NPU_ONLY_DTYPES = {\"float8_e8m0fnu\", \"hifloat8\", \"float4_e2m1fn_x2\"}\n\ndef _resolve_dtype(v):\n if isinstance(v, str):\n if v in _TORCH_NPU_ONLY_DTYPES and hasattr(torch_npu, v):\n return getattr(torch_npu, v)\n if hasattr(torch, v):\n return getattr(torch, v)\n if hasattr(torch_npu, v):\n return getattr(torch_npu, v)\n raise ValueError(f\"unknown dtype specifier: {v}\")\n return v\n\ndef _soc_is_950():\n try:\n return \"950\" in torch.npu.get_device_name(torch.npu.current_device())\n except Exception as e:\n print(f\"[warn] _soc_is_950 detection failed, fallback to NZ path: {e}\")\n return False\n\ndef _cast_list_to_nz(tensor_list):\n if tensor_list is None or _soc_is_950():\n return tensor_list\n result = []\n for t in tensor_list:\n if not isinstance(t, torch.Tensor):\n result.append(t)\n continue\n if t.device.type != 'npu':\n t = t.npu()\n fmt = torch_npu.get_npu_format(t)\n if fmt != 29:\n t = torch_npu.npu_format_cast(t, 29)\n result.append(t)\n return result\n\ndef run(x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, x_scale: torch.Tensor, group_list: torch.Tensor, smooth_scale=None, weight_assist_matrix=None, bias=None, dequant_mode=0, dequant_dtype=0, quant_mode=0, quant_dtype=0, group_list_type=0, tuning_config=None, x_dtype=None, weight_dtype=None, weight_scale_dtype=None, x_scale_dtype=None):\n if group_list is None:\n group_list = torch.tensor([x.shape[0]], dtype=torch.int64, device=x.device)\n if isinstance(group_list, list):\n group_list = torch.tensor(group_list, dtype=torch.int64, device=x.device)\n w_list = [weight]\n ws_list = [weight_scale]\n if dequant_mode != 2:\n w_list = _cast_list_to_nz(w_list)\n if weight_assist_matrix is not None:\n weight_assist_matrix = _cast_list_to_nz(weight_assist_matrix)\n quant_out, quant_scale = torch_npu.npu_grouped_matmul_swiglu_quant_v2(\n x, w_list, ws_list, x_scale, group_list,\n smooth_scale=smooth_scale,\n weight_assist_matrix=weight_assist_matrix,\n bias=bias,\n dequant_mode=dequant_mode,\n dequant_dtype=_resolve_dtype(dequant_dtype),\n quant_mode=quant_mode,\n quant_dtype=_resolve_dtype(quant_dtype),\n group_list_type=group_list_type,\n tuning_config=tuning_config,\n x_dtype=_resolve_dtype(x_dtype),\n weight_dtype=_resolve_dtype(weight_dtype),\n weight_scale_dtype=_resolve_dtype(weight_scale_dtype),\n x_scale_dtype=_resolve_dtype(x_scale_dtype)\n )\n return quant_out, quant_scale\n",
58 "description": "Model that performs Grouped Matmul + SwiGLU + Quant computation using NPU accelerated npu_grouped_matmul_swiglu_quant_v2.\nFuses grouped matrix multiplication, SwiGLU activation, and quantization.\n\nPure PyTorch reference implementation (decomposing the fused operator into basic ops):\nimport torch\n\ndef run(x, weight, weight_scale, x_scale, group_list, smooth_scale=None, weight_assist_matrix=None, bias=None, dequant_mode=0, dequant_dtype=0, quant_mode=0, quant_dtype=0, group_list_type=0, tuning_config=None):\n if dequant_mode is None: dequant_mode = 0\n if dequant_dtype is None: dequant_dtype = 0\n if quant_mode is None: quant_mode = 0\n if quant_dtype is None: quant_dtype = 0\n if group_list_type is None: group_list_type = 0\n if not isinstance(group_list, torch.Tensor):\n group_list = torch.tensor(group_list, dtype=torch.int64, device=x.device)\n m, k = x.shape\n e = len(weight)\n\n if group_list_type == 0:\n cumsum = torch.cat([torch.zeros(1, dtype=group_list.dtype, device=x.device), group_list])\n token_counts = cumsum[1:] - cumsum[:-1]\n else:\n token_counts = group_list\n cumsum = torch.zeros(e + 1, dtype=torch.int64, device=x.device)\n cumsum[1:] = torch.cumsum(token_counts, dim=0)\n\n outputs = []\n for g in range(e):\n start = cumsum[g].item()\n end = cumsum[g + 1].item()\n\n x_g = x[start:end, :]\n x_s_g = x_scale[start:end]\n\n x_deq = x_g.to(torch.float32) * x_s_g.unsqueeze(1)\n\n W_g = weight[g]\n W_s_g = weight_scale[g]\n\n if dequant_mode == 0:\n W_deq = W_g.to(torch.float32) * W_s_g.to(torch.float32)\n elif dequant_mode == 1 and W_s_g.dim() == 2:\n W_deq = W_g.to(torch.float32) * W_s_g.to(torch.float32)\n elif dequant_mode == 1 and W_s_g.dim() == 3:\n k_group = W_s_g.shape[0]\n repeats = W_g.shape[0] // k_group\n W_s_exp = W_s_g.to(torch.float32).repeat_interleave(repeats, dim=0)\n W_deq = W_g.to(torch.float32) * W_s_exp\n else:\n W_deq = W_g.to(torch.float32) * W_s_g.to(torch.float32)\n\n hidden = x_deq @ W_deq\n\n if bias is not None:\n hidden = hidden + bias[g].to(torch.float32)\n\n if weight_assist_matrix is not None and weight_assist_matrix[g] is not None:\n hidden = hidden + weight_assist_matrix[g].to(torch.float32)\n\n n = hidden.shape[1]\n half = n // 2\n left = hidden[:, :half]\n right = hidden[:, half:]\n swiglu_out = torch.nn.functional.silu(left) * right\n\n if smooth_scale is not None:\n smooth_g = smooth_scale[g].to(torch.float32)\n swiglu_out = swiglu_out * smooth_g\n\n abs_max = swiglu_out.abs().max(dim=1, keepdim=True).values\n safe_max = abs_max.clamp(min=1e-12)\n out_scale_g = safe_max / 127.0\n q_out = (swiglu_out / out_scale_g).round().clamp(-128, 127).to(torch.int8)\n out_scale_g = (abs_max / 127.0).squeeze(1)\n\n outputs.append((q_out, out_scale_g))\n\n output = torch.cat([o[0] for o in outputs], dim=0)\n output_scale = torch.cat([o[1] for o in outputs], dim=0)\n\n return output, output_scale\n"67 "description": "Model that performs Grouped Matmul + SwiGLU + Quant computation using NPU accelerated npu_grouped_matmul_swiglu_quant_v2.\nFuses grouped matrix multiplication, SwiGLU activation, and quantization.\n\nPure PyTorch reference implementation (decomposing the fused operator into basic ops):\nimport torch\n\ndef run(x, weight, weight_scale, x_scale, group_list, smooth_scale=None, weight_assist_matrix=None, bias=None, dequant_mode=0, dequant_dtype=0, quant_mode=0, quant_dtype=0, group_list_type=0, tuning_config=None):\n if dequant_mode is None: dequant_mode = 0\n if dequant_dtype is None: dequant_dtype = 0\n if quant_mode is None: quant_mode = 0\n if quant_dtype is None: quant_dtype = 0\n if group_list_type is None: group_list_type = 0\n if not isinstance(group_list, torch.Tensor):\n group_list = torch.tensor(group_list, dtype=torch.int64, device=x.device)\n m, k = x.shape\n e = len(weight)\n\n if group_list_type == 0:\n cumsum = torch.cat([torch.zeros(1, dtype=group_list.dtype, device=x.device), group_list])\n token_counts = cumsum[1:] - cumsum[:-1]\n else:\n token_counts = group_list\n cumsum = torch.zeros(e + 1, dtype=torch.int64, device=x.device)\n cumsum[1:] = torch.cumsum(token_counts, dim=0)\n\n outputs = []\n for g in range(e):\n start = cumsum[g].item()\n end = cumsum[g + 1].item()\n\n x_g = x[start:end, :]\n x_s_g = x_scale[start:end]\n\n x_deq = x_g.to(torch.float32) * x_s_g.unsqueeze(1)\n\n W_g = weight[g]\n W_s_g = weight_scale[g]\n\n if dequant_mode == 0:\n W_deq = W_g.to(torch.float32) * W_s_g.to(torch.float32)\n elif dequant_mode == 1 and W_s_g.dim() == 2:\n W_deq = W_g.to(torch.float32) * W_s_g.to(torch.float32)\n elif dequant_mode == 1 and W_s_g.dim() == 3:\n k_group = W_s_g.shape[0]\n repeats = W_g.shape[0] // k_group\n W_s_exp = W_s_g.to(torch.float32).repeat_interleave(repeats, dim=0)\n W_deq = W_g.to(torch.float32) * W_s_exp\n else:\n W_deq = W_g.to(torch.float32) * W_s_g.to(torch.float32)\n\n hidden = x_deq @ W_deq\n\n if bias is not None:\n hidden = hidden + bias[g].to(torch.float32)\n\n if weight_assist_matrix is not None and weight_assist_matrix[g] is not None:\n hidden = hidden + weight_assist_matrix[g].to(torch.float32)\n\n n = hidden.shape[1]\n half = n // 2\n left = hidden[:, :half]\n right = hidden[:, half:]\n swiglu_out = torch.nn.functional.silu(left) * right\n\n if smooth_scale is not None:\n smooth_g = smooth_scale[g].to(torch.float32)\n swiglu_out = swiglu_out * smooth_g\n\n abs_max = swiglu_out.abs().max(dim=1, keepdim=True).values\n safe_max = abs_max.clamp(min=1e-12)\n out_scale_g = safe_max / 127.0\n q_out = (swiglu_out / out_scale_g).round().clamp(-128, 127).to(torch.int8)\n out_scale_g = (abs_max / 127.0).squeeze(1)\n\n outputs.append((q_out, out_scale_g))\n\n output = torch.cat([o[0] for o in outputs], dim=0)\n output_scale = torch.cat([o[1] for o in outputs], dim=0)\n\n return output, output_scale\n"
59-}68+}
Mnpu-kernelbench/data/kernel_generator/level4/GroupedMatmulSwigluQuantV2/workload.jsonl+2-0
@@ -48,3 +48,5 @@
48{"uuid": "GroupedMatmulSwigluQuantV2-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [384, 2560], "dtype": "int8", "range": [-128, 127]}, "weight": {"type": "random", "shape": [8, 2560, 2560], "dtype": "int8", "range": [-128, 127]}, "weight_scale": {"type": "random", "shape": [8, 2560], "dtype": "float32"}, "x_scale": {"type": "random", "shape": [384], "dtype": "float32"}, "group_list": {"type": "scalar", "value": [48, 48, 48, 48, 48, 48, 48, 48]}, "dequant_mode": {"type": "scalar", "value": 1}, "quant_mode": {"type": "scalar", "value": 1}, "quant_dtype": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 1}}}48{"uuid": "GroupedMatmulSwigluQuantV2-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [384, 2560], "dtype": "int8", "range": [-128, 127]}, "weight": {"type": "random", "shape": [8, 2560, 2560], "dtype": "int8", "range": [-128, 127]}, "weight_scale": {"type": "random", "shape": [8, 2560], "dtype": "float32"}, "x_scale": {"type": "random", "shape": [384], "dtype": "float32"}, "group_list": {"type": "scalar", "value": [48, 48, 48, 48, 48, 48, 48, 48]}, "dequant_mode": {"type": "scalar", "value": 1}, "quant_mode": {"type": "scalar", "value": 1}, "quant_dtype": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 1}}}
49{"uuid": "GroupedMatmulSwigluQuantV2-049", "axes": null, "inputs": {"x": {"type": "random", "shape": [512, 256], "dtype": "int8", "range": [-128, 127]}, "weight": {"type": "random", "shape": [2, 256, 256], "dtype": "int8", "range": [-128, 127]}, "weight_scale": {"type": "random", "shape": [2, 256], "dtype": "float32"}, "x_scale": {"type": "random", "shape": [512], "dtype": "float32"}, "group_list": {"type": "scalar", "value": [256, 512]}, "dequant_mode": {"type": "scalar", "value": 0}, "quant_mode": {"type": "scalar", "value": 0}, "quant_dtype": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 0}}}49{"uuid": "GroupedMatmulSwigluQuantV2-049", "axes": null, "inputs": {"x": {"type": "random", "shape": [512, 256], "dtype": "int8", "range": [-128, 127]}, "weight": {"type": "random", "shape": [2, 256, 256], "dtype": "int8", "range": [-128, 127]}, "weight_scale": {"type": "random", "shape": [2, 256], "dtype": "float32"}, "x_scale": {"type": "random", "shape": [512], "dtype": "float32"}, "group_list": {"type": "scalar", "value": [256, 512]}, "dequant_mode": {"type": "scalar", "value": 0}, "quant_mode": {"type": "scalar", "value": 0}, "quant_dtype": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 0}}}
50{"uuid": "GroupedMatmulSwigluQuantV2-050", "axes": null, "inputs": {"x": {"type": "random", "shape": [64, 384], "dtype": "int8", "range": [-128, 127]}, "weight": {"type": "random", "shape": [4, 384, 384], "dtype": "int8", "range": [-128, 127]}, "weight_scale": {"type": "random", "shape": [4, 384], "dtype": "float32"}, "x_scale": {"type": "random", "shape": [64], "dtype": "float32"}, "group_list": {"type": "scalar", "value": [16, 16, 16, 16]}, "dequant_mode": {"type": "scalar", "value": 1}, "quant_mode": {"type": "scalar", "value": 1}, "quant_dtype": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 1}}}50{"uuid": "GroupedMatmulSwigluQuantV2-050", "axes": null, "inputs": {"x": {"type": "random", "shape": [64, 384], "dtype": "int8", "range": [-128, 127]}, "weight": {"type": "random", "shape": [4, 384, 384], "dtype": "int8", "range": [-128, 127]}, "weight_scale": {"type": "random", "shape": [4, 384], "dtype": "float32"}, "x_scale": {"type": "random", "shape": [64], "dtype": "float32"}, "group_list": {"type": "scalar", "value": [16, 16, 16, 16]}, "dequant_mode": {"type": "scalar", "value": 1}, "quant_mode": {"type": "scalar", "value": 1}, "quant_dtype": {"type": "scalar", "value": 1}, "group_list_type": {"type": "scalar", "value": 1}}}
51+{"uuid": "GroupedMatmulSwigluQuantV2-051", "axes": null, "inputs": {"x": {"type": "random", "shape": [64, 32], "dtype": "float8_e4m3fn"}, "weight": {"type": "random", "shape": [2, 32, 128], "dtype": "float8_e4m3fn"}, "weight_scale": {"type": "random", "shape": [2, 1, 128, 2], "dtype": "int8", "range": [120, 128]}, "x_scale": {"type": "random", "shape": [64, 1, 2], "dtype": "int8", "range": [120, 128]}, "group_list": {"type": "scalar", "value": [32, 32]}, "group_list_type": {"type": "scalar", "value": 1}, "dequant_mode": {"type": "scalar", "value": 2}, "dequant_dtype": {"type": "scalar", "value": "float32"}, "quant_mode": {"type": "scalar", "value": 2}, "quant_dtype": {"type": "scalar", "value": "float8_e4m3fn"}, "weight_scale_dtype": {"type": "scalar", "value": "float8_e8m0fnu"}, "x_scale_dtype": {"type": "scalar", "value": "float8_e8m0fnu"}}}
52+{"uuid": "GroupedMatmulSwigluQuantV2-052", "axes": null, "inputs": {"x": {"type": "random", "shape": [128, 32], "dtype": "float8_e4m3fn"}, "weight": {"type": "random", "shape": [1, 32, 128], "dtype": "float8_e4m3fn"}, "weight_scale": {"type": "random", "shape": [1, 1, 128, 2], "dtype": "int8", "range": [120, 128]}, "x_scale": {"type": "random", "shape": [128, 1, 2], "dtype": "int8", "range": [120, 128]}, "group_list": {"type": "scalar", "value": [128]}, "group_list_type": {"type": "scalar", "value": 0}, "dequant_mode": {"type": "scalar", "value": 2}, "dequant_dtype": {"type": "scalar", "value": "float32"}, "quant_mode": {"type": "scalar", "value": 2}, "quant_dtype": {"type": "scalar", "value": "float8_e4m3fn"}, "weight_scale_dtype": {"type": "scalar", "value": "float8_e8m0fnu"}, "x_scale_dtype": {"type": "scalar", "value": "float8_e8m0fnu"}}}
Mnpu-kernelbench/data/kernel_generator/level4/QuantMatmul/definition.json+1-1
@@ -33,6 +33,6 @@
33 "description": "Quantized matmul output"33 "description": "Quantized matmul output"
34 }34 }
35 },35 },
36- "reference": "import torch\nimport torch.nn as nn\n\ndef run(x1, x2, scale, offset=None, pertoken_scale=None, bias=None, output_dtype=None, group_sizes=None):\n import torch_npu\n _dtype_map = {'float16': torch.float16, 'bfloat16': torch.bfloat16, 'int32': torch.int32, 'int8': torch.int8}\n if isinstance(output_dtype, str):\n output_dtype = _dtype_map.get(output_dtype, output_dtype)\n return torch_npu.npu_quant_matmul(x1, x2, scale, offset=offset, pertoken_scale=pertoken_scale, bias=bias, output_dtype=output_dtype, group_sizes=group_sizes)\n",36+ "reference": "import torch\nimport torch.nn as nn\n\ndef run(x1, x2, scale, offset=None, pertoken_scale=None, bias=None, output_dtype=None, group_sizes=None):\n import torch_npu\n _dtype_map = {'float16': torch.float16, 'bfloat16': torch.bfloat16, 'int32': torch.int32, 'int8': torch.int8}\n if isinstance(output_dtype, str):\n output_dtype = _dtype_map.get(output_dtype, output_dtype)\n if isinstance(scale, torch.Tensor) and scale.dtype == torch.float32 and x1.dtype == torch.int32:\n scale = torch_npu.npu_trans_quant_param(scale)\n return torch_npu.npu_quant_matmul(x1, x2, scale, offset=offset, pertoken_scale=pertoken_scale, bias=bias, output_dtype=output_dtype, group_sizes=group_sizes)\n",
37 "description": "Model that performs quantized matrix multiplication using NPU accelerated npu_quant_matmul.\nSupports int8 quantized matmul with various output data types.\n\ntorch_npu.npu_quant_matmul(x1, x2, scale, *, offset=None, pertoken_scale=None,\n bias=None, output_dtype=None, group_sizes=None) -> Tensor\n\nPyTorch native reference implementation (for correctness analysis only):\nimport torch\n\ndef _crop_float32_to_19bit(scale):\n scale_f32 = scale.to(torch.float32).contiguous()\n bits = scale_f32.view(torch.int32)\n mask = ~((1 << 4) - 1)\n bits = bits & mask\n return bits.view(torch.float32).to(scale.dtype)\n\n_DTYPE_MAP = {\"bfloat16\": torch.bfloat16, \"float16\": torch.float16, \"float32\": torch.float32, \"int32\": torch.int32, \"int8\": torch.int8}\n\ndef run(x1, x2, scale, offset=None, pertoken_scale=None, bias=None, output_dtype=None, group_sizes=None):\n if isinstance(output_dtype, str):\n output_dtype = _DTYPE_MAP.get(output_dtype, getattr(torch, output_dtype, None))\n x1_fp = x1.to(torch.float32)\n x2_fp = x2.to(torch.float32)\n matmul_result = torch.matmul(x1_fp, x2_fp)\n\n if (scale.dtype == torch.float32 and pertoken_scale is None and\n output_dtype not in (torch.bfloat16, torch.int32)):\n scale_fp = _crop_float32_to_19bit(scale)\n else:\n scale_fp = scale.to(torch.float32)\n\n if bias is not None and bias.dtype == torch.int32:\n matmul_result = matmul_result + bias.to(torch.float32)\n\n if pertoken_scale is not None:\n pertoken_scale_fp = pertoken_scale.to(torch.float32)\n if pertoken_scale_fp.dim() == 1:\n pertoken_scale_fp = pertoken_scale_fp.view(-1, 1)\n matmul_result_fp = matmul_result * pertoken_scale_fp * scale_fp\n else:\n matmul_result_fp = matmul_result * scale_fp\n\n if bias is not None and bias.dtype != torch.int32:\n matmul_result_fp = matmul_result_fp + bias.to(torch.float32)\n\n if offset is not None:\n matmul_result_fp = matmul_result_fp + offset.to(torch.float32)\n\n if output_dtype is not None:\n matmul_result_fp = matmul_result_fp.to(output_dtype)\n\n if output_dtype is not None and torch.is_floating_point(matmul_result_fp):\n info = torch.finfo(output_dtype)\n matmul_result_fp = torch.clamp(matmul_result_fp, info.min, info.max)\n\n return matmul_result_fp\n"37 "description": "Model that performs quantized matrix multiplication using NPU accelerated npu_quant_matmul.\nSupports int8 quantized matmul with various output data types.\n\ntorch_npu.npu_quant_matmul(x1, x2, scale, *, offset=None, pertoken_scale=None,\n bias=None, output_dtype=None, group_sizes=None) -> Tensor\n\nPyTorch native reference implementation (for correctness analysis only):\nimport torch\n\ndef _crop_float32_to_19bit(scale):\n scale_f32 = scale.to(torch.float32).contiguous()\n bits = scale_f32.view(torch.int32)\n mask = ~((1 << 4) - 1)\n bits = bits & mask\n return bits.view(torch.float32).to(scale.dtype)\n\n_DTYPE_MAP = {\"bfloat16\": torch.bfloat16, \"float16\": torch.float16, \"float32\": torch.float32, \"int32\": torch.int32, \"int8\": torch.int8}\n\ndef run(x1, x2, scale, offset=None, pertoken_scale=None, bias=None, output_dtype=None, group_sizes=None):\n if isinstance(output_dtype, str):\n output_dtype = _DTYPE_MAP.get(output_dtype, getattr(torch, output_dtype, None))\n x1_fp = x1.to(torch.float32)\n x2_fp = x2.to(torch.float32)\n matmul_result = torch.matmul(x1_fp, x2_fp)\n\n if (scale.dtype == torch.float32 and pertoken_scale is None and\n output_dtype not in (torch.bfloat16, torch.int32)):\n scale_fp = _crop_float32_to_19bit(scale)\n else:\n scale_fp = scale.to(torch.float32)\n\n if bias is not None and bias.dtype == torch.int32:\n matmul_result = matmul_result + bias.to(torch.float32)\n\n if pertoken_scale is not None:\n pertoken_scale_fp = pertoken_scale.to(torch.float32)\n if pertoken_scale_fp.dim() == 1:\n pertoken_scale_fp = pertoken_scale_fp.view(-1, 1)\n matmul_result_fp = matmul_result * pertoken_scale_fp * scale_fp\n else:\n matmul_result_fp = matmul_result * scale_fp\n\n if bias is not None and bias.dtype != torch.int32:\n matmul_result_fp = matmul_result_fp + bias.to(torch.float32)\n\n if offset is not None:\n matmul_result_fp = matmul_result_fp + offset.to(torch.float32)\n\n if output_dtype is not None:\n matmul_result_fp = matmul_result_fp.to(output_dtype)\n\n if output_dtype is not None and torch.is_floating_point(matmul_result_fp):\n info = torch.finfo(output_dtype)\n matmul_result_fp = torch.clamp(matmul_result_fp, info.min, info.max)\n\n return matmul_result_fp\n"
38}38}
Mnpu-kernelbench/data/kernel_generator/level4/QuantMatmul/workload.jsonl+2-0
@@ -55,3 +55,5 @@
55{"uuid": "QuantMatmul-055", "axes": null, "inputs": {"x1": {"type": "random", "shape": [128, 512], "dtype": "int8", "range": [-5, 5]}, "x2": {"type": "random", "shape": [512, 2048], "dtype": "int8", "range": [-5, 5]}, "scale": {"type": "random", "shape": [2048], "dtype": "float32"}, "output_dtype": {"type": "scalar", "value": "float16"}}}55{"uuid": "QuantMatmul-055", "axes": null, "inputs": {"x1": {"type": "random", "shape": [128, 512], "dtype": "int8", "range": [-5, 5]}, "x2": {"type": "random", "shape": [512, 2048], "dtype": "int8", "range": [-5, 5]}, "scale": {"type": "random", "shape": [2048], "dtype": "float32"}, "output_dtype": {"type": "scalar", "value": "float16"}}}
56{"uuid": "QuantMatmul-056", "axes": null, "inputs": {"x1": {"type": "random", "shape": [256, 1024], "dtype": "int8", "range": [-5, 5]}, "x2": {"type": "random", "shape": [1024, 512], "dtype": "int8", "range": [-5, 5]}, "scale": {"type": "random", "shape": [512], "dtype": "bfloat16"}, "output_dtype": {"type": "scalar", "value": "bfloat16"}}}56{"uuid": "QuantMatmul-056", "axes": null, "inputs": {"x1": {"type": "random", "shape": [256, 1024], "dtype": "int8", "range": [-5, 5]}, "x2": {"type": "random", "shape": [1024, 512], "dtype": "int8", "range": [-5, 5]}, "scale": {"type": "random", "shape": [512], "dtype": "bfloat16"}, "output_dtype": {"type": "scalar", "value": "bfloat16"}}}
57{"uuid": "QuantMatmul-057", "axes": null, "inputs": {"x1": {"type": "random", "shape": [256, 1024], "dtype": "int8", "range": [-5, 5]}, "x2": {"type": "random", "shape": [1024, 1024], "dtype": "int8", "range": [-5, 5]}, "scale": {"type": "random", "shape": [1024], "dtype": "float32"}, "output_dtype": {"type": "scalar", "value": "float16"}}}57{"uuid": "QuantMatmul-057", "axes": null, "inputs": {"x1": {"type": "random", "shape": [256, 1024], "dtype": "int8", "range": [-5, 5]}, "x2": {"type": "random", "shape": [1024, 1024], "dtype": "int8", "range": [-5, 5]}, "scale": {"type": "random", "shape": [1024], "dtype": "float32"}, "output_dtype": {"type": "scalar", "value": "float16"}}}
58+{"uuid": "QuantMatmul-058", "axes": null, "inputs": {"x1": {"type": "random", "shape": [256, 1024], "dtype": "float8_e4m3fn"}, "x2": {"type": "random", "shape": [1024, 1024], "dtype": "float8_e4m3fn"}, "scale": {"type": "random", "shape": [1024], "dtype": "float32"}, "output_dtype": {"type": "scalar", "value": "float16"}}}
59+{"uuid": "QuantMatmul-059", "axes": null, "inputs": {"x1": {"type": "random", "shape": [256, 1024], "dtype": "float8_e5m2"}, "x2": {"type": "random", "shape": [1024, 1024], "dtype": "float8_e5m2"}, "scale": {"type": "random", "shape": [1024], "dtype": "float32"}, "output_dtype": {"type": "scalar", "value": "float16"}}}
Mnpu-kernelbench/data/kernel_generator/level4/WeightQuantBatchmatmul/definition.json+2-2
@@ -7,7 +7,7 @@
7 "description": "left matrix (M, K), dtype float16/bfloat16"7 "description": "left matrix (M, K), dtype float16/bfloat16"
8 },8 },
9 "weight": {9 "weight": {
10- "description": "right weight matrix (K, N), dtype int8"10+ "description": "weight matrix (K, N), dtype int8; or int32 where each element holds ONE int4 value (-8~7) — logical shape equals the int32 tensor shape [K, N] (converted to hardware-packed layout via npu_convert_weight_to_int4pack inside reference); antiquant_scale/offset shape must match logical [1, N]"
11 },11 },
12 "antiquant_scale": {12 "antiquant_scale": {
13 "description": "shape varies; supports float16; defaults to ones([1]) when absent"13 "description": "shape varies; supports float16; defaults to ones([1]) when absent"
@@ -36,6 +36,6 @@
36 "description": "Weight-quantized batch matmul output"36 "description": "Weight-quantized batch matmul output"
37 }37 }
38 },38 },
39- "reference": "import torch\nimport torch.nn as nn\n\ndef run(x: torch.Tensor, weight: torch.Tensor, antiquant_scale: torch.Tensor, antiquant_offset=None, quant_scale=None, quant_offset=None, bias=None, antiquant_group_size=0, inner_precise=0):\n \"\"\"\n Performs weight-quantized batch matmul on NPU.\n Args:\n x (Tensor): Left matrix, shape [M, K], dtype float16/bfloat16.\n weight (Tensor): Right matrix (weight), shape [K, N], dtype int8.\n antiquant_scale (Tensor): Dequantization scale for weight, dtype float16/bfloat16.\n antiquant_offset (Tensor, optional): Dequantization offset for weight, dtype float16/bfloat16.\n quant_scale (Tensor, optional): Output quantization scale, dtype float32.\n quant_offset (Tensor, optional): Output quantization offset, dtype float32.\n bias (Tensor, optional): Bias term, shape [1, N] or [N], dtype float16/float32.\n antiquant_group_size (int): Group size for pergroup quantization, default 0.\n inner_precise (int): 0=high precision, 1=high performance, default 0.\n Returns:\n Tensor: Output tensor, same dtype as x.\n \"\"\"\n import torch_npu\n if antiquant_group_size is None:\n antiquant_group_size = 0\n if inner_precise is None:\n inner_precise = 0\n return torch_npu.npu_weight_quant_batchmatmul(x, weight, antiquant_scale, antiquant_offset, quant_scale, quant_offset, bias, antiquant_group_size, inner_precise)\n",39+ "reference": "import torch\nimport torch.nn as nn\n\ndef run(x: torch.Tensor, weight: torch.Tensor, antiquant_scale: torch.Tensor, antiquant_offset=None, quant_scale=None, quant_offset=None, bias=None, antiquant_group_size=0, inner_precise=0):\n \"\"\"\n Performs weight-quantized batch matmul on NPU.\n Args:\n x (Tensor): Left matrix, shape [M, K], dtype float16/bfloat16.\n weight (Tensor): Right matrix (weight), shape [K, N], dtype int8.\n antiquant_scale (Tensor): Dequantization scale for weight, dtype float16/bfloat16.\n antiquant_offset (Tensor, optional): Dequantization offset for weight, dtype float16/bfloat16.\n quant_scale (Tensor, optional): Output quantization scale, dtype float32.\n quant_offset (Tensor, optional): Output quantization offset, dtype float32.\n bias (Tensor, optional): Bias term, shape [1, N] or [N], dtype float16/float32.\n antiquant_group_size (int): Group size for pergroup quantization, default 0.\n inner_precise (int): 0=high precision, 1=high performance, default 0.\n Returns:\n Tensor: Output tensor, same dtype as x.\n \"\"\"\n import torch_npu\n if antiquant_group_size is None:\n antiquant_group_size = 0\n if inner_precise is None:\n inner_precise = 0\n if weight.dtype == torch.int32:\n weight = torch_npu.npu_convert_weight_to_int4pack(weight)\n return torch_npu.npu_weight_quant_batchmatmul(x, weight, antiquant_scale, antiquant_offset, quant_scale, quant_offset, bias, antiquant_group_size, inner_precise)\n",
40 "description": "Model that performs weight-quantized batch matrix multiplication using NPU accelerated npu_weight_quant_batchmatmul.\nSupports pertensor, perchannel, and pergroup quantization for weight matriceu.npu_weight_quant_batchmatmul(x, weight, antiquant_scale, antiquant_offset, quant_scale, quant_offset, bias, antiquant_group_size, inner_precise) -> Tensor\n\nCPU 标杆实现（注释内）—— 来源: analysis/正确性_V2_CPU_修复/v2_golden_cpu.py 的 normal() 路径\n----------------------------------------------------------------------------------------\n设计要点（与朴素 PyTorch 实现的关键差异，是对齐 NPU cube 行为的修复点）：\n\n1. dequant 域按 x.dtype 分流（V3 修16 输入：在 fp16 域做 (weight+offset)*scale，最后再 fp16→fp32（贴合 V2 仓上原行为）\n - bf16 输入：在 fp32 域做 (weight+offset)*scale，再 cast 到 bf16（模拟 bf16 表示精度）→ 回 fp32\n （对应 NPU cube L0C 行为，避免 bf16 加法尾数损失）\n\n2. matmul 用 fp32 split-K 累加，BLOCK_K=16（模拟 NPU cube 单次累加深度），不是一次性 torch.matmul。\n 循环内每个 partial 都是 fp32，最后输出仍是 fp32。\n\n3. bias / quant_sca quant_offset 可选；quant 路径有 clamp 到 [-256,256] / [-128,127] 的边界处理。\n\n4. 标杆假设输入 tensor 可在任意 device，内部统一 .cpu() 计算（CPU 标杆本质）。\n\nPyTorch native implementation of forward function\nimport torch\n\ndef run(x, weight, antiquant_scale, antiquant_offset=None, quant_scale=None, quant_offset=None, bias=None, antiquant_group_size=0, inner_precise=0):\n if antiquant_group_size is None: antiquant_group_size = 0\n if inner_precise is None: ise = 0\n low_dtype = x.dtype\n weight_t = weight\n x_t = x\n scale_t = antiquant_scale\n off_t = antiquant_offset if (antiquant_offset is not None and antiquant_offset.numel() > 0) else None\n\n if low_dtype == torch.bfloat16:\n weight_f32 = weight_t.to(torch.float32)\n scale_f32 = scale_t.to(torch.float32)\n off_f32 = off_t.to(torch.float32) if off_t is not None else None\n if antiquant_group_size:\n num_groups = scale_f32.shape[0]\n gs = antiquant_group_size\n kSize = x_t.shape[-1]\n scale_exp = scale_f32.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, scale_f32.shape[-1])[:kSize, :]\n if off_f32 is not None:\n off_exp = off_f32.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, off_f32.shape[-1])[:kSize, :]\n weight_t = (weight_f32 + off_exp) * scale_exp\n else:\n weight_t = weight_f32 * scale_exp\n else:\n if off_f32 is not None:\n weight_t = (weight_f32 + off_f32) * scale_f32\n else:\n weight_t = weight_f32 * scale_f32\n weight_t = weight_t.to(torch.bfloat16).to(torch.float32)\n else:\n weight_t = weight_t.to(low_dtype)\n scale_t = scale_t.to(low_dtype)\n if off_t is not None:\n off_t = off_t.to(low_dtype)\n if antiquant_group_size:\n num_groups = scale_t.shape[0]\n gs = antiquant_group_size\n kSize = x_t.shape[-1]\n scale_exp = scale_t.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, scale_t.shape[-1])[:kSize, :]\n if off_t is not None:\n off_exp = off_t.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, off_t.shape[-1])[:kSize, :]\n weight_t = (weight_t + off_exp) * scale_exp\n else:\n weight_t = weight_t * scale_exp\n else:\n if off_t is not None:\n weight_t = (weight_t + off_t) * scale_t\n else:\n weight_t = weight_t * scale_t\n weight_t = weight_t.to(torch.float16).to(torch.float32)\n\n x_f32 = x_t.to(torch.float32)\n K = x_f32.shape[-1]\n BLOCK_K = 16\n output = None\n for k0 in range(0, K, BLOCK_K):\n k1 = min(k0 + BLOCK_K, K)\n partial = torch.matmul(x_f32[:, k0:k1], weight_t[k0:k1, :])\n output = partial if output is None else output + partial\n\n if bias is not None and bias.numel() != 0:\n output = output + bias.to(torch.float32)\n if quant_scale is not None and quant_scale.numel() != 0:\n output = torch.clamp(torch.round(output * quant_scale), -256, 256).to(torch.int16)\n if quant_offset is not None and quant_offset.numel() != 0:\n output = torch.clamp(output + torch.clamp(torch.round(quant_offset), -256, 256), -128, 127)\n\n return output.to(low_dtype).to(x.device)\n"40 "description": "Model that performs weight-quantized batch matrix multiplication using NPU accelerated npu_weight_quant_batchmatmul.\nSupports pertensor, perchannel, and pergroup quantization for weight matriceu.npu_weight_quant_batchmatmul(x, weight, antiquant_scale, antiquant_offset, quant_scale, quant_offset, bias, antiquant_group_size, inner_precise) -> Tensor\n\nCPU 标杆实现（注释内）—— 来源: analysis/正确性_V2_CPU_修复/v2_golden_cpu.py 的 normal() 路径\n----------------------------------------------------------------------------------------\n设计要点（与朴素 PyTorch 实现的关键差异，是对齐 NPU cube 行为的修复点）：\n\n1. dequant 域按 x.dtype 分流（V3 修16 输入：在 fp16 域做 (weight+offset)*scale，最后再 fp16→fp32（贴合 V2 仓上原行为）\n - bf16 输入：在 fp32 域做 (weight+offset)*scale，再 cast 到 bf16（模拟 bf16 表示精度）→ 回 fp32\n （对应 NPU cube L0C 行为，避免 bf16 加法尾数损失）\n\n2. matmul 用 fp32 split-K 累加，BLOCK_K=16（模拟 NPU cube 单次累加深度），不是一次性 torch.matmul。\n 循环内每个 partial 都是 fp32，最后输出仍是 fp32。\n\n3. bias / quant_sca quant_offset 可选；quant 路径有 clamp 到 [-256,256] / [-128,127] 的边界处理。\n\n4. 标杆假设输入 tensor 可在任意 device，内部统一 .cpu() 计算（CPU 标杆本质）。\n\nPyTorch native implementation of forward function\nimport torch\n\ndef run(x, weight, antiquant_scale, antiquant_offset=None, quant_scale=None, quant_offset=None, bias=None, antiquant_group_size=0, inner_precise=0):\n if antiquant_group_size is None: antiquant_group_size = 0\n if inner_precise is None: ise = 0\n low_dtype = x.dtype\n weight_t = weight\n x_t = x\n scale_t = antiquant_scale\n off_t = antiquant_offset if (antiquant_offset is not None and antiquant_offset.numel() > 0) else None\n\n if low_dtype == torch.bfloat16:\n weight_f32 = weight_t.to(torch.float32)\n scale_f32 = scale_t.to(torch.float32)\n off_f32 = off_t.to(torch.float32) if off_t is not None else None\n if antiquant_group_size:\n num_groups = scale_f32.shape[0]\n gs = antiquant_group_size\n kSize = x_t.shape[-1]\n scale_exp = scale_f32.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, scale_f32.shape[-1])[:kSize, :]\n if off_f32 is not None:\n off_exp = off_f32.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, off_f32.shape[-1])[:kSize, :]\n weight_t = (weight_f32 + off_exp) * scale_exp\n else:\n weight_t = weight_f32 * scale_exp\n else:\n if off_f32 is not None:\n weight_t = (weight_f32 + off_f32) * scale_f32\n else:\n weight_t = weight_f32 * scale_f32\n weight_t = weight_t.to(torch.bfloat16).to(torch.float32)\n else:\n weight_t = weight_t.to(low_dtype)\n scale_t = scale_t.to(low_dtype)\n if off_t is not None:\n off_t = off_t.to(low_dtype)\n if antiquant_group_size:\n num_groups = scale_t.shape[0]\n gs = antiquant_group_size\n kSize = x_t.shape[-1]\n scale_exp = scale_t.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, scale_t.shape[-1])[:kSize, :]\n if off_t is not None:\n off_exp = off_t.view(num_groups, 1, -1).expand(num_groups, gs, -1).reshape(-1, off_t.shape[-1])[:kSize, :]\n weight_t = (weight_t + off_exp) * scale_exp\n else:\n weight_t = weight_t * scale_exp\n else:\n if off_t is not None:\n weight_t = (weight_t + off_t) * scale_t\n else:\n weight_t = weight_t * scale_t\n weight_t = weight_t.to(torch.float16).to(torch.float32)\n\n x_f32 = x_t.to(torch.float32)\n K = x_f32.shape[-1]\n BLOCK_K = 16\n output = None\n for k0 in range(0, K, BLOCK_K):\n k1 = min(k0 + BLOCK_K, K)\n partial = torch.matmul(x_f32[:, k0:k1], weight_t[k0:k1, :])\n output = partial if output is None else output + partial\n\n if bias is not None and bias.numel() != 0:\n output = output + bias.to(torch.float32)\n if quant_scale is not None and quant_scale.numel() != 0:\n output = torch.clamp(torch.round(output * quant_scale), -256, 256).to(torch.int16)\n if quant_offset is not None and quant_offset.numel() != 0:\n output = torch.clamp(output + torch.clamp(torch.round(quant_offset), -256, 256), -128, 127)\n\n return output.to(low_dtype).to(x.device)\n"
41}41}
Mnpu-kernelbench/data/kernel_generator/level4/WeightQuantBatchmatmul/workload.jsonl+2-0
@@ -46,3 +46,5 @@
46{"uuid": "WeightQuantBatchmatmul-046", "axes": null, "inputs": {"x": {"type": "random", "shape": [2192, 16864], "dtype": "float16"}, "weight": {"type": "random", "shape": [16864, 33712], "dtype": "int8", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [132, 33712], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [132, 33712], "dtype": "float16"}, "bias": {"type": "random", "shape": [33712], "dtype": "float16"}, "antiquant_group_size": {"type": "scalar", "value": 128}, "inner_precise": {"type": "scalar", "value": 0}}}46{"uuid": "WeightQuantBatchmatmul-046", "axes": null, "inputs": {"x": {"type": "random", "shape": [2192, 16864], "dtype": "float16"}, "weight": {"type": "random", "shape": [16864, 33712], "dtype": "int8", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [132, 33712], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [132, 33712], "dtype": "float16"}, "bias": {"type": "random", "shape": [33712], "dtype": "float16"}, "antiquant_group_size": {"type": "scalar", "value": 128}, "inner_precise": {"type": "scalar", "value": 0}}}
47{"uuid": "WeightQuantBatchmatmul-047", "axes": null, "inputs": {"x": {"type": "random", "shape": [2304, 18400], "dtype": "float16"}, "weight": {"type": "random", "shape": [18400, 36784], "dtype": "int8", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [144, 36784], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [144, 36784], "dtype": "float16"}, "bias": {"type": "random", "shape": [36784], "dtype": "float16"}, "antiquant_group_size": {"type": "scalar", "value": 128}, "inner_precise": {"type": "scalar", "value": 0}}}47{"uuid": "WeightQuantBatchmatmul-047", "axes": null, "inputs": {"x": {"type": "random", "shape": [2304, 18400], "dtype": "float16"}, "weight": {"type": "random", "shape": [18400, 36784], "dtype": "int8", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [144, 36784], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [144, 36784], "dtype": "float16"}, "bias": {"type": "random", "shape": [36784], "dtype": "float16"}, "antiquant_group_size": {"type": "scalar", "value": 128}, "inner_precise": {"type": "scalar", "value": 0}}}
48{"uuid": "WeightQuantBatchmatmul-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [2624, 20960], "dtype": "float16"}, "weight": {"type": "random", "shape": [20960, 41904], "dtype": "int8", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [82, 41904], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [82, 41904], "dtype": "float16"}, "bias": {"type": "random", "shape": [41904], "dtype": "float16"}, "antiquant_group_size": {"type": "scalar", "value": 256}, "inner_precise": {"type": "scalar", "value": 0}}}48{"uuid": "WeightQuantBatchmatmul-048", "axes": null, "inputs": {"x": {"type": "random", "shape": [2624, 20960], "dtype": "float16"}, "weight": {"type": "random", "shape": [20960, 41904], "dtype": "int8", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [82, 41904], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [82, 41904], "dtype": "float16"}, "bias": {"type": "random", "shape": [41904], "dtype": "float16"}, "antiquant_group_size": {"type": "scalar", "value": 256}, "inner_precise": {"type": "scalar", "value": 0}}}
49+{"uuid": "WeightQuantBatchmatmul-049", "axes": null, "inputs": {"x": {"type": "random", "shape": [64, 128], "dtype": "float16"}, "weight": {"type": "random", "shape": [128, 256], "dtype": "int32", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [1, 256], "dtype": "float16"}, "antiquant_offset": {"type": "random", "shape": [1, 256], "dtype": "float16"}}}
50+{"uuid": "WeightQuantBatchmatmul-050", "axes": null, "inputs": {"x": {"type": "random", "shape": [128, 64], "dtype": "bfloat16"}, "weight": {"type": "random", "shape": [64, 128], "dtype": "int32", "range": [-8, 8]}, "antiquant_scale": {"type": "random", "shape": [1, 128], "dtype": "bfloat16"}, "antiquant_offset": {"type": "random", "shape": [1, 128], "dtype": "bfloat16"}}}