qbmm v3 golden with new ttk-ops-nn-AtomGit

已合并

qbmm v3 golden with new ttk #5976

he_kan创建于 6月11日从he_kan/ops-nn： qbmm 合入到 cann/ops-nn： master

qbmm v3 golden with new ttk #5976

已合并

he_kan创建于 6月11日

从he_kan/ops-nn qbmm合入到cann/ops-nn master

共 6 个文件变更+1193-419

Amatmul/dual_level_quant_batch_matmul/tests/assets/golden.py+162-0

@@ -0,0 +1,162 @@
	1	+#!/usr/bin/env python3
	2	+# -- coding: UTF-8 --
	3	+# ----------------------------------------------------------------------------
	4	+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
	5	+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
	6	+# CANN Open Software License Agreement Version 2.0 (the "License").
	7	+# Please refer to the License for details. You may not use this file except in compliance with the License.
	8	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
	9	+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
	10	+# See LICENSE in the root of the software repository for the full text of the License.
	11	+# ----------------------------------------------------------------------------
	12	+import numpy as np
	13	+from ml_dtypes import bfloat16
	14	+
	15	+try:
	16	+ from en_dtypes import float4_e2m1
	17	+ _HAS_FLOAT4 = True
	18	+except ImportError:
	19	+ _HAS_FLOAT4 = False
	20	+
	21	+__golden__ = {
	22	+ "kernel": {
	23	+ "dual_level_quant_batch_matmul": "dual_level_quant_batch_matmul_golden"
	24	+ }
	25	+}
	26	+
	27	+_DTYPE_ATTR_MAP = {1: np.float16, 27: bfloat16}
	28	+
	29	+
	30	+def customize_inputs(x1, x2, x1_level0_scale, x1_level1_scale, x2_level0_scale,
	31	+ x2_level1_scale, bias=None, *, dtype=1, transpose_x1=False,
	32	+ transpose_x2=True, level0_group_size=512, level1_group_size=32,
	33	+ **kwargs):
	34	+ input_formats = kwargs.get('input_formats', ())
	35	+ input_ori_shapes = kwargs.get('input_ori_shapes', ())
	36	+ if len(input_formats) > 1 and input_formats[1] == 'FRACTAL_NZ':
	37	+ ori_shape = input_ori_shapes[1] if len(input_ori_shapes) > 1 else None
	38	+ x2 = _nz_to_nd(x2, ori_shape)
	39	+ return x1, x2, x1_level0_scale, x1_level1_scale, x2_level0_scale, x2_level1_scale, bias
	40	+
	41	+
	42	+def pre_compare(outputs, *kwargs):
	43	+ return list(outputs)
	44	+
	45	+
	46	+def dual_level_quant_batch_matmul_golden(x1, x2, x1_level0_scale, x1_level1_scale,
	47	+ x2_level0_scale, x2_level1_scale, bias=None, *,
	48	+ dtype=1, transpose_x1=False, transpose_x2=True,
	49	+ level0_group_size=512, level1_group_size=32,
	50	+ **kwargs):
	51	+ x1, x2, x1_level0_scale, x1_level1_scale, x2_level0_scale, x2_level1_scale, bias = customize_inputs(
	52	+ x1, x2, x1_level0_scale, x1_level1_scale, x2_level0_scale, x2_level1_scale, bias,
	53	+ dtype=dtype, transpose_x1=transpose_x1, transpose_x2=transpose_x2,
	54	+ level0_group_size=level0_group_size, level1_group_size=level1_group_size, **kwargs)
	55	+
	56	+ x1_f32 = x1.astype(np.float32)
	57	+ x2_f32 = x2.astype(np.float32)
	58	+
	59	+ if transpose_x2:
	60	+ x2_f32 = np.swapaxes(x2_f32, -2, -1)
	61	+
	62	+ x1_l0 = x1_level0_scale.astype(np.float32)
	63	+ x2_l0 = x2_level0_scale.astype(np.float32)
	64	+ x1_l1 = _e8m0_to_float(x1_level1_scale)
	65	+ x2_l1 = _e8m0_to_float(x2_level1_scale)
	66	+
	67	+ M, K = x1_f32.shape
	68	+ N = x2_f32.shape[1]
	69	+
	70	+ num_l1_groups = x1_l1.shape[-2] * x1_l1.shape[-1] if x1_l1.ndim >= 2 else x1_l1.shape[0]
	71	+ x1_l1_flat = x1_l1.reshape(M, -1)
	72	+ x2_l1_flat = x2_l1.reshape(N, -1)
	73	+
	74	+ num_l0_groups = x1_l0.shape[1]
	75	+ l1_per_l0 = level0_group_size // level1_group_size
	76	+
	77	+ acc = np.zeros((M, N), dtype=np.float32)
	78	+ for g0 in range(num_l0_groups):
	79	+ acc_l0 = np.zeros((M, N), dtype=np.float32)
	80	+ for g1 in range(l1_per_l0):
	81	+ g1_global = g0 * l1_per_l0 + g1
	82	+ if g1_global >= num_l1_groups:
	83	+ break
	84	+ k_start = g1_global * level1_group_size
	85	+ k_end = min(k_start + level1_group_size, K)
	86	+ if k_end <= k_start:
	87	+ break
	88	+ partial = np.matmul(x1_f32[:, k_start:k_end], x2_f32[k_start:k_end, :])
	89	+ s1 = x1_l1_flat[:, g1_global][:, np.newaxis]
	90	+ s2 = x2_l1_flat[:, g1_global][np.newaxis, :]
	91	+ acc_l0 += partial * s1 * s2
	92	+ acc += x1_l0[:, g0][:, np.newaxis] * x2_l0[g0, :][np.newaxis, :] * acc_l0
	93	+
	94	+ if bias is not None:
	95	+ acc = acc + bias.astype(np.float32).reshape(1, -1)
	96	+
	97	+ output_dtypes = kwargs.get("output_dtypes", None)
	98	+ if output_dtypes is not None:
	99	+ acc = _cast_output_dtype(acc, output_dtypes[0])
	100	+ else:
	101	+ target_dtype = _DTYPE_ATTR_MAP.get(dtype, np.float16)
	102	+ acc = acc.astype(target_dtype)
	103	+
	104	+ return [acc]
	105	+
	106	+
	107	+class DualLevelQuantBatchMatmulAssets:
	108	+
	109	+ golden = dual_level_quant_batch_matmul_golden
	110	+ customize_inputs = customize_inputs
	111	+ pre_compare = pre_compare
	112	+
	113	+ tolerance = {
	114	+ "float16": {
	115	+ "standard": "BenchmarkCompareStandard",
	116	+ "avg_re_rtol": 2.0,
	117	+ "max_re_rtol": 10.0,
	118	+ "rmse_rtol": 2.0,
	119	+ "small_value": 0.001,
	120	+ "small_value_atol": 1e-5,
	121	+ },
	122	+ "bfloat16": {
	123	+ "standard": "BenchmarkCompareStandard",
	124	+ "avg_re_rtol": 2.0,
	125	+ "max_re_rtol": 10.0,
	126	+ "rmse_rtol": 2.0,
	127	+ "small_value": 0.001,
	128	+ "small_value_atol": 1e-5,
	129	+ },
	130	+ }
	131	+
	132	+
	133	+def _e8m0_to_float(arr):
	134	+ raw = arr.view(np.uint8).astype(np.float32)
	135	+ result = np.power(2.0, raw - 127.0)
	136	+ result[raw == 255.0] = np.nan
	137	+ return result
	138	+
	139	+
	140	+def _cast_output_dtype(arr, dtype_name):
	141	+ dtype_map = {"float16": np.float16, "float32": np.float32, "bfloat16": bfloat16}
	142	+ target = dtype_map.get(dtype_name)
	143	+ if target is not None:
	144	+ return arr.astype(target)
	145	+ return arr.astype(dtype_name)
	146	+
	147	+
	148	+def _nz_to_nd(data, ori_shape=None):
	149	+ shape = data.shape
	150	+ batch_dims = len(shape) - 4
	151	+ perm = list(range(batch_dims)) + [batch_dims + 1, batch_dims + 2, batch_dims + 0, batch_dims + 3]
	152	+ data = np.transpose(data, perm)
	153	+ m1 = shape[batch_dims + 1]
	154	+ m0_actual = shape[batch_dims + 2]
	155	+ n1 = shape[batch_dims + 0]
	156	+ n0_actual = shape[batch_dims + 3]
	157	+ data = data.reshape(shape[:batch_dims], m1 m0_actual, n1 * n0_actual)
	158	+ if ori_shape is not None:
	159	+ target_M = ori_shape[-2]
	160	+ target_N = ori_shape[-1]
	161	+ data = data[..., :target_M, :target_N]
	162	+ return data

Amatmul/quant_batch_matmul_inplace_add/tests/assets/golden.py+142-0

@@ -0,0 +1,142 @@
	1	+#!/usr/bin/env python3
	2	+# -- coding: UTF-8 --
	3	+# ----------------------------------------------------------------------------
	4	+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
	5	+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
	6	+# CANN Open Software License Agreement Version 2.0 (the "License").
	7	+# Please refer to the License for details. You may not use this file except in compliance with the License.
	8	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
	9	+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
	10	+# See LICENSE in the root of the software repository for the full text of the License.
	11	+# ----------------------------------------------------------------------------
	12	+import numpy as np
	13	+from ml_dtypes import bfloat16, float8_e5m2, float8_e4m3fn
	14	+
	15	+try:
	16	+ from en_dtypes import hifloat8
	17	+ _HAS_HIFLOAT8 = True
	18	+except ImportError:
	19	+ _HAS_HIFLOAT8 = False
	20	+
	21	+__golden__ = {
	22	+ "kernel": {
	23	+ "quant_batch_matmul_inplace_add": "quant_batch_matmul_inplace_add_golden"
	24	+ }
	25	+}
	26	+
	27	+
	28	+def customize_inputs(x1, x2, x2_scale, y, x1_scale=None, *, transpose_x1=False,
	29	+ transpose_x2=False, group_size=0, **kwargs):
	30	+ return x1, x2, x2_scale, y, x1_scale
	31	+
	32	+
	33	+def pre_compare(outputs, *kwargs):
	34	+ return list(outputs)
	35	+
	36	+
	37	+def quant_batch_matmul_inplace_add_golden(x1, x2, x2_scale, y, x1_scale=None, *,
	38	+ transpose_x1=False, transpose_x2=False,
	39	+ group_size=0, **kwargs):
	40	+ x1, x2, x2_scale, y, x1_scale = customize_inputs(
	41	+ x1, x2, x2_scale, y, x1_scale,
	42	+ transpose_x1=transpose_x1, transpose_x2=transpose_x2,
	43	+ group_size=group_size, **kwargs)
	44	+
	45	+ x1_f32 = _to_float32(x1)
	46	+ x2_f32 = _to_float32(x2)
	47	+
	48	+ if transpose_x1:
	49	+ x1_f32 = np.swapaxes(x1_f32, -2, -1)
	50	+ if transpose_x2:
	51	+ x2_f32 = np.swapaxes(x2_f32, -2, -1)
	52	+
	53	+ is_mx = _is_mx_scale(x2_scale)
	54	+
	55	+ if is_mx:
	56	+ acc = _mxfp8_matmul(x1_f32, x2_f32, x1_scale, x2_scale)
	57	+ else:
	58	+ acc = _per_tensor_matmul(x1_f32, x2_f32, x1_scale, x2_scale)
	59	+
	60	+ y_out = acc + y.astype(np.float32)
	61	+
	62	+ output_dtypes = kwargs.get("output_dtypes", None)
	63	+ if output_dtypes is not None:
	64	+ y_out = _cast_output_dtype(y_out, output_dtypes[0])
	65	+ else:
	66	+ y_out = y_out.astype(np.float32)
	67	+
	68	+ return [y_out]
	69	+
	70	+
	71	+def _to_float32(arr):
	72	+ dtype_str = str(arr.dtype)
	73	+ if "e4m3" in dtype_str or "e5m2" in dtype_str:
	74	+ return arr.astype(np.float32)
	75	+ if "hifloat" in dtype_str or "hif8" in dtype_str:
	76	+ return arr.astype(np.float32)
	77	+ return arr.astype(np.float32)
	78	+
	79	+
	80	+def _is_mx_scale(scale):
	81	+ if scale is None:
	82	+ return False
	83	+ return "e8m0" in str(scale.dtype)
	84	+
	85	+
	86	+def _e8m0_to_float(arr):
	87	+ raw = arr.view(np.uint8).astype(np.float32)
	88	+ result = np.power(2.0, raw - 127.0)
	89	+ result[raw == 255.0] = np.nan
	90	+ return result
	91	+
	92	+
	93	+def _per_tensor_matmul(x1, x2, x1_scale, x2_scale):
	94	+ acc = np.matmul(x1, x2)
	95	+ s1 = x1_scale.astype(np.float32).reshape(-1)[0] if x1_scale is not None else 1.0
	96	+ s2 = x2_scale.astype(np.float32).reshape(-1)[0] if x2_scale is not None else 1.0
	97	+ acc = acc * (s1 * s2)
	98	+ return acc
	99	+
	100	+
	101	+def _mxfp8_matmul(x1, x2, x1_scale, x2_scale):
	102	+ M, K = x1.shape
	103	+ N = x2.shape[1]
	104	+
	105	+ x1s_f = _e8m0_to_float(x1_scale)
	106	+ x2s_f = _e8m0_to_float(x2_scale)
	107	+
	108	+ x1s_flat = x1s_f.reshape(M, -1)
	109	+ x2s_flat = x2s_f.reshape(-1, N)
	110	+
	111	+ num_groups = x1s_flat.shape[1]
	112	+ group_size_k = K // num_groups
	113	+
	114	+ out = np.zeros((M, N), dtype=np.float32)
	115	+ for g in range(num_groups):
	116	+ k_start = g * group_size_k
	117	+ k_end = (g + 1) * group_size_k
	118	+ partial = np.matmul(x1[:, k_start:k_end], x2[k_start:k_end, :])
	119	+ s1 = x1s_flat[:, g][:, np.newaxis]
	120	+ s2 = x2s_flat[g, :][np.newaxis, :]
	121	+ out += partial * s1 * s2
	122	+
	123	+ return out
	124	+
	125	+
	126	+class QuantBatchMatmulInplaceAddAssets:
	127	+
	128	+ golden = quant_batch_matmul_inplace_add_golden
	129	+ customize_inputs = customize_inputs
	130	+ pre_compare = pre_compare
	131	+
	132	+ tolerance = {
	133	+ "float32": {"standard": "IsClose", "rtol": 1e-3, "atol": 1e-3},
	134	+ }
	135	+
	136	+
	137	+def _cast_output_dtype(arr, dtype_name):
	138	+ dtype_map = {"float16": np.float16, "float32": np.float32, "bfloat16": bfloat16}
	139	+ target = dtype_map.get(dtype_name)
	140	+ if target is not None:
	141	+ return arr.astype(target)
	142	+ return arr.astype(dtype_name)

Mmatmul/quant_batch_matmul_v3/tests/assets/golden.py+594-346

	@@ -1,346 +1,594 @@
1	-#!/usr/bin/env python3	1	+#!/usr/bin/env python3
2	-# -- coding: UTF-8 --	2	+# -- coding: UTF-8 --
3	-# ----------------------------------------------------------------------------	3	+# ----------------------------------------------------------------------------
4	-# Copyright (c) 2026 Huawei Technologies Co., Ltd.	4	+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
5	-# This program is free software, you can redistribute it and/or modify it under the terms and conditions of	5	+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
6	-# CANN Open Software License Agreement Version 2.0 (the "License").	6	+# CANN Open Software License Agreement Version 2.0 (the "License").
7	-# Please refer to the License for details. You may not use this file except in compliance with the License.	7	+# Please refer to the License for details. You may not use this file except in compliance with the License.
8	-# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,	8	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
9	-# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.	9	+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
10	-# See LICENSE in the root of the software repository for the full text of the License.	10	+# See LICENSE in the root of the software repository for the full text of the License.
11	-# ----------------------------------------------------------------------------	11	+# ----------------------------------------------------------------------------
12	-__golden__ = {	12	+import numpy as np
13	- "kernel": {	13	+import en_dtypes
14	- "quant_batch_matmul_v3": "quant_batch_matmul_v3_golden"	14	+import ml_dtypes
15	- }	15	+np_fp4_e2m1 = en_dtypes.float4_e2m1
16	-}	16	+np_fp4_e1m2 = en_dtypes.float4_e1m2
17	-	17	+np_hif8 = en_dtypes.hifloat8
18	-import torch	18	+np_mx_scale = en_dtypes.float8_e8m0
19	-import os	19	+np_bfloat16 = ml_dtypes.bfloat16
20	-import numpy as np	20	+np_fp8_e4m3 = ml_dtypes.float8_e4m3fn
21	-from ml_dtypes import bfloat16	21	+np_fp8_e5m2 = ml_dtypes.float8_e5m2
22	-	22	+np_int4 = ml_dtypes.int4
23	-def quant_batch_matmul_v3_golden(x1, x2, scale, offset = None, bias = None, pertoken_scale = None, * ,dtype: int,	23	+
24	- transpose_x1: bool = False, transpose_x2: bool = False,	24	+__golden__ = {
25	- group_size:int = 0, **kwargs):	25	+ "kernel": {
26	-	26	+ "quant_batch_matmul_v3": "quant_batch_matmul_v3_golden"
27	-	27	+ }
28	- isAscend910B = False	28	+ }
29	- if kwargs['short_soc_version'] in ("Ascend910B", "Ascend910_93"):	29	+
30	- isAscend910B = True	30	+
31	- deq_scale = scale	31	+# 参数命名与算子定义(V3 op index)及V5 API的对应关系:
32	-	32	+# x1 → 算子入参0 x1 (左矩阵) → V5: x1
33	- testcase_name = kwargs['testcase_name']	33	+# x2 → 算子入参1 x2 (右矩阵) → V5: x2
34	-	34	+# scale → 算子入参2 scale (x2量化scale) → V5: x2Scale; golden接收的是customize_inputs处理后的deq_scale
35	- x1_dtype = x1.dtype.name	35	+# offset → 算子入参3 offset (x2量化offset) → V5: x2Offset
36	- x2_dtype = x2.dtype.name	36	+# bias → 算子入参4 bias → V5: bias
37	- deq_scale_dtype = deq_scale.dtype.name	37	+# pertoken_scale → 算子入参5 pertoken_scale (x1量化scale) → V5: x1Scale
38	- bias_dtype = None	38	+# y → 算子出参0 y (结果) → V5: out
39	- if bias is not None:	39	+#
40	- bias_dtype = bias.dtype.name	40	+# 内部变量命名遵循V5语义, 与算子入参名存在明显关联:
41	-	41	+# x2_scale → 算子入参scale (x2侧量化scale, customize_inputs后为deq_scale)
42	- output_dtypes = kwargs['output_dtypes']	42	+# x1_scale → 算子入参pertoken_scale (x1侧量化scale)
43	- out_dtype = output_dtypes[0]	43	+# scale_dtype → 算子入参scale的原始dtype (通过kwargs传入)
44	- trans_a = transpose_x1	44	+# y_dtype → 算子出参y的dtype
45	- trans_b = transpose_x2	45	+#
46	-	46	+# T-C场景简化说明:
47	- groups = group_size	47	+# customize_inputs已将UINT64 scale转为float32 deq_scale(含高19位截断, 等价于scale_generate).
48	- group_size_m, group_size_n, group_size_k = unpack_groupsize(groups)	48	+# offset不从uint64 scale中提取, 保留原始输入参数(与TTK框架行为一致:
49	-	49	+# 原始offset输入为None时不参与requant计算, 仅当显式提供时才使用).
50	- # mxfp4、mxfp8，输入先和scale相乘，再去做MMAD	50	+# golden()接收的x2_scale在两种T-C场景下均为float32/bf16,
51	- is_mxFP = pertoken_scale is not None and \	51	+# bias位置由is_bias_vec决定(与TTK golden及_compute_pertoken一致):
52	- x1_dtype in ("float4_e2m1", "float4_e1m2", "float8_e4m3fn", "float8_e5m2") and \	52	+# 非is_bias_vec时bias在×x2Scale前加入(int32域/float32域加法),
53	- deq_scale_dtype in ("float8_e8m0",)	53	+# is_bias_vec时bias在×x2Scale后加入(fixpipe后处理).
54	- # fp8和hif8，pertoken_scale代表2路scale, 2路scale是用fixpipe中运算的，需要进行高19位与操作，否则会出现精度问题	54	+# T-C-static(uint64)和T-C-dynamic(float32/bf16)合并为单个_compute_tc函数.
55	- is_two_scale = pertoken_scale is not None and x1_dtype in ("float8_e4m3fn", "float8_e5m2", "hifloat8") and \	55	+
56	- not is_mxFP and deq_scale.shape[0] == 1 and pertoken_scale.shape[0] == 1	56	+def quant_batch_matmul_v3_golden(x1, x2, scale, offset=None, bias=None, pertoken_scale=None,
57	- # int8，pertoken_scale代表pertoken，pertoken不是在fixpipe中运算的，无需高19位与操作	57	+ *, dtype: int, transpose_x1: bool = False, transpose_x2: bool = False,
58	- is_pertoken = pertoken_scale is not None and x1_dtype in ('int8', "float8_e4m3fn", "float8_e5m2", "hifloat8") and not is_two_scale	58	+ group_size: int = 0, **kwargs):
59	- is_950_compatible = out_dtype == 'bfloat16' and x1_dtype == 'int8' and deq_scale_dtype != 'uint64' and pertoken_scale is None	59	+ scale_dtype = kwargs.get('scale_dtype', _dtype_to_str(scale.dtype))
60	- is_bias_vec = (x1_dtype == 'int8' or x1_dtype == "int4") and bias is not None and bias_dtype in ('bfloat16', "float16", "float32")	60	+ x1, x2, scale, offset, bias, pertoken_scale = customize_inputs(
61	- is_perblock = x1_dtype in ("float8_e4m3fn", "float8_e5m2", "hifloat8") and deq_scale_dtype in ("float32",) and len(deq_scale.shape) > 1 and len(pertoken_scale.shape) > 1	61	+ x1, x2, scale, offset, bias, pertoken_scale,
62	-	62	+ dtype=dtype, transpose_x1=transpose_x1, transpose_x2=transpose_x2,
63	- # mxFP4	63	+ group_size=group_size, **kwargs)
64	- if is_mxFP:	64	+ y_dtype = kwargs.get('output_dtypes', ['float32'])[0]
65	- deq_scale_mx = deq_scale	65	+ x1_dtype = _dtype_to_str(x1.dtype)
66	- pertoken_scale_mx = pertoken_scale	66	+ x2_dtype = _dtype_to_str(x2.dtype)
67	- if x1_dtype in ("float8_e4m3fn", "float8_e5m2"):	67	+ bias_dtype = _dtype_to_str(bias.dtype) if bias is not None else None
68	- x1 = torch.from_numpy(x1.astype(np.float32))	68	+ x2_scale = scale.astype(np.float32) if scale.dtype != np_mx_scale else scale
69	-	69	+ x1_scale = pertoken_scale
70	- if x2_dtype in ("float8_e4m3fn", "float8_e5m2"):	70	+ group_size_m, group_size_n, group_size_k = _unpack_groupsize(group_size)
71	- x2 = torch.from_numpy(x2.astype(np.float32))	71	+ quant_mode = _determine_quant_mode(
72	- # mxFP4单独处理transpose，统一使用(M,K)和(K,N)格式处理mxFP4	72	+ x1_dtype, x2_dtype, x1_scale, x2_scale, scale_dtype, group_size_n)
73	- if trans_a:	73	+ do_scale_gen = _needs_scale_generate(x1_dtype, x2_scale, bias_dtype, scale_dtype)
74	- x1 = np.swapaxes(x1, -1, -2)	74	+ is_bias_vec = x1_dtype in ('int8', 'int4') and \
75	- pertoken_scale_mx = np.swapaxes(pertoken_scale_mx, -1, -2)	75	+ bias_dtype is not None and \
76	- if len(pertoken_scale_mx.shape) == 3:	76	+ bias_dtype in ('bfloat16', 'float16', 'float32')
77	- pertoken_scale_mx = pertoken_scale_mx.reshape(pertoken_scale_mx.shape[0] * pertoken_scale_mx.shape[1], pertoken_scale_mx.shape[2])	77	+ if offset is not None:
78	- pertoken_scale_mx = np.swapaxes(pertoken_scale_mx, -1, -2)	78	+ offset = offset.astype(np.float32)
79	- else:	79	+ if x2_scale.shape[-1] < offset.shape[-1]:
80	- if len(pertoken_scale_mx.shape) == 3:	80	+ offset = offset[0]
81	- pertoken_scale_mx = pertoken_scale_mx.reshape(pertoken_scale_mx.shape[0], pertoken_scale_mx.shape[1] * pertoken_scale_mx.shape[2])	81	+ if quant_mode == "G-G":
82	- if trans_b:	82	+ y = _compute_mx(x1, x2, x2_scale, x1_scale, bias, bias_dtype,
83	- x2 = np.swapaxes(x2, -1, -2)	83	+ transpose_x1, transpose_x2, y_dtype)
84	- if len(deq_scale_mx.shape) == 3:	84	+ elif quant_mode == "B-B":
85	- deq_scale_mx = deq_scale_mx.reshape(deq_scale_mx.shape[0], deq_scale_mx.shape[1] * deq_scale_mx.shape[2])	85	+ y = _compute_perblock(x1, x2, x2_scale, x1_scale,
86	- deq_scale_mx = np.swapaxes(deq_scale_mx, -1, -2)	86	+ group_size_m, group_size_n, group_size_k,
87	- else:	87	+ transpose_x1, transpose_x2, y_dtype, bias)
88	- deq_scale_mx = np.swapaxes(deq_scale_mx, -1, -2)	88	+ else:
89	- if len(deq_scale_mx.shape) == 3:	89	+ if x1_dtype in ('int8', 'int4') and x2_dtype in ('int8', 'int4'):
90	- deq_scale_mx = deq_scale_mx.reshape(deq_scale_mx.shape[0] * deq_scale_mx.shape[1], deq_scale_mx.shape[2])	90	+ x1 = x1.astype(np.int32)
91	-	91	+ x2 = x2.astype(np.int32)
92	- k_dim = x1.shape[-1]	92	+ else:
93	- if ceil_div(k_dim, 32) % 2 != 0:	93	+ x1 = x1.astype(np.float32)
94	- pertoken_scale_mx = pertoken_scale_mx[:, :-1]	94	+ x2 = x2.astype(np.float32)
95	- deq_scale_mx = deq_scale_mx[:-1, :]	95	+ if transpose_x1:
96	-	96	+ x1 = np.transpose(x1, _gen_axes_for_transpose(len(x1.shape) - 2, [1, 0]))
97	- # broadcast，每个数对应32个数	97	+ if transpose_x2:
98	- pertoken_scale_mx_broadcast = np.repeat(pertoken_scale_mx, 32, axis=-1)	98	+ x2 = np.transpose(x2, _gen_axes_for_transpose(len(x2.shape) - 2, [1, 0]))
99	- deq_scale_mx_broadcast = np.repeat(deq_scale_mx, 32, axis=-2)	99	+ matmul_out = np.matmul(x1, x2)
100	- x1_dims = len(x1.shape)	100	+ if y_dtype == 'int8':
101	- x2_dims = len(x2.shape)	101	+ y = _compute_requant(matmul_out, x2_scale, offset, bias, bias_dtype)
102	- x1_pad_len = pertoken_scale_mx_broadcast.shape[-1] - x1.shape[-1]	102	+ elif y_dtype == 'int32':
103	- x2_pad_len = deq_scale_mx_broadcast.shape[-2] - x2.shape[-2]	103	+ y = _compute_int32(matmul_out, bias, bias_dtype)
104	- x1 = np.pad(x1, [(0, 0)] * (x1_dims -1) + [(0, x1_pad_len)], mode='constant', constant_values=0)	104	+ elif quant_mode == "K-C":
105	- x2 = np.pad(x2, [(0, 0)] * (x2_dims -2) + [(0, x2_pad_len)] + [(0, 0)], mode='constant', constant_values=0)	105	+ y = _compute_pertoken(matmul_out, x2_scale, x1_scale, bias, bias_dtype,
106	-	106	+ is_bias_vec, y_dtype, x1_dtype)
107	- x1 = x1 * pertoken_scale_mx_broadcast	107	+ elif quant_mode == "T-C":
108	- x2 = x2 * deq_scale_mx_broadcast	108	+ y = _compute_tc(matmul_out, x2_scale, bias, bias_dtype,
109	-	109	+ do_scale_gen, is_bias_vec, y_dtype)
110	- # 升精度 & 转torch	110	+ else:
111	- if x1_dtype in ("float8_e4m3fn", "float8_e5m2", "float4_e2m1", "float4_e1m2", "hifloat8"):	111	+ if matmul_out.dtype == np.int32:
112	- x1 = torch.from_numpy(x1.astype(np.float32))	112	+ matmul_out = matmul_out.astype(np.float32)
113	- elif x1_dtype in ("int4",):	113	+ y = _cast_output_dtype(matmul_out * x2_scale, y_dtype)
114	- x1 = torch.from_numpy(x1.astype(np.int32)).to(torch.int32)	114	+ return [y]
115	- else:	115	+
116	- x1 = torch.from_numpy(x1).to(torch.int32)	116	+
117	- if x2_dtype in ("float8_e4m3fn", "float8_e5m2", "float4_e2m1", "float4_e1m2", "hifloat8"):	117	+
118	- x2 = torch.from_numpy(x2.astype(np.float32))	118	+def customize_inputs(x1, x2, scale, offset=None, bias=None, pertoken_scale=None,
119	- elif x2_dtype in ("int4",):	119	+ *, dtype: int, transpose_x1: bool = False, transpose_x2: bool = False,
120	- x2 = torch.from_numpy(x2.astype(np.int32)).to(torch.int32)	120	+ group_size: int = 0, **kwargs):
121	- else:	121	+ if scale.dtype in (np.uint64, np.int64):
122	- x2 = torch.from_numpy(x2).to(torch.int32)	122	+ deq_scale = _u64_to_deq_scale(scale)
123	-	123	+ if offset is not None:
124	- if not (is_mxFP):	124	+ offset = _u64_to_offset(scale)
125	- if trans_a:	125	+ else:
126	- array_trans = gen_axes_for_transpose(len(x1.shape) - 2, [1, 0])	126	+ deq_scale = scale
127	- x1 = x1.permute(*array_trans)	127	+ input_formats = kwargs.get('input_formats', ())
128	- if trans_b:	128	+ input_ori_shapes = kwargs.get('input_ori_shapes', ())
129	- array_trans = gen_axes_for_transpose(len(x2.shape) - 2, [1, 0])	129	+ if len(input_formats) > 1 and input_formats[1] == 'FRACTAL_NZ':
130	- x2 = x2.permute(*array_trans)	130	+ ori_shape = input_ori_shapes[1] if len(input_ori_shapes) > 1 else None
131	-	131	+ x2 = _nz_to_nd(x2, ori_shape)
132	- if not is_perblock:	132	+ if len(input_formats) > 2 and input_formats[2] == 'FRACTAL_NZ' and \
133	- out = torch.matmul(x1, x2)	133	+ kwargs.get('scale_dtype') not in ("uint64", "int64"):
134	- else:	134	+ ori_shape = input_ori_shapes[2] if len(input_ori_shapes) > 2 else None
135	- pass	135	+ deq_scale = _nz_to_nd(deq_scale, ori_shape)
136	-	136	+ return x1, x2, deq_scale, offset, bias, pertoken_scale
137	- if is_mxFP: # mxFP不做scale处理	137	+
138	- pass	138	+
139	- elif deq_scale_dtype == "uint64":	139	+def pre_compare(outputs, *kwargs):
140	- deq_scale = np.load(testcase_name + "_deq_scale.npy")	140	+ return list(outputs)
141	- deq_scale_tensor = torch.from_numpy(deq_scale)	141	+
142	- else:	142	+
143	- deq_scale = deq_scale.astype(np.float32)	143	+class QuantBatchMatmulV3Assets:
144	- deq_scale_tensor = torch.from_numpy(deq_scale)	144	+ """quant_batch_matmul_v3 算子的全部资产（每个都是可选的）"""
145	-	145	+ golden = quant_batch_matmul_v3_golden
146	- if offset is not None:	146	+ customize_inputs = customize_inputs
147	- offset = np.load(testcase_name + "_offset.npy")	147	+ pre_compare = pre_compare
148	- offset = torch.from_numpy(offset)	148	+
149	- if deq_scale_tensor.shape[-1] < offset.shape[-1]:	149	+ tolerance = {
150	- offset = offset[0]	150	+ "float32": {
151	-	151	+ "standard": "BenchmarkCompareStandard",
152	- if bias is not None and bias_dtype == "int32":	152	+ "avg_re_rtol": 2.0, "max_re_rtol": 5.0, "rmse_rtol": 2.0,
153	- bias = torch.from_numpy(bias).to(torch.int32)	153	+ "small_value": 1e-6, "small_value_atol": 1e-9,
154	- out = torch.add(out, bias)	154	+ },
155	-	155	+ "float16": {
156	- if is_perblock:	156	+ "standard": "BenchmarkCompareStandard",
157	- pertoken_scale_tensor = torch.from_numpy(pertoken_scale).to(torch.float32)	157	+ "avg_re_rtol": 2.0, "max_re_rtol": 10.0, "rmse_rtol": 2.0,
158	-	158	+ "small_value": 0.001, "small_value_atol": 1e-5,
159	- if trans_a:	159	+ },
160	- array_trans = gen_axes_for_transpose(len(pertoken_scale_tensor.shape) - 2, [1, 0])	160	+ "bfloat16": {
161	- pertoken_scale_tensor = pertoken_scale_tensor.permute(*array_trans)	161	+ "standard": "BenchmarkCompareStandard",
162	- if trans_b:	162	+ "avg_re_rtol": 2.0, "max_re_rtol": 10.0, "rmse_rtol": 2.0,
163	- array_trans = gen_axes_for_transpose(len(deq_scale_tensor.shape) - 2, [1, 0])	163	+ "small_value": 0.001, "small_value_atol": 1e-5,
164	- deq_scale_tensor = deq_scale_tensor.permute(*array_trans)	164	+ },
165	-	165	+ "float8_e5m2": {
166	- batch_x1_shape = x1.shape[:-2]	166	+ "standard": "BenchmarkCompareStandard",
167	- batch_x2_shape = x2.shape[:-2]	167	+ "avg_re_rtol": 2.0, "max_re_rtol": 10.0, "rmse_rtol": 2.0,
168	- batch_x1 = [int(i) for i in batch_x1_shape]	168	+ "small_value": 0.001, "small_value_atol": 1e-5,
169	- batch_x2 = [int(i) for i in batch_x2_shape]	169	+ },
170	- import copy	170	+ "int32": {"standard": "BinaryMatch"},
171	- batch_out = copy.deepcopy(batch_x1) if len(batch_x1) > len(batch_x2) else copy.deepcopy(batch_x2)	171	+ "int8": {"standard": "IsClose", "rtol": 0, "atol": 1},
172	- if batch_x2 != batch_x1 and len(batch_x1) != 0 and len(batch_x2) != 0:	172	+ }
173	- idx= 1	173	+
174	- for item in reversed(batch_out):	174	+
175	- if idx <= len(batch_x1) and idx <= len(batch_x2):	175	+# ----------------------------------------------------------------------------
176	- batch_out[-idx] = max(batch_x1[-idx], batch_x2[-idx])	176	+# 量化模式计算函数 — 每种量化类型一个函数, 函数前注释给出计算基础流程
177	- elif idx <= len(batch_x1):	177	+# ----------------------------------------------------------------------------
178	- batch_out[-idx] = batch_x1[-idx]	178	+
179	- else:	179	+def _compute_mx(x1, x2, x2_scale, x1_scale, bias, bias_dtype,
180	- batch_out[-idx] = batch_x2[-idx]	180	+ transpose_x1, transpose_x2, y_dtype):
181	- idx = idx+1	181	+ # MX(G-G) 计算流程:
182	- if batch_x1 != batch_out:	182	+ # 1. x1/x2转换为float32
183	- new_x1_shape = batch_out + list(x1.shape[-2:])	183	+ # 2. 将x1Scale(E8M0)预乘到x1, 将x2Scale(E8M0)预乘到x2 (含transpose对齐和K轴padding)
184	- x1 = torch.broadcast_to(x1, new_x1_shape)	184	+ # 3. matmul(x1, x2) → 结果已含scale效果
185	- new_s_shape = batch_out + list(pertoken_scale_tensor.shape[-2:])	185	+ # 4. +bias(FP32 opt)
186	- pertoken_scale_tensor = torch.broadcast_to(pertoken_scale_tensor, new_s_shape)	186	+ # 5. cast到y_dtype
187	-	187	+ x2_scale_mx = x2_scale.copy().astype(np.float32)
188	- if batch_x2 != batch_out:	188	+ x1_scale_mx = x1_scale.copy().astype(np.float32)
189	- new_x2_shape = batch_out + list(x2.shape[-2:])	189	+ x1 = x1.astype(np.float32)
190	- x2 = torch.broadcast_to(x2, new_x2_shape)	190	+ x2 = x2.astype(np.float32)
191	- new_s_shape = batch_out + list(deq_scale_tensor.shape[-2:])	191	+
192	- deq_scale_tensor = torch.broadcast_to(deq_scale_tensor, new_s_shape)	192	+ if transpose_x1:
193	-	193	+ x1 = np.swapaxes(x1, -1, -2)
194	- # batch轴合轴	194	+ x1_scale_mx = np.swapaxes(x1_scale_mx, -1, -2)
195	- batch_all = 1	195	+ if len(x1_scale_mx.shape) == 3:
196	- if batch_out:	196	+ x1_scale_mx = x1_scale_mx.reshape(
197	- batch_all = np.prod(batch_out)	197	+ x1_scale_mx.shape[0] * x1_scale_mx.shape[1], x1_scale_mx.shape[2])
198	- x1 = x1.reshape([batch_all] + list(x1.shape[-2:]))	198	+ x1_scale_mx = np.swapaxes(x1_scale_mx, -1, -2)
199	- x2 = x2.reshape([batch_all] + list(x2.shape[-2:]))	199	+ else:
200	- pertoken_scale_tensor = pertoken_scale_tensor.reshape([batch_all] + list(pertoken_scale_tensor.shape[-2:]))	200	+ if len(x1_scale_mx.shape) == 3:
201	- deq_scale_tensor = deq_scale_tensor.reshape([batch_all] + list(deq_scale_tensor.shape[-2:]))	201	+ x1_scale_mx = x1_scale_mx.reshape(
202	-	202	+ x1_scale_mx.shape[0], x1_scale_mx.shape[1] * x1_scale_mx.shape[2])
203	-	203	+
204	- # Ascend_910B新增biasDtype float32 float16,当Ascend_910B时，不执行这段代码	204	+ if transpose_x2:
205	- if not isAscend910B and not is_950_compatible and not is_bias_vec:	205	+ x2 = np.swapaxes(x2, -1, -2)
206	- # fp8 hif8 的bias 处理	206	+ if len(x2_scale_mx.shape) == 3:
207	- if bias is not None and bias_dtype == "float32" and not (is_mxFP):	207	+ x2_scale_mx = x2_scale_mx.reshape(
208	- bias = torch.from_numpy(bias).to(torch.float32)	208	+ x2_scale_mx.shape[0], x2_scale_mx.shape[1] * x2_scale_mx.shape[2])
209	- out = torch.add(out, bias)	209	+ x2_scale_mx = np.swapaxes(x2_scale_mx, -1, -2)
210	- if is_mxFP:	210	+ else:
211	- if bias is not None:	211	+ x2_scale_mx = np.swapaxes(x2_scale_mx, -1, -2)
212	- bias = torch.from_numpy(bias.astype(np.float32))	212	+ if len(x2_scale_mx.shape) == 3:
213	- out = (out + bias).numpy().astype(out_dtype)	213	+ x2_scale_mx = x2_scale_mx.reshape(
214	- else:	214	+ x2_scale_mx.shape[0] * x2_scale_mx.shape[1], x2_scale_mx.shape[2])
215	- out = out.numpy().astype(out_dtype)	215	+
216	- elif is_perblock:	216	+ k_dim = x1.shape[-1]
217	- # 已预处理：x1,x2,deq_scale和pertoken_scale全为非转置	217	+ if _ceil_div(k_dim, 32) % 2 != 0:
218	-	218	+ x1_scale_mx = x1_scale_mx[:, :-1]
219	- m = x1.shape[-2] # 非转置情况下，m是倒数第二维	219	+ x2_scale_mx = x2_scale_mx[:-1, :]
220	- k = x1.shape[-1] # 非转置情况下，k是倒数第一维	220	+
221	- n = x2.shape[-1] # 非转置情况下，n是倒数第一维	221	+ x1_scale_mx_br = np.repeat(x1_scale_mx, 32, axis=-1)
222	- out = torch.zeros(m, n)	222	+ x2_scale_mx_br = np.repeat(x2_scale_mx, 32, axis=-2)
223	- if pertoken_scale_tensor.dim() > 2 or deq_scale_tensor.dim() > 2:	223	+
224	- out = torch.zeros(batch_all, m, n)	224	+ x1_dims = len(x1.shape)
225	- pertoken_scale_tensor_m = torch.repeat_interleave(pertoken_scale_tensor, repeats=group_size_m, dim=-2)	225	+ x2_dims = len(x2.shape)
226	- pertoken_scale_tensor_m = pertoken_scale_tensor_m[..., :m, :]	226	+ x1_pad_len = x1_scale_mx_br.shape[-1] - x1.shape[-1]
227	- deq_scale_tensor_n = torch.repeat_interleave(deq_scale_tensor, repeats=group_size_n, dim=-1)	227	+ x2_pad_len = x2_scale_mx_br.shape[-2] - x2.shape[-2]
228	- deq_scale_tensor_n = deq_scale_tensor_n[..., :n]	228	+ if x1_pad_len > 0:
229	- if pertoken_scale_tensor.dim() > 2 or deq_scale_tensor.dim() > 2:	229	+ x1 = np.pad(x1, [(0, 0)] * (x1_dims - 1) + [(0, x1_pad_len)],
230	- for i in range(batch_all):	230	+ mode='constant', constant_values=0)
231	- for k_idx in range((k + group_size_k - 1) // group_size_k):	231	+ if x2_pad_len > 0:
232	- k_start = k_idx * group_size_k	232	+ x2 = np.pad(x2, [(0, 0)] * (x2_dims - 2) + [(0, x2_pad_len)] + [(0, 0)],
233	- k_end = min((k_idx + 1) * group_size_k, k)	233	+ mode='constant', constant_values=0)
234	- scale_mul = pertoken_scale_tensor_m[i, :, k_idx].unsqueeze(1) * deq_scale_tensor_n[i, k_idx, :].unsqueeze(0)	234	+
235	- out[i] += torch.matmul(x1[i, :, k_start:k_end], x2[i, k_start:k_end, :]) * scale_mul	235	+ x1 = x1 * x1_scale_mx_br
236	- else:	236	+ x2 = x2 * x2_scale_mx_br
237	- for k_idx in range((k + group_size_k - 1) // group_size_k):	237	+
238	- k_start = k_idx * group_size_k	238	+ y = np.matmul(x1, x2)
239	- k_end = min((k_idx + 1) * group_size_k, k)	239	+ if bias is not None:
240	- scale_mul = pertoken_scale_tensor_m[:, k_idx].unsqueeze(1) * deq_scale_tensor_n[k_idx, :].unsqueeze(0)	240	+ y = _cast_output_dtype(y + bias.astype(np.float32), y_dtype)
241	- out += torch.matmul(x1[:, k_start:k_end], x2[k_start:k_end, :]) * scale_mul	241	+ else:
242	- if out_dtype == 'bfloat16':	242	+ y = _cast_output_dtype(y, y_dtype)
243	- out_dtype = bfloat16	243	+ return y
244	- out = (out).numpy().astype(out_dtype) # 暂时只支持fp16/bf16/fp32	244	+
245	-	245	+
246	- else:	246	+def _compute_perblock(x1, x2, x2_scale, x1_scale,
247	- if out_dtype == 'int8':	247	+ group_size_m, group_size_n, group_size_k,
248	- out = f32_2_s9(out * deq_scale_tensor)	248	+ transpose_x1, transpose_x2, y_dtype, bias=None):
249	- if offset is not None:	249	+ # B-B(PerBlock) 计算流程:
250	- out = f32_2_s9(out) + f32_2_s9(offset)	250	+ # 1. x1/x2转换为float32
251	- out = np.clip(out, -128, 127).numpy().astype(out_dtype)	251	+ # 2. 对x1Scale/x2Scale做transpose对齐x1/x2的行列方向
252	- elif out_dtype == 'bfloat16':	252	+ # 3. broadcast对齐batch维度
253	- output_dtype = bfloat16	253	+ # 4. x1Scale沿M轴repeat(groupSizeM), x2Scale沿N轴repeat(groupSizeN)
254	- if is_pertoken:	254	+ # 5. 分K轴block级累加: matmul(x1[:,k_start:k_end], x2[k_start:k_end,:]) × (x1Scale[:,k_idx] × x2Scale[k_idx,:])
255	- pertoken_scale_slice = torch.unsqueeze(torch.from_numpy(pertoken_scale), dim=1).to(torch.float32)	255	+ # 6. cast到y_dtype
256	- out = out * deq_scale_tensor * pertoken_scale_slice	256	+ # 注: B-B不支持bias; 仅FP8/HIF8可走PerBlock(INT8/INT4走V4)
257	- elif is_two_scale:	257	+ x1 = x1.astype(np.float32)
258	- two_scale = scale_generate(pertoken_scale * deq_scale)	258	+ x2 = x2.astype(np.float32)
259	- two_scale_tensor = torch.unsqueeze(torch.from_numpy(two_scale), dim=1).to(torch.float32)	259	+ x1_scale_f = x1_scale.astype(np.float32)
260	- out = out * two_scale_tensor	260	+ x2_scale_f = x2_scale.astype(np.float32)
261	- elif is_950_compatible:	261	+
262	- if deq_scale.shape[0] == 1 and (bias_dtype != "bfloat16" and bias_dtype != "float32"):	262	+ if transpose_x1:
263	- scale = scale_generate(deq_scale)	263	+ x1 = np.transpose(x1, _gen_axes_for_transpose(len(x1.shape) - 2, [1, 0]))
264	- deq_scale_tensor = torch.unsqueeze(torch.from_numpy(scale), dim=1).to(torch.float32)	264	+ x1_scale_f = np.transpose(x1_scale_f, _gen_axes_for_transpose(len(x1_scale_f.shape) - 2, [1, 0]))
265	- out = out * deq_scale_tensor	265	+ if transpose_x2:
266	- else:	266	+ x2 = np.transpose(x2, _gen_axes_for_transpose(len(x2.shape) - 2, [1, 0]))
267	- out = out * deq_scale_tensor	267	+ x2_scale_f = np.transpose(x2_scale_f, _gen_axes_for_transpose(len(x2_scale_f.shape) - 2, [1, 0]))
268	- # Ascend_910B新增biasDtype float32 float16, 这段代码扩展 or bias_dtype == "float32"	268	+
269	- if (isAscend910B or is_950_compatible) and bias is not None and (bias_dtype == "bfloat16" or bias_dtype == "float32"):	269	+ batch_x1 = list(x1.shape[:-2])
270	- bias_fp32 = torch.from_numpy(bias.astype(np.float32))	270	+ batch_x2 = list(x2.shape[:-2])
271	- out = (out + bias_fp32).numpy().astype(output_dtype)	271	+ batch_ps = list(x1_scale_f.shape[:-2])
272	- elif is_bias_vec:	272	+ batch_ds = list(x2_scale_f.shape[:-2])
273	- bias_fp32 = torch.from_numpy(bias.astype(np.float32))	273	+
274	- out = (out + bias_fp32).numpy().astype(output_dtype)	274	+ all_batches = [b for b in [batch_x1, batch_x2, batch_ps, batch_ds] if b]
275	- else:	275	+ if not all_batches:
276	- out = out.numpy().astype(output_dtype)	276	+ batch_out = []
277	- elif out_dtype == 'float16':	277	+ else:
278	- if is_pertoken:	278	+ max_len = max(len(b) for b in all_batches)
279	- pertoken_scale_slice = torch.unsqueeze(torch.from_numpy(pertoken_scale), dim=1).to(torch.float32)	279	+ batch_out = list(all_batches[0])
280	- out = out * deq_scale_tensor * pertoken_scale_slice	280	+ for b in all_batches[1:]:
281	- elif is_two_scale:	281	+ padded_b = [1] * (max_len - len(b)) + b
282	- two_scale = scale_generate(pertoken_scale * deq_scale)	282	+ padded_out = [1] * (max_len - len(batch_out)) + batch_out
283	- two_scale_tensor = torch.unsqueeze(torch.from_numpy(two_scale), dim=1).to(torch.float32)	283	+ for idx in range(max_len):
284	- out = out * two_scale_tensor	284	+ padded_out[idx] = max(padded_out[idx], padded_b[idx])
285	- else:	285	+ batch_out = padded_out
286	- out = (out * deq_scale_tensor)	286	+
287	- # Ascend_910B新增biasDtype float32 float16	287	+ if batch_out:
288	- if (isAscend910B and bias is not None and (bias_dtype == "float16" or bias_dtype == "float32")) or is_bias_vec:	288	+ if batch_x1 != batch_out:
289	- bias_fp32 = torch.from_numpy(bias.astype(np.float32))	289	+ x1 = np.broadcast_to(x1, batch_out + list(x1.shape[-2:]))
290	- out = (out + bias_fp32).numpy().astype(out_dtype)	290	+ if batch_x2 != batch_out:
291	- else:	291	+ x2 = np.broadcast_to(x2, batch_out + list(x2.shape[-2:]))
292	- out = out.numpy().astype(out_dtype)	292	+ if batch_ps != batch_out:
293	- elif out_dtype == 'hifloat8':	293	+ x1_scale_f = np.broadcast_to(x1_scale_f, batch_out + list(x1_scale_f.shape[-2:]))
294	- out = (out * deq_scale_tensor).numpy().astype(out_dtype)	294	+ if batch_ds != batch_out:
295	- elif out_dtype == 'float8_e4m3fn':	295	+ x2_scale_f = np.broadcast_to(x2_scale_f, batch_out + list(x2_scale_f.shape[-2:]))
296	- out = (out * deq_scale_tensor).numpy().astype(out_dtype)	296	+
297	- elif out_dtype == 'float32':	297	+ batch_all = int(np.prod(batch_out))
298	- if is_pertoken:	298	+ x1 = x1.reshape([batch_all] + list(x1.shape[-2:]))
299	- pertoken_scale_slice = torch.unsqueeze(torch.from_numpy(pertoken_scale), dim=1).to(torch.float32)	299	+ x2 = x2.reshape([batch_all] + list(x2.shape[-2:]))
300	- out = (out * deq_scale_tensor * pertoken_scale_slice).numpy().astype(out_dtype)	300	+ x1_scale_f = x1_scale_f.reshape([batch_all] + list(x1_scale_f.shape[-2:]))
301	- elif is_two_scale:	301	+ x2_scale_f = x2_scale_f.reshape([batch_all] + list(x2_scale_f.shape[-2:]))
302	- two_scale = scale_generate(pertoken_scale * deq_scale)	302	+
303	- two_scale_tensor = torch.unsqueeze(torch.from_numpy(two_scale), dim=1).to(torch.float32)	303	+ m = x1.shape[-2]
304	- out = (out * two_scale_tensor).numpy().astype(out_dtype)	304	+ k = x1.shape[-1]
305	- else:	305	+ n = x2.shape[-1]
306	- out = (out * deq_scale_tensor).numpy().astype(out_dtype)	306	+
307	- elif out_dtype == 'int32':	307	+ x1_scale_m = np.repeat(x1_scale_f, group_size_m, axis=-2)
308	- out = out.numpy().astype(out_dtype)	308	+ x1_scale_m = x1_scale_m[..., :m, :]
309	- else:	309	+ x2_scale_n = np.repeat(x2_scale_f, group_size_n, axis=-1)
310	- print("Please check whether this dtype '{out_dtype}' is supported")	310	+ x2_scale_n = x2_scale_n[..., :n]
311	-	311	+
312	- if os.path.exists(testcase_name + "_deq_scale.npy"):	312	+ has_batch = x1.ndim > 2
313	- os.remove(testcase_name + "_deq_scale.npy")	313	+ if has_batch:
314	- if os.path.exists(testcase_name + "_offset.npy"):	314	+ y = np.zeros((batch_all, m, n), dtype=np.float32)
315	- os.remove(testcase_name + "_offset.npy")	315	+ for i in range(batch_all):
316	- return out	316	+ for k_idx in range(_ceil_div(k, group_size_k)):
317	-	317	+ k_start = k_idx * group_size_k
318	-def unpack_groupsize(group_size):	318	+ k_end = min((k_idx + 1) * group_size_k, k)
319	- group_size_M = (group_size >> 32) & 0xFFFF	319	+ x1_s_col = np.expand_dims(x1_scale_m[i, :, k_idx], axis=1)
320	- group_size_N = (group_size >> 16) & 0xFFFF	320	+ x2_s_row = np.expand_dims(x2_scale_n[i, k_idx, :], axis=0)
321	- group_size_K = group_size & 0xFFFF	321	+ scale_mul = x1_s_col * x2_s_row
322	- # 当前只有0和1的情况，后续需要推导再改	322	+ y[i] += np.matmul(x1[i, :, k_start:k_end], x2[i, k_start:k_end, :]) * scale_mul
323	- if group_size_M == 0:	323	+ if batch_out:
324	- group_size_M = 1	324	+ y = y.reshape(batch_out + [m, n])
325	- if group_size_N == 0:	325	+ else:
326	- group_size_N = 1	326	+ y = np.zeros((m, n), dtype=np.float32)
327	- return group_size_M, group_size_N, group_size_K	327	+ for k_idx in range(_ceil_div(k, group_size_k)):
328	-	328	+ k_start = k_idx * group_size_k
329	-def ceil_div(a, b):	329	+ k_end = min((k_idx + 1) * group_size_k, k)
330	- return (a + b - 1) // b	330	+ x1_s_col = np.expand_dims(x1_scale_m[:, k_idx], axis=1)
331	-	331	+ x2_s_row = np.expand_dims(x2_scale_n[k_idx, :], axis=0)
332	-def gen_axes_for_transpose(offset, base):	332	+ scale_mul = x1_s_col * x2_s_row
333	- return [x for x in range(offset)] + [x + offset for x in base]	333	+ y += np.matmul(x1[:, k_start:k_end], x2[k_start:k_end, :]) * scale_mul
334	-	334	+
335	-def f32_2_s9(array):	335	+ if bias is not None:
336	- array_round = np.round(array)	336	+ y = y + bias.astype(np.float32)
337	- array_round_clip = np.clip(array_round, -256, 255)	337	+ return _cast_output_dtype(y, y_dtype)
338	- return array_round_clip	338	+
339	-	339	+
340	-def scale_generate(fp32_deq_scale):	340	+def _compute_pertoken(matmul_out, x2_scale, x1_scale, bias, bias_dtype,
341	- uint32_deq_scale = np.frombuffer(fp32_deq_scale, np.uint32)	341	+ is_bias_vec, y_dtype, x1_dtype):
342	- #与高19位运算，模拟硬件	342	+ # K-C(PerToken参与) 计算流程:
343	- uint32_deq_scale &= 0XFFFFE000	343	+ # 1. INT32 bias前加 (int32域加法, 与硬件fixpipe一致)
344	- fp32_deq_scale = np.frombuffer(uint32_deq_scale, np.float32)	344	+ # 2. 转float32: 判断isDoubleScale → merged_scale或分离scale乘法 (float32域)
345	-	345	+ # 3. float32 bias(非is_bias_vec)和is_bias_vec的bias均在scale乘法后加入 (与硬件fixpipe一致)
346	- return fp32_deq_scale	346	+ # 4. cast到y_dtype
		347	+ out = matmul_out
		348	+ if bias is not None and bias_dtype == "int32":
		349	+ out = out + bias
		350	+
		351	+ out = out.astype(np.float32)
		352	+ is_two_scale = x1_dtype in ("float8_e4m3fn", "float8_e5m2", "hifloat8") and \
		353	+ x1_scale.shape[0] == 1 and x2_scale.shape[0] == 1
		354	+ if is_two_scale:
		355	+ merged_scale = x2_scale.astype(np.float32) * x1_scale.astype(np.float32)
		356	+ merged_scale_slice = np.expand_dims(merged_scale, axis=1)
		357	+ out = out * merged_scale_slice
		358	+ else:
		359	+ x1_scale_slice = np.expand_dims(x1_scale, axis=1).astype(np.float32)
		360	+ out = out * x2_scale.astype(np.float32) * x1_scale_slice
		361	+
		362	+ if not is_bias_vec and bias is not None and bias_dtype == "float32":
		363	+ out = out + bias.astype(np.float32)
		364	+ if is_bias_vec and bias is not None:
		365	+ return _cast_output_dtype(out + bias.astype(np.float32), y_dtype)
		366	+ if y_dtype == 'float32':
		367	+ return out.astype(np.float32)
		368	+ return _cast_output_dtype(out, y_dtype)
		369	+
		370	+
		371	+def _compute_tc(matmul_out, x2_scale, bias, bias_dtype,
		372	+ do_scale_gen, is_bias_vec, y_dtype):
		373	+ # T-C(PerTensor/PerChannel, x1Scale不参与) 计算流程:
		374	+ # customize_inputs已将UINT64 scale转为float32 deq_scale,
		375	+ # T-C-static(uint64)和T-C-dynamic(float32/bf16)合并处理, bias位置由is_bias_vec决定.
		376	+ #
		377	+ # 计算流程:
		378	+ # 1. 非is_bias_vec: +bias (int32域/float32域加法, 反量化前加入, 与TTK golden及_compute_pertoken一致)
		379	+ # 2. 转float32
		380	+ # 3. 若do_scale_gen=True: 截断x2Scale高19位
		381	+ # 4. x2Scale若1维则expand到2维; out = out × x2Scale (float32域乘法)
		382	+ # 5. is_bias_vec时+bf16/fp16/fp32 bias (fixpipe后处理, 反量化后加入)
		383	+ # 6. cast到y_dtype
		384	+ out = matmul_out
		385	+
		386	+ if not is_bias_vec and bias is not None:
		387	+ if bias_dtype == "int32":
		388	+ out = out + bias
		389	+ elif bias_dtype in ("float32", "bfloat16"):
		390	+ out = out.astype(np.float32) + bias.astype(np.float32)
		391	+
		392	+ out = out.astype(np.float32)
		393	+ if do_scale_gen:
		394	+ x2_scale = _scale_generate(x2_scale)
		395	+ if x2_scale.ndim == 1:
		396	+ x2_scale = np.expand_dims(x2_scale, axis=0)
		397	+ out = out * x2_scale.astype(np.float32)
		398	+
		399	+ if is_bias_vec and bias is not None:
		400	+ return _cast_output_dtype(out + bias.astype(np.float32), y_dtype)
		401	+ if y_dtype == 'float32':
		402	+ return out.astype(np.float32)
		403	+ return _cast_output_dtype(out, y_dtype)
		404	+
		405	+
		406	+def _compute_requant(matmul_out, x2_scale, offset, bias, bias_dtype):
		407	+ # Requant(y=INT8) 计算流程 — T-C的输出dtype变体:
		408	+ # 1. +INT32 bias (int32域加法, 与TTK一致, 反量化前加入)
		409	+ # 2. 转float32: f32_2_s9(out × x2Scale) — float32域乘法 + 9bit量化截断
		410	+ # 3. +f32_2_s9(offset) (若有)
		411	+ # 4. clip[-128, 127] → int8
		412	+ # 注: 仅INT8×INT8+UINT64 scale支持requant
		413	+ out = matmul_out
		414	+ if bias is not None and bias_dtype == "int32":
		415	+ out = out + bias
		416	+
		417	+ out = out.astype(np.float32)
		418	+ out = _f32_2_s9(out * x2_scale.astype(np.float32))
		419	+ if offset is not None:
		420	+ out = _f32_2_s9(out) + _f32_2_s9(offset)
		421	+ return np.clip(out, -128, 127).astype(np.int8)
		422	+
		423	+
		424	+def _compute_int32(matmul_out, bias, bias_dtype):
		425	+ # 纯整数(y=INT32) 计算流程 — scale不参与计算的输出dtype变体:
		426	+ # 1. +INT32 bias (int32域加法, 与TTK一致); float32 bias在float32域加法
		427	+ # 2. cast到int32
		428	+ # 注: 仅INT8×INT8支持; scale不参与, bias位置不影响最终结果
		429	+ out = matmul_out
		430	+ if bias is not None:
		431	+ if bias_dtype == "int32":
		432	+ out = out + bias
		433	+ else:
		434	+ out = out.astype(np.float32) + bias.astype(np.float32)
		435	+ return out.astype(np.int32)
		436	+
		437	+
		438	+# ----------------------------------------------------------------------------
		439	+# 量化模式判定与辅助函数
		440	+# ----------------------------------------------------------------------------
		441	+
		442	+def _determine_quant_mode(x1_dtype, x2_dtype, x1_scale, x2_scale,
		443	+ scale_dtype, group_size_n):
		444	+ # 量化模式判定流程对齐op_tiling代码: 双轴(x1+x2)独立判定 → 组合映射
		445	+ # Step1判x2(x2Scale侧): P1(MX)→P2(PerBlock)→P3(PerTensor)→P4(PerChannel)
		446	+ # Step2判x1(x1Scale侧): P0(skip)→P1(MX)→P2(PerBlock)→P3(PERTENSOR/isDoubleScale)→P4(PERTOKEN)
		447	+ # Step3组合映射 → 量化场景名 + 公式
		448	+ # 输出dtype变体(y=INT8/INT32)不影响量化模式判定, 由_compute_requant/_compute_int32处理
		449	+ # T-C不再区分static/dynamic子模式: customize_inputs已将UINT64转为float32 deq_scale,
		450	+ # bias位置由is_bias_vec决定(非is_bias_vec→scale前加, is_bias_vec→scale后加)
		451	+ #
		452	+ # 模式名(与op_tiling代码boolean flags的对应):
		453	+ # G-G = isMxPerGroup (MX_PERGROUP+MX_PERGROUP)
		454	+ # B-B = isPerBlock (PERBLOCK+PERBLOCK, 仅FP8/HIF8)
		455	+ # K-C = isPertoken+isPerChannel/isPerTensor (PERTOKEN+PERCHANNEL/PERTENSOR,
		456	+ # 或PERTENSOR+PERTENSOR(isDoubleScale); 公式统一, numpy广播无需区分)
		457	+ # T-C = isPerTensor/isPerChannel (DEFAULT+PERTENSOR/PERCHANNEL, bias位置由is_bias_vec决定)
		458	+ fp8_hif8 = ("float8_e4m3fn", "float8_e5m2", "hifloat8")
		459	+ fp4_fp8 = ("float4_e2m1", "float4_e1m2", "float8_e4m3fn", "float8_e5m2")
		460	+
		461	+ # Step1 P1 + Step2 P1: MX — both x1 and x2 must be fp4/fp8
		462	+ # FP8×FP4 mx伪量化已去除: x1和x2必须同属fp4或fp8族
		463	+ if x1_scale is not None and \
		464	+ x1_dtype in fp4_fp8 and x2_dtype in fp4_fp8 and \
		465	+ scale_dtype == "float8_e8m0":
		466	+ return "G-G"
		467	+
		468	+ # Step1 P2 + Step2 P2: PerBlock (B-B) — 仅FP8/HIF8, 不含INT8/INT4
		469	+ # 代码条件: isFp8Hif8Input ∧ scaleDtype==FP32 ∧ scaleShapeLen>1
		470	+ # ∧ (scaleShapeLen>2 ∨ scaleDim0!=1 ∨ groupSizeN>1)
		471	+ # isFp8Hif8Input ∧ perTokenScaleDtype==FP32 ∧ pertokenLen>1
		472	+ if x1_scale is not None and \
		473	+ x1_dtype in fp8_hif8 and \
		474	+ scale_dtype == "float32" and \
		475	+ _dtype_to_str(x1_scale.dtype) == "float32" and \
		476	+ x2_scale.ndim > 1 and \
		477	+ (x2_scale.ndim > 2 or x2_scale.shape[0] != 1 or group_size_n > 1) and \
		478	+ x1_scale.ndim > 1:
		479	+ return "B-B"
		480	+
		481	+ # Step2 P3-P4: x1Scale参与 — K-C/T-C动态FP8/HIF8/T-T动态(isDoubleScale)
		482	+ # 公式统一: out = matmul × x2Scale × x1Scale [+bias]
		483	+ if x1_scale is not None:
		484	+ return "K-C"
		485	+
		486	+ # Step2 P0: x1Scale不参与(pertoken_scale=null) — T-C
		487	+ # bias位置由is_bias_vec决定(与TTK golden及_compute_pertoken一致):
		488	+ # 非is_bias_vec → bias在scale前加入 (int32域/float32域加法)
		489	+ # is_bias_vec → bias在scale后加入 (fixpipe后处理)
		490	+ return "T-C"
		491	+
		492	+
		493	+def _needs_scale_generate(x1_dtype, x2_scale, bias_dtype, scale_dtype):
		494	+ # 950硬件对int8/int4输入做fixpipe精度截断(scale_generate高19位掩码)
		495	+ # 条件: int8/int4 + 单行x2Scale + bias不是bf16/f32 + x2Scale原始dtype非uint64/int64
		496	+ # uint64/int64的deq_scale已是截断精度(customize_inputs中_u64_to_deq_scale已做0xFFFFE000掩码),
		497	+ # 即使apply scale_generate也无副作用(已是截断值再截断=原值), 但为避免不必要的计算, 仍跳过
		498	+ if x1_dtype not in ("int8", "int4"):
		499	+ return False
		500	+ if scale_dtype in ("uint64", "int64"):
		501	+ return False
		502	+ if x2_scale.shape[0] != 1:
		503	+ return False
		504	+ if bias_dtype in ("bfloat16", "float32"):
		505	+ return False
		506	+ return True
		507	+
		508	+
		509	+def _u64_to_deq_scale(u64_scale):
		510	+ deq_u32 = u64_scale.astype(np.uint32).copy()
		511	+ deq_u32 &= np.uint32(0xFFFFE000)
		512	+ return deq_u32.view(np.float32).reshape(u64_scale.shape)
		513	+
		514	+
		515	+def _u64_to_offset(u64_scale):
		516	+ raw = (u64_scale.astype(np.uint64) >> np.uint64(37)) & np.uint64(0x1FF)
		517	+ raw = raw.astype(np.int64)
		518	+ sign_mask = np.int64(0x100)
		519	+ raw = np.where(raw & sign_mask, raw - np.int64(0x200), raw)
		520	+ return raw.astype(np.float32).reshape(u64_scale.shape)
		521	+
		522	+
		523	+def _scale_generate(fp32_array):
		524	+ u32 = fp32_array.view(np.uint32).copy()
		525	+ u32 &= np.uint32(0xFFFFE000)
		526	+ return u32.view(np.float32)
		527	+
		528	+
		529	+def _f32_2_s9(array):
		530	+ return np.clip(np.round(array), -256, 255)
		531	+
		532	+
		533	+def _ceil_div(a, b):
		534	+ return (a + b - 1) // b
		535	+
		536	+
		537	+def _unpack_groupsize(group_size):
		538	+ gs_m = (group_size >> 32) & 0xFFFF
		539	+ gs_n = (group_size >> 16) & 0xFFFF
		540	+ gs_k = group_size & 0xFFFF
		541	+ if gs_m == 0:
		542	+ gs_m = 1
		543	+ if gs_n == 0:
		544	+ gs_n = 1
		545	+ if gs_k == 0:
		546	+ gs_k = 1
		547	+ return gs_m, gs_n, gs_k
		548	+
		549	+
		550	+def _gen_axes_for_transpose(offset, base):
		551	+ return [x for x in range(offset)] + [x + offset for x in base]
		552	+
		553	+
		554	+def _cast_output_dtype(arr, dtype_name):
		555	+ dtype_map = {
		556	+ "float16": np.float16, "float32": np.float32, "int32": np.int32,
		557	+ "int8": np.int8, "bfloat16": np_bfloat16,
		558	+ "hifloat8": np_hif8, "float8_e4m3fn": np_fp8_e4m3,
		559	+ "float8_e5m2": np_fp8_e5m2,
		560	+ }
		561	+ target = dtype_map.get(dtype_name)
		562	+ if target is not None:
		563	+ return arr.astype(target)
		564	+ return arr.astype(dtype_name)
		565	+
		566	+
		567	+def _dtype_to_str(dtype):
		568	+ dtype_map = {
		569	+ np.float16: "float16", np.float32: "float32", np.float64: "float64",
		570	+ np.int8: "int8", np.int32: "int32", np.uint64: "uint64",
		571	+ np_bfloat16: "bfloat16", np_int4: "int4",
		572	+ np_fp8_e4m3: "float8_e4m3fn", np_fp8_e5m2: "float8_e5m2",
		573	+ np_hif8: "hifloat8", np_mx_scale: "float8_e8m0",
		574	+ np_fp4_e2m1: "float4_e2m1", np_fp4_e1m2: "float4_e1m2",
		575	+ }
		576	+ return dtype_map.get(dtype, str(dtype))
		577	+
		578	+
		579	+def _nz_to_nd(data, ori_shape):
		580	+ if ori_shape is None:
		581	+ raise ValueError("ori_shape is required for NZ→ND conversion to remove fractal padding")
		582	+ shape = data.shape
		583	+ batch_dims = len(shape) - 4
		584	+ perm = list(range(batch_dims)) + [batch_dims + 1, batch_dims + 2, batch_dims + 0, batch_dims + 3]
		585	+ data = np.transpose(data, perm)
		586	+ m1 = shape[batch_dims + 1]
		587	+ m0_actual = shape[batch_dims + 2]
		588	+ n1 = shape[batch_dims + 0]
		589	+ n0_actual = shape[batch_dims + 3]
		590	+ data = data.reshape(shape[:batch_dims], m1 m0_actual, n1 * n0_actual)
		591	+ target_M = ori_shape[-2]
		592	+ target_N = ori_shape[-1]
		593	+ data = data[..., :target_M, :target_N]
		594	+ return data

Mmatmul/quant_batch_matmul_v3/tests/assets/inputs.py+48-14

	@@ -16,48 +16,82 @@ __input__ = {
16	}	16	}
17		17
18	import numpy as np	18	import numpy as np
		19	+import warnings
		20	+
		21	+UNSIGNED_ONLY_DTYPES = ("float8_e8m0",)
		22	+
		23	+
		24	+def _check_and_fix_unsigned_dtype_nan(tensor, input_index, input_ranges, testcase_name):
		25	+ if tensor is None:
		26	+ return tensor
		27	+ dtype_str = str(tensor.dtype)
		28	+ if not any(d in dtype_str for d in UNSIGNED_ONLY_DTYPES):
		29	+ return tensor
		30	+ fp32_check = tensor.astype(np.float32)
		31	+ nan_count = int(np.isnan(fp32_check).sum())
		32	+ if nan_count == 0:
		33	+ return tensor
		34	+ orig_range = None
		35	+ if input_ranges is not None:
		36	+ try:
		37	+ orig_range = input_ranges[input_index] if input_index < len(input_ranges) else None
		38	+ except (TypeError, IndexError):
		39	+ pass
		40	+ orig_low, orig_high = -10, 10
		41	+ if orig_range is not None and len(orig_range) >= 2:
		42	+ orig_low = orig_range[0] if orig_range[0] is not None else -10
		43	+ orig_high = orig_range[1] if orig_range[1] is not None else 10
		44	+ new_high = max(abs(float(orig_low)), abs(float(orig_high)), 1.0)
		45	+ new_low = max(new_high * 0.001, 1e-6)
		46	+ warnings.warn(
		47	+ f"[{testcase_name}] Input {input_index} dtype={dtype_str} contains {nan_count} NaN "
		48	+ f"values (original range ({orig_low}, {orig_high}) includes negatives). "
		49	+ f"Regenerating with positive range ({new_low}, {new_high})."
		50	+ )
		51	+ new_data = np.random.uniform(new_low, new_high, tensor.shape).astype(np.float32)
		52	+ return new_data.astype(tensor.dtype)
		53	+
19		54
20	def quant_batch_matmul_v3_inputs(x1, x2, scale, offset = None, bias = None, pertoken_scale = None, *, dtype: int,	55	def quant_batch_matmul_v3_inputs(x1, x2, scale, offset = None, bias = None, pertoken_scale = None, *, dtype: int,
21	transpose_x1: bool = False, transpose_x2: bool = False,	56	transpose_x1: bool = False, transpose_x2: bool = False,
22	group_size:int = 0, **kwargs):	57	group_size:int = 0, **kwargs):
23	- # 获取数据
24	input_deq_scale = scale	58	input_deq_scale = scale
25	output_dtypes = kwargs['output_dtypes']	59	output_dtypes = kwargs['output_dtypes']
26	out_dtype = output_dtypes[0]	60	out_dtype = output_dtypes[0]
27	- testcase_name = kwargs['testcase_name']	61	+ testcase_name = kwargs.get('testcase_name', 'unknown')
		62	+ input_ranges = kwargs.get('input_ranges', None)
28		63
29	- # convert scale to uint64	64	+ x1 = _check_and_fix_unsigned_dtype_nan(x1, 0, input_ranges, testcase_name)
30	- if input_deq_scale.dtype == "uint64" and pertoken_scale is None:	65	+ x2 = _check_and_fix_unsigned_dtype_nan(x2, 1, input_ranges, testcase_name)
		66	+ input_deq_scale = _check_and_fix_unsigned_dtype_nan(input_deq_scale, 2, input_ranges, testcase_name)
		67	+ pertoken_scale = _check_and_fix_unsigned_dtype_nan(pertoken_scale, 5, input_ranges, testcase_name)
		68	+
		69	+ if input_deq_scale.dtype in ("uint64", "int64") and pertoken_scale is None:
31	deq_scale_shape = scale.shape	70	deq_scale_shape = scale.shape
32	- input_deq_scale = scale_generate(deq_scale_shape, offset, out_dtype, testcase_name)	71	+ target_dtype = input_deq_scale.dtype
		72	+ input_deq_scale = scale_generate(deq_scale_shape, offset, out_dtype)
		73	+ if target_dtype == "int64":
		74	+ input_deq_scale = input_deq_scale.astype(np.int64)
33		75
34	return x1, x2, input_deq_scale, offset, bias, pertoken_scale	76	return x1, x2, input_deq_scale, offset, bias, pertoken_scale
35		77
36	-def scale_generate(deq_scale_shape, offset, out_dtype, testcase_name):	78	+def scale_generate(deq_scale_shape, offset, out_dtype):
37	has_offset = offset is not None	79	has_offset = offset is not None
38		80
39	fp32_deq_scale = np.random.uniform(low=-5, high=5, size=deq_scale_shape).astype(np.float32)	81	fp32_deq_scale = np.random.uniform(low=-5, high=5, size=deq_scale_shape).astype(np.float32)
40	uint32_deq_scale = np.frombuffer(fp32_deq_scale, np.uint32).reshape(deq_scale_shape)	82	uint32_deq_scale = np.frombuffer(fp32_deq_scale, np.uint32).reshape(deq_scale_shape)
41	- # 与高19位运算，模拟硬件
42	uint32_deq_scale &= 0XFFFFE000	83	uint32_deq_scale &= 0XFFFFE000
43		84
44	if has_offset:	85	if has_offset:
45	offset_shape = offset.shape	86	offset_shape = offset.shape
46	fp32_offset = np.random.uniform(low=-5, high=5, size=offset_shape).astype(np.float32)	87	fp32_offset = np.random.uniform(low=-5, high=5, size=offset_shape).astype(np.float32)
47		88
48	- # dequant
49	if out_dtype != "int8":	89	if out_dtype != "int8":
50	- fp32_deq_scale = np.frombuffer(uint32_deq_scale, np.float32).reshape(deq_scale_shape)
51	- np.save(testcase_name + "_deq_scale.npy", fp32_deq_scale)
52	uint64_deq_scale = np.zeros(deq_scale_shape, np.uint64)	90	uint64_deq_scale = np.zeros(deq_scale_shape, np.uint64)
53	uint64_deq_scale \|= np.uint64(uint32_deq_scale)	91	uint64_deq_scale \|= np.uint64(uint32_deq_scale)
54	- # requant
55	elif out_dtype == "int8":	92	elif out_dtype == "int8":
56	- fp32_deq_scale = np.frombuffer(uint32_deq_scale, np.float32).reshape(deq_scale_shape)
57	- np.save(testcase_name + "_deq_scale.npy", fp32_deq_scale)
58	s9_offset = 0	93	s9_offset = 0
59	if has_offset:	94	if has_offset:
60	- np.save(testcase_name + "_offset.npy", fp32_offset)
61	s9_offset = f32_2_s9(fp32_offset).astype(int).reshape(offset_shape)	95	s9_offset = f32_2_s9(fp32_offset).astype(int).reshape(offset_shape)
62	s9_offset &= 0X1FF	96	s9_offset &= 0X1FF
63	s9_offset = s9_offset[0] if deq_scale_shape[-1] < offset_shape[-1] else s9_offset	97	s9_offset = s9_offset[0] if deq_scale_shape[-1] < offset_shape[-1] else s9_offset

Mmatmul/quant_batch_matmul_v3/tests/st/arch35/ttk_kernel_quant_batch_matmul_v3_st.csv+27-27

Mmatmul/transpose_quant_batch_mat_mul/tests/assets/golden.py+220-32

	@@ -9,49 +9,235 @@
9	# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.	9	# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
10	# See LICENSE in the root of the software repository for the full text of the License.	10	# See LICENSE in the root of the software repository for the full text of the License.
11	# ----------------------------------------------------------------------------	11	# ----------------------------------------------------------------------------
		12	+import ml_dtypes
		13	+import numpy as np
		14	+from ml_dtypes import bfloat16, float8_e5m2, float8_e4m3fn
		15	+
12	__golden__ = {	16	__golden__ = {
13	- "e2e": {	17	+ "kernel": {
14	- "torch_npu.npu_transpose_quant_batchmatmul": "torch_npu_npu_transpose_quant_batchmatmul_golden"	18	+ "transpose_quant_batch_mat_mul": "transpose_quant_batch_mat_mul_golden"
15	- }	19	+ },
		20	+ "e2e": {
		21	+ "torch_npu.npu_transpose_quant_batchmatmul": "torch_npu_npu_transpose_quant_batchmatmul_golden"
		22	+ }
16	}	23	}
17		24
18	-import ml_dtypes	25	+_DTYPE_ATTR_MAP = {1: np.float16, 27: bfloat16}
19	-import torch
20	-import numpy as np
21		26
		27	+_FRACTAL_DIMS = {
		28	+ "float16": (16, 16),
		29	+ "bfloat16": (16, 16),
		30	+ "float32": (16, 8),
		31	+}
		32	+
		33	+
		34	+def customize_inputs(x1, x2, bias=None, x1_scale=None, x2_scale=None, *,
		35	+ dtype=1, group_size=0, perm_x1=(1, 0, 2), perm_x2=(0, 1, 2),
		36	+ perm_y=(1, 0, 2), batch_split_factor=1, **kwargs):
		37	+ input_formats = kwargs.get('input_formats', ())
		38	+ input_ori_shapes = kwargs.get('input_ori_shapes', ())
		39	+ if len(input_formats) > 1 and input_formats[1] == 'FRACTAL_NZ':
		40	+ x2_dtype_str = _dtype_to_str(x2.dtype)
		41	+ ori_shape = input_ori_shapes[1] if len(input_ori_shapes) > 1 else None
		42	+ x2 = _nz_to_nd(x2, x2_dtype_str, ori_shape)
		43	+ return x1, x2, bias, x1_scale, x2_scale
		44	+
		45	+
		46	+def pre_compare(outputs, *kwargs):
		47	+ return list(outputs)
		48	+
		49	+
		50	+def transpose_quant_batch_mat_mul_golden(x1, x2, bias=None, x1_scale=None,
		51	+ x2_scale=None, *, dtype=1, group_size=0,
		52	+ perm_x1=(1, 0, 2), perm_x2=(0, 1, 2),
		53	+ perm_y=(1, 0, 2), batch_split_factor=1,
		54	+ **kwargs):
		55	+ x1, x2, bias, x1_scale, x2_scale = customize_inputs(
		56	+ x1, x2, bias, x1_scale, x2_scale,
		57	+ dtype=dtype, group_size=group_size, perm_x1=perm_x1, perm_x2=perm_x2,
		58	+ perm_y=perm_y, batch_split_factor=batch_split_factor, **kwargs)
		59	+
		60	+ x1_f32 = x1.astype(np.float32)
		61	+ x2_f32 = x2.astype(np.float32)
		62	+
		63	+ x1_t = np.transpose(x1_f32, axes=list(perm_x1))
		64	+
		65	+ if tuple(perm_x2) == (0, 2, 1):
		66	+ x2_t = np.swapaxes(x2_f32, -2, -1)
		67	+ if x2_scale is not None and x2_scale.ndim == 4:
		68	+ x2_scale = np.transpose(x2_scale, (0, 2, 1, 3))
		69	+ else:
		70	+ x2_t = x2_f32
		71	+
		72	+ is_mx = _is_mx_scale(x1_scale)
		73	+
		74	+ if is_mx:
		75	+ acc = _mxfp8_matmul(x1_t, x2_t, x1_scale, x2_scale)
		76	+ else:
		77	+ acc = _kc_matmul(x1_t, x2_t, x1_scale, x2_scale)
		78	+
		79	+ out = np.transpose(acc, axes=list(perm_y))
		80	+
		81	+ output_dtypes = kwargs.get("output_dtypes", None)
		82	+ if output_dtypes is not None:
		83	+ out = _cast_output_dtype(out, output_dtypes[0])
		84	+ else:
		85	+ target_dtype = _DTYPE_ATTR_MAP.get(dtype, np.float16)
		86	+ out = out.astype(target_dtype)
		87	+
		88	+ return [out]
		89	+
		90	+
		91	+def _is_mx_scale(scale):
		92	+ if scale is None:
		93	+ return False
		94	+ dtype_str = str(scale.dtype)
		95	+ return "e8m0" in dtype_str
		96	+
		97	+
		98	+def _e8m0_to_float(arr):
		99	+ raw = arr.view(np.uint8).astype(np.float32)
		100	+ result = np.power(2.0, raw - 127.0)
		101	+ result[raw == 255.0] = np.nan
		102	+ return result
		103	+
		104	+
		105	+def _kc_matmul(x1, x2, x1_scale, x2_scale):
		106	+ acc = np.matmul(x1, x2)
		107	+
		108	+ if x2_scale is not None:
		109	+ acc = acc * x2_scale.astype(np.float32).reshape(1, 1, -1)
		110	+
		111	+ if x1_scale is not None:
		112	+ acc = acc * x1_scale.astype(np.float32).reshape(1, -1, 1)
		113	+
		114	+ return acc
		115	+
		116	+
		117	+def _mxfp8_matmul(x1, x2, x1_scale, x2_scale):
		118	+ B, M, K = x1.shape
		119	+ N = x2.shape[2]
		120	+ group_size_k = 32
		121	+ num_groups = K // group_size_k
		122	+
		123	+ x1_scale_f = _e8m0_to_float(x1_scale)
		124	+ x2_scale_f = _e8m0_to_float(x2_scale)
		125	+
		126	+ x1_scale_flat = x1_scale_f.reshape(M, B, num_groups)
		127	+ x1_scale_flat = np.transpose(x1_scale_flat, (1, 0, 2))
		128	+
		129	+ x2_scale_flat = x2_scale_f.reshape(B, num_groups, N)
		130	+
		131	+ out = np.zeros((B, M, N), dtype=np.float32)
		132	+ for g in range(num_groups):
		133	+ k_start = g * group_size_k
		134	+ k_end = (g + 1) * group_size_k
		135	+ partial = np.matmul(x1[:, :, k_start:k_end], x2[:, k_start:k_end, :])
		136	+ s1 = x1_scale_flat[:, :, g][:, :, np.newaxis]
		137	+ s2 = x2_scale_flat[:, g, :][:, np.newaxis, :]
		138	+ out += partial * s1 * s2
		139	+
		140	+ return out
		141	+
		142	+
		143	+class TransposeQuantBatchMatMulAssets:
		144	+
		145	+ golden = transpose_quant_batch_mat_mul_golden
		146	+ customize_inputs = customize_inputs
		147	+ pre_compare = pre_compare
		148	+
		149	+ tolerance = {
		150	+ "float16": {
		151	+ "standard": "BenchmarkCompareStandard",
		152	+ "avg_re_rtol": 2.0,
		153	+ "max_re_rtol": 10.0,
		154	+ "rmse_rtol": 2.0,
		155	+ "small_value": 0.001,
		156	+ "small_value_atol": 1e-5,
		157	+ },
		158	+ "bfloat16": {
		159	+ "standard": "BenchmarkCompareStandard",
		160	+ "avg_re_rtol": 2.0,
		161	+ "max_re_rtol": 10.0,
		162	+ "rmse_rtol": 2.0,
		163	+ "small_value": 0.001,
		164	+ "small_value_atol": 1e-5,
		165	+ },
		166	+ }
		167	+
		168	+
		169	+def _dtype_to_str(dtype):
		170	+ dtype_map = {
		171	+ np.float16: "float16",
		172	+ np.float32: "float32",
		173	+ np.float64: "float64",
		174	+ bfloat16: "bfloat16",
		175	+ }
		176	+ return dtype_map.get(dtype, str(dtype))
		177	+
		178	+
		179	+def _cast_output_dtype(arr, dtype_name):
		180	+ dtype_map = {"float16": np.float16, "float32": np.float32, "bfloat16": bfloat16}
		181	+ target = dtype_map.get(dtype_name)
		182	+ if target is not None:
		183	+ return arr.astype(target)
		184	+ return arr.astype(dtype_name)
		185	+
		186	+
		187	+def _nz_to_nd(data, dtype_str, ori_shape=None):
		188	+ m0, n0 = _FRACTAL_DIMS.get(dtype_str, (16, 16))
		189	+ shape = data.shape
		190	+ batch_dims = len(shape) - 4
		191	+ perm = list(range(batch_dims)) + [batch_dims + 1, batch_dims + 2, batch_dims + 0, batch_dims + 3]
		192	+ data = np.transpose(data, perm)
		193	+ m1 = shape[batch_dims + 1]
		194	+ m0_actual = shape[batch_dims + 2]
		195	+ n1 = shape[batch_dims + 0]
		196	+ n0_actual = shape[batch_dims + 3]
		197	+ data = data.reshape(shape[:batch_dims], m1 m0_actual, n1 * n0_actual)
		198	+ if ori_shape is not None:
		199	+ target_M = ori_shape[-2]
		200	+ target_N = ori_shape[-1]
		201	+ data = data[..., :target_M, :target_N]
		202	+ return data
		203	+
		204	+
		205	+# ===== E2E golden (torch-based, retained for e2e mode) =====
22		206
23	def torch_npu_npu_transpose_quant_batchmatmul_golden(x1, x2, dtype, *, bias=None, x1_scale=None,	207	def torch_npu_npu_transpose_quant_batchmatmul_golden(x1, x2, dtype, *, bias=None, x1_scale=None,
24	- x2_scale=None, group_sizes=None, perm_x1=None,	208	+ x2_scale=None, group_sizes=None, perm_x1=None,
25	- perm_x2=None, perm_y=None, batch_split_factor=None, **kwargs):	209	+ perm_x2=None, perm_y=None, batch_split_factor=None, **kwargs):
26	- x1 = torch_to_numpy(x1)	210	+ import torch
27	- x2 = torch_to_numpy(x2)	211	+ x1 = _torch_to_numpy(x1)
28	- bias = torch_to_numpy(bias)	212	+ x2 = _torch_to_numpy(x2)
29	- x1_scale = torch_to_numpy(x1_scale)	213	+ bias = _torch_to_numpy(bias)
30	- x2_scale = torch_to_numpy(x2_scale)	214	+ x1_scale = _torch_to_numpy(x1_scale)
		215	+ x2_scale = _torch_to_numpy(x2_scale)
31	x1_dtype = str(x1.dtype)	216	x1_dtype = str(x1.dtype)
32	x2_dtype = str(x2.dtype)	217	x2_dtype = str(x2.dtype)
33	x1_scale_dtype = ""	218	x1_scale_dtype = ""
34	if isinstance(x1_scale, torch.Tensor):	219	if isinstance(x1_scale, torch.Tensor):
35	x1_scale_dtype = str(x1_scale.dtype)	220	x1_scale_dtype = str(x1_scale.dtype)
36	- pertoken_flag = (x1_scale is not None	221	+ pertoken_flag = (x1_scale is not None
37	- and x1_dtype in ("int8", "float8_e4m3fn", "float8_e5m2")	222	+ and x1_dtype in ("int8", "float8_e4m3fn", "float8_e5m2")
38	- and x1_scale_dtype in ("float32",))	223	+ and x1_scale_dtype in ("float32",))
39		224
40	- x1 = transpose_(x1, perm_x1)	225	+ x1 = _transpose_(x1, perm_x1)
41	- x2 = transpose_(x2, perm_x2)	226	+ x2 = _transpose_(x2, perm_x2)
42		227
43	- x1 = x_dtype_cope(x1, x1_dtype)	228	+ x1 = _x_dtype_cope(x1, x1_dtype)
44	- x2 = x_dtype_cope(x2, x2_dtype)	229	+ x2 = _x_dtype_cope(x2, x2_dtype)
45	if pertoken_flag:	230	if pertoken_flag:
46	- out = pertoken_calculation(x1, x2, x1_scale, x2_scale, bias, is_bias_epilogue=False, output_dtype=dtype)	231	+ out = _pertoken_calculation(x1, x2, x1_scale, x2_scale, bias, is_bias_epilogue=False, output_dtype=dtype)
47	else:	232	else:
48	raise TypeError("Please check whether this quantitative method is supported")	233	raise TypeError("Please check whether this quantitative method is supported")
49	- out = transpose_(out, perm_y)	234	+ out = _transpose_(out, perm_y)
50	- out = out_dtype_cope(out, dtype)	235	+ out = _out_dtype_cope(out, dtype)
51	return out	236	return out
52		237
53		238
54	-def pertoken_calculation(x1, x2, x1_scale, x2_scale, bias, is_bias_epilogue=False, output_dtype=2):	239	+def _pertoken_calculation(x1, x2, x1_scale, x2_scale, bias, is_bias_epilogue=False, output_dtype=2):
		240	+ import torch
55	x1 = torch.from_numpy(x1)	241	x1 = torch.from_numpy(x1)
56	x2 = torch.from_numpy(x2)	242	x2 = torch.from_numpy(x2)
57	out = torch.matmul(x1, x2)	243	out = torch.matmul(x1, x2)
	@@ -71,7 +257,7 @@ def pertoken_calculation(x1, x2, x1_scale, x2_scale, bias, is_bias_epilogue=Fals
71	return out.numpy()	257	return out.numpy()
72		258
73		259
74	-def x_dtype_cope(x, x_dtype):	260	+def _x_dtype_cope(x, x_dtype):
75	if x_dtype in ("float8_e4m3fn", "float8_e5m2", "float4_e2m1", "float4_e1m2", "hifloat8"):	261	if x_dtype in ("float8_e4m3fn", "float8_e5m2", "float4_e2m1", "float4_e1m2", "hifloat8"):
76	x = x.astype(np.float32)	262	x = x.astype(np.float32)
77	else:	263	else:
	@@ -79,7 +265,8 @@ def x_dtype_cope(x, x_dtype):
79	return x	265	return x
80		266
81		267
82	-def transpose_(x, array_trans_x):	268	+def _transpose_(x, array_trans_x):
		269	+ import torch
83	if isinstance(x, torch.Tensor):	270	if isinstance(x, torch.Tensor):
84	x = torch.permute(x, array_trans_x)	271	x = torch.permute(x, array_trans_x)
85	elif isinstance(x, np.ndarray):	272	elif isinstance(x, np.ndarray):
	@@ -89,8 +276,8 @@ def transpose_(x, array_trans_x):
89	return x	276	return x
90		277
91		278
92	-def out_dtype_cope(out, output_dtype):	279	+def _out_dtype_cope(out, output_dtype):
93	- from ml_dtypes import bfloat16, float8_e4m3fn	280	+ import torch
94	if output_dtype == torch.float32:	281	if output_dtype == torch.float32:
95	out = out.astype(np.float32)	282	out = out.astype(np.float32)
96	elif output_dtype == torch.float16:	283	elif output_dtype == torch.float16:
	@@ -109,12 +296,13 @@ def out_dtype_cope(out, output_dtype):
109	return out	296	return out
110		297
111		298
112	-def torch_to_numpy(x):	299	+def _torch_to_numpy(x):
		300	+ import torch
113	if isinstance(x, torch.Tensor):	301	if isinstance(x, torch.Tensor):
114	if x.dtype == torch.float8_e5m2:	302	if x.dtype == torch.float8_e5m2:
115	- x = x.to(torch.float32).numpy().astype(ml_dtypes.float8_e5m2)	303	+ x = x.to(torch.float32).numpy().astype(float8_e5m2)
116	elif x.dtype == torch.float8_e4m3fn:	304	elif x.dtype == torch.float8_e4m3fn:
117	- x = x.to(torch.float32).numpy().astype(ml_dtypes.float8_e4m3fn)	305	+ x = x.to(torch.float32).numpy().astype(float8_e4m3fn)
118	else:	306	else:
119	x = x.numpy()	307	x = x.numpy()
120	- return x	308	+ return x

	@@ -1,27 +1,27 @@
1	-testcase_name,op_name,input_shapes,input_ori_shapes,output_shapes,output_ori_shapes,input_dtypes,output_dtypes,input_formats,input_ori_formats,output_formats,output_ori_formats,attributes,input_data_ranges,precision_tolerances,absolute_precision,is_enabled,m_feat,n_feat,k_feat	1	+testcase_name,op_name,input_shapes,input_ori_shapes,output_shapes,output_ori_shapes,input_dtypes,output_dtypes,input_formats,input_ori_formats,output_formats,output_ori_formats,attributes,input_data_ranges,precision_tolerances,absolute_precision,is_enabled,m_feat,n_feat,k_feat
2	-qbmm_BB_case0001,quant_batch_matmul_v3,"((1, 1008), (1008, 144), (8, 2), None, None, (1, 8))","((1, 1008), (1008, 144), (8, 2), None, None, (1, 8))","((1, 144),)","((1, 144),)","('float8_e4m3fn', 'float8_e5m2', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False, 'group_size': 549764202624}","((-3, 3),)","((0.001, 0.001),)",,1,one,multiple_of_16,multiple_of_16	2	+qbmm_BB_case0001,quant_batch_matmul_v3,"((1, 1008), (1008, 144), (8, 2), None, None, (1, 8))","((1, 1008), (1008, 144), (8, 2), None, None, (1, 8))","((1, 144),)","((1, 144),)","('float8_e4m3fn', 'float8_e5m2', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False, 'group_size': 549764202624}","((-3, 3),)","((0.001, 0.001),)",,1,one,multiple_of_16,multiple_of_16
3	-qbmm_BB_case0002,quant_batch_matmul_v3,"((1296, 464), (1296, 1312), (11, 11), None, None, (11, 4))","((1296, 464), (1296, 1312), (11, 11), None, None, (11, 4))","((464, 1312),)","((464, 1312),)","('float8_e5m2', 'float8_e5m2', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False, 'group_size': 549764202624}","((-1000, 1000),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_32,multiple_of_16	3	+qbmm_BB_case0002,quant_batch_matmul_v3,"((1296, 464), (1296, 1312), (11, 11), None, None, (11, 4))","((1296, 464), (1296, 1312), (11, 11), None, None, (11, 4))","((464, 1312),)","((464, 1312),)","('float8_e5m2', 'float8_e5m2', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False, 'group_size': 549764202624}","((-1000, 1000),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_32,multiple_of_16
4	-qbmm_BB_case0004,quant_batch_matmul_v3,"((1663, 883), (861, 883), (7, 7), None, None, (13, 7))","((1663, 883), (861, 883), (7, 7), None, None, (13, 7))","((1663, 861),)","((1663, 861),)","('float8_e4m3fn', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,prime,random,prime	4	+qbmm_BB_case0004,quant_batch_matmul_v3,"((1663, 883), (861, 883), (7, 7), None, None, (13, 7))","((1663, 883), (861, 883), (7, 7), None, None, (13, 7))","((1663, 861),)","((1663, 861),)","('float8_e4m3fn', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,prime,random,prime
5	-qbmm_BB_case0003,quant_batch_matmul_v3,"((944, 1936), (1808, 944), (15, 8), None, None, (8, 16))","((944, 1936), (1808, 944), (15, 8), None, None, (8, 16))","((1936, 1808),)","((1936, 1808),)","('float8_e5m2', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': True, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16	5	+qbmm_BB_case0003,quant_batch_matmul_v3,"((944, 1936), (1808, 944), (15, 8), None, None, (8, 16))","((944, 1936), (1808, 944), (15, 8), None, None, (8, 16))","((1936, 1808),)","((1936, 1808),)","('float8_e5m2', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': True, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16
6	-qbmm_BB_case0015,quant_batch_matmul_v3,"((1741, 1344), (1741, 941), (14, 8), None, None, (14, 11))","((1741, 1344), (1741, 941), (14, 8), None, None, (14, 11))","((1344, 941),)","((1344, 941),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",float32,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,random,random,random	6	+qbmm_BB_case0015,quant_batch_matmul_v3,"((1741, 1344), (1741, 941), (14, 8), None, None, (14, 11))","((1741, 1344), (1741, 941), (14, 8), None, None, (14, 11))","((1344, 941),)","((1344, 941),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",'float32',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,random,random,random
7	-qbmm_BB_case0029,quant_batch_matmul_v3,"((5, 6, 7, 1296, 751), (5, 6, 7, 751, 1), (5, 6, 7, 6, 1), None, None, (5, 6, 7, 11, 6))","((5, 6, 7, 1296, 751), (5, 6, 7, 751, 1), (5, 6, 7, 6, 1), None, None, (5, 6, 7, 11, 6))","((5, 6, 7, 1296, 1),)","((5, 6, 7, 1296, 1),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,one,prime	7	+qbmm_BB_case0029,quant_batch_matmul_v3,"((5, 6, 7, 1296, 751), (5, 6, 7, 751, 1), (5, 6, 7, 6, 1), None, None, (5, 6, 7, 11, 6))","((5, 6, 7, 1296, 751), (5, 6, 7, 751, 1), (5, 6, 7, 6, 1), None, None, (5, 6, 7, 11, 6))","((5, 6, 7, 1296, 1),)","((5, 6, 7, 1296, 1),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False, 'group_size': 549764202624}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,one,prime
8	-qbmm_MXFP4_case0001,quant_batch_matmul_v3,"((197, 960), (854, 960), (854, 15, 2), None, None, (197, 15, 2))","((197, 960), (854, 960), (854, 15, 2), None, None, (197, 15, 2))","((197, 854),)","((197, 854),)","('float4_e2m1', 'float4_e2m1', 'float8_e8m0', 'int8', 'int8', 'float8_e8m0')",float32,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True, 'group_size': 4295032864}","((-3, 3),)","((0.001, 0.001),)",,1,random,random,multiple_of_32	8	+qbmm_MXFP4_case0001,quant_batch_matmul_v3,"((197, 960), (854, 960), (854, 15, 2), None, None, (197, 15, 2))","((197, 960), (854, 960), (854, 15, 2), None, None, (197, 15, 2))","((197, 854),)","((197, 854),)","('float4_e2m1', 'float4_e2m1', 'float8_e8m0', 'int8', 'int8', 'float8_e8m0')",'float32',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True, 'group_size': 4295032864}","((-3, 3),)","((0.001, 0.001),)",,1,random,random,multiple_of_32
9	-qbmm_KT_case0007,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, None, (1,))","((1, 1), (1, 1), (1,), None, None, (1,))","((1, 1),)","((1, 1),)","('float8_e4m3fn', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one	9	+qbmm_KT_case0007,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, None, (1,))","((1, 1), (1, 1), (1,), None, None, (1,))","((1, 1),)","((1, 1),)","('float8_e4m3fn', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one
10	-qbmm_KC_case0027,quant_batch_matmul_v3,"((5, 1, 1, 1), (1, 4, 1, 1), (1,), None, None, (1,))","((5, 1, 1, 1), (1, 4, 1, 1), (1,), None, None, (1,))","((5, 4, 1, 1),)","((5, 4, 1, 1),)","('float8_e5m2', 'float8_e5m2', 'float32', 'int8', 'int8', 'float32')",float32,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one	10	+qbmm_KC_case0027,quant_batch_matmul_v3,"((5, 1, 1, 1), (1, 4, 1, 1), (1,), None, None, (1,))","((5, 1, 1, 1), (1, 4, 1, 1), (1,), None, None, (1,))","((5, 4, 1, 1),)","((5, 4, 1, 1),)","('float8_e5m2', 'float8_e5m2', 'float32', 'int8', 'int8', 'float32')",'float32',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one
11	-qbmm_TC_case0067,quant_batch_matmul_v3,"((5, 1, 576, 560), (1, 3, 560, 256), (256,), None, None, (1,))","((5, 1, 576, 560), (1, 3, 560, 256), (256,), None, None, (1,))","((5, 3, 576, 256),)","((5, 3, 576, 256),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_32,multiple_of_32,multiple_of_16	11	+qbmm_TC_case0067,quant_batch_matmul_v3,"((5, 1, 576, 560), (1, 3, 560, 256), (256,), None, None, (1,))","((5, 1, 576, 560), (1, 3, 560, 256), (256,), None, None, (1,))","((5, 3, 576, 256),)","((5, 3, 576, 256),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_32,multiple_of_32,multiple_of_16
12	-qbmm_KC_case0006,quant_batch_matmul_v3,"((639, 1), (1, 757), (757,), None, None, (639,))","((639, 1), (1, 757), (757,), None, None, (639,))","((639, 757),)","((639, 757),)","('float8_e5m2', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,random,random,one	12	+qbmm_KC_case0006,quant_batch_matmul_v3,"((639, 1), (1, 757), (757,), None, None, (639,))","((639, 1), (1, 757), (757,), None, None, (639,))","((639, 757),)","((639, 757),)","('float8_e5m2', 'float8_e4m3fn', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,random,random,one
13	-qbmm_KC_case0008,quant_batch_matmul_v3,"((1872, 1648), (1744, 1648), (1744,), None, None, (1872,))","((1872, 1648), (1744, 1648), (1744,), None, None, (1872,))","((1872, 1744),)","((1872, 1744),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16	13	+qbmm_KC_case0008,quant_batch_matmul_v3,"((1872, 1648), (1744, 1648), (1744,), None, None, (1872,))","((1872, 1648), (1744, 1648), (1744,), None, None, (1872,))","((1872, 1744),)","((1872, 1744),)","('hifloat8', 'hifloat8', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16
14	-qbmm_TC_case0001,quant_batch_matmul_v3,"((592, 368), (151, 368), (151,), None, None, None)","((592, 368), (151, 368), (151,), None, None, None)","((592, 151),)","((592, 151),)","('int8', 'int8', 'uint64', 'int8', 'int8', 'int8')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,random,multiple_of_16	14	+qbmm_TC_case0001,quant_batch_matmul_v3,"((592, 368), (151, 368), (151,), None, None, None)","((592, 368), (151, 368), (151,), None, None, None)","((592, 151),)","((592, 151),)","('int8', 'int8', 'uint64', 'int8', 'int8', 'int8')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,random,multiple_of_16
15	-qbmm_TC_case0002,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, None, None)","((1, 1), (1, 1), (1,), None, None, None)","((1, 1),)","((1, 1),)","('int8', 'int8', 'uint64', 'int8', 'int8', 'int8')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one	15	+qbmm_TC_case0002,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, None, None)","((1, 1), (1, 1), (1,), None, None, None)","((1, 1),)","((1, 1),)","('int8', 'int8', 'uint64', 'int8', 'int8', 'int8')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one
16	-qbmm_TC_case0015,quant_batch_matmul_v3,"((699, 719), (59, 719), (59,), None, None, None)","((699, 719), (59, 719), (59,), None, None, None)","((699, 59),)","((699, 59),)","('float8_e5m2', 'float8_e4m3fn', 'uint64', 'int8', 'int8', 'int8')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,random,prime,prime	16	+qbmm_TC_case0015,quant_batch_matmul_v3,"((699, 719), (59, 719), (59,), None, None, None)","((699, 719), (59, 719), (59,), None, None, None)","((699, 59),)","((699, 59),)","('float8_e5m2', 'float8_e4m3fn', 'uint64', 'int8', 'int8', 'int8')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,random,prime,prime
17	-qbmm_TC_case0005,quant_batch_matmul_v3,"((257, 447), (257, 941), (941,), None, None, None)","((257, 447), (257, 941), (941,), None, None, None)","((447, 941),)","((447, 941),)","('int8', 'int8', 'bfloat16', 'int8', 'int8', 'int8')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,random,prime,prime	17	+qbmm_TC_case0005,quant_batch_matmul_v3,"((257, 447), (257, 941), (941,), None, None, None)","((257, 447), (257, 941), (941,), None, None, None)","((447, 941),)","((447, 941),)","('int8', 'int8', 'bfloat16', 'int8', 'int8', 'int8')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,random,prime,prime
18	-qbmm_TC_case0012,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, None, None)","((1, 1), (1, 1), (1,), None, None, None)","((1, 1),)","((1, 1),)","('float8_e4m3fn', 'float8_e4m3fn', 'uint64', 'int8', 'int8', 'int8')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one	18	+qbmm_TC_case0012,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, None, None)","((1, 1), (1, 1), (1,), None, None, None)","((1, 1),)","((1, 1),)","('float8_e4m3fn', 'float8_e4m3fn', 'uint64', 'int8', 'int8', 'int8')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one
19	-qbmm_TT_case0001,quant_batch_matmul_v3,"((432, 272), (432, 176), (1,), None, None, None)","((432, 272), (432, 176), (1,), None, None, None)","((272, 176),)","((272, 176),)","('int8', 'int8', 'uint64', 'int8', 'int8', 'int8')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-3, 3),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16	19	+qbmm_TT_case0001,quant_batch_matmul_v3,"((432, 272), (432, 176), (1,), None, None, None)","((432, 272), (432, 176), (1,), None, None, None)","((272, 176),)","((272, 176),)","('int8', 'int8', 'uint64', 'int8', 'int8', 'int8')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-3, 3),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16
20	-qbmm_TT_case0005,quant_batch_matmul_v3,"((691, 272), (691, 592), (1,), None, None, None)","((691, 272), (691, 592), (1,), None, None, None)","((272, 592),)","((272, 592),)","('int8', 'int8', 'bfloat16', 'int8', 'int8', 'int8')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,prime	20	+qbmm_TT_case0005,quant_batch_matmul_v3,"((691, 272), (691, 592), (1,), None, None, None)","((691, 272), (691, 592), (1,), None, None, None)","((272, 592),)","((272, 592),)","('int8', 'int8', 'bfloat16', 'int8', 'int8', 'int8')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,prime
21	-qbmm_KC_case0001,quant_batch_matmul_v3,"((811, 131), (811, 160), (160,), None, None, (131,))","((811, 131), (811, 160), (160,), None, None, (131,))","((131, 160),)","((131, 160),)","('int8', 'int8', 'float32', 'int8', 'int8', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-3, 3),)","((0.001, 0.001),)",,1,prime,multiple_of_32,prime	21	+qbmm_KC_case0001,quant_batch_matmul_v3,"((811, 131), (811, 160), (160,), None, None, (131,))","((811, 131), (811, 160), (160,), None, None, (131,))","((131, 160),)","((131, 160),)","('int8', 'int8', 'float32', 'int8', 'int8', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-3, 3),)","((0.001, 0.001),)",,1,prime,multiple_of_32,prime
22	-qbmm_KC_case0002,quant_batch_matmul_v3,"((194, 1), (1, 1), (1,), None, None, (194,))","((194, 1), (1, 1), (1,), None, None, (194,))","((194, 1),)","((194, 1),)","('int8', 'int8', 'float32', 'int8', 'int8', 'float32')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-1000, 1000),)","((0.001, 0.001),)",,1,random,one,one	22	+qbmm_KC_case0002,quant_batch_matmul_v3,"((194, 1), (1, 1), (1,), None, None, (194,))","((194, 1), (1, 1), (1,), None, None, (194,))","((194, 1),)","((194, 1),)","('int8', 'int8', 'float32', 'int8', 'int8', 'float32')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-1000, 1000),)","((0.001, 0.001),)",,1,random,one,one
23	-qbmm_KT_case0011,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, (1,), (1,))","((1, 1), (1, 1), (1,), None, (1,), (1,))","((1, 1),)","((1, 1),)","('int8', 'int8', 'float32', 'int8', 'float32', 'float32')",float16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one	23	+qbmm_KT_case0011,quant_batch_matmul_v3,"((1, 1), (1, 1), (1,), None, (1,), (1,))","((1, 1), (1, 1), (1,), None, (1,), (1,))","((1, 1),)","((1, 1),)","('int8', 'int8', 'float32', 'int8', 'float32', 'float32')",'float16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,one,one,one
24	-qbmm_KC_case0003,quant_batch_matmul_v3,"((679, 149), (238, 149), (238,), None, None, (679,))","((679, 149), (238, 149), (238,), None, None, (679,))","((679, 238),)","((679, 238),)","('int8', 'int8', 'bfloat16', 'int8', 'int8', 'float32')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,random,random,random	24	+qbmm_KC_case0003,quant_batch_matmul_v3,"((679, 149), (238, 149), (238,), None, None, (679,))","((679, 149), (238, 149), (238,), None, None, (679,))","((679, 238),)","((679, 238),)","('int8', 'int8', 'bfloat16', 'int8', 'int8', 'float32')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,random,random,random
25	-qbmm_KC_case0014,quant_batch_matmul_v3,"((1232, 1348), (1104, 1348), (1104,), None, (1104,), (1232,))","((1232, 1348), (1104, 1348), (1104,), None, (1104,), (1232,))","((1232, 1104),)","((1232, 1104),)","('int8', 'int8', 'float32', 'int8', 'float32', 'float32')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,random	25	+qbmm_KC_case0014,quant_batch_matmul_v3,"((1232, 1348), (1104, 1348), (1104,), None, (1104,), (1232,))","((1232, 1348), (1104, 1348), (1104,), None, (1104,), (1232,))","((1232, 1104),)","((1232, 1104),)","('int8', 'int8', 'float32', 'int8', 'float32', 'float32')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': True}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,random
26	-qbmm_TC_case0027,quant_batch_matmul_v3,"((240, 176), (176, 304), (304,), None, (304,), None)","((240, 176), (176, 304), (304,), None, (304,), None)","((240, 304),)","((240, 304),)","('int8', 'int8', 'uint64', 'int8', 'int32', 'int8')",int8,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16	26	+qbmm_TC_case0027,quant_batch_matmul_v3,"((240, 176), (176, 304), (304,), None, (304,), None)","((240, 176), (176, 304), (304,), None, (304,), None)","((240, 304),)","((240, 304),)","('int8', 'int8', 'uint64', 'int8', 'int32', 'int8')",'int8',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': False, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_16,multiple_of_16,multiple_of_16
27	-qbmm_TC_case0020,quant_batch_matmul_v3,"((443, 128), (443, 320), (320,), None, None, None)","((443, 128), (443, 320), (320,), None, None, None)","((128, 320),)","((128, 320),)","('float8_e4m3fn', 'float8_e5m2', 'uint64', 'int8', 'int8', 'int8')",bfloat16,"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_32,multiple_of_32,prime	27	+qbmm_TC_case0020,quant_batch_matmul_v3,"((443, 128), (443, 320), (320,), None, None, None)","((443, 128), (443, 320), (320,), None, None, None)","((128, 320),)","((128, 320),)","('float8_e4m3fn', 'float8_e5m2', 'uint64', 'int8', 'int8', 'int8')",'bfloat16',"('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND', 'ND', 'ND', 'ND', 'ND', 'ND')","('ND',)","('ND',)","{'dtype': 1, 'transpose_x1': True, 'transpose_x2': False}","((-10, 10),)","((0.001, 0.001),)",,1,multiple_of_32,multiple_of_32,prime