已合并
quant batch matmul v4 modify doc desc #5306
zhangquanxin创建于 5月27日
quant batch matmul v4 modify doc desc #5306
已合并
共 3 个文件变更+19-21
| @@ -897,7 +897,7 @@ aclnnStatus aclnnQuantMatmulV5( | |||
| 897 | |-------|--------|--------|--------|--------|-------------|-------------|------------|---------------------------------------|--|--| | 897 | |-------|--------|--------|--------|--------|-------------|-------------|------------|---------------------------------------|--|--| |
| 898 | |mx 全量化|FLOAT8_E4M3FN/FLOAT8_E5M2|FLOAT8_E4M3FN/FLOAT8_E5M2|<li>非转置：(batch, m, k)</li><li>转置：(batch, k, m)</li>|<li>非转置：(batch, k, n)</li><li>转置：(batch, n, k)</li>|<li>非转置：(m, ceil(k / 64), 2)</li><li>转置：(ceil(k / 64), m, 2)</li>|<li>非转置：(ceil(k / 64), n, 2)</li><li>转置：(n, ceil(k / 64), 2)</li>|(n,)或(batch, 1, n)|null|[1, 1, 32]|4295032864| | 898 | |mx 全量化|FLOAT8_E4M3FN/FLOAT8_E5M2|FLOAT8_E4M3FN/FLOAT8_E5M2|<li>非转置：(batch, m, k)</li><li>转置：(batch, k, m)</li>|<li>非转置：(batch, k, n)</li><li>转置：(batch, n, k)</li>|<li>非转置：(m, ceil(k / 64), 2)</li><li>转置：(ceil(k / 64), m, 2)</li>|<li>非转置：(ceil(k / 64), n, 2)</li><li>转置：(n, ceil(k / 64), 2)</li>|(n,)或(batch, 1, n)|null|[1, 1, 32]|4295032864| |
| 899 | |mx 全量化|FLOAT4_E2M1|FLOAT4_E2M1|<li>非转置：(batch, m, k)</li><li>转置：(batch, k, m)</li>|<li>非转置：(batch, k, n)</li><li>转置：(batch, n, k)</li>|<li>非转置：(m, ceil(k / 64), 2)</li><li>转置：(ceil(k / 64), m, 2)</li>|<li>非转置：(ceil(k / 64), n, 2)</li><li>转置：(n, ceil(k / 64), 2)</li>|(n,)或(batch, 1, n)|null|[1, 1, 32]|4295032864| | 899 | |mx 全量化|FLOAT4_E2M1|FLOAT4_E2M1|<li>非转置：(batch, m, k)</li><li>转置：(batch, k, m)</li>|<li>非转置：(batch, k, n)</li><li>转置：(batch, n, k)</li>|<li>非转置：(m, ceil(k / 64), 2)</li><li>转置：(ceil(k / 64), m, 2)</li>|<li>非转置：(ceil(k / 64), n, 2)</li><li>转置：(n, ceil(k / 64), 2)</li>|(n,)或(batch, 1, n)|null|[1, 1, 32]|4295032864| |
| 900 | - |mx 伪量化|FLOAT8_E4M3FN|FLOAT4_E2M1|(m, k)|(n, k)|(m, ceil(k / 64), 2)|(n, ceil(k / 64), 2)|(1，n)|null|[1, 1, 32]|4295032864| | 900 | + |mx 伪量化|FLOAT8_E4M3FN|FLOAT4_E2M1|(m, k)|(n, k)|(m, ceil(k / 64), 2)|(n, ceil(k / 64), 2)|(1，n)|null|[0, 0, 32]/[1, 1, 32]|32/4295032864| |
| 901 | 901 | ||
| 902 | - 注：上表中gsM、gsK和gsN分别表示groupSizeM、groupSizeK和groupSizeN。gsM、gsK和gsN为0的维度会自动推导，上表中是不用自动推导的情况。 | 902 | - 注：上表中gsM、gsK和gsN分别表示groupSizeM、groupSizeK和groupSizeN。gsM、gsK和gsN为0的维度会自动推导，上表中是不用自动推导的情况。 |
| 903 | - mx全量化场景下，当x2数据类型为FLOAT8_E4M3FN/FLOAT8_E5M2时，x1和x1Scale的转置属性需要保持一致，x2和x2Scale的转置属性需要保持一致。 | 903 | - mx全量化场景下，当x2数据类型为FLOAT8_E4M3FN/FLOAT8_E5M2时，x1和x1Scale的转置属性需要保持一致，x2和x2Scale的转置属性需要保持一致。 |
| @@ -23,27 +23,24 @@ namespace ge { | |||
| 23 | 23 | ||
| 24 | * @par Inputs: | 24 | * @par Inputs: |
| 25 | * @li x1: A matrix Tensor. The shape supports (m, k), and the format supports ND. | 25 | * @li x1: A matrix Tensor. The shape supports (m, k), and the format supports ND. |
| 26 | -* The data type supports float8_e5m2, float8_e4m3fn, int8. The m and k value must be at least 1. The k value | 26 | +* The data type supports float8_e5m2, float8_e4m3fn, int8.\n |
| 27 | -* must be at least 1 and must be a multiple of 64. | 27 | +* In A4W4 pergroup scenario: The data type supports int4. |
| 28 | -* - In A4W4 pergroup scenario: The data type supports int4. | ||
| 29 | * @li x2: A matrix Tensor of quantized weight. The shape supports (n, k), and the format supports ND/FRACTAL_NZ.\n | 28 | * @li x2: A matrix Tensor of quantized weight. The shape supports (n, k), and the format supports ND/FRACTAL_NZ.\n |
| 30 | -* - In MxA8W4 scenario: | 29 | +* In MxA8W4 and A8W4 scenario: the data type supports float4_e2m1. |
| 31 | -* For ND format, the data type supports float4_e2m1. For FRACTAL_NZ format, the data type supports float4_e2m1.\n | 30 | +* The shape must be (k1, n1, n0, k0) (n0 = 16, k0 = 32) when the format is FRACTAL_NZ.\n |
| 32 | -* - In A8W4 scenario: | 31 | +* In A8W8 scenario: The data type supports int8.\n |
| 33 | -* The data type supports float4_e2m1. | 32 | +* In A4W4 pergroup scenario: The data type supports int4. |
| 34 | -* The k, n value must be at least 1 and k, n value must be a multiple of 64. | ||
| 35 | -* The shape must be (k1, n1, n0, k0) (n0 = 16, k0 = 32) when the format is FRACTAL_NZ. | ||
| 36 | -* - In A8W8 scenario: The data type supports int8. | ||
| 37 | -* - In A4W4 pergroup scenario: The data type supports int4. | ||
| 38 | * @li bias: An Optional Tensor. | 33 | * @li bias: An Optional Tensor. |
| 39 | * The shape supprts(1, n), format supports ND, the type supports bfloat16, float16, float32. | 34 | * The shape supprts(1, n), format supports ND, the type supports bfloat16, float16, float32. |
| 40 | * @li x1_scale: An Optional Tensor for quantization parameters. | 35 | * @li x1_scale: An Optional Tensor for quantization parameters. |
| 41 | * The type supports float8_e8m0, float16, bfloat16, format supports ND. | 36 | * The type supports float8_e8m0, float16, bfloat16, format supports ND. |
| 42 | -* The shape supprts(m, k/group_size) when type is float8_e8m0, float32. | 37 | +* The shape supprts(m, k/group_size) when type is float8_e8m0, float32.\n |
| 38 | +* In MxA8W4 scenario: The shape supports (m, ceildiv(k, group_size * 2), 2). The type supports float8_e8m0. | ||
| 43 | * The type bfloat16, bfloat16 is not supported yet. | 39 | * The type bfloat16, bfloat16 is not supported yet. |
| 44 | * @li x2_scale: An Optional Tensor for quantization parameters. | 40 | * @li x2_scale: An Optional Tensor for quantization parameters. |
| 45 | * The type supports float8_e8m0, bfloat16, float16, uint64, format supports ND, The shape supports (n, ceildiv(k, group_size)). | 41 | * The type supports float8_e8m0, bfloat16, float16, uint64, format supports ND, The shape supports (n, ceildiv(k, group_size)). |
| 46 | * - In A8W8 scenario: The shape supports (ceildiv(k, group_size_k), ceildiv(n, group_size_n)). The type supports float32. | 42 | * - In A8W8 scenario: The shape supports (ceildiv(k, group_size_k), ceildiv(n, group_size_n)). The type supports float32. |
| 43 | +* - In MxA8W4 scenario: The shape supports (n, ceildiv(k, group_size * 2), 2). The type supports float8_e8m0. | ||
| 47 | * @li y_scale: An Optional Tensor for quantization parameters. | 44 | * @li y_scale: An Optional Tensor for quantization parameters. |
| 48 | * The type support uint64, format supports ND, The shape supprts(1, n). | 45 | * The type support uint64, format supports ND, The shape supprts(1, n). |
| 49 | * @li x1_offset: An Optional Tensor for quantization parameters. It's not supported yet. | 46 | * @li x1_offset: An Optional Tensor for quantization parameters. It's not supported yet. |
| @@ -58,9 +55,10 @@ namespace ge { | |||
| 58 | * When transpose_x1 is true, x1's shape is (k, m). Currently, it should always be false. | 55 | * When transpose_x1 is true, x1's shape is (k, m). Currently, it should always be false. |
| 59 | * @li transpose_x2: A bool. x2 is transposed if true. Default: false. | 56 | * @li transpose_x2: A bool. x2 is transposed if true. Default: false. |
| 60 | * When transpose_x2 is true, x2's shape is (n, k), x2_scale's shape should be (n, k / group_size). | 57 | * When transpose_x2 is true, x2's shape is (n, k), x2_scale's shape should be (n, k / group_size). |
| 61 | -* Currently, it should always be true. | 58 | +* Currently, it should always be true.\n |
| 62 | * In A8W8 scenario, when transpose_x2 is true, x2_scale's shape should be | 59 | * In A8W8 scenario, when transpose_x2 is true, x2_scale's shape should be |
| 63 | -* (ceildiv(n, group_size_n), ceildiv(k, group_size_k)). | 60 | +* (ceildiv(n, group_size_n), ceildiv(k, group_size_k)).\n |
| 61 | +* In MxA8W4 scenario, when transpose_x2 is true, x2_scale's shape should be (n, ceildiv(k, group_size * 2), 2). | ||
 | |||
| 64 | * @li group_size: An int32. Default: -1. | 62 | * @li group_size: An int32. Default: -1. |
| 65 | 63 | ||
| 66 | * @par Outputs: | 64 | * @par Outputs: |
| @@ -2,7 +2,7 @@ | |||
| 2 | "op_type": "QuantBatchMatmulV4", | 2 | "op_type": "QuantBatchMatmulV4", |
| 3 | "op_list": [ | 3 | "op_list": [ |
| 4 | { | 4 | { |
| 5 | - "bin_filename": "QuantBatchMatmulV4_ND_ND_float8_e4m3fn_float4_e2m1_fp16", | 5 | + "bin_filename": "QuantBatchMatmulV4_ND_ND_float8_e4m3fn_float4_e2m1_out_bf16", |
| 6 | "inputs": [ | 6 | "inputs": [ |
| 7 | { | 7 | { |
| 8 | "name": "x1", | 8 | "name": "x1", |
| @@ -157,7 +157,7 @@ | |||
| 157 | ] | 157 | ] |
| 158 | }, | 158 | }, |
| 159 | { | 159 | { |
| 160 | - "bin_filename": "QuantBatchMatmulV4_ND_NZ_float8_e4m3fn_float4_e2m1_fp16", | 160 | + "bin_filename": "QuantBatchMatmulV4_ND_NZ_float8_e4m3fn_float4_e2m1_out_bf16", |
| 161 | "inputs": [ | 161 | "inputs": [ |
| 162 | { | 162 | { |
| 163 | "name": "x1", | 163 | "name": "x1", |
| @@ -777,7 +777,7 @@ | |||
| 777 | ] | 777 | ] |
| 778 | }, | 778 | }, |
| 779 | { | 779 | { |
| 780 | - "bin_filename": "QuantBatchMatmulV4_ND_ND_float8_e4m3fn_float4_e2m1_fp16_out_fp16", | 780 | + "bin_filename": "QuantBatchMatmulV4_ND_ND_float8_e4m3fn_float4_e2m1_out_fp16", |
| 781 | "inputs": [ | 781 | "inputs": [ |
| 782 | { | 782 | { |
| 783 | "name": "x1", | 783 | "name": "x1", |
| @@ -932,7 +932,7 @@ | |||
| 932 | ] | 932 | ] |
| 933 | }, | 933 | }, |
| 934 | { | 934 | { |
| 935 | - "bin_filename": "QuantBatchMatmulV4_ND_NZ_float8_e4m3fn_float4_e2m1_fp16_out_fp16", | 935 | + "bin_filename": "QuantBatchMatmulV4_ND_NZ_float8_e4m3fn_float4_e2m1_out_fp16", |
| 936 | "inputs": [ | 936 | "inputs": [ |
| 937 | { | 937 | { |
| 938 | "name": "x1", | 938 | "name": "x1", |
| @@ -1087,7 +1087,7 @@ | |||
| 1087 | ] | 1087 | ] |
| 1088 | }, | 1088 | }, |
| 1089 | { | 1089 | { |
| 1090 | - "bin_filename": "QuantBatchMatmulV4_ND_NZ_MX_float8_e4m3fn_float4_e2m1_fp16_out_fp16", | 1090 | + "bin_filename": "QuantBatchMatmulV4_ND_NZ_MX_float8_e4m3fn_float4_e2m1_out_fp16", |
| 1091 | "inputs": [ | 1091 | "inputs": [ |
| 1092 | { | 1092 | { |
| 1093 | "name": "x1", | 1093 | "name": "x1", |
| @@ -1242,7 +1242,7 @@ | |||
| 1242 | ] | 1242 | ] |
| 1243 | }, | 1243 | }, |
| 1244 | { | 1244 | { |
| 1245 | - "bin_filename": "QuantBatchMatmulV4_ND_ND_MX_float8_e4m3fn_float4_e2m1_fp16_out_fp16", | 1245 | + "bin_filename": "QuantBatchMatmulV4_ND_ND_MX_float8_e4m3fn_float4_e2m1_out_fp16", |
| 1246 | "inputs": [ | 1246 | "inputs": [ |
| 1247 | { | 1247 | { |
| 1248 | "name": "x1", | 1248 | "name": "x1", |
这个场景和60行并列，在59行和61行后面分别加个\n