Merged
fix: 加固INTrainingReduceV2、Relu6Grad和NormalizeBBox #13239
raoliang_sacAdd on 21 days ago
fix: 加固INTrainingReduceV2、Relu6Grad和NormalizeBBox #13239
Merged
Total: 32 files changed+758-210
| @@ -2,14 +2,14 @@ | |||
| 2 | 2 | ||
| 3 | ## 产品支持情况 | 3 | ## 产品支持情况 |
| 4 | 4 | ||
| 5 | -|产品 | 是否支持 | | 5 | +| 产品 | 是否支持 | |
| 6 | -|:-------------------------|:----------:| | 6 | +| :--- | :---: | |
| 7 | -| <term>Ascend 950PR&950DT系列产品</term> | √ | | 7 | +| <term>Ascend 950PR&950DT系列产品</term> | √ | |
| 8 | -| <term>Atlas A3系列产品</term> | √ | | 8 | +| <term>Atlas A3系列产品</term> | √ | |
| 9 | -| <term>Atlas A2系列产品</term> | √ | | 9 | +| <term>Atlas A2系列产品</term> | √ | |
| 10 | -| <term>Atlas 200I/500 A2推理产品</term> | × | | 10 | +| <term>Atlas 200I/500 A2推理产品</term> | × | |
| 11 | -| <term>Atlas推理系列产品</term> | × | | 11 | +| <term>Atlas推理系列产品</term> | × | |
| 12 | -| <term>Atlas训练系列产品</term> | × | | 12 | +| <term>Atlas训练系列产品</term> | × | |
| 13 | 13 | ||
| 14 | ## 功能说明 | 14 | ## 功能说明 |
| 15 | 15 | ||
| @@ -26,49 +26,17 @@ | |||
| 26 | $$ | 26 | $$ |
| 27 | 27 | ||
| 28 | 其中: | 28 | 其中: |
| 29 | + | ||
| 29 | - gradients:对应Relu6操作的反向传播梯度。 | 30 | - gradients:对应Relu6操作的反向传播梯度。 |
| 30 | - features:作为输入传递给对应Relu6操作的特征,或其输出,使用任一者产生相同结果。 | 31 | - features:作为输入传递给对应Relu6操作的特征,或其输出,使用任一者产生相同结果。 |
| 31 | 32 | ||
| 32 | ## 参数说明 | 33 | ## 参数说明 |
| 33 | 34 | ||
| 34 | -<table style="undefined;table-layout: fixed; width: 820px"><colgroup> | 35 | +| 参数名 | 输入/输出/属性 | 描述 | 数据类型 | 数据格式 | |
| 35 | - <col style="width: 100px"> | 36 | +| --- | --- | --- | --- | --- | |
| 36 | - <col style="width: 150px"> | 37 | +| gradients | 输入 | 传递给对应Relu6操作的反向传播梯度,公式中的gradients。 | FLOAT16、FLOAT、BFLOAT16 | ND | |
| 37 | - <col style="width: 190px"> | 38 | +| features | 输入 | 作为输入传递给对应Relu6操作的特征,或其输出,使用任一者产生相同结果,公式中的features。与gradients的shape关系见约束说明。 | FLOAT16、FLOAT、BFLOAT16 | ND | |
| 38 | - <col style="width: 260px"> | 39 | +| backprops | 输出 | 公式中的输出张量,shape由输入shape按各产品的广播约束确定,见约束说明。 | FLOAT16、FLOAT、BFLOAT16 | ND | |
| 39 | - <col style="width: 120px"> | ||
| 40 | - </colgroup> | ||
| 41 | - <thead> | ||
| 42 | - <tr> | ||
| 43 | - <th>参数名</th> | ||
| 44 | - <th>输入/输出/属性</th> | ||
| 45 | - <th>描述</th> | ||
| 46 | - <th>数据类型</th> | ||
| 47 | - <th>数据格式</th> | ||
| 48 | - </tr></thead> | ||
| 49 | - <tbody> | ||
| 50 | - <tr> | ||
| 51 | - <td>gradients</td> | ||
| 52 | - <td>输入</td> | ||
| 53 | - <td>传递给对应Relu6操作的反向传播梯度,公式中的gradients。</td> | ||
| 54 | - <td>FLOAT16、FLOAT、BFLOAT16</td> | ||
| 55 | - <td>ND</td> | ||
| 56 | - </tr> | ||
| 57 | - <tr> | ||
| 58 | - <td>features</td> | ||
| 59 | - <td>输入</td> | ||
| 60 | - <td>作为输入传递给对应Relu6操作的特征,或其输出,使用任一者产生相同结果,公式中的features。与gradients的shape关系见约束说明。</td> | ||
| 61 | - <td>FLOAT16、FLOAT、BFLOAT16</td> | ||
| 62 | - <td>ND</td> | ||
| 63 | - </tr> | ||
| 64 | - <tr> | ||
| 65 | - <td>backprops</td> | ||
| 66 | - <td>输出</td> | ||
| 67 | - <td>公式中的输出张量,shape由输入shape按各产品的广播约束确定,见约束说明。</td> | ||
| 68 | - <td>FLOAT16、FLOAT、BFLOAT16</td> | ||
| 69 | - <td>ND</td> | ||
| 70 | - </tr> | ||
| 71 | - </tbody></table> | ||
| 72 | 40 | ||
| 73 | ## 约束说明 | 41 | ## 约束说明 |
| 74 | 42 | ||
| @@ -124,12 +92,12 @@ In0/In1 ─CopyInBrc─ Cast(->fp32) ─+ | |||
| 124 | 把fp16/bf16整体提升到fp32做cmp/and/sel,末端用`CAST_MODE_RINT` | 92 | 把fp16/bf16整体提升到fp32做cmp/and/sel,末端用`CAST_MODE_RINT` |
| 125 | (round-to-nearest-even)回退到原dtype。原因: | 93 | (round-to-nearest-even)回退到原dtype。原因: |
| 126 | 94 | ||
| 127 | -1. 与DSL `relu6_grad.py`在fp16vcmpsel不可用时fallback到fp32的行为一致; | 95 | +1. 比较和选择操作统一使用fp32中间精度; |
| 128 | 2. 上下界0.0和6.0在fp16/bf16中均能精确表达,但cmp走fp32路径更稳健, | 96 | 2. 上下界0.0和6.0在fp16/bf16中均能精确表达,但cmp走fp32路径更稳健, |
| 129 | 且dy是NaN/Inf时fp32中间不会被半精度范围截断。 | 97 | 且dy是NaN/Inf时fp32中间不会被半精度范围截断。 |
| 130 | 98 | ||
| 131 | ## 调用说明 | 99 | ## 调用说明 |
| 132 | 100 | ||
| 133 | -| 调用方式 | 样例代码 | 说明 | | 101 | +| 调用方式 | 样例代码 | 说明 | |
| 134 | -| ---------------- | --------------------------- |-----------------------------------------------| | 102 | +| --- | --- | --- | |
| 135 | -| GE图模式 | [test_geir_relu6_grad.cpp](examples/arch35/test_geir_relu6_grad.cpp) | 通过[算子IR](op_graph/relu6_grad_proto.h)构图方式调用Relu6Grad算子。 | | 103 | +| GE图模式 | [test_geir_relu6_grad.cpp](examples/arch35/test_geir_relu6_grad.cpp) | 通过[算子IR](op_graph/relu6_grad_proto.h)构图方式调用Relu6Grad算子。 | |
| @@ -18,8 +18,15 @@ ge::graphStatus InferDataType4Relu6Grad(gert::InferDataTypeContext* context) | |||
| 18 | return ge::GRAPH_FAILED; | 18 | return ge::GRAPH_FAILED; |
| 19 | } | 19 | } |
| 20 | const auto dtype = context->GetInputDataType(0); | 20 | const auto dtype = context->GetInputDataType(0); |
| 21 | + OP_CHECK_IF(dtype != ge::DT_FLOAT16 && dtype != ge::DT_FLOAT && dtype != ge::DT_BF16, | ||
| 22 | + OP_LOGE_FOR_INVALID_DTYPE(context->GetNodeName(), "gradients", Ops::Base::ToString(dtype).c_str(), | ||
| 23 | + "FLOAT16, FLOAT, BFLOAT16"), | ||
| 24 | + return ge::GRAPH_FAILED); | ||
| 21 | OP_CHECK_IF(dtype != context->GetInputDataType(1), | 25 | OP_CHECK_IF(dtype != context->GetInputDataType(1), |
| 22 | - OP_LOGE(context->GetNodeName(), "gradients and features must have the same dtype."), | 26 | + OP_LOGE_FOR_INVALID_DTYPES_WITH_REASON( |
| 27 | + context->GetNodeName(), "gradients, features", | ||
| 28 | + (Ops::Base::ToString(dtype) + ", " + Ops::Base::ToString(context->GetInputDataType(1))).c_str(), | ||
| 29 | + "gradients and features must have the same dtype"), | ||
| 23 | return ge::GRAPH_FAILED); | 30 | return ge::GRAPH_FAILED); |
| 24 | return context->SetOutputDataType(0, dtype); | 31 | return context->SetOutputDataType(0, dtype); |
| 25 | } | 32 | } |
| @@ -12,6 +12,7 @@ | |||
| 12 | * \file relu6_grad_infershape.cpp | 12 | * \file relu6_grad_infershape.cpp |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | +#include <algorithm> | ||
| 15 | #include "op_host/infershape_broadcast_util.h" | 16 | #include "op_host/infershape_broadcast_util.h" |
| 16 | #include "register/op_impl_registry.h" | 17 | #include "register/op_impl_registry.h" |
| 17 | #include "log/log.h" | 18 | #include "log/log.h" |
| @@ -21,7 +22,33 @@ namespace ops { | |||
| 21 | 22 | ||
| 22 | static ge::graphStatus InferShape4Relu6Grad(gert::InferShapeContext* context) | 23 | static ge::graphStatus InferShape4Relu6Grad(gert::InferShapeContext* context) |
| 23 | { | 24 | { |
| 25 | + if (context == nullptr) { | ||
| 26 | + return ge::GRAPH_FAILED; | ||
| 27 | + } | ||
| 24 | OP_LOGD(context->GetNodeName(), "Begin to do InferShape4Relu6Grad in ops-nn"); | 28 | OP_LOGD(context->GetNodeName(), "Begin to do InferShape4Relu6Grad in ops-nn"); |
| 29 | + constexpr size_t MAX_INPUT_DIM_NUM = 8; | ||
| 30 | + const char* inputNames[] = {"gradients", "features"}; | ||
| 31 | + for (size_t input = 0; input < 2; ++input) { | ||
| 32 | + const auto* shape = context->GetInputShape(input); | ||
| 33 | + OP_CHECK_NULL_WITH_CONTEXT(context, shape); | ||
| 34 | + OP_CHECK_IF(shape->GetDimNum() > MAX_INPUT_DIM_NUM, | ||
| 35 | + OP_LOGE_FOR_INVALID_SHAPEDIM(context->GetNodeName(), inputNames[input], | ||
| 36 | + std::to_string(shape->GetDimNum()).c_str(), "0 ~ 8 dims"), | ||
| 37 | + return ge::GRAPH_FAILED); | ||
| 38 | + } | ||
| 39 | + const auto* gradients = context->GetInputShape(0); | ||
| 40 | + const auto* features = context->GetInputShape(1); | ||
| 41 | + const size_t commonRank = std::min(gradients->GetDimNum(), features->GetDimNum()); | ||
| 42 | + for (size_t offset = 1; offset <= commonRank; ++offset) { | ||
| 43 | + const int64_t gradDim = gradients->GetDim(gradients->GetDimNum() - offset); | ||
| 44 | + const int64_t featDim = features->GetDim(features->GetDimNum() - offset); | ||
| 45 | + OP_CHECK_IF(gradDim >= 0 && featDim >= 0 && gradDim != featDim && gradDim != 1 && featDim != 1, | ||
| 46 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( | ||
| 47 | + context->GetNodeName(), "gradients, features", | ||
| 48 | + Ops::Base::ToString(std::vector<const gert::Shape*>{gradients, features}).c_str(), | ||
| 49 | + "aligned dimensions must be equal or one of them must be 1"), | ||
| 50 | + return ge::GRAPH_FAILED); | ||
| 51 | + } | ||
| 25 | return Ops::Base::InferShape4Broadcast(context); | 52 | return Ops::Base::InferShape4Broadcast(context); |
| 26 | } | 53 | } |
| 27 | 54 | ||
| @@ -60,9 +60,8 @@ struct Relu6Grad { | |||
| 60 | }; | 60 | }; |
| 61 | 61 | ||
| 62 | // Promoted path for half / bfloat16_t: cast inputs to PromoteT (float) for | 62 | // Promoted path for half / bfloat16_t: cast inputs to PromoteT (float) for |
| 63 | -// cmp/and/sel to (a) match the DSL fallback in relu6_grad.py that lifts to | 63 | +// cmp/and/sel to keep boundary comparison against 0 and 6 stable across |
| 64 | -// fp32 when fp16 vcmpsel is unavailable, (b) keep boundary comparison | 64 | +// half/bf16. Result is cast back to T with |
| 65 | -// against 0 and 6 stable across half/bf16. Result is cast back to T with | ||
| 66 | // round-to-nearest-even. | 65 | // round-to-nearest-even. |
| 67 | template <typename T, typename PromoteT> | 66 | template <typename T, typename PromoteT> |
| 68 | struct Relu6GradFloatCast { | 67 | struct Relu6GradFloatCast { |
| @@ -0,0 +1,7 @@ | |||
| 1 | +testcase_name,network_name,op_name,input_shapes,input_dtypes,input_formats,output_shapes,output_dtypes,output_formats,input_ori_shapes,input_ori_formats,output_ori_shapes,output_ori_formats,attributes,input_data_ranges,precision_tolerances,absolute_precision,output_inplace_indexes,output_shape_unknown_indexes,is_enabled,remark,soc_series,priority,dump_file_prefix,manual_input_binaries,manual_golden_binaries | ||
| 2 | +Relu6Grad_st_fp32_tail,UNKNOWN,Relu6Grad,"((33,), (33,))","('float32', 'float32')","('ND', 'ND')","((33,),)","('float32',)","('ND',)","((33,), (33,))","('ND', 'ND')","((33,),)","('ND',)",{},"((-2, 2), (-1, 7))",,1e-08,(),(),True,equal shapes and unaligned tail,,0,,(),() | ||
| 3 | +Relu6Grad_st_fp16_zero_broadcast,UNKNOWN,Relu6Grad,"((1, 3), (2, 1))","('float16', 'float16')","('ND', 'ND')","((2, 3),)","('float16',)","('ND',)","((1, 3), (2, 1))","('ND', 'ND')","((2, 3),)","('ND',)",{},"((-2, 2), (0, 0))",,1e-08,(),(),True,bidirectional broadcast and lower endpoint,,0,,(),() | ||
| 4 | +Relu6Grad_st_bf16_six,UNKNOWN,Relu6Grad,"((2, 3), (2, 3))","('bfloat16', 'bfloat16')","('ND', 'ND')","((2, 3),)","('bfloat16',)","('ND',)","((2, 3), (2, 3))","('ND', 'ND')","((2, 3),)","('ND',)",{},"((-2, 2), (6, 6))",,1e-08,(),(),True,upper endpoint,,0,,(),() | ||
| 5 | +Relu6Grad_st_bf16_rank8,UNKNOWN,Relu6Grad,"((1, 1, 1, 1, 1, 1, 2, 1), (1, 1, 1, 1, 1, 1, 1, 3))","('bfloat16', 'bfloat16')","('ND', 'ND')","((1, 1, 1, 1, 1, 1, 2, 3),)","('bfloat16',)","('ND',)","((1, 1, 1, 1, 1, 1, 2, 1), (1, 1, 1, 1, 1, 1, 1, 3))","('ND', 'ND')","((1, 1, 1, 1, 1, 1, 2, 3),)","('ND',)",{},"((2, 2), (4, 4))",,1e-08,(),(),True,maximum rank and broadcast with nonzero output,,0,,(),() | ||
| 6 | +Relu6Grad_st_fp16_feature_one,UNKNOWN,Relu6Grad,"((17,), (1,))","('float16', 'float16')","('ND', 'ND')","((17,),)","('float16',)","('ND',)","((17,), (1,))","('ND', 'ND')","((17,),)","('ND',)",{},"((-2, 2), (4, 4))",,1e-08,(),(),True,single-element feature broadcast,,0,,(),() | ||
| 7 | +Relu6Grad_st_fp32_inf_outside,UNKNOWN,Relu6Grad,"((3,), (3,))","('float32', 'float32')","('ND', 'ND')","((3,),)","('float32',)","('ND',)","((3,), (3,))","('ND', 'ND')","((3,),)","('ND',)",{},"((float('inf'), float('inf')), (-1, -1))",,1e-08,(),(),True,Inf gradient outside interval selects zero,,0,,(),() | ||
| @@ -45,3 +45,10 @@ TEST(Relu6GradGraphInfer, RejectsMismatchedInputsWithoutLegacyVerifier) | |||
| 45 | CheckDtype(ge::DT_FLOAT16, ge::DT_FLOAT, ge::GRAPH_FAILED); | 45 | CheckDtype(ge::DT_FLOAT16, ge::DT_FLOAT, ge::GRAPH_FAILED); |
| 46 | CheckDtype(ge::DT_BF16, ge::DT_FLOAT16, ge::GRAPH_FAILED); | 46 | CheckDtype(ge::DT_BF16, ge::DT_FLOAT16, ge::GRAPH_FAILED); |
| 47 | } | 47 | } |
| 48 | + | ||
| 49 | +TEST(Relu6GradGraphInfer, RejectsUnsupportedMatchingDataTypes) | ||
| 50 | +{ | ||
| 51 | + CheckDtype(ge::DT_INT32, ge::DT_INT32, ge::GRAPH_FAILED); | ||
| 52 | + CheckDtype(ge::DT_BOOL, ge::DT_BOOL, ge::GRAPH_FAILED); | ||
| 53 | + CheckDtype(ge::DT_DOUBLE, ge::DT_DOUBLE, ge::GRAPH_FAILED); | ||
| 54 | +} | ||
| @@ -27,6 +27,23 @@ protected: | |||
| 27 | static void TearDownTestCase() { std::cout << "relu6grad TearDown" << std::endl; } | 27 | static void TearDownTestCase() { std::cout << "relu6grad TearDown" << std::endl; } |
| 28 | }; | 28 | }; |
| 29 | 29 | ||
| 30 | +TEST_F(relu6grad, relu6grad_infer_shape_rank9_rejected_for_each_input) | ||
| 31 | +{ | ||
| 32 | + auto inferShapeFunc = gert::OpImplRegistry::GetInstance().GetOpImpl("Relu6Grad")->infer_shape; | ||
| 33 | + gert::StorageShape rank9 = {{1, 1, 1, 1, 1, 1, 1, 1, 1}, {1, 1, 1, 1, 1, 1, 1, 1, 1}}; | ||
| 34 | + gert::StorageShape scalar = {{1}, {1}}; | ||
| 35 | + for (size_t input = 0; input < 2; ++input) { | ||
| 36 | + gert::StorageShape output = {{}, {}}; | ||
| 37 | + auto holder = gert::InferShapeContextFaker() | ||
| 38 | + .NodeIoNum(2, 1) | ||
| 39 | + .IrInstanceNum({1, 1}) | ||
| 40 | + .InputShapes({input == 0 ? &rank9 : &scalar, input == 1 ? &rank9 : &scalar}) | ||
| 41 | + .OutputShapes({&output}) | ||
| 42 | + .Build(); | ||
| 43 | + EXPECT_EQ(inferShapeFunc(holder.GetContext<gert::InferShapeContext>()), ge::GRAPH_FAILED); | ||
| 44 | + } | ||
| 45 | +} | ||
| 46 | + | ||
| 30 | TEST_F(relu6grad, relu6grad_infer_shape_same_fp32) | 47 | TEST_F(relu6grad, relu6grad_infer_shape_same_fp32) |
| 31 | { | 48 | { |
| 32 | auto inferShapeFunc = gert::OpImplRegistry::GetInstance().GetOpImpl("Relu6Grad")->infer_shape; | 49 | auto inferShapeFunc = gert::OpImplRegistry::GetInstance().GetOpImpl("Relu6Grad")->infer_shape; |
| @@ -37,25 +37,30 @@ | |||
| 37 | | sum | 输出 | `x`沿空间维求和的结果,保留实例维和通道维,规约维大小为1。 | FLOAT | NC1HWC0、NDC1HWC0、ND | | 37 | | sum | 输出 | `x`沿空间维求和的结果,保留实例维和通道维,规约维大小为1。 | FLOAT | NC1HWC0、NDC1HWC0、ND | |
| 38 | | square_sum | 输出 | `x`沿空间维求平方和的结果,shape与`sum`一致。 | FLOAT | NC1HWC0、NDC1HWC0、ND | | 38 | | square_sum | 输出 | `x`沿空间维求平方和的结果,shape与`sum`一致。 | FLOAT | NC1HWC0、NDC1HWC0、ND | |
| 39 | 39 | ||
| 40 | -### 产品差异说明 | ||
| 41 | - | ||
| 42 | -| 产品 | 静态shape能力 | 动态shape能力 | shape/rank及格式映射 | | ||
| 43 | -| --- | --- | --- | --- | | ||
| 44 | -| <term>Ascend 950PR&950DT系列产品</term> | `x`为NCHW、NCDHW、ND、NHWC或NDHWC,两个输出为ND | 支持上述格式的动态shape及ND动态rank;NHWC/NDHWC输入保留对应格式,NCHW/NCDHW/ND输入使用ND存储,两个输出均为ND | NCHW输入为`[N,C,H,W]`,输出为`[N,C,1,1]`;NCDHW输入为`[N,C,D,H,W]`,输出为`[N,C,1,1,1]`;ND输入的rank为2~8,前两维为N、C,其余维规约为1;NHWC输入为`[N,H,W,C]`,输出为`[N,1,1,C]`;NDHWC输入为`[N,D,H,W,C]`,输出为`[N,1,1,1,C]` | | ||
| 45 | -| <term>Atlas A3系列产品</term><br><term>Atlas A2系列产品</term><br><term>Atlas 200I/500 A2推理产品</term><br><term>Atlas推理系列产品</term><br><term>Atlas训练系列产品</term> | 4维NCHW/NHWC原始格式映射为NC1HWC0,5维NCDHW/NDHWC原始格式映射为NDC1HWC0;输入、输出存储格式一致 | 对应型号已注册的动态shape和动态rank路径使用NC1HWC0或NDC1HWC0存储,格式映射与静态shape一致 | 原始输入为4维或5维;逻辑输出保留N和C,H、W或D、H、W为1;存储格式由图引擎映射 | | ||
| 46 | - | ||
| 47 | ## 约束说明 | 40 | ## 约束说明 |
| 48 | 41 | ||
| 49 | -- NCHW/NHWC输入的逻辑shape为4维,空间维为H、W;NCDHW/NDHWC输入为5维,空间维为D、H、W。 | 42 | +- NCHW、NHWC输入的逻辑shape为4维,空间维为H、W;NCDHW、NDHWC输入为5维,空间维为D、H、W。 |
| 50 | -- <term>Ascend 950PR&950DT系列产品</term>的ND输入rank为2~8,dim0和dim1分别为实例维和通道维,dim2及之后的维度为空间维。 | 43 | +- 动态shape推导中的`-1`表示未知维度,`[-2]`表示未知rank;它们是编译期标记,执行前需解析为满足对应产品约束的实际shape。 |
| 51 | -- <term>Ascend 950PR&950DT系列产品</term>要求`x`的各维度均大于0,不支持空Tensor。 | 44 | +- <term>Ascend 950PR&950DT系列产品</term>: |
| 52 | -- <term>Ascend 950PR&950DT系列产品</term>要求输入元素总数(各维度大小的乘积)不超过`2^63-1`。 | 45 | + - 静态shape支持NCHW、NCDHW、ND、NHWC和NDHWC输入,两个输出均为ND;支持上述格式的动态shape及ND动态rank。动态shape下,NHWC、NDHWC输入保留对应格式,NCHW、NCDHW、ND输入使用ND存储。 |
| 53 | -- <term>Ascend 950PR&950DT系列产品</term>的NHWC/NDHWC输入要求通道行字节数`C*sizeof(x.dtype)`不超过`2^40-1`,即FLOAT16的`C`不超过`2^39-1`,FLOAT的`C`不超过`2^38-1`。 | 46 | + - NCHW输入为`[N,C,H,W]`,输出为`[N,C,1,1]`;NCDHW输入为`[N,C,D,H,W]`,输出为`[N,C,1,1,1]`。 |
| 47 | + - NHWC输入为`[N,H,W,C]`,输出为`[N,1,1,C]`;NDHWC输入为`[N,D,H,W,C]`,输出为`[N,1,1,1,C]`。 | ||
| 48 | + - ND输入rank为2~8,dim0和dim1分别为实例维和通道维,dim2及之后的维度为空间维。 | ||
| 49 | + - 要求`x`的各维度均大于0,不支持空Tensor。 | ||
| 50 | + - 要求输入元素总数(各维度大小的乘积)不超过`2^63-1`。 | ||
| 51 | + - NHWC、NDHWC输入要求通道行字节数`C*sizeof(x.dtype)`不超过`2^40-1`,即FLOAT16的`C`不超过`2^39-1`,FLOAT的`C`不超过`2^38-1`。 | ||
| 52 | + - 按所声明格式的连续存储解释Tensor,不提供独立的stride参数;转置、切片等非连续视图需由上层框架转换为相应格式的连续Tensor。 | ||
| 53 | + - 算子侧参数校验使用`EZ0019`表示非法dtype,`EZ0017`表示非法格式,`EZ0012`表示rank不匹配,`EZ0009`表示非法维度或尺寸乘积越界。GE图模式还可能先由图匹配、类型转换或图编译阶段处理这些参数,具体诊断以GE日志中的阶段和错误信息为准。 | ||
| 54 | +- <term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>、<term>Atlas 200I/500 A2推理产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas训练系列产品</term>: | ||
| 55 | + - 4维NCHW、NHWC原始格式映射为NC1HWC0,5维NCDHW、NDHWC原始格式映射为NDC1HWC0,输入、输出存储格式一致。 | ||
| 56 | + - 对应型号已注册的动态shape和动态rank路径使用NC1HWC0或NDC1HWC0存储,格式映射与静态shape一致。 | ||
| 57 | + - 原始输入为4维或5维;逻辑输出保留N和C,H、W或D、H、W为1;存储格式由图引擎映射。 | ||
| 54 | 58 | ||
| 55 | ## 调用说明 | 59 | ## 调用说明 |
| 56 | 60 | ||
| 57 | 本算子不提供aclnn单算子接口,通过图引擎(Graph Engine,GE)图模式调用。 | 61 | 本算子不提供aclnn单算子接口,通过图引擎(Graph Engine,GE)图模式调用。 |
| 62 | +示例的编译和运行方法参见[算子调用指南](../../docs/zh/invocation/quick_op_invocation.md#ge图模式)。 | ||
| 58 | 63 | ||
| 59 | -| 调用方式 | 调用样例 | 说明 | | 64 | +| 调用方式 | 样例代码 | 说明 | |
| 60 | | --- | --- | --- | | 65 | | --- | --- | --- | |
| 61 | | GE图模式 | [test_geir_in_training_reduce_v2](examples/arch35/test_geir_in_training_reduce_v2.cpp) | 通过[INTrainingReduceV2算子IR](op_graph/in_training_reduce_v2_proto.h)构图并调用算子。 | | 66 | | GE图模式 | [test_geir_in_training_reduce_v2](examples/arch35/test_geir_in_training_reduce_v2.cpp) | 通过[INTrainingReduceV2算子IR](op_graph/in_training_reduce_v2_proto.h)构图并调用算子。 | |
| @@ -45,15 +45,19 @@ inline ge::graphStatus InferShape4INTrainingReduceV2(gert::InferShapeContext* co | |||
| 45 | gert::Shape* squareSumShape = context->GetOutputShape(1); | 45 | gert::Shape* squareSumShape = context->GetOutputShape(1); |
| 46 | OP_CHECK_NULL_WITH_CONTEXT(context, squareSumShape); | 46 | OP_CHECK_NULL_WITH_CONTEXT(context, squareSumShape); |
| 47 | 47 | ||
| 48 | + const auto* xDesc = context->GetInputDesc(0); | ||
| 49 | + OP_CHECK_NULL_WITH_CONTEXT(context, xDesc); | ||
| 50 | + const ge::Format originFormat = xDesc->GetOriginFormat(); | ||
| 51 | + OP_CHECK_IF(originFormat != ge::FORMAT_ND && originFormat != ge::FORMAT_NCHW && originFormat != ge::FORMAT_NHWC && | ||
| 52 | + originFormat != ge::FORMAT_NCDHW && originFormat != ge::FORMAT_NDHWC, | ||
| 53 | + OP_LOGE_FOR_INVALID_FORMAT(context->GetNodeName(), "x", Ops::Base::ToString(originFormat).c_str(), | ||
| 54 | + "ND, NCHW, NHWC, NCDHW or NDHWC"), | ||
| 55 | + return ge::GRAPH_FAILED); | ||
| 48 | if (Ops::Base::IsUnknownRank(*xShape)) { | 56 | if (Ops::Base::IsUnknownRank(*xShape)) { |
| 49 | Ops::Base::SetUnknownRank(*sumShape); | 57 | Ops::Base::SetUnknownRank(*sumShape); |
| 50 | Ops::Base::SetUnknownRank(*squareSumShape); | 58 | Ops::Base::SetUnknownRank(*squareSumShape); |
| 51 | return ge::GRAPH_SUCCESS; | 59 | return ge::GRAPH_SUCCESS; |
| 52 | } | 60 | } |
| 53 | - | ||
| 54 | - const auto* xDesc = context->GetInputDesc(0); | ||
| 55 | - OP_CHECK_NULL_WITH_CONTEXT(context, xDesc); | ||
| 56 | - const ge::Format originFormat = xDesc->GetOriginFormat(); | ||
| 57 | const bool isChannelLast = (originFormat == ge::FORMAT_NHWC || originFormat == ge::FORMAT_NDHWC); | 61 | const bool isChannelLast = (originFormat == ge::FORMAT_NHWC || originFormat == ge::FORMAT_NDHWC); |
| 58 | 62 | ||
| 59 | const size_t rank = xShape->GetDimNum(); | 63 | const size_t rank = xShape->GetDimNum(); |
Mnorm/in_training_reduce_v2/op_host/arch35/in_training_reduce_v2_tiling_ar_full_reduce_arch35.cpp+2-1
| @@ -81,7 +81,8 @@ bool INTrainingReduceV2ARFullReduceTiling::IsCapable() | |||
| 81 | return DoSubRTiling(rAlign, binAddQuotient, elemSize); | 81 | return DoSubRTiling(rAlign, binAddQuotient, elemSize); |
| 82 | } | 82 | } |
| 83 | // 折叠临时缓存单行字节数(保守按字节计) | 83 | // 折叠临时缓存单行字节数(保守按字节计) |
| 84 | - uint64_t binAddBufferOneline = Ops::Base::CeilAlign((binAddQuotient + vlfp32 - 1) / vlfp32, ubfp32) * sizeof(float); | 84 | + uint64_t binAddBufferOneline = Ops::Base::CeilAlign((binAddQuotient + vlfp32 - 1) / vlfp32, ubfp32) * |
| 85 | + sizeof(float) * ACC_COMPONENTS; | ||
| 85 | 86 | ||
| 86 | // 单行 UB 占用(删 gamma/beta/y/mean/var/rstd/meanFp32): | 87 | // 单行 UB 占用(删 gamma/beta/y/mean/var/rstd/meanFp32): |
| 87 | // inQueueX_: rAlign * elemSize # 双 buf → * 2 | 88 | // inQueueX_: rAlign * elemSize # 双 buf → * 2 |
| @@ -158,6 +158,10 @@ ge::graphStatus INTrainingReduceV2RegbaseTilingBase::ParseShapeByFormat() | |||
| 158 | return ge::GRAPH_FAILED); | 158 | return ge::GRAPH_FAILED); |
| 159 | r = nextR; | 159 | r = nextR; |
| 160 | } | 160 | } |
| 161 | + OP_CHECK_IF(r <= 0, | ||
| 162 | + OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(GetNodeName(), "x", ToString(xStorageShape).c_str(), | ||
| 163 | + "The product of spatial dimensions must be positive"), | ||
| 164 | + return ge::GRAPH_FAILED); | ||
| 161 | int64_t totalRows = 0; | 165 | int64_t totalRows = 0; |
| 162 | int64_t totalElements = 0; | 166 | int64_t totalElements = 0; |
| 163 | OP_CHECK_IF(!CheckedMulNonNegative(a1, a0, totalRows), | 167 | OP_CHECK_IF(!CheckedMulNonNegative(a1, a0, totalRows), |
| @@ -203,6 +207,12 @@ ge::graphStatus INTrainingReduceV2RegbaseTilingBase::CheckOutputShapes() | |||
| 203 | const size_t channelAxis = IsChannelLast(format) ? rank - 1 : DIM_1; | 207 | const size_t channelAxis = IsChannelLast(format) ? rank - 1 : DIM_1; |
| 204 | for (uint32_t index : {OUTPUT_SUM_INDEX, OUTPUT_SQUARE_SUM_INDEX}) { | 208 | for (uint32_t index : {OUTPUT_SUM_INDEX, OUTPUT_SQUARE_SUM_INDEX}) { |
| 205 | const char* outputName = index == OUTPUT_SUM_INDEX ? "sum" : "square_sum"; | 209 | const char* outputName = index == OUTPUT_SUM_INDEX ? "sum" : "square_sum"; |
| 210 | + auto outputDesc = context_->GetOutputDesc(index); | ||
| 211 | + OP_CHECK_NULL_WITH_CONTEXT(context_, outputDesc); | ||
| 212 | + const ge::Format outputFormat = outputDesc->GetFormat().GetStorageFormat(); | ||
| 213 | + OP_CHECK_IF(outputFormat != ge::FORMAT_ND, | ||
| 214 | + OP_LOGE_FOR_INVALID_FORMAT(GetNodeName(), outputName, ToString(outputFormat).c_str(), "ND"), | ||
| 215 | + return ge::GRAPH_FAILED); | ||
| 206 | auto outputShape = context_->GetOutputShape(index); | 216 | auto outputShape = context_->GetOutputShape(index); |
| 207 | OP_CHECK_NULL_WITH_CONTEXT(context_, outputShape); | 217 | OP_CHECK_NULL_WITH_CONTEXT(context_, outputShape); |
| 208 | const auto& shape = outputShape->GetStorageShape(); | 218 | const auto& shape = outputShape->GetStorageShape(); |
| @@ -34,15 +34,22 @@ using AscendC::Reg::UpdateMask; | |||
| 34 | 34 | ||
| 35 | constexpr uint32_t ALIGN_32_FACTOR = 32; | 35 | constexpr uint32_t ALIGN_32_FACTOR = 32; |
| 36 | 36 | ||
| 37 | +__simd_callee__ inline void ReduceFullARParts(RegTensor<float>& values, RegTensor<float>& low, MaskReg& valid) | ||
| 38 | +{ | ||
| 39 | + MaskReg full = CreateMask<float, MaskPattern::ALL>(); | ||
| 40 | + ShiftLefts((RegTensor<uint32_t>&)values, (RegTensor<uint32_t>&)values, static_cast<int16_t>(0), valid); | ||
| 41 | + ShiftLefts((RegTensor<uint32_t>&)low, (RegTensor<uint32_t>&)low, static_cast<int16_t>(0), valid); | ||
| 42 | + ReduceWithError(values, low, full); | ||
| 43 | +} | ||
| 44 | + | ||
| 37 | // Preserve horizontal addition errors before rounding each full-AR partial. | 45 | // Preserve horizontal addition errors before rounding each full-AR partial. |
| 38 | __simd_callee__ inline void ReduceFullAR(RegTensor<float>& result, RegTensor<float>& values, MaskReg& valid) | 46 | __simd_callee__ inline void ReduceFullAR(RegTensor<float>& result, RegTensor<float>& values, MaskReg& valid) |
| 39 | { | 47 | { |
| 40 | RegTensor<float> low; | 48 | RegTensor<float> low; |
| 41 | MaskReg full = CreateMask<float, MaskPattern::ALL>(); | 49 | MaskReg full = CreateMask<float, MaskPattern::ALL>(); |
| 42 | MaskReg one = CreateMask<float, MaskPattern::VL1>(); | 50 | MaskReg one = CreateMask<float, MaskPattern::VL1>(); |
| 43 | - ShiftLefts((RegTensor<uint32_t>&)values, (RegTensor<uint32_t>&)values, static_cast<int16_t>(0), valid); | ||
| 44 | Duplicate(low, 0.0f, full); | 51 | Duplicate(low, 0.0f, full); |
| 45 | - ReduceWithError(values, low, full); | 52 | + ReduceFullARParts(values, low, valid); |
| 46 | Add(result, values, low, one); | 53 | Add(result, values, low, one); |
| 47 | } | 54 | } |
| 48 | 55 | ||
| @@ -62,6 +69,7 @@ public: | |||
| 62 | numN_ = tilingData->numN; | 69 | numN_ = tilingData->numN; |
| 63 | numC_ = tilingData->numC; | 70 | numC_ = tilingData->numC; |
| 64 | numR_ = tilingData->numR; | 71 | numR_ = tilingData->numR; |
| 72 | + GetSumScale<T_X>(static_cast<uint64_t>(numR_), sumScale_, sumRestore_); | ||
| 65 | rAlign_ = tilingData->rAlign; | 73 | rAlign_ = tilingData->rAlign; |
| 66 | binaryAddQuotient_ = static_cast<uint32_t>(tilingData->binaryAddQuotient); | 74 | binaryAddQuotient_ = static_cast<uint32_t>(tilingData->binaryAddQuotient); |
| 67 | perCoreCnt_ = tilingData->perCoreCnt; | 75 | perCoreCnt_ = tilingData->perCoreCnt; |
| @@ -219,6 +227,8 @@ private: | |||
| 219 | __aicore__ inline void CalculateSumLessThanVL(__ubuf__ T_X* xInUb, __ubuf__ TSum* sumUb, uint16_t curRows, | 227 | __aicore__ inline void CalculateSumLessThanVL(__ubuf__ T_X* xInUb, __ubuf__ TSum* sumUb, uint16_t curRows, |
| 220 | uint32_t numColAlign, uint32_t reduceNum) | 228 | uint32_t numColAlign, uint32_t reduceNum) |
| 221 | { | 229 | { |
| 230 | + const float scale = sumScale_; | ||
| 231 | + const float restore = sumRestore_; | ||
| 222 | __VEC_SCOPE__ | 232 | __VEC_SCOPE__ |
| 223 | { | 233 | { |
| 224 | RegTensor<float> x; | 234 | RegTensor<float> x; |
| @@ -230,7 +240,9 @@ private: | |||
| 230 | 240 | ||
| 231 | for (uint16_t i = 0; i < curRows; i++) { | 241 | for (uint16_t i = 0; i < curRows; i++) { |
| 232 | LoadTensorForDtypeTIn<T_X>(xInUb, x, pregLoop, i * numColAlign); | 242 | LoadTensorForDtypeTIn<T_X>(xInUb, x, pregLoop, i * numColAlign); |
| 243 | + ScaleSumValue<T_X>(x, scale, pregLoop); | ||
| 233 | ReduceFullAR(sum, x, pregLoop); // Σx:对原始 x 规约 | 244 | ReduceFullAR(sum, x, pregLoop); // Σx:对原始 x 规约 |
| 245 | + ScaleSumValue<T_X>(sum, restore, pregOne); | ||
| 234 | StoreOneElementForDtypeTOut<TSum>(sumUb, sum, pregOne, i); | 246 | StoreOneElementForDtypeTOut<TSum>(sumUb, sum, pregOne, i); |
| 235 | } | 247 | } |
| 236 | } | 248 | } |
| @@ -262,11 +274,14 @@ private: | |||
| 262 | uint32_t numColAlign, uint32_t reduceNum) | 274 | uint32_t numColAlign, uint32_t reduceNum) |
| 263 | { | 275 | { |
| 264 | uint32_t tailLen = reduceNum - VL_FP32; | 276 | uint32_t tailLen = reduceNum - VL_FP32; |
| 277 | + const float scale = sumScale_; | ||
| 278 | + const float restore = sumRestore_; | ||
| 265 | __VEC_SCOPE__ | 279 | __VEC_SCOPE__ |
| 266 | { | 280 | { |
| 267 | RegTensor<float> x; | 281 | RegTensor<float> x; |
| 268 | RegTensor<float> xFold; | 282 | RegTensor<float> xFold; |
| 269 | RegTensor<float> sum; | 283 | RegTensor<float> sum; |
| 284 | + RegTensor<float> low; | ||
| 270 | 285 | ||
| 271 | MaskReg pregFull = CreateMask<float, MaskPattern::ALL>(); | 286 | MaskReg pregFull = CreateMask<float, MaskPattern::ALL>(); |
| 272 | MaskReg pregOne = CreateMask<float, MaskPattern::VL1>(); | 287 | MaskReg pregOne = CreateMask<float, MaskPattern::VL1>(); |
| @@ -275,8 +290,13 @@ private: | |||
| 275 | LoadTensorForDtypeTIn<T_X>(xInUb, x, pregFull, i * numColAlign); | 290 | LoadTensorForDtypeTIn<T_X>(xInUb, x, pregFull, i * numColAlign); |
| 276 | LoadTensorForDtypeTIn<T_X>(xInUb + VL_FP32, xFold, pregTail, i * numColAlign); | 291 | LoadTensorForDtypeTIn<T_X>(xInUb + VL_FP32, xFold, pregTail, i * numColAlign); |
| 277 | ShiftLefts((RegTensor<uint32_t>&)xFold, (RegTensor<uint32_t>&)xFold, static_cast<int16_t>(0), pregTail); | 292 | ShiftLefts((RegTensor<uint32_t>&)xFold, (RegTensor<uint32_t>&)xFold, static_cast<int16_t>(0), pregTail); |
| 278 | - Add(x, x, xFold, pregFull); // 折叠原始 x | 293 | + ScaleSumValue<T_X>(x, scale, pregFull); |
| 279 | - ReduceFullAR(sum, x, pregFull); | 294 | + ScaleSumValue<T_X>(xFold, scale, pregTail); |
| 295 | + Duplicate(low, 0.0f, pregFull); | ||
| 296 | + AccumulateWithError(x, low, xFold, pregFull); | ||
| 297 | + ReduceFullARParts(x, low, pregFull); | ||
| 298 | + Add(sum, x, low, pregOne); | ||
| 299 | + ScaleSumValue<T_X>(sum, restore, pregOne); | ||
| 280 | StoreOneElementForDtypeTOut<TSum>(sumUb, sum, pregOne, i); | 300 | StoreOneElementForDtypeTOut<TSum>(sumUb, sum, pregOne, i); |
| 281 | } | 301 | } |
| 282 | } | 302 | } |
| @@ -312,6 +332,8 @@ private: | |||
| 312 | __aicore__ inline void CalculateSumCommon(__ubuf__ T_X* xInUb, __ubuf__ TSum* sumUb, __ubuf__ float* tmpUb, | 332 | __aicore__ inline void CalculateSumCommon(__ubuf__ T_X* xInUb, __ubuf__ TSum* sumUb, __ubuf__ float* tmpUb, |
| 313 | uint16_t curRows, uint32_t numColAlign, uint32_t reduceNum) | 333 | uint16_t curRows, uint32_t numColAlign, uint32_t reduceNum) |
| 314 | { | 334 | { |
| 335 | + const float scale = sumScale_; | ||
| 336 | + const float restore = sumRestore_; | ||
| 315 | uint32_t binaryAddQuotient = binaryAddQuotient_; | 337 | uint32_t binaryAddQuotient = binaryAddQuotient_; |
| 316 | uint16_t binaryAddQuotientLoop = (binaryAddQuotient + VL_FP32 - 1) / VL_FP32; | 338 | uint16_t binaryAddQuotientLoop = (binaryAddQuotient + VL_FP32 - 1) / VL_FP32; |
| 317 | 339 | ||
| @@ -326,7 +348,7 @@ private: | |||
| 326 | RegTensor<float> xFold; | 348 | RegTensor<float> xFold; |
| 327 | RegTensor<float> sumReg; | 349 | RegTensor<float> sumReg; |
| 328 | RegTensor<float> vSum; | 350 | RegTensor<float> vSum; |
| 329 | - RegTensor<float> partial; | 351 | + RegTensor<float> low; |
| 330 | 352 | ||
| 331 | MaskReg pregFull = CreateMask<float, MaskPattern::ALL>(); | 353 | MaskReg pregFull = CreateMask<float, MaskPattern::ALL>(); |
| 332 | MaskReg pregOne = CreateMask<float, MaskPattern::VL1>(); | 354 | MaskReg pregOne = CreateMask<float, MaskPattern::VL1>(); |
| @@ -337,10 +359,15 @@ private: | |||
| 337 | uint32_t offset = r * VL_FP32 + baseOffset; | 359 | uint32_t offset = r * VL_FP32 + baseOffset; |
| 338 | LoadTensorForDtypeTIn<T_X>(xInUb, x, pregFull, offset); | 360 | LoadTensorForDtypeTIn<T_X>(xInUb, x, pregFull, offset); |
| 339 | LoadTensorForDtypeTIn<T_X>(xInUb + binaryAddQuotient, xFold, pregFull, offset); | 361 | LoadTensorForDtypeTIn<T_X>(xInUb + binaryAddQuotient, xFold, pregFull, offset); |
| 340 | - Add(x, x, xFold, pregFull); | 362 | + ScaleSumValue<T_X>(x, scale, pregFull); |
| 341 | - ReduceFullAR(partial, x, pregFull); | 363 | + ScaleSumValue<T_X>(xFold, scale, pregFull); |
| 364 | + Duplicate(low, 0.0f, pregFull); | ||
| 365 | + AccumulateWithError(x, low, xFold, pregFull); | ||
| 366 | + ReduceFullARParts(x, low, pregFull); | ||
| 342 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( | 367 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( |
| 343 | - tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign + r), partial, pregOne); | 368 | + tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign * 2 + r), x, pregOne); |
| 369 | + DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( | ||
| 370 | + tmpUb + static_cast<uint32_t>((i * 2 + 1) * lastBinaryAddNumAlign + r), low, pregOne); | ||
| 344 | } | 371 | } |
| 345 | uint32_t sregRemainder = binaryAddRemainder - binaryAddRemainderFloorLoop * VL_FP32; | 372 | uint32_t sregRemainder = binaryAddRemainder - binaryAddRemainderFloorLoop * VL_FP32; |
| 346 | for (uint16_t r = 0; | 373 | for (uint16_t r = 0; |
| @@ -352,20 +379,33 @@ private: | |||
| 352 | pregLoop, offset); | 379 | pregLoop, offset); |
| 353 | ShiftLefts((RegTensor<uint32_t>&)xFold, (RegTensor<uint32_t>&)xFold, static_cast<int16_t>(0), | 380 | ShiftLefts((RegTensor<uint32_t>&)xFold, (RegTensor<uint32_t>&)xFold, static_cast<int16_t>(0), |
| 354 | pregLoop); | 381 | pregLoop); |
| 355 | - Add(x, x, xFold, pregFull); | 382 | + ScaleSumValue<T_X>(x, scale, pregFull); |
| 356 | - ReduceFullAR(partial, x, pregFull); | 383 | + ScaleSumValue<T_X>(xFold, scale, pregLoop); |
| 384 | + Duplicate(low, 0.0f, pregFull); | ||
| 385 | + AccumulateWithError(x, low, xFold, pregFull); | ||
| 386 | + ReduceFullARParts(x, low, pregFull); | ||
| 357 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( | 387 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( |
| 358 | - tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign + binaryAddRemainderFloorLoop), partial, | 388 | + tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign * 2 + binaryAddRemainderFloorLoop), x, |
| 359 | pregOne); | 389 | pregOne); |
| 390 | + DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( | ||
| 391 | + tmpUb + | ||
| 392 | + static_cast<uint32_t>((i * 2 + 1) * lastBinaryAddNumAlign + binaryAddRemainderFloorLoop), | ||
| 393 | + low, pregOne); | ||
| 360 | } | 394 | } |
| 361 | for (uint16_t r = 0; r < static_cast<uint16_t>(binaryAddQuotientLoop - binaryAddRemainderCeilLoop); | 395 | for (uint16_t r = 0; r < static_cast<uint16_t>(binaryAddQuotientLoop - binaryAddRemainderCeilLoop); |
| 362 | r++) { | 396 | r++) { |
| 363 | uint32_t offset = r * VL_FP32 + baseOffset; | 397 | uint32_t offset = r * VL_FP32 + baseOffset; |
| 364 | LoadTensorForDtypeTIn<T_X>(xInUb + binaryAddRemainderCeilLoop * VL_FP32, x, pregFull, offset); | 398 | LoadTensorForDtypeTIn<T_X>(xInUb + binaryAddRemainderCeilLoop * VL_FP32, x, pregFull, offset); |
| 365 | - ReduceFullAR(partial, x, pregFull); | 399 | + ScaleSumValue<T_X>(x, scale, pregFull); |
| 400 | + Duplicate(low, 0.0f, pregFull); | ||
| 401 | + ReduceFullARParts(x, low, pregFull); | ||
| 366 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( | 402 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( |
| 367 | - tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign + binaryAddRemainderCeilLoop + r), | 403 | + tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign * 2 + binaryAddRemainderCeilLoop + r), |
| 368 | - partial, pregOne); | 404 | + x, pregOne); |
| 405 | + DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>( | ||
| 406 | + tmpUb + | ||
| 407 | + static_cast<uint32_t>((i * 2 + 1) * lastBinaryAddNumAlign + binaryAddRemainderCeilLoop + r), | ||
| 408 | + low, pregOne); | ||
| 369 | } | 409 | } |
| 370 | } | 410 | } |
| 371 | } // 关闭 store 阶段 __VEC_SCOPE__(与 Σx² 一致:分作用域强制 store→load 顺序,防跨语句冒险)。 | 411 | } // 关闭 store 阶段 __VEC_SCOPE__(与 Σx² 一致:分作用域强制 store→load 顺序,防跨语句冒险)。 |
| @@ -373,21 +413,28 @@ private: | |||
| 373 | { | 413 | { |
| 374 | RegTensor<float> x; | 414 | RegTensor<float> x; |
| 375 | RegTensor<float> acc; | 415 | RegTensor<float> acc; |
| 416 | + RegTensor<float> low; | ||
| 376 | RegTensor<float> vSum; | 417 | RegTensor<float> vSum; |
| 377 | MaskReg pregFull = CreateMask<float, MaskPattern::ALL>(); | 418 | MaskReg pregFull = CreateMask<float, MaskPattern::ALL>(); |
| 378 | MaskReg pregOne = CreateMask<float, MaskPattern::VL1>(); | 419 | MaskReg pregOne = CreateMask<float, MaskPattern::VL1>(); |
| 379 | uint16_t slotSeg = (binaryAddQuotientLoop + VL_FP32 - 1) / VL_FP32; | 420 | uint16_t slotSeg = (binaryAddQuotientLoop + VL_FP32 - 1) / VL_FP32; |
| 380 | for (uint16_t i = 0; i < curRows; ++i) { | 421 | for (uint16_t i = 0; i < curRows; ++i) { |
| 381 | uint32_t sregSlot = binaryAddQuotientLoop; | 422 | uint32_t sregSlot = binaryAddQuotientLoop; |
| 382 | - Duplicate(acc, static_cast<float>(0.0), pregFull); | 423 | + Duplicate(acc, 0.0f, pregFull); |
| 424 | + Duplicate(low, 0.0f, pregFull); | ||
| 383 | for (uint16_t s = 0; s < slotSeg; ++s) { | 425 | for (uint16_t s = 0; s < slotSeg; ++s) { |
| 384 | MaskReg pregSeg = UpdateMask<float>(sregSlot); // 尾段部分槽的有效掩码 | 426 | MaskReg pregSeg = UpdateMask<float>(sregSlot); // 尾段部分槽的有效掩码 |
| 385 | - DataCopy(x, tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign + s * VL_FP32)); | 427 | + DataCopy(x, tmpUb + static_cast<uint32_t>(i * lastBinaryAddNumAlign * 2 + s * VL_FP32)); |
| 386 | // 尾段无效 lane 清零后累加 | 428 | // 尾段无效 lane 清零后累加 |
| 387 | ShiftLefts((RegTensor<uint32_t>&)x, (RegTensor<uint32_t>&)x, static_cast<int16_t>(0), pregSeg); | 429 | ShiftLefts((RegTensor<uint32_t>&)x, (RegTensor<uint32_t>&)x, static_cast<int16_t>(0), pregSeg); |
| 388 | - Add(acc, acc, x, pregFull); | 430 | + AccumulateWithError(acc, low, x, pregFull); |
| 431 | + DataCopy(x, tmpUb + static_cast<uint32_t>((i * 2 + 1) * lastBinaryAddNumAlign + s * VL_FP32)); | ||
| 432 | + ShiftLefts((RegTensor<uint32_t>&)x, (RegTensor<uint32_t>&)x, static_cast<int16_t>(0), pregSeg); | ||
| 433 | + Add(low, low, x, pregFull); | ||
| 389 | } | 434 | } |
| 390 | - ReduceFullAR(vSum, acc, pregFull); | 435 | + ReduceFullARParts(acc, low, pregFull); |
| 436 | + Add(vSum, acc, low, pregOne); | ||
| 437 | + ScaleSumValue<T_X>(vSum, restore, pregOne); | ||
| 391 | StoreOneElementForDtypeTOut<TSum>(sumUb, vSum, pregOne, i); | 438 | StoreOneElementForDtypeTOut<TSum>(sumUb, vSum, pregOne, i); |
| 392 | } | 439 | } |
| 393 | } | 440 | } |
| @@ -500,6 +547,8 @@ private: | |||
| 500 | int64_t numN_{0}; | 547 | int64_t numN_{0}; |
| 501 | int64_t numC_{0}; | 548 | int64_t numC_{0}; |
| 502 | int64_t numR_{0}; | 549 | int64_t numR_{0}; |
| 550 | + float sumScale_{1.0f}; | ||
| 551 | + float sumRestore_{1.0f}; | ||
| 503 | uint64_t rAlign_; | 552 | uint64_t rAlign_; |
| 504 | uint32_t binaryAddQuotient_; | 553 | uint32_t binaryAddQuotient_; |
| 505 | int64_t perCoreCnt_; | 554 | int64_t perCoreCnt_; |
| @@ -42,7 +42,7 @@ template <typename T_X> | |||
| 42 | __simd_callee__ inline void AraLoadAndAccumulate(__ubuf__ T_X* xUb, RegTensor<float>& xReg, RegTensor<float>& squareReg, | 42 | __simd_callee__ inline void AraLoadAndAccumulate(__ubuf__ T_X* xUb, RegTensor<float>& xReg, RegTensor<float>& squareReg, |
| 43 | RegTensor<float>& sumAcc, RegTensor<float>& squareAcc, | 43 | RegTensor<float>& sumAcc, RegTensor<float>& squareAcc, |
| 44 | RegTensor<float>& sumLow, RegTensor<float>& squareLow, MaskReg& pregC, | 44 | RegTensor<float>& sumLow, RegTensor<float>& squareLow, MaskReg& pregC, |
| 45 | - MaskReg& pregFull, uint32_t offset) | 45 | + MaskReg& pregFull, uint32_t offset, float sumScale) |
| 46 | { | 46 | { |
| 47 | if constexpr (IsSameType<T_X, float>::value) { | 47 | if constexpr (IsSameType<T_X, float>::value) { |
| 48 | LoadAlign(xReg, xUb + offset); | 48 | LoadAlign(xReg, xUb + offset); |
| @@ -53,8 +53,9 @@ __simd_callee__ inline void AraLoadAndAccumulate(__ubuf__ T_X* xUb, RegTensor<fl | |||
| 53 | } | 53 | } |
| 54 | // fp32 直载不消费 pregC;用 zeroing mask 显式清掉尾 C 的无效 lane。 | 54 | // fp32 直载不消费 pregC;用 zeroing mask 显式清掉尾 C 的无效 lane。 |
| 55 | ShiftLefts((RegTensor<uint32_t>&)xReg, (RegTensor<uint32_t>&)xReg, static_cast<int16_t>(0), pregC); | 55 | ShiftLefts((RegTensor<uint32_t>&)xReg, (RegTensor<uint32_t>&)xReg, static_cast<int16_t>(0), pregC); |
| 56 | - AccumulateWithError(sumAcc, sumLow, xReg, pregFull); | ||
| 57 | Mul(squareReg, xReg, xReg, pregFull); | 56 | Mul(squareReg, xReg, xReg, pregFull); |
| 57 | + ScaleSumValue<T_X>(xReg, sumScale, pregFull); | ||
| 58 | + AccumulateWithError(sumAcc, sumLow, xReg, pregFull); | ||
| 58 | AccumulateWithError(squareAcc, squareLow, squareReg, pregFull); | 59 | AccumulateWithError(squareAcc, squareLow, squareReg, pregFull); |
| 59 | } | 60 | } |
| 60 | 61 | ||
| @@ -74,7 +75,8 @@ __simd_vf__ inline void AraInitAccumulatorsVf(__ubuf__ float* sumUb, __ubuf__ fl | |||
| 74 | 75 | ||
| 75 | template <typename T_X, uint16_t ROW_TAIL> | 76 | template <typename T_X, uint16_t ROW_TAIL> |
| 76 | __simd_vf__ inline void AraAccumulateChunkVf(__ubuf__ T_X* xUb, __ubuf__ float* sumUb, __ubuf__ float* squareSumUb, | 77 | __simd_vf__ inline void AraAccumulateChunkVf(__ubuf__ T_X* xUb, __ubuf__ float* sumUb, __ubuf__ float* squareSumUb, |
| 77 | - uint16_t cLoops, uint16_t rowGroups, uint32_t currentC, uint32_t rowStride) | 78 | + uint16_t cLoops, uint16_t rowGroups, uint32_t currentC, uint32_t rowStride, |
| 79 | + float sumScale) | ||
| 78 | { | 80 | { |
| 79 | RegTensor<float> globalSum; | 81 | RegTensor<float> globalSum; |
| 80 | RegTensor<float> globalSumLow; | 82 | RegTensor<float> globalSumLow; |
| @@ -127,27 +129,27 @@ __simd_vf__ inline void AraAccumulateChunkVf(__ubuf__ T_X* xUb, __ubuf__ float* | |||
| 127 | for (uint16_t group = 0; group < rowGroups; ++group) { | 129 | for (uint16_t group = 0; group < rowGroups; ++group) { |
| 128 | uint32_t rowBase = static_cast<uint32_t>(group) * ARA_PAIRWISE_GROUP; | 130 | uint32_t rowBase = static_cast<uint32_t>(group) * ARA_PAIRWISE_GROUP; |
| 129 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc0, squareAcc0, sumLow0, squareLow0, pregC, pregFull, | 131 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc0, squareAcc0, sumLow0, squareLow0, pregC, pregFull, |
| 130 | - rowBase * rowStride + cOffset); | 132 | + rowBase * rowStride + cOffset, sumScale); |
| 131 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc1, squareAcc1, sumLow1, squareLow1, pregC, pregFull, | 133 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc1, squareAcc1, sumLow1, squareLow1, pregC, pregFull, |
| 132 | - (rowBase + 1) * rowStride + cOffset); | 134 | + (rowBase + 1) * rowStride + cOffset, sumScale); |
| 133 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc2, squareAcc2, sumLow2, squareLow2, pregC, pregFull, | 135 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc2, squareAcc2, sumLow2, squareLow2, pregC, pregFull, |
| 134 | - (rowBase + 2) * rowStride + cOffset); | 136 | + (rowBase + 2) * rowStride + cOffset, sumScale); |
| 135 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc3, squareAcc3, sumLow3, squareLow3, pregC, pregFull, | 137 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc3, squareAcc3, sumLow3, squareLow3, pregC, pregFull, |
| 136 | - (rowBase + 3) * rowStride + cOffset); | 138 | + (rowBase + 3) * rowStride + cOffset, sumScale); |
| 137 | } | 139 | } |
| 138 | 140 | ||
| 139 | uint32_t tailBase = static_cast<uint32_t>(rowGroups) * ARA_PAIRWISE_GROUP; | 141 | uint32_t tailBase = static_cast<uint32_t>(rowGroups) * ARA_PAIRWISE_GROUP; |
| 140 | if constexpr (ROW_TAIL > 0) { | 142 | if constexpr (ROW_TAIL > 0) { |
| 141 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc0, squareAcc0, sumLow0, squareLow0, pregC, pregFull, | 143 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc0, squareAcc0, sumLow0, squareLow0, pregC, pregFull, |
| 142 | - tailBase * rowStride + cOffset); | 144 | + tailBase * rowStride + cOffset, sumScale); |
| 143 | } | 145 | } |
| 144 | if constexpr (ROW_TAIL > 1) { | 146 | if constexpr (ROW_TAIL > 1) { |
| 145 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc1, squareAcc1, sumLow1, squareLow1, pregC, pregFull, | 147 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc1, squareAcc1, sumLow1, squareLow1, pregC, pregFull, |
| 146 | - (tailBase + 1) * rowStride + cOffset); | 148 | + (tailBase + 1) * rowStride + cOffset, sumScale); |
| 147 | } | 149 | } |
| 148 | if constexpr (ROW_TAIL > 2) { | 150 | if constexpr (ROW_TAIL > 2) { |
| 149 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc2, squareAcc2, sumLow2, squareLow2, pregC, pregFull, | 151 | AraLoadAndAccumulate<T_X>(xUb, xReg, squareReg, sumAcc2, squareAcc2, sumLow2, squareLow2, pregC, pregFull, |
| 150 | - (tailBase + 2) * rowStride + cOffset); | 152 | + (tailBase + 2) * rowStride + cOffset, sumScale); |
| 151 | } | 153 | } |
| 152 | 154 | ||
| 153 | AccumulateWithError(sumAcc0, sumLow0, sumAcc1, pregFull); | 155 | AccumulateWithError(sumAcc0, sumLow0, sumAcc1, pregFull); |
| @@ -180,7 +182,7 @@ __simd_vf__ inline void AraAccumulateChunkVf(__ubuf__ T_X* xUb, __ubuf__ float* | |||
| 180 | template <typename T_X> | 182 | template <typename T_X> |
| 181 | __simd_vf__ inline void AraAccumulateSmallChannelsVf(__ubuf__ T_X* xUb, __ubuf__ float* sumUb, | 183 | __simd_vf__ inline void AraAccumulateSmallChannelsVf(__ubuf__ T_X* xUb, __ubuf__ float* sumUb, |
| 182 | __ubuf__ float* squareSumUb, uint32_t currentR, uint32_t currentC, | 184 | __ubuf__ float* squareSumUb, uint32_t currentR, uint32_t currentC, |
| 183 | - uint32_t rowStride, uint32_t channelOffset) | 185 | + uint32_t rowStride, uint32_t channelOffset, float sumScale) |
| 184 | { | 186 | { |
| 185 | RegTensor<uint32_t> rowOffsets; | 187 | RegTensor<uint32_t> rowOffsets; |
| 186 | RegTensor<uint32_t> offsets; | 188 | RegTensor<uint32_t> offsets; |
| @@ -225,8 +227,9 @@ __simd_vf__ inline void AraAccumulateSmallChannelsVf(__ubuf__ T_X* xUb, __ubuf__ | |||
| 225 | Cast<float, T_X, castTraitB162B32>(xReg, unpacked, pregR); | 227 | Cast<float, T_X, castTraitB162B32>(xReg, unpacked, pregR); |
| 226 | } | 228 | } |
| 227 | ShiftLefts((RegTensor<uint32_t>&)xReg, (RegTensor<uint32_t>&)xReg, static_cast<int16_t>(0), pregR); | 229 | ShiftLefts((RegTensor<uint32_t>&)xReg, (RegTensor<uint32_t>&)xReg, static_cast<int16_t>(0), pregR); |
| 228 | - AccumulateWithError(sumAcc, sumLow, xReg, pregFull); | ||
| 229 | Mul(squareReg, xReg, xReg, pregFull); | 230 | Mul(squareReg, xReg, xReg, pregFull); |
| 231 | + ScaleSumValue<T_X>(xReg, sumScale, pregFull); | ||
| 232 | + AccumulateWithError(sumAcc, sumLow, xReg, pregFull); | ||
| 230 | AccumulateWithError(squareAcc, squareLow, squareReg, pregFull); | 233 | AccumulateWithError(squareAcc, squareLow, squareReg, pregFull); |
| 231 | } | 234 | } |
| 232 | ReduceWithError(sumAcc, sumLow, pregFull); | 235 | ReduceWithError(sumAcc, sumLow, pregFull); |
| @@ -248,7 +251,9 @@ __simd_vf__ inline void AraAccumulateSmallChannelsVf(__ubuf__ T_X* xUb, __ubuf__ | |||
| 248 | } | 251 | } |
| 249 | } | 252 | } |
| 250 | 253 | ||
| 251 | -__simd_vf__ inline void AraFinalizeVf(__ubuf__ float* sumUb, __ubuf__ float* squareUb, uint16_t cLoops) | 254 | +template <typename T_X> |
| 255 | +__simd_vf__ inline void AraFinalizeVf(__ubuf__ float* sumUb, __ubuf__ float* squareUb, uint16_t cLoops, | ||
| 256 | + float sumRestore) | ||
| 252 | { | 257 | { |
| 253 | RegTensor<float> hi; | 258 | RegTensor<float> hi; |
| 254 | RegTensor<float> lo; | 259 | RegTensor<float> lo; |
| @@ -260,6 +265,7 @@ __simd_vf__ inline void AraFinalizeVf(__ubuf__ float* sumUb, __ubuf__ float* squ | |||
| 260 | LoadAlign(lo, sumUb + ARA_CORRECTION_STRIDE + offset); | 265 | LoadAlign(lo, sumUb + ARA_CORRECTION_STRIDE + offset); |
| 261 | ClearNonFiniteCorrection(lo, full); | 266 | ClearNonFiniteCorrection(lo, full); |
| 262 | Add(hi, hi, lo, full); | 267 | Add(hi, hi, lo, full); |
| 268 | + ScaleSumValue<T_X>(hi, sumRestore, full); | ||
| 263 | StoreAlign(sumUb + offset, hi, full); | 269 | StoreAlign(sumUb + offset, hi, full); |
| 264 | LoadAlign(hi, squareUb + offset); | 270 | LoadAlign(hi, squareUb + offset); |
| 265 | LoadAlign(lo, squareUb + ARA_CORRECTION_STRIDE + offset); | 271 | LoadAlign(lo, squareUb + ARA_CORRECTION_STRIDE + offset); |
| @@ -278,6 +284,7 @@ public: | |||
| 278 | { | 284 | { |
| 279 | blockIdx_ = GetBlockIdx(); | 285 | blockIdx_ = GetBlockIdx(); |
| 280 | numR_ = tilingData->numR; | 286 | numR_ = tilingData->numR; |
| 287 | + GetSumScale<T_X>(numR_, sumScale_, sumRestore_); | ||
| 281 | numC_ = tilingData->numC; | 288 | numC_ = tilingData->numC; |
| 282 | totalTiles_ = tilingData->totalTiles; | 289 | totalTiles_ = tilingData->totalTiles; |
| 283 | tilesPerCore_ = tilingData->tilesPerCore; | 290 | tilesPerCore_ = tilingData->tilesPerCore; |
| @@ -338,9 +345,9 @@ public: | |||
| 338 | inQueueX_.FreeTensor(xLocal); | 345 | inQueueX_.FreeTensor(xLocal); |
| 339 | } | 346 | } |
| 340 | 347 | ||
| 341 | - asc_vf_call<AraFinalizeVf>((__ubuf__ float*)sumLocal.GetPhyAddr(), | 348 | + asc_vf_call<AraFinalizeVf<T_X>>((__ubuf__ float*)sumLocal.GetPhyAddr(), |
| 342 | - (__ubuf__ float*)squareSumLocal.GetPhyAddr(), | 349 | + (__ubuf__ float*)squareSumLocal.GetPhyAddr(), |
| 343 | - static_cast<uint16_t>((currentC + VL_FP32 - 1) / VL_FP32)); | 350 | + static_cast<uint16_t>((currentC + VL_FP32 - 1) / VL_FP32), sumRestore_); |
| 344 | outQueueSum_.EnQue<TSum>(sumLocal); | 351 | outQueueSum_.EnQue<TSum>(sumLocal); |
| 345 | outQueueSquareSum_.EnQue<TSum>(squareSumLocal); | 352 | outQueueSquareSum_.EnQue<TSum>(squareSumLocal); |
| 346 | CopyOut(nIdx * numC_ + cOffset, currentC); | 353 | CopyOut(nIdx * numC_ + cOffset, currentC); |
| @@ -397,25 +404,25 @@ private: | |||
| 397 | 404 | ||
| 398 | if (currentC <= ARA_GATHER_MAX_CHANNELS && numR_ >= VL_FP32) { | 405 | if (currentC <= ARA_GATHER_MAX_CHANNELS && numR_ >= VL_FP32) { |
| 399 | asc_vf_call<AraAccumulateSmallChannelsVf<T_X>>(xUb, sumUb, squareSumUb, currentR, currentC, rowStride, | 406 | asc_vf_call<AraAccumulateSmallChannelsVf<T_X>>(xUb, sumUb, squareSumUb, currentR, currentC, rowStride, |
| 400 | - channelOffset); | 407 | + channelOffset, sumScale_); |
| 401 | return; | 408 | return; |
| 402 | } | 409 | } |
| 403 | switch (rowTail) { | 410 | switch (rowTail) { |
| 404 | case 0: | 411 | case 0: |
| 405 | asc_vf_call<AraAccumulateChunkVf<T_X, 0>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, | 412 | asc_vf_call<AraAccumulateChunkVf<T_X, 0>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, |
| 406 | - rowStride); | 413 | + rowStride, sumScale_); |
| 407 | break; | 414 | break; |
| 408 | case 1: | 415 | case 1: |
| 409 | asc_vf_call<AraAccumulateChunkVf<T_X, 1>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, | 416 | asc_vf_call<AraAccumulateChunkVf<T_X, 1>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, |
| 410 | - rowStride); | 417 | + rowStride, sumScale_); |
| 411 | break; | 418 | break; |
| 412 | case 2: | 419 | case 2: |
| 413 | asc_vf_call<AraAccumulateChunkVf<T_X, 2>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, | 420 | asc_vf_call<AraAccumulateChunkVf<T_X, 2>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, |
| 414 | - rowStride); | 421 | + rowStride, sumScale_); |
| 415 | break; | 422 | break; |
| 416 | default: | 423 | default: |
| 417 | asc_vf_call<AraAccumulateChunkVf<T_X, 3>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, | 424 | asc_vf_call<AraAccumulateChunkVf<T_X, 3>>(xUb, sumUb, squareSumUb, cLoops, rowGroups, currentC, |
| 418 | - rowStride); | 425 | + rowStride, sumScale_); |
| 419 | break; | 426 | break; |
| 420 | } | 427 | } |
| 421 | } | 428 | } |
| @@ -448,6 +455,8 @@ private: | |||
| 448 | 455 | ||
| 449 | uint64_t blockIdx_{0}; | 456 | uint64_t blockIdx_{0}; |
| 450 | uint64_t numR_{0}; | 457 | uint64_t numR_{0}; |
| 458 | + float sumScale_{1.0f}; | ||
| 459 | + float sumRestore_{1.0f}; | ||
| 451 | uint64_t numC_{0}; | 460 | uint64_t numC_{0}; |
| 452 | uint64_t totalTiles_{0}; | 461 | uint64_t totalTiles_{0}; |
| 453 | uint64_t tilesPerCore_{0}; | 462 | uint64_t tilesPerCore_{0}; |
| @@ -38,6 +38,26 @@ constexpr uint32_t BLOCK_SIZE = platform::GetUbBlockSize(); | |||
| 38 | constexpr uint32_t BLK_B32 = BLOCK_SIZE / sizeof(float); | 38 | constexpr uint32_t BLK_B32 = BLOCK_SIZE / sizeof(float); |
| 39 | constexpr uint32_t DOUBLE_BUFFER_NUM = 2; | 39 | constexpr uint32_t DOUBLE_BUFFER_NUM = 2; |
| 40 | 40 | ||
| 41 | +// Use one power-of-two scale for the complete reduction, including sub-R | ||
| 42 | +// carries. This prevents finite FLOAT inputs overflowing before cancellation. | ||
| 43 | +template <typename T_IN> | ||
| 44 | +__aicore__ inline void GetSumScale(uint64_t reduceNum, float& scale, float& restore) | ||
| 45 | +{ | ||
| 46 | + if constexpr (IsSameType<T_IN, float>::value) { | ||
| 47 | + uint32_t power = static_cast<uint32_t>(64 - AscendC::CountLeadingZero(reduceNum - 1)); | ||
| 48 | + restore = static_cast<float>(uint64_t{1} << power); | ||
| 49 | + scale = 1.0f / restore; | ||
| 50 | + } | ||
| 51 | +} | ||
| 52 | + | ||
| 53 | +template <typename T_IN> | ||
| 54 | +__simd_callee__ inline void ScaleSumValue(RegTensor<float>& value, float scale, MaskReg& mask) | ||
| 55 | +{ | ||
| 56 | + if constexpr (IsSameType<T_IN, float>::value) { | ||
| 57 | + AscendC::Reg::Muls(value, value, scale, mask); | ||
| 58 | + } | ||
| 59 | +} | ||
| 60 | + | ||
| 41 | // fp16 → fp32 提升(输入侧)。输出恒 fp32,故本算子无 fp32→fp16 回写的 castTraitB322B16。 | 61 | // fp16 → fp32 提升(输入侧)。输出恒 fp32,故本算子无 fp32→fp16 回写的 castTraitB322B16。 |
| 42 | constexpr AscendC::Reg::CastTrait castTraitB162B32 = { | 62 | constexpr AscendC::Reg::CastTrait castTraitB162B32 = { |
| 43 | AscendC::Reg::RegLayout::ZERO, | 63 | AscendC::Reg::RegLayout::ZERO, |
| @@ -56,6 +56,7 @@ public: | |||
| 56 | sqGm_ = &sqGm; | 56 | sqGm_ = &sqGm; |
| 57 | totalRows_ = totalRows; | 57 | totalRows_ = totalRows; |
| 58 | numR_ = numR; | 58 | numR_ = numR; |
| 59 | + GetSumScale<T_X>(static_cast<uint64_t>(numR_), sumScale_, sumRestore_); | ||
| 59 | rFactor_ = rFactor; | 60 | rFactor_ = rFactor; |
| 60 | numChunks_ = numChunks; | 61 | numChunks_ = numChunks; |
| 61 | tailLen_ = tailLen; | 62 | tailLen_ = tailLen; |
| @@ -139,6 +140,7 @@ private: | |||
| 139 | { | 140 | { |
| 140 | __ubuf__ T_X* xUb = (__ubuf__ T_X*)xLocal.GetPhyAddr(); | 141 | __ubuf__ T_X* xUb = (__ubuf__ T_X*)xLocal.GetPhyAddr(); |
| 141 | uint32_t numSeg = (count + VL_FP32 - 1) / VL_FP32; | 142 | uint32_t numSeg = (count + VL_FP32 - 1) / VL_FP32; |
| 143 | + const float scale = sumScale_; | ||
| 142 | __VEC_SCOPE__ | 144 | __VEC_SCOPE__ |
| 143 | { | 145 | { |
| 144 | RegTensor<float> xReg; | 146 | RegTensor<float> xReg; |
| @@ -162,9 +164,10 @@ private: | |||
| 162 | MaskReg pregSeg = UpdateMask<float>(sreg); | 164 | MaskReg pregSeg = UpdateMask<float>(sreg); |
| 163 | LoadTensorForDtypeTIn<T_X>(xUb, xReg, pregSeg, s * VL_FP32); | 165 | LoadTensorForDtypeTIn<T_X>(xUb, xReg, pregSeg, s * VL_FP32); |
| 164 | ShiftLefts((RegTensor<uint32_t>&)xReg, (RegTensor<uint32_t>&)xReg, static_cast<int16_t>(0), pregSeg); | 166 | ShiftLefts((RegTensor<uint32_t>&)xReg, (RegTensor<uint32_t>&)xReg, static_cast<int16_t>(0), pregSeg); |
| 165 | - AccumulateWithError(sumAccReg, sumLow, xReg, pregFull); // Σx:累进原始 x | 167 | + Mul(sqReg, xReg, xReg, pregFull); // HIGH-1:先逐元素平方 |
| 166 | - Mul(sqReg, xReg, xReg, pregFull); // HIGH-1:先逐元素平方 | 168 | + ScaleSumValue<T_X>(xReg, scale, pregFull); |
| 167 | - AccumulateWithError(sqAccReg, sqLow, sqReg, pregFull); // Σx²:累进已平方值 | 169 | + AccumulateWithError(sumAccReg, sumLow, xReg, pregFull); |
| 170 | + AccumulateWithError(sqAccReg, sqLow, sqReg, pregFull); // Σx²:累进已平方值 | ||
| 168 | } | 171 | } |
| 169 | ReduceWithError(sumAccReg, sumLow, pregFull); | 172 | ReduceWithError(sumAccReg, sumLow, pregFull); |
| 170 | ReduceWithError(sqAccReg, sqLow, pregFull); | 173 | ReduceWithError(sqAccReg, sqLow, pregFull); |
| @@ -184,6 +187,7 @@ private: | |||
| 184 | __ubuf__ float* sqDst, uint32_t nChunks) | 187 | __ubuf__ float* sqDst, uint32_t nChunks) |
| 185 | { | 188 | { |
| 186 | uint32_t numSeg = (nChunks + VL_FP32 - 1) / VL_FP32; | 189 | uint32_t numSeg = (nChunks + VL_FP32 - 1) / VL_FP32; |
| 190 | + const float restore = sumRestore_; | ||
| 187 | __VEC_SCOPE__ | 191 | __VEC_SCOPE__ |
| 188 | { | 192 | { |
| 189 | RegTensor<float> sumReg; | 193 | RegTensor<float> sumReg; |
| @@ -225,6 +229,7 @@ private: | |||
| 225 | ReduceWithError(sqAccReg, sqLow, pregFull); | 229 | ReduceWithError(sqAccReg, sqLow, pregFull); |
| 226 | if constexpr (FINAL_OUTPUT) { | 230 | if constexpr (FINAL_OUTPUT) { |
| 227 | Add(sumAccReg, sumAccReg, sumLow, pregOne); | 231 | Add(sumAccReg, sumAccReg, sumLow, pregOne); |
| 232 | + ScaleSumValue<T_X>(sumAccReg, restore, pregOne); | ||
| 228 | Add(sqAccReg, sqAccReg, sqLow, pregOne); | 233 | Add(sqAccReg, sqAccReg, sqLow, pregOne); |
| 229 | } else { | 234 | } else { |
| 230 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>(sumDst + correctionStride_, sumLow, pregOne); | 235 | DataCopy<float, StoreDist::DIST_FIRST_ELEMENT_B32>(sumDst + correctionStride_, sumLow, pregOne); |
| @@ -259,6 +264,8 @@ private: | |||
| 259 | GlobalTensor<T_SUM>* sqGm_{nullptr}; | 264 | GlobalTensor<T_SUM>* sqGm_{nullptr}; |
| 260 | int64_t totalRows_{0}; | 265 | int64_t totalRows_{0}; |
| 261 | int64_t numR_{0}; | 266 | int64_t numR_{0}; |
| 267 | + float sumScale_{1.0f}; | ||
| 268 | + float sumRestore_{1.0f}; | ||
| 262 | uint32_t rFactor_{0}; | 269 | uint32_t rFactor_{0}; |
| 263 | int64_t numChunks_{0}; | 270 | int64_t numChunks_{0}; |
| 264 | uint32_t tailLen_{0}; | 271 | uint32_t tailLen_{0}; |
| @@ -25,11 +25,18 @@ using namespace INTrainingReduceV2Ops; | |||
| 25 | #define TILINGKEY_AR_FULL_REDUCE 200000 | 25 | #define TILINGKEY_AR_FULL_REDUCE 200000 |
| 26 | #define TILINGKEY_ARA_REDUCE 400000 | 26 | #define TILINGKEY_ARA_REDUCE 400000 |
| 27 | 27 | ||
| 28 | +// Expand the key into one literal without expanding the reserved TILING_KEY_VAR token. | ||
| 29 | +#define IN_TRAINING_REDUCE_V2_STRINGIFY_IMPL(expression) #expression | ||
| 30 | +#define IN_TRAINING_REDUCE_V2_TILING_EXPR(key) IN_TRAINING_REDUCE_V2_STRINGIFY_IMPL(TILING_KEY_VAR == key) | ||
| 31 | + | ||
| 28 | extern "C" __global__ __aicore__ void in_training_reduce_v2(GM_ADDR x, GM_ADDR sum, GM_ADDR square_sum, | 32 | extern "C" __global__ __aicore__ void in_training_reduce_v2(GM_ADDR x, GM_ADDR sum, GM_ADDR square_sum, |
| 29 | GM_ADDR workspace, GM_ADDR tiling) | 33 | GM_ADDR workspace, GM_ADDR tiling) |
| 30 | { | 34 | { |
| 31 | KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY); | 35 | KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY); |
| 32 | REGISTER_TILING_DEFAULT(INTrainingReduceV2ARFullReduceTilingData); | 36 | REGISTER_TILING_DEFAULT(INTrainingReduceV2ARFullReduceTilingData); |
| 37 | + // OOM instrumentation locates tensor bounds after the registered TilingData. | ||
| 38 | + REGISTER_TILING_FOR_TILINGKEY(IN_TRAINING_REDUCE_V2_TILING_EXPR(TILINGKEY_ARA_REDUCE), | ||
| 39 | + INTrainingReduceV2ARAReduceTilingData); | ||
| 33 | if (TILING_KEY_IS(TILINGKEY_AR_FULL_REDUCE)) { | 40 | if (TILING_KEY_IS(TILINGKEY_AR_FULL_REDUCE)) { |
| 34 | GET_TILING_DATA_WITH_STRUCT(INTrainingReduceV2ARFullReduceTilingData, tilingData, tiling); | 41 | GET_TILING_DATA_WITH_STRUCT(INTrainingReduceV2ARFullReduceTilingData, tilingData, tiling); |
| 35 | INTrainingReduceV2ARFullReduce<DTYPE_X> op(&tilingData); | 42 | INTrainingReduceV2ARFullReduce<DTYPE_X> op(&tilingData); |
| @@ -0,0 +1,213 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ---------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +"""Generate portable TTK regression fixtures for reduction precision boundaries. | ||
| 12 | + | ||
| 13 | +Use --output-dir in the task's MyTestCase checkout. No reference output is | ||
| 14 | +embedded: the repository Torch FP64 golden and the GPU composition remain the | ||
| 15 | +independent oracles. --large additionally exercises grouped sub-R carries. | ||
| 16 | +""" | ||
| 17 | + | ||
| 18 | +import argparse | ||
| 19 | +import csv | ||
| 20 | +import hashlib | ||
| 21 | +import json | ||
| 22 | +from pathlib import Path | ||
| 23 | + | ||
| 24 | +import numpy as np | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +def generate(output_dir, large=False): | ||
| 28 | + output_dir.mkdir(parents=True, exist_ok=True) | ||
| 29 | + inputs = output_dir / "precision-inputs" | ||
| 30 | + inputs.mkdir(exist_ok=True) | ||
| 31 | + cases = ( | ||
| 32 | + Path(__file__).parents[1] / "st/arch35/ttk_kernel_in_training_reduce_v2_st.csv" | ||
| 33 | + ) | ||
| 34 | + with cases.open() as source: | ||
| 35 | + template = next(csv.DictReader(source)) | ||
| 36 | + rows, manifest = [], [] | ||
| 37 | + | ||
| 38 | + def add(name, shape, layout, dtype, values, purpose): | ||
| 39 | + np_dtype = np.dtype(dtype) | ||
| 40 | + count = int(np.prod(shape, dtype=object)) | ||
| 41 | + pattern = np.asarray(values, dtype=np_dtype) | ||
| 42 | + path = inputs / (name + ".bin") | ||
| 43 | + # Stream repeated input patterns so grouped tests need no large generator allocation. | ||
| 44 | + chunk = np.tile(pattern, max(1, (1 << 18) // pattern.size)) | ||
| 45 | + digest = hashlib.sha256() | ||
| 46 | + with path.open("wb") as target: | ||
| 47 | + written = 0 | ||
| 48 | + while written < count: | ||
| 49 | + data = chunk[: min(chunk.size, count - written)].tobytes() | ||
| 50 | + target.write(data) | ||
| 51 | + digest.update(data) | ||
| 52 | + written += len(data) // np_dtype.itemsize | ||
| 53 | + channel = len(shape) - 1 if layout in ("NHWC", "NDHWC") else 1 | ||
| 54 | + output_shape = tuple( | ||
| 55 | + value if axis in (0, channel) else 1 for axis, value in enumerate(shape) | ||
| 56 | + ) | ||
| 57 | + row = dict(template) | ||
| 58 | + row.update( | ||
| 59 | + testcase_name=name, | ||
| 60 | + input_shapes=repr((shape,)), | ||
| 61 | + input_ori_shapes=repr((shape,)), | ||
| 62 | + input_formats=repr((layout,)), | ||
| 63 | + input_ori_formats=repr((layout,)), | ||
| 64 | + input_dtypes=repr((dtype,)), | ||
| 65 | + output_shapes=repr((output_shape, output_shape)), | ||
| 66 | + output_ori_shapes=repr((output_shape, output_shape)), | ||
| 67 | + manual_input_binaries=repr((str(path.resolve()),)), | ||
| 68 | + remark=purpose, | ||
| 69 | + input_data_ranges="((None,None),)", | ||
| 70 | + ) | ||
| 71 | + rows.append(row) | ||
| 72 | + manifest.append( | ||
| 73 | + { | ||
| 74 | + "case": name, | ||
| 75 | + "shape": shape, | ||
| 76 | + "layout": layout, | ||
| 77 | + "dtype": dtype, | ||
| 78 | + "purpose": purpose, | ||
| 79 | + "input_sha256": digest.hexdigest(), | ||
| 80 | + } | ||
| 81 | + ) | ||
| 82 | + | ||
| 83 | + for dtype in ("float16", "float32"): | ||
| 84 | + info = np.finfo(dtype) | ||
| 85 | + tiny = np.nextafter(np.array(0, dtype=dtype), np.array(1, dtype=dtype)).item() | ||
| 86 | + below_normal = np.nextafter( | ||
| 87 | + np.array(info.tiny, dtype=dtype), np.array(0, dtype=dtype) | ||
| 88 | + ).item() | ||
| 89 | + denormals = [ | ||
| 90 | + tiny, | ||
| 91 | + -tiny, | ||
| 92 | + below_normal, | ||
| 93 | + -below_normal, | ||
| 94 | + info.tiny, | ||
| 95 | + -info.tiny, | ||
| 96 | + tiny, | ||
| 97 | + tiny, | ||
| 98 | + ] | ||
| 99 | + patterns = { | ||
| 100 | + "denorm": denormals, | ||
| 101 | + "posinf": [1, np.inf, -2, 3], | ||
| 102 | + "neginf": [1, -np.inf, -2, 3], | ||
| 103 | + "bothinf": [1, np.inf, -np.inf, 3], | ||
| 104 | + "nan": [1, np.nan, -2, 3], | ||
| 105 | + "cancel": [info.max, info.max, -info.max, -info.max], | ||
| 106 | + } | ||
| 107 | + for layout, shape in ( | ||
| 108 | + ("ND", (1, 3, 129)), | ||
| 109 | + ("NHWC", (1, 1, 129, 7)), | ||
| 110 | + ("NDHWC", (1, 1, 1, 129, 65)), | ||
| 111 | + ): | ||
| 112 | + for kind, values in patterns.items(): | ||
| 113 | + # Channel-last inputs use whole repeated rows. Each channel sees | ||
| 114 | + # the complete finite/special pattern instead of a constant column. | ||
| 115 | + if layout != "ND": | ||
| 116 | + values = np.repeat(values, shape[-1]) | ||
| 117 | + add( | ||
| 118 | + f"precision_{dtype}_{layout}_{kind}", | ||
| 119 | + shape, | ||
| 120 | + layout, | ||
| 121 | + dtype, | ||
| 122 | + values, | ||
| 123 | + "Explicit " + kind + " and finite mixture; AR or actual C>1 ARA", | ||
| 124 | + ) | ||
| 125 | + add( | ||
| 126 | + f"precision_{dtype}_single", | ||
| 127 | + (1, 1), | ||
| 128 | + "ND", | ||
| 129 | + dtype, | ||
| 130 | + [tiny], | ||
| 131 | + "numel=1 minimum legal input", | ||
| 132 | + ) | ||
| 133 | + add( | ||
| 134 | + f"precision_{dtype}_subr_cancel", | ||
| 135 | + (1, 1, 32769), | ||
| 136 | + "ND", | ||
| 137 | + dtype, | ||
| 138 | + patterns["cancel"], | ||
| 139 | + "Finite extreme cancellation across sub-R chunks", | ||
| 140 | + ) | ||
| 141 | + add( | ||
| 142 | + f"precision_{dtype}_subr_bothinf", | ||
| 143 | + (1, 1, 32769), | ||
| 144 | + "ND", | ||
| 145 | + dtype, | ||
| 146 | + patterns["bothinf"], | ||
| 147 | + "Opposite infinities and finite values across sub-R chunks", | ||
| 148 | + ) | ||
| 149 | + add( | ||
| 150 | + f"precision_{dtype}_ara_chunks_cancel", | ||
| 151 | + (1, 1, 1, 4097, 65), | ||
| 152 | + "NDHWC", | ||
| 153 | + dtype, | ||
| 154 | + np.repeat(patterns["cancel"], 65), | ||
| 155 | + "Finite extreme cancellation across ARA R chunks and C tails", | ||
| 156 | + ) | ||
| 157 | + for reduce_size in ( | ||
| 158 | + 63, | ||
| 159 | + 64, | ||
| 160 | + 65, | ||
| 161 | + 127, | ||
| 162 | + 128, | ||
| 163 | + 129, | ||
| 164 | + 817, | ||
| 165 | + 4236, | ||
| 166 | + 16383, | ||
| 167 | + 16384, | ||
| 168 | + 16385, | ||
| 169 | + 32769, | ||
| 170 | + ): | ||
| 171 | + add( | ||
| 172 | + f"precision_{dtype}_r{reduce_size}", | ||
| 173 | + (2, 3, reduce_size), | ||
| 174 | + "ND", | ||
| 175 | + dtype, | ||
| 176 | + [1, 0.125, -0.5, -0.25, 2, -1], | ||
| 177 | + "VL, folding and full/sub-R capacity boundaries", | ||
| 178 | + ) | ||
| 179 | + for channels in (8, 16, 17, 63, 64, 65, 127, 128, 129): | ||
| 180 | + add( | ||
| 181 | + f"precision_{dtype}_c{channels}", | ||
| 182 | + (2, 1, 4097, channels), | ||
| 183 | + "NHWC", | ||
| 184 | + dtype, | ||
| 185 | + [1, 0.125, -0.5, -0.25, 2, -1], | ||
| 186 | + "ARA gather, channel/vector tails and multi-chunk R", | ||
| 187 | + ) | ||
| 188 | + if large: | ||
| 189 | + add( | ||
| 190 | + f"precision_{dtype}_grouped", | ||
| 191 | + (1, 1, (1 << 28) + 1), | ||
| 192 | + "ND", | ||
| 193 | + dtype, | ||
| 194 | + [1, -0.25, 0.5, -0.125], | ||
| 195 | + "Grouped sub-R: multiple groups, carry, slot tail and chunk tail", | ||
| 196 | + ) | ||
| 197 | + csv_path = output_dir / "precision-regression.csv" | ||
| 198 | + with csv_path.open("w") as target: | ||
| 199 | + writer = csv.DictWriter(target, fieldnames=list(template), lineterminator="\n") | ||
| 200 | + writer.writeheader() | ||
| 201 | + writer.writerows(rows) | ||
| 202 | + (output_dir / "precision-manifest.json").write_text( | ||
| 203 | + json.dumps(manifest, indent=2) + "\n" | ||
| 204 | + ) | ||
| 205 | + print(f"Generated {len(rows)} cases in {csv_path}") | ||
| 206 | + | ||
| 207 | + | ||
| 208 | +if __name__ == "__main__": | ||
| 209 | + parser = argparse.ArgumentParser(description=__doc__) | ||
| 210 | + parser.add_argument("--output-dir", type=Path, required=True) | ||
| 211 | + parser.add_argument("--large", action="store_true") | ||
| 212 | + options = parser.parse_args() | ||
| 213 | + generate(options.output_dir, options.large) | ||
| @@ -21,10 +21,10 @@ Only one registration key is needed: kernel and GEIR both resolve the snake-case | |||
| 21 | operator name and share :class:`INTrainingReduceV2KernelSpec`. The legacy | 21 | operator name and share :class:`INTrainingReduceV2KernelSpec`. The legacy |
| 22 | ``__golden__`` entry remains available and calls the same Torch computation core. | 22 | ``__golden__`` entry remains available and calls the same Torch computation core. |
| 23 | 23 | ||
| 24 | -The reference math uses Torch operators. Promote mode computes both statistics | 24 | +The TestSpec reference computes both statistics in FP64 and preserves FP64 |
| 25 | -in FP64 and preserves FP64 outputs. TTK's FP16-to-FP32 input promotion alone is | 25 | +outputs independently of the comparison mode. TTK's FP16-to-FP32 input promotion |
| 26 | -insufficient for a high-precision reference to this operator's FP32 accumulation | 26 | +alone is insufficient for this operator's FP32 accumulation and outputs. The |
| 27 | -and outputs. Other modes follow the kernel's FP32 computation and output contract. | 27 | +legacy entry retains its historical FP32 output contract outside Promote mode. |
| 28 | """ | 28 | """ |
| 29 | 29 | ||
| 30 | import numpy as np | 30 | import numpy as np |
| @@ -187,7 +187,9 @@ class INTrainingReduceV2KernelSpec: | |||
| 187 | """Shared kernel/GEIR TestSpec; parameters follow def.cpp (``x`` only).""" | 187 | """Shared kernel/GEIR TestSpec; parameters follow def.cpp (``x`` only).""" |
| 188 | 188 | ||
| 189 | def golden(x, **kwargs): | 189 | def golden(x, **kwargs): |
| 190 | - return _kernel_golden(x, **kwargs) | 190 | + reference_kwargs = dict(kwargs) |
| 191 | + reference_kwargs["golden_mode"] = "Promote" | ||
| 192 | + return _kernel_golden(x, **reference_kwargs) | ||
| 191 | 193 | ||
| 192 | third_party = {"torch": _INTrainingReduceV2Compose} | 194 | third_party = {"torch": _INTrainingReduceV2Compose} |
| 193 | tolerance = _TOL | 195 | tolerance = _TOL |
| @@ -0,0 +1,21 @@ | |||
| 1 | +testcase_name,network_name,op_name,input_shapes,input_dtypes,input_formats,output_shapes,output_dtypes,output_formats,input_ori_shapes,input_ori_formats,output_ori_shapes,output_ori_formats,attributes,input_data_ranges,precision_tolerances,absolute_precision,output_inplace_indexes,output_shape_unknown_indexes,is_enabled,remark,soc_series,priority,dump_file_prefix,manual_input_binaries,manual_golden_binaries | ||
| 2 | +h31_rank2_single_float16,UNKNOWN,in_training_reduce_v2,"((1, 1),)","('float16',)","('ND',)","((1, 1), (1, 1))","('float32', 'float32')","('ND', 'ND')","((1, 1),)","('ND',)","((1, 1), (1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 3 | +h31_geir_float16_nd_rank8,UNKNOWN,in_training_reduce_v2,"((2, 3, 2, 3, 2, 3, 2, 3),)","('float16',)","('ND',)","((2, 3, 1, 1, 1, 1, 1, 1), (2, 3, 1, 1, 1, 1, 1, 1))","('float32', 'float32')","('ND', 'ND')","((2, 3, 2, 3, 2, 3, 2, 3),)","('ND',)","((2, 3, 1, 1, 1, 1, 1, 1), (2, 3, 1, 1, 1, 1, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 4 | +h31_geir_float16_nchw_rank4,UNKNOWN,in_training_reduce_v2,"((2, 3, 5, 7),)","('float16',)","('NCHW',)","((2, 3, 1, 1), (2, 3, 1, 1))","('float32', 'float32')","('ND', 'ND')","((2, 3, 5, 7),)","('NCHW',)","((2, 3, 1, 1), (2, 3, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 5 | +h31_geir_float16_ncdhw_rank5,UNKNOWN,in_training_reduce_v2,"((2, 3, 2, 5, 7),)","('float16',)","('NCDHW',)","((2, 3, 1, 1, 1), (2, 3, 1, 1, 1))","('float32', 'float32')","('ND', 'ND')","((2, 3, 2, 5, 7),)","('NCDHW',)","((2, 3, 1, 1, 1), (2, 3, 1, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 6 | +h31_c_tail_float16_c67,UNKNOWN,in_training_reduce_v2,"((1, 5, 13, 67),)","('float16',)","('NHWC',)","((1, 1, 1, 67), (1, 1, 1, 67))","('float32', 'float32')","('ND', 'ND')","((1, 5, 13, 67),)","('NHWC',)","((1, 1, 1, 67), (1, 1, 1, 67))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 7 | +h31_geir_float16_ndhwc_rank5,UNKNOWN,in_training_reduce_v2,"((2, 2, 5, 7, 3),)","('float16',)","('NDHWC',)","((2, 1, 1, 1, 3), (2, 1, 1, 1, 3))","('float32', 'float32')","('ND', 'ND')","((2, 2, 5, 7, 3),)","('NDHWC',)","((2, 1, 1, 1, 3), (2, 1, 1, 1, 3))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 8 | +h31_smallc_float16_r63,UNKNOWN,in_training_reduce_v2,"((1, 1, 63, 7),)","('float16',)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('float32', 'float32')","('ND', 'ND')","((1, 1, 63, 7),)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 9 | +h31_smallc_float16_r64,UNKNOWN,in_training_reduce_v2,"((1, 1, 64, 7),)","('float16',)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('float32', 'float32')","('ND', 'ND')","((1, 1, 64, 7),)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 10 | +h31_smallc_float16_r65,UNKNOWN,in_training_reduce_v2,"((1, 1, 65, 7),)","('float16',)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('float32', 'float32')","('ND', 'ND')","((1, 1, 65, 7),)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 11 | +h31_subr_float16_tail65,UNKNOWN,in_training_reduce_v2,"((1, 1, 131137),)","('float16',)","('ND',)","((1, 1, 1), (1, 1, 1))","('float32', 'float32')","('ND', 'ND')","((1, 1, 131137),)","('ND',)","((1, 1, 1), (1, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 12 | +h31_rank2_single_float32,UNKNOWN,in_training_reduce_v2,"((1, 1),)","('float32',)","('ND',)","((1, 1), (1, 1))","('float32', 'float32')","('ND', 'ND')","((1, 1),)","('ND',)","((1, 1), (1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 13 | +h31_geir_float32_nd_rank8,UNKNOWN,in_training_reduce_v2,"((2, 3, 2, 3, 2, 3, 2, 3),)","('float32',)","('ND',)","((2, 3, 1, 1, 1, 1, 1, 1), (2, 3, 1, 1, 1, 1, 1, 1))","('float32', 'float32')","('ND', 'ND')","((2, 3, 2, 3, 2, 3, 2, 3),)","('ND',)","((2, 3, 1, 1, 1, 1, 1, 1), (2, 3, 1, 1, 1, 1, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 14 | +h31_geir_float32_nchw_rank4,UNKNOWN,in_training_reduce_v2,"((2, 3, 5, 7),)","('float32',)","('NCHW',)","((2, 3, 1, 1), (2, 3, 1, 1))","('float32', 'float32')","('ND', 'ND')","((2, 3, 5, 7),)","('NCHW',)","((2, 3, 1, 1), (2, 3, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 15 | +h31_geir_float32_ncdhw_rank5,UNKNOWN,in_training_reduce_v2,"((2, 3, 2, 5, 7),)","('float32',)","('NCDHW',)","((2, 3, 1, 1, 1), (2, 3, 1, 1, 1))","('float32', 'float32')","('ND', 'ND')","((2, 3, 2, 5, 7),)","('NCDHW',)","((2, 3, 1, 1, 1), (2, 3, 1, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 16 | +h31_c_tail_float32_c67,UNKNOWN,in_training_reduce_v2,"((1, 5, 13, 67),)","('float32',)","('NHWC',)","((1, 1, 1, 67), (1, 1, 1, 67))","('float32', 'float32')","('ND', 'ND')","((1, 5, 13, 67),)","('NHWC',)","((1, 1, 1, 67), (1, 1, 1, 67))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 17 | +h31_geir_float32_ndhwc_rank5,UNKNOWN,in_training_reduce_v2,"((2, 2, 5, 7, 3),)","('float32',)","('NDHWC',)","((2, 1, 1, 1, 3), (2, 1, 1, 1, 3))","('float32', 'float32')","('ND', 'ND')","((2, 2, 5, 7, 3),)","('NDHWC',)","((2, 1, 1, 1, 3), (2, 1, 1, 1, 3))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,GEIR format and rank regression; const and dynamic shapes,,0,,(),() | ||
| 18 | +h31_smallc_float32_r63,UNKNOWN,in_training_reduce_v2,"((1, 1, 63, 7),)","('float32',)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('float32', 'float32')","('ND', 'ND')","((1, 1, 63, 7),)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 19 | +h31_smallc_float32_r64,UNKNOWN,in_training_reduce_v2,"((1, 1, 64, 7),)","('float32',)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('float32', 'float32')","('ND', 'ND')","((1, 1, 64, 7),)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 20 | +h31_smallc_float32_r65,UNKNOWN,in_training_reduce_v2,"((1, 1, 65, 7),)","('float32',)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('float32', 'float32')","('ND', 'ND')","((1, 1, 65, 7),)","('NHWC',)","((1, 1, 1, 7), (1, 1, 1, 7))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| 21 | +h31_subr_float32_tail65,UNKNOWN,in_training_reduce_v2,"((1, 1, 131137),)","('float32',)","('ND',)","((1, 1, 1), (1, 1, 1))","('float32', 'float32')","('ND', 'ND')","((1, 1, 131137),)","('ND',)","((1, 1, 1), (1, 1, 1))","('ND', 'ND')",{},"((1, 2),)",,1e-08,(),(),True,Targeted check of reported tail-mask behavior; UpdateMask updates count by reference,,0,,(),() | ||
| @@ -5,11 +5,11 @@ | |||
| 5 | | 产品 | 是否支持 | | 5 | | 产品 | 是否支持 | |
| 6 | | :----------------------------------------- | :------: | | 6 | | :----------------------------------------- | :------: | |
| 7 | | <term>Ascend 950PR&950DT系列产品</term> | √ | | 7 | | <term>Ascend 950PR&950DT系列产品</term> | √ | |
| 8 | -| <term>Atlas A3系列产品</term>| × | | 8 | +| <term>Atlas A3系列产品</term>| √ | |
| 9 | -| <term>Atlas A2系列产品</term> | × | | 9 | +| <term>Atlas A2系列产品</term> | √ | |
| 10 | -| <term>Atlas 200I/500 A2推理产品</term> | × | | 10 | +| <term>Atlas 200I/500 A2推理产品</term> | √ | |
| 11 | -| <term>Atlas推理系列产品</term> | × | | 11 | +| <term>Atlas推理系列产品</term> | √ | |
| 12 | -| <term>Atlas训练系列产品</term> | × | | 12 | +| <term>Atlas训练系列产品</term> | √ | |
| 13 | 13 | ||
| 14 | ## 功能说明 | 14 | ## 功能说明 |
| 15 | 15 | ||
| @@ -37,12 +37,6 @@ | |||
| 37 | | y | 输出 | 换算后的相对坐标,shape和数据类型与`boxes`一致。 | FLOAT、FLOAT16 | ND | | 37 | | y | 输出 | 换算后的相对坐标,shape和数据类型与`boxes`一致。 | FLOAT、FLOAT16 | ND | |
| 38 | | reversed_box | 可选属性 | 指定`boxes`的坐标轴位置,默认值为false。 | BOOL | - | | 38 | | reversed_box | 可选属性 | 指定`boxes`的坐标轴位置,默认值为false。 | BOOL | - | |
| 39 | 39 | ||
| 40 | -### 产品差异说明 | ||
| 41 | - | ||
| 42 | -| 产品 | 静态shape能力 | 动态shape能力 | shape/rank及空Tensor限制 | | ||
| 43 | -| --- | --- | --- | --- | | ||
| 44 | -| <term>Ascend 950PR&950DT系列产品</term> | 输入、输出均为ND | 支持动态shape和动态rank,输入、输出均为ND | `boxes`的rank为2~8;支持batch或非坐标维为0的空Tensor,坐标维必须为4 | | ||
| 45 | - | ||
| 46 | ## 约束说明 | 40 | ## 约束说明 |
| 47 | 41 | ||
| 48 | - `shape_hw`的batch必须与`boxes`的batch一致。 | 42 | - `shape_hw`的batch必须与`boxes`的batch一致。 |
| @@ -62,6 +56,6 @@ | |||
| 62 | 56 | ||
| 63 | 本算子不提供aclnn单算子接口,通过图引擎(Graph Engine,GE)图模式调用。 | 57 | 本算子不提供aclnn单算子接口,通过图引擎(Graph Engine,GE)图模式调用。 |
| 64 | 58 | ||
| 65 | -| 调用方式 | 调用样例 | 说明 | | 59 | +| 调用方式 | 样例代码 | 说明 | |
| 66 | | --- | --- | --- | | 60 | | --- | --- | --- | |
| 67 | | GE图模式 | [test_geir_normalize_bbox.cpp](examples/arch35/test_geir_normalize_bbox.cpp) | 通过[NormalizeBBox算子IR](op_graph/normalize_bbox_proto.h)构图并调用算子。 | | 61 | | GE图模式 | [test_geir_normalize_bbox.cpp](examples/arch35/test_geir_normalize_bbox.cpp) | 通过[NormalizeBBox算子IR](op_graph/normalize_bbox_proto.h)构图并调用算子。 | |
| @@ -21,15 +21,28 @@ | |||
| 21 | namespace ops { | 21 | namespace ops { |
| 22 | namespace { | 22 | namespace { |
| 23 | constexpr size_t INPUT_BOXES_IDX = 0; | 23 | constexpr size_t INPUT_BOXES_IDX = 0; |
| 24 | +constexpr size_t INPUT_SHAPE_HW_IDX = 1; | ||
| 24 | constexpr size_t OUTPUT_Y_IDX = 0; | 25 | constexpr size_t OUTPUT_Y_IDX = 0; |
| 25 | } // namespace | 26 | } // namespace |
| 26 | 27 | ||
| 27 | -static ge::graphStatus NormalizeBBoxInferDataType(gert::InferDataTypeContext* context) | 28 | +ge::graphStatus NormalizeBBoxInferDataType(gert::InferDataTypeContext* context) |
| 28 | { | 29 | { |
| 30 | + if (context == nullptr) { | ||
| 31 | + return ge::GRAPH_FAILED; | ||
| 32 | + } | ||
| 29 | OP_LOGD(context, "NormalizeBBoxInferDataType enter"); | 33 | OP_LOGD(context, "NormalizeBBoxInferDataType enter"); |
| 30 | - context->SetOutputDataType(OUTPUT_Y_IDX, context->GetInputDataType(INPUT_BOXES_IDX)); | 34 | + const auto boxesDtype = context->GetInputDataType(INPUT_BOXES_IDX); |
| 35 | + OP_CHECK_IF(boxesDtype != ge::DT_FLOAT16 && boxesDtype != ge::DT_FLOAT, | ||
| 36 | + OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON(context->GetNodeName(), "boxes", Ops::Base::ToString(boxesDtype), | ||
| 37 | + "only float16 and float32 are supported"), | ||
| 38 | + return ge::GRAPH_FAILED); | ||
| 39 | + OP_CHECK_IF(context->GetInputDataType(INPUT_SHAPE_HW_IDX) != ge::DT_INT32, | ||
| 40 | + OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON( | ||
| 41 | + context->GetNodeName(), "shape_hw", | ||
| 42 | + Ops::Base::ToString(context->GetInputDataType(INPUT_SHAPE_HW_IDX)), "only int32 is supported"), | ||
| 43 | + return ge::GRAPH_FAILED); | ||
| 31 | OP_LOGD(context, "NormalizeBBoxInferDataType end"); | 44 | OP_LOGD(context, "NormalizeBBoxInferDataType end"); |
| 32 | - return ge::GRAPH_SUCCESS; | 45 | + return context->SetOutputDataType(OUTPUT_Y_IDX, boxesDtype); |
| 33 | } | 46 | } |
| 34 | 47 | ||
| 35 | IMPL_OP(NormalizeBBox).InferDataType(NormalizeBBoxInferDataType); | 48 | IMPL_OP(NormalizeBBox).InferDataType(NormalizeBBoxInferDataType); |
| @@ -21,14 +21,70 @@ | |||
| 21 | using namespace ge; | 21 | using namespace ge; |
| 22 | namespace ops { | 22 | namespace ops { |
| 23 | static constexpr size_t INPUT_BOXES_IDX = 0; | 23 | static constexpr size_t INPUT_BOXES_IDX = 0; |
| 24 | +static constexpr size_t INPUT_SHAPE_HW_IDX = 1; | ||
| 24 | static constexpr size_t OUTPUT_Y_IDX = 0; | 25 | static constexpr size_t OUTPUT_Y_IDX = 0; |
| 26 | +static constexpr size_t MAX_BOXES_RANK = 8; | ||
| 27 | + | ||
| 28 | +static ge::graphStatus CheckNormalizeBBoxShapes(gert::InferShapeContext* context, const gert::Shape& boxesShape, | ||
| 29 | + const gert::Shape& shapeHwShape) | ||
| 30 | +{ | ||
| 31 | + const bool unknownBoxesRank = Ops::Base::IsUnknownRank(boxesShape); | ||
| 32 | + const bool unknownHwRank = Ops::Base::IsUnknownRank(shapeHwShape); | ||
| 33 | + if (!unknownHwRank) { | ||
| 34 | + OP_CHECK_IF(shapeHwShape.GetDimNum() != 2, | ||
| 35 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( | ||
| 36 | + context->GetNodeName(), "shape_hw", std::to_string(shapeHwShape.GetDimNum()), "rank must be 2"), | ||
| 37 | + return ge::GRAPH_FAILED); | ||
| 38 | + const auto width = shapeHwShape.GetDim(1); | ||
| 39 | + OP_CHECK_IF(width >= 0 && width != 3, | ||
| 40 | + OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(context->GetNodeName(), "shape_hw", | ||
| 41 | + Ops::Base::ToString(shapeHwShape), "dimension 1 must be 3"), | ||
| 42 | + return ge::GRAPH_FAILED); | ||
| 43 | + } | ||
| 44 | + if (!unknownBoxesRank) { | ||
| 45 | + const auto rank = boxesShape.GetDimNum(); | ||
| 46 | + OP_CHECK_IF(rank < 2 || rank > MAX_BOXES_RANK, | ||
| 47 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(context->GetNodeName(), "boxes", std::to_string(rank), | ||
| 48 | + "rank must be in [2, 8]"), | ||
| 49 | + return ge::GRAPH_FAILED); | ||
| 50 | + const auto* attrs = context->GetAttrs(); | ||
| 51 | + const auto* reversed = attrs == nullptr ? nullptr : attrs->GetAttrPointer<bool>(0); | ||
| 52 | + const size_t coordinateAxis = reversed != nullptr && *reversed ? 1 : rank - 1; | ||
| 53 | + const auto coordinateSize = boxesShape.GetDim(coordinateAxis); | ||
| 54 | + OP_CHECK_IF( | ||
| 55 | + coordinateSize >= 0 && coordinateSize != 4, | ||
| 56 | + OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(context->GetNodeName(), "boxes", Ops::Base::ToString(boxesShape), | ||
| 57 | + "coordinate dimension must be 4"), | ||
| 58 | + return ge::GRAPH_FAILED); | ||
| 59 | + } | ||
| 60 | + if (!unknownBoxesRank && !unknownHwRank) { | ||
| 61 | + const auto boxesBatch = boxesShape.GetDim(0); | ||
| 62 | + const auto hwBatch = shapeHwShape.GetDim(0); | ||
| 63 | + OP_CHECK_IF(boxesBatch >= 0 && hwBatch >= 0 && boxesBatch != hwBatch, | ||
| 64 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( | ||
| 65 | + context->GetNodeName(), "boxes, shape_hw", | ||
| 66 | + Ops::Base::ToString(boxesShape) + ", " + Ops::Base::ToString(shapeHwShape), | ||
| 67 | + "batch dimensions must match"), | ||
| 68 | + return ge::GRAPH_FAILED); | ||
| 69 | + } | ||
| 70 | + return ge::GRAPH_SUCCESS; | ||
| 71 | +} | ||
| 25 | 72 | ||
| 26 | static ge::graphStatus NormalizeBBoxInferShape(gert::InferShapeContext* context) | 73 | static ge::graphStatus NormalizeBBoxInferShape(gert::InferShapeContext* context) |
| 27 | { | 74 | { |
| 75 | + if (context == nullptr) { | ||
| 76 | + return ge::GRAPH_FAILED; | ||
| 77 | + } | ||
| 28 | OP_LOGD(context, "Begin to do NormalizeBBoxInferShape"); | 78 | OP_LOGD(context, "Begin to do NormalizeBBoxInferShape"); |
| 29 | 79 | ||
| 30 | const gert::Shape* boxesShape = context->GetInputShape(INPUT_BOXES_IDX); | 80 | const gert::Shape* boxesShape = context->GetInputShape(INPUT_BOXES_IDX); |
| 31 | OP_CHECK_NULL_WITH_CONTEXT(context, boxesShape); | 81 | OP_CHECK_NULL_WITH_CONTEXT(context, boxesShape); |
| 82 | + const gert::Shape* shapeHwShape = context->GetInputShape(INPUT_SHAPE_HW_IDX); | ||
| 83 | + OP_CHECK_NULL_WITH_CONTEXT(context, shapeHwShape); | ||
| 84 | + const auto shapeStatus = CheckNormalizeBBoxShapes(context, *boxesShape, *shapeHwShape); | ||
| 85 | + if (shapeStatus != ge::GRAPH_SUCCESS) { | ||
| 86 | + return shapeStatus; | ||
| 87 | + } | ||
| 32 | gert::Shape* yShape = context->GetOutputShape(OUTPUT_Y_IDX); | 88 | gert::Shape* yShape = context->GetOutputShape(OUTPUT_Y_IDX); |
| 33 | OP_CHECK_NULL_WITH_CONTEXT(context, yShape); | 89 | OP_CHECK_NULL_WITH_CONTEXT(context, yShape); |
| 34 | if (Ops::Base::IsUnknownRank(*boxesShape)) { | 90 | if (Ops::Base::IsUnknownRank(*boxesShape)) { |
| @@ -224,11 +224,13 @@ ge::graphStatus NormalizeBBoxTilingForRegbase::GetShapeAttrsInfo() | |||
| 224 | ge::graphStatus NormalizeBBoxTilingForRegbase::ComputeTileLen() | 224 | ge::graphStatus NormalizeBBoxTilingForRegbase::ComputeTileLen() |
| 225 | { | 225 | { |
| 226 | uint64_t divisorBlocks = reversedBox_ ? 2 : 1; | 226 | uint64_t divisorBlocks = reversedBox_ ? 2 : 1; |
| 227 | - uint64_t ubBufFactor = 2 + 2 + divisorBlocks; | 227 | + const uint64_t computeDtypeSize = sizeof(float); |
| 228 | + const uint64_t computeBytesPerElem = boxesDType_ == ge::DT_FLOAT16 ? computeDtypeSize : 0; | ||
| 229 | + const uint64_t ubBytesPerElem = 4 * boxesDtypeSize_ + divisorBlocks * computeDtypeSize + computeBytesPerElem; | ||
| 228 | uint64_t blockAlignElems = VREPEAT_SIZE / boxesDtypeSize_; | 230 | uint64_t blockAlignElems = VREPEAT_SIZE / boxesDtypeSize_; |
| 229 | uint64_t tileLen = 0; | 231 | uint64_t tileLen = 0; |
| 230 | if (ubSize_ > UB_RESERVE) { | 232 | if (ubSize_ > UB_RESERVE) { |
| 231 | - tileLen = (ubSize_ - UB_RESERVE) / (ubBufFactor * boxesDtypeSize_); | 233 | + tileLen = (ubSize_ - UB_RESERVE) / ubBytesPerElem; |
| 232 | } | 234 | } |
| 233 | tileLen = tileLen / blockAlignElems * blockAlignElems; | 235 | tileLen = tileLen / blockAlignElems * blockAlignElems; |
| 234 | if (tileLen > MAX_TILE_LEN) { | 236 | if (tileLen > MAX_TILE_LEN) { |
| @@ -241,9 +243,10 @@ ge::graphStatus NormalizeBBoxTilingForRegbase::ComputeTileLen() | |||
| 241 | tilingData_.boxesBufferBytes = tileLen * boxesDtypeSize_; | 243 | tilingData_.boxesBufferBytes = tileLen * boxesDtypeSize_; |
| 242 | tilingData_.shapeBufferBytes = BLOCK_SIZE; | 244 | tilingData_.shapeBufferBytes = BLOCK_SIZE; |
| 243 | tilingData_.hwCastBufferBytes = BLOCK_SIZE * 2; | 245 | tilingData_.hwCastBufferBytes = BLOCK_SIZE * 2; |
| 244 | - tilingData_.divisorBufferBytes = tileLen * boxesDtypeSize_ * divisorBlocks; | 246 | + tilingData_.divisorBufferBytes = tileLen * computeDtypeSize * divisorBlocks; |
| 245 | const uint64_t requiredUb = UB_RESERVE + tilingData_.boxesBufferBytes * 2 * 2 + tilingData_.shapeBufferBytes + | 247 | const uint64_t requiredUb = UB_RESERVE + tilingData_.boxesBufferBytes * 2 * 2 + tilingData_.shapeBufferBytes + |
| 246 | - tilingData_.hwCastBufferBytes + tilingData_.divisorBufferBytes; | 248 | + tilingData_.hwCastBufferBytes + tilingData_.divisorBufferBytes + |
| 249 | + tileLen * computeBytesPerElem; | ||
| 247 | OP_CHECK_IF(requiredUb > ubSize_, | 250 | OP_CHECK_IF(requiredUb > ubSize_, |
| 248 | OP_LOGE(opName_, "UB is too small: required=%luB available=%luB.", requiredUb, ubSize_), | 251 | OP_LOGE(opName_, "UB is too small: required=%luB available=%luB.", requiredUb, ubSize_), |
| 249 | return ge::GRAPH_FAILED); | 252 | return ge::GRAPH_FAILED); |
| @@ -32,18 +32,12 @@ constexpr int64_t NB_BLOCK_SIZE = 32; | |||
| 32 | constexpr int64_t NB_VREPEAT_SIZE = 256; | 32 | constexpr int64_t NB_VREPEAT_SIZE = 256; |
| 33 | constexpr int64_t NB_DOUBLE_BUFFER = 2; | 33 | constexpr int64_t NB_DOUBLE_BUFFER = 2; |
| 34 | constexpr uint64_t NB_ODD_LANE_MASK = 0xAAAAAAAAAAAAAAAAULL; // odd lanes (x-system coords) -> w | 34 | constexpr uint64_t NB_ODD_LANE_MASK = 0xAAAAAAAAAAAAAAAAULL; // odd lanes (x-system coords) -> w |
| 35 | -constexpr int64_t NB_MASK_ELEM_WIDTH = 128; // mask Duplicate covers 128 CompT elements per repeat | 35 | +constexpr int64_t NB_MASK_ELEM_WIDTH = 128; // divisor coverage alignment (two fp32 repeats) |
| 36 | constexpr size_t NB_MASK_WORD_COUNT = 2; | 36 | constexpr size_t NB_MASK_WORD_COUNT = 2; |
| 37 | constexpr uint8_t NB_DUPLICATE_REPEAT_COUNT = 1; | 37 | constexpr uint8_t NB_DUPLICATE_REPEAT_COUNT = 1; |
| 38 | constexpr uint8_t NB_DUPLICATE_BLOCK_STRIDE = 1; | 38 | constexpr uint8_t NB_DUPLICATE_BLOCK_STRIDE = 1; |
| 39 | constexpr uint8_t NB_DUPLICATE_REPEAT_STRIDE = 8; | 39 | constexpr uint8_t NB_DUPLICATE_REPEAT_STRIDE = 8; |
| 40 | 40 | ||
| 41 | -// Compute type: boxes dtype is half or float, Div runs in-place (CompT == T). | ||
| 42 | -template <typename T> | ||
| 43 | -struct DivCompute { | ||
| 44 | - using Type = T; | ||
| 45 | -}; | ||
| 46 | - | ||
| 47 | template <typename T, bool reversedBox> | 41 | template <typename T, bool reversedBox> |
| 48 | class NormalizeBBoxKernel { | 42 | class NormalizeBBoxKernel { |
| 49 | public: | 43 | public: |
| @@ -52,7 +46,8 @@ public: | |||
| 52 | __aicore__ inline void Init(GM_ADDR boxes, GM_ADDR shape_hw, GM_ADDR y); | 46 | __aicore__ inline void Init(GM_ADDR boxes, GM_ADDR shape_hw, GM_ADDR y); |
| 53 | __aicore__ inline void Process(); | 47 | __aicore__ inline void Process(); |
| 54 | 48 | ||
| 55 | - using CompT = typename DivCompute<T>::Type; | 49 | + // Keep int32 image dimensions in fp32, including dimensions above 65504. |
| 50 | + using CompT = float; | ||
| 56 | 51 | ||
| 57 | private: | 52 | private: |
| 58 | __aicore__ inline void LoadHW(int64_t b, CompT& h, CompT& w); | 53 | __aicore__ inline void LoadHW(int64_t b, CompT& h, CompT& w); |
| @@ -82,8 +77,9 @@ private: | |||
| 82 | TQue<QuePosition::VECIN, NB_DOUBLE_BUFFER> boxesInQue_; | 77 | TQue<QuePosition::VECIN, NB_DOUBLE_BUFFER> boxesInQue_; |
| 83 | TQue<QuePosition::VECOUT, NB_DOUBLE_BUFFER> boxesOutQue_; | 78 | TQue<QuePosition::VECOUT, NB_DOUBLE_BUFFER> boxesOutQue_; |
| 84 | TQue<QuePosition::VECIN, 1> shapeInQue_; // shape_hw 3 int32 per batch | 79 | TQue<QuePosition::VECIN, 1> shapeInQue_; // shape_hw 3 int32 per batch |
| 85 | - TBuf<TPosition::VECCALC> hwCastBuf_; // int32 -> f32 [-> half] scratch for h/w | 80 | + TBuf<TPosition::VECCALC> hwCastBuf_; // int32 -> f32 scratch for h/w |
| 86 | TBuf<TPosition::VECCALC> divisorBuf_; // divisor block(s), in CompT | 81 | TBuf<TPosition::VECCALC> divisorBuf_; // divisor block(s), in CompT |
| 82 | + TBuf<TPosition::VECCALC> computeBuf_; // fp16 boxes -> fp32 division scratch | ||
| 87 | 83 | ||
| 88 | GlobalTensor<T> boxesGm_; | 84 | GlobalTensor<T> boxesGm_; |
| 89 | GlobalTensor<T> yGm_; | 85 | GlobalTensor<T> yGm_; |
| @@ -126,14 +122,15 @@ __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::Init(GM_ADDR boxes, | |||
| 126 | tpipe_.InitBuffer(boxesInQue_, NB_DOUBLE_BUFFER, tiling_.boxesBufferBytes); | 122 | tpipe_.InitBuffer(boxesInQue_, NB_DOUBLE_BUFFER, tiling_.boxesBufferBytes); |
| 127 | tpipe_.InitBuffer(boxesOutQue_, NB_DOUBLE_BUFFER, tiling_.boxesBufferBytes); | 123 | tpipe_.InitBuffer(boxesOutQue_, NB_DOUBLE_BUFFER, tiling_.boxesBufferBytes); |
| 128 | tpipe_.InitBuffer(shapeInQue_, 1, tiling_.shapeBufferBytes); // 3 int32 fits in one 32B block | 124 | tpipe_.InitBuffer(shapeInQue_, 1, tiling_.shapeBufferBytes); // 3 int32 fits in one 32B block |
| 129 | - tpipe_.InitBuffer(hwCastBuf_, tiling_.hwCastBufferBytes); // f32 + half scratch | 125 | + tpipe_.InitBuffer(hwCastBuf_, tiling_.hwCastBufferBytes); |
| 130 | // divisor is in CompT: normal needs 1 full-width block; reversed needs divH + divW (2 blocks) | 126 | // divisor is in CompT: normal needs 1 full-width block; reversed needs divH + divW (2 blocks) |
| 131 | tpipe_.InitBuffer(divisorBuf_, tiling_.divisorBufferBytes); | 127 | tpipe_.InitBuffer(divisorBuf_, tiling_.divisorBufferBytes); |
| 128 | + if constexpr (std::is_same_v<T, half>) { | ||
| 129 | + tpipe_.InitBuffer(computeBuf_, tileLen_ * sizeof(CompT)); | ||
| 130 | + } | ||
| 132 | } | 131 | } |
| 133 | 132 | ||
| 134 | -// Load shape_hw[b] = [h, w, *] (int32) -> Cast to CompT scalars h, w. | 133 | +// Load shape_hw[b] = [h, w, *] without narrowing the int32 dimensions to fp16. |
| 135 | -// CompT is float for float and half for half. The divisor is built in CompT, so | ||
| 136 | -// the whole divide runs in CompT (cast chain: int32->f32->[f16 for half boxes]->divide). | ||
| 137 | template <typename T, bool reversedBox> | 134 | template <typename T, bool reversedBox> |
| 138 | __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::LoadHW(int64_t b, CompT& h, CompT& w) | 135 | __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::LoadHW(int64_t b, CompT& h, CompT& w) |
| 139 | { | 136 | { |
| @@ -149,21 +146,10 @@ __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::LoadHW(int64_t b, Co | |||
| 149 | PipeBarrier<PIPE_V>(); | 146 | PipeBarrier<PIPE_V>(); |
| 150 | 147 | ||
| 151 | event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 148 | event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 152 | - if constexpr (std::is_same_v<CompT, half>) { | 149 | + SetFlag<HardEvent::V_S>(eventVS); |
| 153 | - LocalTensor<half> hwHalf = hwCastBuf_.Get<half>()[16]; // separate 32B block (byte 32) | 150 | + WaitFlag<HardEvent::V_S>(eventVS); |
| 154 | - Cast(hwHalf, hwF32, RoundMode::CAST_RINT, 3); // float32 -> half | 151 | + h = hwF32.GetValue(0); |
| 155 | - PipeBarrier<PIPE_V>(); | 152 | + w = hwF32.GetValue(1); |
| 156 | - SetFlag<HardEvent::V_S>(eventVS); | ||
| 157 | - WaitFlag<HardEvent::V_S>(eventVS); | ||
| 158 | - h = hwHalf.GetValue(0); | ||
| 159 | - w = hwHalf.GetValue(1); | ||
| 160 | - } else { | ||
| 161 | - // CompT == float: read h/w directly as float | ||
| 162 | - SetFlag<HardEvent::V_S>(eventVS); | ||
| 163 | - WaitFlag<HardEvent::V_S>(eventVS); | ||
| 164 | - h = hwF32.GetValue(0); | ||
| 165 | - w = hwF32.GetValue(1); | ||
| 166 | - } | ||
| 167 | shapeInQue_.FreeTensor(shapeLocal); | 153 | shapeInQue_.FreeTensor(shapeLocal); |
| 168 | } | 154 | } |
| 169 | 155 | ||
| @@ -176,10 +162,8 @@ __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::BuildDivisorNormal(c | |||
| 176 | Duplicate(divisor, h, divWidth); // all lanes = h | 162 | Duplicate(divisor, h, divWidth); // all lanes = h |
| 177 | PipeBarrier<PIPE_V>(); | 163 | PipeBarrier<PIPE_V>(); |
| 178 | // overwrite odd lanes (x-system coords) with w via masked Duplicate. | 164 | // overwrite odd lanes (x-system coords) with w via masked Duplicate. |
| 179 | - // DAV_3510 Vector lane count is dtype-dependent: 128 for 2-byte (half), 64 for 4-byte (float). | 165 | + // Each 256B repeat holds 64 fp32 divisors and uses the first 64 mask bits. |
| 180 | - // mask[2] has 128 bits, but only the first 64 are effective for 4-byte CompT on 3510, | 166 | + // Fill every repeat separately so all odd lanes receive the width. |
| 181 | - // so elements 64+ would keep h instead of w if we used a single 128-element repeat. | ||
| 182 | - // Process in chunks of 256/sizeof(CompT) elements (64 for float, 128 for half) to cover all dtypes. | ||
| 183 | constexpr int32_t maskElemWidth = NB_VREPEAT_SIZE / static_cast<int32_t>(sizeof(CompT)); | 167 | constexpr int32_t maskElemWidth = NB_VREPEAT_SIZE / static_cast<int32_t>(sizeof(CompT)); |
| 184 | uint64_t mask[NB_MASK_WORD_COUNT] = {NB_ODD_LANE_MASK, NB_ODD_LANE_MASK}; | 168 | uint64_t mask[NB_MASK_WORD_COUNT] = {NB_ODD_LANE_MASK, NB_ODD_LANE_MASK}; |
| 185 | for (int32_t off = 0; off < divWidth; off += maskElemWidth) { | 169 | for (int32_t off = 0; off < divWidth; off += maskElemWidth) { |
| @@ -198,7 +182,7 @@ __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::BuildConstBlock(cons | |||
| 198 | } | 182 | } |
| 199 | 183 | ||
| 200 | // CopyIn one tile -> Div by divisor -> CopyOut. | 184 | // CopyIn one tile -> Div by divisor -> CopyOut. |
| 201 | -// half/float: Div runs directly in T (CompT == T). | 185 | +// fp16 inputs are widened before division and rounded once on output. |
| 202 | template <typename T, bool reversedBox> | 186 | template <typename T, bool reversedBox> |
| 203 | __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::CopyDivide(int64_t gmOffset, int64_t processLen, | 187 | __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::CopyDivide(int64_t gmOffset, int64_t processLen, |
| 204 | const LocalTensor<CompT>& divisor) | 188 | const LocalTensor<CompT>& divisor) |
| @@ -211,7 +195,16 @@ __aicore__ inline void NormalizeBBoxKernel<T, reversedBox>::CopyDivide(int64_t g | |||
| 211 | 195 | ||
| 212 | boxesIn = boxesInQue_.template DeQue<T>(); | 196 | boxesIn = boxesInQue_.template DeQue<T>(); |
| 213 | LocalTensor<T> boxesOut = boxesOutQue_.template AllocTensor<T>(); | 197 | LocalTensor<T> boxesOut = boxesOutQue_.template AllocTensor<T>(); |
| 214 | - Div(boxesOut, boxesIn, divisor, static_cast<int32_t>(processLen)); | 198 | + if constexpr (std::is_same_v<T, half>) { |
| 199 | + LocalTensor<CompT> compute = computeBuf_.Get<CompT>(); | ||
| 200 | + Cast(compute, boxesIn, RoundMode::CAST_NONE, static_cast<int32_t>(processLen)); | ||
| 201 | + PipeBarrier<PIPE_V>(); | ||
| 202 | + Div(compute, compute, divisor, static_cast<int32_t>(processLen)); | ||
| 203 | + PipeBarrier<PIPE_V>(); | ||
| 204 | + Cast(boxesOut, compute, RoundMode::CAST_RINT, static_cast<int32_t>(processLen)); | ||
| 205 | + } else { | ||
| 206 | + Div(boxesOut, boxesIn, divisor, static_cast<int32_t>(processLen)); | ||
| 207 | + } | ||
| 215 | boxesInQue_.FreeTensor(boxesIn); | 208 | boxesInQue_.FreeTensor(boxesIn); |
| 216 | boxesOutQue_.EnQue(boxesOut); | 209 | boxesOutQue_.EnQue(boxesOut); |
| 217 | 210 | ||
| @@ -41,7 +41,7 @@ struct NormalizeBBoxTilingData { | |||
| 41 | int64_t shapeElements; // shape_hw 总元素数(batch*3) | 41 | int64_t shapeElements; // shape_hw 总元素数(batch*3) |
| 42 | uint64_t boxesBufferBytes; // boxes input/output 各 queue 单 buffer 字节数 | 42 | uint64_t boxesBufferBytes; // boxes input/output 各 queue 单 buffer 字节数 |
| 43 | uint64_t shapeBufferBytes; // shape_hw queue 字节数 | 43 | uint64_t shapeBufferBytes; // shape_hw queue 字节数 |
| 44 | - uint64_t hwCastBufferBytes; // int32->fp32/half 临时 buffer 字节数 | 44 | + uint64_t hwCastBufferBytes; // int32->fp32 临时 buffer 字节数 |
| 45 | uint64_t divisorBufferBytes; // divisor buffer 字节数 | 45 | uint64_t divisorBufferBytes; // divisor buffer 字节数 |
| 46 | }; | 46 | }; |
| 47 | #pragma pack(pop) | 47 | #pragma pack(pop) |
| @@ -11,10 +11,9 @@ | |||
| 11 | 11 | ||
| 12 | """NormalizeBBox kernel/GEIR golden (TestSpec multi-path form). | 12 | """NormalizeBBox kernel/GEIR golden (TestSpec multi-path form). |
| 13 | 13 | ||
| 14 | -The mathematical golden uses Torch tensor composition and honors TTK Promote: | 14 | +The mathematical reference preserves TTK Promote and performs at least fp32 |
| 15 | -it computes in the dtype presented by the framework and never narrows a promoted | 15 | +division, so int32 image dimensions are never narrowed to fp16. The independent |
| 16 | -fp64/fp32 input. The independent third-party compose mirrors the arch35 kernel's | 16 | +GPU composition follows the formula rather than reproducing kernel casts. |
| 17 | -dtype-dependent int32 -> fp32 [-> fp16] divisor construction and division path. | ||
| 18 | """ | 17 | """ |
| 19 | 18 | ||
| 20 | import numpy as np | 19 | import numpy as np |
| @@ -73,6 +72,8 @@ def _compute(boxes, shape_hw, reversed_box=False): | |||
| 73 | active reference precision and never cuts a promoted input back down. | 72 | active reference precision and never cuts a promoted input back down. |
| 74 | """ | 73 | """ |
| 75 | boxes_tensor = _as_tensor(boxes) | 74 | boxes_tensor = _as_tensor(boxes) |
| 75 | + if boxes_tensor.dtype == torch.float16: | ||
| 76 | + boxes_tensor = boxes_tensor.to(torch.float32) | ||
| 76 | shape_tensor = _as_tensor(shape_hw) | 77 | shape_tensor = _as_tensor(shape_hw) |
| 77 | 78 | ||
| 78 | batch = boxes_tensor.shape[0] | 79 | batch = boxes_tensor.shape[0] |
| @@ -108,7 +109,7 @@ def _numpy_outputs(outputs, output_dtypes=None): | |||
| 108 | 109 | ||
| 109 | 110 | ||
| 110 | class _NormalizeBBoxCompose: | 111 | class _NormalizeBBoxCompose: |
| 111 | - """Independent Torch compose matching the arch35 kernel operation order.""" | 112 | + """Independent GPU reference for coordinate normalization.""" |
| 112 | 113 | ||
| 113 | def __init__(self, reversed_box=False, **kwargs): | 114 | def __init__(self, reversed_box=False, **kwargs): |
| 114 | self.reversed_box = _resolve_reversed_box(reversed_box, kwargs) | 115 | self.reversed_box = _resolve_reversed_box(reversed_box, kwargs) |
| @@ -121,14 +122,9 @@ class _NormalizeBBoxCompose: | |||
| 121 | f"got {output_dtype}" | 122 | f"got {output_dtype}" |
| 122 | ) | 123 | ) |
| 123 | 124 | ||
| 124 | - # NormalizeBBoxKernel::LoadHW, line for line at dtype level: | 125 | + # Image dimensions may exceed the finite fp16 range even when the |
| 125 | - # shape input is int32, then Cast int32 -> fp32, and fp16 boxes take one | 126 | + # normalized coordinates are representable. Divide before narrowing. |
| 126 | - # additional Cast fp32 -> fp16 before Div; fp32 boxes stay in fp32. | 127 | + hw_compute = shape_hw.to(dtype=torch.float32) |
| 127 | - shape_i32 = shape_hw.to(dtype=torch.int32) | ||
| 128 | - hw_f32 = shape_i32.to(dtype=torch.float32) | ||
| 129 | - hw_compute = ( | ||
| 130 | - hw_f32.to(dtype=torch.float16) if output_dtype == torch.float16 else hw_f32 | ||
| 131 | - ) | ||
| 132 | height = hw_compute[:, 0] | 128 | height = hw_compute[:, 0] |
| 133 | width = hw_compute[:, 1] | 129 | width = hw_compute[:, 1] |
| 134 | divisor4 = torch.stack((height, width, height, width), dim=1) | 130 | divisor4 = torch.stack((height, width, height, width), dim=1) |
| @@ -141,8 +137,7 @@ class _NormalizeBBoxCompose: | |||
| 141 | divisor_shape = (batch,) + (1,) * (rank - 2) + (4,) | 137 | divisor_shape = (batch,) + (1,) * (rank - 2) + (4,) |
| 142 | divisor = torch.reshape(divisor4, divisor_shape) | 138 | divisor = torch.reshape(divisor4, divisor_shape) |
| 143 | 139 | ||
| 144 | - # NormalizeBBoxKernel::CopyDivide performs Div directly in boxes dtype. | 140 | + output = torch.div(boxes.to(torch.float32), divisor) |
| 145 | - output = torch.div(boxes, divisor) | ||
| 146 | return [output.to(dtype=output_dtype)] | 141 | return [output.to(dtype=output_dtype)] |
| 147 | 142 | ||
| 148 | 143 | ||
| @@ -168,6 +163,7 @@ def normalize_bbox_golden(boxes, shape_hw, reversed_box=False, **kwargs): | |||
| 168 | input_is_tensor = isinstance(boxes, torch.Tensor) | 163 | input_is_tensor = isinstance(boxes, torch.Tensor) |
| 169 | reversed_value = _resolve_reversed_box(reversed_box, kwargs) | 164 | reversed_value = _resolve_reversed_box(reversed_box, kwargs) |
| 170 | output = _compute(boxes, shape_hw, reversed_value)[0] | 165 | output = _compute(boxes, shape_hw, reversed_value)[0] |
| 166 | + output = output.to(dtype=_as_tensor(boxes).dtype) | ||
| 171 | if input_is_tensor: | 167 | if input_is_tensor: |
| 172 | return output | 168 | return output |
| 173 | return output.detach().cpu().numpy() | 169 | return output.detach().cpu().numpy() |
| @@ -0,0 +1,9 @@ | |||
| 1 | +testcase_name,op_name,input_shapes,input_dtypes,input_formats,output_shapes,output_dtypes,output_formats,input_ori_shapes,input_ori_formats,output_ori_shapes,output_ori_formats,attributes,input_data_ranges,is_enabled,remark | ||
| 2 | +nb_float16_normal,normalize_bbox,"((2, 3, 4), (2, 3))","('float16', 'int32')","('ND', 'ND')","((2, 3, 4),)","('float16',)","('ND',)","((2, 3, 4), (2, 3))","('ND', 'ND')","((2, 3, 4),)","('ND',)",{'reversed_box': False},"((-30, 100), (10, 200))",TRUE, | ||
| 3 | +nb_float16_reversed,normalize_bbox,"((2, 4, 7), (2, 3))","('float16', 'int32')","('ND', 'ND')","((2, 4, 7),)","('float16',)","('ND',)","((2, 4, 7), (2, 3))","('ND', 'ND')","((2, 4, 7),)","('ND',)",{'reversed_box': True},"((-30, 100), (10, 200))",TRUE, | ||
| 4 | +nb_float32_normal,normalize_bbox,"((2, 3, 4), (2, 3))","('float32', 'int32')","('ND', 'ND')","((2, 3, 4),)","('float32',)","('ND',)","((2, 3, 4), (2, 3))","('ND', 'ND')","((2, 3, 4),)","('ND',)",{'reversed_box': False},"((-30, 100), (10, 200))",TRUE, | ||
| 5 | +nb_float32_rank8,normalize_bbox,"((1, 1, 2, 1, 1, 1, 1, 4), (1, 3))","('float32', 'int32')","('ND', 'ND')","((1, 1, 2, 1, 1, 1, 1, 4),)","('float32',)","('ND',)","((1, 1, 2, 1, 1, 1, 1, 4), (1, 3))","('ND', 'ND')","((1, 1, 2, 1, 1, 1, 1, 4),)","('ND',)",{'reversed_box': False},"((-20, 20), (11, 200))",TRUE, | ||
| 6 | +nb_float16_large_hw,normalize_bbox,"((1, 3, 4), (1, 3))","('float16', 'int32')","('ND', 'ND')","((1, 3, 4),)","('float16',)","('ND',)","((1, 3, 4), (1, 3))","('ND', 'ND')","((1, 3, 4),)","('ND',)",{'reversed_box': False},"((1000, 1000), (65536, 65536))",TRUE,int32 divisor above float16 range; expected 1000/65536 | ||
| 7 | +nb_float32_large_hw,normalize_bbox,"((1, 3, 4), (1, 3))","('float32', 'int32')","('ND', 'ND')","((1, 3, 4),)","('float32',)","('ND',)","((1, 3, 4), (1, 3))","('ND', 'ND')","((1, 3, 4),)","('ND',)",{'reversed_box': False},"((1000, 1000), (65536, 65536))",TRUE,int32 divisor above float16 range; expected 1000/65536 | ||
| 8 | +nb_float16_split_num_tail,normalize_bbox,"((1, 1027, 4), (1, 3))","('float16', 'int32')","('ND', 'ND')","((1, 1027, 4),)","('float16',)","('ND',)","((1, 1027, 4), (1, 3))","('ND', 'ND')","((1, 1027, 4),)","('ND',)",{'reversed_box': False},"((-20, 20), (3, 233))",TRUE, | ||
| 9 | +nb_float32_empty_num,normalize_bbox,"((2, 4, 0), (2, 3))","('float32', 'int32')","('ND', 'ND')","((2, 4, 0),)","('float32',)","('ND',)","((2, 4, 0), (2, 3))","('ND', 'ND')","((2, 4, 0),)","('ND',)",{'reversed_box': True},"((1, 2), (10, 200))",TRUE, | ||
| @@ -10,4 +10,5 @@ | |||
| 10 | 10 | ||
| 11 | # 显式声明子目录,避免 file(GLOB) 对新增目录不敏感导致静默跳过编译 | 11 | # 显式声明子目录,避免 file(GLOB) 对新增目录不敏感导致静默跳过编译 |
| 12 | add_subdirectory(op_host) | 12 | add_subdirectory(op_host) |
| 13 | +add_subdirectory(op_graph) | ||
| 13 | add_subdirectory(framework) | 14 | add_subdirectory(framework) |
| @@ -0,0 +1,11 @@ | |||
| 1 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 2 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 3 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 4 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 5 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 6 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 7 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 8 | + | ||
| 9 | +if(UT_TEST_ALL OR OP_GRAPH_UT) | ||
| 10 | + add_modules_ut_sources(HOSTNAME ${OP_GRAPH_MODULE_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR}) | ||
| 11 | +endif() | ||
| @@ -0,0 +1,44 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +#include <gtest/gtest.h> | ||
| 12 | +#include "base/context_builder/op_infer_datatype_context_builder.h" | ||
| 13 | + | ||
| 14 | +namespace ops { | ||
| 15 | +ge::graphStatus NormalizeBBoxInferDataType(gert::InferDataTypeContext* context); | ||
| 16 | +} | ||
| 17 | + | ||
| 18 | +TEST(NormalizeBBoxGraphInfer, validates_both_input_dtypes) | ||
| 19 | +{ | ||
| 20 | + struct Case { | ||
| 21 | + ge::DataType boxes; | ||
| 22 | + ge::DataType shapeHw; | ||
| 23 | + bool valid; | ||
| 24 | + }; | ||
| 25 | + const Case cases[] = { | ||
| 26 | + {ge::DT_FLOAT16, ge::DT_INT32, true}, {ge::DT_FLOAT, ge::DT_INT32, true}, {ge::DT_BF16, ge::DT_INT32, false}, | ||
| 27 | + {ge::DT_BOOL, ge::DT_INT32, false}, {ge::DT_INT32, ge::DT_INT32, false}, {ge::DT_FLOAT, ge::DT_BF16, false}, | ||
| 28 | + {ge::DT_FLOAT, ge::DT_BOOL, false}, {ge::DT_FLOAT, ge::DT_FLOAT, false}, | ||
| 29 | + }; | ||
| 30 | + for (const auto& item : cases) { | ||
| 31 | + SCOPED_TRACE(static_cast<int>(item.boxes)); | ||
| 32 | + gert::OpInferDataTypeContextBuilder builder; | ||
| 33 | + builder.OpType("NormalizeBBox").OpName("normalize_bbox_dtype_check").IONum(2, 1); | ||
| 34 | + builder.InputTensorDesc(0, item.boxes, ge::FORMAT_ND, ge::FORMAT_ND); | ||
| 35 | + builder.InputTensorDesc(1, item.shapeHw, ge::FORMAT_ND, ge::FORMAT_ND); | ||
| 36 | + builder.OutputTensorDesc(0, ge::FORMAT_ND, ge::FORMAT_ND); | ||
| 37 | + auto holder = builder.Build(); | ||
| 38 | + ASSERT_NE(holder.GetContext(), nullptr); | ||
| 39 | + EXPECT_EQ(ops::NormalizeBBoxInferDataType(holder.GetContext()), | ||
| 40 | + item.valid ? ge::GRAPH_SUCCESS : ge::GRAPH_FAILED); | ||
| 41 | + if (item.valid) | ||
| 42 | + EXPECT_EQ(holder.GetContext()->GetOutputDataType(0), item.boxes); | ||
| 43 | + } | ||
| 44 | +} | ||
| @@ -10,16 +10,7 @@ | |||
| 10 | 10 | ||
| 11 | /*! | 11 | /*! |
| 12 | * \file test_normalize_bbox_infershape.cpp | 12 | * \file test_normalize_bbox_infershape.cpp |
| 13 | - * \brief NormalizeBBox InferShape UT (iteration-1 core path) | 13 | + * \brief NormalizeBBox shape validation and propagation tests. |
| 14 | - * | ||
| 15 | - * Coverage: | ||
| 16 | - * - InferShape: y.shape == boxes.shape for normal (batch,num,4) and reversed (batch,4,num) layouts | ||
| 17 | - * | ||
| 18 | - * InferDataType (y.dtype == boxes.dtype) now lives in | ||
| 19 | - * op_graph/normalize_bbox_graph_infer.cpp: it is a graph-only deliverable. The op_graph | ||
| 20 | - * UT module links graph_plugin_obj only and has no access to the tests/ut/common | ||
| 21 | - * infershape fakers, so it is not covered here -- same as every other op in this repo | ||
| 22 | - * that keeps InferDataType in op_graph (bn_infer_grad / lp_norm_update / in_infer_v2). | ||
| 23 | */ | 14 | */ |
| 24 | 15 | ||
| 25 | #include <gtest/gtest.h> | 16 | #include <gtest/gtest.h> |
| @@ -28,6 +19,7 @@ | |||
| 28 | #include "infershape_test_util.h" | 19 | #include "infershape_test_util.h" |
| 29 | #include "kernel_run_context_facker.h" | 20 | #include "kernel_run_context_facker.h" |
| 30 | #include "log/log.h" | 21 | #include "log/log.h" |
| 22 | +#include "base/context_builder/op_infer_shape_context_builder.h" | ||
| 31 | 23 | ||
| 32 | class NormalizeBBoxProto : public testing::Test { | 24 | class NormalizeBBoxProto : public testing::Test { |
| 33 | protected: | 25 | protected: |
| @@ -70,6 +62,7 @@ TEST_F(NormalizeBBoxProto, normalize_bbox_infershape_reversed_layout) | |||
| 70 | auto holder = gert::InferShapeContextFaker() | 62 | auto holder = gert::InferShapeContextFaker() |
| 71 | .NodeIoNum(2, 1) | 63 | .NodeIoNum(2, 1) |
| 72 | .IrInstanceNum({1, 1}) | 64 | .IrInstanceNum({1, 1}) |
| 65 | + .NodeAttrs({{"reversed_box", Ops::NN::AnyValue::CreateFrom<bool>(true)}}) | ||
| 73 | .InputShapes({&boxesShape, &shapeHwShape}) | 66 | .InputShapes({&boxesShape, &shapeHwShape}) |
| 74 | .OutputShapes({&yShape}) | 67 | .OutputShapes({&yShape}) |
| 75 | .Build(); | 68 | .Build(); |
| @@ -145,12 +138,6 @@ TEST_F(NormalizeBBoxProto, normalize_bbox_infershape_unknown_rank) | |||
| 145 | ASSERT_EQ(Ops::Base::ToString(*context->GetOutputShape(0)), "[-2]"); | 138 | ASSERT_EQ(Ops::Base::ToString(*context->GetOutputShape(0)), "[-2]"); |
| 146 | } | 139 | } |
| 147 | 140 | ||
| 148 | -// ============================================================================================ | ||
| 149 | -// Iteration-3: round out InferShape (layout) coverage. | ||
| 150 | -// InferShape copies boxes->y verbatim (layout/dtype agnostic). dtype *rejection* lives | ||
| 151 | -// in the host tiling layer, NOT in the proto layer. | ||
| 152 | -// ============================================================================================ | ||
| 153 | - | ||
| 154 | // InferShape: reversed large layout (batch, 4, num) preserved verbatim | 141 | // InferShape: reversed large layout (batch, 4, num) preserved verbatim |
| 155 | TEST_F(NormalizeBBoxProto, normalize_bbox_infershape_reversed_large_shape) | 142 | TEST_F(NormalizeBBoxProto, normalize_bbox_infershape_reversed_large_shape) |
| 156 | { | 143 | { |
| @@ -164,6 +151,7 @@ TEST_F(NormalizeBBoxProto, normalize_bbox_infershape_reversed_large_shape) | |||
| 164 | auto holder = gert::InferShapeContextFaker() | 151 | auto holder = gert::InferShapeContextFaker() |
| 165 | .NodeIoNum(2, 1) | 152 | .NodeIoNum(2, 1) |
| 166 | .IrInstanceNum({1, 1}) | 153 | .IrInstanceNum({1, 1}) |
| 154 | + .NodeAttrs({{"reversed_box", Ops::NN::AnyValue::CreateFrom<bool>(true)}}) | ||
| 167 | .InputShapes({&boxesShape, &shapeHwShape}) | 155 | .InputShapes({&boxesShape, &shapeHwShape}) |
| 168 | .OutputShapes({&yShape}) | 156 | .OutputShapes({&yShape}) |
| 169 | .Build(); | 157 | .Build(); |
| @@ -172,3 +160,56 @@ TEST_F(NormalizeBBoxProto, normalize_bbox_infershape_reversed_large_shape) | |||
| 172 | auto context = holder.GetContext<gert::InferShapeContext>(); | 160 | auto context = holder.GetContext<gert::InferShapeContext>(); |
| 173 | ASSERT_EQ(Ops::Base::ToString(*context->GetOutputShape(0)), "[8, 4, 4096]"); | 161 | ASSERT_EQ(Ops::Base::ToString(*context->GetOutputShape(0)), "[8, 4, 4096]"); |
| 174 | } | 162 | } |
| 163 | + | ||
| 164 | +TEST_F(NormalizeBBoxProto, rejects_known_invalid_shapes_and_preserves_dynamic_shapes) | ||
| 165 | +{ | ||
| 166 | + struct Case { | ||
| 167 | + std::vector<int64_t> boxes; | ||
| 168 | + std::vector<int64_t> shapeHw; | ||
| 169 | + bool reversed; | ||
| 170 | + bool valid; | ||
| 171 | + }; | ||
| 172 | + const std::vector<Case> cases = { | ||
| 173 | + {{4}, {4, 3}, false, false}, | ||
| 174 | + {{1, 1, 1, 1, 1, 1, 1, 1, 4}, {1, 3}, false, false}, | ||
| 175 | + {{2, 3, 4}, {6}, false, false}, | ||
| 176 | + {{2, 3, 4}, {2, 1, 3}, false, false}, | ||
| 177 | + {{2, 3, 4}, {2, 2}, false, false}, | ||
| 178 | + {{2, 3, 4}, {3, 3}, false, false}, | ||
| 179 | + {{2, 3, 3}, {2, 3}, false, false}, | ||
| 180 | + {{2, 3, 4}, {2, 3}, true, false}, | ||
| 181 | + {{-1, -1, 4}, {-1, 3}, false, true}, | ||
| 182 | + {{2, 3, -1}, {2, 3}, false, true}, | ||
| 183 | + {{-2}, {-2}, false, true}, | ||
| 184 | + {{2, 0, 4}, {2, 3}, false, true}, | ||
| 185 | + {{1, 1, 1, 1, 1, 1, 1, 4}, {1, 3}, false, true}, | ||
| 186 | + }; | ||
| 187 | + const auto* impl = gert::OpImplRegistry::GetInstance().GetOpImpl("NormalizeBBox"); | ||
| 188 | + ASSERT_NE(impl, nullptr); | ||
| 189 | + ASSERT_NE(impl->infer_shape, nullptr); | ||
| 190 | + for (size_t index = 0; index < cases.size(); ++index) { | ||
| 191 | + SCOPED_TRACE(index); | ||
| 192 | + const auto& item = cases[index]; | ||
| 193 | + gert::StorageShape boxesShape, hwShape; | ||
| 194 | + for (size_t i = 0; i < item.boxes.size(); ++i) { | ||
| 195 | + boxesShape.MutableOriginShape().AppendDim(item.boxes[i]); | ||
| 196 | + boxesShape.MutableStorageShape().AppendDim(item.boxes[i]); | ||
| 197 | + } | ||
| 198 | + for (size_t i = 0; i < item.shapeHw.size(); ++i) { | ||
| 199 | + hwShape.MutableOriginShape().AppendDim(item.shapeHw[i]); | ||
| 200 | + hwShape.MutableStorageShape().AppendDim(item.shapeHw[i]); | ||
| 201 | + } | ||
| 202 | + gert::Tensor boxes(boxesShape, {ge::FORMAT_ND, ge::FORMAT_ND, {}}, ge::DT_FLOAT); | ||
| 203 | + gert::Tensor hw(hwShape, {ge::FORMAT_ND, ge::FORMAT_ND, {}}, ge::DT_INT32); | ||
| 204 | + gert::OpInferShapeContextBuilder builder; | ||
| 205 | + builder.OpType("NormalizeBBox").OpName("normalize_bbox_shape_check").IONum(2, 1); | ||
| 206 | + builder.InputTensors({&boxes, &hw}).AppendAttr(item.reversed); | ||
| 207 | + builder.OutputTensorDesc(0, ge::DT_FLOAT, ge::FORMAT_ND, ge::FORMAT_ND); | ||
| 208 | + auto holder = builder.Build(); | ||
| 209 | + ASSERT_NE(holder.GetContext(), nullptr); | ||
| 210 | + EXPECT_EQ(impl->infer_shape(holder.GetContext()), item.valid ? ge::GRAPH_SUCCESS : ge::GRAPH_FAILED); | ||
| 211 | + if (item.valid) { | ||
| 212 | + EXPECT_EQ(*holder.GetContext()->GetOutputShape(0), boxesShape.GetOriginShape()); | ||
| 213 | + } | ||
| 214 | + } | ||
| 215 | +} | ||
| @@ -703,7 +703,8 @@ TEST_F(NormalizeBBoxTiling, tiling_tilelen_ub_budget_capped) | |||
| 703 | 703 | ||
| 704 | // Additional op_host branch coverage. | 704 | // Additional op_host branch coverage. |
| 705 | // Constants mirrored from normalize_bbox_regbase_tiling.cpp: | 705 | // Constants mirrored from normalize_bbox_regbase_tiling.cpp: |
| 706 | -// UB_RESERVE=32*1024=32768, UB_BUF_FACTOR=5(normal)/6(reversed), VREPEAT_SIZE=256, | 706 | +// UB_RESERVE=32*1024=32768, VREPEAT_SIZE=256, |
| 707 | +// per-element UB: fp32=20/24B, fp16=16/20B (normal/reversed), | ||
| 707 | // MAX_TILE_LEN=8192. | 708 | // MAX_TILE_LEN=8192. |
| 708 | // blockAlignElems = VREPEAT_SIZE/dsize -> fp32:64, fp16:128 (also the tileLen floor clamp). | 709 | // blockAlignElems = VREPEAT_SIZE/dsize -> fp32:64, fp16:128 (also the tileLen floor clamp). |
| 709 | // Covered branches: | 710 | // Covered branches: |
| @@ -731,18 +732,24 @@ TEST_F(NormalizeBBoxTiling, tiling_fail_platform_zero_ubsize) | |||
| 731 | } | 732 | } |
| 732 | 733 | ||
| 733 | // ---- tileLen minimum viable UB: reserve + explicit minimum buffers -------------------------- | 734 | // ---- tileLen minimum viable UB: reserve + explicit minimum buffers -------------------------- |
| 734 | -// Normal fp32/fp16 both need 1376B beyond UB_RESERVE at their minimum 256B-repeat tile: | 735 | +// Minimum tiles use 1024B for boxes queues and 96B for image dimensions. |
| 735 | -// boxes queues 1024B + shape 32B + cast 64B + divisor 256B. | 736 | +// FP32 adds a 256B divisor; FP16 adds a 512B divisor and 512B fp32 compute buffer. |
| 736 | TEST_F(NormalizeBBoxTiling, tiling_tilelen_small_ub_clamp_to_blockalign) | 737 | TEST_F(NormalizeBBoxTiling, tiling_tilelen_small_ub_clamp_to_blockalign) |
| 737 | { | 738 | { |
| 738 | constexpr uint64_t kUbReserve = 32u * 1024u; | 739 | constexpr uint64_t kUbReserve = 32u * 1024u; |
| 739 | - constexpr uint64_t kMinimumBuffers = 1376u; | 740 | + constexpr uint64_t kMinimumFp32Buffers = 1376u; |
| 740 | - auto f32 = RunTiling({1, 4096, 4}, {1, 3}, NORMAL_LAYOUT, ge::DT_FLOAT, CORE_NUM, kUbReserve + kMinimumBuffers); | 741 | + constexpr uint64_t kMinimumFp16Buffers = 2144u; |
| 741 | - auto f16 = RunTiling({1, 4096, 4}, {1, 3}, NORMAL_LAYOUT, ge::DT_FLOAT16, CORE_NUM, kUbReserve + kMinimumBuffers); | 742 | + auto f32 = RunTiling({1, 4096, 4}, {1, 3}, NORMAL_LAYOUT, ge::DT_FLOAT, CORE_NUM, kUbReserve + kMinimumFp32Buffers); |
| 743 | + auto f16 = RunTiling({1, 4096, 4}, {1, 3}, NORMAL_LAYOUT, ge::DT_FLOAT16, CORE_NUM, | ||
| 744 | + kUbReserve + kMinimumFp16Buffers); | ||
| 742 | ASSERT_EQ(f32.status, ge::GRAPH_SUCCESS); | 745 | ASSERT_EQ(f32.status, ge::GRAPH_SUCCESS); |
| 743 | ASSERT_EQ(f16.status, ge::GRAPH_SUCCESS); | 746 | ASSERT_EQ(f16.status, ge::GRAPH_SUCCESS); |
| 744 | EXPECT_EQ(f32.tilingData.tileLen, 64u); // clamped to blockAlignElems (fp32) | 747 | EXPECT_EQ(f32.tilingData.tileLen, 64u); // clamped to blockAlignElems (fp32) |
| 745 | EXPECT_EQ(f16.tilingData.tileLen, 128u); // clamped to blockAlignElems (fp16) | 748 | EXPECT_EQ(f16.tilingData.tileLen, 128u); // clamped to blockAlignElems (fp16) |
| 749 | + EXPECT_EQ(f16.tilingData.divisorBufferBytes, 512u); | ||
| 750 | + const auto insufficient = RunTiling({1, 4096, 4}, {1, 3}, NORMAL_LAYOUT, ge::DT_FLOAT16, CORE_NUM, | ||
| 751 | + kUbReserve + kMinimumFp16Buffers - 1); | ||
| 752 | + EXPECT_EQ(insufficient.status, ge::GRAPH_FAILED); | ||
| 746 | } | 753 | } |
| 747 | 754 | ||
| 748 | // ---- UB below the reserved/runtime requirement must be rejected ----------------------------- | 755 | // ---- UB below the reserved/runtime requirement must be rejected ----------------------------- |