Merged
[CANNBot]:dequant_rope_quant_kvcache新增支持ascend950 ascendc实现 #10481
yulianjieAdd on Aug 24
[CANNBot]:dequant_rope_quant_kvcache新增支持ascend950 ascendc实现 #10481
Merged
Merged fromlianjieyu/ops-transformerdevintocann/ops-transformermaster
Total: 5 files changed+1096-166
| @@ -4,7 +4,7 @@ | |||
| 4 | 4 | ||
| 5 | |产品 | 是否支持 | | 5 | |产品 | 是否支持 | |
| 6 | |:-------------------------|:----------:| | 6 | |:-------------------------|:----------:| |
| 7 | -| <term>Ascend 950PR/Ascend 950DT</term> | × | | 7 | +| <term>Ascend 950PR/Ascend 950DT</term> | √ | |
| 8 | | <term>Atlas A3 训练系列产品/Atlas A3 推理系列产品</term> | √ | | 8 | | <term>Atlas A3 训练系列产品/Atlas A3 推理系列产品</term> | √ | |
| 9 | | <term>Atlas A2 训练系列产品/Atlas A2 推理系列产品</term> | √ | | 9 | | <term>Atlas A2 训练系列产品/Atlas A2 推理系列产品</term> | √ | |
| 10 | | <term>Atlas 200I/500 A2 推理产品</term> | × | | 10 | | <term>Atlas 200I/500 A2 推理产品</term> | × | |
| @@ -18,51 +18,51 @@ | |||
| 18 | - 算子功能:对输入张量(x)进行dequant(可选)后,按`sizeSplits`(为切分的长度)对尾轴进行切分,划分为q、k、vOut,对q、k进行旋转位置编码,生成qOut和kOut,之后对kOut和vOut进行量化并按照`indices`更新到kCacheRef和vCacheRef上。 | 18 | - 算子功能:对输入张量(x)进行dequant(可选)后,按`sizeSplits`(为切分的长度)对尾轴进行切分,划分为q、k、vOut,对q、k进行旋转位置编码,生成qOut和kOut,之后对kOut和vOut进行量化并按照`indices`更新到kCacheRef和vCacheRef上。 |
| 19 | 19 | ||
| 20 | - 计算公式: | 20 | - 计算公式: |
| 21 | - | 21 | + |
| 22 | $$ | 22 | $$ |
| 23 | dequantX = Dequant(x,weightScaleOptional,activationScaleOptional,biasOptional) | 23 | dequantX = Dequant(x,weightScaleOptional,activationScaleOptional,biasOptional) |
| 24 | $$ | 24 | $$ |
| 25 | - | 25 | + |
| 26 | $$ | 26 | $$ |
| 27 | q,k,vOut = SplitTensor(dequantX,dim=-1,`sizeSplits`) | 27 | q,k,vOut = SplitTensor(dequantX,dim=-1,`sizeSplits`) |
| 28 | $$ | 28 | $$ |
| 29 | - | 29 | + |
| 30 | $$ | 30 | $$ |
| 31 | qOut,kOut = ApplyRotaryPosEmb(q,k,cos,sin) | 31 | qOut,kOut = ApplyRotaryPosEmb(q,k,cos,sin) |
| 32 | $$ | 32 | $$ |
| 33 | - | 33 | + |
| 34 | $$ | 34 | $$ |
| 35 | quantK = Quant(kOut,scaleK,offsetKOptional) | 35 | quantK = Quant(kOut,scaleK,offsetKOptional) |
| 36 | $$ | 36 | $$ |
| 37 | - | 37 | + |
| 38 | $$ | 38 | $$ |
| 39 | quantV = Quant(vOut,scaleV,offsetVOptional) | 39 | quantV = Quant(vOut,scaleV,offsetVOptional) |
| 40 | $$ | 40 | $$ |
| 41 | - | 41 | + |
| 42 | 如果cacheModeOptional为contiguous则: | 42 | 如果cacheModeOptional为contiguous则: |
| 43 | - | 43 | + |
| 44 | $$ | 44 | $$ |
| 45 | kCacheRef[i][indices[i]]=quantK[i] | 45 | kCacheRef[i][indices[i]]=quantK[i] |
| 46 | $$ | 46 | $$ |
| 47 | - | 47 | + |
| 48 | $$ | 48 | $$ |
| 49 | vCacheRef[i][indices[i]]=quantV[i] | 49 | vCacheRef[i][indices[i]]=quantV[i] |
| 50 | $$ | 50 | $$ |
| 51 | - | 51 | + |
| 52 | 如果cacheModeOptional为page则: | 52 | 如果cacheModeOptional为page则: |
| 53 | - | 53 | + |
| 54 | $$ | 54 | $$ |
| 55 | kCacheRefView=kCacheRef.view(-1,kCacheRef[-2],kCacheRef[-1]) | 55 | kCacheRefView=kCacheRef.view(-1,kCacheRef[-2],kCacheRef[-1]) |
| 56 | $$ | 56 | $$ |
| 57 | - | 57 | + |
| 58 | $$ | 58 | $$ |
| 59 | vCacheRefView=vCacheRef.view(-1,vCacheRef[-2],vCacheRef[-1]) | 59 | vCacheRefView=vCacheRef.view(-1,vCacheRef[-2],vCacheRef[-1]) |
| 60 | $$ | 60 | $$ |
| 61 | - | 61 | + |
| 62 | $$ | 62 | $$ |
| 63 | kCacheRefView[indices[i]]=quantK[i] | 63 | kCacheRefView[indices[i]]=quantK[i] |
| 64 | $$ | 64 | $$ |
| 65 | - | 65 | + |
| 66 | $$ | 66 | $$ |
| 67 | vCacheRefView[indices[i]]=quantV[i] | 67 | vCacheRefView[indices[i]]=quantV[i] |
| 68 | $$ | 68 | $$ |
| @@ -0,0 +1,13 @@ | |||
| 1 | +; 该文件主要影响 opc 工具 编译二进制kernel时, --simplified_key_mode 选项中填写的值,格式如下所示: | ||
| 2 | +; [某算子] | ||
| 3 | +; default=xx | ||
| 4 | +; ascendxx=xx | ||
| 5 | +; 其中,default为默认mode,ascnedxx为可选mode,如果不同芯片有差异化要求时,需要配置; | ||
| 6 | +; 1)如果没有配置:非ascendC算子继续按空处理,即opc编译命令中不添加 --simplified_key_mode 选项,AscendC算子按照 simplified_key_mode=0 处理 | ||
| 7 | +; 2)如果仅有default配置:各个版本按default配置 | ||
| 8 | +; 3)如果仅有某些平台的配置,没有default配置:对应平台的按照配置的值传递,非对应平台的:非AscendC算子继续按空处理,AscendC算子按照 simplified_key_mode=0 处理 | ||
| 9 | +; 4)如果default配置和平台配置都有:对应平台的使用平台的配置,非对应的平台的以default值配置。 | ||
| 10 | +; 5)对于自定义simplified key的情况,需要在binary_simplified_key_mode.ini 文件中显式配置为None,不传入 --simplified_key_mode 选项,由opc工具和FE框架自行判断使用何种模式 | ||
| 11 | +; 6)是否是AscendC算子,由 ops/build-in/tbe/op_info_cfg/parser/ascendc_config.json 中配置的算子名字和对于的平台决定 | ||
| 12 | +[DequantRopeQuantKvcache] | ||
| 13 | +default=0 | ||
| @@ -16,15 +16,15 @@ | |||
| 16 | 16 | ||
| 17 | namespace ops { | 17 | namespace ops { |
| 18 | 18 | ||
| 19 | -static const std::vector<ge::DataType> XDtypeList = { | 19 | +static const std::vector<ge::DataType> XDtypeList = {{ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 20 | - {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, | 20 | + ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, |
| 21 | - ge::DT_INT32, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, | 21 | + ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_INT32, |
| 22 | - ge::DT_INT32}}; | 22 | + ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}}; |
| 23 | 23 | ||
| 24 | -static const std::vector<ge::DataType> cosDtypeList = { | 24 | +static const std::vector<ge::DataType> cosDtypeList = {{ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 25 | - {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, | 25 | + ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, |
| 26 | - ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, | 26 | + ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, |
| 27 | - ge::DT_BF16}}; | 27 | + ge::DT_BF16, ge::DT_BF16, ge::DT_BF16}}; |
| 28 | 28 | ||
| 29 | static const std::vector<ge::DataType> biasDtypeList = { | 29 | static const std::vector<ge::DataType> biasDtypeList = { |
| 30 | {ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT, | 30 | {ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT, |
| @@ -42,10 +42,10 @@ static const std::vector<ge::DataType> indicesDtypeList = { | |||
| 42 | {ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, | 42 | {ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, |
| 43 | ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}}; | 43 | ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}}; |
| 44 | 44 | ||
| 45 | -static const std::vector<ge::Format> formatList = { | 45 | +static const std::vector<ge::Format> formatList = {{ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 46 | - {ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 46 | + ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 47 | - ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, | 47 | + ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, |
| 48 | - ge::FORMAT_ND, ge::FORMAT_ND}}; | 48 | + ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}}; |
| 49 | 49 | ||
| 50 | static const std::vector<ge::DataType> XDtypeListKirin = { | 50 | static const std::vector<ge::DataType> XDtypeListKirin = { |
| 51 | {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}}; | 51 | {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}}; |
| @@ -70,7 +70,8 @@ static const std::vector<ge::Format> formatListKirin = { | |||
| 70 | 70 | ||
| 71 | class DequantRopeQuantKvcache : public OpDef { | 71 | class DequantRopeQuantKvcache : public OpDef { |
| 72 | public: | 72 | public: |
| 73 | - explicit DequantRopeQuantKvcache(const char* name) : OpDef(name) | 73 | + explicit DequantRopeQuantKvcache(const char *name) |
| 74 | + : OpDef(name) | ||
| 74 | { | 75 | { |
| 75 | this->Input("x") | 76 | this->Input("x") |
| 76 | .ParamType(REQUIRED) | 77 | .ParamType(REQUIRED) |
| @@ -170,6 +171,7 @@ public: | |||
| 170 | this->Attr("cache_mode").AttrType(OPTIONAL).String("contiguous"); | 171 | this->Attr("cache_mode").AttrType(OPTIONAL).String("contiguous"); |
| 171 | this->AICore().AddConfig("ascend910b"); | 172 | this->AICore().AddConfig("ascend910b"); |
| 172 | this->AICore().AddConfig("ascend910_93"); | 173 | this->AICore().AddConfig("ascend910_93"); |
| 174 | + this->AICore().AddConfig("ascend950"); | ||
| 173 | 175 | ||
| 174 | OpAICoreConfig config_kirin = GetKirinCoreConfig(); | 176 | OpAICoreConfig config_kirin = GetKirinCoreConfig(); |
| 175 | this->AICore().AddConfig("kirinx90", config_kirin); | 177 | this->AICore().AddConfig("kirinx90", config_kirin); |
| @@ -70,7 +70,7 @@ constexpr int64_t DATA_TYPE_INT32 = 2; | |||
| 70 | constexpr int64_t DATA_TYPE_BF16 = 3; | 70 | constexpr int64_t DATA_TYPE_BF16 = 3; |
| 71 | 71 | ||
| 72 | template <typename T1, typename T2> | 72 | template <typename T1, typename T2> |
| 73 | -inline auto UpAlign(const T1& a, const T2& b) -> T1 | 73 | +inline auto UpAlign(const T1 &a, const T2 &b) -> T1 |
| 74 | { | 74 | { |
| 75 | if (b != 0) { | 75 | if (b != 0) { |
| 76 | return (a + b - 1) / b * b; | 76 | return (a + b - 1) / b * b; |
| @@ -79,7 +79,7 @@ inline auto UpAlign(const T1& a, const T2& b) -> T1 | |||
| 79 | } | 79 | } |
| 80 | 80 | ||
| 81 | template <typename T> | 81 | template <typename T> |
| 82 | -static auto GetRem(const T& value1, const T& value2) -> T | 82 | +static auto GetRem(const T &value1, const T &value2) -> T |
| 83 | { | 83 | { |
| 84 | if (value2 == 0) { | 84 | if (value2 == 0) { |
| 85 | return value2; | 85 | return value2; |
| @@ -88,7 +88,7 @@ static auto GetRem(const T& value1, const T& value2) -> T | |||
| 88 | } | 88 | } |
| 89 | 89 | ||
| 90 | template <typename T> | 90 | template <typename T> |
| 91 | -static auto GetCeilInt(const T& value1, const T& value2) -> T | 91 | +static auto GetCeilInt(const T &value1, const T &value2) -> T |
| 92 | { | 92 | { |
| 93 | if (value2 == 0) { | 93 | if (value2 == 0) { |
| 94 | return value2; | 94 | return value2; |
| @@ -97,7 +97,7 @@ static auto GetCeilInt(const T& value1, const T& value2) -> T | |||
| 97 | } | 97 | } |
| 98 | 98 | ||
| 99 | template <typename T> | 99 | template <typename T> |
| 100 | -static auto GetDiv(const T& value1, const T& value2) -> T | 100 | +static auto GetDiv(const T &value1, const T &value2) -> T |
| 101 | { | 101 | { |
| 102 | if (value2 == 0) { | 102 | if (value2 == 0) { |
| 103 | return value2; | 103 | return value2; |
| @@ -106,17 +106,18 @@ static auto GetDiv(const T& value1, const T& value2) -> T | |||
| 106 | } | 106 | } |
| 107 | 107 | ||
| 108 | template <typename T> | 108 | template <typename T> |
| 109 | -std::string Shape2String(const T& shape) { | 109 | +std::string Shape2String(const T &shape) |
| 110 | - std::ostringstream oss; | 110 | +{ |
| 111 | - oss << "["; | 111 | + std::ostringstream oss; |
| 112 | - if (shape.GetDimNum() > 0) { | 112 | + oss << "["; |
| 113 | - for (size_t i = 0; i < shape.GetDimNum() - 1; ++i) { | 113 | + if (shape.GetDimNum() > 0) { |
| 114 | - oss << shape.GetDim(i) << ", "; | 114 | + for (size_t i = 0; i < shape.GetDimNum() - 1; ++i) { |
| 115 | + oss << shape.GetDim(i) << ", "; | ||
| 116 | + } | ||
| 117 | + oss << shape.GetDim(shape.GetDimNum() - 1); | ||
| 115 | } | 118 | } |
| 116 | - oss << shape.GetDim(shape.GetDimNum() - 1); | 119 | + oss << "]"; |
| 117 | - } | 120 | + return oss.str(); |
| 118 | - oss << "]"; | ||
| 119 | - return oss.str(); | ||
| 120 | } | 121 | } |
| 121 | 122 | ||
| 122 | struct TilingInfoDRQK { | 123 | struct TilingInfoDRQK { |
| @@ -143,14 +144,14 @@ struct TilingInfoDRQK { | |||
| 143 | int64_t hasAS = 0; | 144 | int64_t hasAS = 0; |
| 144 | int64_t biasDataType = 0; | 145 | int64_t biasDataType = 0; |
| 145 | int64_t isDequant = 0; | 146 | int64_t isDequant = 0; |
| 146 | - const int64_t* attrData; | 147 | + const int64_t *attrData; |
| 147 | }; | 148 | }; |
| 148 | } // namespace | 149 | } // namespace |
| 149 | using namespace std; | 150 | using namespace std; |
| 150 | 151 | ||
| 151 | namespace optiling { | 152 | namespace optiling { |
| 152 | 153 | ||
| 153 | -static void DQRKPrintParam(gert::TilingContext* context, DequantRopeQuantKvcacheTilingData& tiling) | 154 | +static void DQRKPrintParam(gert::TilingContext *context, DequantRopeQuantKvcacheTilingData &tiling) |
| 154 | { | 155 | { |
| 155 | auto nodeName = context->GetNodeName(); | 156 | auto nodeName = context->GetNodeName(); |
| 156 | OP_LOGD(nodeName, ">>>>>>>>>>>>>>> Start to print DequantRopeQuantKvcache tiling data <<<<<<<<<<<<<<<<"); | 157 | OP_LOGD(nodeName, ">>>>>>>>>>>>>>> Start to print DequantRopeQuantKvcache tiling data <<<<<<<<<<<<<<<<"); |
| @@ -177,42 +178,35 @@ static void DQRKPrintParam(gert::TilingContext* context, DequantRopeQuantKvcache | |||
| 177 | OP_LOGD(nodeName, ">>> hasAS: %ld", tiling.get_hasAS()); | 178 | OP_LOGD(nodeName, ">>> hasAS: %ld", tiling.get_hasAS()); |
| 178 | OP_LOGD(nodeName, ">>> batch: %ld", tiling.get_batch()); | 179 | OP_LOGD(nodeName, ">>> batch: %ld", tiling.get_batch()); |
| 179 | } | 180 | } |
| 180 | -ge::graphStatus checkOptDtype(const uint64_t index, gert::TilingContext* context, const ge::char_t* nodeName_) | 181 | +ge::graphStatus checkOptDtype(const uint64_t index, gert::TilingContext *context, const ge::char_t *nodeName_) |
| 181 | { | 182 | { |
| 182 | auto optDesc = context->GetOptionalInputDesc(index); | 183 | auto optDesc = context->GetOptionalInputDesc(index); |
| 183 | if (optDesc != nullptr) { | 184 | if (optDesc != nullptr) { |
| 184 | auto optDtype = optDesc->GetDataType(); | 185 | auto optDtype = optDesc->GetDataType(); |
| 185 | - OP_CHECK_IF( | 186 | + OP_CHECK_IF(optDtype != ge::DT_FLOAT, OP_LOGE(nodeName_, "input %lu 's dtype should be float.", index), |
| 186 | - optDtype != ge::DT_FLOAT, | 187 | + return ge::GRAPH_FAILED); |
| 187 | - OP_LOGE(nodeName_, "input %lu 's dtype should be float.", index), | ||
| 188 | - return ge::GRAPH_FAILED); | ||
| 189 | } | 188 | } |
| 190 | return ge::GRAPH_SUCCESS; | 189 | return ge::GRAPH_SUCCESS; |
| 191 | } | 190 | } |
| 192 | -ge::graphStatus checkDRQKDim( | 191 | +ge::graphStatus checkDRQKDim(const gert::StorageShape *inShape, const std::vector<int64_t> &realShape, |
| 193 | - const gert::StorageShape* inShape, const std::vector<int64_t>& realShape, const ge::char_t* nodeName_, | 192 | + const ge::char_t *nodeName_, std::string logName) |
| 194 | - std::string logName) | ||
| 195 | { | 193 | { |
| 196 | - OP_CHECK_IF( | 194 | + OP_CHECK_IF(inShape->GetStorageShape().GetDimNum() != realShape.size(), |
| 197 | - inShape->GetStorageShape().GetDimNum() != realShape.size(), | 195 | + OP_LOGE(nodeName_, "%s's dim size should be %zu but got %zu", logName.c_str(), realShape.size(), |
| 198 | - OP_LOGE( | 196 | + inShape->GetStorageShape().GetDimNum()), |
| 199 | - nodeName_, "%s's dim size should be %zu but got %zu", logName.c_str(), realShape.size(), | 197 | + return ge::GRAPH_FAILED); |
| 200 | - inShape->GetStorageShape().GetDimNum()), | ||
| 201 | - return ge::GRAPH_FAILED); | ||
| 202 | for (uint64_t dimI = 0; dimI < realShape.size(); dimI++) { | 198 | for (uint64_t dimI = 0; dimI < realShape.size(); dimI++) { |
| 203 | - OP_CHECK_IF( | 199 | + OP_CHECK_IF(inShape->GetStorageShape().GetDim(dimI) != realShape[dimI], |
| 204 | - inShape->GetStorageShape().GetDim(dimI) != realShape[dimI], | 200 | + OP_LOGE(nodeName_, "%s's dim[%lu] should be [%ld] but got %ld.", logName.c_str(), dimI, |
| 205 | - OP_LOGE( | 201 | + realShape[dimI], inShape->GetStorageShape().GetDim(dimI)), |
| 206 | - nodeName_, "%s's dim[%lu] should be [%ld] but got %ld.", logName.c_str(), dimI, realShape[dimI], | 202 | + return ge::GRAPH_FAILED); |
| 207 | - inShape->GetStorageShape().GetDim(dimI)), | ||
| 208 | - return ge::GRAPH_FAILED); | ||
| 209 | } | 203 | } |
| 210 | return ge::GRAPH_SUCCESS; | 204 | return ge::GRAPH_SUCCESS; |
| 211 | } | 205 | } |
| 212 | 206 | ||
| 213 | -bool CheckPaCacheMode(const gert::RuntimeAttrs* attrs) | 207 | +bool CheckPaCacheMode(const gert::RuntimeAttrs *attrs) |
| 214 | { | 208 | { |
| 215 | - const char* tmpmode = attrs->GetStr(CACHE_MODE_IDX); | 209 | + const char *tmpmode = attrs->GetStr(CACHE_MODE_IDX); |
| 216 | if (tmpmode == nullptr) { | 210 | if (tmpmode == nullptr) { |
| 217 | return false; | 211 | return false; |
| 218 | } | 212 | } |
| @@ -221,7 +215,7 @@ bool CheckPaCacheMode(const gert::RuntimeAttrs* attrs) | |||
| 221 | return (cacheMode == "page"); | 215 | return (cacheMode == "page"); |
| 222 | } | 216 | } |
| 223 | 217 | ||
| 224 | -ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& tilingInfoDRQK) | 218 | +ge::graphStatus getDRQKShapeInfo(gert::TilingContext *context, TilingInfoDRQK &tilingInfoDRQK) |
| 225 | { | 219 | { |
| 226 | auto platformInfo = context->GetPlatformInfo(); | 220 | auto platformInfo = context->GetPlatformInfo(); |
| 227 | auto nodeName = context->GetNodeName(); | 221 | auto nodeName = context->GetNodeName(); |
| @@ -238,27 +232,23 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 238 | auto attr = attrs->GetAttrPointer<gert::ContinuousVector>(0); | 232 | auto attr = attrs->GetAttrPointer<gert::ContinuousVector>(0); |
| 239 | OP_CHECK_IF( | 233 | OP_CHECK_IF( |
| 240 | attr->GetSize() != QKV_NUM, | 234 | attr->GetSize() != QKV_NUM, |
| 241 | - OP_LOGE( | 235 | + OP_LOGE(context->GetNodeName(), "the attr size_splits's size %zu is invalid, it should be 3", attr->GetSize()), |
| 242 | - context->GetNodeName(), "the attr size_splits's size %zu is invalid, it should be 3", attr->GetSize()), | ||
| 243 | return ge::GRAPH_FAILED); | 236 | return ge::GRAPH_FAILED); |
| 244 | - tilingInfoDRQK.attrData = reinterpret_cast<const int64_t*>(attr->GetData()); | 237 | + tilingInfoDRQK.attrData = reinterpret_cast<const int64_t *>(attr->GetData()); |
| 245 | int64_t qHiddensize = tilingInfoDRQK.attrData[0]; | 238 | int64_t qHiddensize = tilingInfoDRQK.attrData[0]; |
| 246 | int64_t kHiddensize = tilingInfoDRQK.attrData[1]; | 239 | int64_t kHiddensize = tilingInfoDRQK.attrData[1]; |
| 247 | int64_t vHiddensize = tilingInfoDRQK.attrData[2]; | 240 | int64_t vHiddensize = tilingInfoDRQK.attrData[2]; |
| 248 | int64_t qkvHiddensize = qHiddensize + kHiddensize + vHiddensize; | 241 | int64_t qkvHiddensize = qHiddensize + kHiddensize + vHiddensize; |
| 249 | OP_CHECK_IF( | 242 | OP_CHECK_IF( |
| 250 | kHiddensize != vHiddensize, | 243 | kHiddensize != vHiddensize, |
| 251 | - OP_LOGE( | 244 | + OP_LOGE(context->GetNodeName(), "the attr size_splits's kHiddensize %ld is not same with vHiddensize %ld", |
| 252 | - context->GetNodeName(), "the attr size_splits's kHiddensize %ld is not same with vHiddensize %ld", | 245 | + kHiddensize, vHiddensize), |
| 253 | - kHiddensize, vHiddensize), | ||
| 254 | return ge::GRAPH_FAILED); | 246 | return ge::GRAPH_FAILED); |
| 255 | 247 | ||
| 256 | - OP_CHECK_IF( | 248 | + OP_CHECK_IF((qHiddensize < 0) || (kHiddensize < 0), |
| 257 | - (qHiddensize < 0) || (kHiddensize < 0), | 249 | + OP_LOGE(context->GetNodeName(), "size_splits[0] %ld size_splits[1] %ld should not less than 0", |
| 258 | - OP_LOGE( | 250 | + qHiddensize, kHiddensize), |
| 259 | - context->GetNodeName(), "size_splits[0] %ld size_splits[1] %ld should not less than 0", qHiddensize, | 251 | + return ge::GRAPH_FAILED); |
| 260 | - kHiddensize), | ||
| 261 | - return ge::GRAPH_FAILED); | ||
| 262 | tilingInfoDRQK.ifKVout = *attrs->GetAttrPointer<bool>(IFKV_INDEX) == true ? 1 : 0; | 252 | tilingInfoDRQK.ifKVout = *attrs->GetAttrPointer<bool>(IFKV_INDEX) == true ? 1 : 0; |
| 263 | 253 | ||
| 264 | auto quantKOffsetDesc = context->GetOptionalInputDesc(QUANT_K_OFFSET_INPUT_INDEX); | 254 | auto quantKOffsetDesc = context->GetOptionalInputDesc(QUANT_K_OFFSET_INPUT_INDEX); |
| @@ -273,11 +263,10 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 273 | auto optDesc = context->GetOptionalInputDesc(BIAS_INPUT_INDEX); | 263 | auto optDesc = context->GetOptionalInputDesc(BIAS_INPUT_INDEX); |
| 274 | if (optDesc != nullptr) { | 264 | if (optDesc != nullptr) { |
| 275 | auto optDtype = optDesc->GetDataType(); | 265 | auto optDtype = optDesc->GetDataType(); |
| 276 | - OP_CHECK_IF( | 266 | + OP_CHECK_IF((optDtype != ge::DT_FLOAT && optDtype != ge::DT_FLOAT16 && optDtype != ge::DT_BF16 && |
| 277 | - (optDtype != ge::DT_FLOAT && optDtype != ge::DT_FLOAT16 && optDtype != ge::DT_BF16 && | 267 | + optDtype != ge::DT_INT32), |
| 278 | - optDtype != ge::DT_INT32), | 268 | + OP_LOGE(nodeName, "input bias 's dtype should be in{float float16 bfloat16 int32}"), |
| 279 | - OP_LOGE(nodeName, "input bias 's dtype should be in{float float16 bfloat16 int32}"), | 269 | + return ge::GRAPH_FAILED); |
| 280 | - return ge::GRAPH_FAILED); | ||
| 281 | if (optDtype == ge::DT_FLOAT) { | 270 | if (optDtype == ge::DT_FLOAT) { |
| 282 | tilingInfoDRQK.biasDataType = DATA_TYPE_FLOAT; | 271 | tilingInfoDRQK.biasDataType = DATA_TYPE_FLOAT; |
| 283 | } else if (optDtype == ge::DT_FLOAT16) { | 272 | } else if (optDtype == ge::DT_FLOAT16) { |
| @@ -296,31 +285,27 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 296 | auto as = context->GetOptionalInputDesc(ACTIVATION_INPUT_INDEX); | 285 | auto as = context->GetOptionalInputDesc(ACTIVATION_INPUT_INDEX); |
| 297 | tilingInfoDRQK.hasAS = (as == nullptr) ? 0 : 1; | 286 | tilingInfoDRQK.hasAS = (as == nullptr) ? 0 : 1; |
| 298 | 287 | ||
| 299 | - const gert::StorageShape* xShape = context->GetInputShape(X_INPUT_INDEX); | 288 | + const gert::StorageShape *xShape = context->GetInputShape(X_INPUT_INDEX); |
| 300 | 289 | ||
| 301 | OP_CHECK_NULL_WITH_CONTEXT(context, xShape); | 290 | OP_CHECK_NULL_WITH_CONTEXT(context, xShape); |
| 302 | int64_t xShapeDimNum = xShape->GetStorageShape().GetDimNum(); | 291 | int64_t xShapeDimNum = xShape->GetStorageShape().GetDimNum(); |
| 303 | - OP_CHECK_IF( | 292 | + OP_CHECK_IF((xShapeDimNum != INPUT_X_DIM && xShapeDimNum != BASE_2), |
| 304 | - (xShapeDimNum != INPUT_X_DIM && xShapeDimNum != BASE_2), | 293 | + OP_LOGE(context->GetNodeName(), "x's dim [%ld] is not 3D or 2D.", xShapeDimNum), |
| 305 | - OP_LOGE(context->GetNodeName(), "x's dim [%ld] is not 3D or 2D.", xShapeDimNum), | 294 | + return ge::GRAPH_FAILED); |
| 306 | - return ge::GRAPH_FAILED); | ||
| 307 | tilingInfoDRQK.batch = xShape->GetStorageShape().GetDim(0); | 295 | tilingInfoDRQK.batch = xShape->GetStorageShape().GetDim(0); |
| 308 | tilingInfoDRQK.seqlen = (xShapeDimNum == BASE_2) ? 1 : xShape->GetStorageShape().GetDim(1); | 296 | tilingInfoDRQK.seqlen = (xShapeDimNum == BASE_2) ? 1 : xShape->GetStorageShape().GetDim(1); |
| 309 | tilingInfoDRQK.isPA = CheckPaCacheMode(attrs); | 297 | tilingInfoDRQK.isPA = CheckPaCacheMode(attrs); |
| 310 | 298 | ||
| 311 | - OP_CHECK_IF( | 299 | + OP_CHECK_IF(xShape->GetStorageShape().GetDim(xShapeDimNum - 1) != qkvHiddensize, |
| 312 | - xShape->GetStorageShape().GetDim(xShapeDimNum - 1) != qkvHiddensize, | 300 | + OP_LOGE(context->GetNodeName(), "x's dim[-1] [%ld] should be [%ld].", |
| 313 | - OP_LOGE( | 301 | + xShape->GetStorageShape().GetDim(xShapeDimNum - 1), qkvHiddensize), |
| 314 | - context->GetNodeName(), "x's dim[-1] [%ld] should be [%ld].", | 302 | + return ge::GRAPH_FAILED); |
| 315 | - xShape->GetStorageShape().GetDim(xShapeDimNum - 1), qkvHiddensize), | 303 | + const gert::StorageShape *cacheShape = context->GetInputShape(KVCACHE_INPUT_INDEX); |
| 316 | - return ge::GRAPH_FAILED); | ||
| 317 | - const gert::StorageShape* cacheShape = context->GetInputShape(KVCACHE_INPUT_INDEX); | ||
| 318 | OP_CHECK_NULL_WITH_CONTEXT(context, cacheShape); | 304 | OP_CHECK_NULL_WITH_CONTEXT(context, cacheShape); |
| 319 | 305 | ||
| 320 | OP_CHECK_IF( | 306 | OP_CHECK_IF( |
| 321 | cacheShape->GetStorageShape().GetDimNum() != INPUT_CACHE_DIM, | 307 | cacheShape->GetStorageShape().GetDimNum() != INPUT_CACHE_DIM, |
| 322 | - OP_LOGE( | 308 | + OP_LOGE(context->GetNodeName(), "cacheK's dim [%zu] is not 4D.", cacheShape->GetStorageShape().GetDimNum()), |
| 323 | - context->GetNodeName(), "cacheK's dim [%zu] is not 4D.", cacheShape->GetStorageShape().GetDimNum()), | ||
| 324 | return ge::GRAPH_FAILED); | 309 | return ge::GRAPH_FAILED); |
| 325 | 310 | ||
| 326 | int64_t xHiddenSize = xShape->GetStorageShape().GetDim(xShapeDimNum - 1); | 311 | int64_t xHiddenSize = xShape->GetStorageShape().GetDim(xShapeDimNum - 1); |
| @@ -331,32 +316,26 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 331 | int64_t quantShapeSize = tilingInfoDRQK.kvHeadNum * tilingInfoDRQK.hiddenSize; | 316 | int64_t quantShapeSize = tilingInfoDRQK.kvHeadNum * tilingInfoDRQK.hiddenSize; |
| 332 | OP_CHECK_IF( | 317 | OP_CHECK_IF( |
| 333 | cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize != kHiddensize, | 318 | cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize != kHiddensize, |
| 334 | - OP_LOGE( | 319 | + OP_LOGE(context->GetNodeName(), "cacheK's headdim * hiddenSize should same with [%ld] but got [%ld].", |
| 335 | - context->GetNodeName(), "cacheK's headdim * hiddenSize should same with [%ld] but got [%ld].", kHiddensize, | 320 | + kHiddensize, cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize), |
| 336 | - cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize), | ||
| 337 | return ge::GRAPH_FAILED); | 321 | return ge::GRAPH_FAILED); |
| 338 | OP_CHECK_IF( | 322 | OP_CHECK_IF( |
| 339 | tilingInfoDRQK.hiddenSize % FP16_ONE_BLOCK_NUM != 0, | 323 | tilingInfoDRQK.hiddenSize % FP16_ONE_BLOCK_NUM != 0, |
| 340 | - OP_LOGE( | 324 | + OP_LOGE(context->GetNodeName(), "hiddenSize [%ld] should be a multiple of 16", tilingInfoDRQK.hiddenSize), |
| 341 | - context->GetNodeName(), "hiddenSize [%ld] should be a multiple of 16", tilingInfoDRQK.hiddenSize), | ||
| 342 | - return ge::GRAPH_FAILED); | ||
| 343 | - OP_CHECK_IF( | ||
| 344 | - qHiddensize % tilingInfoDRQK.hiddenSize != 0, | ||
| 345 | - OP_LOGE( | ||
| 346 | - context->GetNodeName(), "qhiddenSize [%ld] should be a multiple of hiddenSize", qHiddensize), | ||
| 347 | - return ge::GRAPH_FAILED); | ||
| 348 | - OP_CHECK_IF( | ||
| 349 | - vHiddensize % tilingInfoDRQK.hiddenSize != 0, | ||
| 350 | - OP_LOGE( | ||
| 351 | - context->GetNodeName(), "vhiddenSize [%ld] should be a multiple of hiddenSize", vHiddensize), | ||
| 352 | return ge::GRAPH_FAILED); | 325 | return ge::GRAPH_FAILED); |
| 326 | + OP_CHECK_IF(qHiddensize % tilingInfoDRQK.hiddenSize != 0, | ||
| 327 | + OP_LOGE(context->GetNodeName(), "qhiddenSize [%ld] should be a multiple of hiddenSize", qHiddensize), | ||
| 328 | + return ge::GRAPH_FAILED); | ||
| 329 | + OP_CHECK_IF(vHiddensize % tilingInfoDRQK.hiddenSize != 0, | ||
| 330 | + OP_LOGE(context->GetNodeName(), "vhiddenSize [%ld] should be a multiple of hiddenSize", vHiddensize), | ||
| 331 | + return ge::GRAPH_FAILED); | ||
| 353 | tilingInfoDRQK.qHeadNum = GetDiv(tilingInfoDRQK.attrData[0], tilingInfoDRQK.hiddenSize); | 332 | tilingInfoDRQK.qHeadNum = GetDiv(tilingInfoDRQK.attrData[0], tilingInfoDRQK.hiddenSize); |
| 354 | - const gert::StorageShape* cosShape = context->GetInputShape(COS_INDEX); | 333 | + const gert::StorageShape *cosShape = context->GetInputShape(COS_INDEX); |
| 355 | - const gert::StorageShape* sinShape = context->GetInputShape(SIN_INDEX); | 334 | + const gert::StorageShape *sinShape = context->GetInputShape(SIN_INDEX); |
| 356 | - const gert::StorageShape* vCacheShape = context->GetInputShape(VCACHE_INPUT_INDEX); | 335 | + const gert::StorageShape *vCacheShape = context->GetInputShape(VCACHE_INPUT_INDEX); |
| 357 | - const gert::StorageShape* indicesShape = context->GetInputShape(INDICES_INPUT_INDEX); | 336 | + const gert::StorageShape *indicesShape = context->GetInputShape(INDICES_INPUT_INDEX); |
| 358 | - const gert::StorageShape* kScaleShape = context->GetInputShape(QUANT_K_SCALE_INPUT_INDEX); | 337 | + const gert::StorageShape *kScaleShape = context->GetInputShape(QUANT_K_SCALE_INPUT_INDEX); |
| 359 | - const gert::StorageShape* vScaleShape = context->GetInputShape(QUANT_V_SCALE_INPUT_INDEX); | 338 | + const gert::StorageShape *vScaleShape = context->GetInputShape(QUANT_V_SCALE_INPUT_INDEX); |
| 360 | 339 | ||
| 361 | std::vector<int64_t> expSinCos = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, 1, tilingInfoDRQK.hiddenSize}; | 340 | std::vector<int64_t> expSinCos = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, 1, tilingInfoDRQK.hiddenSize}; |
| 362 | std::vector<int64_t> expX = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, qkvHiddensize}; | 341 | std::vector<int64_t> expX = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, qkvHiddensize}; |
| @@ -376,29 +355,23 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 376 | (checkDRQKDim(indicesShape, {tilingInfoDRQK.batch}, nodeName, "input indices") == ge::GRAPH_FAILED)) { | 355 | (checkDRQKDim(indicesShape, {tilingInfoDRQK.batch}, nodeName, "input indices") == ge::GRAPH_FAILED)) { |
| 377 | return ge::GRAPH_FAILED; | 356 | return ge::GRAPH_FAILED; |
| 378 | } | 357 | } |
| 379 | - OP_CHECK_IF( | 358 | + OP_CHECK_IF(cacheShape->GetStorageShape() != vCacheShape->GetStorageShape(), |
| 380 | - cacheShape->GetStorageShape() != vCacheShape->GetStorageShape(), | 359 | + OP_LOGE(context->GetNodeName(), "kcache shape:%s should be equal with vcache shape:%s", |
| 381 | - OP_LOGE( | 360 | + Shape2String(cacheShape->GetStorageShape()).c_str(), |
| 382 | - context->GetNodeName(), "kcache shape:%s should be equal with vcache shape:%s", | 361 | + Shape2String(vCacheShape->GetStorageShape()).c_str()), |
| 383 | - Shape2String(cacheShape->GetStorageShape()).c_str(), | 362 | + return ge::GRAPH_FAILED); |
| 384 | - Shape2String(vCacheShape->GetStorageShape()).c_str()), | 363 | + OP_CHECK_IF((kScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize || |
| 385 | - return ge::GRAPH_FAILED); | 364 | + vScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize), |
| 386 | - OP_CHECK_IF( | 365 | + OP_LOGE(context->GetNodeName(), |
| 387 | - (kScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize || | 366 | + "kScaleShape shape size:%ld or vScaleShape shape size:%ld should be equal with :%ld", |
| 388 | - vScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize), | 367 | + kScaleShape->GetStorageShape().GetShapeSize(), vScaleShape->GetStorageShape().GetShapeSize(), |
| 389 | - OP_LOGE( | 368 | + quantShapeSize), |
| 390 | - context->GetNodeName(), | 369 | + return ge::GRAPH_FAILED); |
| 391 | - "kScaleShape shape size:%ld or vScaleShape shape size:%ld should be equal with :%ld", | ||
| 392 | - kScaleShape->GetStorageShape().GetShapeSize(), vScaleShape->GetStorageShape().GetShapeSize(), | ||
| 393 | - quantShapeSize), | ||
| 394 | - return ge::GRAPH_FAILED); | ||
| 395 | if (!tilingInfoDRQK.isPA) { | 370 | if (!tilingInfoDRQK.isPA) { |
| 396 | - OP_CHECK_IF( | 371 | + OP_CHECK_IF(vCacheShape->GetStorageShape().GetDim(0) < xShape->GetStorageShape().GetDim(0), |
| 397 | - vCacheShape->GetStorageShape().GetDim(0) < xShape->GetStorageShape().GetDim(0), | 372 | + OP_LOGE(context->GetNodeName(), "vcache's dim[0]:%ld should not be less than input x's dim[0]:%ld", |
| 398 | - OP_LOGE( | 373 | + vCacheShape->GetStorageShape().GetDim(0), xShape->GetStorageShape().GetDim(0)), |
| 399 | - context->GetNodeName(), "vcache's dim[0]:%ld should not be less than input x's dim[0]:%ld", | 374 | + return ge::GRAPH_FAILED); |
| 400 | - vCacheShape->GetStorageShape().GetDim(0), xShape->GetStorageShape().GetDim(0)), | ||
| 401 | - return ge::GRAPH_FAILED); | ||
| 402 | } | 375 | } |
| 403 | 376 | ||
| 404 | auto xDesc = context->GetInputDesc(0); | 377 | auto xDesc = context->GetInputDesc(0); |
| @@ -418,17 +391,15 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 418 | auto scaleDesc = context->GetInputDesc(QUANT_K_SCALE_INPUT_INDEX); | 391 | auto scaleDesc = context->GetInputDesc(QUANT_K_SCALE_INPUT_INDEX); |
| 419 | OP_CHECK_NULL_WITH_CONTEXT(context, scaleDesc); | 392 | OP_CHECK_NULL_WITH_CONTEXT(context, scaleDesc); |
| 420 | if (tilingInfoDRQK.hasQuantOffset == 1) { | 393 | if (tilingInfoDRQK.hasQuantOffset == 1) { |
| 421 | - const gert::StorageShape* kOffsetShape = context->GetOptionalInputShape(QUANT_K_OFFSET_INPUT_INDEX); | 394 | + const gert::StorageShape *kOffsetShape = context->GetOptionalInputShape(QUANT_K_OFFSET_INPUT_INDEX); |
| 422 | - const gert::StorageShape* vOffsetShape = context->GetOptionalInputShape(QUANT_V_OFFSET_INPUT_INDEX); | 395 | + const gert::StorageShape *vOffsetShape = context->GetOptionalInputShape(QUANT_V_OFFSET_INPUT_INDEX); |
| 423 | - OP_CHECK_IF( | 396 | + OP_CHECK_IF((kOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize || |
| 424 | - (kOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize || | 397 | + vOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize), |
| 425 | - vOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize), | 398 | + OP_LOGE(context->GetNodeName(), |
| 426 | - OP_LOGE( | 399 | + "kOffsetShape shape size:%ld or vOffsetShape shape size:%ld should be equal with :%ld", |
| 427 | - context->GetNodeName(), | 400 | + kOffsetShape->GetStorageShape().GetShapeSize(), |
| 428 | - "kOffsetShape shape size:%ld or vOffsetShape shape size:%ld should be equal with :%ld", | 401 | + vOffsetShape->GetStorageShape().GetShapeSize(), quantShapeSize), |
| 429 | - kOffsetShape->GetStorageShape().GetShapeSize(), vOffsetShape->GetStorageShape().GetShapeSize(), | 402 | + return ge::GRAPH_FAILED); |
| 430 | - quantShapeSize), | ||
| 431 | - return ge::GRAPH_FAILED); | ||
| 432 | tilingInfoDRQK.scalesBtyeSize = ge::GetSizeByDataType(scaleDesc->GetDataType()); | 403 | tilingInfoDRQK.scalesBtyeSize = ge::GetSizeByDataType(scaleDesc->GetDataType()); |
| 433 | } | 404 | } |
| 434 | auto wsDesc = context->GetOptionalInputDesc(WEIGHT_SCALES_INPUT_INDEX); | 405 | auto wsDesc = context->GetOptionalInputDesc(WEIGHT_SCALES_INPUT_INDEX); |
| @@ -457,25 +428,21 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t | |||
| 457 | 428 | ||
| 458 | int64_t unUsedSize = tilingInfoDRQK.ubSize - remainSize; | 429 | int64_t unUsedSize = tilingInfoDRQK.ubSize - remainSize; |
| 459 | 430 | ||
| 460 | - OP_CHECK_IF( | 431 | + OP_CHECK_IF((unUsedSize <= 0), |
| 461 | - (unUsedSize <= 0), | 432 | + OP_LOGE(context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].", |
| 462 | - OP_LOGE( | 433 | + xHiddenSize, HIDDEN_SIZE_LIMIT_LENGTH), |
| 463 | - context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].", xHiddenSize, | 434 | + return ge::GRAPH_FAILED); |
| 464 | - HIDDEN_SIZE_LIMIT_LENGTH), | ||
| 465 | - return ge::GRAPH_FAILED); | ||
| 466 | 435 | ||
| 467 | tilingInfoDRQK.onceS = GetDiv(unUsedSize, onceSSize); | 436 | tilingInfoDRQK.onceS = GetDiv(unUsedSize, onceSSize); |
| 468 | 437 | ||
| 469 | - OP_CHECK_IF( | 438 | + OP_CHECK_IF((tilingInfoDRQK.onceS == 0), |
| 470 | - (tilingInfoDRQK.onceS == 0), | 439 | + OP_LOGE(context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].", |
| 471 | - OP_LOGE( | 440 | + xHiddenSize, HIDDEN_SIZE_LIMIT_LENGTH), |
| 472 | - context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].", xHiddenSize, | 441 | + return ge::GRAPH_FAILED); |
| 473 | - HIDDEN_SIZE_LIMIT_LENGTH), | ||
| 474 | - return ge::GRAPH_FAILED); | ||
| 475 | return ge::GRAPH_SUCCESS; | 442 | return ge::GRAPH_SUCCESS; |
| 476 | } | 443 | } |
| 477 | 444 | ||
| 478 | -static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* context) | 445 | +static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext *context) |
| 479 | { | 446 | { |
| 480 | TilingInfoDRQK tilingInfoDRQK; | 447 | TilingInfoDRQK tilingInfoDRQK; |
| 481 | auto nodeName = context->GetNodeName(); | 448 | auto nodeName = context->GetNodeName(); |
| @@ -485,6 +452,21 @@ static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* contex | |||
| 485 | return ge::GRAPH_FAILED; | 452 | return ge::GRAPH_FAILED; |
| 486 | } | 453 | } |
| 487 | int64_t taskNum = tilingInfoDRQK.batch * tilingInfoDRQK.seqlen; | 454 | int64_t taskNum = tilingInfoDRQK.batch * tilingInfoDRQK.seqlen; |
| 455 | + | ||
| 456 | +#if (__NPU_ARCH__ == 3510) | ||
| 457 | + if (taskNum == 0) { | ||
| 458 | + OP_LOGD(nodeName, "taskNum is 0, empty tensor, set block dim to 0."); | ||
| 459 | + context->SetBlockDim(0); | ||
| 460 | + DequantRopeQuantKvcacheTilingData emptyTiling; | ||
| 461 | + emptyTiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); | ||
| 462 | + context->GetRawTilingData()->SetDataSize(emptyTiling.GetDataSize()); | ||
| 463 | + size_t *currentWorkspace = context->GetWorkspaceSizes(1); | ||
| 464 | + OP_CHECK_NULL_WITH_CONTEXT(context, currentWorkspace); | ||
| 465 | + currentWorkspace[0] = MINIMAL_WORKSPACE; | ||
| 466 | + return ge::GRAPH_SUCCESS; | ||
| 467 | + } | ||
| 468 | +#endif | ||
| 469 | + | ||
| 488 | int64_t taskNumRem = GetRem(taskNum, tilingInfoDRQK.coreNum); | 470 | int64_t taskNumRem = GetRem(taskNum, tilingInfoDRQK.coreNum); |
| 489 | int64_t frontCoreNum = taskNumRem != 0 ? taskNumRem : tilingInfoDRQK.coreNum; | 471 | int64_t frontCoreNum = taskNumRem != 0 ? taskNumRem : tilingInfoDRQK.coreNum; |
| 490 | int64_t tailCoreNum = taskNum <= tilingInfoDRQK.coreNum ? 0 : tilingInfoDRQK.coreNum - frontCoreNum; | 472 | int64_t tailCoreNum = taskNum <= tilingInfoDRQK.coreNum ? 0 : tilingInfoDRQK.coreNum - frontCoreNum; |
| @@ -524,7 +506,7 @@ static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* contex | |||
| 524 | context->SetBlockDim(numBlocks); | 506 | context->SetBlockDim(numBlocks); |
| 525 | context->SetTilingKey(tilingKey); | 507 | context->SetTilingKey(tilingKey); |
| 526 | 508 | ||
| 527 | - size_t* currentWorkspace = context->GetWorkspaceSizes(1); | 509 | + size_t *currentWorkspace = context->GetWorkspaceSizes(1); |
| 528 | OP_CHECK_NULL_WITH_CONTEXT(context, currentWorkspace); | 510 | OP_CHECK_NULL_WITH_CONTEXT(context, currentWorkspace); |
| 529 | 511 | ||
| 530 | currentWorkspace[0] = MINIMAL_WORKSPACE; | 512 | currentWorkspace[0] = MINIMAL_WORKSPACE; |
| @@ -536,7 +518,7 @@ static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* contex | |||
| 536 | return ge::GRAPH_SUCCESS; | 518 | return ge::GRAPH_SUCCESS; |
| 537 | } | 519 | } |
| 538 | 520 | ||
| 539 | -static ge::graphStatus TilingPrepareForDequantRopeQuantKvcache(gert::TilingParseContext* context) | 521 | +static ge::graphStatus TilingPrepareForDequantRopeQuantKvcache(gert::TilingParseContext *context) |
| 540 | { | 522 | { |
| 541 | (void)context; | 523 | (void)context; |
| 542 | return ge::GRAPH_SUCCESS; | 524 | return ge::GRAPH_SUCCESS; |