Merged
[CANNBot]:dequant_rope_quant_kvcache新增支持ascend950 ascendc实现 #10481
[CANNBot]:dequant_rope_quant_kvcache新增支持ascend950 ascendc实现 #10481
Merged
yulianjieAdd on Aug 24
Total: 5 files changed+1096-166
@@ -4,7 +4,7 @@
4 4 
5|产品 | 是否支持 |5|产品 | 是否支持 |
6|:-------------------------|:----------:|6|:-------------------------|:----------:|
7-| <term>Ascend 950PR/Ascend 950DT</term> | × |7+| <term>Ascend 950PR/Ascend 950DT</term> | √ |
8| <term>Atlas A3 训练系列产品/Atlas A3 推理系列产品</term> | √ |8| <term>Atlas A3 训练系列产品/Atlas A3 推理系列产品</term> | √ |
9| <term>Atlas A2 训练系列产品/Atlas A2 推理系列产品</term> | √ |9| <term>Atlas A2 训练系列产品/Atlas A2 推理系列产品</term> | √ |
10| <term>Atlas 200I/500 A2 推理产品</term> | × |10| <term>Atlas 200I/500 A2 推理产品</term> | × |
@@ -18,51 +18,51 @@
18- 算子功能:对输入张量(x)进行dequant(可选)后,按`sizeSplits`(为切分的长度)对尾轴进行切分,划分为q、k、vOut,对q、k进行旋转位置编码,生成qOut和kOut,之后对kOut和vOut进行量化并按照`indices`更新到kCacheRef和vCacheRef上。18- 算子功能:对输入张量(x)进行dequant(可选)后,按`sizeSplits`(为切分的长度)对尾轴进行切分,划分为q、k、vOut,对q、k进行旋转位置编码,生成qOut和kOut,之后对kOut和vOut进行量化并按照`indices`更新到kCacheRef和vCacheRef上。
19 19 
20- 计算公式:20- 计算公式:
21- 21+ 
22 $$22 $$
23 dequantX = Dequant(x,weightScaleOptional,activationScaleOptional,biasOptional)23 dequantX = Dequant(x,weightScaleOptional,activationScaleOptional,biasOptional)
24 $$24 $$
25- 25+ 
26 $$26 $$
27 q,k,vOut = SplitTensor(dequantX,dim=-1,`sizeSplits`)27 q,k,vOut = SplitTensor(dequantX,dim=-1,`sizeSplits`)
28 $$28 $$
29- 29+ 
30 $$30 $$
31 qOut,kOut = ApplyRotaryPosEmb(q,k,cos,sin)31 qOut,kOut = ApplyRotaryPosEmb(q,k,cos,sin)
32 $$32 $$
33- 33+ 
34 $$34 $$
35 quantK = Quant(kOut,scaleK,offsetKOptional)35 quantK = Quant(kOut,scaleK,offsetKOptional)
36 $$36 $$
37- 37+ 
38 $$38 $$
39 quantV = Quant(vOut,scaleV,offsetVOptional)39 quantV = Quant(vOut,scaleV,offsetVOptional)
40 $$40 $$
41- 41+ 
42 如果cacheModeOptional为contiguous则:42 如果cacheModeOptional为contiguous则:
43- 43+ 
44 $$44 $$
45 kCacheRef[i][indices[i]]=quantK[i]45 kCacheRef[i][indices[i]]=quantK[i]
46 $$46 $$
47- 47+ 
48 $$48 $$
49 vCacheRef[i][indices[i]]=quantV[i]49 vCacheRef[i][indices[i]]=quantV[i]
50 $$50 $$
51- 51+ 
52 如果cacheModeOptional为page则:52 如果cacheModeOptional为page则:
53- 53+ 
54 $$54 $$
55 kCacheRefView=kCacheRef.view(-1,kCacheRef[-2],kCacheRef[-1])55 kCacheRefView=kCacheRef.view(-1,kCacheRef[-2],kCacheRef[-1])
56 $$56 $$
57- 57+ 
58 $$58 $$
59 vCacheRefView=vCacheRef.view(-1,vCacheRef[-2],vCacheRef[-1])59 vCacheRefView=vCacheRef.view(-1,vCacheRef[-2],vCacheRef[-1])
60 $$60 $$
61- 61+ 
62 $$62 $$
63 kCacheRefView[indices[i]]=quantK[i]63 kCacheRefView[indices[i]]=quantK[i]
64 $$64 $$
65- 65+ 
66 $$66 $$
67 vCacheRefView[indices[i]]=quantV[i]67 vCacheRefView[indices[i]]=quantV[i]
68 $$68 $$
@@ -0,0 +1,13 @@
1+; 该文件主要影响 opc 工具 编译二进制kernel时, --simplified_key_mode 选项中填写的值,格式如下所示:
2+; [某算子]
3+; default=xx
4+; ascendxx=xx
5+; 其中,default为默认mode,ascnedxx为可选mode,如果不同芯片有差异化要求时,需要配置;
6+; 1)如果没有配置:非ascendC算子继续按空处理,即opc编译命令中不添加 --simplified_key_mode 选项,AscendC算子按照 simplified_key_mode=0 处理
7+; 2)如果仅有default配置:各个版本按default配置
8+; 3)如果仅有某些平台的配置,没有default配置:对应平台的按照配置的值传递,非对应平台的:非AscendC算子继续按空处理,AscendC算子按照 simplified_key_mode=0 处理
9+; 4)如果default配置和平台配置都有:对应平台的使用平台的配置,非对应的平台的以default值配置。
10+; 5)对于自定义simplified key的情况,需要在binary_simplified_key_mode.ini 文件中显式配置为None,不传入 --simplified_key_mode 选项,由opc工具和FE框架自行判断使用何种模式
11+; 6)是否是AscendC算子,由 ops/build-in/tbe/op_info_cfg/parser/ascendc_config.json 中配置的算子名字和对于的平台决定
12+[DequantRopeQuantKvcache]
13+default=0
@@ -16,15 +16,15 @@
16 16 
17namespace ops {17namespace ops {
18 18 
19-static const std::vector<ge::DataType> XDtypeList = {19+static const std::vector<ge::DataType> XDtypeList = {{ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16,
20- {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32,20+ ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32,
21- ge::DT_INT32, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32,21+ ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_INT32,
22- ge::DT_INT32}};22+ ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}};
23 23 
24-static const std::vector<ge::DataType> cosDtypeList = {24+static const std::vector<ge::DataType> cosDtypeList = {{ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16,
25- {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16,25+ ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16,
26- ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16,26+ ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16, ge::DT_BF16,
27- ge::DT_BF16}};27+ ge::DT_BF16, ge::DT_BF16, ge::DT_BF16}};
28 28 
29static const std::vector<ge::DataType> biasDtypeList = {29static const std::vector<ge::DataType> biasDtypeList = {
30 {ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT,30 {ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_BF16, ge::DT_INT32, ge::DT_FLOAT,
@@ -42,10 +42,10 @@ static const std::vector<ge::DataType> indicesDtypeList = {
42 {ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32,42 {ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32,
43 ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}};43 ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}};
44 44 
45-static const std::vector<ge::Format> formatList = {45+static const std::vector<ge::Format> formatList = {{ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,
46- {ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,46+ ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,
47- ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,47+ ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND,
48- ge::FORMAT_ND, ge::FORMAT_ND}};48+ ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}};
49 49 
50static const std::vector<ge::DataType> XDtypeListKirin = {50static const std::vector<ge::DataType> XDtypeListKirin = {
51 {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}};51 {ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_FLOAT16, ge::DT_INT32, ge::DT_INT32, ge::DT_INT32}};
@@ -70,7 +70,8 @@ static const std::vector<ge::Format> formatListKirin = {
70 70 
71class DequantRopeQuantKvcache : public OpDef {71class DequantRopeQuantKvcache : public OpDef {
72public:72public:
73- explicit DequantRopeQuantKvcache(const char* name) : OpDef(name)73+ explicit DequantRopeQuantKvcache(const char *name)
74+ : OpDef(name)
74 {75 {
75 this->Input("x")76 this->Input("x")
76 .ParamType(REQUIRED)77 .ParamType(REQUIRED)
@@ -170,6 +171,7 @@ public:
170 this->Attr("cache_mode").AttrType(OPTIONAL).String("contiguous");171 this->Attr("cache_mode").AttrType(OPTIONAL).String("contiguous");
171 this->AICore().AddConfig("ascend910b");172 this->AICore().AddConfig("ascend910b");
172 this->AICore().AddConfig("ascend910_93");173 this->AICore().AddConfig("ascend910_93");
174+ this->AICore().AddConfig("ascend950");
173 175 
174 OpAICoreConfig config_kirin = GetKirinCoreConfig();176 OpAICoreConfig config_kirin = GetKirinCoreConfig();
175 this->AICore().AddConfig("kirinx90", config_kirin);177 this->AICore().AddConfig("kirinx90", config_kirin);
@@ -70,7 +70,7 @@ constexpr int64_t DATA_TYPE_INT32 = 2;
70constexpr int64_t DATA_TYPE_BF16 = 3;70constexpr int64_t DATA_TYPE_BF16 = 3;
71 71 
72template <typename T1, typename T2>72template <typename T1, typename T2>
73-inline auto UpAlign(const T1& a, const T2& b) -> T173+inline auto UpAlign(const T1 &a, const T2 &b) -> T1
74{74{
75 if (b != 0) {75 if (b != 0) {
76 return (a + b - 1) / b * b;76 return (a + b - 1) / b * b;
@@ -79,7 +79,7 @@ inline auto UpAlign(const T1& a, const T2& b) -> T1
79}79}
80 80 
81template <typename T>81template <typename T>
82-static auto GetRem(const T& value1, const T& value2) -> T82+static auto GetRem(const T &value1, const T &value2) -> T
83{83{
84 if (value2 == 0) {84 if (value2 == 0) {
85 return value2;85 return value2;
@@ -88,7 +88,7 @@ static auto GetRem(const T& value1, const T& value2) -> T
88}88}
89 89 
90template <typename T>90template <typename T>
91-static auto GetCeilInt(const T& value1, const T& value2) -> T91+static auto GetCeilInt(const T &value1, const T &value2) -> T
92{92{
93 if (value2 == 0) {93 if (value2 == 0) {
94 return value2;94 return value2;
@@ -97,7 +97,7 @@ static auto GetCeilInt(const T& value1, const T& value2) -> T
97}97}
98 98 
99template <typename T>99template <typename T>
100-static auto GetDiv(const T& value1, const T& value2) -> T100+static auto GetDiv(const T &value1, const T &value2) -> T
101{101{
102 if (value2 == 0) {102 if (value2 == 0) {
103 return value2;103 return value2;
@@ -106,17 +106,18 @@ static auto GetDiv(const T& value1, const T& value2) -> T
106}106}
107 107 
108template <typename T>108template <typename T>
109-std::string Shape2String(const T& shape) {109+std::string Shape2String(const T &shape)
110- std::ostringstream oss;110+{
111- oss << "[";111+ std::ostringstream oss;
112- if (shape.GetDimNum() > 0) {112+ oss << "[";
113- for (size_t i = 0; i < shape.GetDimNum() - 1; ++i) {113+ if (shape.GetDimNum() > 0) {
114- oss << shape.GetDim(i) << ", ";114+ for (size_t i = 0; i < shape.GetDimNum() - 1; ++i) {
115+ oss << shape.GetDim(i) << ", ";
116+ }
117+ oss << shape.GetDim(shape.GetDimNum() - 1);
115 }118 }
116- oss << shape.GetDim(shape.GetDimNum() - 1);119+ oss << "]";
117- }120+ return oss.str();
118- oss << "]";
119- return oss.str();
120}121}
121 122 
122struct TilingInfoDRQK {123struct TilingInfoDRQK {
@@ -143,14 +144,14 @@ struct TilingInfoDRQK {
143 int64_t hasAS = 0;144 int64_t hasAS = 0;
144 int64_t biasDataType = 0;145 int64_t biasDataType = 0;
145 int64_t isDequant = 0;146 int64_t isDequant = 0;
146- const int64_t* attrData;147+ const int64_t *attrData;
147};148};
148} // namespace149} // namespace
149using namespace std;150using namespace std;
150 151 
151namespace optiling {152namespace optiling {
152 153 
153-static void DQRKPrintParam(gert::TilingContext* context, DequantRopeQuantKvcacheTilingData& tiling)154+static void DQRKPrintParam(gert::TilingContext *context, DequantRopeQuantKvcacheTilingData &tiling)
154{155{
155 auto nodeName = context->GetNodeName();156 auto nodeName = context->GetNodeName();
156 OP_LOGD(nodeName, ">>>>>>>>>>>>>>> Start to print DequantRopeQuantKvcache tiling data <<<<<<<<<<<<<<<<");157 OP_LOGD(nodeName, ">>>>>>>>>>>>>>> Start to print DequantRopeQuantKvcache tiling data <<<<<<<<<<<<<<<<");
@@ -177,42 +178,35 @@ static void DQRKPrintParam(gert::TilingContext* context, DequantRopeQuantKvcache
177 OP_LOGD(nodeName, ">>> hasAS: %ld", tiling.get_hasAS());178 OP_LOGD(nodeName, ">>> hasAS: %ld", tiling.get_hasAS());
178 OP_LOGD(nodeName, ">>> batch: %ld", tiling.get_batch());179 OP_LOGD(nodeName, ">>> batch: %ld", tiling.get_batch());
179}180}
180-ge::graphStatus checkOptDtype(const uint64_t index, gert::TilingContext* context, const ge::char_t* nodeName_)181+ge::graphStatus checkOptDtype(const uint64_t index, gert::TilingContext *context, const ge::char_t *nodeName_)
181{182{
182 auto optDesc = context->GetOptionalInputDesc(index);183 auto optDesc = context->GetOptionalInputDesc(index);
183 if (optDesc != nullptr) {184 if (optDesc != nullptr) {
184 auto optDtype = optDesc->GetDataType();185 auto optDtype = optDesc->GetDataType();
185- OP_CHECK_IF(186+ OP_CHECK_IF(optDtype != ge::DT_FLOAT, OP_LOGE(nodeName_, "input %lu 's dtype should be float.", index),
186- optDtype != ge::DT_FLOAT,187+ return ge::GRAPH_FAILED);
187- OP_LOGE(nodeName_, "input %lu 's dtype should be float.", index),
188- return ge::GRAPH_FAILED);
189 }188 }
190 return ge::GRAPH_SUCCESS;189 return ge::GRAPH_SUCCESS;
191}190}
192-ge::graphStatus checkDRQKDim(191+ge::graphStatus checkDRQKDim(const gert::StorageShape *inShape, const std::vector<int64_t> &realShape,
193- const gert::StorageShape* inShape, const std::vector<int64_t>& realShape, const ge::char_t* nodeName_,192+ const ge::char_t *nodeName_, std::string logName)
194- std::string logName)
195{193{
196- OP_CHECK_IF(194+ OP_CHECK_IF(inShape->GetStorageShape().GetDimNum() != realShape.size(),
197- inShape->GetStorageShape().GetDimNum() != realShape.size(),195+ OP_LOGE(nodeName_, "%s's dim size should be %zu but got %zu", logName.c_str(), realShape.size(),
198- OP_LOGE(196+ inShape->GetStorageShape().GetDimNum()),
199- nodeName_, "%s's dim size should be %zu but got %zu", logName.c_str(), realShape.size(),197+ return ge::GRAPH_FAILED);
200- inShape->GetStorageShape().GetDimNum()),
201- return ge::GRAPH_FAILED);
202 for (uint64_t dimI = 0; dimI < realShape.size(); dimI++) {198 for (uint64_t dimI = 0; dimI < realShape.size(); dimI++) {
203- OP_CHECK_IF(199+ OP_CHECK_IF(inShape->GetStorageShape().GetDim(dimI) != realShape[dimI],
204- inShape->GetStorageShape().GetDim(dimI) != realShape[dimI],200+ OP_LOGE(nodeName_, "%s's dim[%lu] should be [%ld] but got %ld.", logName.c_str(), dimI,
205- OP_LOGE(201+ realShape[dimI], inShape->GetStorageShape().GetDim(dimI)),
206- nodeName_, "%s's dim[%lu] should be [%ld] but got %ld.", logName.c_str(), dimI, realShape[dimI],202+ return ge::GRAPH_FAILED);
207- inShape->GetStorageShape().GetDim(dimI)),
208- return ge::GRAPH_FAILED);
209 }203 }
210 return ge::GRAPH_SUCCESS;204 return ge::GRAPH_SUCCESS;
211}205}
212 206 
213-bool CheckPaCacheMode(const gert::RuntimeAttrs* attrs)207+bool CheckPaCacheMode(const gert::RuntimeAttrs *attrs)
214{208{
215- const char* tmpmode = attrs->GetStr(CACHE_MODE_IDX);209+ const char *tmpmode = attrs->GetStr(CACHE_MODE_IDX);
216 if (tmpmode == nullptr) {210 if (tmpmode == nullptr) {
217 return false;211 return false;
218 }212 }
@@ -221,7 +215,7 @@ bool CheckPaCacheMode(const gert::RuntimeAttrs* attrs)
221 return (cacheMode == "page");215 return (cacheMode == "page");
222}216}
223 217 
224-ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& tilingInfoDRQK)218+ge::graphStatus getDRQKShapeInfo(gert::TilingContext *context, TilingInfoDRQK &tilingInfoDRQK)
225{219{
226 auto platformInfo = context->GetPlatformInfo();220 auto platformInfo = context->GetPlatformInfo();
227 auto nodeName = context->GetNodeName();221 auto nodeName = context->GetNodeName();
@@ -238,27 +232,23 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
238 auto attr = attrs->GetAttrPointer<gert::ContinuousVector>(0);232 auto attr = attrs->GetAttrPointer<gert::ContinuousVector>(0);
239 OP_CHECK_IF(233 OP_CHECK_IF(
240 attr->GetSize() != QKV_NUM,234 attr->GetSize() != QKV_NUM,
241- OP_LOGE(235+ OP_LOGE(context->GetNodeName(), "the attr size_splits's size %zu is invalid, it should be 3", attr->GetSize()),
242- context->GetNodeName(), "the attr size_splits's size %zu is invalid, it should be 3", attr->GetSize()),
243 return ge::GRAPH_FAILED);236 return ge::GRAPH_FAILED);
244- tilingInfoDRQK.attrData = reinterpret_cast<const int64_t*>(attr->GetData());237+ tilingInfoDRQK.attrData = reinterpret_cast<const int64_t *>(attr->GetData());
245 int64_t qHiddensize = tilingInfoDRQK.attrData[0];238 int64_t qHiddensize = tilingInfoDRQK.attrData[0];
246 int64_t kHiddensize = tilingInfoDRQK.attrData[1];239 int64_t kHiddensize = tilingInfoDRQK.attrData[1];
247 int64_t vHiddensize = tilingInfoDRQK.attrData[2];240 int64_t vHiddensize = tilingInfoDRQK.attrData[2];
248 int64_t qkvHiddensize = qHiddensize + kHiddensize + vHiddensize;241 int64_t qkvHiddensize = qHiddensize + kHiddensize + vHiddensize;
249 OP_CHECK_IF(242 OP_CHECK_IF(
250 kHiddensize != vHiddensize,243 kHiddensize != vHiddensize,
251- OP_LOGE(244+ OP_LOGE(context->GetNodeName(), "the attr size_splits's kHiddensize %ld is not same with vHiddensize %ld",
252- context->GetNodeName(), "the attr size_splits's kHiddensize %ld is not same with vHiddensize %ld",245+ kHiddensize, vHiddensize),
253- kHiddensize, vHiddensize),
254 return ge::GRAPH_FAILED);246 return ge::GRAPH_FAILED);
255 247 
256- OP_CHECK_IF(248+ OP_CHECK_IF((qHiddensize < 0) || (kHiddensize < 0),
257- (qHiddensize < 0) || (kHiddensize < 0),249+ OP_LOGE(context->GetNodeName(), "size_splits[0] %ld size_splits[1] %ld should not less than 0",
258- OP_LOGE(250+ qHiddensize, kHiddensize),
259- context->GetNodeName(), "size_splits[0] %ld size_splits[1] %ld should not less than 0", qHiddensize,251+ return ge::GRAPH_FAILED);
260- kHiddensize),
261- return ge::GRAPH_FAILED);
262 tilingInfoDRQK.ifKVout = *attrs->GetAttrPointer<bool>(IFKV_INDEX) == true ? 1 : 0;252 tilingInfoDRQK.ifKVout = *attrs->GetAttrPointer<bool>(IFKV_INDEX) == true ? 1 : 0;
263 253 
264 auto quantKOffsetDesc = context->GetOptionalInputDesc(QUANT_K_OFFSET_INPUT_INDEX);254 auto quantKOffsetDesc = context->GetOptionalInputDesc(QUANT_K_OFFSET_INPUT_INDEX);
@@ -273,11 +263,10 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
273 auto optDesc = context->GetOptionalInputDesc(BIAS_INPUT_INDEX);263 auto optDesc = context->GetOptionalInputDesc(BIAS_INPUT_INDEX);
274 if (optDesc != nullptr) {264 if (optDesc != nullptr) {
275 auto optDtype = optDesc->GetDataType();265 auto optDtype = optDesc->GetDataType();
276- OP_CHECK_IF(266+ OP_CHECK_IF((optDtype != ge::DT_FLOAT && optDtype != ge::DT_FLOAT16 && optDtype != ge::DT_BF16 &&
277- (optDtype != ge::DT_FLOAT && optDtype != ge::DT_FLOAT16 && optDtype != ge::DT_BF16 &&267+ optDtype != ge::DT_INT32),
278- optDtype != ge::DT_INT32),268+ OP_LOGE(nodeName, "input bias 's dtype should be in{float float16 bfloat16 int32}"),
279- OP_LOGE(nodeName, "input bias 's dtype should be in{float float16 bfloat16 int32}"),269+ return ge::GRAPH_FAILED);
280- return ge::GRAPH_FAILED);
281 if (optDtype == ge::DT_FLOAT) {270 if (optDtype == ge::DT_FLOAT) {
282 tilingInfoDRQK.biasDataType = DATA_TYPE_FLOAT;271 tilingInfoDRQK.biasDataType = DATA_TYPE_FLOAT;
283 } else if (optDtype == ge::DT_FLOAT16) {272 } else if (optDtype == ge::DT_FLOAT16) {
@@ -296,31 +285,27 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
296 auto as = context->GetOptionalInputDesc(ACTIVATION_INPUT_INDEX);285 auto as = context->GetOptionalInputDesc(ACTIVATION_INPUT_INDEX);
297 tilingInfoDRQK.hasAS = (as == nullptr) ? 0 : 1;286 tilingInfoDRQK.hasAS = (as == nullptr) ? 0 : 1;
298 287 
299- const gert::StorageShape* xShape = context->GetInputShape(X_INPUT_INDEX);288+ const gert::StorageShape *xShape = context->GetInputShape(X_INPUT_INDEX);
300 289 
301 OP_CHECK_NULL_WITH_CONTEXT(context, xShape);290 OP_CHECK_NULL_WITH_CONTEXT(context, xShape);
302 int64_t xShapeDimNum = xShape->GetStorageShape().GetDimNum();291 int64_t xShapeDimNum = xShape->GetStorageShape().GetDimNum();
303- OP_CHECK_IF(292+ OP_CHECK_IF((xShapeDimNum != INPUT_X_DIM && xShapeDimNum != BASE_2),
304- (xShapeDimNum != INPUT_X_DIM && xShapeDimNum != BASE_2),293+ OP_LOGE(context->GetNodeName(), "x's dim [%ld] is not 3D or 2D.", xShapeDimNum),
305- OP_LOGE(context->GetNodeName(), "x's dim [%ld] is not 3D or 2D.", xShapeDimNum),294+ return ge::GRAPH_FAILED);
306- return ge::GRAPH_FAILED);
307 tilingInfoDRQK.batch = xShape->GetStorageShape().GetDim(0);295 tilingInfoDRQK.batch = xShape->GetStorageShape().GetDim(0);
308 tilingInfoDRQK.seqlen = (xShapeDimNum == BASE_2) ? 1 : xShape->GetStorageShape().GetDim(1);296 tilingInfoDRQK.seqlen = (xShapeDimNum == BASE_2) ? 1 : xShape->GetStorageShape().GetDim(1);
309 tilingInfoDRQK.isPA = CheckPaCacheMode(attrs);297 tilingInfoDRQK.isPA = CheckPaCacheMode(attrs);
310 298 
311- OP_CHECK_IF(299+ OP_CHECK_IF(xShape->GetStorageShape().GetDim(xShapeDimNum - 1) != qkvHiddensize,
312- xShape->GetStorageShape().GetDim(xShapeDimNum - 1) != qkvHiddensize,300+ OP_LOGE(context->GetNodeName(), "x's dim[-1] [%ld] should be [%ld].",
313- OP_LOGE(301+ xShape->GetStorageShape().GetDim(xShapeDimNum - 1), qkvHiddensize),
314- context->GetNodeName(), "x's dim[-1] [%ld] should be [%ld].",302+ return ge::GRAPH_FAILED);
315- xShape->GetStorageShape().GetDim(xShapeDimNum - 1), qkvHiddensize),303+ const gert::StorageShape *cacheShape = context->GetInputShape(KVCACHE_INPUT_INDEX);
316- return ge::GRAPH_FAILED);
317- const gert::StorageShape* cacheShape = context->GetInputShape(KVCACHE_INPUT_INDEX);
318 OP_CHECK_NULL_WITH_CONTEXT(context, cacheShape);304 OP_CHECK_NULL_WITH_CONTEXT(context, cacheShape);
319 305 
320 OP_CHECK_IF(306 OP_CHECK_IF(
321 cacheShape->GetStorageShape().GetDimNum() != INPUT_CACHE_DIM,307 cacheShape->GetStorageShape().GetDimNum() != INPUT_CACHE_DIM,
322- OP_LOGE(308+ OP_LOGE(context->GetNodeName(), "cacheK's dim [%zu] is not 4D.", cacheShape->GetStorageShape().GetDimNum()),
323- context->GetNodeName(), "cacheK's dim [%zu] is not 4D.", cacheShape->GetStorageShape().GetDimNum()),
324 return ge::GRAPH_FAILED);309 return ge::GRAPH_FAILED);
325 310 
326 int64_t xHiddenSize = xShape->GetStorageShape().GetDim(xShapeDimNum - 1);311 int64_t xHiddenSize = xShape->GetStorageShape().GetDim(xShapeDimNum - 1);
@@ -331,32 +316,26 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
331 int64_t quantShapeSize = tilingInfoDRQK.kvHeadNum * tilingInfoDRQK.hiddenSize;316 int64_t quantShapeSize = tilingInfoDRQK.kvHeadNum * tilingInfoDRQK.hiddenSize;
332 OP_CHECK_IF(317 OP_CHECK_IF(
333 cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize != kHiddensize,318 cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize != kHiddensize,
334- OP_LOGE(319+ OP_LOGE(context->GetNodeName(), "cacheK's headdim * hiddenSize should same with [%ld] but got [%ld].",
335- context->GetNodeName(), "cacheK's headdim * hiddenSize should same with [%ld] but got [%ld].", kHiddensize,320+ kHiddensize, cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize),
336- cacheShape->GetStorageShape().GetDim(KVCACHE_HEAD_DIM_INDEX) * tilingInfoDRQK.hiddenSize),
337 return ge::GRAPH_FAILED);321 return ge::GRAPH_FAILED);
338 OP_CHECK_IF(322 OP_CHECK_IF(
339 tilingInfoDRQK.hiddenSize % FP16_ONE_BLOCK_NUM != 0,323 tilingInfoDRQK.hiddenSize % FP16_ONE_BLOCK_NUM != 0,
340- OP_LOGE(324+ OP_LOGE(context->GetNodeName(), "hiddenSize [%ld] should be a multiple of 16", tilingInfoDRQK.hiddenSize),
341- context->GetNodeName(), "hiddenSize [%ld] should be a multiple of 16", tilingInfoDRQK.hiddenSize),
342- return ge::GRAPH_FAILED);
343- OP_CHECK_IF(
344- qHiddensize % tilingInfoDRQK.hiddenSize != 0,
345- OP_LOGE(
346- context->GetNodeName(), "qhiddenSize [%ld] should be a multiple of hiddenSize", qHiddensize),
347- return ge::GRAPH_FAILED);
348- OP_CHECK_IF(
349- vHiddensize % tilingInfoDRQK.hiddenSize != 0,
350- OP_LOGE(
351- context->GetNodeName(), "vhiddenSize [%ld] should be a multiple of hiddenSize", vHiddensize),
352 return ge::GRAPH_FAILED);325 return ge::GRAPH_FAILED);
326+ OP_CHECK_IF(qHiddensize % tilingInfoDRQK.hiddenSize != 0,
327+ OP_LOGE(context->GetNodeName(), "qhiddenSize [%ld] should be a multiple of hiddenSize", qHiddensize),
328+ return ge::GRAPH_FAILED);
329+ OP_CHECK_IF(vHiddensize % tilingInfoDRQK.hiddenSize != 0,
330+ OP_LOGE(context->GetNodeName(), "vhiddenSize [%ld] should be a multiple of hiddenSize", vHiddensize),
331+ return ge::GRAPH_FAILED);
353 tilingInfoDRQK.qHeadNum = GetDiv(tilingInfoDRQK.attrData[0], tilingInfoDRQK.hiddenSize);332 tilingInfoDRQK.qHeadNum = GetDiv(tilingInfoDRQK.attrData[0], tilingInfoDRQK.hiddenSize);
354- const gert::StorageShape* cosShape = context->GetInputShape(COS_INDEX);333+ const gert::StorageShape *cosShape = context->GetInputShape(COS_INDEX);
355- const gert::StorageShape* sinShape = context->GetInputShape(SIN_INDEX);334+ const gert::StorageShape *sinShape = context->GetInputShape(SIN_INDEX);
356- const gert::StorageShape* vCacheShape = context->GetInputShape(VCACHE_INPUT_INDEX);335+ const gert::StorageShape *vCacheShape = context->GetInputShape(VCACHE_INPUT_INDEX);
357- const gert::StorageShape* indicesShape = context->GetInputShape(INDICES_INPUT_INDEX);336+ const gert::StorageShape *indicesShape = context->GetInputShape(INDICES_INPUT_INDEX);
358- const gert::StorageShape* kScaleShape = context->GetInputShape(QUANT_K_SCALE_INPUT_INDEX);337+ const gert::StorageShape *kScaleShape = context->GetInputShape(QUANT_K_SCALE_INPUT_INDEX);
359- const gert::StorageShape* vScaleShape = context->GetInputShape(QUANT_V_SCALE_INPUT_INDEX);338+ const gert::StorageShape *vScaleShape = context->GetInputShape(QUANT_V_SCALE_INPUT_INDEX);
360 339 
361 std::vector<int64_t> expSinCos = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, 1, tilingInfoDRQK.hiddenSize};340 std::vector<int64_t> expSinCos = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, 1, tilingInfoDRQK.hiddenSize};
362 std::vector<int64_t> expX = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, qkvHiddensize};341 std::vector<int64_t> expX = {tilingInfoDRQK.batch, tilingInfoDRQK.seqlen, qkvHiddensize};
@@ -376,29 +355,23 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
376 (checkDRQKDim(indicesShape, {tilingInfoDRQK.batch}, nodeName, "input indices") == ge::GRAPH_FAILED)) {355 (checkDRQKDim(indicesShape, {tilingInfoDRQK.batch}, nodeName, "input indices") == ge::GRAPH_FAILED)) {
377 return ge::GRAPH_FAILED;356 return ge::GRAPH_FAILED;
378 }357 }
379- OP_CHECK_IF(358+ OP_CHECK_IF(cacheShape->GetStorageShape() != vCacheShape->GetStorageShape(),
380- cacheShape->GetStorageShape() != vCacheShape->GetStorageShape(),359+ OP_LOGE(context->GetNodeName(), "kcache shape:%s should be equal with vcache shape:%s",
381- OP_LOGE(360+ Shape2String(cacheShape->GetStorageShape()).c_str(),
382- context->GetNodeName(), "kcache shape:%s should be equal with vcache shape:%s",361+ Shape2String(vCacheShape->GetStorageShape()).c_str()),
383- Shape2String(cacheShape->GetStorageShape()).c_str(),362+ return ge::GRAPH_FAILED);
384- Shape2String(vCacheShape->GetStorageShape()).c_str()),363+ OP_CHECK_IF((kScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize ||
385- return ge::GRAPH_FAILED);364+ vScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize),
386- OP_CHECK_IF(365+ OP_LOGE(context->GetNodeName(),
387- (kScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize ||366+ "kScaleShape shape size:%ld or vScaleShape shape size:%ld should be equal with :%ld",
388- vScaleShape->GetStorageShape().GetShapeSize() != quantShapeSize),367+ kScaleShape->GetStorageShape().GetShapeSize(), vScaleShape->GetStorageShape().GetShapeSize(),
389- OP_LOGE(368+ quantShapeSize),
390- context->GetNodeName(),369+ return ge::GRAPH_FAILED);
391- "kScaleShape shape size:%ld or vScaleShape shape size:%ld should be equal with :%ld",
392- kScaleShape->GetStorageShape().GetShapeSize(), vScaleShape->GetStorageShape().GetShapeSize(),
393- quantShapeSize),
394- return ge::GRAPH_FAILED);
395 if (!tilingInfoDRQK.isPA) {370 if (!tilingInfoDRQK.isPA) {
396- OP_CHECK_IF(371+ OP_CHECK_IF(vCacheShape->GetStorageShape().GetDim(0) < xShape->GetStorageShape().GetDim(0),
397- vCacheShape->GetStorageShape().GetDim(0) < xShape->GetStorageShape().GetDim(0),372+ OP_LOGE(context->GetNodeName(), "vcache's dim[0]:%ld should not be less than input x's dim[0]:%ld",
398- OP_LOGE(373+ vCacheShape->GetStorageShape().GetDim(0), xShape->GetStorageShape().GetDim(0)),
399- context->GetNodeName(), "vcache's dim[0]:%ld should not be less than input x's dim[0]:%ld",374+ return ge::GRAPH_FAILED);
400- vCacheShape->GetStorageShape().GetDim(0), xShape->GetStorageShape().GetDim(0)),
401- return ge::GRAPH_FAILED);
402 }375 }
403 376 
404 auto xDesc = context->GetInputDesc(0);377 auto xDesc = context->GetInputDesc(0);
@@ -418,17 +391,15 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
418 auto scaleDesc = context->GetInputDesc(QUANT_K_SCALE_INPUT_INDEX);391 auto scaleDesc = context->GetInputDesc(QUANT_K_SCALE_INPUT_INDEX);
419 OP_CHECK_NULL_WITH_CONTEXT(context, scaleDesc);392 OP_CHECK_NULL_WITH_CONTEXT(context, scaleDesc);
420 if (tilingInfoDRQK.hasQuantOffset == 1) {393 if (tilingInfoDRQK.hasQuantOffset == 1) {
421- const gert::StorageShape* kOffsetShape = context->GetOptionalInputShape(QUANT_K_OFFSET_INPUT_INDEX);394+ const gert::StorageShape *kOffsetShape = context->GetOptionalInputShape(QUANT_K_OFFSET_INPUT_INDEX);
422- const gert::StorageShape* vOffsetShape = context->GetOptionalInputShape(QUANT_V_OFFSET_INPUT_INDEX);395+ const gert::StorageShape *vOffsetShape = context->GetOptionalInputShape(QUANT_V_OFFSET_INPUT_INDEX);
423- OP_CHECK_IF(396+ OP_CHECK_IF((kOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize ||
424- (kOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize ||397+ vOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize),
425- vOffsetShape->GetStorageShape().GetShapeSize() != quantShapeSize),398+ OP_LOGE(context->GetNodeName(),
426- OP_LOGE(399+ "kOffsetShape shape size:%ld or vOffsetShape shape size:%ld should be equal with :%ld",
427- context->GetNodeName(),400+ kOffsetShape->GetStorageShape().GetShapeSize(),
428- "kOffsetShape shape size:%ld or vOffsetShape shape size:%ld should be equal with :%ld",401+ vOffsetShape->GetStorageShape().GetShapeSize(), quantShapeSize),
429- kOffsetShape->GetStorageShape().GetShapeSize(), vOffsetShape->GetStorageShape().GetShapeSize(),402+ return ge::GRAPH_FAILED);
430- quantShapeSize),
431- return ge::GRAPH_FAILED);
432 tilingInfoDRQK.scalesBtyeSize = ge::GetSizeByDataType(scaleDesc->GetDataType());403 tilingInfoDRQK.scalesBtyeSize = ge::GetSizeByDataType(scaleDesc->GetDataType());
433 }404 }
434 auto wsDesc = context->GetOptionalInputDesc(WEIGHT_SCALES_INPUT_INDEX);405 auto wsDesc = context->GetOptionalInputDesc(WEIGHT_SCALES_INPUT_INDEX);
@@ -457,25 +428,21 @@ ge::graphStatus getDRQKShapeInfo(gert::TilingContext* context, TilingInfoDRQK& t
457 428 
458 int64_t unUsedSize = tilingInfoDRQK.ubSize - remainSize;429 int64_t unUsedSize = tilingInfoDRQK.ubSize - remainSize;
459 430 
460- OP_CHECK_IF(431+ OP_CHECK_IF((unUsedSize <= 0),
461- (unUsedSize <= 0),432+ OP_LOGE(context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].",
462- OP_LOGE(433+ xHiddenSize, HIDDEN_SIZE_LIMIT_LENGTH),
463- context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].", xHiddenSize,434+ return ge::GRAPH_FAILED);
464- HIDDEN_SIZE_LIMIT_LENGTH),
465- return ge::GRAPH_FAILED);
466 435 
467 tilingInfoDRQK.onceS = GetDiv(unUsedSize, onceSSize);436 tilingInfoDRQK.onceS = GetDiv(unUsedSize, onceSSize);
468 437 
469- OP_CHECK_IF(438+ OP_CHECK_IF((tilingInfoDRQK.onceS == 0),
470- (tilingInfoDRQK.onceS == 0),439+ OP_LOGE(context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].",
471- OP_LOGE(440+ xHiddenSize, HIDDEN_SIZE_LIMIT_LENGTH),
472- context->GetNodeName(), "The elements num of x dim[-1] [%ld] should be less than [%ld].", xHiddenSize,441+ return ge::GRAPH_FAILED);
473- HIDDEN_SIZE_LIMIT_LENGTH),
474- return ge::GRAPH_FAILED);
475 return ge::GRAPH_SUCCESS;442 return ge::GRAPH_SUCCESS;
476}443}
477 444 
478-static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* context)445+static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext *context)
479{446{
480 TilingInfoDRQK tilingInfoDRQK;447 TilingInfoDRQK tilingInfoDRQK;
481 auto nodeName = context->GetNodeName();448 auto nodeName = context->GetNodeName();
@@ -485,6 +452,21 @@ static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* contex
485 return ge::GRAPH_FAILED;452 return ge::GRAPH_FAILED;
486 }453 }
487 int64_t taskNum = tilingInfoDRQK.batch * tilingInfoDRQK.seqlen;454 int64_t taskNum = tilingInfoDRQK.batch * tilingInfoDRQK.seqlen;
455+ 
456+#if (__NPU_ARCH__ == 3510)
457+ if (taskNum == 0) {
458+ OP_LOGD(nodeName, "taskNum is 0, empty tensor, set block dim to 0.");
459+ context->SetBlockDim(0);
460+ DequantRopeQuantKvcacheTilingData emptyTiling;
461+ emptyTiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity());
462+ context->GetRawTilingData()->SetDataSize(emptyTiling.GetDataSize());
463+ size_t *currentWorkspace = context->GetWorkspaceSizes(1);
464+ OP_CHECK_NULL_WITH_CONTEXT(context, currentWorkspace);
465+ currentWorkspace[0] = MINIMAL_WORKSPACE;
466+ return ge::GRAPH_SUCCESS;
467+ }
468+#endif
469+ 
488 int64_t taskNumRem = GetRem(taskNum, tilingInfoDRQK.coreNum);470 int64_t taskNumRem = GetRem(taskNum, tilingInfoDRQK.coreNum);
489 int64_t frontCoreNum = taskNumRem != 0 ? taskNumRem : tilingInfoDRQK.coreNum;471 int64_t frontCoreNum = taskNumRem != 0 ? taskNumRem : tilingInfoDRQK.coreNum;
490 int64_t tailCoreNum = taskNum <= tilingInfoDRQK.coreNum ? 0 : tilingInfoDRQK.coreNum - frontCoreNum;472 int64_t tailCoreNum = taskNum <= tilingInfoDRQK.coreNum ? 0 : tilingInfoDRQK.coreNum - frontCoreNum;
@@ -524,7 +506,7 @@ static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* contex
524 context->SetBlockDim(numBlocks);506 context->SetBlockDim(numBlocks);
525 context->SetTilingKey(tilingKey);507 context->SetTilingKey(tilingKey);
526 508 
527- size_t* currentWorkspace = context->GetWorkspaceSizes(1);509+ size_t *currentWorkspace = context->GetWorkspaceSizes(1);
528 OP_CHECK_NULL_WITH_CONTEXT(context, currentWorkspace);510 OP_CHECK_NULL_WITH_CONTEXT(context, currentWorkspace);
529 511 
530 currentWorkspace[0] = MINIMAL_WORKSPACE;512 currentWorkspace[0] = MINIMAL_WORKSPACE;
@@ -536,7 +518,7 @@ static ge::graphStatus TilingDequantRopeQuantKvcache(gert::TilingContext* contex
536 return ge::GRAPH_SUCCESS;518 return ge::GRAPH_SUCCESS;
537}519}
538 520 
539-static ge::graphStatus TilingPrepareForDequantRopeQuantKvcache(gert::TilingParseContext* context)521+static ge::graphStatus TilingPrepareForDequantRopeQuantKvcache(gert::TilingParseContext *context)
540{522{
541 (void)context;523 (void)context;
542 return ge::GRAPH_SUCCESS;524 return ge::GRAPH_SUCCESS;