qbmm算子，x2 input, 外轴大于65536，weightnz场景，部分case有精度问题-ops-nn-AtomGit

Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/quant_batch_matmul_v3_basic_tiling.cpp+14-0

	@@ -833,6 +833,20 @@ bool QuantBatchMatmulV3BasicTiling::SetBase(const std::vector<uint64_t> &mBases,
833	}	833	}
834	OP_TILING_CHECK(!GetBaseK(basicTiling_.baseM, basicTiling_.baseN),	834	OP_TILING_CHECK(!GetBaseK(basicTiling_.baseM, basicTiling_.baseN),
835	CUBE_INNER_ERR_REPORT(inputParams_.opName, "GetBaseK failed"), return false);	835	CUBE_INNER_ERR_REPORT(inputParams_.opName, "GetBaseK failed"), return false);
		836	+ return CapBaseNForATransWeightNz();
		837	+}
		838	+
		839	+bool QuantBatchMatmulV3BasicTiling::CapBaseNForATransWeightNz()
		840	+{
		841	+ // Weight NZ n1,k1,k0,n0: cube fetches N in n0=32 steps; baseN must not exceed 32 for transA.
		842	+ if (inputParams_.transA && !inputParams_.transB && inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ &&
		843	+ basicTiling_.baseN > ONE_BLK_SIZE) {
		844	+ basicTiling_.baseN = ONE_BLK_SIZE;
		845	+ uint64_t totalCnt = GetTotalCnt(basicTiling_.baseM, basicTiling_.baseN);
		846	+ basicTiling_.usedCoreNum = std::min(totalCnt, aicoreParams_.aicNum);
		847	+ OP_TILING_CHECK(!GetBaseK(basicTiling_.baseM, basicTiling_.baseN),
		848	+ CUBE_INNER_ERR_REPORT(inputParams_.opName, "GetBaseK failed after cap baseN"), return false);
		849	+ }
836	return true;	850	return true;
837	}	851	}
838		852

Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/quant_batch_matmul_v3_basic_tiling.h+2-1

	@@ -139,6 +139,7 @@ protected:
139	void DivisibleCoreLayout(uint64_t mCnt, uint64_t nCnt, uint64_t& calcOrder, uint64_t round) const;	139	void DivisibleCoreLayout(uint64_t mCnt, uint64_t nCnt, uint64_t& calcOrder, uint64_t round) const;
140	void DetermineCalcOrder();	140	void DetermineCalcOrder();
141	void SetCalcOrderinMNClashCase(uint64_t mTotalCnt, uint64_t nTotalCnt);	141	void SetCalcOrderinMNClashCase(uint64_t mTotalCnt, uint64_t nTotalCnt);
		142	+ bool CapBaseNForATransWeightNz();
142		143
143	private:	144	private:
144	gert::TilingContext *context;	145	gert::TilingContext *context;
	@@ -147,4 +148,4 @@ private:
147	};	148	};
148		149
149	} // namespace optiling	150	} // namespace optiling
150	-#endif // QUANT_BATCH_MATMUL_V3_BASIC_TILING_H	151	+#endif // QUANT_BATCH_MATMUL_V3_BASIC_TILING_H

Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_bf16.h+5-9

	@@ -277,11 +277,9 @@ protected:
277	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *	277	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *
278	DequantBmm::Align(k_, K0_INT8);	278	DequantBmm::Align(k_, K0_INT8);
279	} else {	279	} else {
280	- // n1, k1, k0, n0	280	+ // n1, k1, k0, n0 — column stride is Align(k, 16) for nd_to_nz layout (k0=16)
281	- offsetB_ =	281	+ offsetB_ = nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM) +
282	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM) +	282	+ batchBOffset * DequantBmm::Align(n_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
283	- batchBOffset * DequantBmm::Align(n_, K0_INT8) *
284	- DequantBmm::Align(k_, BMM_BLOCK_NUM);
285	}	283	}
286	}	284	}
287		285
	@@ -321,8 +319,7 @@ protected:
321	if constexpr (bTrans) {	319	if constexpr (bTrans) {
322	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;	320	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;
323	} else {	321	} else {
324	- offsetB_ -=	322	+ offsetB_ -= nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM);
325	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
326	}	323	}
327	}	324	}
328	offsetBias_ -= nOffset;	325	offsetBias_ -= nOffset;
	@@ -341,8 +338,7 @@ protected:
341	if constexpr (bTrans) {	338	if constexpr (bTrans) {
342	offsetB_ += DequantBmm::Align(singleTimeN_, BMM_BLOCK_NUM) * K0_INT8;	339	offsetB_ += DequantBmm::Align(singleTimeN_, BMM_BLOCK_NUM) * K0_INT8;
343	} else {	340	} else {
344	- offsetB_ +=	341	+ offsetB_ += singleTimeN_ * DequantBmm::Align(k_, BMM_BLOCK_NUM);
345	- DequantBmm::Align(singleTimeN_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
346	}	342	}
347	}	343	}
348		344

Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_bf16_opt.h+5-9

	@@ -281,11 +281,9 @@ protected:
281	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *	281	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *
282	DequantBmm::Align(k_, K0_INT8);	282	DequantBmm::Align(k_, K0_INT8);
283	} else {	283	} else {
284	- // n1, k1, k0, n0	284	+ // n1, k1, k0, n0 — column stride is Align(k, 16) for nd_to_nz layout (k0=16)
285	- offsetB_ =	285	+ offsetB_ = nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM) +
286	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM) +	286	+ batchBOffset * DequantBmm::Align(n_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
287	- batchBOffset * DequantBmm::Align(n_, K0_INT8) *
288	- DequantBmm::Align(k_, BMM_BLOCK_NUM);
289	}	287	}
290	}	288	}
291		289
	@@ -325,8 +323,7 @@ protected:
325	if constexpr (bTrans) {	323	if constexpr (bTrans) {
326	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;	324	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;
327	} else {	325	} else {
328	- offsetB_ -=	326	+ offsetB_ -= nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM);
329	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
330	}	327	}
331	}	328	}
332	offsetBias_ -= nOffset;	329	offsetBias_ -= nOffset;
	@@ -345,8 +342,7 @@ protected:
345	if constexpr (bTrans) {	342	if constexpr (bTrans) {
346	offsetB_ += DequantBmm::Align(singleTimeN_, BMM_BLOCK_NUM) * K0_INT8;	343	offsetB_ += DequantBmm::Align(singleTimeN_, BMM_BLOCK_NUM) * K0_INT8;
347	} else {	344	} else {
348	- offsetB_ +=	345	+ offsetB_ += singleTimeN_ * DequantBmm::Align(k_, BMM_BLOCK_NUM);
349	- DequantBmm::Align(singleTimeN_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
350	}	346	}
351	}	347	}
352		348

Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_pertoken.h+26-11

	@@ -121,7 +121,7 @@ public:
121	uint32_t gmUseN = n_ - nCoreIndx * singleCoreN_;	121	uint32_t gmUseN = n_ - nCoreIndx * singleCoreN_;
122	uint32_t singleCoreNUpdate = gmUseN < singleCoreN_ ? gmUseN : singleCoreN_;	122	uint32_t singleCoreNUpdate = gmUseN < singleCoreN_ ? gmUseN : singleCoreN_;
123		123
124	- mm.SetOrgShape(m_, n_, k_);	124	+ SetMatmulOrgShape();
125	uint32_t mLoops = DequantBmm::CeilDiv(singleCoreMUpdate, singleTimeM_);	125	uint32_t mLoops = DequantBmm::CeilDiv(singleCoreMUpdate, singleTimeM_);
126	uint32_t nLoops = DequantBmm::CeilDiv(singleCoreNUpdate, singleTimeN_);	126	uint32_t nLoops = DequantBmm::CeilDiv(singleCoreNUpdate, singleTimeN_);
127	for (uint32_t i = 0; i < singleCoreBatchUpdate; ++i) {	127	for (uint32_t i = 0; i < singleCoreBatchUpdate; ++i) {
	@@ -248,6 +248,25 @@ protected:
248	ubTmpBuffer_ = tilingData->params.needUbBuffer;	248	ubTmpBuffer_ = tilingData->params.needUbBuffer;
249	}	249	}
250		250
		251	+ __aicore__ inline void SetMatmulOrgShape()
		252	+ {
		253	+ if constexpr (DequantBmm::GetFormat(fFormat) == CubeFormat::NZ &&
		254	+ DequantBmm::GetFormat(wFormat) == CubeFormat::NZ) {
		255	+ mm.SetOrgShape(DequantBmm::Align(m_, BMM_BLOCK_NUM), DequantBmm::Align(n_, BMM_BLOCK_NUM),
		256	+ DequantBmm::Align(k_, K0_INT8), DequantBmm::Align(k_, K0_INT8), n_);
		257	+ } else if constexpr (DequantBmm::GetFormat(fFormat) == CubeFormat::NZ) {
		258	+ mm.SetOrgShape(DequantBmm::Align(m_, K0_INT8), n_, DequantBmm::Align(k_, BMM_BLOCK_NUM), k_, n_);
		259	+ } else if constexpr (DequantBmm::GetFormat(wFormat) == CubeFormat::NZ) {
		260	+ if constexpr (bTrans) {
		261	+ mm.SetOrgShape(m_, DequantBmm::Align(n_, BMM_BLOCK_NUM), k_, DequantBmm::Align(k_, K0_INT8), n_);
		262	+ } else {
		263	+ mm.SetOrgShape(m_, DequantBmm::Align(n_, BMM_BLOCK_NUM), k_, DequantBmm::Align(k_, BMM_BLOCK_NUM), n_);
		264	+ }
		265	+ } else {
		266	+ mm.SetOrgShape(m_, n_, k_);
		267	+ }
		268	+ }
		269	+
251	__aicore__ inline void CalcOffset(uint32_t &batchIndex, uint32_t &batchCoreIndx, uint32_t &mCoreIndx,	270	__aicore__ inline void CalcOffset(uint32_t &batchIndex, uint32_t &batchCoreIndx, uint32_t &mCoreIndx,
252	uint32_t &nCoreIndx)	271	uint32_t &nCoreIndx)
253	{	272	{
	@@ -284,11 +303,9 @@ protected:
284	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *	303	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *
285	DequantBmm::Align(k_, K0_INT8);	304	DequantBmm::Align(k_, K0_INT8);
286	} else {	305	} else {
287	- // n1, k1, k0, n0	306	+ // n1, k1, k0, n0 — column stride is Align(k, 16) for nd_to_nz layout (k0=16)
288	- offsetB_ =	307	+ offsetB_ = nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM) +
289	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM) +	308	+ batchBOffset * DequantBmm::Align(n_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
290	- batchBOffset * DequantBmm::Align(n_, K0_INT8) *
291	- DequantBmm::Align(k_, BMM_BLOCK_NUM);
292	}	309	}
293	}	310	}
294		311
	@@ -330,8 +347,7 @@ protected:
330	if constexpr (bTrans) {	347	if constexpr (bTrans) {
331	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;	348	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;
332	} else {	349	} else {
333	- offsetB_ -=	350	+ offsetB_ -= nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM);
334	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
335	}	351	}
336	}	352	}
337	offsetBias_ -= nOffset;	353	offsetBias_ -= nOffset;
	@@ -350,8 +366,7 @@ protected:
350	if constexpr (bTrans) {	366	if constexpr (bTrans) {
351	offsetB_ += DequantBmm::Align(singleTimeN_, BMM_BLOCK_NUM) * K0_INT8;	367	offsetB_ += DequantBmm::Align(singleTimeN_, BMM_BLOCK_NUM) * K0_INT8;
352	} else {	368	} else {
353	- offsetB_ +=	369	+ offsetB_ += singleTimeN_ * DequantBmm::Align(k_, BMM_BLOCK_NUM);
354	- DequantBmm::Align(singleTimeN_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
355	}	370	}
356	}	371	}
357		372
	@@ -593,4 +608,4 @@ protected:
593	};	608	};
594	} // namespace AscendC	609	} // namespace AscendC
595		610
596	-#endif // QUANT_BATCH_MATMUL_V3_PERTOKEN_H	611	+#endif // QUANT_BATCH_MATMUL_V3_PERTOKEN_H

Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_pertoken_basic.h+29-1

	@@ -50,8 +50,10 @@ public:
50	return;	50	return;
51	}	51	}
52	pipe_ = tPipe;	52	pipe_ = tPipe;
		53	+ mm_.SetSubBlockIdx(0);
53	mm_.Init(&(tilingData->matmulTiling), pipe_);	54	mm_.Init(&(tilingData->matmulTiling), pipe_);
54	InitTilingData(tilingData);	55	InitTilingData(tilingData);
		56	+ SetMatmulOrgShape();
55		57
56	InitGlobalBuffers(x1, x2, scale, bias, pertokenScale, y, workSpace);	58	InitGlobalBuffers(x1, x2, scale, bias, pertokenScale, y, workSpace);
57	InitLocalBuffers();	59	InitLocalBuffers();
	@@ -140,6 +142,28 @@ private:
140	ubTmpBuffer_ = tilingData->params.needUbBuffer;	142	ubTmpBuffer_ = tilingData->params.needUbBuffer;
141	}	143	}
142		144
		145	+ __aicore__ inline void SetMatmulOrgShape()
		146	+ {
		147	+ if constexpr (DequantBmm::GetFormat(x1Format) == CubeFormat::NZ &&
		148	+ DequantBmm::GetFormat(x2Format) == CubeFormat::NZ) {
		149	+ mm_.SetOrgShape(DequantBmm::Align(m_, BMM_BLOCK_NUM), DequantBmm::Align(n_, BMM_BLOCK_NUM),
		150	+ DequantBmm::Align(ka_, K0_INT8), DequantBmm::Align(ka_, K0_INT8), n_);
		151	+ } else if constexpr (DequantBmm::GetFormat(x1Format) == CubeFormat::NZ) {
		152	+ mm_.SetOrgShape(DequantBmm::Align(m_, K0_INT8), n_, DequantBmm::Align(ka_, BMM_BLOCK_NUM), ka_, n_);
		153	+ } else if constexpr (DequantBmm::GetFormat(x2Format) == CubeFormat::NZ) {
		154	+ if constexpr (bTrans) {
		155	+ // k1,n1,n0,k0 layout: orgN aligned, last param is full N (see mat_mul_v3 / weight_quant NZ)
		156	+ mm_.SetOrgShape(m_, DequantBmm::Align(n_, BMM_BLOCK_NUM), ka_, DequantBmm::Align(ka_, K0_INT8), n_);
		157	+ } else {
		158	+ // n1,k1,k0,n0 layout (k0=16): orgK aligned to k fractal
		159	+ mm_.SetOrgShape(m_, DequantBmm::Align(n_, BMM_BLOCK_NUM), ka_, DequantBmm::Align(ka_, BMM_BLOCK_NUM),
		160	+ n_);
		161	+ }
		162	+ } else {
		163	+ mm_.SetOrgShape(m_, n_, ka_);
		164	+ }
		165	+ }
		166	+
143	__aicore__ inline void InitGlobalBuffers(GM_ADDR x1, GM_ADDR x2, GM_ADDR scale, GM_ADDR bias, GM_ADDR pertokenScale,	167	__aicore__ inline void InitGlobalBuffers(GM_ADDR x1, GM_ADDR x2, GM_ADDR scale, GM_ADDR bias, GM_ADDR pertokenScale,
144	GM_ADDR y, GM_ADDR workSpace)	168	GM_ADDR y, GM_ADDR workSpace)
145	{	169	{
	@@ -260,7 +284,11 @@ private:
260		284
261	__aicore__ inline void BasicMMCompute(uint32_t baseM, uint32_t baseN)	285	__aicore__ inline void BasicMMCompute(uint32_t baseM, uint32_t baseN)
262	{	286	{
		287	+ if constexpr (DequantBmm::GetFormat(x2Format) == CubeFormat::NZ && !bTrans) {
		288	+ mm_.SetOrgShape(m_, baseN, ka_, DequantBmm::Align(ka_, BMM_BLOCK_NUM), n_);
		289	+ }
263	mm_.SetSingleShape(baseM, baseN, ka_);	290	mm_.SetSingleShape(baseM, baseN, ka_);
		291	+ mm_.SetTail(baseM, baseN, ka_);
264	mm_.SetTensorA(xGm_[offset_.offsetA], aTrans);	292	mm_.SetTensorA(xGm_[offset_.offsetA], aTrans);
265	mm_.SetTensorB(weightGm_[offset_.offsetB], bTrans);	293	mm_.SetTensorB(weightGm_[offset_.offsetB], bTrans);
266		294
	@@ -354,4 +382,4 @@ private:
354		382
355	} // namespace AscendC	383	} // namespace AscendC
356		384
357	-#endif // QUANT_BATCH_MATMUL_V3_PERTOKEN_BASIC_H	385	+#endif // QUANT_BATCH_MATMUL_V3_PERTOKEN_BASIC_H

Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_pertoken_opt.h+28-10

	@@ -39,6 +39,7 @@ public:
39	}	39	}
40	mm.SetSubBlockIdx(0);	40	mm.SetSubBlockIdx(0);
41	mm.Init(&(tilingData->matmulTiling), tPipe);	41	mm.Init(&(tilingData->matmulTiling), tPipe);
		42	+ SetMatmulOrgShape();
42	pipe_ = tPipe;	43	pipe_ = tPipe;
43		44
44	// init global buffer	45	// init global buffer
	@@ -260,7 +261,27 @@ protected:
260	ubTmpBuffer_ = tilingData->params.needUbBuffer;	261	ubTmpBuffer_ = tilingData->params.needUbBuffer;
261	}	262	}
262		263
263	- __aicore__ inline void CalcOffset(uint32_t batchIndex, uint32_t batchCoreIndx, uint32_t mCoreIndx, uint32_t nCoreIndx)	264	+ __aicore__ inline void SetMatmulOrgShape()
		265	+ {
		266	+ if constexpr (DequantBmm::GetFormat(fFormat) == CubeFormat::NZ &&
		267	+ DequantBmm::GetFormat(wFormat) == CubeFormat::NZ) {
		268	+ mm.SetOrgShape(DequantBmm::Align(m_, BMM_BLOCK_NUM), DequantBmm::Align(n_, BMM_BLOCK_NUM),
		269	+ DequantBmm::Align(k_, K0_INT8), DequantBmm::Align(k_, K0_INT8), n_);
		270	+ } else if constexpr (DequantBmm::GetFormat(fFormat) == CubeFormat::NZ) {
		271	+ mm.SetOrgShape(DequantBmm::Align(m_, K0_INT8), n_, DequantBmm::Align(k_, BMM_BLOCK_NUM), k_, n_);
		272	+ } else if constexpr (DequantBmm::GetFormat(wFormat) == CubeFormat::NZ) {
		273	+ if constexpr (bTrans) {
		274	+ mm.SetOrgShape(m_, DequantBmm::Align(n_, BMM_BLOCK_NUM), k_, DequantBmm::Align(k_, K0_INT8), n_);
		275	+ } else {
		276	+ mm.SetOrgShape(m_, DequantBmm::Align(n_, BMM_BLOCK_NUM), k_, DequantBmm::Align(k_, BMM_BLOCK_NUM), n_);
		277	+ }
		278	+ } else {
		279	+ mm.SetOrgShape(m_, n_, k_);
		280	+ }
		281	+ }
		282	+
		283	+ __aicore__ inline void CalcOffset(uint32_t batchIndex, uint32_t batchCoreIndx, uint32_t mCoreIndx,
		284	+ uint32_t nCoreIndx)
264	{	285	{
265	uint64_t batchAOffset = static_cast<uint64_t>((batchCoreIndx * (singleCoreBatch_) + batchIndex) % batchA_);	286	uint64_t batchAOffset = static_cast<uint64_t>((batchCoreIndx * (singleCoreBatch_) + batchIndex) % batchA_);
266	uint64_t batchBOffset = static_cast<uint64_t>((batchCoreIndx * (singleCoreBatch_) + batchIndex) % batchB_);	287	uint64_t batchBOffset = static_cast<uint64_t>((batchCoreIndx * (singleCoreBatch_) + batchIndex) % batchB_);
	@@ -295,11 +316,9 @@ protected:
295	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *	316	batchBOffset * DequantBmm::Align(n_, BMM_BLOCK_NUM) *
296	DequantBmm::Align(k_, K0_INT8);	317	DequantBmm::Align(k_, K0_INT8);
297	} else {	318	} else {
298	- // n1, k1, k0, n0	319	+ // n1, k1, k0, n0 — column stride is Align(k, 16) for nd_to_nz layout (k0=16)
299	- offsetB_ =	320	+ offsetB_ = nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM) +
300	- DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM) +	321	+ batchBOffset * DequantBmm::Align(n_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
301	- batchBOffset * DequantBmm::Align(n_, K0_INT8) *
302	- DequantBmm::Align(k_, BMM_BLOCK_NUM);
303	}	322	}
304	}	323	}
305		324
	@@ -355,7 +374,7 @@ protected:
355	if constexpr (bTrans) {	374	if constexpr (bTrans) {
356	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;	375	offsetB_ -= DequantBmm::Align(nOffset, BMM_BLOCK_NUM) * K0_INT8;
357	} else {	376	} else {
358	- offsetB_ -= DequantBmm::Align(nOffset, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);	377	+ offsetB_ -= nOffset * DequantBmm::Align(k_, BMM_BLOCK_NUM);
359	}	378	}
360	}	379	}
361	offsetBias_ -= nOffset;	380	offsetBias_ -= nOffset;
	@@ -374,8 +393,7 @@ protected:
374	if constexpr (bTrans) {	393	if constexpr (bTrans) {
375	offsetB_ += DequantBmm::Align(baseN_, BMM_BLOCK_NUM) * K0_INT8;	394	offsetB_ += DequantBmm::Align(baseN_, BMM_BLOCK_NUM) * K0_INT8;
376	} else {	395	} else {
377	- offsetB_ +=	396	+ offsetB_ += baseN_ * DequantBmm::Align(k_, BMM_BLOCK_NUM);
378	- DequantBmm::Align(baseN_, K0_INT8) * DequantBmm::Align(k_, BMM_BLOCK_NUM);
379	}	397	}
380	}	398	}
381		399
	@@ -638,4 +656,4 @@ protected:
638	};	656	};
639	} // namespace AscendC	657	} // namespace AscendC
640		658
641	-#endif // QUANT_BATCH_MATMUL_V3_PERTOKEN_OPT_H	659	+#endif // QUANT_BATCH_MATMUL_V3_PERTOKEN_OPT_H

Mmatmul/quant_batch_matmul_v3/op_kernel/quant_batch_matmul_v3_update.h+4-2

	@@ -60,7 +60,9 @@ __aicore__ inline void QuantBatchMatmulV3Update::Init(const TCubeTiling *mmTilin
60	}	60	}
61	if constexpr (bTrans) { // (n, k)	61	if constexpr (bTrans) { // (n, k)
62	info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, DequantBmm::GetC0Size<x2Type>());	62	info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, DequantBmm::GetC0Size<x2Type>());
63	- } else { // (k, n)	63	+ } else if constexpr (DequantBmm::GetFormat(x2Format) == CubeFormat::NZ) { // (n1,k1,k0,n0), k0=16
		64	+ info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, BMM_BLOCK_NUM);
		65	+ } else { // (k, n) ND
64	info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, BMM_BLOCK_NUM);	66	info_.alignedKbSize = DequantBmm::Align(mmTiling_->Kb, BMM_BLOCK_NUM);
65	}	67	}
66	}	68	}
	@@ -140,4 +142,4 @@ __aicore__ inline void QuantBatchMatmulV3Update::CalcGMOffset(QBmmBaseBlockArgs
140	offset.offsetPertoken = mOffset;	142	offset.offsetPertoken = mOffset;
141	}	143	}
142	} // namespace AscendC	144	} // namespace AscendC
143	-#endif // QUANT_BATCH_MATMUL_V3_UPDATE_H	145	+#endif // QUANT_BATCH_MATMUL_V3_UPDATE_H

Mmatmul/quant_batch_matmul_v3/tests/ut/op_host/test_quant_batch_matmul_v3.csv+1-1

	@@ -85,7 +85,7 @@ Ascend910B4,QUANTMM910B4/TestQuantBatchMatmulV3Tiling.generalTest/15,quant_batch
85	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/60,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,24,8192,1280,0,1,1,1,0,1,INT8,INT8,FLOAT,FLOAT,FLOAT,FLOAT16,ND,NZ,TRUE,20,26,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 24 64 0 32 64 0 0 0 0 0 0 0 20 24 1280 8192 8192 32 64 8192 32 64 512 16 4 1 1 1 32768 1 0 393216 8192 0 1 1 1 1 16 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,	85	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/60,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,24,8192,1280,0,1,1,1,0,1,INT8,INT8,FLOAT,FLOAT,FLOAT,FLOAT16,ND,NZ,TRUE,20,26,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 24 64 0 32 64 0 0 0 0 0 0 0 20 24 1280 8192 8192 32 64 8192 32 64 512 16 4 1 1 1 32768 1 0 393216 8192 0 1 1 1 1 16 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,
86	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/61,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,24,8192,1280,0,1,1,1,1,1,INT8,INT8,FLOAT,FLOAT,FLOAT,FLOAT16,ND,NZ,TRUE,20,27,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 24 64 0 32 64 0 0 0 0 0 0 0 20 24 1280 8192 8192 32 64 8192 32 64 512 16 4 1 1 1 32768 1 0 393216 8192 0 1 1 1 1 16 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,	86	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/61,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,24,8192,1280,0,1,1,1,1,1,INT8,INT8,FLOAT,FLOAT,FLOAT,FLOAT16,ND,NZ,TRUE,20,27,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 24 64 0 32 64 0 0 0 0 0 0 0 20 24 1280 8192 8192 32 64 8192 32 64 512 16 4 1 1 1 32768 1 0 393216 8192 0 1 1 1 1 16 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,
87	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/62,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,1024,11264,1664,0,1,1,0,1,1,INT8,INT8,BF16,FLOAT,INT32,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 31 192 47616 0 0 3 0 0 0 0 0 0 24 1024 1664 11264 11264 128 192 11264 128 192 128 8 8 1 1 1 0 0 0 295936 131072 0 1 1 1 1 4 4 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 8 9 2 1 0 0 0 0 0 0,FALSE,	87	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/62,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,1024,11264,1664,0,1,1,0,1,1,INT8,INT8,BF16,FLOAT,INT32,BF16,ND,NZ,TRUE,24,21,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 31 192 47616 0 0 3 0 0 0 0 0 0 24 1024 1664 11264 11264 128 192 11264 128 192 128 8 8 1 1 1 0 0 0 295936 131072 0 1 1 1 1 4 4 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 8 9 2 1 0 0 0 0 0 0,FALSE,
88	-Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/63,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,1024,11264,1664,0,1,1,1,0,1,INT8,INT8,BF16,FLOAT,INT32,BF16,ND,NZ,TRUE,24,22,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 45 128 46080 0 0 3 0 0 0 0 0 0 24 1024 1664 11264 11264 256 128 11264 256 128 128 4 12 1 1 1 0 0 0 197632 131072 0 1 1 1 1 2 6 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 4 13 0 1 0 0 0 0 0 0,FALSE,	88	+Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/63,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,1024,11264,1664,0,1,1,1,0,1,INT8,INT8,BF16,FLOAT,INT32,BF16,ND,NZ,TRUE,24,22,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 182 32 46592 0 0 3 0 0 0 0 0 0 24 1024 1664 11264 11264 256 32 11264 256 32 128 4 4 1 1 1 0 0 0 197632 131072 0 1 1 1 1 2 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 4 52 0 1 0 0 0 0 0 0,FALSE,
89	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/64,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,1024,11264,1664,0,1,1,1,1,1,INT8,INT8,BF16,FLOAT,INT32,BF16,ND,ND,TRUE,24,23,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 45 128 46080 0 0 3 0 0 0 0 0 0 24 1024 1664 11264 11264 256 128 11264 256 128 128 4 12 1 1 1 0 0 0 197248 81920 0 1 1 1 1 2 6 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 4 13 0 1 0 0 0 0 0 0,FALSE,	89	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/64,quant_batch_matmul_v3_nz,fanhua,,,2,2,2,0,0,0,1024,11264,1664,0,1,1,1,1,1,INT8,INT8,BF16,FLOAT,INT32,BF16,ND,ND,TRUE,24,23,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 45 128 46080 0 0 3 0 0 0 0 0 0 24 1024 1664 11264 11264 256 128 11264 256 128 128 4 12 1 1 1 0 0 0 197248 81920 0 1 1 1 1 2 6 0 0 2 2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 4 13 0 1 0 0 0 0 0 0,FALSE,
90	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/65,quant_batch_matmul_v3_nz,fanhua,,,3,3,3,1,1,1,32,6912,11264,0,1,1,0,0,1,INT8,INT8,BF16,FLOAT,BF16,BF16,ND,NZ,TRUE,22,24,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 14 512 0 32 512 27 0 0 0 0 0 0 22 32 11264 6912 6912 32 512 6912 32 512 64 108 4 1 1 1 65536 1 0 352256 65536 0 1 1 1 1 108 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,	90	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/65,quant_batch_matmul_v3_nz,fanhua,,,3,3,3,1,1,1,32,6912,11264,0,1,1,0,0,1,INT8,INT8,BF16,FLOAT,BF16,BF16,ND,NZ,TRUE,22,24,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 14 512 0 32 512 27 0 0 0 0 0 0 22 32 11264 6912 6912 32 512 6912 32 512 64 108 4 1 1 1 65536 1 0 352256 65536 0 1 1 1 1 108 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,
91	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/66,quant_batch_matmul_v3_nz,fanhua,,,3,3,3,1,1,1,32,6912,11264,0,1,1,1,0,1,INT8,INT8,BF16,FLOAT,BF16,BF16,ND,ND,TRUE,22,26,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 14 512 0 32 512 27 0 0 0 0 0 0 22 32 11264 6912 6912 32 512 6912 32 512 64 108 4 1 1 1 65536 1 0 352256 65536 0 1 1 1 1 108 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,	91	Ascend910B2,QUANTMM910B/TestQuantBatchMatmulV3Tiling.generalTest/66,quant_batch_matmul_v3_nz,fanhua,,,3,3,3,1,1,1,32,6912,11264,0,1,1,1,0,1,INT8,INT8,BF16,FLOAT,BF16,BF16,ND,ND,TRUE,22,26,1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 14 512 0 32 512 27 0 0 0 0 0 0 22 32 11264 6912 6912 32 512 6912 32 512 64 108 4 1 1 1 65536 1 0 352256 65536 0 1 1 1 1 108 2 0 0 2 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,FALSE,