Merged
fix issue #13356
liuyun_njAdd on 20 days ago
fix issue #13356
Merged
Merged fromliuyun_nj/ops-nn_1masterintocann/ops-nnmaster
Total: 9 files changed+437-146
| @@ -97,45 +97,47 @@ enum class DATA_TYPE : uint32_t { FP16 = 1, FP32 = 2, BF16 = 3 }; | |||
| 97 | * | 97 | * |
| 98 | * UB occupancy composition | 98 | * UB occupancy composition |
| 99 | * 1. NORMAL CASE (COL=LAST_DIM, ROW=?) | 99 | * 1. NORMAL CASE (COL=LAST_DIM, ROW=?) |
| 100 | - * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL*BN + | 100 | + * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL + |
| 101 | - * DT*COUNT(BIAS_BROADCAST?BIAS:NULL)*COL*BN + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + | 101 | + * DT*COUNT(BIAS_BROADCAST?BIAS:NULL)*COL + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + |
| 102 | * DT_FLOAT*COUNT(x_buf,y_buf,z_buf)*ROW*COL + DT_FLOAT*64 + UB_RESERVED UB = DT*(3+(ENABLE_X_OUT?1:0))*ROW*LAST_DIM*BN | 102 | * DT_FLOAT*COUNT(x_buf,y_buf,z_buf)*ROW*COL + DT_FLOAT*64 + UB_RESERVED UB = DT*(3+(ENABLE_X_OUT?1:0))*ROW*LAST_DIM*BN |
| 103 | - * + GAMMADT*2*LAST_DIM*BN + 4*ROW*2*BN + 4*3*ROW*LAST_DIM + 4*64 + UB_RESERVED UB = | 103 | + * + GAMMADT*2*LAST_DIM + 4*ROW*2*BN + 4*3*ROW*LAST_DIM + 4*64 + UB_RESERVED UB = |
| 104 | - * ROW*DT*(3+(ENABLE_X_OUT?1:0))*LAST_DIM*BN + GAMMADT*2*LAST_DIM*BN + ROW*4*2*BN + ROW*4*3*LAST_DIM + 4*64 + | 104 | + * ROW*DT*(3+(ENABLE_X_OUT?1:0))*LAST_DIM*BN + GAMMADT*2*LAST_DIM + ROW*4*2*BN + ROW*4*3*LAST_DIM + 4*64 + |
| 105 | - * UB_RESERVED UB = ROW*(DT*(3+(ENABLE_X_OUT?1:0))*LAST_DIM*BN + 4*2*BN + 4*3*LAST_DIM) + GAMMADT*2*LAST_DIM*BN + 4*64 + | 105 | + * UB_RESERVED UB = ROW*(DT*(3+(ENABLE_X_OUT?1:0))*LAST_DIM*BN + 4*2*BN + 4*3*LAST_DIM) + GAMMADT*2*LAST_DIM + 4*64 + |
| 106 | - * UB_RESERVED ROW = (UB - (GAMMADT*2*LAST_DIM*BN + 4*64 + UB_RESERVED)) / (DT*(3+(ENABLE_X_OUT?1:0))*LAST_DIM*BN + | 106 | + * UB_RESERVED ROW = (UB - (GAMMADT*2*LAST_DIM + 4*64 + UB_RESERVED)) / (DT*(3+(ENABLE_X_OUT?1:0))*LAST_DIM*BN + |
| 107 | * 4*2*BN + 4*3*LAST_DIM) | 107 | * 4*2*BN + 4*3*LAST_DIM) |
| 108 | * 2. NORMAL CASE WITH SINGLE ROW (ROW=1, COL=LAST_DIM) | 108 | * 2. NORMAL CASE WITH SINGLE ROW (ROW=1, COL=LAST_DIM) |
| 109 | - * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL*BN + | 109 | + * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL + |
| 110 | * DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + DT_FLOAT*COUNT(x_buf,y_buf)*COL + DT_FLOAT*64 + UB_RESERVED UB = | 110 | * DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + DT_FLOAT*COUNT(x_buf,y_buf)*COL + DT_FLOAT*64 + UB_RESERVED UB = |
| 111 | - * DT*(3+(ENABLE_X_OUT?1:0))*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM*BN + DT_FLOAT*1*2*BN + DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 | 111 | + * DT*(3+(ENABLE_X_OUT?1:0))*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM + DT_FLOAT*1*2*BN + DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 |
| 112 | - * + UB_RESERVED UB >? DT*(3+(ENABLE_X_OUT?1:0))*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM*BN + DT_FLOAT*1*2*BN + | 112 | + * + UB_RESERVED UB >? DT*(3+(ENABLE_X_OUT?1:0))*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM + DT_FLOAT*1*2*BN + |
| 113 | * DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 + UB_RESERVED | 113 | * DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 + UB_RESERVED |
| 114 | * 3. NORMAL CASE WITH SINGLE ROW EXT SHARE X/Y VECIN/OUT QUEUE (ROW=1, COL=LAST_DIM) | 114 | * 3. NORMAL CASE WITH SINGLE ROW EXT SHARE X/Y VECIN/OUT QUEUE (ROW=1, COL=LAST_DIM) |
| 115 | - * UB = DT*COUNT(X1,Y)*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL*BN + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + | 115 | + * UB = DT*COUNT(X1,Y)*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + |
| 116 | - * DT_FLOAT*COUNT(x_buf,y_buf)*COL + DT_FLOAT*64 + UB_RESERVED UB = DT*(2)*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM*BN + | 116 | + * DT_FLOAT*COUNT(x_buf,y_buf)*COL + DT_FLOAT*64 + UB_RESERVED UB = DT*(2)*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM + |
| 117 | - * DT_FLOAT*1*2*BN + DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 + UB_RESERVED UB >? DT*(2)*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM*BN | 117 | + * DT_FLOAT*1*2*BN + DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 + UB_RESERVED UB >? DT*(2)*1*LAST_DIM*BN + GAMMADT*2*LAST_DIM |
| 118 | * + DT_FLOAT*1*2*BN + DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 + UB_RESERVED | 118 | * + DT_FLOAT*1*2*BN + DT_FLOAT*2*LAST_DIM + DT_FLOAT*64 + UB_RESERVED |
| 119 | * 4. SLICE REDUCE AXIS WITH FULL LOAD GAMMA/BETA/X (ROW=1, COL=?) | 119 | * 4. SLICE REDUCE AXIS WITH FULL LOAD GAMMA/BETA/X (ROW=1, COL=?) |
| 120 | - * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*LAST_DIM*BN + | 120 | + * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*LAST_DIM + |
| 121 | - * DT*COUNT(BIAS_BROADCAST?BIAS:NULL)*LAST_DIM*BN + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + DT_FLOAT*COUNT(x_buf)*LAST_DIM + | 121 | + * DT*COUNT(BIAS_BROADCAST?BIAS:NULL)*LAST_DIM + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + DT_FLOAT*COUNT(x_buf)*LAST_DIM + |
| 122 | * DT_FLOAT*COUNT(y_buf,z_buf)*COL + DT_FLOAT*64 + DT_FLOAT*COUNT(mean_buf,rstd_buf) + UB_RESERVED UB = | 122 | * DT_FLOAT*COUNT(y_buf,z_buf)*COL + DT_FLOAT*64 + DT_FLOAT*COUNT(mean_buf,rstd_buf) + UB_RESERVED UB = |
| 123 | - * DT*(3+(ENABLE_X_OUT?1:0))*1*COL*BN + GAMMADT*2*LAST_DIM*BN + DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + DT_FLOAT*2*COL + | 123 | + * DT*(3+(ENABLE_X_OUT?1:0))*1*COL*BN + GAMMADT*2*LAST_DIM + DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + DT_FLOAT*2*COL + |
| 124 | - * DT_FLOAT*64 + DT_FLOAT*2 + UB_RESERVED UB = COL*DT*(3+(ENABLE_X_OUT?1:0))*1*BN + GAMMADT*2*LAST_DIM*BN + | 124 | + * DT_FLOAT*64 + DT_FLOAT*2 + UB_RESERVED UB = COL*DT*(3+(ENABLE_X_OUT?1:0))*1*BN + GAMMADT*2*LAST_DIM + |
| 125 | * DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + COL*DT_FLOAT*2 + DT_FLOAT*64 + DT_FLOAT*2 + UB_RESERVED UB = | 125 | * DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + COL*DT_FLOAT*2 + DT_FLOAT*64 + DT_FLOAT*2 + UB_RESERVED UB = |
| 126 | - * COL*(DT*(3+(ENABLE_X_OUT?1:0))*1*BN + DT_FLOAT*2) + GAMMADT*2*LAST_DIM*BN + DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + | 126 | + * COL*(DT*(3+(ENABLE_X_OUT?1:0))*1*BN + DT_FLOAT*2) + GAMMADT*2*LAST_DIM + DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + |
| 127 | - * DT_FLOAT*64 + DT_FLOAT*2 + UB_RESERVED COL = (UB - (GAMMADT*2*LAST_DIM*BN + DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + | 127 | + * DT_FLOAT*64 + DT_FLOAT*2 + UB_RESERVED COL = (UB - (GAMMADT*2*LAST_DIM + DT_FLOAT*1*2*BN + DT_FLOAT*1*LAST_DIM + |
| 128 | * DT_FLOAT*64 + DT_FLOAT*2 + DT_FLOAT*64 + UB_RESERVED)) / (DT*(3+(ENABLE_X_OUT?1:0))*1*BN + DT_FLOAT*2) | 128 | * DT_FLOAT*64 + DT_FLOAT*2 + DT_FLOAT*64 + UB_RESERVED)) / (DT*(3+(ENABLE_X_OUT?1:0))*1*BN + DT_FLOAT*2) |
| 129 | * 5. SLICE EXT (ROW=1, COL=?) | 129 | * 5. SLICE EXT (ROW=1, COL=?) |
| 130 | - * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL*BN + | 130 | + * UB = DT*COUNT(X1,X2,Y,(ENABLE_X_OUT?X:NULL))*ROW*COL*BN + GAMMADT*COUNT(Gamma,Beta)*COL + |
| 131 | - * DT*COUNT(BIAS_BROADCAST?BIAS:NULL)*COL*BN + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + DT_FLOAT*COUNT(x_buf,y_buf,z_buf)*COL | 131 | + * DT*COUNT(BIAS_BROADCAST?BIAS:NULL)*COL + DT_FLOAT*ROW*COUNT(Mean,Rstd)*BN + DT_FLOAT*COUNT(x_buf,y_buf,z_buf)*COL |
| 132 | - * + DT_FLOAT*64 + UB_RESERVED UB = DT*(3+(ENABLE_X_OUT?1:0))*1*COL*BN + GAMMADT*2*COL*BN + | 132 | + * + DT_FLOAT*64 + UB_RESERVED UB = DT*(3+(ENABLE_X_OUT?1:0))*1*COL*BN + GAMMADT*2*COL + |
| 133 | - * DT*(BIAS_BROADCAST?1:0)*COL*BN + DT_FLOAT*1*2*BN + DT_FLOAT*3*COL + DT_FLOAT*64 + UB_RESERVED UB = | 133 | + * DT*(BIAS_BROADCAST?1:0)*COL + DT_FLOAT*1*2*BN + DT_FLOAT*3*COL + DT_FLOAT*64 + UB_RESERVED UB = |
| 134 | - * COL*DT*(3+(ENABLE_X_OUT?1:0))*1*BN + COL*(GAMMADT*2*BN + DT*(BIAS_BROADCAST?1:0)*BN) + DT_FLOAT*1*2*BN + | 134 | + * COL*DT*(3+(ENABLE_X_OUT?1:0))*1*BN + COL*(GAMMADT*2 + DT*(BIAS_BROADCAST?1:0)) + DT_FLOAT*1*2*BN + |
| 135 | - * COL*DT_FLOAT*3 + DT_FLOAT*64 + UB_RESERVED UB = COL*(DT*(3+(ENABLE_X_OUT?1:0))*1*BN + GAMMADT*(2*BN) + | 135 | + * COL*DT_FLOAT*3 + DT_FLOAT*64 + UB_RESERVED UB = COL*(DT*(3+(ENABLE_X_OUT?1:0))*1*BN + GAMMADT*2 + |
| 136 | - * DT*(BIAS_BROADCAST?1:0)*BN + DT_FLOAT*3) + DT_FLOAT*1*2*BN + DT_FLOAT*64 + UB_RESERVED COL = (UB - (DT_FLOAT*1*2*BN + | 136 | + * DT*(BIAS_BROADCAST?1:0) + DT_FLOAT*3) + DT_FLOAT*1*2*BN + DT_FLOAT*64 + UB_RESERVED COL = (UB - (DT_FLOAT*1*2*BN + |
| 137 | - * DT_FLOAT*64 + UB_RESERVED)) / (DT*(3+(ENABLE_X_OUT?1:0))*1*BN + GAMMADT*2*BN + DT*(BIAS_BROADCAST?1:0)*BN + | 137 | + * DT_FLOAT*64 + UB_RESERVED)) / (DT*(3+(ENABLE_X_OUT?1:0))*1*BN + GAMMADT*2 + DT*(BIAS_BROADCAST?1:0) + |
| 138 | * DT_FLOAT*3) | 138 | * DT_FLOAT*3) |
| 139 | + * 注:gamma/beta/bias 为常驻参数,在 Process() 起始处加载一次并全核复用,不参与 | ||
| 140 | + * CopyIn->Compute->CopyOut 轮转,UB 预算中其大小不乘 bufferNum(双缓冲对它们无意义)。 | ||
| 139 | */ | 141 | */ |
| 140 | inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataType, ge::DataType betagammaDataType, | 142 | inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataType, ge::DataType betagammaDataType, |
| 141 | int32_t bufferNum, int64_t numCol, bool enableXOut, enum BIAS_TYPE biasType, | 143 | int32_t bufferNum, int64_t numCol, bool enableXOut, enum BIAS_TYPE biasType, |
| @@ -166,7 +168,7 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 166 | // Tensor(n_dim != 1): x1(dtype)/x2(dtype)/y(dtype)/x_que(dtype)/x(fp32)/z(fp32)/y(fp32) | 168 | // Tensor(n_dim != 1): x1(dtype)/x2(dtype)/y(dtype)/x_que(dtype)/x(fp32)/z(fp32)/y(fp32) |
| 167 | // Tensor(n_dim == 1): beta(float32)/gamma(float32)/bias(dtype) | 169 | // Tensor(n_dim == 1): beta(float32)/gamma(float32)/bias(dtype) |
| 168 | // other Tensor: transposeSrcBuf(int16)/transposeDstBuf(int16)/orBuf(int16) | 170 | // other Tensor: transposeSrcBuf(int16)/transposeDstBuf(int16)/orBuf(int16) |
| 169 | - auto betaGammaBufSize = sizeof(float) * 2 * numColAligned * bufferNum; | 171 | + auto betaGammaBufSize = sizeof(float) * 2 * numColAligned; |
| 170 | auto preRowInputOutputBufSize = dtSize * 3 * numColAligned * bufferNum; // x1/x2/y | 172 | auto preRowInputOutputBufSize = dtSize * 3 * numColAligned * bufferNum; // x1/x2/y |
| 171 | auto preRowTmpBufSize = sizeof(float) * 2 * numColAligned; // x_buf/z_buf | 173 | auto preRowTmpBufSize = sizeof(float) * 2 * numColAligned; // x_buf/z_buf |
| 172 | auto preRowReduceBufSize = sizeof(float) * 64; // y_buf | 174 | auto preRowReduceBufSize = sizeof(float) * 64; // y_buf |
| @@ -190,8 +192,8 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 190 | if (isSpecialReduceTiling) { | 192 | if (isSpecialReduceTiling) { |
| 191 | // Tensor(n_dim != 1): x1(dtype)/x2(dtype)/y(dtype)/mean(fp32)/rstd(fp32)/x_que(dtype)/x(fp32)/z(fp32)/y(fp32) | 193 | // Tensor(n_dim != 1): x1(dtype)/x2(dtype)/y(dtype)/mean(fp32)/rstd(fp32)/x_que(dtype)/x(fp32)/z(fp32)/y(fp32) |
| 192 | // Tensor(n_dim == 1): beta(float32)/gamma(float32)/bias(dtype) | 194 | // Tensor(n_dim == 1): beta(float32)/gamma(float32)/bias(dtype) |
| 193 | - auto betaGammaBufSize = sizeof(float) * 2 * numColAligned * bufferNum; | 195 | + auto betaGammaBufSize = sizeof(float) * 2 * numColAligned; |
| 194 | - auto biasBufSize = dtSize * biasNum * numColAligned * bufferNum; | 196 | + auto biasBufSize = dtSize * biasNum * numColAligned; |
| 195 | auto preRowInputOutputBufSize = static_cast<unsigned long>(dtSize) * 3U * | 197 | auto preRowInputOutputBufSize = static_cast<unsigned long>(dtSize) * 3U * |
| 196 | static_cast<unsigned long>(numColAligned) * | 198 | static_cast<unsigned long>(numColAligned) * |
| 197 | static_cast<unsigned long>(bufferNum) + | 199 | static_cast<unsigned long>(bufferNum) + |
| @@ -210,19 +212,17 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 210 | } | 212 | } |
| 211 | } | 213 | } |
| 212 | // try 910 normal tiling case: | 214 | // try 910 normal tiling case: |
| 213 | - auto tmpRow = static_cast<double>(static_cast<long>(maxUbSize) - | 215 | + auto tmpRow = static_cast<double>( |
| 214 | - (static_cast<long>(betagammaDtSize) * 2L * static_cast<long>(numColAligned) * | 216 | + static_cast<long>(maxUbSize) - |
| 215 | - static_cast<long>(bufferNum) + | 217 | + (static_cast<long>(betagammaDtSize) * 2L * static_cast<long>(numColAligned) + |
| 216 | - static_cast<long>(dtSize) * static_cast<long>(biasNum) * | 218 | + static_cast<long>(dtSize) * static_cast<long>(biasNum) * static_cast<long>(numColAligned) + |
| 217 | - static_cast<long>(numColAligned) * static_cast<long>(bufferNum) + | 219 | + 4L * 64L + static_cast<long>(UB_RESERVED_BYTE))) / |
| 218 | - 4L * 64L + static_cast<long>(UB_RESERVED_BYTE))) / | ||
| 219 | static_cast<double>(dtSize * (3 + additionalOutputNum) * numColAligned * bufferNum + | 220 | static_cast<double>(dtSize * (3 + additionalOutputNum) * numColAligned * bufferNum + |
| 220 | 4 * 2 * bufferNum + 4 * 3 * numColAligned); | 221 | 4 * 2 * bufferNum + 4 * 3 * numColAligned); |
| 221 | // try normal tiling special case, 2 is fp16 or bf16 dtype size | 222 | // try normal tiling special case, 2 is fp16 or bf16 dtype size |
| 222 | bool isSpecialTiling = (!is310P) && (isBroadcastBias) && isAllInputFP16; | 223 | bool isSpecialTiling = (!is310P) && (isBroadcastBias) && isAllInputFP16; |
| 223 | // beta, gamma, bias on ub, count(beta, gamma) = 2, mul(4, 64) required by mean and rstd | 224 | // beta, gamma, bias on ub, count(beta, gamma) = 2, mul(4, 64) required by mean and rstd |
| 224 | - int64_t ubCacheSize = sizeof(float) * 2 * numColAligned * bufferNum + dtSize * numColAligned * bufferNum + 4 * 64 + | 225 | + int64_t ubCacheSize = sizeof(float) * 2 * numColAligned + dtSize * numColAligned + 4 * 64 + UB_RESERVED_BYTE; |
| 225 | - UB_RESERVED_BYTE; | ||
| 226 | // ubsize for one row compute, we have 3 TQue and 2 fp32 TBuf, and count(mean, rstd) = 2 TQues, which size is 4 | 226 | // ubsize for one row compute, we have 3 TQue and 2 fp32 TBuf, and count(mean, rstd) = 2 TQues, which size is 4 |
| 227 | // Bytes | 227 | // Bytes |
| 228 | int64_t preRowSize = dtSize * 3 * numColAligned * bufferNum + sizeof(float) * 2 * bufferNum + | 228 | int64_t preRowSize = dtSize * 3 * numColAligned * bufferNum + sizeof(float) * 2 * bufferNum + |
| @@ -245,9 +245,7 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 245 | int64_t singleRowQueSize = static_cast<int64_t>(dtSize) * (3L + static_cast<int64_t>(additionalOutputNum)) * 1L * | 245 | int64_t singleRowQueSize = static_cast<int64_t>(dtSize) * (3L + static_cast<int64_t>(additionalOutputNum)) * 1L * |
| 246 | static_cast<int64_t>(numCol) * static_cast<int64_t>(bufferNum); | 246 | static_cast<int64_t>(numCol) * static_cast<int64_t>(bufferNum); |
| 247 | // 2 tmp TBuf + 2 weight TBuf, fp32 dtsize is 4. | 247 | // 2 tmp TBuf + 2 weight TBuf, fp32 dtsize is 4. |
| 248 | - int64_t singleRowBufSize = (4L * 2L + | 248 | + int64_t singleRowBufSize = (4L * 2L + static_cast<int64_t>(betagammaDtSize) * 2L) * static_cast<int64_t>(numCol); |
| 249 | - static_cast<int64_t>(betagammaDtSize) * 2L * static_cast<int64_t>(bufferNum)) * | ||
| 250 | - static_cast<int64_t>(numCol); | ||
| 251 | // 64 for rep elements, fp32 dtsize is 4. | 249 | // 64 for rep elements, fp32 dtsize is 4. |
| 252 | int64_t singleRowConstSize = static_cast<int64_t>(4 * 1 * 2 * bufferNum + 4 * 64 + UB_RESERVED_BYTE); | 250 | int64_t singleRowConstSize = static_cast<int64_t>(4 * 1 * 2 * bufferNum + 4 * 64 + UB_RESERVED_BYTE); |
| 253 | uint64_t tmpUbSize = static_cast<uint64_t>(singleRowQueSize + singleRowBufSize + singleRowConstSize); | 251 | uint64_t tmpUbSize = static_cast<uint64_t>(singleRowQueSize + singleRowBufSize + singleRowConstSize); |
| @@ -261,7 +259,7 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 261 | singleRowQueSize = static_cast<int64_t>(dtSize) * OUTPUT_RSTD_INDEX * 1 * static_cast<int64_t>(numCol) * | 259 | singleRowQueSize = static_cast<int64_t>(dtSize) * OUTPUT_RSTD_INDEX * 1 * static_cast<int64_t>(numCol) * |
| 262 | static_cast<int64_t>(bufferNum); | 260 | static_cast<int64_t>(bufferNum); |
| 263 | // 2 tmp TBuf + 2 weight TBuf, fp32 dtsize is 4. | 261 | // 2 tmp TBuf + 2 weight TBuf, fp32 dtsize is 4. |
| 264 | - singleRowBufSize = static_cast<int64_t>(4 * 2 + betagammaDtSize * 2 * bufferNum) * static_cast<int64_t>(numCol); | 262 | + singleRowBufSize = static_cast<int64_t>(4 * 2 + betagammaDtSize * 2) * static_cast<int64_t>(numCol); |
| 265 | // 2 tmp TBuf + 2 weight TBuf, fp32 dtsize is 4, 64 is tmpBuffer size for compute. | 263 | // 2 tmp TBuf + 2 weight TBuf, fp32 dtsize is 4, 64 is tmpBuffer size for compute. |
| 266 | singleRowConstSize = static_cast<int64_t>(4 * 1 * 2 * bufferNum + 4 * 64 + UB_RESERVED_BYTE); | 264 | singleRowConstSize = static_cast<int64_t>(4 * 1 * 2 * bufferNum + 4 * 64 + UB_RESERVED_BYTE); |
| 267 | tmpUbSize = static_cast<uint64_t>(singleRowQueSize + singleRowBufSize + singleRowConstSize); | 265 | tmpUbSize = static_cast<uint64_t>(singleRowQueSize + singleRowBufSize + singleRowConstSize); |
| @@ -315,7 +313,7 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 315 | int64_t ubColNumSlice = static_cast<int64_t>(dtSize * (3 + additionalOutputNum) * bufferNum + 4 * 2); | 313 | int64_t ubColNumSlice = static_cast<int64_t>(dtSize * (3 + additionalOutputNum) * bufferNum + 4 * 2); |
| 316 | // 2 weight Buf, 1 bias Buf, fp32 dtsize is 4, rep element for fp 32 is 64. | 314 | // 2 weight Buf, 1 bias Buf, fp32 dtsize is 4, rep element for fp 32 is 64. |
| 317 | int64_t ubConstSlice = static_cast<int64_t>(betagammaDtSize) * 2 * static_cast<int64_t>(numCol) + | 315 | int64_t ubConstSlice = static_cast<int64_t>(betagammaDtSize) * 2 * static_cast<int64_t>(numCol) + |
| 318 | - static_cast<int64_t>(dtSize * biasNum * bufferNum) * static_cast<int64_t>(numCol) + | 316 | + static_cast<int64_t>(dtSize * biasNum) * static_cast<int64_t>(numCol) + |
| 319 | static_cast<int64_t>(dtSize * 2 * bufferNum) + 4 * static_cast<int64_t>(numCol) + | 317 | static_cast<int64_t>(dtSize * 2 * bufferNum) + 4 * static_cast<int64_t>(numCol) + |
| 320 | static_cast<int64_t>(4 * 64 + UB_RESERVED_BYTE); | 318 | static_cast<int64_t>(4 * 64 + UB_RESERVED_BYTE); |
| 321 | double tmpCol = static_cast<double>(static_cast<int64_t>(maxUbSize) - ubConstSlice) / | 319 | double tmpCol = static_cast<double>(static_cast<int64_t>(maxUbSize) - ubConstSlice) / |
| @@ -335,8 +333,7 @@ inline TILING_TYPE AddLayerNormTilingImpl(uint64_t maxUbSize, ge::DataType dataT | |||
| 335 | ubColNumSlice = static_cast<int64_t>(dtSize) * | 333 | ubColNumSlice = static_cast<int64_t>(dtSize) * |
| 336 | (static_cast<int64_t>(OUTPUT_X_INDEX) + static_cast<int64_t>(additionalOutputNum)) * | 334 | (static_cast<int64_t>(OUTPUT_X_INDEX) + static_cast<int64_t>(additionalOutputNum)) * |
| 337 | static_cast<int64_t>(bufferNum) + | 335 | static_cast<int64_t>(bufferNum) + |
| 338 | - static_cast<int64_t>(betagammaDtSize) * 2L * static_cast<int64_t>(bufferNum) + | 336 | + static_cast<int64_t>(betagammaDtSize) * 2L + static_cast<int64_t>(dtSize) * biasNum + |
| 339 | - static_cast<int64_t>(dtSize) * biasNum * static_cast<int64_t>(bufferNum) + | ||
| 340 | static_cast<int64_t>(FOUR_NUM) * static_cast<int64_t>(OUTPUT_X_INDEX); | 337 | static_cast<int64_t>(FOUR_NUM) * static_cast<int64_t>(OUTPUT_X_INDEX); |
| 341 | tmpCol = static_cast<double>(static_cast<int64_t>(maxUbSize) - ubConstSlice) / static_cast<double>(ubColNumSlice); | 338 | tmpCol = static_cast<double>(static_cast<int64_t>(maxUbSize) - ubConstSlice) / static_cast<double>(ubColNumSlice); |
| 342 | colPerTime = floor(tmpCol); | 339 | colPerTime = floor(tmpCol); |
| @@ -123,7 +123,7 @@ aclnnStatus aclnnGemmaRmsNorm( | |||
| 123 | <td>rstdOut(aclTensor*)</td> | 123 | <td>rstdOut(aclTensor*)</td> |
| 124 | <td>输出</td> | 124 | <td>输出</td> |
| 125 | <td>表示x的标准差倒数,用于归一化操作,对应公式中的`Rms(x)`的倒数。</td> | 125 | <td>表示x的标准差倒数,用于归一化操作,对应公式中的`Rms(x)`的倒数。</td> |
| 126 | - <td><ul><li>支持空Tensor。</li><li>shape与入参`x`的shape前几维保持一致,前几维指`x`的维度减去`gamma`的维度,表示不需要norm的维度。</li></ul></td> | 126 | + <td><ul><li>支持空Tensor。</li><li>shape与入参`x`的shape前几维保持一致,前几维指`x`的维度减去`gamma`的维度,表示不需要norm的维度。</li><li>元素数必须等于归一化行数,即`x`的元素数除以`gamma`的元素数。</li></ul></td> |
| 127 | <td>FLOAT32</td> | 127 | <td>FLOAT32</td> |
| 128 | <td>ND</td> | 128 | <td>ND</td> |
| 129 | <td>1-8</td> | 129 | <td>1-8</td> |
| @@ -116,6 +116,183 @@ TEST_F(GemmaRmsNormTiling, gemma_rms_norm_tiling_0001) | |||
| 116 | // dlog_setlevel(0, 3, 0); | 116 | // dlog_setlevel(0, 3, 0); |
| 117 | } | 117 | } |
| 118 | 118 | ||
| 119 | +TEST_F(GemmaRmsNormTiling, gemma_rms_norm_tiling_rstd_numel_mismatch_failed) | ||
| 120 | +{ | ||
| 121 | + // 非 regbase 平台(NpuArch=2201, 910B):rstdOut 元素数必须等于归一化行数 x.numel / gamma.numel, | ||
| 122 | + // 否则内核按行数写 rstd 会产生越界写(issue #7333)。此处声明 1 个元素而 rows=8,tiling 应失败。 | ||
| 123 | + gert::StorageShape input_shape = {{8, 128}, {8, 128}}; | ||
| 124 | + gert::StorageShape gamma_shape = {{128}, {128}}; | ||
| 125 | + gert::StorageShape out_shape = {{8, 128}, {8, 128}}; | ||
| 126 | + gert::StorageShape rstd_shape = {{1, 1}, {1, 1}}; | ||
| 127 | + | ||
| 128 | + string compile_info_string = R"({ | ||
| 129 | + "hardware_info": {"BT_SIZE": 0, "load3d_constraints": "1", | ||
| 130 | + "Intrinsic_fix_pipe_l0c2out": false, "Intrinsic_data_move_l12ub": true, "Intrinsic_data_move_l0c2ub": true, "Intrinsic_data_move_out2l1_nd2nz": false, | ||
| 131 | + "UB_SIZE": 196608, "L2_SIZE": 33554432, "L1_SIZE": 524288, | ||
| 132 | + "L0A_SIZE": 65536, "L0B_SIZE": 65536, "L0C_SIZE": 131072, | ||
| 133 | + "CORE_NUM": 40} | ||
| 134 | + })"; | ||
| 135 | + map<string, string> soc_infos; | ||
| 136 | + map<string, string> aicore_spec; | ||
| 137 | + map<string, string> intrinsics; | ||
| 138 | + GetPlatFormInfos(compile_info_string.c_str(), soc_infos, aicore_spec, intrinsics); | ||
| 139 | + std::map<std::string, std::string> soc_version_infos = {{"Short_SoC_version", "Ascend910B"}, {"NpuArch", "2201"}}; | ||
| 140 | + | ||
| 141 | + // platform info | ||
| 142 | + fe::PlatFormInfos platform_info; | ||
| 143 | + platform_info.Init(); | ||
| 144 | + // compile info | ||
| 145 | + struct Tiling4RmsNormCompileInfo { | ||
| 146 | + uint32_t totalCoreNum = 40; | ||
| 147 | + uint64_t totalUbSize = 196608; | ||
| 148 | + platform_ascendc::SocVersion curSocVersion = platform_ascendc::SocVersion::ASCEND910B; | ||
| 149 | + }; | ||
| 150 | + Tiling4RmsNormCompileInfo compile_info; | ||
| 151 | + std::string op_type("GemmaRmsNorm"); | ||
| 152 | + ASSERT_NE(gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str()), nullptr); | ||
| 153 | + auto tiling_func = gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str())->tiling; | ||
| 154 | + auto tiling_parse_func = gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str())->tiling_parse; | ||
| 155 | + | ||
| 156 | + // tilingParseFunc simulate | ||
| 157 | + auto kernel_holder = gert::KernelRunContextFaker() | ||
| 158 | + .KernelIONum(2, 1) | ||
| 159 | + .Inputs({const_cast<char*>(compile_info_string.c_str()), | ||
| 160 | + reinterpret_cast<void*>(&platform_info)}) | ||
| 161 | + .Outputs({&compile_info}) | ||
| 162 | + .Build(); | ||
| 163 | + | ||
| 164 | + ASSERT_TRUE(kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->Init()); | ||
| 165 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("SoCInfo", soc_infos); | ||
| 166 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("AICoreSpec", aicore_spec); | ||
| 167 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetCoreNumByCoreType("AICore"); | ||
| 168 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("AICoreintrinsicDtypeMap", | ||
| 169 | + intrinsics); | ||
| 170 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("version", | ||
| 171 | + soc_version_infos); | ||
| 172 | + | ||
| 173 | + ASSERT_EQ(tiling_parse_func(kernel_holder.GetContext<gert::KernelContext>()), ge::GRAPH_SUCCESS); | ||
| 174 | + | ||
| 175 | + // tilingFunc simulate | ||
| 176 | + auto param = gert::TilingData::CreateCap(4096); | ||
| 177 | + auto workspace_size_holer = gert::ContinuousVector::Create<size_t>(4096); | ||
| 178 | + auto ws_size = reinterpret_cast<gert::ContinuousVector*>(workspace_size_holer.get()); | ||
| 179 | + ASSERT_NE(param, nullptr); | ||
| 180 | + compile_info.curSocVersion = platform_ascendc::SocVersion::ASCEND910B; | ||
| 181 | + auto holder = gert::TilingContextFaker() | ||
| 182 | + .NodeIoNum(2, 2) | ||
| 183 | + .IrInstanceNum({1, 1}) | ||
| 184 | + .InputShapes({&input_shape, &gamma_shape}) | ||
| 185 | + .OutputShapes({&out_shape, &rstd_shape}) | ||
| 186 | + .CompileInfo(&compile_info) | ||
| 187 | + .PlatformInfo(reinterpret_cast<char*>(&platform_info)) | ||
| 188 | + .NodeInputTd(0, ge::DT_FLOAT16, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 189 | + .NodeInputTd(1, ge::DT_FLOAT16, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 190 | + .NodeOutputTd(0, ge::DT_FLOAT16, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 191 | + .NodeOutputTd(1, ge::DT_FLOAT, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 192 | + .NodeAttrs({{"epsilon", Ops::NN::AnyValue::CreateFrom<float>(0.01)}}) | ||
| 193 | + .TilingData(param.get()) | ||
| 194 | + .Workspace(ws_size) | ||
| 195 | + .Build(); | ||
| 196 | + | ||
| 197 | + gert::TilingContext* tiling_context = holder.GetContext<gert::TilingContext>(); | ||
| 198 | + ASSERT_NE(tiling_context->GetPlatformInfo(), nullptr); | ||
| 199 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("SoCInfo", soc_infos); | ||
| 200 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("AICoreSpec", aicore_spec); | ||
| 201 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetCoreNumByCoreType("AICore"); | ||
| 202 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("AICoreintrinsicDtypeMap", intrinsics); | ||
| 203 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("version", soc_version_infos); | ||
| 204 | + | ||
| 205 | + EXPECT_EQ(tiling_func(tiling_context), ge::GRAPH_FAILED); | ||
| 206 | +} | ||
| 207 | + | ||
| 208 | +TEST_F(GemmaRmsNormTiling, gemma_rms_norm_tiling_rstd_numel_as_rows_success) | ||
| 209 | +{ | ||
| 210 | + // 正向对照:rstdOut 元素数等于归一化行数(8)时 tiling 正常,不被新增校验误伤。 | ||
| 211 | + gert::StorageShape input_shape = {{8, 128}, {8, 128}}; | ||
| 212 | + gert::StorageShape gamma_shape = {{128}, {128}}; | ||
| 213 | + gert::StorageShape out_shape = {{8, 128}, {8, 128}}; | ||
| 214 | + gert::StorageShape rstd_shape = {{8, 1}, {8, 1}}; | ||
| 215 | + | ||
| 216 | + string compile_info_string = R"({ | ||
| 217 | + "hardware_info": {"BT_SIZE": 0, "load3d_constraints": "1", | ||
| 218 | + "Intrinsic_fix_pipe_l0c2out": false, "Intrinsic_data_move_l12ub": true, "Intrinsic_data_move_l0c2ub": true, "Intrinsic_data_move_out2l1_nd2nz": false, | ||
| 219 | + "UB_SIZE": 196608, "L2_SIZE": 33554432, "L1_SIZE": 524288, | ||
| 220 | + "L0A_SIZE": 65536, "L0B_SIZE": 65536, "L0C_SIZE": 131072, | ||
| 221 | + "CORE_NUM": 40} | ||
| 222 | + })"; | ||
| 223 | + map<string, string> soc_infos; | ||
| 224 | + map<string, string> aicore_spec; | ||
| 225 | + map<string, string> intrinsics; | ||
| 226 | + GetPlatFormInfos(compile_info_string.c_str(), soc_infos, aicore_spec, intrinsics); | ||
| 227 | + std::map<std::string, std::string> soc_version_infos = {{"Short_SoC_version", "Ascend910B"}, {"NpuArch", "2201"}}; | ||
| 228 | + | ||
| 229 | + // platform info | ||
| 230 | + fe::PlatFormInfos platform_info; | ||
| 231 | + platform_info.Init(); | ||
| 232 | + // compile info | ||
| 233 | + struct Tiling4RmsNormCompileInfo { | ||
| 234 | + uint32_t totalCoreNum = 40; | ||
| 235 | + uint64_t totalUbSize = 196608; | ||
| 236 | + platform_ascendc::SocVersion curSocVersion = platform_ascendc::SocVersion::ASCEND910B; | ||
| 237 | + }; | ||
| 238 | + Tiling4RmsNormCompileInfo compile_info; | ||
| 239 | + std::string op_type("GemmaRmsNorm"); | ||
| 240 | + ASSERT_NE(gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str()), nullptr); | ||
| 241 | + auto tiling_func = gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str())->tiling; | ||
| 242 | + auto tiling_parse_func = gert::OpImplRegistry::GetInstance().GetOpImpl(op_type.c_str())->tiling_parse; | ||
| 243 | + | ||
| 244 | + // tilingParseFunc simulate | ||
| 245 | + auto kernel_holder = gert::KernelRunContextFaker() | ||
| 246 | + .KernelIONum(2, 1) | ||
| 247 | + .Inputs({const_cast<char*>(compile_info_string.c_str()), | ||
| 248 | + reinterpret_cast<void*>(&platform_info)}) | ||
| 249 | + .Outputs({&compile_info}) | ||
| 250 | + .Build(); | ||
| 251 | + | ||
| 252 | + ASSERT_TRUE(kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->Init()); | ||
| 253 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("SoCInfo", soc_infos); | ||
| 254 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("AICoreSpec", aicore_spec); | ||
| 255 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetCoreNumByCoreType("AICore"); | ||
| 256 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("AICoreintrinsicDtypeMap", | ||
| 257 | + intrinsics); | ||
| 258 | + kernel_holder.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("version", | ||
| 259 | + soc_version_infos); | ||
| 260 | + | ||
| 261 | + ASSERT_EQ(tiling_parse_func(kernel_holder.GetContext<gert::KernelContext>()), ge::GRAPH_SUCCESS); | ||
| 262 | + | ||
| 263 | + // tilingFunc simulate | ||
| 264 | + auto param = gert::TilingData::CreateCap(4096); | ||
| 265 | + auto workspace_size_holer = gert::ContinuousVector::Create<size_t>(4096); | ||
| 266 | + auto ws_size = reinterpret_cast<gert::ContinuousVector*>(workspace_size_holer.get()); | ||
| 267 | + ASSERT_NE(param, nullptr); | ||
| 268 | + compile_info.curSocVersion = platform_ascendc::SocVersion::ASCEND910B; | ||
| 269 | + auto holder = gert::TilingContextFaker() | ||
| 270 | + .NodeIoNum(2, 2) | ||
| 271 | + .IrInstanceNum({1, 1}) | ||
| 272 | + .InputShapes({&input_shape, &gamma_shape}) | ||
| 273 | + .OutputShapes({&out_shape, &rstd_shape}) | ||
| 274 | + .CompileInfo(&compile_info) | ||
| 275 | + .PlatformInfo(reinterpret_cast<char*>(&platform_info)) | ||
| 276 | + .NodeInputTd(0, ge::DT_FLOAT16, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 277 | + .NodeInputTd(1, ge::DT_FLOAT16, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 278 | + .NodeOutputTd(0, ge::DT_FLOAT16, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 279 | + .NodeOutputTd(1, ge::DT_FLOAT, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 280 | + .NodeAttrs({{"epsilon", Ops::NN::AnyValue::CreateFrom<float>(0.01)}}) | ||
| 281 | + .TilingData(param.get()) | ||
| 282 | + .Workspace(ws_size) | ||
| 283 | + .Build(); | ||
| 284 | + | ||
| 285 | + gert::TilingContext* tiling_context = holder.GetContext<gert::TilingContext>(); | ||
| 286 | + ASSERT_NE(tiling_context->GetPlatformInfo(), nullptr); | ||
| 287 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("SoCInfo", soc_infos); | ||
| 288 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("AICoreSpec", aicore_spec); | ||
| 289 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetCoreNumByCoreType("AICore"); | ||
| 290 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("AICoreintrinsicDtypeMap", intrinsics); | ||
| 291 | + holder.GetContext<gert::TilingContext>()->GetPlatformInfo()->SetPlatformRes("version", soc_version_infos); | ||
| 292 | + | ||
| 293 | + EXPECT_EQ(tiling_func(tiling_context), ge::GRAPH_SUCCESS); | ||
| 294 | +} | ||
| 295 | + | ||
| 119 | TEST_F(GemmaRmsNormTiling, gemma_rms_norm_tiling_0002) | 296 | TEST_F(GemmaRmsNormTiling, gemma_rms_norm_tiling_0002) |
| 120 | { | 297 | { |
| 121 | // dlog_setlevel(0,0,0); | 298 | // dlog_setlevel(0,0,0); |
| @@ -209,4 +386,4 @@ TEST_F(GemmaRmsNormTiling, gemma_rms_norm_tiling_0002) | |||
| 209 | auto tiling_key = tiling_context->GetTilingKey(); | 386 | auto tiling_key = tiling_context->GetTilingKey(); |
| 210 | ASSERT_EQ(tiling_key, 1); | 387 | ASSERT_EQ(tiling_key, 1); |
| 211 | // dlog_setlevel(0, 3, 0); | 388 | // dlog_setlevel(0, 3, 0); |
| 212 | -} | 389 | +} |
| @@ -108,7 +108,7 @@ aclnnStatus aclnnGroupNorm( | |||
| 108 | <td>gamma(aclTensor*)</td> | 108 | <td>gamma(aclTensor*)</td> |
| 109 | <td>输入</td> | 109 | <td>输入</td> |
| 110 | <td>表示进行归一化计算的缩放因子(权重),对应公式中的`γ`。</td> | 110 | <td>表示进行归一化计算的缩放因子(权重),对应公式中的`γ`。</td> |
| 111 | - <td><ul><li>支持空Tensor。</li><li>数据类型与self相同。</li><li>元素数量需与C相同。</li></ul></td> | 111 | + <td><ul><li>支持空Tensor。</li><li>可选输入:允许传空指针,为空时按γ=1代入计算。</li><li>数据类型与self相同。</li><li>元素数量需与C相同。</li></ul></td> |
| 112 | <td>FLOAT32、FLOAT16、BFLOAT16</td> | 112 | <td>FLOAT32、FLOAT16、BFLOAT16</td> |
| 113 | <td>ND</td> | 113 | <td>ND</td> |
| 114 | <td>1</td> | 114 | <td>1</td> |
| @@ -118,7 +118,7 @@ aclnnStatus aclnnGroupNorm( | |||
| 118 | <td>beta(aclTensor*)</td> | 118 | <td>beta(aclTensor*)</td> |
| 119 | <td>输入</td> | 119 | <td>输入</td> |
| 120 | <td>表示进行归一化计算的偏移量,对应公式中的`β`。</td> | 120 | <td>表示进行归一化计算的偏移量,对应公式中的`β`。</td> |
| 121 | - <td><ul><li>支持空Tensor。</li><li>数据类型与self相同。</li><li>元素数量需与C相同。</li></ul></td> | 121 | + <td><ul><li>支持空Tensor。</li><li>可选输入:允许传空指针,为空时按β=0代入计算。</li><li>数据类型与self相同。</li><li>元素数量需与C相同。</li></ul></td> |
| 122 | <td>FLOAT32、FLOAT16、BFLOAT16</td> | 122 | <td>FLOAT32、FLOAT16、BFLOAT16</td> |
| 123 | <td>ND</td> | 123 | <td>ND</td> |
| 124 | <td>1</td> | 124 | <td>1</td> |
| @@ -220,6 +220,35 @@ static bool CheckInputShape4RmsNorm(const gert::TilingContext* context) | |||
| 220 | return false; | 220 | return false; |
| 221 | } | 221 | } |
| 222 | } | 222 | } |
| 223 | + // 非regbase平台:rstd元素数必须等于归一化行数 x.numel / gamma.numel, | ||
| 224 | + // 否则内核按行数写rstd会产生越界写(aclnnGemmaRmsNorm #7333)。 | ||
| 225 | + if (!Ops::NN::OpTiling::IsRegbaseSocVersion(context)) { | ||
| 226 | + const gert::Shape& xStorageShape = xShape->GetStorageShape(); | ||
| 227 | + const gert::Shape& gammaStorageShape = gammaShape->GetStorageShape(); | ||
| 228 | + const gert::Shape& rstdStorageShape = rstdShape->GetStorageShape(); | ||
| 229 | + bool allShapeKnown = true; | ||
| 230 | + for (size_t i = 0; i < xStorageShape.GetDimNum(); i++) { | ||
| 231 | + allShapeKnown = allShapeKnown && xStorageShape.GetDim(i) >= 0; | ||
| 232 | + } | ||
| 233 | + for (size_t i = 0; i < gammaStorageShape.GetDimNum(); i++) { | ||
| 234 | + allShapeKnown = allShapeKnown && gammaStorageShape.GetDim(i) >= 0; | ||
| 235 | + } | ||
| 236 | + for (size_t i = 0; i < rstdStorageShape.GetDimNum(); i++) { | ||
| 237 | + allShapeKnown = allShapeKnown && rstdStorageShape.GetDim(i) >= 0; | ||
| 238 | + } | ||
| 239 | + if (allShapeKnown) { | ||
| 240 | + int64_t xNumel = xStorageShape.GetShapeSize(); | ||
| 241 | + int64_t gammaNumel = gammaStorageShape.GetShapeSize(); | ||
| 242 | + int64_t rstdNumel = rstdStorageShape.GetShapeSize(); | ||
| 243 | + int64_t rowNum = (gammaNumel != 0) ? xNumel / gammaNumel : 0; | ||
| 244 | + OP_TILING_CHECK(rstdNumel != rowNum, | ||
| 245 | + OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON( | ||
| 246 | + context->GetNodeName(), "rstd", ToString(rstdStorageShape).c_str(), | ||
| 247 | + "the element number of output rstd must be equal to the number of normalized rows " | ||
| 248 | + "(x.numel / gamma.numel)"), | ||
| 249 | + return false); | ||
| 250 | + } | ||
| 251 | + } | ||
| 223 | return true; | 252 | return true; |
| 224 | } | 253 | } |
| 225 | 254 | ||
| @@ -2006,4 +2006,24 @@ TEST(RmsNormTemplatesTiling, ExistingFallbackAndInvalidPlatform) | |||
| 2006 | optiling::RMSNORM_REGBASE_SPLIT); | 2006 | optiling::RMSNORM_REGBASE_SPLIT); |
| 2007 | EXPECT_EQ(RunTiling(1, 0, ge::DT_FLOAT16, ge::DT_FLOAT16, 0, "GemmaRmsNorm").status, ge::GRAPH_FAILED); | 2007 | EXPECT_EQ(RunTiling(1, 0, ge::DT_FLOAT16, ge::DT_FLOAT16, 0, "GemmaRmsNorm").status, ge::GRAPH_FAILED); |
| 2008 | } | 2008 | } |
| 2009 | + | ||
| 2010 | +TEST(RmsNormTemplatesTiling, NonRegbaseRstdNumelMustEqualRows) | ||
| 2011 | +{ | ||
| 2012 | + // 非 regbase 平台(910B,NpuArch=2201):rstdOut 元素数必须等于归一化行数 x.numel / gamma.numel, | ||
| 2013 | + // 否则内核按行数写 rstd 会产生越界写(issue #7333)。声明 1 个元素而 rows=8 时 tiling 应失败。 | ||
| 2014 | + auto bad = RunShapeTiling({{8, 128}, {8, 128}}, {{128}, {128}}, {{1, 1}, {1, 1}}, ge::DT_FLOAT16, ge::DT_FLOAT, 0, | ||
| 2015 | + "RmsNorm", 64, 253952, "Ascend910B", "2201"); | ||
| 2016 | + EXPECT_EQ(bad.status, ge::GRAPH_FAILED); | ||
| 2017 | + auto badGemma = RunShapeTiling({{8, 128}, {8, 128}}, {{128}, {128}}, {{1, 1}, {1, 1}}, ge::DT_FLOAT16, ge::DT_FLOAT, | ||
| 2018 | + 0, "GemmaRmsNorm", 64, 253952, "Ascend910B", "2201"); | ||
| 2019 | + EXPECT_EQ(badGemma.status, ge::GRAPH_FAILED); | ||
| 2020 | + | ||
| 2021 | + // 合法形态(rstdOut 元素数等于行数)不被新增校验误伤。 | ||
| 2022 | + auto good = RunShapeTiling({{8, 128}, {8, 128}}, {{128}, {128}}, {{8, 1}, {8, 1}}, ge::DT_FLOAT16, ge::DT_FLOAT, 0, | ||
| 2023 | + "RmsNorm", 64, 253952, "Ascend910B", "2201"); | ||
| 2024 | + EXPECT_EQ(good.status, ge::GRAPH_SUCCESS); | ||
| 2025 | + auto goodGemma = RunShapeTiling({{8, 128}, {8, 128}}, {{128}, {128}}, {{8, 1}, {8, 1}}, ge::DT_FLOAT16, | ||
| 2026 | + ge::DT_FLOAT, 0, "GemmaRmsNorm", 64, 253952, "Ascend910B", "2201"); | ||
| 2027 | + EXPECT_EQ(goodGemma.status, ge::GRAPH_SUCCESS); | ||
| 2028 | +} | ||
| 2009 | } // namespace | 2029 | } // namespace |
| @@ -181,7 +181,7 @@ aclnnStatus aclnnRmsNormQuant( | |||
| 181 | - <term>Atlas推理系列产品</term>、<term>Atlas 200I/500 A2推理产品</term>:入参`x`、`gamma`、`beta`、`scale`的数据类型仅支持FLOAT16;`scale`的shape为`[1]`;出参`y`的数据类型仅支持INT8。 | 181 | - <term>Atlas推理系列产品</term>、<term>Atlas 200I/500 A2推理产品</term>:入参`x`、`gamma`、`beta`、`scale`的数据类型仅支持FLOAT16;`scale`的shape为`[1]`;出参`y`的数据类型仅支持INT8。 |
| 182 | <!-- end id7 --> | 182 | <!-- end id7 --> |
| 183 | <!-- npu="A3,910b" id8 --> | 183 | <!-- npu="A3,910b" id8 --> |
| 184 | - - <term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>:入参`x`、`gamma`、`beta`、`scale`的数据类型仅支持FLOAT16,BFLOAT16;`offset`仅支持INT8;`scale`的shape为`[1]`;出参`y`的数据类型仅支持INT8、INT4。 | 184 | + - <term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>:入参`x`、`gamma`、`beta`、`scale`的数据类型仅支持FLOAT16,BFLOAT16;`offset`仅支持INT8;`scale`的shape为`[1]`;出参`y`的数据类型仅支持INT8、INT32(INT4量化结果通过INT32承载,此时`y`的最后一维为`x`最后一维的1/8,不支持直接以INT4数据类型声明输出)。 |
| 185 | <!-- end id8 --> | 185 | <!-- end id8 --> |
| 186 | 186 | ||
| 187 | <!-- npu="950" id15 --> | 187 | <!-- npu="950" id15 --> |
| @@ -267,12 +267,9 @@ aclnnStatus aclnnRmsNormQuant( | |||
| 267 | 267 | ||
| 268 | ## 约束说明 | 268 | ## 约束说明 |
| 269 | 269 | ||
| 270 | -<!-- npu="950" id16 --> | 270 | +<!-- npu="950,A3,910b,310p,310b" id16 --> |
| 271 | -- <term>Ascend 950PR&950DT系列产品</term>:空Tensor是shape中存在0的Tensor,不等同于空指针或标量。满足其余shape、数据类型约束时,支持`x`与`y`均为空Tensor,第一段接口返回0字节workspace,第二段接口不执行计算。必选参数仍须提供Tensor对象。 | 271 | +- <term>Ascend 950PR&950DT系列产品</term>、<term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas 200I/500 A2推理产品</term>:空Tensor是shape中存在0的Tensor,不等同于空指针或标量。满足其余shape、数据类型约束时,支持`x`与`y`均为空Tensor,第一段接口返回0字节workspace,第二段接口不执行计算。必选参数仍须提供Tensor对象。 |
| 272 | <!-- end id16 --> | 272 | <!-- end id16 --> |
| 273 | -<!-- npu="A3,910b,310p,310b" id17 --> | ||
| 274 | -- <term>Atlas A3系列产品</term>、<term>Atlas A2系列产品</term>、<term>Atlas推理系列产品</term>、<term>Atlas 200I/500 A2推理产品</term>:不支持空Tensor。 | ||
| 275 | -<!-- end id17 --> | ||
| 276 | 273 | ||
| 277 | - 当输入参数scale或offset的值为NaN,或者RmsNorm的计算结果(对应计算公式中的$quant\_in_i$)为NaN时,该接口的最终输出结果为0。 | 274 | - 当输入参数scale或offset的值为NaN,或者RmsNorm的计算结果(对应计算公式中的$quant\_in_i$)为NaN时,该接口的最终输出结果为0。 |
| 278 | 275 | ||
| @@ -285,6 +282,9 @@ aclnnStatus aclnnRmsNormQuant( | |||
| 285 | <!-- npu="950" id11 --> | 282 | <!-- npu="950" id11 --> |
| 286 | - <term>Ascend 950PR&950DT系列产品</term>:当`y`的数据类型为INT32时,`y`的最后一维必须是`x`最后一维的1/8。 | 283 | - <term>Ascend 950PR&950DT系列产品</term>:当`y`的数据类型为INT32时,`y`的最后一维必须是`x`最后一维的1/8。 |
| 287 | <!-- end id11 --> | 284 | <!-- end id11 --> |
| 285 | +<!-- npu="A3,910b" id18 --> | ||
| 286 | +- <term>Atlas A2系列产品</term>、<term>Atlas A3系列产品</term>:INT4量化结果须以INT32数据类型声明,此时`y`的最后一维必须是`x`最后一维的1/8;不支持直接以INT4数据类型声明输出。 | ||
| 287 | +<!-- end id18 --> | ||
| 288 | - 各产品型号支持数据类型说明: | 288 | - 各产品型号支持数据类型说明: |
| 289 | 289 | ||
| 290 | <!-- npu="A3,910b" id12 --> | 290 | <!-- npu="A3,910b" id12 --> |
| @@ -294,8 +294,8 @@ aclnnStatus aclnnRmsNormQuant( | |||
| 294 | | --------- | ------------- | ------------- | ------------- | -------------- | --------- |--------- | | 294 | | --------- | ------------- | ------------- | ------------- | -------------- | --------- |--------- | |
| 295 | | FLOAT16 | FLOAT16 | FLOAT16 | FLOAT16 | INT8 | DOUBLE |INT8 | | 295 | | FLOAT16 | FLOAT16 | FLOAT16 | FLOAT16 | INT8 | DOUBLE |INT8 | |
| 296 | | BFLOAT16 | BFLOAT16 | BFLOAT16 | BFLOAT16 | INT8 | DOUBLE |INT8 | | 296 | | BFLOAT16 | BFLOAT16 | BFLOAT16 | BFLOAT16 | INT8 | DOUBLE |INT8 | |
| 297 | - | FLOAT16 | FLOAT16 | FLOAT16 | FLOAT16 | INT8 | DOUBLE |INT4 | | 297 | + | FLOAT16 | FLOAT16 | FLOAT16 | FLOAT16 | INT8 | DOUBLE |INT32(承载INT4) | |
| 298 | - | BFLOAT16 | BFLOAT16 | BFLOAT16 | BFLOAT16 | INT8 | DOUBLE |INT4 | | 298 | + | BFLOAT16 | BFLOAT16 | BFLOAT16 | BFLOAT16 | INT8 | DOUBLE |INT32(承载INT4) | |
| 299 | <!-- end id12 --> | 299 | <!-- end id12 --> |
| 300 | 300 | ||
| 301 | <!-- npu="310p,310b" id13 --> | 301 | <!-- npu="310p,310b" id13 --> |
| @@ -258,28 +258,29 @@ static bool CheckShapeDim(RmsNormQuantInputTensor& inputTensor, const aclTensor* | |||
| 258 | 258 | ||
| 259 | int64_t scaleDimNum = static_cast<int64_t>(inputTensor.scale->GetViewShape().GetDimNum()); | 259 | int64_t scaleDimNum = static_cast<int64_t>(inputTensor.scale->GetViewShape().GetDimNum()); |
| 260 | if (scaleDimNum != 1) { | 260 | if (scaleDimNum != 1) { |
| 261 | - OP_LOGE(ACLNN_ERR_PARAM_INVALID, "DimNum of scale must be 1, but get [%ld]", scaleDimNum); | 261 | + OP_LOGE_FOR_INVALID_SHAPEDIM("aclnnRmsNormQuant", "scale", std::to_string(scaleDimNum).c_str(), "1"); |
| 262 | return false; | 262 | return false; |
| 263 | } | 263 | } |
| 264 | int64_t gammaDimNum = static_cast<int64_t>(inputTensor.gamma->GetViewShape().GetDimNum()); | 264 | int64_t gammaDimNum = static_cast<int64_t>(inputTensor.gamma->GetViewShape().GetDimNum()); |
| 265 | int64_t gammaLastDim = inputTensor.gamma->GetViewShape().GetDim(gammaDimNum - 1); | 265 | int64_t gammaLastDim = inputTensor.gamma->GetViewShape().GetDim(gammaDimNum - 1); |
| 266 | int64_t xDimNum = static_cast<int64_t>(inputTensor.x->GetViewShape().GetDimNum()); | 266 | int64_t xDimNum = static_cast<int64_t>(inputTensor.x->GetViewShape().GetDimNum()); |
| 267 | int64_t xLastDim = inputTensor.x->GetViewShape().GetDim(xDimNum - 1); | 267 | int64_t xLastDim = inputTensor.x->GetViewShape().GetDim(xDimNum - 1); |
| 268 | - int64_t scaleLength = inputTensor.scale->GetViewShape().GetDim(0); | ||
| 269 | if (Ops::NN::AclnnUtil::IsRegbase()) { | 268 | if (Ops::NN::AclnnUtil::IsRegbase()) { |
| 270 | - OP_CHECK(scaleLength == 1 || scaleLength == xLastDim, | 269 | + int64_t scaleLength = inputTensor.scale->GetViewShape().GetDim(0); |
| 271 | - OP_LOGE(ACLNN_ERR_PARAM_INVALID, | 270 | + OP_CHECK( |
| 272 | - "The length of scale must be 1 or x last dimension (%ld), " | 271 | + scaleLength == 1 || scaleLength == xLastDim, |
| 273 | - "but got %ld.", | 272 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( |
| 274 | - xLastDim, scaleLength), | 273 | + "aclnnRmsNormQuant", "scale", std::to_string(scaleLength).c_str(), |
| 275 | - return false); | 274 | + (std::string("the length of scale must be 1 or the last dim of x (") + std::to_string(xLastDim) + ")") |
| 275 | + .c_str()), | ||
| 276 | + return false); | ||
| 276 | } else { | 277 | } else { |
| 277 | - // A2/A3 的 kernel 仅按标量读取 scale(只取 scale[0] 广播到全部输出),只支持 shape 为 [1] 的 scale; | 278 | + int64_t scaleLength = inputTensor.scale->GetViewShape().GetDim(0); |
| 278 | - // 空 scale(长度 0)时 kernel 会从空 tensor 地址读取,必须一并拦截。 | 279 | + OP_CHECK( |
| 279 | - OP_CHECK(scaleLength == 1, | 280 | + scaleLength == 1, |
| 280 | - OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The length of scale must be 1 on this platform, but got %ld.", | 281 | + OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnRmsNormQuant", "scale", std::to_string(scaleLength).c_str(), |
| 281 | - scaleLength), | 282 | + "the length of scale must be 1 on this platform"), |
| 282 | - return false); | 283 | + return false); |
| 283 | } | 284 | } |
| 284 | if (gammaLastDim != xLastDim) { | 285 | if (gammaLastDim != xLastDim) { |
| 285 | OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnRmsNormQuant", "x", std::to_string(xLastDim).c_str(), | 286 | OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnRmsNormQuant", "x", std::to_string(xLastDim).c_str(), |
| @@ -291,16 +292,14 @@ static bool CheckShapeDim(RmsNormQuantInputTensor& inputTensor, const aclTensor* | |||
| 291 | int64_t yDimNum = static_cast<int64_t>(y->GetViewShape().GetDimNum()); | 292 | int64_t yDimNum = static_cast<int64_t>(y->GetViewShape().GetDimNum()); |
| 292 | int64_t outLastDim = y->GetViewShape().GetDim(yDimNum - 1); | 293 | int64_t outLastDim = y->GetViewShape().GetDim(yDimNum - 1); |
| 293 | if (outDtype == op::DataType::DT_INT32) { | 294 | if (outDtype == op::DataType::DT_INT32) { |
| 294 | - if (Ops::NN::AclnnUtil::IsRegbase()) { | 295 | + OP_CHECK(CheckShapeDimWithFrontN(inputTensor.x->GetViewShape(), y->GetViewShape(), xDimNum - 1), |
| 295 | - OP_CHECK(CheckShapeDimWithFrontN(inputTensor.x->GetViewShape(), y->GetViewShape(), xDimNum - 1), | 296 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( |
| 296 | - OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( | 297 | + "aclnnRmsNormQuant", "x,y", |
| 297 | - "aclnnRmsNormQuant", "x,y", | 298 | + (std::string(op::ToString(inputTensor.x->GetViewShape()).GetString()) + ", " + |
| 298 | - (std::string(op::ToString(inputTensor.x->GetViewShape()).GetString()) + ", " + | 299 | + op::ToString(y->GetViewShape()).GetString()) |
| 299 | - op::ToString(y->GetViewShape()).GetString()) | 300 | + .c_str(), |
| 300 | - .c_str(), | 301 | + "the first n-1 dims of x and y should be the same"), |
| 301 | - "the first n-1 dims of x and y should be the same"), | 302 | + return false); |
| 302 | - return false); | ||
| 303 | - } | ||
| 304 | OP_CHECK( | 303 | OP_CHECK( |
| 305 | xLastDim == outLastDim * INT4_NUMS_IN_INT32_SPACE, | 304 | xLastDim == outLastDim * INT4_NUMS_IN_INT32_SPACE, |
| 306 | OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnRmsNormQuant", "y", std::to_string(outLastDim).c_str(), | 305 | OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON("aclnnRmsNormQuant", "y", std::to_string(outLastDim).c_str(), |
| @@ -330,10 +329,26 @@ static bool CheckShapeDim(RmsNormQuantInputTensor& inputTensor, const aclTensor* | |||
| 330 | "x shape should be the same as y shape"), | 329 | "x shape should be the same as y shape"), |
| 331 | return false); | 330 | return false); |
| 332 | } else { | 331 | } else { |
| 332 | + // A2/A3:INT4 量化结果须以 INT32 打包张量声明(y 最后一维为 x 最后一维的 1/8), | ||
| 333 | + // 不支持直接以 INT4 数据类型声明输出。 | ||
| 334 | + OP_CHECK(outDtype != op::DataType::DT_INT4, | ||
| 335 | + OP_LOGE_FOR_INVALID_DTYPE_WITH_REASON( | ||
| 336 | + "aclnnRmsNormQuant", "y", op::ToString(outDtype).GetString(), | ||
| 337 | + "on this platform int4 output must be declared as int32 with the last dim of y equal to " | ||
| 338 | + "1/8 of the last dim of x"), | ||
| 339 | + return false); | ||
| 340 | + OP_CHECK(CheckShapeDimWithFrontN(inputTensor.x->GetViewShape(), y->GetViewShape(), xDimNum), | ||
| 341 | + OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON( | ||
| 342 | + "aclnnRmsNormQuant", "x,y", | ||
| 343 | + (std::string(op::ToString(inputTensor.x->GetViewShape()).GetString()) + ", " + | ||
| 344 | + op::ToString(y->GetViewShape()).GetString()) | ||
| 345 | + .c_str(), | ||
| 346 | + "x shape should be the same as y shape"), | ||
| 347 | + return false); | ||
| 333 | OP_CHECK(xLastDim == outLastDim, | 348 | OP_CHECK(xLastDim == outLastDim, |
| 334 | OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( | 349 | OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON( |
| 335 | "aclnnRmsNormQuant", "x", std::to_string(xLastDim).c_str(), | 350 | "aclnnRmsNormQuant", "x", std::to_string(xLastDim).c_str(), |
| 336 | - "if outType is int4 or int8, x last dim must be equal with out last dim"), | 351 | + "if outType is int8, x last dim must be equal with out last dim"), |
| 337 | return false); | 352 | return false); |
| 338 | } | 353 | } |
| 339 | } | 354 | } |
| @@ -476,6 +491,13 @@ aclnnStatus aclnnRmsNormQuantGetWorkspaceSize(const aclTensor* x, const aclTenso | |||
| 476 | if (!isRegbase) { | 491 | if (!isRegbase) { |
| 477 | const auto paramRet = CheckParams(inputTensorOri, y); | 492 | const auto paramRet = CheckParams(inputTensorOri, y); |
| 478 | CHECK_RET(paramRet == ACLNN_SUCCESS, paramRet); | 493 | CHECK_RET(paramRet == ACLNN_SUCCESS, paramRet); |
| 494 | + if (x->IsEmpty()) { | ||
| 495 | + // Shape validation guarantees an empty y. 空Tensor在所有平台均按成功空转处理: | ||
| 496 | + // 返回0字节workspace,第二段不执行计算。 | ||
| 497 | + *workspaceSize = 0; | ||
| 498 | + uniqueExecutor.ReleaseTo(executor); | ||
| 499 | + return ACLNN_SUCCESS; | ||
| 500 | + } | ||
| 479 | } | 501 | } |
| 480 | int32_t yType = y->GetDataType(); | 502 | int32_t yType = y->GetDataType(); |
| 481 | if (yType == op::DataType::DT_INT32) { | 503 | if (yType == op::DataType::DT_INT32) { |
| @@ -126,71 +126,6 @@ TEST_F(l2_rms_norm_quant_test, ascend950_rank2_scale_returns_param_invalid) | |||
| 126 | EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_ERR_PARAM_INVALID); | 126 | EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_ERR_PARAM_INVALID); |
| 127 | } | 127 | } |
| 128 | 128 | ||
| 129 | -TEST_F(l2_rms_norm_quant_test, non_regbase_rank2_scale_returns_param_invalid) | ||
| 130 | -{ | ||
| 131 | - // A2/A3 的 kernel 仅按标量读取 scale(只取 scale[0] 广播到全部输出), | ||
| 132 | - // 2 维 scale 会静默产出错误量化结果,必须在第一段拦截。 | ||
| 133 | - for (auto socVersion : {op::SocVersion::ASCEND910B, op::SocVersion::ASCEND910_93}) { | ||
| 134 | - op::SocVersionManager versionManager(socVersion); | ||
| 135 | - SCOPED_TRACE("soc version: " + to_string(static_cast<int>(socVersion))); | ||
| 136 | - TensorCase testCase; | ||
| 137 | - testCase.dtypes[3] = ACL_FLOAT16; | ||
| 138 | - testCase.dtypes[4] = ACL_INT8; | ||
| 139 | - for (const auto& scaleShape : {vector<int64_t>{2, 16}, vector<int64_t>{1, 16}}) { | ||
| 140 | - SCOPED_TRACE("rank-2 scale shape: " + to_string(scaleShape[0]) + "x" + to_string(scaleShape[1])); | ||
| 141 | - testCase.shapes[3] = scaleShape; | ||
| 142 | - testCase.shapes[4] = scaleShape; | ||
| 143 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_PARAM_INVALID); | ||
| 144 | - } | ||
| 145 | - testCase.shapes[3] = {1}; | ||
| 146 | - testCase.shapes[4] = {1}; | ||
| 147 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_SUCCESS); | ||
| 148 | - } | ||
| 149 | -} | ||
| 150 | - | ||
| 151 | -TEST_F(l2_rms_norm_quant_test, non_regbase_scale_length_must_be_one) | ||
| 152 | -{ | ||
| 153 | - // A2/A3 文档约束 scale shape 为 [1]:1 维多元素(regbase 上合法的 [H])与空 scale(长度 0) | ||
| 154 | - // 在非 regbase 上都会导致 kernel 只读 scale[0] 或从空 tensor 地址读取,必须拦截。 | ||
| 155 | - for (auto socVersion : {op::SocVersion::ASCEND910B, op::SocVersion::ASCEND910_93}) { | ||
| 156 | - op::SocVersionManager versionManager(socVersion); | ||
| 157 | - SCOPED_TRACE("soc version: " + to_string(static_cast<int>(socVersion))); | ||
| 158 | - TensorCase testCase; | ||
| 159 | - testCase.dtypes[3] = ACL_FLOAT16; | ||
| 160 | - testCase.dtypes[4] = ACL_INT8; | ||
| 161 | - // 1 维 [H](H == xLastDim,regbase 合法)在非 regbase 上非法 | ||
| 162 | - testCase.shapes[3] = {16}; | ||
| 163 | - testCase.shapes[4] = {16}; | ||
| 164 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_PARAM_INVALID); | ||
| 165 | - // 空 scale(长度 0)非法 | ||
| 166 | - testCase.shapes[3] = {0}; | ||
| 167 | - testCase.shapes[4] = {0}; | ||
| 168 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_PARAM_INVALID); | ||
| 169 | - // x 为空但 scale 为 [1]:保持既有第一段成功行为(仅校验 scale 长度,不联动 x) | ||
| 170 | - testCase.shapes[0] = {0, 16}; | ||
| 171 | - testCase.shapes[3] = {1}; | ||
| 172 | - testCase.shapes[4] = {1}; | ||
| 173 | - testCase.shapes[5] = {0, 16}; | ||
| 174 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_SUCCESS); | ||
| 175 | - } | ||
| 176 | -} | ||
| 177 | - | ||
| 178 | -TEST_F(l2_rms_norm_quant_test, regbase_scale_length_one_or_x_last_dim_is_unchanged) | ||
| 179 | -{ | ||
| 180 | - // regbase 上 [1] 与 [H](== xLastDim)保持合法,空 scale(长度 0)保持非法。 | ||
| 181 | - op::SocVersionManager versionManager(op::SocVersion::ASCEND950); | ||
| 182 | - TensorCase testCase; | ||
| 183 | - for (int64_t scaleLength : {1, 16}) { | ||
| 184 | - SCOPED_TRACE("regbase scale length: " + to_string(scaleLength)); | ||
| 185 | - testCase.shapes[3] = {scaleLength}; | ||
| 186 | - testCase.shapes[4] = {scaleLength}; | ||
| 187 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_SUCCESS); | ||
| 188 | - } | ||
| 189 | - testCase.shapes[3] = {0}; | ||
| 190 | - testCase.shapes[4] = {0}; | ||
| 191 | - EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_PARAM_INVALID); | ||
| 192 | -} | ||
| 193 | - | ||
| 194 | TEST_F(l2_rms_norm_quant_test, ascend950_invalid_norm_leading_dim_returns_param_invalid) | 129 | TEST_F(l2_rms_norm_quant_test, ascend950_invalid_norm_leading_dim_returns_param_invalid) |
| 195 | { | 130 | { |
| 196 | op::SocVersionManager versionManager(op::SocVersion::ASCEND950); | 131 | op::SocVersionManager versionManager(op::SocVersion::ASCEND950); |
| @@ -279,3 +214,114 @@ TEST_F(l2_rms_norm_quant_test, legacy_norm_preprocessing_behavior_is_unchanged) | |||
| 279 | EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_INNER_NULLPTR); | 214 | EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_INNER_NULLPTR); |
| 280 | } | 215 | } |
| 281 | } | 216 | } |
| 217 | + | ||
| 218 | +TEST_F(l2_rms_norm_quant_test, ascend910b_scale_dim_or_length_invalid_returns_param_invalid) | ||
| 219 | +{ | ||
| 220 | + op::SocVersionManager versionManager(op::SocVersion::ASCEND910B); | ||
| 221 | + TensorCase testCase; | ||
| 222 | + testCase.dtypes[3] = ACL_FLOAT16; | ||
| 223 | + testCase.dtypes[4] = ACL_INT8; | ||
| 224 | + | ||
| 225 | + // scale=[1,1] is 2D, A2/A3 requires scale dim == 1. | ||
| 226 | + testCase.shapes[3] = {1, 1}; | ||
| 227 | + testCase.shapes[4] = {1, 1}; | ||
| 228 | + EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_PARAM_INVALID); | ||
| 229 | + | ||
| 230 | + // scale=[3] has length != 1, A2/A3 requires scale shape == [1]. | ||
| 231 | + testCase.shapes[3] = {3}; | ||
| 232 | + testCase.shapes[4] = {3}; | ||
| 233 | + EXPECT_EQ(RunTensorCase(testCase), ACLNN_ERR_PARAM_INVALID); | ||
| 234 | +} | ||
| 235 | + | ||
| 236 | +TEST_F(l2_rms_norm_quant_test, ascend910b_x_y_front_dim_mismatch_returns_param_invalid) | ||
| 237 | +{ | ||
| 238 | + op::SocVersionManager versionManager(op::SocVersion::ASCEND910B); | ||
| 239 | + auto xDesc = TensorDesc({1, 3, 5}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 240 | + auto gammaDesc = TensorDesc({1, 5}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 241 | + auto betaDesc = TensorDesc({1, 5}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 242 | + auto scaleDesc = TensorDesc({1}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 243 | + auto offsetDesc = TensorDesc({1}, ACL_INT8, ACL_FORMAT_ND); | ||
| 244 | + auto yDesc = TensorDesc({3, 5}, ACL_INT8, ACL_FORMAT_ND); | ||
| 245 | + | ||
| 246 | + auto ut = OP_API_UT(aclnnRmsNormQuant, INPUT(xDesc, gammaDesc, betaDesc, scaleDesc, offsetDesc, 1e-6, yDesc), | ||
| 247 | + OUTPUT()); | ||
| 248 | + uint64_t workspaceSize = 0; | ||
| 249 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_ERR_PARAM_INVALID); | ||
| 250 | +} | ||
| 251 | + | ||
| 252 | +TEST_F(l2_rms_norm_quant_test, ascend910b_empty_tensor_is_successful_noop) | ||
| 253 | +{ | ||
| 254 | + // 空 Tensor 在所有平台按成功空转处理(issue #7402):第一段返回 0 且 workspaceSize 为 0 字节, | ||
| 255 | + // 第二段不执行计算。 | ||
| 256 | + op::SocVersionManager versionManager(op::SocVersion::ASCEND910B); | ||
| 257 | + auto xDesc = TensorDesc({0, 5}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 258 | + auto gammaDesc = TensorDesc({1, 5}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 259 | + auto betaDesc = TensorDesc({1, 5}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 260 | + auto scaleDesc = TensorDesc({1}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 261 | + auto offsetDesc = TensorDesc({1}, ACL_INT8, ACL_FORMAT_ND); | ||
| 262 | + auto yDesc = TensorDesc({0, 5}, ACL_INT8, ACL_FORMAT_ND); | ||
| 263 | + | ||
| 264 | + auto ut = OP_API_UT(aclnnRmsNormQuant, INPUT(xDesc, gammaDesc, betaDesc, scaleDesc, offsetDesc, 1e-6, yDesc), | ||
| 265 | + OUTPUT()); | ||
| 266 | + uint64_t workspaceSize = 1; | ||
| 267 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_SUCCESS); | ||
| 268 | + EXPECT_EQ(workspaceSize, static_cast<uint64_t>(0)); | ||
| 269 | +} | ||
| 270 | + | ||
| 271 | +TEST_F(l2_rms_norm_quant_test, ascend910b_empty_tensor_int32_packed_is_successful_noop) | ||
| 272 | +{ | ||
| 273 | + // INT32(承载 INT4)输出的空 Tensor 同样成功空转:y 末维为 x 末维的 1/8,x=[0,16] 时 y=[0,2]。 | ||
| 274 | + op::SocVersionManager versionManager(op::SocVersion::ASCEND910B); | ||
| 275 | + auto xDesc = TensorDesc({0, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 276 | + auto gammaDesc = TensorDesc({1, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 277 | + auto betaDesc = TensorDesc({1, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 278 | + auto scaleDesc = TensorDesc({1}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 279 | + auto offsetDesc = TensorDesc({1}, ACL_INT8, ACL_FORMAT_ND); | ||
| 280 | + auto yDesc = TensorDesc({0, 2}, ACL_INT32, ACL_FORMAT_ND); | ||
| 281 | + | ||
| 282 | + auto ut = OP_API_UT(aclnnRmsNormQuant, INPUT(xDesc, gammaDesc, betaDesc, scaleDesc, offsetDesc, 1e-6, yDesc), | ||
| 283 | + OUTPUT()); | ||
| 284 | + uint64_t workspaceSize = 1; | ||
| 285 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_SUCCESS); | ||
| 286 | + EXPECT_EQ(workspaceSize, static_cast<uint64_t>(0)); | ||
| 287 | +} | ||
| 288 | + | ||
| 289 | +TEST_F(l2_rms_norm_quant_test, ascend950_empty_scale_returns_param_invalid) | ||
| 290 | +{ | ||
| 291 | + // regbase 平台:scale shape 为 [0](空 scale)时 kernel 会从空 tensor 地址读取,必须在第一段拦截。 | ||
| 292 | + op::SocVersionManager versionManager(op::SocVersion::ASCEND950); | ||
| 293 | + auto xDesc = TensorDesc({2, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 294 | + auto gammaDesc = TensorDesc({16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 295 | + auto betaDesc = TensorDesc({16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 296 | + auto scaleDesc = TensorDesc({0}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 297 | + auto offsetDesc = TensorDesc({0}, ACL_FLOAT, ACL_FORMAT_ND); | ||
| 298 | + auto yDesc = TensorDesc({2, 16}, ACL_INT8, ACL_FORMAT_ND); | ||
| 299 | + | ||
| 300 | + auto ut = OP_API_UT(aclnnRmsNormQuant, INPUT(xDesc, gammaDesc, betaDesc, scaleDesc, offsetDesc, 1e-6, yDesc), | ||
| 301 | + OUTPUT()); | ||
| 302 | + uint64_t workspaceSize = 0; | ||
| 303 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_ERR_PARAM_INVALID); | ||
| 304 | +} | ||
| 305 | + | ||
| 306 | +TEST_F(l2_rms_norm_quant_test, ascend910b_int4_output_must_be_int32_packed) | ||
| 307 | +{ | ||
| 308 | + op::SocVersionManager versionManager(op::SocVersion::ASCEND910B); | ||
| 309 | + auto xDesc = TensorDesc({3, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 310 | + auto gammaDesc = TensorDesc({1, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 311 | + auto betaDesc = TensorDesc({1, 16}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 312 | + auto scaleDesc = TensorDesc({1}, ACL_FLOAT16, ACL_FORMAT_ND); | ||
| 313 | + auto offsetDesc = TensorDesc({1}, ACL_INT8, ACL_FORMAT_ND); | ||
| 314 | + auto yInt4Desc = TensorDesc({3, 16}, ACL_INT4, ACL_FORMAT_ND); | ||
| 315 | + | ||
| 316 | + // A2/A3 上 INT4 输出须以 INT32 打包张量声明,直接传 INT4 dtype 应在首段返回 161002。 | ||
| 317 | + auto ut = OP_API_UT(aclnnRmsNormQuant, INPUT(xDesc, gammaDesc, betaDesc, scaleDesc, offsetDesc, 1e-6, yInt4Desc), | ||
| 318 | + OUTPUT()); | ||
| 319 | + uint64_t workspaceSize = 0; | ||
| 320 | + EXPECT_EQ(ut.TestGetWorkspaceSize(&workspaceSize), ACLNN_ERR_PARAM_INVALID); | ||
| 321 | + | ||
| 322 | + // INT32(承载 INT4)时,y 末维必须为 x 末维的 1/8(x=16 时 y 末维必须为 2),否则 161002。 | ||
| 323 | + auto yInt32WrongDesc = TensorDesc({3, 3}, ACL_INT32, ACL_FORMAT_ND); | ||
| 324 | + auto utWrong = OP_API_UT(aclnnRmsNormQuant, | ||
| 325 | + INPUT(xDesc, gammaDesc, betaDesc, scaleDesc, offsetDesc, 1e-6, yInt32WrongDesc), OUTPUT()); | ||
| 326 | + EXPECT_EQ(utWrong.TestGetWorkspaceSize(&workspaceSize), ACLNN_ERR_PARAM_INVALID); | ||
| 327 | +} | ||