已合并
FlatQuant修复大shape内存检测问题 #3989
陈海杰创建于 4月20日
FlatQuant修复大shape内存检测问题 #3989
已合并
共 2 个文件变更+2-3
| @@ -89,8 +89,7 @@ public: | |||
| 89 | if (shape.K * shape.M - shape.Mceil < 0) { | 89 | if (shape.K * shape.M - shape.Mceil < 0) { |
| 90 | invalidK = -1; | 90 | invalidK = -1; |
| 91 | } else { | 91 | } else { |
| 92 | - invalidK = | 92 | + invalidK = (shape.K * shape.M - shape.Mceil) / shape.M; |
| 93 | - static_cast<int64_t>(static_cast<float>(shape.K * shape.M - shape.Mceil) / static_cast<float>(shape.M)); | ||
| 94 | } | 93 | } |
| 95 | } | 94 | } |
| 96 | 95 | ||
| @@ -83,7 +83,7 @@ public: | |||
| 83 | matmulInfo.splitCount2 = matmulInfo.splitCount * splitN; | 83 | matmulInfo.splitCount2 = matmulInfo.splitCount * splitN; |
| 84 | matmulInfo.splitCount1 = matmulInfo.splitCount * splitM; | 84 | matmulInfo.splitCount1 = matmulInfo.splitCount * splitM; |
| 85 | } | 85 | } |
| 86 | - invalidK = static_cast<int64_t>(static_cast<float>(shape.K * shape.M - shape.Mceil) / static_cast<float>(shape.M)); | 86 | + invalidK = (shape.K * shape.M - shape.Mceil) / shape.M; |
| 87 | } | 87 | } |
| 88 | 88 | ||
| 89 | aifunc void Process(){ | 89 | aifunc void Process(){ |