已合并
[AscendNPU IR] [PlanMemory] Fix: fix planmemory level1 reuse on pong buffers #1450
SL25创建于 7月8日
[AscendNPU IR] [PlanMemory] Fix: fix planmemory level1 reuse on pong buffers #1450
已合并
共 2 个文件变更+149-34
| @@ -715,8 +715,8 @@ void MemLivenessAnalysis::UpdateOperandGenInfo(OpInfo *opInfo, Value operand) { | |||
| 715 | genKillMap[opInfo].gen.push_back(operand); | 715 | genKillMap[opInfo].gen.push_back(operand); |
| 716 | buffer2status[iter_buffer->first] = BufferStatus::GENED; | 716 | buffer2status[iter_buffer->first] = BufferStatus::GENED; |
| 717 | } else if (iter_buffer->second == BufferStatus::KILLED) { | 717 | } else if (iter_buffer->second == BufferStatus::KILLED) { |
| 718 | - llvm::report_fatal_error("The buffer memory has been released and cannot be used " | 718 | + llvm::report_fatal_error( |
| 719 | - "again! "); | 719 | + "The buffer memory has been released and cannot be used again! "); |
| 720 | } | 720 | } |
| 721 | } | 721 | } |
| 722 | 722 | ||
| @@ -894,7 +894,7 @@ bool MemLivenessAnalysis::IsBlockAfter(Block *afterBlock, | |||
| 894 | } | 894 | } |
| 895 | 895 | ||
| 896 | bool MemLivenessAnalysis::IsDeadAfterOp(Value value, | 896 | bool MemLivenessAnalysis::IsDeadAfterOp(Value value, |
| 897 | - Operation *operation) const { | 897 | + Operation *operation) const { |
| 898 | auto *moduleBlock = utils::getTopLevelModuleOp(operation).getBody(); | 898 | auto *moduleBlock = utils::getTopLevelModuleOp(operation).getBody(); |
| 899 | // trace all blocks that contains ifOp until moduleBlock. | 899 | // trace all blocks that contains ifOp until moduleBlock. |
| 900 | DenseMap<Block *, Operation *> block2Op; | 900 | DenseMap<Block *, Operation *> block2Op; |
| @@ -920,7 +920,8 @@ bool MemLivenessAnalysis::IsDeadAfterOp(Value value, | |||
| 920 | // check whether parent ops are same, ex: if then ... else ... | 920 | // check whether parent ops are same, ex: if then ... else ... |
| 921 | if (currentOp == op && userChildOp != nullptr && | 921 | if (currentOp == op && userChildOp != nullptr && |
| 922 | currChildIt != parentToChild.end() && | 922 | currChildIt != parentToChild.end() && |
| 923 | - IsBlockAfter(userChildOp->getBlock(), currChildIt->second->getBlock())) { | 923 | + IsBlockAfter(userChildOp->getBlock(), |
| 924 | + currChildIt->second->getBlock())) { | ||
| 924 | return false; | 925 | return false; |
| 925 | } | 926 | } |
| 926 | // once different parent ops in same block, check the order | 927 | // once different parent ops in same block, check the order |
| @@ -941,8 +942,7 @@ bool MemLivenessAnalysis::IsDeadAfterOp(Value value, | |||
| 941 | bool MemLivenessAnalysis::AllDeadAfter(Operation *op, SetVector<Value> aliasVec, | 942 | bool MemLivenessAnalysis::AllDeadAfter(Operation *op, SetVector<Value> aliasVec, |
| 942 | Liveness live) const { | 943 | Liveness live) const { |
| 943 | for (auto aliasBuffer : aliasVec) { | 944 | for (auto aliasBuffer : aliasVec) { |
| 944 | - if (!live.isDeadAfter(aliasBuffer, op) || | 945 | + if (!live.isDeadAfter(aliasBuffer, op) || !IsDeadAfterOp(aliasBuffer, op)) { |
| 945 | - !IsDeadAfterOp(aliasBuffer, op)) { | ||
| 946 | return false; | 946 | return false; |
| 947 | } | 947 | } |
| 948 | } | 948 | } |
| @@ -2034,10 +2034,14 @@ bool MemPlan::VerifyConflictStage1( | |||
| 2034 | // Multi-buffer reuse multi buffer | 2034 | // Multi-buffer reuse multi buffer |
| 2035 | 2035 | ||
| 2036 | // Multi-buffer case: require enough multibuffer entries for all buffer | 2036 | // Multi-buffer case: require enough multibuffer entries for all buffer |
| 2037 | - // instances. | 2037 | + // instances, and only first buffer(not other relation entries) can reuse in |
| 2038 | - if (e->multiBufferNum > 1 && otherBufferEntries.size() < e->multiBufferNum) { | 2038 | + // level1. |
| 2039 | + auto otherBufferEntriesSize = otherBufferEntries.size(); | ||
| 2040 | + if (e->multiBufferNum > 1 && | ||
| 2041 | + (otherBufferEntriesSize < e->multiBufferNum - 1 || | ||
| 2042 | + e->otherBufferRelationEntries.empty())) { | ||
| 2039 | // Not enough historical multibuffer entries to match current multi-buffer | 2043 | // Not enough historical multibuffer entries to match current multi-buffer |
| 2040 | - // requirement. | 2044 | + // requirement, or current entry is not first buffer. |
| 2041 | return true; | 2045 | return true; |
| 2042 | } | 2046 | } |
| 2043 | 2047 | ||
| @@ -2045,7 +2049,7 @@ bool MemPlan::VerifyConflictStage1( | |||
| 2045 | // entry conflicts with historical records at its offset, the whole | 2049 | // entry conflicts with historical records at its offset, the whole |
| 2046 | // multi-buffer reuse fails. Only when all required multibuffer entries are | 2050 | // multi-buffer reuse fails. Only when all required multibuffer entries are |
| 2047 | // conflict-free can we reuse (return false). | 2051 | // conflict-free can we reuse (return false). |
| 2048 | - for (uint32_t i = 0; i < e->multiBufferNum; ++i) { | 2052 | + for (uint32_t i = 0; i < otherBufferEntriesSize; ++i) { |
| 2049 | StorageEntry *multiRelationMultiBufferEntry = otherBufferEntries[i]; | 2053 | StorageEntry *multiRelationMultiBufferEntry = otherBufferEntries[i]; |
| 2050 | if (!multiRelationMultiBufferEntry) { | 2054 | if (!multiRelationMultiBufferEntry) { |
| 2051 | return true; | 2055 | return true; |
| @@ -2126,33 +2130,35 @@ bool MemPlan::IsBufferLifeVecConflict(PlanRecord &r, uint64_t offset, | |||
| 2126 | 2130 | ||
| 2127 | void MemPlan::PlanRelationOtherBufferEntryAddress( | 2131 | void MemPlan::PlanRelationOtherBufferEntryAddress( |
| 2128 | llvm::ArrayRef<uint64_t> otherBufferOffsets, StorageEntry *e) { | 2132 | llvm::ArrayRef<uint64_t> otherBufferOffsets, StorageEntry *e) { |
| 2129 | - if (e->multiBufferNum == 1) { | 2133 | + // First loop: assign offsets to existing otherBufferRelationEntries. |
| 2130 | - // Single-buffer entry expanded to use multibuffer. Create | 2134 | + // For each multibuffer index from 1 to multiBufferNum, the corresponding |
| 2131 | - // one extra StorageEntry per multibuffer offset and store in | 2135 | + // relation entry (at index i-1) gets the offset at index i-1. |
| 2132 | - // firstBufferEntry2RelationOtherBufferEntry as a vector. | 2136 | + for (size_t i = 1; i < e->multiBufferNum; ++i) { |
| 2133 | - auto &vec = firstBufferEntry2RelationOtherBufferEntry[e]; | 2137 | + if (i - 1 >= e->otherBufferRelationEntries.size() || |
| 2134 | - vec.clear(); | 2138 | + i - 1 >= otherBufferOffsets.size()) { |
| 2135 | - vec.reserve(otherBufferOffsets.size()); | 2139 | + continue; |
| 2136 | - for (uint64_t offset : otherBufferOffsets) { | ||
| 2137 | - auto entry = std::make_unique<StorageEntry>(); | ||
| 2138 | - entry->bufInfo = e->bufInfo; | ||
| 2139 | - entry->bufferLifeVec = e->bufferLifeVec; | ||
| 2140 | - entry->alignedConstBits = e->alignedConstBits; | ||
| 2141 | - entry->inplaceBuffers = e->inplaceBuffers; | ||
| 2142 | - entry->multiBufferNum = e->multiBufferNum; | ||
| 2143 | - entry->bitsOffset = offset; | ||
| 2144 | - vec.push_back(std::move(entry)); | ||
| 2145 | } | 2140 | } |
| 2146 | - } else if (e->multiBufferNum > 1 && !e->otherBufferRelationEntries.empty()) { | 2141 | + if (StorageEntry *re = e->otherBufferRelationEntries[i - 1]) { |
| 2147 | - // Multi-buffer entry: assign each relation entry its multibuffer offset | 2142 | + re->bitsOffset = otherBufferOffsets[i - 1]; |
| 2148 | - // from otherBufferOffsets. | ||
| 2149 | - for (size_t i = 0; i < e->otherBufferRelationEntries.size() && | ||
| 2150 | - i < otherBufferOffsets.size(); | ||
| 2151 | - ++i) { | ||
| 2152 | - if (StorageEntry *re = e->otherBufferRelationEntries[i]) | ||
| 2153 | - re->bitsOffset = otherBufferOffsets[i]; | ||
| 2154 | } | 2143 | } |
| 2155 | } | 2144 | } |
| 2145 | + | ||
| 2146 | + // Second loop: create new StorageEntries for the remaining offsets (from | ||
| 2147 | + // index multiBufferNum-1 to the end of otherBufferOffsets) and store them | ||
| 2148 | + // in firstBufferEntry2RelationOtherBufferEntry as a vector. | ||
| 2149 | + auto &vec = firstBufferEntry2RelationOtherBufferEntry[e]; | ||
| 2150 | + vec.clear(); | ||
| 2151 | + vec.reserve(otherBufferOffsets.size()); | ||
| 2152 | + for (size_t i = e->multiBufferNum - 1; i < otherBufferOffsets.size(); ++i) { | ||
| 2153 | + auto entry = std::make_unique<StorageEntry>(); | ||
| 2154 | + entry->bufInfo = e->bufInfo; | ||
| 2155 | + entry->bufferLifeVec = e->bufferLifeVec; | ||
| 2156 | + entry->alignedConstBits = e->alignedConstBits; | ||
| 2157 | + entry->inplaceBuffers = e->inplaceBuffers; | ||
| 2158 | + entry->multiBufferNum = e->multiBufferNum; | ||
| 2159 | + entry->bitsOffset = otherBufferOffsets[i]; | ||
| 2160 | + vec.push_back(std::move(entry)); | ||
| 2161 | + } | ||
| 2156 | } | 2162 | } |
| 2157 | 2163 | ||
| 2158 | bool MemPlan::VerifyConflictStageCommon( | 2164 | bool MemPlan::VerifyConflictStageCommon( |
| @@ -1088,6 +1088,115 @@ module { | |||
| 1088 | } | 1088 | } |
| 1089 | } | 1089 | } |
| 1090 | 1090 | ||
| 1091 | +// ----- | ||
| 1092 | +module { | ||
| 1093 | + // CHECK-LABEL: func.func @test_mem_level1_single_buffer_reuse | ||
| 1094 | + func.func @test_mem_level1_single_buffer_reuse(%src1_gm: memref<6144xf32, #hivm.address_space<gm>>, | ||
| 1095 | + %src2_gm: memref<6144xf32, #hivm.address_space<gm>>, | ||
| 1096 | + %dst1_gm: memref<6144xf32, #hivm.address_space<gm>>, | ||
| 1097 | + %dst2_gm: memref<6144xf32, #hivm.address_space<gm>>) { | ||
| 1098 | + // CHECK-NOT: memref.alloc() | ||
| 1099 | + %c0 = arith.constant 0 : index | ||
| 1100 | + %c4 = arith.constant 4 : index | ||
| 1101 | + %c16 = arith.constant 16 : index | ||
| 1102 | + scf.for %i0 = %c0 to %c16 step %c4 { | ||
| 1103 | + %src1_ub = memref.alloc() : memref<6144xf32, #hivm.address_space<ub>> | ||
| 1104 | + annotation.mark %src1_ub {hivm.multi_buffer = 4 : i32} : memref<6144xf32, #hivm.address_space<ub>> | ||
| 1105 | + %src2_ub = memref.alloc() : memref<6144xf32, #hivm.address_space<ub>> | ||
| 1106 | + %dst1_ub = memref.alloc() : memref<6144xf32, #hivm.address_space<ub>> | ||
| 1107 | + annotation.mark %dst1_ub {hivm.multi_buffer = 4 : i32} : memref<6144xf32, #hivm.address_space<ub>> | ||
| 1108 | + %dst2_ub = memref.alloc() : memref<6144xf32, #hivm.address_space<ub>> | ||
| 1109 | + hivm.hir.load ins(%src1_gm : memref<6144xf32, #hivm.address_space<gm>>) | ||
| 1110 | + outs(%src1_ub : memref<6144xf32, #hivm.address_space<ub>>) | ||
| 1111 | + hivm.hir.vadd ins(%src1_ub, %src1_ub : memref<6144xf32, #hivm.address_space<ub>>, memref<6144xf32, #hivm.address_space<ub>>) | ||
| 1112 | + outs(%dst1_ub : memref<6144xf32, #hivm.address_space<ub>>) | ||
| 1113 | + hivm.hir.store ins(%dst1_ub : memref<6144xf32,#hivm.address_space<ub>>) | ||
| 1114 | + outs(%dst1_gm: memref<6144xf32,#hivm.address_space<gm>>) | ||
| 1115 | + hivm.hir.load ins(%src2_gm : memref<6144xf32, #hivm.address_space<gm>>) | ||
| 1116 | + outs(%src2_ub : memref<6144xf32, #hivm.address_space<ub>>) | ||
| 1117 | + hivm.hir.vadd ins(%src2_ub, %src2_ub : memref<6144xf32, #hivm.address_space<ub>>, memref<6144xf32, #hivm.address_space<ub>>) | ||
| 1118 | + outs(%dst2_ub : memref<6144xf32, #hivm.address_space<ub>>) | ||
| 1119 | + hivm.hir.store ins(%dst2_ub : memref<6144xf32,#hivm.address_space<ub>>) | ||
| 1120 | + outs(%dst2_gm: memref<6144xf32,#hivm.address_space<gm>>) | ||
| 1121 | + } | ||
| 1122 | + return | ||
| 1123 | + } | ||
| 1124 | +} | ||
| 1125 | + | ||
| 1126 | +// ----- | ||
| 1127 | +module { | ||
| 1128 | + // CHECK-LABEL: func.func @test_mem_level1_equal_multi_buffers_reuse | ||
| 1129 | + func.func @test_mem_level1_equal_multi_buffers_reuse(%src1_gm: memref<5120xf32, #hivm.address_space<gm>>, | ||
| 1130 | + %src2_gm: memref<5120xf32, #hivm.address_space<gm>>, | ||
| 1131 | + %dst1_gm: memref<5120xf32, #hivm.address_space<gm>>, | ||
| 1132 | + %dst2_gm: memref<5120xf32, #hivm.address_space<gm>>) { | ||
| 1133 | + // CHECK-NOT: memref.alloc() | ||
| 1134 | + %c0 = arith.constant 0 : index | ||
| 1135 | + %c4 = arith.constant 4 : index | ||
| 1136 | + %c16 = arith.constant 16 : index | ||
| 1137 | + scf.for %i0 = %c0 to %c16 step %c4 { | ||
| 1138 | + %src1_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1139 | + annotation.mark %src1_ub {hivm.multi_buffer = 4 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1140 | + %src2_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1141 | + annotation.mark %src2_ub {hivm.multi_buffer = 4 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1142 | + %dst1_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1143 | + annotation.mark %dst1_ub {hivm.multi_buffer = 4 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1144 | + %dst2_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1145 | + annotation.mark %dst2_ub {hivm.multi_buffer = 4 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1146 | + hivm.hir.load ins(%src1_gm : memref<5120xf32, #hivm.address_space<gm>>) | ||
| 1147 | + outs(%src1_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1148 | + hivm.hir.vadd ins(%src1_ub, %src1_ub : memref<5120xf32, #hivm.address_space<ub>>, memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1149 | + outs(%dst1_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1150 | + hivm.hir.store ins(%dst1_ub : memref<5120xf32,#hivm.address_space<ub>>) | ||
| 1151 | + outs(%dst1_gm: memref<5120xf32,#hivm.address_space<gm>>) | ||
| 1152 | + hivm.hir.load ins(%src2_gm : memref<5120xf32, #hivm.address_space<gm>>) | ||
| 1153 | + outs(%src2_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1154 | + hivm.hir.vadd ins(%src2_ub, %src2_ub : memref<5120xf32, #hivm.address_space<ub>>, memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1155 | + outs(%dst2_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1156 | + hivm.hir.store ins(%dst2_ub : memref<5120xf32,#hivm.address_space<ub>>) | ||
| 1157 | + outs(%dst2_gm: memref<5120xf32,#hivm.address_space<gm>>) | ||
| 1158 | + } | ||
| 1159 | + return | ||
| 1160 | + } | ||
| 1161 | +} | ||
| 1162 | + | ||
| 1163 | +// ----- | ||
| 1164 | +module { | ||
| 1165 | + // CHECK-LABEL: func.func @test_mem_level1_less_multi_buffers_reuse | ||
| 1166 | + func.func @test_mem_level1_less_multi_buffers_reuse(%src1_gm: memref<5120xf32, #hivm.address_space<gm>>, | ||
| 1167 | + %src2_gm: memref<5120xf32, #hivm.address_space<gm>>, | ||
| 1168 | + %dst1_gm: memref<5120xf32, #hivm.address_space<gm>>, | ||
| 1169 | + %dst2_gm: memref<5120xf32, #hivm.address_space<gm>>) { | ||
| 1170 | + // CHECK-NOT: memref.alloc() | ||
| 1171 | + %c0 = arith.constant 0 : index | ||
| 1172 | + %c4 = arith.constant 4 : index | ||
| 1173 | + %c16 = arith.constant 16 : index | ||
| 1174 | + scf.for %i0 = %c0 to %c16 step %c4 { | ||
| 1175 | + %src1_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1176 | + annotation.mark %src1_ub {hivm.multi_buffer = 4 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1177 | + %src2_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1178 | + annotation.mark %src2_ub {hivm.multi_buffer = 2 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1179 | + %dst1_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1180 | + annotation.mark %dst1_ub {hivm.multi_buffer = 4 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1181 | + %dst2_ub = memref.alloc() : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1182 | + annotation.mark %dst2_ub {hivm.multi_buffer = 2 : i32} : memref<5120xf32, #hivm.address_space<ub>> | ||
| 1183 | + hivm.hir.load ins(%src1_gm : memref<5120xf32, #hivm.address_space<gm>>) | ||
| 1184 | + outs(%src1_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1185 | + hivm.hir.vadd ins(%src1_ub, %src1_ub : memref<5120xf32, #hivm.address_space<ub>>, memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1186 | + outs(%dst1_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1187 | + hivm.hir.store ins(%dst1_ub : memref<5120xf32,#hivm.address_space<ub>>) | ||
| 1188 | + outs(%dst1_gm: memref<5120xf32,#hivm.address_space<gm>>) | ||
| 1189 | + hivm.hir.load ins(%src2_gm : memref<5120xf32, #hivm.address_space<gm>>) | ||
| 1190 | + outs(%src2_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1191 | + hivm.hir.vadd ins(%src2_ub, %src2_ub : memref<5120xf32, #hivm.address_space<ub>>, memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1192 | + outs(%dst2_ub : memref<5120xf32, #hivm.address_space<ub>>) | ||
| 1193 | + hivm.hir.store ins(%dst2_ub : memref<5120xf32,#hivm.address_space<ub>>) | ||
| 1194 | + outs(%dst2_gm: memref<5120xf32,#hivm.address_space<gm>>) | ||
| 1195 | + } | ||
| 1196 | + return | ||
| 1197 | + } | ||
| 1198 | +} | ||
| 1199 | + | ||
| 1091 | // ----- | 1200 | // ----- |
| 1092 | module { | 1201 | module { |
| 1093 | // CHECK-LABEL: func.func @test_infer_mem_allocate_loop_conflict | 1202 | // CHECK-LABEL: func.func @test_infer_mem_allocate_loop_conflict |