已合并
[VFFusion] Avoid moving region-bearing ops that have anchors inside. #1365
hujiajun创建于 7月5日
[VFFusion] Avoid moving region-bearing ops that have anchors inside. #1365
已合并
共 2 个文件变更+53-21
| @@ -501,23 +501,44 @@ static void computeConflictLists( | |||
| 501 | } | 501 | } |
| 502 | } | 502 | } |
| 503 | 503 | ||
| 504 | - if (isa<hivm::CopyOp, memref::CopyOp>(op)) { | 504 | + // Only region-bearing ops or explicit sync/copy ops need the |
| 505 | - auto copyOp = cast<CopyOpInterface>(op); | 505 | + // previous/following scan and the body walk below. |
| 506 | - DenseSet<Operation *> previousOps; | 506 | + if (op->getNumRegions() == 0 && |
| 507 | - DenseSet<Operation *> followingOps; | 507 | + !isa<hivm::SyncBlockOp, hivm::SyncBlockSetOp, |
| 508 | - findPreviousAndFollowingFusableOpOf(op, block, previousOps, followingOps); | 508 | + hivm::SyncBlockWaitOp, hivm::CreateSyncBlockLockOp, |
| 509 | - computeConflictListsForCopyOpOperand(fusableOpInfoMap, previousOps, followingOps, copyOp.getTarget()); | 509 | + hivm::SyncBlockLockOp, hivm::SyncBlockUnlockOp>(op) && |
| 510 | - computeConflictListsForCopyOpOperand(fusableOpInfoMap, previousOps, followingOps, copyOp.getSource()); | 510 | + !isa<hivm::CopyOp, memref::CopyOp>(op)) |
| 511 | - } | 511 | + return; |
| 512 | 512 | ||
| 513 | - if (isa<hivm::SyncBlockOp, hivm::SyncBlockSetOp, | 513 | + DenseSet<Operation *> previousOps; |
| 514 | - hivm::SyncBlockWaitOp, hivm::CreateSyncBlockLockOp, | 514 | + DenseSet<Operation *> followingOps; |
| 515 | - hivm::SyncBlockLockOp, hivm::SyncBlockUnlockOp, scf::ForOp, | 515 | + findPreviousAndFollowingFusableOpOf(op, block, previousOps, |
| 516 | - scf::WhileOp, scf::IfOp, scope::ScopeOp>(op)) { | 516 | + followingOps); |
| 517 | - DenseSet<Operation *> previousOps; | 517 | + |
| 518 | - DenseSet<Operation *> followingOps; | 518 | + // Walk the op body: CopyOps create operand-specific conflicts; |
| 519 | - findPreviousAndFollowingFusableOpOf(op, block, previousOps, | 519 | + // sync ops trigger a full barrier between all previous and |
| 520 | - followingOps); | 520 | + // following fusable ops. |
| 521 | + auto walker = [&fusableOpInfoMap, &previousOps, | ||
| 522 | + &followingOps](Operation *op) { | ||
| 523 | + if (isa<hivm::CopyOp, memref::CopyOp>(op)) { | ||
| 524 | + auto copyOp = cast<CopyOpInterface>(op); | ||
| 525 | + computeConflictListsForCopyOpOperand(fusableOpInfoMap, | ||
| 526 | + previousOps, followingOps, | ||
| 527 | + copyOp.getTarget()); | ||
| 528 | + computeConflictListsForCopyOpOperand(fusableOpInfoMap, | ||
| 529 | + previousOps, followingOps, | ||
| 530 | + copyOp.getSource()); | ||
| 531 | + return WalkResult::advance(); | ||
| 532 | + } | ||
| 533 | + if (isa<hivm::AnchorOp>(op)) | ||
| 534 | + return WalkResult::interrupt(); | ||
| 535 | + return isa<hivm::SyncBlockOp, hivm::SyncBlockSetOp, | ||
| 536 | + hivm::SyncBlockWaitOp, hivm::CreateSyncBlockLockOp, | ||
| 537 | + hivm::SyncBlockLockOp, hivm::SyncBlockUnlockOp>(op) | ||
| 538 | + ? WalkResult::interrupt() | ||
| 539 | + : WalkResult::advance(); | ||
| 540 | + }; | ||
| 541 | + if (op->walk(walker).wasInterrupted()) { | ||
| 521 | for (auto previousOp : previousOps) { | 542 | for (auto previousOp : previousOps) { |
| 522 | for (auto followingOp : followingOps) { | 543 | for (auto followingOp : followingOps) { |
| 523 | fusableOpInfoMap[previousOp].conflictList.insert(followingOp); | 544 | fusableOpInfoMap[previousOp].conflictList.insert(followingOp); |
| @@ -1237,6 +1258,10 @@ void AutoVectorizeV2::planFuseSiblingForLeafNodes( | |||
| 1237 | if (llvm::any_of(leafNodeGroup, isVsstbPatternTransposeOp) && | 1258 | if (llvm::any_of(leafNodeGroup, isVsstbPatternTransposeOp) && |
| 1238 | hasRankReducingIndexingMap(leafNode)) | 1259 | hasRankReducingIndexingMap(leafNode)) |
| 1239 | continue; | 1260 | continue; |
| 1261 | + // Don't fuse vsstb leaf into a group with rank-reducing indexing_map. | ||
| 1262 | + if (llvm::any_of(leafNodeGroup, hasRankReducingIndexingMap) && | ||
| 1263 | + isVsstbPatternTransposeOp(leafNode)) | ||
| 1264 | + continue; | ||
| 1240 | // All leafNodes within a group have the same shape and do not conflict | 1265 | // All leafNodes within a group have the same shape and do not conflict |
| 1241 | // with each other. | 1266 | // with each other. |
| 1242 | auto leafNodeInfo = fusableOpInfoMap[leafNode]; | 1267 | auto leafNodeInfo = fusableOpInfoMap[leafNode]; |
| @@ -5,10 +5,17 @@ | |||
| 5 | // loops (scf.for barrier). | 5 | // loops (scf.for barrier). |
| 6 | // producer stays standalone loop(no fusion opportunity). | 6 | // producer stays standalone loop(no fusion opportunity). |
| 7 | // CHECK-LABEL: func.func @many_users_no_fusion_opportunity | 7 | // CHECK-LABEL: func.func @many_users_no_fusion_opportunity |
| 8 | -// CHECK: "outlined-loop-target-3" | 8 | + |
| 9 | -// CHECK: "outlined-loop-target-1" | 9 | +// CHECK: scf.for |
| 10 | +// CHECK: scf.for | ||
| 11 | +// CHECK-COUNT-2: arith.mulf | ||
| 12 | +// CHECK: {reductionLoop} | ||
| 10 | // CHECK: "outlined-loop-target-2" | 13 | // CHECK: "outlined-loop-target-2" |
| 11 | -// CHECK-NOT: "outlined-loop-target-4" | 14 | + |
| 15 | +// CHECK: scf.for | ||
| 16 | +// CHECK-COUNT-2: arith.addf | ||
| 17 | +// CHECK: "outlined-loop-target-1" | ||
| 18 | +// CHECK-NOT: "outlined-loop-target-3" | ||
| 12 | 19 | ||
| 13 | #map = affine_map<(d0, d1) -> (d0, d1)> | 20 | #map = affine_map<(d0, d1) -> (d0, d1)> |
| 14 | #map1 = affine_map<(d0, d1) -> (d0)> | 21 | #map1 = affine_map<(d0, d1) -> (d0)> |
| @@ -24,8 +31,8 @@ module { | |||
| 24 | %producer = linalg.generic {indexing_maps = [#map, #map1, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0, %arg1 : tensor<1x16xi32>, tensor<1xf32>) outs(%1 : tensor<1xf32>) { | 31 | %producer = linalg.generic {indexing_maps = [#map, #map1, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0, %arg1 : tensor<1x16xi32>, tensor<1xf32>) outs(%1 : tensor<1xf32>) { |
| 25 | ^bb0(%in: i32, %in_0: f32, %out: f32): | 32 | ^bb0(%in: i32, %in_0: f32, %out: f32): |
| 26 | %7 = arith.sitofp %in : i32 to f32 | 33 | %7 = arith.sitofp %in : i32 to f32 |
| 27 | - %8 = arith.addf %7, %in_0 : f32 | 34 | + %8 = arith.mulf %7, %in_0 : f32 |
| 28 | - %9 = arith.addf %8, %out : f32 | 35 | + %9 = arith.mulf %8, %out : f32 |
| 29 | linalg.yield %9 : f32 | 36 | linalg.yield %9 : f32 |
| 30 | } -> tensor<1xf32> | 37 | } -> tensor<1xf32> |
| 31 | %3 = tensor.empty() : tensor<1xf32> | 38 | %3 = tensor.empty() : tensor<1xf32> |