已合并
[AscendNPU-IR] Reapply "enable flatten by default" (!2326) & fix performance degradation caused by this patch #2583
kiansitov创建于 6 天前
[AscendNPU-IR] Reapply "enable flatten by default" (!2326) & fix performance degradation caused by this patch #2583
已合并
共 8 个文件变更+378-21
| @@ -67,6 +67,123 @@ struct FoldTransferReadAfterWriteAndInsertSlice | |||
| 67 | } | 67 | } |
| 68 | }; | 68 | }; |
| 69 | 69 | ||
| 70 | /// Return the sizes of the leading all-true region of `mask`, if they are | ||
| 71 | /// statically known. Supports `vector.constant_mask` and `vector.create_mask` | ||
| 72 | /// with constant bounds. | ||
| 73 | SmallVector<int64_t, 4> getStaticMaskSizes(Value mask) { | ||
| 74 | SmallVector<int64_t, 4> sizes; | ||
| 75 | |||
| 76 | if (auto constMask = mask.getDefiningOp<vector::ConstantMaskOp>()) { | ||
| 77 | for (Attribute attr : constMask.getMaskDimSizes().getValue()) | ||
| 78 | sizes.push_back(llvm::cast<IntegerAttr>(attr).getInt()); | ||
| 79 | |||
| 80 | return sizes; | ||
| 81 | } | ||
| 82 | |||
| 83 | if (auto createMask = mask.getDefiningOp<vector::CreateMaskOp>()) { | ||
| 84 | for (Value bound : createMask.getOperands()) { | ||
| 85 | std::optional<int64_t> cst = getConstantIntValue(bound); | ||
| 86 | if (!cst) | ||
| 87 | return {}; | ||
| 88 | |||
| 89 | sizes.push_back(*cst); | ||
| 90 | } | ||
| 91 | } | ||
| 92 | |||
| 93 | return sizes; | ||
| 94 | } | ||
| 95 | |||
| 96 | /// Replace the transfer_read by a broadcast of the source vector of the | ||
| 97 | /// transfer_write, when the read vector is wider than the written one and the | ||
| 98 | /// extra lanes are cut off by a static mask. | ||
| 99 | /// Example: | ||
| 100 | /// ``` | ||
| 101 | /// %6 = vector.multi_reduction <add>, %4, %5 [2] | ||
| 102 | /// : vector<1x1x64xf32> to vector<1x1xf32> | ||
| 103 | /// %7 = vector.transfer_write %6, %extracted_slice_3[%c0, %c0] | ||
| 104 | /// : vector<1x1xf32>, tensor<1x1xf32> | ||
| 105 | /// %8 = vector.constant_mask [1, 1] : vector<1x64xi1> | ||
| 106 | /// %10 = vector.transfer_read %7[%c0, %c0], %cst_0, %8 | ||
| 107 | /// : tensor<1x1xf32>, vector<1x64xf32> | ||
| 108 | /// %11 = arith.divf %9, %10 : vector<1x64xf32> | ||
| 109 | /// ``` | ||
| 110 | /// To: | ||
| 111 | /// ``` | ||
| 112 | /// %6 = vector.multi_reduction <add>, %4, %5 [2] | ||
| 113 | /// : vector<1x1x64xf32> to vector<1x1xf32> | ||
| 114 | /// %10 = vector.broadcast %6 : vector<1x1xf32> to vector<1x64xf32> | ||
| 115 | /// %11 = arith.divf %9, %10 : vector<1x64xf32> | ||
| 116 | /// ``` | ||
| 117 | struct FoldWidenedTransferReadAfterWrite | ||
| 118 | : public OpRewritePattern<vector::TransferReadOp> { | ||
| 119 | using OpRewritePattern::OpRewritePattern; | ||
| 120 | |||
| 121 | LogicalResult matchAndRewrite(vector::TransferReadOp readOp, | ||
| 122 | PatternRewriter &rewriter) const override { | ||
| 123 | if (readOp.hasOutOfBoundsDim() || | ||
| 124 | !llvm::isa<RankedTensorType>(readOp.getShapedType())) | ||
| 125 | return failure(); | ||
| 126 | |||
| 127 | auto defWrite = readOp.getSource().getDefiningOp<vector::TransferWriteOp>(); | ||
| 128 | if (!defWrite) | ||
| 129 | return failure(); | ||
| 130 | |||
| 131 | // The write has to define every element of the region it claims to write. | ||
| 132 | if (defWrite.getMask() || defWrite.hasOutOfBoundsDim()) | ||
| 133 | return failure(); | ||
| 134 | |||
| 135 | // Same location, same layout. Restricting to minor identity maps keeps the | ||
| 136 | // element correspondence trivial once the vector shapes differ. | ||
| 137 | if (readOp.getIndices() != defWrite.getIndices() || | ||
| 138 | readOp.getPermutationMap() != defWrite.getPermutationMap() || | ||
| 139 | !readOp.getPermutationMap().isMinorIdentity()) | ||
| 140 | return failure(); | ||
| 141 | |||
| 142 | VectorType readType = readOp.getVectorType(); | ||
| 143 | VectorType writeType = defWrite.getVectorType(); | ||
| 144 | if (readType.getElementType() != writeType.getElementType() || | ||
| 145 | readType.getRank() != writeType.getRank() || readType.getRank() == 0) | ||
| 146 | return failure(); | ||
| 147 | |||
| 148 | // Every element consumed by the read must have been produced by the write, | ||
| 149 | // i.e. the masked-in region of the read has to fit into the written vector. | ||
| 150 | SmallVector<int64_t, 4> activeSizes(readType.getShape()); | ||
| 151 | |||
| 152 | if (Value mask = readOp.getMask()) { | ||
| 153 | auto maskSizes = getStaticMaskSizes(mask); | ||
| 154 | if (maskSizes.empty() || maskSizes.size() != activeSizes.size()) | ||
| 155 | return failure(); | ||
| 156 | |||
| 157 | activeSizes = maskSizes; | ||
| 158 | } | ||
| 159 | |||
| 160 | for (int64_t dim = 0, rank = readType.getRank(); dim < rank; ++dim) { | ||
| 161 | if (activeSizes[dim] > writeType.getDimSize(dim) || | ||
| 162 | writeType.getDimSize(dim) > readType.getDimSize(dim)) | ||
| 163 | return failure(); | ||
| 164 | } | ||
| 165 | |||
| 166 | if (readType == writeType) { | ||
| 167 | rewriter.replaceOp(readOp, defWrite.getVector()); | ||
| 168 | return success(); | ||
| 169 | } | ||
| 170 | |||
| 171 | // Widening. `vector.broadcast` can only stretch dimensions of size 1, and | ||
| 172 | // VecBroadcastOpPattern in VectorToHIVMAVE lowers a vector source only when | ||
| 173 | // it holds a single element and the result is a single row. | ||
| 174 | if (writeType.getNumElements() != 1 || | ||
| 175 | readType.getNumElements() != readType.getShape().back()) | ||
| 176 | return failure(); | ||
| 177 | |||
| 178 | Location loc = readOp->getLoc(); | ||
| 179 | vector::BroadcastOp broadcast = rewriter.create<vector::BroadcastOp>( | ||
| 180 | loc, readType, defWrite.getVector()); | ||
| 181 | rewriter.replaceOp(readOp, broadcast); | ||
| 182 | |||
| 183 | return success(); | ||
| 184 | } | ||
| 185 | }; | ||
| 186 | |||
| 70 | struct RemoveRedundantWriteAndReadPairPass | 187 | struct RemoveRedundantWriteAndReadPairPass |
| 71 | : public impl::RemoveRedundantWriteAndReadPairBase< | 188 | : public impl::RemoveRedundantWriteAndReadPairBase< |
| 72 | RemoveRedundantWriteAndReadPairPass> { | 189 | RemoveRedundantWriteAndReadPairPass> { |
| @@ -80,6 +197,7 @@ void RemoveRedundantWriteAndReadPairPass::runOnOperation() { | |||
| 80 | auto *ctx = &getContext(); | 197 | auto *ctx = &getContext(); |
| 81 | RewritePatternSet patterns(ctx); | 198 | RewritePatternSet patterns(ctx); |
| 82 | patterns.add<FoldTransferReadAfterWriteAndInsertSlice>(ctx); | 199 | patterns.add<FoldTransferReadAfterWriteAndInsertSlice>(ctx); |
| 200 | patterns.add<FoldWidenedTransferReadAfterWrite>(ctx); | ||
| 83 | 201 | ||
| 84 | if (failed(applyPatternsGreedily(func, std::move(patterns)))) { | 202 | if (failed(applyPatternsGreedily(func, std::move(patterns)))) { |
| 85 | signalPassFailure(); | 203 | signalPassFailure(); |
| @@ -51,14 +51,12 @@ static void hivmAVEOptimizationPipeline( | |||
| 51 | OptimizeReductionLoopHIVMAVEOptions optimizeReductionLoopOptions; | 51 | OptimizeReductionLoopHIVMAVEOptions optimizeReductionLoopOptions; |
| 52 | optimizeReductionLoopOptions.maxSplit = | 52 | optimizeReductionLoopOptions.maxSplit = |
| 53 | hivmAVEPipelineOptions.maxReductionSplitNum; | 53 | hivmAVEPipelineOptions.maxReductionSplitNum; |
| 54 | // Vsstb packing depends on adjacent store order; reduction splitting may | ||
| 55 | // pair non-adjacent IVs (e.g. i and i + half) and hide that pattern. | ||
| 56 | pm.nest<func::FuncOp>().addPass(hivmave::createProcessVsstbPass()); | ||
| 57 | pm.nest<func::FuncOp>().addPass( | 54 | pm.nest<func::FuncOp>().addPass( |
| 58 | hivmave::createOptimizeReductionLoopHIVMAVEPass( | 55 | hivmave::createOptimizeReductionLoopHIVMAVEPass( |
| 59 | optimizeReductionLoopOptions)); | 56 | optimizeReductionLoopOptions)); |
| 60 | if (hivmAVEPipelineOptions.enableAveLoopOptimize) | 57 | if (hivmAVEPipelineOptions.enableAveLoopOptimize) |
| 61 | pm.nest<func::FuncOp>().addPass(hivmave::createAveLoopOptimizePass()); | 58 | pm.nest<func::FuncOp>().addPass(hivmave::createAveLoopOptimizePass()); |
| 59 | pm.nest<func::FuncOp>().addPass(hivmave::createProcessVsstbPass()); | ||
| 62 | pm.nest<func::FuncOp>().addPass(hivmave::createLegalizeOptHIVMAVEPass()); | 60 | pm.nest<func::FuncOp>().addPass(hivmave::createLegalizeOptHIVMAVEPass()); |
| 63 | pm.nest<func::FuncOp>().addPass( | 61 | pm.nest<func::FuncOp>().addPass( |
| 64 | hivmave::createReplaceWithVectorScalarPass()); | 62 | hivmave::createReplaceWithVectorScalarPass()); |
| @@ -71,8 +71,11 @@ void PropagateReshapePass::runOnOperation() { | |||
| 71 | options.skipScope = skipScope; | 71 | options.skipScope = skipScope; |
| 72 | options.maxUnitDimsForPropagation = maxUnitDimsForPropagation; | 72 | options.maxUnitDimsForPropagation = maxUnitDimsForPropagation; |
| 73 | 73 | ||
| 74 | // Do not apply pass if flattening will not be applied. | ||
| 74 | if (auto coreType = mlir::hivm::queryFuncCoreType(f); | 75 | if (auto coreType = mlir::hivm::queryFuncCoreType(f); |
| 75 | coreType && *coreType == mlir::hivm::TFuncCoreType::AIC) | 76 | coreType.has_value() && |
| 77 | (coreType.value() == mlir::hivm::TFuncCoreType::AIC || | ||
| 78 | coreType.value() == mlir::hivm::TFuncCoreType::AIV)) | ||
| 76 | return; | 79 | return; |
| 77 | 80 | ||
| 78 | if (options.forRegbased && options.skipScope && hasScopeOperation(f)) | 81 | if (options.forRegbased && options.skipScope && hasScopeOperation(f)) |
| @@ -107,4 +110,4 @@ createPropagateReshapePass(const PropagateReshapeOptions &options) { | |||
| 107 | } | 110 | } |
| 108 | 111 | ||
| 109 | } // namespace tensor | 112 | } // namespace tensor |
| 110 | } // namespace mlir | 113 | } // namespace mlir |
| @@ -1084,6 +1084,13 @@ class TransferReadToGatheringLoadPattern | |||
| 1084 | for (unsigned i = 0; i < destShape.size(); ++i) | 1084 | for (unsigned i = 0; i < destShape.size(); ++i) |
| 1085 | totalDestSize *= destShape[i]; | 1085 | totalDestSize *= destShape[i]; |
| 1086 | 1086 | ||
| 1087 | // permMap maps the memref index space onto the result vector index space, | ||
| 1088 | // so recovering the memref indices of a result element requires the | ||
| 1089 | // *inverse* map. For a 2-D transpose they're the same, but not for a | ||
| 1090 | // 3-D one, like (d0, d1, d2) -> (d1, d2, d0) | ||
| 1091 | AffineMap invPermMap = inversePermutation(permMap); | ||
| 1092 | assert(invPermMap && "expected an invertible permutation map"); | ||
| 1093 | |||
| 1087 | unsigned elementBitWidth = memrefType.getElementTypeBitWidth(); | 1094 | unsigned elementBitWidth = memrefType.getElementTypeBitWidth(); |
| 1088 | SmallVector<int64_t, 0> composeIndices(destShape.size(), 0); | 1095 | SmallVector<int64_t, 0> composeIndices(destShape.size(), 0); |
| 1089 | gatherIndices.reserve(totalDestSize); | 1096 | gatherIndices.reserve(totalDestSize); |
| @@ -1101,7 +1108,7 @@ class TransferReadToGatheringLoadPattern | |||
| 1101 | composeIndices[dim - 1] += 1; | 1108 | composeIndices[dim - 1] += 1; |
| 1102 | } | 1109 | } |
| 1103 | 1110 | ||
| 1104 | auto composeResult = permMap.compose(composeIndices); | 1111 | auto composeResult = invPermMap.compose(composeIndices); |
| 1105 | if (composeResult.size() != strides.size()) | 1112 | if (composeResult.size() != strides.size()) |
| 1106 | llvm::report_fatal_error("Unexpected dimension mismatch"); | 1113 | llvm::report_fatal_error("Unexpected dimension mismatch"); |
| 1107 | 1114 | ||
| @@ -303,24 +303,13 @@ static void buildDelayedHFusionRegBaseVectorizePipeline( | |||
| 303 | if (config.getDisableHfusionVectorize()) { | 303 | if (config.getDisableHfusionVectorize()) { |
| 304 | return; | 304 | return; |
| 305 | } | 305 | } |
| 306 | // inferMixedCV populates enableMixedCV before this delayed HFusion pipeline | ||
| 307 | // is built; adjust only the local HFusion options consumed by flatten. | ||
| 308 | BiShengIRCompileMainConfig hfusionConfig = config; | ||
| 309 | auto ®isteredOptions = llvm::cl::getRegisteredOptions(); | ||
| 310 | auto enableFlattenOpt = registeredOptions.find("enable-flatten"); | ||
| 311 | bool hasExplicitEnableFlatten = | ||
| 312 | enableFlattenOpt != registeredOptions.end() && | ||
| 313 | enableFlattenOpt->second->getNumOccurrences() != 0; | ||
| 314 | if (hfusionConfig.shouldEnableMixedCV() && !hasExplicitEnableFlatten) { | ||
| 315 | hfusionConfig.setEnableFlatten(false); | ||
| 316 | } | ||
| 317 | 306 | ||
| 318 | HIVMAggregatedDecomposeOpOptions decomposeOption; | 307 | HIVMAggregatedDecomposeOpOptions decomposeOption; |
| 319 | decomposeOption.decomposePhase = bishengir::DecomposePhase::NO_CONSTRAINT; | 308 | decomposeOption.decomposePhase = bishengir::DecomposePhase::NO_CONSTRAINT; |
| 320 | pm.nest<func::FuncOp>().addPass( | 309 | pm.nest<func::FuncOp>().addPass( |
| 321 | mlir::hivm::createHIVMAggregatedDecomposeOpPass(decomposeOption)); | 310 | mlir::hivm::createHIVMAggregatedDecomposeOpPass(decomposeOption)); |
| 322 | hfusion::HFusionPipelineOptions hfusionPipelineOptions; | 311 | hfusion::HFusionPipelineOptions hfusionPipelineOptions; |
| 323 | setupHFusionPipelineOptions(hfusionPipelineOptions, hfusionConfig); | 312 | setupHFusionPipelineOptions(hfusionPipelineOptions, config); |
| 324 | ExecutionEngineHIVMToUpstreamConversionOptions upstreamOptions; | 313 | ExecutionEngineHIVMToUpstreamConversionOptions upstreamOptions; |
| 325 | upstreamOptions.convertToNamedOp = | 314 | upstreamOptions.convertToNamedOp = |
| 326 | hacc::utils::isRegBasedArch(config.getTarget()); | 315 | hacc::utils::isRegBasedArch(config.getTarget()); |
| @@ -13,7 +13,7 @@ | |||
| 13 | #map1 = affine_map<(d0) -> (d0 * 28672 + 8192)> | 13 | #map1 = affine_map<(d0) -> (d0 * 28672 + 8192)> |
| 14 | #map2 = affine_map<(d0) -> (d0 * 28672 + 12288)> | 14 | #map2 = affine_map<(d0) -> (d0 * 28672 + 12288)> |
| 15 | module { | 15 | module { |
| 16 | func.func @scf_for_propagate(%arg0: memref<?xi8>, %arg1: memref<?xf32>, %arg2: memref<?xf32>, %arg3: f32, %arg4: i32, %arg5: tensor<64x64xf32>, %arg6: tensor<64x64xf32>, %arg7: tensor<64x32xf32>, %arg8: tensor<64xf32>, %arg9: tensor<64xf32>, %arg10: tensor<64xf32>, %arg11: i64, %arg12: index, %arg13: i32, %arg14: f32, %arg15: i32, %arg16: i64, %arg17: i64, %arg18: i32, %arg19: i32, %arg20: index, %arg21: index, %arg22: f32, %arg23: f32, %arg24: i32) attributes {WorkspaceArgIdx = 0 : i64, func_dyn_memref_args = dense<[false, true, true, true, true, true, true, false, false, false, false]> : vector<11xi1>, global_kernel = "local", hacc.entry, hacc.function_kind = #hacc.function_kind<DEVICE>, hivm.func_core_type = #hivm.func_core_type<AIV>, hivm.part_of_mix, mix_mode = "mix"} { | 16 | func.func @scf_for_propagate(%arg0: memref<?xi8>, %arg1: memref<?xf32>, %arg2: memref<?xf32>, %arg3: f32, %arg4: i32, %arg5: tensor<64x64xf32>, %arg6: tensor<64x64xf32>, %arg7: tensor<64x32xf32>, %arg8: tensor<64xf32>, %arg9: tensor<64xf32>, %arg10: tensor<64xf32>, %arg11: i64, %arg12: index, %arg13: i32, %arg14: f32, %arg15: i32, %arg16: i64, %arg17: i64, %arg18: i32, %arg19: i32, %arg20: index, %arg21: index, %arg22: f32, %arg23: f32, %arg24: i32) attributes {WorkspaceArgIdx = 0 : i64, func_dyn_memref_args = dense<[false, true, true, true, true, true, true, false, false, false, false]> : vector<11xi1>, global_kernel = "local", hacc.entry, hacc.function_kind = #hacc.function_kind<DEVICE>, hivm.func_core_type = #hivm.func_core_type<MIX>, hivm.part_of_mix, mix_mode = "mix"} { |
| 17 | scf.for %arg25 = %arg19 to %arg18 step %arg24 : i32 { | 17 | scf.for %arg25 = %arg19 to %arg18 step %arg24 : i32 { |
| 18 | %0 = arith.divsi %arg25, %arg18 : i32 | 18 | %0 = arith.divsi %arg25, %arg18 : i32 |
| 19 | %1 = arith.remsi %arg25, %arg18 : i32 | 19 | %1 = arith.remsi %arg25, %arg18 : i32 |
| @@ -87,4 +87,4 @@ module { | |||
| 87 | } | 87 | } |
| 88 | return | 88 | return |
| 89 | } | 89 | } |
| 90 | } | 90 | } |
| @@ -68,4 +68,206 @@ func.func @shape_cast_different_vector_shape(%arg0: tensor<2x64xf16>, %arg1: ten | |||
| 68 | // CHECK: %[[ADD:.*]] = arith.addf | 68 | // CHECK: %[[ADD:.*]] = arith.addf |
| 69 | // CHECK-NOT: vector.transfer_read | 69 | // CHECK-NOT: vector.transfer_read |
| 70 | // CHECK: %[[CAST:.*]] = vector.shape_cast %[[ADD]] : vector<1x64xf16> to vector<64xf16> | 70 | // CHECK: %[[CAST:.*]] = vector.shape_cast %[[ADD]] : vector<1x64xf16> to vector<64xf16> |
| 71 | // CHECK: arith.mulf %[[CAST]], %[[CAST]] | 71 | // CHECK: arith.mulf %[[CAST]], %[[CAST]] |
| 72 | |||
| 73 | // Test 3: widened masked read right after the write -- the read is replaced by a | ||
| 74 | // broadcast of the written vector and the write becomes dead. | ||
| 75 | func.func @fold_widened_read_after_write(%arg0: tensor<16x1xf32>, %arg1: tensor<16x1x16xf32>, %arg2: tensor<16x1xf32>, %arg7: tensor<16x1xf32>) -> tensor<16x1xf32> attributes {hivm.vector_function} { | ||
| 76 | %cst = arith.constant dense<0.000000e+00> : vector<1x1x64xf32> | ||
| 77 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 78 | %c1 = arith.constant 1 : index | ||
| 79 | %c16 = arith.constant 16 : index | ||
| 80 | %c0 = arith.constant 0 : index | ||
| 81 | %0 = scf.for %arg8 = %c0 to %c16 step %c1 iter_args(%arg9 = %arg7) -> (tensor<16x1xf32>) { | ||
| 82 | %extracted_slice = tensor.extract_slice %arg9[%arg8, 0] [1, 1] [1, 1] : tensor<16x1xf32> to tensor<1x1xf32> | ||
| 83 | %extracted_slice_1 = tensor.extract_slice %arg0[%arg8, 0] [1, 1] [1, 1] : tensor<16x1xf32> to tensor<1x1xf32> | ||
| 84 | %extracted_slice_2 = tensor.extract_slice %arg1[%arg8, 0, 0] [1, 1, 16] [1, 1, 1] : tensor<16x1x16xf32> to tensor<1x1x16xf32> | ||
| 85 | %extracted_slice_3 = tensor.extract_slice %arg2[%arg8, 0] [1, 1] [1, 1] : tensor<16x1xf32> to tensor<1x1xf32> | ||
| 86 | %1 = vector.constant_mask [1, 1, 16] : vector<1x1x64xi1> | ||
| 87 | %2 = vector.transfer_read %extracted_slice_2[%c0, %c0, %c0], %cst_0, %1 {in_bounds = [true, true, true]} : tensor<1x1x16xf32>, vector<1x1x64xf32> | ||
| 88 | %3 = arith.select %1, %2, %cst : vector<1x1x64xi1>, vector<1x1x64xf32> | ||
| 89 | %4 = arith.addf %3, %cst {reductionOp} : vector<1x1x64xf32> | ||
| 90 | %5 = vector.transfer_read %extracted_slice_3[%c0, %c0], %cst_0 {in_bounds = [true, true]} : tensor<1x1xf32>, vector<1x1xf32> | ||
| 91 | %6 = vector.multi_reduction <add>, %4, %5 {withoutInitMergeOp} [2] : vector<1x1x64xf32> to vector<1x1xf32> | ||
| 92 | %7 = vector.transfer_write %6, %extracted_slice_3[%c0, %c0] {in_bounds = [true, true]} : vector<1x1xf32>, tensor<1x1xf32> | ||
| 93 | %8 = vector.constant_mask [1, 1] : vector<1x64xi1> | ||
| 94 | %9 = vector.transfer_read %extracted_slice_1[%c0, %c0], %cst_0, %8 {in_bounds = [true, true]} : tensor<1x1xf32>, vector<1x64xf32> | ||
| 95 | %10 = vector.transfer_read %7[%c0, %c0], %cst_0, %8 {in_bounds = [true, true]} : tensor<1x1xf32>, vector<1x64xf32> | ||
| 96 | %11 = arith.divf %9, %10 : vector<1x64xf32> | ||
| 97 | %12 = vector.transfer_write %11, %extracted_slice[%c0, %c0], %8 {in_bounds = [true, true]} : vector<1x64xf32>, tensor<1x1xf32> | ||
| 98 | %inserted_slice = tensor.insert_slice %12 into %arg9[%arg8, 0] [1, 1] [1, 1] : tensor<1x1xf32> into tensor<16x1xf32> | ||
| 99 | scf.yield %inserted_slice : tensor<16x1xf32> | ||
| 100 | } | ||
| 101 | return %0 : tensor<16x1xf32> | ||
| 102 | } | ||
| 103 | |||
| 104 | // CHECK-LABEL: @fold_widened_read_after_write | ||
| 105 | // CHECK: %[[RED:.*]] = vector.multi_reduction <add> | ||
| 106 | // CHECK-NOT: vector.transfer_write %[[RED]] | ||
| 107 | // CHECK: %[[BC:.*]] = vector.broadcast %[[RED]] : vector<1x1xf32> to vector<1x64xf32> | ||
| 108 | // CHECK: arith.divf %{{.*}}, %[[BC]] : vector<1x64xf32> | ||
| 109 | |||
| 110 | // Test 4: rank 4, constant_mask. The written vector<1x1x1x1xf32> covers the | ||
| 111 | // whole masked-in region [1, 1, 1, 1] of the read, so the read is replaced by a | ||
| 112 | // broadcast and the write dies. | ||
| 113 | func.func @fold_widened_read_rank_4(%arg0: tensor<8x4x2x1xf32>, %arg1: tensor<8x4x2x64xf32>, %arg2: tensor<8x4x2x64xf32>) -> tensor<8x4x2x64xf32> attributes {hivm.vector_function} { | ||
| 114 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 115 | %c0 = arith.constant 0 : index | ||
| 116 | %c1 = arith.constant 1 : index | ||
| 117 | %c8 = arith.constant 8 : index | ||
| 118 | %0 = scf.for %arg3 = %c0 to %c8 step %c1 iter_args(%arg4 = %arg2) -> (tensor<8x4x2x64xf32>) { | ||
| 119 | %scale_slice = tensor.extract_slice %arg0[%arg3, 0, 0, 0] [1, 1, 1, 1] [1, 1, 1, 1] : tensor<8x4x2x1xf32> to tensor<1x1x1x1xf32> | ||
| 120 | %row_slice = tensor.extract_slice %arg1[%arg3, 0, 0, 0] [1, 1, 1, 64] [1, 1, 1, 1] : tensor<8x4x2x64xf32> to tensor<1x1x1x64xf32> | ||
| 121 | %out_slice = tensor.extract_slice %arg4[%arg3, 0, 0, 0] [1, 1, 1, 64] [1, 1, 1, 1] : tensor<8x4x2x64xf32> to tensor<1x1x1x64xf32> | ||
| 122 | %1 = vector.transfer_read %scale_slice[%c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true, true]} : tensor<1x1x1x1xf32>, vector<1x1x1x1xf32> | ||
| 123 | %2 = arith.addf %1, %1 : vector<1x1x1x1xf32> | ||
| 124 | %3 = vector.transfer_write %2, %scale_slice[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<1x1x1x1xf32>, tensor<1x1x1x1xf32> | ||
| 125 | %mask = vector.constant_mask [1, 1, 1, 1] : vector<1x1x1x64xi1> | ||
| 126 | %4 = vector.transfer_read %3[%c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true]} : tensor<1x1x1x1xf32>, vector<1x1x1x64xf32> | ||
| 127 | %5 = vector.transfer_read %row_slice[%c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true, true]} : tensor<1x1x1x64xf32>, vector<1x1x1x64xf32> | ||
| 128 | %6 = arith.divf %5, %4 : vector<1x1x1x64xf32> | ||
| 129 | %7 = vector.transfer_write %6, %out_slice[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<1x1x1x64xf32>, tensor<1x1x1x64xf32> | ||
| 130 | %inserted_slice = tensor.insert_slice %7 into %arg4[%arg3, 0, 0, 0] [1, 1, 1, 64] [1, 1, 1, 1] : tensor<1x1x1x64xf32> into tensor<8x4x2x64xf32> | ||
| 131 | scf.yield %inserted_slice : tensor<8x4x2x64xf32> | ||
| 132 | } | ||
| 133 | return %0 : tensor<8x4x2x64xf32> | ||
| 134 | } | ||
| 135 | |||
| 136 | // CHECK-LABEL: func.func @fold_widened_read_rank_4 | ||
| 137 | // CHECK: %[[ADD:.*]] = arith.addf | ||
| 138 | // CHECK-NOT: vector.transfer_write %[[ADD]] | ||
| 139 | // CHECK: %[[BC:.*]] = vector.broadcast %[[ADD]] : vector<1x1x1x1xf32> to vector<1x1x1x64xf32> | ||
| 140 | // CHECK: %[[ROW:.*]] = vector.transfer_read | ||
| 141 | // CHECK: arith.divf %[[ROW]], %[[BC]] | ||
| 142 | |||
| 143 | // Test 5: rank 5, create_mask with constant bounds. Same fold, exercises the | ||
| 144 | // create_mask branch of getStaticMaskSizes. | ||
| 145 | func.func @fold_widened_read_rank_5(%arg0: tensor<2x1x1x1x1xf32>, %arg1: tensor<2x1x1x1x128xf32>) -> vector<1x1x1x1x128xf32> attributes {hivm.vector_function} { | ||
| 146 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 147 | %c0 = arith.constant 0 : index | ||
| 148 | %c1 = arith.constant 1 : index | ||
| 149 | %scale_slice = tensor.extract_slice %arg0[0, 0, 0, 0, 0] [1, 1, 1, 1, 1] [1, 1, 1, 1, 1] : tensor<2x1x1x1x1xf32> to tensor<1x1x1x1x1xf32> | ||
| 150 | %row_slice = tensor.extract_slice %arg1[0, 0, 0, 0, 0] [1, 1, 1, 1, 128] [1, 1, 1, 1, 1] : tensor<2x1x1x1x128xf32> to tensor<1x1x1x1x128xf32> | ||
| 151 | %0 = vector.transfer_read %scale_slice[%c0, %c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true, true, true]} : tensor<1x1x1x1x1xf32>, vector<1x1x1x1x1xf32> | ||
| 152 | %1 = arith.mulf %0, %0 : vector<1x1x1x1x1xf32> | ||
| 153 | %2 = vector.transfer_write %1, %scale_slice[%c0, %c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x1x1x1xf32>, tensor<1x1x1x1x1xf32> | ||
| 154 | %mask = vector.create_mask %c1, %c1, %c1, %c1, %c1 : vector<1x1x1x1x128xi1> | ||
| 155 | %3 = vector.transfer_read %2[%c0, %c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true, true]} : tensor<1x1x1x1x1xf32>, vector<1x1x1x1x128xf32> | ||
| 156 | %4 = vector.transfer_read %row_slice[%c0, %c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true, true, true]} : tensor<1x1x1x1x128xf32>, vector<1x1x1x1x128xf32> | ||
| 157 | %5 = arith.subf %4, %3 : vector<1x1x1x1x128xf32> | ||
| 158 | return %5 : vector<1x1x1x1x128xf32> | ||
| 159 | } | ||
| 160 | |||
| 161 | // CHECK-LABEL: func.func @fold_widened_read_rank_5 | ||
| 162 | // CHECK: %[[MUL:.*]] = arith.mulf | ||
| 163 | // CHECK-NOT: vector.transfer_write %[[MUL]] | ||
| 164 | // CHECK: %[[BC:.*]] = vector.broadcast %[[MUL]] : vector<1x1x1x1x1xf32> to vector<1x1x1x1x128xf32> | ||
| 165 | // CHECK: %[[ROW:.*]] = vector.transfer_read | ||
| 166 | // CHECK: arith.subf %[[ROW]], %[[BC]] | ||
| 167 | |||
| 168 | // Negative test 1: the read consumes more elements than the write produced | ||
| 169 | // (mask is [1, 2] while only 1x1 was written), so nothing must be folded. | ||
| 170 | func.func @no_fold_read_escapes_write(%arg0: tensor<1x2xf32>, %v: vector<1x1xf32>) -> vector<1x64xf32> attributes {hivm.vector_function} { | ||
| 171 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 172 | %c0 = arith.constant 0 : index | ||
| 173 | %m = vector.constant_mask [1, 2] : vector<1x64xi1> | ||
| 174 | %w = vector.transfer_write %v, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<1x1xf32>, tensor<1x2xf32> | ||
| 175 | %r = vector.transfer_read %w[%c0, %c0], %cst_0, %m {in_bounds = [true, true]} : tensor<1x2xf32>, vector<1x64xf32> | ||
| 176 | return %r : vector<1x64xf32> | ||
| 177 | } | ||
| 178 | |||
| 179 | // CHECK-LABEL: @no_fold_read_escapes_write | ||
| 180 | // CHECK: vector.transfer_write | ||
| 181 | // CHECK: vector.transfer_read | ||
| 182 | // CHECK-NOT: vector.broadcast | ||
| 183 | |||
| 184 | // Negative test 2: rank 4, the read consumes elements the write never produced. The mask | ||
| 185 | // is [1, 1, 2, 1] while the write only covered 1 element in dim 2, so element | ||
| 186 | // [0, 0, 1, 0] still comes from the original tensor. Must not fold. | ||
| 187 | func.func @no_fold_read_escapes_write_rank_4(%arg0: tensor<1x1x2x1xf32>, %v: vector<1x1x1x1xf32>) -> vector<1x1x2x64xf32> attributes {hivm.vector_function} { | ||
| 188 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 189 | %c0 = arith.constant 0 : index | ||
| 190 | %0 = vector.transfer_write %v, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<1x1x1x1xf32>, tensor<1x1x2x1xf32> | ||
| 191 | %mask = vector.constant_mask [1, 1, 2, 1] : vector<1x1x2x64xi1> | ||
| 192 | %1 = vector.transfer_read %0[%c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true]} : tensor<1x1x2x1xf32>, vector<1x1x2x64xf32> | ||
| 193 | return %1 : vector<1x1x2x64xf32> | ||
| 194 | } | ||
| 195 | |||
| 196 | // CHECK-LABEL: func.func @no_fold_read_escapes_write_rank_4 | ||
| 197 | // CHECK: vector.transfer_write | ||
| 198 | // CHECK: vector.transfer_read | ||
| 199 | // CHECK-NOT: vector.broadcast | ||
| 200 | // CHECK: return | ||
| 201 | |||
| 202 | // Negative test 3: rank 4, the region checks pass but the result is not a single row | ||
| 203 | // (dim 2 is 4). VecBroadcastOpPattern in VectorToHIVMAVE builds the dup type | ||
| 204 | // from the trailing dimension only, so such a broadcast would not legalize. | ||
| 205 | // Must not fold. | ||
| 206 | func.func @no_fold_multi_row_result_rank_4(%arg0: tensor<1x1x1x1xf32>, %v: vector<1x1x1x1xf32>) -> vector<1x1x4x64xf32> attributes {hivm.vector_function} { | ||
| 207 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 208 | %c0 = arith.constant 0 : index | ||
| 209 | %0 = vector.transfer_write %v, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<1x1x1x1xf32>, tensor<1x1x1x1xf32> | ||
| 210 | %mask = vector.constant_mask [1, 1, 1, 1] : vector<1x1x4x64xi1> | ||
| 211 | %1 = vector.transfer_read %0[%c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true]} : tensor<1x1x1x1xf32>, vector<1x1x4x64xf32> | ||
| 212 | return %1 : vector<1x1x4x64xf32> | ||
| 213 | } | ||
| 214 | |||
| 215 | // CHECK-LABEL: func.func @no_fold_multi_row_result_rank_4 | ||
| 216 | // CHECK: vector.transfer_write | ||
| 217 | // CHECK: vector.transfer_read | ||
| 218 | // CHECK-NOT: vector.broadcast | ||
| 219 | // CHECK: return | ||
| 220 | |||
| 221 | // Negative test 4: rank 4, the written vector holds more than one element, so widening | ||
| 222 | // cannot be expressed as a single broadcast. Must not fold. | ||
| 223 | func.func @no_fold_multi_element_write_rank_4(%arg0: tensor<1x1x2x1xf32>, %v: vector<1x1x2x1xf32>) -> vector<1x1x2x64xf32> attributes {hivm.vector_function} { | ||
| 224 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 225 | %c0 = arith.constant 0 : index | ||
| 226 | %0 = vector.transfer_write %v, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<1x1x2x1xf32>, tensor<1x1x2x1xf32> | ||
| 227 | %mask = vector.constant_mask [1, 1, 2, 1] : vector<1x1x2x64xi1> | ||
| 228 | %1 = vector.transfer_read %0[%c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true]} : tensor<1x1x2x1xf32>, vector<1x1x2x64xf32> | ||
| 229 | return %1 : vector<1x1x2x64xf32> | ||
| 230 | } | ||
| 231 | |||
| 232 | // CHECK-LABEL: func.func @no_fold_multi_element_write_rank_4 | ||
| 233 | // CHECK: vector.transfer_write | ||
| 234 | // CHECK: vector.transfer_read | ||
| 235 | // CHECK-NOT: vector.broadcast | ||
| 236 | // CHECK: return | ||
| 237 | |||
| 238 | // Negative test 5: rank 4, the mask bound in dim 2 is dynamic, so the masked-in region | ||
| 239 | // cannot be proven to fit into the written vector. Must not fold. | ||
| 240 | func.func @no_fold_dynamic_mask_rank_4(%arg0: tensor<1x1x2x1xf32>, %v: vector<1x1x1x1xf32>, %n: index) -> vector<1x1x2x64xf32> attributes {hivm.vector_function} { | ||
| 241 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 242 | %c0 = arith.constant 0 : index | ||
| 243 | %c1 = arith.constant 1 : index | ||
| 244 | %0 = vector.transfer_write %v, %arg0[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<1x1x1x1xf32>, tensor<1x1x2x1xf32> | ||
| 245 | %mask = vector.create_mask %c1, %c1, %n, %c1 : vector<1x1x2x64xi1> | ||
| 246 | %1 = vector.transfer_read %0[%c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true]} : tensor<1x1x2x1xf32>, vector<1x1x2x64xf32> | ||
| 247 | return %1 : vector<1x1x2x64xf32> | ||
| 248 | } | ||
| 249 | |||
| 250 | // CHECK-LABEL: func.func @no_fold_dynamic_mask_rank_4 | ||
| 251 | // CHECK: vector.transfer_write | ||
| 252 | // CHECK: vector.transfer_read | ||
| 253 | // CHECK-NOT: vector.broadcast | ||
| 254 | // CHECK: return | ||
| 255 | |||
| 256 | // Negative test 6: rank 4, the write itself is masked, so it does not define the whole | ||
| 257 | // region the read consumes. Must not fold. | ||
| 258 | func.func @no_fold_masked_write_rank_4(%arg0: tensor<1x1x1x1xf32>, %v: vector<1x1x1x1xf32>, %n: index) -> vector<1x1x1x64xf32> attributes {hivm.vector_function} { | ||
| 259 | %cst_0 = arith.constant 0.000000e+00 : f32 | ||
| 260 | %c0 = arith.constant 0 : index | ||
| 261 | %c1 = arith.constant 1 : index | ||
| 262 | %wmask = vector.create_mask %c1, %c1, %c1, %n : vector<1x1x1x1xi1> | ||
| 263 | %0 = vector.transfer_write %v, %arg0[%c0, %c0, %c0, %c0], %wmask {in_bounds = [true, true, true, true]} : vector<1x1x1x1xf32>, tensor<1x1x1x1xf32> | ||
| 264 | %mask = vector.constant_mask [1, 1, 1, 1] : vector<1x1x1x64xi1> | ||
| 265 | %1 = vector.transfer_read %0[%c0, %c0, %c0, %c0], %cst_0, %mask {in_bounds = [true, true, true, true]} : tensor<1x1x1x1xf32>, vector<1x1x1x64xf32> | ||
| 266 | return %1 : vector<1x1x1x64xf32> | ||
| 267 | } | ||
| 268 | |||
| 269 | // CHECK-LABEL: func.func @no_fold_masked_write_rank_4 | ||
| 270 | // CHECK: vector.transfer_write | ||
| 271 | // CHECK: vector.transfer_read | ||
| 272 | // CHECK-NOT: vector.broadcast | ||
| 273 | // CHECK: return | ||
| @@ -0,0 +1,40 @@ | |||
| 1 | // RUN: bishengir-opt %s -normalize-vector -split-input-file | FileCheck %s | ||
| 2 | |||
| 3 | // CHECK-LABEL: func.func @gather_3d_cyclic_perm_masked | ||
| 4 | // CHECK-DAG: %[[PAD:.*]] = arith.constant dense<0.000000e+00> : vector<64xf32> | ||
| 5 | // CHECK-DAG: %[[INDEX:.*]] = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 0,{{.*}}]> : vector<64xi32> | ||
| 6 | // CHECK: %[[MASK:.*]] = vector.constant_mask [16] : vector<64xi1> | ||
| 7 | // CHECK: %[[GATHER:.*]] = vector.gather %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] [%[[INDEX]]], %[[MASK]], %[[PAD]] : memref<16x1x1xf32, strided<[16, 16, 1], offset: ?>, #hivm.address_space<ub>>, vector<64xi32>, vector<64xi1>, vector<64xf32> into vector<64xf32> | ||
| 8 | // CHECK: vector.transfer_write %[[GATHER]] | ||
| 9 | func.func @gather_3d_cyclic_perm_masked(%arg0: memref<16x1x16xf32, #hivm.address_space<ub>>, %arg1: memref<1x16x16xf32, #hivm.address_space<ub>>) attributes {hivm.func_core_type = #hivm.func_core_type<AIV>, hivm.vector_function, no_inline} { | ||
| 10 | %cst = arith.constant 0.000000e+00 : f32 | ||
| 11 | %c1 = arith.constant 1 : index | ||
| 12 | %c16 = arith.constant 16 : index | ||
| 13 | %c0 = arith.constant 0 : index | ||
| 14 | scf.for %arg2 = %c0 to %c16 step %c1 { | ||
| 15 | %subview = memref.subview %arg1[0, %arg2, 0] [1, 1, 16] [1, 1, 1] : memref<1x16x16xf32, #hivm.address_space<ub>> to memref<1x1x16xf32, strided<[256, 16, 1], offset: ?>, #hivm.address_space<ub>> | ||
| 16 | %subview_0 = memref.subview %arg0[0, 0, %arg2] [16, 1, 1] [1, 1, 1] : memref<16x1x16xf32, #hivm.address_space<ub>> to memref<16x1x1xf32, strided<[16, 16, 1], offset: ?>, #hivm.address_space<ub>> | ||
| 17 | %0 = vector.constant_mask [16, 1, 1] : vector<64x1x1xi1> | ||
| 18 | %1 = vector.transfer_read %subview_0[%c0, %c0, %c0], %cst, %0 {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1, d2) -> (d1, d2, d0)>} : memref<16x1x1xf32, strided<[16, 16, 1], offset: ?>, #hivm.address_space<ub>>, vector<1x1x64xf32> | ||
| 19 | %2 = vector.constant_mask [16] : vector<64xi1> | ||
| 20 | %subview_1 = memref.subview %subview[0, 0, 0] [1, 1, 16] [1, 1, 1] : memref<1x1x16xf32, strided<[256, 16, 1], offset: ?>, #hivm.address_space<ub>> to memref<16xf32, affine_map<(d0)[s0] -> (d0 + s0)>, #hivm.address_space<ub>> | ||
| 21 | %3 = vector.shape_cast %1 : vector<1x1x64xf32> to vector<64xf32> | ||
| 22 | vector.transfer_write %3, %subview_1[%c0], %2 {in_bounds = [true]} : vector<64xf32>, memref<16xf32, affine_map<(d0)[s0] -> (d0 + s0)>, #hivm.address_space<ub>> | ||
| 23 | } | ||
| 24 | return | ||
| 25 | } | ||
| 26 | |||
| 27 | // CHECK-LABEL: func.func @gather_3d_cyclic_perm_full_mask | ||
| 28 | // CHECK-DAG: %[[PAD:.*]] = arith.constant dense<0.000000e+00> : vector<32xf32> | ||
| 29 | // CHECK-DAG: %[[INDEX:.*]] = arith.constant dense<[0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29, 2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31]> : vector<32xi32> | ||
| 30 | // CHECK: %[[MASK:.*]] = vector.constant_mask [32] : vector<32xi1> | ||
| 31 | // CHECK: %[[GATHER:.*]] = vector.gather %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] [%[[INDEX]]], %[[MASK]], %[[PAD]] : memref<8x4x1xf32, #hivm.address_space<ub>>, vector<32xi32>, vector<32xi1>, vector<32xf32> into vector<32xf32> | ||
| 32 | // CHECK: vector.transfer_write %[[GATHER]] | ||
| 33 | func.func @gather_3d_cyclic_perm_full_mask(%arg0: memref<8x4x1xf32, #hivm.address_space<ub>>, %arg1: memref<32xf32, #hivm.address_space<ub>>) attributes {hivm.vector_function} { | ||
| 34 | %cst = arith.constant 0.000000e+00 : f32 | ||
| 35 | %c0 = arith.constant 0 : index | ||
| 36 | %0 = vector.transfer_read %arg0[%c0, %c0, %c0], %cst {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1, d2) -> (d1, d2, d0)>} : memref<8x4x1xf32, #hivm.address_space<ub>>, vector<4x1x8xf32> | ||
 | |||
| 37 | %1 = vector.shape_cast %0 : vector<4x1x8xf32> to vector<32xf32> | ||
| 38 | vector.transfer_write %1, %arg1[%c0] {in_bounds = [true]} : vector<32xf32>, memref<32xf32, #hivm.address_space<ub>> | ||
| 39 | return | ||
| 40 | } | ||
🟡 Medium Priority
在第二个测试用例
@gather_3d_cyclic_perm_full_mask中：失效模式：如果 normalize-vector pass 正确实现了标准 MLIR 语义，则生成的 gather 索引与 CHECK 不匹配，测试将失败。反之，如果测试当前通过（说明 pass 实际行为与 CHECK 一致），则 pass 对 permutation_map 的解释与输入 IR 的语义不一致。无论如何，这是一个测试正确性 bug。
修法方向：将输入的
permutation_map改为affine_map<(d0, d1, d2) -> (d2, d0, d1)>，或将 CHECK 的 gather 索引改为[0,1,2,3,4,5,6,7, 1,2,3,4,5,6,7,8, 2,3,4,5,6,7,8,9, 3,4,5,6,7,8,9,10]。推荐前者，因为(d2,d0,d1)使 memref 所有维度一一对应（dim0 size 8↔8, dim1 size 4↔4, dim2 size 1↔1），且in_bounds = [true, true, true]能真正成立。建议：将第 36 行的 permutation_map 从
(d1, d2, d0)改为(d2, d0, d1)，或相应更新第 29 行的 CHECK 索引。推荐改为(d2, d0, d1)，因为这样 memref 各维度和结果各维度一一对应，in_bounds = [true, true, true]语义正确。