Closed
perf(foreach_cosh): arch35 小列表多核并行与大数据量访存展开 #13873
liangtongxueAdd on 15 days agoClosed at 4 days ago
perf(foreach_cosh): arch35 小列表多核并行与大数据量访存展开 #13873
Closed
Total: 5 files changed+302-30
| @@ -13,6 +13,7 @@ | |||
| 13 | * \brief tiling implementation for foreach_cosh | 13 | * \brief tiling implementation for foreach_cosh |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | +#include <limits> | ||
| 16 | #include "log/log.h" | 17 | #include "log/log.h" |
| 17 | #include "util/math_util.h" | 18 | #include "util/math_util.h" |
| 18 | #include "util/platform_util.h" | 19 | #include "util/platform_util.h" |
| @@ -28,7 +29,7 @@ using namespace Ops::NN::OpTiling; | |||
| 28 | constexpr uint32_t WS_SYS_SIZE = 0U; | 29 | constexpr uint32_t WS_SYS_SIZE = 0U; |
| 29 | constexpr int32_t INPUT_IDX_0 = 0; | 30 | constexpr int32_t INPUT_IDX_0 = 0; |
| 30 | constexpr uint32_t DCACHE_SIZE = 128 * 1024; | 31 | constexpr uint32_t DCACHE_SIZE = 128 * 1024; |
| 31 | -constexpr int64_t MIN_PER_CORE = 1024; | 32 | +constexpr int64_t ALIGN_SIZE = 32; |
| 32 | constexpr int32_t MAX_TENSOR_NUM = 256; | 33 | constexpr int32_t MAX_TENSOR_NUM = 256; |
| 33 | constexpr int32_t MAX_SUPPORT_RANK = 8; | 34 | constexpr int32_t MAX_SUPPORT_RANK = 8; |
| 34 | 35 | ||
| @@ -85,7 +86,8 @@ static ge::graphStatus ForeachCoshTilingFunc(gert::TilingContext* context) | |||
| 85 | tiling->tensorCount = static_cast<int32_t>(tensorNum); | 86 | tiling->tensorCount = static_cast<int32_t>(tensorNum); |
| 86 | int64_t totalElements = 0; | 87 | int64_t totalElements = 0; |
| 87 | 88 | ||
| 88 | - // 5. Validate a single dtype across the list and accumulate element counts. | 89 | + // 5. Validate a single dtype across the list and accumulate prefix starts so the |
| 90 | + // kernel can map any flattened index back to its tensor with a binary search. | ||
| 89 | for (uint64_t i = 0; i < tensorNum; i++) { | 91 | for (uint64_t i = 0; i < tensorNum; i++) { |
| 90 | // The tiling key and SIMT kernel interpret the whole list with element 0's type. | 92 | // The tiling key and SIMT kernel interpret the whole list with element 0's type. |
| 91 | if (i > 0) { | 93 | if (i > 0) { |
| @@ -106,18 +108,41 @@ static ge::graphStatus ForeachCoshTilingFunc(gert::TilingContext* context) | |||
| 106 | storageShape.GetDimNum(), i, MAX_SUPPORT_RANK), | 108 | storageShape.GetDimNum(), i, MAX_SUPPORT_RANK), |
| 107 | return ge::GRAPH_FAILED); | 109 | return ge::GRAPH_FAILED); |
| 108 | int64_t numel = storageShape.GetShapeSize(); | 110 | int64_t numel = storageShape.GetShapeSize(); |
| 109 | - tiling->tensorElements[i] = numel; | 111 | + OP_CHECK_IF(numel < 0 || totalElements > std::numeric_limits<int64_t>::max() - numel, |
| 112 | + OP_LOGE(context, "invalid shape size[%ld] for tensor %lu of foreach_cosh", numel, i), | ||
| 113 | + return ge::GRAPH_FAILED); | ||
| 114 | + tiling->tensorStarts[i] = totalElements; | ||
| 110 | totalElements += numel; | 115 | totalElements += numel; |
| 111 | } | 116 | } |
| 117 | + tiling->tensorStarts[tensorNum] = totalElements; | ||
| 112 | tiling->totalElements = totalElements; | 118 | tiling->totalElements = totalElements; |
| 113 | 119 | ||
| 114 | - // 6. Calculate core split (an empty list yields a single no-op core). | 120 | + // 6. Calculate core split over the flattened element range. The kernel gives each |
| 121 | + // core a contiguous chunk, so an empty list is a single no-op core. | ||
| 115 | if (totalElements == 0) { | 122 | if (totalElements == 0) { |
| 116 | tiling->needCoreNum = 1; | 123 | tiling->needCoreNum = 1; |
| 124 | + tiling->perCoreElements = 0; | ||
| 125 | + tiling->lastCoreElements = 0; | ||
| 117 | context->SetBlockDim(1); | 126 | context->SetBlockDim(1); |
| 118 | } else { | 127 | } else { |
| 119 | - int64_t needCoreNum = std::min(coreNum, Ops::Base::CeilDiv(totalElements, MIN_PER_CORE)); | 128 | + int64_t perCoreElements = Ops::Base::CeilDiv(totalElements, coreNum); |
| 129 | + // Align only when the chunk is large enough: for small tensor lists (e.g. tens of | ||
| 130 | + // 0-dim scalars) rounding up to 32 would squash needCoreNum to 1 and serialise the | ||
| 131 | + // per-tensor VF launches on a single core (measured ~0.7us per tensor). Small lists | ||
| 132 | + // keep the unaligned chunk so needCoreNum stays close to min(coreNum, totalElements). | ||
| 133 | + if (perCoreElements >= ALIGN_SIZE) { | ||
| 134 | + perCoreElements = Ops::Base::CeilAlign(perCoreElements, ALIGN_SIZE); | ||
| 135 | + } | ||
| 136 | + OP_CHECK_IF(perCoreElements <= 0, OP_LOGE(context, "perCoreElements[%ld] should be positive", perCoreElements), | ||
| 137 | + return ge::GRAPH_FAILED); | ||
| 138 | + int64_t needCoreNum = Ops::Base::CeilDiv(totalElements, perCoreElements); | ||
| 139 | + OP_CHECK_IF( | ||
| 140 | + needCoreNum > std::numeric_limits<int32_t>::max(), | ||
| 141 | + OP_LOGE(context, "needCoreNum[%ld] exceeds the INT32_MAX limit of foreach_cosh tiling data", needCoreNum), | ||
| 142 | + return ge::GRAPH_FAILED); | ||
| 143 | + tiling->perCoreElements = perCoreElements; | ||
| 120 | tiling->needCoreNum = static_cast<int32_t>(needCoreNum); | 144 | tiling->needCoreNum = static_cast<int32_t>(needCoreNum); |
| 145 | + tiling->lastCoreElements = totalElements - perCoreElements * (needCoreNum - 1); | ||
| 121 | context->SetBlockDim(static_cast<uint32_t>(needCoreNum)); | 146 | context->SetBlockDim(static_cast<uint32_t>(needCoreNum)); |
| 122 | } | 147 | } |
| 123 | 148 | ||
| @@ -27,13 +27,11 @@ __global__ __aicore__ void foreach_cosh(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, | |||
| 27 | REGISTER_TILING_DEFAULT(ForeachCoshTilingData); | 27 | REGISTER_TILING_DEFAULT(ForeachCoshTilingData); |
| 28 | GET_TILING_DATA_WITH_STRUCT(ForeachCoshTilingData, tilingData, tiling); | 28 | GET_TILING_DATA_WITH_STRUCT(ForeachCoshTilingData, tilingData, tiling); |
| 29 | 29 | ||
| 30 | - const ForeachCoshTilingData* tilingGm = &tilingData; | ||
| 31 | - | ||
| 32 | if constexpr (schMode == static_cast<uint32_t>(ForeachCoshTilingKey::TILING_KEY_FLOAT)) { | 30 | if constexpr (schMode == static_cast<uint32_t>(ForeachCoshTilingKey::TILING_KEY_FLOAT)) { |
| 33 | - NsForeachCosh::Process<float>(x, y, tilingGm); | 31 | + NsForeachCosh::Process<float>(x, y, tiling, &tilingData); |
| 34 | } else if constexpr (schMode == static_cast<uint32_t>(ForeachCoshTilingKey::TILING_KEY_FLOAT16)) { | 32 | } else if constexpr (schMode == static_cast<uint32_t>(ForeachCoshTilingKey::TILING_KEY_FLOAT16)) { |
| 35 | - NsForeachCosh::Process<half>(x, y, tilingGm); | 33 | + NsForeachCosh::Process<half>(x, y, tiling, &tilingData); |
| 36 | } else if constexpr (schMode == static_cast<uint32_t>(ForeachCoshTilingKey::TILING_KEY_BF16)) { | 34 | } else if constexpr (schMode == static_cast<uint32_t>(ForeachCoshTilingKey::TILING_KEY_BF16)) { |
| 37 | - NsForeachCosh::Process<bfloat16_t>(x, y, tilingGm); | 35 | + NsForeachCosh::Process<bfloat16_t>(x, y, tiling, &tilingData); |
| 38 | } | 36 | } |
| 39 | } | 37 | } |
| @@ -11,6 +11,15 @@ | |||
| 11 | /*! | 11 | /*! |
| 12 | * \file foreach_cosh_simt.h | 12 | * \file foreach_cosh_simt.h |
| 13 | * \brief SIMT kernel implementation for foreach_cosh | 13 | * \brief SIMT kernel implementation for foreach_cosh |
| 14 | + * | ||
| 15 | + * The tensor list is treated as one flattened element range. Tiling assigns each core a | ||
| 16 | + * contiguous chunk [coreStart, coreEnd); a core then either | ||
| 17 | + * - large segment: walks the intersecting tensors (binary search for the first one) and | ||
| 18 | + * launches one VF per tensor slice, with an 8x per-thread unroll so that eight | ||
| 19 | + * independent loads are in flight and the bandwidth-bound path covers the GM latency; | ||
| 20 | + * - small segment (<= FLAT_SEG_MAX_ELEMENTS, e.g. lists of 0-dim scalars): launches a | ||
| 21 | + * single VF for the whole segment in which every thread resolves its own tensor via | ||
| 22 | + * the tiling prefix table, removing the serial per-tensor VF_CALL chain. | ||
| 14 | */ | 23 | */ |
| 15 | 24 | ||
| 16 | #ifndef FOREACH_COSH_SIMT_H | 25 | #ifndef FOREACH_COSH_SIMT_H |
| @@ -30,8 +39,20 @@ constexpr int TENSOR_PTR_SHIFT = 3; | |||
| 30 | 39 | ||
| 31 | using namespace AscendC; | 40 | using namespace AscendC; |
| 32 | 41 | ||
| 42 | +// fixed thread count per core for every VF launch | ||
| 33 | constexpr uint32_t THREAD_NUM = 512; | 43 | constexpr uint32_t THREAD_NUM = 512; |
| 34 | 44 | ||
| 45 | +// per-thread memory-level-parallelism unroll for the bandwidth-bound large segments: | ||
| 46 | +// eight independent loads are issued back to back before the first store, growing the | ||
| 47 | +// in-flight bytes per warp 8x so the stream can cover the GM access latency | ||
| 48 | +constexpr int64_t UNROLL_COUNT = 8; | ||
| 49 | + | ||
| 50 | +// segments no larger than this are handled by one flat VF whose threads resolve their own | ||
| 51 | +// tensor through the prefix table: for such short segments the per-element lookup cost is | ||
| 52 | +// negligible while the single launch removes the serial per-tensor VF_CALL chain that | ||
| 53 | +// dominates small tensor lists (measured ~0.7us per tensor before this change) | ||
| 54 | +constexpr int64_t FLAT_SEG_MAX_ELEMENTS = 4096; | ||
| 55 | + | ||
| 35 | /** | 56 | /** |
| 36 | * \brief Get the GM address of the idx-th tensor in a TensorList | 57 | * \brief Get the GM address of the idx-th tensor in a TensorList |
| 37 | */ | 58 | */ |
| @@ -75,38 +96,123 @@ __simt_callee__ inline T FromFloat32(float val) | |||
| 75 | } | 96 | } |
| 76 | 97 | ||
| 77 | /** | 98 | /** |
| 78 | - * \brief SIMT VF kernel: compute cosh for all elements across all tensors | 99 | + * \brief VF over one contiguous slice of a single tensor (large-segment path) |
| 79 | - * cosh(x) = (exp(x) + exp(-x)) / 2 | 100 | + * cosh(x) = (exp(x) + exp(-x)) / 2, computed via coshf in float32 |
| 80 | */ | 101 | */ |
| 81 | template <typename T> | 102 | template <typename T> |
| 82 | -__simt_vf__ __aicore__ LAUNCH_BOUND(THREAD_NUM) inline void OpForeachCoshSimt(int32_t tensorId, int64_t count, | 103 | +__simt_vf__ __aicore__ LAUNCH_BOUND(THREAD_NUM) inline void ForeachCoshTensorSegSimt(int32_t tensorId, |
| 83 | - GM_ADDR xList, GM_ADDR yList) | 104 | + int64_t localStart, int64_t count, |
| 105 | + GM_ADDR xList, GM_ADDR yList) | ||
| 84 | { | 106 | { |
| 107 | + // all threads issue the same two dependent loads, so the pointer table is fetched once | ||
| 85 | __gm__ T* xData = SimtGetTensorAddr<T>(xList, tensorId); | 108 | __gm__ T* xData = SimtGetTensorAddr<T>(xList, tensorId); |
| 86 | __gm__ T* yData = SimtGetTensorAddr<T>(yList, tensorId); | 109 | __gm__ T* yData = SimtGetTensorAddr<T>(yList, tensorId); |
| 87 | - uint64_t tid = static_cast<uint64_t>(AscendC::Simt::GetBlockIdx() * AscendC::Simt::GetThreadNum() + | 110 | + const int64_t stride = static_cast<int64_t>(AscendC::Simt::GetThreadNum()); |
| 88 | - AscendC::Simt::GetThreadIdx()); | 111 | + const int64_t end = localStart + count; |
| 89 | - uint64_t stride = static_cast<uint64_t>(AscendC::Simt::GetThreadNum() * AscendC::Simt::GetBlockNum()); | 112 | + const int64_t unrollEnd = end - (UNROLL_COUNT - 1) * stride; |
| 90 | - for (uint64_t idx = tid; idx < static_cast<uint64_t>(count); idx += stride) { | 113 | + int64_t idx = localStart + static_cast<int64_t>(AscendC::Simt::GetThreadIdx()); |
| 91 | - T xVal = xData[idx]; | 114 | + for (; idx < unrollEnd; idx += UNROLL_COUNT * stride) { |
| 92 | - float xF32 = ToFloat32<T>(xVal); | 115 | + T val[UNROLL_COUNT]; |
| 93 | - float yF32 = coshf(xF32); | 116 | +#pragma unroll |
| 94 | - yData[idx] = FromFloat32<T>(yF32); | 117 | + for (int64_t k = 0; k < UNROLL_COUNT; k++) { |
| 118 | + val[k] = xData[idx + k * stride]; | ||
| 119 | + } | ||
| 120 | +#pragma unroll | ||
| 121 | + for (int64_t k = 0; k < UNROLL_COUNT; k++) { | ||
| 122 | + yData[idx + k * stride] = FromFloat32<T>(coshf(ToFloat32<T>(val[k]))); | ||
| 123 | + } | ||
| 124 | + } | ||
| 125 | + for (; idx < end; idx += stride) { | ||
| 126 | + yData[idx] = FromFloat32<T>(coshf(ToFloat32<T>(xData[idx]))); | ||
| 95 | } | 127 | } |
| 96 | } | 128 | } |
| 97 | 129 | ||
| 98 | /** | 130 | /** |
| 99 | - * \brief Process entry: launch SIMT VF for foreach_cosh | 131 | + * \brief VF over a whole small core segment, crossing tensor boundaries |
| 132 | + * (small-list path): every thread locates its own tensor through the tiling | ||
| 133 | + * prefix table, so a N-tensor list costs one VF launch instead of N | ||
| 100 | */ | 134 | */ |
| 101 | template <typename T> | 135 | template <typename T> |
| 102 | -__aicore__ inline void Process(GM_ADDR x, GM_ADDR y, const ForeachCoshTilingData* tilingGm) | 136 | +__simt_vf__ __aicore__ LAUNCH_BOUND(THREAD_NUM) inline void ForeachCoshFlatSegSimt(int64_t coreStart, int64_t coreEnd, |
| 137 | + GM_ADDR xList, GM_ADDR yList, | ||
| 138 | + GM_ADDR tiling) | ||
| 103 | { | 139 | { |
| 104 | - for (int32_t tensorId = 0; tensorId < tilingGm->tensorCount; tensorId++) { | 140 | + __gm__ const ForeachCoshTilingData* tilingGm = reinterpret_cast<__gm__ const ForeachCoshTilingData*>(tiling); |
| 105 | - int64_t count = tilingGm->tensorElements[tensorId]; | 141 | + const int32_t tensorCount = tilingGm->tensorCount; |
| 106 | - if (count <= 0) { | 142 | + const int64_t stride = static_cast<int64_t>(AscendC::Simt::GetThreadNum()); |
| 107 | - continue; | 143 | + for (int64_t flat = coreStart + static_cast<int64_t>(AscendC::Simt::GetThreadIdx()); flat < coreEnd; |
| 144 | + flat += stride) { | ||
| 145 | + // locate the tensor owning this flat index: the last t with tensorStarts[t] <= flat. | ||
| 146 | + // Empty tensors hold equal starts, so the "last" rule skips them naturally. | ||
| 147 | + int32_t lo = 0; | ||
| 148 | + int32_t hi = tensorCount - 1; | ||
| 149 | + while (lo < hi) { | ||
| 150 | + const int32_t mid = (lo + hi + 1) >> 1; | ||
| 151 | + if (tilingGm->tensorStarts[mid] <= flat) { | ||
| 152 | + lo = mid; | ||
| 153 | + } else { | ||
| 154 | + hi = mid - 1; | ||
| 155 | + } | ||
| 156 | + } | ||
| 157 | + const int64_t localIdx = flat - tilingGm->tensorStarts[lo]; | ||
| 158 | + __gm__ T* xData = SimtGetTensorAddr<T>(xList, lo); | ||
| 159 | + __gm__ T* yData = SimtGetTensorAddr<T>(yList, lo); | ||
| 160 | + yData[localIdx] = FromFloat32<T>(coshf(ToFloat32<T>(xData[localIdx]))); | ||
| 161 | + } | ||
| 162 | +} | ||
| 163 | + | ||
| 164 | +/** | ||
| 165 | + * \brief Process entry: per-core contiguous chunk of the flattened element range | ||
| 166 | + */ | ||
| 167 | +template <typename T> | ||
| 168 | +__aicore__ inline void Process(GM_ADDR x, GM_ADDR y, GM_ADDR tiling, const ForeachCoshTilingData* tilingData) | ||
| 169 | +{ | ||
| 170 | + const int64_t totalElements = tilingData->totalElements; | ||
| 171 | + if (totalElements <= 0) { | ||
| 172 | + return; | ||
| 173 | + } | ||
| 174 | + const int32_t blockIdx = static_cast<int32_t>(AscendC::GetBlockIdx()); | ||
| 175 | + const int64_t coreStart = static_cast<int64_t>(blockIdx) * tilingData->perCoreElements; | ||
| 176 | + int64_t coreEnd; | ||
| 177 | + if (blockIdx == tilingData->needCoreNum - 1) { | ||
| 178 | + coreEnd = coreStart + tilingData->lastCoreElements; | ||
| 179 | + } else { | ||
| 180 | + coreEnd = coreStart + tilingData->perCoreElements; | ||
| 181 | + } | ||
| 182 | + if (coreStart >= coreEnd) { | ||
| 183 | + return; | ||
| 184 | + } | ||
| 185 | + | ||
| 186 | + if (coreEnd - coreStart <= FLAT_SEG_MAX_ELEMENTS) { | ||
| 187 | + AscendC::Simt::VF_CALL<ForeachCoshFlatSegSimt<T>>(AscendC::Simt::Dim3(THREAD_NUM), coreStart, coreEnd, x, y, | ||
| 188 | + tiling); | ||
| 189 | + return; | ||
| 190 | + } | ||
| 191 | + | ||
| 192 | + // large segment: binary search the first tensor intersecting [coreStart, coreEnd) on the | ||
| 193 | + // local tiling copy, then walk forward only over the intersecting ones | ||
| 194 | + int32_t lo = 0; | ||
| 195 | + int32_t hi = tilingData->tensorCount - 1; | ||
| 196 | + while (lo < hi) { | ||
| 197 | + const int32_t mid = (lo + hi + 1) >> 1; | ||
| 198 | + if (tilingData->tensorStarts[mid] <= coreStart) { | ||
| 199 | + lo = mid; | ||
| 200 | + } else { | ||
| 201 | + hi = mid - 1; | ||
| 202 | + } | ||
| 203 | + } | ||
| 204 | + for (int32_t t = lo; t < tilingData->tensorCount; t++) { | ||
| 205 | + const int64_t tensorStart = tilingData->tensorStarts[t]; | ||
| 206 | + if (tensorStart >= coreEnd) { | ||
| 207 | + break; | ||
| 208 | + } | ||
| 209 | + const int64_t tensorEnd = tilingData->tensorStarts[t + 1]; | ||
| 210 | + const int64_t segStart = (coreStart > tensorStart) ? coreStart : tensorStart; | ||
| 211 | + const int64_t segEnd = (coreEnd < tensorEnd) ? coreEnd : tensorEnd; | ||
| 212 | + if (segStart < segEnd) { | ||
| 213 | + AscendC::Simt::VF_CALL<ForeachCoshTensorSegSimt<T>>(AscendC::Simt::Dim3(THREAD_NUM), t, | ||
| 214 | + segStart - tensorStart, segEnd - segStart, x, y); | ||
| 108 | } | 215 | } |
| 109 | - AscendC::Simt::VF_CALL<OpForeachCoshSimt<T>>(AscendC::Simt::Dim3(THREAD_NUM), tensorId, count, x, y); | ||
| 110 | } | 216 | } |
| 111 | } | 217 | } |
| 112 | 218 | ||
| @@ -16,11 +16,21 @@ | |||
| 16 | #ifndef FOREACH_COSH_TILING_DATA_H | 16 | #ifndef FOREACH_COSH_TILING_DATA_H |
| 17 | #define FOREACH_COSH_TILING_DATA_H | 17 | #define FOREACH_COSH_TILING_DATA_H |
| 18 | 18 | ||
| 19 | +#include <cstdint> | ||
| 20 | + | ||
| 21 | +constexpr int32_t FOREACH_COSH_MAX_TENSOR_NUM = 256; | ||
| 22 | + | ||
| 19 | struct ForeachCoshTilingData { | 23 | struct ForeachCoshTilingData { |
| 20 | int32_t needCoreNum = 0; | 24 | int32_t needCoreNum = 0; |
| 21 | int32_t tensorCount = 0; | 25 | int32_t tensorCount = 0; |
| 22 | int64_t totalElements = 0; | 26 | int64_t totalElements = 0; |
| 23 | - int64_t tensorElements[256] = {0}; | 27 | + // per-core contiguous chunk of the flattened element range; the last core uses lastCoreElements |
| 28 | + int64_t perCoreElements = 0; | ||
| 29 | + int64_t lastCoreElements = 0; | ||
| 30 | + // prefix starts of each tensor in the flattened range: | ||
| 31 | + // tensorStarts[0] == 0, tensorStarts[i] is the first flat index of tensor i, | ||
| 32 | + // tensorStarts[tensorCount] == totalElements (so empty tensors hold equal starts) | ||
| 33 | + int64_t tensorStarts[FOREACH_COSH_MAX_TENSOR_NUM + 1] = {0}; | ||
| 24 | }; | 34 | }; |
| 25 | 35 | ||
| 26 | #endif // FOREACH_COSH_TILING_DATA_H | 36 | #endif // FOREACH_COSH_TILING_DATA_H |
| @@ -96,3 +96,136 @@ TEST_F(ForeachCoshTilingTest, foreach_cosh_bfloat16) | |||
| 96 | { | 96 | { |
| 97 | DoCase({4, 4}, ge::DT_BF16, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_BF16)); | 97 | DoCase({4, 4}, ge::DT_BF16, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_BF16)); |
| 98 | } | 98 | } |
| 99 | + | ||
| 100 | +namespace { | ||
| 101 | +constexpr int64_t MOCK_CORE_NUM = 64; | ||
| 102 | +constexpr int64_t ALIGN_SIZE_UT = 32; | ||
| 103 | + | ||
| 104 | +int64_t CeilDivExpect(int64_t a, int64_t b) { return (a + b - 1) / b; } | ||
| 105 | + | ||
| 106 | +int64_t CeilAlignExpect(int64_t a, int64_t b) { return CeilDivExpect(a, b) * b; } | ||
| 107 | +} // namespace | ||
| 108 | + | ||
| 109 | +// list-shaped tiling: per-tensor shapes are 1-D of the given numel, mirroring the | ||
| 110 | +// scalar-list / multi-tensor workloads of issue #65; expected values are derived from | ||
| 111 | +// MOCK_CORE_NUM so they stay coupled to the mocked platform instead of hard-coded | ||
| 112 | +static void DoListCase(const std::vector<int64_t>& tensorNumels, ge::DataType dt, uint64_t expectKey) | ||
| 113 | +{ | ||
| 114 | + fe::PlatFormInfos pf; | ||
| 115 | + std::map<std::string, std::string> soc, ai, intr; | ||
| 116 | + GetPlatFormInfos(R"({"hardware_info":{"UB_SIZE":253952,"CORE_NUM":64}})", soc, ai, intr); | ||
| 117 | + pf.Init(); | ||
| 118 | + | ||
| 119 | + struct ForeachCoshCompileInfo { | ||
| 120 | + } ci; | ||
| 121 | + auto opType = std::string("ForeachCosh"); | ||
| 122 | + auto tilingFn = gert::OpImplRegistry::GetInstance().GetOpImpl(opType.c_str())->tiling; | ||
| 123 | + auto tilingParseFn = gert::OpImplRegistry::GetInstance().GetOpImpl(opType.c_str())->tiling_parse; | ||
| 124 | + ASSERT_NE(tilingFn, nullptr); | ||
| 125 | + ASSERT_NE(tilingParseFn, nullptr); | ||
| 126 | + | ||
| 127 | + std::string ciStr = R"({"device_id":null})"; | ||
| 128 | + auto kh = gert::KernelRunContextFaker() | ||
| 129 | + .KernelIONum(2, 1) | ||
| 130 | + .Inputs({const_cast<char*>(ciStr.c_str()), reinterpret_cast<void*>(&pf)}) | ||
| 131 | + .Outputs({&ci}) | ||
| 132 | + .Build(); | ||
| 133 | + kh.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->Init(); | ||
| 134 | + kh.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("SoCInfo", soc); | ||
| 135 | + kh.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("AICoreSpec", ai); | ||
| 136 | + kh.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetCoreNumByCoreType("AICore"); | ||
| 137 | + kh.GetContext<gert::TilingParseContext>()->GetPlatformInfo()->SetPlatformRes("AICoreintrinsicDtypeMap", intr); | ||
| 138 | + ASSERT_EQ(tilingParseFn(kh.GetContext<gert::KernelContext>()), ge::GRAPH_SUCCESS); | ||
| 139 | + | ||
| 140 | + const uint32_t tensorNum = static_cast<uint32_t>(tensorNumels.size()); | ||
| 141 | + std::vector<gert::StorageShape> inputShapes(tensorNum); | ||
| 142 | + std::vector<gert::StorageShape> outputShapes(tensorNum); | ||
| 143 | + int64_t totalElements = 0; | ||
| 144 | + for (uint32_t i = 0; i < tensorNum; i++) { | ||
| 145 | + inputShapes[i] = {{tensorNumels[i]}, {tensorNumels[i]}}; | ||
| 146 | + outputShapes[i] = {{tensorNumels[i]}, {tensorNumels[i]}}; | ||
| 147 | + totalElements += tensorNumels[i]; | ||
| 148 | + } | ||
| 149 | + std::vector<gert::StorageShape*> inputShapeRefs; | ||
| 150 | + std::vector<gert::StorageShape*> outputShapeRefs; | ||
| 151 | + for (auto& shape : inputShapes) { | ||
| 152 | + inputShapeRefs.push_back(&shape); | ||
| 153 | + } | ||
| 154 | + for (auto& shape : outputShapes) { | ||
| 155 | + outputShapeRefs.push_back(&shape); | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + auto param = gert::TilingData::CreateCap(4096); | ||
| 159 | + auto wsh = gert::ContinuousVector::Create<size_t>(4096); | ||
| 160 | + auto th = gert::TilingContextFaker() | ||
| 161 | + .NodeIoNum(1, 1) | ||
| 162 | + .IrInstanceNum({tensorNum}, {tensorNum}) | ||
| 163 | + .InputShapes(inputShapeRefs) | ||
| 164 | + .OutputShapes(outputShapeRefs) | ||
| 165 | + .CompileInfo(&ci) | ||
| 166 | + .PlatformInfo(reinterpret_cast<char*>(&pf)) | ||
| 167 | + .NodeInputTd(0, dt, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 168 | + .NodeOutputTd(0, dt, ge::FORMAT_ND, ge::FORMAT_ND) | ||
| 169 | + .TilingData(param.get()) | ||
| 170 | + .Workspace(reinterpret_cast<gert::ContinuousVector*>(wsh.get())) | ||
| 171 | + .Build(); | ||
| 172 | + auto* ctx = th.GetContext<gert::TilingContext>(); | ||
| 173 | + ctx->GetPlatformInfo()->SetPlatformRes("SoCInfo", soc); | ||
| 174 | + ctx->GetPlatformInfo()->SetPlatformRes("AICoreSpec", ai); | ||
| 175 | + ctx->GetPlatformInfo()->SetCoreNumByCoreType("AICore"); | ||
| 176 | + ctx->GetPlatformInfo()->SetPlatformRes("AICoreintrinsicDtypeMap", intr); | ||
| 177 | + | ||
| 178 | + EXPECT_EQ(tilingFn(ctx), ge::GRAPH_SUCCESS); | ||
| 179 | + EXPECT_EQ(ctx->GetTilingKey(), expectKey); | ||
| 180 | + | ||
| 181 | + auto rawTilingData = ctx->GetRawTilingData(); | ||
| 182 | + ASSERT_NE(rawTilingData, nullptr); | ||
| 183 | + ASSERT_GE(rawTilingData->GetDataSize(), sizeof(ForeachCoshTilingData)); | ||
| 184 | + const auto* tilingData = reinterpret_cast<const ForeachCoshTilingData*>(rawTilingData->GetData()); | ||
| 185 | + EXPECT_EQ(tilingData->tensorCount, static_cast<int32_t>(tensorNum)); | ||
| 186 | + EXPECT_EQ(tilingData->totalElements, totalElements); | ||
| 187 | + | ||
| 188 | + int64_t expectPerCore = CeilDivExpect(totalElements, MOCK_CORE_NUM); | ||
| 189 | + if (expectPerCore >= ALIGN_SIZE_UT) { | ||
| 190 | + expectPerCore = CeilAlignExpect(expectPerCore, ALIGN_SIZE_UT); | ||
| 191 | + } | ||
| 192 | + const int64_t expectNeedCore = CeilDivExpect(totalElements, expectPerCore); | ||
| 193 | + const int64_t expectLastCore = totalElements - expectPerCore * (expectNeedCore - 1); | ||
| 194 | + EXPECT_EQ(tilingData->perCoreElements, expectPerCore); | ||
| 195 | + EXPECT_EQ(tilingData->needCoreNum, static_cast<int32_t>(expectNeedCore)); | ||
| 196 | + EXPECT_EQ(tilingData->lastCoreElements, expectLastCore); | ||
| 197 | + | ||
| 198 | + // prefix starts: tensorStarts[i] is the flat start of tensor i, tensorStarts[N] the total | ||
| 199 | + int64_t prefix = 0; | ||
| 200 | + for (uint32_t i = 0; i < tensorNum; i++) { | ||
| 201 | + EXPECT_EQ(tilingData->tensorStarts[i], prefix); | ||
| 202 | + prefix += tensorNumels[i]; | ||
| 203 | + } | ||
| 204 | + EXPECT_EQ(tilingData->tensorStarts[tensorNum], totalElements); | ||
| 205 | +} | ||
| 206 | + | ||
| 207 | +// 128 0-dim-scalar-like tensors: the small-list regime of issue #65 must keep | ||
| 208 | +// needCoreNum close to min(coreNum, totalElements) instead of being squashed to 1 | ||
| 209 | +TEST_F(ForeachCoshTilingTest, foreach_cosh_scalar_list_multicore) | ||
| 210 | +{ | ||
| 211 | + std::vector<int64_t> numels(128, 1); | ||
| 212 | + DoListCase(numels, ge::DT_FLOAT16, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_FLOAT16)); | ||
| 213 | +} | ||
| 214 | + | ||
| 215 | +// mid-size list: perCoreElements crosses the 32-alignment threshold | ||
| 216 | +TEST_F(ForeachCoshTilingTest, foreach_cosh_mid_list_aligned_chunk) | ||
| 217 | +{ | ||
| 218 | + DoListCase({1000, 1000, 1000}, ge::DT_FLOAT, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_FLOAT)); | ||
| 219 | +} | ||
| 220 | + | ||
| 221 | +// single large tensor: aligned full-core split with a shorter last core | ||
| 222 | +TEST_F(ForeachCoshTilingTest, foreach_cosh_large_tensor_full_cores) | ||
| 223 | +{ | ||
| 224 | + DoListCase({125898626}, ge::DT_BF16, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_BF16)); | ||
| 225 | +} | ||
| 226 | + | ||
| 227 | +// empty tensor inside the list holds equal prefix starts and must not break the table | ||
| 228 | +TEST_F(ForeachCoshTilingTest, foreach_cosh_list_with_empty_tensor) | ||
| 229 | +{ | ||
| 230 | + DoListCase({4, 0, 8}, ge::DT_FLOAT, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_FLOAT)); | ||
| 231 | +} | ||
【review】 问题分类:可测 位置:foreach/foreach_cosh/tests/ut/op_host/arch35/test_foreach_cosh_tiling.cpp:236,代码:DoListCase({4, 0, 8}, ge::DT_FLOAT, GET_TPL_TILING_KEY(FOREACH_COSH_TPL_SCH_MODE_FLOAT)); 问题:新增用例只读取 host 生成的 tensorStarts、perCoreElements 等字段,没有运行本次重写的两套 SIMT kernel。含空张量用例只是验证重复前缀,不能验证 ForeachCoshFlatSegSimt 的“取最后一个起点”查找;大张量用例也不能验证 8 路展开的尾部写回。把二分查找的 <= 改成 <,或删掉展开后的尾循环,这些新增断言仍全部通过。 可能导致的结果:张量边界恰好落在核起点、连续空张量及不足 8×线程数的尾段,都是此次分段算法的新边界;出现索引映射或尾部遗漏回归时,现有测试无法发现输出错位、越界或漏写。 建议的修改方法:增加实际执行 kernel 并逐张量对照 cosh 的数值回归:输入使用不同张量可区分的值,包含首部/连续/末尾空张量、恰落张量边界的 coreStart、每核 4096 与超过 4096 的分流边界,以及 4096±1 等展开尾长;输出预填哨兵并检查全部有效元素和边界。覆盖 FLOAT/FLOAT16/BFLOAT16,使上述查找或尾循环变异能被测试拦截。