Merged
[SectionStreamK] 支持S1G不合轴和KV不同Dim #10563
FiguraDogeAdd on Aug 25
[SectionStreamK] 支持S1G不合轴和KV不同Dim #10563
Merged
Merged fromChengjiaWu/ops-transformermasterintocann/ops-transformermaster
Total: 13 files changed+452-380
| @@ -36,7 +36,8 @@ public: | |||
| 36 | [[nodiscard]] virtual uint32_t GetGroupSize() const = 0; | 36 | [[nodiscard]] virtual uint32_t GetGroupSize() const = 0; |
| 37 | [[nodiscard]] virtual uint32_t GetQueryHeadNum() const = 0; | 37 | [[nodiscard]] virtual uint32_t GetQueryHeadNum() const = 0; |
| 38 | [[nodiscard]] virtual uint32_t GetKvHeadNum() const = 0; | 38 | [[nodiscard]] virtual uint32_t GetKvHeadNum() const = 0; |
| 39 | - [[nodiscard]] virtual uint32_t GetHeadDim() const = 0; | 39 | + [[nodiscard]] virtual uint32_t GetHeadDimQk() const = 0; |
| 40 | + [[nodiscard]] virtual uint32_t GetHeadDimV() const = 0; | ||
| 40 | [[nodiscard]] virtual uint32_t GetQuerySeqSize() const = 0; | 41 | [[nodiscard]] virtual uint32_t GetQuerySeqSize() const = 0; |
| 41 | [[nodiscard]] virtual uint32_t GetQuerySeqSize(uint32_t batchIdx) const = 0; | 42 | [[nodiscard]] virtual uint32_t GetQuerySeqSize(uint32_t batchIdx) const = 0; |
F | |||
| 42 | [[nodiscard]] virtual uint32_t GetKvSeqSize() const = 0; | 43 | [[nodiscard]] virtual uint32_t GetKvSeqSize() const = 0; |
| @@ -79,9 +80,14 @@ public: | |||
| 79 | return kvHeadNum; | 80 | return kvHeadNum; |
| 80 | } | 81 | } |
| 81 | 82 | ||
| 82 | - [[nodiscard]] uint32_t GetHeadDim() const override | 83 | + [[nodiscard]] uint32_t GetHeadDimQk() const override |
| 83 | { | 84 | { |
| 84 | - return headDim; | 85 | + return headDimQk; |
| 86 | + } | ||
| 87 | + | ||
| 88 | + [[nodiscard]] uint32_t GetHeadDimV() const override | ||
| 89 | + { | ||
| 90 | + return headDimV; | ||
| 85 | } | 91 | } |
| 86 | 92 | ||
| 87 | [[nodiscard]] uint32_t GetQuerySeqSize() const override | 93 | [[nodiscard]] uint32_t GetQuerySeqSize() const override |
| @@ -202,7 +208,8 @@ public: | |||
| 202 | uint32_t querySeqSize{0U}; | 208 | uint32_t querySeqSize{0U}; |
| 203 | uint32_t kvHeadNum{0U}; | 209 | uint32_t kvHeadNum{0U}; |
| 204 | uint32_t kvSeqSize{0U}; | 210 | uint32_t kvSeqSize{0U}; |
| 205 | - uint32_t headDim{64U}; | 211 | + uint32_t headDimQk{64U}; |
| 212 | + uint32_t headDimV{64U}; | ||
| 206 | bool attenMaskFlag{false}; | 213 | bool attenMaskFlag{false}; |
| 207 | uint32_t sparseMode{0U}; | 214 | uint32_t sparseMode{0U}; |
| 208 | uint32_t preToken{0U}; | 215 | uint32_t preToken{0U}; |
| @@ -38,6 +38,12 @@ enum class SocVersion : uint32_t { | |||
| 38 | BUTT | 38 | BUTT |
| 39 | }; | 39 | }; |
| 40 | 40 | ||
| 41 | +enum class OutputLayout : uint8_t { | ||
| 42 | + BN2_S1G = 0, | ||
| 43 | + BN1_S1, | ||
| 44 | + BUTT | ||
| 45 | +}; | ||
| 46 | + | ||
| 41 | struct DeviceInfo { | 47 | struct DeviceInfo { |
| 42 | uint32_t aicCoreMaxNum{0U}; // At most amount of aic core would be turned on, this is not final used number | 48 | uint32_t aicCoreMaxNum{0U}; // At most amount of aic core would be turned on, this is not final used number |
| 43 | uint32_t aicCoreMinNum{0U}; // At least amount of aic core would be turned on, this is not final used number | 49 | uint32_t aicCoreMinNum{0U}; // At least amount of aic core would be turned on, this is not final used number |
| @@ -55,6 +61,7 @@ struct GeneralBalanceParam { | |||
| 55 | // full_block_cost is costFunc(mBaseSize, s2BaseSize) | 61 | // full_block_cost is costFunc(mBaseSize, s2BaseSize) |
| 56 | int64_t fdLeastBlock{0U}; // if noFd.maxCost <= fdLeastBlock * full_block_cost, then choose no fd | 62 | int64_t fdLeastBlock{0U}; // if noFd.maxCost <= fdLeastBlock * full_block_cost, then choose no fd |
| 57 | CostFunc costFunc{nullptr}; // Customize cost func. Set nullptr to use default cost func | 63 | CostFunc costFunc{nullptr}; // Customize cost func. Set nullptr to use default cost func |
| 64 | + OutputLayout outputLayout{OutputLayout::BN2_S1G}; | ||
| 58 | }; | 65 | }; |
| 59 | 66 | ||
| 60 | enum class SparseMode : uint8_t { | 67 | enum class SparseMode : uint8_t { |
| @@ -48,16 +48,16 @@ inline uint32_t SectionStreamK::Compute(const DeviceInfo &deviceInfo, const IBas | |||
| 48 | for (size_t i = 0; i < result.sectionNum; ++i) { | 48 | for (size_t i = 0; i < result.sectionNum; ++i) { |
| 49 | SectionStreamKFaResult tmpFa(deviceInfo.aicCoreMaxNum); | 49 | SectionStreamKFaResult tmpFa(deviceInfo.aicCoreMaxNum); |
| 50 | tmpFa.usedCoreNum = implResult[i].usedCoreNum; | 50 | tmpFa.usedCoreNum = implResult[i].usedCoreNum; |
| 51 | - tmpFa.bN2End = implResult[i].bN2End; | 51 | + tmpFa.bNEnd = implResult[i].bNEnd; |
| 52 | - tmpFa.gS1End = implResult[i].gS1End; | 52 | + tmpFa.mEnd = implResult[i].mEnd; |
| 53 | tmpFa.s2End = implResult[i].s2End; | 53 | tmpFa.s2End = implResult[i].s2End; |
| 54 | tmpFa.firstFdDataWorkspaceIdx = implResult[i].firstFdDataWorkspaceIdx; | 54 | tmpFa.firstFdDataWorkspaceIdx = implResult[i].firstFdDataWorkspaceIdx; |
| 55 | result.sectionFaResult.emplace_back(tmpFa); | 55 | result.sectionFaResult.emplace_back(tmpFa); |
| 56 | 56 | ||
| 57 | SectionStreamKFdResult tmpFd(deviceInfo.aicCoreMaxNum, deviceInfo.aivCoreMaxNum); | 57 | SectionStreamKFdResult tmpFd(deviceInfo.aicCoreMaxNum, deviceInfo.aivCoreMaxNum); |
| 58 | tmpFd.usedVecNum = implResult[i].usedVecNum; | 58 | tmpFd.usedVecNum = implResult[i].usedVecNum; |
| 59 | - tmpFd.bN2Idx = implResult[i].bN2Idx; | 59 | + tmpFd.bNIdx = implResult[i].bNIdx; |
| 60 | - tmpFd.gS1Idx = implResult[i].mIdx; | 60 | + tmpFd.mIdx = implResult[i].mIdx; |
| 61 | tmpFd.workspaceIdx = implResult[i].workspaceIdx; | 61 | tmpFd.workspaceIdx = implResult[i].workspaceIdx; |
| 62 | tmpFd.s2SplitNum = implResult[i].s2SplitNum; | 62 | tmpFd.s2SplitNum = implResult[i].s2SplitNum; |
| 63 | tmpFd.taskIdx = implResult[i].taskIdx; | 63 | tmpFd.taskIdx = implResult[i].taskIdx; |
| @@ -35,22 +35,24 @@ struct SectionStreamKParam : GeneralBalanceParam { | |||
| 35 | 35 | ||
| 36 | struct SectionStreamKFaResult { | 36 | struct SectionStreamKFaResult { |
| 37 | uint32_t usedCoreNum{0U}; // FA中使用的AIC数量 | 37 | uint32_t usedCoreNum{0U}; // FA中使用的AIC数量 |
| 38 | - std::vector<uint32_t> bN2End{}; // 每个核处理数据的BN2结束点 | 38 | + std::vector<uint32_t> bNEnd{}; // 每个核处理数据的BN结束点 |
| 39 | - std::vector<uint32_t> gS1End{}; // 每个核处理数据的GS1结束点 | 39 | + std::vector<uint32_t> mEnd{}; // 每个核处理数据的M结束点 |
| 40 | std::vector<uint32_t> s2End{}; // 每个核处理数据的S2结束点 | 40 | std::vector<uint32_t> s2End{}; // 每个核处理数据的S2结束点 |
| 41 | std::vector<uint32_t> firstFdDataWorkspaceIdx{}; // FD块在workspace中的位置 | 41 | std::vector<uint32_t> firstFdDataWorkspaceIdx{}; // FD块在workspace中的位置 |
| 42 | 42 | ||
| 43 | explicit SectionStreamKFaResult(uint32_t aicNum) | 43 | explicit SectionStreamKFaResult(uint32_t aicNum) |
| 44 | - : bN2End(aicNum), gS1End(aicNum), s2End(aicNum), firstFdDataWorkspaceIdx(aicNum) | 44 | + : bNEnd(aicNum), |
| 45 | - { | 45 | + mEnd(aicNum), |
| 46 | - } | 46 | + s2End(aicNum), |
| 47 | + firstFdDataWorkspaceIdx(aicNum) | ||
| 48 | + {} | ||
| 47 | }; | 49 | }; |
| 48 | 50 | ||
| 49 | struct SectionStreamKFdResult { | 51 | struct SectionStreamKFdResult { |
| 50 | uint32_t usedVecNum{0U}; // 归约过程中使用的AIV数量 | 52 | uint32_t usedVecNum{0U}; // 归约过程中使用的AIV数量 |
| 51 | // 1、归约任务的索引信息 | 53 | // 1、归约任务的索引信息 |
| 52 | - std::vector<uint32_t> bN2Idx{}; // 每个归约任务的BN2索引,脚标为归约任务的序号,最大为核数-1 | 54 | + std::vector<uint32_t> bNIdx{}; // 每个归约任务的BN索引,脚标为归约任务的序号,最大为核数-1 |
| 53 | - std::vector<uint32_t> gS1Idx{}; // 每个归约任务的GS1索引,脚标为归约任务的序号 | 55 | + std::vector<uint32_t> mIdx{}; // 每个归约任务的m索引,脚标为归约任务的序号 |
| 54 | std::vector<uint32_t> workspaceIdx{}; // 每个归约任务在workspace中的存放位置 | 56 | std::vector<uint32_t> workspaceIdx{}; // 每个归约任务在workspace中的存放位置 |
| 55 | std::vector<uint32_t> s2SplitNum{}; // 每个归约任务的S2核间切分份数,脚标为归约任务的序号 | 57 | std::vector<uint32_t> s2SplitNum{}; // 每个归约任务的S2核间切分份数,脚标为归约任务的序号 |
| 56 | // 2、FD kernel阶段,归约任务的分核信息 | 58 | // 2、FD kernel阶段,归约任务的分核信息 |
| @@ -59,10 +61,14 @@ struct SectionStreamKFdResult { | |||
| 59 | std::vector<uint32_t> mLen{}; // 每个AIV处理的归约任务的M轴行数 | 61 | std::vector<uint32_t> mLen{}; // 每个AIV处理的归约任务的M轴行数 |
| 60 | 62 | ||
| 61 | explicit SectionStreamKFdResult(uint32_t aicNum, uint32_t aivNum) | 63 | explicit SectionStreamKFdResult(uint32_t aicNum, uint32_t aivNum) |
| 62 | - : bN2Idx(aicNum), gS1Idx(aicNum), workspaceIdx(aicNum), s2SplitNum(aicNum), taskIdx(aivNum), mStart(aivNum), | 64 | + : bNIdx(aicNum), |
| 65 | + mIdx(aicNum), | ||
| 66 | + workspaceIdx(aicNum), | ||
| 67 | + s2SplitNum(aicNum), | ||
| 68 | + taskIdx(aivNum), | ||
| 69 | + mStart(aivNum), | ||
| 63 | mLen(aivNum) | 70 | mLen(aivNum) |
| 64 | - { | 71 | + {} |
| 65 | - } | ||
| 66 | }; | 72 | }; |
| 67 | 73 | ||
| 68 | struct SectionStreamKResult { | 74 | struct SectionStreamKResult { |
| @@ -22,14 +22,15 @@ | |||
| 22 | using namespace std; | 22 | using namespace std; |
| 23 | using namespace optiling; | 23 | using namespace optiling; |
| 24 | 24 | ||
| 25 | -#define CHECK_RET(cond) ((cond) ? true :(false)) | 25 | +#define CHECK_RET(cond) ((cond) ? true : (false)) |
| 26 | 26 | ||
| 27 | -#define LOG_PRINT(message, ...) \ | 27 | +#define LOG_PRINT(message, ...) \ |
| 28 | - do { \ | 28 | + do { \ |
| 29 | - printf(message, ##__VA_ARGS__); \ | 29 | + printf(message, ##__VA_ARGS__); \ |
| 30 | } while (0) | 30 | } while (0) |
| 31 | 31 | ||
| 32 | -int64_t GetShapeSize(const std::vector<int64_t> &shape) { | 32 | +int64_t GetShapeSize(const std::vector<int64_t> &shape) |
| 33 | +{ | ||
| 33 | int64_t shapeSize = 1; | 34 | int64_t shapeSize = 1; |
| 34 | for (auto i : shape) { | 35 | for (auto i : shape) { |
| 35 | shapeSize *= i; | 36 | shapeSize *= i; |
| @@ -37,7 +38,8 @@ int64_t GetShapeSize(const std::vector<int64_t> &shape) { | |||
| 37 | return shapeSize; | 38 | return shapeSize; |
| 38 | } | 39 | } |
| 39 | 40 | ||
| 40 | -int Init(int32_t deviceId, aclrtStream *stream) { | 41 | +int Init(int32_t deviceId, aclrtStream *stream) |
| 42 | +{ | ||
| 41 | // Fixed writing method, AscendCL initialization. | 43 | // Fixed writing method, AscendCL initialization. |
| 42 | auto ret = aclInit(nullptr); | 44 | auto ret = aclInit(nullptr); |
| 43 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 45 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| @@ -57,8 +59,9 @@ int Init(int32_t deviceId, aclrtStream *stream) { | |||
| 57 | return 0; | 59 | return 0; |
| 58 | } | 60 | } |
| 59 | 61 | ||
| 60 | -static void DumpMeta(void* data) { | 62 | +static void DumpMeta(void *data) |
| 61 | - int32_t sectionNum = static_cast<int32_t>(((int32_t*)data)[0]); | 63 | +{ |
| 64 | + int32_t sectionNum = static_cast<int32_t>(((int32_t *)data)[0]); | ||
| 62 | optiling::detail::FaMetadata faMetadata(data, sectionNum); | 65 | optiling::detail::FaMetadata faMetadata(data, sectionNum); |
| 63 | printf("sectionNum:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_SECTION_NUM_INDEX)); | 66 | printf("sectionNum:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_SECTION_NUM_INDEX)); |
| 64 | printf("isFd:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_IS_FD_INDEX)); | 67 | printf("isFd:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_IS_FD_INDEX)); |
| @@ -66,24 +69,24 @@ static void DumpMeta(void* data) { | |||
| 66 | printf("s2BaseSize:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_S2_BASE_SIZE_INDEX)); | 69 | printf("s2BaseSize:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_S2_BASE_SIZE_INDEX)); |
| 67 | for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) { | 70 | for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) { |
| 68 | // FA Metadata Generate | 71 | // FA Metadata Generate |
| 69 | - printf("sectionIdx:%d\n",sectionId); | 72 | + printf("sectionIdx:%d\n", sectionId); |
| 70 | for (size_t i = 0; i < AIC_CORE_NUM; ++i) { | 73 | for (size_t i = 0; i < AIC_CORE_NUM; ++i) { |
| 71 | // FA start | 74 | // FA start |
| 72 | - printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX)); | 75 | + printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX)); |
| 73 | printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX)); | 76 | printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX)); |
| 74 | printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX)); | 77 | printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX)); |
| 75 | // FA end | 78 | // FA end |
| 76 | - printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX)); | 79 | + printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX)); |
| 77 | printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX)); | 80 | printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX)); |
| 78 | printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX)); | 81 | printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX)); |
| 79 | // FA idx | 82 | // FA idx |
| 80 | - printf("first fd data ws idx: %d\n",faMetadata.GetFaMetadata(sectionId, i, | 83 | + printf("first fd data ws idx: %d\n", |
| 81 | - optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX)); | 84 | + faMetadata.GetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX)); |
| 82 | } | 85 | } |
| 83 | 86 | ||
| 84 | // FD Metadata Generate | 87 | // FD Metadata Generate |
| 85 | for (size_t i = 0; i < AIV_CORE_NUM; ++i) { | 88 | for (size_t i = 0; i < AIV_CORE_NUM; ++i) { |
| 86 | - printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX)); | 89 | + printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX)); |
| 87 | printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX)); | 90 | printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX)); |
| 88 | printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX)); | 91 | printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX)); |
| 89 | printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX)); | 92 | printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX)); |
| @@ -95,7 +98,8 @@ static void DumpMeta(void* data) { | |||
| 95 | 98 | ||
| 96 | template <typename T> | 99 | template <typename T> |
| 97 | int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr, | 100 | int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr, |
| 98 | - aclDataType dataType, aclTensor **tensor) { | 101 | + aclDataType dataType, aclTensor **tensor) |
| 102 | +{ | ||
| 99 | auto size = GetShapeSize(shape) * sizeof(T); | 103 | auto size = GetShapeSize(shape) * sizeof(T); |
| 100 | // Call aclrtMalloc to request device side memory. | 104 | // Call aclrtMalloc to request device side memory. |
| 101 | auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST); | 105 | auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST); |
| @@ -122,7 +126,8 @@ int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> & | |||
| 122 | return 0; | 126 | return 0; |
| 123 | } | 127 | } |
| 124 | 128 | ||
| 125 | -int main() { | 129 | +int main() |
| 130 | +{ | ||
| 126 | // 1. (Fixed writing method) device/stream initialization. Refer to AscendCL's list of external interfaces. | 131 | // 1. (Fixed writing method) device/stream initialization. Refer to AscendCL's list of external interfaces. |
| 127 | // Fill in the deviceId based on your actual device. | 132 | // Fill in the deviceId based on your actual device. |
| 128 | int32_t deviceId = 0; | 133 | int32_t deviceId = 0; |
| @@ -145,10 +150,10 @@ int main() { | |||
| 145 | int32_t preTokens = -1; | 150 | int32_t preTokens = -1; |
| 146 | int32_t nextTokens = -1; | 151 | int32_t nextTokens = -1; |
| 147 | string layout_query = "BSND"; | 152 | string layout_query = "BSND"; |
| 148 | - char layoutQuery[256]; // 256: default length | 153 | + char layoutQuery[256]; // 256: default length |
| 149 | strcpy_s(layoutQuery, sizeof(layoutQuery), layout_query.c_str()); | 154 | strcpy_s(layoutQuery, sizeof(layoutQuery), layout_query.c_str()); |
| 150 | string layout_kv = "BSND"; | 155 | string layout_kv = "BSND"; |
| 151 | - char layoutKv[256]; // 256: default length | 156 | + char layoutKv[256]; // 256: default length |
| 152 | strcpy_s(layoutKv, sizeof(layoutKv), layout_kv.c_str()); | 157 | strcpy_s(layoutKv, sizeof(layoutKv), layout_kv.c_str()); |
| 153 | int32_t sparseMode = 3; | 158 | int32_t sparseMode = 3; |
| 154 | int32_t attentionMode = 0; | 159 | int32_t attentionMode = 0; |
| @@ -163,9 +168,9 @@ int main() { | |||
| 163 | void *actualSeqLengthsQueryDeviceAddr = nullptr; | 168 | void *actualSeqLengthsQueryDeviceAddr = nullptr; |
| 164 | void *actualSeqLengthsKvDeviceAddr = nullptr; | 169 | void *actualSeqLengthsKvDeviceAddr = nullptr; |
| 165 | void *metadataDeviceAddr = nullptr; | 170 | void *metadataDeviceAddr = nullptr; |
| 166 | - aclTensor* actualSeqLengthsQueryTensor = nullptr; | 171 | + aclTensor *actualSeqLengthsQueryTensor = nullptr; |
| 167 | - aclTensor* actualSeqLengthsKvTensor = nullptr; | 172 | + aclTensor *actualSeqLengthsKvTensor = nullptr; |
| 168 | - aclTensor* metadataTensor = nullptr; | 173 | + aclTensor *metadataTensor = nullptr; |
| 169 | int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape); | 174 | int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape); |
| 170 | int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape); | 175 | int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape); |
| 171 | int64_t metadataShapeSize = GetShapeSize(metadataShape); | 176 | int64_t metadataShapeSize = GetShapeSize(metadataShape); |
| @@ -175,40 +180,34 @@ int main() { | |||
| 175 | 180 | ||
| 176 | // Create actual_seq_lengths_query aclTensor. | 181 | // Create actual_seq_lengths_query aclTensor. |
| 177 | ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr, | 182 | ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr, |
| 178 | - aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor); | 183 | + aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor); |
| 179 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 184 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 180 | return ret; | 185 | return ret; |
| 181 | } | 186 | } |
| 182 | // Create actual_seq_lengths_kv aclTensor. | 187 | // Create actual_seq_lengths_kv aclTensor. |
| 183 | ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr, | 188 | ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr, |
| 184 | - aclDataType::ACL_INT32, &actualSeqLengthsKvTensor); | 189 | + aclDataType::ACL_INT32, &actualSeqLengthsKvTensor); |
| 185 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 190 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 186 | return ret; | 191 | return ret; |
| 187 | } | 192 | } |
| 188 | // Create metadata aclTensor. | 193 | // Create metadata aclTensor. |
| 189 | - ret = CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor); | 194 | + ret = |
| 195 | + CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor); | ||
| 190 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 196 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 191 | return ret; | 197 | return ret; |
| 192 | } | 198 | } |
| 193 | 199 | ||
| 194 | - aclOpExecutor* executor = nullptr; | 200 | + aclOpExecutor *executor = nullptr; |
| 195 | uint64_t workspaceSize = 0; | 201 | uint64_t workspaceSize = 0; |
| 196 | - void* workspaceAddr = nullptr; | 202 | + void *workspaceAddr = nullptr; |
| 197 | 203 | ||
| 198 | char socVersion[] = "ascend950"; | 204 | char socVersion[] = "ascend950"; |
| 199 | printf("start aclnnFlashAttnMetadata\n"); | 205 | printf("start aclnnFlashAttnMetadata\n"); |
| 200 | - ret = aclnnFlashAttnMetadataGetWorkspaceSize( | 206 | + ret = aclnnFlashAttnMetadataGetWorkspaceSize(nullptr, nullptr, nullptr, nullptr, batchSize, qS, kvS, numHeads, |
| 201 | - nullptr, nullptr, | 207 | + numKeyValueHeads, headDim, sparseMode, preTokens, nextTokens, "BSND", |
| 202 | - nullptr, nullptr, | 208 | + "BSND", "BSND", metadataTensor, &workspaceSize, &executor); |
| 203 | - batchSize, qS, kvS, | ||
| 204 | - numHeads, numKeyValueHeads, | ||
| 205 | - headDim, sparseMode, preTokens, nextTokens, | ||
| 206 | - "BSND", "BSND", "BSND", | ||
| 207 | - metadataTensor, | ||
| 208 | - &workspaceSize, &executor); | ||
| 209 | if (ret != ACL_SUCCESS) { | 209 | if (ret != ACL_SUCCESS) { |
| 210 | - printf("aclnnFlashAttnMetadataGetWorkspaceSize %d\n", | 210 | + printf("aclnnFlashAttnMetadataGetWorkspaceSize %d\n", ret); |
| 211 | - ret); | ||
| 212 | return -1; | 211 | return -1; |
| 213 | } | 212 | } |
| 214 | 213 | ||
| @@ -224,10 +223,8 @@ int main() { | |||
| 224 | return -1; | 223 | return -1; |
| 225 | } | 224 | } |
| 226 | 225 | ||
| 227 | - ret = aclrtMemcpy(metadataHostData.data(), | 226 | + ret = aclrtMemcpy(metadataHostData.data(), metadataHostData.size() * sizeof(metadataHostData[0]), |
| 228 | - metadataHostData.size() * sizeof(metadataHostData[0]), metadataDeviceAddr, | 227 | + metadataDeviceAddr, metadataShapeSize, ACL_MEMCPY_DEVICE_TO_HOST); |
| 229 | - metadataShapeSize, | ||
| 230 | - ACL_MEMCPY_DEVICE_TO_HOST); | ||
| 231 | if (ret != ACL_SUCCESS) { | 228 | if (ret != ACL_SUCCESS) { |
| 232 | printf("aclrtMemcpy %d\n", ret); | 229 | printf("aclrtMemcpy %d\n", ret); |
| 233 | return -1; | 230 | return -1; |
| @@ -38,16 +38,16 @@ constexpr uint32_t HEAD_M_BASE_SIZE_INDEX = 2U; | |||
| 38 | constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U; | 38 | constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U; |
| 39 | 39 | ||
| 40 | // FA Metadata Index Definitions | 40 | // FA Metadata Index Definitions |
| 41 | -constexpr uint32_t FA_BN2_START_INDEX = 0U; | 41 | +constexpr uint32_t FA_BN_START_INDEX = 0U; |
| 42 | constexpr uint32_t FA_M_START_INDEX = 1U; | 42 | constexpr uint32_t FA_M_START_INDEX = 1U; |
| 43 | constexpr uint32_t FA_S2_START_INDEX = 2U; | 43 | constexpr uint32_t FA_S2_START_INDEX = 2U; |
| 44 | -constexpr uint32_t FA_BN2_END_INDEX = 3U; | 44 | +constexpr uint32_t FA_BN_END_INDEX = 3U; |
| 45 | constexpr uint32_t FA_M_END_INDEX = 4U; | 45 | constexpr uint32_t FA_M_END_INDEX = 4U; |
| 46 | constexpr uint32_t FA_S2_END_INDEX = 5U; | 46 | constexpr uint32_t FA_S2_END_INDEX = 5U; |
| 47 | constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U; | 47 | constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U; |
| 48 | 48 | ||
| 49 | // FD Metadata Index Definitions | 49 | // FD Metadata Index Definitions |
| 50 | -constexpr uint32_t FD_BN2_IDX_INDEX = 0U; | 50 | +constexpr uint32_t FD_BN_IDX_INDEX = 0U; |
| 51 | constexpr uint32_t FD_M_IDX_INDEX = 1U; | 51 | constexpr uint32_t FD_M_IDX_INDEX = 1U; |
| 52 | constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U; | 52 | constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U; |
| 53 | constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U; | 53 | constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U; |
| @@ -58,13 +58,13 @@ namespace detail { | |||
| 58 | struct FaMetadata { | 58 | struct FaMetadata { |
| 59 | uint32_t sectionNum; | 59 | uint32_t sectionNum; |
| 60 | FA_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE]; | 60 | FA_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE]; |
| 61 | - FA_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE]; | 61 | + FA_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE]; |
| 62 | - FA_METADATA_T *fdMetadata; // [sectionNum][AIV_CORE_NUM][FD_METADATA_STRIDE]; | 62 | + FA_METADATA_T *fdMetadata; // [sectionNum][AIV_CORE_NUM][FD_METADATA_STRIDE]; |
| 63 | FaMetadata(void *metadataPtr, uint32_t sectionNum) | 63 | FaMetadata(void *metadataPtr, uint32_t sectionNum) |
| 64 | - :sectionNum(sectionNum), | 64 | + : sectionNum(sectionNum), |
| 65 | - headMetadata(static_cast<FA_METADATA_T*>(metadataPtr)), | 65 | + headMetadata(static_cast<FA_METADATA_T *>(metadataPtr)), |
| 66 | - faMetadata(headMetadata + HEAD_METADATA_STRIDE), | 66 | + faMetadata(headMetadata + HEAD_METADATA_STRIDE), |
| 67 | - fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_STRIDE) | 67 | + fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_STRIDE) |
| 68 | { | 68 | { |
| 69 | headMetadata[0] = sectionNum; | 69 | headMetadata[0] = sectionNum; |
| 70 | } | 70 | } |
| @@ -57,12 +57,10 @@ bool FlashAttnMetadataCpuKernel::Prepare(CpuKernelContext &ctx) | |||
| 57 | 57 | ||
| 58 | KERNEL_CHECK_FALSE((metadata_ != nullptr && metadata_->GetData() != nullptr), false, "metadata is empty"); | 58 | KERNEL_CHECK_FALSE((metadata_ != nullptr && metadata_->GetData() != nullptr), false, "metadata is empty"); |
| 59 | 59 | ||
| 60 | - bool requiredAttrs = GetAttrValue(ctx, "num_heads_q", numHeadsQ_) && | 60 | + bool requiredAttrs = |
| 61 | - GetAttrValue(ctx, "num_heads_kv", numHeadsKv_) && | 61 | + GetAttrValue(ctx, "num_heads_q", numHeadsQ_) && GetAttrValue(ctx, "num_heads_kv", numHeadsKv_) && |
| 62 | - GetAttrValue(ctx, "head_dim", headDim_) && | 62 | + GetAttrValue(ctx, "head_dim", headDim_) && GetAttrValue(ctx, "soc_version", socVersion_) && |
| 63 | - GetAttrValue(ctx, "soc_version", socVersion_) && | 63 | + GetAttrValue(ctx, "aic_core_num", aicCoreNum_) && GetAttrValue(ctx, "aiv_core_num", aivCoreNum_); |
| 64 | - GetAttrValue(ctx, "aic_core_num", aicCoreNum_) && | ||
| 65 | - GetAttrValue(ctx, "aiv_core_num", aivCoreNum_); | ||
| 66 | KERNEL_CHECK_FALSE(requiredAttrs, false, "Missing Required attrs missing!"); | 64 | KERNEL_CHECK_FALSE(requiredAttrs, false, "Missing Required attrs missing!"); |
| 67 | 65 | ||
| 68 | // attributes optional | 66 | // attributes optional |
| @@ -122,8 +120,8 @@ bool FlashAttnMetadataCpuKernel::CheckActualQuerySeq() | |||
| 122 | for (size_t i = 1; i < cuSeqlensQ.size(); ++i) { | 120 | for (size_t i = 1; i < cuSeqlensQ.size(); ++i) { |
| 123 | if (cuSeqlensQ[i] < cuSeqlensQ[i - 1]) { | 121 | if (cuSeqlensQ[i] < cuSeqlensQ[i - 1]) { |
| 124 | KERNEL_LOG_ERROR( | 122 | KERNEL_LOG_ERROR( |
| 125 | - "The %zuth element of cuSeqlensQ must not be less than the %zuth element, but got %ld and %ld", | 123 | + "The %zuth element of cuSeqlensQ must not be less than the %zuth element, but got %ld and %ld", i, |
| 126 | - i, i - 1, cuSeqlensQ[i], cuSeqlensQ[i - 1]); | 124 | + i - 1, cuSeqlensQ[i], cuSeqlensQ[i - 1]); |
| 127 | return false; | 125 | return false; |
| 128 | } | 126 | } |
| 129 | } | 127 | } |
| @@ -151,8 +149,8 @@ bool FlashAttnMetadataCpuKernel::CheckActualKvSeq() | |||
| 151 | 149 | ||
| 152 | for (size_t i = 0; i < sequsedKv.size(); ++i) { | 150 | for (size_t i = 0; i < sequsedKv.size(); ++i) { |
| 153 | if (sequsedKv[i] < 0) { | 151 | if (sequsedKv[i] < 0) { |
| 154 | - KERNEL_LOG_ERROR("The elements of sequsedKv must be non-negative, but %zuth element is %ld", | 152 | + KERNEL_LOG_ERROR("The elements of sequsedKv must be non-negative, but %zuth element is %ld", i, |
| 155 | - i, sequsedKv[i]); | 153 | + sequsedKv[i]); |
| 156 | return false; | 154 | return false; |
| 157 | } | 155 | } |
| 158 | } | 156 | } |
| @@ -167,8 +165,8 @@ bool FlashAttnMetadataCpuKernel::CheckActualKvSeq() | |||
| 167 | for (size_t i = 1; i < cuSeqlensKv.size(); ++i) { | 165 | for (size_t i = 1; i < cuSeqlensKv.size(); ++i) { |
| 168 | if (cuSeqlensKv[i] < cuSeqlensKv[i - 1]) { | 166 | if (cuSeqlensKv[i] < cuSeqlensKv[i - 1]) { |
| 169 | KERNEL_LOG_ERROR( | 167 | KERNEL_LOG_ERROR( |
| 170 | - "The %zuth element of cuSeqlensKv must not be less than the %zuth element, but got %ld and %ld", | 168 | + "The %zuth element of cuSeqlensKv must not be less than the %zuth element, but got %ld and %ld", i, |
| 171 | - i, i - 1, cuSeqlensKv[i], cuSeqlensKv[i - 1]); | 169 | + i - 1, cuSeqlensKv[i], cuSeqlensKv[i - 1]); |
| 172 | return false; | 170 | return false; |
| 173 | } | 171 | } |
| 174 | } | 172 | } |
| @@ -200,10 +198,9 @@ void FlashAttnMetadataCpuKernel::InitLoadBalanceParams() | |||
| 200 | } else if (baseInfo.layoutQuery == load_balance::Layout::TND) { | 198 | } else if (baseInfo.layoutQuery == load_balance::Layout::TND) { |
| 201 | qlayout = optiling::flash_attn::fa_tiling_util::LAYOUT_TND; | 199 | qlayout = optiling::flash_attn::fa_tiling_util::LAYOUT_TND; |
| 202 | } | 200 | } |
| 203 | - optiling::flash_attn::fa_tiling_util::AdjustSinnerAndSouter(baseInfo.headDim, maxSeqlenQ_, | 201 | + optiling::flash_attn::fa_tiling_util::AdjustSinnerAndSouter(baseInfo.headDimQk, maxSeqlenQ_, maxSeqlenKv_, |
| 204 | - maxSeqlenKv_, baseInfo.sparseMode, baseInfo.preToken, | 202 | + baseInfo.sparseMode, baseInfo.preToken, |
| 205 | - baseInfo.nextToken, qlayout, | 203 | + baseInfo.nextToken, qlayout, mBaseSize_, s2BaseSize_); |
| 206 | - mBaseSize_, s2BaseSize_); | ||
| 207 | mBaseSize_ *= (aivCoreNum_ / aicCoreNum_); | 204 | mBaseSize_ *= (aivCoreNum_ / aicCoreNum_); |
| 208 | param.mBaseSize = mBaseSize_; | 205 | param.mBaseSize = mBaseSize_; |
| 209 | param.s2BaseSize = s2BaseSize_; | 206 | param.s2BaseSize = s2BaseSize_; |
| @@ -211,6 +208,7 @@ void FlashAttnMetadataCpuKernel::InitLoadBalanceParams() | |||
| 211 | param.fdTolerance = 10; // 10: tolerance block | 208 | param.fdTolerance = 10; // 10: tolerance block |
| 212 | param.fdLeastBlock = 3; // 3: least block | 209 | param.fdLeastBlock = 3; // 3: least block |
| 213 | param.fdOn = true; | 210 | param.fdOn = true; |
| 211 | + param.outputLayout = load_balance::OutputLayout::BN2_S1G; | ||
| 214 | } | 212 | } |
| 215 | 213 | ||
| 216 | void FlashAttnMetadataCpuKernel::InitBaseInfo() | 214 | void FlashAttnMetadataCpuKernel::InitBaseInfo() |
| @@ -220,7 +218,8 @@ void FlashAttnMetadataCpuKernel::InitBaseInfo() | |||
| 220 | baseInfo.queryHeadNum = numHeadsQ_; | 218 | baseInfo.queryHeadNum = numHeadsQ_; |
| 221 | baseInfo.kvSeqSize = maxSeqlenKv_; | 219 | baseInfo.kvSeqSize = maxSeqlenKv_; |
| 222 | baseInfo.kvHeadNum = numHeadsKv_; | 220 | baseInfo.kvHeadNum = numHeadsKv_; |
| 223 | - baseInfo.headDim = headDim_; | 221 | + baseInfo.headDimQk = headDim_; |
| 222 | + baseInfo.headDimV = headDim_; | ||
| 224 | load_balance::SparseMode maskMode = load_balance::SparseMode::BUTT; | 223 | load_balance::SparseMode maskMode = load_balance::SparseMode::BUTT; |
| 225 | if (maskMode_ != 0) { | 224 | if (maskMode_ != 0) { |
| 226 | maskMode = static_cast<load_balance::SparseMode>(maskMode_); | 225 | maskMode = static_cast<load_balance::SparseMode>(maskMode_); |
| @@ -263,15 +262,15 @@ bool FlashAttnMetadataCpuKernel::GenMetadata(load_balance::SectionStreamKResult | |||
| 263 | for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) { | 262 | for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) { |
| 264 | auto &prevFaRes = (secIdx == 0U) ? dummyHead : splitRes.sectionFaResult[secIdx - 1U]; | 263 | auto &prevFaRes = (secIdx == 0U) ? dummyHead : splitRes.sectionFaResult[secIdx - 1U]; |
| 265 | auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U; | 264 | auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U; |
| 266 | - FA_METADATA_T bn2Start = (aicIdx == 0) ? prevFaRes.bN2End[prevLastCore] : faRes.bN2End[aicIdx - 1U]; | 265 | + FA_METADATA_T bnStart = (aicIdx == 0) ? prevFaRes.bNEnd[prevLastCore] : faRes.bNEnd[aicIdx - 1U]; |
| 267 | - FA_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.gS1End[prevLastCore] : faRes.gS1End[aicIdx - 1U]; | 266 | + FA_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.mEnd[prevLastCore] : faRes.mEnd[aicIdx - 1U]; |
| 268 | FA_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U]; | 267 | FA_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U]; |
| 269 | 268 | ||
| 270 | - faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN2_START_INDEX, bn2Start); | 269 | + faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN_START_INDEX, bnStart); |
| 271 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_START_INDEX, mStart); | 270 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_START_INDEX, mStart); |
| 272 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_START_INDEX, s2Start); | 271 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_START_INDEX, s2Start); |
| 273 | - faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN2_END_INDEX, faRes.bN2End[aicIdx]); | 272 | + faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN_END_INDEX, faRes.bNEnd[aicIdx]); |
| 274 | - faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_END_INDEX, faRes.gS1End[aicIdx]); | 273 | + faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_END_INDEX, faRes.mEnd[aicIdx]); |
| 275 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_END_INDEX, faRes.s2End[aicIdx]); | 274 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_END_INDEX, faRes.s2End[aicIdx]); |
| 276 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, | 275 | faMetadata.SetFaMetadata(secIdx, aicIdx, FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, |
| 277 | faRes.firstFdDataWorkspaceIdx[aicIdx]); | 276 | faRes.firstFdDataWorkspaceIdx[aicIdx]); |
| @@ -280,8 +279,8 @@ bool FlashAttnMetadataCpuKernel::GenMetadata(load_balance::SectionStreamKResult | |||
| 280 | auto &fdRes = splitRes.sectionFdResult[secIdx]; | 279 | auto &fdRes = splitRes.sectionFdResult[secIdx]; |
| 281 | for (uint32_t aivIdx = 0; aivIdx < fdRes.usedVecNum; ++aivIdx) { | 280 | for (uint32_t aivIdx = 0; aivIdx < fdRes.usedVecNum; ++aivIdx) { |
| 282 | uint32_t t = fdRes.taskIdx[aivIdx]; | 281 | uint32_t t = fdRes.taskIdx[aivIdx]; |
| 283 | - faMetadata.SetFdMetadata(secIdx, aivIdx, FD_BN2_IDX_INDEX, fdRes.bN2Idx[t]); | 282 | + faMetadata.SetFdMetadata(secIdx, aivIdx, FD_BN_IDX_INDEX, fdRes.bNIdx[t]); |
| 284 | - faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_IDX_INDEX, fdRes.gS1Idx[t]); | 283 | + faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_IDX_INDEX, fdRes.mIdx[t]); |
| 285 | faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_IDX_INDEX, fdRes.workspaceIdx[t]); | 284 | faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_IDX_INDEX, fdRes.workspaceIdx[t]); |
| 286 | faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_NUM_INDEX, fdRes.s2SplitNum[t]); | 285 | faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_NUM_INDEX, fdRes.s2SplitNum[t]); |
| 287 | faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_START_INDEX, fdRes.mStart[aivIdx]); | 286 | faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_START_INDEX, fdRes.mStart[aivIdx]); |
| @@ -6,8 +6,8 @@ | |||
| 6 | * this file except in compliance with the License. THIS SOFTWARE IS PROVIDED ON | 6 | * this file except in compliance with the License. THIS SOFTWARE IS PROVIDED ON |
| 7 | * AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 7 | * AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 8 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS | 8 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS |
| 9 | - * FOR A PARTICULAR PURPOSE. See LICENSE in the software repository for the full | 9 | + * FOR A PARTICULAR PURPOSE. See LICENSE in the root of the software repository |
| 10 | - * text of the License. | 10 | + * for the full text of the License. |
| 11 | */ | 11 | */ |
| 12 | 12 | ||
| 13 | #include <iostream> | 13 | #include <iostream> |
| @@ -22,14 +22,15 @@ | |||
| 22 | using namespace std; | 22 | using namespace std; |
| 23 | using namespace optiling; | 23 | using namespace optiling; |
| 24 | 24 | ||
| 25 | -#define CHECK_RET(cond) ((cond) ? true :(false)) | 25 | +#define CHECK_RET(cond) ((cond) ? true : (false)) |
| 26 | 26 | ||
| 27 | -#define LOG_PRINT(message, ...) \ | 27 | +#define LOG_PRINT(message, ...) \ |
| 28 | - do { \ | 28 | + do { \ |
| 29 | - printf(message, ##__VA_ARGS__); \ | 29 | + printf(message, ##__VA_ARGS__); \ |
| 30 | } while (0) | 30 | } while (0) |
| 31 | 31 | ||
| 32 | -int64_t GetShapeSize(const std::vector<int64_t> &shape) { | 32 | +int64_t GetShapeSize(const std::vector<int64_t> &shape) |
| 33 | +{ | ||
| 33 | int64_t shapeSize = 1; | 34 | int64_t shapeSize = 1; |
| 34 | for (auto i : shape) { | 35 | for (auto i : shape) { |
| 35 | shapeSize *= i; | 36 | shapeSize *= i; |
| @@ -37,46 +38,49 @@ int64_t GetShapeSize(const std::vector<int64_t> &shape) { | |||
| 37 | return shapeSize; | 38 | return shapeSize; |
| 38 | } | 39 | } |
| 39 | 40 | ||
| 40 | -int Init(int32_t deviceId, aclrtStream *stream) { | 41 | +int Init(int32_t deviceId, aclrtStream *stream) |
| 42 | +{ | ||
| 41 | auto ret = aclInit(nullptr); | 43 | auto ret = aclInit(nullptr); |
| 42 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 44 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 43 | - LOG_PRINT("aclInit failed. ERROR: %d\n", ret); | 45 | + LOG_PRINT("aclInit failed. ERROR: %d\n", ret); |
| 44 | return ret; | 46 | return ret; |
| 45 | } | 47 | } |
| 46 | ret = aclrtSetDevice(deviceId); | 48 | ret = aclrtSetDevice(deviceId); |
| 47 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 49 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 48 | - LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); | 50 | + LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); |
| 49 | return ret; | 51 | return ret; |
| 50 | } | 52 | } |
| 51 | ret = aclrtCreateStream(stream); | 53 | ret = aclrtCreateStream(stream); |
| 52 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 54 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 53 | - LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); | 55 | + LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); |
| 54 | return ret; | 56 | return ret; |
| 55 | } | 57 | } |
| 56 | return 0; | 58 | return 0; |
| 57 | } | 59 | } |
| 58 | 60 | ||
| 59 | -static void DumpMeta(void* data) { | 61 | +static void DumpMeta(void *data) |
| 60 | - int32_t sectionNum = static_cast<int32_t>(((int32_t*)data)[0]); | 62 | +{ |
| 63 | + int32_t sectionNum = static_cast<int32_t>(((int32_t *)data)[0]); | ||
| 61 | optiling::detail::FaMetaData faMetadata(data, sectionNum); | 64 | optiling::detail::FaMetaData faMetadata(data, sectionNum); |
| 62 | printf("sectionNum:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_SECTION_NUM_INDEX)); | 65 | printf("sectionNum:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_SECTION_NUM_INDEX)); |
| 63 | printf("isFd:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_IS_FD_INDEX)); | 66 | printf("isFd:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_IS_FD_INDEX)); |
| 64 | printf("mBaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_M_BASE_SIZE_INDEX)); | 67 | printf("mBaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_M_BASE_SIZE_INDEX)); |
| 65 | printf("s2BaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_S2_BASE_SIZE_INDEX)); | 68 | printf("s2BaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_S2_BASE_SIZE_INDEX)); |
| 66 | for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) { | 69 | for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) { |
| 67 | - printf("sectionIdx:%d\n",sectionId); | 70 | + printf("sectionIdx:%d\n", sectionId); |
| 68 | for (size_t i = 0; i < AIC_CORE_NUM; ++i) { | 71 | for (size_t i = 0; i < AIC_CORE_NUM; ++i) { |
| 69 | - printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX)); | 72 | + printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX)); |
| 70 | printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX)); | 73 | printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX)); |
| 71 | printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX)); | 74 | printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX)); |
| 72 | - printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX)); | 75 | + printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX)); |
| 73 | printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX)); | 76 | printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX)); |
| 74 | printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX)); | 77 | printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX)); |
| 75 | - printf("first fd data ws idx: %d\n",faMetadata.GetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX)); | 78 | + printf("first fd data ws idx: %d\n", |
| 79 | + faMetadata.GetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX)); | ||
| 76 | } | 80 | } |
| 77 | 81 | ||
| 78 | for (size_t i = 0; i < AIV_CORE_NUM; ++i) { | 82 | for (size_t i = 0; i < AIV_CORE_NUM; ++i) { |
| 79 | - printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX)); | 83 | + printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX)); |
| 80 | printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX)); | 84 | printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX)); |
| 81 | printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX)); | 85 | printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX)); |
| 82 | printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX)); | 86 | printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX)); |
| @@ -88,16 +92,17 @@ static void DumpMeta(void* data) { | |||
| 88 | 92 | ||
| 89 | template <typename T> | 93 | template <typename T> |
| 90 | int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr, | 94 | int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr, |
| 91 | - aclDataType dataType, aclTensor **tensor) { | 95 | + aclDataType dataType, aclTensor **tensor) |
| 96 | +{ | ||
| 92 | auto size = GetShapeSize(shape) * sizeof(T); | 97 | auto size = GetShapeSize(shape) * sizeof(T); |
| 93 | auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST); | 98 | auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST); |
| 94 | - if (!CHECK_RET(ret == ACL_SUCCESS)) { | 99 | + if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 95 | - LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret); | 100 | + LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret); |
| 96 | return ret; | 101 | return ret; |
| 97 | } | 102 | } |
| 98 | ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE); | 103 | ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE); |
| 99 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 104 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 100 | - LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret); | 105 | + LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret); |
| 101 | return ret; | 106 | return ret; |
| 102 | } | 107 | } |
| 103 | 108 | ||
| @@ -111,12 +116,13 @@ int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> & | |||
| 111 | return 0; | 116 | return 0; |
| 112 | } | 117 | } |
| 113 | 118 | ||
| 114 | -int main() { | 119 | +int main() |
| 120 | +{ | ||
| 115 | int32_t deviceId = 0; | 121 | int32_t deviceId = 0; |
| 116 | aclrtStream stream; | 122 | aclrtStream stream; |
| 117 | auto ret = Init(deviceId, &stream); | 123 | auto ret = Init(deviceId, &stream); |
| 118 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 124 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 119 | - LOG_PRINT("Init acl failed. ERROR: %d\n", ret); | 125 | + LOG_PRINT("Init acl failed. ERROR: %d\n", ret); |
| 120 | return ret; | 126 | return ret; |
| 121 | } | 127 | } |
| 122 | 128 | ||
| @@ -147,9 +153,9 @@ int main() { | |||
| 147 | void *actualSeqLengthsQueryDeviceAddr = nullptr; | 153 | void *actualSeqLengthsQueryDeviceAddr = nullptr; |
| 148 | void *actualSeqLengthsKvDeviceAddr = nullptr; | 154 | void *actualSeqLengthsKvDeviceAddr = nullptr; |
| 149 | void *metadataDeviceAddr = nullptr; | 155 | void *metadataDeviceAddr = nullptr; |
| 150 | - aclTensor* actualSeqLengthsQueryTensor = nullptr; | 156 | + aclTensor *actualSeqLengthsQueryTensor = nullptr; |
| 151 | - aclTensor* actualSeqLengthsKvTensor = nullptr; | 157 | + aclTensor *actualSeqLengthsKvTensor = nullptr; |
| 152 | - aclTensor* metadataTensor = nullptr; | 158 | + aclTensor *metadataTensor = nullptr; |
| 153 | int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape); | 159 | int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape); |
| 154 | int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape); | 160 | int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape); |
| 155 | int64_t metadataShapeSize = GetShapeSize(metadataShape); | 161 | int64_t metadataShapeSize = GetShapeSize(metadataShape); |
| @@ -157,34 +163,30 @@ int main() { | |||
| 157 | std::vector<int32_t> actualSeqLengthsKvHostData(actualSeqLengthsKvShapeSize, kvS); | 163 | std::vector<int32_t> actualSeqLengthsKvHostData(actualSeqLengthsKvShapeSize, kvS); |
| 158 | std::vector<int32_t> metadataHostData(metadataShapeSize, 0); | 164 | std::vector<int32_t> metadataHostData(metadataShapeSize, 0); |
| 159 | 165 | ||
| 160 | - ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr, aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor); | 166 | + ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr, |
| 167 | + aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor); | ||
| 161 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 168 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 162 | return ret; | 169 | return ret; |
| 163 | } | 170 | } |
| 164 | - ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr, aclDataType::ACL_INT32, &actualSeqLengthsKvTensor); | 171 | + ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr, |
| 172 | + aclDataType::ACL_INT32, &actualSeqLengthsKvTensor); | ||
| 165 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 173 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 166 | return ret; | 174 | return ret; |
| 167 | } | 175 | } |
| 168 | - ret = CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor); | 176 | + ret = |
| 177 | + CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor); | ||
| 169 | if (!CHECK_RET(ret == ACL_SUCCESS)) { | 178 | if (!CHECK_RET(ret == ACL_SUCCESS)) { |
| 170 | return ret; | 179 | return ret; |
| 171 | } | 180 | } |
| 172 | 181 | ||
| 173 | - aclOpExecutor* executor = nullptr; | 182 | + aclOpExecutor *executor = nullptr; |
| 174 | uint64_t workspaceSize = 0; | 183 | uint64_t workspaceSize = 0; |
| 175 | - void* workspaceAddr = nullptr; | 184 | + void *workspaceAddr = nullptr; |
| 176 | 185 | ||
| 177 | printf("start aclnnQuantFlashAttnMetadata\n"); | 186 | printf("start aclnnQuantFlashAttnMetadata\n"); |
| 178 | ret = aclnnQuantFlashAttnMetadataGetWorkspaceSize( | 187 | ret = aclnnQuantFlashAttnMetadataGetWorkspaceSize( |
| 179 | - nullptr, nullptr, | 188 | + nullptr, nullptr, nullptr, nullptr, nullptr, batchSize, qS, kvS, numHeads, numKeyValueHeads, headDim, quantMode, |
| 180 | - nullptr, nullptr, | 189 | + sparseMode, preTokens, nextTokens, "BSND", "BSND", "BSND", "BSND", metadataTensor, &workspaceSize, &executor); |
| 181 | - nullptr, | ||
| 182 | - batchSize, qS, kvS, | ||
| 183 | - numHeads, numKeyValueHeads, | ||
| 184 | - headDim, quantMode, sparseMode, preTokens, nextTokens, | ||
| 185 | - "BSND", "BSND", "BSND", "BSND", | ||
| 186 | - metadataTensor, | ||
| 187 | - &workspaceSize, &executor); | ||
| 188 | if (ret != ACL_SUCCESS) { | 190 | if (ret != ACL_SUCCESS) { |
| 189 | printf("aclnnQuantFlashAttnMetadataGetWorkspaceSize %d\n", ret); | 191 | printf("aclnnQuantFlashAttnMetadataGetWorkspaceSize %d\n", ret); |
| 190 | return -1; | 192 | return -1; |
| @@ -202,15 +204,13 @@ int main() { | |||
| 202 | return -1; | 204 | return -1; |
| 203 | } | 205 | } |
| 204 | 206 | ||
| 205 | - ret = aclrtMemcpy(metadataHostData.data(), | 207 | + ret = aclrtMemcpy(metadataHostData.data(), metadataHostData.size() * sizeof(metadataHostData[0]), |
| 206 | - metadataHostData.size() * sizeof(metadataHostData[0]), metadataDeviceAddr, | 208 | + metadataDeviceAddr, metadataShapeSize, ACL_MEMCPY_DEVICE_TO_HOST); |
| 207 | - metadataShapeSize, | ||
| 208 | - ACL_MEMCPY_DEVICE_TO_HOST); | ||
| 209 | if (ret != ACL_SUCCESS) { | 209 | if (ret != ACL_SUCCESS) { |
| 210 | printf("aclrtMemcpy %d\n", ret); | 210 | printf("aclrtMemcpy %d\n", ret); |
| 211 | return -1; | 211 | return -1; |
| 212 | } | 212 | } |
| 213 | - | 213 | + |
| 214 | DumpMeta(&metadataHostData[0]); | 214 | DumpMeta(&metadataHostData[0]); |
| 215 | 215 | ||
| 216 | aclDestroyTensor(metadataTensor); | 216 | aclDestroyTensor(metadataTensor); |
| @@ -35,15 +35,15 @@ constexpr uint32_t HEAD_IS_FD_INDEX = 1U; | |||
| 35 | constexpr uint32_t HEAD_M_BASE_SIZE_INDEX = 2U; | 35 | constexpr uint32_t HEAD_M_BASE_SIZE_INDEX = 2U; |
| 36 | constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U; | 36 | constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U; |
| 37 | 37 | ||
| 38 | -constexpr uint32_t FA_BN2_START_INDEX = 0U; | 38 | +constexpr uint32_t FA_BN_START_INDEX = 0U; |
| 39 | constexpr uint32_t FA_M_START_INDEX = 1U; | 39 | constexpr uint32_t FA_M_START_INDEX = 1U; |
| 40 | constexpr uint32_t FA_S2_START_INDEX = 2U; | 40 | constexpr uint32_t FA_S2_START_INDEX = 2U; |
| 41 | -constexpr uint32_t FA_BN2_END_INDEX = 3U; | 41 | +constexpr uint32_t FA_BN_END_INDEX = 3U; |
| 42 | constexpr uint32_t FA_M_END_INDEX = 4U; | 42 | constexpr uint32_t FA_M_END_INDEX = 4U; |
| 43 | constexpr uint32_t FA_S2_END_INDEX = 5U; | 43 | constexpr uint32_t FA_S2_END_INDEX = 5U; |
| 44 | constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U; | 44 | constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U; |
| 45 | 45 | ||
| 46 | -constexpr uint32_t FD_BN2_IDX_INDEX = 0U; | 46 | +constexpr uint32_t FD_BN_IDX_INDEX = 0U; |
| 47 | constexpr uint32_t FD_M_IDX_INDEX = 1U; | 47 | constexpr uint32_t FD_M_IDX_INDEX = 1U; |
| 48 | constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U; | 48 | constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U; |
| 49 | constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U; | 49 | constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U; |
| @@ -57,12 +57,12 @@ struct FaMetaData { | |||
| 57 | uint32_t *faMetadata; | 57 | uint32_t *faMetadata; |
| 58 | uint32_t *fdMetadata; | 58 | uint32_t *fdMetadata; |
| 59 | FaMetaData(void *metadataPtr, uint32_t sectionNum) | 59 | FaMetaData(void *metadataPtr, uint32_t sectionNum) |
| 60 | - :sectionNum(sectionNum), | 60 | + : sectionNum(sectionNum), |
| 61 | - headMedata(static_cast<uint32_t*>(metadataPtr)), | 61 | + headMedata(static_cast<uint32_t *>(metadataPtr)), |
| 62 | - faMetadata(headMedata + HEAD_METADATA_SIZE), | 62 | + faMetadata(headMedata + HEAD_METADATA_SIZE), |
| 63 | - fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_SIZE) | 63 | + fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_SIZE) |
| 64 | { | 64 | { |
| 65 | - headMedata[0] = sectionNum; | 65 | + headMedata[0] = sectionNum; |
| 66 | } | 66 | } |
| 67 | 67 | ||
| 68 | void SetHeadMedata(uint32_t metaIdx, uint32_t val) | 68 | void SetHeadMedata(uint32_t metaIdx, uint32_t val) |
| @@ -195,7 +195,8 @@ bool QuantFlashAttnMetadataCpuKernel::ParamsInit() | |||
| 195 | baseInfo.queryHeadNum = numHeadsQ_; | 195 | baseInfo.queryHeadNum = numHeadsQ_; |
| 196 | bool isDecode = (layoutQDescale_ == "N2TGD"); | 196 | bool isDecode = (layoutQDescale_ == "N2TGD"); |
| 197 | baseInfo.kvHeadNum = isDecode ? numHeadsKv_ : numHeadsQ_; | 197 | baseInfo.kvHeadNum = isDecode ? numHeadsKv_ : numHeadsQ_; |
| 198 | - baseInfo.headDim = headDim_; | 198 | + baseInfo.headDimQk = headDim_; |
| 199 | + baseInfo.headDimV = headDim_; | ||
| 199 | baseInfo.attenMaskFlag = (maskMode_ != 0); | 200 | baseInfo.attenMaskFlag = (maskMode_ != 0); |
| 200 | baseInfo.sparseMode = static_cast<uint32_t>(maskMode_); | 201 | baseInfo.sparseMode = static_cast<uint32_t>(maskMode_); |
| 201 | baseInfo.preToken = winLeft_ == -1 ? std::numeric_limits<uint32_t>::max() : winLeft_; | 202 | baseInfo.preToken = winLeft_ == -1 ? std::numeric_limits<uint32_t>::max() : winLeft_; |
| @@ -207,7 +208,7 @@ bool QuantFlashAttnMetadataCpuKernel::ParamsInit() | |||
| 207 | baseInfo.kvType = load_balance::DataType::INT8; | 208 | baseInfo.kvType = load_balance::DataType::INT8; |
| 208 | } | 209 | } |
| 209 | if (quantMode_ == 1) { // TODO:用adjust函数代替 | 210 | if (quantMode_ == 1) { // TODO:用adjust函数代替 |
| 210 | - if (baseInfo.headDim == 256) { | 211 | + if (baseInfo.headDimQk == 256) { |
| 211 | mBaseSize_ = 64U; | 212 | mBaseSize_ = 64U; |
| 212 | s2BaseSize_ = 256U; | 213 | s2BaseSize_ = 256U; |
| 213 | } else { | 214 | } else { |
| @@ -224,6 +225,7 @@ bool QuantFlashAttnMetadataCpuKernel::ParamsInit() | |||
| 224 | param.l2Byte = 0; | 225 | param.l2Byte = 0; |
| 225 | param.fdTolerance = 300; | 226 | param.fdTolerance = 300; |
| 226 | param.fdOn = 0; | 227 | param.fdOn = 0; |
| 228 | + param.outputLayout = load_balance::OutputLayout::BN2_S1G; | ||
| 227 | 229 | ||
| 228 | // 校验 v_descale: quantMode=1(MxFp8) 且 TND layout 下, dim0 应等于 sum(ceil(seqused_kv[i] / 64)) | 230 | // 校验 v_descale: quantMode=1(MxFp8) 且 TND layout 下, dim0 应等于 sum(ceil(seqused_kv[i] / 64)) |
| 229 | if (quantMode_ == 1 && vDescale_ != nullptr && vDescale_->GetTensorShape() != nullptr) { | 231 | if (quantMode_ == 1 && vDescale_ != nullptr && vDescale_->GetTensorShape() != nullptr) { |
| @@ -280,16 +282,16 @@ bool QuantFlashAttnMetadataCpuKernel::GenMetaData(SectionStreamKResult &splitRes | |||
| 280 | 282 | ||
| 281 | for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) { | 283 | for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) { |
| 282 | for (uint32_t i = 0; i < AIC_CORE_NUM; ++i) { | 284 | for (uint32_t i = 0; i < AIC_CORE_NUM; ++i) { |
| 283 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX, 0U); | 285 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX, 0U); |
| 284 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, 0U); | 286 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, 0U); |
| 285 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, 0U); | 287 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, 0U); |
| 286 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX, 0U); | 288 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX, 0U); |
| 287 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, 0U); | 289 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, 0U); |
| 288 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, 0U); | 290 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, 0U); |
| 289 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, 0U); | 291 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, 0U); |
| 290 | } | 292 | } |
| 291 | for (uint32_t i = 0; i < AIV_CORE_NUM; ++i) { | 293 | for (uint32_t i = 0; i < AIV_CORE_NUM; ++i) { |
| 292 | - faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX, 0U); | 294 | + faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX, 0U); |
| 293 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, 0U); | 295 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, 0U); |
| 294 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX, 0U); | 296 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX, 0U); |
| 295 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, 0U); | 297 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, 0U); |
| @@ -300,20 +302,20 @@ bool QuantFlashAttnMetadataCpuKernel::GenMetaData(SectionStreamKResult &splitRes | |||
| 300 | auto faSplitRes = splitRes.sectionFaResult[sectionId]; | 302 | auto faSplitRes = splitRes.sectionFaResult[sectionId]; |
| 301 | for (uint32_t i = 0; i < faSplitRes.usedCoreNum; ++i) { | 303 | for (uint32_t i = 0; i < faSplitRes.usedCoreNum; ++i) { |
| 302 | if (i > 0) { | 304 | if (i > 0) { |
| 303 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX, faSplitRes.bN2End[i - 1]); | 305 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX, faSplitRes.bNEnd[i - 1]); |
| 304 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, faSplitRes.gS1End[i - 1]); | 306 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, faSplitRes.mEnd[i - 1]); |
| 305 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, faSplitRes.s2End[i - 1]); | 307 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, faSplitRes.s2End[i - 1]); |
| 306 | } else if (sectionId > 0) { | 308 | } else if (sectionId > 0) { |
| 307 | auto preFaSplitRes = splitRes.sectionFaResult[sectionId - 1]; | 309 | auto preFaSplitRes = splitRes.sectionFaResult[sectionId - 1]; |
| 308 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX, | 310 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX, |
| 309 | - preFaSplitRes.bN2End[preFaSplitRes.usedCoreNum - 1]); | 311 | + preFaSplitRes.bNEnd[preFaSplitRes.usedCoreNum - 1]); |
| 310 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, | 312 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, |
| 311 | - preFaSplitRes.gS1End[preFaSplitRes.usedCoreNum - 1]); | 313 | + preFaSplitRes.mEnd[preFaSplitRes.usedCoreNum - 1]); |
| 312 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, | 314 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, |
| 313 | preFaSplitRes.s2End[preFaSplitRes.usedCoreNum - 1]); | 315 | preFaSplitRes.s2End[preFaSplitRes.usedCoreNum - 1]); |
| 314 | } | 316 | } |
| 315 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX, faSplitRes.bN2End[i]); | 317 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX, faSplitRes.bNEnd[i]); |
| 316 | - faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, faSplitRes.gS1End[i]); | 318 | + faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, faSplitRes.mEnd[i]); |
| 317 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, faSplitRes.s2End[i]); | 319 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, faSplitRes.s2End[i]); |
| 318 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, | 320 | faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, |
| 319 | faSplitRes.firstFdDataWorkspaceIdx[i]); | 321 | faSplitRes.firstFdDataWorkspaceIdx[i]); |
| @@ -322,8 +324,8 @@ bool QuantFlashAttnMetadataCpuKernel::GenMetaData(SectionStreamKResult &splitRes | |||
| 322 | auto fdSplitRes = splitRes.sectionFdResult[sectionId]; | 324 | auto fdSplitRes = splitRes.sectionFdResult[sectionId]; |
| 323 | for (uint32_t i = 0; i < fdSplitRes.usedVecNum; ++i) { | 325 | for (uint32_t i = 0; i < fdSplitRes.usedVecNum; ++i) { |
| 324 | uint32_t curTaskIdx = fdSplitRes.taskIdx[i]; | 326 | uint32_t curTaskIdx = fdSplitRes.taskIdx[i]; |
| 325 | - faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX, fdSplitRes.bN2Idx[curTaskIdx]); | 327 | + faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX, fdSplitRes.bNIdx[curTaskIdx]); |
| 326 | - faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, fdSplitRes.gS1Idx[curTaskIdx]); | 328 | + faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, fdSplitRes.mIdx[curTaskIdx]); |
| 327 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX, | 329 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX, |
| 328 | fdSplitRes.workspaceIdx[curTaskIdx]); | 330 | fdSplitRes.workspaceIdx[curTaskIdx]); |
| 329 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, fdSplitRes.s2SplitNum[curTaskIdx]); | 331 | faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, fdSplitRes.s2SplitNum[curTaskIdx]); |
| @@ -33,19 +33,18 @@ constexpr uint32_t AIV_RESERVED_METADATA_STRIDE = 16U; | |||
| 33 | constexpr uint64_t GetMetadataRequiredElems(uint32_t sectionNum) | 33 | constexpr uint64_t GetMetadataRequiredElems(uint32_t sectionNum) |
| 34 | { | 34 | { |
| 35 | return static_cast<uint64_t>(HEAD_METADATA_STRIDE) + | 35 | return static_cast<uint64_t>(HEAD_METADATA_STRIDE) + |
| 36 | - static_cast<uint64_t>(sectionNum) * | 36 | + static_cast<uint64_t>(sectionNum) * (static_cast<uint64_t>(AIC_CORE_NUM) * FA_METADATA_STRIDE + |
| 37 | - (static_cast<uint64_t>(AIC_CORE_NUM) * FA_METADATA_STRIDE + | 37 | + static_cast<uint64_t>(AIV_CORE_NUM) * AIV_RESERVED_METADATA_STRIDE); |
| 38 | - static_cast<uint64_t>(AIV_CORE_NUM) * AIV_RESERVED_METADATA_STRIDE); | ||
| 39 | } | 38 | } |
| 40 | 39 | ||
| 41 | // Head Metadata Index Definitions | 40 | // Head Metadata Index Definitions |
| 42 | constexpr uint32_t HEAD_SECTION_NUM_INDEX = 0U; | 41 | constexpr uint32_t HEAD_SECTION_NUM_INDEX = 0U; |
| 43 | 42 | ||
| 44 | // Section-based FA Metadata Index (0-based, matching AICPU flash_attn_metadata format) | 43 | // Section-based FA Metadata Index (0-based, matching AICPU flash_attn_metadata format) |
| 45 | -constexpr uint32_t SLI_SEC_BN2_START_INDEX = 0U; | 44 | +constexpr uint32_t SLI_SEC_BN_START_INDEX = 0U; |
| 46 | constexpr uint32_t SLI_SEC_M_START_INDEX = 1U; | 45 | constexpr uint32_t SLI_SEC_M_START_INDEX = 1U; |
| 47 | constexpr uint32_t SLI_SEC_S2_START_INDEX = 2U; | 46 | constexpr uint32_t SLI_SEC_S2_START_INDEX = 2U; |
| 48 | -constexpr uint32_t SLI_SEC_BN2_END_INDEX = 3U; | 47 | +constexpr uint32_t SLI_SEC_BN_END_INDEX = 3U; |
| 49 | constexpr uint32_t SLI_SEC_M_END_INDEX = 4U; | 48 | constexpr uint32_t SLI_SEC_M_END_INDEX = 4U; |
| 50 | constexpr uint32_t SLI_SEC_S2_END_INDEX = 5U; | 49 | constexpr uint32_t SLI_SEC_S2_END_INDEX = 5U; |
| 51 | 50 | ||
| @@ -54,7 +53,7 @@ struct SliMetadata { | |||
| 54 | uint32_t sectionNum; | 53 | uint32_t sectionNum; |
| 55 | SLI_METADATA_T *metadata; | 54 | SLI_METADATA_T *metadata; |
| 56 | SLI_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE]; | 55 | SLI_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE]; |
| 57 | - SLI_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE]; | 56 | + SLI_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE]; |
| 58 | SliMetadata(void *metadataPtr, uint32_t sectionNum) | 57 | SliMetadata(void *metadataPtr, uint32_t sectionNum) |
| 59 | : sectionNum(sectionNum), | 58 | : sectionNum(sectionNum), |
| 60 | metadata(static_cast<SLI_METADATA_T *>(metadataPtr)), | 59 | metadata(static_cast<SLI_METADATA_T *>(metadataPtr)), |
| @@ -162,7 +162,8 @@ bool StemIndexerMetadataCpuKernel::GenerateBaseInfo(StemIndexerBaseInfo &baseInf | |||
| 162 | baseInfo.querySeqSize = 0; | 162 | baseInfo.querySeqSize = 0; |
| 163 | baseInfo.kvHeadNum = numHeadsKv_; | 163 | baseInfo.kvHeadNum = numHeadsKv_; |
| 164 | baseInfo.kvSeqSize = 0; | 164 | baseInfo.kvSeqSize = 0; |
| 165 | - baseInfo.headDim = headDim_; | 165 | + baseInfo.headDimQk = headDim_; |
| 166 | + baseInfo.headDimV = headDim_; | ||
| 166 | baseInfo.attenMaskFlag = causal_; | 167 | baseInfo.attenMaskFlag = causal_; |
| 167 | baseInfo.sparseMode = (causal_) ? static_cast<uint32_t>(load_balance::SparseMode::RIGHT_DOWN_CAUSAL) : | 168 | baseInfo.sparseMode = (causal_) ? static_cast<uint32_t>(load_balance::SparseMode::RIGHT_DOWN_CAUSAL) : |
| 168 | static_cast<uint32_t>(load_balance::SparseMode::BUTT); | 169 | static_cast<uint32_t>(load_balance::SparseMode::BUTT); |
| @@ -192,6 +193,7 @@ bool StemIndexerMetadataCpuKernel::GenerateSectionStreamKParam(load_balance::Sec | |||
| 192 | param.mBaseSize = STEM_M_BASE_SIZE; | 193 | param.mBaseSize = STEM_M_BASE_SIZE; |
| 193 | param.s2BaseSize = STEM_S2_BASE_SIZE; | 194 | param.s2BaseSize = STEM_S2_BASE_SIZE; |
| 194 | param.fdOn = false; | 195 | param.fdOn = false; |
| 196 | + param.outputLayout = load_balance::OutputLayout::BN2_S1G; | ||
| 195 | return true; | 197 | return true; |
| 196 | } | 198 | } |
| 197 | 199 | ||
| @@ -219,15 +221,15 @@ bool StemIndexerMetadataCpuKernel::GenMetadata(SectionStreamKResult &result) | |||
| 219 | for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) { | 221 | for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) { |
| 220 | auto &prevFaRes = (secIdx == 0U) ? dummyHead : result.sectionFaResult[secIdx - 1U]; | 222 | auto &prevFaRes = (secIdx == 0U) ? dummyHead : result.sectionFaResult[secIdx - 1U]; |
| 221 | auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U; | 223 | auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U; |
| 222 | - SLI_METADATA_T bn2Start = (aicIdx == 0) ? prevFaRes.bN2End[prevLastCore] : faRes.bN2End[aicIdx - 1U]; | 224 | + SLI_METADATA_T bn2Start = (aicIdx == 0) ? prevFaRes.bNEnd[prevLastCore] : faRes.bNEnd[aicIdx - 1U]; |
| 223 | - SLI_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.gS1End[prevLastCore] : faRes.gS1End[aicIdx - 1U]; | 225 | + SLI_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.mEnd[prevLastCore] : faRes.mEnd[aicIdx - 1U]; |
| 224 | SLI_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U]; | 226 | SLI_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U]; |
| 225 | 227 | ||
| 226 | - sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN2_START_INDEX, bn2Start); | 228 | + sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN_START_INDEX, bn2Start); |
| 227 | sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_START_INDEX, mStart); | 229 | sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_START_INDEX, mStart); |
| 228 | sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_START_INDEX, s2Start); | 230 | sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_START_INDEX, s2Start); |
| 229 | - sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN2_END_INDEX, faRes.bN2End[aicIdx]); | 231 | + sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN_END_INDEX, faRes.bNEnd[aicIdx]); |
| 230 | - sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_END_INDEX, faRes.gS1End[aicIdx]); | 232 | + sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_END_INDEX, faRes.mEnd[aicIdx]); |
| 231 | sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_END_INDEX, faRes.s2End[aicIdx]); | 233 | sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_END_INDEX, faRes.s2End[aicIdx]); |
| 232 | } | 234 | } |
| 233 | } | 235 | } |
空评论,避免合入