Merged
[SectionStreamK] 支持S1G不合轴和KV不同Dim #10563
[SectionStreamK] 支持S1G不合轴和KV不同Dim #10563
Merged
FiguraDogeAdd on Aug 25
Total: 13 files changed+452-380
@@ -36,7 +36,8 @@ public:
36 [[nodiscard]] virtual uint32_t GetGroupSize() const = 0;36 [[nodiscard]] virtual uint32_t GetGroupSize() const = 0;
37 [[nodiscard]] virtual uint32_t GetQueryHeadNum() const = 0;37 [[nodiscard]] virtual uint32_t GetQueryHeadNum() const = 0;
38 [[nodiscard]] virtual uint32_t GetKvHeadNum() const = 0;38 [[nodiscard]] virtual uint32_t GetKvHeadNum() const = 0;
39- [[nodiscard]] virtual uint32_t GetHeadDim() const = 0;39+ [[nodiscard]] virtual uint32_t GetHeadDimQk() const = 0;
40+ [[nodiscard]] virtual uint32_t GetHeadDimV() const = 0;
40 [[nodiscard]] virtual uint32_t GetQuerySeqSize() const = 0;41 [[nodiscard]] virtual uint32_t GetQuerySeqSize() const = 0;
41 [[nodiscard]] virtual uint32_t GetQuerySeqSize(uint32_t batchIdx) const = 0;42 [[nodiscard]] virtual uint32_t GetQuerySeqSize(uint32_t batchIdx) const = 0;
F

空评论,避免合入

likedislike
42 [[nodiscard]] virtual uint32_t GetKvSeqSize() const = 0;43 [[nodiscard]] virtual uint32_t GetKvSeqSize() const = 0;
@@ -79,9 +80,14 @@ public:
79 return kvHeadNum;80 return kvHeadNum;
80 }81 }
81 82 
82- [[nodiscard]] uint32_t GetHeadDim() const override83+ [[nodiscard]] uint32_t GetHeadDimQk() const override
83 {84 {
84- return headDim;85+ return headDimQk;
86+ }
87+ 
88+ [[nodiscard]] uint32_t GetHeadDimV() const override
89+ {
90+ return headDimV;
85 }91 }
86 92 
87 [[nodiscard]] uint32_t GetQuerySeqSize() const override93 [[nodiscard]] uint32_t GetQuerySeqSize() const override
@@ -202,7 +208,8 @@ public:
202 uint32_t querySeqSize{0U};208 uint32_t querySeqSize{0U};
203 uint32_t kvHeadNum{0U};209 uint32_t kvHeadNum{0U};
204 uint32_t kvSeqSize{0U};210 uint32_t kvSeqSize{0U};
205- uint32_t headDim{64U};211+ uint32_t headDimQk{64U};
212+ uint32_t headDimV{64U};
206 bool attenMaskFlag{false};213 bool attenMaskFlag{false};
207 uint32_t sparseMode{0U};214 uint32_t sparseMode{0U};
208 uint32_t preToken{0U};215 uint32_t preToken{0U};
@@ -38,6 +38,12 @@ enum class SocVersion : uint32_t {
38 BUTT38 BUTT
39};39};
40 40 
41+enum class OutputLayout : uint8_t {
42+ BN2_S1G = 0,
43+ BN1_S1,
44+ BUTT
45+};
46+ 
41struct DeviceInfo {47struct DeviceInfo {
42 uint32_t aicCoreMaxNum{0U}; // At most amount of aic core would be turned on, this is not final used number48 uint32_t aicCoreMaxNum{0U}; // At most amount of aic core would be turned on, this is not final used number
43 uint32_t aicCoreMinNum{0U}; // At least amount of aic core would be turned on, this is not final used number49 uint32_t aicCoreMinNum{0U}; // At least amount of aic core would be turned on, this is not final used number
@@ -55,6 +61,7 @@ struct GeneralBalanceParam {
55 // full_block_cost is costFunc(mBaseSize, s2BaseSize)61 // full_block_cost is costFunc(mBaseSize, s2BaseSize)
56 int64_t fdLeastBlock{0U}; // if noFd.maxCost <= fdLeastBlock * full_block_cost, then choose no fd62 int64_t fdLeastBlock{0U}; // if noFd.maxCost <= fdLeastBlock * full_block_cost, then choose no fd
57 CostFunc costFunc{nullptr}; // Customize cost func. Set nullptr to use default cost func63 CostFunc costFunc{nullptr}; // Customize cost func. Set nullptr to use default cost func
64+ OutputLayout outputLayout{OutputLayout::BN2_S1G};
58};65};
59 66 
60enum class SparseMode : uint8_t {67enum class SparseMode : uint8_t {
@@ -48,16 +48,16 @@ inline uint32_t SectionStreamK::Compute(const DeviceInfo &deviceInfo, const IBas
48 for (size_t i = 0; i < result.sectionNum; ++i) {48 for (size_t i = 0; i < result.sectionNum; ++i) {
49 SectionStreamKFaResult tmpFa(deviceInfo.aicCoreMaxNum);49 SectionStreamKFaResult tmpFa(deviceInfo.aicCoreMaxNum);
50 tmpFa.usedCoreNum = implResult[i].usedCoreNum;50 tmpFa.usedCoreNum = implResult[i].usedCoreNum;
51- tmpFa.bN2End = implResult[i].bN2End;51+ tmpFa.bNEnd = implResult[i].bNEnd;
52- tmpFa.gS1End = implResult[i].gS1End;52+ tmpFa.mEnd = implResult[i].mEnd;
53 tmpFa.s2End = implResult[i].s2End;53 tmpFa.s2End = implResult[i].s2End;
54 tmpFa.firstFdDataWorkspaceIdx = implResult[i].firstFdDataWorkspaceIdx;54 tmpFa.firstFdDataWorkspaceIdx = implResult[i].firstFdDataWorkspaceIdx;
55 result.sectionFaResult.emplace_back(tmpFa);55 result.sectionFaResult.emplace_back(tmpFa);
56 56 
57 SectionStreamKFdResult tmpFd(deviceInfo.aicCoreMaxNum, deviceInfo.aivCoreMaxNum);57 SectionStreamKFdResult tmpFd(deviceInfo.aicCoreMaxNum, deviceInfo.aivCoreMaxNum);
58 tmpFd.usedVecNum = implResult[i].usedVecNum;58 tmpFd.usedVecNum = implResult[i].usedVecNum;
59- tmpFd.bN2Idx = implResult[i].bN2Idx;59+ tmpFd.bNIdx = implResult[i].bNIdx;
60- tmpFd.gS1Idx = implResult[i].mIdx;60+ tmpFd.mIdx = implResult[i].mIdx;
61 tmpFd.workspaceIdx = implResult[i].workspaceIdx;61 tmpFd.workspaceIdx = implResult[i].workspaceIdx;
62 tmpFd.s2SplitNum = implResult[i].s2SplitNum;62 tmpFd.s2SplitNum = implResult[i].s2SplitNum;
63 tmpFd.taskIdx = implResult[i].taskIdx;63 tmpFd.taskIdx = implResult[i].taskIdx;
@@ -35,22 +35,24 @@ struct SectionStreamKParam : GeneralBalanceParam {
35 35 
36struct SectionStreamKFaResult {36struct SectionStreamKFaResult {
37 uint32_t usedCoreNum{0U}; // FA中使用的AIC数量37 uint32_t usedCoreNum{0U}; // FA中使用的AIC数量
38- std::vector<uint32_t> bN2End{}; // 每个核处理数据的BN2结束点38+ std::vector<uint32_t> bNEnd{}; // 每个核处理数据的BN结束点
39- std::vector<uint32_t> gS1End{}; // 每个核处理数据的GS1结束点39+ std::vector<uint32_t> mEnd{}; // 每个核处理数据的M结束点
40 std::vector<uint32_t> s2End{}; // 每个核处理数据的S2结束点40 std::vector<uint32_t> s2End{}; // 每个核处理数据的S2结束点
41 std::vector<uint32_t> firstFdDataWorkspaceIdx{}; // FD块在workspace中的位置41 std::vector<uint32_t> firstFdDataWorkspaceIdx{}; // FD块在workspace中的位置
42 42 
43 explicit SectionStreamKFaResult(uint32_t aicNum)43 explicit SectionStreamKFaResult(uint32_t aicNum)
44- : bN2End(aicNum), gS1End(aicNum), s2End(aicNum), firstFdDataWorkspaceIdx(aicNum)44+ : bNEnd(aicNum),
45- {45+ mEnd(aicNum),
46- }46+ s2End(aicNum),
47+ firstFdDataWorkspaceIdx(aicNum)
48+ {}
47};49};
48 50 
49struct SectionStreamKFdResult {51struct SectionStreamKFdResult {
50 uint32_t usedVecNum{0U}; // 归约过程中使用的AIV数量52 uint32_t usedVecNum{0U}; // 归约过程中使用的AIV数量
51 // 1、归约任务的索引信息53 // 1、归约任务的索引信息
52- std::vector<uint32_t> bN2Idx{}; // 每个归约任务的BN2索引,脚标为归约任务的序号,最大为核数-154+ std::vector<uint32_t> bNIdx{}; // 每个归约任务的BN索引,脚标为归约任务的序号,最大为核数-1
53- std::vector<uint32_t> gS1Idx{}; // 每个归约任务的GS1索引,脚标为归约任务的序号55+ std::vector<uint32_t> mIdx{}; // 每个归约任务的m索引,脚标为归约任务的序号
54 std::vector<uint32_t> workspaceIdx{}; // 每个归约任务在workspace中的存放位置56 std::vector<uint32_t> workspaceIdx{}; // 每个归约任务在workspace中的存放位置
55 std::vector<uint32_t> s2SplitNum{}; // 每个归约任务的S2核间切分份数,脚标为归约任务的序号57 std::vector<uint32_t> s2SplitNum{}; // 每个归约任务的S2核间切分份数,脚标为归约任务的序号
56 // 2、FD kernel阶段,归约任务的分核信息58 // 2、FD kernel阶段,归约任务的分核信息
@@ -59,10 +61,14 @@ struct SectionStreamKFdResult {
59 std::vector<uint32_t> mLen{}; // 每个AIV处理的归约任务的M轴行数61 std::vector<uint32_t> mLen{}; // 每个AIV处理的归约任务的M轴行数
60 62 
61 explicit SectionStreamKFdResult(uint32_t aicNum, uint32_t aivNum)63 explicit SectionStreamKFdResult(uint32_t aicNum, uint32_t aivNum)
62- : bN2Idx(aicNum), gS1Idx(aicNum), workspaceIdx(aicNum), s2SplitNum(aicNum), taskIdx(aivNum), mStart(aivNum),64+ : bNIdx(aicNum),
65+ mIdx(aicNum),
66+ workspaceIdx(aicNum),
67+ s2SplitNum(aicNum),
68+ taskIdx(aivNum),
69+ mStart(aivNum),
63 mLen(aivNum)70 mLen(aivNum)
64- {71+ {}
65- }
66};72};
67 73 
68struct SectionStreamKResult {74struct SectionStreamKResult {
@@ -22,14 +22,15 @@
22using namespace std;22using namespace std;
23using namespace optiling;23using namespace optiling;
24 24 
25-#define CHECK_RET(cond) ((cond) ? true :(false))25+#define CHECK_RET(cond) ((cond) ? true : (false))
26 26 
27-#define LOG_PRINT(message, ...) \27+#define LOG_PRINT(message, ...) \
28- do { \28+ do { \
29- printf(message, ##__VA_ARGS__); \29+ printf(message, ##__VA_ARGS__); \
30 } while (0)30 } while (0)
31 31 
32-int64_t GetShapeSize(const std::vector<int64_t> &shape) {32+int64_t GetShapeSize(const std::vector<int64_t> &shape)
33+{
33 int64_t shapeSize = 1;34 int64_t shapeSize = 1;
34 for (auto i : shape) {35 for (auto i : shape) {
35 shapeSize *= i;36 shapeSize *= i;
@@ -37,7 +38,8 @@ int64_t GetShapeSize(const std::vector<int64_t> &shape) {
37 return shapeSize;38 return shapeSize;
38}39}
39 40 
40-int Init(int32_t deviceId, aclrtStream *stream) {41+int Init(int32_t deviceId, aclrtStream *stream)
42+{
41 // Fixed writing method, AscendCL initialization.43 // Fixed writing method, AscendCL initialization.
42 auto ret = aclInit(nullptr);44 auto ret = aclInit(nullptr);
43 if (!CHECK_RET(ret == ACL_SUCCESS)) {45 if (!CHECK_RET(ret == ACL_SUCCESS)) {
@@ -57,8 +59,9 @@ int Init(int32_t deviceId, aclrtStream *stream) {
57 return 0;59 return 0;
58}60}
59 61 
60-static void DumpMeta(void* data) {62+static void DumpMeta(void *data)
61- int32_t sectionNum = static_cast<int32_t>(((int32_t*)data)[0]);63+{
64+ int32_t sectionNum = static_cast<int32_t>(((int32_t *)data)[0]);
62 optiling::detail::FaMetadata faMetadata(data, sectionNum);65 optiling::detail::FaMetadata faMetadata(data, sectionNum);
63 printf("sectionNum:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_SECTION_NUM_INDEX));66 printf("sectionNum:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_SECTION_NUM_INDEX));
64 printf("isFd:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_IS_FD_INDEX));67 printf("isFd:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_IS_FD_INDEX));
@@ -66,24 +69,24 @@ static void DumpMeta(void* data) {
66 printf("s2BaseSize:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_S2_BASE_SIZE_INDEX));69 printf("s2BaseSize:%d\n", faMetadata.GetHeadMetadata(optiling::HEAD_S2_BASE_SIZE_INDEX));
67 for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) {70 for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) {
68 // FA Metadata Generate71 // FA Metadata Generate
69- printf("sectionIdx:%d\n",sectionId);72+ printf("sectionIdx:%d\n", sectionId);
70 for (size_t i = 0; i < AIC_CORE_NUM; ++i) {73 for (size_t i = 0; i < AIC_CORE_NUM; ++i) {
71 // FA start74 // FA start
72- printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX));75+ printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX));
73 printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX));76 printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX));
74 printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX));77 printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX));
75 // FA end78 // FA end
76- printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX));79+ printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX));
77 printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX));80 printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX));
78 printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX));81 printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX));
79 // FA idx82 // FA idx
80- printf("first fd data ws idx: %d\n",faMetadata.GetFaMetadata(sectionId, i,83+ printf("first fd data ws idx: %d\n",
81- optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX));84+ faMetadata.GetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX));
82 }85 }
83 86 
84 // FD Metadata Generate87 // FD Metadata Generate
85 for (size_t i = 0; i < AIV_CORE_NUM; ++i) {88 for (size_t i = 0; i < AIV_CORE_NUM; ++i) {
86- printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX));89+ printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX));
87 printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX));90 printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX));
88 printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX));91 printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX));
89 printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX));92 printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX));
@@ -95,7 +98,8 @@ static void DumpMeta(void* data) {
95 98 
96template <typename T>99template <typename T>
97int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr,100int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr,
98- aclDataType dataType, aclTensor **tensor) {101+ aclDataType dataType, aclTensor **tensor)
102+{
99 auto size = GetShapeSize(shape) * sizeof(T);103 auto size = GetShapeSize(shape) * sizeof(T);
100 // Call aclrtMalloc to request device side memory.104 // Call aclrtMalloc to request device side memory.
101 auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);105 auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);
@@ -122,7 +126,8 @@ int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &
122 return 0;126 return 0;
123}127}
124 128 
125-int main() {129+int main()
130+{
126 // 1. (Fixed writing method) device/stream initialization. Refer to AscendCL's list of external interfaces.131 // 1. (Fixed writing method) device/stream initialization. Refer to AscendCL's list of external interfaces.
127 // Fill in the deviceId based on your actual device.132 // Fill in the deviceId based on your actual device.
128 int32_t deviceId = 0;133 int32_t deviceId = 0;
@@ -145,10 +150,10 @@ int main() {
145 int32_t preTokens = -1;150 int32_t preTokens = -1;
146 int32_t nextTokens = -1;151 int32_t nextTokens = -1;
147 string layout_query = "BSND";152 string layout_query = "BSND";
148- char layoutQuery[256]; // 256: default length153+ char layoutQuery[256]; // 256: default length
149 strcpy_s(layoutQuery, sizeof(layoutQuery), layout_query.c_str());154 strcpy_s(layoutQuery, sizeof(layoutQuery), layout_query.c_str());
150 string layout_kv = "BSND";155 string layout_kv = "BSND";
151- char layoutKv[256]; // 256: default length156+ char layoutKv[256]; // 256: default length
152 strcpy_s(layoutKv, sizeof(layoutKv), layout_kv.c_str());157 strcpy_s(layoutKv, sizeof(layoutKv), layout_kv.c_str());
153 int32_t sparseMode = 3;158 int32_t sparseMode = 3;
154 int32_t attentionMode = 0;159 int32_t attentionMode = 0;
@@ -163,9 +168,9 @@ int main() {
163 void *actualSeqLengthsQueryDeviceAddr = nullptr;168 void *actualSeqLengthsQueryDeviceAddr = nullptr;
164 void *actualSeqLengthsKvDeviceAddr = nullptr;169 void *actualSeqLengthsKvDeviceAddr = nullptr;
165 void *metadataDeviceAddr = nullptr;170 void *metadataDeviceAddr = nullptr;
166- aclTensor* actualSeqLengthsQueryTensor = nullptr;171+ aclTensor *actualSeqLengthsQueryTensor = nullptr;
167- aclTensor* actualSeqLengthsKvTensor = nullptr;172+ aclTensor *actualSeqLengthsKvTensor = nullptr;
168- aclTensor* metadataTensor = nullptr;173+ aclTensor *metadataTensor = nullptr;
169 int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape);174 int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape);
170 int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape);175 int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape);
171 int64_t metadataShapeSize = GetShapeSize(metadataShape);176 int64_t metadataShapeSize = GetShapeSize(metadataShape);
@@ -175,40 +180,34 @@ int main() {
175 180 
176 // Create actual_seq_lengths_query aclTensor.181 // Create actual_seq_lengths_query aclTensor.
177 ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr,182 ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr,
178- aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor);183+ aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor);
179 if (!CHECK_RET(ret == ACL_SUCCESS)) {184 if (!CHECK_RET(ret == ACL_SUCCESS)) {
180 return ret;185 return ret;
181 }186 }
182 // Create actual_seq_lengths_kv aclTensor.187 // Create actual_seq_lengths_kv aclTensor.
183 ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr,188 ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr,
184- aclDataType::ACL_INT32, &actualSeqLengthsKvTensor);189+ aclDataType::ACL_INT32, &actualSeqLengthsKvTensor);
185 if (!CHECK_RET(ret == ACL_SUCCESS)) {190 if (!CHECK_RET(ret == ACL_SUCCESS)) {
186 return ret;191 return ret;
187 }192 }
188 // Create metadata aclTensor.193 // Create metadata aclTensor.
189- ret = CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor);194+ ret =
195+ CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor);
190 if (!CHECK_RET(ret == ACL_SUCCESS)) {196 if (!CHECK_RET(ret == ACL_SUCCESS)) {
191 return ret;197 return ret;
192 }198 }
193 199 
194- aclOpExecutor* executor = nullptr;200+ aclOpExecutor *executor = nullptr;
195 uint64_t workspaceSize = 0;201 uint64_t workspaceSize = 0;
196- void* workspaceAddr = nullptr;202+ void *workspaceAddr = nullptr;
197 203 
198 char socVersion[] = "ascend950";204 char socVersion[] = "ascend950";
199 printf("start aclnnFlashAttnMetadata\n");205 printf("start aclnnFlashAttnMetadata\n");
200- ret = aclnnFlashAttnMetadataGetWorkspaceSize(206+ ret = aclnnFlashAttnMetadataGetWorkspaceSize(nullptr, nullptr, nullptr, nullptr, batchSize, qS, kvS, numHeads,
201- nullptr, nullptr,207+ numKeyValueHeads, headDim, sparseMode, preTokens, nextTokens, "BSND",
202- nullptr, nullptr,208+ "BSND", "BSND", metadataTensor, &workspaceSize, &executor);
203- batchSize, qS, kvS,
204- numHeads, numKeyValueHeads,
205- headDim, sparseMode, preTokens, nextTokens,
206- "BSND", "BSND", "BSND",
207- metadataTensor,
208- &workspaceSize, &executor);
209 if (ret != ACL_SUCCESS) {209 if (ret != ACL_SUCCESS) {
210- printf("aclnnFlashAttnMetadataGetWorkspaceSize %d\n",210+ printf("aclnnFlashAttnMetadataGetWorkspaceSize %d\n", ret);
211- ret);
212 return -1;211 return -1;
213 }212 }
214 213 
@@ -224,10 +223,8 @@ int main() {
224 return -1;223 return -1;
225 }224 }
226 225 
227- ret = aclrtMemcpy(metadataHostData.data(),226+ ret = aclrtMemcpy(metadataHostData.data(), metadataHostData.size() * sizeof(metadataHostData[0]),
228- metadataHostData.size() * sizeof(metadataHostData[0]), metadataDeviceAddr,227+ metadataDeviceAddr, metadataShapeSize, ACL_MEMCPY_DEVICE_TO_HOST);
229- metadataShapeSize,
230- ACL_MEMCPY_DEVICE_TO_HOST);
231 if (ret != ACL_SUCCESS) {228 if (ret != ACL_SUCCESS) {
232 printf("aclrtMemcpy %d\n", ret);229 printf("aclrtMemcpy %d\n", ret);
233 return -1;230 return -1;
@@ -38,16 +38,16 @@ constexpr uint32_t HEAD_M_BASE_SIZE_INDEX = 2U;
38constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U;38constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U;
39 39 
40// FA Metadata Index Definitions40// FA Metadata Index Definitions
41-constexpr uint32_t FA_BN2_START_INDEX = 0U;41+constexpr uint32_t FA_BN_START_INDEX = 0U;
42constexpr uint32_t FA_M_START_INDEX = 1U;42constexpr uint32_t FA_M_START_INDEX = 1U;
43constexpr uint32_t FA_S2_START_INDEX = 2U;43constexpr uint32_t FA_S2_START_INDEX = 2U;
44-constexpr uint32_t FA_BN2_END_INDEX = 3U;44+constexpr uint32_t FA_BN_END_INDEX = 3U;
45constexpr uint32_t FA_M_END_INDEX = 4U;45constexpr uint32_t FA_M_END_INDEX = 4U;
46constexpr uint32_t FA_S2_END_INDEX = 5U;46constexpr uint32_t FA_S2_END_INDEX = 5U;
47constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U;47constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U;
48 48 
49// FD Metadata Index Definitions49// FD Metadata Index Definitions
50-constexpr uint32_t FD_BN2_IDX_INDEX = 0U;50+constexpr uint32_t FD_BN_IDX_INDEX = 0U;
51constexpr uint32_t FD_M_IDX_INDEX = 1U;51constexpr uint32_t FD_M_IDX_INDEX = 1U;
52constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U;52constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U;
53constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U;53constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U;
@@ -58,13 +58,13 @@ namespace detail {
58struct FaMetadata {58struct FaMetadata {
59 uint32_t sectionNum;59 uint32_t sectionNum;
60 FA_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE];60 FA_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE];
61- FA_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE];61+ FA_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE];
62- FA_METADATA_T *fdMetadata; // [sectionNum][AIV_CORE_NUM][FD_METADATA_STRIDE];62+ FA_METADATA_T *fdMetadata; // [sectionNum][AIV_CORE_NUM][FD_METADATA_STRIDE];
63 FaMetadata(void *metadataPtr, uint32_t sectionNum)63 FaMetadata(void *metadataPtr, uint32_t sectionNum)
64- :sectionNum(sectionNum),64+ : sectionNum(sectionNum),
65- headMetadata(static_cast<FA_METADATA_T*>(metadataPtr)),65+ headMetadata(static_cast<FA_METADATA_T *>(metadataPtr)),
66- faMetadata(headMetadata + HEAD_METADATA_STRIDE),66+ faMetadata(headMetadata + HEAD_METADATA_STRIDE),
67- fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_STRIDE)67+ fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_STRIDE)
68 {68 {
69 headMetadata[0] = sectionNum;69 headMetadata[0] = sectionNum;
70 }70 }
@@ -57,12 +57,10 @@ bool FlashAttnMetadataCpuKernel::Prepare(CpuKernelContext &ctx)
57 57 
58 KERNEL_CHECK_FALSE((metadata_ != nullptr && metadata_->GetData() != nullptr), false, "metadata is empty");58 KERNEL_CHECK_FALSE((metadata_ != nullptr && metadata_->GetData() != nullptr), false, "metadata is empty");
59 59 
60- bool requiredAttrs = GetAttrValue(ctx, "num_heads_q", numHeadsQ_) &&60+ bool requiredAttrs =
61- GetAttrValue(ctx, "num_heads_kv", numHeadsKv_) &&61+ GetAttrValue(ctx, "num_heads_q", numHeadsQ_) && GetAttrValue(ctx, "num_heads_kv", numHeadsKv_) &&
62- GetAttrValue(ctx, "head_dim", headDim_) &&62+ GetAttrValue(ctx, "head_dim", headDim_) && GetAttrValue(ctx, "soc_version", socVersion_) &&
63- GetAttrValue(ctx, "soc_version", socVersion_) &&63+ GetAttrValue(ctx, "aic_core_num", aicCoreNum_) && GetAttrValue(ctx, "aiv_core_num", aivCoreNum_);
64- GetAttrValue(ctx, "aic_core_num", aicCoreNum_) &&
65- GetAttrValue(ctx, "aiv_core_num", aivCoreNum_);
66 KERNEL_CHECK_FALSE(requiredAttrs, false, "Missing Required attrs missing!");64 KERNEL_CHECK_FALSE(requiredAttrs, false, "Missing Required attrs missing!");
67 65 
68 // attributes optional66 // attributes optional
@@ -122,8 +120,8 @@ bool FlashAttnMetadataCpuKernel::CheckActualQuerySeq()
122 for (size_t i = 1; i < cuSeqlensQ.size(); ++i) {120 for (size_t i = 1; i < cuSeqlensQ.size(); ++i) {
123 if (cuSeqlensQ[i] < cuSeqlensQ[i - 1]) {121 if (cuSeqlensQ[i] < cuSeqlensQ[i - 1]) {
124 KERNEL_LOG_ERROR(122 KERNEL_LOG_ERROR(
125- "The %zuth element of cuSeqlensQ must not be less than the %zuth element, but got %ld and %ld",123+ "The %zuth element of cuSeqlensQ must not be less than the %zuth element, but got %ld and %ld", i,
126- i, i - 1, cuSeqlensQ[i], cuSeqlensQ[i - 1]);124+ i - 1, cuSeqlensQ[i], cuSeqlensQ[i - 1]);
127 return false;125 return false;
128 }126 }
129 }127 }
@@ -151,8 +149,8 @@ bool FlashAttnMetadataCpuKernel::CheckActualKvSeq()
151 149 
152 for (size_t i = 0; i < sequsedKv.size(); ++i) {150 for (size_t i = 0; i < sequsedKv.size(); ++i) {
153 if (sequsedKv[i] < 0) {151 if (sequsedKv[i] < 0) {
154- KERNEL_LOG_ERROR("The elements of sequsedKv must be non-negative, but %zuth element is %ld",152+ KERNEL_LOG_ERROR("The elements of sequsedKv must be non-negative, but %zuth element is %ld", i,
155- i, sequsedKv[i]);153+ sequsedKv[i]);
156 return false;154 return false;
157 }155 }
158 }156 }
@@ -167,8 +165,8 @@ bool FlashAttnMetadataCpuKernel::CheckActualKvSeq()
167 for (size_t i = 1; i < cuSeqlensKv.size(); ++i) {165 for (size_t i = 1; i < cuSeqlensKv.size(); ++i) {
168 if (cuSeqlensKv[i] < cuSeqlensKv[i - 1]) {166 if (cuSeqlensKv[i] < cuSeqlensKv[i - 1]) {
169 KERNEL_LOG_ERROR(167 KERNEL_LOG_ERROR(
170- "The %zuth element of cuSeqlensKv must not be less than the %zuth element, but got %ld and %ld",168+ "The %zuth element of cuSeqlensKv must not be less than the %zuth element, but got %ld and %ld", i,
171- i, i - 1, cuSeqlensKv[i], cuSeqlensKv[i - 1]);169+ i - 1, cuSeqlensKv[i], cuSeqlensKv[i - 1]);
172 return false;170 return false;
173 }171 }
174 }172 }
@@ -200,10 +198,9 @@ void FlashAttnMetadataCpuKernel::InitLoadBalanceParams()
200 } else if (baseInfo.layoutQuery == load_balance::Layout::TND) {198 } else if (baseInfo.layoutQuery == load_balance::Layout::TND) {
201 qlayout = optiling::flash_attn::fa_tiling_util::LAYOUT_TND;199 qlayout = optiling::flash_attn::fa_tiling_util::LAYOUT_TND;
202 }200 }
203- optiling::flash_attn::fa_tiling_util::AdjustSinnerAndSouter(baseInfo.headDim, maxSeqlenQ_,201+ optiling::flash_attn::fa_tiling_util::AdjustSinnerAndSouter(baseInfo.headDimQk, maxSeqlenQ_, maxSeqlenKv_,
204- maxSeqlenKv_, baseInfo.sparseMode, baseInfo.preToken,202+ baseInfo.sparseMode, baseInfo.preToken,
205- baseInfo.nextToken, qlayout,203+ baseInfo.nextToken, qlayout, mBaseSize_, s2BaseSize_);
206- mBaseSize_, s2BaseSize_);
207 mBaseSize_ *= (aivCoreNum_ / aicCoreNum_);204 mBaseSize_ *= (aivCoreNum_ / aicCoreNum_);
208 param.mBaseSize = mBaseSize_;205 param.mBaseSize = mBaseSize_;
209 param.s2BaseSize = s2BaseSize_;206 param.s2BaseSize = s2BaseSize_;
@@ -211,6 +208,7 @@ void FlashAttnMetadataCpuKernel::InitLoadBalanceParams()
211 param.fdTolerance = 10; // 10: tolerance block208 param.fdTolerance = 10; // 10: tolerance block
212 param.fdLeastBlock = 3; // 3: least block209 param.fdLeastBlock = 3; // 3: least block
213 param.fdOn = true;210 param.fdOn = true;
211+ param.outputLayout = load_balance::OutputLayout::BN2_S1G;
214}212}
215 213 
216void FlashAttnMetadataCpuKernel::InitBaseInfo()214void FlashAttnMetadataCpuKernel::InitBaseInfo()
@@ -220,7 +218,8 @@ void FlashAttnMetadataCpuKernel::InitBaseInfo()
220 baseInfo.queryHeadNum = numHeadsQ_;218 baseInfo.queryHeadNum = numHeadsQ_;
221 baseInfo.kvSeqSize = maxSeqlenKv_;219 baseInfo.kvSeqSize = maxSeqlenKv_;
222 baseInfo.kvHeadNum = numHeadsKv_;220 baseInfo.kvHeadNum = numHeadsKv_;
223- baseInfo.headDim = headDim_;221+ baseInfo.headDimQk = headDim_;
222+ baseInfo.headDimV = headDim_;
224 load_balance::SparseMode maskMode = load_balance::SparseMode::BUTT;223 load_balance::SparseMode maskMode = load_balance::SparseMode::BUTT;
225 if (maskMode_ != 0) {224 if (maskMode_ != 0) {
226 maskMode = static_cast<load_balance::SparseMode>(maskMode_);225 maskMode = static_cast<load_balance::SparseMode>(maskMode_);
@@ -263,15 +262,15 @@ bool FlashAttnMetadataCpuKernel::GenMetadata(load_balance::SectionStreamKResult
263 for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) {262 for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) {
264 auto &prevFaRes = (secIdx == 0U) ? dummyHead : splitRes.sectionFaResult[secIdx - 1U];263 auto &prevFaRes = (secIdx == 0U) ? dummyHead : splitRes.sectionFaResult[secIdx - 1U];
265 auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U;264 auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U;
266- FA_METADATA_T bn2Start = (aicIdx == 0) ? prevFaRes.bN2End[prevLastCore] : faRes.bN2End[aicIdx - 1U];265+ FA_METADATA_T bnStart = (aicIdx == 0) ? prevFaRes.bNEnd[prevLastCore] : faRes.bNEnd[aicIdx - 1U];
267- FA_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.gS1End[prevLastCore] : faRes.gS1End[aicIdx - 1U];266+ FA_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.mEnd[prevLastCore] : faRes.mEnd[aicIdx - 1U];
268 FA_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U];267 FA_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U];
269 268 
270- faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN2_START_INDEX, bn2Start);269+ faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN_START_INDEX, bnStart);
271 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_START_INDEX, mStart);270 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_START_INDEX, mStart);
272 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_START_INDEX, s2Start);271 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_START_INDEX, s2Start);
273- faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN2_END_INDEX, faRes.bN2End[aicIdx]);272+ faMetadata.SetFaMetadata(secIdx, aicIdx, FA_BN_END_INDEX, faRes.bNEnd[aicIdx]);
274- faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_END_INDEX, faRes.gS1End[aicIdx]);273+ faMetadata.SetFaMetadata(secIdx, aicIdx, FA_M_END_INDEX, faRes.mEnd[aicIdx]);
275 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_END_INDEX, faRes.s2End[aicIdx]);274 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_S2_END_INDEX, faRes.s2End[aicIdx]);
276 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX,275 faMetadata.SetFaMetadata(secIdx, aicIdx, FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX,
277 faRes.firstFdDataWorkspaceIdx[aicIdx]);276 faRes.firstFdDataWorkspaceIdx[aicIdx]);
@@ -280,8 +279,8 @@ bool FlashAttnMetadataCpuKernel::GenMetadata(load_balance::SectionStreamKResult
280 auto &fdRes = splitRes.sectionFdResult[secIdx];279 auto &fdRes = splitRes.sectionFdResult[secIdx];
281 for (uint32_t aivIdx = 0; aivIdx < fdRes.usedVecNum; ++aivIdx) {280 for (uint32_t aivIdx = 0; aivIdx < fdRes.usedVecNum; ++aivIdx) {
282 uint32_t t = fdRes.taskIdx[aivIdx];281 uint32_t t = fdRes.taskIdx[aivIdx];
283- faMetadata.SetFdMetadata(secIdx, aivIdx, FD_BN2_IDX_INDEX, fdRes.bN2Idx[t]);282+ faMetadata.SetFdMetadata(secIdx, aivIdx, FD_BN_IDX_INDEX, fdRes.bNIdx[t]);
284- faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_IDX_INDEX, fdRes.gS1Idx[t]);283+ faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_IDX_INDEX, fdRes.mIdx[t]);
285 faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_IDX_INDEX, fdRes.workspaceIdx[t]);284 faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_IDX_INDEX, fdRes.workspaceIdx[t]);
286 faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_NUM_INDEX, fdRes.s2SplitNum[t]);285 faMetadata.SetFdMetadata(secIdx, aivIdx, FD_WORKSPACE_NUM_INDEX, fdRes.s2SplitNum[t]);
287 faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_START_INDEX, fdRes.mStart[aivIdx]);286 faMetadata.SetFdMetadata(secIdx, aivIdx, FD_M_START_INDEX, fdRes.mStart[aivIdx]);
@@ -6,8 +6,8 @@
6 * this file except in compliance with the License. THIS SOFTWARE IS PROVIDED ON6 * this file except in compliance with the License. THIS SOFTWARE IS PROVIDED ON
7 * AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,7 * AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
8 * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS8 * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS
9- * FOR A PARTICULAR PURPOSE. See LICENSE in the software repository for the full9+ * FOR A PARTICULAR PURPOSE. See LICENSE in the root of the software repository
10- * text of the License.10+ * for the full text of the License.
11 */11 */
12 12 
13#include <iostream>13#include <iostream>
@@ -22,14 +22,15 @@
22using namespace std;22using namespace std;
23using namespace optiling;23using namespace optiling;
24 24 
25-#define CHECK_RET(cond) ((cond) ? true :(false))25+#define CHECK_RET(cond) ((cond) ? true : (false))
26 26 
27-#define LOG_PRINT(message, ...) \27+#define LOG_PRINT(message, ...) \
28- do { \28+ do { \
29- printf(message, ##__VA_ARGS__); \29+ printf(message, ##__VA_ARGS__); \
30 } while (0)30 } while (0)
31 31 
32-int64_t GetShapeSize(const std::vector<int64_t> &shape) {32+int64_t GetShapeSize(const std::vector<int64_t> &shape)
33+{
33 int64_t shapeSize = 1;34 int64_t shapeSize = 1;
34 for (auto i : shape) {35 for (auto i : shape) {
35 shapeSize *= i;36 shapeSize *= i;
@@ -37,46 +38,49 @@ int64_t GetShapeSize(const std::vector<int64_t> &shape) {
37 return shapeSize;38 return shapeSize;
38}39}
39 40 
40-int Init(int32_t deviceId, aclrtStream *stream) {41+int Init(int32_t deviceId, aclrtStream *stream)
42+{
41 auto ret = aclInit(nullptr);43 auto ret = aclInit(nullptr);
42 if (!CHECK_RET(ret == ACL_SUCCESS)) {44 if (!CHECK_RET(ret == ACL_SUCCESS)) {
43- LOG_PRINT("aclInit failed. ERROR: %d\n", ret); 45+ LOG_PRINT("aclInit failed. ERROR: %d\n", ret);
44 return ret;46 return ret;
45 }47 }
46 ret = aclrtSetDevice(deviceId);48 ret = aclrtSetDevice(deviceId);
47 if (!CHECK_RET(ret == ACL_SUCCESS)) {49 if (!CHECK_RET(ret == ACL_SUCCESS)) {
48- LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); 50+ LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret);
49 return ret;51 return ret;
50 }52 }
51 ret = aclrtCreateStream(stream);53 ret = aclrtCreateStream(stream);
52 if (!CHECK_RET(ret == ACL_SUCCESS)) {54 if (!CHECK_RET(ret == ACL_SUCCESS)) {
53- LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); 55+ LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret);
54 return ret;56 return ret;
55 }57 }
56 return 0;58 return 0;
57}59}
58 60 
59-static void DumpMeta(void* data) {61+static void DumpMeta(void *data)
60- int32_t sectionNum = static_cast<int32_t>(((int32_t*)data)[0]);62+{
63+ int32_t sectionNum = static_cast<int32_t>(((int32_t *)data)[0]);
61 optiling::detail::FaMetaData faMetadata(data, sectionNum);64 optiling::detail::FaMetaData faMetadata(data, sectionNum);
62 printf("sectionNum:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_SECTION_NUM_INDEX));65 printf("sectionNum:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_SECTION_NUM_INDEX));
63 printf("isFd:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_IS_FD_INDEX));66 printf("isFd:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_IS_FD_INDEX));
64 printf("mBaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_M_BASE_SIZE_INDEX));67 printf("mBaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_M_BASE_SIZE_INDEX));
65 printf("s2BaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_S2_BASE_SIZE_INDEX));68 printf("s2BaseSize:%d\n", faMetadata.GetHeadMedata(optiling::HEAD_S2_BASE_SIZE_INDEX));
66 for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) {69 for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) {
67- printf("sectionIdx:%d\n",sectionId);70+ printf("sectionIdx:%d\n", sectionId);
68 for (size_t i = 0; i < AIC_CORE_NUM; ++i) {71 for (size_t i = 0; i < AIC_CORE_NUM; ++i) {
69- printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX));72+ printf("bn2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX));
70 printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX));73 printf("m start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX));
71 printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX));74 printf("s2 start: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX));
72- printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX));75+ printf("bn2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX));
73 printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX));76 printf("m end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX));
74 printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX));77 printf("s2 end: %d\n", faMetadata.GetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX));
75- printf("first fd data ws idx: %d\n",faMetadata.GetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX));78+ printf("first fd data ws idx: %d\n",
79+ faMetadata.GetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX));
76 }80 }
77 81 
78 for (size_t i = 0; i < AIV_CORE_NUM; ++i) {82 for (size_t i = 0; i < AIV_CORE_NUM; ++i) {
79- printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX));83+ printf("bn2 idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX));
80 printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX));84 printf("m idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX));
81 printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX));85 printf("fd workspace idx: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX));
82 printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX));86 printf("fd workspace num: %d\n", faMetadata.GetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX));
@@ -88,16 +92,17 @@ static void DumpMeta(void* data) {
88 92 
89template <typename T>93template <typename T>
90int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr,94int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr,
91- aclDataType dataType, aclTensor **tensor) {95+ aclDataType dataType, aclTensor **tensor)
96+{
92 auto size = GetShapeSize(shape) * sizeof(T);97 auto size = GetShapeSize(shape) * sizeof(T);
93 auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);98 auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);
94- if (!CHECK_RET(ret == ACL_SUCCESS)) { 99+ if (!CHECK_RET(ret == ACL_SUCCESS)) {
95- LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret); 100+ LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret);
96 return ret;101 return ret;
97 }102 }
98 ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE);103 ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE);
99 if (!CHECK_RET(ret == ACL_SUCCESS)) {104 if (!CHECK_RET(ret == ACL_SUCCESS)) {
100- LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret); 105+ LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret);
101 return ret;106 return ret;
102 }107 }
103 108 
@@ -111,12 +116,13 @@ int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &
111 return 0;116 return 0;
112}117}
113 118 
114-int main() {119+int main()
120+{
115 int32_t deviceId = 0;121 int32_t deviceId = 0;
116 aclrtStream stream;122 aclrtStream stream;
117 auto ret = Init(deviceId, &stream);123 auto ret = Init(deviceId, &stream);
118 if (!CHECK_RET(ret == ACL_SUCCESS)) {124 if (!CHECK_RET(ret == ACL_SUCCESS)) {
119- LOG_PRINT("Init acl failed. ERROR: %d\n", ret); 125+ LOG_PRINT("Init acl failed. ERROR: %d\n", ret);
120 return ret;126 return ret;
121 }127 }
122 128 
@@ -147,9 +153,9 @@ int main() {
147 void *actualSeqLengthsQueryDeviceAddr = nullptr;153 void *actualSeqLengthsQueryDeviceAddr = nullptr;
148 void *actualSeqLengthsKvDeviceAddr = nullptr;154 void *actualSeqLengthsKvDeviceAddr = nullptr;
149 void *metadataDeviceAddr = nullptr;155 void *metadataDeviceAddr = nullptr;
150- aclTensor* actualSeqLengthsQueryTensor = nullptr;156+ aclTensor *actualSeqLengthsQueryTensor = nullptr;
151- aclTensor* actualSeqLengthsKvTensor = nullptr;157+ aclTensor *actualSeqLengthsKvTensor = nullptr;
152- aclTensor* metadataTensor = nullptr;158+ aclTensor *metadataTensor = nullptr;
153 int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape);159 int64_t actualSeqLengthsQueryShapeSize = GetShapeSize(actualSeqLengthsQueryShape);
154 int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape);160 int64_t actualSeqLengthsKvShapeSize = GetShapeSize(actualSeqLengthsKvShape);
155 int64_t metadataShapeSize = GetShapeSize(metadataShape);161 int64_t metadataShapeSize = GetShapeSize(metadataShape);
@@ -157,34 +163,30 @@ int main() {
157 std::vector<int32_t> actualSeqLengthsKvHostData(actualSeqLengthsKvShapeSize, kvS);163 std::vector<int32_t> actualSeqLengthsKvHostData(actualSeqLengthsKvShapeSize, kvS);
158 std::vector<int32_t> metadataHostData(metadataShapeSize, 0);164 std::vector<int32_t> metadataHostData(metadataShapeSize, 0);
159 165 
160- ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr, aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor);166+ ret = CreateAclTensor(actualSeqLengthsQueryHostData, actualSeqLengthsQueryShape, &actualSeqLengthsQueryDeviceAddr,
167+ aclDataType::ACL_INT32, &actualSeqLengthsQueryTensor);
161 if (!CHECK_RET(ret == ACL_SUCCESS)) {168 if (!CHECK_RET(ret == ACL_SUCCESS)) {
162 return ret;169 return ret;
163 }170 }
164- ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr, aclDataType::ACL_INT32, &actualSeqLengthsKvTensor);171+ ret = CreateAclTensor(actualSeqLengthsKvHostData, actualSeqLengthsKvShape, &actualSeqLengthsKvDeviceAddr,
172+ aclDataType::ACL_INT32, &actualSeqLengthsKvTensor);
165 if (!CHECK_RET(ret == ACL_SUCCESS)) {173 if (!CHECK_RET(ret == ACL_SUCCESS)) {
166 return ret;174 return ret;
167 }175 }
168- ret = CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor);176+ ret =
177+ CreateAclTensor(metadataHostData, metadataShape, &metadataDeviceAddr, aclDataType::ACL_INT32, &metadataTensor);
169 if (!CHECK_RET(ret == ACL_SUCCESS)) {178 if (!CHECK_RET(ret == ACL_SUCCESS)) {
170 return ret;179 return ret;
171 }180 }
172 181 
173- aclOpExecutor* executor = nullptr;182+ aclOpExecutor *executor = nullptr;
174 uint64_t workspaceSize = 0;183 uint64_t workspaceSize = 0;
175- void* workspaceAddr = nullptr;184+ void *workspaceAddr = nullptr;
176 185 
177 printf("start aclnnQuantFlashAttnMetadata\n");186 printf("start aclnnQuantFlashAttnMetadata\n");
178 ret = aclnnQuantFlashAttnMetadataGetWorkspaceSize(187 ret = aclnnQuantFlashAttnMetadataGetWorkspaceSize(
179- nullptr, nullptr,188+ nullptr, nullptr, nullptr, nullptr, nullptr, batchSize, qS, kvS, numHeads, numKeyValueHeads, headDim, quantMode,
180- nullptr, nullptr,189+ sparseMode, preTokens, nextTokens, "BSND", "BSND", "BSND", "BSND", metadataTensor, &workspaceSize, &executor);
181- nullptr,
182- batchSize, qS, kvS,
183- numHeads, numKeyValueHeads,
184- headDim, quantMode, sparseMode, preTokens, nextTokens,
185- "BSND", "BSND", "BSND", "BSND",
186- metadataTensor,
187- &workspaceSize, &executor);
188 if (ret != ACL_SUCCESS) {190 if (ret != ACL_SUCCESS) {
189 printf("aclnnQuantFlashAttnMetadataGetWorkspaceSize %d\n", ret);191 printf("aclnnQuantFlashAttnMetadataGetWorkspaceSize %d\n", ret);
190 return -1;192 return -1;
@@ -202,15 +204,13 @@ int main() {
202 return -1;204 return -1;
203 }205 }
204 206 
205- ret = aclrtMemcpy(metadataHostData.data(),207+ ret = aclrtMemcpy(metadataHostData.data(), metadataHostData.size() * sizeof(metadataHostData[0]),
206- metadataHostData.size() * sizeof(metadataHostData[0]), metadataDeviceAddr,208+ metadataDeviceAddr, metadataShapeSize, ACL_MEMCPY_DEVICE_TO_HOST);
207- metadataShapeSize,
208- ACL_MEMCPY_DEVICE_TO_HOST);
209 if (ret != ACL_SUCCESS) {209 if (ret != ACL_SUCCESS) {
210 printf("aclrtMemcpy %d\n", ret);210 printf("aclrtMemcpy %d\n", ret);
211 return -1;211 return -1;
212 }212 }
213- 213+ 
214 DumpMeta(&metadataHostData[0]);214 DumpMeta(&metadataHostData[0]);
215 215 
216 aclDestroyTensor(metadataTensor);216 aclDestroyTensor(metadataTensor);
@@ -35,15 +35,15 @@ constexpr uint32_t HEAD_IS_FD_INDEX = 1U;
35constexpr uint32_t HEAD_M_BASE_SIZE_INDEX = 2U;35constexpr uint32_t HEAD_M_BASE_SIZE_INDEX = 2U;
36constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U;36constexpr uint32_t HEAD_S2_BASE_SIZE_INDEX = 3U;
37 37 
38-constexpr uint32_t FA_BN2_START_INDEX = 0U;38+constexpr uint32_t FA_BN_START_INDEX = 0U;
39constexpr uint32_t FA_M_START_INDEX = 1U;39constexpr uint32_t FA_M_START_INDEX = 1U;
40constexpr uint32_t FA_S2_START_INDEX = 2U;40constexpr uint32_t FA_S2_START_INDEX = 2U;
41-constexpr uint32_t FA_BN2_END_INDEX = 3U;41+constexpr uint32_t FA_BN_END_INDEX = 3U;
42constexpr uint32_t FA_M_END_INDEX = 4U;42constexpr uint32_t FA_M_END_INDEX = 4U;
43constexpr uint32_t FA_S2_END_INDEX = 5U;43constexpr uint32_t FA_S2_END_INDEX = 5U;
44constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U;44constexpr uint32_t FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX = 6U;
45 45 
46-constexpr uint32_t FD_BN2_IDX_INDEX = 0U;46+constexpr uint32_t FD_BN_IDX_INDEX = 0U;
47constexpr uint32_t FD_M_IDX_INDEX = 1U;47constexpr uint32_t FD_M_IDX_INDEX = 1U;
48constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U;48constexpr uint32_t FD_WORKSPACE_IDX_INDEX = 2U;
49constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U;49constexpr uint32_t FD_WORKSPACE_NUM_INDEX = 3U;
@@ -57,12 +57,12 @@ struct FaMetaData {
57 uint32_t *faMetadata;57 uint32_t *faMetadata;
58 uint32_t *fdMetadata;58 uint32_t *fdMetadata;
59 FaMetaData(void *metadataPtr, uint32_t sectionNum)59 FaMetaData(void *metadataPtr, uint32_t sectionNum)
60- :sectionNum(sectionNum),60+ : sectionNum(sectionNum),
61- headMedata(static_cast<uint32_t*>(metadataPtr)),61+ headMedata(static_cast<uint32_t *>(metadataPtr)),
62- faMetadata(headMedata + HEAD_METADATA_SIZE),62+ faMetadata(headMedata + HEAD_METADATA_SIZE),
63- fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_SIZE)63+ fdMetadata(faMetadata + sectionNum * AIC_CORE_NUM * FA_METADATA_SIZE)
64 {64 {
65- headMedata[0] = sectionNum;65+ headMedata[0] = sectionNum;
66 }66 }
67 67 
68 void SetHeadMedata(uint32_t metaIdx, uint32_t val)68 void SetHeadMedata(uint32_t metaIdx, uint32_t val)
@@ -195,7 +195,8 @@ bool QuantFlashAttnMetadataCpuKernel::ParamsInit()
195 baseInfo.queryHeadNum = numHeadsQ_;195 baseInfo.queryHeadNum = numHeadsQ_;
196 bool isDecode = (layoutQDescale_ == "N2TGD");196 bool isDecode = (layoutQDescale_ == "N2TGD");
197 baseInfo.kvHeadNum = isDecode ? numHeadsKv_ : numHeadsQ_;197 baseInfo.kvHeadNum = isDecode ? numHeadsKv_ : numHeadsQ_;
198- baseInfo.headDim = headDim_;198+ baseInfo.headDimQk = headDim_;
199+ baseInfo.headDimV = headDim_;
199 baseInfo.attenMaskFlag = (maskMode_ != 0);200 baseInfo.attenMaskFlag = (maskMode_ != 0);
200 baseInfo.sparseMode = static_cast<uint32_t>(maskMode_);201 baseInfo.sparseMode = static_cast<uint32_t>(maskMode_);
201 baseInfo.preToken = winLeft_ == -1 ? std::numeric_limits<uint32_t>::max() : winLeft_;202 baseInfo.preToken = winLeft_ == -1 ? std::numeric_limits<uint32_t>::max() : winLeft_;
@@ -207,7 +208,7 @@ bool QuantFlashAttnMetadataCpuKernel::ParamsInit()
207 baseInfo.kvType = load_balance::DataType::INT8;208 baseInfo.kvType = load_balance::DataType::INT8;
208 }209 }
209 if (quantMode_ == 1) { // TODO:用adjust函数代替210 if (quantMode_ == 1) { // TODO:用adjust函数代替
210- if (baseInfo.headDim == 256) {211+ if (baseInfo.headDimQk == 256) {
211 mBaseSize_ = 64U;212 mBaseSize_ = 64U;
212 s2BaseSize_ = 256U;213 s2BaseSize_ = 256U;
213 } else {214 } else {
@@ -224,6 +225,7 @@ bool QuantFlashAttnMetadataCpuKernel::ParamsInit()
224 param.l2Byte = 0;225 param.l2Byte = 0;
225 param.fdTolerance = 300;226 param.fdTolerance = 300;
226 param.fdOn = 0;227 param.fdOn = 0;
228+ param.outputLayout = load_balance::OutputLayout::BN2_S1G;
227 229 
228 // 校验 v_descale: quantMode=1(MxFp8) 且 TND layout 下, dim0 应等于 sum(ceil(seqused_kv[i] / 64))230 // 校验 v_descale: quantMode=1(MxFp8) 且 TND layout 下, dim0 应等于 sum(ceil(seqused_kv[i] / 64))
229 if (quantMode_ == 1 && vDescale_ != nullptr && vDescale_->GetTensorShape() != nullptr) {231 if (quantMode_ == 1 && vDescale_ != nullptr && vDescale_->GetTensorShape() != nullptr) {
@@ -280,16 +282,16 @@ bool QuantFlashAttnMetadataCpuKernel::GenMetaData(SectionStreamKResult &splitRes
280 282 
281 for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) {283 for (uint32_t sectionId = 0; sectionId < sectionNum; ++sectionId) {
282 for (uint32_t i = 0; i < AIC_CORE_NUM; ++i) {284 for (uint32_t i = 0; i < AIC_CORE_NUM; ++i) {
283- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX, 0U);285+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX, 0U);
284 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, 0U);286 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, 0U);
285 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, 0U);287 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, 0U);
286- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX, 0U);288+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX, 0U);
287 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, 0U);289 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, 0U);
288 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, 0U);290 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, 0U);
289 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, 0U);291 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX, 0U);
290 }292 }
291 for (uint32_t i = 0; i < AIV_CORE_NUM; ++i) {293 for (uint32_t i = 0; i < AIV_CORE_NUM; ++i) {
292- faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX, 0U);294+ faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX, 0U);
293 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, 0U);295 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, 0U);
294 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX, 0U);296 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX, 0U);
295 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, 0U);297 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, 0U);
@@ -300,20 +302,20 @@ bool QuantFlashAttnMetadataCpuKernel::GenMetaData(SectionStreamKResult &splitRes
300 auto faSplitRes = splitRes.sectionFaResult[sectionId];302 auto faSplitRes = splitRes.sectionFaResult[sectionId];
301 for (uint32_t i = 0; i < faSplitRes.usedCoreNum; ++i) {303 for (uint32_t i = 0; i < faSplitRes.usedCoreNum; ++i) {
302 if (i > 0) {304 if (i > 0) {
303- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX, faSplitRes.bN2End[i - 1]);305+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX, faSplitRes.bNEnd[i - 1]);
304- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, faSplitRes.gS1End[i - 1]);306+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX, faSplitRes.mEnd[i - 1]);
305 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, faSplitRes.s2End[i - 1]);307 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX, faSplitRes.s2End[i - 1]);
306 } else if (sectionId > 0) {308 } else if (sectionId > 0) {
307 auto preFaSplitRes = splitRes.sectionFaResult[sectionId - 1];309 auto preFaSplitRes = splitRes.sectionFaResult[sectionId - 1];
308- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_START_INDEX,310+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_START_INDEX,
309- preFaSplitRes.bN2End[preFaSplitRes.usedCoreNum - 1]);311+ preFaSplitRes.bNEnd[preFaSplitRes.usedCoreNum - 1]);
310 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX,312 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_START_INDEX,
311- preFaSplitRes.gS1End[preFaSplitRes.usedCoreNum - 1]);313+ preFaSplitRes.mEnd[preFaSplitRes.usedCoreNum - 1]);
312 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX,314 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_START_INDEX,
313 preFaSplitRes.s2End[preFaSplitRes.usedCoreNum - 1]);315 preFaSplitRes.s2End[preFaSplitRes.usedCoreNum - 1]);
314 }316 }
315- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN2_END_INDEX, faSplitRes.bN2End[i]);317+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_BN_END_INDEX, faSplitRes.bNEnd[i]);
316- faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, faSplitRes.gS1End[i]);318+ faMetadata.SetFaMetadata(sectionId, i, optiling::FA_M_END_INDEX, faSplitRes.mEnd[i]);
317 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, faSplitRes.s2End[i]);319 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_S2_END_INDEX, faSplitRes.s2End[i]);
318 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX,320 faMetadata.SetFaMetadata(sectionId, i, optiling::FA_FIRST_FD_DATA_WORKSPACE_IDX_INDEX,
319 faSplitRes.firstFdDataWorkspaceIdx[i]);321 faSplitRes.firstFdDataWorkspaceIdx[i]);
@@ -322,8 +324,8 @@ bool QuantFlashAttnMetadataCpuKernel::GenMetaData(SectionStreamKResult &splitRes
322 auto fdSplitRes = splitRes.sectionFdResult[sectionId];324 auto fdSplitRes = splitRes.sectionFdResult[sectionId];
323 for (uint32_t i = 0; i < fdSplitRes.usedVecNum; ++i) {325 for (uint32_t i = 0; i < fdSplitRes.usedVecNum; ++i) {
324 uint32_t curTaskIdx = fdSplitRes.taskIdx[i];326 uint32_t curTaskIdx = fdSplitRes.taskIdx[i];
325- faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN2_IDX_INDEX, fdSplitRes.bN2Idx[curTaskIdx]);327+ faMetadata.SetFdMetadata(sectionId, i, optiling::FD_BN_IDX_INDEX, fdSplitRes.bNIdx[curTaskIdx]);
326- faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, fdSplitRes.gS1Idx[curTaskIdx]);328+ faMetadata.SetFdMetadata(sectionId, i, optiling::FD_M_IDX_INDEX, fdSplitRes.mIdx[curTaskIdx]);
327 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX,329 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_IDX_INDEX,
328 fdSplitRes.workspaceIdx[curTaskIdx]);330 fdSplitRes.workspaceIdx[curTaskIdx]);
329 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, fdSplitRes.s2SplitNum[curTaskIdx]);331 faMetadata.SetFdMetadata(sectionId, i, optiling::FD_WORKSPACE_NUM_INDEX, fdSplitRes.s2SplitNum[curTaskIdx]);
@@ -33,19 +33,18 @@ constexpr uint32_t AIV_RESERVED_METADATA_STRIDE = 16U;
33constexpr uint64_t GetMetadataRequiredElems(uint32_t sectionNum)33constexpr uint64_t GetMetadataRequiredElems(uint32_t sectionNum)
34{34{
35 return static_cast<uint64_t>(HEAD_METADATA_STRIDE) +35 return static_cast<uint64_t>(HEAD_METADATA_STRIDE) +
36- static_cast<uint64_t>(sectionNum) *36+ static_cast<uint64_t>(sectionNum) * (static_cast<uint64_t>(AIC_CORE_NUM) * FA_METADATA_STRIDE +
37- (static_cast<uint64_t>(AIC_CORE_NUM) * FA_METADATA_STRIDE +37+ static_cast<uint64_t>(AIV_CORE_NUM) * AIV_RESERVED_METADATA_STRIDE);
38- static_cast<uint64_t>(AIV_CORE_NUM) * AIV_RESERVED_METADATA_STRIDE);
39}38}
40 39 
41// Head Metadata Index Definitions40// Head Metadata Index Definitions
42constexpr uint32_t HEAD_SECTION_NUM_INDEX = 0U;41constexpr uint32_t HEAD_SECTION_NUM_INDEX = 0U;
43 42 
44// Section-based FA Metadata Index (0-based, matching AICPU flash_attn_metadata format)43// Section-based FA Metadata Index (0-based, matching AICPU flash_attn_metadata format)
45-constexpr uint32_t SLI_SEC_BN2_START_INDEX = 0U;44+constexpr uint32_t SLI_SEC_BN_START_INDEX = 0U;
46constexpr uint32_t SLI_SEC_M_START_INDEX = 1U;45constexpr uint32_t SLI_SEC_M_START_INDEX = 1U;
47constexpr uint32_t SLI_SEC_S2_START_INDEX = 2U;46constexpr uint32_t SLI_SEC_S2_START_INDEX = 2U;
48-constexpr uint32_t SLI_SEC_BN2_END_INDEX = 3U;47+constexpr uint32_t SLI_SEC_BN_END_INDEX = 3U;
49constexpr uint32_t SLI_SEC_M_END_INDEX = 4U;48constexpr uint32_t SLI_SEC_M_END_INDEX = 4U;
50constexpr uint32_t SLI_SEC_S2_END_INDEX = 5U;49constexpr uint32_t SLI_SEC_S2_END_INDEX = 5U;
51 50 
@@ -54,7 +53,7 @@ struct SliMetadata {
54 uint32_t sectionNum;53 uint32_t sectionNum;
55 SLI_METADATA_T *metadata;54 SLI_METADATA_T *metadata;
56 SLI_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE];55 SLI_METADATA_T *headMetadata; // [HEAD_METADATA_STRIDE];
57- SLI_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE];56+ SLI_METADATA_T *faMetadata; // [sectionNum][AIC_CORE_NUM][FA_METADATA_STRIDE];
58 SliMetadata(void *metadataPtr, uint32_t sectionNum)57 SliMetadata(void *metadataPtr, uint32_t sectionNum)
59 : sectionNum(sectionNum),58 : sectionNum(sectionNum),
60 metadata(static_cast<SLI_METADATA_T *>(metadataPtr)),59 metadata(static_cast<SLI_METADATA_T *>(metadataPtr)),
@@ -162,7 +162,8 @@ bool StemIndexerMetadataCpuKernel::GenerateBaseInfo(StemIndexerBaseInfo &baseInf
162 baseInfo.querySeqSize = 0;162 baseInfo.querySeqSize = 0;
163 baseInfo.kvHeadNum = numHeadsKv_;163 baseInfo.kvHeadNum = numHeadsKv_;
164 baseInfo.kvSeqSize = 0;164 baseInfo.kvSeqSize = 0;
165- baseInfo.headDim = headDim_;165+ baseInfo.headDimQk = headDim_;
166+ baseInfo.headDimV = headDim_;
166 baseInfo.attenMaskFlag = causal_;167 baseInfo.attenMaskFlag = causal_;
167 baseInfo.sparseMode = (causal_) ? static_cast<uint32_t>(load_balance::SparseMode::RIGHT_DOWN_CAUSAL) :168 baseInfo.sparseMode = (causal_) ? static_cast<uint32_t>(load_balance::SparseMode::RIGHT_DOWN_CAUSAL) :
168 static_cast<uint32_t>(load_balance::SparseMode::BUTT);169 static_cast<uint32_t>(load_balance::SparseMode::BUTT);
@@ -192,6 +193,7 @@ bool StemIndexerMetadataCpuKernel::GenerateSectionStreamKParam(load_balance::Sec
192 param.mBaseSize = STEM_M_BASE_SIZE;193 param.mBaseSize = STEM_M_BASE_SIZE;
193 param.s2BaseSize = STEM_S2_BASE_SIZE;194 param.s2BaseSize = STEM_S2_BASE_SIZE;
194 param.fdOn = false;195 param.fdOn = false;
196+ param.outputLayout = load_balance::OutputLayout::BN2_S1G;
195 return true;197 return true;
196}198}
197 199 
@@ -219,15 +221,15 @@ bool StemIndexerMetadataCpuKernel::GenMetadata(SectionStreamKResult &result)
219 for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) {221 for (uint32_t aicIdx = 0; aicIdx < faRes.usedCoreNum; ++aicIdx) {
220 auto &prevFaRes = (secIdx == 0U) ? dummyHead : result.sectionFaResult[secIdx - 1U];222 auto &prevFaRes = (secIdx == 0U) ? dummyHead : result.sectionFaResult[secIdx - 1U];
221 auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U;223 auto prevLastCore = (secIdx == 0U) ? 0U : prevFaRes.usedCoreNum - 1U;
222- SLI_METADATA_T bn2Start = (aicIdx == 0) ? prevFaRes.bN2End[prevLastCore] : faRes.bN2End[aicIdx - 1U];224+ SLI_METADATA_T bn2Start = (aicIdx == 0) ? prevFaRes.bNEnd[prevLastCore] : faRes.bNEnd[aicIdx - 1U];
223- SLI_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.gS1End[prevLastCore] : faRes.gS1End[aicIdx - 1U];225+ SLI_METADATA_T mStart = (aicIdx == 0) ? prevFaRes.mEnd[prevLastCore] : faRes.mEnd[aicIdx - 1U];
224 SLI_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U];226 SLI_METADATA_T s2Start = (aicIdx == 0) ? prevFaRes.s2End[prevLastCore] : faRes.s2End[aicIdx - 1U];
225 227 
226- sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN2_START_INDEX, bn2Start);228+ sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN_START_INDEX, bn2Start);
227 sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_START_INDEX, mStart);229 sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_START_INDEX, mStart);
228 sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_START_INDEX, s2Start);230 sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_START_INDEX, s2Start);
229- sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN2_END_INDEX, faRes.bN2End[aicIdx]);231+ sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_BN_END_INDEX, faRes.bNEnd[aicIdx]);
230- sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_END_INDEX, faRes.gS1End[aicIdx]);232+ sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_M_END_INDEX, faRes.mEnd[aicIdx]);
231 sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_END_INDEX, faRes.s2End[aicIdx]);233 sliMetadata.SetFaMetadata(secIdx, aicIdx, optiling::SLI_SEC_S2_END_INDEX, faRes.s2End[aicIdx]);
232 }234 }
233 }235 }