feat(rdma): 支持多 AIV 共享 HNS1825 队列-pto-isa-AtomGit

Mdocs/isa/comm/TPUT_ASYNC_NOTIFY.md+1-1

	@@ -256,7 +256,7 @@ payload, the caller must ensure visibility according to the target platform and
256	not submit concurrently to one group. With `queue_num == N`, at most `kSdmaMaxChannelGroups / N` groups are valid.	256	not submit concurrently to one group. With `queue_num == N`, at most `kSdmaMaxChannelGroups / N` groups are valid.
257	- Concurrent `Set` producers should use separate remote signals. Use `AtomicAdd` when multiple AIVs share one signal	257	- Concurrent `Set` producers should use separate remote signals. Use `AtomicAdd` when multiple AIVs share one signal
258	as a completion counter. Payload destination ranges must not overlap in either mode.	258	as a completion counter. Payload destination ranges must not overlap in either mode.
259	-- For URMA and RDMA, submissions to the same peer/QP must be serialized even when callers constructed separate	259	+- For URMA, submissions to the same peer/QP must be serialized even when callers constructed separate
260	sessions over the same workspace. Different peers use independent queues.	260	sessions over the same workspace. Different peers use independent queues.
261	- Complete all earlier events before rebuilding a session or reusing its backend queue.	261	- Complete all earlier events before rebuilding a session or reusing its backend queue.
262	- When using URMA, complete the last event for every active peer/QP and synchronize every host stream that uses the	262	- When using URMA, complete the last event for every active peer/QP and synchronize every host stream that uses the

Mdocs/isa/comm/TPUT_ASYNC_NOTIFY_zh.md+1-1

	@@ -245,7 +245,7 @@ Event，也会覆盖此前尚未完成的操作。URMA或RDMA访问不同peer时
245	Group。`queue_num` 为 `N` 时，合法Group数最多为 `kSdmaMaxChannelGroups / N`。	245	Group。`queue_num` 为 `N` 时，合法Group数最多为 `kSdmaMaxChannelGroups / N`。
246	- 多个AIV并发 `Set` 时，应为每个生产者使用独立signal；使用同一个signal做完成计数时应使用	246	- 多个AIV并发 `Set` 时，应为每个生产者使用独立signal；使用同一个signal做完成计数时应使用
247	`AtomicAdd`。无论哪种模式，各AIV的payload目的地址范围都不得重叠。	247	`AtomicAdd`。无论哪种模式，各AIV的payload目的地址范围都不得重叠。
248	-- 对URMA和RDMA，即使调用方基于同一workspace构建了不同Session，对同一peer/QP的提交也必须串行；	248	+- 对URMA，即使调用方基于同一workspace构建了不同Session，对同一peer/QP的提交也必须串行；
249	不同peer使用独立队列。	249	不同peer使用独立队列。
250	- 重新构建Session或复用后端队列前，必须先完成此前全部Event。	250	- 重新构建Session或复用后端队列前，必须先完成此前全部Event。
251	- 使用URMA时，释放通信资源前必须完成每个活动peer/QP的最后一个Event，并同步所有使用该	251	- 使用URMA时，释放通信资源前必须完成每个活动peer/QP的最后一个Event，并同步所有使用该

Mpkg_inc/pto/comm/async/rdma/backends/hns_1825/hns_1825_backend.hpp+307-115

	@@ -11,11 +11,16 @@ See LICENSE in the root of the software repository for the full text of the Lice
11	// HNS_1825 (Hi1825) RoCE RDMA device-side backend for A5.	11	// HNS_1825 (Hi1825) RoCE RDMA device-side backend for A5.
12	//	12	//
13	// Posts RDMA WRITE / READ from AIV using the RdmaInfo table supplied via RdmaExecContext.	13	// Posts RDMA WRITE / READ from AIV using the RdmaInfo table supplied via RdmaExecContext.
14	-// WQE/CQE staging uses the RDMA session's UB scratch; atomics are not supported.	14	+// WQE/CQE staging uses each AIV's own UB scratch. Local scalar atomics coordinate
		15	+// shared queues; remote RDMA atomic operations are not supported.
15		16
16	#ifndef PTO_COMM_ASYNC_RDMA_BACKENDS_HNS_1825_BACKEND_HPP	17	#ifndef PTO_COMM_ASYNC_RDMA_BACKENDS_HNS_1825_BACKEND_HPP
17	#define PTO_COMM_ASYNC_RDMA_BACKENDS_HNS_1825_BACKEND_HPP	18	#define PTO_COMM_ASYNC_RDMA_BACKENDS_HNS_1825_BACKEND_HPP
18		19
		20	+#if defined(__NPU_ARCH__) && __NPU_ARCH__ != 0 && __NPU_ARCH__ != 3510
		21	+#error "The HNS1825 shared-queue backend requires Ascend950 (__NPU_ARCH__ == 3510)."
		22	+#endif
		23	+
19	#include "pto/pto-inst.hpp"	24	#include "pto/pto-inst.hpp"
20	#include "pto/comm/async_common/async_types.hpp"	25	#include "pto/comm/async_common/async_types.hpp"
21	#include "pto/comm/async/rdma/rdma_device_common.hpp"	26	#include "pto/comm/async/rdma/rdma_device_common.hpp"

Mpkg_inc/pto/comm/async/rdma/backends/hns_1825/hns_1825_types.hpp+22-0

	@@ -22,6 +22,19 @@ namespace comm {
22	namespace rdma {	22	namespace rdma {
23	namespace hns_1825 {	23	namespace hns_1825 {
24		24
		25	+// One zero-initialized, 64-byte record per peer/QP. All device accesses
		26	+// to these fields use scalar atomics; session-local UB is never shared.
		27	+struct QueueState {
		28	+ uint64_t reserveHead;
		29	+ uint64_t readyHead;
		30	+ uint64_t postedHead;
		31	+ uint64_t completedHead;
		32	+ uint64_t postLock;
		33	+ uint64_t cqLock;
		34	+ uint64_t error;
		35	+ uint64_t reserved;
		36	+};
		37	+
25	struct SqContext {	38	struct SqContext {
26	uint32_t wqn;	39	uint32_t wqn;
27	uint64_t bufAddr;	40	uint64_t bufAddr;
	@@ -38,6 +51,7 @@ struct SqContext {
38	uint8_t mtuShift{4};	51	uint8_t mtuShift{4};
39	uint8_t dbCos{0x7};	52	uint8_t dbCos{0x7};
40	uint8_t reserved[6];	53	uint8_t reserved[6];
		54	+ uint64_t stateAddr;
41	};	55	};
42		56
43	struct CqContext {	57	struct CqContext {
	@@ -141,6 +155,14 @@ constexpr uint32_t kSessionBuildError = 0x20000;
141	constexpr uint32_t kInvalidArgumentError = 0x20001;	155	constexpr uint32_t kInvalidArgumentError = 0x20001;
142	constexpr uint32_t kInvalidContextError = 0x20002;	156	constexpr uint32_t kInvalidContextError = 0x20002;
143	constexpr uint32_t kCqeError = 0x20003;	157	constexpr uint32_t kCqeError = 0x20003;
		158	+// Handles retain their peer/u32-index encoding. Stop before the index can wrap
		159	+// into the zero/no-op handle or alias a live event from an earlier generation.
		160	+constexpr uint32_t kQueueIndexExhaustedError = 0x20004;
		161	+constexpr uint64_t kMaxQueueIndex = 0xffffffffULL;
		162	+constexpr uint32_t kCqProgressBatch = 8;
		163	+constexpr uint32_t kPostSendBatch = 32;
		164	+
		165	+static_assert(sizeof(QueueState) == 64, "unexpected HNS1825 queue-state layout");
144		166
145	#if defined(PTO_NPU_ARCH_A2A3) \|\| defined(__DAV_C220_VEC__) \|\| defined(__DAV_C220_CUBE__)	167	#if defined(PTO_NPU_ARCH_A2A3) \|\| defined(__DAV_C220_VEC__) \|\| defined(__DAV_C220_CUBE__)
146	constexpr uint64_t kCycleToTimeBase = 50;	168	constexpr uint64_t kCycleToTimeBase = 50;

Mpkg_inc/pto/comm/async/rdma/backends/hns_1825/hns_1825_workspace_manager.hpp+2-0

	@@ -97,6 +97,7 @@ private:
97	RoceCqCtx* scq{nullptr};	97	RoceCqCtx* scq{nullptr};
98	RoceCqCtx* rcq{nullptr};	98	RoceCqCtx* rcq{nullptr};
99	RdmaMemInfo* memory{nullptr};	99	RdmaMemInfo* memory{nullptr};
		100	+ uint64_t stateDeviceBase{0};
100	};	101	};
101		102
102	template <typename... Args>	103	template <typename... Args>
	@@ -150,6 +151,7 @@ private:
150	static bool ReadU32Device(uint64_t deviceAddress, uint32_t& value);	151	static bool ReadU32Device(uint64_t deviceAddress, uint32_t& value);
151	static bool WriteU32Device(uint64_t deviceAddress, uint32_t value);	152	static bool WriteU32Device(uint64_t deviceAddress, uint32_t value);
152	bool InitializeQueueMirrors(const host::SqContext& sq, const host::CqContext& cq, uint32_t peer) const;	153	bool InitializeQueueMirrors(const host::SqContext& sq, const host::CqContext& cq, uint32_t peer) const;
		154	+ bool InitializeSqOwners(const host::SqContext& sq, uint32_t peer) const;
153	bool PrepareRdmaInfoLayout(std::vector<uint8_t>& hostBuffer, RdmaInfoHostLayout& layout);	155	bool PrepareRdmaInfoLayout(std::vector<uint8_t>& hostBuffer, RdmaInfoHostLayout& layout);
154	bool FillLocalMemoryInfo(RdmaMemInfo* memory);	156	bool FillLocalMemoryInfo(RdmaMemInfo* memory);
155	bool FillPeerRdmaInfo(size_t channelIndex, RdmaInfoHostLayout& layout);	157	bool FillPeerRdmaInfo(size_t channelIndex, RdmaInfoHostLayout& layout);

Mpkg_inc/pto/comm/async/rdma/backends/hns_1825/hns_1825_workspace_manager_info.hpp+27-5

	@@ -100,18 +100,39 @@ inline bool WorkspaceManager::InitializeQueueMirrors(
100	return true;	100	return true;
101	}	101	}
102		102
		103	+inline bool WorkspaceManager::InitializeSqOwners(const host::SqContext& sq, uint32_t peer) const
		104	+{
		105	+ const auto context = DecodeRoceSqContext(sq);
		106	+ // These channels belong exclusively to this workspace and have not posted
		107	+ // any work. Every slot must stop NIC prefetch before its first publication.
		108	+ std::vector<uint8_t> ring(static_cast<size_t>(context.depth) * kWqebbSize, 0);
		109	+ for (size_t offset = 0; offset < ring.size(); offset += kWqebbSize) {
		110	+ ring[offset] = 0xc0; // fixed control bit + invalid owner for lap zero
		111	+ }
		112	+ if (aclrtMemcpy(
		113	+ reinterpret_cast<void*>(context.sqVa), ring.size(), ring.data(), ring.size(), ACL_MEMCPY_HOST_TO_DEVICE) !=
		114	+ ACL_SUCCESS) {
		115	+ std::cerr << "[RoCE] peer " << peer << " SQ owner initialization failed" << std::endl;
		116	+ return false;
		117	+ }
		118	+ return true;
		119	+}
		120	+
103	inline bool WorkspaceManager::PrepareRdmaInfoLayout(std::vector<uint8_t>& hostBuffer, RdmaInfoHostLayout& layout)	121	inline bool WorkspaceManager::PrepareRdmaInfoLayout(std::vector<uint8_t>& hostBuffer, RdmaInfoHostLayout& layout)
104	{	122	{
105	constexpr uint32_t kQpNum = 1;	123	constexpr uint32_t kQpNum = 1;
106	const size_t sqBytes = sizeof(RoceSqCtx) * rankCount_ * kQpNum;	124	const size_t sqBytes = sizeof(RoceSqCtx) * rankCount_ * kQpNum;
107	const size_t cqBytes = sizeof(RoceCqCtx) * rankCount_ * kQpNum;	125	const size_t cqBytes = sizeof(RoceCqCtx) * rankCount_ * kQpNum;
108	const size_t memBytes = sizeof(RdmaMemInfo) * rankCount_;	126	const size_t memBytes = sizeof(RdmaMemInfo) * rankCount_;
109	- layout.totalSize = sizeof(RdmaInfo) + 2 * sqBytes + 2 * cqBytes + memBytes;	127	+ const size_t metadataBytes = sizeof(RdmaInfo) + 2 * sqBytes + 2 * cqBytes + memBytes;
		128	+ const size_t stateOffset = (metadataBytes + sizeof(QueueState) - 1) & ~(sizeof(QueueState) - 1);
		129	+ layout.totalSize = stateOffset + sizeof(QueueState) * rankCount_ * kQpNum;
110	if (aclrtMalloc(&rdmaInfoDevice_, layout.totalSize, ACL_MEM_MALLOC_HUGE_FIRST) != 0 \|\| rdmaInfoDevice_ == nullptr) {	130	if (aclrtMalloc(&rdmaInfoDevice_, layout.totalSize, ACL_MEM_MALLOC_HUGE_FIRST) != 0 \|\| rdmaInfoDevice_ == nullptr) {
111	std::cerr << "[RoCE] aclrtMalloc(rdmaInfo) failed" << std::endl;	131	std::cerr << "[RoCE] aclrtMalloc(rdmaInfo) failed" << std::endl;
112	return false;	132	return false;
113	}	133	}
114	hostBuffer.assign(layout.totalSize, 0);	134	hostBuffer.assign(layout.totalSize, 0);
		135	+ layout.stateDeviceBase = reinterpret_cast<uint64_t>(rdmaInfoDevice_) + stateOffset;
115	layout.info = reinterpret_cast<RdmaInfo*>(hostBuffer.data());	136	layout.info = reinterpret_cast<RdmaInfo*>(hostBuffer.data());
116	layout.info->magic = kRdmaWorkspaceMagic;	137	layout.info->magic = kRdmaWorkspaceMagic;
117	layout.info->version = kRdmaWorkspaceVersion;	138	layout.info->version = kRdmaWorkspaceVersion;
	@@ -200,11 +221,12 @@ inline bool WorkspaceManager::FillPeerRdmaInfo(size_t channelIndex, RdmaInfoHost
200	return false;	221	return false;
201	}	222	}
202	CopyRoceSq(layout.sq[peer], sq);	223	CopyRoceSq(layout.sq[peer], sq);
		224	+ layout.sq[peer].stateAddr = layout.stateDeviceBase + sizeof(QueueState) * peer;
203	layout.rq[peer] = layout.sq[peer];	225	layout.rq[peer] = layout.sq[peer];
204	host::CqContext cq{};	226	host::CqContext cq{};
205	if (entity.cqNum == 0 \|\| entity.cqContextAddr == nullptr \|\|	227	if (entity.cqNum == 0 \|\| entity.cqContextAddr == nullptr \|\|
206	!ReadDeviceStruct(entity.cqContextAddr, &cq, sizeof(cq)) \|\| !ValidateCqContext(cq, peer) \|\|	228	!ReadDeviceStruct(entity.cqContextAddr, &cq, sizeof(cq)) \|\| !ValidateCqContext(cq, peer) \|\|
207	- !InitializeQueueMirrors(sq, cq, peer)) {	229	+ !InitializeQueueMirrors(sq, cq, peer) \|\| !InitializeSqOwners(sq, peer)) {
208	return false;	230	return false;
209	}	231	}
210	CopyRoceCq(layout.scq[peer], cq);	232	CopyRoceCq(layout.scq[peer], cq);
	@@ -323,9 +345,9 @@ inline bool WorkspaceManager::ValidateCqContext(const host::CqContext& cq, uint3
323	const uint32_t cqeSize = context.cqeSize == 0 ? kHns1825DefaultCqeSize : context.cqeSize;	345	const uint32_t cqeSize = context.cqeSize == 0 ? kHns1825DefaultCqeSize : context.cqeSize;
324	const uint64_t cqRing = static_cast<uint64_t>(context.cqDepth);	346	const uint64_t cqRing = static_cast<uint64_t>(context.cqDepth);
325	const bool supportedCqeSize = cqeSize == sizeof(Hns1825Cqe) \|\| cqeSize == kHns1825DefaultCqeSize;	347	const bool supportedCqeSize = cqeSize == sizeof(Hns1825Cqe) \|\| cqeSize == kHns1825DefaultCqeSize;
326	- if (cq.type != host::CQ_CONTEXT_TYPE_ROCE \|\| !IsPowerOfTwo(cqRing) \|\| !supportedCqeSize \|\| context.cqVa == 0 \|\|	348	+ if (cq.type != host::CQ_CONTEXT_TYPE_ROCE \|\| !IsPowerOfTwo(cqRing) \|\| cqRing < 4 \|\| !supportedCqeSize \|\|
327	- context.headAddr == 0 \|\| context.tailAddr == 0 \|\| context.dbSwVa == 0 \|\| !IsCacheLineAligned(context.cqVa) \|\|	349	+ context.cqVa == 0 \|\| context.headAddr == 0 \|\| context.tailAddr == 0 \|\| context.dbSwVa == 0 \|\|
328	- !IsU32Aligned(context.tailAddr) \|\| !IsU32Aligned(context.dbSwVa)) {	350	+ !IsCacheLineAligned(context.cqVa) \|\| !IsU32Aligned(context.tailAddr) \|\| !IsU32Aligned(context.dbSwVa)) {
329	std::cerr << "[RoCE] peer " << peer << " has an unusable RoCE CQ context: type=" << static_cast<int>(cq.type)	351	std::cerr << "[RoCE] peer " << peer << " has an unusable RoCE CQ context: type=" << static_cast<int>(cq.type)
330	<< " depth=" << context.cqDepth << " cqeSize=" << context.cqeSize << std::endl;	352	<< " depth=" << context.cqDepth << " cqeSize=" << context.cqeSize << std::endl;
331	return false;	353	return false;

Mpkg_inc/pto/comm/async/rdma/rdma_workspace_types.hpp+1-1

	@@ -23,7 +23,7 @@ namespace comm {
23	namespace rdma {	23	namespace rdma {
24		24
25	constexpr uint32_t kRdmaWorkspaceMagic = 0x52444d41U; // "RDMA"	25	constexpr uint32_t kRdmaWorkspaceMagic = 0x52444d41U; // "RDMA"
26	-constexpr uint32_t kRdmaWorkspaceVersion = 1;	26	+constexpr uint32_t kRdmaWorkspaceVersion = 2;
27		27
28	enum class RdmaDbMode : int32_t {	28	enum class RdmaDbMode : int32_t {
29	INVALID_DB = -1,	29	INVALID_DB = -1,

Mtests/npu/a5/comm/st/testcase/tput_async_rdma/README.md+3-0

	@@ -21,6 +21,9 @@ python3 tests/script/run_st.py -r npu -v a5 -t comm/tget_async_rdma -d -n 2
21	python3 tests/script/run_st.py -r npu -v a5 -t comm/tput_async_notify_rdma -d -n 2	21	python3 tests/script/run_st.py -r npu -v a5 -t comm/tput_async_notify_rdma -d -n 2
22	```	22	```
23		23
		24	+`TPutAsyncRdma.Vec_Int32_MultiAiv` checks 16 AIVs sharing one peer/QP, covering PUT/notify traffic, ring wraparound,
		25	+and data/event correctness. Add `-g TPutAsyncRdma.Vec_Int32_MultiAiv` to the `tput_async_rdma` command to run it alone.
		26	+
24	For the first `TPUT_ASYNC_NOTIFY` validation, run only the focused two-rank case:	27	For the first `TPUT_ASYNC_NOTIFY` validation, run only the focused two-rank case:
25		28
26	```bash	29	```bash

Mtests/npu/a5/comm/st/testcase/tput_async_rdma/README_zh.md+3-0

	@@ -20,6 +20,9 @@ python3 tests/script/run_st.py -r npu -v a5 -t comm/tget_async_rdma -d -n 2
20	python3 tests/script/run_st.py -r npu -v a5 -t comm/tput_async_notify_rdma -d -n 2	20	python3 tests/script/run_st.py -r npu -v a5 -t comm/tput_async_notify_rdma -d -n 2
21	```	21	```
22		22
		23	+`TPutAsyncRdma.Vec_Int32_MultiAiv` 验证16个AIV共享同一peer/QP时的PUT/notify、队列回绕及数据/Event正确性。
		24	+在 `tput_async_rdma` 命令中加 `-g TPutAsyncRdma.Vec_Int32_MultiAiv` 可单独运行。
		25	+
23	首次验证`TPUT_ASYNC_NOTIFY`时，先只运行2个rank的定向用例：	26	首次验证`TPUT_ASYNC_NOTIFY`时，先只运行2个rank的定向用例：
24		27
25	```bash	28	```bash

Mtests/npu/a5/comm/st/testcase/tput_async_rdma/backends/rdma_test_backend.hpp+36-1

	@@ -13,6 +13,7 @@ See LICENSE in the root of the software repository for the full text of the Lice
13		13
14	#include <cstdint>	14	#include <cstdint>
15	#include <cstdlib>	15	#include <cstdlib>
		16	+#include <acl/acl.h>
16		17
17	#ifdef PTO_RDMA_BACKEND_HNS_1825_SUPPORTED	18	#ifdef PTO_RDMA_BACKEND_HNS_1825_SUPPORTED
18	#include "hns_1825/hns_1825_bootstrap.hpp"	19	#include "hns_1825/hns_1825_bootstrap.hpp"
	@@ -29,6 +30,38 @@ namespace test {
29	#ifdef PTO_RDMA_BACKEND_HNS_1825_SUPPORTED	30	#ifdef PTO_RDMA_BACKEND_HNS_1825_SUPPORTED
30	using BackendBootstrap = hns_1825::bootstrap::BootstrapConfig;	31	using BackendBootstrap = hns_1825::bootstrap::BootstrapConfig;
31		32
		33	+struct BackendQueueSnapshot {
		34	+ uint32_t sqDepth{0};
		35	+ uint32_t cqDepth{0};
		36	+ hns_1825::QueueState state{};
		37	+};
		38	+
		39	+// Read only while the test stream is idle; the driver owns both ring depths.
		40	+inline bool ReadBackendQueueSnapshot(const void* workspace, uint32_t peer, BackendQueueSnapshot& snapshot)
		41	+{
		42	+ RdmaInfo info{};
		43	+ if (aclrtMemcpy(&info, sizeof(info), workspace, sizeof(info), ACL_MEMCPY_DEVICE_TO_HOST) != ACL_SUCCESS \|\|
		44	+ peer >= info.rankCount \|\| info.qpNum != 1 \|\| info.sqPtr == 0 \|\| info.scqPtr == 0) {
		45	+ return false;
		46	+ }
		47	+ hns_1825::SqContext sq{};
		48	+ hns_1825::CqContext cq{};
		49	+ if (aclrtMemcpy(
		50	+ &sq, sizeof(sq), reinterpret_cast<const void>(info.sqPtr + peer sizeof(sq)), sizeof(sq),
		51	+ ACL_MEMCPY_DEVICE_TO_HOST) != ACL_SUCCESS \|\|
		52	+ aclrtMemcpy(
		53	+ &cq, sizeof(cq), reinterpret_cast<const void>(info.scqPtr + peer sizeof(cq)), sizeof(cq),
		54	+ ACL_MEMCPY_DEVICE_TO_HOST) != ACL_SUCCESS \|\|
		55	+ sq.stateAddr == 0) {
		56	+ return false;
		57	+ }
		58	+ snapshot.sqDepth = sq.depth;
		59	+ snapshot.cqDepth = cq.depth;
		60	+ return aclrtMemcpy(
		61	+ &snapshot.state, sizeof(snapshot.state), reinterpret_cast<const void*>(sq.stateAddr),
		62	+ sizeof(snapshot.state), ACL_MEMCPY_DEVICE_TO_HOST) == ACL_SUCCESS;
		63	+}
		64	+
32	inline bool BackendVerboseEnabled()	65	inline bool BackendVerboseEnabled()
33	{	66	{
34	const char* verbose = std::getenv("PTO_ROCE_VERBOSE");	67	const char* verbose = std::getenv("PTO_ROCE_VERBOSE");
	@@ -39,13 +72,15 @@ inline const char* DescribeBackendCompletionStatus(uint32_t status)
39	{	72	{
40	switch (status) {	73	switch (status) {
41	case hns_1825::kHns1825PollCqTimeoutError:	74	case hns_1825::kHns1825PollCqTimeoutError:
42	- return "poll_cq_timeout";	75	+ return "queue_progress_timeout";
43	case hns_1825::kHns1825InvalidArgumentError:	76	case hns_1825::kHns1825InvalidArgumentError:
44	return "invalid_transfer";	77	return "invalid_transfer";
45	case hns_1825::kHns1825InvalidContextError:	78	case hns_1825::kHns1825InvalidContextError:
46	return "invalid_context";	79	return "invalid_context";
47	case hns_1825::kHns1825CqeError:	80	case hns_1825::kHns1825CqeError:
48	return "cqe_error_without_syndrome";	81	return "cqe_error_without_syndrome";
		82	+ case hns_1825::kQueueIndexExhaustedError:
		83	+ return "queue_index_exhausted";
49	default:	84	default:
50	return nullptr;	85	return nullptr;
51	}	86	}

Mtests/npu/a5/comm/st/testcase/tput_async_rdma/main.cpp+12-2

	@@ -28,10 +28,10 @@ void ExpectPutResult(int nRanks = 2, int nDevices = 2)
28	template <typename T, size_t count>	28	template <typename T, size_t count>
29	void ExpectPutPlan(	29	void ExpectPutPlan(
30	int elemOffset, int elemCount, int operationCount, RdmaCompletionMode completionMode, int nRanks = 2,	30	int elemOffset, int elemCount, int operationCount, RdmaCompletionMode completionMode, int nRanks = 2,
31	- int nDevices = 2)	31	+ int nDevices = 2, int blockCount = 1)
32	{	32	{
33	RdmaTestResult result = RunPutAsyncRdmaRootPutPlan<T, count>(	33	RdmaTestResult result = RunPutAsyncRdmaRootPutPlan<T, count>(
34	- nRanks, nDevices, 0, 0, elemOffset, elemCount, operationCount, completionMode);	34	+ nRanks, nDevices, 0, 0, elemOffset, elemCount, operationCount, completionMode, blockCount);
35	if (result == RdmaTestResult::SKIPPED) {	35	if (result == RdmaTestResult::SKIPPED) {
36	GTEST_SKIP() << "RDMA runtime prerequisites are unavailable on at least one rank";	36	GTEST_SKIP() << "RDMA runtime prerequisites are unavailable on at least one rank";
37	}	37	}
	@@ -86,6 +86,16 @@ TEST(TPutAsyncRdma, Vec_Int32_MultiWqe_WaitLast)
86	ExpectPutPlan<int32_t, 4096>(0, 256, 16, RdmaCompletionMode::STATUS_WAIT_LAST);	86	ExpectPutPlan<int32_t, 4096>(0, 256, 16, RdmaCompletionMode::STATUS_WAIT_LAST);
87	}	87	}
88		88
		89	+TEST(TPutAsyncRdma, Vec_Int32_MultiAiv)
		90	+{
		91	+ SKIP_IF_RANKS_LT(2);
		92	+ const RdmaTestResult result = RunPutAsyncRdmaConcurrent(2, 2, 0, 0);
		93	+ if (result == RdmaTestResult::SKIPPED) {
		94	+ GTEST_SKIP() << "RDMA runtime prerequisites are unavailable on at least one rank";
		95	+ }
		96	+ ASSERT_EQ(result, RdmaTestResult::PASSED);
		97	+}
		98	+
89	// Cover the public AsyncEvent dispatch, including Test before/after Wait.	99	// Cover the public AsyncEvent dispatch, including Test before/after Wait.
90	TEST(TPutAsyncRdma, Vec_Float_PublicEventWaitTest)	100	TEST(TPutAsyncRdma, Vec_Float_PublicEventWaitTest)
91	{	101	{

Mtests/npu/a5/comm/st/testcase/tput_async_rdma/tput_async_rdma_kernel.cpp+363-41

 #include <utility>
 #include <vector>
++#include <acl/acl.h>
 #include <pto/pto-inst.hpp>
--#include "../common.hpp"
++#include "../comm_mpi.h"
 #include "tput_async_rdma_kernel.h"
 #include "pto/common/pto_tile.hpp"
 #ifdef PTO_RDMA_SUPPORTED

Mtests/npu/a5/comm/st/testcase/tput_async_rdma/tput_async_rdma_kernel.h+3-1

	@@ -30,10 +30,12 @@ RdmaTestResult RunPutAsyncRdmaRootPut(int n_ranks, int n_devices, int first_rank
30		30
31	RdmaTestResult RunPutAsyncNotifyRdmaSet(int n_ranks, int n_devices, int first_rank_id, int first_device_id);	31	RdmaTestResult RunPutAsyncNotifyRdmaSet(int n_ranks, int n_devices, int first_rank_id, int first_device_id);
32		32
		33	+RdmaTestResult RunPutAsyncRdmaConcurrent(int n_ranks, int n_devices, int first_rank_id, int first_device_id);
		34	+
33	template <typename T, size_t count>	35	template <typename T, size_t count>
34	RdmaTestResult RunPutAsyncRdmaRootPutPlan(	36	RdmaTestResult RunPutAsyncRdmaRootPutPlan(
35	int n_ranks, int n_devices, int first_rank_id, int first_device_id, int elem_offset, int elem_count,	37	int n_ranks, int n_devices, int first_rank_id, int first_device_id, int elem_offset, int elem_count,
36	- int operation_count, RdmaCompletionMode completion_mode);	38	+ int operation_count, RdmaCompletionMode completion_mode, int block_count = 1);
37		39
38	// Shared host entry point used by the RDMA GET test target.	40	// Shared host entry point used by the RDMA GET test target.
39	template <typename T, size_t count>	41	template <typename T, size_t count>