Open
引入 URMA 统一资源框架,接入社区独占实现并支持共享资源池与有序收发 #41
引入 URMA 统一资源框架,接入社区独占实现并支持共享资源池与有序收发 #41
Open
CarbonadoRainAdd on 2 days ago
Total: 41 files changed+8506-314
@@ -27,6 +27,7 @@ option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF)
27option(WITH_SNAPPY "With snappy" OFF)27option(WITH_SNAPPY "With snappy" OFF)
28option(WITH_RDMA "With RDMA" OFF)28option(WITH_RDMA "With RDMA" OFF)
29option(WITH_URMA "With URMA (openEuler Unified Remote Memory Access)" OFF)29option(WITH_URMA "With URMA (openEuler Unified Remote Memory Access)" OFF)
30+option(WITH_TCMALLOC "Link tcmalloc" OFF)
30set(URMA_USE_MOCK "AUTO" CACHE STRING31set(URMA_USE_MOCK "AUTO" CACHE STRING
31 "URMA mock mode: AUTO, ON, or OFF (OFF requires liburma)")32 "URMA mock mode: AUTO, ON, or OFF (OFF requires liburma)")
32set_property(CACHE URMA_USE_MOCK PROPERTY STRINGS AUTO ON OFF)33set_property(CACHE URMA_USE_MOCK PROPERTY STRINGS AUTO ON OFF)
@@ -187,12 +188,25 @@ list(APPEND BRPC_COMMON_DEFINITIONS
187 __const__=__unused__188 __const__=__unused__
188 _GNU_SOURCE189 _GNU_SOURCE
189 USE_SYMBOLIZE190 USE_SYMBOLIZE
190- NO_TCMALLOC
191 __STDC_FORMAT_MACROS191 __STDC_FORMAT_MACROS
192 __STDC_LIMIT_MACROS192 __STDC_LIMIT_MACROS
193 __STDC_CONSTANT_MACROS193 __STDC_CONSTANT_MACROS
194)194)
195 195 
196+if(WITH_TCMALLOC)
197+ find_path(TCMALLOC_INCLUDE_PATH NAMES gperftools/tcmalloc.h)
198+ find_library(TCMALLOC_LIB NAMES tcmalloc)
199+ if(NOT TCMALLOC_INCLUDE_PATH OR NOT TCMALLOC_LIB)
200+ message(FATAL_ERROR "WITH_TCMALLOC requires gperftools headers and libtcmalloc")
201+ endif()
202+ if(WITH_ASAN)
203+ message(FATAL_ERROR "tcmalloc and AddressSanitizer cannot be enabled together")
204+ endif()
205+ list(APPEND BRPC_COMMON_INCLUDE_DIRS ${TCMALLOC_INCLUDE_PATH})
206+else()
207+ list(APPEND BRPC_COMMON_DEFINITIONS NO_TCMALLOC)
208+endif()
209+ 
196if(WITH_ASAN)210if(WITH_ASAN)
197 list(APPEND BRPC_COMMON_COMPILE_OPTIONS -fsanitize=address)211 list(APPEND BRPC_COMMON_COMPILE_OPTIONS -fsanitize=address)
198 list(APPEND BRPC_COMMON_LINK_OPTIONS -fsanitize=address)212 list(APPEND BRPC_COMMON_LINK_OPTIONS -fsanitize=address)
@@ -468,6 +482,9 @@ if(WITH_UBRING)
468endif()482endif()
469 483 
470set(BRPC_PRIVATE_LIBS "-lgflags -lprotobuf -lleveldb -lprotoc -lssl -lcrypto -ldl -lz")484set(BRPC_PRIVATE_LIBS "-lgflags -lprotobuf -lleveldb -lprotoc -lssl -lcrypto -ldl -lz")
485+if(WITH_TCMALLOC)
486+ set(BRPC_PRIVATE_LIBS "${BRPC_PRIVATE_LIBS} -ltcmalloc")
487+endif()
471 488 
472if(WITH_GLOG)489if(WITH_GLOG)
473 set(DYNAMIC_LIB ${GLOG_LIB} ${DYNAMIC_LIB})490 set(DYNAMIC_LIB ${GLOG_LIB} ${DYNAMIC_LIB})
@@ -508,6 +525,9 @@ endif()
508list(REMOVE_DUPLICATES BRPC_COMMON_INCLUDE_DIRS)525list(REMOVE_DUPLICATES BRPC_COMMON_INCLUDE_DIRS)
509 526 
510add_library(brpc_common_config INTERFACE)527add_library(brpc_common_config INTERFACE)
528+if(WITH_TCMALLOC)
529+ target_link_libraries(brpc_common_config INTERFACE ${TCMALLOC_LIB})
530+endif()
511# Generated protobuf headers live in the build directory. Search it before src/531# Generated protobuf headers live in the build directory. Search it before src/
512# so leftover in-source *.pb.h (gitignored, produced by Make) cannot shadow532# so leftover in-source *.pb.h (gitignored, produced by Make) cannot shadow
513# updated .proto files such as errno.proto.533# updated .proto files such as errno.proto.
@@ -1,195 +1,153 @@
1-# UrmaTransport:基于 URMA 的远程内存 RPC1+# URMA 共享队列传输
2 2 
3-UrmaTransport 是使用 openEuler3+URMA 通过统一连接资源框架支持两种后端:`--urma_resource_mode=exclusive`(默认)保留每连接独占 Jetty 的实现;`--urma_resource_mode=pooled` 使用共享 JFS/JFR。下文池化架构、配置和示例均指显式选择 `pooled` 的情况。两种后端使用各自的资源描述;两端模式一致时建立 URMA,模式不匹配时协商回退到 TCP。详细边界见 [URMA 连接资源抽象](urma_resources.md)。
4-[URMA](https://atomgit.com/openeuler/umdk)(Unified Remote Memory Access)SDK
5-实现的传输层。它是
6-[#3217](https://github.com/apache/brpc/discussions/3217) 中提出的路线 B,
7-与基于 OBMM 的 UBRing 传输(路线 A,
8-[#3226](https://github.com/apache/brpc/issues/3226))互补,承担大包/跨节点
9-高吞吐场景,共同构成路线 C(双后端)。
10 4 
11-## 技术背景5+UrmaTransport 使用 openEuler [UMDK](https://atomgit.com/openeuler/umdk) 的
6+URMA 接口承载 brpc 消息。当前数据通路采用可靠消息模式 `URMA_TM_RM`、
7+CTP 和独立的 JFS/JFR,通过共享硬件队列服务多个逻辑连接。
12 8 
13-URMA 在 UMDK 支持的设备上提供 verbs 风格接口。当前实现创建可靠消息9+## 资源组织
14-(`URMA_TM_RM`)Jetty,并使用 CTP 传输路径;通过
15-`urma_post_jetty_send_wr` 提交发送 WR,通过 `urma_post_jfr_wr` 提交接收
16-WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获取。
17 10 
18-## 编译配置11+`UrmaMemoryPool` 管理已注册内存,`UrmaResourcePool` 管理队列及其生命周期,
12+`UrmaPooledConnection` 保存分片顺序和连接资源,`UrmaReceiveAdapter` 负责
13+接收路由与交付。`UrmaEndpoint` 保留握手、输入缓存和消息解析。
14+各层通过内存池工厂、资源池工厂和事件适配接口连接。
19 15 
20-### CMake 编译16+JFS 是发送队列,JFR 是接收队列。每个 slot 独立持有 JFC、JFCE 和用于
17+事件分发的 Socket。进程默认创建 48 个 JFS、64 个 JFR,队列数量不随
18+逻辑连接数量线性增长。每个连接保存分配的本地 JFR、对端资源引用、
19+SocketId、收发序号以及重排状态,不单独创建一套硬件队列。
21 20 
22-```bash21+对端 JFR 导入缓存使用 EID、UASID 和 JFR ID 共同作为键,以引用计数
23-# 带 URMA 支持编译 brpc22+共享导入结果。连接销毁后,未完成发送仍持有相关资源引用,直到完成处理
24-cmake -B build -DWITH_URMA=ON23+释放它们。
25-make -C build -j$(nproc)
26 24 
27-# 编译 urma_performance 示例25+## 注册内存池
28-cd example/urma_performance
29-cmake -B build
30-make -C build -j$(nproc)
31-```
32 26 
33-`WITH_URMA=ON` 使用上游 UMDK 头文件进行编译。CMake 优先使用系统安装的27+默认内存池使用 mmap 分配连续内存并注册为 URMA segment,以固定大小的
34-SDK;找不到头文件时,会参照 Mooncake 的 mock 构建方式下载固定版本的28+块进行分配和回收。数据池的默认块大小为 8KiB,供 IOBuf 使用;可用数据
35-UMDK commit,可通过 `DOWNLOAD_URMA_HEADERS=OFF` 禁止下载。找到 `liburma` 时使用29+空间需扣除 IOBuf 的块头。CTP 池提供独立的 4KiB 块,用于接收
36-真实硬件数据通路,否则链接 brpc 的 mock,使 URMA 代码和测试仍可在无硬件30+body;扣除块内 IOBuf 元数据后仍可容纳完整的 4032B body。独立 header
37-环境编译。`URMA_USE_MOCK` 默认为 `AUTO`;设为 `ON` 可强制使用 mock,设为31+池的块大小为 64B,不经过 IOBuf 块分配器。发送直接引用已注册 IOBuf,
38-`OFF` 则要求找到 `liburma` 并排除 mock。32+包括 8KiB 数据块和收到的 4KiB CTP 块,不为切换内存池而复制 body。
39 33 
40-### Make 编译34+空闲块分为 64 个分片,每个分片分别加锁。分配从当前线程对应的分片开始,
35+必要时扫描其他分片。内存池初始化后不自动扩容;耗尽时返回失败。
36+注销注册内存前必须先结束 DMA。仍被应用 IOBuf 引用的数据块需要保留
37+其映射及回收器;CTP 池在所有应用引用归还前同时保留注册信息。
41 38 
42-```bash39+`UrmaMemoryPool` 提供分配、回收、地址归属、segment 查询和注销接口。
43-sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \40+可在全局初始化前通过 `SetUrmaMemoryPoolFactory` 更换实现,由全局初始化
44- --with-urma41+创建并安装数据池、CTP 池与 header 池。替代实现必须保持内存注册及引用生命周期
45-make -j442+约束。用户注册数据通过 `RegisterMemoryForUrma` 和
43+`DeregisterMemoryForUrma` 管理;发送期间不得注销仍被使用的内存。
46 44 
47-# 强制使用 mock,或强制使用真实 liburma45+## 建链与协商
48-sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \
49- --with-urma --with-urma-mock
50-sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \
51- --with-urma --without-urma-mock
52-```
53 46 
54-默认根据 `liburma` 是否存在自动选择;`--without-urma-mock` 要求配置路径中47+连接先通过 TCP 完成握手,再切换到 URMA 数据通路。握手交换接收块大小、
55-存在 `liburma`。48+接收队列信息、带版本的 SocketId、EID/UASID、传输路径类型和 segment
49+描述。客户端支持二进制 v2 和 Protobuf v3 两种握手编码,默认使用 v2。
50+共享队列在 v2 使用 implementation=2,在 v3 使用 resource_mode=2;
51+独占模式对应值为1。双方校验远端资源及地址范围。
56 52 
57-### Bazel 编译53+协商与资源导入完成后才发布 `URMA_ON` 状态。模式不匹配时完整消费 Hello,
54+通过拒绝响应或 ACK=0 协商 TCP 回退,避免握手残留字节进入业务解析。
55+I/O 错误及非法资源描述仍按相应错误路径处理。并发连接各自持有握手对象,
56+避免复用仍在进行中的回调状态。
58 57 
59-```bash58+## 发送路径
60-# 编译带 URMA 支持的 brpc
61-bazel build --define=BRPC_WITH_URMA=true //:brpc
62 59 
63-# 仅展示关闭下载的开关;头文件仍须由工具链或本地 BUILD 目标声明60+每个 SEND 分片带有 64B header,包含目标 SocketId、16位序号和消息类型,
64-bazel build --define=BRPC_WITH_URMA=true \61+多字节字段使用网络字节序。CTP 的单次消息总长度不超过 4096B,因此
65- --define=BRPC_DOWNLOAD_URMA_HEADERS=false //:brpc62+header 之后最多承载 4032B body;实际长度还受双方接收能力约束。
66 63 
67-# 编译并运行 URMA 单元测试(使用 mock,不需要 URMA 硬件)64+多 SGE 发送使用一个 header SGE,加上一个或多个 body SGE。
68-bazel test --define=BRPC_WITH_URMA=true \65+发送函数跨输入 IOBuf 及其数据块收集 body,直到达到总 SGE 上限或 body
69- --define=BRPC_URMA_USE_MOCK=true //test:brpc_unittests66+长度上限。已注册 body 直接引用原内存,不为拼接报文复制整个 payload。
67+未注册的片段不能直接作为 DMA 源。
70 68 
71-# 使用真实 liburma;要求链接器可找到 -lurma69+接收采用 header/body 两个 SGE。
72-bazel build --define=BRPC_WITH_URMA=true \
73- --define=BRPC_URMA_USE_MOCK=false //:brpc
74-```
75 70 
76-Bazel 会自动获取固定 commit 的 UMDK 头文件,并通过 `@umdk//:urma_headers`71+默认按 SocketId 哈希确定首选 JFS。首选容量不足时,从最多4个不重复备选中
77-提供给 URMA 目标。设置 `BRPC_DOWNLOAD_URMA_HEADERS=false` 后,Bazel 不再依赖72+选择在途请求最少者,一次最多尝试首选和一个备选。关闭 `urma_jfs_affinity`
78-该外部仓库;实际构建仍需通过工具链或本地 BUILD 目标声明73+时按批轮询首选。每个 JFS 的 MPSC 请求队列由单一执行者推进,单次最多
79-`urma_api.h`、`urma_types.h` 和 `urma_ubagg.h`。仅通过74+链式提交 `urma_io_tx_batch_max` 个 WR;仅收集已发布请求,不等待凑满。
80-`--copt=-I/path` 指向工作区外目录,在默认 Bazel 沙箱下不会把未声明的头文件
81-带入编译输入。设置
82-`BRPC_URMA_USE_MOCK=true` 强制加入 mock;设置为 `false` 则排除 mock 并链接
83-`liburma`。未指定时,URMA 构建默认使用 mock。
84 75 
85-## 使用76+请求发布到资源队列即视为接受,调用方移除对应的连续字节前缀。资源层持有
77+header、SGE、数据和导入资源,直到提交及 CQE 引用均释放。暂未成功 post
78+的后缀保留在队列中,由完成事件继续推进;若容量已在 post 返回前归还,
79+直接重试后缀。共享 SQ 的等待者登记在首选 JFS,按容量通知,退出时全部唤醒。
86 80 
87-通过在 channel / server 上设置 `socket_mode` 选择传输层:81+## 接收、重排与交付
88 82 
89-```cpp83+每个 JFR 预投递多个接收上下文。接收完成后检查 header 和数据长度,
90-// 客户端84+接收适配器使用目标 SocketId 查找逻辑连接,在池化连接资源中按分片序号
91-brpc::ChannelOptions opt;85+恢复有序字节流,并在交付锁内再次核对资源身份后进入 Endpoint 输入缓存。
92-opt.socket_mode = brpc::SOCKET_MODE_URMA;86+顺序到达的分片走快速路径,不插入重排 map;乱序分片暂存,缺口补齐后
93-opt.protocol = "baidu_std"; // URMA 仅支持 baidu_std87+连续交付。序号处理支持16位回绕,重复或非法分片会使连接失败。
94-brpc::Channel channel;
95-channel.Init("127.0.0.1:8003", &opt);
96 88 
97-// 服务端89+接收上下文及 header 可在补收时复用,已交给应用的 body 由 IOBuf 引用
98-brpc::ServerOptions sopt;90+保活。IOBuf 交接使用 movable 转移引用,避免按值浅拷贝带来的额外引用
99-sopt.socket_mode = brpc::SOCKET_MODE_URMA;91+维护;这不等同于复制 payload。
100-server.Start(port, &sopt);
101-```
102 92 
103-若对端不支持 URMA(例如 TCP 客户端连接 URMA 服务端),在 4 字节 magic93+默认按一次 CQ poll 的结果收集需要解析的连接,完成该批次的接收处理后,
104-握手后透明回退到 TCP,应用代码无需改动。94+每个连接调用一次消息解析入口。此批次按连接合并解析调用,不改变消息
95+协议边界或同一连接内的字节顺序。
105 96 
106-## 架构97+完成事件处理支持延后执行最后一条消息:先完成 CQ 事件交接并释放相关锁,
98+再允许整个交付范围内至多一条消息在当前 bthread 执行。其余消息通过
99+消息调度机制处理,避免业务回调持有共享队列锁。
107 100 
108-UrmaTransport 沿用与 `RdmaTransport` / `UBShmTransport` 一致的两层设计:101+## 并发与退出
109 102 
110-```103+JFS 的发送执行者串行提交 WR,JFR/JFC 使用各自的接收和完成执行者。
111-UrmaTransport : public Transport (urma_transport.{h,cpp})104+操作屏障协调 post、poll 与销毁;TX 使用原子登记数组,RX 使用固定上下文
112- +-- std::shared_ptr<TcpTransport> (回退路径)105+数组和 posted 标记。Socket 和资源池共同持有 slot 引用,防止排队的事件
113- +-- urma::UrmaEndpoint* (URMA 数据路径)106+回调访问已释放对象。逻辑连接仍使用重排锁和交付锁维护字节流顺序。
114- +-- UrmaState { URMA_ON, URMA_OFF, URMA_UNKNOWN }
115 107 
116-urma::UrmaEndpoint : public SocketUser (urma/urma_endpoint.{h,cpp})108+关闭时先停止事件与队列活动,唤醒发送等待者,并按资源依赖释放本地队列、
117- +-- UrmaResource { jfc, jfce, jfr, jetty, remote_jetty, remote_seg }109+完成上下文和导入资源。内存注销以 DMA 结束为前提,存储释放还需保证应用引用不再使用它;不能
118- +-- 握手状态机(C/S 对称,在 TCP fd 上驱动)110+为了回收对象而提前释放设备或应用仍可能访问的内存。
119- +-- 发送路径:urma_post_jetty_send_wr(URMA_OPC_SEND)
120- +-- 接收路径:urma_poll_jfc -> HandleCompletion -> InputMessenger
121- +-- 双窗口信用流控(_remote_rq_window_size / _sq_window_size)
122-```
123 111 
124-### 建链流程(双平面)112+## 构建与启用
125 113 
126-与 RDMA / UBRing 一致,控制面为 TCP,数据面为 URMA:114+在真实设备上启用该模块需要 UMDK 头文件和 `liburma`。CMake 使用
115+`WITH_URMA=ON`、`URMA_USE_MOCK=OFF`,
116+优先查找安装的 SDK;找不到头文件时可下载固定版本的 UMDK 头文件,
117+`DOWNLOAD_URMA_HEADERS=OFF` 可禁止下载。可通过 `URMA_ROOT`、
118+`URMA_INCLUDE_PATH`、`URMA_BOND_INCLUDE_PATH` 和 `URMA_LIB` 指定 SDK。
127 119 
128-1. TCP 连接建立。120+Make 使用 `config_brpc.sh` 的 `--with-urma --without-urma-mock`,
129-2. `UrmaConnect::StartConnect` 起客户端握手 bthread。121+并通过 `--headers`、`--libs` 提供依赖路径。Bazel 使用 `BRPC_WITH_URMA=true`、
130-3. 双方在 TCP fd 上交换 `UrmaHello` 消息(v2 二进制 magic `URMA`,122+`BRPC_URMA_USE_MOCK=false`,链接真实的 `liburma`;
131- v3 protobuf magic `URM3`),携带本地 EID、jetty id、recv buffer 数量、123+关闭 `BRPC_DOWNLOAD_URMA_HEADERS` 后,需要由工具链或本地依赖声明头文件。
132- 以及扁平化的 buffer 池 segment。
133-4. 双方先调用 `urma_import_seg` **再**调用 `urma_import_jetty`,为远端 EID
134- 建立传输路径(TP)路由。跳过 `import_seg` 会导致首个 SEND 被硬件以
135- `URMA_CR_RNR_RETRY_CNT_EXC_ERR` 拒绝。
136-5. 4 字节 ACK(`HELLO_ACK_URMA_OK = 0x1`)确认双方均要 URMA。
137-6. 成功后 TCP fd 仅保留用于 epoll 生命周期和回退,数据走 URMA。
138 124 
139-### 内存管理125+客户端和服务端分别通过 `ChannelOptions::use_urma` 与
126+`ServerOptions::use_urma` 启用传输。当前支持 `baidu_std` 协议,使用
127+非 bonding 设备及事件驱动模式;不支持忙轮询、端到端 credit/ACK 流控,
128+也不提供大包单边 READ/WRITE。大于单次 CTP 上限的数据继续分片 SEND。
140 129 
141-申请一大段 `mmap` 内存,用 `urma_register_seg` 一次性注册,再切成固定大小130+## 主要运行参数
142-buffer(默认 8KB)。劫持 `butil::iobuf::blockmem_allocate` 使每个 IOBuf
143-block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs 构建
144-`urma_sge_t`,无需逐消息注册(与 RDMA `block_pool` 设计一致)。用户注册
145-内存通过 `urma::RegisterMemoryForUrma` / `DeregisterMemoryForUrma` 支持。
146 131 
147-## 配置132+| 参数 | 默认值 | 含义 |
133+| --- | --- | --- |
134+| `urma_device` | 空 | 指定设备;空值选择枚举到的首个设备 |
135+| `urma_io_jfs_cnt` | 48 | 进程共享发送队列数量 |
136+| `urma_io_jfr_cnt` | 64 | 进程共享接收队列数量 |
137+| `urma_io_sq_depth` | 128 | 每个发送队列的在途槽位数 |
138+| `urma_io_rx_depth` | 512 | 每个接收队列的预投递深度 |
139+| `urma_jfs_affinity` | true | 按 SocketId 哈希选择固定首选 JFS |
140+| `urma_io_tx_batch_max` | 8 | 每批准备与链式提交的 WR 上限,范围1至32 |
141+| `urma_io_jfs_lock_free` | false | 设置 UMDK JFS 单执行者属性 |
142+| `urma_io_jfr_lock_free` | false | 设置 UMDK JFR 单执行者属性 |
143+| `urma_io_jfc_lock_free` | false | 设置 UMDK JFC 单执行者属性 |
144+| `urma_buffer_size` | 8192 | 数据池块大小,包含 IOBuf 块头 |
145+| `urma_ctp_buf_count` | 48000 | 4KiB CTP 块数量;必须大于接收预投递总数 |
146+| `urma_max_sge` | 0 | 0采用设备能力;正值限制发送 SGE 数 |
147+| `urma_cqe_poll_once` | 32 | 每次 poll 最多取出的 CQE 数 |
148+| `urma_rx_batch_parse` | true | 同一 poll 中按连接合并解析调用 |
149+| `urma_client_handshake_version` | 2 | 客户端握手编码,支持2或3 |
148 150 
149-所有 flag 使用 `urma_` 前缀(对标 RDMA 的 `rdma_` 前缀):151+队列数量、深度与内存池参数应在全局初始化前确定。接收预投递需要充足
150- 152+CTP 块,header 数量需要覆盖接收和发送在途槽位;初始化会校验组合是否
151-| Flag | 默认 | 用途 |153+可用。硬件能力同样约束队列深度、SGE 数和消息大小。
152-|------|------|------|
153-| `--urma_use_polling` | false | 轮询 JFC 而非事件模式 |
154-| `--urma_poller_num` | 1 | 每 bthread tag 的轮询器数(轮询模式) |
155-| `--urma_disable_bthread` | false | 内联处理消息(不起 bthread) |
156-| `--urma_sq_size` | 128 | 本地 JFS 深度 [16, 4096] |
157-| `--urma_rq_size` | 128 | 本地 JFR 深度 [16, 4096] |
158-| `--urma_cqe_poll_once` | 32 | 每次 `urma_poll_jfc` 的上限 |
159-| `--urma_recv_zerocopy` | true | 大于 `--urma_zerocopy_min_size` 的接收零拷贝 |
160-| `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 |
161-| `--urma_device` | "" | URMA 设备名(空=首个) |
162-| `--urma_max_sge` | 0 | 每 WR SGE 上限(0=设备上限) |
163-| `--urma_bonding_mode` | 0 | bonding 模式:0=standalone,1=active-backup,2=balance |
164-| `--urma_bonding_level` | 0 | bonding 层级:0=IODIE,1=port |
165-| `--urma_prepared_jetty_cnt` | 8 | 预连接 Jetty+CQ 请求数量;会根据 `RLIMIT_NOFILE` 自动限制 |
166-| `--urma_buffer_size` | 8192 | 池中每个 buffer 大小(字节) |
167-| `--urma_buffer_count` | 65536 | 池中 buffer 数量 |
168-| `--urma_poller_yield` | false | 忙轮询循环中主动让出 bthread |
169-| `--urma_client_handshake_version` | 2 | 客户端握手版本(2=二进制,3=protobuf) |
170- 
171-设备名以 `bonding` 开头时,brpc 会在创建 context 后、创建 segment 和队列
172-前配置 provider。默认 standalone+IODIE 配置与 UMDK 性能工具保持一致。
173-bonding 支持需要 provider 扩展头文件 `urma_ubagg.h`。
174- 
175-## 与 UBRing 协同
176- 
177-UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing
178-(`SOCKET_MODE_UBRING`)对**小包和同机 IPC**最优(亚微秒、零系统调用)。
179-混合负载可按服务选择传输层:
180- 
181-| 场景 | 建议 `socket_mode` |
182-|------|---------------------|
183-| 同机 IPC | `SOCKET_MODE_UBRING` |
184-| 跨节点小包(< 64KB) | `SOCKET_MODE_UBRING`(UBS-Mem)或 `SOCKET_MODE_URMA` |
185-| 跨节点大包(>= 64KB) | `SOCKET_MODE_URMA` |
186-| 传统 RoCE/IB 数据中心 | `SOCKET_MODE_URMA` 或 `SOCKET_MODE_RDMA` |
187- 
188-单连接内按包大小自动分流的方案(路线 C 方案 B)作为后续演进方向。
189- 
190-## 限制
191- 
192-- 仅支持 `baidu_std` 协议(与 RDMA 一致)。SSL、RTMP、NSHEAD、MONGO 在
193- `ContextInitOrDie` 阶段拒绝。
194-- 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。
195-- 当前实现面向 Linux。
@@ -0,0 +1,120 @@
1+# URMA 连接资源抽象
2+ 
3+`UrmaEndpoint` 负责 TCP 协商、逻辑字节流和消息解析,`UrmaConnectionResource` 负责连接资源,`UrmaResourceProvider` 负责资源创建及退出。`--urma_resource_mode=exclusive` 为默认模式,每连接独占 Jetty/JFR/JFC/JFCE;`--urma_resource_mode=pooled` 将同一进程中的连接映射到共享 JFS/JFR。两种后端共用 Endpoint 和握手状态机,独占路径不构造池化交付上下文。模式在启动时选择,不支持运行期间切换。
4+ 
5+## 职责
6+ 
7+| 层次 | 职责 |
8+|---|---|
9+| `UrmaEndpoint` | TCP 握手与原有失败/回退路径、连接状态、发送转交、输入缓存、协议解析和消息交付 |
10+| `UrmaConnectionResource` | 连接资源描述、对端绑定、接受并消费发送前缀、可写等待、关闭和资源诊断 |
11+| `UrmaExclusiveConnection` | 原 SEND/RECV、SGE、环形 buffer、双 credit 窗口、纯 ACK、CQ 处理和接收预投 |
12+| `UrmaExclusiveResourceProvider` | 独占资源创建、预创建 Jetty 集合、按 bthread tag 管理 polling、停止与回收 |
13+| `urma_helper` | 公共设备/context、注册内存与 IOBuf 分配服务 |
14+ 
15+公共接口定义在 `src/brpc/urma/urma_resource.h`,不暴露 native URMA 结构。独占实现及其私有状态位于 `urma_exclusive_resource.cpp`。以后接入另一种资源实现时,应实现相同接口及回调契约,而不是复制 Endpoint 的握手和输入处理代码。
16+ 
17+以下“发送与等待”至“构建与验证”说明独占后端的保留语义;共享队列的差异见“池化后端”。
18+ 
19+## 发送与等待
20+ 
21+`Send(IOBuf**, size_t)` 直接消费已接受的连续字节前缀,返回该前缀的长度。构造 SGE 时通过 `cutn` 将数据引用交给 SQ 槽位,保留到发送完成;Endpoint 仅转交调用,不再执行独立的 `pop_front`。调用方串行执行同一连接的数据发送,并在整个调用期间持有资源句柄。
22+ 
23+每个 WR 保留有界的输入切片记录。构造或提交失败时,仅将当前未提交 WR 的切片恢复到原 IOBuf 的前端;已成功提交的前缀保持消费,未接受的后缀及用户注册内存元数据保持不变。后续 WR 失败时仍返回前面已接受的正长度,错误通过携带来源资源的回调上报。
24+ 
25+空输入返回 0;非空输入在无容量时返回 `-1/EAGAIN`。已关闭或未绑定资源不能报告可写。发送仍逐 WR 提交,不引入请求队列、链式提交、凑批或 payload 复制。
26+ 
27+原本地 SQ 窗口、远端接收 credit 和三个纯 ACK 预留 WR 保留。可写等待仅用于绑定成功后的发送背压,并且只接受所属 Socket 的 `_epollout_butex`。等待前读取 butex 计数,再检查可写条件;资源层包装 `butex_wait`,不另建等待者集合。TX 完成、接收 credit、Close 通过 `Socket::WakeAsEpollOut` 先递增计数再唤醒,避免通知先于等待时丢失。`BindPeer` 不触发可写通知,但仍刷新提前接收积累的信用。SQ 槽位先释放数据引用,再公布可用容量。
28+ 
29+数据 SEND 与 CQ 路径的 ACK 不获取资源层发布锁,底层 provider 必须支持同一 Jetty 的并发提交。纯 ACK 预算通过原子 CAS 预留,避免绑定后的初始信用刷新与 CQ 并发时超额使用;这部分软件状态不依赖 provider 内部锁。初始化、绑定与关闭之间仍使用仅用于管理路径的生命周期锁。
30+ 
31+## 接收与交付
32+ 
33+原 CQ drain、ACK、rearm、最终 drain、解析的顺序保留。单次 drain 内聚合连续字节,通过一次资源回调交给 Endpoint,使用 `IOBuf::movable()` 交接所有权。接收复制阈值、零拷贝补收和 peer credit 格式不变。
34+ 
35+CQ 入口持有资源句柄,直接调用 `DrainCq`;不经交付策略回调往返,也不构造空的交付上下文。独占模式仍使用原来的 parser-local `last_msg` 和 `UrmaTransport::QueueMessage`;event、polling、`urma_disable_bthread` 及 coroutine 选项的调度语义不变。池化模式的 deferred 尾消息规则不影响此路径。
36+ 
37+资源句柄在本地 Hello 发出之前发布。握手期到达的数据可以进入 Endpoint 输入缓存,但只有状态达到 ESTABLISHED 才解析;握手完成主动推进 pending 字节,不依赖下一次 CQE。
38+ 
39+## 握手与生命周期
40+ 
41+v2 二进制 Hello、v3 protobuf、Jetty 标识和接收配额编码保持原格式。角色和执行属性通过 `UrmaConnectionOptions` 传入资源层:
42+ 
43+- 客户端、非 bonding 服务端在创建资源时预投接收。
44+- bonding 服务端在导入完成后预投,再由 Endpoint 发出 Hello。
45+- 对端 segment 导入始终先于 Jetty 导入。
46+ 
47+Endpoint、已进入的调用和 CQ 执行者持有资源句柄。CQ carrier 仅持有弱引用,不长期引用 Endpoint。数据和错误回调携带来源资源,Endpoint 核对其仍为当前实例后处理;Reset 摘下旧句柄后在锁外 Close。旧实例的数据和错误回调不能影响 Reset 后的新实例。
48+ 
49+资源句柄仅延长连接资源及其硬件的生命周期,并不延长 Socket、Endpoint 的生命周期。资源只保存带版本的 `SocketId`;可写通知时用 `Socket::Address` 临时取得 `SocketUniquePtr`,成功后再访问 butex,失效则忽略通知。等待期间同样持有临时 Socket 引用。资源不长期保存 Socket/butex 裸指针,也不持有指向 Socket 的永久强引用,避免迟到通知访问回收后复用的对象,以及 Socket→Endpoint→resource→Socket 的引用环。Socket 失败路径仍负责唤醒自身等待者。
50+ 
51+Close 幂等,发布关闭状态、移除事件入口并唤醒等待者。随后进入的调用拒绝发送;已通过关闭检查的 SEND/ACK 可以完成当前投递。Close 不等待正在执行的发送或自身回调,实际硬件及数据引用的回收在执行中的句柄退出后进行。硬件销毁失败时同时保留硬件与数据引用,等待 provider 后续 Shutdown 重试。
52+ 
53+polling 的初始化、停止和 Shutdown 属于控制路径;在本 provider 的 poller 内调用停止/初始化会被拒绝,避免等待自身。停止与启动串行执行,Shutdown 先禁止新建连接,再在创建锁之外等待 poller 退出。
54+ 
55+绑定的参数校验失败可更正描述后重试;开始导入或预投后的失败会关闭该资源实例,不能在半初始化资源上重新绑定。
56+ 
57+`Shutdown()` 返回 false 时,拥有者必须保留 provider 及公共设备/内存依赖,禁止新连接;返回 true 后才可以销毁 provider。公共注册内存还需检查在用块及注销结果。沿用原进程退出策略,不注册可能早于 IOBuf TLS 析构的 atexit 回收函数。
58+ 
59+## 构建与验证
60+ 
61+沿用现有 `WITH_URMA`、`URMA_USE_MOCK` 和 SDK 路径配置,不新增必选分配器。`WITH_URMA=OFF` 不启用该实现。
62+ 
63+`test/brpc_urma_unittest.cpp` 覆盖原握手/SDK mock,以及资源描述、发送输入与前缀所有权、可写等待/关闭、提前接收缓存、Reset 来源校验和多连接 v2/v3 RPC。资源集成测试使用 link-time mock;应分别以 event 和 `--urma_use_polling=true` 运行。真实 provider 还需进行跨进程收发验证,mock 不能替代 DMA、硬件销毁和 bonding 的实机验证。
64+ 
65+本框架尚未接入 RDMA 或单边读写。池化后端不支持 polling 和 bonding;独占模式保留这两条已有路径。
66+ 
67+ 
68+## 池化后端
69+ 
70+`UrmaPooledResourceProvider` 拥有共享队列及 CQ 入口。每个 `UrmaPooledConnection` 保存所属 JFR、对端 JFR/segment 引用、对端 SocketId、发送序号及首选 JFS。连接创建登记使用哈希表,避免每创建一个连接扫描全部存活连接。逻辑连接的 Close 只禁止本连接的新操作并唤醒其 Socket;共享队列由 provider 在所有连接句柄退出后关闭。
71+ 
72+发送首选 JFS 在连接创建时按 SocketId 哈希确定。首选放不下时,从最多4个不重复备选中选择 inflight 最小者;大于5个 JFS 时使用 pthread TLS xoshiro256++ 抽样。最多尝试首选和一个备选,按可用容量接受前缀,不扫描全池。一次 SEND 采用一个 64 B header SGE 加多个原始 IOBuf SGE,累计正文达到 4032 B 或 SGE 上限即提交。发布到 JFS 请求队列即表示接受,对应连续前缀从输入移除。每批请求持有 WR、SGE、数据及导入资源;MPSC owner 负责链式 post。已接受但暂未 post 的后缀由资源层继续推进,不让调用者重复提交。TX 上下文仅持有轻量导入资源;来源连接弱引用在绑定时存入该每连接资源,避免逐 WR 重复维护引用和扩大上下文。迟到错误通过导入资源取得来源,且仅在来源仍是 Socket 当前资源时生效。
73+ 
74+共享 SQ 等待在固定首选 JFS 登记:先登记,再检查容量;首选的 CQE/owner 推进唤醒其等待者,不因其他备选队列完成而全池唤醒。登记和取消使用每 JFS 的等待者锁,正常 post/poll 不使用该锁。登记节点仅在 WaitWritable 调用期间存在,并持有所属 Socket 的临时引用。独占模式继续直接使用所属 Socket 的 butex,不引入共享等待者表。
75+ 
76+共享 CQ 入口临时持有 provider 状态,原子 admission 状态同时记录活动回调数和停止标志。Shutdown 先禁止新回调进入,已有回调未退出时返回 false,不持有关闭锁等待 CQ;回调引用在 slot 操作屏障退出及尾消息处理之后释放,避免执行中的 CQ 资源被析构。
77+ 
78+共享 JFR 先检查 CQE、取出 header 和正文并补收,再交给 `UrmaReceiveAdapter`。
79+适配器通过 SocketId 查找一次 Socket,取得当前池化资源句柄,在资源的重排锁内
80+校验 JFR 和序号并组装连续字节。交付时获取 Endpoint 交付锁,重新核对来源仍为
81+当前资源,再以 `IOBuf::movable()` 进入输入流。Reset 可以替换资源,旧回调因
82+身份不匹配而被丢弃;Close 和被替换资源的析构在 Endpoint 锁外执行。
83+一次 poll 最多处理 `BRPC_URMA_MAX_CQE_PER_POLL` 条完成,批次至多持有同样
84+数量的 Socket 引用;逐片追加后每个连接解析一次,不等待额外 CQE。
85+ 
86+`UrmaDeferredMessage` 的生命周期跨越整个共享 CQ 回调,只保留最新的一条 parser-last 消息;被替换的候选正常入队。解析通过 `InputMessageClosure` 返回尾消息,适配器在重排和交付锁外更新 deferred;最终消息只在 slot 操作屏障退出、事件所有权交还之后执行,并使用所属连接的 bthread 本地存储。错误退出不在持锁或事件尚未释放时执行业务。独占 CQ 仍直接调用 `on_received`,没有空的 batch 回调或资源层往返。
87+ 
88+池化 v2 使用 implementation=2、94 B 报文,独占 v2 使用 implementation=1、86 B 报文。读取固定前缀后再决定剩余长度,避免混合模式等待不存在的扩展字段。v3 保持字段号12的 SocketId 和字段号13的 uint32 资源模式,字段13名称为 resource_mode,默认1代表独占、2代表池化。模式不匹配时通过完整拒绝 Hello 或 ACK=0 回退 TCP;非法报文与 I/O 错误仍按错误路径处理。此协商不表示任意历史池化版本均支持 TCP 回退。
89+ 
90+池化内存使用独立的 8 KiB IOBuf、4 KiB CTP 和 64 B header 池;原始已注册 IOBuf 多 SGE 发送保持零拷贝。独占模式保留原有单块大小配置及分配器行为,默认内存和资源参数不因选择框架而改变。池化数据块数量默认 24000,独占默认 65536;显式设置 `urma_buffer_count` 时使用用户值。
91+ 
92+`test/brpc_urma_pooled_unittest.cpp` 单独运行池化用例,避免同一进程重复替换 IOBuf 全局分配器。覆盖池资源独立性、分片及回绕、共享 SQ 等待、JFS affinity、并发握手、延后尾消息和 RX batch 边界;非池化用例保持独立。
93+ 
94+### 池化连接代次限制
95+ 
96+池化线格式目前只携带目标 SocketId 和 16 位分片序号。健康检查复活可能保留 SocketId,而共享 JFR 在逻辑连接重置后仍存在;同一 JFR 上迟到的旧接收分片缺少额外连接代次标识。这是已有池化协议的限制,不能将独占回调的来源句柄校验等同于池化网络接收的代次隔离。解决需要扩展握手及 header 代次,或证明重连前的接收已静默;本轮保留原池化线格式,不宣称已提供该保证。TX 错误回调仍使用来源资源弱引用校验。
97+ 
98+## JFS/JFR slot 去锁迁移
99+ 
100+从 jfs_发送优化/brpc_bk 的 dfad2f8e 之后迁移共享发送 owner、分流和固定context方案,范围止于1935f1a9。未迁移后续接收重排/解析去锁;重排状态移入池化连接资源后仍受锁保护。独占后端保留原调度语义,握手新增资源模式不匹配时的 TCP 回退。独立8K/4K CTP/64B注册池保留,未混入旧仓库的单池分配器或额外TLS修改。
101+ 
102+- 每 JFS 使用 exchange+link MPSC 请求链及 RUNNING/KICK owner 状态,发送者与 TX CQE 都可以推进,无额外常驻发送 bthread。
103+- SQ 区分已接受请求量和硬件在途量。并发选路可能少量超出软件预算,由 owner 按硬件容量推进。
104+- TX context/批次使用 ObjectPool。context 的提交引用与 CQE 引用协调,覆盖 CQE 早于 post 返回;已提交context通过原子固定登记数组回收。
105+- RX context 使用固定数组及 posted 标记,由当前 JFR 单执行者修改,移除 pending set 和 slot mutex。
106+- 池和 slot 使用操作屏障协调 shutdown,停止接纳并等待活跃操作退出后,按硬件删除成功的边界回收DMA数据。
107+- 保留抽象版来源资源弱引用校验,晚到 TX 错误不影响替换后的资源实例。
108+- 暂时 post 失败且仍有在途 WR 时保留后缀,后续 CQE 重试;已观察到已有 WR 或本次前缀提前完成时立即重试。没有容量进展且无未来完成可推进时上报连接错误,避免永久挂起。组装提交链始终停在首个失败上下文,保证提交与出队对应同一个连续前缀。
109+ 
110+新增运行参数:
111+ 
112+| 参数 | 默认 | 含义 |
113+|---|---:|---|
114+| urma_jfs_affinity | true | 固定哈希首选;关闭时按批轮询首选 |
115+| urma_io_tx_batch_max | 8 | 每次准备/链式post的WR上限,1至32,不等待凑满 |
116+| urma_io_jfs_lock_free | false | 向UMDK声明JFS由单owner提交 |
117+| urma_io_jfr_lock_free | false | 向UMDK声明JFR由单接收执行者补投 |
118+| urma_io_jfc_lock_free | false | 向UMDK声明JFC由单执行者poll |
119+ 
120+上述UMDK lock_free参数仅设置池化队列的创建属性;Endpoint锁与独占资源配置不受影响。
@@ -0,0 +1,23 @@
1+# 池化连接接收重排职责
2+ 
3+重排状态归属 `UrmaPooledConnection`,每条逻辑连接独立维护 `_deliver_base`、`_reorder_queue` 和重排锁,不另建 SocketId 到接收流的全局 map。
4+ 
5+接收路径:共享 JFR CQE → 读取目标 SocketId / seq → UrmaReceiveAdapter::OnFragment 只查找一次 Socket 并临时持有引用 → 取得当前池化连接 → AssembleFragmentLocked 校验和合并连续前缀 → 复用 Socket 引用交付 Endpoint → BRPC 解析。
6+ 
7+Endpoint 不再包含分片序号、重排队列或 ReorderAndDispatch。池化通过 on_fragment 进入适配器;独占通过 on_received 进入同一适配器并只查找一次 Socket。Endpoint 不引用 UrmaDeferredMessage / UrmaRxParseBatch;这两个上下文、批次 Socket 引用和延迟分发均由适配器管理。Endpoint 通过函数参数返回 InputMessageClosure,不保存延迟输出槽或 CQ 批次。Endpoint 在分发锁内检查关闭标记、连接状态和 Socket 有效性。池化资源在一次连接生命周期内保持绑定不变;独占回调仍比较传入资源身份。
8+ 
9+同一连接的重排锁覆盖片段组装至有序交付,防止并发回调再次打乱字节流。Close 只标记关闭,不同步等待该锁;否则可能与 Endpoint 分发锁形成锁顺序冲突。关闭后新片段被拒绝,已有回调通过 Socket 引用保活绑定资源,缓存随连接资源最终销毁释放。只有旧访问者全部退出后的健康检查 Reset 才能清空绑定并建立新连接,不复用旧 seq 状态。详见 [稳定资源绑定约束](stable_connection_binding.md)。
10+ 
11+序号保持 16 位及前向半区间规则,按期望 seq 查找缓存以支持回绕。重复或过期 seq 通过资源错误回调报告 EPROTO。CQ 每轮批量解析和延迟业务分发继续保留,纯乱序缓存不会触发无数据的解析。
12+ 
13+验证:CMake 动态链接真实 liburma、tcmalloc 构建成功。池化 mock 70 项通过;独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过。145 服务端 / 151 客户端,NUMA1 CPU 96–111,udmac0d1e2,event / multi SGE,4 连接、QD10、目标 500 QPS,1K / 100K 双边收发各 60 个请求零错误。短迭代统计用于功能验证,不用于时延结论。命令和日志位于项目根目录 performance_results/20261114_池化重排下沉。
14+ 
15+ 
16+接收路由查找次数:每条池化 CQE 进入 OnFragment 时一次;后续 HandleFragment、重排合并、Deliver 和批次 Flush 均复用已有引用。乱序片段暂存不产生第二次查找。非池化 OnReceived 一次。这里指消息所属逻辑 Socket 的查找,不含事件分发器对 JFR 载体 Socket 的路由。池化连接不保存 Socket 引用,也不调用 Endpoint。适配器持有重排锁直到本次有序数据交付完成,避免两个并发执行者在返回连续数据后再次颠倒交付顺序。
17+ 
18+ReceiveAdapter 版本验证:池化 71 项通过(增加禁止 RX 返回旧 lookup_connection 回调的测试),独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过;145/151 真实设备的 1K / 100K 双边收发各 60 请求零错误。完整命令与日志:performance_results/20261114_ReceiveAdapter验证。
19+ 
20+ 
21+最后消息的所有权:ProcessNewMessage 保持原有实现;URMA event 模式的 QueueMessage(last=true) 保留调用者的最后消息(池化 usercode_in_pthread 模式仍按原规则排队,polling 仍全部排队)。适配器以局部 InputMessageClosure 调用 Endpoint 解析,解析结束并释放分发/重排锁后,将最后消息接入 CQ 回调的延迟上下文;若替换了旧候选,就将旧候选通过其原 Socket Transport 后台入队并 flush。CQ 回调交还事件权且操作保护结束后才执行最终候选。Endpoint 和 Transport 均无持久的 deferred_slot 字段。握手和独占路径的局部 Closure 声明在分发锁之前,保证析构执行发生在解锁后。
22+ 
23+Closure 返回版本验证:池化 71 项通过;独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过;145/151、NUMA1 CPU96–111、udmac0d1e2,1K / 100K、4 连接、QD10 双边收发各 60 请求零错误。命令及日志位于 performance_results/20261114_Closure返回验证。ProcessNewMessage 的 cpp/h 相对基线没有实质修改。
@@ -0,0 +1,36 @@
1+# 抽象版池化 JFS/JFR 去锁迁移验证
2+ 
3+目标基线:3ad1ae2520201a57f131718df5d0de1b0519afd8。
4+ 
5+源方案:jfs_发送优化/brpc_bk,dfad2f8e为验证基线,后续到1935f1a9为本轮参考范围;不迁移9373647b之后的Endpoint重排/解析去锁。
6+ 
7+实现范围为共享队列MPSC owner、最大32 WR链式post、SocketId哈希首选、最多4个备选抽样、首选JFS等待、TX引用及原子登记数组、RX固定数组和slot操作屏障。保留本仓库独立CTP内存池、来源资源校验、Endpoint两把锁和独占后端。
8+ 
9+构建使用145 AArch64、GCC12.3.1、CMake Release、动态libbrpc.so、真实URMA SDK、tcmalloc、CPU frequency计时,无打点;原包保存在package/bin,新包保存在unlock-package/bin。Windows源码修改后同步Linux编译,未在Linux更改源码。
10+ 
11+单元测试采用链接期mock(测试可执行文件编译mock_urma.cpp并导出符号,动态库仍为同一个待验证libbrpc.so),用于验证队列及所有权,不替代真实硬件测试。
12+ 
13+已通过池化70项单元测试,包括8项新增的链式批次/owner/关闭回收测试;独占event模式40项通过、2项polling专用跳过。7项关键并发测试重复100轮全部通过;开启三项SDK lock_free后池化70项全部通过;独占polling模式41项通过、1项多tag条件用例跳过。真实硬件三轮交替对照结果在后续验证记录中补充。
14+ 
15+原测试中“post失败后缀仍归调用者”的断言已按新队列语义调整:进入资源队列即为接受,当前未能post的后缀仍由资源层持有并由后续CQE推进,不重复消耗或重复发送。
16+ 
17+## 145/151 三轮交替性能对照
18+ 
19+条件:NUMA1 CPU96-111、udmac0d1e2、multi/event、4 JFS/4 JFR、单连接QD10、目标500QPS、每组20秒;请求响应等长,max_random_delay_us不设置(默认5000)。两版同CMake动态链接构建。修改后显式开启3项UMDK lock_free,batch_max=32。
20+ 
21+| 消息大小 | 修改前报告均值三轮平均 µs | 修改后报告均值三轮平均 µs | 差值 µs |
22+|---|---:|---:|---:|
23+| 1K | 31.00 | 32.33 | +1.33 |
24+| 4K | 35.00 | 35.33 | +0.33 |
25+| 64K | 82.00 | 89.33 | +7.33 |
26+| 100K | 107.67 | 120.33 | +12.67 |
27+ 
28+24组均无错误,实测约500QPS。未观测到低负载时延收益,大包存在可重复增加。上述数值是三次程序报告的简单平均;时延沿用默认10秒窗口,不能视为合并样本统计。组合测试同时包含owner/context/选路和UMDK选项,不能单独归因某把锁。
29+ 
30+batch_max=1的单轮补测为64K 106µs、100K 155µs,反而更慢。因而不能简单把默认32的劣化归因为准备批次延后首个post;频繁进入提交/owner/context管理也可能增加成本,需进一步打点才能归因。默认配置未据此改动。
31+ 
32+补测batch_max=8时,64K均值81µs/P99 109µs,100K均值108µs/P99 154µs,均无错误。接近基线均值,但仅各一轮,不宣称稳定收益;默认batch上限仍32。
33+ 
34+## batch8完整一轮复测
35+ 
36+相同设备、绑核、QD/QPS和20秒时长,1K/4K/64K/100K均值分别31/35/82/109µs,P99分别42/45/119/155µs,40028次完成请求、错误0。接近修改前31/35/82/107.67µs的三轮平均。只以运行参数设为8,代码默认32未修改;不宣称单轮差异代表稳定提升。
@@ -0,0 +1,34 @@
1+# 资源模式与跨模式 TCP 回退
2+ 
3+v2 保留 hello_ver=2 和二进制 impl_ver:1 为独占(86B),2 为池化(94B)。v3 将 protobuf 字段 13 改名为 resource_mode,仍为 uint32:1 为独占、2 为池化,默认 1。字段编号和已有数值保持不变;非池化发送时仍省略此字段和 socket_id,保留旧非池化 Hello 格式。池化发送 resource_mode=2 和 socket_id。resource_mode 表达硬件资源拓扑,不表达 protobuf 格式版本。
4+ 
5+解析器完整消费 Hello 后返回对端描述,由握手状态机判断模式是否一致。v2 对合法长度的未知版本/实现号也会消费完整 Hello;非法长度仍报协议错误。v3 对未知模式及缺少必要 socket_id 的报文仍报错。正常模式一致时沿用原有资源导入和 ACK。
6+ 
7+跨模式流程:
8+ 
9+1. 客户端发送其本地模式 Hello。
10+2. 服务端读完 Hello,发现资源模式不匹配,不创建/导入对端连接资源。
11+3. 服务端发送同一握手格式的完整拒绝 Hello:Jetty ID、buffer 等资源信息为 0。
12+4. 此拒绝 Hello 不等待 ACK。服务端转入 TCP,客户端完整读取拒绝响应后也转入 TCP。
13+5. 双方继续使用原 TCP 连接发送 BRPC 数据,关闭本地 URMA 连接资源。池化进程的全局资源池仍保留。
14+ 
15+客户端若收到有效但模式不同的资源描述,则先发送 ACK=0 通知等待 ACK 的对端,再转入 TCP。以前的池化客户端禁止 TCP 回退分支已移除。I/O 错误、非法资源描述等错误仍按其原有错误路径处理,不能把残留握手数据直接交给业务解析。
16+ 
17+ub_test 的 SGE 校验新增独占资源分支:独占模式没有 g_urma_pool,不能用池化 JFS 是否存在作为启动条件。
18+ 
19+## 验证
20+ 
21+CMake 动态 libbrpc.so + libtcmalloc,真实 liburma 构建。池化回归 72 项通过;独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过。新增测试确认 v2 和 v3 解析对端模式之后不会消费后面的 NEXT 字节。
22+ 
23+真实设备:145 服务端 / 151 客户端,NUMA1 CPU96–111、udmac0d1e2,单连接 QD10,目标500QPS,50 次迭代(含收尾实际60请求)。以下每项均测试1K与100K,零错误,并在服务端日志确认 resource mode mismatch / falling back to TCP:
24+ 
25+| Hello | 客户端 | 服务端 | 结果 |
26+|---|---|---|---|
27+| v2 | 独占 | 池化 | TCP 收发成功 |
28+| v2 | 池化 | 独占 | TCP 收发成功 |
29+| v3 | 独占 | 池化 | TCP 收发成功 |
30+| v3 | 池化 | 独占 | TCP 收发成功 |
31+ 
32+结果、逐项命令及日志:项目根目录 performance_results/20261114_跨模式TCP回退 下的 *_r2 目录。第一次尝试被 ub_test 原有的仅池化启动检查阻止,没有发出 Hello,单独保留为排查记录。
33+ 
34+以上跨模式实测针对两端均使用本次代码的版本;没有据此声称任意历史池化版本都支持回退。
@@ -0,0 +1,42 @@
1+# URMA 连接资源稳定绑定与无资源锁访问
2+ 
3+## 生命周期约束
4+ 
5+每次有效连接生命周期内,Endpoint 的连接资源只发布一次。发布后不清空、不替换,连接失败和 TCP 回退仅执行逻辑关闭。资源对象由 Endpoint 持有,通常在 Socket 最终回收、销毁 Endpoint 时释放。
6+ 
7+健康检查恢复是明确的生命周期边界:`Socket::Revive()` 先等待引用数达到预期值,退出旧连接的访问者,再调用 `UrmaTransport::Reset()`。Reset 可释放旧绑定并允许下一次连接重新发布资源。不能在接收、发送、调试查询或握手线程仍访问旧绑定时并发调用 Reset;不能把任意时刻的 Reset 当作关闭接口。该前提也适用于直接调用 Endpoint 的测试和内部调用者。
8+ 
9+Socket 引用保证 Endpoint 和绑定资源在处理期间有效。资源后端不长期持有 Socket 引用。资源指针不能被保存到超出 Socket 保活范围的后台任务中;这类异步任务需保留独立的资源所有权。
10+ 
11+## 发布与读取
12+ 
13+- `_connection_resource` 是拥有资源的 shared_ptr,只在发布前写入;稳定期读操作不与写操作并发。
14+- `_published_resource` 是非拥有型原子指针。所有初始化完成后 release-store 发布,访问者 acquire-load 后借用。它不承担资源回收,也不通过原子 shared_ptr 隐藏锁。
15+- `BorrowConnectionResource()` 不获取 mutex、不增加共享引用计数。正常收发和接收适配器使用该入口。
16+- `GetConnectionResource()` 保留给需要独立所有权的接口:先 acquire 观察发布,再复制稳定的 owner。复制本身仍会更新 shared_ptr 引用计数。
17+- 资源发布与逻辑关闭由已有 `_dispatch_mutex` 串行;关闭标记阻止关闭后再发布。重复分配返回 EALREADY,关闭后分配返回 ECANCELED,不能替换原绑定。
18+ 
19+## 接收路径
20+ 
21+```text
22+Socket::Address(一次,临时保活)
23+ → acquire-load 已发布资源,借用池化连接
24+ → 重排锁:检查 JFR/seq,整理连续字节
25+ → 解析锁:检查连接状态,拼入输入流/登记批量解析
26+```
27+ 
28+正常连续交付分片由四次锁获取减少到两次。不再获取 `_resource_mutex`,不再复制资源 shared_ptr,也不再校验资源是否被替换。重排锁、解析锁继续保留;批量解析仍按批次获取解析锁。类型转换仍使用 dynamic_cast,不将类型安全与本次生命周期优化混在一起。
29+ 
30+`Close()` 标记资源关闭;`DeallocateResources()` 在解析锁内标记 Endpoint 逻辑关闭,之后在锁外调用 Close,保留绑定。晚到片段由资源关闭状态拒绝,交付和解析还会检查 Endpoint 关闭状态。关闭不等待重排锁,避免与接收路径形成反向锁顺序。
31+ 
32+## 在途 WR 与回收
33+ 
34+Socket 保活只覆盖当前处理过程,不能代替已提交 WR 的 DMA 所有权。TX context 中的 IOBuf、导入资源引用和完成回收逻辑继续保留,直到设备完成或原有安全退出流程允许回收。
35+ 
36+池化连接资源对象延迟释放意味着连接失败/TCP 回退后,导入引用和乱序 IOBuf 可能保留至最终回收或恢复 Reset。共享 JFS/JFR 的生命周期仍由进程级资源池管理。非池化 Close 仍按原实现停止 CQ 事件入口,硬件对象最终析构随资源所有权释放;本次不改变硬件清理的安全要求。
37+ 
38+## 验证
39+ 
40+新增/调整测试覆盖逻辑关闭后绑定保留、重复关闭、晚到 RX 拒绝、关闭与阻塞在重排锁的接收回调并发、回调退出后 Reset 回收,以及原有在途 TX 导入资源保活。CMake Release 动态链接真实 liburma/tcmalloc 构建通过;池化软件 provider 73 项通过,非池化 event 40 项通过/2 项配置相关跳过,polling 41 项通过/1 项配置相关跳过。回归日志和真实设备性能命令位于项目根目录 `performance_results/20261116_资源绑定去锁对照/`。
41+ 
42+真实设备交错两轮测试:QD10、500QPS、20秒,1K 平均时延由34.0变为33.5µs,P99两轮均值48.0变为53.0µs;100K平均由118.5降为111.5µs(减少7µs,约5.9%),P99两轮均值168.5降为159.5µs。该结果为本次组合修改收益,不代表单次锁操作耗时。所有真实设备测试零错误,命令和完整分布见上述性能结果目录。
@@ -0,0 +1,64 @@
1+# Licensed to the Apache Software Foundation (ASF) under one
2+# or more contributor license agreements. See the NOTICE file
3+# distributed with this work for additional information
4+# regarding copyright ownership. The ASF licenses this file
5+# to you under the Apache License, Version 2.0 (the
6+# "License"); you may not use this file except in compliance
7+# with the License. You may obtain a copy of the License at
8+#
9+# http://www.apache.org/licenses/LICENSE-2.0
10+#
11+# Unless required by applicable law or agreed to in writing,
12+# software distributed under the License is distributed on an
13+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+# KIND, either express or implied. See the License for the
15+# specific language governing permissions and limitations
16+# under the License.
17+ 
18+cmake_minimum_required(VERSION 3.16...3.28)
19+project(ub_test C CXX)
20+ 
21+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
22+ 
23+option(LINK_SO "Whether examples are linked dynamically" OFF)
24+ 
25+brpc_example_find_common_deps(DYNAMIC_LIB)
26+option(WITH_TCMALLOC "Link tcmalloc" OFF)
27+if(WITH_TCMALLOC)
28+ find_library(TCMALLOC_LIB NAMES tcmalloc REQUIRED)
29+ list(APPEND DYNAMIC_LIB ${TCMALLOC_LIB})
30+endif()
31+ 
32+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto)
33+set(BRPC_EXAMPLE_WITH_URMA ON)
34+find_path(URMA_INCLUDE_PATH NAMES urma_api.h
35+ HINTS ENV URMA_ROOT
36+ PATHS /usr/include /usr/local/include
37+ PATH_SUFFIXES ub/umdk/urma umdk/urma urma
38+ src/urma/lib/urma/core/include)
39+if(NOT URMA_INCLUDE_PATH)
40+ message(FATAL_ERROR
41+ "Fail to find urma_api.h. Install UMDK headers or set URMA_ROOT.")
42+endif()
43+find_library(URMA_LIB NAMES urma
44+ HINTS ENV URMA_ROOT
45+ PATH_SUFFIXES lib lib64)
46+if(URMA_LIB)
47+ list(APPEND DYNAMIC_LIB ${URMA_LIB})
48+else()
49+ message(STATUS
50+ "liburma not found; using the URMA implementation linked into brpc")
51+endif()
52+ 
53+# Match the original matrix's BUTIL_USE_CPU_FREQUENCY=true build.
54+option(WITH_CPU_FREQUENCY "Use CPU frequency for benchmark timing" ON)
55+foreach(role client server)
56+ add_executable(ub_test_${role} ${role}.cpp ${PROTO_SRC} ${PROTO_HEADER})
57+ brpc_example_configure_target(ub_test_${role})
58+ target_compile_definitions(ub_test_${role} PRIVATE
59+ BUTIL_USE_CPU_FREQUENCY=$<BOOL:${WITH_CPU_FREQUENCY}>)
60+ target_link_libraries(ub_test_${role} PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
61+ set_target_properties(ub_test_${role} PROPERTIES
62+ BUILD_WITH_INSTALL_RPATH TRUE
63+ INSTALL_RPATH "$ORIGIN;$ORIGIN/../common")
64+endforeach()
@@ -0,0 +1,137 @@
1+#!/usr/bin/env python3
2+"""Build pooled multi-SGE URMA and a relocatable binary bundle.
3+ 
4+Requires Linux, CMake, a C++ compiler, Protobuf and UMDK. gperftools is optional.
5+The transport sources are used unchanged. Run on the destination architecture.
6+"""
7+ 
8+import argparse
9+import hashlib
10+import json
11+import os
12+from pathlib import Path
13+import shutil
14+import subprocess
15+import tarfile
16+import tempfile
17+ 
18+ 
19+def sha256(path):
20+ return hashlib.sha256(path.read_bytes()).hexdigest()
21+ 
22+ 
23+def run(args):
24+ print('+ ' + ' '.join(map(str, args)), flush=True)
25+ subprocess.run(list(map(str, args)), check=True)
26+ 
27+ 
28+def cache_value(build, name):
29+ for line in (build / 'CMakeCache.txt').read_text().splitlines():
30+ if line.startswith(name + ':'):
31+ return line.split('=', 1)[1]
32+ raise RuntimeError('Missing CMake cache entry: ' + name)
33+ 
34+ 
35+def install_file(source, destination, strip=False):
36+ # Replacing the inode avoids truncating an executable or mapped library
37+ # when rebuilding a package in an existing work directory.
38+ fd, name = tempfile.mkstemp(prefix='.' + destination.name, dir=destination.parent)
39+ os.close(fd)
40+ temporary = Path(name)
41+ try:
42+ shutil.copy2(source, temporary)
43+ if strip:
44+ run(['strip', '--strip-unneeded', temporary])
45+ temporary.replace(destination)
46+ finally:
47+ if temporary.exists():
48+ temporary.unlink()
49+ 
50+ 
51+def main():
52+ parser = argparse.ArgumentParser(description=__doc__)
53+ parser.add_argument('--source-root', type=Path,
54+ default=Path(__file__).resolve().parents[2])
55+ parser.add_argument('--work-dir', type=Path, required=True)
56+ parser.add_argument('--prefix', default='')
57+ parser.add_argument('--jobs', type=int, default=24)
58+ parser.add_argument('--urma-include', default='/usr/include/ub/umdk/urma')
59+ parser.add_argument('--urma-bond-include', default='/usr/local/include/ub/umdk/urma')
60+ parser.add_argument('--urma-library', default='/usr/lib64/liburma.so')
61+ parser.add_argument('--with-tcmalloc', action='store_true')
62+ args = parser.parse_args()
63+ source = args.source_root.resolve()
64+ work = args.work_dir.resolve()
65+ runtime = work / 'runtime'
66+ (runtime / 'common').mkdir(parents=True, exist_ok=True)
67+ for mode in ('multi',):
68+ build = work / ('build-' + mode)
69+ run(['cmake', '-S', source, '-B', build,
70+ '-DCMAKE_BUILD_TYPE=Release', '-DWITH_DEBUG_SYMBOLS=OFF',
71+ '-DWITH_URMA=ON', '-DURMA_USE_MOCK=OFF', '-DWITH_TCMALLOC=' + ('ON' if args.with_tcmalloc else 'OFF'),
72+ '-DWITH_CPU_FREQUENCY=ON', '-DBUILD_UNIT_TESTS=OFF',
73+ '-DBUILD_BRPC_TOOLS=OFF', '-DBUILD_SHARED_LIBS=ON',
74+ '-DCMAKE_PREFIX_PATH=' + args.prefix,
75+ '-DURMA_INCLUDE_PATH=' + args.urma_include,
76+ '-DURMA_BOND_INCLUDE_PATH=' + args.urma_bond_include,
77+ '-DURMA_LIB=' + args.urma_library,
78+ '-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON',
79+ '-DCMAKE_INSTALL_RPATH=$ORIGIN/../common'])
80+ run(['cmake', '--build', build, '--target', 'brpc-shared', '-j', args.jobs])
81+ target = runtime / mode
82+ target.mkdir(exist_ok=True)
83+ install_file(build / 'output/lib/libbrpc.so', target / 'libbrpc.so', strip=True)
84+ allocator_options = ['-DWITH_TCMALLOC=OFF']
85+ if args.with_tcmalloc:
86+ tcmalloc = Path(cache_value(work / 'build-multi', 'TCMALLOC_LIB'))
87+ install_file(tcmalloc.resolve(), runtime / 'common/libtcmalloc.so.4')
88+ allocator_options = ['-DWITH_TCMALLOC=ON', '-DTCMALLOC_LIB=' + str(tcmalloc)]
89+ else:
90+ (runtime / 'common/libtcmalloc.so.4').unlink(missing_ok=True)
91+ example = work / 'build-example'
92+ run(['cmake', '-S', source / 'example/ub_test', '-B', example,
93+ '-DCMAKE_BUILD_TYPE=Release', '-DLINK_SO=ON', '-DWITH_CPU_FREQUENCY=ON',
94+ '-DCMAKE_PREFIX_PATH=' + args.prefix,
95+ '-DBRPC_LIB=' + str(work / 'build-multi/output/lib/libbrpc.so'),
96+ '-DBRPC_INCLUDE_PATH=' + str(work / 'build-multi/output/include'),
97+ '-DURMA_INCLUDE_PATH=' + args.urma_include,
98+ '-DURMA_LIB=' + args.urma_library] + allocator_options)
99+ run(['cmake', '--build', example, '-j', min(args.jobs, 8)])
100+ for mode in ('multi',):
101+ for role in ('client', 'server'):
102+ dest = runtime / mode / ('ub_test_' + role)
103+ install_file(example / dest.name, dest, strip=True)
104+ dependencies = {}
105+ for mode in ('multi',):
106+ env = dict(os.environ, LD_LIBRARY_PATH=str(runtime / mode) + ':' + str(runtime / 'common'))
107+ for role in ('client', 'server'):
108+ name = mode + '/ub_test_' + role
109+ dep = subprocess.check_output(['ldd', str(runtime / name)], env=env, text=True)
110+ if 'not found' in dep:
111+ raise RuntimeError(name + ': unresolved runtime dependency\n' + dep)
112+ dependencies[name] = dep
113+ (work / 'dependencies.json').write_text(json.dumps(dependencies, indent=2) + '\n')
114+ manifest = {
115+ 'upstream_repository': 'dwh110/brpc',
116+ 'upstream_commit': 'b4dff13b0de19a12dc3a1b82902a20d4b51b1bb1',
117+ 'architecture': subprocess.check_output(['uname', '-m'], text=True).strip(),
118+ 'with_cpu_frequency': True, 'with_tcmalloc': args.with_tcmalloc, 'urma_mock': False,
119+ 'mode_selection': 'Multi-SGE with shared URMA resources',
120+ 'files': {str(p.relative_to(runtime)): sha256(p)
121+ for p in sorted(runtime.rglob('*'))
122+ if p.is_file() and p != runtime / 'manifest.json'
123+ and p.relative_to(runtime).parts[0] in ('multi', 'common')},
124+ 'source_sha256': {str(p.relative_to(source)): sha256(p)
125+ for folder in ('src/brpc/urma', 'example/ub_test')
126+ for p in sorted((source / folder).rglob('*'))
127+ if p.is_file() and '__pycache__' not in p.parts},
128+ }
129+ (runtime / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
130+ with tarfile.open(work / 'ub-test-runtime-aarch64.tar.gz', 'w:gz') as archive:
131+ for name in ('multi', 'common', 'manifest.json'):
132+ archive.add(runtime / name, arcname='runtime/' + name)
133+ print('BINARY_PACKAGE_READY ' + str(work / 'ub-test-runtime-aarch64.tar.gz'), flush=True)
134+ 
135+ 
136+if __name__ == '__main__':
137+ main()
@@ -0,0 +1,440 @@
1+#!/usr/bin/env python3
2+"""Run the dwh110/brpc ub_test matrix using the pooled multi-SGE build.
3+ 
4+Upstream: b4dff13b0de19a12dc3a1b82902a20d4b51b1bb1/run_ub_test_matrix.py.
5+Preserved workload: QPS 1000/500; global QD 10; 20 seconds; req_size=rsp_size
6+1KiB,4KiB,8KiB,100KiB,200KiB,1MiB,8MiB; CPUs 96-111; memory node 1;
7+RPC timeout 2000ms, connect timeout 6000ms, max_retry=10, initial_tokens=0.
8+Only the transport selection and orchestration are adapted. Event mode is
9+required by pooled URMA. Uses Python's standard library and OpenSSH, no Paramiko.
10+ 
11+The binary archive contains runtime/multi/ub_test_{client,server},
12+the corresponding libbrpc.so, an optional common/libtcmalloc.so.4, and manifest.json.
13+No sources are uploaded. Existing system UMDK/Protobuf dependencies are required.
14+Use --dry-run to inspect every command without contacting either host.
15+"""
16+ 
17+import argparse
18+import csv
19+import json
20+import os
21+from pathlib import Path
22+import re
23+import shlex
24+import subprocess
25+import sys
26+import time
27+ 
28+ 
29+UPSTREAM_COMMIT = 'b4dff13b0de19a12dc3a1b82902a20d4b51b1bb1'
30+QPS_VALUES = [1000, 500]
31+SIZES = [1024, 4096, 8192, 102400, 204800, 1048576, 8388608]
32+METRICS = ['Avg-Latency', '50th-Latency', '90th-Latency', '99th-Latency',
33+ '99.9th-Latency', '99.99th-Latency', 'Max-Latency', 'Throughput',
34+ 'QPS', 'Server_CPU_avg', 'Server_CPU_max', 'Client_CPU_avg',
35+ 'Client_CPU_max', 'Client_Mem_avg', 'Client_Mem_max', 'Error_rate']
36+CSV_HEADER = ['urma_sge_mode', 'expected_qps', 'size'] + METRICS
37+NUMBER = r'([0-9]+(?:\.[0-9]*)?(?:[eE][+-]?[0-9]+)?)'
38+ 
39+ 
40+def positive_int(value):
41+ number = int(value)
42+ if number <= 0:
43+ raise argparse.ArgumentTypeError('must be positive')
44+ return number
45+ 
46+ 
47+def parse_result(output):
48+ line = next((s for s in output.splitlines() if 'Avg-Latency:' in s), '')
49+ result = {}
50+ for field in METRICS[:9]:
51+ m = re.search(re.escape(field) + r':\s*' + NUMBER, line)
52+ result[field] = m.group(1) if m else 'N/A'
53+ for label, prefix in [('Server CPU', 'Server_CPU'), ('Client CPU', 'Client_CPU'),
54+ ('Client Memory', 'Client_Mem')]:
55+ m = re.search(re.escape(label) + r'\(avg/max\):\s*' + NUMBER + '/' + NUMBER, line)
56+ result[prefix + '_avg'] = m.group(1) if m else 'N/A'
57+ result[prefix + '_max'] = m.group(2) if m else 'N/A'
58+ m = re.search(r'Error rate:\s*' + NUMBER, line)
59+ result['Error_rate'] = m.group(1) if m else 'N/A'
60+ totals = {}
61+ for label in ('Total Sent', 'Total Errors', 'Total Requests'):
62+ m = re.search(re.escape(label) + r':\s*(\d+)', line)
63+ totals[label] = int(m.group(1)) if m else None
64+ return line, result, totals
65+ 
66+ 
67+def command(args, role, mode, qps, size):
68+ root = getattr(args, role + '_dir')
69+ binary = root + '/runtime/' + mode + '/ub_test_' + role
70+ cmd = ['numactl', '-C', args.cpus, '-m', '1', binary,
71+ '--use_urma=true', '--urma_sge_mode=' + mode,
72+ '--urma_device=' + args.device, '--urma_use_polling=false',
73+ '--urma_resource_mode=pooled']
74+ if getattr(args, 'io_jfs_count', None) is not None:
75+ cmd.append('--urma_io_jfs_cnt=' + str(args.io_jfs_count))
76+ if role == 'server':
77+ cmd += ['--port', str(args.port), '--rsp_size=' + str(size), '--num_threads=16']
78+ else:
79+ cmd += ['--servers=' + args.server_address + ':' + str(args.port),
80+ '--rpc_timeout_ms=2000', '--connect_timeout_ms=6000',
81+ '--test_seconds=20', '--max_retry=10', '--queue_depth=10',
82+ '--req_size=' + str(size), '--dummy_port=0',
83+ '--expected_qps=' + str(qps), '--initial_tokens=0']
84+ return cmd
85+ 
86+ 
87+class Remote:
88+ def __init__(self, host, key, ssh_multiplexing='auto'):
89+ self.host = host
90+ self.multiplexing = (sys.platform != 'win32' if ssh_multiplexing == 'auto'
91+ else ssh_multiplexing == 'on')
92+ self.options = ['-i', str(Path(key).expanduser()), '-o', 'BatchMode=yes',
93+ '-o', 'ConnectTimeout=10']
94+ if self.multiplexing:
95+ self.options += ['-o', 'ControlMaster=auto', '-o', 'ControlPersist=600',
96+ '-o', 'ControlPath=/tmp/brpc-ub-matrix-%C']
97+ else:
98+ # Windows OpenSSH cannot use the Unix control socket. Explicitly
99+ # disable all three options, including inherited ~/.ssh/config.
100+ self.options += ['-o', 'ControlMaster=no', '-o', 'ControlPersist=no',
101+ '-o', 'ControlPath=none']
102+ 
103+ def python(self, source, timeout=30):
104+ proc = subprocess.run(['ssh'] + self.options + [self.host, 'python3 -'],
105+ input=source, text=True, encoding='utf-8',
106+ capture_output=True, timeout=timeout)
107+ if proc.returncode:
108+ message = self.host + ': ' + proc.stderr + proc.stdout
109+ if self.multiplexing and 'getsockname failed' in proc.stderr.lower():
110+ message += '\nSSH control socket failed; use --ssh-multiplexing off.'
111+ raise RuntimeError(message)
112+ return json.loads(proc.stdout)
113+ 
114+ def put(self, local, remote):
115+ subprocess.run(['scp'] + self.options + [str(local), self.host + ':' + shlex.quote(remote)],
116+ check=True, timeout=120)
117+ 
118+ def read(self, path):
119+ return self.python('import json,pathlib\np=pathlib.Path(' + repr(path) +
120+ ')\nprint(json.dumps(p.read_text() if p.exists() else ""))\n')
121+ 
122+ def close(self):
123+ if self.multiplexing:
124+ try:
125+ subprocess.run(['ssh'] + self.options + ['-O', 'exit', self.host],
126+ capture_output=True, timeout=10)
127+ except (OSError, subprocess.SubprocessError):
128+ pass # Cleanup must not hide the original SSH/test failure.
129+ 
130+ 
131+def deploy(remote, root, archive):
132+ remote.python('import pathlib,json\npathlib.Path(' + repr(root) +
133+ ').mkdir(parents=True,exist_ok=True)\nprint(json.dumps(True))\n')
134+ dest = root + '/ub-test-runtime.upload.tar.gz'
135+ remote.put(archive, dest)
136+ remote.python('''import json,pathlib,tarfile
137+root=pathlib.Path(ROOT).resolve()
138+archive=root/'ub-test-runtime.upload.tar.gz'
139+with tarfile.open(archive) as a:
140+ for member in a.getmembers():
141+ path=(root/member.name).resolve()
142+ if root not in path.parents or member.issym() or member.islnk():
143+ raise RuntimeError('Unsafe archive member: '+member.name)
144+ a.extractall(root)
145+archive.unlink()
146+print(json.dumps(True))
147+'''.replace('ROOT', repr(root)), timeout=60)
148+ 
149+ 
150+def preflight(remote, root, cpus, port, server):
151+ return remote.python('''import hashlib,json,os,pathlib,subprocess
152+root=pathlib.Path(ROOT);runtime=root/'runtime'
153+manifest=json.loads((runtime/'manifest.json').read_text())
154+hashes={name:hashlib.sha256((runtime/name).read_bytes()).hexdigest() for name in manifest['files']}
155+if hashes!=manifest['files']:raise RuntimeError('Binary manifest mismatch')
156+deps={}
157+for mode in ['multi']:
158+ env=dict(os.environ,LD_LIBRARY_PATH=str(runtime/mode)+':'+str(runtime/'common'))
159+ for role in ['client','server']:
160+ key=mode+'/ub_test_'+role
161+ deps[key]=subprocess.check_output(['ldd',str(runtime/key)],text=True,env=env)
162+ if 'not found' in deps[key]:raise RuntimeError(deps[key])
163+binding=subprocess.check_output(['numactl','-C',CPUS,'-m','1','--show'],text=True,stderr=subprocess.STDOUT)
164+if SERVER:
165+ listeners=subprocess.check_output(['ss','-H','-ltn'],text=True)
166+ if any(s.split()[3].rsplit(':',1)[-1]==str(PORT) for s in listeners.splitlines()):
167+ raise RuntimeError('Test port already in use; refusing to stop its owner')
168+(root/'logs').mkdir(exist_ok=True)
169+print(json.dumps({'manifest':manifest,'hashes':hashes,'dependencies':deps,'binding':binding,
170+ 'architecture':subprocess.check_output(['uname','-m'],text=True).strip()}))
171+'''.replace('ROOT', repr(root)).replace('CPUS', repr(cpus))
172+ .replace('SERVER', repr(server)).replace('PORT', str(port)), timeout=60)
173+ 
174+ 
175+def launch(remote, root, name, role, mode, argv):
176+ spec = {'args': argv, 'log': root + '/logs/' + name + '-' + role + '.log',
177+ 'pid': root + '/logs/' + name + '-' + role + '.pid',
178+ 'exit': root + '/logs/' + name + '-' + role + '.exit',
179+ 'library_path': root + '/runtime/' + mode + ':' + root + '/runtime/common'}
180+ # The supervisor is sent in the SSH command, never deployed as a source file.
181+ supervisor = '''import json,os,pathlib,subprocess,sys
182+s=json.loads(sys.argv[1]);env=dict(os.environ,LD_LIBRARY_PATH=s['library_path'])
183+with open(s['log'],'w') as f:
184+ p=subprocess.Popen(s['args'],stdin=subprocess.DEVNULL,stdout=f,stderr=subprocess.STDOUT,env=env)
185+ pathlib.Path(s['pid']).write_text(str(p.pid))
186+ code=p.wait()
187+pathlib.Path(s['exit']).write_text(str(code))
188+'''
189+ result = remote.python('''import json,pathlib,subprocess,time
190+s=SPEC
191+for key in ['pid','exit']:
192+ p=pathlib.Path(s[key])
193+ if p.exists():p.unlink()
194+p=subprocess.Popen(['python3','-c',SUPERVISOR,json.dumps(s)],stdin=subprocess.DEVNULL,
195+ stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True)
196+for _ in range(100):
197+ if pathlib.Path(s['pid']).exists():break
198+ time.sleep(.02)
199+print(json.dumps({**s,'pid':int(pathlib.Path(s['pid']).read_text()),'supervisor_pid':p.pid}))
200+'''.replace('SPEC', repr(spec)).replace('SUPERVISOR', repr(supervisor)))
201+ return result
202+ 
203+ 
204+def process_info(remote, proc):
205+ return remote.python('''import collections,json,os,pathlib,re
206+p=pathlib.Path('/proc')/str(PID)
207+threads=[]
208+for t in (p/'task').iterdir():
209+ try:threads.append(sorted(os.sched_getaffinity(int(t.name))))
210+ except ProcessLookupError:pass
211+maps=(p/'maps').read_text();numa=(p/'numa_maps').read_text()
212+policies=collections.Counter(s.split()[1] for s in numa.splitlines() if len(s.split())>1)
213+manifest=json.loads((p/'exe').resolve().parent.parent.joinpath('manifest.json').read_text())
214+print(json.dumps({'thread_affinities':threads,'numa_policies':dict(policies),
215+ 'expects_tcmalloc':'common/libtcmalloc.so.4' in manifest['files'],
216+ 'libraries':sorted({s.split()[-1] for s in maps.splitlines() if 'libbrpc.so' in s or 'libtcmalloc' in s}),
217+ 'status':(p/'status').read_text()}))
218+'''.replace('PID', str(proc['pid'])))
219+ 
220+ 
221+def socket_snapshot(remote, port):
222+ return remote.python('''import json,re,urllib.request
223+opener=urllib.request.build_opener(urllib.request.ProxyHandler({}))
224+base='http://127.0.0.1:PORT'
225+def get(path):return opener.open(base+path+'?console=1',timeout=3).read().decode()
226+connections=get('/connections');sockets=[]
227+for line in connections.splitlines():
228+ m=re.search(r'(\d+)\s*$',line)
229+ if not m:continue
230+ detail=get('/sockets/'+m.group(1))
231+ if 'state=ESTABLISHED' in detail:
232+ sockets.append({'id':m.group(1),'detail':detail})
233+print(json.dumps({'connections':connections,'urma_sockets':sockets}))
234+'''.replace('PORT', str(port)))
235+ 
236+ 
237+def stop(remote, proc):
238+ executable = next(s for s in proc['args'] if '/ub_test_' in s)
239+ return remote.python('''import json,os,pathlib,signal,time
240+p=pathlib.Path('/proc')/str(PID)
241+def alive():
242+ try:return (p/'stat').read_text().split()[2]!='Z' and (p/'cmdline').read_bytes().split(bytes([0]))[0].decode()==EXE
243+ except (FileNotFoundError,ProcessLookupError):return False
244+if alive():
245+ try:os.kill(PID,signal.SIGINT)
246+ except ProcessLookupError:pass
247+ for _ in range(50):
248+ if not alive():break
249+ time.sleep(.1)
250+ if alive():
251+ try:os.kill(PID,signal.SIGKILL)
252+ except ProcessLookupError:pass
253+ for _ in range(20):
254+ if not alive():break
255+ time.sleep(.05)
256+print(json.dumps({'stopped':not alive()}))
257+'''.replace('PID', str(proc['pid'])).replace('EXE', repr(executable)))
258+ 
259+ 
260+def expected_cpus(value):
261+ cpus = set()
262+ for part in value.split(','):
263+ ends = part.split('-')
264+ cpus.update(range(int(ends[0]), int(ends[-1]) + 1))
265+ return cpus
266+ 
267+ 
268+def statistics_failures(returncode, line, metrics, totals):
269+ failures = []
270+ if returncode != 0:
271+ failures.append('client exit code')
272+ if not line or any(v == 'N/A' for v in metrics.values()):
273+ failures.append('incomplete statistics')
274+ # Rounded "Error rate: 0%" must not hide a small non-zero error count.
275+ if totals['Total Errors'] != 0:
276+ failures.append('RPC errors')
277+ if not totals['Total Requests']:
278+ failures.append('no completed RPCs')
279+ return failures
280+ 
281+ 
282+def run_test(args, remotes, mode, qps, size):
283+ name = '{}-q{}-s{}'.format(mode, qps, size)
284+ out = Path(args.output)
285+ result = {'name': name, 'sge_mode': mode, 'expected_qps': qps, 'size': size,
286+ 'io_jfs_count': getattr(args, 'io_jfs_count', None),
287+ 'valid': False, 'processes': {}, 'cleanup': {}}
288+ logs = {}
289+ try:
290+ server = launch(remotes['server'], args.server_dir, name, 'server', mode,
291+ command(args, 'server', mode, qps, size))
292+ result['processes']['server'] = server
293+ time.sleep(3) # Preserve the original pre-client startup interval.
294+ server_log = remotes['server'].read(server['log'])
295+ marker = 'UB_TEST_TRANSPORT sge_mode=' + mode
296+ if remotes['server'].read(server['exit']).strip():
297+ raise RuntimeError('Server exited during startup: ' + server_log[-1500:])
298+ if marker not in server_log:
299+ raise RuntimeError('Server did not initialize the requested URMA build: ' + server_log[-1500:])
300+ client = launch(remotes['client'], args.client_dir, name, 'client', mode,
301+ command(args, 'client', mode, qps, size))
302+ result['processes']['client'] = client
303+ deadline = time.monotonic() + (300 if size >= 1048576 else 120)
304+ time.sleep(3)
305+ # Auditing must not shorten a benchmark if a process has exited or a
306+ # diagnostics request fails. Keep its final statistics and raw logs.
307+ audit_failures = []
308+ result['process_info'] = {}
309+ for role, proc in result['processes'].items():
310+ try:
311+ result['process_info'][role] = process_info(remotes[role], proc)
312+ except Exception as error:
313+ audit_failures.append(role + ' process audit: ' + str(error))
314+ try:
315+ result['sockets'] = socket_snapshot(remotes['server'], args.port)
316+ except Exception as error:
317+ result['sockets'] = {'urma_sockets': []}
318+ audit_failures.append('socket audit: ' + str(error))
319+ while time.monotonic() < deadline:
320+ code = remotes['client'].read(client['exit'])
321+ if code.strip():
322+ result['client_returncode'] = int(code)
323+ break
324+ time.sleep(1)
325+ else:
326+ raise TimeoutError('Client exceeded the original matrix timeout')
327+ logs['client'] = remotes['client'].read(client['log'])
328+ logs['server'] = remotes['server'].read(server['log'])
329+ line, metrics, totals = parse_result(logs['client'])
330+ result.update(result_line=line, metrics=metrics, totals=totals)
331+ failures = statistics_failures(result['client_returncode'], line, metrics, totals) + audit_failures
332+ if len(result['sockets']['urma_sockets']) != 1: failures.append('expected one established URMA connection')
333+ for role, info in result['process_info'].items():
334+ if marker not in logs[role] or 'tp=CTP' not in logs[role]: failures.append(role + ' transport')
335+ if result['io_jfs_count'] is not None:
336+ counts = re.findall(r'\bjfs_count=(\d+)\b', logs[role])
337+ if counts != [str(result['io_jfs_count'])]:
338+ failures.append(role + ' JFS count')
339+ if not info['thread_affinities'] or any(set(c) != expected_cpus(args.cpus) for c in info['thread_affinities']):
340+ failures.append(role + ' CPU affinity')
341+ if set(info['numa_policies']) != {'bind:1'}: failures.append(role + ' memory binding')
342+ root = getattr(args, role + '_dir')
343+ if root + '/runtime/' + mode + '/libbrpc.so' not in info['libraries']:
344+ failures.append(role + ' libbrpc')
345+ if (info.get('expects_tcmalloc', False) and
346+ root + '/runtime/common/libtcmalloc.so.4' not in info['libraries']):
347+ failures.append(role + ' tcmalloc')
348+ result['failures'] = failures
349+ result['valid'] = not failures
350+ except Exception as error:
351+ result['exception'] = str(error)
352+ finally:
353+ for role in ('client', 'server'):
354+ proc = result['processes'].get(role)
355+ if proc:
356+ try:
357+ result['cleanup'][role] = stop(remotes[role], proc)
358+ if not result['cleanup'][role]['stopped']:
359+ result['valid'] = False
360+ logs[role] = remotes[role].read(proc['log'])
361+ except Exception as error:
362+ result['cleanup'][role] = {'stopped': False, 'error': str(error)}
363+ result['valid'] = False
364+ for role, log in logs.items():
365+ (out / (name + '-' + role + '.log')).write_text(log)
366+ (out / (name + '.json')).write_text(json.dumps(result, indent=2) + '\n')
367+ print(json.dumps({k: result.get(k) for k in ('name', 'valid', 'metrics', 'failures', 'exception')}), flush=True)
368+ time.sleep(2) # Preserve the original inter-case interval.
369+ return result
370+ 
371+ 
372+def main():
373+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
374+ parser.add_argument('--server', required=True, help='SSH user@host of the server/build node')
375+ parser.add_argument('--client', required=True, help='SSH user@host of the client node')
376+ parser.add_argument('--key', default='~/.ssh/id_ed25519')
377+ parser.add_argument('--ssh-multiplexing', choices=['auto', 'off', 'on'], default='auto',
378+ help='SSH connection reuse: auto disables it on Windows; off disables it everywhere')
379+ parser.add_argument('--server-address', help='RPC address; defaults to the host in --server')
380+ parser.add_argument('--server-dir', required=True, help='Absolute remote directory containing runtime/')
381+ parser.add_argument('--client-dir', required=True)
382+ parser.add_argument('--device', required=True)
383+ parser.add_argument('--port', type=int, default=10918)
384+ parser.add_argument('--cpus', default='96-111')
385+ parser.add_argument('--io-jfs-count', type=positive_int,
386+ help='Override shared JFS count on both nodes; otherwise use the library default')
387+ parser.add_argument('--modes', nargs='+', choices=['multi'], default=['multi'])
388+ parser.add_argument('--qps', nargs='+', type=int, choices=QPS_VALUES, default=QPS_VALUES)
389+ parser.add_argument('--sizes', nargs='+', type=int, choices=SIZES, default=SIZES)
390+ parser.add_argument('--deploy', type=Path, help='Deploy only the runtime binary archive to both nodes')
391+ parser.add_argument('--output', default='ub_test_results')
392+ parser.add_argument('--dry-run', action='store_true')
393+ args = parser.parse_args()
394+ args.server_address = args.server_address or args.server.rsplit('@', 1)[-1]
395+ if not all(p.startswith('/') for p in (args.server_dir, args.client_dir)):
396+ parser.error('Remote runtime directories must be absolute')
397+ plan = [{'sge_mode': m, 'qps': q, 'size': s,
398+ 'server': command(args, 'server', m, q, s),
399+ 'client': command(args, 'client', m, q, s)}
400+ for m in args.modes for q in args.qps for s in args.sizes]
401+ if args.dry_run:
402+ print(json.dumps(plan, indent=2))
403+ return 0
404+ out = Path(args.output)
405+ out.mkdir(parents=True, exist_ok=True)
406+ if (out / 'results.json').exists():
407+ parser.error('Output already contains results; choose a new output directory')
408+ (out / 'plan.json').write_text(json.dumps({'upstream_commit': UPSTREAM_COMMIT, 'cases': plan,
409+ 'transport_changes': 'multi-SGE transport; event mode; pooled transport defaults; selected device',
410+ 'statistics': 'Unchanged upstream ub_test calculations; CSV retains original columns except SGE mode'}, indent=2) + '\n')
411+ remotes = {role: Remote(getattr(args, role), args.key, args.ssh_multiplexing)
412+ for role in ('server', 'client')}
413+ results = []
414+ try:
415+ if args.deploy:
416+ for role, remote in remotes.items():
417+ deploy(remote, getattr(args, role + '_dir'), args.deploy)
418+ audit = {role: preflight(remote, getattr(args, role + '_dir'), args.cpus, args.port, role == 'server')
419+ for role, remote in remotes.items()}
420+ if audit['server']['hashes'] != audit['client']['hashes']:
421+ raise RuntimeError('The two nodes have different runtime binaries')
422+ (out / 'preflight.json').write_text(json.dumps(audit, indent=2) + '\n')
423+ for case in plan:
424+ results.append(run_test(args, remotes, case['sge_mode'], case['qps'], case['size']))
425+ (out / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
426+ with (out / 'ub_test_results.csv').open('w', newline='') as stream:
427+ writer = csv.writer(stream)
428+ writer.writerow(CSV_HEADER)
429+ for r in results:
430+ writer.writerow([r['sge_mode'], r['expected_qps'], r['size']] +
431+ [r.get('metrics', {}).get(k, 'ERROR') for k in METRICS])
432+ print(json.dumps({'completed': len(results), 'valid': sum(r['valid'] for r in results),
433+ 'output': str(out.resolve())}), flush=True)
434+ return 0 if all(r['valid'] for r in results) else 1
435+ finally:
436+ for remote in remotes.values(): remote.close()
437+ 
438+ 
439+if __name__ == '__main__':
440+ raise SystemExit(main())
@@ -0,0 +1,242 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information
4+// regarding copyright ownership. The ASF licenses this file
5+// to you under the Apache License, Version 2.0 (the
6+// "License"); you may not use this file except in compliance
7+// with the License. You may obtain a copy of the License at
8+//
9+// http://www.apache.org/licenses/LICENSE-2.0
10+//
11+// Unless required by applicable law or agreed to in writing,
12+// software distributed under the License is distributed on an
13+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+// KIND, either express or implied. See the License for the
15+// specific language governing permissions and limitations
16+// under the License.
17+ 
18+#include <fcntl.h>
19+#include <pthread.h>
20+#include <gflags/gflags.h>
21+#include "butil/atomicops.h"
22+#include "butil/logging.h"
23+#include "butil/time.h"
24+#include "brpc/server.h"
25+#include "brpc/socket_mode.h"
26+#include "brpc/urma/urma_helper.h"
27+#include "bvar/variable.h"
28+#include "bvar/latency_recorder.h"
29+#include "test.pb.h"
30+#include "sge_mode.h"
31+ 
32+DEFINE_int32(port, 8002, "TCP Port of this server");
33+DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)");
34+DEFINE_bool(use_rdma, false, "Use RDMA or not");
35+ 
36+DEFINE_bool(server_ignore_oc, false, "Server ignore eovercrowded, false by default");
37+DEFINE_int32(num_threads, 5, "The max number of threads are used");
38+DEFINE_int32(max_concurrency, 128, "max concurrency");
39+DEFINE_int32(server_bthread_concurrency, 5, "server bthread concurrency");
40+DEFINE_int64(rsp_size, 0, "Response name size in bytes (0 means echo the request name back)");
41+DEFINE_int32(stats_timeout_seconds, 3, "Timeout in seconds before collecting statistics. (default 3)");
42+DEFINE_int32(test_seconds, 40, "Test duration in seconds (should match client)");
43+ 
44+pthread_mutex_t g_metrics_lock = PTHREAD_MUTEX_INITIALIZER;
45+std::string g_cpu_usage_str; // cached describe_exposed("process_cpu_usage") value
46+butil::atomic<bool> g_cpu_usage_ready(false); // claim token: at most one response per 1s cycle
47+butil::atomic<uint64_t> g_total_cnt(0);
48+butil::atomic<uint64_t> g_total_bytes(0);
49+butil::atomic<uint64_t> g_start_time(0);
50+butil::atomic<uint64_t> g_last_request_time(0);
51+butil::atomic<uint64_t> g_end_time(0);
52+bvar::LatencyRecorder g_server_latency_recorder("server_latency");
53+bvar::LatencyRecorder g_server_cpu_recorder("server_cpu");
54+bvar::LatencyRecorder g_server_memory_recorder("server_memory");
55+std::string g_name;
56+volatile bool g_stop = false;
57+brpc::Server* g_server = nullptr;
58+ 
59+static void* StatsPrinter(void* arg) {
60+ brpc::Server* server = static_cast<brpc::Server*>(arg);
61+ 
62+ while (!g_stop) {
63+ bthread_usleep(100000);
64+ 
65+ // Sample process CPU/memory every 100ms, off the RPC hot path.
66+ const std::string cpu_usage_str = bvar::Variable::describe_exposed("process_cpu_usage");
67+ const double cpu_percent = atof(cpu_usage_str.c_str()) * 100;
68+ const double memory_mb = atof(
69+ bvar::Variable::describe_exposed("process_memory_resident").c_str()) / 1024 / 1024;
70+ 
71+ pthread_mutex_lock(&g_metrics_lock);
72+ g_cpu_usage_str = cpu_usage_str;
73+ g_cpu_usage_ready.store(true, butil::memory_order_release);
74+ pthread_mutex_unlock(&g_metrics_lock);
75+ 
76+ g_server_cpu_recorder << cpu_percent;
77+ g_server_memory_recorder << memory_mb;
78+ 
79+ uint64_t now = butil::monotonic_time_us();
80+ uint64_t last = g_last_request_time.load(butil::memory_order_relaxed);
81+ uint64_t end = g_end_time.load(butil::memory_order_relaxed);
82+ uint64_t start_time = g_start_time.load(butil::memory_order_relaxed);
83+ 
84+ if (end == 0 && start_time > 0) {
85+ uint64_t test_duration_us = now - start_time;
86+ uint64_t trigger_time_us = ((uint64_t)FLAGS_test_seconds + FLAGS_stats_timeout_seconds) * 1000000;
87+ 
88+ if (test_duration_us >= trigger_time_us) {
89+ uint64_t end_expected = 0;
90+ if (g_end_time.compare_exchange_strong(end_expected, last)) {
91+ double duration_s = (last - start_time) / 1000000.0;
92+ uint64_t total_cnt = g_total_cnt.load(butil::memory_order_relaxed);
93+ uint64_t total_bytes = g_total_bytes.load(butil::memory_order_relaxed);
94+ 
95+ std::cout << "========== Final Report ==========" << std::endl;
96+ std::cout << "Duration: " << duration_s << "s" << std::endl;
97+ std::cout << "Total Requests: " << total_cnt << std::endl;
98+ if (duration_s > 0) {
99+ std::cout << "QPS: " << total_cnt / duration_s << std::endl;
100+ std::cout << "Bandwidth: " << total_bytes / 1024.0 / 1024.0 / duration_s << "MB/s" << std::endl;
101+ }
102+ std::cout << "Latency(avg/p99/p999/p9999): "
103+ << g_server_latency_recorder.latency(10) << "/"
104+ << g_server_latency_recorder.latency_percentile(0.99) << "/"
105+ << g_server_latency_recorder.latency_percentile(0.999) << "/"
106+ << g_server_latency_recorder.latency_percentile(0.9999) << "us" << std::endl;
107+ std::cout << "CPU(avg/max): "
108+ << g_server_cpu_recorder.latency(10) << "/"
109+ << g_server_cpu_recorder.max_latency() << "%" << std::endl;
110+ std::cout << "Memory(avg/max): "
111+ << g_server_memory_recorder.latency(10) << "/"
112+ << g_server_memory_recorder.max_latency() << "MB" << std::endl;
113+ std::cout << "==================================" << std::endl;
114+ return nullptr;
115+ }
116+ }
117+ }
118+ }
119+ return nullptr;
120+}
121+ 
122+namespace test {
123+class PerfTestServiceImpl : public PerfTestService {
124+public:
125+ PerfTestServiceImpl() {}
126+ 
127+ ~PerfTestServiceImpl() {}
128+ 
129+ void Test(google::protobuf::RpcController* cntl_base,
130+ const PerfTestRequest* request,
131+ PerfTestResponse* response,
132+ google::protobuf::Closure* done) {
133+ brpc::ClosureGuard done_guard(done);
134+ uint64_t now = butil::monotonic_time_us();
135+ 
136+ uint64_t expected = 0;
137+ if (g_start_time.compare_exchange_strong(expected, now)) {
138+ bthread_t tid;
139+ bthread_start_background(&tid, &BTHREAD_ATTR_NORMAL, StatsPrinter, g_server);
140+ }
141+ g_last_request_time.store(now, butil::memory_order_relaxed);
142+ 
143+ if (g_cpu_usage_ready.exchange(false, butil::memory_order_acq_rel)) {
144+ pthread_mutex_lock(&g_metrics_lock);
145+ response->set_cpu_usage(g_cpu_usage_str);
146+ pthread_mutex_unlock(&g_metrics_lock);
147+ } else {
148+ response->set_cpu_usage(""); // proto2 required field: must be set explicitly
149+ }
150+ 
151+ brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base);
152+ 
153+ if (FLAGS_rsp_size == 0) {
154+ response->set_name(request->name());
155+ } else {
156+ response->set_name(g_name);
157+ }
158+ if (request->echo_attachment()) {
159+ cntl->response_attachment().append(cntl->request_attachment());
160+ g_total_bytes.fetch_add(cntl->response_attachment().size(), butil::memory_order_relaxed);
161+ } else {
162+ g_total_bytes.fetch_add(response->name().size(), butil::memory_order_relaxed);
163+ }
164+ 
165+ g_total_cnt.fetch_add(1, butil::memory_order_relaxed);
166+ g_server_latency_recorder << cntl->latency_us();
167+ }
168+};
169+}
170+ 
171+namespace bthread {
172+ DECLARE_int32(bthread_concurrency);
173+}
174+DEFINE_string(urma_sge_mode, "auto", "Expected loaded URMA build: auto or multi");
175+ 
176+constexpr int FD_NUM = 20000;
177+ 
178+int main(int argc, char* argv[]) {
179+ // 预申请fd 规避扩容时延
180+ int dummy_fd = open("/dev/null", O_RDONLY);
181+ int new_fd = dup2(dummy_fd, FD_NUM);
182+ if (dummy_fd != -1) close(dummy_fd);
183+ if (new_fd != -1) close(new_fd);
184+ 
185+ GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
186+ 
187+ bthread::FLAGS_bthread_concurrency = FLAGS_server_bthread_concurrency;
188+ 
189+ // Initialize URMA before any IOBuf operations to ensure all IOBuf blocks
190+ // are allocated from the URMA pool, not the original malloc.
191+ if (FLAGS_use_urma) {
192+ brpc::urma::GlobalUrmaInitializeOrDie();
193+ if (!ValidateUrmaSgeMode(FLAGS_urma_sge_mode)) {
194+ return 2;
195+ }
196+ }
197+ 
198+ brpc::Server server;
199+ test::PerfTestServiceImpl perf_test_service_impl;
200+ 
201+ g_name.resize(FLAGS_rsp_size, 'r');
202+ if (FLAGS_rsp_size == 0) {
203+ std::cout << "server rsp_size=0: echoing request name in response" << std::endl;
204+ } else {
205+ std::cout << "server rsp/name len is " << g_name.size() << "B" << std::endl;
206+ }
207+ std::cout << "URMA: " << (FLAGS_use_urma ? "yes" : "no")
208+ << ", URMA_SGE_MODE: " << FLAGS_urma_sge_mode
209+ << std::endl;
210+ 
211+ g_total_cnt.store(0, butil::memory_order_relaxed);
212+ g_total_bytes.store(0, butil::memory_order_relaxed);
213+ g_start_time.store(0, butil::memory_order_relaxed);
214+ g_last_request_time.store(0, butil::memory_order_relaxed);
215+ g_end_time.store(0, butil::memory_order_relaxed);
216+ 
217+ if (server.AddService(&perf_test_service_impl,
218+ brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
219+ LOG(ERROR) << "Fail to add service";
220+ return -1;
221+ }
222+ brpc::ServerOptions options;
223+ if (FLAGS_use_urma) {
224+ options.socket_mode = brpc::SOCKET_MODE_URMA;
225+ } else if (FLAGS_use_rdma) {
226+ options.socket_mode = brpc::SOCKET_MODE_RDMA;
227+ } else {
228+ options.socket_mode = brpc::SOCKET_MODE_TCP;
229+ }
230+ options.max_concurrency = FLAGS_max_concurrency;
231+ options.num_threads = FLAGS_num_threads;
232+ options.ignore_eovercrowded = FLAGS_server_ignore_oc;
233+ if (server.Start(FLAGS_port, &options) != 0) {
234+ LOG(ERROR) << "Fail to start EchoServer";
235+ return -1;
236+ }
237+ g_server = &server;
238+ 
239+ server.RunUntilAskedToQuit();
240+ g_stop = true;
241+ return 0;
242+}
@@ -0,0 +1,70 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information.
4+// Licensed under the Apache License, Version 2.0 (the "License");
5+// you may not use this file except in compliance with the License.
6+// You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0
7+// Unless required by applicable law or agreed to in writing, software
8+// distributed under the License is distributed on an "AS IS" BASIS,
9+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+ 
11+#ifndef EXAMPLE_UB_TEST_SGE_MODE_H
12+#define EXAMPLE_UB_TEST_SGE_MODE_H
13+ 
14+#include <iostream>
15+#include <string>
16+#include "brpc/urma/urma_helper.h"
17+#include "brpc/urma/urma_resource_pool.h"
18+#include "brpc/urma/urma_resource.h"
19+ 
20+// Verify the loaded pooled transport before starting benchmark traffic.
21+inline bool ValidateUrmaSgeMode(const std::string& requested) {
22+ if (!brpc::urma::UsingPooledUrmaResources()) {
23+ const int max_sge = brpc::urma::GetUrmaMaxSge();
24+ const std::string actual = max_sge >= 2 ? "multi" : "single";
25+ if (!brpc::urma::GetUrmaContext() || max_sge < 1 ||
26+ (requested != "auto" && requested != actual)) return false;
27+ std::cout << "UB_TEST_TRANSPORT resource_mode=exclusive sge_mode=" << actual
28+ << " max_sge=" << max_sge << std::endl;
29+ return true;
30+ }
31+ auto* pool = brpc::urma::g_urma_pool;
32+ if (!pool || pool->io_jfs.empty() || !pool->io_jfs.front() ||
33+ !pool->io_jfs.front()->jfs) {
34+ std::cerr << "URMA resource pool is unavailable; refusing TCP fallback"
35+ << std::endl;
36+ return false;
37+ }
38+ const uint32_t max_sge = pool->io_jfs.front()->jfs->jfs_cfg.max_sge;
39+ const std::string actual = "multi";
40+ if (max_sge < 2) { return false; }
41+ if (requested != "auto" && requested != actual) {
42+ std::cerr << "Requested SGE mode " << requested << ", loaded " << actual
43+ << "; select the matching libbrpc build" << std::endl;
44+ return false;
45+ }
46+ auto* context = brpc::urma::GetUrmaContext();
47+ urma_device_attr_t attr = {};
48+ if (!context || urma_query_device(context->dev, &attr) != URMA_SUCCESS) {
49+ std::cerr << "Cannot verify URMA device CTP capabilities" << std::endl;
50+ return false;
51+ }
52+ const int priority = brpc::urma::FindUrmaPriorityForTpType(attr, URMA_CTP);
53+ for (const auto* slot : pool->io_jfs) {
54+ if (priority < 0 || !slot || !slot->jfs ||
55+ slot->jfs->jfs_cfg.max_sge != max_sge ||
56+ slot->jfs->jfs_cfg.priority != priority) {
57+ std::cerr << "Inconsistent JFS SGE configuration or non-CTP transport"
58+ << std::endl;
59+ return false;
60+ }
61+ }
62+ std::cout << "UB_TEST_TRANSPORT sge_mode=" << actual
63+ << " max_sge=" << max_sge << " tp=CTP priority=" << priority
64+ << " jfs_count=" << pool->io_jfs.size()
65+ << " jfr_count=" << pool->io_jfr.size()
66+ << std::endl;
67+ return true;
68+}
69+ 
70+#endif // EXAMPLE_UB_TEST_SGE_MODE_H
@@ -0,0 +1,35 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information
4+// regarding copyright ownership. The ASF licenses this file
5+// to you under the Apache License, Version 2.0 (the
6+// "License"); you may not use this file except in compliance
7+// with the License. You may obtain a copy of the License at
8+//
9+// http://www.apache.org/licenses/LICENSE-2.0
10+//
11+// Unless required by applicable law or agreed to in writing,
12+// software distributed under the License is distributed on an
13+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+// KIND, either express or implied. See the License for the
15+// specific language governing permissions and limitations
16+// under the License.
17+ 
18+syntax="proto2";
19+option cc_generic_services = true;
20+ 
21+package test;
22+ 
23+message PerfTestRequest {
24+ required bool echo_attachment = 1;
25+ required bytes name = 2;
26+};
27+ 
28+message PerfTestResponse {
29+ required string cpu_usage = 1;
30+ required bytes name = 2;
31+};
32+ 
33+service PerfTestService {
34+ rpc Test(PerfTestRequest) returns (PerfTestResponse);
35+};
@@ -23,6 +23,11 @@ include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
23option(LINK_SO "Whether examples are linked dynamically" OFF)23option(LINK_SO "Whether examples are linked dynamically" OFF)
24 24 
25brpc_example_find_common_deps(DYNAMIC_LIB)25brpc_example_find_common_deps(DYNAMIC_LIB)
26+option(WITH_TCMALLOC "Link tcmalloc" OFF)
27+if(WITH_TCMALLOC)
28+ find_library(TCMALLOC_LIB NAMES tcmalloc REQUIRED)
29+ list(APPEND DYNAMIC_LIB ${TCMALLOC_LIB})
30+endif()
26 31 
27protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto)32protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto)
28set(BRPC_EXAMPLE_WITH_URMA ON)33set(BRPC_EXAMPLE_WITH_URMA ON)
@@ -52,3 +57,7 @@ brpc_example_configure_target(urma_performance_server)
52 57 
53target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})58target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
54target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})59target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
60+ 
61+add_executable(urma_connection_stress connection_stress.cpp ${PROTO_SRC} ${PROTO_HEADER})
62+brpc_example_configure_target(urma_connection_stress)
63+target_link_libraries(urma_connection_stress PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
@@ -0,0 +1,371 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information
4+// regarding copyright ownership. The ASF licenses this file
5+// to you under the Apache License, Version 2.0 (the
6+// "License"); you may not use this file except in compliance
7+// with the License. You may obtain a copy of the License at
8+//
9+// http://www.apache.org/licenses/LICENSE-2.0
10+//
11+// Unless required by applicable law or agreed to in writing,
12+// software distributed under the License is distributed on an
13+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+// KIND, either express or implied. See the License for the
15+// specific language governing permissions and limitations
16+// under the License.
17+ 
18+// Bounded workers drive independent, persistent URMA connections. Each sweep
19+// visits every connection and checks its identity and response byte for byte.
20+#include <sys/resource.h>
21+#include <dirent.h>
22+#include <unistd.h>
23+#include <algorithm>
24+#include <atomic>
25+#include <fstream>
26+#include <functional>
27+#include <memory>
28+#include <set>
29+#include <sstream>
30+#include <string>
31+#include <vector>
32+#include <gflags/gflags.h>
33+#include "bthread/bthread.h"
34+#include "butil/logging.h"
35+#include "butil/time.h"
36+#include "brpc/channel.h"
37+#include "brpc/closure_guard.h"
38+#include "brpc/controller.h"
39+#include "brpc/details/controller_private_accessor.h"
40+#include "brpc/server.h"
41+#include "brpc/socket.h"
42+#include "brpc/urma/urma_helper.h"
43+#include "test.pb.h"
44+ 
45+DEFINE_string(mode, "client", "client or server");
46+DEFINE_string(servers, "127.0.0.1:18083,127.0.0.1:18084",
47+ "Comma-separated control endpoints of the SAME server process");
48+DEFINE_string(ports, "18083,18084", "Server listening ports");
49+DEFINE_int32(connections, 40000, "Simultaneously live independent connections");
50+DEFINE_int32(concurrency, 128, "Maximum outstanding RPCs / bthread workers");
51+DEFINE_int32(rounds, 3, "Complete sweeps after connection warmup");
52+DEFINE_int32(churn_percent, 10, "Percent of connections replaced after each sweep");
53+DEFINE_int32(idle_seconds, 10, "Idle hold after warmup and each sweep");
54+DEFINE_int32(timeout_ms, 10000, "Per-RPC timeout, with retries disabled");
55+DEFINE_string(sizes, "64,4032,4096,8192,65536", "Attachment sizes per sweep");
56+DEFINE_int32(report_seconds, 5, "Server resource report interval");
57+DEFINE_int32(server_threads, 16, "Server bthread worker threads");
58+ 
59+static std::vector<std::string> Split(const std::string& value) {
60+ std::istringstream in(value);
61+ std::vector<std::string> result;
62+ std::string part;
63+ while (std::getline(in, part, ',')) { result.push_back(part); }
64+ return result;
65+}
66+ 
67+static bool ParseNumbers(const std::string& value, size_t max,
68+ std::vector<size_t>* out) {
69+ for (const auto& item : Split(value)) {
70+ if (item.empty() || item.find_first_not_of("0123456789") != std::string::npos) {
71+ return false;
72+ }
73+ try {
74+ const auto n = std::stoull(item);
75+ if (n > max) { return false; }
76+ out->push_back(n);
77+ } catch (...) { return false; }
78+ }
79+ return !out->empty();
80+}
81+ 
82+static void Report(const char* phase, size_t connections) {
83+ size_t rss = 0, peak = 0, threads = 0, fds = 0;
84+ std::ifstream status("/proc/self/status");
85+ std::string line;
86+ while (std::getline(status, line)) {
87+ std::istringstream row(line);
88+ std::string key;
89+ size_t value;
90+ if (!(row >> key >> value)) { continue; }
91+ if (key == "VmRSS:") { rss = value; }
92+ if (key == "VmHWM:") { peak = value; }
93+ if (key == "Threads:") { threads = value; }
94+ }
95+ DIR* directory = opendir("/proc/self/fd");
96+ if (directory) {
97+ while (dirent* entry = readdir(directory)) {
98+ if (entry->d_name[0] != '.') { ++fds; }
99+ }
100+ closedir(directory);
101+ --fds; // The directory descriptor itself.
102+ }
103+ LOG(INFO) << "STRESS_RESOURCE phase=" << phase << " connections=" << connections
104+ << " rss_kib=" << rss << " peak_rss_kib=" << peak
105+ << " threads=" << threads << " fds=" << fds;
106+}
107+ 
108+class StressService : public test::PerfTestService {
109+public:
110+ std::function<size_t()> connection_count;
111+ void Test(google::protobuf::RpcController* controller,
112+ const test::PerfTestRequest* request,
113+ test::PerfTestResponse* response,
114+ google::protobuf::Closure* done) override {
115+ brpc::ClosureGuard guard(done);
116+ auto* cntl = static_cast<brpc::Controller*>(controller);
117+ brpc::Socket* socket = brpc::ControllerPrivateAccessor(cntl).get_sending_socket();
118+ std::ostringstream debug;
119+ if (socket) { brpc::Socket::DebugSocket(debug, socket->id()); }
120+ if (!socket || socket->socket_mode() != brpc::SOCKET_MODE_URMA ||
121+ debug.str().find("state=ESTABLISHED") == std::string::npos) {
122+ cntl->SetFailed("Established URMA required; TCP fallback rejected");
123+ return;
124+ }
125+ response->set_cpu_usage("URMA_ESTABLISHED");
126+ response->set_verification_socket_id(socket->id());
127+ if (request->query_connections()) {
128+ response->set_active_connections(connection_count());
129+ }
130+ if (request->echo_attachment()) {
131+ cntl->response_attachment().append(cntl->request_attachment());
132+ }
133+ }
134+};
135+ 
136+static int RunServer() {
137+ std::vector<size_t> ports;
138+ if (!ParseNumbers(FLAGS_ports, 65535, &ports) ||
139+ std::set<size_t>(ports.begin(), ports.end()).size() != ports.size() ||
140+ std::find(ports.begin(), ports.end(), 0) != ports.end()) { return 1; }
141+ StressService service;
142+ std::vector<std::unique_ptr<brpc::Server>> servers;
143+ service.connection_count = [&servers]() {
144+ size_t count = 0;
145+ for (const auto& server : servers) {
146+ brpc::ServerStatistics stat;
147+ server->GetStat(&stat);
148+ count += stat.connection_count;
149+ }
150+ return count;
151+ };
152+ for (auto port : ports) {
153+ std::unique_ptr<brpc::Server> server(new brpc::Server);
154+ brpc::ServerOptions options;
155+ options.socket_mode = brpc::SOCKET_MODE_URMA;
156+ options.idle_timeout_sec = -1;
157+ options.num_threads = FLAGS_server_threads;
158+ if (server->AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE) != 0 ||
159+ server->Start(port, &options) != 0 || !brpc::urma::IsUrmaAvailable()) {
160+ return 1;
161+ }
162+ servers.emplace_back(std::move(server));
163+ }
164+ LOG(INFO) << "STRESS_READY ports=" << FLAGS_ports;
165+ int ticks = 0;
166+ while (!brpc::IsAskedToQuit()) {
167+ if (ticks++ % FLAGS_report_seconds == 0) {
168+ Report("server", service.connection_count());
169+ }
170+ sleep(1);
171+ }
172+ for (auto& server : servers) { server->Stop(0); }
173+ for (auto& server : servers) { server->Join(); }
174+ return 0;
175+}
176+ 
177+struct Connection {
178+ std::unique_ptr<brpc::Channel> channel;
179+ uint64_t peer_id = brpc::INVALID_SOCKET_ID;
180+ uint64_t retired_id = brpc::INVALID_SOCKET_ID;
181+ unsigned generation = 0;
182+};
183+ 
184+struct Sweep {
185+ std::vector<Connection>* connections;
186+ const std::vector<std::string>* servers;
187+ const std::vector<size_t>* sizes;
188+ int round;
189+ std::atomic<size_t> next{0};
190+ std::atomic<size_t> passed{0};
191+ std::atomic<uint64_t> bytes{0};
192+ std::atomic<bool> failed{false};
193+ 
194+ static void* Run(void* arg) {
195+ auto& sweep = *static_cast<Sweep*>(arg);
196+ while (!sweep.failed.load()) {
197+ const size_t index = sweep.next.fetch_add(1);
198+ if (index >= sweep.connections->size()) { break; }
199+ Connection& connection = (*sweep.connections)[index];
200+ if (!connection.channel) {
201+ brpc::ChannelOptions options;
202+ options.socket_mode = brpc::SOCKET_MODE_URMA;
203+ options.connection_type = "single";
204+ options.connection_group = "urma-stress-" + std::to_string(index) +
205+ "-" + std::to_string(connection.generation);
206+ options.timeout_ms = FLAGS_timeout_ms;
207+ options.connect_timeout_ms = FLAGS_timeout_ms;
208+ options.max_retry = 0;
209+ connection.channel.reset(new brpc::Channel);
210+ if (connection.channel->Init(
211+ (*sweep.servers)[index % sweep.servers->size()].c_str(), &options) != 0) {
212+ LOG(ERROR) << "STRESS_FAILED channel_init index=" << index;
213+ sweep.failed = true;
214+ break;
215+ }
216+ }
217+ const size_t size = sweep.round == 0 ? 64 :
218+ (*sweep.sizes)[(index + sweep.round - 1) % sweep.sizes->size()];
219+ std::string payload(size, '\0');
220+ uint32_t state = 0x9e3779b9u ^ static_cast<uint32_t>(index) ^
221+ (static_cast<uint32_t>(sweep.round) * 0x85ebca6bu) ^ connection.generation;
222+ for (char& byte : payload) {
223+ state ^= state << 13;
224+ state ^= state >> 17;
225+ state ^= state << 5;
226+ byte = static_cast<char>(state);
227+ }
228+ brpc::Controller cntl;
229+ cntl.set_log_id((uint64_t(sweep.round) << 32) | index);
230+ cntl.request_attachment().append(payload);
231+ test::PerfTestRequest request;
232+ test::PerfTestResponse response;
233+ request.set_echo_attachment(true);
234+ test::PerfTestService_Stub stub(connection.channel.get());
235+ stub.Test(&cntl, &request, &response, nullptr);
236+ const uint64_t peer = response.verification_socket_id();
237+ if (cntl.Failed() || response.cpu_usage() != "URMA_ESTABLISHED" ||
238+ !response.has_verification_socket_id() ||
239+ peer == brpc::INVALID_SOCKET_ID || peer == connection.retired_id ||
240+ (connection.peer_id != brpc::INVALID_SOCKET_ID && peer != connection.peer_id) ||
241+ cntl.response_attachment().to_string() != payload) {
242+ LOG(ERROR) << "STRESS_FAILED round=" << sweep.round << " index=" << index
243+ << " bytes=" << size << " error=" << cntl.ErrorText()
244+ << " peer=" << peer << " previous=" << connection.peer_id
245+ << " received=" << cntl.response_attachment().size();
246+ sweep.failed = true;
247+ break;
248+ }
249+ connection.peer_id = peer;
250+ sweep.bytes.fetch_add(size);
251+ sweep.passed.fetch_add(1);
252+ }
253+ return nullptr;
254+ }
255+};
256+ 
257+static bool CheckServerConnections(Connection& connection, size_t expected) {
258+ const int64_t deadline = butil::monotonic_time_us() + int64_t(FLAGS_timeout_ms) * 1000;
259+ size_t observed = 0;
260+ do {
261+ brpc::Controller cntl;
262+ test::PerfTestRequest request;
263+ test::PerfTestResponse response;
264+ request.set_echo_attachment(false);
265+ request.set_query_connections(true);
266+ test::PerfTestService_Stub stub(connection.channel.get());
267+ stub.Test(&cntl, &request, &response, nullptr);
268+ if (cntl.Failed() || response.cpu_usage() != "URMA_ESTABLISHED" ||
269+ !response.has_active_connections() ||
270+ response.verification_socket_id() != connection.peer_id) {
271+ LOG(ERROR) << "STRESS_FAILED connection query: " << cntl.ErrorText();
272+ return false;
273+ }
274+ observed = response.active_connections();
275+ if (observed == expected) { return true; }
276+ usleep(10000);
277+ } while (butil::monotonic_time_us() < deadline);
278+ LOG(ERROR) << "STRESS_FAILED server connections=" << observed << " expected=" << expected;
279+ return false;
280+}
281+ 
282+static int RunClient() {
283+ auto servers = Split(FLAGS_servers);
284+ std::vector<size_t> sizes;
285+ if (servers.empty() || !ParseNumbers(FLAGS_sizes, 1024 * 1024, &sizes)) { return 1; }
286+ std::vector<Connection> connections(FLAGS_connections);
287+ Report("before_connect", 0);
288+ uint64_t total = 0;
289+ for (int round = 0; round <= FLAGS_rounds; ++round) {
290+ Sweep sweep;
291+ sweep.connections = &connections;
292+ sweep.servers = &servers;
293+ sweep.sizes = &sizes;
294+ sweep.round = round;
295+ const int64_t start = butil::monotonic_time_us();
296+ std::vector<bthread_t> workers;
297+ for (int i = 0; i < std::min(FLAGS_concurrency, FLAGS_connections); ++i) {
298+ bthread_t id;
299+ if (bthread_start_background(&id, nullptr, Sweep::Run, &sweep) != 0) {
300+ sweep.failed = true;
301+ break;
302+ }
303+ workers.push_back(id);
304+ }
305+ for (auto id : workers) { bthread_join(id, nullptr); }
306+ if (sweep.failed || sweep.passed != connections.size()) { return 1; }
307+ // All endpoints must belong to one process, so SocketIds are globally unique.
308+ std::set<uint64_t> peers;
309+ for (const auto& connection : connections) { peers.insert(connection.peer_id); }
310+ if (peers.size() != connections.size()) {
311+ LOG(ERROR) << "STRESS_FAILED connections unexpectedly share sockets unique="
312+ << peers.size();
313+ return 1;
314+ }
315+ if (!CheckServerConnections(connections.front(), connections.size())) { return 1; }
316+ total += sweep.passed;
317+ LOG(INFO) << "STRESS_SWEEP_PASS round=" << round << " verified=" << sweep.passed
318+ << " unique_peer_socket_ids=" << peers.size()
319+ << " bytes=" << sweep.bytes
320+ << " elapsed_us=" << butil::monotonic_time_us() - start;
321+ // Measure the real idle footprint without forcing allocator reclamation.
322+ sleep(FLAGS_idle_seconds);
323+ Report(round == 0 ? "connected_idle" : "sweep_idle", connections.size());
324+ if (round > 0 && round < FLAGS_rounds && FLAGS_churn_percent > 0) {
325+ size_t changed = 0;
326+ for (size_t i = 0; i < connections.size(); ++i) {
327+ if ((i + round * 17) % 100 >= size_t(FLAGS_churn_percent)) { continue; }
328+ auto& connection = connections[i];
329+ connection.channel.reset();
330+ connection.retired_id = connection.peer_id;
331+ connection.peer_id = brpc::INVALID_SOCKET_ID;
332+ ++connection.generation;
333+ ++changed;
334+ }
335+ LOG(INFO) << "STRESS_CHURN closed=" << changed;
336+ }
337+ }
338+ for (size_t i = 1; i < connections.size(); ++i) { connections[i].channel.reset(); }
339+ if (!CheckServerConnections(connections.front(), 1)) { return 1; }
340+ LOG(INFO) << "STRESS_CLEANUP_PASS server_connections=1 monitoring_connection_only";
341+ connections.clear();
342+ sleep(FLAGS_idle_seconds);
343+ Report("released", 0);
344+ LOG(INFO) << "STRESS_PASS connections=" << FLAGS_connections
345+ << " verified_rpcs=" << total << " rounds=" << FLAGS_rounds
346+ << " concurrency=" << FLAGS_concurrency;
347+ return 0;
348+}
349+ 
350+int main(int argc, char** argv) {
351+ gflags::ParseCommandLineFlags(&argc, &argv, true);
352+ if (FLAGS_connections <= 0 || FLAGS_concurrency <= 0 || FLAGS_rounds <= 0 ||
353+ FLAGS_timeout_ms <= 0 || FLAGS_report_seconds <= 0 || FLAGS_idle_seconds < 0 ||
354+ FLAGS_server_threads <= 0 ||
355+ FLAGS_churn_percent < 0 || FLAGS_churn_percent > 100) { return 1; }
356+ // Raise only this test process's soft fd limit, never host-wide settings.
357+ rlimit limit;
358+ if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { return 1; }
359+ const rlim_t required = static_cast<rlim_t>(FLAGS_connections) + 4096;
360+ if (limit.rlim_cur < required) {
361+ if (limit.rlim_max < required) {
362+ LOG(ERROR) << "Need RLIMIT_NOFILE >= " << required;
363+ return 1;
364+ }
365+ limit.rlim_cur = required;
366+ if (setrlimit(RLIMIT_NOFILE, &limit) != 0) { return 1; }
367+ }
368+ if (FLAGS_mode == "server") { return RunServer(); }
369+ if (FLAGS_mode == "client") { return RunClient(); }
370+ return 1;
371+}
@@ -23,10 +23,14 @@ package test;
23 23 
24message PerfTestRequest {24message PerfTestRequest {
25 required bool echo_attachment = 1;25 required bool echo_attachment = 1;
26+ optional bool query_connections = 2;
26}27}
27 28 
28message PerfTestResponse {29message PerfTestResponse {
29 required string cpu_usage = 1;30 required string cpu_usage = 1;
31+ // Used by the connection stress test to detect connection sharing/reconnects.
32+ optional fixed64 verification_socket_id = 2;
33+ optional uint64 active_connections = 3;
30}34}
31 35 
32service PerfTestService {36service PerfTestService {
@@ -64,8 +64,11 @@ class RdmaHandshakeServerV3;
64}64}
65 65 
66namespace urma {66namespace urma {
67+class UrmaReceiveAdapter;
67class UrmaEndpoint;68class UrmaEndpoint;
68class UrmaExclusiveConnection;69class UrmaExclusiveConnection;
70+class UrmaPooledConnection;
71+class UrmaPooledResourceProvider;
69class UrmaConnect;72class UrmaConnect;
70class UrmaHandshakeClientV2;73class UrmaHandshakeClientV2;
71class UrmaHandshakeServerV2;74class UrmaHandshakeServerV2;
@@ -352,7 +355,10 @@ friend class rdma::RdmaHandshakeServerV2;
352friend class rdma::RdmaHandshakeClientV3;355friend class rdma::RdmaHandshakeClientV3;
353friend class rdma::RdmaHandshakeServerV3;356friend class rdma::RdmaHandshakeServerV3;
354friend class urma::UrmaEndpoint;357friend class urma::UrmaEndpoint;
358+friend class urma::UrmaReceiveAdapter;
355friend class urma::UrmaExclusiveConnection;359friend class urma::UrmaExclusiveConnection;
360+friend class urma::UrmaPooledConnection;
361+friend class urma::UrmaPooledResourceProvider;
356friend class urma::UrmaConnect;362friend class urma::UrmaConnect;
357friend class urma::UrmaHandshakeClientV2;363friend class urma::UrmaHandshakeClientV2;
358friend class urma::UrmaHandshakeServerV2;364friend class urma::UrmaHandshakeServerV2;
@@ -55,11 +55,11 @@ struct JfcState {
55 std::mutex mutex;55 std::mutex mutex;
56 std::deque<urma_cr_t> completions;56 std::deque<urma_cr_t> completions;
57 bool event_pending{false};57 bool event_pending{false};
58+ bool armed{false};
58};59};
59 60 
60struct PendingRecv {61struct PendingRecv {
61- uint64_t addr;62+ std::vector<urma_sge_t> sge;
62- uint32_t len;
63 uint64_t user_ctx;63 uint64_t user_ctx;
64};64};
65 65 
@@ -70,6 +70,9 @@ std::map<urma_context_t *, int> context_map;
70std::map<urma_jfce_t *, int> jfce_map;70std::map<urma_jfce_t *, int> jfce_map;
71std::map<urma_jfc_t *, JfcState *> jfc_state_map;71std::map<urma_jfc_t *, JfcState *> jfc_state_map;
72std::map<urma_jfr_t *, int> jfr_map;72std::map<urma_jfr_t *, int> jfr_map;
73+std::map<uint32_t, urma_jfr_t*> jfr_id_map;
74+std::map<urma_jfs_t*, int> jfs_map;
75+std::map<urma_target_jetty_t*, bool> target_is_jfr;
73// Side-table: JFR -> the JFC it was created with (used to route recv76// Side-table: JFR -> the JFC it was created with (used to route recv
74// completions to the right JfcState, since the JFR is an opaque handle).77// completions to the right JfcState, since the JFR is an opaque handle).
75std::map<urma_jfr_t *, urma_jfc_t *> jfr_jfc_map;78std::map<urma_jfr_t *, urma_jfc_t *> jfr_jfc_map;
@@ -98,7 +101,8 @@ void PushCompletion(urma_jfc_t* jfc, JfcState* state,
98 {101 {
99 std::lock_guard<std::mutex> lock(state->mutex);102 std::lock_guard<std::mutex> lock(state->mutex);
100 state->completions.push_back(completion);103 state->completions.push_back(completion);
101- if (!state->event_pending) {104+ if (state->armed && !state->event_pending) {
105+ state->armed = false;
102 state->event_pending = true;106 state->event_pending = true;
103 signal = true;107 signal = true;
104 }108 }
@@ -154,6 +158,9 @@ urma_status_t urma_uninit(void) {
154 }158 }
155 jfc_state_map.clear();159 jfc_state_map.clear();
156 jfr_map.clear();160 jfr_map.clear();
161+ jfr_id_map.clear();
162+ jfs_map.clear();
163+ target_is_jfr.clear();
157 jfr_jfc_map.clear();164 jfr_jfc_map.clear();
158 jfr_recv_map.clear();165 jfr_recv_map.clear();
159 seg_map.clear();166 seg_map.clear();
@@ -375,10 +382,12 @@ urma_jfr_t *urma_create_jfr(urma_context_t *ctx, urma_jfr_cfg_t *cfg) {
375 if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) {382 if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) {
376 return nullptr;383 return nullptr;
377 }384 }
378- // Opaque handle (avoids partially initializing urma_jfr_t's pthread385+ urma_jfr_t* jfr = new urma_jfr_t{};
379- // members). brpc's endpoint reads back jfr_cfg.jfc from the stored cfg386+ jfr->urma_ctx = ctx;
380- // pointer below, so we keep a side-table mapping jfr -> cfg.jfc.387+ jfr->jfr_cfg = *cfg;
381- urma_jfr_t *jfr = reinterpret_cast<urma_jfr_t *>(new int(1));388+ jfr->jfr_id.id = next_jetty_id.fetch_add(1);
389+ jfr->jfr_id.eid = mock_eid_info.eid;
390+ jfr_id_map[jfr->jfr_id.id] = jfr;
382 jfr_map[jfr] = 1;391 jfr_map[jfr] = 1;
383 // Stash the JFC for urma_post_jfr_wr's completion routing.392 // Stash the JFC for urma_post_jfr_wr's completion routing.
384 jfr_jfc_map[jfr] = cfg->jfc;393 jfr_jfc_map[jfr] = cfg->jfc;
@@ -391,10 +400,11 @@ urma_status_t urma_delete_jfr(urma_jfr_t *jfr) {
391 if (!jfr || jfr_map.find(jfr) == jfr_map.end()) {400 if (!jfr || jfr_map.find(jfr) == jfr_map.end()) {
392 return URMA_EINVAL;401 return URMA_EINVAL;
393 }402 }
403+ jfr_id_map.erase(jfr->jfr_id.id);
394 jfr_map.erase(jfr);404 jfr_map.erase(jfr);
395 jfr_jfc_map.erase(jfr);405 jfr_jfc_map.erase(jfr);
396 jfr_recv_map.erase(jfr);406 jfr_recv_map.erase(jfr);
397- delete reinterpret_cast<int *>(jfr);407+ delete jfr;
398 return URMA_SUCCESS;408 return URMA_SUCCESS;
399}409}
400 410 
@@ -549,6 +559,7 @@ urma_status_t urma_unimport_jetty(urma_target_jetty_t *tjetty) {
549 return URMA_EINVAL;559 return URMA_EINVAL;
550 }560 }
551 target_jetty_map.erase(tjetty);561 target_jetty_map.erase(tjetty);
562+ target_is_jfr.erase(tjetty);
552 delete tjetty;563 delete tjetty;
553 return URMA_SUCCESS;564 return URMA_SUCCESS;
554}565}
@@ -682,10 +693,10 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr,
682 }693 }
683 uint32_t copied = 0;694 uint32_t copied = 0;
684 for (uint32_t i = 0;695 for (uint32_t i = 0;
685- i < current->send.src.num_sge && copied < recv.len; ++i) {696+ i < current->send.src.num_sge && copied < recv.sge[0].len; ++i) {
686 const urma_sge_t& sge = current->send.src.sge[i];697 const urma_sge_t& sge = current->send.src.sge[i];
687- const uint32_t n = std::min(sge.len, recv.len - copied);698+ const uint32_t n = std::min(sge.len, recv.sge[0].len - copied);
688- std::memcpy(reinterpret_cast<void*>(recv.addr + copied),699+ std::memcpy(reinterpret_cast<void*>(recv.sge[0].addr + copied),
689 reinterpret_cast<const void*>(sge.addr), n);700 reinterpret_cast<const void*>(sge.addr), n);
690 copied += n;701 copied += n;
691 }702 }
@@ -725,6 +736,128 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr,
725 return URMA_SUCCESS;736 return URMA_SUCCESS;
726}737}
727 738 
739+ 
740+// Called under g_rw_mutex. Complete only after every DMA read/copy is done.
741+static urma_status_t PostSend(urma_jfc_t* local_jfc, urma_jfs_wr_t* wr,
742+ urma_jfs_wr_t** bad) {
743+ if (!wr || !jfc_state_map.count(local_jfc)) {
744+ if (bad) { *bad = wr; }
745+ return URMA_EINVAL;
746+ }
747+ for (auto* current = wr; current; current = current->next) {
748+ auto* target = current->tjetty;
749+ if (!target || !target_jetty_map.count(target)) {
750+ if (bad) { *bad = current; }
751+ return URMA_EINVAL;
752+ }
753+ urma_jfr_t* remote = nullptr;
754+ if (target_is_jfr[target]) {
755+ auto it = jfr_id_map.find(target->id.id);
756+ if (it != jfr_id_map.end()) { remote = it->second; }
757+ } else {
758+ auto it = jetty_id_map.find(target->id.id);
759+ if (it != jetty_id_map.end()) { remote = it->second->jetty_cfg.shared.jfr; }
760+ }
761+ auto recv_it = jfr_recv_map.find(remote);
762+ if (recv_it == jfr_recv_map.end() || recv_it->second.empty()) {
763+ if (bad) { *bad = current; }
764+ return URMA_EAGAIN;
765+ }
766+ size_t total = 0, capacity = 0;
767+ for (uint32_t i = 0; i < current->send.src.num_sge; ++i) {
768+ total += current->send.src.sge[i].len;
769+ }
770+ for (const auto& sge : recv_it->second.front().sge) { capacity += sge.len; }
771+ if (total > capacity || (target->tp_type == URMA_CTP && total > 4096)) {
772+ if (bad) { *bad = current; }
773+ return URMA_EINVAL;
774+ }
775+ PendingRecv recv = std::move(recv_it->second.front());
776+ recv_it->second.pop_front();
777+ // Scatter the concatenated send bytes into the posted receive SGEs.
778+ // This covers N->2 as well as the compile-time compatibility 1->2 path.
779+ size_t dst_index = 0, dst_offset = 0;
780+ for (uint32_t i = 0; i < current->send.src.num_sge; ++i) {
781+ const auto& src = current->send.src.sge[i];
782+ size_t offset = 0;
783+ while (offset < src.len) {
784+ auto& dst = recv.sge[dst_index];
785+ const size_t n = std::min<size_t>(src.len - offset, dst.len - dst_offset);
786+ std::memcpy(reinterpret_cast<void*>(dst.addr + dst_offset),
787+ reinterpret_cast<const void*>(src.addr + offset), n);
788+ offset += n;
789+ dst_offset += n;
790+ if (dst_offset == dst.len) { ++dst_index; dst_offset = 0; }
791+ }
792+ }
793+ urma_jfc_t* remote_jfc = jfr_jfc_map[remote];
794+ urma_cr_t rx{};
795+ rx.status = URMA_CR_SUCCESS;
796+ rx.user_ctx = recv.user_ctx;
797+ rx.flag.bs.s_r = 1;
798+ rx.completion_len = total;
799+ rx.opcode = current->opcode == URMA_OPC_SEND_IMM
800+ ? URMA_CR_OPC_SEND_WITH_IMM : URMA_CR_OPC_SEND;
801+ rx.imm_data = current->send.imm_data;
802+ PushCompletion(remote_jfc, jfc_state_map[remote_jfc], rx);
803+ if (current->flag.bs.complete_enable) {
804+ urma_cr_t tx{};
805+ tx.status = URMA_CR_SUCCESS;
806+ tx.user_ctx = current->user_ctx;
807+ tx.flag.bs.s_r = 0;
808+ PushCompletion(local_jfc, jfc_state_map[local_jfc], tx);
809+ }
810+ }
811+ if (bad) { *bad = nullptr; }
812+ return URMA_SUCCESS;
813+}
814+ 
815+urma_jfs_t* urma_create_jfs(urma_context_t* ctx, urma_jfs_cfg_t* cfg) {
816+ std::unique_lock<std::shared_mutex> lock(g_rw_mutex);
817+ if (!ctx || !cfg || !context_map.count(ctx) || !jfc_state_map.count(cfg->jfc)) {
818+ return nullptr;
819+ }
820+ auto* jfs = new urma_jfs_t{};
821+ jfs->urma_ctx = ctx;
822+ jfs->jfs_cfg = *cfg;
823+ jfs->jfs_id.id = next_jetty_id.fetch_add(1);
824+ jfs->jfs_id.eid = mock_eid_info.eid;
825+ jfs_map[jfs] = 1;
826+ return jfs;
827+}
828+urma_status_t urma_delete_jfs(urma_jfs_t* jfs) {
829+ std::unique_lock<std::shared_mutex> lock(g_rw_mutex);
830+ if (!jfs_map.erase(jfs)) { return URMA_EINVAL; }
831+ delete jfs;
832+ return URMA_SUCCESS;
833+}
834+urma_target_jetty_t* urma_import_jfr(urma_context_t* ctx,
835+ urma_rjfr_t* remote, urma_token_t* token) {
836+ std::unique_lock<std::shared_mutex> lock(g_rw_mutex);
837+ if (!ctx || !remote || !token || !context_map.count(ctx)) { return nullptr; }
838+ auto* target = new urma_target_jetty_t{};
839+ target->id = remote->jfr_id;
840+ target->tp_type = remote->tp_type;
841+ target_jetty_map[target] = 1;
842+ target_is_jfr[target] = true;
843+ return target;
844+}
845+urma_status_t urma_unimport_jfr(urma_target_jetty_t* target) {
846+ std::unique_lock<std::shared_mutex> lock(g_rw_mutex);
847+ if (!target_jetty_map.erase(target)) { return URMA_EINVAL; }
848+ target_is_jfr.erase(target);
849+ delete target;
850+ return URMA_SUCCESS;
851+}
852+urma_status_t urma_post_jfs_wr(urma_jfs_t* jfs, urma_jfs_wr_t* wr,
853+ urma_jfs_wr_t** bad) {
854+ std::unique_lock<std::shared_mutex> lock(g_rw_mutex);
855+ if (!jfs_map.count(jfs) || !wr || wr->send.src.num_sge > jfs->jfs_cfg.max_sge) {
856+ if (bad) { *bad = wr; }
857+ return URMA_EINVAL;
858+ }
859+ return PostSend(jfs->jfs_cfg.jfc, wr, bad);
860+}
728urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr,861urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr,
729 urma_jfr_wr_t **bad_wr) {862 urma_jfr_wr_t **bad_wr) {
730 std::unique_lock<std::shared_mutex> lock(g_rw_mutex);863 std::unique_lock<std::shared_mutex> lock(g_rw_mutex);
@@ -736,16 +869,18 @@ urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr,
736 return URMA_EINVAL;869 return URMA_EINVAL;
737 }870 }
738 for (urma_jfr_wr_t* current = wr; current; current = current->next) {871 for (urma_jfr_wr_t* current = wr; current; current = current->next) {
739- if (current->src.num_sge == 0 || !current->src.sge) {872+ if (current->src.num_sge == 0 || !current->src.sge ||
873+ current->src.num_sge > jfr->jfr_cfg.max_sge ||
874+ recv_it->second.size() >= jfr->jfr_cfg.depth) {
740 if (bad_wr) {875 if (bad_wr) {
741 *bad_wr = current;876 *bad_wr = current;
742 }877 }
743 return URMA_EINVAL;878 return URMA_EINVAL;
744 }879 }
745- recv_it->second.push_back(PendingRecv{880+ PendingRecv recv;
746- current->src.sge[0].addr,881+ recv.user_ctx = current->user_ctx;
747- current->src.sge[0].len,882+ recv.sge.assign(current->src.sge, current->src.sge + current->src.num_sge);
748- current->user_ctx});883+ recv_it->second.push_back(std::move(recv));
749 }884 }
750 if (bad_wr) {885 if (bad_wr) {
751 *bad_wr = nullptr;886 *bad_wr = nullptr;
@@ -786,13 +921,15 @@ int urma_poll_jfc(urma_jfc_t *jfc, int num_entries, urma_cr_t *cr_list) {
786 cr_list[count++] = state->completions.front();921 cr_list[count++] = state->completions.front();
787 state->completions.pop_front();922 state->completions.pop_front();
788 }923 }
789- if (state->completions.empty()) {
790- state->event_pending = false;
791- }
792 return count;924 return count;
793}925}
794 926 
795-urma_status_t urma_rearm_jfc(urma_jfc_t*, bool) {927+urma_status_t urma_rearm_jfc(urma_jfc_t* jfc, bool) {
928+ std::shared_lock<std::shared_mutex> lock(g_rw_mutex);
929+ auto it = jfc_state_map.find(jfc);
930+ if (it == jfc_state_map.end()) { return URMA_EINVAL; }
931+ std::lock_guard<std::mutex> state_lock(it->second->mutex);
932+ it->second->armed = true;
796 return URMA_SUCCESS;933 return URMA_SUCCESS;
797}934}
798 935 
@@ -0,0 +1,48 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information
4+// regarding copyright ownership. The ASF licenses this file
5+// to you under the Apache License, Version 2.0 (the
6+// "License"); you may not use this file except in compliance
7+// with the License. You may obtain a copy of the License at
8+//
9+// http://www.apache.org/licenses/LICENSE-2.0
10+//
11+// Unless required by applicable law or agreed to in writing,
12+// software distributed under the License is distributed on an
13+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+// KIND, either express or implied. See the License for the
15+// specific language governing permissions and limitations
16+// under the License.
17+ 
18+#ifndef BRPC_URMA_DELIVERY_H
19+#define BRPC_URMA_DELIVERY_H
20+#include "brpc/socket.h"
21+#include "brpc/urma/urma_resource.h"
22+#define BRPC_URMA_MAX_CQE_PER_POLL 32
23+#if BRPC_WITH_URMA
24+namespace brpc {
25+namespace urma {
26+// Callback-owned, bounded to one message across all logical connections.
27+// Declare before the CQ lock so destruction always happens after unlocking.
28+struct UrmaDeferredMessage {
29+ UrmaDeferredMessage() = default;
30+ InputMessageBase* message{nullptr};
31+ bool event_released{false};
32+ ~UrmaDeferredMessage() noexcept(false);
33+private:
34+ DISALLOW_COPY_AND_ASSIGN(UrmaDeferredMessage);
35+};
36+ 
37+// One poll result, never a wait for additional completions. Socket references
38+// keep endpoint storage alive until its accumulated bytes are dispatched.
39+struct UrmaRxParseBatch {
40+ SocketUniquePtr sockets[BRPC_URMA_MAX_CQE_PER_POLL];
41+ size_t size{0};
42+ void Add(SocketUniquePtr&);
43+};
44+ 
45+}
46+}
47+#endif
48+#endif
Msrc/brpc/urma/urma_endpoint.cpp+140-84File content is under review, please try again later
@@ -18,6 +18,7 @@
18#ifndef BRPC_URMA_ENDPOINT_H18#ifndef BRPC_URMA_ENDPOINT_H
19#define BRPC_URMA_ENDPOINT_H19#define BRPC_URMA_ENDPOINT_H
20 20 
21+#include <atomic>
21#include <cstdint>22#include <cstdint>
22#include <functional>23#include <functional>
23#include <ostream>24#include <ostream>
@@ -44,6 +45,7 @@ class UrmaTransport;
44namespace urma {45namespace urma {
45 46 
46class UrmaHandshake;47class UrmaHandshake;
48+class UrmaReceiveAdapter;
47struct ParsedHello;49struct ParsedHello;
48 50 
49DECLARE_bool(urma_use_polling);51DECLARE_bool(urma_use_polling);
@@ -77,8 +79,9 @@ private:
77// behind the connection-resource interface.79// behind the connection-resource interface.
78class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint {80class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint {
79 friend class UrmaConnect;81 friend class UrmaConnect;
82+ friend class UrmaReceiveAdapter;
80 friend class Socket;83 friend class Socket;
81- friend class UrmaTransport;84+ friend class ::brpc::UrmaTransport;
82 friend class UrmaHandshakeClientV2;85 friend class UrmaHandshakeClientV2;
83 friend class UrmaHandshakeServerV2;86 friend class UrmaHandshakeServerV2;
84 friend class UrmaHandshakeClientV3;87 friend class UrmaHandshakeClientV3;
@@ -109,12 +112,6 @@ public:
109 112 
110 int WaitWritable(butil::atomic<int>* butex, int expected,113 int WaitWritable(butil::atomic<int>* butex, int expected,
111 const timespec* deadline);114 const timespec* deadline);
112- static UrmaConnectionHandle LookupConnectionResource(SocketId id);
113- static void OnReceived(SocketId id, const UrmaConnectionHandle& source,
114- butil::IOBuf data);
115- static void OnResourceError(SocketId id, const UrmaConnectionHandle& source,
116- int error);
117- 
118 // For debug: dump endpoint state to @os.115 // For debug: dump endpoint state to @os.
119 void DebugInfo(std::ostream& os,116 void DebugInfo(std::ostream& os,
120 butil::StringPiece connector = "\n") const;117 butil::StringPiece connector = "\n") const;
@@ -180,8 +177,9 @@ private:
180 static void* ProcessHandshakeAtClient(void* arg);177 static void* ProcessHandshakeAtClient(void* arg);
181 static void* ProcessHandshakeAtServer(void* arg);178 static void* ProcessHandshakeAtServer(void* arg);
182 179 
183- // Allocate / deallocate the per-connection URMA resources (JFCE/JFC/JFR/180+ // Publish per-connection resources once. DeallocateResources logically
184- // Jetty, plus the CQ socket). Returns 0 on success.181+ // closes without detaching; destruction/quiescent Reset releases ownership.
182+ // AllocateResources returns 0 on success.
185 int AllocateResources(UrmaConnectionRole role);183 int AllocateResources(UrmaConnectionRole role);
186 void DeallocateResources();184 void DeallocateResources();
187 185 
@@ -198,10 +196,17 @@ private:
198 // after ESTABLISHED, matching the connection-ready boundary seen by user196 // after ESTABLISHED, matching the connection-ready boundary seen by user
199 // code.197 // code.
200 void DispatchReceivedBytes(SocketUniquePtr& s, ssize_t bytes);198 void DispatchReceivedBytes(SocketUniquePtr& s, ssize_t bytes);
201- void DispatchReceivedBytesLocked(SocketUniquePtr& s, ssize_t bytes);199+ // Caller holds _dispatch_mutex and owns last_msg through dispatch.
200+ void ParseReceivedBytesLocked(SocketUniquePtr& s, ssize_t bytes,
201+ InputMessageClosure& last_msg);
202+ // Caller must keep the Socket alive. Reset requires quiescent old users.
203+ UrmaConnectionResource* BorrowConnectionResource() const {
204+ return _published_resource.load(std::memory_order_acquire);
205+ }
202 UrmaConnectionHandle GetConnectionResource() const;206 UrmaConnectionHandle GetConnectionResource() const;
203 207 
204 inline void TryReadOnTcp();208 inline void TryReadOnTcp();
209+ int SendFallbackHello(int version);
205 void FallbackToTcp(UrmaTransport* transport, bool process_tcp);210 void FallbackToTcp(UrmaTransport* transport, bool process_tcp);
206 void FailHandshake(UrmaTransport* transport, int error,211 void FailHandshake(UrmaTransport* transport, int error,
207 const char* reason);212 const char* reason);
@@ -220,8 +225,13 @@ private:
220 butil::atomic<int64_t> _pending_received_bytes{0};225 butil::atomic<int64_t> _pending_received_bytes{0};
221 butil::Mutex _dispatch_mutex;226 butil::Mutex _dispatch_mutex;
222 227 
223- mutable butil::Mutex _resource_mutex;228+ // Write once before release-publication; never detach on failure/fallback.
229+ // Socket references pin this owner. Only destruction or quiescent Reset
230+ // may release it. TX contexts retain their independent DMA/import owners.
224 UrmaConnectionHandle _connection_resource;231 UrmaConnectionHandle _connection_resource;
232+ std::atomic<UrmaConnectionResource*> _published_resource{nullptr};
233+ // Protected by _dispatch_mutex; prevents publication after logical close.
234+ bool _resources_closed{false};
225 InputMessengerProcessor _input_processor;235 InputMessengerProcessor _input_processor;
226 236 
227 // Butex for waking the handshake bthread parked in ReadFromFd.237 // Butex for waking the handshake bthread parked in ReadFromFd.
@@ -1596,6 +1596,9 @@ int CreateUrmaResourceProvider(UrmaResourceMode mode,
1596 errno = EINVAL;1596 errno = EINVAL;
1597 return -1;1597 return -1;
1598 }1598 }
1599+ if (mode == UrmaResourceMode::POOLED) {
1600+ return CreatePooledUrmaResourceProvider(out);
1601+ }
1599 if (mode != UrmaResourceMode::EXCLUSIVE) {1602 if (mode != UrmaResourceMode::EXCLUSIVE) {
1600 errno = ENOTSUP;1603 errno = ENOTSUP;
1601 return -1;1604 return -1;
@@ -1606,13 +1609,22 @@ int CreateUrmaResourceProvider(UrmaResourceMode mode,
1606 1609 
1607// Intentionally process-lived, like the common registered-memory service.1610// Intentionally process-lived, like the common registered-memory service.
1608// No atexit cleanup: IOBuf TLS chains may outlive atexit handlers.1611// No atexit cleanup: IOBuf TLS chains may outlive atexit handlers.
1612+DEFINE_string(urma_resource_mode, "exclusive",
1613+ "URMA resource backend: exclusive or pooled (startup only)");
1614+bool UsingPooledUrmaResources() { return FLAGS_urma_resource_mode == "pooled"; }
1609static auto* g_provider = new std::unique_ptr<UrmaResourceProvider>;1615static auto* g_provider = new std::unique_ptr<UrmaResourceProvider>;
1610int InitializeUrmaResourceProvider(const UrmaResourceCallbacks& callbacks) {1616int InitializeUrmaResourceProvider(const UrmaResourceCallbacks& callbacks) {
1611 if (*g_provider) {1617 if (*g_provider) {
1612 errno = EALREADY;1618 errno = EALREADY;
1613 return -1;1619 return -1;
1614 }1620 }
1615- if (CreateUrmaResourceProvider(UrmaResourceMode::EXCLUSIVE, g_provider) !=1621+ if (FLAGS_urma_resource_mode != "exclusive" && !UsingPooledUrmaResources()) {
1622+ errno = EINVAL;
1623+ return -1;
1624+ }
1625+ const auto mode = UsingPooledUrmaResources() ? UrmaResourceMode::POOLED :
1626+ UrmaResourceMode::EXCLUSIVE;
1627+ if (CreateUrmaResourceProvider(mode, g_provider) !=
1616 0)1628 0)
1617 return -1;1629 return -1;
1618 return (*g_provider)->Initialize(callbacks);1630 return (*g_provider)->Initialize(callbacks);
@@ -105,6 +105,7 @@ void HelloMessage::Serialize(void* buf) const {
105 write64(seg_va);105 write64(seg_va);
106 write64(seg_len);106 write64(seg_len);
107 write32(seg_token_id);107 write32(seg_token_id);
108+ if (impl_ver == IMPL_POOLED_VERSION) write64(socket_id);
108}109}
109 110 
110void HelloMessage::Deserialize(const void* buf) {111void HelloMessage::Deserialize(const void* buf) {
@@ -145,6 +146,7 @@ void HelloMessage::Deserialize(const void* buf) {
145 seg_va = read64();146 seg_va = read64();
146 seg_len = read64();147 seg_len = read64();
147 seg_token_id = read32();148 seg_token_id = read32();
149+ if (impl_ver == IMPL_POOLED_VERSION) socket_id = read64();
148}150}
149 151 
150} // namespace v2_wire152} // namespace v2_wire
@@ -165,10 +167,11 @@ constexpr uint32_t MAX_V3_PB_SIZE = 4096;
165} // namespace167} // namespace
166 168 
167bool ValidHello(const ParsedHello& h) {169bool ValidHello(const ParsedHello& h) {
168- if (h.buffer_size < MIN_BUFFER_SIZE) {170+ if (h.pooled && (h.socket_id == INVALID_SOCKET_ID || h.tp_type != URMA_CTP)) return false;
171+ if (h.buffer_size < (h.pooled ? 1024 - sizeof(butil::IOBuf::Block) : MIN_BUFFER_SIZE)) {
169 return false;172 return false;
170 }173 }
171- if (h.recv_buffer_cnt < MIN_BUFFER_CNT || h.recv_buffer_cnt > MAX_BUFFER_CNT) {174+ if (h.recv_buffer_cnt < (h.pooled ? 1 : MIN_BUFFER_CNT) || h.recv_buffer_cnt > MAX_BUFFER_CNT) {
172 return false;175 return false;
173 }176 }
174 if (h.jetty_id == 0) {177 if (h.jetty_id == 0) {
@@ -193,17 +196,29 @@ bool ValidHello(const ParsedHello& h) {
193// and translates it into ParsedHello.196// and translates it into ParsedHello.
194int ReadBodyAndNegotiate(UrmaEndpoint* ep, ParsedHello* out, bool* negotiated) {197int ReadBodyAndNegotiate(UrmaEndpoint* ep, ParsedHello* out, bool* negotiated) {
195 *negotiated = false;198 *negotiated = false;
196- uint8_t body[v2_wire::HELLO_BODY_LEN];199+ uint8_t body[v2_wire::POOLED_BODY_LEN]{};
197- if (ep->ReadFromFd(body, v2_wire::HELLO_BODY_LEN) < 0) {200+ if (ep->ReadFromFd(body, 6) < 0) return -1;
198- return -1;201+ uint16_t length, version, implementation;
202+ std::memcpy(&length, body, 2);
203+ std::memcpy(&version, body + 2, 2);
204+ std::memcpy(&implementation, body + 4, 2);
205+ length = butil::NetToHost16(length);
206+ implementation = butil::NetToHost16(implementation);
207+ const bool pooled = implementation == v2_wire::IMPL_POOLED_VERSION;
208+ const size_t packet_size = pooled ? v2_wire::POOLED_PACKET_LEN : v2_wire::HELLO_PACKET_LEN;
209+ if (length < packet_size || length > v2_wire::HELLO_MSG_LEN_MAX) {
210+ errno = EPROTO; return -1;
199 }211 }
212+ if (butil::NetToHost16(version) != v2_wire::HELLO_V2_VERSION ||
213+ (implementation != v2_wire::IMPL_V2_VERSION && !pooled)) {
214+ return DrainBytes(ep, length - 4 - 6);
215+ }
216+ if (ep->ReadFromFd(body + 6, packet_size - 4 - 6) < 0) return -1;
200 v2_wire::HelloMessage m;217 v2_wire::HelloMessage m;
201 m.Deserialize(body);218 m.Deserialize(body);
202- if (m.msg_len < v2_wire::HELLO_MSG_LEN_MIN ||
203- m.msg_len > v2_wire::HELLO_MSG_LEN_MAX ||
204- m.hello_ver != v2_wire::HELLO_V2_VERSION ||
205- m.impl_ver != v2_wire::IMPL_V2_VERSION) { return 0; }
206 ParsedHello p;219 ParsedHello p;
220+ p.pooled = pooled;
221+ p.socket_id = m.socket_id;
207 p.buffer_size = m.buffer_size;222 p.buffer_size = m.buffer_size;
208 p.recv_buffer_cnt = m.recv_buffer_cnt;223 p.recv_buffer_cnt = m.recv_buffer_cnt;
209 p.jetty_id = m.jetty_id;224 p.jetty_id = m.jetty_id;
@@ -215,15 +230,13 @@ int ReadBodyAndNegotiate(UrmaEndpoint* ep, ParsedHello* out, bool* negotiated) {
215 p.seg_va = m.seg_va;230 p.seg_va = m.seg_va;
216 p.seg_len = m.seg_len;231 p.seg_len = m.seg_len;
217 p.seg_token_id = m.seg_token_id;232 p.seg_token_id = m.seg_token_id;
218- if (!ValidHello(p)) {
219- return 0;
220- }
221 // Drain trailing bytes if msg_len advertises more than the fixed body.233 // Drain trailing bytes if msg_len advertises more than the fixed body.
222- if (m.msg_len > v2_wire::HELLO_PACKET_LEN) {234+ if (m.msg_len > packet_size) {
223- if (DrainBytes(ep, m.msg_len - v2_wire::HELLO_PACKET_LEN) < 0) {235+ if (DrainBytes(ep, m.msg_len - packet_size) < 0) {
224 return -1;236 return -1;
225 }237 }
226 }238 }
239+ if (!ValidHello(p)) return 0;
227 *out = p;240 *out = p;
228 *negotiated = true;241 *negotiated = true;
229 return 0;242 return 0;
@@ -248,10 +261,10 @@ int DrainBytes(UrmaEndpoint* ep, size_t n) {
248int UrmaHandshakeClientV2::SendLocalHello() {261int UrmaHandshakeClientV2::SendLocalHello() {
249 v2_wire::HelloMessage m;262 v2_wire::HelloMessage m;
250 _ep->FillLocalHelloV2(&m);263 _ep->FillLocalHelloV2(&m);
251- uint8_t packet[v2_wire::HELLO_PACKET_LEN];264+ uint8_t packet[v2_wire::POOLED_PACKET_LEN]{};
252 std::memcpy(packet, "URMA", 4);265 std::memcpy(packet, "URMA", 4);
253 m.Serialize(packet + 4);266 m.Serialize(packet + 4);
254- return _ep->WriteToFd(packet, v2_wire::HELLO_PACKET_LEN);267+ return _ep->WriteToFd(packet, m.msg_len);
255}268}
256 269 
257int UrmaHandshakeClientV2::ReceiveAndParseRemoteHello(ParsedHello* out,270int UrmaHandshakeClientV2::ReceiveAndParseRemoteHello(ParsedHello* out,
@@ -288,10 +301,10 @@ int UrmaHandshakeServerV2::SendLocalHello() {
288 m.jetty_id = 0;301 m.jetty_id = 0;
289 m.buffer_size = 0;302 m.buffer_size = 0;
290 }303 }
291- uint8_t packet[v2_wire::HELLO_PACKET_LEN];304+ uint8_t packet[v2_wire::POOLED_PACKET_LEN]{};
292 std::memcpy(packet, "URMA", 4);305 std::memcpy(packet, "URMA", 4);
293 m.Serialize(packet + 4);306 m.Serialize(packet + 4);
294- return _ep->WriteToFd(packet, v2_wire::HELLO_PACKET_LEN);307+ return _ep->WriteToFd(packet, m.msg_len);
295}308}
296 309 
297// ============================================================================310// ============================================================================
@@ -35,6 +35,8 @@ class UrmaEndpoint;
35// then uses ApplyRemoteHello to size its send/recv windows and to drive35// then uses ApplyRemoteHello to size its send/recv windows and to drive
36// urma_import_seg / urma_import_jetty.36// urma_import_seg / urma_import_jetty.
37struct ParsedHello {37struct ParsedHello {
38+ bool pooled = false;
39+ uint64_t socket_id = UINT64_MAX;
38 uint32_t buffer_size = 0; // peer recv buffer size in bytes (per WR)40 uint32_t buffer_size = 0; // peer recv buffer size in bytes (per WR)
39 uint32_t recv_buffer_cnt = 0; // peer recv buffer count (RQ depth - 1)41 uint32_t recv_buffer_cnt = 0; // peer recv buffer count (RQ depth - 1)
40 uint32_t jetty_id = 0; // peer jetty id42 uint32_t jetty_id = 0; // peer jetty id
@@ -67,9 +69,13 @@ constexpr size_t HELLO_MSG_LEN_MIN = HELLO_PACKET_LEN;
67constexpr size_t HELLO_MSG_LEN_MAX = 4096;69constexpr size_t HELLO_MSG_LEN_MAX = 4096;
68constexpr uint16_t HELLO_V2_VERSION = 2;70constexpr uint16_t HELLO_V2_VERSION = 2;
69constexpr uint16_t IMPL_V2_VERSION = 1;71constexpr uint16_t IMPL_V2_VERSION = 1;
72+constexpr uint16_t IMPL_POOLED_VERSION = 2;
73+constexpr size_t POOLED_BODY_LEN = HELLO_BODY_LEN + 8;
74+constexpr size_t POOLED_PACKET_LEN = HELLO_PACKET_LEN + 8;
70 75 
71// The serializable struct. Aligned so it can be reinterpreted as raw bytes.76// The serializable struct. Aligned so it can be reinterpreted as raw bytes.
72struct HelloMessage {77struct HelloMessage {
78+ uint64_t socket_id = UINT64_MAX;
73 uint16_t msg_len; // total packet length (incl. magic)79 uint16_t msg_len; // total packet length (incl. magic)
74 uint16_t hello_ver;80 uint16_t hello_ver;
75 uint16_t impl_ver;81 uint16_t impl_ver;
@@ -29,7 +29,7 @@ option java_outer_classname = "UrmaHandshakeProto";
29message UrmaHello {29message UrmaHello {
30 required uint32 buffer_size = 1; // recv buffer size in bytes (per WR)30 required uint32 buffer_size = 1; // recv buffer size in bytes (per WR)
31 required uint32 recv_buffer_cnt = 2; // number of recv buffers posted31 required uint32 recv_buffer_cnt = 2; // number of recv buffers posted
32- required uint32 jetty_id = 3; // local jetty id32+ required uint32 jetty_id = 3; // Jetty, or shared JFR for resource_mode=2
33 required bytes eid = 4; // 16-byte local EID (network order)33 required bytes eid = 4; // 16-byte local EID (network order)
34 required uint32 uasid = 5; // local uasid34 required uint32 uasid = 5; // local uasid
35 required uint32 tp_type = 6; // urma_tp_type_t35 required uint32 tp_type = 6; // urma_tp_type_t
@@ -44,4 +44,6 @@ message UrmaHello {
44 required uint64 seg_va = 9; // segment virtual address44 required uint64 seg_va = 9; // segment virtual address
45 required uint64 seg_len = 10; // segment length in bytes45 required uint64 seg_len = 10; // segment length in bytes
46 required uint32 seg_token_id = 11; // segment token id46 required uint32 seg_token_id = 11; // segment token id
47+ optional uint64 socket_id = 12;
48+ optional uint32 resource_mode = 13 [default = 1]; // 1=exclusive, 2=pooled
47}49}
@@ -46,6 +46,8 @@
46#include "urma_types.h"46#include "urma_types.h"
47#include "brpc/urma/urma_bonding.h"47#include "brpc/urma/urma_bonding.h"
48#include "brpc/urma/urma_endpoint.h"48#include "brpc/urma/urma_endpoint.h"
49+#include "brpc/urma/urma_memory_pool.h"
50+#include "brpc/urma/urma_resource_pool.h"
49 51 
50DECLARE_int32(task_group_ntags);52DECLARE_int32(task_group_ntags);
51 53 
@@ -94,6 +96,9 @@ DEFINE_int32(urma_prepared_jetty_cnt, 8,
94DEFINE_int32(urma_buffer_size, 8 * 1024,96DEFINE_int32(urma_buffer_size, 8 * 1024,
95 "Per-buffer size in the URMA buffer pool (bytes). "97 "Per-buffer size in the URMA buffer pool (bytes). "
96 "Must match IOBuf block size to keep zero-copy working.");98 "Must match IOBuf block size to keep zero-copy working.");
99+DEFINE_int32(urma_ctp_buf_count, 48000, "Registered 4 KiB CTP blocks (pooled mode)");
100+DEFINE_int32(urma_header_buf_count, 65536, "Registered 64 B headers (pooled mode)");
101+DEFINE_int32(urma_memory_pool_limit_mb, 400, "Registered pool limit in MiB (pooled mode)");
97DEFINE_int32(urma_buffer_count, 65536,102DEFINE_int32(urma_buffer_count, 65536,
98 "Number of buffers in the URMA buffer pool.");103 "Number of buffers in the URMA buffer pool.");
99 104 
@@ -398,6 +403,7 @@ bool InitPool() {
398// All pool buffers share the same segment, so we return g_pool_seg for any403// All pool buffers share the same segment, so we return g_pool_seg for any
399// pool address; the per-WR length selects the slice.404// pool address; the per-WR length selects the slice.
400urma_target_seg_t* GetPoolSegFor(void* buf) {405urma_target_seg_t* GetPoolSegFor(void* buf) {
406+ if (UsingPooledUrmaResources()) { return GetPooledMemorySegFor(buf); }
401 if (g_skip_urma_init || !g_pool_seg || !g_pool_base) {407 if (g_skip_urma_init || !g_pool_seg || !g_pool_base) {
402 return nullptr;408 return nullptr;
403 }409 }
@@ -418,6 +424,9 @@ static void GlobalRelease() {
418 LOG(WARNING) << "Retain URMA services until resource shutdown completes";424 LOG(WARNING) << "Retain URMA services until resource shutdown completes";
419 return;425 return;
420 }426 }
427+ if (UsingPooledUrmaResources() && !ReleaseUrmaMemoryPools()) {
428+ return;
429+ }
421 if ((g_pool && g_pool->outstanding.load(butil::memory_order_acquire) != 0) ||430 if ((g_pool && g_pool->outstanding.load(butil::memory_order_acquire) != 0) ||
422 (g_pool_seg && urma_unregister_seg(g_pool_seg) != URMA_SUCCESS)) {431 (g_pool_seg && urma_unregister_seg(g_pool_seg) != URMA_SUCCESS)) {
423 LOG(WARNING) << "Retain URMA memory while registered blocks are live";432 LOG(WARNING) << "Retain URMA memory while registered blocks are live";
@@ -483,6 +492,36 @@ static bool GlobalUrmaInitializeImpl() {
483 return false;492 return false;
484 }493 }
485 494 
495+ const bool pooled = UsingPooledUrmaResources();
496+ gflags::CommandLineFlagInfo count_info;
497+ const size_t pool_count = pooled &&
498+ gflags::GetCommandLineFlagInfo("urma_buffer_count", &count_info) &&
499+ count_info.is_default ? 24000 : FLAGS_urma_buffer_count;
500+ if (pooled) {
501+ const int64_t rx_slots = int64_t(FLAGS_urma_io_jfr_cnt) * FLAGS_urma_io_rx_depth;
502+ const int64_t tx_slots = int64_t(FLAGS_urma_io_jfs_cnt) * FLAGS_urma_io_sq_depth;
503+ const int64_t pool_bytes = int64_t(FLAGS_urma_buffer_size) * pool_count +
504+ int64_t(64) * FLAGS_urma_header_buf_count +
505+ int64_t(URMA_CTP_BLOCK_SIZE) * FLAGS_urma_ctp_buf_count;
506+ if (FLAGS_urma_io_jfs_cnt <= 0 || FLAGS_urma_io_jfr_cnt <= 0 ||
507+ FLAGS_urma_io_sq_depth <= 0 || FLAGS_urma_io_rx_depth <= 0 ||
508+ FLAGS_urma_io_sq_depth > 65535 || FLAGS_urma_io_rx_depth > 65535 ||
509+ FLAGS_urma_buffer_size <= static_cast<int>(URMA_CTP_BLOCK_SIZE) ||
510+ FLAGS_urma_buffer_size % 64 != 0 ||
511+ pool_count <= 0 || FLAGS_urma_ctp_buf_count <= rx_slots ||
512+ FLAGS_urma_header_buf_count < rx_slots + tx_slots ||
513+ FLAGS_urma_memory_pool_limit_mb <= 0 ||
514+ pool_bytes > int64_t(FLAGS_urma_memory_pool_limit_mb) * 1024 * 1024) {
515+ LOG(ERROR) << "Invalid URMA queue, buffer, or poller configuration";
516+ errno = EINVAL;
517+ return false;
518+ }
519+ 
520+ if (FLAGS_urma_use_polling || IsBondingDeviceName(FLAGS_urma_device.c_str())) {
521+ errno = ENOTSUP;
522+ return false;
523+ }
524+ }
486 urma_init_attr_t init_attr{};525 urma_init_attr_t init_attr{};
487 const urma_status_t status = urma_init(&init_attr);526 const urma_status_t status = urma_init(&init_attr);
488 if (status != URMA_SUCCESS && status != URMA_EEXIST) {527 if (status != URMA_SUCCESS && status != URMA_EEXIST) {
@@ -583,7 +622,7 @@ static bool GlobalUrmaInitializeImpl() {
583 g_max_sge = FLAGS_urma_max_sge;622 g_max_sge = FLAGS_urma_max_sge;
584 }623 }
585 }624 }
586- g_recv_block_size =625+ g_recv_block_size = pooled ? URMA_CTP_MAX_MESSAGE_SIZE - sizeof(UrmaMsgHeader) :
587 static_cast<size_t>(FLAGS_urma_buffer_size) -626 static_cast<size_t>(FLAGS_urma_buffer_size) -
588 sizeof(butil::IOBuf::Block);627 sizeof(butil::IOBuf::Block);
589 628 
@@ -596,6 +635,13 @@ static bool GlobalUrmaInitializeImpl() {
596 return false;635 return false;
597 }636 }
598 637 
638+ if (pooled) {
639+ if (g_max_sge < 2 || IsUrmaBondingDevice() ||
640+ !InitializeUrmaMemoryPools(g_context, FLAGS_urma_buffer_size,
641+ pool_count, FLAGS_urma_header_buf_count, FLAGS_urma_ctp_buf_count)) {
642+ return false;
643+ }
644+ } else {
599 // Buffer pool.645 // Buffer pool.
600 g_pool = new BufferPool();646 g_pool = new BufferPool();
601 g_pool_buffer_size = static_cast<size_t>(FLAGS_urma_buffer_size);647 g_pool_buffer_size = static_cast<size_t>(FLAGS_urma_buffer_size);
@@ -620,6 +666,8 @@ static bool GlobalUrmaInitializeImpl() {
620 butil::iobuf::blockmem_deallocate = PoolDeallocate;666 butil::iobuf::blockmem_deallocate = PoolDeallocate;
621 butil::SetDefaultBlockSize(g_pool_buffer_size);667 butil::SetDefaultBlockSize(g_pool_buffer_size);
622 668 
669+ }
670+ 
623 if (UrmaEndpoint::GlobalInitialize() != 0) {671 if (UrmaEndpoint::GlobalInitialize() != 0) {
624 LOG(ERROR) << "Fail to initialize URMA endpoint resources";672 LOG(ERROR) << "Fail to initialize URMA endpoint resources";
625 return false;673 return false;
@@ -634,8 +682,9 @@ static bool GlobalUrmaInitializeImpl() {
634 LOG(INFO) << "URMA initialized: device=" << device_name682 LOG(INFO) << "URMA initialized: device=" << device_name
635 << " bonding=" << g_is_bonding_device683 << " bonding=" << g_is_bonding_device
636 << " max_sge=" << g_max_sge684 << " max_sge=" << g_max_sge
637- << " buffer_size=" << g_pool_buffer_size685+ << " pooled=" << pooled
638- << " buffer_count=" << g_pool->buffer_count();686+ << " buffer_size=" << FLAGS_urma_buffer_size
687+ << " buffer_count=" << pool_count;
639 return true;688 return true;
640}689}
641 690 
@@ -0,0 +1,291 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information
4+// regarding copyright ownership. The ASF licenses this file
5+// to you under the Apache License, Version 2.0 (the
6+// "License"); you may not use this file except in compliance
7+// with the License. You may obtain a copy of the License at
8+//
9+// http://www.apache.org/licenses/LICENSE-2.0
10+//
11+// Unless required by applicable law or agreed to in writing,
12+// software distributed under the License is distributed on an
13+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+// KIND, either express or implied. See the License for the
15+// specific language governing permissions and limitations
16+// under the License.
17+ 
18+#include "brpc/urma/urma_memory_pool.h"
19+ 
20+#if BRPC_WITH_URMA
21+#include <sys/mman.h>
22+#include <pthread.h>
23+#include <atomic>
24+#include <cerrno>
25+#include <cstdlib>
26+#include <functional>
27+#include <limits>
28+#include <vector>
29+#include "butil/iobuf.h"
30+#include "butil/logging.h"
31+#include "butil/scoped_lock.h"
32+#include "butil/synchronization/lock.h"
33+ 
34+namespace butil {
35+namespace iobuf {
36+extern void* (*blockmem_allocate)(size_t);
37+extern void (*blockmem_deallocate)(void*);
38+}
39+}
40+ 
41+namespace brpc {
42+namespace urma {
43+namespace {
44+const size_t kShards = 64;
45+class RegisteredPool : public UrmaMemoryPool {
46+public:
47+ RegisteredPool(size_t size, size_t count)
48+ : _block_size(size), _count(count) {}
49+ ~RegisteredPool() override {
50+ // Never invalidate DMA or application-owned blocks on a failed cleanup.
51+ if (_outstanding.load() == 0 && Unregister() && _base) {
52+ munmap(_base, _block_size * _count);
53+ }
54+ }
55+ bool Init(urma_context_t* context) {
56+ if (!context || !_count || _block_size < sizeof(void*) ||
57+ _block_size % alignof(void*) ||
58+ _count > std::numeric_limits<size_t>::max() / _block_size) {
59+ errno = EINVAL;
60+ return false;
61+ }
62+ _base = mmap(nullptr, _block_size * _count, PROT_READ | PROT_WRITE,
63+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
64+ if (_base == MAP_FAILED) {
65+ _base = nullptr;
66+ return false;
67+ }
68+ urma_seg_cfg_t cfg{};
69+ cfg.va = reinterpret_cast<uint64_t>(_base);
70+ cfg.len = _block_size * _count;
71+ cfg.flag.bs.token_policy = URMA_TOKEN_NONE;
72+ cfg.flag.bs.cacheable = URMA_NON_CACHEABLE;
73+ cfg.flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE |
74+ URMA_ACCESS_ATOMIC;
75+ _seg = urma_register_seg(context, &cfg);
76+ if (!_seg) {
77+ return false;
78+ }
79+ _used.assign(_count, 0);
80+ for (size_t i = 0; i < _count; ++i) {
81+ _free[i % kShards].push_back(static_cast<char*>(_base) +
82+ i * _block_size);
83+ }
84+ return true;
85+ }
86+ void* Allocate(size_t size) override {
87+ if (!_seg) { errno = ENODEV; return nullptr; }
88+ if (size > _block_size) {
89+ errno = EMSGSIZE;
90+ return nullptr;
91+ }
92+ const size_t start = std::hash<pthread_t>()(pthread_self()) % kShards;
93+ for (size_t i = 0; i < kShards; ++i) {
94+ const size_t shard = (start + i) % kShards;
95+ BAIDU_SCOPED_LOCK(_mutex[shard]);
96+ if (_free[shard].empty()) {
97+ continue;
98+ }
99+ void* p = _free[shard].back();
100+ _free[shard].pop_back();
101+ _used[Index(p)] = 1;
102+ _outstanding.fetch_add(1, std::memory_order_relaxed);
103+ return p;
104+ }
105+ errno = ENOMEM;
106+ return nullptr;
107+ }
108+ void Deallocate(void* p) override {
109+ if (!p) {
110+ return;
111+ }
112+ CHECK(Owns(p));
113+ CHECK_EQ((reinterpret_cast<uintptr_t>(p) -
114+ reinterpret_cast<uintptr_t>(_base)) % _block_size, 0u);
115+ const size_t index = Index(p);
116+ {
117+ BAIDU_SCOPED_LOCK(_mutex[index % kShards]);
118+ CHECK(_used[index]);
119+ _used[index] = 0;
120+ _free[index % kShards].push_back(p);
121+ }
122+ // Publish the last return only after releasing the shard lock.
123+ _outstanding.fetch_sub(1, std::memory_order_release);
124+ }
125+ bool Owns(const void* p) const override {
126+ const uintptr_t base = reinterpret_cast<uintptr_t>(_base);
127+ const uintptr_t addr = reinterpret_cast<uintptr_t>(p);
128+ return _base && addr >= base && addr - base < _block_size * _count;
129+ }
130+ urma_target_seg_t* SegmentFor(const void* p) const override {
131+ return !p || Owns(p) ? _seg : nullptr;
132+ }
133+ size_t outstanding() const override { return _outstanding.load(); }
134+ bool Unregister() override {
135+ if (_seg && urma_unregister_seg(_seg) != URMA_SUCCESS) {
136+ return false;
137+ }
138+ _seg = nullptr;
139+ return true;
140+ }
141+private:
142+ size_t Index(const void* p) const {
143+ return (reinterpret_cast<uintptr_t>(p) -
144+ reinterpret_cast<uintptr_t>(_base)) / _block_size;
145+ }
146+ size_t _block_size;
147+ size_t _count;
148+ void* _base{nullptr};
149+ urma_target_seg_t* _seg{nullptr};
150+ butil::Mutex _mutex[kShards];
151+ std::vector<void*> _free[kShards];
152+ std::vector<char> _used;
153+ std::atomic<size_t> _outstanding{0};
154+};
155+ 
156+// These allocators intentionally live until process exit: IOBuf has TLS block
157+// caches. On rollback unregister DMA but retain mappings and the deallocator
158+// while any application/TLS block still refers to them.
159+UrmaMemoryPoolFactory g_factory = CreateUrmaMemoryPool;
160+UrmaMemoryPool* g_data_pool = nullptr;
161+UrmaMemoryPool* g_header_pool = nullptr;
162+UrmaMemoryPool* g_ctp_pool = nullptr;
163+void* (*g_original_allocate)(size_t) = nullptr;
164+void (*g_original_deallocate)(void*) = nullptr;
165+size_t g_original_block_size = 0;
166+void* AllocateIOBuf(size_t size) {
167+ // Explicit CTP blocks must remain registered; never fall back to malloc.
168+ if (size == URMA_CTP_BLOCK_SIZE && g_ctp_pool) { return AllocCtpBuffer(); }
169+ void* p = g_data_pool->Allocate(size);
170+ return p ? p : g_original_allocate(size);
171+}
172+void DeallocateIOBuf(void* p) {
173+ if (g_ctp_pool && g_ctp_pool->Owns(p)) {
174+ g_ctp_pool->Deallocate(p);
175+ } else if (g_data_pool->Owns(p)) {
176+ g_data_pool->Deallocate(p);
177+ } else {
178+ g_original_deallocate(p);
179+ }
180+}
181+} // namespace
182+ 
183+std::unique_ptr<UrmaMemoryPool> CreateUrmaMemoryPool(
184+ urma_context_t* context, size_t size, size_t count) {
185+ std::unique_ptr<RegisteredPool> pool(new RegisteredPool(size, count));
186+ if (!pool->Init(context)) {
187+ return nullptr;
188+ }
189+ return std::unique_ptr<UrmaMemoryPool>(pool.release());
190+}
191+ 
192+bool SetUrmaMemoryPoolFactory(UrmaMemoryPoolFactory factory) {
193+ if (g_data_pool || g_header_pool || g_ctp_pool) { errno = EBUSY; return false; }
194+ g_factory = factory ? factory : CreateUrmaMemoryPool;
195+ return true;
196+}
197+bool InitializeUrmaMemoryPools(urma_context_t* context, size_t size,
198+ size_t count, size_t headers, size_t ctp_count) {
199+ if (g_data_pool || g_header_pool || g_ctp_pool) {
200+ errno = EALREADY;
201+ return false;
202+ }
203+ auto data = g_factory(context, size, count);
204+ auto header = g_factory(context, 64, headers);
205+ auto ctp = g_factory(context, URMA_CTP_BLOCK_SIZE, ctp_count);
206+ if (!data || !header || !ctp) {
207+ return false;
208+ }
209+ return InstallUrmaMemoryPools(std::move(data), std::move(header),
210+ std::move(ctp), size);
211+}
212+ 
213+bool InstallUrmaMemoryPools(std::unique_ptr<UrmaMemoryPool> data,
214+ std::unique_ptr<UrmaMemoryPool> header,
215+ std::unique_ptr<UrmaMemoryPool> ctp, size_t size) {
216+ if (g_data_pool || g_header_pool || g_ctp_pool || !data || !header || !ctp ||
217+ size <= sizeof(butil::IOBuf::Block)) {
218+ errno = EINVAL;
219+ return false;
220+ }
221+ g_data_pool = data.release();
222+ g_header_pool = header.release();
223+ g_ctp_pool = ctp.release();
224+ g_original_allocate = butil::iobuf::blockmem_allocate;
225+ g_original_deallocate = butil::iobuf::blockmem_deallocate;
226+ g_original_block_size = butil::GetDefaultBlockSize();
227+ butil::iobuf::blockmem_allocate = AllocateIOBuf;
228+ butil::iobuf::blockmem_deallocate = DeallocateIOBuf;
229+ butil::SetDefaultBlockSize(size);
230+ return true;
231+}
232+ 
233+bool ReleaseUrmaMemoryPools() {
234+ // Delivered IOBufs may still own CTP blocks after queues have stopped.
235+ // Their deleters require both the pool object and its registration.
236+ if (g_ctp_pool) {
237+ if (g_ctp_pool->outstanding() != 0 || !g_ctp_pool->Unregister()) {
238+ return false;
239+ }
240+ delete g_ctp_pool;
241+ g_ctp_pool = nullptr;
242+ }
243+ if (g_header_pool) {
244+ if (g_header_pool->outstanding() != 0 || !g_header_pool->Unregister()) {
245+ return false;
246+ }
247+ delete g_header_pool;
248+ g_header_pool = nullptr;
249+ }
250+ if (g_data_pool) {
251+ if (!g_data_pool->Unregister()) { return false; }
252+ butil::SetDefaultBlockSize(g_original_block_size);
253+ if (g_data_pool->outstanding() == 0) {
254+ butil::iobuf::blockmem_allocate = g_original_allocate;
255+ butil::iobuf::blockmem_deallocate = g_original_deallocate;
256+ delete g_data_pool;
257+ g_data_pool = nullptr;
258+ }
259+ }
260+ return true;
261+}
262+void* AllocHeaderBuffer() {
263+ return g_header_pool ? g_header_pool->Allocate(64) : nullptr;
264+}
265+void FreeHeaderBuffer(void* p) {
266+ if (p) { g_header_pool->Deallocate(p); }
267+}
268+urma_target_seg_t* GetHeaderPoolSegFor(void* p) {
269+ return g_header_pool ? g_header_pool->SegmentFor(p) : nullptr;
270+}
271+void* AllocCtpBuffer() {
272+ if (!g_ctp_pool) { errno = ENODEV; return nullptr; }
273+ return g_ctp_pool->Allocate(URMA_CTP_BLOCK_SIZE);
274+}
275+void FreeCtpBuffer(void* p) {
276+ if (p) { g_ctp_pool->Deallocate(p); }
277+}
278+urma_target_seg_t* GetCtpPoolSegFor(void* p) {
279+ return g_ctp_pool ? g_ctp_pool->SegmentFor(p) : nullptr;
280+}
281+urma_target_seg_t* GetPooledMemorySegFor(void* p) {
282+ // nullptr keeps advertising the standard pool in the handshake. Address
283+ // lookup includes CTP buffers, so echoing received data stays zero-copy.
284+ if (p && g_ctp_pool && g_ctp_pool->Owns(p)) {
285+ return g_ctp_pool->SegmentFor(p);
286+ }
287+ return g_data_pool ? g_data_pool->SegmentFor(p) : nullptr;
288+}
289+} // namespace urma
290+} // namespace brpc
291+#endif
@@ -0,0 +1,75 @@
1+// Licensed to the Apache Software Foundation (ASF) under one
2+// or more contributor license agreements. See the NOTICE file
3+// distributed with this work for additional information
4+// regarding copyright ownership. The ASF licenses this file
5+// to you under the Apache License, Version 2.0 (the
6+// "License"); you may not use this file except in compliance
7+// with the License. You may obtain a copy of the License at
8+//
9+// http://www.apache.org/licenses/LICENSE-2.0
10+//
11+// Unless required by applicable law or agreed to in writing,
12+// software distributed under the License is distributed on an
13+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+// KIND, either express or implied. See the License for the
15+// specific language governing permissions and limitations
16+// under the License.
17+ 
18+#ifndef BRPC_URMA_MEMORY_POOL_H
19+#define BRPC_URMA_MEMORY_POOL_H
20+ 
21+#if BRPC_WITH_URMA
22+#include <cstddef>
23+#include <memory>
24+#include "urma_api.h"
25+ 
26+namespace brpc {
27+namespace urma {
28+ 
29+constexpr size_t URMA_CTP_BLOCK_SIZE = 4096;
30+ 
31+// Replacement boundary for registered storage. Allocate never returns an
32+// unregistered buffer. The owner must stop DMA before Unregister/ destruction.
33+class UrmaMemoryPool {
34+public:
35+ virtual ~UrmaMemoryPool() = default;
36+ virtual void* Allocate(size_t size) = 0;
37+ virtual void Deallocate(void* buffer) = 0;
38+ virtual urma_target_seg_t* SegmentFor(const void* address) const = 0;
39+ virtual bool Owns(const void* address) const = 0;
40+ virtual size_t outstanding() const = 0;
41+ virtual bool Unregister() = 0;
42+};
43+ 
44+// Fixed-size, sharded, registered mmap allocator; also used for raw 64B
45+// headers. Headers never enter the IOBuf block allocator.
46+std::unique_ptr<UrmaMemoryPool> CreateUrmaMemoryPool(
47+ urma_context_t* context, size_t block_size, size_t block_count);
48+ 
49+// Install alternative allocators before URMA initialization. Ownership moves to
50+// the process; the data allocator's block size becomes IOBuf's default size.
51+bool InstallUrmaMemoryPools(std::unique_ptr<UrmaMemoryPool> data,
52+ std::unique_ptr<UrmaMemoryPool> headers,
53+ std::unique_ptr<UrmaMemoryPool> ctp,
54+ size_t block_size);
55+typedef std::unique_ptr<UrmaMemoryPool> (*UrmaMemoryPoolFactory)(
56+ urma_context_t*, size_t, size_t);
57+// Call before global initialization; nullptr restores the default allocator.
58+bool SetUrmaMemoryPoolFactory(UrmaMemoryPoolFactory factory);
59+bool InitializeUrmaMemoryPools(urma_context_t* context,
60+ size_t block_size, size_t block_count,
61+ size_t header_count, size_t ctp_count);
62+// Returns false if a registered segment must remain alive.
63+bool ReleaseUrmaMemoryPools();
64+void* AllocHeaderBuffer();
65+void FreeHeaderBuffer(void* buffer);
66+urma_target_seg_t* GetHeaderPoolSegFor(void* buffer);
67+void* AllocCtpBuffer();
68+void FreeCtpBuffer(void* buffer);
69+urma_target_seg_t* GetCtpPoolSegFor(void* buffer);
70+urma_target_seg_t* GetPooledMemorySegFor(void* buffer);
71+ 
72+} // namespace urma
73+} // namespace brpc
74+#endif
75+#endif