Open
引入 URMA 统一资源框架,接入社区独占实现并支持共享资源池与有序收发 #41
CarbonadoRainAdd on 2 days ago
引入 URMA 统一资源框架,接入社区独占实现并支持共享资源池与有序收发 #41
Open
Total: 41 files changed+8506-314
| @@ -27,6 +27,7 @@ option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF) | |||
| 27 | option(WITH_SNAPPY "With snappy" OFF) | 27 | option(WITH_SNAPPY "With snappy" OFF) |
| 28 | option(WITH_RDMA "With RDMA" OFF) | 28 | option(WITH_RDMA "With RDMA" OFF) |
| 29 | option(WITH_URMA "With URMA (openEuler Unified Remote Memory Access)" OFF) | 29 | option(WITH_URMA "With URMA (openEuler Unified Remote Memory Access)" OFF) |
| 30 | +option(WITH_TCMALLOC "Link tcmalloc" OFF) | ||
| 30 | set(URMA_USE_MOCK "AUTO" CACHE STRING | 31 | set(URMA_USE_MOCK "AUTO" CACHE STRING |
| 31 | "URMA mock mode: AUTO, ON, or OFF (OFF requires liburma)") | 32 | "URMA mock mode: AUTO, ON, or OFF (OFF requires liburma)") |
| 32 | set_property(CACHE URMA_USE_MOCK PROPERTY STRINGS AUTO ON OFF) | 33 | set_property(CACHE URMA_USE_MOCK PROPERTY STRINGS AUTO ON OFF) |
| @@ -187,12 +188,25 @@ list(APPEND BRPC_COMMON_DEFINITIONS | |||
| 187 | __const__=__unused__ | 188 | __const__=__unused__ |
| 188 | _GNU_SOURCE | 189 | _GNU_SOURCE |
| 189 | USE_SYMBOLIZE | 190 | USE_SYMBOLIZE |
| 190 | - NO_TCMALLOC | ||
| 191 | __STDC_FORMAT_MACROS | 191 | __STDC_FORMAT_MACROS |
| 192 | __STDC_LIMIT_MACROS | 192 | __STDC_LIMIT_MACROS |
| 193 | __STDC_CONSTANT_MACROS | 193 | __STDC_CONSTANT_MACROS |
| 194 | ) | 194 | ) |
| 195 | 195 | ||
| 196 | +if(WITH_TCMALLOC) | ||
| 197 | + find_path(TCMALLOC_INCLUDE_PATH NAMES gperftools/tcmalloc.h) | ||
| 198 | + find_library(TCMALLOC_LIB NAMES tcmalloc) | ||
| 199 | + if(NOT TCMALLOC_INCLUDE_PATH OR NOT TCMALLOC_LIB) | ||
| 200 | + message(FATAL_ERROR "WITH_TCMALLOC requires gperftools headers and libtcmalloc") | ||
| 201 | + endif() | ||
| 202 | + if(WITH_ASAN) | ||
| 203 | + message(FATAL_ERROR "tcmalloc and AddressSanitizer cannot be enabled together") | ||
| 204 | + endif() | ||
| 205 | + list(APPEND BRPC_COMMON_INCLUDE_DIRS ${TCMALLOC_INCLUDE_PATH}) | ||
| 206 | +else() | ||
| 207 | + list(APPEND BRPC_COMMON_DEFINITIONS NO_TCMALLOC) | ||
| 208 | +endif() | ||
| 209 | + | ||
| 196 | if(WITH_ASAN) | 210 | if(WITH_ASAN) |
| 197 | list(APPEND BRPC_COMMON_COMPILE_OPTIONS -fsanitize=address) | 211 | list(APPEND BRPC_COMMON_COMPILE_OPTIONS -fsanitize=address) |
| 198 | list(APPEND BRPC_COMMON_LINK_OPTIONS -fsanitize=address) | 212 | list(APPEND BRPC_COMMON_LINK_OPTIONS -fsanitize=address) |
| @@ -468,6 +482,9 @@ if(WITH_UBRING) | |||
| 468 | endif() | 482 | endif() |
| 469 | 483 | ||
| 470 | set(BRPC_PRIVATE_LIBS "-lgflags -lprotobuf -lleveldb -lprotoc -lssl -lcrypto -ldl -lz") | 484 | set(BRPC_PRIVATE_LIBS "-lgflags -lprotobuf -lleveldb -lprotoc -lssl -lcrypto -ldl -lz") |
| 485 | +if(WITH_TCMALLOC) | ||
| 486 | + set(BRPC_PRIVATE_LIBS "${BRPC_PRIVATE_LIBS} -ltcmalloc") | ||
| 487 | +endif() | ||
| 471 | 488 | ||
| 472 | if(WITH_GLOG) | 489 | if(WITH_GLOG) |
| 473 | set(DYNAMIC_LIB ${GLOG_LIB} ${DYNAMIC_LIB}) | 490 | set(DYNAMIC_LIB ${GLOG_LIB} ${DYNAMIC_LIB}) |
| @@ -508,6 +525,9 @@ endif() | |||
| 508 | list(REMOVE_DUPLICATES BRPC_COMMON_INCLUDE_DIRS) | 525 | list(REMOVE_DUPLICATES BRPC_COMMON_INCLUDE_DIRS) |
| 509 | 526 | ||
| 510 | add_library(brpc_common_config INTERFACE) | 527 | add_library(brpc_common_config INTERFACE) |
| 528 | +if(WITH_TCMALLOC) | ||
| 529 | + target_link_libraries(brpc_common_config INTERFACE ${TCMALLOC_LIB}) | ||
| 530 | +endif() | ||
| 511 | # Generated protobuf headers live in the build directory. Search it before src/ | 531 | # Generated protobuf headers live in the build directory. Search it before src/ |
| 512 | # so leftover in-source *.pb.h (gitignored, produced by Make) cannot shadow | 532 | # so leftover in-source *.pb.h (gitignored, produced by Make) cannot shadow |
| 513 | # updated .proto files such as errno.proto. | 533 | # updated .proto files such as errno.proto. |
| @@ -1,195 +1,153 @@ | |||
| 1 | -# UrmaTransport:基于 URMA 的远程内存 RPC | 1 | +# URMA 共享队列传输 |
| 2 | 2 | ||
| 3 | -UrmaTransport 是使用 openEuler | 3 | +URMA 通过统一连接资源框架支持两种后端:`--urma_resource_mode=exclusive`(默认)保留每连接独占 Jetty 的实现;`--urma_resource_mode=pooled` 使用共享 JFS/JFR。下文池化架构、配置和示例均指显式选择 `pooled` 的情况。两种后端使用各自的资源描述;两端模式一致时建立 URMA,模式不匹配时协商回退到 TCP。详细边界见 [URMA 连接资源抽象](urma_resources.md)。 |
| 4 | -[URMA](https://atomgit.com/openeuler/umdk)(Unified Remote Memory Access)SDK | ||
| 5 | -实现的传输层。它是 | ||
| 6 | -[#3217](https://github.com/apache/brpc/discussions/3217) 中提出的路线 B, | ||
| 7 | -与基于 OBMM 的 UBRing 传输(路线 A, | ||
| 8 | -[#3226](https://github.com/apache/brpc/issues/3226))互补,承担大包/跨节点 | ||
| 9 | -高吞吐场景,共同构成路线 C(双后端)。 | ||
| 10 | 4 | ||
| 11 | -## 技术背景 | 5 | +UrmaTransport 使用 openEuler [UMDK](https://atomgit.com/openeuler/umdk) 的 |
| 6 | +URMA 接口承载 brpc 消息。当前数据通路采用可靠消息模式 `URMA_TM_RM`、 | ||
| 7 | +CTP 和独立的 JFS/JFR,通过共享硬件队列服务多个逻辑连接。 | ||
| 12 | 8 | ||
| 13 | -URMA 在 UMDK 支持的设备上提供 verbs 风格接口。当前实现创建可靠消息 | 9 | +## 资源组织 |
| 14 | -(`URMA_TM_RM`)Jetty,并使用 CTP 传输路径;通过 | ||
| 15 | -`urma_post_jetty_send_wr` 提交发送 WR,通过 `urma_post_jfr_wr` 提交接收 | ||
| 16 | -WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获取。 | ||
| 17 | 10 | ||
| 18 | -## 编译配置 | 11 | +`UrmaMemoryPool` 管理已注册内存,`UrmaResourcePool` 管理队列及其生命周期, |
| 12 | +`UrmaPooledConnection` 保存分片顺序和连接资源,`UrmaReceiveAdapter` 负责 | ||
| 13 | +接收路由与交付。`UrmaEndpoint` 保留握手、输入缓存和消息解析。 | ||
| 14 | +各层通过内存池工厂、资源池工厂和事件适配接口连接。 | ||
| 19 | 15 | ||
| 20 | -### CMake 编译 | 16 | +JFS 是发送队列,JFR 是接收队列。每个 slot 独立持有 JFC、JFCE 和用于 |
| 17 | +事件分发的 Socket。进程默认创建 48 个 JFS、64 个 JFR,队列数量不随 | ||
| 18 | +逻辑连接数量线性增长。每个连接保存分配的本地 JFR、对端资源引用、 | ||
| 19 | +SocketId、收发序号以及重排状态,不单独创建一套硬件队列。 | ||
| 21 | 20 | ||
| 22 | -```bash | 21 | +对端 JFR 导入缓存使用 EID、UASID 和 JFR ID 共同作为键,以引用计数 |
| 23 | -# 带 URMA 支持编译 brpc | 22 | +共享导入结果。连接销毁后,未完成发送仍持有相关资源引用,直到完成处理 |
| 24 | -cmake -B build -DWITH_URMA=ON | 23 | +释放它们。 |
| 25 | -make -C build -j$(nproc) | ||
| 26 | 24 | ||
| 27 | -# 编译 urma_performance 示例 | 25 | +## 注册内存池 |
| 28 | -cd example/urma_performance | ||
| 29 | -cmake -B build | ||
| 30 | -make -C build -j$(nproc) | ||
| 31 | -``` | ||
| 32 | 26 | ||
| 33 | -`WITH_URMA=ON` 使用上游 UMDK 头文件进行编译。CMake 优先使用系统安装的 | 27 | +默认内存池使用 mmap 分配连续内存并注册为 URMA segment,以固定大小的 |
| 34 | -SDK;找不到头文件时,会参照 Mooncake 的 mock 构建方式下载固定版本的 | 28 | +块进行分配和回收。数据池的默认块大小为 8KiB,供 IOBuf 使用;可用数据 |
| 35 | -UMDK commit,可通过 `DOWNLOAD_URMA_HEADERS=OFF` 禁止下载。找到 `liburma` 时使用 | 29 | +空间需扣除 IOBuf 的块头。CTP 池提供独立的 4KiB 块,用于接收 |
| 36 | -真实硬件数据通路,否则链接 brpc 的 mock,使 URMA 代码和测试仍可在无硬件 | 30 | +body;扣除块内 IOBuf 元数据后仍可容纳完整的 4032B body。独立 header |
| 37 | -环境编译。`URMA_USE_MOCK` 默认为 `AUTO`;设为 `ON` 可强制使用 mock,设为 | 31 | +池的块大小为 64B,不经过 IOBuf 块分配器。发送直接引用已注册 IOBuf, |
| 38 | -`OFF` 则要求找到 `liburma` 并排除 mock。 | 32 | +包括 8KiB 数据块和收到的 4KiB CTP 块,不为切换内存池而复制 body。 |
| 39 | 33 | ||
| 40 | -### Make 编译 | 34 | +空闲块分为 64 个分片,每个分片分别加锁。分配从当前线程对应的分片开始, |
| 35 | +必要时扫描其他分片。内存池初始化后不自动扩容;耗尽时返回失败。 | ||
| 36 | +注销注册内存前必须先结束 DMA。仍被应用 IOBuf 引用的数据块需要保留 | ||
| 37 | +其映射及回收器;CTP 池在所有应用引用归还前同时保留注册信息。 | ||
| 41 | 38 | ||
| 42 | -```bash | 39 | +`UrmaMemoryPool` 提供分配、回收、地址归属、segment 查询和注销接口。 |
| 43 | -sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \ | 40 | +可在全局初始化前通过 `SetUrmaMemoryPoolFactory` 更换实现,由全局初始化 |
| 44 | - --with-urma | 41 | +创建并安装数据池、CTP 池与 header 池。替代实现必须保持内存注册及引用生命周期 |
| 45 | -make -j4 | 42 | +约束。用户注册数据通过 `RegisterMemoryForUrma` 和 |
| 43 | +`DeregisterMemoryForUrma` 管理;发送期间不得注销仍被使用的内存。 | ||
| 46 | 44 | ||
| 47 | -# 强制使用 mock,或强制使用真实 liburma | 45 | +## 建链与协商 |
| 48 | -sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \ | ||
| 49 | - --with-urma --with-urma-mock | ||
| 50 | -sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \ | ||
| 51 | - --with-urma --without-urma-mock | ||
| 52 | -``` | ||
| 53 | 46 | ||
| 54 | -默认根据 `liburma` 是否存在自动选择;`--without-urma-mock` 要求配置路径中 | 47 | +连接先通过 TCP 完成握手,再切换到 URMA 数据通路。握手交换接收块大小、 |
| 55 | -存在 `liburma`。 | 48 | +接收队列信息、带版本的 SocketId、EID/UASID、传输路径类型和 segment |
| 49 | +描述。客户端支持二进制 v2 和 Protobuf v3 两种握手编码,默认使用 v2。 | ||
| 50 | +共享队列在 v2 使用 implementation=2,在 v3 使用 resource_mode=2; | ||
| 51 | +独占模式对应值为1。双方校验远端资源及地址范围。 | ||
| 56 | 52 | ||
| 57 | -### Bazel 编译 | 53 | +协商与资源导入完成后才发布 `URMA_ON` 状态。模式不匹配时完整消费 Hello, |
| 54 | +通过拒绝响应或 ACK=0 协商 TCP 回退,避免握手残留字节进入业务解析。 | ||
| 55 | +I/O 错误及非法资源描述仍按相应错误路径处理。并发连接各自持有握手对象, | ||
| 56 | +避免复用仍在进行中的回调状态。 | ||
| 58 | 57 | ||
| 59 | -```bash | 58 | +## 发送路径 |
| 60 | -# 编译带 URMA 支持的 brpc | ||
| 61 | -bazel build --define=BRPC_WITH_URMA=true //:brpc | ||
| 62 | 59 | ||
| 63 | -# 仅展示关闭下载的开关;头文件仍须由工具链或本地 BUILD 目标声明 | 60 | +每个 SEND 分片带有 64B header,包含目标 SocketId、16位序号和消息类型, |
| 64 | -bazel build --define=BRPC_WITH_URMA=true \ | 61 | +多字节字段使用网络字节序。CTP 的单次消息总长度不超过 4096B,因此 |
| 65 | - --define=BRPC_DOWNLOAD_URMA_HEADERS=false //:brpc | 62 | +header 之后最多承载 4032B body;实际长度还受双方接收能力约束。 |
| 66 | 63 | ||
| 67 | -# 编译并运行 URMA 单元测试(使用 mock,不需要 URMA 硬件) | 64 | +多 SGE 发送使用一个 header SGE,加上一个或多个 body SGE。 |
| 68 | -bazel test --define=BRPC_WITH_URMA=true \ | 65 | +发送函数跨输入 IOBuf 及其数据块收集 body,直到达到总 SGE 上限或 body |
| 69 | - --define=BRPC_URMA_USE_MOCK=true //test:brpc_unittests | 66 | +长度上限。已注册 body 直接引用原内存,不为拼接报文复制整个 payload。 |
| 67 | +未注册的片段不能直接作为 DMA 源。 | ||
| 70 | 68 | ||
| 71 | -# 使用真实 liburma;要求链接器可找到 -lurma | 69 | +接收采用 header/body 两个 SGE。 |
| 72 | -bazel build --define=BRPC_WITH_URMA=true \ | ||
| 73 | - --define=BRPC_URMA_USE_MOCK=false //:brpc | ||
| 74 | -``` | ||
| 75 | 70 | ||
| 76 | -Bazel 会自动获取固定 commit 的 UMDK 头文件,并通过 `@umdk//:urma_headers` | 71 | +默认按 SocketId 哈希确定首选 JFS。首选容量不足时,从最多4个不重复备选中 |
| 77 | -提供给 URMA 目标。设置 `BRPC_DOWNLOAD_URMA_HEADERS=false` 后,Bazel 不再依赖 | 72 | +选择在途请求最少者,一次最多尝试首选和一个备选。关闭 `urma_jfs_affinity` |
| 78 | -该外部仓库;实际构建仍需通过工具链或本地 BUILD 目标声明 | 73 | +时按批轮询首选。每个 JFS 的 MPSC 请求队列由单一执行者推进,单次最多 |
| 79 | -`urma_api.h`、`urma_types.h` 和 `urma_ubagg.h`。仅通过 | 74 | +链式提交 `urma_io_tx_batch_max` 个 WR;仅收集已发布请求,不等待凑满。 |
| 80 | -`--copt=-I/path` 指向工作区外目录,在默认 Bazel 沙箱下不会把未声明的头文件 | ||
| 81 | -带入编译输入。设置 | ||
| 82 | -`BRPC_URMA_USE_MOCK=true` 强制加入 mock;设置为 `false` 则排除 mock 并链接 | ||
| 83 | -`liburma`。未指定时,URMA 构建默认使用 mock。 | ||
| 84 | 75 | ||
| 85 | -## 使用 | 76 | +请求发布到资源队列即视为接受,调用方移除对应的连续字节前缀。资源层持有 |
| 77 | +header、SGE、数据和导入资源,直到提交及 CQE 引用均释放。暂未成功 post | ||
| 78 | +的后缀保留在队列中,由完成事件继续推进;若容量已在 post 返回前归还, | ||
| 79 | +直接重试后缀。共享 SQ 的等待者登记在首选 JFS,按容量通知,退出时全部唤醒。 | ||
| 86 | 80 | ||
| 87 | -通过在 channel / server 上设置 `socket_mode` 选择传输层: | 81 | +## 接收、重排与交付 |
| 88 | 82 | ||
| 89 | -```cpp | 83 | +每个 JFR 预投递多个接收上下文。接收完成后检查 header 和数据长度, |
| 90 | -// 客户端 | 84 | +接收适配器使用目标 SocketId 查找逻辑连接,在池化连接资源中按分片序号 |
| 91 | -brpc::ChannelOptions opt; | 85 | +恢复有序字节流,并在交付锁内再次核对资源身份后进入 Endpoint 输入缓存。 |
| 92 | -opt.socket_mode = brpc::SOCKET_MODE_URMA; | 86 | +顺序到达的分片走快速路径,不插入重排 map;乱序分片暂存,缺口补齐后 |
| 93 | -opt.protocol = "baidu_std"; // URMA 仅支持 baidu_std | 87 | +连续交付。序号处理支持16位回绕,重复或非法分片会使连接失败。 |
| 94 | -brpc::Channel channel; | ||
| 95 | -channel.Init("127.0.0.1:8003", &opt); | ||
| 96 | 88 | ||
| 97 | -// 服务端 | 89 | +接收上下文及 header 可在补收时复用,已交给应用的 body 由 IOBuf 引用 |
| 98 | -brpc::ServerOptions sopt; | 90 | +保活。IOBuf 交接使用 movable 转移引用,避免按值浅拷贝带来的额外引用 |
| 99 | -sopt.socket_mode = brpc::SOCKET_MODE_URMA; | 91 | +维护;这不等同于复制 payload。 |
| 100 | -server.Start(port, &sopt); | ||
| 101 | -``` | ||
| 102 | 92 | ||
| 103 | -若对端不支持 URMA(例如 TCP 客户端连接 URMA 服务端),在 4 字节 magic | 93 | +默认按一次 CQ poll 的结果收集需要解析的连接,完成该批次的接收处理后, |
| 104 | -握手后透明回退到 TCP,应用代码无需改动。 | 94 | +每个连接调用一次消息解析入口。此批次按连接合并解析调用,不改变消息 |
| 95 | +协议边界或同一连接内的字节顺序。 | ||
| 105 | 96 | ||
| 106 | -## 架构 | 97 | +完成事件处理支持延后执行最后一条消息:先完成 CQ 事件交接并释放相关锁, |
| 98 | +再允许整个交付范围内至多一条消息在当前 bthread 执行。其余消息通过 | ||
| 99 | +消息调度机制处理,避免业务回调持有共享队列锁。 | ||
| 107 | 100 | ||
| 108 | -UrmaTransport 沿用与 `RdmaTransport` / `UBShmTransport` 一致的两层设计: | 101 | +## 并发与退出 |
| 109 | 102 | ||
| 110 | -``` | 103 | +JFS 的发送执行者串行提交 WR,JFR/JFC 使用各自的接收和完成执行者。 |
| 111 | -UrmaTransport : public Transport (urma_transport.{h,cpp}) | 104 | +操作屏障协调 post、poll 与销毁;TX 使用原子登记数组,RX 使用固定上下文 |
| 112 | - +-- std::shared_ptr<TcpTransport> (回退路径) | 105 | +数组和 posted 标记。Socket 和资源池共同持有 slot 引用,防止排队的事件 |
| 113 | - +-- urma::UrmaEndpoint* (URMA 数据路径) | 106 | +回调访问已释放对象。逻辑连接仍使用重排锁和交付锁维护字节流顺序。 |
| 114 | - +-- UrmaState { URMA_ON, URMA_OFF, URMA_UNKNOWN } | ||
| 115 | 107 | ||
| 116 | -urma::UrmaEndpoint : public SocketUser (urma/urma_endpoint.{h,cpp}) | 108 | +关闭时先停止事件与队列活动,唤醒发送等待者,并按资源依赖释放本地队列、 |
| 117 | - +-- UrmaResource { jfc, jfce, jfr, jetty, remote_jetty, remote_seg } | 109 | +完成上下文和导入资源。内存注销以 DMA 结束为前提,存储释放还需保证应用引用不再使用它;不能 |
| 118 | - +-- 握手状态机(C/S 对称,在 TCP fd 上驱动) | 110 | +为了回收对象而提前释放设备或应用仍可能访问的内存。 |
| 119 | - +-- 发送路径:urma_post_jetty_send_wr(URMA_OPC_SEND) | ||
| 120 | - +-- 接收路径:urma_poll_jfc -> HandleCompletion -> InputMessenger | ||
| 121 | - +-- 双窗口信用流控(_remote_rq_window_size / _sq_window_size) | ||
| 122 | -``` | ||
| 123 | 111 | ||
| 124 | -### 建链流程(双平面) | 112 | +## 构建与启用 |
| 125 | 113 | ||
| 126 | -与 RDMA / UBRing 一致,控制面为 TCP,数据面为 URMA: | 114 | +在真实设备上启用该模块需要 UMDK 头文件和 `liburma`。CMake 使用 |
| 115 | +`WITH_URMA=ON`、`URMA_USE_MOCK=OFF`, | ||
| 116 | +优先查找安装的 SDK;找不到头文件时可下载固定版本的 UMDK 头文件, | ||
| 117 | +`DOWNLOAD_URMA_HEADERS=OFF` 可禁止下载。可通过 `URMA_ROOT`、 | ||
| 118 | +`URMA_INCLUDE_PATH`、`URMA_BOND_INCLUDE_PATH` 和 `URMA_LIB` 指定 SDK。 | ||
| 127 | 119 | ||
| 128 | -1. TCP 连接建立。 | 120 | +Make 使用 `config_brpc.sh` 的 `--with-urma --without-urma-mock`, |
| 129 | -2. `UrmaConnect::StartConnect` 起客户端握手 bthread。 | 121 | +并通过 `--headers`、`--libs` 提供依赖路径。Bazel 使用 `BRPC_WITH_URMA=true`、 |
| 130 | -3. 双方在 TCP fd 上交换 `UrmaHello` 消息(v2 二进制 magic `URMA`, | 122 | +`BRPC_URMA_USE_MOCK=false`,链接真实的 `liburma`; |
| 131 | - v3 protobuf magic `URM3`),携带本地 EID、jetty id、recv buffer 数量、 | 123 | +关闭 `BRPC_DOWNLOAD_URMA_HEADERS` 后,需要由工具链或本地依赖声明头文件。 |
| 132 | - 以及扁平化的 buffer 池 segment。 | ||
| 133 | -4. 双方先调用 `urma_import_seg` **再**调用 `urma_import_jetty`,为远端 EID | ||
| 134 | - 建立传输路径(TP)路由。跳过 `import_seg` 会导致首个 SEND 被硬件以 | ||
| 135 | - `URMA_CR_RNR_RETRY_CNT_EXC_ERR` 拒绝。 | ||
| 136 | -5. 4 字节 ACK(`HELLO_ACK_URMA_OK = 0x1`)确认双方均要 URMA。 | ||
| 137 | -6. 成功后 TCP fd 仅保留用于 epoll 生命周期和回退,数据走 URMA。 | ||
| 138 | 124 | ||
| 139 | -### 内存管理 | 125 | +客户端和服务端分别通过 `ChannelOptions::use_urma` 与 |
| 126 | +`ServerOptions::use_urma` 启用传输。当前支持 `baidu_std` 协议,使用 | ||
| 127 | +非 bonding 设备及事件驱动模式;不支持忙轮询、端到端 credit/ACK 流控, | ||
| 128 | +也不提供大包单边 READ/WRITE。大于单次 CTP 上限的数据继续分片 SEND。 | ||
| 140 | 129 | ||
| 141 | -申请一大段 `mmap` 内存,用 `urma_register_seg` 一次性注册,再切成固定大小 | 130 | +## 主要运行参数 |
| 142 | -buffer(默认 8KB)。劫持 `butil::iobuf::blockmem_allocate` 使每个 IOBuf | ||
| 143 | -block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs 构建 | ||
| 144 | -`urma_sge_t`,无需逐消息注册(与 RDMA `block_pool` 设计一致)。用户注册 | ||
| 145 | -内存通过 `urma::RegisterMemoryForUrma` / `DeregisterMemoryForUrma` 支持。 | ||
| 146 | 131 | ||
| 147 | -## 配置 | 132 | +| 参数 | 默认值 | 含义 | |
| 133 | +| --- | --- | --- | | ||
| 134 | +| `urma_device` | 空 | 指定设备;空值选择枚举到的首个设备 | | ||
| 135 | +| `urma_io_jfs_cnt` | 48 | 进程共享发送队列数量 | | ||
| 136 | +| `urma_io_jfr_cnt` | 64 | 进程共享接收队列数量 | | ||
| 137 | +| `urma_io_sq_depth` | 128 | 每个发送队列的在途槽位数 | | ||
| 138 | +| `urma_io_rx_depth` | 512 | 每个接收队列的预投递深度 | | ||
| 139 | +| `urma_jfs_affinity` | true | 按 SocketId 哈希选择固定首选 JFS | | ||
| 140 | +| `urma_io_tx_batch_max` | 8 | 每批准备与链式提交的 WR 上限,范围1至32 | | ||
| 141 | +| `urma_io_jfs_lock_free` | false | 设置 UMDK JFS 单执行者属性 | | ||
| 142 | +| `urma_io_jfr_lock_free` | false | 设置 UMDK JFR 单执行者属性 | | ||
| 143 | +| `urma_io_jfc_lock_free` | false | 设置 UMDK JFC 单执行者属性 | | ||
| 144 | +| `urma_buffer_size` | 8192 | 数据池块大小,包含 IOBuf 块头 | | ||
| 145 | +| `urma_ctp_buf_count` | 48000 | 4KiB CTP 块数量;必须大于接收预投递总数 | | ||
| 146 | +| `urma_max_sge` | 0 | 0采用设备能力;正值限制发送 SGE 数 | | ||
| 147 | +| `urma_cqe_poll_once` | 32 | 每次 poll 最多取出的 CQE 数 | | ||
| 148 | +| `urma_rx_batch_parse` | true | 同一 poll 中按连接合并解析调用 | | ||
| 149 | +| `urma_client_handshake_version` | 2 | 客户端握手编码,支持2或3 | | ||
| 148 | 150 | ||
| 149 | -所有 flag 使用 `urma_` 前缀(对标 RDMA 的 `rdma_` 前缀): | 151 | +队列数量、深度与内存池参数应在全局初始化前确定。接收预投递需要充足 |
| 150 | - | 152 | +CTP 块,header 数量需要覆盖接收和发送在途槽位;初始化会校验组合是否 |
| 151 | -| Flag | 默认 | 用途 | | 153 | +可用。硬件能力同样约束队列深度、SGE 数和消息大小。 |
| 152 | -|------|------|------| | ||
| 153 | -| `--urma_use_polling` | false | 轮询 JFC 而非事件模式 | | ||
| 154 | -| `--urma_poller_num` | 1 | 每 bthread tag 的轮询器数(轮询模式) | | ||
| 155 | -| `--urma_disable_bthread` | false | 内联处理消息(不起 bthread) | | ||
| 156 | -| `--urma_sq_size` | 128 | 本地 JFS 深度 [16, 4096] | | ||
| 157 | -| `--urma_rq_size` | 128 | 本地 JFR 深度 [16, 4096] | | ||
| 158 | -| `--urma_cqe_poll_once` | 32 | 每次 `urma_poll_jfc` 的上限 | | ||
| 159 | -| `--urma_recv_zerocopy` | true | 大于 `--urma_zerocopy_min_size` 的接收零拷贝 | | ||
| 160 | -| `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 | | ||
| 161 | -| `--urma_device` | "" | URMA 设备名(空=首个) | | ||
| 162 | -| `--urma_max_sge` | 0 | 每 WR SGE 上限(0=设备上限) | | ||
| 163 | -| `--urma_bonding_mode` | 0 | bonding 模式:0=standalone,1=active-backup,2=balance | | ||
| 164 | -| `--urma_bonding_level` | 0 | bonding 层级:0=IODIE,1=port | | ||
| 165 | -| `--urma_prepared_jetty_cnt` | 8 | 预连接 Jetty+CQ 请求数量;会根据 `RLIMIT_NOFILE` 自动限制 | | ||
| 166 | -| `--urma_buffer_size` | 8192 | 池中每个 buffer 大小(字节) | | ||
| 167 | -| `--urma_buffer_count` | 65536 | 池中 buffer 数量 | | ||
| 168 | -| `--urma_poller_yield` | false | 忙轮询循环中主动让出 bthread | | ||
| 169 | -| `--urma_client_handshake_version` | 2 | 客户端握手版本(2=二进制,3=protobuf) | | ||
| 170 | - | ||
| 171 | -设备名以 `bonding` 开头时,brpc 会在创建 context 后、创建 segment 和队列 | ||
| 172 | -前配置 provider。默认 standalone+IODIE 配置与 UMDK 性能工具保持一致。 | ||
| 173 | -bonding 支持需要 provider 扩展头文件 `urma_ubagg.h`。 | ||
| 174 | - | ||
| 175 | -## 与 UBRing 协同 | ||
| 176 | - | ||
| 177 | -UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing | ||
| 178 | -(`SOCKET_MODE_UBRING`)对**小包和同机 IPC**最优(亚微秒、零系统调用)。 | ||
| 179 | -混合负载可按服务选择传输层: | ||
| 180 | - | ||
| 181 | -| 场景 | 建议 `socket_mode` | | ||
| 182 | -|------|---------------------| | ||
| 183 | -| 同机 IPC | `SOCKET_MODE_UBRING` | | ||
| 184 | -| 跨节点小包(< 64KB) | `SOCKET_MODE_UBRING`(UBS-Mem)或 `SOCKET_MODE_URMA` | | ||
| 185 | -| 跨节点大包(>= 64KB) | `SOCKET_MODE_URMA` | | ||
| 186 | -| 传统 RoCE/IB 数据中心 | `SOCKET_MODE_URMA` 或 `SOCKET_MODE_RDMA` | | ||
| 187 | - | ||
| 188 | -单连接内按包大小自动分流的方案(路线 C 方案 B)作为后续演进方向。 | ||
| 189 | - | ||
| 190 | -## 限制 | ||
| 191 | - | ||
| 192 | -- 仅支持 `baidu_std` 协议(与 RDMA 一致)。SSL、RTMP、NSHEAD、MONGO 在 | ||
| 193 | - `ContextInitOrDie` 阶段拒绝。 | ||
| 194 | -- 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 | ||
| 195 | -- 当前实现面向 Linux。 | ||
| @@ -0,0 +1,120 @@ | |||
| 1 | +# URMA 连接资源抽象 | ||
| 2 | + | ||
| 3 | +`UrmaEndpoint` 负责 TCP 协商、逻辑字节流和消息解析,`UrmaConnectionResource` 负责连接资源,`UrmaResourceProvider` 负责资源创建及退出。`--urma_resource_mode=exclusive` 为默认模式,每连接独占 Jetty/JFR/JFC/JFCE;`--urma_resource_mode=pooled` 将同一进程中的连接映射到共享 JFS/JFR。两种后端共用 Endpoint 和握手状态机,独占路径不构造池化交付上下文。模式在启动时选择,不支持运行期间切换。 | ||
| 4 | + | ||
| 5 | +## 职责 | ||
| 6 | + | ||
| 7 | +| 层次 | 职责 | | ||
| 8 | +|---|---| | ||
| 9 | +| `UrmaEndpoint` | TCP 握手与原有失败/回退路径、连接状态、发送转交、输入缓存、协议解析和消息交付 | | ||
| 10 | +| `UrmaConnectionResource` | 连接资源描述、对端绑定、接受并消费发送前缀、可写等待、关闭和资源诊断 | | ||
| 11 | +| `UrmaExclusiveConnection` | 原 SEND/RECV、SGE、环形 buffer、双 credit 窗口、纯 ACK、CQ 处理和接收预投 | | ||
| 12 | +| `UrmaExclusiveResourceProvider` | 独占资源创建、预创建 Jetty 集合、按 bthread tag 管理 polling、停止与回收 | | ||
| 13 | +| `urma_helper` | 公共设备/context、注册内存与 IOBuf 分配服务 | | ||
| 14 | + | ||
| 15 | +公共接口定义在 `src/brpc/urma/urma_resource.h`,不暴露 native URMA 结构。独占实现及其私有状态位于 `urma_exclusive_resource.cpp`。以后接入另一种资源实现时,应实现相同接口及回调契约,而不是复制 Endpoint 的握手和输入处理代码。 | ||
| 16 | + | ||
| 17 | +以下“发送与等待”至“构建与验证”说明独占后端的保留语义;共享队列的差异见“池化后端”。 | ||
| 18 | + | ||
| 19 | +## 发送与等待 | ||
| 20 | + | ||
| 21 | +`Send(IOBuf**, size_t)` 直接消费已接受的连续字节前缀,返回该前缀的长度。构造 SGE 时通过 `cutn` 将数据引用交给 SQ 槽位,保留到发送完成;Endpoint 仅转交调用,不再执行独立的 `pop_front`。调用方串行执行同一连接的数据发送,并在整个调用期间持有资源句柄。 | ||
| 22 | + | ||
| 23 | +每个 WR 保留有界的输入切片记录。构造或提交失败时,仅将当前未提交 WR 的切片恢复到原 IOBuf 的前端;已成功提交的前缀保持消费,未接受的后缀及用户注册内存元数据保持不变。后续 WR 失败时仍返回前面已接受的正长度,错误通过携带来源资源的回调上报。 | ||
| 24 | + | ||
| 25 | +空输入返回 0;非空输入在无容量时返回 `-1/EAGAIN`。已关闭或未绑定资源不能报告可写。发送仍逐 WR 提交,不引入请求队列、链式提交、凑批或 payload 复制。 | ||
| 26 | + | ||
| 27 | +原本地 SQ 窗口、远端接收 credit 和三个纯 ACK 预留 WR 保留。可写等待仅用于绑定成功后的发送背压,并且只接受所属 Socket 的 `_epollout_butex`。等待前读取 butex 计数,再检查可写条件;资源层包装 `butex_wait`,不另建等待者集合。TX 完成、接收 credit、Close 通过 `Socket::WakeAsEpollOut` 先递增计数再唤醒,避免通知先于等待时丢失。`BindPeer` 不触发可写通知,但仍刷新提前接收积累的信用。SQ 槽位先释放数据引用,再公布可用容量。 | ||
| 28 | + | ||
| 29 | +数据 SEND 与 CQ 路径的 ACK 不获取资源层发布锁,底层 provider 必须支持同一 Jetty 的并发提交。纯 ACK 预算通过原子 CAS 预留,避免绑定后的初始信用刷新与 CQ 并发时超额使用;这部分软件状态不依赖 provider 内部锁。初始化、绑定与关闭之间仍使用仅用于管理路径的生命周期锁。 | ||
| 30 | + | ||
| 31 | +## 接收与交付 | ||
| 32 | + | ||
| 33 | +原 CQ drain、ACK、rearm、最终 drain、解析的顺序保留。单次 drain 内聚合连续字节,通过一次资源回调交给 Endpoint,使用 `IOBuf::movable()` 交接所有权。接收复制阈值、零拷贝补收和 peer credit 格式不变。 | ||
| 34 | + | ||
| 35 | +CQ 入口持有资源句柄,直接调用 `DrainCq`;不经交付策略回调往返,也不构造空的交付上下文。独占模式仍使用原来的 parser-local `last_msg` 和 `UrmaTransport::QueueMessage`;event、polling、`urma_disable_bthread` 及 coroutine 选项的调度语义不变。池化模式的 deferred 尾消息规则不影响此路径。 | ||
| 36 | + | ||
| 37 | +资源句柄在本地 Hello 发出之前发布。握手期到达的数据可以进入 Endpoint 输入缓存,但只有状态达到 ESTABLISHED 才解析;握手完成主动推进 pending 字节,不依赖下一次 CQE。 | ||
| 38 | + | ||
| 39 | +## 握手与生命周期 | ||
| 40 | + | ||
| 41 | +v2 二进制 Hello、v3 protobuf、Jetty 标识和接收配额编码保持原格式。角色和执行属性通过 `UrmaConnectionOptions` 传入资源层: | ||
| 42 | + | ||
| 43 | +- 客户端、非 bonding 服务端在创建资源时预投接收。 | ||
| 44 | +- bonding 服务端在导入完成后预投,再由 Endpoint 发出 Hello。 | ||
| 45 | +- 对端 segment 导入始终先于 Jetty 导入。 | ||
| 46 | + | ||
| 47 | +Endpoint、已进入的调用和 CQ 执行者持有资源句柄。CQ carrier 仅持有弱引用,不长期引用 Endpoint。数据和错误回调携带来源资源,Endpoint 核对其仍为当前实例后处理;Reset 摘下旧句柄后在锁外 Close。旧实例的数据和错误回调不能影响 Reset 后的新实例。 | ||
| 48 | + | ||
| 49 | +资源句柄仅延长连接资源及其硬件的生命周期,并不延长 Socket、Endpoint 的生命周期。资源只保存带版本的 `SocketId`;可写通知时用 `Socket::Address` 临时取得 `SocketUniquePtr`,成功后再访问 butex,失效则忽略通知。等待期间同样持有临时 Socket 引用。资源不长期保存 Socket/butex 裸指针,也不持有指向 Socket 的永久强引用,避免迟到通知访问回收后复用的对象,以及 Socket→Endpoint→resource→Socket 的引用环。Socket 失败路径仍负责唤醒自身等待者。 | ||
| 50 | + | ||
| 51 | +Close 幂等,发布关闭状态、移除事件入口并唤醒等待者。随后进入的调用拒绝发送;已通过关闭检查的 SEND/ACK 可以完成当前投递。Close 不等待正在执行的发送或自身回调,实际硬件及数据引用的回收在执行中的句柄退出后进行。硬件销毁失败时同时保留硬件与数据引用,等待 provider 后续 Shutdown 重试。 | ||
| 52 | + | ||
| 53 | +polling 的初始化、停止和 Shutdown 属于控制路径;在本 provider 的 poller 内调用停止/初始化会被拒绝,避免等待自身。停止与启动串行执行,Shutdown 先禁止新建连接,再在创建锁之外等待 poller 退出。 | ||
| 54 | + | ||
| 55 | +绑定的参数校验失败可更正描述后重试;开始导入或预投后的失败会关闭该资源实例,不能在半初始化资源上重新绑定。 | ||
| 56 | + | ||
| 57 | +`Shutdown()` 返回 false 时,拥有者必须保留 provider 及公共设备/内存依赖,禁止新连接;返回 true 后才可以销毁 provider。公共注册内存还需检查在用块及注销结果。沿用原进程退出策略,不注册可能早于 IOBuf TLS 析构的 atexit 回收函数。 | ||
| 58 | + | ||
| 59 | +## 构建与验证 | ||
| 60 | + | ||
| 61 | +沿用现有 `WITH_URMA`、`URMA_USE_MOCK` 和 SDK 路径配置,不新增必选分配器。`WITH_URMA=OFF` 不启用该实现。 | ||
| 62 | + | ||
| 63 | +`test/brpc_urma_unittest.cpp` 覆盖原握手/SDK mock,以及资源描述、发送输入与前缀所有权、可写等待/关闭、提前接收缓存、Reset 来源校验和多连接 v2/v3 RPC。资源集成测试使用 link-time mock;应分别以 event 和 `--urma_use_polling=true` 运行。真实 provider 还需进行跨进程收发验证,mock 不能替代 DMA、硬件销毁和 bonding 的实机验证。 | ||
| 64 | + | ||
| 65 | +本框架尚未接入 RDMA 或单边读写。池化后端不支持 polling 和 bonding;独占模式保留这两条已有路径。 | ||
| 66 | + | ||
| 67 | + | ||
| 68 | +## 池化后端 | ||
| 69 | + | ||
| 70 | +`UrmaPooledResourceProvider` 拥有共享队列及 CQ 入口。每个 `UrmaPooledConnection` 保存所属 JFR、对端 JFR/segment 引用、对端 SocketId、发送序号及首选 JFS。连接创建登记使用哈希表,避免每创建一个连接扫描全部存活连接。逻辑连接的 Close 只禁止本连接的新操作并唤醒其 Socket;共享队列由 provider 在所有连接句柄退出后关闭。 | ||
| 71 | + | ||
| 72 | +发送首选 JFS 在连接创建时按 SocketId 哈希确定。首选放不下时,从最多4个不重复备选中选择 inflight 最小者;大于5个 JFS 时使用 pthread TLS xoshiro256++ 抽样。最多尝试首选和一个备选,按可用容量接受前缀,不扫描全池。一次 SEND 采用一个 64 B header SGE 加多个原始 IOBuf SGE,累计正文达到 4032 B 或 SGE 上限即提交。发布到 JFS 请求队列即表示接受,对应连续前缀从输入移除。每批请求持有 WR、SGE、数据及导入资源;MPSC owner 负责链式 post。已接受但暂未 post 的后缀由资源层继续推进,不让调用者重复提交。TX 上下文仅持有轻量导入资源;来源连接弱引用在绑定时存入该每连接资源,避免逐 WR 重复维护引用和扩大上下文。迟到错误通过导入资源取得来源,且仅在来源仍是 Socket 当前资源时生效。 | ||
| 73 | + | ||
| 74 | +共享 SQ 等待在固定首选 JFS 登记:先登记,再检查容量;首选的 CQE/owner 推进唤醒其等待者,不因其他备选队列完成而全池唤醒。登记和取消使用每 JFS 的等待者锁,正常 post/poll 不使用该锁。登记节点仅在 WaitWritable 调用期间存在,并持有所属 Socket 的临时引用。独占模式继续直接使用所属 Socket 的 butex,不引入共享等待者表。 | ||
| 75 | + | ||
| 76 | +共享 CQ 入口临时持有 provider 状态,原子 admission 状态同时记录活动回调数和停止标志。Shutdown 先禁止新回调进入,已有回调未退出时返回 false,不持有关闭锁等待 CQ;回调引用在 slot 操作屏障退出及尾消息处理之后释放,避免执行中的 CQ 资源被析构。 | ||
| 77 | + | ||
| 78 | +共享 JFR 先检查 CQE、取出 header 和正文并补收,再交给 `UrmaReceiveAdapter`。 | ||
| 79 | +适配器通过 SocketId 查找一次 Socket,取得当前池化资源句柄,在资源的重排锁内 | ||
| 80 | +校验 JFR 和序号并组装连续字节。交付时获取 Endpoint 交付锁,重新核对来源仍为 | ||
| 81 | +当前资源,再以 `IOBuf::movable()` 进入输入流。Reset 可以替换资源,旧回调因 | ||
| 82 | +身份不匹配而被丢弃;Close 和被替换资源的析构在 Endpoint 锁外执行。 | ||
| 83 | +一次 poll 最多处理 `BRPC_URMA_MAX_CQE_PER_POLL` 条完成,批次至多持有同样 | ||
| 84 | +数量的 Socket 引用;逐片追加后每个连接解析一次,不等待额外 CQE。 | ||
| 85 | + | ||
| 86 | +`UrmaDeferredMessage` 的生命周期跨越整个共享 CQ 回调,只保留最新的一条 parser-last 消息;被替换的候选正常入队。解析通过 `InputMessageClosure` 返回尾消息,适配器在重排和交付锁外更新 deferred;最终消息只在 slot 操作屏障退出、事件所有权交还之后执行,并使用所属连接的 bthread 本地存储。错误退出不在持锁或事件尚未释放时执行业务。独占 CQ 仍直接调用 `on_received`,没有空的 batch 回调或资源层往返。 | ||
| 87 | + | ||
| 88 | +池化 v2 使用 implementation=2、94 B 报文,独占 v2 使用 implementation=1、86 B 报文。读取固定前缀后再决定剩余长度,避免混合模式等待不存在的扩展字段。v3 保持字段号12的 SocketId 和字段号13的 uint32 资源模式,字段13名称为 resource_mode,默认1代表独占、2代表池化。模式不匹配时通过完整拒绝 Hello 或 ACK=0 回退 TCP;非法报文与 I/O 错误仍按错误路径处理。此协商不表示任意历史池化版本均支持 TCP 回退。 | ||
| 89 | + | ||
| 90 | +池化内存使用独立的 8 KiB IOBuf、4 KiB CTP 和 64 B header 池;原始已注册 IOBuf 多 SGE 发送保持零拷贝。独占模式保留原有单块大小配置及分配器行为,默认内存和资源参数不因选择框架而改变。池化数据块数量默认 24000,独占默认 65536;显式设置 `urma_buffer_count` 时使用用户值。 | ||
| 91 | + | ||
| 92 | +`test/brpc_urma_pooled_unittest.cpp` 单独运行池化用例,避免同一进程重复替换 IOBuf 全局分配器。覆盖池资源独立性、分片及回绕、共享 SQ 等待、JFS affinity、并发握手、延后尾消息和 RX batch 边界;非池化用例保持独立。 | ||
| 93 | + | ||
| 94 | +### 池化连接代次限制 | ||
| 95 | + | ||
| 96 | +池化线格式目前只携带目标 SocketId 和 16 位分片序号。健康检查复活可能保留 SocketId,而共享 JFR 在逻辑连接重置后仍存在;同一 JFR 上迟到的旧接收分片缺少额外连接代次标识。这是已有池化协议的限制,不能将独占回调的来源句柄校验等同于池化网络接收的代次隔离。解决需要扩展握手及 header 代次,或证明重连前的接收已静默;本轮保留原池化线格式,不宣称已提供该保证。TX 错误回调仍使用来源资源弱引用校验。 | ||
| 97 | + | ||
| 98 | +## JFS/JFR slot 去锁迁移 | ||
| 99 | + | ||
| 100 | +从 jfs_发送优化/brpc_bk 的 dfad2f8e 之后迁移共享发送 owner、分流和固定context方案,范围止于1935f1a9。未迁移后续接收重排/解析去锁;重排状态移入池化连接资源后仍受锁保护。独占后端保留原调度语义,握手新增资源模式不匹配时的 TCP 回退。独立8K/4K CTP/64B注册池保留,未混入旧仓库的单池分配器或额外TLS修改。 | ||
| 101 | + | ||
| 102 | +- 每 JFS 使用 exchange+link MPSC 请求链及 RUNNING/KICK owner 状态,发送者与 TX CQE 都可以推进,无额外常驻发送 bthread。 | ||
| 103 | +- SQ 区分已接受请求量和硬件在途量。并发选路可能少量超出软件预算,由 owner 按硬件容量推进。 | ||
| 104 | +- TX context/批次使用 ObjectPool。context 的提交引用与 CQE 引用协调,覆盖 CQE 早于 post 返回;已提交context通过原子固定登记数组回收。 | ||
| 105 | +- RX context 使用固定数组及 posted 标记,由当前 JFR 单执行者修改,移除 pending set 和 slot mutex。 | ||
| 106 | +- 池和 slot 使用操作屏障协调 shutdown,停止接纳并等待活跃操作退出后,按硬件删除成功的边界回收DMA数据。 | ||
| 107 | +- 保留抽象版来源资源弱引用校验,晚到 TX 错误不影响替换后的资源实例。 | ||
| 108 | +- 暂时 post 失败且仍有在途 WR 时保留后缀,后续 CQE 重试;已观察到已有 WR 或本次前缀提前完成时立即重试。没有容量进展且无未来完成可推进时上报连接错误,避免永久挂起。组装提交链始终停在首个失败上下文,保证提交与出队对应同一个连续前缀。 | ||
| 109 | + | ||
| 110 | +新增运行参数: | ||
| 111 | + | ||
| 112 | +| 参数 | 默认 | 含义 | | ||
| 113 | +|---|---:|---| | ||
| 114 | +| urma_jfs_affinity | true | 固定哈希首选;关闭时按批轮询首选 | | ||
| 115 | +| urma_io_tx_batch_max | 8 | 每次准备/链式post的WR上限,1至32,不等待凑满 | | ||
| 116 | +| urma_io_jfs_lock_free | false | 向UMDK声明JFS由单owner提交 | | ||
| 117 | +| urma_io_jfr_lock_free | false | 向UMDK声明JFR由单接收执行者补投 | | ||
| 118 | +| urma_io_jfc_lock_free | false | 向UMDK声明JFC由单执行者poll | | ||
| 119 | + | ||
| 120 | +上述UMDK lock_free参数仅设置池化队列的创建属性;Endpoint锁与独占资源配置不受影响。 | ||
| @@ -0,0 +1,23 @@ | |||
| 1 | +# 池化连接接收重排职责 | ||
| 2 | + | ||
| 3 | +重排状态归属 `UrmaPooledConnection`,每条逻辑连接独立维护 `_deliver_base`、`_reorder_queue` 和重排锁,不另建 SocketId 到接收流的全局 map。 | ||
| 4 | + | ||
| 5 | +接收路径:共享 JFR CQE → 读取目标 SocketId / seq → UrmaReceiveAdapter::OnFragment 只查找一次 Socket 并临时持有引用 → 取得当前池化连接 → AssembleFragmentLocked 校验和合并连续前缀 → 复用 Socket 引用交付 Endpoint → BRPC 解析。 | ||
| 6 | + | ||
| 7 | +Endpoint 不再包含分片序号、重排队列或 ReorderAndDispatch。池化通过 on_fragment 进入适配器;独占通过 on_received 进入同一适配器并只查找一次 Socket。Endpoint 不引用 UrmaDeferredMessage / UrmaRxParseBatch;这两个上下文、批次 Socket 引用和延迟分发均由适配器管理。Endpoint 通过函数参数返回 InputMessageClosure,不保存延迟输出槽或 CQ 批次。Endpoint 在分发锁内检查关闭标记、连接状态和 Socket 有效性。池化资源在一次连接生命周期内保持绑定不变;独占回调仍比较传入资源身份。 | ||
| 8 | + | ||
| 9 | +同一连接的重排锁覆盖片段组装至有序交付,防止并发回调再次打乱字节流。Close 只标记关闭,不同步等待该锁;否则可能与 Endpoint 分发锁形成锁顺序冲突。关闭后新片段被拒绝,已有回调通过 Socket 引用保活绑定资源,缓存随连接资源最终销毁释放。只有旧访问者全部退出后的健康检查 Reset 才能清空绑定并建立新连接,不复用旧 seq 状态。详见 [稳定资源绑定约束](stable_connection_binding.md)。 | ||
| 10 | + | ||
| 11 | +序号保持 16 位及前向半区间规则,按期望 seq 查找缓存以支持回绕。重复或过期 seq 通过资源错误回调报告 EPROTO。CQ 每轮批量解析和延迟业务分发继续保留,纯乱序缓存不会触发无数据的解析。 | ||
| 12 | + | ||
| 13 | +验证:CMake 动态链接真实 liburma、tcmalloc 构建成功。池化 mock 70 项通过;独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过。145 服务端 / 151 客户端,NUMA1 CPU 96–111,udmac0d1e2,event / multi SGE,4 连接、QD10、目标 500 QPS,1K / 100K 双边收发各 60 个请求零错误。短迭代统计用于功能验证,不用于时延结论。命令和日志位于项目根目录 performance_results/20261114_池化重排下沉。 | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +接收路由查找次数:每条池化 CQE 进入 OnFragment 时一次;后续 HandleFragment、重排合并、Deliver 和批次 Flush 均复用已有引用。乱序片段暂存不产生第二次查找。非池化 OnReceived 一次。这里指消息所属逻辑 Socket 的查找,不含事件分发器对 JFR 载体 Socket 的路由。池化连接不保存 Socket 引用,也不调用 Endpoint。适配器持有重排锁直到本次有序数据交付完成,避免两个并发执行者在返回连续数据后再次颠倒交付顺序。 | ||
| 17 | + | ||
| 18 | +ReceiveAdapter 版本验证:池化 71 项通过(增加禁止 RX 返回旧 lookup_connection 回调的测试),独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过;145/151 真实设备的 1K / 100K 双边收发各 60 请求零错误。完整命令与日志:performance_results/20261114_ReceiveAdapter验证。 | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +最后消息的所有权:ProcessNewMessage 保持原有实现;URMA event 模式的 QueueMessage(last=true) 保留调用者的最后消息(池化 usercode_in_pthread 模式仍按原规则排队,polling 仍全部排队)。适配器以局部 InputMessageClosure 调用 Endpoint 解析,解析结束并释放分发/重排锁后,将最后消息接入 CQ 回调的延迟上下文;若替换了旧候选,就将旧候选通过其原 Socket Transport 后台入队并 flush。CQ 回调交还事件权且操作保护结束后才执行最终候选。Endpoint 和 Transport 均无持久的 deferred_slot 字段。握手和独占路径的局部 Closure 声明在分发锁之前,保证析构执行发生在解锁后。 | ||
| 22 | + | ||
| 23 | +Closure 返回版本验证:池化 71 项通过;独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过;145/151、NUMA1 CPU96–111、udmac0d1e2,1K / 100K、4 连接、QD10 双边收发各 60 请求零错误。命令及日志位于 performance_results/20261114_Closure返回验证。ProcessNewMessage 的 cpp/h 相对基线没有实质修改。 | ||
| @@ -0,0 +1,36 @@ | |||
| 1 | +# 抽象版池化 JFS/JFR 去锁迁移验证 | ||
| 2 | + | ||
| 3 | +目标基线:3ad1ae2520201a57f131718df5d0de1b0519afd8。 | ||
| 4 | + | ||
| 5 | +源方案:jfs_发送优化/brpc_bk,dfad2f8e为验证基线,后续到1935f1a9为本轮参考范围;不迁移9373647b之后的Endpoint重排/解析去锁。 | ||
| 6 | + | ||
| 7 | +实现范围为共享队列MPSC owner、最大32 WR链式post、SocketId哈希首选、最多4个备选抽样、首选JFS等待、TX引用及原子登记数组、RX固定数组和slot操作屏障。保留本仓库独立CTP内存池、来源资源校验、Endpoint两把锁和独占后端。 | ||
| 8 | + | ||
| 9 | +构建使用145 AArch64、GCC12.3.1、CMake Release、动态libbrpc.so、真实URMA SDK、tcmalloc、CPU frequency计时,无打点;原包保存在package/bin,新包保存在unlock-package/bin。Windows源码修改后同步Linux编译,未在Linux更改源码。 | ||
| 10 | + | ||
| 11 | +单元测试采用链接期mock(测试可执行文件编译mock_urma.cpp并导出符号,动态库仍为同一个待验证libbrpc.so),用于验证队列及所有权,不替代真实硬件测试。 | ||
| 12 | + | ||
| 13 | +已通过池化70项单元测试,包括8项新增的链式批次/owner/关闭回收测试;独占event模式40项通过、2项polling专用跳过。7项关键并发测试重复100轮全部通过;开启三项SDK lock_free后池化70项全部通过;独占polling模式41项通过、1项多tag条件用例跳过。真实硬件三轮交替对照结果在后续验证记录中补充。 | ||
| 14 | + | ||
| 15 | +原测试中“post失败后缀仍归调用者”的断言已按新队列语义调整:进入资源队列即为接受,当前未能post的后缀仍由资源层持有并由后续CQE推进,不重复消耗或重复发送。 | ||
| 16 | + | ||
| 17 | +## 145/151 三轮交替性能对照 | ||
| 18 | + | ||
| 19 | +条件:NUMA1 CPU96-111、udmac0d1e2、multi/event、4 JFS/4 JFR、单连接QD10、目标500QPS、每组20秒;请求响应等长,max_random_delay_us不设置(默认5000)。两版同CMake动态链接构建。修改后显式开启3项UMDK lock_free,batch_max=32。 | ||
| 20 | + | ||
| 21 | +| 消息大小 | 修改前报告均值三轮平均 µs | 修改后报告均值三轮平均 µs | 差值 µs | | ||
| 22 | +|---|---:|---:|---:| | ||
| 23 | +| 1K | 31.00 | 32.33 | +1.33 | | ||
| 24 | +| 4K | 35.00 | 35.33 | +0.33 | | ||
| 25 | +| 64K | 82.00 | 89.33 | +7.33 | | ||
| 26 | +| 100K | 107.67 | 120.33 | +12.67 | | ||
| 27 | + | ||
| 28 | +24组均无错误,实测约500QPS。未观测到低负载时延收益,大包存在可重复增加。上述数值是三次程序报告的简单平均;时延沿用默认10秒窗口,不能视为合并样本统计。组合测试同时包含owner/context/选路和UMDK选项,不能单独归因某把锁。 | ||
| 29 | + | ||
| 30 | +batch_max=1的单轮补测为64K 106µs、100K 155µs,反而更慢。因而不能简单把默认32的劣化归因为准备批次延后首个post;频繁进入提交/owner/context管理也可能增加成本,需进一步打点才能归因。默认配置未据此改动。 | ||
| 31 | + | ||
| 32 | +补测batch_max=8时,64K均值81µs/P99 109µs,100K均值108µs/P99 154µs,均无错误。接近基线均值,但仅各一轮,不宣称稳定收益;默认batch上限仍32。 | ||
| 33 | + | ||
| 34 | +## batch8完整一轮复测 | ||
| 35 | + | ||
| 36 | +相同设备、绑核、QD/QPS和20秒时长,1K/4K/64K/100K均值分别31/35/82/109µs,P99分别42/45/119/155µs,40028次完成请求、错误0。接近修改前31/35/82/107.67µs的三轮平均。只以运行参数设为8,代码默认32未修改;不宣称单轮差异代表稳定提升。 | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | +# 资源模式与跨模式 TCP 回退 | ||
| 2 | + | ||
| 3 | +v2 保留 hello_ver=2 和二进制 impl_ver:1 为独占(86B),2 为池化(94B)。v3 将 protobuf 字段 13 改名为 resource_mode,仍为 uint32:1 为独占、2 为池化,默认 1。字段编号和已有数值保持不变;非池化发送时仍省略此字段和 socket_id,保留旧非池化 Hello 格式。池化发送 resource_mode=2 和 socket_id。resource_mode 表达硬件资源拓扑,不表达 protobuf 格式版本。 | ||
| 4 | + | ||
| 5 | +解析器完整消费 Hello 后返回对端描述,由握手状态机判断模式是否一致。v2 对合法长度的未知版本/实现号也会消费完整 Hello;非法长度仍报协议错误。v3 对未知模式及缺少必要 socket_id 的报文仍报错。正常模式一致时沿用原有资源导入和 ACK。 | ||
| 6 | + | ||
| 7 | +跨模式流程: | ||
| 8 | + | ||
| 9 | +1. 客户端发送其本地模式 Hello。 | ||
| 10 | +2. 服务端读完 Hello,发现资源模式不匹配,不创建/导入对端连接资源。 | ||
| 11 | +3. 服务端发送同一握手格式的完整拒绝 Hello:Jetty ID、buffer 等资源信息为 0。 | ||
| 12 | +4. 此拒绝 Hello 不等待 ACK。服务端转入 TCP,客户端完整读取拒绝响应后也转入 TCP。 | ||
| 13 | +5. 双方继续使用原 TCP 连接发送 BRPC 数据,关闭本地 URMA 连接资源。池化进程的全局资源池仍保留。 | ||
| 14 | + | ||
| 15 | +客户端若收到有效但模式不同的资源描述,则先发送 ACK=0 通知等待 ACK 的对端,再转入 TCP。以前的池化客户端禁止 TCP 回退分支已移除。I/O 错误、非法资源描述等错误仍按其原有错误路径处理,不能把残留握手数据直接交给业务解析。 | ||
| 16 | + | ||
| 17 | +ub_test 的 SGE 校验新增独占资源分支:独占模式没有 g_urma_pool,不能用池化 JFS 是否存在作为启动条件。 | ||
| 18 | + | ||
| 19 | +## 验证 | ||
| 20 | + | ||
| 21 | +CMake 动态 libbrpc.so + libtcmalloc,真实 liburma 构建。池化回归 72 项通过;独占 event 40 项通过、2 项条件跳过,polling 41 项通过、1 项条件跳过。新增测试确认 v2 和 v3 解析对端模式之后不会消费后面的 NEXT 字节。 | ||
| 22 | + | ||
| 23 | +真实设备:145 服务端 / 151 客户端,NUMA1 CPU96–111、udmac0d1e2,单连接 QD10,目标500QPS,50 次迭代(含收尾实际60请求)。以下每项均测试1K与100K,零错误,并在服务端日志确认 resource mode mismatch / falling back to TCP: | ||
| 24 | + | ||
| 25 | +| Hello | 客户端 | 服务端 | 结果 | | ||
| 26 | +|---|---|---|---| | ||
| 27 | +| v2 | 独占 | 池化 | TCP 收发成功 | | ||
| 28 | +| v2 | 池化 | 独占 | TCP 收发成功 | | ||
| 29 | +| v3 | 独占 | 池化 | TCP 收发成功 | | ||
| 30 | +| v3 | 池化 | 独占 | TCP 收发成功 | | ||
| 31 | + | ||
| 32 | +结果、逐项命令及日志:项目根目录 performance_results/20261114_跨模式TCP回退 下的 *_r2 目录。第一次尝试被 ub_test 原有的仅池化启动检查阻止,没有发出 Hello,单独保留为排查记录。 | ||
| 33 | + | ||
| 34 | +以上跨模式实测针对两端均使用本次代码的版本;没有据此声称任意历史池化版本都支持回退。 | ||
| @@ -0,0 +1,42 @@ | |||
| 1 | +# URMA 连接资源稳定绑定与无资源锁访问 | ||
| 2 | + | ||
| 3 | +## 生命周期约束 | ||
| 4 | + | ||
| 5 | +每次有效连接生命周期内,Endpoint 的连接资源只发布一次。发布后不清空、不替换,连接失败和 TCP 回退仅执行逻辑关闭。资源对象由 Endpoint 持有,通常在 Socket 最终回收、销毁 Endpoint 时释放。 | ||
| 6 | + | ||
| 7 | +健康检查恢复是明确的生命周期边界:`Socket::Revive()` 先等待引用数达到预期值,退出旧连接的访问者,再调用 `UrmaTransport::Reset()`。Reset 可释放旧绑定并允许下一次连接重新发布资源。不能在接收、发送、调试查询或握手线程仍访问旧绑定时并发调用 Reset;不能把任意时刻的 Reset 当作关闭接口。该前提也适用于直接调用 Endpoint 的测试和内部调用者。 | ||
| 8 | + | ||
| 9 | +Socket 引用保证 Endpoint 和绑定资源在处理期间有效。资源后端不长期持有 Socket 引用。资源指针不能被保存到超出 Socket 保活范围的后台任务中;这类异步任务需保留独立的资源所有权。 | ||
| 10 | + | ||
| 11 | +## 发布与读取 | ||
| 12 | + | ||
| 13 | +- `_connection_resource` 是拥有资源的 shared_ptr,只在发布前写入;稳定期读操作不与写操作并发。 | ||
| 14 | +- `_published_resource` 是非拥有型原子指针。所有初始化完成后 release-store 发布,访问者 acquire-load 后借用。它不承担资源回收,也不通过原子 shared_ptr 隐藏锁。 | ||
| 15 | +- `BorrowConnectionResource()` 不获取 mutex、不增加共享引用计数。正常收发和接收适配器使用该入口。 | ||
| 16 | +- `GetConnectionResource()` 保留给需要独立所有权的接口:先 acquire 观察发布,再复制稳定的 owner。复制本身仍会更新 shared_ptr 引用计数。 | ||
| 17 | +- 资源发布与逻辑关闭由已有 `_dispatch_mutex` 串行;关闭标记阻止关闭后再发布。重复分配返回 EALREADY,关闭后分配返回 ECANCELED,不能替换原绑定。 | ||
| 18 | + | ||
| 19 | +## 接收路径 | ||
| 20 | + | ||
| 21 | +```text | ||
| 22 | +Socket::Address(一次,临时保活) | ||
| 23 | + → acquire-load 已发布资源,借用池化连接 | ||
| 24 | + → 重排锁:检查 JFR/seq,整理连续字节 | ||
| 25 | + → 解析锁:检查连接状态,拼入输入流/登记批量解析 | ||
| 26 | +``` | ||
| 27 | + | ||
| 28 | +正常连续交付分片由四次锁获取减少到两次。不再获取 `_resource_mutex`,不再复制资源 shared_ptr,也不再校验资源是否被替换。重排锁、解析锁继续保留;批量解析仍按批次获取解析锁。类型转换仍使用 dynamic_cast,不将类型安全与本次生命周期优化混在一起。 | ||
| 29 | + | ||
| 30 | +`Close()` 标记资源关闭;`DeallocateResources()` 在解析锁内标记 Endpoint 逻辑关闭,之后在锁外调用 Close,保留绑定。晚到片段由资源关闭状态拒绝,交付和解析还会检查 Endpoint 关闭状态。关闭不等待重排锁,避免与接收路径形成反向锁顺序。 | ||
| 31 | + | ||
| 32 | +## 在途 WR 与回收 | ||
| 33 | + | ||
| 34 | +Socket 保活只覆盖当前处理过程,不能代替已提交 WR 的 DMA 所有权。TX context 中的 IOBuf、导入资源引用和完成回收逻辑继续保留,直到设备完成或原有安全退出流程允许回收。 | ||
| 35 | + | ||
| 36 | +池化连接资源对象延迟释放意味着连接失败/TCP 回退后,导入引用和乱序 IOBuf 可能保留至最终回收或恢复 Reset。共享 JFS/JFR 的生命周期仍由进程级资源池管理。非池化 Close 仍按原实现停止 CQ 事件入口,硬件对象最终析构随资源所有权释放;本次不改变硬件清理的安全要求。 | ||
| 37 | + | ||
| 38 | +## 验证 | ||
| 39 | + | ||
| 40 | +新增/调整测试覆盖逻辑关闭后绑定保留、重复关闭、晚到 RX 拒绝、关闭与阻塞在重排锁的接收回调并发、回调退出后 Reset 回收,以及原有在途 TX 导入资源保活。CMake Release 动态链接真实 liburma/tcmalloc 构建通过;池化软件 provider 73 项通过,非池化 event 40 项通过/2 项配置相关跳过,polling 41 项通过/1 项配置相关跳过。回归日志和真实设备性能命令位于项目根目录 `performance_results/20261116_资源绑定去锁对照/`。 | ||
| 41 | + | ||
| 42 | +真实设备交错两轮测试:QD10、500QPS、20秒,1K 平均时延由34.0变为33.5µs,P99两轮均值48.0变为53.0µs;100K平均由118.5降为111.5µs(减少7µs,约5.9%),P99两轮均值168.5降为159.5µs。该结果为本次组合修改收益,不代表单次锁操作耗时。所有真实设备测试零错误,命令和完整分布见上述性能结果目录。 | ||
| @@ -0,0 +1,64 @@ | |||
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +# or more contributor license agreements. See the NOTICE file | ||
| 3 | +# distributed with this work for additional information | ||
| 4 | +# regarding copyright ownership. The ASF licenses this file | ||
| 5 | +# to you under the Apache License, Version 2.0 (the | ||
| 6 | +# "License"); you may not use this file except in compliance | ||
| 7 | +# with the License. You may obtain a copy of the License at | ||
| 8 | +# | ||
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +# | ||
| 11 | +# Unless required by applicable law or agreed to in writing, | ||
| 12 | +# software distributed under the License is distributed on an | ||
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +# KIND, either express or implied. See the License for the | ||
| 15 | +# specific language governing permissions and limitations | ||
| 16 | +# under the License. | ||
| 17 | + | ||
| 18 | +cmake_minimum_required(VERSION 3.16...3.28) | ||
| 19 | +project(ub_test C CXX) | ||
| 20 | + | ||
| 21 | +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake) | ||
| 22 | + | ||
| 23 | +option(LINK_SO "Whether examples are linked dynamically" OFF) | ||
| 24 | + | ||
| 25 | +brpc_example_find_common_deps(DYNAMIC_LIB) | ||
| 26 | +option(WITH_TCMALLOC "Link tcmalloc" OFF) | ||
| 27 | +if(WITH_TCMALLOC) | ||
| 28 | + find_library(TCMALLOC_LIB NAMES tcmalloc REQUIRED) | ||
| 29 | + list(APPEND DYNAMIC_LIB ${TCMALLOC_LIB}) | ||
| 30 | +endif() | ||
| 31 | + | ||
| 32 | +protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto) | ||
| 33 | +set(BRPC_EXAMPLE_WITH_URMA ON) | ||
| 34 | +find_path(URMA_INCLUDE_PATH NAMES urma_api.h | ||
| 35 | + HINTS ENV URMA_ROOT | ||
| 36 | + PATHS /usr/include /usr/local/include | ||
| 37 | + PATH_SUFFIXES ub/umdk/urma umdk/urma urma | ||
| 38 | + src/urma/lib/urma/core/include) | ||
| 39 | +if(NOT URMA_INCLUDE_PATH) | ||
| 40 | + message(FATAL_ERROR | ||
| 41 | + "Fail to find urma_api.h. Install UMDK headers or set URMA_ROOT.") | ||
| 42 | +endif() | ||
| 43 | +find_library(URMA_LIB NAMES urma | ||
| 44 | + HINTS ENV URMA_ROOT | ||
| 45 | + PATH_SUFFIXES lib lib64) | ||
| 46 | +if(URMA_LIB) | ||
| 47 | + list(APPEND DYNAMIC_LIB ${URMA_LIB}) | ||
| 48 | +else() | ||
| 49 | + message(STATUS | ||
| 50 | + "liburma not found; using the URMA implementation linked into brpc") | ||
| 51 | +endif() | ||
| 52 | + | ||
| 53 | +# Match the original matrix's BUTIL_USE_CPU_FREQUENCY=true build. | ||
| 54 | +option(WITH_CPU_FREQUENCY "Use CPU frequency for benchmark timing" ON) | ||
| 55 | +foreach(role client server) | ||
| 56 | + add_executable(ub_test_${role} ${role}.cpp ${PROTO_SRC} ${PROTO_HEADER}) | ||
| 57 | + brpc_example_configure_target(ub_test_${role}) | ||
| 58 | + target_compile_definitions(ub_test_${role} PRIVATE | ||
| 59 | + BUTIL_USE_CPU_FREQUENCY=$<BOOL:${WITH_CPU_FREQUENCY}>) | ||
| 60 | + target_link_libraries(ub_test_${role} PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) | ||
| 61 | + set_target_properties(ub_test_${role} PROPERTIES | ||
| 62 | + BUILD_WITH_INSTALL_RPATH TRUE | ||
| 63 | + INSTALL_RPATH "$ORIGIN;$ORIGIN/../common") | ||
| 64 | +endforeach() | ||
| @@ -0,0 +1,137 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +"""Build pooled multi-SGE URMA and a relocatable binary bundle. | ||
| 3 | + | ||
| 4 | +Requires Linux, CMake, a C++ compiler, Protobuf and UMDK. gperftools is optional. | ||
| 5 | +The transport sources are used unchanged. Run on the destination architecture. | ||
| 6 | +""" | ||
| 7 | + | ||
| 8 | +import argparse | ||
| 9 | +import hashlib | ||
| 10 | +import json | ||
| 11 | +import os | ||
| 12 | +from pathlib import Path | ||
| 13 | +import shutil | ||
| 14 | +import subprocess | ||
| 15 | +import tarfile | ||
| 16 | +import tempfile | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +def sha256(path): | ||
| 20 | + return hashlib.sha256(path.read_bytes()).hexdigest() | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def run(args): | ||
| 24 | + print('+ ' + ' '.join(map(str, args)), flush=True) | ||
| 25 | + subprocess.run(list(map(str, args)), check=True) | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +def cache_value(build, name): | ||
| 29 | + for line in (build / 'CMakeCache.txt').read_text().splitlines(): | ||
| 30 | + if line.startswith(name + ':'): | ||
| 31 | + return line.split('=', 1)[1] | ||
| 32 | + raise RuntimeError('Missing CMake cache entry: ' + name) | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +def install_file(source, destination, strip=False): | ||
| 36 | + # Replacing the inode avoids truncating an executable or mapped library | ||
| 37 | + # when rebuilding a package in an existing work directory. | ||
| 38 | + fd, name = tempfile.mkstemp(prefix='.' + destination.name, dir=destination.parent) | ||
| 39 | + os.close(fd) | ||
| 40 | + temporary = Path(name) | ||
| 41 | + try: | ||
| 42 | + shutil.copy2(source, temporary) | ||
| 43 | + if strip: | ||
| 44 | + run(['strip', '--strip-unneeded', temporary]) | ||
| 45 | + temporary.replace(destination) | ||
| 46 | + finally: | ||
| 47 | + if temporary.exists(): | ||
| 48 | + temporary.unlink() | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +def main(): | ||
| 52 | + parser = argparse.ArgumentParser(description=__doc__) | ||
| 53 | + parser.add_argument('--source-root', type=Path, | ||
| 54 | + default=Path(__file__).resolve().parents[2]) | ||
| 55 | + parser.add_argument('--work-dir', type=Path, required=True) | ||
| 56 | + parser.add_argument('--prefix', default='') | ||
| 57 | + parser.add_argument('--jobs', type=int, default=24) | ||
| 58 | + parser.add_argument('--urma-include', default='/usr/include/ub/umdk/urma') | ||
| 59 | + parser.add_argument('--urma-bond-include', default='/usr/local/include/ub/umdk/urma') | ||
| 60 | + parser.add_argument('--urma-library', default='/usr/lib64/liburma.so') | ||
| 61 | + parser.add_argument('--with-tcmalloc', action='store_true') | ||
| 62 | + args = parser.parse_args() | ||
| 63 | + source = args.source_root.resolve() | ||
| 64 | + work = args.work_dir.resolve() | ||
| 65 | + runtime = work / 'runtime' | ||
| 66 | + (runtime / 'common').mkdir(parents=True, exist_ok=True) | ||
| 67 | + for mode in ('multi',): | ||
| 68 | + build = work / ('build-' + mode) | ||
| 69 | + run(['cmake', '-S', source, '-B', build, | ||
| 70 | + '-DCMAKE_BUILD_TYPE=Release', '-DWITH_DEBUG_SYMBOLS=OFF', | ||
| 71 | + '-DWITH_URMA=ON', '-DURMA_USE_MOCK=OFF', '-DWITH_TCMALLOC=' + ('ON' if args.with_tcmalloc else 'OFF'), | ||
| 72 | + '-DWITH_CPU_FREQUENCY=ON', '-DBUILD_UNIT_TESTS=OFF', | ||
| 73 | + '-DBUILD_BRPC_TOOLS=OFF', '-DBUILD_SHARED_LIBS=ON', | ||
| 74 | + '-DCMAKE_PREFIX_PATH=' + args.prefix, | ||
| 75 | + '-DURMA_INCLUDE_PATH=' + args.urma_include, | ||
| 76 | + '-DURMA_BOND_INCLUDE_PATH=' + args.urma_bond_include, | ||
| 77 | + '-DURMA_LIB=' + args.urma_library, | ||
| 78 | + '-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON', | ||
| 79 | + '-DCMAKE_INSTALL_RPATH=$ORIGIN/../common']) | ||
| 80 | + run(['cmake', '--build', build, '--target', 'brpc-shared', '-j', args.jobs]) | ||
| 81 | + target = runtime / mode | ||
| 82 | + target.mkdir(exist_ok=True) | ||
| 83 | + install_file(build / 'output/lib/libbrpc.so', target / 'libbrpc.so', strip=True) | ||
| 84 | + allocator_options = ['-DWITH_TCMALLOC=OFF'] | ||
| 85 | + if args.with_tcmalloc: | ||
| 86 | + tcmalloc = Path(cache_value(work / 'build-multi', 'TCMALLOC_LIB')) | ||
| 87 | + install_file(tcmalloc.resolve(), runtime / 'common/libtcmalloc.so.4') | ||
| 88 | + allocator_options = ['-DWITH_TCMALLOC=ON', '-DTCMALLOC_LIB=' + str(tcmalloc)] | ||
| 89 | + else: | ||
| 90 | + (runtime / 'common/libtcmalloc.so.4').unlink(missing_ok=True) | ||
| 91 | + example = work / 'build-example' | ||
| 92 | + run(['cmake', '-S', source / 'example/ub_test', '-B', example, | ||
| 93 | + '-DCMAKE_BUILD_TYPE=Release', '-DLINK_SO=ON', '-DWITH_CPU_FREQUENCY=ON', | ||
| 94 | + '-DCMAKE_PREFIX_PATH=' + args.prefix, | ||
| 95 | + '-DBRPC_LIB=' + str(work / 'build-multi/output/lib/libbrpc.so'), | ||
| 96 | + '-DBRPC_INCLUDE_PATH=' + str(work / 'build-multi/output/include'), | ||
| 97 | + '-DURMA_INCLUDE_PATH=' + args.urma_include, | ||
| 98 | + '-DURMA_LIB=' + args.urma_library] + allocator_options) | ||
| 99 | + run(['cmake', '--build', example, '-j', min(args.jobs, 8)]) | ||
| 100 | + for mode in ('multi',): | ||
| 101 | + for role in ('client', 'server'): | ||
| 102 | + dest = runtime / mode / ('ub_test_' + role) | ||
| 103 | + install_file(example / dest.name, dest, strip=True) | ||
| 104 | + dependencies = {} | ||
| 105 | + for mode in ('multi',): | ||
| 106 | + env = dict(os.environ, LD_LIBRARY_PATH=str(runtime / mode) + ':' + str(runtime / 'common')) | ||
| 107 | + for role in ('client', 'server'): | ||
| 108 | + name = mode + '/ub_test_' + role | ||
| 109 | + dep = subprocess.check_output(['ldd', str(runtime / name)], env=env, text=True) | ||
| 110 | + if 'not found' in dep: | ||
| 111 | + raise RuntimeError(name + ': unresolved runtime dependency\n' + dep) | ||
| 112 | + dependencies[name] = dep | ||
| 113 | + (work / 'dependencies.json').write_text(json.dumps(dependencies, indent=2) + '\n') | ||
| 114 | + manifest = { | ||
| 115 | + 'upstream_repository': 'dwh110/brpc', | ||
| 116 | + 'upstream_commit': 'b4dff13b0de19a12dc3a1b82902a20d4b51b1bb1', | ||
| 117 | + 'architecture': subprocess.check_output(['uname', '-m'], text=True).strip(), | ||
| 118 | + 'with_cpu_frequency': True, 'with_tcmalloc': args.with_tcmalloc, 'urma_mock': False, | ||
| 119 | + 'mode_selection': 'Multi-SGE with shared URMA resources', | ||
| 120 | + 'files': {str(p.relative_to(runtime)): sha256(p) | ||
| 121 | + for p in sorted(runtime.rglob('*')) | ||
| 122 | + if p.is_file() and p != runtime / 'manifest.json' | ||
| 123 | + and p.relative_to(runtime).parts[0] in ('multi', 'common')}, | ||
| 124 | + 'source_sha256': {str(p.relative_to(source)): sha256(p) | ||
| 125 | + for folder in ('src/brpc/urma', 'example/ub_test') | ||
| 126 | + for p in sorted((source / folder).rglob('*')) | ||
| 127 | + if p.is_file() and '__pycache__' not in p.parts}, | ||
| 128 | + } | ||
| 129 | + (runtime / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') | ||
| 130 | + with tarfile.open(work / 'ub-test-runtime-aarch64.tar.gz', 'w:gz') as archive: | ||
| 131 | + for name in ('multi', 'common', 'manifest.json'): | ||
| 132 | + archive.add(runtime / name, arcname='runtime/' + name) | ||
| 133 | + print('BINARY_PACKAGE_READY ' + str(work / 'ub-test-runtime-aarch64.tar.gz'), flush=True) | ||
| 134 | + | ||
| 135 | + | ||
| 136 | +if __name__ == '__main__': | ||
| 137 | + main() | ||
| @@ -0,0 +1,440 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +"""Run the dwh110/brpc ub_test matrix using the pooled multi-SGE build. | ||
| 3 | + | ||
| 4 | +Upstream: b4dff13b0de19a12dc3a1b82902a20d4b51b1bb1/run_ub_test_matrix.py. | ||
| 5 | +Preserved workload: QPS 1000/500; global QD 10; 20 seconds; req_size=rsp_size | ||
| 6 | +1KiB,4KiB,8KiB,100KiB,200KiB,1MiB,8MiB; CPUs 96-111; memory node 1; | ||
| 7 | +RPC timeout 2000ms, connect timeout 6000ms, max_retry=10, initial_tokens=0. | ||
| 8 | +Only the transport selection and orchestration are adapted. Event mode is | ||
| 9 | +required by pooled URMA. Uses Python's standard library and OpenSSH, no Paramiko. | ||
| 10 | + | ||
| 11 | +The binary archive contains runtime/multi/ub_test_{client,server}, | ||
| 12 | +the corresponding libbrpc.so, an optional common/libtcmalloc.so.4, and manifest.json. | ||
| 13 | +No sources are uploaded. Existing system UMDK/Protobuf dependencies are required. | ||
| 14 | +Use --dry-run to inspect every command without contacting either host. | ||
| 15 | +""" | ||
| 16 | + | ||
| 17 | +import argparse | ||
| 18 | +import csv | ||
| 19 | +import json | ||
| 20 | +import os | ||
| 21 | +from pathlib import Path | ||
| 22 | +import re | ||
| 23 | +import shlex | ||
| 24 | +import subprocess | ||
| 25 | +import sys | ||
| 26 | +import time | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +UPSTREAM_COMMIT = 'b4dff13b0de19a12dc3a1b82902a20d4b51b1bb1' | ||
| 30 | +QPS_VALUES = [1000, 500] | ||
| 31 | +SIZES = [1024, 4096, 8192, 102400, 204800, 1048576, 8388608] | ||
| 32 | +METRICS = ['Avg-Latency', '50th-Latency', '90th-Latency', '99th-Latency', | ||
| 33 | + '99.9th-Latency', '99.99th-Latency', 'Max-Latency', 'Throughput', | ||
| 34 | + 'QPS', 'Server_CPU_avg', 'Server_CPU_max', 'Client_CPU_avg', | ||
| 35 | + 'Client_CPU_max', 'Client_Mem_avg', 'Client_Mem_max', 'Error_rate'] | ||
| 36 | +CSV_HEADER = ['urma_sge_mode', 'expected_qps', 'size'] + METRICS | ||
| 37 | +NUMBER = r'([0-9]+(?:\.[0-9]*)?(?:[eE][+-]?[0-9]+)?)' | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +def positive_int(value): | ||
| 41 | + number = int(value) | ||
| 42 | + if number <= 0: | ||
| 43 | + raise argparse.ArgumentTypeError('must be positive') | ||
| 44 | + return number | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +def parse_result(output): | ||
| 48 | + line = next((s for s in output.splitlines() if 'Avg-Latency:' in s), '') | ||
| 49 | + result = {} | ||
| 50 | + for field in METRICS[:9]: | ||
| 51 | + m = re.search(re.escape(field) + r':\s*' + NUMBER, line) | ||
| 52 | + result[field] = m.group(1) if m else 'N/A' | ||
| 53 | + for label, prefix in [('Server CPU', 'Server_CPU'), ('Client CPU', 'Client_CPU'), | ||
| 54 | + ('Client Memory', 'Client_Mem')]: | ||
| 55 | + m = re.search(re.escape(label) + r'\(avg/max\):\s*' + NUMBER + '/' + NUMBER, line) | ||
| 56 | + result[prefix + '_avg'] = m.group(1) if m else 'N/A' | ||
| 57 | + result[prefix + '_max'] = m.group(2) if m else 'N/A' | ||
| 58 | + m = re.search(r'Error rate:\s*' + NUMBER, line) | ||
| 59 | + result['Error_rate'] = m.group(1) if m else 'N/A' | ||
| 60 | + totals = {} | ||
| 61 | + for label in ('Total Sent', 'Total Errors', 'Total Requests'): | ||
| 62 | + m = re.search(re.escape(label) + r':\s*(\d+)', line) | ||
| 63 | + totals[label] = int(m.group(1)) if m else None | ||
| 64 | + return line, result, totals | ||
| 65 | + | ||
| 66 | + | ||
| 67 | +def command(args, role, mode, qps, size): | ||
| 68 | + root = getattr(args, role + '_dir') | ||
| 69 | + binary = root + '/runtime/' + mode + '/ub_test_' + role | ||
| 70 | + cmd = ['numactl', '-C', args.cpus, '-m', '1', binary, | ||
| 71 | + '--use_urma=true', '--urma_sge_mode=' + mode, | ||
| 72 | + '--urma_device=' + args.device, '--urma_use_polling=false', | ||
| 73 | + '--urma_resource_mode=pooled'] | ||
| 74 | + if getattr(args, 'io_jfs_count', None) is not None: | ||
| 75 | + cmd.append('--urma_io_jfs_cnt=' + str(args.io_jfs_count)) | ||
| 76 | + if role == 'server': | ||
| 77 | + cmd += ['--port', str(args.port), '--rsp_size=' + str(size), '--num_threads=16'] | ||
| 78 | + else: | ||
| 79 | + cmd += ['--servers=' + args.server_address + ':' + str(args.port), | ||
| 80 | + '--rpc_timeout_ms=2000', '--connect_timeout_ms=6000', | ||
| 81 | + '--test_seconds=20', '--max_retry=10', '--queue_depth=10', | ||
| 82 | + '--req_size=' + str(size), '--dummy_port=0', | ||
| 83 | + '--expected_qps=' + str(qps), '--initial_tokens=0'] | ||
| 84 | + return cmd | ||
| 85 | + | ||
| 86 | + | ||
| 87 | +class Remote: | ||
| 88 | + def __init__(self, host, key, ssh_multiplexing='auto'): | ||
| 89 | + self.host = host | ||
| 90 | + self.multiplexing = (sys.platform != 'win32' if ssh_multiplexing == 'auto' | ||
| 91 | + else ssh_multiplexing == 'on') | ||
| 92 | + self.options = ['-i', str(Path(key).expanduser()), '-o', 'BatchMode=yes', | ||
| 93 | + '-o', 'ConnectTimeout=10'] | ||
| 94 | + if self.multiplexing: | ||
| 95 | + self.options += ['-o', 'ControlMaster=auto', '-o', 'ControlPersist=600', | ||
| 96 | + '-o', 'ControlPath=/tmp/brpc-ub-matrix-%C'] | ||
| 97 | + else: | ||
| 98 | + # Windows OpenSSH cannot use the Unix control socket. Explicitly | ||
| 99 | + # disable all three options, including inherited ~/.ssh/config. | ||
| 100 | + self.options += ['-o', 'ControlMaster=no', '-o', 'ControlPersist=no', | ||
| 101 | + '-o', 'ControlPath=none'] | ||
| 102 | + | ||
| 103 | + def python(self, source, timeout=30): | ||
| 104 | + proc = subprocess.run(['ssh'] + self.options + [self.host, 'python3 -'], | ||
| 105 | + input=source, text=True, encoding='utf-8', | ||
| 106 | + capture_output=True, timeout=timeout) | ||
| 107 | + if proc.returncode: | ||
| 108 | + message = self.host + ': ' + proc.stderr + proc.stdout | ||
| 109 | + if self.multiplexing and 'getsockname failed' in proc.stderr.lower(): | ||
| 110 | + message += '\nSSH control socket failed; use --ssh-multiplexing off.' | ||
| 111 | + raise RuntimeError(message) | ||
| 112 | + return json.loads(proc.stdout) | ||
| 113 | + | ||
| 114 | + def put(self, local, remote): | ||
| 115 | + subprocess.run(['scp'] + self.options + [str(local), self.host + ':' + shlex.quote(remote)], | ||
| 116 | + check=True, timeout=120) | ||
| 117 | + | ||
| 118 | + def read(self, path): | ||
| 119 | + return self.python('import json,pathlib\np=pathlib.Path(' + repr(path) + | ||
| 120 | + ')\nprint(json.dumps(p.read_text() if p.exists() else ""))\n') | ||
| 121 | + | ||
| 122 | + def close(self): | ||
| 123 | + if self.multiplexing: | ||
| 124 | + try: | ||
| 125 | + subprocess.run(['ssh'] + self.options + ['-O', 'exit', self.host], | ||
| 126 | + capture_output=True, timeout=10) | ||
| 127 | + except (OSError, subprocess.SubprocessError): | ||
| 128 | + pass # Cleanup must not hide the original SSH/test failure. | ||
| 129 | + | ||
| 130 | + | ||
| 131 | +def deploy(remote, root, archive): | ||
| 132 | + remote.python('import pathlib,json\npathlib.Path(' + repr(root) + | ||
| 133 | + ').mkdir(parents=True,exist_ok=True)\nprint(json.dumps(True))\n') | ||
| 134 | + dest = root + '/ub-test-runtime.upload.tar.gz' | ||
| 135 | + remote.put(archive, dest) | ||
| 136 | + remote.python('''import json,pathlib,tarfile | ||
| 137 | +root=pathlib.Path(ROOT).resolve() | ||
| 138 | +archive=root/'ub-test-runtime.upload.tar.gz' | ||
| 139 | +with tarfile.open(archive) as a: | ||
| 140 | + for member in a.getmembers(): | ||
| 141 | + path=(root/member.name).resolve() | ||
| 142 | + if root not in path.parents or member.issym() or member.islnk(): | ||
| 143 | + raise RuntimeError('Unsafe archive member: '+member.name) | ||
| 144 | + a.extractall(root) | ||
| 145 | +archive.unlink() | ||
| 146 | +print(json.dumps(True)) | ||
| 147 | +'''.replace('ROOT', repr(root)), timeout=60) | ||
| 148 | + | ||
| 149 | + | ||
| 150 | +def preflight(remote, root, cpus, port, server): | ||
| 151 | + return remote.python('''import hashlib,json,os,pathlib,subprocess | ||
| 152 | +root=pathlib.Path(ROOT);runtime=root/'runtime' | ||
| 153 | +manifest=json.loads((runtime/'manifest.json').read_text()) | ||
| 154 | +hashes={name:hashlib.sha256((runtime/name).read_bytes()).hexdigest() for name in manifest['files']} | ||
| 155 | +if hashes!=manifest['files']:raise RuntimeError('Binary manifest mismatch') | ||
| 156 | +deps={} | ||
| 157 | +for mode in ['multi']: | ||
| 158 | + env=dict(os.environ,LD_LIBRARY_PATH=str(runtime/mode)+':'+str(runtime/'common')) | ||
| 159 | + for role in ['client','server']: | ||
| 160 | + key=mode+'/ub_test_'+role | ||
| 161 | + deps[key]=subprocess.check_output(['ldd',str(runtime/key)],text=True,env=env) | ||
| 162 | + if 'not found' in deps[key]:raise RuntimeError(deps[key]) | ||
| 163 | +binding=subprocess.check_output(['numactl','-C',CPUS,'-m','1','--show'],text=True,stderr=subprocess.STDOUT) | ||
| 164 | +if SERVER: | ||
| 165 | + listeners=subprocess.check_output(['ss','-H','-ltn'],text=True) | ||
| 166 | + if any(s.split()[3].rsplit(':',1)[-1]==str(PORT) for s in listeners.splitlines()): | ||
| 167 | + raise RuntimeError('Test port already in use; refusing to stop its owner') | ||
| 168 | +(root/'logs').mkdir(exist_ok=True) | ||
| 169 | +print(json.dumps({'manifest':manifest,'hashes':hashes,'dependencies':deps,'binding':binding, | ||
| 170 | + 'architecture':subprocess.check_output(['uname','-m'],text=True).strip()})) | ||
| 171 | +'''.replace('ROOT', repr(root)).replace('CPUS', repr(cpus)) | ||
| 172 | + .replace('SERVER', repr(server)).replace('PORT', str(port)), timeout=60) | ||
| 173 | + | ||
| 174 | + | ||
| 175 | +def launch(remote, root, name, role, mode, argv): | ||
| 176 | + spec = {'args': argv, 'log': root + '/logs/' + name + '-' + role + '.log', | ||
| 177 | + 'pid': root + '/logs/' + name + '-' + role + '.pid', | ||
| 178 | + 'exit': root + '/logs/' + name + '-' + role + '.exit', | ||
| 179 | + 'library_path': root + '/runtime/' + mode + ':' + root + '/runtime/common'} | ||
| 180 | + # The supervisor is sent in the SSH command, never deployed as a source file. | ||
| 181 | + supervisor = '''import json,os,pathlib,subprocess,sys | ||
| 182 | +s=json.loads(sys.argv[1]);env=dict(os.environ,LD_LIBRARY_PATH=s['library_path']) | ||
| 183 | +with open(s['log'],'w') as f: | ||
| 184 | + p=subprocess.Popen(s['args'],stdin=subprocess.DEVNULL,stdout=f,stderr=subprocess.STDOUT,env=env) | ||
| 185 | + pathlib.Path(s['pid']).write_text(str(p.pid)) | ||
| 186 | + code=p.wait() | ||
| 187 | +pathlib.Path(s['exit']).write_text(str(code)) | ||
| 188 | +''' | ||
| 189 | + result = remote.python('''import json,pathlib,subprocess,time | ||
| 190 | +s=SPEC | ||
| 191 | +for key in ['pid','exit']: | ||
| 192 | + p=pathlib.Path(s[key]) | ||
| 193 | + if p.exists():p.unlink() | ||
| 194 | +p=subprocess.Popen(['python3','-c',SUPERVISOR,json.dumps(s)],stdin=subprocess.DEVNULL, | ||
| 195 | + stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True) | ||
| 196 | +for _ in range(100): | ||
| 197 | + if pathlib.Path(s['pid']).exists():break | ||
| 198 | + time.sleep(.02) | ||
| 199 | +print(json.dumps({**s,'pid':int(pathlib.Path(s['pid']).read_text()),'supervisor_pid':p.pid})) | ||
| 200 | +'''.replace('SPEC', repr(spec)).replace('SUPERVISOR', repr(supervisor))) | ||
| 201 | + return result | ||
| 202 | + | ||
| 203 | + | ||
| 204 | +def process_info(remote, proc): | ||
| 205 | + return remote.python('''import collections,json,os,pathlib,re | ||
| 206 | +p=pathlib.Path('/proc')/str(PID) | ||
| 207 | +threads=[] | ||
| 208 | +for t in (p/'task').iterdir(): | ||
| 209 | + try:threads.append(sorted(os.sched_getaffinity(int(t.name)))) | ||
| 210 | + except ProcessLookupError:pass | ||
| 211 | +maps=(p/'maps').read_text();numa=(p/'numa_maps').read_text() | ||
| 212 | +policies=collections.Counter(s.split()[1] for s in numa.splitlines() if len(s.split())>1) | ||
| 213 | +manifest=json.loads((p/'exe').resolve().parent.parent.joinpath('manifest.json').read_text()) | ||
| 214 | +print(json.dumps({'thread_affinities':threads,'numa_policies':dict(policies), | ||
| 215 | + 'expects_tcmalloc':'common/libtcmalloc.so.4' in manifest['files'], | ||
| 216 | + 'libraries':sorted({s.split()[-1] for s in maps.splitlines() if 'libbrpc.so' in s or 'libtcmalloc' in s}), | ||
| 217 | + 'status':(p/'status').read_text()})) | ||
| 218 | +'''.replace('PID', str(proc['pid']))) | ||
| 219 | + | ||
| 220 | + | ||
| 221 | +def socket_snapshot(remote, port): | ||
| 222 | + return remote.python('''import json,re,urllib.request | ||
| 223 | +opener=urllib.request.build_opener(urllib.request.ProxyHandler({})) | ||
| 224 | +base='http://127.0.0.1:PORT' | ||
| 225 | +def get(path):return opener.open(base+path+'?console=1',timeout=3).read().decode() | ||
| 226 | +connections=get('/connections');sockets=[] | ||
| 227 | +for line in connections.splitlines(): | ||
| 228 | + m=re.search(r'(\d+)\s*$',line) | ||
| 229 | + if not m:continue | ||
| 230 | + detail=get('/sockets/'+m.group(1)) | ||
| 231 | + if 'state=ESTABLISHED' in detail: | ||
| 232 | + sockets.append({'id':m.group(1),'detail':detail}) | ||
| 233 | +print(json.dumps({'connections':connections,'urma_sockets':sockets})) | ||
| 234 | +'''.replace('PORT', str(port))) | ||
| 235 | + | ||
| 236 | + | ||
| 237 | +def stop(remote, proc): | ||
| 238 | + executable = next(s for s in proc['args'] if '/ub_test_' in s) | ||
| 239 | + return remote.python('''import json,os,pathlib,signal,time | ||
| 240 | +p=pathlib.Path('/proc')/str(PID) | ||
| 241 | +def alive(): | ||
| 242 | + try:return (p/'stat').read_text().split()[2]!='Z' and (p/'cmdline').read_bytes().split(bytes([0]))[0].decode()==EXE | ||
| 243 | + except (FileNotFoundError,ProcessLookupError):return False | ||
| 244 | +if alive(): | ||
| 245 | + try:os.kill(PID,signal.SIGINT) | ||
| 246 | + except ProcessLookupError:pass | ||
| 247 | + for _ in range(50): | ||
| 248 | + if not alive():break | ||
| 249 | + time.sleep(.1) | ||
| 250 | + if alive(): | ||
| 251 | + try:os.kill(PID,signal.SIGKILL) | ||
| 252 | + except ProcessLookupError:pass | ||
| 253 | + for _ in range(20): | ||
| 254 | + if not alive():break | ||
| 255 | + time.sleep(.05) | ||
| 256 | +print(json.dumps({'stopped':not alive()})) | ||
| 257 | +'''.replace('PID', str(proc['pid'])).replace('EXE', repr(executable))) | ||
| 258 | + | ||
| 259 | + | ||
| 260 | +def expected_cpus(value): | ||
| 261 | + cpus = set() | ||
| 262 | + for part in value.split(','): | ||
| 263 | + ends = part.split('-') | ||
| 264 | + cpus.update(range(int(ends[0]), int(ends[-1]) + 1)) | ||
| 265 | + return cpus | ||
| 266 | + | ||
| 267 | + | ||
| 268 | +def statistics_failures(returncode, line, metrics, totals): | ||
| 269 | + failures = [] | ||
| 270 | + if returncode != 0: | ||
| 271 | + failures.append('client exit code') | ||
| 272 | + if not line or any(v == 'N/A' for v in metrics.values()): | ||
| 273 | + failures.append('incomplete statistics') | ||
| 274 | + # Rounded "Error rate: 0%" must not hide a small non-zero error count. | ||
| 275 | + if totals['Total Errors'] != 0: | ||
| 276 | + failures.append('RPC errors') | ||
| 277 | + if not totals['Total Requests']: | ||
| 278 | + failures.append('no completed RPCs') | ||
| 279 | + return failures | ||
| 280 | + | ||
| 281 | + | ||
| 282 | +def run_test(args, remotes, mode, qps, size): | ||
| 283 | + name = '{}-q{}-s{}'.format(mode, qps, size) | ||
| 284 | + out = Path(args.output) | ||
| 285 | + result = {'name': name, 'sge_mode': mode, 'expected_qps': qps, 'size': size, | ||
| 286 | + 'io_jfs_count': getattr(args, 'io_jfs_count', None), | ||
| 287 | + 'valid': False, 'processes': {}, 'cleanup': {}} | ||
| 288 | + logs = {} | ||
| 289 | + try: | ||
| 290 | + server = launch(remotes['server'], args.server_dir, name, 'server', mode, | ||
| 291 | + command(args, 'server', mode, qps, size)) | ||
| 292 | + result['processes']['server'] = server | ||
| 293 | + time.sleep(3) # Preserve the original pre-client startup interval. | ||
| 294 | + server_log = remotes['server'].read(server['log']) | ||
| 295 | + marker = 'UB_TEST_TRANSPORT sge_mode=' + mode | ||
| 296 | + if remotes['server'].read(server['exit']).strip(): | ||
| 297 | + raise RuntimeError('Server exited during startup: ' + server_log[-1500:]) | ||
| 298 | + if marker not in server_log: | ||
| 299 | + raise RuntimeError('Server did not initialize the requested URMA build: ' + server_log[-1500:]) | ||
| 300 | + client = launch(remotes['client'], args.client_dir, name, 'client', mode, | ||
| 301 | + command(args, 'client', mode, qps, size)) | ||
| 302 | + result['processes']['client'] = client | ||
| 303 | + deadline = time.monotonic() + (300 if size >= 1048576 else 120) | ||
| 304 | + time.sleep(3) | ||
| 305 | + # Auditing must not shorten a benchmark if a process has exited or a | ||
| 306 | + # diagnostics request fails. Keep its final statistics and raw logs. | ||
| 307 | + audit_failures = [] | ||
| 308 | + result['process_info'] = {} | ||
| 309 | + for role, proc in result['processes'].items(): | ||
| 310 | + try: | ||
| 311 | + result['process_info'][role] = process_info(remotes[role], proc) | ||
| 312 | + except Exception as error: | ||
| 313 | + audit_failures.append(role + ' process audit: ' + str(error)) | ||
| 314 | + try: | ||
| 315 | + result['sockets'] = socket_snapshot(remotes['server'], args.port) | ||
| 316 | + except Exception as error: | ||
| 317 | + result['sockets'] = {'urma_sockets': []} | ||
| 318 | + audit_failures.append('socket audit: ' + str(error)) | ||
| 319 | + while time.monotonic() < deadline: | ||
| 320 | + code = remotes['client'].read(client['exit']) | ||
| 321 | + if code.strip(): | ||
| 322 | + result['client_returncode'] = int(code) | ||
| 323 | + break | ||
| 324 | + time.sleep(1) | ||
| 325 | + else: | ||
| 326 | + raise TimeoutError('Client exceeded the original matrix timeout') | ||
| 327 | + logs['client'] = remotes['client'].read(client['log']) | ||
| 328 | + logs['server'] = remotes['server'].read(server['log']) | ||
| 329 | + line, metrics, totals = parse_result(logs['client']) | ||
| 330 | + result.update(result_line=line, metrics=metrics, totals=totals) | ||
| 331 | + failures = statistics_failures(result['client_returncode'], line, metrics, totals) + audit_failures | ||
| 332 | + if len(result['sockets']['urma_sockets']) != 1: failures.append('expected one established URMA connection') | ||
| 333 | + for role, info in result['process_info'].items(): | ||
| 334 | + if marker not in logs[role] or 'tp=CTP' not in logs[role]: failures.append(role + ' transport') | ||
| 335 | + if result['io_jfs_count'] is not None: | ||
| 336 | + counts = re.findall(r'\bjfs_count=(\d+)\b', logs[role]) | ||
| 337 | + if counts != [str(result['io_jfs_count'])]: | ||
| 338 | + failures.append(role + ' JFS count') | ||
| 339 | + if not info['thread_affinities'] or any(set(c) != expected_cpus(args.cpus) for c in info['thread_affinities']): | ||
| 340 | + failures.append(role + ' CPU affinity') | ||
| 341 | + if set(info['numa_policies']) != {'bind:1'}: failures.append(role + ' memory binding') | ||
| 342 | + root = getattr(args, role + '_dir') | ||
| 343 | + if root + '/runtime/' + mode + '/libbrpc.so' not in info['libraries']: | ||
| 344 | + failures.append(role + ' libbrpc') | ||
| 345 | + if (info.get('expects_tcmalloc', False) and | ||
| 346 | + root + '/runtime/common/libtcmalloc.so.4' not in info['libraries']): | ||
| 347 | + failures.append(role + ' tcmalloc') | ||
| 348 | + result['failures'] = failures | ||
| 349 | + result['valid'] = not failures | ||
| 350 | + except Exception as error: | ||
| 351 | + result['exception'] = str(error) | ||
| 352 | + finally: | ||
| 353 | + for role in ('client', 'server'): | ||
| 354 | + proc = result['processes'].get(role) | ||
| 355 | + if proc: | ||
| 356 | + try: | ||
| 357 | + result['cleanup'][role] = stop(remotes[role], proc) | ||
| 358 | + if not result['cleanup'][role]['stopped']: | ||
| 359 | + result['valid'] = False | ||
| 360 | + logs[role] = remotes[role].read(proc['log']) | ||
| 361 | + except Exception as error: | ||
| 362 | + result['cleanup'][role] = {'stopped': False, 'error': str(error)} | ||
| 363 | + result['valid'] = False | ||
| 364 | + for role, log in logs.items(): | ||
| 365 | + (out / (name + '-' + role + '.log')).write_text(log) | ||
| 366 | + (out / (name + '.json')).write_text(json.dumps(result, indent=2) + '\n') | ||
| 367 | + print(json.dumps({k: result.get(k) for k in ('name', 'valid', 'metrics', 'failures', 'exception')}), flush=True) | ||
| 368 | + time.sleep(2) # Preserve the original inter-case interval. | ||
| 369 | + return result | ||
| 370 | + | ||
| 371 | + | ||
| 372 | +def main(): | ||
| 373 | + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | ||
| 374 | + parser.add_argument('--server', required=True, help='SSH user@host of the server/build node') | ||
| 375 | + parser.add_argument('--client', required=True, help='SSH user@host of the client node') | ||
| 376 | + parser.add_argument('--key', default='~/.ssh/id_ed25519') | ||
| 377 | + parser.add_argument('--ssh-multiplexing', choices=['auto', 'off', 'on'], default='auto', | ||
| 378 | + help='SSH connection reuse: auto disables it on Windows; off disables it everywhere') | ||
| 379 | + parser.add_argument('--server-address', help='RPC address; defaults to the host in --server') | ||
| 380 | + parser.add_argument('--server-dir', required=True, help='Absolute remote directory containing runtime/') | ||
| 381 | + parser.add_argument('--client-dir', required=True) | ||
| 382 | + parser.add_argument('--device', required=True) | ||
| 383 | + parser.add_argument('--port', type=int, default=10918) | ||
| 384 | + parser.add_argument('--cpus', default='96-111') | ||
| 385 | + parser.add_argument('--io-jfs-count', type=positive_int, | ||
| 386 | + help='Override shared JFS count on both nodes; otherwise use the library default') | ||
| 387 | + parser.add_argument('--modes', nargs='+', choices=['multi'], default=['multi']) | ||
| 388 | + parser.add_argument('--qps', nargs='+', type=int, choices=QPS_VALUES, default=QPS_VALUES) | ||
| 389 | + parser.add_argument('--sizes', nargs='+', type=int, choices=SIZES, default=SIZES) | ||
| 390 | + parser.add_argument('--deploy', type=Path, help='Deploy only the runtime binary archive to both nodes') | ||
| 391 | + parser.add_argument('--output', default='ub_test_results') | ||
| 392 | + parser.add_argument('--dry-run', action='store_true') | ||
| 393 | + args = parser.parse_args() | ||
| 394 | + args.server_address = args.server_address or args.server.rsplit('@', 1)[-1] | ||
| 395 | + if not all(p.startswith('/') for p in (args.server_dir, args.client_dir)): | ||
| 396 | + parser.error('Remote runtime directories must be absolute') | ||
| 397 | + plan = [{'sge_mode': m, 'qps': q, 'size': s, | ||
| 398 | + 'server': command(args, 'server', m, q, s), | ||
| 399 | + 'client': command(args, 'client', m, q, s)} | ||
| 400 | + for m in args.modes for q in args.qps for s in args.sizes] | ||
| 401 | + if args.dry_run: | ||
| 402 | + print(json.dumps(plan, indent=2)) | ||
| 403 | + return 0 | ||
| 404 | + out = Path(args.output) | ||
| 405 | + out.mkdir(parents=True, exist_ok=True) | ||
| 406 | + if (out / 'results.json').exists(): | ||
| 407 | + parser.error('Output already contains results; choose a new output directory') | ||
| 408 | + (out / 'plan.json').write_text(json.dumps({'upstream_commit': UPSTREAM_COMMIT, 'cases': plan, | ||
| 409 | + 'transport_changes': 'multi-SGE transport; event mode; pooled transport defaults; selected device', | ||
| 410 | + 'statistics': 'Unchanged upstream ub_test calculations; CSV retains original columns except SGE mode'}, indent=2) + '\n') | ||
| 411 | + remotes = {role: Remote(getattr(args, role), args.key, args.ssh_multiplexing) | ||
| 412 | + for role in ('server', 'client')} | ||
| 413 | + results = [] | ||
| 414 | + try: | ||
| 415 | + if args.deploy: | ||
| 416 | + for role, remote in remotes.items(): | ||
| 417 | + deploy(remote, getattr(args, role + '_dir'), args.deploy) | ||
| 418 | + audit = {role: preflight(remote, getattr(args, role + '_dir'), args.cpus, args.port, role == 'server') | ||
| 419 | + for role, remote in remotes.items()} | ||
| 420 | + if audit['server']['hashes'] != audit['client']['hashes']: | ||
| 421 | + raise RuntimeError('The two nodes have different runtime binaries') | ||
| 422 | + (out / 'preflight.json').write_text(json.dumps(audit, indent=2) + '\n') | ||
| 423 | + for case in plan: | ||
| 424 | + results.append(run_test(args, remotes, case['sge_mode'], case['qps'], case['size'])) | ||
| 425 | + (out / 'results.json').write_text(json.dumps(results, indent=2) + '\n') | ||
| 426 | + with (out / 'ub_test_results.csv').open('w', newline='') as stream: | ||
| 427 | + writer = csv.writer(stream) | ||
| 428 | + writer.writerow(CSV_HEADER) | ||
| 429 | + for r in results: | ||
| 430 | + writer.writerow([r['sge_mode'], r['expected_qps'], r['size']] + | ||
| 431 | + [r.get('metrics', {}).get(k, 'ERROR') for k in METRICS]) | ||
| 432 | + print(json.dumps({'completed': len(results), 'valid': sum(r['valid'] for r in results), | ||
| 433 | + 'output': str(out.resolve())}), flush=True) | ||
| 434 | + return 0 if all(r['valid'] for r in results) else 1 | ||
| 435 | + finally: | ||
| 436 | + for remote in remotes.values(): remote.close() | ||
| 437 | + | ||
| 438 | + | ||
| 439 | +if __name__ == '__main__': | ||
| 440 | + raise SystemExit(main()) | ||
| @@ -0,0 +1,242 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information | ||
| 4 | +// regarding copyright ownership. The ASF licenses this file | ||
| 5 | +// to you under the Apache License, Version 2.0 (the | ||
| 6 | +// "License"); you may not use this file except in compliance | ||
| 7 | +// with the License. You may obtain a copy of the License at | ||
| 8 | +// | ||
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +// | ||
| 11 | +// Unless required by applicable law or agreed to in writing, | ||
| 12 | +// software distributed under the License is distributed on an | ||
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +// KIND, either express or implied. See the License for the | ||
| 15 | +// specific language governing permissions and limitations | ||
| 16 | +// under the License. | ||
| 17 | + | ||
| 18 | +#include <fcntl.h> | ||
| 19 | +#include <pthread.h> | ||
| 20 | +#include <gflags/gflags.h> | ||
| 21 | +#include "butil/atomicops.h" | ||
| 22 | +#include "butil/logging.h" | ||
| 23 | +#include "butil/time.h" | ||
| 24 | +#include "brpc/server.h" | ||
| 25 | +#include "brpc/socket_mode.h" | ||
| 26 | +#include "brpc/urma/urma_helper.h" | ||
| 27 | +#include "bvar/variable.h" | ||
| 28 | +#include "bvar/latency_recorder.h" | ||
| 29 | +#include "test.pb.h" | ||
| 30 | +#include "sge_mode.h" | ||
| 31 | + | ||
| 32 | +DEFINE_int32(port, 8002, "TCP Port of this server"); | ||
| 33 | +DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); | ||
| 34 | +DEFINE_bool(use_rdma, false, "Use RDMA or not"); | ||
| 35 | + | ||
| 36 | +DEFINE_bool(server_ignore_oc, false, "Server ignore eovercrowded, false by default"); | ||
| 37 | +DEFINE_int32(num_threads, 5, "The max number of threads are used"); | ||
| 38 | +DEFINE_int32(max_concurrency, 128, "max concurrency"); | ||
| 39 | +DEFINE_int32(server_bthread_concurrency, 5, "server bthread concurrency"); | ||
| 40 | +DEFINE_int64(rsp_size, 0, "Response name size in bytes (0 means echo the request name back)"); | ||
| 41 | +DEFINE_int32(stats_timeout_seconds, 3, "Timeout in seconds before collecting statistics. (default 3)"); | ||
| 42 | +DEFINE_int32(test_seconds, 40, "Test duration in seconds (should match client)"); | ||
| 43 | + | ||
| 44 | +pthread_mutex_t g_metrics_lock = PTHREAD_MUTEX_INITIALIZER; | ||
| 45 | +std::string g_cpu_usage_str; // cached describe_exposed("process_cpu_usage") value | ||
| 46 | +butil::atomic<bool> g_cpu_usage_ready(false); // claim token: at most one response per 1s cycle | ||
| 47 | +butil::atomic<uint64_t> g_total_cnt(0); | ||
| 48 | +butil::atomic<uint64_t> g_total_bytes(0); | ||
| 49 | +butil::atomic<uint64_t> g_start_time(0); | ||
| 50 | +butil::atomic<uint64_t> g_last_request_time(0); | ||
| 51 | +butil::atomic<uint64_t> g_end_time(0); | ||
| 52 | +bvar::LatencyRecorder g_server_latency_recorder("server_latency"); | ||
| 53 | +bvar::LatencyRecorder g_server_cpu_recorder("server_cpu"); | ||
| 54 | +bvar::LatencyRecorder g_server_memory_recorder("server_memory"); | ||
| 55 | +std::string g_name; | ||
| 56 | +volatile bool g_stop = false; | ||
| 57 | +brpc::Server* g_server = nullptr; | ||
| 58 | + | ||
| 59 | +static void* StatsPrinter(void* arg) { | ||
| 60 | + brpc::Server* server = static_cast<brpc::Server*>(arg); | ||
| 61 | + | ||
| 62 | + while (!g_stop) { | ||
| 63 | + bthread_usleep(100000); | ||
| 64 | + | ||
| 65 | + // Sample process CPU/memory every 100ms, off the RPC hot path. | ||
| 66 | + const std::string cpu_usage_str = bvar::Variable::describe_exposed("process_cpu_usage"); | ||
| 67 | + const double cpu_percent = atof(cpu_usage_str.c_str()) * 100; | ||
| 68 | + const double memory_mb = atof( | ||
| 69 | + bvar::Variable::describe_exposed("process_memory_resident").c_str()) / 1024 / 1024; | ||
| 70 | + | ||
| 71 | + pthread_mutex_lock(&g_metrics_lock); | ||
| 72 | + g_cpu_usage_str = cpu_usage_str; | ||
| 73 | + g_cpu_usage_ready.store(true, butil::memory_order_release); | ||
| 74 | + pthread_mutex_unlock(&g_metrics_lock); | ||
| 75 | + | ||
| 76 | + g_server_cpu_recorder << cpu_percent; | ||
| 77 | + g_server_memory_recorder << memory_mb; | ||
| 78 | + | ||
| 79 | + uint64_t now = butil::monotonic_time_us(); | ||
| 80 | + uint64_t last = g_last_request_time.load(butil::memory_order_relaxed); | ||
| 81 | + uint64_t end = g_end_time.load(butil::memory_order_relaxed); | ||
| 82 | + uint64_t start_time = g_start_time.load(butil::memory_order_relaxed); | ||
| 83 | + | ||
| 84 | + if (end == 0 && start_time > 0) { | ||
| 85 | + uint64_t test_duration_us = now - start_time; | ||
| 86 | + uint64_t trigger_time_us = ((uint64_t)FLAGS_test_seconds + FLAGS_stats_timeout_seconds) * 1000000; | ||
| 87 | + | ||
| 88 | + if (test_duration_us >= trigger_time_us) { | ||
| 89 | + uint64_t end_expected = 0; | ||
| 90 | + if (g_end_time.compare_exchange_strong(end_expected, last)) { | ||
| 91 | + double duration_s = (last - start_time) / 1000000.0; | ||
| 92 | + uint64_t total_cnt = g_total_cnt.load(butil::memory_order_relaxed); | ||
| 93 | + uint64_t total_bytes = g_total_bytes.load(butil::memory_order_relaxed); | ||
| 94 | + | ||
| 95 | + std::cout << "========== Final Report ==========" << std::endl; | ||
| 96 | + std::cout << "Duration: " << duration_s << "s" << std::endl; | ||
| 97 | + std::cout << "Total Requests: " << total_cnt << std::endl; | ||
| 98 | + if (duration_s > 0) { | ||
| 99 | + std::cout << "QPS: " << total_cnt / duration_s << std::endl; | ||
| 100 | + std::cout << "Bandwidth: " << total_bytes / 1024.0 / 1024.0 / duration_s << "MB/s" << std::endl; | ||
| 101 | + } | ||
| 102 | + std::cout << "Latency(avg/p99/p999/p9999): " | ||
| 103 | + << g_server_latency_recorder.latency(10) << "/" | ||
| 104 | + << g_server_latency_recorder.latency_percentile(0.99) << "/" | ||
| 105 | + << g_server_latency_recorder.latency_percentile(0.999) << "/" | ||
| 106 | + << g_server_latency_recorder.latency_percentile(0.9999) << "us" << std::endl; | ||
| 107 | + std::cout << "CPU(avg/max): " | ||
| 108 | + << g_server_cpu_recorder.latency(10) << "/" | ||
| 109 | + << g_server_cpu_recorder.max_latency() << "%" << std::endl; | ||
| 110 | + std::cout << "Memory(avg/max): " | ||
| 111 | + << g_server_memory_recorder.latency(10) << "/" | ||
| 112 | + << g_server_memory_recorder.max_latency() << "MB" << std::endl; | ||
| 113 | + std::cout << "==================================" << std::endl; | ||
| 114 | + return nullptr; | ||
| 115 | + } | ||
| 116 | + } | ||
| 117 | + } | ||
| 118 | + } | ||
| 119 | + return nullptr; | ||
| 120 | +} | ||
| 121 | + | ||
| 122 | +namespace test { | ||
| 123 | +class PerfTestServiceImpl : public PerfTestService { | ||
| 124 | +public: | ||
| 125 | + PerfTestServiceImpl() {} | ||
| 126 | + | ||
| 127 | + ~PerfTestServiceImpl() {} | ||
| 128 | + | ||
| 129 | + void Test(google::protobuf::RpcController* cntl_base, | ||
| 130 | + const PerfTestRequest* request, | ||
| 131 | + PerfTestResponse* response, | ||
| 132 | + google::protobuf::Closure* done) { | ||
| 133 | + brpc::ClosureGuard done_guard(done); | ||
| 134 | + uint64_t now = butil::monotonic_time_us(); | ||
| 135 | + | ||
| 136 | + uint64_t expected = 0; | ||
| 137 | + if (g_start_time.compare_exchange_strong(expected, now)) { | ||
| 138 | + bthread_t tid; | ||
| 139 | + bthread_start_background(&tid, &BTHREAD_ATTR_NORMAL, StatsPrinter, g_server); | ||
| 140 | + } | ||
| 141 | + g_last_request_time.store(now, butil::memory_order_relaxed); | ||
| 142 | + | ||
| 143 | + if (g_cpu_usage_ready.exchange(false, butil::memory_order_acq_rel)) { | ||
| 144 | + pthread_mutex_lock(&g_metrics_lock); | ||
| 145 | + response->set_cpu_usage(g_cpu_usage_str); | ||
| 146 | + pthread_mutex_unlock(&g_metrics_lock); | ||
| 147 | + } else { | ||
| 148 | + response->set_cpu_usage(""); // proto2 required field: must be set explicitly | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base); | ||
| 152 | + | ||
| 153 | + if (FLAGS_rsp_size == 0) { | ||
| 154 | + response->set_name(request->name()); | ||
| 155 | + } else { | ||
| 156 | + response->set_name(g_name); | ||
| 157 | + } | ||
| 158 | + if (request->echo_attachment()) { | ||
| 159 | + cntl->response_attachment().append(cntl->request_attachment()); | ||
| 160 | + g_total_bytes.fetch_add(cntl->response_attachment().size(), butil::memory_order_relaxed); | ||
| 161 | + } else { | ||
| 162 | + g_total_bytes.fetch_add(response->name().size(), butil::memory_order_relaxed); | ||
| 163 | + } | ||
| 164 | + | ||
| 165 | + g_total_cnt.fetch_add(1, butil::memory_order_relaxed); | ||
| 166 | + g_server_latency_recorder << cntl->latency_us(); | ||
| 167 | + } | ||
| 168 | +}; | ||
| 169 | +} | ||
| 170 | + | ||
| 171 | +namespace bthread { | ||
| 172 | + DECLARE_int32(bthread_concurrency); | ||
| 173 | +} | ||
| 174 | +DEFINE_string(urma_sge_mode, "auto", "Expected loaded URMA build: auto or multi"); | ||
| 175 | + | ||
| 176 | +constexpr int FD_NUM = 20000; | ||
| 177 | + | ||
| 178 | +int main(int argc, char* argv[]) { | ||
| 179 | + // 预申请fd 规避扩容时延 | ||
| 180 | + int dummy_fd = open("/dev/null", O_RDONLY); | ||
| 181 | + int new_fd = dup2(dummy_fd, FD_NUM); | ||
| 182 | + if (dummy_fd != -1) close(dummy_fd); | ||
| 183 | + if (new_fd != -1) close(new_fd); | ||
| 184 | + | ||
| 185 | + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); | ||
| 186 | + | ||
| 187 | + bthread::FLAGS_bthread_concurrency = FLAGS_server_bthread_concurrency; | ||
| 188 | + | ||
| 189 | + // Initialize URMA before any IOBuf operations to ensure all IOBuf blocks | ||
| 190 | + // are allocated from the URMA pool, not the original malloc. | ||
| 191 | + if (FLAGS_use_urma) { | ||
| 192 | + brpc::urma::GlobalUrmaInitializeOrDie(); | ||
| 193 | + if (!ValidateUrmaSgeMode(FLAGS_urma_sge_mode)) { | ||
| 194 | + return 2; | ||
| 195 | + } | ||
| 196 | + } | ||
| 197 | + | ||
| 198 | + brpc::Server server; | ||
| 199 | + test::PerfTestServiceImpl perf_test_service_impl; | ||
| 200 | + | ||
| 201 | + g_name.resize(FLAGS_rsp_size, 'r'); | ||
| 202 | + if (FLAGS_rsp_size == 0) { | ||
| 203 | + std::cout << "server rsp_size=0: echoing request name in response" << std::endl; | ||
| 204 | + } else { | ||
| 205 | + std::cout << "server rsp/name len is " << g_name.size() << "B" << std::endl; | ||
| 206 | + } | ||
| 207 | + std::cout << "URMA: " << (FLAGS_use_urma ? "yes" : "no") | ||
| 208 | + << ", URMA_SGE_MODE: " << FLAGS_urma_sge_mode | ||
| 209 | + << std::endl; | ||
| 210 | + | ||
| 211 | + g_total_cnt.store(0, butil::memory_order_relaxed); | ||
| 212 | + g_total_bytes.store(0, butil::memory_order_relaxed); | ||
| 213 | + g_start_time.store(0, butil::memory_order_relaxed); | ||
| 214 | + g_last_request_time.store(0, butil::memory_order_relaxed); | ||
| 215 | + g_end_time.store(0, butil::memory_order_relaxed); | ||
| 216 | + | ||
| 217 | + if (server.AddService(&perf_test_service_impl, | ||
| 218 | + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { | ||
| 219 | + LOG(ERROR) << "Fail to add service"; | ||
| 220 | + return -1; | ||
| 221 | + } | ||
| 222 | + brpc::ServerOptions options; | ||
| 223 | + if (FLAGS_use_urma) { | ||
| 224 | + options.socket_mode = brpc::SOCKET_MODE_URMA; | ||
| 225 | + } else if (FLAGS_use_rdma) { | ||
| 226 | + options.socket_mode = brpc::SOCKET_MODE_RDMA; | ||
| 227 | + } else { | ||
| 228 | + options.socket_mode = brpc::SOCKET_MODE_TCP; | ||
| 229 | + } | ||
| 230 | + options.max_concurrency = FLAGS_max_concurrency; | ||
| 231 | + options.num_threads = FLAGS_num_threads; | ||
| 232 | + options.ignore_eovercrowded = FLAGS_server_ignore_oc; | ||
| 233 | + if (server.Start(FLAGS_port, &options) != 0) { | ||
| 234 | + LOG(ERROR) << "Fail to start EchoServer"; | ||
| 235 | + return -1; | ||
| 236 | + } | ||
| 237 | + g_server = &server; | ||
| 238 | + | ||
| 239 | + server.RunUntilAskedToQuit(); | ||
| 240 | + g_stop = true; | ||
| 241 | + return 0; | ||
| 242 | +} | ||
| @@ -0,0 +1,70 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information. | ||
| 4 | +// Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 5 | +// you may not use this file except in compliance with the License. | ||
| 6 | +// You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0 | ||
| 7 | +// Unless required by applicable law or agreed to in writing, software | ||
| 8 | +// distributed under the License is distributed on an "AS IS" BASIS, | ||
| 9 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 10 | + | ||
| 11 | +#ifndef EXAMPLE_UB_TEST_SGE_MODE_H | ||
| 12 | +#define EXAMPLE_UB_TEST_SGE_MODE_H | ||
| 13 | + | ||
| 14 | +#include <iostream> | ||
| 15 | +#include <string> | ||
| 16 | +#include "brpc/urma/urma_helper.h" | ||
| 17 | +#include "brpc/urma/urma_resource_pool.h" | ||
| 18 | +#include "brpc/urma/urma_resource.h" | ||
| 19 | + | ||
| 20 | +// Verify the loaded pooled transport before starting benchmark traffic. | ||
| 21 | +inline bool ValidateUrmaSgeMode(const std::string& requested) { | ||
| 22 | + if (!brpc::urma::UsingPooledUrmaResources()) { | ||
| 23 | + const int max_sge = brpc::urma::GetUrmaMaxSge(); | ||
| 24 | + const std::string actual = max_sge >= 2 ? "multi" : "single"; | ||
| 25 | + if (!brpc::urma::GetUrmaContext() || max_sge < 1 || | ||
| 26 | + (requested != "auto" && requested != actual)) return false; | ||
| 27 | + std::cout << "UB_TEST_TRANSPORT resource_mode=exclusive sge_mode=" << actual | ||
| 28 | + << " max_sge=" << max_sge << std::endl; | ||
| 29 | + return true; | ||
| 30 | + } | ||
| 31 | + auto* pool = brpc::urma::g_urma_pool; | ||
| 32 | + if (!pool || pool->io_jfs.empty() || !pool->io_jfs.front() || | ||
| 33 | + !pool->io_jfs.front()->jfs) { | ||
| 34 | + std::cerr << "URMA resource pool is unavailable; refusing TCP fallback" | ||
| 35 | + << std::endl; | ||
| 36 | + return false; | ||
| 37 | + } | ||
| 38 | + const uint32_t max_sge = pool->io_jfs.front()->jfs->jfs_cfg.max_sge; | ||
| 39 | + const std::string actual = "multi"; | ||
| 40 | + if (max_sge < 2) { return false; } | ||
| 41 | + if (requested != "auto" && requested != actual) { | ||
| 42 | + std::cerr << "Requested SGE mode " << requested << ", loaded " << actual | ||
| 43 | + << "; select the matching libbrpc build" << std::endl; | ||
| 44 | + return false; | ||
| 45 | + } | ||
| 46 | + auto* context = brpc::urma::GetUrmaContext(); | ||
| 47 | + urma_device_attr_t attr = {}; | ||
| 48 | + if (!context || urma_query_device(context->dev, &attr) != URMA_SUCCESS) { | ||
| 49 | + std::cerr << "Cannot verify URMA device CTP capabilities" << std::endl; | ||
| 50 | + return false; | ||
| 51 | + } | ||
| 52 | + const int priority = brpc::urma::FindUrmaPriorityForTpType(attr, URMA_CTP); | ||
| 53 | + for (const auto* slot : pool->io_jfs) { | ||
| 54 | + if (priority < 0 || !slot || !slot->jfs || | ||
| 55 | + slot->jfs->jfs_cfg.max_sge != max_sge || | ||
| 56 | + slot->jfs->jfs_cfg.priority != priority) { | ||
| 57 | + std::cerr << "Inconsistent JFS SGE configuration or non-CTP transport" | ||
| 58 | + << std::endl; | ||
| 59 | + return false; | ||
| 60 | + } | ||
| 61 | + } | ||
| 62 | + std::cout << "UB_TEST_TRANSPORT sge_mode=" << actual | ||
| 63 | + << " max_sge=" << max_sge << " tp=CTP priority=" << priority | ||
| 64 | + << " jfs_count=" << pool->io_jfs.size() | ||
| 65 | + << " jfr_count=" << pool->io_jfr.size() | ||
| 66 | + << std::endl; | ||
| 67 | + return true; | ||
| 68 | +} | ||
| 69 | + | ||
| 70 | +#endif // EXAMPLE_UB_TEST_SGE_MODE_H | ||
| @@ -0,0 +1,35 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information | ||
| 4 | +// regarding copyright ownership. The ASF licenses this file | ||
| 5 | +// to you under the Apache License, Version 2.0 (the | ||
| 6 | +// "License"); you may not use this file except in compliance | ||
| 7 | +// with the License. You may obtain a copy of the License at | ||
| 8 | +// | ||
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +// | ||
| 11 | +// Unless required by applicable law or agreed to in writing, | ||
| 12 | +// software distributed under the License is distributed on an | ||
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +// KIND, either express or implied. See the License for the | ||
| 15 | +// specific language governing permissions and limitations | ||
| 16 | +// under the License. | ||
| 17 | + | ||
| 18 | +syntax="proto2"; | ||
| 19 | +option cc_generic_services = true; | ||
| 20 | + | ||
| 21 | +package test; | ||
| 22 | + | ||
| 23 | +message PerfTestRequest { | ||
| 24 | + required bool echo_attachment = 1; | ||
| 25 | + required bytes name = 2; | ||
| 26 | +}; | ||
| 27 | + | ||
| 28 | +message PerfTestResponse { | ||
| 29 | + required string cpu_usage = 1; | ||
| 30 | + required bytes name = 2; | ||
| 31 | +}; | ||
| 32 | + | ||
| 33 | +service PerfTestService { | ||
| 34 | + rpc Test(PerfTestRequest) returns (PerfTestResponse); | ||
| 35 | +}; | ||
| @@ -23,6 +23,11 @@ include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake) | |||
| 23 | option(LINK_SO "Whether examples are linked dynamically" OFF) | 23 | option(LINK_SO "Whether examples are linked dynamically" OFF) |
| 24 | 24 | ||
| 25 | brpc_example_find_common_deps(DYNAMIC_LIB) | 25 | brpc_example_find_common_deps(DYNAMIC_LIB) |
| 26 | +option(WITH_TCMALLOC "Link tcmalloc" OFF) | ||
| 27 | +if(WITH_TCMALLOC) | ||
| 28 | + find_library(TCMALLOC_LIB NAMES tcmalloc REQUIRED) | ||
| 29 | + list(APPEND DYNAMIC_LIB ${TCMALLOC_LIB}) | ||
| 30 | +endif() | ||
| 26 | 31 | ||
| 27 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto) | 32 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER test.proto) |
| 28 | set(BRPC_EXAMPLE_WITH_URMA ON) | 33 | set(BRPC_EXAMPLE_WITH_URMA ON) |
| @@ -52,3 +57,7 @@ brpc_example_configure_target(urma_performance_server) | |||
| 52 | 57 | ||
| 53 | target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) | 58 | target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) |
| 54 | target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) | 59 | target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) |
| 60 | + | ||
| 61 | +add_executable(urma_connection_stress connection_stress.cpp ${PROTO_SRC} ${PROTO_HEADER}) | ||
| 62 | +brpc_example_configure_target(urma_connection_stress) | ||
| 63 | +target_link_libraries(urma_connection_stress PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) | ||
| @@ -0,0 +1,371 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information | ||
| 4 | +// regarding copyright ownership. The ASF licenses this file | ||
| 5 | +// to you under the Apache License, Version 2.0 (the | ||
| 6 | +// "License"); you may not use this file except in compliance | ||
| 7 | +// with the License. You may obtain a copy of the License at | ||
| 8 | +// | ||
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +// | ||
| 11 | +// Unless required by applicable law or agreed to in writing, | ||
| 12 | +// software distributed under the License is distributed on an | ||
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +// KIND, either express or implied. See the License for the | ||
| 15 | +// specific language governing permissions and limitations | ||
| 16 | +// under the License. | ||
| 17 | + | ||
| 18 | +// Bounded workers drive independent, persistent URMA connections. Each sweep | ||
| 19 | +// visits every connection and checks its identity and response byte for byte. | ||
| 20 | +#include <sys/resource.h> | ||
| 21 | +#include <dirent.h> | ||
| 22 | +#include <unistd.h> | ||
| 23 | +#include <algorithm> | ||
| 24 | +#include <atomic> | ||
| 25 | +#include <fstream> | ||
| 26 | +#include <functional> | ||
| 27 | +#include <memory> | ||
| 28 | +#include <set> | ||
| 29 | +#include <sstream> | ||
| 30 | +#include <string> | ||
| 31 | +#include <vector> | ||
| 32 | +#include <gflags/gflags.h> | ||
| 33 | +#include "bthread/bthread.h" | ||
| 34 | +#include "butil/logging.h" | ||
| 35 | +#include "butil/time.h" | ||
| 36 | +#include "brpc/channel.h" | ||
| 37 | +#include "brpc/closure_guard.h" | ||
| 38 | +#include "brpc/controller.h" | ||
| 39 | +#include "brpc/details/controller_private_accessor.h" | ||
| 40 | +#include "brpc/server.h" | ||
| 41 | +#include "brpc/socket.h" | ||
| 42 | +#include "brpc/urma/urma_helper.h" | ||
| 43 | +#include "test.pb.h" | ||
| 44 | + | ||
| 45 | +DEFINE_string(mode, "client", "client or server"); | ||
| 46 | +DEFINE_string(servers, "127.0.0.1:18083,127.0.0.1:18084", | ||
| 47 | + "Comma-separated control endpoints of the SAME server process"); | ||
| 48 | +DEFINE_string(ports, "18083,18084", "Server listening ports"); | ||
| 49 | +DEFINE_int32(connections, 40000, "Simultaneously live independent connections"); | ||
| 50 | +DEFINE_int32(concurrency, 128, "Maximum outstanding RPCs / bthread workers"); | ||
| 51 | +DEFINE_int32(rounds, 3, "Complete sweeps after connection warmup"); | ||
| 52 | +DEFINE_int32(churn_percent, 10, "Percent of connections replaced after each sweep"); | ||
| 53 | +DEFINE_int32(idle_seconds, 10, "Idle hold after warmup and each sweep"); | ||
| 54 | +DEFINE_int32(timeout_ms, 10000, "Per-RPC timeout, with retries disabled"); | ||
| 55 | +DEFINE_string(sizes, "64,4032,4096,8192,65536", "Attachment sizes per sweep"); | ||
| 56 | +DEFINE_int32(report_seconds, 5, "Server resource report interval"); | ||
| 57 | +DEFINE_int32(server_threads, 16, "Server bthread worker threads"); | ||
| 58 | + | ||
| 59 | +static std::vector<std::string> Split(const std::string& value) { | ||
| 60 | + std::istringstream in(value); | ||
| 61 | + std::vector<std::string> result; | ||
| 62 | + std::string part; | ||
| 63 | + while (std::getline(in, part, ',')) { result.push_back(part); } | ||
| 64 | + return result; | ||
| 65 | +} | ||
| 66 | + | ||
| 67 | +static bool ParseNumbers(const std::string& value, size_t max, | ||
| 68 | + std::vector<size_t>* out) { | ||
| 69 | + for (const auto& item : Split(value)) { | ||
| 70 | + if (item.empty() || item.find_first_not_of("0123456789") != std::string::npos) { | ||
| 71 | + return false; | ||
| 72 | + } | ||
| 73 | + try { | ||
| 74 | + const auto n = std::stoull(item); | ||
| 75 | + if (n > max) { return false; } | ||
| 76 | + out->push_back(n); | ||
| 77 | + } catch (...) { return false; } | ||
| 78 | + } | ||
| 79 | + return !out->empty(); | ||
| 80 | +} | ||
| 81 | + | ||
| 82 | +static void Report(const char* phase, size_t connections) { | ||
| 83 | + size_t rss = 0, peak = 0, threads = 0, fds = 0; | ||
| 84 | + std::ifstream status("/proc/self/status"); | ||
| 85 | + std::string line; | ||
| 86 | + while (std::getline(status, line)) { | ||
| 87 | + std::istringstream row(line); | ||
| 88 | + std::string key; | ||
| 89 | + size_t value; | ||
| 90 | + if (!(row >> key >> value)) { continue; } | ||
| 91 | + if (key == "VmRSS:") { rss = value; } | ||
| 92 | + if (key == "VmHWM:") { peak = value; } | ||
| 93 | + if (key == "Threads:") { threads = value; } | ||
| 94 | + } | ||
| 95 | + DIR* directory = opendir("/proc/self/fd"); | ||
| 96 | + if (directory) { | ||
| 97 | + while (dirent* entry = readdir(directory)) { | ||
| 98 | + if (entry->d_name[0] != '.') { ++fds; } | ||
| 99 | + } | ||
| 100 | + closedir(directory); | ||
| 101 | + --fds; // The directory descriptor itself. | ||
| 102 | + } | ||
| 103 | + LOG(INFO) << "STRESS_RESOURCE phase=" << phase << " connections=" << connections | ||
| 104 | + << " rss_kib=" << rss << " peak_rss_kib=" << peak | ||
| 105 | + << " threads=" << threads << " fds=" << fds; | ||
| 106 | +} | ||
| 107 | + | ||
| 108 | +class StressService : public test::PerfTestService { | ||
| 109 | +public: | ||
| 110 | + std::function<size_t()> connection_count; | ||
| 111 | + void Test(google::protobuf::RpcController* controller, | ||
| 112 | + const test::PerfTestRequest* request, | ||
| 113 | + test::PerfTestResponse* response, | ||
| 114 | + google::protobuf::Closure* done) override { | ||
| 115 | + brpc::ClosureGuard guard(done); | ||
| 116 | + auto* cntl = static_cast<brpc::Controller*>(controller); | ||
| 117 | + brpc::Socket* socket = brpc::ControllerPrivateAccessor(cntl).get_sending_socket(); | ||
| 118 | + std::ostringstream debug; | ||
| 119 | + if (socket) { brpc::Socket::DebugSocket(debug, socket->id()); } | ||
| 120 | + if (!socket || socket->socket_mode() != brpc::SOCKET_MODE_URMA || | ||
| 121 | + debug.str().find("state=ESTABLISHED") == std::string::npos) { | ||
| 122 | + cntl->SetFailed("Established URMA required; TCP fallback rejected"); | ||
| 123 | + return; | ||
| 124 | + } | ||
| 125 | + response->set_cpu_usage("URMA_ESTABLISHED"); | ||
| 126 | + response->set_verification_socket_id(socket->id()); | ||
| 127 | + if (request->query_connections()) { | ||
| 128 | + response->set_active_connections(connection_count()); | ||
| 129 | + } | ||
| 130 | + if (request->echo_attachment()) { | ||
| 131 | + cntl->response_attachment().append(cntl->request_attachment()); | ||
| 132 | + } | ||
| 133 | + } | ||
| 134 | +}; | ||
| 135 | + | ||
| 136 | +static int RunServer() { | ||
| 137 | + std::vector<size_t> ports; | ||
| 138 | + if (!ParseNumbers(FLAGS_ports, 65535, &ports) || | ||
| 139 | + std::set<size_t>(ports.begin(), ports.end()).size() != ports.size() || | ||
| 140 | + std::find(ports.begin(), ports.end(), 0) != ports.end()) { return 1; } | ||
| 141 | + StressService service; | ||
| 142 | + std::vector<std::unique_ptr<brpc::Server>> servers; | ||
| 143 | + service.connection_count = [&servers]() { | ||
| 144 | + size_t count = 0; | ||
| 145 | + for (const auto& server : servers) { | ||
| 146 | + brpc::ServerStatistics stat; | ||
| 147 | + server->GetStat(&stat); | ||
| 148 | + count += stat.connection_count; | ||
| 149 | + } | ||
| 150 | + return count; | ||
| 151 | + }; | ||
| 152 | + for (auto port : ports) { | ||
| 153 | + std::unique_ptr<brpc::Server> server(new brpc::Server); | ||
| 154 | + brpc::ServerOptions options; | ||
| 155 | + options.socket_mode = brpc::SOCKET_MODE_URMA; | ||
| 156 | + options.idle_timeout_sec = -1; | ||
| 157 | + options.num_threads = FLAGS_server_threads; | ||
| 158 | + if (server->AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE) != 0 || | ||
| 159 | + server->Start(port, &options) != 0 || !brpc::urma::IsUrmaAvailable()) { | ||
| 160 | + return 1; | ||
| 161 | + } | ||
| 162 | + servers.emplace_back(std::move(server)); | ||
| 163 | + } | ||
| 164 | + LOG(INFO) << "STRESS_READY ports=" << FLAGS_ports; | ||
| 165 | + int ticks = 0; | ||
| 166 | + while (!brpc::IsAskedToQuit()) { | ||
| 167 | + if (ticks++ % FLAGS_report_seconds == 0) { | ||
| 168 | + Report("server", service.connection_count()); | ||
| 169 | + } | ||
| 170 | + sleep(1); | ||
| 171 | + } | ||
| 172 | + for (auto& server : servers) { server->Stop(0); } | ||
| 173 | + for (auto& server : servers) { server->Join(); } | ||
| 174 | + return 0; | ||
| 175 | +} | ||
| 176 | + | ||
| 177 | +struct Connection { | ||
| 178 | + std::unique_ptr<brpc::Channel> channel; | ||
| 179 | + uint64_t peer_id = brpc::INVALID_SOCKET_ID; | ||
| 180 | + uint64_t retired_id = brpc::INVALID_SOCKET_ID; | ||
| 181 | + unsigned generation = 0; | ||
| 182 | +}; | ||
| 183 | + | ||
| 184 | +struct Sweep { | ||
| 185 | + std::vector<Connection>* connections; | ||
| 186 | + const std::vector<std::string>* servers; | ||
| 187 | + const std::vector<size_t>* sizes; | ||
| 188 | + int round; | ||
| 189 | + std::atomic<size_t> next{0}; | ||
| 190 | + std::atomic<size_t> passed{0}; | ||
| 191 | + std::atomic<uint64_t> bytes{0}; | ||
| 192 | + std::atomic<bool> failed{false}; | ||
| 193 | + | ||
| 194 | + static void* Run(void* arg) { | ||
| 195 | + auto& sweep = *static_cast<Sweep*>(arg); | ||
| 196 | + while (!sweep.failed.load()) { | ||
| 197 | + const size_t index = sweep.next.fetch_add(1); | ||
| 198 | + if (index >= sweep.connections->size()) { break; } | ||
| 199 | + Connection& connection = (*sweep.connections)[index]; | ||
| 200 | + if (!connection.channel) { | ||
| 201 | + brpc::ChannelOptions options; | ||
| 202 | + options.socket_mode = brpc::SOCKET_MODE_URMA; | ||
| 203 | + options.connection_type = "single"; | ||
| 204 | + options.connection_group = "urma-stress-" + std::to_string(index) + | ||
| 205 | + "-" + std::to_string(connection.generation); | ||
| 206 | + options.timeout_ms = FLAGS_timeout_ms; | ||
| 207 | + options.connect_timeout_ms = FLAGS_timeout_ms; | ||
| 208 | + options.max_retry = 0; | ||
| 209 | + connection.channel.reset(new brpc::Channel); | ||
| 210 | + if (connection.channel->Init( | ||
| 211 | + (*sweep.servers)[index % sweep.servers->size()].c_str(), &options) != 0) { | ||
| 212 | + LOG(ERROR) << "STRESS_FAILED channel_init index=" << index; | ||
| 213 | + sweep.failed = true; | ||
| 214 | + break; | ||
| 215 | + } | ||
| 216 | + } | ||
| 217 | + const size_t size = sweep.round == 0 ? 64 : | ||
| 218 | + (*sweep.sizes)[(index + sweep.round - 1) % sweep.sizes->size()]; | ||
| 219 | + std::string payload(size, '\0'); | ||
| 220 | + uint32_t state = 0x9e3779b9u ^ static_cast<uint32_t>(index) ^ | ||
| 221 | + (static_cast<uint32_t>(sweep.round) * 0x85ebca6bu) ^ connection.generation; | ||
| 222 | + for (char& byte : payload) { | ||
| 223 | + state ^= state << 13; | ||
| 224 | + state ^= state >> 17; | ||
| 225 | + state ^= state << 5; | ||
| 226 | + byte = static_cast<char>(state); | ||
| 227 | + } | ||
| 228 | + brpc::Controller cntl; | ||
| 229 | + cntl.set_log_id((uint64_t(sweep.round) << 32) | index); | ||
| 230 | + cntl.request_attachment().append(payload); | ||
| 231 | + test::PerfTestRequest request; | ||
| 232 | + test::PerfTestResponse response; | ||
| 233 | + request.set_echo_attachment(true); | ||
| 234 | + test::PerfTestService_Stub stub(connection.channel.get()); | ||
| 235 | + stub.Test(&cntl, &request, &response, nullptr); | ||
| 236 | + const uint64_t peer = response.verification_socket_id(); | ||
| 237 | + if (cntl.Failed() || response.cpu_usage() != "URMA_ESTABLISHED" || | ||
| 238 | + !response.has_verification_socket_id() || | ||
| 239 | + peer == brpc::INVALID_SOCKET_ID || peer == connection.retired_id || | ||
| 240 | + (connection.peer_id != brpc::INVALID_SOCKET_ID && peer != connection.peer_id) || | ||
| 241 | + cntl.response_attachment().to_string() != payload) { | ||
| 242 | + LOG(ERROR) << "STRESS_FAILED round=" << sweep.round << " index=" << index | ||
| 243 | + << " bytes=" << size << " error=" << cntl.ErrorText() | ||
| 244 | + << " peer=" << peer << " previous=" << connection.peer_id | ||
| 245 | + << " received=" << cntl.response_attachment().size(); | ||
| 246 | + sweep.failed = true; | ||
| 247 | + break; | ||
| 248 | + } | ||
| 249 | + connection.peer_id = peer; | ||
| 250 | + sweep.bytes.fetch_add(size); | ||
| 251 | + sweep.passed.fetch_add(1); | ||
| 252 | + } | ||
| 253 | + return nullptr; | ||
| 254 | + } | ||
| 255 | +}; | ||
| 256 | + | ||
| 257 | +static bool CheckServerConnections(Connection& connection, size_t expected) { | ||
| 258 | + const int64_t deadline = butil::monotonic_time_us() + int64_t(FLAGS_timeout_ms) * 1000; | ||
| 259 | + size_t observed = 0; | ||
| 260 | + do { | ||
| 261 | + brpc::Controller cntl; | ||
| 262 | + test::PerfTestRequest request; | ||
| 263 | + test::PerfTestResponse response; | ||
| 264 | + request.set_echo_attachment(false); | ||
| 265 | + request.set_query_connections(true); | ||
| 266 | + test::PerfTestService_Stub stub(connection.channel.get()); | ||
| 267 | + stub.Test(&cntl, &request, &response, nullptr); | ||
| 268 | + if (cntl.Failed() || response.cpu_usage() != "URMA_ESTABLISHED" || | ||
| 269 | + !response.has_active_connections() || | ||
| 270 | + response.verification_socket_id() != connection.peer_id) { | ||
| 271 | + LOG(ERROR) << "STRESS_FAILED connection query: " << cntl.ErrorText(); | ||
| 272 | + return false; | ||
| 273 | + } | ||
| 274 | + observed = response.active_connections(); | ||
| 275 | + if (observed == expected) { return true; } | ||
| 276 | + usleep(10000); | ||
| 277 | + } while (butil::monotonic_time_us() < deadline); | ||
| 278 | + LOG(ERROR) << "STRESS_FAILED server connections=" << observed << " expected=" << expected; | ||
| 279 | + return false; | ||
| 280 | +} | ||
| 281 | + | ||
| 282 | +static int RunClient() { | ||
| 283 | + auto servers = Split(FLAGS_servers); | ||
| 284 | + std::vector<size_t> sizes; | ||
| 285 | + if (servers.empty() || !ParseNumbers(FLAGS_sizes, 1024 * 1024, &sizes)) { return 1; } | ||
| 286 | + std::vector<Connection> connections(FLAGS_connections); | ||
| 287 | + Report("before_connect", 0); | ||
| 288 | + uint64_t total = 0; | ||
| 289 | + for (int round = 0; round <= FLAGS_rounds; ++round) { | ||
| 290 | + Sweep sweep; | ||
| 291 | + sweep.connections = &connections; | ||
| 292 | + sweep.servers = &servers; | ||
| 293 | + sweep.sizes = &sizes; | ||
| 294 | + sweep.round = round; | ||
| 295 | + const int64_t start = butil::monotonic_time_us(); | ||
| 296 | + std::vector<bthread_t> workers; | ||
| 297 | + for (int i = 0; i < std::min(FLAGS_concurrency, FLAGS_connections); ++i) { | ||
| 298 | + bthread_t id; | ||
| 299 | + if (bthread_start_background(&id, nullptr, Sweep::Run, &sweep) != 0) { | ||
| 300 | + sweep.failed = true; | ||
| 301 | + break; | ||
| 302 | + } | ||
| 303 | + workers.push_back(id); | ||
| 304 | + } | ||
| 305 | + for (auto id : workers) { bthread_join(id, nullptr); } | ||
| 306 | + if (sweep.failed || sweep.passed != connections.size()) { return 1; } | ||
| 307 | + // All endpoints must belong to one process, so SocketIds are globally unique. | ||
| 308 | + std::set<uint64_t> peers; | ||
| 309 | + for (const auto& connection : connections) { peers.insert(connection.peer_id); } | ||
| 310 | + if (peers.size() != connections.size()) { | ||
| 311 | + LOG(ERROR) << "STRESS_FAILED connections unexpectedly share sockets unique=" | ||
| 312 | + << peers.size(); | ||
| 313 | + return 1; | ||
| 314 | + } | ||
| 315 | + if (!CheckServerConnections(connections.front(), connections.size())) { return 1; } | ||
| 316 | + total += sweep.passed; | ||
| 317 | + LOG(INFO) << "STRESS_SWEEP_PASS round=" << round << " verified=" << sweep.passed | ||
| 318 | + << " unique_peer_socket_ids=" << peers.size() | ||
| 319 | + << " bytes=" << sweep.bytes | ||
| 320 | + << " elapsed_us=" << butil::monotonic_time_us() - start; | ||
| 321 | + // Measure the real idle footprint without forcing allocator reclamation. | ||
| 322 | + sleep(FLAGS_idle_seconds); | ||
| 323 | + Report(round == 0 ? "connected_idle" : "sweep_idle", connections.size()); | ||
| 324 | + if (round > 0 && round < FLAGS_rounds && FLAGS_churn_percent > 0) { | ||
| 325 | + size_t changed = 0; | ||
| 326 | + for (size_t i = 0; i < connections.size(); ++i) { | ||
| 327 | + if ((i + round * 17) % 100 >= size_t(FLAGS_churn_percent)) { continue; } | ||
| 328 | + auto& connection = connections[i]; | ||
| 329 | + connection.channel.reset(); | ||
| 330 | + connection.retired_id = connection.peer_id; | ||
| 331 | + connection.peer_id = brpc::INVALID_SOCKET_ID; | ||
| 332 | + ++connection.generation; | ||
| 333 | + ++changed; | ||
| 334 | + } | ||
| 335 | + LOG(INFO) << "STRESS_CHURN closed=" << changed; | ||
| 336 | + } | ||
| 337 | + } | ||
| 338 | + for (size_t i = 1; i < connections.size(); ++i) { connections[i].channel.reset(); } | ||
| 339 | + if (!CheckServerConnections(connections.front(), 1)) { return 1; } | ||
| 340 | + LOG(INFO) << "STRESS_CLEANUP_PASS server_connections=1 monitoring_connection_only"; | ||
| 341 | + connections.clear(); | ||
| 342 | + sleep(FLAGS_idle_seconds); | ||
| 343 | + Report("released", 0); | ||
| 344 | + LOG(INFO) << "STRESS_PASS connections=" << FLAGS_connections | ||
| 345 | + << " verified_rpcs=" << total << " rounds=" << FLAGS_rounds | ||
| 346 | + << " concurrency=" << FLAGS_concurrency; | ||
| 347 | + return 0; | ||
| 348 | +} | ||
| 349 | + | ||
| 350 | +int main(int argc, char** argv) { | ||
| 351 | + gflags::ParseCommandLineFlags(&argc, &argv, true); | ||
| 352 | + if (FLAGS_connections <= 0 || FLAGS_concurrency <= 0 || FLAGS_rounds <= 0 || | ||
| 353 | + FLAGS_timeout_ms <= 0 || FLAGS_report_seconds <= 0 || FLAGS_idle_seconds < 0 || | ||
| 354 | + FLAGS_server_threads <= 0 || | ||
| 355 | + FLAGS_churn_percent < 0 || FLAGS_churn_percent > 100) { return 1; } | ||
| 356 | + // Raise only this test process's soft fd limit, never host-wide settings. | ||
| 357 | + rlimit limit; | ||
| 358 | + if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { return 1; } | ||
| 359 | + const rlim_t required = static_cast<rlim_t>(FLAGS_connections) + 4096; | ||
| 360 | + if (limit.rlim_cur < required) { | ||
| 361 | + if (limit.rlim_max < required) { | ||
| 362 | + LOG(ERROR) << "Need RLIMIT_NOFILE >= " << required; | ||
| 363 | + return 1; | ||
| 364 | + } | ||
| 365 | + limit.rlim_cur = required; | ||
| 366 | + if (setrlimit(RLIMIT_NOFILE, &limit) != 0) { return 1; } | ||
| 367 | + } | ||
| 368 | + if (FLAGS_mode == "server") { return RunServer(); } | ||
| 369 | + if (FLAGS_mode == "client") { return RunClient(); } | ||
| 370 | + return 1; | ||
| 371 | +} | ||
| @@ -23,10 +23,14 @@ package test; | |||
| 23 | 23 | ||
| 24 | message PerfTestRequest { | 24 | message PerfTestRequest { |
| 25 | required bool echo_attachment = 1; | 25 | required bool echo_attachment = 1; |
| 26 | + optional bool query_connections = 2; | ||
| 26 | } | 27 | } |
| 27 | 28 | ||
| 28 | message PerfTestResponse { | 29 | message PerfTestResponse { |
| 29 | required string cpu_usage = 1; | 30 | required string cpu_usage = 1; |
| 31 | + // Used by the connection stress test to detect connection sharing/reconnects. | ||
| 32 | + optional fixed64 verification_socket_id = 2; | ||
| 33 | + optional uint64 active_connections = 3; | ||
| 30 | } | 34 | } |
| 31 | 35 | ||
| 32 | service PerfTestService { | 36 | service PerfTestService { |
| @@ -64,8 +64,11 @@ class RdmaHandshakeServerV3; | |||
| 64 | } | 64 | } |
| 65 | 65 | ||
| 66 | namespace urma { | 66 | namespace urma { |
| 67 | +class UrmaReceiveAdapter; | ||
| 67 | class UrmaEndpoint; | 68 | class UrmaEndpoint; |
| 68 | class UrmaExclusiveConnection; | 69 | class UrmaExclusiveConnection; |
| 70 | +class UrmaPooledConnection; | ||
| 71 | +class UrmaPooledResourceProvider; | ||
| 69 | class UrmaConnect; | 72 | class UrmaConnect; |
| 70 | class UrmaHandshakeClientV2; | 73 | class UrmaHandshakeClientV2; |
| 71 | class UrmaHandshakeServerV2; | 74 | class UrmaHandshakeServerV2; |
| @@ -352,7 +355,10 @@ friend class rdma::RdmaHandshakeServerV2; | |||
| 352 | friend class rdma::RdmaHandshakeClientV3; | 355 | friend class rdma::RdmaHandshakeClientV3; |
| 353 | friend class rdma::RdmaHandshakeServerV3; | 356 | friend class rdma::RdmaHandshakeServerV3; |
| 354 | friend class urma::UrmaEndpoint; | 357 | friend class urma::UrmaEndpoint; |
| 358 | +friend class urma::UrmaReceiveAdapter; | ||
| 355 | friend class urma::UrmaExclusiveConnection; | 359 | friend class urma::UrmaExclusiveConnection; |
| 360 | +friend class urma::UrmaPooledConnection; | ||
| 361 | +friend class urma::UrmaPooledResourceProvider; | ||
| 356 | friend class urma::UrmaConnect; | 362 | friend class urma::UrmaConnect; |
| 357 | friend class urma::UrmaHandshakeClientV2; | 363 | friend class urma::UrmaHandshakeClientV2; |
| 358 | friend class urma::UrmaHandshakeServerV2; | 364 | friend class urma::UrmaHandshakeServerV2; |
| @@ -55,11 +55,11 @@ struct JfcState { | |||
| 55 | std::mutex mutex; | 55 | std::mutex mutex; |
| 56 | std::deque<urma_cr_t> completions; | 56 | std::deque<urma_cr_t> completions; |
| 57 | bool event_pending{false}; | 57 | bool event_pending{false}; |
| 58 | + bool armed{false}; | ||
| 58 | }; | 59 | }; |
| 59 | 60 | ||
| 60 | struct PendingRecv { | 61 | struct PendingRecv { |
| 61 | - uint64_t addr; | 62 | + std::vector<urma_sge_t> sge; |
| 62 | - uint32_t len; | ||
| 63 | uint64_t user_ctx; | 63 | uint64_t user_ctx; |
| 64 | }; | 64 | }; |
| 65 | 65 | ||
| @@ -70,6 +70,9 @@ std::map<urma_context_t *, int> context_map; | |||
| 70 | std::map<urma_jfce_t *, int> jfce_map; | 70 | std::map<urma_jfce_t *, int> jfce_map; |
| 71 | std::map<urma_jfc_t *, JfcState *> jfc_state_map; | 71 | std::map<urma_jfc_t *, JfcState *> jfc_state_map; |
| 72 | std::map<urma_jfr_t *, int> jfr_map; | 72 | std::map<urma_jfr_t *, int> jfr_map; |
| 73 | +std::map<uint32_t, urma_jfr_t*> jfr_id_map; | ||
| 74 | +std::map<urma_jfs_t*, int> jfs_map; | ||
| 75 | +std::map<urma_target_jetty_t*, bool> target_is_jfr; | ||
| 73 | // Side-table: JFR -> the JFC it was created with (used to route recv | 76 | // Side-table: JFR -> the JFC it was created with (used to route recv |
| 74 | // completions to the right JfcState, since the JFR is an opaque handle). | 77 | // completions to the right JfcState, since the JFR is an opaque handle). |
| 75 | std::map<urma_jfr_t *, urma_jfc_t *> jfr_jfc_map; | 78 | std::map<urma_jfr_t *, urma_jfc_t *> jfr_jfc_map; |
| @@ -98,7 +101,8 @@ void PushCompletion(urma_jfc_t* jfc, JfcState* state, | |||
| 98 | { | 101 | { |
| 99 | std::lock_guard<std::mutex> lock(state->mutex); | 102 | std::lock_guard<std::mutex> lock(state->mutex); |
| 100 | state->completions.push_back(completion); | 103 | state->completions.push_back(completion); |
| 101 | - if (!state->event_pending) { | 104 | + if (state->armed && !state->event_pending) { |
| 105 | + state->armed = false; | ||
| 102 | state->event_pending = true; | 106 | state->event_pending = true; |
| 103 | signal = true; | 107 | signal = true; |
| 104 | } | 108 | } |
| @@ -154,6 +158,9 @@ urma_status_t urma_uninit(void) { | |||
| 154 | } | 158 | } |
| 155 | jfc_state_map.clear(); | 159 | jfc_state_map.clear(); |
| 156 | jfr_map.clear(); | 160 | jfr_map.clear(); |
| 161 | + jfr_id_map.clear(); | ||
| 162 | + jfs_map.clear(); | ||
| 163 | + target_is_jfr.clear(); | ||
| 157 | jfr_jfc_map.clear(); | 164 | jfr_jfc_map.clear(); |
| 158 | jfr_recv_map.clear(); | 165 | jfr_recv_map.clear(); |
| 159 | seg_map.clear(); | 166 | seg_map.clear(); |
| @@ -375,10 +382,12 @@ urma_jfr_t *urma_create_jfr(urma_context_t *ctx, urma_jfr_cfg_t *cfg) { | |||
| 375 | if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) { | 382 | if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) { |
| 376 | return nullptr; | 383 | return nullptr; |
| 377 | } | 384 | } |
| 378 | - // Opaque handle (avoids partially initializing urma_jfr_t's pthread | 385 | + urma_jfr_t* jfr = new urma_jfr_t{}; |
| 379 | - // members). brpc's endpoint reads back jfr_cfg.jfc from the stored cfg | 386 | + jfr->urma_ctx = ctx; |
| 380 | - // pointer below, so we keep a side-table mapping jfr -> cfg.jfc. | 387 | + jfr->jfr_cfg = *cfg; |
| 381 | - urma_jfr_t *jfr = reinterpret_cast<urma_jfr_t *>(new int(1)); | 388 | + jfr->jfr_id.id = next_jetty_id.fetch_add(1); |
| 389 | + jfr->jfr_id.eid = mock_eid_info.eid; | ||
| 390 | + jfr_id_map[jfr->jfr_id.id] = jfr; | ||
| 382 | jfr_map[jfr] = 1; | 391 | jfr_map[jfr] = 1; |
| 383 | // Stash the JFC for urma_post_jfr_wr's completion routing. | 392 | // Stash the JFC for urma_post_jfr_wr's completion routing. |
| 384 | jfr_jfc_map[jfr] = cfg->jfc; | 393 | jfr_jfc_map[jfr] = cfg->jfc; |
| @@ -391,10 +400,11 @@ urma_status_t urma_delete_jfr(urma_jfr_t *jfr) { | |||
| 391 | if (!jfr || jfr_map.find(jfr) == jfr_map.end()) { | 400 | if (!jfr || jfr_map.find(jfr) == jfr_map.end()) { |
| 392 | return URMA_EINVAL; | 401 | return URMA_EINVAL; |
| 393 | } | 402 | } |
| 403 | + jfr_id_map.erase(jfr->jfr_id.id); | ||
| 394 | jfr_map.erase(jfr); | 404 | jfr_map.erase(jfr); |
| 395 | jfr_jfc_map.erase(jfr); | 405 | jfr_jfc_map.erase(jfr); |
| 396 | jfr_recv_map.erase(jfr); | 406 | jfr_recv_map.erase(jfr); |
| 397 | - delete reinterpret_cast<int *>(jfr); | 407 | + delete jfr; |
| 398 | return URMA_SUCCESS; | 408 | return URMA_SUCCESS; |
| 399 | } | 409 | } |
| 400 | 410 | ||
| @@ -549,6 +559,7 @@ urma_status_t urma_unimport_jetty(urma_target_jetty_t *tjetty) { | |||
| 549 | return URMA_EINVAL; | 559 | return URMA_EINVAL; |
| 550 | } | 560 | } |
| 551 | target_jetty_map.erase(tjetty); | 561 | target_jetty_map.erase(tjetty); |
| 562 | + target_is_jfr.erase(tjetty); | ||
| 552 | delete tjetty; | 563 | delete tjetty; |
| 553 | return URMA_SUCCESS; | 564 | return URMA_SUCCESS; |
| 554 | } | 565 | } |
| @@ -682,10 +693,10 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, | |||
| 682 | } | 693 | } |
| 683 | uint32_t copied = 0; | 694 | uint32_t copied = 0; |
| 684 | for (uint32_t i = 0; | 695 | for (uint32_t i = 0; |
| 685 | - i < current->send.src.num_sge && copied < recv.len; ++i) { | 696 | + i < current->send.src.num_sge && copied < recv.sge[0].len; ++i) { |
| 686 | const urma_sge_t& sge = current->send.src.sge[i]; | 697 | const urma_sge_t& sge = current->send.src.sge[i]; |
| 687 | - const uint32_t n = std::min(sge.len, recv.len - copied); | 698 | + const uint32_t n = std::min(sge.len, recv.sge[0].len - copied); |
| 688 | - std::memcpy(reinterpret_cast<void*>(recv.addr + copied), | 699 | + std::memcpy(reinterpret_cast<void*>(recv.sge[0].addr + copied), |
| 689 | reinterpret_cast<const void*>(sge.addr), n); | 700 | reinterpret_cast<const void*>(sge.addr), n); |
| 690 | copied += n; | 701 | copied += n; |
| 691 | } | 702 | } |
| @@ -725,6 +736,128 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, | |||
| 725 | return URMA_SUCCESS; | 736 | return URMA_SUCCESS; |
| 726 | } | 737 | } |
| 727 | 738 | ||
| 739 | + | ||
| 740 | +// Called under g_rw_mutex. Complete only after every DMA read/copy is done. | ||
| 741 | +static urma_status_t PostSend(urma_jfc_t* local_jfc, urma_jfs_wr_t* wr, | ||
| 742 | + urma_jfs_wr_t** bad) { | ||
| 743 | + if (!wr || !jfc_state_map.count(local_jfc)) { | ||
| 744 | + if (bad) { *bad = wr; } | ||
| 745 | + return URMA_EINVAL; | ||
| 746 | + } | ||
| 747 | + for (auto* current = wr; current; current = current->next) { | ||
| 748 | + auto* target = current->tjetty; | ||
| 749 | + if (!target || !target_jetty_map.count(target)) { | ||
| 750 | + if (bad) { *bad = current; } | ||
| 751 | + return URMA_EINVAL; | ||
| 752 | + } | ||
| 753 | + urma_jfr_t* remote = nullptr; | ||
| 754 | + if (target_is_jfr[target]) { | ||
| 755 | + auto it = jfr_id_map.find(target->id.id); | ||
| 756 | + if (it != jfr_id_map.end()) { remote = it->second; } | ||
| 757 | + } else { | ||
| 758 | + auto it = jetty_id_map.find(target->id.id); | ||
| 759 | + if (it != jetty_id_map.end()) { remote = it->second->jetty_cfg.shared.jfr; } | ||
| 760 | + } | ||
| 761 | + auto recv_it = jfr_recv_map.find(remote); | ||
| 762 | + if (recv_it == jfr_recv_map.end() || recv_it->second.empty()) { | ||
| 763 | + if (bad) { *bad = current; } | ||
| 764 | + return URMA_EAGAIN; | ||
| 765 | + } | ||
| 766 | + size_t total = 0, capacity = 0; | ||
| 767 | + for (uint32_t i = 0; i < current->send.src.num_sge; ++i) { | ||
| 768 | + total += current->send.src.sge[i].len; | ||
| 769 | + } | ||
| 770 | + for (const auto& sge : recv_it->second.front().sge) { capacity += sge.len; } | ||
| 771 | + if (total > capacity || (target->tp_type == URMA_CTP && total > 4096)) { | ||
| 772 | + if (bad) { *bad = current; } | ||
| 773 | + return URMA_EINVAL; | ||
| 774 | + } | ||
| 775 | + PendingRecv recv = std::move(recv_it->second.front()); | ||
| 776 | + recv_it->second.pop_front(); | ||
| 777 | + // Scatter the concatenated send bytes into the posted receive SGEs. | ||
| 778 | + // This covers N->2 as well as the compile-time compatibility 1->2 path. | ||
| 779 | + size_t dst_index = 0, dst_offset = 0; | ||
| 780 | + for (uint32_t i = 0; i < current->send.src.num_sge; ++i) { | ||
| 781 | + const auto& src = current->send.src.sge[i]; | ||
| 782 | + size_t offset = 0; | ||
| 783 | + while (offset < src.len) { | ||
| 784 | + auto& dst = recv.sge[dst_index]; | ||
| 785 | + const size_t n = std::min<size_t>(src.len - offset, dst.len - dst_offset); | ||
| 786 | + std::memcpy(reinterpret_cast<void*>(dst.addr + dst_offset), | ||
| 787 | + reinterpret_cast<const void*>(src.addr + offset), n); | ||
| 788 | + offset += n; | ||
| 789 | + dst_offset += n; | ||
| 790 | + if (dst_offset == dst.len) { ++dst_index; dst_offset = 0; } | ||
| 791 | + } | ||
| 792 | + } | ||
| 793 | + urma_jfc_t* remote_jfc = jfr_jfc_map[remote]; | ||
| 794 | + urma_cr_t rx{}; | ||
| 795 | + rx.status = URMA_CR_SUCCESS; | ||
| 796 | + rx.user_ctx = recv.user_ctx; | ||
| 797 | + rx.flag.bs.s_r = 1; | ||
| 798 | + rx.completion_len = total; | ||
| 799 | + rx.opcode = current->opcode == URMA_OPC_SEND_IMM | ||
| 800 | + ? URMA_CR_OPC_SEND_WITH_IMM : URMA_CR_OPC_SEND; | ||
| 801 | + rx.imm_data = current->send.imm_data; | ||
| 802 | + PushCompletion(remote_jfc, jfc_state_map[remote_jfc], rx); | ||
| 803 | + if (current->flag.bs.complete_enable) { | ||
| 804 | + urma_cr_t tx{}; | ||
| 805 | + tx.status = URMA_CR_SUCCESS; | ||
| 806 | + tx.user_ctx = current->user_ctx; | ||
| 807 | + tx.flag.bs.s_r = 0; | ||
| 808 | + PushCompletion(local_jfc, jfc_state_map[local_jfc], tx); | ||
| 809 | + } | ||
| 810 | + } | ||
| 811 | + if (bad) { *bad = nullptr; } | ||
| 812 | + return URMA_SUCCESS; | ||
| 813 | +} | ||
| 814 | + | ||
| 815 | +urma_jfs_t* urma_create_jfs(urma_context_t* ctx, urma_jfs_cfg_t* cfg) { | ||
| 816 | + std::unique_lock<std::shared_mutex> lock(g_rw_mutex); | ||
| 817 | + if (!ctx || !cfg || !context_map.count(ctx) || !jfc_state_map.count(cfg->jfc)) { | ||
| 818 | + return nullptr; | ||
| 819 | + } | ||
| 820 | + auto* jfs = new urma_jfs_t{}; | ||
| 821 | + jfs->urma_ctx = ctx; | ||
| 822 | + jfs->jfs_cfg = *cfg; | ||
| 823 | + jfs->jfs_id.id = next_jetty_id.fetch_add(1); | ||
| 824 | + jfs->jfs_id.eid = mock_eid_info.eid; | ||
| 825 | + jfs_map[jfs] = 1; | ||
| 826 | + return jfs; | ||
| 827 | +} | ||
| 828 | +urma_status_t urma_delete_jfs(urma_jfs_t* jfs) { | ||
| 829 | + std::unique_lock<std::shared_mutex> lock(g_rw_mutex); | ||
| 830 | + if (!jfs_map.erase(jfs)) { return URMA_EINVAL; } | ||
| 831 | + delete jfs; | ||
| 832 | + return URMA_SUCCESS; | ||
| 833 | +} | ||
| 834 | +urma_target_jetty_t* urma_import_jfr(urma_context_t* ctx, | ||
| 835 | + urma_rjfr_t* remote, urma_token_t* token) { | ||
| 836 | + std::unique_lock<std::shared_mutex> lock(g_rw_mutex); | ||
| 837 | + if (!ctx || !remote || !token || !context_map.count(ctx)) { return nullptr; } | ||
| 838 | + auto* target = new urma_target_jetty_t{}; | ||
| 839 | + target->id = remote->jfr_id; | ||
| 840 | + target->tp_type = remote->tp_type; | ||
| 841 | + target_jetty_map[target] = 1; | ||
| 842 | + target_is_jfr[target] = true; | ||
| 843 | + return target; | ||
| 844 | +} | ||
| 845 | +urma_status_t urma_unimport_jfr(urma_target_jetty_t* target) { | ||
| 846 | + std::unique_lock<std::shared_mutex> lock(g_rw_mutex); | ||
| 847 | + if (!target_jetty_map.erase(target)) { return URMA_EINVAL; } | ||
| 848 | + target_is_jfr.erase(target); | ||
| 849 | + delete target; | ||
| 850 | + return URMA_SUCCESS; | ||
| 851 | +} | ||
| 852 | +urma_status_t urma_post_jfs_wr(urma_jfs_t* jfs, urma_jfs_wr_t* wr, | ||
| 853 | + urma_jfs_wr_t** bad) { | ||
| 854 | + std::unique_lock<std::shared_mutex> lock(g_rw_mutex); | ||
| 855 | + if (!jfs_map.count(jfs) || !wr || wr->send.src.num_sge > jfs->jfs_cfg.max_sge) { | ||
| 856 | + if (bad) { *bad = wr; } | ||
| 857 | + return URMA_EINVAL; | ||
| 858 | + } | ||
| 859 | + return PostSend(jfs->jfs_cfg.jfc, wr, bad); | ||
| 860 | +} | ||
| 728 | urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr, | 861 | urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr, |
| 729 | urma_jfr_wr_t **bad_wr) { | 862 | urma_jfr_wr_t **bad_wr) { |
| 730 | std::unique_lock<std::shared_mutex> lock(g_rw_mutex); | 863 | std::unique_lock<std::shared_mutex> lock(g_rw_mutex); |
| @@ -736,16 +869,18 @@ urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr, | |||
| 736 | return URMA_EINVAL; | 869 | return URMA_EINVAL; |
| 737 | } | 870 | } |
| 738 | for (urma_jfr_wr_t* current = wr; current; current = current->next) { | 871 | for (urma_jfr_wr_t* current = wr; current; current = current->next) { |
| 739 | - if (current->src.num_sge == 0 || !current->src.sge) { | 872 | + if (current->src.num_sge == 0 || !current->src.sge || |
| 873 | + current->src.num_sge > jfr->jfr_cfg.max_sge || | ||
| 874 | + recv_it->second.size() >= jfr->jfr_cfg.depth) { | ||
| 740 | if (bad_wr) { | 875 | if (bad_wr) { |
| 741 | *bad_wr = current; | 876 | *bad_wr = current; |
| 742 | } | 877 | } |
| 743 | return URMA_EINVAL; | 878 | return URMA_EINVAL; |
| 744 | } | 879 | } |
| 745 | - recv_it->second.push_back(PendingRecv{ | 880 | + PendingRecv recv; |
| 746 | - current->src.sge[0].addr, | 881 | + recv.user_ctx = current->user_ctx; |
| 747 | - current->src.sge[0].len, | 882 | + recv.sge.assign(current->src.sge, current->src.sge + current->src.num_sge); |
| 748 | - current->user_ctx}); | 883 | + recv_it->second.push_back(std::move(recv)); |
| 749 | } | 884 | } |
| 750 | if (bad_wr) { | 885 | if (bad_wr) { |
| 751 | *bad_wr = nullptr; | 886 | *bad_wr = nullptr; |
| @@ -786,13 +921,15 @@ int urma_poll_jfc(urma_jfc_t *jfc, int num_entries, urma_cr_t *cr_list) { | |||
| 786 | cr_list[count++] = state->completions.front(); | 921 | cr_list[count++] = state->completions.front(); |
| 787 | state->completions.pop_front(); | 922 | state->completions.pop_front(); |
| 788 | } | 923 | } |
| 789 | - if (state->completions.empty()) { | ||
| 790 | - state->event_pending = false; | ||
| 791 | - } | ||
| 792 | return count; | 924 | return count; |
| 793 | } | 925 | } |
| 794 | 926 | ||
| 795 | -urma_status_t urma_rearm_jfc(urma_jfc_t*, bool) { | 927 | +urma_status_t urma_rearm_jfc(urma_jfc_t* jfc, bool) { |
| 928 | + std::shared_lock<std::shared_mutex> lock(g_rw_mutex); | ||
| 929 | + auto it = jfc_state_map.find(jfc); | ||
| 930 | + if (it == jfc_state_map.end()) { return URMA_EINVAL; } | ||
| 931 | + std::lock_guard<std::mutex> state_lock(it->second->mutex); | ||
| 932 | + it->second->armed = true; | ||
| 796 | return URMA_SUCCESS; | 933 | return URMA_SUCCESS; |
| 797 | } | 934 | } |
| 798 | 935 | ||
| @@ -0,0 +1,48 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information | ||
| 4 | +// regarding copyright ownership. The ASF licenses this file | ||
| 5 | +// to you under the Apache License, Version 2.0 (the | ||
| 6 | +// "License"); you may not use this file except in compliance | ||
| 7 | +// with the License. You may obtain a copy of the License at | ||
| 8 | +// | ||
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +// | ||
| 11 | +// Unless required by applicable law or agreed to in writing, | ||
| 12 | +// software distributed under the License is distributed on an | ||
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +// KIND, either express or implied. See the License for the | ||
| 15 | +// specific language governing permissions and limitations | ||
| 16 | +// under the License. | ||
| 17 | + | ||
| 18 | +#ifndef BRPC_URMA_DELIVERY_H | ||
| 19 | +#define BRPC_URMA_DELIVERY_H | ||
| 20 | +#include "brpc/socket.h" | ||
| 21 | +#include "brpc/urma/urma_resource.h" | ||
| 22 | +#define BRPC_URMA_MAX_CQE_PER_POLL 32 | ||
| 23 | +#if BRPC_WITH_URMA | ||
| 24 | +namespace brpc { | ||
| 25 | +namespace urma { | ||
| 26 | +// Callback-owned, bounded to one message across all logical connections. | ||
| 27 | +// Declare before the CQ lock so destruction always happens after unlocking. | ||
| 28 | +struct UrmaDeferredMessage { | ||
| 29 | + UrmaDeferredMessage() = default; | ||
| 30 | + InputMessageBase* message{nullptr}; | ||
| 31 | + bool event_released{false}; | ||
| 32 | + ~UrmaDeferredMessage() noexcept(false); | ||
| 33 | +private: | ||
| 34 | + DISALLOW_COPY_AND_ASSIGN(UrmaDeferredMessage); | ||
| 35 | +}; | ||
| 36 | + | ||
| 37 | +// One poll result, never a wait for additional completions. Socket references | ||
| 38 | +// keep endpoint storage alive until its accumulated bytes are dispatched. | ||
| 39 | +struct UrmaRxParseBatch { | ||
| 40 | + SocketUniquePtr sockets[BRPC_URMA_MAX_CQE_PER_POLL]; | ||
| 41 | + size_t size{0}; | ||
| 42 | + void Add(SocketUniquePtr&); | ||
| 43 | +}; | ||
| 44 | + | ||
| 45 | +} | ||
| 46 | +} | ||
| 47 | +#endif | ||
| 48 | +#endif | ||
| @@ -18,6 +18,7 @@ | |||
| 18 | #ifndef BRPC_URMA_ENDPOINT_H | 18 | #ifndef BRPC_URMA_ENDPOINT_H |
| 19 | #define BRPC_URMA_ENDPOINT_H | 19 | #define BRPC_URMA_ENDPOINT_H |
| 20 | 20 | ||
| 21 | +#include <atomic> | ||
| 21 | #include <cstdint> | 22 | #include <cstdint> |
| 22 | #include <functional> | 23 | #include <functional> |
| 23 | #include <ostream> | 24 | #include <ostream> |
| @@ -44,6 +45,7 @@ class UrmaTransport; | |||
| 44 | namespace urma { | 45 | namespace urma { |
| 45 | 46 | ||
| 46 | class UrmaHandshake; | 47 | class UrmaHandshake; |
| 48 | +class UrmaReceiveAdapter; | ||
| 47 | struct ParsedHello; | 49 | struct ParsedHello; |
| 48 | 50 | ||
| 49 | DECLARE_bool(urma_use_polling); | 51 | DECLARE_bool(urma_use_polling); |
| @@ -77,8 +79,9 @@ private: | |||
| 77 | // behind the connection-resource interface. | 79 | // behind the connection-resource interface. |
| 78 | class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint { | 80 | class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint { |
| 79 | friend class UrmaConnect; | 81 | friend class UrmaConnect; |
| 82 | + friend class UrmaReceiveAdapter; | ||
| 80 | friend class Socket; | 83 | friend class Socket; |
| 81 | - friend class UrmaTransport; | 84 | + friend class ::brpc::UrmaTransport; |
| 82 | friend class UrmaHandshakeClientV2; | 85 | friend class UrmaHandshakeClientV2; |
| 83 | friend class UrmaHandshakeServerV2; | 86 | friend class UrmaHandshakeServerV2; |
| 84 | friend class UrmaHandshakeClientV3; | 87 | friend class UrmaHandshakeClientV3; |
| @@ -109,12 +112,6 @@ public: | |||
| 109 | 112 | ||
| 110 | int WaitWritable(butil::atomic<int>* butex, int expected, | 113 | int WaitWritable(butil::atomic<int>* butex, int expected, |
| 111 | const timespec* deadline); | 114 | const timespec* deadline); |
| 112 | - static UrmaConnectionHandle LookupConnectionResource(SocketId id); | ||
| 113 | - static void OnReceived(SocketId id, const UrmaConnectionHandle& source, | ||
| 114 | - butil::IOBuf data); | ||
| 115 | - static void OnResourceError(SocketId id, const UrmaConnectionHandle& source, | ||
| 116 | - int error); | ||
| 117 | - | ||
| 118 | // For debug: dump endpoint state to @os. | 115 | // For debug: dump endpoint state to @os. |
| 119 | void DebugInfo(std::ostream& os, | 116 | void DebugInfo(std::ostream& os, |
| 120 | butil::StringPiece connector = "\n") const; | 117 | butil::StringPiece connector = "\n") const; |
| @@ -180,8 +177,9 @@ private: | |||
| 180 | static void* ProcessHandshakeAtClient(void* arg); | 177 | static void* ProcessHandshakeAtClient(void* arg); |
| 181 | static void* ProcessHandshakeAtServer(void* arg); | 178 | static void* ProcessHandshakeAtServer(void* arg); |
| 182 | 179 | ||
| 183 | - // Allocate / deallocate the per-connection URMA resources (JFCE/JFC/JFR/ | 180 | + // Publish per-connection resources once. DeallocateResources logically |
| 184 | - // Jetty, plus the CQ socket). Returns 0 on success. | 181 | + // closes without detaching; destruction/quiescent Reset releases ownership. |
| 182 | + // AllocateResources returns 0 on success. | ||
| 185 | int AllocateResources(UrmaConnectionRole role); | 183 | int AllocateResources(UrmaConnectionRole role); |
| 186 | void DeallocateResources(); | 184 | void DeallocateResources(); |
| 187 | 185 | ||
| @@ -198,10 +196,17 @@ private: | |||
| 198 | // after ESTABLISHED, matching the connection-ready boundary seen by user | 196 | // after ESTABLISHED, matching the connection-ready boundary seen by user |
| 199 | // code. | 197 | // code. |
| 200 | void DispatchReceivedBytes(SocketUniquePtr& s, ssize_t bytes); | 198 | void DispatchReceivedBytes(SocketUniquePtr& s, ssize_t bytes); |
| 201 | - void DispatchReceivedBytesLocked(SocketUniquePtr& s, ssize_t bytes); | 199 | + // Caller holds _dispatch_mutex and owns last_msg through dispatch. |
| 200 | + void ParseReceivedBytesLocked(SocketUniquePtr& s, ssize_t bytes, | ||
| 201 | + InputMessageClosure& last_msg); | ||
| 202 | + // Caller must keep the Socket alive. Reset requires quiescent old users. | ||
| 203 | + UrmaConnectionResource* BorrowConnectionResource() const { | ||
| 204 | + return _published_resource.load(std::memory_order_acquire); | ||
| 205 | + } | ||
| 202 | UrmaConnectionHandle GetConnectionResource() const; | 206 | UrmaConnectionHandle GetConnectionResource() const; |
| 203 | 207 | ||
| 204 | inline void TryReadOnTcp(); | 208 | inline void TryReadOnTcp(); |
| 209 | + int SendFallbackHello(int version); | ||
| 205 | void FallbackToTcp(UrmaTransport* transport, bool process_tcp); | 210 | void FallbackToTcp(UrmaTransport* transport, bool process_tcp); |
| 206 | void FailHandshake(UrmaTransport* transport, int error, | 211 | void FailHandshake(UrmaTransport* transport, int error, |
| 207 | const char* reason); | 212 | const char* reason); |
| @@ -220,8 +225,13 @@ private: | |||
| 220 | butil::atomic<int64_t> _pending_received_bytes{0}; | 225 | butil::atomic<int64_t> _pending_received_bytes{0}; |
| 221 | butil::Mutex _dispatch_mutex; | 226 | butil::Mutex _dispatch_mutex; |
| 222 | 227 | ||
| 223 | - mutable butil::Mutex _resource_mutex; | 228 | + // Write once before release-publication; never detach on failure/fallback. |
| 229 | + // Socket references pin this owner. Only destruction or quiescent Reset | ||
| 230 | + // may release it. TX contexts retain their independent DMA/import owners. | ||
| 224 | UrmaConnectionHandle _connection_resource; | 231 | UrmaConnectionHandle _connection_resource; |
| 232 | + std::atomic<UrmaConnectionResource*> _published_resource{nullptr}; | ||
| 233 | + // Protected by _dispatch_mutex; prevents publication after logical close. | ||
| 234 | + bool _resources_closed{false}; | ||
| 225 | InputMessengerProcessor _input_processor; | 235 | InputMessengerProcessor _input_processor; |
| 226 | 236 | ||
| 227 | // Butex for waking the handshake bthread parked in ReadFromFd. | 237 | // Butex for waking the handshake bthread parked in ReadFromFd. |
| @@ -1596,6 +1596,9 @@ int CreateUrmaResourceProvider(UrmaResourceMode mode, | |||
| 1596 | errno = EINVAL; | 1596 | errno = EINVAL; |
| 1597 | return -1; | 1597 | return -1; |
| 1598 | } | 1598 | } |
| 1599 | + if (mode == UrmaResourceMode::POOLED) { | ||
| 1600 | + return CreatePooledUrmaResourceProvider(out); | ||
| 1601 | + } | ||
| 1599 | if (mode != UrmaResourceMode::EXCLUSIVE) { | 1602 | if (mode != UrmaResourceMode::EXCLUSIVE) { |
| 1600 | errno = ENOTSUP; | 1603 | errno = ENOTSUP; |
| 1601 | return -1; | 1604 | return -1; |
| @@ -1606,13 +1609,22 @@ int CreateUrmaResourceProvider(UrmaResourceMode mode, | |||
| 1606 | 1609 | ||
| 1607 | // Intentionally process-lived, like the common registered-memory service. | 1610 | // Intentionally process-lived, like the common registered-memory service. |
| 1608 | // No atexit cleanup: IOBuf TLS chains may outlive atexit handlers. | 1611 | // No atexit cleanup: IOBuf TLS chains may outlive atexit handlers. |
| 1612 | +DEFINE_string(urma_resource_mode, "exclusive", | ||
| 1613 | + "URMA resource backend: exclusive or pooled (startup only)"); | ||
| 1614 | +bool UsingPooledUrmaResources() { return FLAGS_urma_resource_mode == "pooled"; } | ||
| 1609 | static auto* g_provider = new std::unique_ptr<UrmaResourceProvider>; | 1615 | static auto* g_provider = new std::unique_ptr<UrmaResourceProvider>; |
| 1610 | int InitializeUrmaResourceProvider(const UrmaResourceCallbacks& callbacks) { | 1616 | int InitializeUrmaResourceProvider(const UrmaResourceCallbacks& callbacks) { |
| 1611 | if (*g_provider) { | 1617 | if (*g_provider) { |
| 1612 | errno = EALREADY; | 1618 | errno = EALREADY; |
| 1613 | return -1; | 1619 | return -1; |
| 1614 | } | 1620 | } |
| 1615 | - if (CreateUrmaResourceProvider(UrmaResourceMode::EXCLUSIVE, g_provider) != | 1621 | + if (FLAGS_urma_resource_mode != "exclusive" && !UsingPooledUrmaResources()) { |
| 1622 | + errno = EINVAL; | ||
| 1623 | + return -1; | ||
| 1624 | + } | ||
| 1625 | + const auto mode = UsingPooledUrmaResources() ? UrmaResourceMode::POOLED : | ||
| 1626 | + UrmaResourceMode::EXCLUSIVE; | ||
| 1627 | + if (CreateUrmaResourceProvider(mode, g_provider) != | ||
| 1616 | 0) | 1628 | 0) |
| 1617 | return -1; | 1629 | return -1; |
| 1618 | return (*g_provider)->Initialize(callbacks); | 1630 | return (*g_provider)->Initialize(callbacks); |
| @@ -105,6 +105,7 @@ void HelloMessage::Serialize(void* buf) const { | |||
| 105 | write64(seg_va); | 105 | write64(seg_va); |
| 106 | write64(seg_len); | 106 | write64(seg_len); |
| 107 | write32(seg_token_id); | 107 | write32(seg_token_id); |
| 108 | + if (impl_ver == IMPL_POOLED_VERSION) write64(socket_id); | ||
| 108 | } | 109 | } |
| 109 | 110 | ||
| 110 | void HelloMessage::Deserialize(const void* buf) { | 111 | void HelloMessage::Deserialize(const void* buf) { |
| @@ -145,6 +146,7 @@ void HelloMessage::Deserialize(const void* buf) { | |||
| 145 | seg_va = read64(); | 146 | seg_va = read64(); |
| 146 | seg_len = read64(); | 147 | seg_len = read64(); |
| 147 | seg_token_id = read32(); | 148 | seg_token_id = read32(); |
| 149 | + if (impl_ver == IMPL_POOLED_VERSION) socket_id = read64(); | ||
| 148 | } | 150 | } |
| 149 | 151 | ||
| 150 | } // namespace v2_wire | 152 | } // namespace v2_wire |
| @@ -165,10 +167,11 @@ constexpr uint32_t MAX_V3_PB_SIZE = 4096; | |||
| 165 | } // namespace | 167 | } // namespace |
| 166 | 168 | ||
| 167 | bool ValidHello(const ParsedHello& h) { | 169 | bool ValidHello(const ParsedHello& h) { |
| 168 | - if (h.buffer_size < MIN_BUFFER_SIZE) { | 170 | + if (h.pooled && (h.socket_id == INVALID_SOCKET_ID || h.tp_type != URMA_CTP)) return false; |
| 171 | + if (h.buffer_size < (h.pooled ? 1024 - sizeof(butil::IOBuf::Block) : MIN_BUFFER_SIZE)) { | ||
| 169 | return false; | 172 | return false; |
| 170 | } | 173 | } |
| 171 | - if (h.recv_buffer_cnt < MIN_BUFFER_CNT || h.recv_buffer_cnt > MAX_BUFFER_CNT) { | 174 | + if (h.recv_buffer_cnt < (h.pooled ? 1 : MIN_BUFFER_CNT) || h.recv_buffer_cnt > MAX_BUFFER_CNT) { |
| 172 | return false; | 175 | return false; |
| 173 | } | 176 | } |
| 174 | if (h.jetty_id == 0) { | 177 | if (h.jetty_id == 0) { |
| @@ -193,17 +196,29 @@ bool ValidHello(const ParsedHello& h) { | |||
| 193 | // and translates it into ParsedHello. | 196 | // and translates it into ParsedHello. |
| 194 | int ReadBodyAndNegotiate(UrmaEndpoint* ep, ParsedHello* out, bool* negotiated) { | 197 | int ReadBodyAndNegotiate(UrmaEndpoint* ep, ParsedHello* out, bool* negotiated) { |
| 195 | *negotiated = false; | 198 | *negotiated = false; |
| 196 | - uint8_t body[v2_wire::HELLO_BODY_LEN]; | 199 | + uint8_t body[v2_wire::POOLED_BODY_LEN]{}; |
| 197 | - if (ep->ReadFromFd(body, v2_wire::HELLO_BODY_LEN) < 0) { | 200 | + if (ep->ReadFromFd(body, 6) < 0) return -1; |
| 198 | - return -1; | 201 | + uint16_t length, version, implementation; |
| 202 | + std::memcpy(&length, body, 2); | ||
| 203 | + std::memcpy(&version, body + 2, 2); | ||
| 204 | + std::memcpy(&implementation, body + 4, 2); | ||
| 205 | + length = butil::NetToHost16(length); | ||
| 206 | + implementation = butil::NetToHost16(implementation); | ||
| 207 | + const bool pooled = implementation == v2_wire::IMPL_POOLED_VERSION; | ||
| 208 | + const size_t packet_size = pooled ? v2_wire::POOLED_PACKET_LEN : v2_wire::HELLO_PACKET_LEN; | ||
| 209 | + if (length < packet_size || length > v2_wire::HELLO_MSG_LEN_MAX) { | ||
| 210 | + errno = EPROTO; return -1; | ||
| 199 | } | 211 | } |
| 212 | + if (butil::NetToHost16(version) != v2_wire::HELLO_V2_VERSION || | ||
| 213 | + (implementation != v2_wire::IMPL_V2_VERSION && !pooled)) { | ||
| 214 | + return DrainBytes(ep, length - 4 - 6); | ||
| 215 | + } | ||
| 216 | + if (ep->ReadFromFd(body + 6, packet_size - 4 - 6) < 0) return -1; | ||
| 200 | v2_wire::HelloMessage m; | 217 | v2_wire::HelloMessage m; |
| 201 | m.Deserialize(body); | 218 | m.Deserialize(body); |
| 202 | - if (m.msg_len < v2_wire::HELLO_MSG_LEN_MIN || | ||
| 203 | - m.msg_len > v2_wire::HELLO_MSG_LEN_MAX || | ||
| 204 | - m.hello_ver != v2_wire::HELLO_V2_VERSION || | ||
| 205 | - m.impl_ver != v2_wire::IMPL_V2_VERSION) { return 0; } | ||
| 206 | ParsedHello p; | 219 | ParsedHello p; |
| 220 | + p.pooled = pooled; | ||
| 221 | + p.socket_id = m.socket_id; | ||
| 207 | p.buffer_size = m.buffer_size; | 222 | p.buffer_size = m.buffer_size; |
| 208 | p.recv_buffer_cnt = m.recv_buffer_cnt; | 223 | p.recv_buffer_cnt = m.recv_buffer_cnt; |
| 209 | p.jetty_id = m.jetty_id; | 224 | p.jetty_id = m.jetty_id; |
| @@ -215,15 +230,13 @@ int ReadBodyAndNegotiate(UrmaEndpoint* ep, ParsedHello* out, bool* negotiated) { | |||
| 215 | p.seg_va = m.seg_va; | 230 | p.seg_va = m.seg_va; |
| 216 | p.seg_len = m.seg_len; | 231 | p.seg_len = m.seg_len; |
| 217 | p.seg_token_id = m.seg_token_id; | 232 | p.seg_token_id = m.seg_token_id; |
| 218 | - if (!ValidHello(p)) { | ||
| 219 | - return 0; | ||
| 220 | - } | ||
| 221 | // Drain trailing bytes if msg_len advertises more than the fixed body. | 233 | // Drain trailing bytes if msg_len advertises more than the fixed body. |
| 222 | - if (m.msg_len > v2_wire::HELLO_PACKET_LEN) { | 234 | + if (m.msg_len > packet_size) { |
| 223 | - if (DrainBytes(ep, m.msg_len - v2_wire::HELLO_PACKET_LEN) < 0) { | 235 | + if (DrainBytes(ep, m.msg_len - packet_size) < 0) { |
| 224 | return -1; | 236 | return -1; |
| 225 | } | 237 | } |
| 226 | } | 238 | } |
| 239 | + if (!ValidHello(p)) return 0; | ||
| 227 | *out = p; | 240 | *out = p; |
| 228 | *negotiated = true; | 241 | *negotiated = true; |
| 229 | return 0; | 242 | return 0; |
| @@ -248,10 +261,10 @@ int DrainBytes(UrmaEndpoint* ep, size_t n) { | |||
| 248 | int UrmaHandshakeClientV2::SendLocalHello() { | 261 | int UrmaHandshakeClientV2::SendLocalHello() { |
| 249 | v2_wire::HelloMessage m; | 262 | v2_wire::HelloMessage m; |
| 250 | _ep->FillLocalHelloV2(&m); | 263 | _ep->FillLocalHelloV2(&m); |
| 251 | - uint8_t packet[v2_wire::HELLO_PACKET_LEN]; | 264 | + uint8_t packet[v2_wire::POOLED_PACKET_LEN]{}; |
| 252 | std::memcpy(packet, "URMA", 4); | 265 | std::memcpy(packet, "URMA", 4); |
| 253 | m.Serialize(packet + 4); | 266 | m.Serialize(packet + 4); |
| 254 | - return _ep->WriteToFd(packet, v2_wire::HELLO_PACKET_LEN); | 267 | + return _ep->WriteToFd(packet, m.msg_len); |
| 255 | } | 268 | } |
| 256 | 269 | ||
| 257 | int UrmaHandshakeClientV2::ReceiveAndParseRemoteHello(ParsedHello* out, | 270 | int UrmaHandshakeClientV2::ReceiveAndParseRemoteHello(ParsedHello* out, |
| @@ -288,10 +301,10 @@ int UrmaHandshakeServerV2::SendLocalHello() { | |||
| 288 | m.jetty_id = 0; | 301 | m.jetty_id = 0; |
| 289 | m.buffer_size = 0; | 302 | m.buffer_size = 0; |
| 290 | } | 303 | } |
| 291 | - uint8_t packet[v2_wire::HELLO_PACKET_LEN]; | 304 | + uint8_t packet[v2_wire::POOLED_PACKET_LEN]{}; |
| 292 | std::memcpy(packet, "URMA", 4); | 305 | std::memcpy(packet, "URMA", 4); |
| 293 | m.Serialize(packet + 4); | 306 | m.Serialize(packet + 4); |
| 294 | - return _ep->WriteToFd(packet, v2_wire::HELLO_PACKET_LEN); | 307 | + return _ep->WriteToFd(packet, m.msg_len); |
| 295 | } | 308 | } |
| 296 | 309 | ||
| 297 | // ============================================================================ | 310 | // ============================================================================ |
| @@ -35,6 +35,8 @@ class UrmaEndpoint; | |||
| 35 | // then uses ApplyRemoteHello to size its send/recv windows and to drive | 35 | // then uses ApplyRemoteHello to size its send/recv windows and to drive |
| 36 | // urma_import_seg / urma_import_jetty. | 36 | // urma_import_seg / urma_import_jetty. |
| 37 | struct ParsedHello { | 37 | struct ParsedHello { |
| 38 | + bool pooled = false; | ||
| 39 | + uint64_t socket_id = UINT64_MAX; | ||
| 38 | uint32_t buffer_size = 0; // peer recv buffer size in bytes (per WR) | 40 | uint32_t buffer_size = 0; // peer recv buffer size in bytes (per WR) |
| 39 | uint32_t recv_buffer_cnt = 0; // peer recv buffer count (RQ depth - 1) | 41 | uint32_t recv_buffer_cnt = 0; // peer recv buffer count (RQ depth - 1) |
| 40 | uint32_t jetty_id = 0; // peer jetty id | 42 | uint32_t jetty_id = 0; // peer jetty id |
| @@ -67,9 +69,13 @@ constexpr size_t HELLO_MSG_LEN_MIN = HELLO_PACKET_LEN; | |||
| 67 | constexpr size_t HELLO_MSG_LEN_MAX = 4096; | 69 | constexpr size_t HELLO_MSG_LEN_MAX = 4096; |
| 68 | constexpr uint16_t HELLO_V2_VERSION = 2; | 70 | constexpr uint16_t HELLO_V2_VERSION = 2; |
| 69 | constexpr uint16_t IMPL_V2_VERSION = 1; | 71 | constexpr uint16_t IMPL_V2_VERSION = 1; |
| 72 | +constexpr uint16_t IMPL_POOLED_VERSION = 2; | ||
| 73 | +constexpr size_t POOLED_BODY_LEN = HELLO_BODY_LEN + 8; | ||
| 74 | +constexpr size_t POOLED_PACKET_LEN = HELLO_PACKET_LEN + 8; | ||
| 70 | 75 | ||
| 71 | // The serializable struct. Aligned so it can be reinterpreted as raw bytes. | 76 | // The serializable struct. Aligned so it can be reinterpreted as raw bytes. |
| 72 | struct HelloMessage { | 77 | struct HelloMessage { |
| 78 | + uint64_t socket_id = UINT64_MAX; | ||
| 73 | uint16_t msg_len; // total packet length (incl. magic) | 79 | uint16_t msg_len; // total packet length (incl. magic) |
| 74 | uint16_t hello_ver; | 80 | uint16_t hello_ver; |
| 75 | uint16_t impl_ver; | 81 | uint16_t impl_ver; |
| @@ -29,7 +29,7 @@ option java_outer_classname = "UrmaHandshakeProto"; | |||
| 29 | message UrmaHello { | 29 | message UrmaHello { |
| 30 | required uint32 buffer_size = 1; // recv buffer size in bytes (per WR) | 30 | required uint32 buffer_size = 1; // recv buffer size in bytes (per WR) |
| 31 | required uint32 recv_buffer_cnt = 2; // number of recv buffers posted | 31 | required uint32 recv_buffer_cnt = 2; // number of recv buffers posted |
| 32 | - required uint32 jetty_id = 3; // local jetty id | 32 | + required uint32 jetty_id = 3; // Jetty, or shared JFR for resource_mode=2 |
| 33 | required bytes eid = 4; // 16-byte local EID (network order) | 33 | required bytes eid = 4; // 16-byte local EID (network order) |
| 34 | required uint32 uasid = 5; // local uasid | 34 | required uint32 uasid = 5; // local uasid |
| 35 | required uint32 tp_type = 6; // urma_tp_type_t | 35 | required uint32 tp_type = 6; // urma_tp_type_t |
| @@ -44,4 +44,6 @@ message UrmaHello { | |||
| 44 | required uint64 seg_va = 9; // segment virtual address | 44 | required uint64 seg_va = 9; // segment virtual address |
| 45 | required uint64 seg_len = 10; // segment length in bytes | 45 | required uint64 seg_len = 10; // segment length in bytes |
| 46 | required uint32 seg_token_id = 11; // segment token id | 46 | required uint32 seg_token_id = 11; // segment token id |
| 47 | + optional uint64 socket_id = 12; | ||
| 48 | + optional uint32 resource_mode = 13 [default = 1]; // 1=exclusive, 2=pooled | ||
| 47 | } | 49 | } |
| @@ -46,6 +46,8 @@ | |||
| 46 | #include "urma_types.h" | 46 | #include "urma_types.h" |
| 47 | #include "brpc/urma/urma_bonding.h" | 47 | #include "brpc/urma/urma_bonding.h" |
| 48 | #include "brpc/urma/urma_endpoint.h" | 48 | #include "brpc/urma/urma_endpoint.h" |
| 49 | +#include "brpc/urma/urma_memory_pool.h" | ||
| 50 | +#include "brpc/urma/urma_resource_pool.h" | ||
| 49 | 51 | ||
| 50 | DECLARE_int32(task_group_ntags); | 52 | DECLARE_int32(task_group_ntags); |
| 51 | 53 | ||
| @@ -94,6 +96,9 @@ DEFINE_int32(urma_prepared_jetty_cnt, 8, | |||
| 94 | DEFINE_int32(urma_buffer_size, 8 * 1024, | 96 | DEFINE_int32(urma_buffer_size, 8 * 1024, |
| 95 | "Per-buffer size in the URMA buffer pool (bytes). " | 97 | "Per-buffer size in the URMA buffer pool (bytes). " |
| 96 | "Must match IOBuf block size to keep zero-copy working."); | 98 | "Must match IOBuf block size to keep zero-copy working."); |
| 99 | +DEFINE_int32(urma_ctp_buf_count, 48000, "Registered 4 KiB CTP blocks (pooled mode)"); | ||
| 100 | +DEFINE_int32(urma_header_buf_count, 65536, "Registered 64 B headers (pooled mode)"); | ||
| 101 | +DEFINE_int32(urma_memory_pool_limit_mb, 400, "Registered pool limit in MiB (pooled mode)"); | ||
| 97 | DEFINE_int32(urma_buffer_count, 65536, | 102 | DEFINE_int32(urma_buffer_count, 65536, |
| 98 | "Number of buffers in the URMA buffer pool."); | 103 | "Number of buffers in the URMA buffer pool."); |
| 99 | 104 | ||
| @@ -398,6 +403,7 @@ bool InitPool() { | |||
| 398 | // All pool buffers share the same segment, so we return g_pool_seg for any | 403 | // All pool buffers share the same segment, so we return g_pool_seg for any |
| 399 | // pool address; the per-WR length selects the slice. | 404 | // pool address; the per-WR length selects the slice. |
| 400 | urma_target_seg_t* GetPoolSegFor(void* buf) { | 405 | urma_target_seg_t* GetPoolSegFor(void* buf) { |
| 406 | + if (UsingPooledUrmaResources()) { return GetPooledMemorySegFor(buf); } | ||
| 401 | if (g_skip_urma_init || !g_pool_seg || !g_pool_base) { | 407 | if (g_skip_urma_init || !g_pool_seg || !g_pool_base) { |
| 402 | return nullptr; | 408 | return nullptr; |
| 403 | } | 409 | } |
| @@ -418,6 +424,9 @@ static void GlobalRelease() { | |||
| 418 | LOG(WARNING) << "Retain URMA services until resource shutdown completes"; | 424 | LOG(WARNING) << "Retain URMA services until resource shutdown completes"; |
| 419 | return; | 425 | return; |
| 420 | } | 426 | } |
| 427 | + if (UsingPooledUrmaResources() && !ReleaseUrmaMemoryPools()) { | ||
| 428 | + return; | ||
| 429 | + } | ||
| 421 | if ((g_pool && g_pool->outstanding.load(butil::memory_order_acquire) != 0) || | 430 | if ((g_pool && g_pool->outstanding.load(butil::memory_order_acquire) != 0) || |
| 422 | (g_pool_seg && urma_unregister_seg(g_pool_seg) != URMA_SUCCESS)) { | 431 | (g_pool_seg && urma_unregister_seg(g_pool_seg) != URMA_SUCCESS)) { |
| 423 | LOG(WARNING) << "Retain URMA memory while registered blocks are live"; | 432 | LOG(WARNING) << "Retain URMA memory while registered blocks are live"; |
| @@ -483,6 +492,36 @@ static bool GlobalUrmaInitializeImpl() { | |||
| 483 | return false; | 492 | return false; |
| 484 | } | 493 | } |
| 485 | 494 | ||
| 495 | + const bool pooled = UsingPooledUrmaResources(); | ||
| 496 | + gflags::CommandLineFlagInfo count_info; | ||
| 497 | + const size_t pool_count = pooled && | ||
| 498 | + gflags::GetCommandLineFlagInfo("urma_buffer_count", &count_info) && | ||
| 499 | + count_info.is_default ? 24000 : FLAGS_urma_buffer_count; | ||
| 500 | + if (pooled) { | ||
| 501 | + const int64_t rx_slots = int64_t(FLAGS_urma_io_jfr_cnt) * FLAGS_urma_io_rx_depth; | ||
| 502 | + const int64_t tx_slots = int64_t(FLAGS_urma_io_jfs_cnt) * FLAGS_urma_io_sq_depth; | ||
| 503 | + const int64_t pool_bytes = int64_t(FLAGS_urma_buffer_size) * pool_count + | ||
| 504 | + int64_t(64) * FLAGS_urma_header_buf_count + | ||
| 505 | + int64_t(URMA_CTP_BLOCK_SIZE) * FLAGS_urma_ctp_buf_count; | ||
| 506 | + if (FLAGS_urma_io_jfs_cnt <= 0 || FLAGS_urma_io_jfr_cnt <= 0 || | ||
| 507 | + FLAGS_urma_io_sq_depth <= 0 || FLAGS_urma_io_rx_depth <= 0 || | ||
| 508 | + FLAGS_urma_io_sq_depth > 65535 || FLAGS_urma_io_rx_depth > 65535 || | ||
| 509 | + FLAGS_urma_buffer_size <= static_cast<int>(URMA_CTP_BLOCK_SIZE) || | ||
| 510 | + FLAGS_urma_buffer_size % 64 != 0 || | ||
| 511 | + pool_count <= 0 || FLAGS_urma_ctp_buf_count <= rx_slots || | ||
| 512 | + FLAGS_urma_header_buf_count < rx_slots + tx_slots || | ||
| 513 | + FLAGS_urma_memory_pool_limit_mb <= 0 || | ||
| 514 | + pool_bytes > int64_t(FLAGS_urma_memory_pool_limit_mb) * 1024 * 1024) { | ||
| 515 | + LOG(ERROR) << "Invalid URMA queue, buffer, or poller configuration"; | ||
| 516 | + errno = EINVAL; | ||
| 517 | + return false; | ||
| 518 | + } | ||
| 519 | + | ||
| 520 | + if (FLAGS_urma_use_polling || IsBondingDeviceName(FLAGS_urma_device.c_str())) { | ||
| 521 | + errno = ENOTSUP; | ||
| 522 | + return false; | ||
| 523 | + } | ||
| 524 | + } | ||
| 486 | urma_init_attr_t init_attr{}; | 525 | urma_init_attr_t init_attr{}; |
| 487 | const urma_status_t status = urma_init(&init_attr); | 526 | const urma_status_t status = urma_init(&init_attr); |
| 488 | if (status != URMA_SUCCESS && status != URMA_EEXIST) { | 527 | if (status != URMA_SUCCESS && status != URMA_EEXIST) { |
| @@ -583,7 +622,7 @@ static bool GlobalUrmaInitializeImpl() { | |||
| 583 | g_max_sge = FLAGS_urma_max_sge; | 622 | g_max_sge = FLAGS_urma_max_sge; |
| 584 | } | 623 | } |
| 585 | } | 624 | } |
| 586 | - g_recv_block_size = | 625 | + g_recv_block_size = pooled ? URMA_CTP_MAX_MESSAGE_SIZE - sizeof(UrmaMsgHeader) : |
| 587 | static_cast<size_t>(FLAGS_urma_buffer_size) - | 626 | static_cast<size_t>(FLAGS_urma_buffer_size) - |
| 588 | sizeof(butil::IOBuf::Block); | 627 | sizeof(butil::IOBuf::Block); |
| 589 | 628 | ||
| @@ -596,6 +635,13 @@ static bool GlobalUrmaInitializeImpl() { | |||
| 596 | return false; | 635 | return false; |
| 597 | } | 636 | } |
| 598 | 637 | ||
| 638 | + if (pooled) { | ||
| 639 | + if (g_max_sge < 2 || IsUrmaBondingDevice() || | ||
| 640 | + !InitializeUrmaMemoryPools(g_context, FLAGS_urma_buffer_size, | ||
| 641 | + pool_count, FLAGS_urma_header_buf_count, FLAGS_urma_ctp_buf_count)) { | ||
| 642 | + return false; | ||
| 643 | + } | ||
| 644 | + } else { | ||
| 599 | // Buffer pool. | 645 | // Buffer pool. |
| 600 | g_pool = new BufferPool(); | 646 | g_pool = new BufferPool(); |
| 601 | g_pool_buffer_size = static_cast<size_t>(FLAGS_urma_buffer_size); | 647 | g_pool_buffer_size = static_cast<size_t>(FLAGS_urma_buffer_size); |
| @@ -620,6 +666,8 @@ static bool GlobalUrmaInitializeImpl() { | |||
| 620 | butil::iobuf::blockmem_deallocate = PoolDeallocate; | 666 | butil::iobuf::blockmem_deallocate = PoolDeallocate; |
| 621 | butil::SetDefaultBlockSize(g_pool_buffer_size); | 667 | butil::SetDefaultBlockSize(g_pool_buffer_size); |
| 622 | 668 | ||
| 669 | + } | ||
| 670 | + | ||
| 623 | if (UrmaEndpoint::GlobalInitialize() != 0) { | 671 | if (UrmaEndpoint::GlobalInitialize() != 0) { |
| 624 | LOG(ERROR) << "Fail to initialize URMA endpoint resources"; | 672 | LOG(ERROR) << "Fail to initialize URMA endpoint resources"; |
| 625 | return false; | 673 | return false; |
| @@ -634,8 +682,9 @@ static bool GlobalUrmaInitializeImpl() { | |||
| 634 | LOG(INFO) << "URMA initialized: device=" << device_name | 682 | LOG(INFO) << "URMA initialized: device=" << device_name |
| 635 | << " bonding=" << g_is_bonding_device | 683 | << " bonding=" << g_is_bonding_device |
| 636 | << " max_sge=" << g_max_sge | 684 | << " max_sge=" << g_max_sge |
| 637 | - << " buffer_size=" << g_pool_buffer_size | 685 | + << " pooled=" << pooled |
| 638 | - << " buffer_count=" << g_pool->buffer_count(); | 686 | + << " buffer_size=" << FLAGS_urma_buffer_size |
| 687 | + << " buffer_count=" << pool_count; | ||
| 639 | return true; | 688 | return true; |
| 640 | } | 689 | } |
| 641 | 690 | ||
| @@ -0,0 +1,291 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information | ||
| 4 | +// regarding copyright ownership. The ASF licenses this file | ||
| 5 | +// to you under the Apache License, Version 2.0 (the | ||
| 6 | +// "License"); you may not use this file except in compliance | ||
| 7 | +// with the License. You may obtain a copy of the License at | ||
| 8 | +// | ||
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +// | ||
| 11 | +// Unless required by applicable law or agreed to in writing, | ||
| 12 | +// software distributed under the License is distributed on an | ||
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +// KIND, either express or implied. See the License for the | ||
| 15 | +// specific language governing permissions and limitations | ||
| 16 | +// under the License. | ||
| 17 | + | ||
| 18 | +#include "brpc/urma/urma_memory_pool.h" | ||
| 19 | + | ||
| 20 | +#if BRPC_WITH_URMA | ||
| 21 | +#include <sys/mman.h> | ||
| 22 | +#include <pthread.h> | ||
| 23 | +#include <atomic> | ||
| 24 | +#include <cerrno> | ||
| 25 | +#include <cstdlib> | ||
| 26 | +#include <functional> | ||
| 27 | +#include <limits> | ||
| 28 | +#include <vector> | ||
| 29 | +#include "butil/iobuf.h" | ||
| 30 | +#include "butil/logging.h" | ||
| 31 | +#include "butil/scoped_lock.h" | ||
| 32 | +#include "butil/synchronization/lock.h" | ||
| 33 | + | ||
| 34 | +namespace butil { | ||
| 35 | +namespace iobuf { | ||
| 36 | +extern void* (*blockmem_allocate)(size_t); | ||
| 37 | +extern void (*blockmem_deallocate)(void*); | ||
| 38 | +} | ||
| 39 | +} | ||
| 40 | + | ||
| 41 | +namespace brpc { | ||
| 42 | +namespace urma { | ||
| 43 | +namespace { | ||
| 44 | +const size_t kShards = 64; | ||
| 45 | +class RegisteredPool : public UrmaMemoryPool { | ||
| 46 | +public: | ||
| 47 | + RegisteredPool(size_t size, size_t count) | ||
| 48 | + : _block_size(size), _count(count) {} | ||
| 49 | + ~RegisteredPool() override { | ||
| 50 | + // Never invalidate DMA or application-owned blocks on a failed cleanup. | ||
| 51 | + if (_outstanding.load() == 0 && Unregister() && _base) { | ||
| 52 | + munmap(_base, _block_size * _count); | ||
| 53 | + } | ||
| 54 | + } | ||
| 55 | + bool Init(urma_context_t* context) { | ||
| 56 | + if (!context || !_count || _block_size < sizeof(void*) || | ||
| 57 | + _block_size % alignof(void*) || | ||
| 58 | + _count > std::numeric_limits<size_t>::max() / _block_size) { | ||
| 59 | + errno = EINVAL; | ||
| 60 | + return false; | ||
| 61 | + } | ||
| 62 | + _base = mmap(nullptr, _block_size * _count, PROT_READ | PROT_WRITE, | ||
| 63 | + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | ||
| 64 | + if (_base == MAP_FAILED) { | ||
| 65 | + _base = nullptr; | ||
| 66 | + return false; | ||
| 67 | + } | ||
| 68 | + urma_seg_cfg_t cfg{}; | ||
| 69 | + cfg.va = reinterpret_cast<uint64_t>(_base); | ||
| 70 | + cfg.len = _block_size * _count; | ||
| 71 | + cfg.flag.bs.token_policy = URMA_TOKEN_NONE; | ||
| 72 | + cfg.flag.bs.cacheable = URMA_NON_CACHEABLE; | ||
| 73 | + cfg.flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | | ||
| 74 | + URMA_ACCESS_ATOMIC; | ||
| 75 | + _seg = urma_register_seg(context, &cfg); | ||
| 76 | + if (!_seg) { | ||
| 77 | + return false; | ||
| 78 | + } | ||
| 79 | + _used.assign(_count, 0); | ||
| 80 | + for (size_t i = 0; i < _count; ++i) { | ||
| 81 | + _free[i % kShards].push_back(static_cast<char*>(_base) + | ||
| 82 | + i * _block_size); | ||
| 83 | + } | ||
| 84 | + return true; | ||
| 85 | + } | ||
| 86 | + void* Allocate(size_t size) override { | ||
| 87 | + if (!_seg) { errno = ENODEV; return nullptr; } | ||
| 88 | + if (size > _block_size) { | ||
| 89 | + errno = EMSGSIZE; | ||
| 90 | + return nullptr; | ||
| 91 | + } | ||
| 92 | + const size_t start = std::hash<pthread_t>()(pthread_self()) % kShards; | ||
| 93 | + for (size_t i = 0; i < kShards; ++i) { | ||
| 94 | + const size_t shard = (start + i) % kShards; | ||
| 95 | + BAIDU_SCOPED_LOCK(_mutex[shard]); | ||
| 96 | + if (_free[shard].empty()) { | ||
| 97 | + continue; | ||
| 98 | + } | ||
| 99 | + void* p = _free[shard].back(); | ||
| 100 | + _free[shard].pop_back(); | ||
| 101 | + _used[Index(p)] = 1; | ||
| 102 | + _outstanding.fetch_add(1, std::memory_order_relaxed); | ||
| 103 | + return p; | ||
| 104 | + } | ||
| 105 | + errno = ENOMEM; | ||
| 106 | + return nullptr; | ||
| 107 | + } | ||
| 108 | + void Deallocate(void* p) override { | ||
| 109 | + if (!p) { | ||
| 110 | + return; | ||
| 111 | + } | ||
| 112 | + CHECK(Owns(p)); | ||
| 113 | + CHECK_EQ((reinterpret_cast<uintptr_t>(p) - | ||
| 114 | + reinterpret_cast<uintptr_t>(_base)) % _block_size, 0u); | ||
| 115 | + const size_t index = Index(p); | ||
| 116 | + { | ||
| 117 | + BAIDU_SCOPED_LOCK(_mutex[index % kShards]); | ||
| 118 | + CHECK(_used[index]); | ||
| 119 | + _used[index] = 0; | ||
| 120 | + _free[index % kShards].push_back(p); | ||
| 121 | + } | ||
| 122 | + // Publish the last return only after releasing the shard lock. | ||
| 123 | + _outstanding.fetch_sub(1, std::memory_order_release); | ||
| 124 | + } | ||
| 125 | + bool Owns(const void* p) const override { | ||
| 126 | + const uintptr_t base = reinterpret_cast<uintptr_t>(_base); | ||
| 127 | + const uintptr_t addr = reinterpret_cast<uintptr_t>(p); | ||
| 128 | + return _base && addr >= base && addr - base < _block_size * _count; | ||
| 129 | + } | ||
| 130 | + urma_target_seg_t* SegmentFor(const void* p) const override { | ||
| 131 | + return !p || Owns(p) ? _seg : nullptr; | ||
| 132 | + } | ||
| 133 | + size_t outstanding() const override { return _outstanding.load(); } | ||
| 134 | + bool Unregister() override { | ||
| 135 | + if (_seg && urma_unregister_seg(_seg) != URMA_SUCCESS) { | ||
| 136 | + return false; | ||
| 137 | + } | ||
| 138 | + _seg = nullptr; | ||
| 139 | + return true; | ||
| 140 | + } | ||
| 141 | +private: | ||
| 142 | + size_t Index(const void* p) const { | ||
| 143 | + return (reinterpret_cast<uintptr_t>(p) - | ||
| 144 | + reinterpret_cast<uintptr_t>(_base)) / _block_size; | ||
| 145 | + } | ||
| 146 | + size_t _block_size; | ||
| 147 | + size_t _count; | ||
| 148 | + void* _base{nullptr}; | ||
| 149 | + urma_target_seg_t* _seg{nullptr}; | ||
| 150 | + butil::Mutex _mutex[kShards]; | ||
| 151 | + std::vector<void*> _free[kShards]; | ||
| 152 | + std::vector<char> _used; | ||
| 153 | + std::atomic<size_t> _outstanding{0}; | ||
| 154 | +}; | ||
| 155 | + | ||
| 156 | +// These allocators intentionally live until process exit: IOBuf has TLS block | ||
| 157 | +// caches. On rollback unregister DMA but retain mappings and the deallocator | ||
| 158 | +// while any application/TLS block still refers to them. | ||
| 159 | +UrmaMemoryPoolFactory g_factory = CreateUrmaMemoryPool; | ||
| 160 | +UrmaMemoryPool* g_data_pool = nullptr; | ||
| 161 | +UrmaMemoryPool* g_header_pool = nullptr; | ||
| 162 | +UrmaMemoryPool* g_ctp_pool = nullptr; | ||
| 163 | +void* (*g_original_allocate)(size_t) = nullptr; | ||
| 164 | +void (*g_original_deallocate)(void*) = nullptr; | ||
| 165 | +size_t g_original_block_size = 0; | ||
| 166 | +void* AllocateIOBuf(size_t size) { | ||
| 167 | + // Explicit CTP blocks must remain registered; never fall back to malloc. | ||
| 168 | + if (size == URMA_CTP_BLOCK_SIZE && g_ctp_pool) { return AllocCtpBuffer(); } | ||
| 169 | + void* p = g_data_pool->Allocate(size); | ||
| 170 | + return p ? p : g_original_allocate(size); | ||
| 171 | +} | ||
| 172 | +void DeallocateIOBuf(void* p) { | ||
| 173 | + if (g_ctp_pool && g_ctp_pool->Owns(p)) { | ||
| 174 | + g_ctp_pool->Deallocate(p); | ||
| 175 | + } else if (g_data_pool->Owns(p)) { | ||
| 176 | + g_data_pool->Deallocate(p); | ||
| 177 | + } else { | ||
| 178 | + g_original_deallocate(p); | ||
| 179 | + } | ||
| 180 | +} | ||
| 181 | +} // namespace | ||
| 182 | + | ||
| 183 | +std::unique_ptr<UrmaMemoryPool> CreateUrmaMemoryPool( | ||
| 184 | + urma_context_t* context, size_t size, size_t count) { | ||
| 185 | + std::unique_ptr<RegisteredPool> pool(new RegisteredPool(size, count)); | ||
| 186 | + if (!pool->Init(context)) { | ||
| 187 | + return nullptr; | ||
| 188 | + } | ||
| 189 | + return std::unique_ptr<UrmaMemoryPool>(pool.release()); | ||
| 190 | +} | ||
| 191 | + | ||
| 192 | +bool SetUrmaMemoryPoolFactory(UrmaMemoryPoolFactory factory) { | ||
| 193 | + if (g_data_pool || g_header_pool || g_ctp_pool) { errno = EBUSY; return false; } | ||
| 194 | + g_factory = factory ? factory : CreateUrmaMemoryPool; | ||
| 195 | + return true; | ||
| 196 | +} | ||
| 197 | +bool InitializeUrmaMemoryPools(urma_context_t* context, size_t size, | ||
| 198 | + size_t count, size_t headers, size_t ctp_count) { | ||
| 199 | + if (g_data_pool || g_header_pool || g_ctp_pool) { | ||
| 200 | + errno = EALREADY; | ||
| 201 | + return false; | ||
| 202 | + } | ||
| 203 | + auto data = g_factory(context, size, count); | ||
| 204 | + auto header = g_factory(context, 64, headers); | ||
| 205 | + auto ctp = g_factory(context, URMA_CTP_BLOCK_SIZE, ctp_count); | ||
| 206 | + if (!data || !header || !ctp) { | ||
| 207 | + return false; | ||
| 208 | + } | ||
| 209 | + return InstallUrmaMemoryPools(std::move(data), std::move(header), | ||
| 210 | + std::move(ctp), size); | ||
| 211 | +} | ||
| 212 | + | ||
| 213 | +bool InstallUrmaMemoryPools(std::unique_ptr<UrmaMemoryPool> data, | ||
| 214 | + std::unique_ptr<UrmaMemoryPool> header, | ||
| 215 | + std::unique_ptr<UrmaMemoryPool> ctp, size_t size) { | ||
| 216 | + if (g_data_pool || g_header_pool || g_ctp_pool || !data || !header || !ctp || | ||
| 217 | + size <= sizeof(butil::IOBuf::Block)) { | ||
| 218 | + errno = EINVAL; | ||
| 219 | + return false; | ||
| 220 | + } | ||
| 221 | + g_data_pool = data.release(); | ||
| 222 | + g_header_pool = header.release(); | ||
| 223 | + g_ctp_pool = ctp.release(); | ||
| 224 | + g_original_allocate = butil::iobuf::blockmem_allocate; | ||
| 225 | + g_original_deallocate = butil::iobuf::blockmem_deallocate; | ||
| 226 | + g_original_block_size = butil::GetDefaultBlockSize(); | ||
| 227 | + butil::iobuf::blockmem_allocate = AllocateIOBuf; | ||
| 228 | + butil::iobuf::blockmem_deallocate = DeallocateIOBuf; | ||
| 229 | + butil::SetDefaultBlockSize(size); | ||
| 230 | + return true; | ||
| 231 | +} | ||
| 232 | + | ||
| 233 | +bool ReleaseUrmaMemoryPools() { | ||
| 234 | + // Delivered IOBufs may still own CTP blocks after queues have stopped. | ||
| 235 | + // Their deleters require both the pool object and its registration. | ||
| 236 | + if (g_ctp_pool) { | ||
| 237 | + if (g_ctp_pool->outstanding() != 0 || !g_ctp_pool->Unregister()) { | ||
| 238 | + return false; | ||
| 239 | + } | ||
| 240 | + delete g_ctp_pool; | ||
| 241 | + g_ctp_pool = nullptr; | ||
| 242 | + } | ||
| 243 | + if (g_header_pool) { | ||
| 244 | + if (g_header_pool->outstanding() != 0 || !g_header_pool->Unregister()) { | ||
| 245 | + return false; | ||
| 246 | + } | ||
| 247 | + delete g_header_pool; | ||
| 248 | + g_header_pool = nullptr; | ||
| 249 | + } | ||
| 250 | + if (g_data_pool) { | ||
| 251 | + if (!g_data_pool->Unregister()) { return false; } | ||
| 252 | + butil::SetDefaultBlockSize(g_original_block_size); | ||
| 253 | + if (g_data_pool->outstanding() == 0) { | ||
| 254 | + butil::iobuf::blockmem_allocate = g_original_allocate; | ||
| 255 | + butil::iobuf::blockmem_deallocate = g_original_deallocate; | ||
| 256 | + delete g_data_pool; | ||
| 257 | + g_data_pool = nullptr; | ||
| 258 | + } | ||
| 259 | + } | ||
| 260 | + return true; | ||
| 261 | +} | ||
| 262 | +void* AllocHeaderBuffer() { | ||
| 263 | + return g_header_pool ? g_header_pool->Allocate(64) : nullptr; | ||
| 264 | +} | ||
| 265 | +void FreeHeaderBuffer(void* p) { | ||
| 266 | + if (p) { g_header_pool->Deallocate(p); } | ||
| 267 | +} | ||
| 268 | +urma_target_seg_t* GetHeaderPoolSegFor(void* p) { | ||
| 269 | + return g_header_pool ? g_header_pool->SegmentFor(p) : nullptr; | ||
| 270 | +} | ||
| 271 | +void* AllocCtpBuffer() { | ||
| 272 | + if (!g_ctp_pool) { errno = ENODEV; return nullptr; } | ||
| 273 | + return g_ctp_pool->Allocate(URMA_CTP_BLOCK_SIZE); | ||
| 274 | +} | ||
| 275 | +void FreeCtpBuffer(void* p) { | ||
| 276 | + if (p) { g_ctp_pool->Deallocate(p); } | ||
| 277 | +} | ||
| 278 | +urma_target_seg_t* GetCtpPoolSegFor(void* p) { | ||
| 279 | + return g_ctp_pool ? g_ctp_pool->SegmentFor(p) : nullptr; | ||
| 280 | +} | ||
| 281 | +urma_target_seg_t* GetPooledMemorySegFor(void* p) { | ||
| 282 | + // nullptr keeps advertising the standard pool in the handshake. Address | ||
| 283 | + // lookup includes CTP buffers, so echoing received data stays zero-copy. | ||
| 284 | + if (p && g_ctp_pool && g_ctp_pool->Owns(p)) { | ||
| 285 | + return g_ctp_pool->SegmentFor(p); | ||
| 286 | + } | ||
| 287 | + return g_data_pool ? g_data_pool->SegmentFor(p) : nullptr; | ||
| 288 | +} | ||
| 289 | +} // namespace urma | ||
| 290 | +} // namespace brpc | ||
| 291 | +#endif | ||
| @@ -0,0 +1,75 @@ | |||
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one | ||
| 2 | +// or more contributor license agreements. See the NOTICE file | ||
| 3 | +// distributed with this work for additional information | ||
| 4 | +// regarding copyright ownership. The ASF licenses this file | ||
| 5 | +// to you under the Apache License, Version 2.0 (the | ||
| 6 | +// "License"); you may not use this file except in compliance | ||
| 7 | +// with the License. You may obtain a copy of the License at | ||
| 8 | +// | ||
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +// | ||
| 11 | +// Unless required by applicable law or agreed to in writing, | ||
| 12 | +// software distributed under the License is distributed on an | ||
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| 14 | +// KIND, either express or implied. See the License for the | ||
| 15 | +// specific language governing permissions and limitations | ||
| 16 | +// under the License. | ||
| 17 | + | ||
| 18 | +#ifndef BRPC_URMA_MEMORY_POOL_H | ||
| 19 | +#define BRPC_URMA_MEMORY_POOL_H | ||
| 20 | + | ||
| 21 | +#if BRPC_WITH_URMA | ||
| 22 | +#include <cstddef> | ||
| 23 | +#include <memory> | ||
| 24 | +#include "urma_api.h" | ||
| 25 | + | ||
| 26 | +namespace brpc { | ||
| 27 | +namespace urma { | ||
| 28 | + | ||
| 29 | +constexpr size_t URMA_CTP_BLOCK_SIZE = 4096; | ||
| 30 | + | ||
| 31 | +// Replacement boundary for registered storage. Allocate never returns an | ||
| 32 | +// unregistered buffer. The owner must stop DMA before Unregister/ destruction. | ||
| 33 | +class UrmaMemoryPool { | ||
| 34 | +public: | ||
| 35 | + virtual ~UrmaMemoryPool() = default; | ||
| 36 | + virtual void* Allocate(size_t size) = 0; | ||
| 37 | + virtual void Deallocate(void* buffer) = 0; | ||
| 38 | + virtual urma_target_seg_t* SegmentFor(const void* address) const = 0; | ||
| 39 | + virtual bool Owns(const void* address) const = 0; | ||
| 40 | + virtual size_t outstanding() const = 0; | ||
| 41 | + virtual bool Unregister() = 0; | ||
| 42 | +}; | ||
| 43 | + | ||
| 44 | +// Fixed-size, sharded, registered mmap allocator; also used for raw 64B | ||
| 45 | +// headers. Headers never enter the IOBuf block allocator. | ||
| 46 | +std::unique_ptr<UrmaMemoryPool> CreateUrmaMemoryPool( | ||
| 47 | + urma_context_t* context, size_t block_size, size_t block_count); | ||
| 48 | + | ||
| 49 | +// Install alternative allocators before URMA initialization. Ownership moves to | ||
| 50 | +// the process; the data allocator's block size becomes IOBuf's default size. | ||
| 51 | +bool InstallUrmaMemoryPools(std::unique_ptr<UrmaMemoryPool> data, | ||
| 52 | + std::unique_ptr<UrmaMemoryPool> headers, | ||
| 53 | + std::unique_ptr<UrmaMemoryPool> ctp, | ||
| 54 | + size_t block_size); | ||
| 55 | +typedef std::unique_ptr<UrmaMemoryPool> (*UrmaMemoryPoolFactory)( | ||
| 56 | + urma_context_t*, size_t, size_t); | ||
| 57 | +// Call before global initialization; nullptr restores the default allocator. | ||
| 58 | +bool SetUrmaMemoryPoolFactory(UrmaMemoryPoolFactory factory); | ||
| 59 | +bool InitializeUrmaMemoryPools(urma_context_t* context, | ||
| 60 | + size_t block_size, size_t block_count, | ||
| 61 | + size_t header_count, size_t ctp_count); | ||
| 62 | +// Returns false if a registered segment must remain alive. | ||
| 63 | +bool ReleaseUrmaMemoryPools(); | ||
| 64 | +void* AllocHeaderBuffer(); | ||
| 65 | +void FreeHeaderBuffer(void* buffer); | ||
| 66 | +urma_target_seg_t* GetHeaderPoolSegFor(void* buffer); | ||
| 67 | +void* AllocCtpBuffer(); | ||
| 68 | +void FreeCtpBuffer(void* buffer); | ||
| 69 | +urma_target_seg_t* GetCtpPoolSegFor(void* buffer); | ||
| 70 | +urma_target_seg_t* GetPooledMemorySegFor(void* buffer); | ||
| 71 | + | ||
| 72 | +} // namespace urma | ||
| 73 | +} // namespace brpc | ||
| 74 | +#endif | ||
| 75 | +#endif | ||