已合并
同步 CheckerL2 最新代码到 hccl_vm #5403
git_qk创建于 14 天前
同步 CheckerL2 最新代码到 hccl_vm #5403
已合并
共 820 个文件变更+146382-41373
| @@ -45,12 +45,6 @@ if(ENABLE_COVERAGE) | |||
| 45 | endif() | 45 | endif() |
| 46 | 46 | ||
| 47 | option(BUILD_DEVICE_ARM "Build device_arm (ARM cross-compile)" OFF) | 47 | option(BUILD_DEVICE_ARM "Build device_arm (ARM cross-compile)" OFF) |
| 48 | -option(BUILD_A6_CCU_INSTR "Build A6 ccu instruction (ARM cross-compile)" ON) | ||
| 49 | - | ||
| 50 | -if(BUILD_A6_CCU_INSTR) | ||
| 51 | - add_definitions(-DBUILD_A6_CCU_INSTR) | ||
| 52 | -endif() | ||
| 53 | - | ||
| 54 | set(HCOMM_VM_ROOT_PATH ${CMAKE_CURRENT_SOURCE_DIR}) | 48 | set(HCOMM_VM_ROOT_PATH ${CMAKE_CURRENT_SOURCE_DIR}) |
| 55 | 49 | ||
| 56 | if(NOT DEFINED ASCEND_CANN_PACKAGE_PATH) | 50 | if(NOT DEFINED ASCEND_CANN_PACKAGE_PATH) |
| @@ -181,12 +175,59 @@ include(CPack) | |||
| 181 | # 编译DEVICE-ARM-交叉编译版本 | 175 | # 编译DEVICE-ARM-交叉编译版本 |
| 182 | if(BUILD_DEVICE_ARM) | 176 | if(BUILD_DEVICE_ARM) |
| 183 | message(STATUS "BUILD_DEVICE_ARM start") | 177 | message(STATUS "BUILD_DEVICE_ARM start") |
| 184 | - set(DEVICE_ARM_BUILD_DIR ${CMAKE_SOURCE_DIR}/build_device) | 178 | + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/cann_hcc_validation.cmake") |
| 179 | + | ||
| 180 | + if(NOT DEFINED CANN_HCC_ROOT OR CANN_HCC_ROOT STREQUAL "") | ||
| 181 | + set(_cann_hcc_candidates | ||
| 182 | + "${ASCEND_CANN_PACKAGE_PATH}/tools/hcc" | ||
| 183 | + "${ASCEND_CANN_PACKAGE_PATH}/toolkit/toolchain/hcc" | ||
| 184 | + ) | ||
| 185 | + set(_cann_hcc_candidate_errors "") | ||
| 186 | + foreach(_candidate IN LISTS _cann_hcc_candidates) | ||
| 187 | + hcclvm_validate_cann_hcc("${_candidate}" _candidate_is_complete _candidate_missing_path) | ||
| 188 | + if(_candidate_is_complete) | ||
| 189 | + set(CANN_HCC_ROOT "${_candidate}") | ||
| 190 | + break() | ||
| 191 | + endif() | ||
| 192 | + list(APPEND _cann_hcc_candidate_errors | ||
| 193 | + "${_candidate}: missing ${_candidate_missing_path}" | ||
| 194 | + ) | ||
| 195 | + endforeach() | ||
| 196 | + endif() | ||
| 197 | + | ||
| 198 | + if(NOT DEFINED CANN_HCC_ROOT OR CANN_HCC_ROOT STREQUAL "") | ||
| 199 | + list(JOIN _cann_hcc_candidate_errors "\n " _cann_hcc_error_details) | ||
| 200 | + message(FATAL_ERROR | ||
| 201 | + "Complete CANN HCC toolchain was not found under ${ASCEND_CANN_PACKAGE_PATH}.\n" | ||
| 202 | + "Tried:\n ${_cann_hcc_error_details}\n" | ||
| 203 | + "Expected tools/hcc or toolkit/toolchain/hcc, or set CANN_HCC_ROOT explicitly.") | ||
| 204 | + endif() | ||
| 205 | + | ||
| 206 | + hcclvm_validate_cann_hcc("${CANN_HCC_ROOT}" _cann_hcc_is_complete _cann_hcc_missing_path) | ||
| 207 | + if(NOT _cann_hcc_is_complete) | ||
| 208 | + message(FATAL_ERROR "Incomplete CANN HCC toolchain: ${_cann_hcc_missing_path} was not found") | ||
| 209 | + endif() | ||
| 210 | + | ||
| 211 | + message(STATUS "CANN HCC root: ${CANN_HCC_ROOT}") | ||
| 212 | + message(STATUS "CANN HCC sysroot: ${CANN_HCC_ROOT}/sysroot") | ||
| 213 | + | ||
| 214 | + # 交叉编译 aarch64 版 yaml-cpp 静态库, 供 bin/device 直接读取 log_config.yaml | ||
| 215 | + include(cmake/third_party/yaml-cpp-aarch64.cmake) | ||
| 216 | + if(TARGET third_party_yaml_cpp_aarch64) | ||
| 217 | + set(_yaml_cpp_aarch64_dep third_party_yaml_cpp_aarch64) | ||
| 218 | + else() | ||
| 219 | + set(_yaml_cpp_aarch64_dep "") | ||
| 220 | + endif() | ||
| 221 | + | ||
| 222 | + # 与旧系统交叉 GCC 的 CMake cache 隔离，避免 CMake 忽略首次配置后传入的 toolchain file。 | ||
| 223 | + set(DEVICE_ARM_BUILD_DIR ${CMAKE_SOURCE_DIR}/build_device/hcc) | ||
| 185 | set(DEVICE_ARM_INSTALL_DIR ${CMAKE_SOURCE_DIR}/hccl_vm_install) | 224 | set(DEVICE_ARM_INSTALL_DIR ${CMAKE_SOURCE_DIR}/hccl_vm_install) |
| 186 | include(ExternalProject) | 225 | include(ExternalProject) |
| 187 | ExternalProject_Add(device_arm_external | 226 | ExternalProject_Add(device_arm_external |
| 188 | SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/device_arm | 227 | SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/device_arm |
| 189 | CMAKE_ARGS | 228 | CMAKE_ARGS |
| 229 | + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_SOURCE_DIR}/cmake/toolchains/aarch64-cann-hcc.cmake | ||
| 230 | + -DCANN_HCC_ROOT=${CANN_HCC_ROOT} | ||
| 190 | -DCMAKE_INSTALL_PREFIX=${DEVICE_ARM_INSTALL_DIR} | 231 | -DCMAKE_INSTALL_PREFIX=${DEVICE_ARM_INSTALL_DIR} |
| 191 | -DHCCL_VM_SRC=${HCCL_VM_SRC} | 232 | -DHCCL_VM_SRC=${HCCL_VM_SRC} |
| 192 | -DTHRID_PARTY_DIR=${THRID_PARTY_DIR} | 233 | -DTHRID_PARTY_DIR=${THRID_PARTY_DIR} |
| @@ -195,7 +236,8 @@ if(BUILD_DEVICE_ARM) | |||
| 195 | -DCANN_ARCH_DIR=${CANN_ARCH_DIR} | 236 | -DCANN_ARCH_DIR=${CANN_ARCH_DIR} |
| 196 | -DHCOMM_CODE_ROOT=${HCOMM_CODE_ROOT} | 237 | -DHCOMM_CODE_ROOT=${HCOMM_CODE_ROOT} |
| 197 | -DHCCL_CODE_ROOT=${HCCL_CODE_ROOT} | 238 | -DHCCL_CODE_ROOT=${HCCL_CODE_ROOT} |
| 198 | - | 239 | + -DYAMLCPP_AARCH64_INSTALL_PATH=${YAMLCPP_AARCH64_INSTALL_PATH} |
| 240 | + DEPENDS ${_yaml_cpp_aarch64_dep} | ||
| 199 | BINARY_DIR ${DEVICE_ARM_BUILD_DIR} | 241 | BINARY_DIR ${DEVICE_ARM_BUILD_DIR} |
| 200 | INSTALL_DIR ${DEVICE_ARM_INSTALL_DIR} | 242 | INSTALL_DIR ${DEVICE_ARM_INSTALL_DIR} |
| 201 | BUILD_ALWAYS ON | 243 | BUILD_ALWAYS ON |
| @@ -26,9 +26,10 @@ chmod +x Ascend-cann-950-ops_9.1.0_linux-x86_64.run | |||
| 26 | ./Ascend-cann-950-ops_9.1.0_linux-x86_64.run --install --install-path=/home/workspace/Ascend | 26 | ./Ascend-cann-950-ops_9.1.0_linux-x86_64.run --install --install-path=/home/workspace/Ascend |
| 27 | ``` | 27 | ``` |
| 28 | 28 | ||
| 29 | + | ||
| 29 | ### 2.2 hccl_test编译 | 30 | ### 2.2 hccl_test编译 |
| 30 | 31 | ||
| 31 | -hccl_test是昇腾官方提供的HCCL性能测试工具，详见[HCCL性能测试工具](https://www.hiascend.com/document/redirect/CANNCommunityToolHcclTest)，HCCL-VM支持在虚拟环境中运行hccl_test用例。请先参照[hccl_test用例构建](#42-hccl-test用例构建)章节进行用例二进制程序的编译。 | 32 | +hccl_test是昇腾官方提供的HCCL性能测试工具，详见[HCCL性能测试工具](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/910beta1/devaids/hccltool/HCCLpertest_16_0001.html)，HCCL-VM支持在虚拟环境中运行hccl_test用例。请先参照[hccl_test用例构建](#42-hccl-test用例构建)章节进行用例二进制程序的编译。 |
| 32 | 33 | ||
| 33 | 备注：可选，未来支持Pytorch用例。 | 34 | 备注：可选，未来支持Pytorch用例。 |
| 34 | 35 | ||
| @@ -36,39 +37,7 @@ hccl_test是昇腾官方提供的HCCL性能测试工具，详见[HCCL性能测 | |||
| 36 | 37 | ||
| 37 | ## 3. 快速上手 | 38 | ## 3. 快速上手 |
| 38 | 39 | ||
| 39 | -### 3.1 一键安装 | 40 | +### 3.1 手动构建&安装 |
| 40 | - | ||
| 41 | -一行完成依赖安装、源码拉取、CANN 检测与编译（默认使用 `main` profile，对应 hcomm/hccl 的 `master` 主线）。工作目录与手动安装保持一致，用 `/home/workspace`（后文示例路径均以此为准）： | ||
| 42 | - | ||
| 43 | -```bash | ||
| 44 | -# 创建并进入工作目录（脚本默认装到当前目录） | ||
| 45 | -mkdir -p /home/workspace && cd /home/workspace | ||
| 46 | -curl -fsSL https://raw.gitcode.com/cann/hcomm/raw/master/test/hccl_vm/hccl_vm_installer | bash | ||
| 47 | -``` | ||
| 48 | - | ||
| 49 | -也可下载后本地运行（便于先审阅或离线分发）：`bash hccl_vm_installer`；或用 `--workspace` 显式指定：`... | bash -s -- --workspace /home/workspace`。 | ||
| 50 | - | ||
| 51 | -**前提**：x86_64 Linux；工具链需满足 hcomm build.md 要求——gcc/g++ 7.3.0–13.3.x、cmake ≥ 3.16.0（同时约束宿主与 aarch64 交叉编译器）。Ubuntu 22.04 / 24.04 开箱即用；更高版本默认 gcc（14/15）超范围，脚本会告警并继续尝试，建议在满足范围的环境编译。 | ||
| 52 | - | ||
| 53 | -**CANN**：脚本只在工作目录 `<workspace>/Ascend`（或 `--ascend-path` 指定的路径）探测 CANN；未找到即自动下载配套版本并安装到该处，root 与普通用户行为一致。`--offline` 只检测、从不下载；内网无公网时自动回退为打印自备 CANN 引导。 | ||
| 54 | - | ||
| 55 | -**hccl_test**：默认一并编译 OpenMPI 与 hccl_test 性能测试工具，`--skip-hccl-test` 可关闭。 | ||
| 56 | - | ||
| 57 | -**常用参数**： | ||
| 58 | - | ||
| 59 | -- `--profile <名>`：配套方案（默认 `main`，`--list-profiles` 列全部） | ||
| 60 | -- `--workspace <路径>`：工作目录，源码／编译／产物所在（默认当前目录） | ||
| 61 | -- `--ascend-path <路径>`：指定 CANN 目录，有则复用、无则装到此处 | ||
| 62 | -- `--reinstall-cann`：重新下载覆盖现有 CANN（版本不匹配时用；默认保留） | ||
| 63 | -- `--offline`：只用现有 CANN、从不下载 | ||
| 64 | -- `--skip-hccl-test`：跳过 hccl_test 编译 | ||
| 65 | -- `-h`：完整帮助 | ||
| 66 | - | ||
| 67 | -完成后工具位于 `/home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin/hccl-vm`。删除工作目录即可清理本工具产物（apt 安装的系统依赖如需卸载请自行 `apt remove`），本工具不改动 CANN。（若用 `--workspace` 指定了别的目录，后文示例中的 `/home/workspace` 请相应替换。） | ||
| 68 | - | ||
| 69 | -> 一键安装已自动完成 `build.sh` 编译与 `build_pkg.sh` 子包安装（含 AICPU/AIV 所需的设备侧符号），装完后 [使用示例](#33-使用示例) 中 CCU/AICPU/AIV 各模式均可直接运行，无需再单独执行 `build_pkg.sh`。 | ||
| 70 | - | ||
| 71 | -### 3.2 手动构建&安装 | ||
| 72 | 41 | ||
| 73 | ```bash | 42 | ```bash |
| 74 | # 1. 创建工作目录 | 43 | # 1. 创建工作目录 |
| @@ -94,136 +63,143 @@ bash ./build.sh --full | |||
| 94 | bash build_pkg.sh | 63 | bash build_pkg.sh |
| 95 | ``` | 64 | ``` |
| 96 | 65 | ||
| 97 | -### 3.3 使用示例 | 66 | +### 3.2 使用示例 |
| 98 | 67 | ||
| 99 | -#### 3.3.1 环境配置 | 68 | +#### 3.2.1 环境配置 |
| 100 | 69 | ||
| 101 | 请参照[hccl_rootinfo文件内容](#47-hccl_rootinfojson文件)，创建并配置hccl_rootinfo.json文件。 | 70 | 请参照[hccl_rootinfo文件内容](#47-hccl_rootinfojson文件)，创建并配置hccl_rootinfo.json文件。 |
| 102 | 71 | ||
| 103 | -#### 3.3.2 CCU模式 | 72 | +#### 3.2.2 CCU模式 |
| 104 | 73 | ||
| 105 | -1. 配置环境变量。 | 74 | +1. 环境变量配置。 |
| 106 | 75 | ||
| 107 | - ```bash | 76 | +```bash |
| 108 | - # 进入工具安装目录 | 77 | +# 进入工具安装目录 |
| 109 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install | 78 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install |
| 110 | - source /home/workspace/Ascend/cann/set_env.sh | 79 | +source /home/workspace/Ascend/cann/set_env.sh |
| 111 | - export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | 80 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH |
| 112 | - export RANK_TABLE_FILE=$(pwd)/data/ranktable.json | 81 | +export RANK_TABLE_FILE=$(pwd)/data/ranktable.json |
| 113 | - export HCCL_OP_EXPANSION_MODE="CCU_SCHED" | 82 | +export HCCL_OP_EXPANSION_MODE="CCU_SCHED" |
| 114 | - ``` | 83 | +``` |
| 115 | 84 | ||
| 116 | -2. 执行。 | 85 | +2. 执行 |
| 117 | 86 | ||
| 118 | - ```bash | ||
| 119 | - # 需要进入到新的bin文件目录下执行hccl-vm | ||
| 120 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | ||
| 121 | - | ||
| 122 | - # 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 | ||
| 123 | - ./hccl-vm start ascend950_cluster_32_server_normal.yaml | ||
| 124 | - | ||
| 125 | - # 如需启用runner插件（可选） | ||
| 126 | - (hvm)$> hccl-vm plugin install @runner | ||
| 127 | - | ||
| 128 | - # 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | ||
| 129 | - (hvm)$> hccl-vm mock-comm 112 | ||
| 130 | - (hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 131 | - | ||
| 132 | - # 执行checker校验 | ||
| 133 | - (hvm)$> hccl-vm plugin run @checker | ||
| 134 | - | ||
| 135 | - # 退出工具终端 | ||
| 136 | - (hvm)$> exit | ||
| 137 | - ``` | ||
| 138 | 87 | ||
| 139 | -3. 验证hccl_test用例运行结果。 | 88 | +```bash |
| 89 | +# 需要进入到新的bin文件目录下执行hccl-vm | ||
| 90 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | ||
| 140 | 91 | ||
| 141 | - [Runner结果查看](#491-runner插件结果) | 92 | +# 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 |
| 142 | - [Checker结果查看](#492-checker插件结果) | 93 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml |
| 143 | 94 | ||
| 144 | -#### 3.3.3 AICPU模式 | 95 | +# 如需启用runner插件（可选） |
| 96 | +(hvm)$> hccl-vm plugin install @runner | ||
| 97 | + | ||
| 98 | +# 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | ||
| 99 | +(hvm)$> hccl-vm mock-comm 112 | ||
| 100 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 101 | + | ||
| 102 | +# 执行checker校验 | ||
| 103 | +(hvm)$> hccl-vm plugin run @checker | ||
| 104 | + | ||
| 105 | +# 退出工具终端 | ||
| 106 | +(hvm)$> exit | ||
| 107 | +``` | ||
| 108 | + | ||
| 109 | +3. 验证hccl_test用例运行结果 | ||
| 110 | +[Runner结果查看](#491-runner插件结果) | ||
| 111 | +[Checker结果查看](#492-checker插件结果) | ||
| 112 | + | ||
| 113 | +#### 3.2.3 AICPU模式 | ||
| 145 | 114 | ||
| 146 | AICPU展开模式需要将算法展开步骤放到设备侧执行，因此hccl-vm工具需要将HCCL的设备侧的符号编译并模拟执行。由于设备侧符号是ARM架构的，因此在X86环境上编译时需要借助交叉编译器，运行时需要借助QEMU实现AICPU模式的模拟运行。 | 115 | AICPU展开模式需要将算法展开步骤放到设备侧执行，因此hccl-vm工具需要将HCCL的设备侧的符号编译并模拟执行。由于设备侧符号是ARM架构的，因此在X86环境上编译时需要借助交叉编译器，运行时需要借助QEMU实现AICPU模式的模拟运行。 |
| 147 | 116 | ||
| 148 | 1. 环境变量配置。 | 117 | 1. 环境变量配置。 |
| 149 | 118 | ||
| 150 | - ```bash | 119 | +```bash |
| 151 | - # 进入工具安装目录 | 120 | +# 进入工具安装目录 |
| 152 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install | 121 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install |
| 153 | - source /home/workspace/Ascend/cann/set_env.sh | 122 | +source /home/workspace/Ascend/cann/set_env.sh |
| 154 | - export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | 123 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH |
| 155 | - export RANK_TABLE_FILE=$(pwd)/data/ranktable.json | 124 | +export RANK_TABLE_FILE=$(pwd)/data/ranktable.json |
| 156 | - export HCCL_OP_EXPANSION_MODE="AI_CPU" | 125 | +export HCCL_OP_EXPANSION_MODE="AI_CPU" |
| 157 | - ``` | 126 | +``` |
| 158 | 127 | ||
| 159 | 2. 执行 | 128 | 2. 执行 |
| 160 | 129 | ||
| 161 | - ```bash | 130 | +```bash |
| 162 | - # 需要进入到新的bin文件目录下执行hccl-vm | 131 | +# 需要进入到新的bin文件目录下执行hccl-vm |
| 163 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | 132 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin |
| 164 | - | 133 | + |
| 165 | - # 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 | 134 | +# 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 |
| 166 | - ./hccl-vm start ascend950_cluster_32_server_normal.yaml | 135 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml |
| 167 | - | 136 | + |
| 168 | - # 如需启用runner插件（可选） | 137 | +# 如需启用runner插件（可选） |
| 169 | - (hvm)$> hccl-vm plugin install @runner | 138 | +(hvm)$> hccl-vm plugin install @runner |
| 170 | - | 139 | + |
| 171 | - # 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | 140 | +# 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） |
| 172 | - (hvm)$> hccl-vm mock-comm 112 | 141 | +(hvm)$> hccl-vm mock-comm 112 |
| 173 | - (hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | 142 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt |
| 174 | - | 143 | + |
| 175 | - # 执行checker校验 | 144 | +# 执行checker校验 |
| 176 | - (hvm)$> hccl-vm plugin run @checker | 145 | +(hvm)$> hccl-vm plugin run @checker |
| 177 | - | 146 | + |
| 178 | - # 退出工具终端 | 147 | +# 退出工具终端 |
| 179 | - (hvm)$> exit | 148 | +(hvm)$> exit |
| 180 | - ``` | 149 | +``` |
| 181 | 150 | ||
| 182 | 3. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) [Checker结果查看](#492-checker插件结果) | 151 | 3. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) [Checker结果查看](#492-checker插件结果) |
| 183 | 152 | ||
| 184 | -#### 3.3.4 AIV模式 | 153 | +#### 3.2.4 HostDPU模式 |
| 154 | + | ||
| 155 | +HostDPU展开模式运行时的环境变量配置等均与AICPU模式相同，仅在集群规模和通信域规模上有如下差异： | ||
| 156 | + | ||
| 157 | +1. 集群规格配置需使用 `./hccl-vm start ascend950_cluster_32_server_normal_hostdpu.yaml` | ||
| 158 | +2. 需要Server数量 >= 2，即不支持单Server的通信域（不支持 112.yaml、114.yaml、118.yaml等） | ||
| 159 | + | ||
| 160 | +#### 3.2.5 AIV模式 | ||
| 185 | 161 | ||
| 186 | 1. 环境变量配置。 | 162 | 1. 环境变量配置。 |
| 187 | 163 | ||
| 188 | - ```bash | 164 | +```bash |
| 189 | - # 进入工具安装目录 | 165 | +# 进入工具安装目录 |
| 190 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install | 166 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install |
| 191 | - source /home/workspace/Ascend/cann/set_env.sh | 167 | +source /home/workspace/Ascend/cann/set_env.sh |
| 192 | - export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | 168 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH |
| 193 | - export RANK_TABLE_FILE=$(pwd)/data/ranktable.json | 169 | +export RANK_TABLE_FILE=$(pwd)/data/ranktable.json |
| 194 | - export HCCL_OP_EXPANSION_MODE="AIV" | 170 | +export HCCL_OP_EXPANSION_MODE="AIV" |
| 195 | - ``` | 171 | +``` |
| 196 | 172 | ||
| 197 | 2. 执行 | 173 | 2. 执行 |
| 198 | 174 | ||
| 199 | - ```bash | 175 | +```bash |
| 200 | - # 需要进入到新的bin文件目录下执行hccl-vm | 176 | +# 需要进入到新的bin文件目录下执行hccl-vm |
| 201 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | 177 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin |
| 202 | - | 178 | + |
| 203 | - # 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 | 179 | +# 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 |
| 204 | - ./hccl-vm start ascend950_cluster_32_server_normal.yaml | 180 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml |
| 205 | - | 181 | + |
| 206 | - # 如需启用runner插件（可选） | 182 | +# 如需启用runner插件（可选） |
| 207 | - (hvm)$> hccl-vm plugin install @runner | 183 | +(hvm)$> hccl-vm plugin install @runner |
| 208 | - | 184 | + |
| 209 | - # 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | 185 | +# 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） |
| 210 | - (hvm)$> hccl-vm mock-comm 112 | 186 | +(hvm)$> hccl-vm mock-comm 112 |
| 211 | - (hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | 187 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt |
| 212 | - | 188 | + |
| 213 | - # 执行checker校验 | 189 | +# 执行checker校验 |
| 214 | - (hvm)$> hccl-vm plugin run @checker | 190 | +(hvm)$> hccl-vm plugin run @checker |
| 215 | - | 191 | + |
| 216 | - # 退出工具终端 | 192 | +# 退出工具终端 |
| 217 | - (hvm)$> exit | 193 | +(hvm)$> exit |
| 218 | - ``` | 194 | +``` |
| 219 | 195 | ||
| 220 | 3. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) [Checker结果查看](#492-checker插件结果) | 196 | 3. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) [Checker结果查看](#492-checker插件结果) |
| 221 | 197 | ||
| 222 | -### 3.4 Pytorch用例示例 | 198 | +### 3.3 Pytorch用例示例 |
| 223 | 199 | ||
| 224 | 暂不支持。 | 200 | 暂不支持。 |
| 225 | 201 | ||
| 226 | -### 3.5 hccl代码修改验证示例 | 202 | +### 3.4 hccl代码修改验证示例 |
| 227 | 203 | ||
| 228 | 若您修改了CANN的算子包代码，如新增了算法类型，为保证您的修改生效，需要按照如下步骤操作执行。build_pkg.sh脚本帮助用户执行编包、装包、拷贝Device侧依赖符号，执行前需设置环境变量: | 204 | 若您修改了CANN的算子包代码，如新增了算法类型，为保证您的修改生效，需要按照如下步骤操作执行。build_pkg.sh脚本帮助用户执行编包、装包、拷贝Device侧依赖符号，执行前需设置环境变量: |
| 229 | 205 | ||
| @@ -239,7 +215,7 @@ export HCOMM_CODE_HOME=/home/workspace/hcomm | |||
| 239 | 1. 若您更新/修改了CANN hccl仓代码，请执行`bash build_pkg.sh --install hccl`。 | 215 | 1. 若您更新/修改了CANN hccl仓代码，请执行`bash build_pkg.sh --install hccl`。 |
| 240 | 2. 若您更新/修改了CANN hcomm仓代码，请执行`bash build_pkg.sh --install hcomm`。 | 216 | 2. 若您更新/修改了CANN hcomm仓代码，请执行`bash build_pkg.sh --install hcomm`。 |
| 241 | 3. 若您同时更新/修改了CANN hccl、hcomm仓代码，请执行`bash build_pkg.sh --full`。 | 217 | 3. 若您同时更新/修改了CANN hccl、hcomm仓代码，请执行`bash build_pkg.sh --full`。 |
| 242 | -4. 参考[使用示例](#33-使用示例)步骤，重新运行用例。 | 218 | +4. 参考[使用示例](#32-使用示例)步骤，重新运行用例。 |
| 243 | 219 | ||
| 244 | --- | 220 | --- |
| 245 | 221 | ||
| @@ -263,31 +239,31 @@ hccl_test用例源码在CANN包安装目录下，支持OpenMPI和MPICH两种环 | |||
| 263 | 239 | ||
| 264 | 1. 安装OpenMPI | 240 | 1. 安装OpenMPI |
| 265 | 241 | ||
| 266 | - ```bash | 242 | +```bash |
| 267 | - sudo apt-get update | 243 | +sudo apt-get update |
| 268 | - sudo apt install openmpi-bin libopenmpi-dev | 244 | +sudo apt install openmpi-bin libopenmpi-dev |
| 269 | - ``` | 245 | +``` |
| 270 | 246 | ||
| 271 | 2. 编译hccl_test | 247 | 2. 编译hccl_test |
| 272 | 248 | ||
| 273 | - ```bash | 249 | +```bash |
| 274 | - # 修改CANN安装目录权限 | 250 | +# 修改CANN安装目录权限 |
| 275 | - chmod -R 755 /home/workspace/Ascend | 251 | +chmod -R 755 /home/workspace/Ascend |
| 276 | - | 252 | + |
| 277 | - # 进入hccl_test用例源码目录 | 253 | +# 进入hccl_test用例源码目录 |
| 278 | - cd /home/workspace/Ascend/cann/tools/hccl_test | 254 | +cd /home/workspace/Ascend/cann/tools/hccl_test |
| 279 | - | 255 | + |
| 280 | - # 设置CANN环境变量 | 256 | +# 设置CANN环境变量 |
| 281 | - source /home/workspace/Ascend/cann/set_env.sh | 257 | +source /home/workspace/Ascend/cann/set_env.sh |
| 282 | - | 258 | + |
| 283 | - # 临时修改Makefile脚本 | 259 | +# 临时修改Makefile脚本 |
| 284 | - if ! grep -q '\-lmpi_cxx' Makefile; then | 260 | +if ! grep -q '\-lmpi_cxx' Makefile; then |
| 285 | - sed -i 's/-lmpi/-lmpi -lmpi_cxx/g' Makefile | 261 | + sed -i 's/-lmpi/-lmpi -lmpi_cxx/g' Makefile |
| 286 | - fi | 262 | +fi |
| 287 | - | 263 | + |
| 288 | - # 编译hccl_test用例 | 264 | +# 编译hccl_test用例 |
| 289 | - MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi make ASCEND_DIR=${ASCEND_HOME_PATH} | 265 | +MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi make ASCEND_DIR=${ASCEND_HOME_PATH} |
| 290 | - ``` | 266 | +``` |
| 291 | 267 | ||
| 292 | #### 4.2.2 MPICH环境编译 | 268 | #### 4.2.2 MPICH环境编译 |
| 293 | 269 | ||
| @@ -369,8 +345,8 @@ links: | |||
| 369 | - **port_group**: 描述哪些port合并为一个portGroup，相同portGroup的port对应IP地址相同。没有配置的port，则默认每个port为一个portGroup。 | 345 | - **port_group**: 描述哪些port合并为一个portGroup，相同portGroup的port对应IP地址相同。没有配置的port，则默认每个port为一个portGroup。 |
| 370 | - **links**: 链路配置表，描述Server/Pod内的所有NPU卡之间，以及NPU与交换机之间的连接关系。 | 346 | - **links**: 链路配置表，描述Server/Pod内的所有NPU卡之间，以及NPU与交换机之间的连接关系。 |
| 371 | - **NPU直连关系**: 工具提供了两种方式配置NPU直连关系： | 347 | - **NPU直连关系**: 工具提供了两种方式配置NPU直连关系： |
| 372 | - - **link_mode == "fullmesh"**: 表示所有Device基于一个Die的Port进行全连接。后续有新增典型的连接方式，可以新增link_mode类型，如"ring"。 | 348 | + - **link_mode == "fullmesh"**: 表示所有Device基于一个Die的Port进行全连接。后续有新增典型的连接方式，可以新增link_mode类型，如"ring"。 |
| 373 | - - **link_mode == "enum"**: 枚举法。当Server/Pod内的NPU连接方式比较复杂时，可以通过枚举所有的链路关系来描述。 | 349 | + - **link_mode == "enum"**: 枚举法。当Server/Pod内的NPU连接方式比较复杂时，可以通过枚举所有的链路关系来描述。 |
| 374 | - **NPU与交换机连接关系**: 用户可以通过枚举法配置NPU与交换机之间的连接关系。 | 350 | - **NPU与交换机连接关系**: 用户可以通过枚举法配置NPU与交换机之间的连接关系。 |
| 375 | - **device_to_device_links**: 描述NPU与NPU之间的连接关系。 | 351 | - **device_to_device_links**: 描述NPU与NPU之间的连接关系。 |
| 376 | - **device_to_switch_links**: 描述NPU与交换机之间的连接关系。 | 352 | - **device_to_switch_links**: 描述NPU与交换机之间的连接关系。 |
| @@ -438,6 +414,46 @@ topology: | |||
| 438 | - 配置通信域时，工具会根据指定的通信域配置编号重新生成topo.json和ranktable.json文件。 | 414 | - 配置通信域时，工具会根据指定的通信域配置编号重新生成topo.json和ranktable.json文件。 |
| 439 | - 上述通信域配置yaml文件中，ranks字段表示每个server实际跑的rank的local id（即device的物理ID）列表。 | 415 | - 上述通信域配置yaml文件中，ranks字段表示每个server实际跑的rank的local id（即device的物理ID）列表。 |
| 440 | 416 | ||
| 417 | +当通信域规模较大时，servers条目中的server选择字段与rank选择字段可自由组合使用，支持以下字段： | ||
| 418 | + | ||
| 419 | +- **server选择字段**（二选一，不可同时配置）： | ||
| 420 | + - `serId`: 单个server的id或server的id列表（如`3`或`[0, 1]`，列表表示列举的每个server）； | ||
| 421 | + - `serId_range`: server的id范围，值为`[<起始id>, <结束id>]`（闭区间，仅支持列表形式），表示该超节点下id从起始至结束的所有server。 | ||
| 422 | +- **rank选择字段**（二选一，不可同时配置）： | ||
| 423 | + - `ranks`: 每个server实际跑的rank的local id列表； | ||
| 424 | + - `ranks_range`: local id范围，值为`[<起始id>, <结束id>]`（闭区间，仅支持列表形式），表示每个server选取id从起始至结束的所有local id。 | ||
| 425 | + | ||
| 426 | +如下示例（2个超节点、共4个server、32个rank）中，同一文件内多种组合混用： | ||
| 427 | + | ||
| 428 | +```yaml | ||
| 429 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 430 | +meta: | ||
| 431 | + podNum: 2 # 总的超节点数 | ||
| 432 | + serNum: 4 # 总的server数 | ||
| 433 | + rankNum: 32 # 总的rank数 | ||
| 434 | + | ||
| 435 | +# 2. 详细拓扑结构 | ||
| 436 | +topology: | ||
| 437 | + - podId: 0 | ||
| 438 | + servers: | ||
| 439 | + # serId为id列表，表示pod 0下server 0和server 1 | ||
| 440 | + - serId: [0, 1] | ||
| 441 | + ranks: [0, 1, 2, 3, 4, 5, 6, 7] | ||
| 442 | + - podId: 1 | ||
| 443 | + servers: | ||
| 444 | + - serId: 0 | ||
| 445 | + ranks_range: [0, 7] | ||
| 446 | + - serId: 1 | ||
| 447 | + ranks_range: [0, 7] | ||
| 448 | +``` | ||
| 449 | + | ||
| 450 | +**注意事项**： | ||
| 451 | + | ||
| 452 | +- serId与serId_range不可同时配置，ranks与ranks_range不可同时配置。 | ||
| 453 | +- serId_range与ranks_range仅支持`[<起始id>, <结束id>]`列表形式（闭区间），不支持`<起始id>-<结束id>`格式；起始id不能大于结束id，且范围宽度（结束id-起始id+1）不能超过1024。 | ||
| 454 | +- 一个serId_range条目会展开为该超节点下id从起始至结束的所有server，一个serId列表条目表示列举的每个server，每个server的ranks均相同。 | ||
| 455 | +- 若server条目中未配置rank选择字段，则该server的ranks为空列表；若未配置server选择字段，则默认serId为0。 | ||
| 456 | + | ||
| 441 | #### 4.3.4 topo和ranktable.json文件说明 | 457 | #### 4.3.4 topo和ranktable.json文件说明 |
| 442 | 458 | ||
| 443 | `topo.json` 和 `ranktable.json` 文件不需要手动创建，工具会根据以下信息自动生成： | 459 | `topo.json` 和 `ranktable.json` 文件不需要手动创建，工具会根据以下信息自动生成： |
| @@ -672,9 +688,9 @@ Runner插件支持通过 `hccl-vm plugin install/uninstall` 命令进行安装 | |||
| 672 | Checker插件，即算法分析器插件：功能是将hccl生成的所有task形成一个DAG图，并且通过分析DAG图，判断是否存在内存冲突；通过模拟执行DAG图，判断是否存在语义错误等问题。 | 688 | Checker插件，即算法分析器插件：功能是将hccl生成的所有task形成一个DAG图，并且通过分析DAG图，判断是否存在内存冲突；通过模拟执行DAG图，判断是否存在语义错误等问题。 |
| 673 | 算法分析器插件，是由用户自行通过命令启动执行。 | 689 | 算法分析器插件，是由用户自行通过命令启动执行。 |
| 674 | 690 | ||
| 675 | -Checker插件正处于新旧交替阶段，Checker V3为原Checker的重构版，主要提高了校验性能，在默认情况下将会运行新Checker（Checker V3）。Checker V3大图校验默认开启：每个sync window内的多个算子会合并成一张大图，执行跨算子同步资源冲突校验。大图校验与老Checker、新Checker的开关相互独立，可以通过修改Checker的`manifest.json`文件中的配置参数进行调整。 | 691 | +Checker 插件当前使用 Checker V3 执行单算子校验。Checker V3 大图校验默认开启：每个 sync window 内的多个算子会合并成一张大图，执行跨算子同步资源冲突校验。单算子 Checker V3 和大图校验可以通过 `manifest.json` 中的配置参数分别控制；老版本 Checker 已移除，`enable_old_checker` 不再是有效配置项。 |
| 676 | 692 | ||
| 677 | -```bash | 693 | +```json |
| 678 | 694 | ||
| 679 | # 配置文件位于 /pathto/hccl_vm_install/plugin/checker/manifest.json | 695 | # 配置文件位于 /pathto/hccl_vm_install/plugin/checker/manifest.json |
| 680 | 696 | ||
| @@ -686,11 +702,11 @@ Checker插件正处于新旧交替阶段，Checker V3为原Checker的重构版 | |||
| 686 | "min_core_version": "1.0.0" | 702 | "min_core_version": "1.0.0" |
| 687 | }, | 703 | }, |
| 688 | "setting": { // Checker插件配置项 | 704 | "setting": { // Checker插件配置项 |
| 689 | - "enable_new_checker": true, // 是否启用新Checker（Checker V3，默认开启） | 705 | + "enable_new_checker": true, // 是否启用 Checker V3 单算子校验（默认开启） |
| 690 | - "enable_old_checker": false, // 是否启用老Checker（默认关闭） | 706 | + "enable_big_graph_checker": true, // 是否启用 Checker V3 大图校验（默认开启，与单算子校验独立） |
| 691 | - "enable_big_graph_checker": true, // 是否启用Checker V3大图校验（默认开启，与old/new checker独立） | 707 | + "enable_insight_dump": false, // 是否输出 Checker V3 Insight 数据（默认关闭） |
| 692 | - "enable_insight_dump": false, // 是否启用可视化数据输出（默认关闭，仅支持老Checker） | 708 | + "enable_memory_snapshot_dump": false, // 是否输出内存快照数据（默认关闭，需要先开启 Insight 数据输出） |
| 693 | - "enable_memory_snapshot_dump": false // 是否启用可视化内存快照数据输出（默认关闭，仅支持老Checker，需要先开启可视化数据输出"enable_insight_dump"） | 709 | + "enable_dag_graphviz_dump": false // 是否输出大图 Graphviz DOT 文件（默认关闭） |
| 694 | } | 710 | } |
| 695 | } | 711 | } |
| 696 | ``` | 712 | ``` |
| @@ -770,7 +786,6 @@ data_size(Bytes): | aveg_time(us): | alg_bandwidth(GB/s): | check_result: | |||
| 770 | ``` | 786 | ``` |
| 771 | 787 | ||
| 772 | --- | 788 | --- |
| 773 | - | ||
| 774 | ### 4.10 大块内存复用（仅校验模式） | 789 | ### 4.10 大块内存复用（仅校验模式） |
| 775 | 790 | ||
| 776 | 仅校验模式用于大规模集群仅运行 Checker 校验的场景。开启后，单块 200MB 到 4GB 的大内存申请复用同一块 4GB 共享区 `HcclCommPool`，各 rank 共享、允许互相覆盖，以此大幅降低 `/dev/shm` 占用。此时大块内容不保证正确，仅适用于不读取缓冲区数据的 Checker V3 校验链路，需要数值正确的结果时请勿开启。 | 791 | 仅校验模式用于大规模集群仅运行 Checker 校验的场景。开启后，单块 200MB 到 4GB 的大内存申请复用同一块 4GB 共享区 `HcclCommPool`，各 rank 共享、允许互相覆盖，以此大幅降低 `/dev/shm` 占用。此时大块内容不保证正确，仅适用于不读取缓冲区数据的 Checker V3 校验链路，需要数值正确的结果时请勿开启。 |
| @@ -783,7 +798,7 @@ data_size(Bytes): | aveg_time(us): | alg_bandwidth(GB/s): | check_result: | |||
| 783 | ./hccl-vm start ascend950_cluster_32_server_normal.yaml --check-only | 798 | ./hccl-vm start ascend950_cluster_32_server_normal.yaml --check-only |
| 784 | ``` | 799 | ``` |
| 785 | 800 | ||
| 786 | ---- | 801 | +*** |
| 787 | 802 | ||
| 788 | ## 5 附录 | 803 | ## 5 附录 |
| 789 | 804 | ||
| @@ -28,7 +28,7 @@ chmod +x Ascend-cann-950-ops_9.1.0_linux-x86_64.run | |||
| 28 | 28 | ||
| 29 | ### 2.2 hccl_test Compilation | 29 | ### 2.2 hccl_test Compilation |
| 30 | 30 | ||
| 31 | -hccl_test is the official HCCL performance test tool provided by Ascend. For details, refer to [HCCL Performance Test Tool](https://www.hiascend.com/en/document/redirect/CANNCommunityToolHcclTest). HCCL-VM supports running hccl_test cases in the virtual environment. First follow the [hccl_test Case Build](#42-hccl-test-case-build) section to compile the case binary program. | 31 | +hccl_test is the official HCCL performance test tool provided by Ascend. For details, refer to [HCCL Performance Test Tool](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/910beta1/devaids/hccltool/HCCLpertest_16_0001.html). HCCL-VM supports running hccl_test cases in the virtual environment. First follow the [hccl_test Case Build](#42-hccl-test-case-build) section to compile the case binary program. |
| 32 | 32 | ||
| 33 | Note: Optional. PyTorch cases will be supported in the future. | 33 | Note: Optional. PyTorch cases will be supported in the future. |
| 34 | 34 | ||
| @@ -36,39 +36,7 @@ Note: Optional. PyTorch cases will be supported in the future. | |||
| 36 | 36 | ||
| 37 | ## 3. Quick Start | 37 | ## 3. Quick Start |
| 38 | 38 | ||
| 39 | -### 3.1 One-Click Installation | 39 | +### 3.1 Manual Build & Installation |
| 40 | - | ||
| 41 | -One command completes dependency installation, source code retrieval, CANN detection, and compilation (the default uses the `main` profile, corresponding to the `master` branch of hcomm/hccl). The working directory remains consistent with manual installation, using `/home/workspace` (all sample paths below follow this convention): | ||
| 42 | - | ||
| 43 | -```bash | ||
| 44 | -# Create and enter the working directory (the script installs to the current directory by default) | ||
| 45 | -mkdir -p /home/workspace && cd /home/workspace | ||
| 46 | -curl -fsSL https://raw.gitcode.com/cann/hcomm/raw/master/test/hccl_vm/hccl_vm_installer | bash | ||
| 47 | -``` | ||
| 48 | - | ||
| 49 | -You may also download and run locally (for review or offline distribution): `bash hccl_vm_installer`; or specify explicitly with `--workspace`: `... | bash -s -- --workspace /home/workspace`. | ||
| 50 | - | ||
| 51 | -**Prerequisites**: x86_64 Linux; the toolchain must meet hcomm build.md requirements: gcc/g++ 7.3.0-13.3.x, cmake >= 3.16.0 (constraints apply to both host and aarch64 cross-compilers). Ubuntu 22.04 / 24.04 work out of the box; newer versions with default gcc (14/15) exceed the supported range. The script issues a warning and continues. Compile in an environment that meets the version range. | ||
| 52 | - | ||
| 53 | -**CANN**: The script probes CANN only in the working directory `<workspace>/Ascend` (or the path specified by `--ascend-path`). If CANN is not found, the script downloads and installs the matching version to that location. Behavior is identical for root and regular users. `--offline` only detects and never downloads. On internal networks without public internet access, the script falls back to printing a self-service CANN preparation guide. | ||
| 54 | - | ||
| 55 | -**hccl_test**: The script compiles OpenMPI and the hccl_test performance test tool by default. Use `--skip-hccl-test` to disable this. | ||
| 56 | - | ||
| 57 | -**Common Parameters**: | ||
| 58 | - | ||
| 59 | -- `--profile <name>`: configuration profile (default `main`, use `--list-profiles` to list all) | ||
| 60 | -- `--workspace <path>`: working directory for source code, compilation, and artifacts (default is the current directory) | ||
| 61 | -- `--ascend-path <path>`: specify the CANN directory; reuse if present, install there if absent | ||
| 62 | -- `--reinstall-cann`: re-download and overwrite existing CANN (use when versions do not match; default preserves existing) | ||
| 63 | -- `--offline`: use only existing CANN, never download | ||
| 64 | -- `--skip-hccl-test`: skip hccl_test compilation | ||
| 65 | -- `-h`: full help | ||
| 66 | - | ||
| 67 | -After completion, the tool resides at `/home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin/hccl-vm`. Delete the working directory to clean up tool artifacts (use `apt remove` manually to uninstall system dependencies installed by apt). This tool does not modify CANN. (If you specified a different directory with `--workspace`, replace `/home/workspace` in the examples below accordingly.) | ||
| 68 | - | ||
| 69 | -> The one-click installation automatically completes `build.sh` compilation and `build_pkg.sh` sub-package installation (including device-side symbols required by AICPU/AIV). After installation, all CCU/AICPU/AIV modes in [Usage Examples](#33-usage-examples) run directly. No separate `build_pkg.sh` execution is needed. | ||
| 70 | - | ||
| 71 | -### 3.2 Manual Build & Installation | ||
| 72 | 40 | ||
| 73 | ```bash | 41 | ```bash |
| 74 | # 1. Create the working directory | 42 | # 1. Create the working directory |
| @@ -94,128 +62,128 @@ bash ./build.sh --full | |||
| 94 | bash build_pkg.sh | 62 | bash build_pkg.sh |
| 95 | ``` | 63 | ``` |
| 96 | 64 | ||
| 97 | -### 3.3 Usage Examples | 65 | +### 3.2 Usage Examples |
| 98 | 66 | ||
| 99 | -#### 3.3.1 Environment Configuration | 67 | +#### 3.2.1 Environment Configuration |
| 100 | 68 | ||
| 101 | Refer to [hccl_rootinfo File Content](#47-hccl_rootinfojson-file) to create and configure the hccl_rootinfo.json file. | 69 | Refer to [hccl_rootinfo File Content](#47-hccl_rootinfojson-file) to create and configure the hccl_rootinfo.json file. |
| 102 | 70 | ||
| 103 | -#### 3.3.2 CCU Mode | 71 | +#### 3.2.2 CCU Mode |
| 104 | 72 | ||
| 105 | -1. Configure environment variables. | 73 | +1. Environment variable configuration |
| 106 | 74 | ||
| 107 | - ```bash | 75 | +```bash |
| 108 | - # Enter the tool installation directory | 76 | +# Enter the tool installation directory |
| 109 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install | 77 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install |
| 110 | - source /home/workspace/Ascend/cann/set_env.sh | 78 | +source /home/workspace/Ascend/cann/set_env.sh |
| 111 | - export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | 79 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH |
| 112 | - export RANK_TABLE_FILE=$(pwd)/data/ranktable.json | 80 | +export RANK_TABLE_FILE=$(pwd)/data/ranktable.json |
| 113 | - export HCCL_OP_EXPANSION_MODE="CCU_SCHED" | 81 | +export HCCL_OP_EXPANSION_MODE="CCU_SCHED" |
| 114 | - ``` | 82 | +``` |
| 115 | 83 | ||
| 116 | -2. Execute. | 84 | +2. Execute |
| 117 | 85 | ||
| 118 | - ```bash | ||
| 119 | - # Enter the new bin directory to execute hccl-vm | ||
| 120 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | ||
| 121 | - | ||
| 122 | - # Select the Ascend cluster topology configuration file, start the tool, initialize the cluster environment, and enter the tool command line | ||
| 123 | - ./hccl-vm start ascend950_cluster_32_server_normal.yaml | ||
| 124 | - | ||
| 125 | - # Enable the runner plugin if needed (optional) | ||
| 126 | - (hvm)$> hccl-vm plugin install @runner | ||
| 127 | - | ||
| 128 | - # Select the communication domain configuration file for this operator execution (run hccl_test cases in a cluster environment with 1 supernode, 1 server, and 1 NPU) | ||
| 129 | - (hvm)$> hccl-vm mock-comm 112 | ||
| 130 | - (hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 131 | - | ||
| 132 | - # Execute checker verification | ||
| 133 | - (hvm)$> hccl-vm plugin run @checker | ||
| 134 | - | ||
| 135 | - # Exit the tool terminal | ||
| 136 | - (hvm)$> exit | ||
| 137 | - ``` | ||
| 138 | 86 | ||
| 139 | -3. Verify hccl_test case execution results. | 87 | +```bash |
| 88 | +# Enter the new bin directory to execute hccl-vm | ||
| 89 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | ||
| 140 | 90 | ||
| 141 | - [View Runner Results](#491-runner-plugin-results) | 91 | +# Select the Ascend cluster topology configuration file, start the tool, initialize the cluster environment, and enter the tool command line |
| 142 | - [View Checker Results](#492-checker-plugin-results) | 92 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml |
| 143 | 93 | ||
| 144 | -#### 3.3.3 AICPU Mode | 94 | +# Enable the runner plugin if needed (optional) |
| 95 | +(hvm)$> hccl-vm plugin install @runner | ||
| 96 | + | ||
| 97 | +# Select the communication domain configuration file for this operator execution (run hccl_test cases in a cluster environment with 1 supernode, 1 server, and 1 NPU) | ||
| 98 | +(hvm)$> hccl-vm mock-comm 112 | ||
| 99 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 100 | + | ||
| 101 | +# Execute checker verification | ||
| 102 | +(hvm)$> hccl-vm plugin run @checker | ||
| 103 | + | ||
| 104 | +# Exit the tool terminal | ||
| 105 | +(hvm)$> exit | ||
| 106 | +``` | ||
| 107 | + | ||
| 108 | +3. Verify hccl_test case execution results | ||
| 109 | +[View Runner Results](#491-runner-plugin-results) | ||
| 110 | +[View Checker Results](#492-checker-plugin-results) | ||
| 111 | + | ||
| 112 | +#### 3.2.3 AICPU Mode | ||
| 145 | 113 | ||
| 146 | The AICPU expansion mode executes algorithm expansion steps on the device side. Therefore the hccl-vm tool compiles and simulates HCCL device-side symbols. Device-side symbols use the ARM architecture. Compilation on x86 environments requires a cross-compiler. Execution requires QEMU to simulate AICPU mode. | 114 | The AICPU expansion mode executes algorithm expansion steps on the device side. Therefore the hccl-vm tool compiles and simulates HCCL device-side symbols. Device-side symbols use the ARM architecture. Compilation on x86 environments requires a cross-compiler. Execution requires QEMU to simulate AICPU mode. |
| 147 | 115 | ||
| 148 | -1. Configure environment variables. | 116 | +1. Environment variable configuration |
| 149 | 117 | ||
| 150 | - ```bash | 118 | +```bash |
| 151 | - # Enter the tool installation directory | 119 | +# Enter the tool installation directory |
| 152 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install | 120 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install |
| 153 | - source /home/workspace/Ascend/cann/set_env.sh | 121 | +source /home/workspace/Ascend/cann/set_env.sh |
| 154 | - export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | 122 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH |
| 155 | - export RANK_TABLE_FILE=$(pwd)/data/ranktable.json | 123 | +export RANK_TABLE_FILE=$(pwd)/data/ranktable.json |
| 156 | - export HCCL_OP_EXPANSION_MODE="AI_CPU" | 124 | +export HCCL_OP_EXPANSION_MODE="AI_CPU" |
| 157 | - ``` | 125 | +``` |
| 158 | 126 | ||
| 159 | 2. Execute | 127 | 2. Execute |
| 160 | 128 | ||
| 161 | - ```bash | 129 | +```bash |
| 162 | - # Enter the new bin directory to execute hccl-vm | 130 | +# Enter the new bin directory to execute hccl-vm |
| 163 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | 131 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin |
| 164 | - | 132 | + |
| 165 | - # Select the Ascend cluster topology configuration file, start the tool, initialize the cluster environment, and enter the tool command line | 133 | +# Select the Ascend cluster topology configuration file, start the tool, initialize the cluster environment, and enter the tool command line |
| 166 | - ./hccl-vm start ascend950_cluster_32_server_normal.yaml | 134 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml |
| 167 | - | 135 | + |
| 168 | - # Enable the runner plugin if needed (optional) | 136 | +# Enable the runner plugin if needed (optional) |
| 169 | - (hvm)$> hccl-vm plugin install @runner | 137 | +(hvm)$> hccl-vm plugin install @runner |
| 170 | - | 138 | + |
| 171 | - # Select the communication domain configuration file for this operator execution (run hccl_test cases in a cluster environment with 1 supernode, 1 server, and 1 NPU) | 139 | +# Select the communication domain configuration file for this operator execution (run hccl_test cases in a cluster environment with 1 supernode, 1 server, and 1 NPU) |
| 172 | - (hvm)$> hccl-vm mock-comm 112 | 140 | +(hvm)$> hccl-vm mock-comm 112 |
| 173 | - (hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | 141 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt |
| 174 | - | 142 | + |
| 175 | - # Execute checker verification | 143 | +# Execute checker verification |
| 176 | - (hvm)$> hccl-vm plugin run @checker | 144 | +(hvm)$> hccl-vm plugin run @checker |
| 177 | - | 145 | + |
| 178 | - # Exit the tool terminal | 146 | +# Exit the tool terminal |
| 179 | - (hvm)$> exit | 147 | +(hvm)$> exit |
| 180 | - ``` | 148 | +``` |
| 181 | 149 | ||
| 182 | 3. Verify hccl_test case execution results [View Runner Results](#491-runner-plugin-results) [View Checker Results](#492-checker-plugin-results) | 150 | 3. Verify hccl_test case execution results [View Runner Results](#491-runner-plugin-results) [View Checker Results](#492-checker-plugin-results) |
| 183 | 151 | ||
| 184 | -#### 3.3.4 AIV Mode | 152 | +#### 3.2.4 AIV Mode |
| 185 | 153 | ||
| 186 | -1. Configure environment variables. | 154 | +1. Environment variable configuration |
| 187 | 155 | ||
| 188 | - ```bash | 156 | +```bash |
| 189 | - # Enter the tool installation directory | 157 | +# Enter the tool installation directory |
| 190 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install | 158 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install |
| 191 | - source /home/workspace/Ascend/cann/set_env.sh | 159 | +source /home/workspace/Ascend/cann/set_env.sh |
| 192 | - export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | 160 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH |
| 193 | - export RANK_TABLE_FILE=$(pwd)/data/ranktable.json | 161 | +export RANK_TABLE_FILE=$(pwd)/data/ranktable.json |
| 194 | - export HCCL_OP_EXPANSION_MODE="AIV" | 162 | +export HCCL_OP_EXPANSION_MODE="AIV" |
| 195 | - ``` | 163 | +``` |
| 196 | 164 | ||
| 197 | 2. Execute | 165 | 2. Execute |
| 198 | 166 | ||
| 199 | - ```bash | 167 | +```bash |
| 200 | - # Enter the new bin directory to execute hccl-vm | 168 | +# Enter the new bin directory to execute hccl-vm |
| 201 | - cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin | 169 | +cd /home/workspace/hcomm/test/hccl_vm/hccl_vm_install/bin |
| 202 | - | 170 | + |
| 203 | - # Select the Ascend cluster topology configuration file, start the tool, initialize the cluster environment, and enter the tool command line | 171 | +# Select the Ascend cluster topology configuration file, start the tool, initialize the cluster environment, and enter the tool command line |
| 204 | - ./hccl-vm start ascend950_cluster_32_server_normal.yaml | 172 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml |
| 205 | - | 173 | + |
| 206 | - # Enable the runner plugin if needed (optional) | 174 | +# Enable the runner plugin if needed (optional) |
| 207 | - (hvm)$> hccl-vm plugin install @runner | 175 | +(hvm)$> hccl-vm plugin install @runner |
| 208 | - | 176 | + |
| 209 | - # Select the communication domain configuration file for this operator execution (run hccl_test cases in a cluster environment with 1 supernode, 1 server, and 1 NPU) | 177 | +# Select the communication domain configuration file for this operator execution (run hccl_test cases in a cluster environment with 1 supernode, 1 server, and 1 NPU) |
| 210 | - (hvm)$> hccl-vm mock-comm 112 | 178 | +(hvm)$> hccl-vm mock-comm 112 |
| 211 | - (hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | 179 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt |
| 212 | - | 180 | + |
| 213 | - # Execute checker verification | 181 | +# Execute checker verification |
| 214 | - (hvm)$> hccl-vm plugin run @checker | 182 | +(hvm)$> hccl-vm plugin run @checker |
| 215 | - | 183 | + |
| 216 | - # Exit the tool terminal | 184 | +# Exit the tool terminal |
| 217 | - (hvm)$> exit | 185 | +(hvm)$> exit |
| 218 | - ``` | 186 | +``` |
| 219 | 187 | ||
| 220 | 3. Verify hccl_test case execution results [View Runner Results](#491-runner-plugin-results) [View Checker Results](#492-checker-plugin-results) | 188 | 3. Verify hccl_test case execution results [View Runner Results](#491-runner-plugin-results) [View Checker Results](#492-checker-plugin-results) |
| 221 | 189 | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | name: "ascend950_cluster_32_server_normal" | 19 | name: "ascend950_cluster_32_server_normal" |
| 12 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | 20 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" |
| 13 | 21 | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | name: "ascend950_cluster_32_server_normal" | 19 | name: "ascend950_cluster_32_server_normal" |
| 12 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | 20 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" |
| 13 | 21 | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | name: "ascend950_cluster_32_server_normal" | 19 | name: "ascend950_cluster_32_server_normal" |
| 12 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | 20 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" |
| 13 | 21 | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | name: "ascend950_cluster_32_server_normal" | 19 | name: "ascend950_cluster_32_server_normal" |
| 12 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | 20 | description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" |
| 13 | 21 | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | name: "ascend960_cluster_32_server_normal" | 19 | name: "ascend960_cluster_32_server_normal" |
| 12 | description: "昇腾960 normal组网：32个超级节点，每个超级节点1个服务器" | 20 | description: "昇腾960 normal组网：32个超级节点，每个超级节点1个服务器" |
| 13 | 21 | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | type: "server_intra_links" | 19 | type: "server_intra_links" |
| 12 | name: "ascend950_links_topo_normal" | 20 | name: "ascend950_links_topo_normal" |
| 13 | description: "ascend950芯片NORMAL类型拓扑连接关系描述文件" | 21 | description: "ascend950芯片NORMAL类型拓扑连接关系描述文件" |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | type: "server_intra_links" | 19 | type: "server_intra_links" |
| 12 | name: "ascend960_links_topo_normal" | 20 | name: "ascend960_links_topo_normal" |
| 13 | description: "ascend960芯片NORMAL类型拓扑连接关系描述文件" | 21 | description: "ascend960芯片NORMAL类型拓扑连接关系描述文件" |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,7 +8,15 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 18 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 19 | + | ||
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| 14 | serNum: 1 | 22 | serNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | 19 | # 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 |
| 12 | meta: | 20 | meta: |
| 13 | podNum: 1 | 21 | podNum: 1 |
| @@ -170,7 +170,6 @@ build_host() { | |||
| 170 | } | 170 | } |
| 171 | 171 | ||
| 172 | build_device() { | 172 | build_device() { |
| 173 | - mkdir -p build_device | ||
| 174 | mkdir -p build | 173 | mkdir -p build |
| 175 | cd build | 174 | cd build |
| 176 | cmake -DBUILD_DEVICE_ARM=ON .. && make -j16 | 175 | cmake -DBUILD_DEVICE_ARM=ON .. && make -j16 |
| @@ -235,9 +235,6 @@ echo "--------------------------" | |||
| 235 | # 第二步：构建并安装 HCOMM | 235 | # 第二步：构建并安装 HCOMM |
| 236 | # ========================================== | 236 | # ========================================== |
| 237 | 237 | ||
| 238 | -# 关闭 Linux 系统对 Python 的保护锁，解决 pip 装包时报错 | ||
| 239 | -sudo rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED | ||
| 240 | - | ||
| 241 | if [ "$BUILD_HCOMM" = true ]; then | 238 | if [ "$BUILD_HCOMM" = true ]; then |
| 242 | echo "正在开始构建 HCOMM..." | 239 | echo "正在开始构建 HCOMM..." |
| 243 | cd "$HCOMM_CODE_HOME" || exit 1 | 240 | cd "$HCOMM_CODE_HOME" || exit 1 |
| @@ -255,7 +252,7 @@ if [ "$BUILD_HCOMM" = true ]; then | |||
| 255 | fi | 252 | fi |
| 256 | 253 | ||
| 257 | echo "找到安装包: $CANN_HCOMM_PACKAGE" | 254 | echo "找到安装包: $CANN_HCOMM_PACKAGE" |
| 258 | - yes y | "$CANN_HCOMM_PACKAGE" --full --install-path="$ASCEND_INSTALL_PATH" | 255 | + yes y | "$CANN_HCOMM_PACKAGE" --full --pylocal --install-path="$ASCEND_INSTALL_PATH" |
| 259 | else | 256 | else |
| 260 | echo "错误: HCOMM 构建失败" | 257 | echo "错误: HCOMM 构建失败" |
| 261 | exit 1 | 258 | exit 1 |
| @@ -284,7 +281,7 @@ if [ "$BUILD_HCCL" = true ]; then | |||
| 284 | fi | 281 | fi |
| 285 | 282 | ||
| 286 | echo "找到安装包: $CANN_HCCL_PACKAGE" | 283 | echo "找到安装包: $CANN_HCCL_PACKAGE" |
| 287 | - yes y | "$CANN_HCCL_PACKAGE" --full --install-path="$ASCEND_INSTALL_PATH" | 284 | + yes y | "$CANN_HCCL_PACKAGE" --full --pylocal --install-path="$ASCEND_INSTALL_PATH" |
| 288 | else | 285 | else |
| 289 | echo "错误: HCCL 构建失败" | 286 | echo "错误: HCCL 构建失败" |
| 290 | exit 1 | 287 | exit 1 |
| @@ -322,7 +319,7 @@ else | |||
| 322 | echo "跳过 aicpu_hcomm.tar.gz 处理..." | 319 | echo "跳过 aicpu_hcomm.tar.gz 处理..." |
| 323 | fi | 320 | fi |
| 324 | 321 | ||
| 325 | -# 修正目录和文件的权限，确保 .so 有足够权限被 dlopen 加载 | 322 | +# 确保当前用户可以读写目录内容，并保留必要的执行权限 |
| 326 | -sudo chmod -R 755 "$AICPU_DEPLOY_DIR" | 323 | +chmod -R u+rwX "$AICPU_DEPLOY_DIR" |
| 327 | 324 | ||
| 328 | echo "所有任务均已执行完成！" | 325 | echo "所有任务均已执行完成！" |
| @@ -0,0 +1,44 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +include_guard(GLOBAL) | ||
| 12 | + | ||
| 13 | +function(hcclvm_validate_cann_hcc hcc_root result_var missing_path_var) | ||
| 14 | + set(_cann_hcc_target aarch64-target-linux-gnu) | ||
| 15 | + set(_cann_hcc_bin "${hcc_root}/bin") | ||
| 16 | + set(_cann_hcc_sysroot "${hcc_root}/sysroot") | ||
| 17 | + set(_cann_hcc_required_paths | ||
| 18 | + "${_cann_hcc_bin}/${_cann_hcc_target}-gcc" | ||
| 19 | + "${_cann_hcc_bin}/${_cann_hcc_target}-g++" | ||
| 20 | + "${_cann_hcc_bin}/${_cann_hcc_target}-ar" | ||
| 21 | + "${_cann_hcc_bin}/${_cann_hcc_target}-ranlib" | ||
| 22 | + "${_cann_hcc_bin}/${_cann_hcc_target}-strip" | ||
| 23 | + "${_cann_hcc_bin}/${_cann_hcc_target}-ld" | ||
| 24 | + "${_cann_hcc_bin}/${_cann_hcc_target}-nm" | ||
| 25 | + "${_cann_hcc_bin}/${_cann_hcc_target}-objcopy" | ||
| 26 | + "${_cann_hcc_sysroot}/lib64/ld-linux-aarch64.so.1" | ||
| 27 | + "${_cann_hcc_sysroot}/lib64/libc.so.6" | ||
| 28 | + "${_cann_hcc_sysroot}/usr/lib64/libstdc++.so.6" | ||
| 29 | + "${_cann_hcc_sysroot}/usr/lib64/libgcc_s.so.1" | ||
| 30 | + ) | ||
| 31 | + | ||
| 32 | + set(_cann_hcc_is_complete TRUE) | ||
| 33 | + set(_cann_hcc_missing_path "") | ||
| 34 | + foreach(_required_path IN LISTS _cann_hcc_required_paths) | ||
| 35 | + if(NOT EXISTS "${_required_path}") | ||
| 36 | + set(_cann_hcc_is_complete FALSE) | ||
| 37 | + set(_cann_hcc_missing_path "${_required_path}") | ||
| 38 | + break() | ||
| 39 | + endif() | ||
| 40 | + endforeach() | ||
| 41 | + | ||
| 42 | + set(${result_var} "${_cann_hcc_is_complete}" PARENT_SCOPE) | ||
| 43 | + set(${missing_path_var} "${_cann_hcc_missing_path}" PARENT_SCOPE) | ||
| 44 | +endfunction() | ||
| @@ -0,0 +1,105 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +# 交叉编译 aarch64 版本的 yaml-cpp 静态库, 供 device_arm (bin/device) 链接使用。 | ||
| 12 | +# 复用 host 版 (yaml-cpp.cmake) 的同一份源码包 third_party/yaml-cpp-0.8.0.tar.gz, | ||
| 13 | +# 但使用 CANN HCC 工具链 (cmake/toolchains/aarch64-cann-hcc.cmake) 交叉编译, | ||
| 14 | +# 产物安装到 third_party/yaml-cpp-aarch64/, 与 host 版 (third_party/yaml-cpp/) 物理隔离。 | ||
| 15 | +# | ||
| 16 | +# 调用前提: 顶层 CMakeLists.txt 的 BUILD_DEVICE_ARM 分支已校验 CANN_HCC_ROOT 并可定位 toolchain file。 | ||
| 17 | + | ||
| 18 | +include_guard(GLOBAL) | ||
| 19 | + | ||
| 20 | +# 装载点与源码包 (与 host 版 yaml-cpp.cmake 保持同一份 tarball, 离线可用) | ||
| 21 | +set(YAMLCPP_AARCH64_FILE "yaml-cpp-0.8.0.tar.gz") | ||
| 22 | +set(YAMLCPP_AARCH64_PKG_PATH ${CMAKE_SOURCE_DIR}/third_party/${YAMLCPP_AARCH64_FILE}) | ||
| 23 | +set(YAMLCPP_AARCH64_INSTALL_PATH ${CMAKE_SOURCE_DIR}/third_party/yaml-cpp-aarch64) | ||
| 24 | +set(YAMLCPP_AARCH64_INCLUDE_DIR ${YAMLCPP_AARCH64_INSTALL_PATH}/include) | ||
| 25 | + | ||
| 26 | +# 优先使用 lib64, 兼容 lib | ||
| 27 | +set(YAMLCPP_AARCH64_LIB_DIR "") | ||
| 28 | +if(EXISTS "${YAMLCPP_AARCH64_INSTALL_PATH}/lib64/libyaml-cpp.a") | ||
| 29 | + set(YAMLCPP_AARCH64_LIB_DIR "${YAMLCPP_AARCH64_INSTALL_PATH}/lib64") | ||
| 30 | +elseif(EXISTS "${YAMLCPP_AARCH64_INSTALL_PATH}/lib/libyaml-cpp.a") | ||
| 31 | + set(YAMLCPP_AARCH64_LIB_DIR "${YAMLCPP_AARCH64_INSTALL_PATH}/lib") | ||
| 32 | +endif() | ||
| 33 | + | ||
| 34 | +message(STATUS "[ThirdParty][aarch64] YAMLCPP_AARCH64_INSTALL_PATH=${YAMLCPP_AARCH64_INSTALL_PATH}") | ||
| 35 | +message(STATUS "[ThirdParty][aarch64] YAMLCPP_AARCH64_LIB_DIR=${YAMLCPP_AARCH64_LIB_DIR}") | ||
| 36 | + | ||
| 37 | +# 增量缓存: 头文件 + 静态库均已存在则跳过交叉编译 | ||
| 38 | +if(YAMLCPP_AARCH64_LIB_DIR AND EXISTS "${YAMLCPP_AARCH64_INCLUDE_DIR}/yaml-cpp/yaml.h") | ||
| 39 | + message(STATUS "[ThirdParty][aarch64] yaml-cpp (aarch64) found in ${YAMLCPP_AARCH64_INSTALL_PATH}") | ||
| 40 | +else() | ||
| 41 | + # 与 host 版 yaml-cpp.cmake 对齐: 本地 tarball 缺失时回退在线下载 (下载在 build 阶段执行) | ||
| 42 | + if(EXISTS ${YAMLCPP_AARCH64_PKG_PATH}) | ||
| 43 | + message(STATUS "[ThirdParty][aarch64] Found local yaml-cpp package: ${YAMLCPP_AARCH64_PKG_PATH}") | ||
| 44 | + set(YAMLCPP_AARCH64_PROJECT_URL ${YAMLCPP_AARCH64_PKG_PATH}) | ||
| 45 | + else() | ||
| 46 | + if(NOT DEFINED YAMLCPP_URL) | ||
| 47 | + set(YAMLCPP_URL "https://raw.gitcode.com/src-openeuler/yaml-cpp/blobs/d1ead4fff417073b9cdbf98b8b55eb0efc00b0ba/yaml-cpp-0.8.0.tar.gz") | ||
| 48 | + endif() | ||
| 49 | + message(STATUS "[ThirdParty][aarch64] Downloading yaml-cpp from ${YAMLCPP_URL}") | ||
| 50 | + set(YAMLCPP_AARCH64_PROJECT_URL ${YAMLCPP_URL}) | ||
| 51 | + endif() | ||
| 52 | + message(STATUS "[ThirdParty][aarch64] yaml-cpp (aarch64) not found, will cross-build from source") | ||
| 53 | + | ||
| 54 | + # 编译选项与 host 版对齐 (_GLIBCXX_USE_CXX11_ABI=0 必须, 与 device CMakeLists 一致) | ||
| 55 | + set(YAMLCPP_AARCH64_CXXFLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -O2 -D_FORTIFY_SOURCE=2 -fPIC -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") | ||
| 56 | + set(YAMLCPP_AARCH64_CFLAGS "-D_GLIBCXX_USE_CXX11_ABI=0 -O2 -D_FORTIFY_SOURCE=2 -fPIC -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") | ||
| 57 | + | ||
| 58 | + set(YAMLCPP_AARCH64_TOOLCHAIN_FILE ${CMAKE_SOURCE_DIR}/cmake/toolchains/aarch64-cann-hcc.cmake) | ||
| 59 | + | ||
| 60 | + set(YAMLCPP_AARCH64_OPTS | ||
| 61 | + -DCMAKE_TOOLCHAIN_FILE=${YAMLCPP_AARCH64_TOOLCHAIN_FILE} | ||
| 62 | + -DCANN_HCC_ROOT=${CANN_HCC_ROOT} | ||
| 63 | + -DCMAKE_CXX_FLAGS=${YAMLCPP_AARCH64_CXXFLAGS} | ||
| 64 | + -DCMAKE_C_FLAGS=${YAMLCPP_AARCH64_CFLAGS} | ||
| 65 | + -DCMAKE_INSTALL_PREFIX=${YAMLCPP_AARCH64_INSTALL_PATH} | ||
| 66 | + -DCMAKE_INSTALL_LIBDIR=lib64 | ||
| 67 | + -DCMAKE_CXX_STANDARD=17 | ||
| 68 | + -DYAML_CPP_BUILD_TESTS=OFF | ||
| 69 | + -DYAML_CPP_BUILD_TOOLS=OFF | ||
| 70 | + -DYAML_CPP_BUILD_CONTRIB=OFF | ||
| 71 | + -DBUILD_SHARED_LIBS=OFF | ||
| 72 | + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 | ||
| 73 | + ) | ||
| 74 | + | ||
| 75 | + # DOWNLOAD_EXTRACT_TIMESTAMP 为 CMake 3.24+ 引入的选项(配套策略 CMP0135), | ||
| 76 | + # 旧版本(如3.22)不识别该关键字, 会将其与TRUE误解析为URL的值, 导致报错: | ||
| 77 | + # "At least one entry of URL is a path (invalid in a list)", 故仅在高版本传入。 | ||
| 78 | + # (新旧版本的默认行为一致: 提取时更新源码时间戳) | ||
| 79 | + set(YAMLCPP_AARCH64_DOWNLOAD_OPTS "") | ||
| 80 | + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24) | ||
| 81 | + list(APPEND YAMLCPP_AARCH64_DOWNLOAD_OPTS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) | ||
| 82 | + endif() | ||
| 83 | + | ||
| 84 | + include(ExternalProject) | ||
| 85 | + ExternalProject_Add(third_party_yaml_cpp_aarch64 | ||
| 86 | + URL ${YAMLCPP_AARCH64_PROJECT_URL} | ||
| 87 | + ${YAMLCPP_AARCH64_DOWNLOAD_OPTS} | ||
| 88 | + DOWNLOAD_DIR ${CMAKE_SOURCE_DIR}/third_party | ||
| 89 | + DOWNLOAD_NO_PROGRESS TRUE | ||
| 90 | + CONFIGURE_COMMAND ${CMAKE_COMMAND} ${YAMLCPP_AARCH64_OPTS} <SOURCE_DIR> | ||
| 91 | + BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} | ||
| 92 | + INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install | ||
| 93 | + EXCLUDE_FROM_ALL TRUE | ||
| 94 | + ) | ||
| 95 | + set(YAMLCPP_AARCH64_LIB_DIR "${YAMLCPP_AARCH64_INSTALL_PATH}/lib64") | ||
| 96 | +endif() | ||
| 97 | + | ||
| 98 | +if(NOT EXISTS ${YAMLCPP_AARCH64_INCLUDE_DIR}) | ||
| 99 | + file(MAKE_DIRECTORY "${YAMLCPP_AARCH64_INCLUDE_DIR}") | ||
| 100 | +endif() | ||
| 101 | + | ||
| 102 | +# 暴露给 device_arm 子工程使用的安装路径 (经顶层 CMakeLists 的 CMAKE_ARGS 透传) | ||
| 103 | +set(YAMLCPP_AARCH64_INSTALL_PATH ${YAMLCPP_AARCH64_INSTALL_PATH} CACHE INTERNAL "aarch64 yaml-cpp install path") | ||
| 104 | +set(YAMLCPP_AARCH64_INCLUDE_DIR ${YAMLCPP_AARCH64_INCLUDE_DIR} CACHE INTERNAL "aarch64 yaml-cpp include dir") | ||
| 105 | +set(YAMLCPP_AARCH64_LIB_DIR ${YAMLCPP_AARCH64_LIB_DIR} CACHE INTERNAL "aarch64 yaml-cpp lib dir") | ||
| @@ -61,10 +61,19 @@ if(NOT yaml_cpp_FOUND) | |||
| 61 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 | 61 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 |
| 62 | ) | 62 | ) |
| 63 | 63 | ||
| 64 | + # DOWNLOAD_EXTRACT_TIMESTAMP 为 CMake 3.24+ 引入的选项(配套策略 CMP0135), | ||
| 65 | + # 旧版本(如3.22)不识别该关键字, 会将其与TRUE误解析为URL的值, 导致报错: | ||
| 66 | + # "At least one entry of URL is a path (invalid in a list)", 故仅在高版本传入。 | ||
| 67 | + # (新旧版本的默认行为一致: 提取时更新源码时间戳) | ||
| 68 | + set(YAMLCPP_DOWNLOAD_OPTS "") | ||
| 69 | + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24) | ||
| 70 | + list(APPEND YAMLCPP_DOWNLOAD_OPTS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) | ||
| 71 | + endif() | ||
| 72 | + | ||
| 64 | include(ExternalProject) | 73 | include(ExternalProject) |
| 65 | ExternalProject_Add(third_party_yaml_cpp | 74 | ExternalProject_Add(third_party_yaml_cpp |
| 66 | URL ${YAMLCPP_PROJECT_URL} | 75 | URL ${YAMLCPP_PROJECT_URL} |
| 67 | - DOWNLOAD_EXTRACT_TIMESTAMP TRUE | 76 | + ${YAMLCPP_DOWNLOAD_OPTS} |
| 68 | DOWNLOAD_DIR ${CMAKE_SOURCE_DIR}/third_party | 77 | DOWNLOAD_DIR ${CMAKE_SOURCE_DIR}/third_party |
| 69 | DOWNLOAD_NO_PROGRESS TRUE | 78 | DOWNLOAD_NO_PROGRESS TRUE |
| 70 | CONFIGURE_COMMAND ${CMAKE_COMMAND} ${YAMLCPP_OPTS} <SOURCE_DIR> | 79 | CONFIGURE_COMMAND ${CMAKE_COMMAND} ${YAMLCPP_OPTS} <SOURCE_DIR> |
| @@ -0,0 +1,47 @@ | |||
| 1 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ----------------------------------------------------------------------------------------------------------- | ||
| 10 | + | ||
| 11 | +set(CMAKE_SYSTEM_NAME Linux) | ||
| 12 | +set(CMAKE_SYSTEM_PROCESSOR aarch64) | ||
| 13 | + | ||
| 14 | +if(NOT DEFINED CANN_HCC_ROOT OR CANN_HCC_ROOT STREQUAL "") | ||
| 15 | + message(FATAL_ERROR "CANN_HCC_ROOT is not set") | ||
| 16 | +endif() | ||
| 17 | +set(CANN_HCC_ROOT "${CANN_HCC_ROOT}" CACHE PATH "CANN HCC toolchain root" FORCE) | ||
| 18 | +list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CANN_HCC_ROOT) | ||
| 19 | + | ||
| 20 | +set(_cann_hcc_target aarch64-target-linux-gnu) | ||
| 21 | +set(_cann_hcc_bin "${CANN_HCC_ROOT}/bin") | ||
| 22 | +set(_cann_hcc_sysroot "${CANN_HCC_ROOT}/sysroot") | ||
| 23 | + | ||
| 24 | +include("${CMAKE_CURRENT_LIST_DIR}/../cann_hcc_validation.cmake") | ||
| 25 | +hcclvm_validate_cann_hcc("${CANN_HCC_ROOT}" _cann_hcc_is_complete _cann_hcc_missing_path) | ||
| 26 | +if(NOT _cann_hcc_is_complete) | ||
| 27 | + message(FATAL_ERROR "Incomplete CANN HCC toolchain: ${_cann_hcc_missing_path} was not found") | ||
| 28 | +endif() | ||
| 29 | + | ||
| 30 | +set(CMAKE_C_COMPILER "${_cann_hcc_bin}/${_cann_hcc_target}-gcc" CACHE FILEPATH "CANN HCC C compiler" FORCE) | ||
| 31 | +set(CMAKE_CXX_COMPILER "${_cann_hcc_bin}/${_cann_hcc_target}-g++" CACHE FILEPATH "CANN HCC C++ compiler" FORCE) | ||
| 32 | +set(CMAKE_LINKER "${_cann_hcc_bin}/${_cann_hcc_target}-ld" CACHE FILEPATH "CANN HCC linker" FORCE) | ||
| 33 | +set(CMAKE_AR "${_cann_hcc_bin}/${_cann_hcc_target}-ar" CACHE FILEPATH "CANN HCC archiver" FORCE) | ||
| 34 | +set(CMAKE_RANLIB "${_cann_hcc_bin}/${_cann_hcc_target}-ranlib" CACHE FILEPATH "CANN HCC ranlib" FORCE) | ||
| 35 | +set(CMAKE_STRIP "${_cann_hcc_bin}/${_cann_hcc_target}-strip" CACHE FILEPATH "CANN HCC strip" FORCE) | ||
| 36 | +set(CMAKE_LD "${_cann_hcc_bin}/${_cann_hcc_target}-ld" CACHE FILEPATH "CANN HCC ld" FORCE) | ||
| 37 | +set(CMAKE_NM "${_cann_hcc_bin}/${_cann_hcc_target}-nm" CACHE FILEPATH "CANN HCC nm" FORCE) | ||
| 38 | +set(CMAKE_OBJCOPY "${_cann_hcc_bin}/${_cann_hcc_target}-objcopy" CACHE FILEPATH "CANN HCC objcopy" FORCE) | ||
| 39 | + | ||
| 40 | +set(CMAKE_SYSROOT "${_cann_hcc_sysroot}" CACHE PATH "CANN HCC sysroot" FORCE) | ||
| 41 | +set(CMAKE_FIND_ROOT_PATH "${_cann_hcc_sysroot}" CACHE PATH "CANN HCC target search root" FORCE) | ||
| 42 | +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) | ||
| 43 | +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) | ||
| 44 | +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) | ||
| 45 | +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) | ||
| 46 | + | ||
| 47 | +set(CMAKE_SKIP_RPATH TRUE) | ||
| @@ -8,6 +8,14 @@ | |||
| 8 | # See LICENSE in the root of the software repository for the full text of the License. | 8 | # See LICENSE in the root of the software repository for the full text of the License. |
| 9 | # ----------------------------------------------------------------------------------------------------------- | 9 | # ----------------------------------------------------------------------------------------------------------- |
| 10 | 10 | ||
| 11 | +# You may not use this file except in compliance with the License. | ||
| 12 | +# You may obtain a copy of the License at | ||
| 13 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 18 | + | ||
| 11 | hccl_vm: | 19 | hccl_vm: |
| 12 | console_level: 2 | 20 | console_level: 2 |
| 13 | file_level: 1 | 21 | file_level: 1 |
| @@ -47,3 +55,15 @@ checker: | |||
| 47 | file_base_name: app_log | 55 | file_base_name: app_log |
| 48 | file_suffix: .log | 56 | file_suffix: .log |
| 49 | enable_compress: false | 57 | enable_compress: false |
| 58 | + | ||
| 59 | +# 拓扑生成脚本(generate_cluster_topo.sh/generate_server_topo.sh/allocate_eid.sh)日志配置 | ||
| 60 | +# console_level: 脚本控制台日志级别, 与C++日志级别定义一致: | ||
| 61 | +# 0-trace 1-debug 2-info 3-warn 4-error 5-critical 6-off | ||
| 62 | +# debug(1): 输出全部日志(含逐server/逐die/逐端口明细) | ||
| 63 | +# info(2): 输出步骤与汇总信息(默认) | ||
| 64 | +# warn(3): 仅输出警告与错误 | ||
| 65 | +# error(4): 仅输出错误 | ||
| 66 | +# off(6): 不输出任何日志 | ||
| 67 | +script: | ||
| 68 | + console_level: 2 | ||
| 69 | + | ||
| @@ -7,7 +7,6 @@ | |||
| 7 | ## 1 Checker 是什么 | 7 | ## 1 Checker 是什么 |
| 8 | 8 | ||
| 9 | Checker 是一个静态验证工具，它不介入算子运行，而是通过读取算子执行后留下的记录，重建执行图并进行静态分析，判断本次算子执行在逻辑上是否正确。 | 9 | Checker 是一个静态验证工具，它不介入算子运行，而是通过读取算子执行后留下的记录，重建执行图并进行静态分析，判断本次算子执行在逻辑上是否正确。 |
| 10 | - | ||
| 11 | - Checker 输入：算子信息 + 各 Rank 的任务数据 + CCU指令序列 | 10 | - Checker 输入：算子信息 + 各 Rank 的任务数据 + CCU指令序列 |
| 12 | - Checker 输出：校验结论(成功/失败) + 错误日志 | 11 | - Checker 输出：校验结论(成功/失败) + 错误日志 |
| 13 | 12 | ||
| @@ -50,7 +49,6 @@ flowchart TD | |||
| 50 | --- | 49 | --- |
| 51 | 50 | ||
| 52 | ## 3 任务图 | 51 | ## 3 任务图 |
| 53 | - | ||
| 54 | 任务图是 Checker 的核心数据结构，它用来表达一个算子执行所生成的任务节点及其依赖关系，任务图是在成图阶段基于 Checker 输入生成的 | 52 | 任务图是 Checker 的核心数据结构，它用来表达一个算子执行所生成的任务节点及其依赖关系，任务图是在成图阶段基于 Checker 输入生成的 |
| 55 | 53 | ||
| 56 | ### 3.1 节点与边 | 54 | ### 3.1 节点与边 |
| @@ -101,14 +99,13 @@ flowchart LR | |||
| 101 | ### 3.2 任务图示例 | 99 | ### 3.2 任务图示例 |
| 102 | 100 | ||
| 103 | #### 3.2.1 AICPU模式 | 101 | #### 3.2.1 AICPU模式 |
| 104 | - | ||
| 105 | AICPU 模式下，任务图通常由 `RECORD`、`WAIT` 和 `TRANS_MEM` 这几类节点组合而成。下面给出一个典型的 2-rank、每个 rank 含两条 stream 的 `AllGather` 示例，按实际执行序列展示 stream 内顺序边，并用虚线表示 `RECORD` 到 `WAIT` 的同步依赖。 | 102 | AICPU 模式下，任务图通常由 `RECORD`、`WAIT` 和 `TRANS_MEM` 这几类节点组合而成。下面给出一个典型的 2-rank、每个 rank 含两条 stream 的 `AllGather` 示例，按实际执行序列展示 stream 内顺序边，并用虚线表示 `RECORD` 到 `WAIT` 的同步依赖。 |
| 106 | 103 | ||
| 107 | ```mermaid | 104 | ```mermaid |
| 108 | flowchart LR | 105 | flowchart LR |
| 109 | subgraph R0["rank0"] | 106 | subgraph R0["rank0"] |
| 110 | direction TB | 107 | direction TB |
| 111 | - subgraph R0S0["stream0 主流"] | 108 | + subgraph R0S0["算子主流（streamId 由 opDetails.streamId 指定）"] |
| 112 | direction LR | 109 | direction LR |
| 113 | R0REC0["RECORD\nnotifyId=101"] --> R0WAI1S0["WAIT\nnotifyId=101"] | 110 | R0REC0["RECORD\nnotifyId=101"] --> R0WAI1S0["WAIT\nnotifyId=101"] |
| 114 | end | 111 | end |
| @@ -120,7 +117,7 @@ flowchart LR | |||
| 120 | 117 | ||
| 121 | subgraph R1["rank1"] | 118 | subgraph R1["rank1"] |
| 122 | direction TB | 119 | direction TB |
| 123 | - subgraph R1S0["stream0 主流"] | 120 | + subgraph R1S0["算子主流（streamId 由 opDetails.streamId 指定）"] |
| 124 | direction LR | 121 | direction LR |
| 125 | R1REC6["RECORD\nnotifyId=106"] --> R1WAI7["WAIT\nnotifyId=107"] | 122 | R1REC6["RECORD\nnotifyId=106"] --> R1WAI7["WAIT\nnotifyId=107"] |
| 126 | end | 123 | end |
| @@ -141,7 +138,6 @@ flowchart LR | |||
| 141 | ``` | 138 | ``` |
| 142 | 139 | ||
| 143 | #### 3.2.2 CCU模式 | 140 | #### 3.2.2 CCU模式 |
| 144 | - | ||
| 145 | 在 CCU 模式下，Checker 会把 CCU 指令展开成 CCU 子图。下面沿用 2-rank `AllReduce` 的数据流，省略了 CCU 子图外部的同步操作，只保留 CCU 子图内部的任务序列。CCU 使用同步字段从 `notifyId` 变为 `cke` / `mask`，中间缓冲区使用的是 CCU 的 `MS` 类型。 | 141 | 在 CCU 模式下，Checker 会把 CCU 指令展开成 CCU 子图。下面沿用 2-rank `AllReduce` 的数据流，省略了 CCU 子图外部的同步操作，只保留 CCU 子图内部的任务序列。CCU 使用同步字段从 `notifyId` 变为 `cke` / `mask`，中间缓冲区使用的是 CCU 的 `MS` 类型。 |
| 146 | 142 | ||
| 147 | ```mermaid | 143 | ```mermaid |
| @@ -169,57 +165,51 @@ flowchart LR | |||
| 169 | ``` | 165 | ``` |
| 170 | 166 | ||
| 171 | #### 3.2.3 Graphviz 简易可视化 | 167 | #### 3.2.3 Graphviz 简易可视化 |
| 172 | - | ||
| 173 | Checker 提供了任务图导出能力，用于把任务图输出成 Graphviz 的 `.dot` 文件。 | 168 | Checker 提供了任务图导出能力，用于把任务图输出成 Graphviz 的 `.dot` 文件。 |
| 174 | 169 | ||
| 175 | 它导出的内容不只是“有哪些节点”，还会尽量把调试时常用的信息直接放进图里： | 170 | 它导出的内容不只是“有哪些节点”，还会尽量把调试时常用的信息直接放进图里： |
| 176 | - | ||
| 177 | - 按 `rank / stream` 排布节点，便于观察同一执行队列上的顺序关系 | 171 | - 按 `rank / stream` 排布节点，便于观察同一执行队列上的顺序关系 |
| 178 | - 用实线表示普通依赖边，用虚线表示 `RECORD -> WAIT` 这类同步依赖边 | 172 | - 用实线表示普通依赖边，用虚线表示 `RECORD -> WAIT` 这类同步依赖边 |
| 179 | - 节点标签中会带上任务类型、`nodeId`、位置信息，以及内存片段、`notifyId` 或 `cke/mask` 等关键字段 | 173 | - 节点标签中会带上任务类型、`nodeId`、位置信息，以及内存片段、`notifyId` 或 `cke/mask` 等关键字段 |
| 180 | 174 | ||
| 181 | 使用方式如下： | 175 | 使用方式如下： |
| 182 | - | ||
| 183 | - Checker 在成图完成后会自动尝试导出该 `.dot` 文件，不需要额外开关 | 176 | - Checker 在成图完成后会自动尝试导出该 `.dot` 文件，不需要额外开关 |
| 184 | - 导出成功后，可在日志中搜索 `[GraphvizDot]`，日志里会打印最终落盘路径，通常为`hccl_vm_install/data/` | 177 | - 导出成功后，可在日志中搜索 `[GraphvizDot]`，日志里会打印最终落盘路径，通常为`hccl_vm_install/data/` |
| 185 | - 输出文件名格式为 `TaskGraph_YYYYMMDDHHMMSS.dot` | 178 | - 输出文件名格式为 `TaskGraph_YYYYMMDDHHMMSS.dot` |
| 186 | 179 | ||
| 187 | > 拿到 `.dot` 文件后，可使用 `Microsoft VS Code` 相关的插件如 `Graphviz Interactive Preview` 来实现即时浏览。 | 180 | > 拿到 `.dot` 文件后，可使用 `Microsoft VS Code` 相关的插件如 `Graphviz Interactive Preview` 来实现即时浏览。 |
| 188 | 181 | ||
| 182 | + | ||
| 189 | --- | 183 | --- |
| 190 | 184 | ||
| 191 | ## 4. 单任务校验 | 185 | ## 4. 单任务校验 |
| 192 | - | ||
| 193 | 此阶段主要检查单条任务的内存区间是否合法，并校验从流的头尾结构 | 186 | 此阶段主要检查单条任务的内存区间是否合法，并校验从流的头尾结构 |
| 194 | 187 | ||
| 195 | ### 4.1 内存区间 (MemSlice) | 188 | ### 4.1 内存区间 (MemSlice) |
| 196 | 189 | ||
| 197 | 内存搬运与规约类的任务中最重要的信息就是内存区间 (MemSlice)，一个内存区间由以下信息组成： | 190 | 内存搬运与规约类的任务中最重要的信息就是内存区间 (MemSlice)，一个内存区间由以下信息组成： |
| 198 | 191 | ||
| 199 | -```text | 192 | +``` |
| 200 | MemSlice = { rankId, type, offset, len } | 193 | MemSlice = { rankId, type, offset, len } |
| 201 | ``` | 194 | ``` |
| 202 | 195 | ||
| 203 | - `rankId` 表示内存属于哪个 rank | 196 | - `rankId` 表示内存属于哪个 rank |
| 204 | - `type` 表示内存类型 | 197 | - `type` 表示内存类型 |
| 205 | - | ||
| 206 | | 内存类型 | 用途 | | 198 | | 内存类型 | 用途 | |
| 207 | |----------|------| | 199 | |----------|------| |
| 208 | | INPUT | 算子输入BUFFER | | 200 | | INPUT | 算子输入BUFFER | |
| 209 | | OUTPUT | 算子输出BUFFER | | 201 | | OUTPUT | 算子输出BUFFER | |
| 210 | | CCL | CCL_BUFFER | | 202 | | CCL | CCL_BUFFER | |
| 211 | | MS_CCU | CCU MS | | 203 | | MS_CCU | CCU MS | |
| 212 | - | ||
| 213 | - `offset` 和 `len` 共同确定内存访问区间 | 204 | - `offset` 和 `len` 共同确定内存访问区间 |
| 214 | - `offset` 表示本次访问在该内存片段上的起始地址 | 205 | - `offset` 表示本次访问在该内存片段上的起始地址 |
| 215 | - `len` 表示本次内存访问的长度 | 206 | - `len` 表示本次内存访问的长度 |
| 216 | - 访问区间采用半开表示，即 `[offset, offset + length)` 为本次访问的内存段 | 207 | - 访问区间采用半开表示，即 `[offset, offset + length)` 为本次访问的内存段 |
| 217 | 208 | ||
| 209 | + | ||
| 218 | 单条任务内存区间的校验点如下： | 210 | 单条任务内存区间的校验点如下： |
| 219 | - | 211 | +- `offset + length` 不能溢出 `uint64` 上界，否则会报错 |
| 220 | -- `offset + length` 不能溢出 `uint64` 上界，否则会报错。 | 212 | +- 同一任务内部的多个 MemSlice 在相同 `(rankId, memType)` 下不能有区间重叠，否则会报错 |
| 221 | -- 同一任务内部的多个 MemSlice 在相同 `(rankId, memType)` 下不能有区间重叠，否则会报错。 | ||
| 222 | - | ||
| 223 | ```mermaid | 213 | ```mermaid |
| 224 | gantt | 214 | gantt |
| 225 | title MemSlice 区间对比 | 215 | title MemSlice 区间对比 |
| @@ -235,10 +225,8 @@ MemSlice = { rankId, type, offset, len } | |||
| 235 | Slice A 0x000-0x600 : crit, 0, 600 | 225 | Slice A 0x000-0x600 : crit, 0, 600 |
| 236 | Slice B 0x400-0x800 : crit, 400, 800 | 226 | Slice B 0x400-0x800 : crit, 400, 800 |
| 237 | ``` | 227 | ``` |
| 238 | - | 228 | +- 不同 `type` 是相互独立的地址空间，同一 `offset` 在不同 `type` 下不视为重叠 |
| 239 | -- 不同 `type` 是相互独立的地址空间，同一 `offset` 在不同 `type` 下不视为重叠。 | 229 | +- `offset + length` 不能越过当前 `type` 地址空间的边界 |
| 240 | -- `offset + length` 不能越过当前 `type` 地址空间的边界。 | ||
| 241 | - | ||
| 242 | ```mermaid | 230 | ```mermaid |
| 243 | gantt | 231 | gantt |
| 244 | title MemSlice 边界检查 | 232 | title MemSlice 边界检查 |
| @@ -257,15 +245,14 @@ MemSlice = { rankId, type, offset, len } | |||
| 257 | 245 | ||
| 258 | ### 4.2 从流（slave stream）结构校验 | 246 | ### 4.2 从流（slave stream）结构校验 |
| 259 | 247 | ||
| 260 | -从流用于执行算子的辅助任务，例如数据预搬运。在HCCL编程模型中主流通过同步任务 `RECORD -> WAIT` 触发从流，从流完成后，再通过另一组同步任务通知主流，所以从流必须满足固定的首尾结构：首任务为 `WAIT` && 末任务为 `RECORD`。 | 248 | +从流用于执行算子的辅助任务，例如数据预搬运。当前算子的主流由 `opDetails.streamId` 指定，不再假定为 `stream0`。在HCCL编程模型中主流通过同步任务 `RECORD -> WAIT` 触发从流，从流完成后，再通过另一组同步任务通知主流，所以从流必须满足固定的首尾结构：首任务为 `WAIT` && 末任务为 `RECORD` |
| 261 | 249 | ||
| 262 | 下图为一个错误示例，标红的节点都是违规节点： | 250 | 下图为一个错误示例，标红的节点都是违规节点： |
| 263 | - | ||
| 264 | ```mermaid | 251 | ```mermaid |
| 265 | flowchart LR | 252 | flowchart LR |
| 266 | START(["start"]) | 253 | START(["start"]) |
| 267 | 254 | ||
| 268 | - subgraph MAIN["rank0 / stream0（主流）"] | 255 | + subgraph MAIN["rank0 / 算子主流（streamId 由 opDetails.streamId 指定）"] |
| 269 | M0["TRANS_MEM"] | 256 | M0["TRANS_MEM"] |
| 270 | M1["RECORD\nnotifyId=10\n触发从流"] | 257 | M1["RECORD\nnotifyId=10\n触发从流"] |
| 271 | M2["WAIT\nnotifyId=11\n等待从流完成"] | 258 | M2["WAIT\nnotifyId=11\n等待从流完成"] |
| @@ -297,7 +284,6 @@ flowchart LR | |||
| 297 | ### 5.1 内存冲突的判定标准 | 284 | ### 5.1 内存冲突的判定标准 |
| 298 | 285 | ||
| 299 | 两个内存访问类任务节点同时满足以下三个条件时，会判定为内存冲突： | 286 | 两个内存访问类任务节点同时满足以下三个条件时，会判定为内存冲突： |
| 300 | - | ||
| 301 | 1. 两个节点可能并发执行 (任务图上两个节点之间不存在路径) | 287 | 1. 两个节点可能并发执行 (任务图上两个节点之间不存在路径) |
| 302 | 2. 访问的内存地址区间重叠 | 288 | 2. 访问的内存地址区间重叠 |
| 303 | 3. 至少一方是写操作 | 289 | 3. 至少一方是写操作 |
| @@ -305,7 +291,6 @@ flowchart LR | |||
| 305 | Checker会高效地校验每一对内存访问类任务节点，保证不发生漏报 | 291 | Checker会高效地校验每一对内存访问类任务节点，保证不发生漏报 |
| 306 | 292 | ||
| 307 | ### 5.2 内存冲突示例 | 293 | ### 5.2 内存冲突示例 |
| 308 | - | ||
| 309 | 下图为一个存在内存冲突的任务图示例： | 294 | 下图为一个存在内存冲突的任务图示例： |
| 310 | 295 | ||
| 311 | ```mermaid | 296 | ```mermaid |
| @@ -366,7 +351,6 @@ flowchart LR | |||
| 366 | | `node X, action=read/write` | 任务图中的节点 ID 与本次访问的读写类型。只要两条访问中至少一条是 `write`，就可能报冲突 | | 351 | | `node X, action=read/write` | 任务图中的节点 ID 与本次访问的读写类型。只要两条访问中至少一条是 `write`，就可能报冲突 | |
| 367 | | `access range : [start,end)` | 这条访问自身覆盖的完整地址区间，不一定与 `Overlap range` 完全相同 | | 352 | | `access range : [start,end)` | 这条访问自身覆盖的完整地址区间，不一定与 `Overlap range` 完全相同 | |
| 368 | | `task :` | 具体任务详情（任务类型、节点 ID、位置、src/dst 内存区间等） | | 353 | | `task :` | 具体任务详情（任务类型、节点 ID、位置、src/dst 内存区间等） | |
| 369 | - | ||
| 370 | --- | 354 | --- |
| 371 | 355 | ||
| 372 | ## 6. 语义校验 | 356 | ## 6. 语义校验 |
| @@ -377,7 +361,7 @@ flowchart LR | |||
| 377 | 361 | ||
| 378 | 语义校验过程中，Checker 会为每段内存维护数据来源记录： | 362 | 语义校验过程中，Checker 会为每段内存维护数据来源记录： |
| 379 | 363 | ||
| 380 | -```text | 364 | +``` |
| 381 | BufferSemantic = { | 365 | BufferSemantic = { |
| 382 | startAddr: 内存区间起始地址（offset） | 366 | startAddr: 内存区间起始地址（offset） |
| 383 | size: 内存区间长度（len） | 367 | size: 内存区间长度（len） |
| @@ -618,7 +602,7 @@ flowchart LR | |||
| 618 | 602 | ||
| 619 | 以 4-rank `AllReduce` 为例： | 603 | 以 4-rank `AllReduce` 为例： |
| 620 | 604 | ||
| 621 | -```text | 605 | +``` |
| 622 | rank0.OUTPUT[0,L) 期望： | 606 | rank0.OUTPUT[0,L) 期望： |
| 623 | sources = { rank0.INPUT, rank1.INPUT, rank2.INPUT, rank3.INPUT } | 607 | sources = { rank0.INPUT, rank1.INPUT, rank2.INPUT, rank3.INPUT } |
| 624 | reduceType = SUM | 608 | reduceType = SUM |
| @@ -635,7 +619,7 @@ rank0.OUTPUT[0,L) 期望： | |||
| 635 | 619 | ||
| 636 | 每个 rank 的 INPUT 已有自己的初始语义（来源指向自身）： | 620 | 每个 rank 的 INPUT 已有自己的初始语义（来源指向自身）： |
| 637 | 621 | ||
| 638 | -```text | 622 | +``` |
| 639 | rank0.INPUT[0, 100): srcBufs = { (rank0, INPUT, 0) } | 623 | rank0.INPUT[0, 100): srcBufs = { (rank0, INPUT, 0) } |
| 640 | rank1.INPUT[0, 100): srcBufs = { (rank1, INPUT, 0) } | 624 | rank1.INPUT[0, 100): srcBufs = { (rank1, INPUT, 0) } |
| 641 | rank0.OUTPUT: 空 | 625 | rank0.OUTPUT: 空 |
| @@ -7,7 +7,6 @@ This document introduces the basic concepts and processing flow of Checker. The | |||
| 7 | ## 1 What Is Checker | 7 | ## 1 What Is Checker |
| 8 | 8 | ||
| 9 | Checker is a static verification tool. It does not intervene in operator execution. Instead, it reads the records left after operator execution, rebuilds the execution graph, and performs static analysis to determine whether the operator execution is logically correct. | 9 | Checker is a static verification tool. It does not intervene in operator execution. Instead, it reads the records left after operator execution, rebuilds the execution graph, and performs static analysis to determine whether the operator execution is logically correct. |
| 10 | - | ||
| 11 | - Checker input: operator information + task data from each rank + CCU instruction sequence | 10 | - Checker input: operator information + task data from each rank + CCU instruction sequence |
| 12 | - Checker output: verification result (pass/fail) + error log | 11 | - Checker output: verification result (pass/fail) + error log |
| 13 | 12 | ||
| @@ -50,7 +49,6 @@ flowchart TD | |||
| 50 | --- | 49 | --- |
| 51 | 50 | ||
| 52 | ## 3 Task Graph | 51 | ## 3 Task Graph |
| 53 | - | ||
| 54 | The task graph is the core data structure of Checker. It represents the task nodes generated by an operator execution and their dependency relationships. The task graph is generated during the graph generation phase based on Checker input. | 52 | The task graph is the core data structure of Checker. It represents the task nodes generated by an operator execution and their dependency relationships. The task graph is generated during the graph generation phase based on Checker input. |
| 55 | 53 | ||
| 56 | ### 3.1 Nodes and Edges | 54 | ### 3.1 Nodes and Edges |
| @@ -101,14 +99,13 @@ flowchart LR | |||
| 101 | ### 3.2 Task Graph Samples | 99 | ### 3.2 Task Graph Samples |
| 102 | 100 | ||
| 103 | #### 3.2.1 AICPU Mode | 101 | #### 3.2.1 AICPU Mode |
| 104 | - | ||
| 105 | In AICPU mode, the task graph typically consists of `RECORD`, `WAIT`, and `TRANS_MEM` nodes. Below is a typical 2-rank `AllGather` sample where each rank contains two streams. Sequential edges within each stream follow the actual execution sequence. Dashed lines represent `RECORD` to `WAIT` synchronization dependencies. | 102 | In AICPU mode, the task graph typically consists of `RECORD`, `WAIT`, and `TRANS_MEM` nodes. Below is a typical 2-rank `AllGather` sample where each rank contains two streams. Sequential edges within each stream follow the actual execution sequence. Dashed lines represent `RECORD` to `WAIT` synchronization dependencies. |
| 106 | 103 | ||
| 107 | ```mermaid | 104 | ```mermaid |
| 108 | flowchart LR | 105 | flowchart LR |
| 109 | subgraph R0["rank0"] | 106 | subgraph R0["rank0"] |
| 110 | direction TB | 107 | direction TB |
| 111 | - subgraph R0S0["stream0 Main Stream"] | 108 | + subgraph R0S0["Operator Main Stream (streamId specified by opDetails.streamId)"] |
| 112 | direction LR | 109 | direction LR |
| 113 | R0REC0["RECORD\nnotifyId=101"] --> R0WAI1S0["WAIT\nnotifyId=101"] | 110 | R0REC0["RECORD\nnotifyId=101"] --> R0WAI1S0["WAIT\nnotifyId=101"] |
| 114 | end | 111 | end |
| @@ -120,7 +117,7 @@ flowchart LR | |||
| 120 | 117 | ||
| 121 | subgraph R1["rank1"] | 118 | subgraph R1["rank1"] |
| 122 | direction TB | 119 | direction TB |
| 123 | - subgraph R1S0["stream0 Main Stream"] | 120 | + subgraph R1S0["Operator Main Stream (streamId specified by opDetails.streamId)"] |
| 124 | direction LR | 121 | direction LR |
| 125 | R1REC6["RECORD\nnotifyId=106"] --> R1WAI7["WAIT\nnotifyId=107"] | 122 | R1REC6["RECORD\nnotifyId=106"] --> R1WAI7["WAIT\nnotifyId=107"] |
| 126 | end | 123 | end |
| @@ -141,7 +138,6 @@ flowchart LR | |||
| 141 | ``` | 138 | ``` |
| 142 | 139 | ||
| 143 | #### 3.2.2 CCU Mode | 140 | #### 3.2.2 CCU Mode |
| 144 | - | ||
| 145 | In CCU mode, Checker expands CCU instructions into a CCU subgraph. The sample below follows a 2-rank `AllReduce` data flow. It omits synchronization operations outside the CCU subgraph and retains only the task sequence inside the CCU subgraph. CCU uses synchronization fields `cke` / `mask` instead of `notifyId`. The intermediate buffer uses the CCU `MS` type. | 141 | In CCU mode, Checker expands CCU instructions into a CCU subgraph. The sample below follows a 2-rank `AllReduce` data flow. It omits synchronization operations outside the CCU subgraph and retains only the task sequence inside the CCU subgraph. CCU uses synchronization fields `cke` / `mask` instead of `notifyId`. The intermediate buffer uses the CCU `MS` type. |
| 146 | 142 | ||
| 147 | ```mermaid | 143 | ```mermaid |
| @@ -169,56 +165,50 @@ flowchart LR | |||
| 169 | ``` | 165 | ``` |
| 170 | 166 | ||
| 171 | #### 3.2.3 Simple Graphviz Visualization | 167 | #### 3.2.3 Simple Graphviz Visualization |
| 172 | - | ||
| 173 | Checker provides a task graph export feature that outputs the task graph as a Graphviz `.dot` file. | 168 | Checker provides a task graph export feature that outputs the task graph as a Graphviz `.dot` file. |
| 174 | 169 | ||
| 175 | The exported content includes not only the node list but also debugging information embedded directly into the graph: | 170 | The exported content includes not only the node list but also debugging information embedded directly into the graph: |
| 176 | - | ||
| 177 | - Nodes are arranged by `rank / stream` for easy observation of sequential relationships on the same execution queue | 171 | - Nodes are arranged by `rank / stream` for easy observation of sequential relationships on the same execution queue |
| 178 | - Solid lines represent normal dependency edges. Dashed lines represent `RECORD -> WAIT` synchronization dependency edges | 172 | - Solid lines represent normal dependency edges. Dashed lines represent `RECORD -> WAIT` synchronization dependency edges |
| 179 | - Node labels include the task type, `nodeId`, position information, and key fields such as memory slices, `notifyId`, or `cke/mask` | 173 | - Node labels include the task type, `nodeId`, position information, and key fields such as memory slices, `notifyId`, or `cke/mask` |
| 180 | 174 | ||
| 181 | Usage instructions: | 175 | Usage instructions: |
| 182 | - | ||
| 183 | - Checker automatically exports the `.dot` file after graph generation completes. No extra switch is needed | 176 | - Checker automatically exports the `.dot` file after graph generation completes. No extra switch is needed |
| 184 | - After a successful export, search for `[GraphvizDot]` in the log. The log prints the final output path, typically `hccl_vm_install/data/` | 177 | - After a successful export, search for `[GraphvizDot]` in the log. The log prints the final output path, typically `hccl_vm_install/data/` |
| 185 | - The output file name format is `TaskGraph_YYYYMMDDHHMMSS.dot` | 178 | - The output file name format is `TaskGraph_YYYYMMDDHHMMSS.dot` |
| 186 | 179 | ||
| 187 | > After obtaining the `.dot` file, use a `Microsoft VS Code` plugin such as `Graphviz Interactive Preview` for interactive browsing. | 180 | > After obtaining the `.dot` file, use a `Microsoft VS Code` plugin such as `Graphviz Interactive Preview` for interactive browsing. |
| 188 | 181 | ||
| 182 | + | ||
| 189 | --- | 183 | --- |
| 190 | 184 | ||
| 191 | ## 4. Single Task Check | 185 | ## 4. Single Task Check |
| 192 | - | ||
| 193 | This phase checks whether the memory range of each task is valid and verifies the head-tail structure of slave streams. | 186 | This phase checks whether the memory range of each task is valid and verifies the head-tail structure of slave streams. |
| 194 | 187 | ||
| 195 | ### 4.1 Memory Range (MemSlice) | 188 | ### 4.1 Memory Range (MemSlice) |
| 196 | 189 | ||
| 197 | The most important information in memory transfer and reduction tasks is the memory range (MemSlice). A memory range consists of the following fields: | 190 | The most important information in memory transfer and reduction tasks is the memory range (MemSlice). A memory range consists of the following fields: |
| 198 | 191 | ||
| 199 | -```text | 192 | +``` |
| 200 | MemSlice = { rankId, type, offset, len } | 193 | MemSlice = { rankId, type, offset, len } |
| 201 | ``` | 194 | ``` |
| 202 | 195 | ||
| 203 | - `rankId` indicates which rank the memory belongs to | 196 | - `rankId` indicates which rank the memory belongs to |
| 204 | - `type` indicates the memory type | 197 | - `type` indicates the memory type |
| 205 | - | ||
| 206 | | Memory Type | Purpose | | 198 | | Memory Type | Purpose | |
| 207 | |----------|------| | 199 | |----------|------| |
| 208 | | INPUT | Operator input BUFFER | | 200 | | INPUT | Operator input BUFFER | |
| 209 | | OUTPUT | Operator output BUFFER | | 201 | | OUTPUT | Operator output BUFFER | |
| 210 | | CCL | CCL_BUFFER | | 202 | | CCL | CCL_BUFFER | |
| 211 | | MS_CCU | CCU MS | | 203 | | MS_CCU | CCU MS | |
| 212 | - | ||
| 213 | - `offset` and `len` together define the memory access range | 204 | - `offset` and `len` together define the memory access range |
| 214 | - `offset` indicates the start address of the current access within the memory segment | 205 | - `offset` indicates the start address of the current access within the memory segment |
| 215 | - `len` indicates the length of the current memory access | 206 | - `len` indicates the length of the current memory access |
| 216 | - The access range uses a half-open representation: `[offset, offset + length)` is the memory segment for the current access | 207 | - The access range uses a half-open representation: `[offset, offset + length)` is the memory segment for the current access |
| 217 | 208 | ||
| 218 | The validation points for a single task memory range are as follows: | 209 | The validation points for a single task memory range are as follows: |
| 219 | - | 210 | +- `offset + length` must not overflow the `uint64` upper bound. Otherwise an error is reported |
| 220 | -- `offset + length` must not overflow the `uint64` upper bound. Otherwise an error is reported. | 211 | +- Multiple MemSlice entries within the same task must not have overlapping ranges under the same `(rankId, memType)`. Otherwise an error is reported |
| 221 | -- Multiple MemSlice entries within the same task must not have overlapping ranges under the same `(rankId, memType)`. Otherwise an error is reported. | ||
| 222 | 212 | ||
| 223 | ```mermaid | 213 | ```mermaid |
| 224 | gantt | 214 | gantt |
| @@ -236,8 +226,8 @@ The validation points for a single task memory range are as follows: | |||
| 236 | Slice B 0x400-0x800 : crit, 400, 800 | 226 | Slice B 0x400-0x800 : crit, 400, 800 |
| 237 | ``` | 227 | ``` |
| 238 | 228 | ||
| 239 | -- Different `type` values represent independent address spaces. The same `offset` under different `type` values does not count as overlap. | 229 | +- Different `type` values represent independent address spaces. The same `offset` under different `type` values does not count as overlap |
| 240 | -- `offset + length` must not exceed the boundary of the current `type` address space. | 230 | +- `offset + length` must not exceed the boundary of the current `type` address space |
| 241 | 231 | ||
| 242 | ```mermaid | 232 | ```mermaid |
| 243 | gantt | 233 | gantt |
| @@ -256,16 +246,14 @@ The validation points for a single task memory range are as follows: | |||
| 256 | ``` | 246 | ``` |
| 257 | 247 | ||
| 258 | ### 4.2 Slave Stream Structure Check | 248 | ### 4.2 Slave Stream Structure Check |
| 259 | - | 249 | +A slave stream executes auxiliary tasks for the operator, such as data pre-fetching. The current operator's main stream is specified by `opDetails.streamId`, not assumed to be `stream0`. In the HCCL programming model, the main stream triggers a slave stream through a synchronization task `RECORD -> WAIT`. After the slave stream completes, it notifies the main stream through another synchronization task pair. Therefore the slave stream must satisfy a fixed head-tail structure: the first task is `WAIT` and the last task is `RECORD` |
| 260 | -A slave stream executes auxiliary tasks for the operator, such as data pre-fetching. In the HCCL programming model, the main stream triggers a slave stream through a synchronization task `RECORD -> WAIT`. After the slave stream completes, it notifies the main stream through another synchronization task pair. Therefore the slave stream must satisfy a fixed head-tail structure: the first task is `WAIT` and the last task is `RECORD`. | ||
| 261 | 250 | ||
| 262 | The diagram below shows an error sample. The highlighted nodes are violation nodes: | 251 | The diagram below shows an error sample. The highlighted nodes are violation nodes: |
| 263 | - | ||
| 264 | ```mermaid | 252 | ```mermaid |
| 265 | flowchart LR | 253 | flowchart LR |
| 266 | START(["start"]) | 254 | START(["start"]) |
| 267 | 255 | ||
| 268 | - subgraph MAIN["rank0 / stream0 (main stream)"] | 256 | + subgraph MAIN["rank0 / Operator Main Stream (streamId specified by opDetails.streamId)"] |
| 269 | M0["TRANS_MEM"] | 257 | M0["TRANS_MEM"] |
| 270 | M1["RECORD\nnotifyId=10\ntrigger slave stream"] | 258 | M1["RECORD\nnotifyId=10\ntrigger slave stream"] |
| 271 | M2["WAIT\nnotifyId=11\nwait for slave stream"] | 259 | M2["WAIT\nnotifyId=11\nwait for slave stream"] |
| @@ -293,7 +281,6 @@ flowchart LR | |||
| 293 | ## 5. Memory Conflict Check | 281 | ## 5. Memory Conflict Check |
| 294 | 282 | ||
| 295 | This phase checks whether the task graph contains possible memory conflicts. A memory conflict occurs when multiple memory operations access the same memory segment at the same time and at least one operation is a write. When a memory conflict occurs, the value of the conflicting memory segment is nondeterministic. This causes precision issues in collective communication operators. | 283 | This phase checks whether the task graph contains possible memory conflicts. A memory conflict occurs when multiple memory operations access the same memory segment at the same time and at least one operation is a write. When a memory conflict occurs, the value of the conflicting memory segment is nondeterministic. This causes precision issues in collective communication operators. |
| 296 | - | ||
| 297 | ### 5.1 Memory Conflict Criteria | 284 | ### 5.1 Memory Conflict Criteria |
| 298 | 285 | ||
| 299 | Two memory access task nodes are flagged as a memory conflict when all three conditions below are met: | 286 | Two memory access task nodes are flagged as a memory conflict when all three conditions below are met: |
| @@ -1,7 +1,7 @@ | |||
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: UTF-8 -*- | 2 | # -*- coding: UTF-8 -*- |
| 3 | # ----------------------------------------------------------------------------------------------------------- | 3 | # ----------------------------------------------------------------------------------------------------------- |
| 4 | -# Copyright (c) 2025 Huawei Technologies Co., Ltd. | 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 5 | # This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 5 | # This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 6 | # CANN Open Software License Agreement Version 2.0 (the "License"). | 6 | # CANN Open Software License Agreement Version 2.0 (the "License"). |
| 7 | # Please refer to the License for details. You may not use this file except in compliance with the License. | 7 | # Please refer to the License for details. You may not use this file except in compliance with the License. |
| @@ -10,6 +10,15 @@ | |||
| 10 | # See LICENSE in the root of the software repository for the full text of the License. | 10 | # See LICENSE in the root of the software repository for the full text of the License. |
| 11 | # ----------------------------------------------------------------------------------------------------------- | 11 | # ----------------------------------------------------------------------------------------------------------- |
| 12 | 12 | ||
| 13 | +# You may not use this file except in compliance with the License. | ||
| 14 | +# You may obtain a copy of the License at | ||
| 15 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 16 | +# Unless required by applicable law or agreed to in writing, software | ||
| 17 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 18 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 19 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 20 | + | ||
| 21 | +#!/usr/bin/env python3 | ||
| 13 | """ | 22 | """ |
| 14 | FAQ Markdown 转 HTML 转换脚本 | 23 | FAQ Markdown 转 HTML 转换脚本 |
| 15 | 24 | ||
| @@ -1,7 +1,7 @@ | |||
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: UTF-8 -*- | 2 | # -*- coding: UTF-8 -*- |
| 3 | # ----------------------------------------------------------------------------------------------------------- | 3 | # ----------------------------------------------------------------------------------------------------------- |
| 4 | -# Copyright (c) 2025 Huawei Technologies Co., Ltd. | 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 5 | # This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 5 | # This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 6 | # CANN Open Software License Agreement Version 2.0 (the "License"). | 6 | # CANN Open Software License Agreement Version 2.0 (the "License"). |
| 7 | # Please refer to the License for details. You may not use this file except in compliance with the License. | 7 | # Please refer to the License for details. You may not use this file except in compliance with the License. |
| @@ -10,6 +10,15 @@ | |||
| 10 | # See LICENSE in the root of the software repository for the full text of the License. | 10 | # See LICENSE in the root of the software repository for the full text of the License. |
| 11 | # ----------------------------------------------------------------------------------------------------------- | 11 | # ----------------------------------------------------------------------------------------------------------- |
| 12 | 12 | ||
| 13 | +# You may not use this file except in compliance with the License. | ||
| 14 | +# You may obtain a copy of the License at | ||
| 15 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 16 | +# Unless required by applicable law or agreed to in writing, software | ||
| 17 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 18 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 19 | +# See the License for the specific language governing permissions and limitations under the License. | ||
| 20 | + | ||
| 21 | +#!/usr/bin/env python3 | ||
| 13 | """ | 22 | """ |
| 14 | FAQ Markdown 转 HTML 转换脚本 | 23 | FAQ Markdown 转 HTML 转换脚本 |
| 15 | 24 | ||
| @@ -1302,10 +1302,10 @@ | |||
| 1302 | "ErrorCode: 902" | 1302 | "ErrorCode: 902" |
| 1303 | ], | 1303 | ], |
| 1304 | "logPatterns": [ | 1304 | "logPatterns": [ |
| 1305 | - "[Main] [ErrorCode: 902] This op is skipped because both the new checker and the old checker are disabled, opIndex=47, newCheckerEnabled=0, oldCheckerEnabled=0" | 1305 | + "[Main] [ErrorCode: 902] This op is skipped because the new checker is disabled, opIndex=47, newCheckerEnabled=0" |
| 1306 | ], | 1306 | ], |
| 1307 | - "symptom": "配置开关或运行策略不满足当前 op 的执行条件，例如新旧 checker 同时被关闭。", | 1307 | + "symptom": "`enable_new_checker` 被关闭，当前算子未执行 Checker V3 单算子校验。老版本 Checker 已移除，不能通过其他开关接管该算子校验。", |
| 1308 | - "guidance": "【可能原因】\n1. manifest.json 或运行参数将 checker 关闭了。\n\n【排查步骤】\n1. 优先检查开关配置，确认至少启动一个 Checker。", | 1308 | + "guidance": "【可能原因】\n1. `manifest.json` 中的 `enable_new_checker` 被设置为 `false`。\n\n【排查步骤】\n1. 检查 `manifest.json` 中的 `enable_new_checker`，需要执行单算子校验时将其设置为 `true`。\n2. 如果只需要跨算子同步资源校验，可以单独开启 `enable_big_graph_checker`；该开关不能替代单算子 Checker V3。", |
| 1309 | "diagram": "", | 1309 | "diagram": "", |
| 1310 | "diagramType": "", | 1310 | "diagramType": "", |
| 1311 | "sourceFile": "checker_faq.md" | 1311 | "sourceFile": "checker_faq.md" |
| @@ -24,6 +24,8 @@ | |||
| 24 | | L3 | 网络性能建模 | ⏳远期 | | 24 | | L3 | 网络性能建模 | ⏳远期 | |
| 25 | | L4 | 故障注入诊断 | ⏳远期 | | 25 | | L4 | 故障注入诊断 | ⏳远期 | |
| 26 | 26 | ||
| 27 | + | ||
| 28 | + | ||
| 27 | #### 1.2.1 使用场景 | 29 | #### 1.2.1 使用场景 |
| 28 | 30 | ||
| 29 | | 角色 | 用途 | | 31 | | 角色 | 用途 | |
| @@ -32,6 +34,7 @@ | |||
| 32 | | 测试 | 运行HCCL API测试用例 | | 34 | | 测试 | 运行HCCL API测试用例 | |
| 33 | | 开发 | 执行Runtime/Driver级用例 | | 35 | | 开发 | 执行Runtime/Driver级用例 | |
| 34 | 36 | ||
| 37 | + | ||
| 35 | #### 1.2.2 输入形式 | 38 | #### 1.2.2 输入形式 |
| 36 | 39 | ||
| 37 | - **配置驱动** | 40 | - **配置驱动** |
| @@ -81,8 +84,10 @@ C --> D[CPU执行] | |||
| 81 | - L2：全局事件序列化。 | 84 | - L2：全局事件序列化。 |
| 82 | - L3/L4：固定种子随机源或统一时钟。 | 85 | - L3/L4：固定种子随机源或统一时钟。 |
| 83 | 86 | ||
| 87 | + | ||
| 84 | ### 3.2 典型工作流 | 88 | ### 3.2 典型工作流 |
| 85 | 89 | ||
| 90 | + | ||
| 86 | #### 3.2.1 开源贡献者自定义算子验证流程(LLT) | 91 | #### 3.2.1 开源贡献者自定义算子验证流程(LLT) |
| 87 | 92 | ||
| 88 | 1. **环境准备** | 93 | 1. **环境准备** |
| @@ -491,10 +496,12 @@ while HcclAllReduceInner(is_running=True): | |||
| 491 | 496 | ||
| 492 | ### 4.6 FR4持久化 | 497 | ### 4.6 FR4持久化 |
| 493 | 498 | ||
| 499 | + | ||
| 494 | #### 4.6.1 `L2阶段`Story: 模拟器提供持久化接口，以便开发者测试结束快速定位问题 | 500 | #### 4.6.1 `L2阶段`Story: 模拟器提供持久化接口，以便开发者测试结束快速定位问题 |
| 495 | 501 | ||
| 496 | ### 4.7 FR5控制器 | 502 | ### 4.7 FR5控制器 |
| 497 | 503 | ||
| 504 | + | ||
| 498 | 考虑到易用性，用户在一般在终端中使用HCCL测试程序，因此控制器通过命令行实例程序承载。 | 505 | 考虑到易用性，用户在一般在终端中使用HCCL测试程序，因此控制器通过命令行实例程序承载。 |
| 499 | 506 | ||
| 500 | #### 4.7.1控制器约定 | 507 | #### 4.7.1控制器约定 |
| @@ -511,6 +518,7 @@ while HcclAllReduceInner(is_running=True): | |||
| 511 | 518 | ||
| 512 | ##### 成功场景: 系统模型发现 | 519 | ##### 成功场景: 系统模型发现 |
| 513 | 520 | ||
| 521 | + | ||
| 514 | 1. 用户通过`hccl-vm --list-topologies`查看支持模拟的系统。 | 522 | 1. 用户通过`hccl-vm --list-topologies`查看支持模拟的系统。 |
| 515 | 2. 命令会打印出cloud_matrix, atlas900等所有内置系统模型的列表，但不启动模拟器。 | 523 | 2. 命令会打印出cloud_matrix, atlas900等所有内置系统模型的列表，但不启动模拟器。 |
| 516 | 524 | ||
| @@ -546,7 +554,7 @@ while HcclAllReduceInner(is_running=True): | |||
| 546 | 1. 用户执行`hccl-vm --topology=non_existent_topo`。 | 554 | 1. 用户执行`hccl-vm --topology=non_existent_topo`。 |
| 547 | 2. `hccl-vm`输出错误信息后正常终止，如“错误：未找到名为 'non_existent_topo' 的系统。可用系统：cloud_matrix, atlas900”。 | 555 | 2. `hccl-vm`输出错误信息后正常终止，如“错误：未找到名为 'non_existent_topo' 的系统。可用系统：cloud_matrix, atlas900”。 |
| 548 | 556 | ||
| 549 | -##### 失败场景: 拓扑配置文件格式错误 | 557 | +##### 失败场景: 拓扑配置文件格式错误 |
| 550 | 558 | ||
| 551 | 1. 用户提供的拓扑配置文件内容不符合YAML规范或缺少必要字段，导致`hccl-vm`在初始化时解析失败。 | 559 | 1. 用户提供的拓扑配置文件内容不符合YAML规范或缺少必要字段，导致`hccl-vm`在初始化时解析失败。 |
| 552 | 2. 模拟器向标准错误输出一条明确的错误信息，如“错误：解析拓扑文件a.yaml失败，原因：...”，并立即终止整个程序的执行。 | 560 | 2. 模拟器向标准错误输出一条明确的错误信息，如“错误：解析拓扑文件a.yaml失败，原因：...”，并立即终止整个程序的执行。 |
| @@ -676,6 +684,7 @@ while HcclAllReduceInner(is_running=True): | |||
| 676 | 684 | ||
| 677 | 上板测试程序，会跑在真实的卡上。这时，需要用户手动或者脚本在多个服务器上启动同一份测试程序，如果要跑模拟器则需要在每个跑测试程序的服务器上都准备一份模拟器的控制器。而模拟器内部是需要对进行整个网络拓扑乃至硬件环境进行统一建模的，因此需要对控制器做分布式设计。 | 685 | 上板测试程序，会跑在真实的卡上。这时，需要用户手动或者脚本在多个服务器上启动同一份测试程序，如果要跑模拟器则需要在每个跑测试程序的服务器上都准备一份模拟器的控制器。而模拟器内部是需要对进行整个网络拓扑乃至硬件环境进行统一建模的，因此需要对控制器做分布式设计。 |
| 678 | 686 | ||
| 687 | + | ||
| 679 | #### 4.8.1 `L2阶段`Story: 用户可以通过k8s等集群管理软件，在多个服务器上协同运行HCCL真实用例 | 688 | #### 4.8.1 `L2阶段`Story: 用户可以通过k8s等集群管理软件，在多个服务器上协同运行HCCL真实用例 |
| 680 | 689 | ||
| 681 | ## 5 **附录** | 690 | ## 5 **附录** |
| @@ -686,12 +695,14 @@ while HcclAllReduceInner(is_running=True): | |||
| 686 | 695 | ||
| 687 | #### 5.1.1`L1阶段` | 696 | #### 5.1.1`L1阶段` |
| 688 | 697 | ||
| 698 | + | ||
| 689 | - HCCL通信域管理（25个API） | 699 | - HCCL通信域管理（25个API） |
| 690 | - HCCL控制面编程（23个API） | 700 | - HCCL控制面编程（23个API） |
| 691 | - HCCL AICPU编程（8个API） | 701 | - HCCL AICPU编程（8个API） |
| 692 | 702 | ||
| 693 | #### 5.1.2 `L2阶段` | 703 | #### 5.1.2 `L2阶段` |
| 694 | 704 | ||
| 705 | + | ||
| 695 | ### 5.2 **附录B: 拓扑配置文件格式(Schema)** | 706 | ### 5.2 **附录B: 拓扑配置文件格式(Schema)** |
| 696 | 707 | ||
| 697 | 拓扑配置文件采用YAML格式，用于描述模拟的硬件环境。文件必须包含以下字段： | 708 | 拓扑配置文件采用YAML格式，用于描述模拟的硬件环境。文件必须包含以下字段： |
| @@ -706,6 +717,7 @@ while HcclAllReduceInner(is_running=True): | |||
| 706 | 717 | ||
| 707 | **示例: `my_topo.yaml`** | 718 | **示例: `my_topo.yaml`** |
| 708 | 719 | ||
| 720 | + | ||
| 709 | ```yaml | 721 | ```yaml |
| 710 | ## 描述一个4卡的环形连接拓扑 | 722 | ## 描述一个4卡的环形连接拓扑 |
| 711 | npus: 4 | 723 | npus: 4 |
| @@ -765,3 +777,4 @@ extern "C" SimValidationResult post_HcclAllReduceInner_hook(const SimCommContext | |||
| 765 | ``` | 777 | ``` |
| 766 | 778 | ||
| 767 | ### 5.4 **附录D: 系统模型** | 779 | ### 5.4 **附录D: 系统模型** |
| 780 | + | ||
| @@ -24,6 +24,7 @@ Implement layered simulation depth: | |||
| 24 | | L3 | Network performance modeling | ⏳ Long-term | | 24 | | L3 | Network performance modeling | ⏳ Long-term | |
| 25 | | L4 | Fault injection diagnosis | ⏳ Long-term | | 25 | | L4 | Fault injection diagnosis | ⏳ Long-term | |
| 26 | 26 | ||
| 27 | + | ||
| 27 | #### 1.2.1 Usage Scenarios | 28 | #### 1.2.1 Usage Scenarios |
| 28 | 29 | ||
| 29 | | Role | Purpose | | 30 | | Role | Purpose | |
| @@ -81,6 +82,7 @@ C --> D[CPU Execution] | |||
| 81 | - L2: Global event serialization. | 82 | - L2: Global event serialization. |
| 82 | - L3/L4: Fixed-seed random sources or a unified clock. | 83 | - L3/L4: Fixed-seed random sources or a unified clock. |
| 83 | 84 | ||
| 85 | + | ||
| 84 | ### 3.2 Typical Workflow | 86 | ### 3.2 Typical Workflow |
| 85 | 87 | ||
| 86 | #### 3.2.1 Open Source Contributor Custom Operator Verification Process (LLT) | 88 | #### 3.2.1 Open Source Contributor Custom Operator Verification Process (LLT) |
| @@ -491,10 +493,12 @@ while HcclAllReduceInner(is_running=True): | |||
| 491 | 493 | ||
| 492 | ### 4.6 FR4 Persistence | 494 | ### 4.6 FR4 Persistence |
| 493 | 495 | ||
| 496 | + | ||
| 494 | #### 4.6.1 `L2 Phase` Story: The simulator provides persistence interfaces so developers can quickly locate issues after testing ends | 497 | #### 4.6.1 `L2 Phase` Story: The simulator provides persistence interfaces so developers can quickly locate issues after testing ends |
| 495 | 498 | ||
| 496 | ### 4.7 FR5 Controller | 499 | ### 4.7 FR5 Controller |
| 497 | 500 | ||
| 501 | + | ||
| 498 | The user generally uses HCCL test programs in a terminal for ease of use. Therefore, the controller runs through a command-line instance program. | 502 | The user generally uses HCCL test programs in a terminal for ease of use. Therefore, the controller runs through a command-line instance program. |
| 499 | 503 | ||
| 500 | #### 4.7.1 Controller Conventions | 504 | #### 4.7.1 Controller Conventions |
| @@ -686,12 +690,14 @@ Convention: The simulator provides Host offline implementations for the APIs lis | |||
| 686 | 690 | ||
| 687 | #### 5.1.1 `L1 Phase` | 691 | #### 5.1.1 `L1 Phase` |
| 688 | 692 | ||
| 693 | + | ||
| 689 | - HCCL communication domain management (25 APIs) | 694 | - HCCL communication domain management (25 APIs) |
| 690 | - HCCL control plane programming (23 APIs) | 695 | - HCCL control plane programming (23 APIs) |
| 691 | - HCCL AICPU programming (8 APIs) | 696 | - HCCL AICPU programming (8 APIs) |
| 692 | 697 | ||
| 693 | #### 5.1.2 `L2 Phase` | 698 | #### 5.1.2 `L2 Phase` |
| 694 | 699 | ||
| 700 | + | ||
| 695 | ### 5.2 **Appendix B: Topology Configuration File Format (Schema)** | 701 | ### 5.2 **Appendix B: Topology Configuration File Format (Schema)** |
| 696 | 702 | ||
| 697 | The topology configuration file uses YAML format and describes the simulated hardware environment. The file must contain the following fields: | 703 | The topology configuration file uses YAML format and describes the simulated hardware environment. The file must contain the following fields: |
| @@ -706,6 +712,7 @@ Each `link` object contains: | |||
| 706 | 712 | ||
| 707 | **Sample: `my_topo.yaml`** | 713 | **Sample: `my_topo.yaml`** |
| 708 | 714 | ||
| 715 | + | ||
| 709 | ```yaml | 716 | ```yaml |
| 710 | ## Describe a 4-NPU ring connection topology | 717 | ## Describe a 4-NPU ring connection topology |
| 711 | npus: 4 | 718 | npus: 4 |
| @@ -765,3 +772,4 @@ extern "C" SimValidationResult post_HcclAllReduceInner_hook(const SimCommContext | |||
| 765 | ``` | 772 | ``` |
| 766 | 773 | ||
| 767 | ### 5.4 **Appendix D: System Models** | 774 | ### 5.4 **Appendix D: System Models** |
| 775 | + | ||
| @@ -328,8 +328,8 @@ struct FileHeader { | |||
| 328 | | 1 | ROCE | RDMA over Converged Ethernet | | 328 | | 1 | ROCE | RDMA over Converged Ethernet | |
| 329 | | 2 | PCIE | PCIe 通信 | | 329 | | 2 | PCIE | PCIe 通信 | |
| 330 | | 3 | SIO | Socket I/O | | 330 | | 3 | SIO | Socket I/O | |
| 331 | -| 4 | UB_CTP | UB CTP 协议 | | 331 | +| 4 | UBC_CTP | UBC CTP 协议 | |
| 332 | -| 5 | UBC_TP | UBC TP 协议（历史兼容值，不建议新开发使用） | | 332 | +| 5 | UBC_TP | UBC TP 协议 | |
| 333 | | 6 | UB_MEM | UB 内存协议 | | 333 | | 6 | UB_MEM | UB 内存协议 | |
| 334 | 334 | ||
| 335 | ### 6.3 缓冲区类型 (BufferType) | 335 | ### 6.3 缓冲区类型 (BufferType) |
| @@ -328,8 +328,8 @@ Each MicrocodeInstrInner contains: | |||
| 328 | | 1 | ROCE | RDMA over Converged Ethernet | | 328 | | 1 | ROCE | RDMA over Converged Ethernet | |
| 329 | | 2 | PCIE | PCIe communication | | 329 | | 2 | PCIE | PCIe communication | |
| 330 | | 3 | SIO | Socket I/O | | 330 | | 3 | SIO | Socket I/O | |
| 331 | -| 4 | UB_CTP | UB CTP protocol | | 331 | +| 4 | UBC_CTP | UBC CTP protocol | |
| 332 | -| 5 | UBC_TP | UBC TP protocol (legacy compatibility value, not recommended for new development) | | 332 | +| 5 | UBC_TP | UBC TP protocol | |
| 333 | | 6 | UB_MEM | UB memory protocol | | 333 | | 6 | UB_MEM | UB memory protocol | |
| 334 | 334 | ||
| 335 | ### 6.3 Buffer Type (BufferType) | 335 | ### 6.3 Buffer Type (BufferType) |
| @@ -7,7 +7,6 @@ | |||
| 7 | 本文档列出项目中通过 `#include` 实际引用的所有三方头文件，标注其来源（CANN / HCCL / HCOMM）及在磁盘上的实际路径。 | 7 | 本文档列出项目中通过 `#include` 实际引用的所有三方头文件，标注其来源（CANN / HCCL / HCOMM）及在磁盘上的实际路径。 |
| 8 | 8 | ||
| 9 | **路径约定**： | 9 | **路径约定**： |
| 10 | - | ||
| 11 | - CANN 根目录：`/home/teamserver/workspace/Ascend/cann-9.1.0/`（下文以 `CANN/` 缩写） | 10 | - CANN 根目录：`/home/teamserver/workspace/Ascend/cann-9.1.0/`（下文以 `CANN/` 缩写） |
| 12 | - HCOMM 根目录：`/home/teamserver/workspace/hcomm/`（下文以 `HCOMM/` 缩写） | 11 | - HCOMM 根目录：`/home/teamserver/workspace/hcomm/`（下文以 `HCOMM/` 缩写） |
| 13 | - HCCL 根目录：`/home/teamserver/workspace/hccl/`（下文以 `HCCL/` 缩写） | 12 | - HCCL 根目录：`/home/teamserver/workspace/hccl/`（下文以 `HCCL/` 缩写） |
| @@ -30,7 +29,7 @@ | |||
| 30 | 29 | ||
| 31 | | #include | 实际文件路径 | 引用位置 | | 30 | | #include | 实际文件路径 | 引用位置 | |
| 32 | |----------|------------|---------| | 31 | |----------|------------|---------| |
| 33 | -| `hccl/hccl_types.h` | `CANN/include/hccl/hccl_types.h` | proxy/aclrt_kernel_stub.cc、hccl_op_stub.cc、hccl_proxy_common.h；plugin/runner/ccu_executor/ccu_fp16.h；common/sim_data_dump.cc；device_arm/proxy/device_sqe_parse_stub.h；test/proxy/ | | 32 | +| `hccl/hccl_types.h` | `CANN/include/hccl/hccl_types.h` | proxy/aclrt_kernel_stub.cc、hccl_op_stub.cc、hccl_proxy_common.h；plugin/runner/ccu_executor/ccu_fp16.h；common/sim_data_dump.cc；device_arm/level2/proxy/device_sqe_parse_stub.h；test/proxy/ | |
| 34 | | `hccl/hccl.h` | `CANN/include/hccl/hccl.h` | **proxy/hccl_op_stub.cc**；test/proxy/hccl_comm_stub_test.cc | | 33 | | `hccl/hccl.h` | `CANN/include/hccl/hccl.h` | **proxy/hccl_op_stub.cc**；test/proxy/hccl_comm_stub_test.cc | |
| 35 | | `hccl/hcom.h` | `CANN/include/hccl/hcom.h` | proxy/hccl_inner_stub.cc；test/proxy/hccl_inner_stub_test.cc | | 34 | | `hccl/hcom.h` | `CANN/include/hccl/hcom.h` | proxy/hccl_inner_stub.cc；test/proxy/hccl_inner_stub_test.cc | |
| 36 | | `hccl/base.h` | `CANN/include/hccl/base.h` | proxy/aclrt_context_stub.cc；plugin/runner/ccu_executor/ccu_fp16.h；plugin/checker/header/external/task_param.h；include/sim_ip_address.h | | 35 | | `hccl/base.h` | `CANN/include/hccl/base.h` | proxy/aclrt_context_stub.cc；plugin/runner/ccu_executor/ccu_fp16.h；plugin/checker/header/external/task_param.h；include/sim_ip_address.h | |
| @@ -49,13 +48,13 @@ | |||
| 49 | |----------|------------|---------| | 48 | |----------|------------|---------| |
| 50 | | `"runtime/base.h"` | `CANN/pkg_inc/runtime/runtime/base.h` | proxy/hccl_comm_stub.cc、aclrt_runtime_config.cc、aclrt_notify_stub.cc、hccp_stub.cc、aclrt_device_stub.cc、hccp_ra_socket_stub.cc、aclrt_stream_stub.cc、aclrt_stub.cc；test/proxy/ | | 49 | | `"runtime/base.h"` | `CANN/pkg_inc/runtime/runtime/base.h` | proxy/hccl_comm_stub.cc、aclrt_runtime_config.cc、aclrt_notify_stub.cc、hccp_stub.cc、aclrt_device_stub.cc、hccp_ra_socket_stub.cc、aclrt_stream_stub.cc、aclrt_stub.cc；test/proxy/ | |
| 51 | | `"runtime/event.h"` | `CANN/pkg_inc/runtime/runtime/event.h` | proxy/aclrt_runtime_config.cc、aclrt_notify_stub.cc | | 50 | | `"runtime/event.h"` | `CANN/pkg_inc/runtime/runtime/event.h` | proxy/aclrt_runtime_config.cc、aclrt_notify_stub.cc | |
| 52 | -| `"runtime/rt.h"` | `CANN/pkg_inc/runtime/runtime/rt.h` | device_arm/proxy/device_sqe_parse_stub.h | | 51 | +| `"runtime/rt.h"` | `CANN/pkg_inc/runtime/runtime/rt.h` | device_arm/level2/proxy/device_sqe_parse_stub.h | |
| 53 | 52 | ||
| 54 | ### 1.5 CANN Profiling 接口（`CANN/pkg_inc/profiling/`） | 53 | ### 1.5 CANN Profiling 接口（`CANN/pkg_inc/profiling/`） |
| 55 | 54 | ||
| 56 | | #include | 实际文件路径 | 引用位置 | | 55 | | #include | 实际文件路径 | 引用位置 | |
| 57 | |----------|------------|---------| | 56 | |----------|------------|---------| |
| 58 | -| `"aprof_pub.h"` | `CANN/pkg_inc/profiling/aprof_pub.h` | proxy/aprofiling_stub.cc；device_arm/proxy/aprofiling_stub.cc；test/proxy/aprofiling_stub_test.cc | | 57 | +| `"aprof_pub.h"` | `CANN/pkg_inc/profiling/aprof_pub.h` | proxy/aprofiling_stub.cc；device_arm/level2/proxy/aprofiling_stub.cc；test/proxy/aprofiling_stub_test.cc | |
| 59 | 58 | ||
| 60 | ### 1.6 CANN Trace 接口（`CANN/pkg_inc/trace/`） | 59 | ### 1.6 CANN Trace 接口（`CANN/pkg_inc/trace/`） |
| 61 | 60 | ||
| @@ -68,7 +67,7 @@ | |||
| 68 | 67 | ||
| 69 | | #include | 实际文件路径 | 引用位置 | | 68 | | #include | 实际文件路径 | 引用位置 | |
| 70 | |----------|------------|---------| | 69 | |----------|------------|---------| |
| 71 | -| `"ascend_hal.h"` | `CANN/include/driver/ascend_hal.h` | proxy/adapter_rts_stub.cc、aclrt_new_stub.cc、ascend_hal_stub.cc；device_arm/proxy/device_sqe_parse_stub.h、ascend_hal_stub.cc；test/device_arm/ | | 70 | +| `"ascend_hal.h"` | `CANN/include/driver/ascend_hal.h` | proxy/adapter_rts_stub.cc、aclrt_new_stub.cc、ascend_hal_stub.cc；device_arm/level2/proxy/device_sqe_parse_stub.h、ascend_hal_stub.cc；test/device_arm/ | |
| 72 | 71 | ||
| 73 | ### 1.8 CANN Base 接口（`CANN/pkg_inc/base/`） | 72 | ### 1.8 CANN Base 接口（`CANN/pkg_inc/base/`） |
| 74 | 73 | ||
| @@ -175,12 +174,11 @@ | |||
| 175 | 174 | ||
| 176 | **原始来源**：HCOMM 仓库的 `hccl_aiv_utils.h` 头文件 | 175 | **原始来源**：HCOMM 仓库的 `hccl_aiv_utils.h` 头文件 |
| 177 | 176 | ||
| 178 | -**备份位置**：`src/proxy/aclrt_kernel_stub.cc` 第 597-628 行 | 177 | +**备份位置**：`src/proxy/level2/aclrt_kernel_stub.cc` 第 597-628 行 |
| 179 | 178 | ||
| 180 | **说明**：`AivOpArgs` 是 AIV 集合通信算子的参数结构体，用于在 host 端与 device 端之间传递 AIV kernel 的执行参数。工具原本通过 `dlsym` 动态加载 HCOMM 库中的 `ops_hccl::ExecuteKernelLaunch()` 函数（符号名：`_ZN8ops_hccl19ExecuteKernelLaunchERKNS_9AivOpArgsE`），该函数接收此结构体作为参数。为了解除对 HCOMM 运行时库的依赖，工具在项目内部做了结构体的完整备份。 | 179 | **说明**：`AivOpArgs` 是 AIV 集合通信算子的参数结构体，用于在 host 端与 device 端之间传递 AIV kernel 的执行参数。工具原本通过 `dlsym` 动态加载 HCOMM 库中的 `ops_hccl::ExecuteKernelLaunch()` 函数（符号名：`_ZN8ops_hccl19ExecuteKernelLaunchERKNS_9AivOpArgsE`），该函数接收此结构体作为参数。为了解除对 HCOMM 运行时库的依赖，工具在项目内部做了结构体的完整备份。 |
| 181 | 180 | ||
| 182 | **结构体定义**： | 181 | **结构体定义**： |
| 183 | - | ||
| 184 | ```cpp | 182 | ```cpp |
| 185 | struct AivOpArgs { | 183 | struct AivOpArgs { |
| 186 | HcclCMDType cmdType = HcclCMDType::HCCL_CMD_MAX; // 集合通信命令类型 | 184 | HcclCMDType cmdType = HcclCMDType::HCCL_CMD_MAX; // 集合通信命令类型 |
| @@ -220,7 +218,7 @@ struct AivOpArgs { | |||
| 220 | 218 | ||
| 221 | **原始来源**：HCOMM 仓库的同名头文件 | 219 | **原始来源**：HCOMM 仓库的同名头文件 |
| 222 | 220 | ||
| 223 | -**备份位置**：`src/proxy/aiv_kernel/hccl_op_stub/aiv_communication_base_v2.h` | 221 | +**备份位置**：`src/proxy/level2/aiv_kernel/hccl_op_stub/aiv_communication_base_v2.h` |
| 224 | 222 | ||
| 225 | **说明**：此头文件定义了 AIV 集合通信算子的基类 `AivCommBase`，以及相关的内核参数宏定义（`KERNEL_ARGS_DEF`、`KERNEL_ARGS_CALL` 等）。工具原本直接 include HCOMM 仓库中的该文件，为了解除依赖，在项目内部做了完整备份。 | 223 | **说明**：此头文件定义了 AIV 集合通信算子的基类 `AivCommBase`，以及相关的内核参数宏定义（`KERNEL_ARGS_DEF`、`KERNEL_ARGS_CALL` 等）。工具原本直接 include HCOMM 仓库中的该文件，为了解除依赖，在项目内部做了完整备份。 |
| 226 | 224 | ||
| @@ -240,7 +238,7 @@ struct AivOpArgs { | |||
| 240 | 238 | ||
| 241 | ### 5.3 AIV 算子 Op 头文件（HCOMM / HCCL 依赖） | 239 | ### 5.3 AIV 算子 Op 头文件（HCOMM / HCCL 依赖） |
| 242 | 240 | ||
| 243 | -工具中每种集合通信算子的 AIV kernel 实现文件（`src/proxy/aiv_kernel/hccl_op_stub/*/aiv_communication_v2.cc`）通过 `#include` 引用了对应算子的 op 头文件。这些头文件**不存在于项目内部**，分别来自 HCOMM 和 HCCL 仓库： | 241 | +工具中每种集合通信算子的 AIV kernel 实现文件（`src/proxy/level2/aiv_kernel/hccl_op_stub/*/aiv_communication_v2.cc`）通过 `#include` 引用了对应算子的 op 头文件。这些头文件**不存在于项目内部**，分别来自 HCOMM 和 HCCL 仓库： |
| 244 | 242 | ||
| 245 | | #include | 来源 | 实际文件路径 | 引用位置 | | 243 | | #include | 来源 | 实际文件路径 | 引用位置 | |
| 246 | |----------|------|------------|---------| | 244 | |----------|------|------------|---------| |
| @@ -250,14 +248,14 @@ struct AivOpArgs { | |||
| 250 | | `"aiv_broadcast_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/broadcast/` | hccl_op_stub/broadcast/aiv_communication_v2.cc | | 248 | | `"aiv_broadcast_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/broadcast/` | hccl_op_stub/broadcast/aiv_communication_v2.cc | |
| 251 | | `"aiv_all_to_all_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | | 249 | | `"aiv_all_to_all_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | |
| 252 | | `"aiv_all_to_all_v_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | | 250 | | `"aiv_all_to_all_v_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | |
| 253 | -| `"aiv_scatter_op.h"` | HCCL | `hccl/src/ops/scatter/algorithm/template/aiv/kernel/` | hccl_op_stub/scatter/aiv_communication_v2.cc | | 251 | +| `"aiv_scatter_op.h"` | HCCL | `hccl/src/ops/scatter/template/aiv/kernel/` | hccl_op_stub/scatter/aiv_communication_v2.cc | |
| 254 | -| `"aiv_reduce_op.h"` | HCCL | `hccl/src/ops/reduce/algorithm/template/aiv/kernel/` | hccl_op_stub/reduce/aiv_communication_v2.cc | | 252 | +| `"aiv_reduce_op.h"` | HCCL | `hccl/src/ops/reduce/template/aiv/kernel/` | hccl_op_stub/reduce/aiv_communication_v2.cc | |
| 255 | 253 | ||
| 256 | **依赖风险**：若 HCOMM 或 HCCL 侧算子 op 头文件的接口签名、模板参数或成员函数发生变化，工具的 AIV kernel 编译将失败，或导致运行时行为不一致。 | 254 | **依赖风险**：若 HCOMM 或 HCCL 侧算子 op 头文件的接口签名、模板参数或成员函数发生变化，工具的 AIV kernel 编译将失败，或导致运行时行为不一致。 |
| 257 | 255 | ||
| 258 | ### 5.4 AscendC 接口依赖 | 256 | ### 5.4 AscendC 接口依赖 |
| 259 | 257 | ||
| 260 | -工具依赖 CANN 包提供的 AscendC 编程接口来模拟 AIV kernel 的执行。项目中的 AscendC 接口全部通过 `src/proxy/aiv_kernel/ascendc_stub/` 目录下的本地 stub 文件实现，而非直接使用 CANN SDK 中的 AscendC 头文件。 | 258 | +工具依赖 CANN 包提供的 AscendC 编程接口来模拟 AIV kernel 的执行。项目中的 AscendC 接口全部通过 `src/proxy/level2/aiv_kernel/ascendc_stub/` 目录下的本地 stub 文件实现，而非直接使用 CANN SDK 中的 AscendC 头文件。 |
| 261 | 259 | ||
| 262 | **使用的 AscendC 接口**（通过 stub 实现）： | 260 | **使用的 AscendC 接口**（通过 stub 实现）： |
| 263 | 261 | ||
| @@ -271,7 +269,6 @@ struct AivOpArgs { | |||
| 271 | | `ascendc_utils_stub.h` | `PipeBarrier` 等工具函数 | ascendc_stub/kernel_operator.h | | 269 | | `ascendc_utils_stub.h` | `PipeBarrier` 等工具函数 | ascendc_stub/kernel_operator.h | |
| 272 | 270 | ||
| 273 | **依赖风险**： | 271 | **依赖风险**： |
| 274 | - | ||
| 275 | 1. AscendC 接口 API 变化（如类成员函数签名变更、模板参数调整）可能导致 stub 实现与真实接口不匹配 | 272 | 1. AscendC 接口 API 变化（如类成员函数签名变更、模板参数调整）可能导致 stub 实现与真实接口不匹配 |
| 276 | 2. AscendC 新增数据类型（如 `fp8_e4m3fn_t`、`fp8_e5m2_t`、`hifloat8_t`）需要同步更新 stub 支持 | 273 | 2. AscendC 新增数据类型（如 `fp8_e4m3fn_t`、`fp8_e5m2_t`、`hifloat8_t`）需要同步更新 stub 支持 |
| 277 | 3. `AivCommBase` 类内部大量使用 AscendC 模板类（`GlobalTensor<T>`、`LocalTensor<T>`、`TQueBind` 等），这些类型的语义变化会影响 AIV 仿真的正确性 | 274 | 3. `AivCommBase` 类内部大量使用 AscendC 模板类（`GlobalTensor<T>`、`LocalTensor<T>`、`TQueBind` 等），这些类型的语义变化会影响 AIV 仿真的正确性 |
| @@ -284,11 +281,11 @@ struct AivOpArgs { | |||
| 284 | 281 | ||
| 285 | ### 6.1 AICPU 交互数据结构 | 282 | ### 6.1 AICPU 交互数据结构 |
| 286 | 283 | ||
| 287 | -工具在 `src/device_arm/aicpu_args_stub.h` 中对以下数据结构做了本地备份，以解除对 HCOMM 运行时库的强依赖： | 284 | +工具在 `src/device_arm/common/aicpu_args_stub.h` 中对以下数据结构做了本地备份，以解除对 HCOMM 运行时库的强依赖： |
| 288 | 285 | ||
| 289 | | 数据结构 | 用途 | 对应 kernel 函数 | | 286 | | 数据结构 | 用途 | 对应 kernel 函数 | |
| 290 | |---------|------|-----------------| | 287 | |---------|------|-----------------| |
| 291 | -| `CommAicpuParam` | 通信域初始化参数（hcomId、设备 ID、H2D/D2H 传输参数） | `RunAicpuCommInit` | | 288 | +| `CommAicpuParam` | 通信域初始化参数（hcomId、设备 ID、H2D/D2H 传输参数） | `RunAicpuIndOpCommInit` | |
| 292 | | `HDCommunicateParams` | H2D/D2H 控制传输参数（deviceAddr、readCacheAddr） | 被 `CommAicpuParam` 引用 | | 289 | | `HDCommunicateParams` | H2D/D2H 控制传输参数（deviceAddr、readCacheAddr） | 被 `CommAicpuParam` 引用 | |
| 293 | | `ThreadMgrAicpuParam` | 线程管理参数（threadNum、序列化 threadParam 数组、deviceHandle） | `RunAicpuIndOpThreadInit`、`RunAicpuThreadSupplementNotify` | | 290 | | `ThreadMgrAicpuParam` | 线程管理参数（threadNum、序列化 threadParam 数组、deviceHandle） | `RunAicpuIndOpThreadInit`、`RunAicpuThreadSupplementNotify` | |
| 294 | | `AicpuTsThread` | AICPU 线程信息（streamType、notifyLoadType、devId） | 被 `ThreadMgrAicpuParam` 引用 | | 291 | | `AicpuTsThread` | AICPU 线程信息（streamType、notifyLoadType、devId） | 被 `ThreadMgrAicpuParam` 引用 | |
| @@ -311,15 +308,14 @@ struct AivOpArgs { | |||
| 311 | 308 | ||
| 312 | | dlsym 符号名 | 所在库 | 功能 | | 309 | | dlsym 符号名 | 所在库 | 功能 | |
| 313 | |-------------|-------|------| | 310 | |-------------|-------|------| |
| 314 | -| `RunAicpuCommInit` | `libccl_kernel.so` | AICPU 通信域初始化 | | 311 | +| `RunAicpuIndOpCommInit` | `libccl_kernel.so` | AICPU 通信域初始化 | |
| 315 | | `RunAicpuIndOpThreadInit` | `libccl_kernel.so` | AICPU 线程初始化 | | 312 | | `RunAicpuIndOpThreadInit` | `libccl_kernel.so` | AICPU 线程初始化 | |
| 316 | | `RunAicpuIndOpChannelInitV2` | `libccl_kernel.so` | AICPU 通道初始化 V2 | | 313 | | `RunAicpuIndOpChannelInitV2` | `libccl_kernel.so` | AICPU 通道初始化 V2 | |
| 317 | -| `RunAicpuDfxInitV2` | `libccl_kernel.so` | AICPU DFX 算子信息初始化 V2 | | 314 | +| `RunAicpuDfxOpInfoInitV2` | `libccl_kernel.so` | AICPU DFX 算子信息初始化 V2 | |
| 318 | | `RunAicpuThreadSupplementNotify` | `libccl_kernel.so` | AICPU 资源补充通知 | | 315 | | `RunAicpuThreadSupplementNotify` | `libccl_kernel.so` | AICPU 资源补充通知 | |
| 319 | | `HcclLaunchAicpuKernel` | `libscatter_aicpu_kernel.so` | AICPU 集合通信 kernel 启动 | | 316 | | `HcclLaunchAicpuKernel` | `libscatter_aicpu_kernel.so` | AICPU 集合通信 kernel 启动 | |
| 320 | 317 | ||
| 321 | **依赖风险**： | 318 | **依赖风险**： |
| 322 | - | ||
| 323 | 1. 若 HCOMM 侧上述 kernel 函数的签名或行为发生变化，工具的 `dlsym` 调用可能失败或产生错误结果 | 319 | 1. 若 HCOMM 侧上述 kernel 函数的签名或行为发生变化，工具的 `dlsym` 调用可能失败或产生错误结果 |
| 324 | 2. 若 `libccl_kernel.so` 或 `libscatter_aicpu_kernel.so` 新增/移除 kernel 函数，工具需同步更新函数指针列表 | 320 | 2. 若 `libccl_kernel.so` 或 `libscatter_aicpu_kernel.so` 新增/移除 kernel 函数，工具需同步更新函数指针列表 |
| 325 | 3. `CommAicpuParam`、`ThreadMgrAicpuParam`、`HcclChannelUrmaRes` 等本地备份结构体若与 HCOMM 侧实际定义不一致，将导致地址转换错误和内存越界 | 321 | 3. `CommAicpuParam`、`ThreadMgrAicpuParam`、`HcclChannelUrmaRes` 等本地备份结构体若与 HCOMM 侧实际定义不一致，将导致地址转换错误和内存越界 |
| @@ -330,13 +326,12 @@ struct AivOpArgs { | |||
| 330 | 326 | ||
| 331 | | 硬编码 SO 名称 | 加载的 dlsym 符号 | 说明 | | 327 | | 硬编码 SO 名称 | 加载的 dlsym 符号 | 说明 | |
| 332 | |--------------|-----------------|------| | 328 | |--------------|-----------------|------| |
| 333 | -| `libccl_kernel.so` | `RunAicpuCommInit`、`RunAicpuIndOpThreadInit`、`RunAicpuIndOpChannelInitV2`、`RunAicpuDfxInitV2`、`RunAicpuThreadSupplementNotify` | HCOMM 通信框架核心库 | | 329 | +| `libccl_kernel.so` | `RunAicpuIndOpCommInit`、`RunAicpuIndOpThreadInit`、`RunAicpuIndOpChannelInitV2`、`RunAicpuDfxOpInfoInitV2`、`RunAicpuThreadSupplementNotify` | HCOMM 通信框架核心库 | |
| 334 | | `libscatter_aicpu_kernel.so` | `HcclLaunchAicpuKernel` | AICPU 集合通信算子 kernel 库 | | 330 | | `libscatter_aicpu_kernel.so` | `HcclLaunchAicpuKernel` | AICPU 集合通信算子 kernel 库 | |
| 335 | | `libslog.so` | （日志库依赖） | CANN 安全日志库 | | 331 | | `libslog.so` | （日志库依赖） | CANN 安全日志库 | |
| 336 | | `libc_sec.so` | （安全库依赖） | CANN 安全函数库 | | 332 | | `libc_sec.so` | （安全库依赖） | CANN 安全函数库 | |
| 337 | 333 | ||
| 338 | **依赖风险**： | 334 | **依赖风险**： |
| 339 | - | ||
| 340 | 1. **SO 名称变更**：若 HCOMM 侧重命名或拆分上述 SO（如将 `libccl_kernel.so` 重命名），工具的 `dlopen` 将加载失败，导致所有 AICPU kernel 无法执行 | 335 | 1. **SO 名称变更**：若 HCOMM 侧重命名或拆分上述 SO（如将 `libccl_kernel.so` 重命名），工具的 `dlopen` 将加载失败，导致所有 AICPU kernel 无法执行 |
| 341 | 2. **算子 kernel SO 名称歧义**：当前无论算子类型如何（AllReduce、AllGather、ReduceScatter 等），算子 kernel 的 SO 名称固定为 `libscatter_aicpu_kernel.so`。这一命名可能引起歧义——名称中的 "scatter" 暗示仅适用于 Scatter 算子，但实际上承载了所有集合通信算子的 kernel。若 HCOMM 侧后续按算子类型拆分 SO（如 `liballreduce_aicpu_kernel.so`），工具将无法适配 | 336 | 2. **算子 kernel SO 名称歧义**：当前无论算子类型如何（AllReduce、AllGather、ReduceScatter 等），算子 kernel 的 SO 名称固定为 `libscatter_aicpu_kernel.so`。这一命名可能引起歧义——名称中的 "scatter" 暗示仅适用于 Scatter 算子，但实际上承载了所有集合通信算子的 kernel。若 HCOMM 侧后续按算子类型拆分 SO（如 `liballreduce_aicpu_kernel.so`），工具将无法适配 |
| 342 | 3. **用户自定义算子场景**：HCCL 业务层未对用户自定义算子的 SO 名称做严格规定，用户可能自行编译出私有 SO（如 `libcustom_alltoall_kernel.so`）。工具目前仅识别上述固定 SO 名称，对于用户自定义算子的私有 SO 将无法加载和仿真 | 337 | 3. **用户自定义算子场景**：HCCL 业务层未对用户自定义算子的 SO 名称做严格规定，用户可能自行编译出私有 SO（如 `libcustom_alltoall_kernel.so`）。工具目前仅识别上述固定 SO 名称，对于用户自定义算子的私有 SO 将无法加载和仿真 |
| @@ -355,3 +350,5 @@ struct AivOpArgs { | |||
| 355 | | **AIV 算子 Op** | 8 个头文件（6 个 HCOMM + 2 个 HCCL） | 仅 proxy/aiv_kernel 模块（AIV 算子仿真） | | 350 | | **AIV 算子 Op** | 8 个头文件（6 个 HCOMM + 2 个 HCCL） | 仅 proxy/aiv_kernel 模块（AIV 算子仿真） | |
| 356 | | **AscendC 接口** | 6 个 stub 头文件 | 仅 proxy/aiv_kernel 模块（AIV 算子仿真） | | 351 | | **AscendC 接口** | 6 个 stub 头文件 | 仅 proxy/aiv_kernel 模块（AIV 算子仿真） | |
| 357 | | **AICPU 备份** | 8 个本地结构体 + 6 个 kernel 函数 + 4 个 SO 库名 | 仅 device_arm 模块（AICPU 模式 host-device 交互） | | 352 | | **AICPU 备份** | 8 个本地结构体 + 6 个 kernel 函数 + 4 个 SO 库名 | 仅 device_arm 模块（AICPU 模式 host-device 交互） | |
| 353 | + | ||
| 354 | + | ||
| @@ -7,7 +7,6 @@ | |||
| 7 | This document lists all third-party header files that the project references through `#include`. Each entry marks its origin (CANN / HCCL / HCOMM) and its actual path on disk. | 7 | This document lists all third-party header files that the project references through `#include`. Each entry marks its origin (CANN / HCCL / HCOMM) and its actual path on disk. |
| 8 | 8 | ||
| 9 | **Path Conventions**: | 9 | **Path Conventions**: |
| 10 | - | ||
| 11 | - CANN root directory: `/home/teamserver/workspace/Ascend/cann-9.1.0/` (abbreviated as `CANN/` below) | 10 | - CANN root directory: `/home/teamserver/workspace/Ascend/cann-9.1.0/` (abbreviated as `CANN/` below) |
| 12 | - HCOMM root directory: `/home/teamserver/workspace/hcomm/` (abbreviated as `HCOMM/` below) | 11 | - HCOMM root directory: `/home/teamserver/workspace/hcomm/` (abbreviated as `HCOMM/` below) |
| 13 | - HCCL root directory: `/home/teamserver/workspace/hccl/` (abbreviated as `HCCL/` below) | 12 | - HCCL root directory: `/home/teamserver/workspace/hccl/` (abbreviated as `HCCL/` below) |
| @@ -30,7 +29,7 @@ This document lists all third-party header files that the project references thr | |||
| 30 | 29 | ||
| 31 | | #include | Actual File Path | Reference Locations | | 30 | | #include | Actual File Path | Reference Locations | |
| 32 | |----------|------------|---------| | 31 | |----------|------------|---------| |
| 33 | -| `hccl/hccl_types.h` | `CANN/include/hccl/hccl_types.h` | proxy/aclrt_kernel_stub.cc, hccl_op_stub.cc, hccl_proxy_common.h; plugin/runner/ccu_executor/ccu_fp16.h; common/sim_data_dump.cc; device_arm/proxy/device_sqe_parse_stub.h; test/proxy/ | | 32 | +| `hccl/hccl_types.h` | `CANN/include/hccl/hccl_types.h` | proxy/aclrt_kernel_stub.cc, hccl_op_stub.cc, hccl_proxy_common.h; plugin/runner/ccu_executor/ccu_fp16.h; common/sim_data_dump.cc; device_arm/level2/proxy/device_sqe_parse_stub.h; test/proxy/ | |
| 34 | | `hccl/hccl.h` | `CANN/include/hccl/hccl.h` | **proxy/hccl_op_stub.cc**; test/proxy/hccl_comm_stub_test.cc | | 33 | | `hccl/hccl.h` | `CANN/include/hccl/hccl.h` | **proxy/hccl_op_stub.cc**; test/proxy/hccl_comm_stub_test.cc | |
| 35 | | `hccl/hcom.h` | `CANN/include/hccl/hcom.h` | proxy/hccl_inner_stub.cc; test/proxy/hccl_inner_stub_test.cc | | 34 | | `hccl/hcom.h` | `CANN/include/hccl/hcom.h` | proxy/hccl_inner_stub.cc; test/proxy/hccl_inner_stub_test.cc | |
| 36 | | `hccl/base.h` | `CANN/include/hccl/base.h` | proxy/aclrt_context_stub.cc; plugin/runner/ccu_executor/ccu_fp16.h; plugin/checker/header/external/task_param.h; include/sim_ip_address.h | | 35 | | `hccl/base.h` | `CANN/include/hccl/base.h` | proxy/aclrt_context_stub.cc; plugin/runner/ccu_executor/ccu_fp16.h; plugin/checker/header/external/task_param.h; include/sim_ip_address.h | |
| @@ -49,13 +48,13 @@ This document lists all third-party header files that the project references thr | |||
| 49 | |----------|------------|---------| | 48 | |----------|------------|---------| |
| 50 | | `"runtime/base.h"` | `CANN/pkg_inc/runtime/runtime/base.h` | proxy/hccl_comm_stub.cc, aclrt_runtime_config.cc, aclrt_notify_stub.cc, hccp_stub.cc, aclrt_device_stub.cc, hccp_ra_socket_stub.cc, aclrt_stream_stub.cc, aclrt_stub.cc; test/proxy/ | | 49 | | `"runtime/base.h"` | `CANN/pkg_inc/runtime/runtime/base.h` | proxy/hccl_comm_stub.cc, aclrt_runtime_config.cc, aclrt_notify_stub.cc, hccp_stub.cc, aclrt_device_stub.cc, hccp_ra_socket_stub.cc, aclrt_stream_stub.cc, aclrt_stub.cc; test/proxy/ | |
| 51 | | `"runtime/event.h"` | `CANN/pkg_inc/runtime/runtime/event.h` | proxy/aclrt_runtime_config.cc, aclrt_notify_stub.cc | | 50 | | `"runtime/event.h"` | `CANN/pkg_inc/runtime/runtime/event.h` | proxy/aclrt_runtime_config.cc, aclrt_notify_stub.cc | |
| 52 | -| `"runtime/rt.h"` | `CANN/pkg_inc/runtime/runtime/rt.h` | device_arm/proxy/device_sqe_parse_stub.h | | 51 | +| `"runtime/rt.h"` | `CANN/pkg_inc/runtime/runtime/rt.h` | device_arm/level2/proxy/device_sqe_parse_stub.h | |
| 53 | 52 | ||
| 54 | ### 1.5 CANN Profiling Interfaces (`CANN/pkg_inc/profiling/`) | 53 | ### 1.5 CANN Profiling Interfaces (`CANN/pkg_inc/profiling/`) |
| 55 | 54 | ||
| 56 | | #include | Actual File Path | Reference Locations | | 55 | | #include | Actual File Path | Reference Locations | |
| 57 | |----------|------------|---------| | 56 | |----------|------------|---------| |
| 58 | -| `"aprof_pub.h"` | `CANN/pkg_inc/profiling/aprof_pub.h` | proxy/aprofiling_stub.cc; device_arm/proxy/aprofiling_stub.cc; test/proxy/aprofiling_stub_test.cc | | 57 | +| `"aprof_pub.h"` | `CANN/pkg_inc/profiling/aprof_pub.h` | proxy/aprofiling_stub.cc; device_arm/level2/proxy/aprofiling_stub.cc; test/proxy/aprofiling_stub_test.cc | |
| 59 | 58 | ||
| 60 | ### 1.6 CANN Trace Interfaces (`CANN/pkg_inc/trace/`) | 59 | ### 1.6 CANN Trace Interfaces (`CANN/pkg_inc/trace/`) |
| 61 | 60 | ||
| @@ -68,7 +67,7 @@ This document lists all third-party header files that the project references thr | |||
| 68 | 67 | ||
| 69 | | #include | Actual File Path | Reference Locations | | 68 | | #include | Actual File Path | Reference Locations | |
| 70 | |----------|------------|---------| | 69 | |----------|------------|---------| |
| 71 | -| `"ascend_hal.h"` | `CANN/include/driver/ascend_hal.h` | proxy/adapter_rts_stub.cc, aclrt_new_stub.cc, ascend_hal_stub.cc; device_arm/proxy/device_sqe_parse_stub.h, ascend_hal_stub.cc; test/device_arm/ | | 70 | +| `"ascend_hal.h"` | `CANN/include/driver/ascend_hal.h` | proxy/adapter_rts_stub.cc, aclrt_new_stub.cc, ascend_hal_stub.cc; device_arm/level2/proxy/device_sqe_parse_stub.h, ascend_hal_stub.cc; test/device_arm/ | |
| 72 | 71 | ||
| 73 | ### 1.8 CANN Base Interfaces (`CANN/pkg_inc/base/`) | 72 | ### 1.8 CANN Base Interfaces (`CANN/pkg_inc/base/`) |
| 74 | 73 | ||
| @@ -175,12 +174,11 @@ This section describes the dependencies for AIV (AI Vector Core) operator simula | |||
| 175 | 174 | ||
| 176 | **Original Source**: the `hccl_aiv_utils.h` header file from the HCOMM repository | 175 | **Original Source**: the `hccl_aiv_utils.h` header file from the HCOMM repository |
| 177 | 176 | ||
| 178 | -**Backup Location**: `src/proxy/aclrt_kernel_stub.cc` lines 597-628 | 177 | +**Backup Location**: `src/proxy/level2/aclrt_kernel_stub.cc` lines 597-628 |
| 179 | 178 | ||
| 180 | **Description**: `AivOpArgs` is the parameter structure for AIV collective communication operators. The structure passes AIV kernel execution parameters between the host side and the device side. The tool originally loaded the `ops_hccl::ExecuteKernelLaunch()` function from the HCOMM library through `dlsym` (symbol name: `_ZN8ops_hccl19ExecuteKernelLaunchERKNS_9AivOpArgsE`). That function accepts this structure as a parameter. The tool maintains a complete backup of the structure to remove the dependency on the HCOMM runtime library. | 179 | **Description**: `AivOpArgs` is the parameter structure for AIV collective communication operators. The structure passes AIV kernel execution parameters between the host side and the device side. The tool originally loaded the `ops_hccl::ExecuteKernelLaunch()` function from the HCOMM library through `dlsym` (symbol name: `_ZN8ops_hccl19ExecuteKernelLaunchERKNS_9AivOpArgsE`). That function accepts this structure as a parameter. The tool maintains a complete backup of the structure to remove the dependency on the HCOMM runtime library. |
| 181 | 180 | ||
| 182 | **Structure Definition**: | 181 | **Structure Definition**: |
| 183 | - | ||
| 184 | ```cpp | 182 | ```cpp |
| 185 | struct AivOpArgs { | 183 | struct AivOpArgs { |
| 186 | HcclCMDType cmdType = HcclCMDType::HCCL_CMD_MAX; // collective communication command type | 184 | HcclCMDType cmdType = HcclCMDType::HCCL_CMD_MAX; // collective communication command type |
| @@ -220,7 +218,7 @@ struct AivOpArgs { | |||
| 220 | 218 | ||
| 221 | **Original Source**: the header file with the same name from the HCOMM repository | 219 | **Original Source**: the header file with the same name from the HCOMM repository |
| 222 | 220 | ||
| 223 | -**Backup Location**: `src/proxy/aiv_kernel/hccl_op_stub/aiv_communication_base_v2.h` | 221 | +**Backup Location**: `src/proxy/level2/aiv_kernel/hccl_op_stub/aiv_communication_base_v2.h` |
| 224 | 222 | ||
| 225 | **Description**: This header file defines the base class `AivCommBase` for AIV collective communication operators and the related kernel parameter macros (`KERNEL_ARGS_DEF`, `KERNEL_ARGS_CALL`, and so on). The tool originally included this file directly from the HCOMM repository. The tool maintains a complete backup to remove the dependency. | 223 | **Description**: This header file defines the base class `AivCommBase` for AIV collective communication operators and the related kernel parameter macros (`KERNEL_ARGS_DEF`, `KERNEL_ARGS_CALL`, and so on). The tool originally included this file directly from the HCOMM repository. The tool maintains a complete backup to remove the dependency. |
| 226 | 224 | ||
| @@ -240,7 +238,7 @@ struct AivOpArgs { | |||
| 240 | 238 | ||
| 241 | ### 5.3 AIV Operator Op Header Files (HCOMM / HCCL Dependencies) | 239 | ### 5.3 AIV Operator Op Header Files (HCOMM / HCCL Dependencies) |
| 242 | 240 | ||
| 243 | -Each AIV kernel implementation file for a collective communication operator in the tool (`src/proxy/aiv_kernel/hccl_op_stub/*/aiv_communication_v2.cc`) includes the corresponding operator op header file through `#include`. These header files **do not exist within the project**. They come from the HCOMM and HCCL repositories respectively: | 241 | +Each AIV kernel implementation file for a collective communication operator in the tool (`src/proxy/level2/aiv_kernel/hccl_op_stub/*/aiv_communication_v2.cc`) includes the corresponding operator op header file through `#include`. These header files **do not exist within the project**. They come from the HCOMM and HCCL repositories respectively: |
| 244 | 242 | ||
| 245 | | #include | Source | Actual File Path | Reference Locations | | 243 | | #include | Source | Actual File Path | Reference Locations | |
| 246 | |----------|------|------------|---------| | 244 | |----------|------|------------|---------| |
| @@ -250,14 +248,14 @@ Each AIV kernel implementation file for a collective communication operator in t | |||
| 250 | | `"aiv_broadcast_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/broadcast/` | hccl_op_stub/broadcast/aiv_communication_v2.cc | | 248 | | `"aiv_broadcast_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/broadcast/` | hccl_op_stub/broadcast/aiv_communication_v2.cc | |
| 251 | | `"aiv_all_to_all_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | | 249 | | `"aiv_all_to_all_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | |
| 252 | | `"aiv_all_to_all_v_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | | 250 | | `"aiv_all_to_all_v_op.h"` | HCOMM | `hcomm/src/legacy/ascend910/algorithm/base/alg_aiv_template/all_to_all/` | hccl_op_stub/all_to_all_v/aiv_communication_v2.cc | |
| 253 | -| `"aiv_scatter_op.h"` | HCCL | `hccl/src/ops/scatter/algorithm/template/aiv/kernel/` | hccl_op_stub/scatter/aiv_communication_v2.cc | | 251 | +| `"aiv_scatter_op.h"` | HCCL | `hccl/src/ops/scatter/template/aiv/kernel/` | hccl_op_stub/scatter/aiv_communication_v2.cc | |
| 254 | -| `"aiv_reduce_op.h"` | HCCL | `hccl/src/ops/reduce/algorithm/template/aiv/kernel/` | hccl_op_stub/reduce/aiv_communication_v2.cc | | 252 | +| `"aiv_reduce_op.h"` | HCCL | `hccl/src/ops/reduce/template/aiv/kernel/` | hccl_op_stub/reduce/aiv_communication_v2.cc | |
| 255 | 253 | ||
| 256 | **Dependency Risk**: Changes to the interface signatures, template parameters, or member functions in the operator op header files on the HCOMM or HCCL side will cause AIV kernel compilation failures or lead to inconsistent runtime behavior. | 254 | **Dependency Risk**: Changes to the interface signatures, template parameters, or member functions in the operator op header files on the HCOMM or HCCL side will cause AIV kernel compilation failures or lead to inconsistent runtime behavior. |
| 257 | 255 | ||
| 258 | ### 5.4 AscendC Interface Dependencies | 256 | ### 5.4 AscendC Interface Dependencies |
| 259 | 257 | ||
| 260 | -The tool depends on the AscendC programming interfaces provided by the CANN package to simulate AIV kernel execution. All AscendC interfaces in the project use local stub files under the `src/proxy/aiv_kernel/ascendc_stub/` directory. The tool does not use the AscendC header files from the CANN SDK directly. | 258 | +The tool depends on the AscendC programming interfaces provided by the CANN package to simulate AIV kernel execution. All AscendC interfaces in the project use local stub files under the `src/proxy/level2/aiv_kernel/ascendc_stub/` directory. The tool does not use the AscendC header files from the CANN SDK directly. |
| 261 | 259 | ||
| 262 | **AscendC Interfaces in Use** (implemented through stubs): | 260 | **AscendC Interfaces in Use** (implemented through stubs): |
| 263 | 261 | ||
| @@ -284,11 +282,11 @@ This section describes the dependencies for simulating host-device interaction i | |||
| 284 | 282 | ||
| 285 | ### 6.1 AICPU Interaction Data Structures | 283 | ### 6.1 AICPU Interaction Data Structures |
| 286 | 284 | ||
| 287 | -The tool maintains local backups of the following data structures in `src/device_arm/aicpu_args_stub.h` to remove the strong dependency on the HCOMM runtime library: | 285 | +The tool maintains local backups of the following data structures in `src/device_arm/common/aicpu_args_stub.h` to remove the strong dependency on the HCOMM runtime library: |
| 288 | 286 | ||
| 289 | | Data Structure | Purpose | Corresponding Kernel Function | | 287 | | Data Structure | Purpose | Corresponding Kernel Function | |
| 290 | |---------|------|-----------------| | 288 | |---------|------|-----------------| |
| 291 | -| `CommAicpuParam` | Communication domain initialization parameters (hcomId, device ID, H2D/D2H transfer parameters) | `RunAicpuCommInit` | | 289 | +| `CommAicpuParam` | Communication domain initialization parameters (hcomId, device ID, H2D/D2H transfer parameters) | `RunAicpuIndOpCommInit` | |
| 292 | | `HDCommunicateParams` | H2D/D2H control transfer parameters (deviceAddr, readCacheAddr) | referenced by `CommAicpuParam` | | 290 | | `HDCommunicateParams` | H2D/D2H control transfer parameters (deviceAddr, readCacheAddr) | referenced by `CommAicpuParam` | |
| 293 | | `ThreadMgrAicpuParam` | Thread management parameters (threadNum, serialized threadParam array, deviceHandle) | `RunAicpuIndOpThreadInit`, `RunAicpuThreadSupplementNotify` | | 291 | | `ThreadMgrAicpuParam` | Thread management parameters (threadNum, serialized threadParam array, deviceHandle) | `RunAicpuIndOpThreadInit`, `RunAicpuThreadSupplementNotify` | |
| 294 | | `AicpuTsThread` | AICPU thread information (streamType, notifyLoadType, devId) | referenced by `ThreadMgrAicpuParam` | | 292 | | `AicpuTsThread` | AICPU thread information (streamType, notifyLoadType, devId) | referenced by `ThreadMgrAicpuParam` | |
| @@ -311,15 +309,14 @@ The tool dynamically loads the following kernel functions from the HCOMM runtime | |||
| 311 | 309 | ||
| 312 | | dlsym Symbol Name | Library | Function | | 310 | | dlsym Symbol Name | Library | Function | |
| 313 | |-------------|-------|------| | 311 | |-------------|-------|------| |
| 314 | -| `RunAicpuCommInit` | `libccl_kernel.so` | AICPU communication domain initialization | | 312 | +| `RunAicpuIndOpCommInit` | `libccl_kernel.so` | AICPU communication domain initialization | |
| 315 | | `RunAicpuIndOpThreadInit` | `libccl_kernel.so` | AICPU thread initialization | | 313 | | `RunAicpuIndOpThreadInit` | `libccl_kernel.so` | AICPU thread initialization | |
| 316 | | `RunAicpuIndOpChannelInitV2` | `libccl_kernel.so` | AICPU channel initialization V2 | | 314 | | `RunAicpuIndOpChannelInitV2` | `libccl_kernel.so` | AICPU channel initialization V2 | |
| 317 | -| `RunAicpuDfxInitV2` | `libccl_kernel.so` | AICPU DFX operator information initialization V2 | | 315 | +| `RunAicpuDfxOpInfoInitV2` | `libccl_kernel.so` | AICPU DFX operator information initialization V2 | |
| 318 | | `RunAicpuThreadSupplementNotify` | `libccl_kernel.so` | AICPU resource supplement notification | | 316 | | `RunAicpuThreadSupplementNotify` | `libccl_kernel.so` | AICPU resource supplement notification | |
| 319 | | `HcclLaunchAicpuKernel` | `libscatter_aicpu_kernel.so` | AICPU collective communication kernel launch | | 317 | | `HcclLaunchAicpuKernel` | `libscatter_aicpu_kernel.so` | AICPU collective communication kernel launch | |
| 320 | 318 | ||
| 321 | **Dependency Risks**: | 319 | **Dependency Risks**: |
| 322 | - | ||
| 323 | 1. Changes to the signatures or behavior of the above kernel functions on the HCOMM side may cause `dlsym` calls to fail or produce incorrect results | 320 | 1. Changes to the signatures or behavior of the above kernel functions on the HCOMM side may cause `dlsym` calls to fail or produce incorrect results |
| 324 | 2. If `libccl_kernel.so` or `libscatter_aicpu_kernel.so` adds or removes kernel functions, the tool must update its function pointer list accordingly | 321 | 2. If `libccl_kernel.so` or `libscatter_aicpu_kernel.so` adds or removes kernel functions, the tool must update its function pointer list accordingly |
| 325 | 3. Inconsistencies between the locally backed-up structures (`CommAicpuParam`, `ThreadMgrAicpuParam`, `HcclChannelUrmaRes`, and so on) and the actual definitions on the HCOMM side will cause address translation errors and memory out-of-bounds access | 322 | 3. Inconsistencies between the locally backed-up structures (`CommAicpuParam`, `ThreadMgrAicpuParam`, `HcclChannelUrmaRes`, and so on) and the actual definitions on the HCOMM side will cause address translation errors and memory out-of-bounds access |
| @@ -330,13 +327,12 @@ The `InitKernelFuncHandle` function hardcodes the following SO library names to | |||
| 330 | 327 | ||
| 331 | | Hardcoded SO Name | Loaded dlsym Symbols | Description | | 328 | | Hardcoded SO Name | Loaded dlsym Symbols | Description | |
| 332 | |--------------|-----------------|------| | 329 | |--------------|-----------------|------| |
| 333 | -| `libccl_kernel.so` | `RunAicpuCommInit`, `RunAicpuIndOpThreadInit`, `RunAicpuIndOpChannelInitV2`, `RunAicpuDfxInitV2`, `RunAicpuThreadSupplementNotify` | HCOMM communication framework core library | | 330 | +| `libccl_kernel.so` | `RunAicpuIndOpCommInit`, `RunAicpuIndOpThreadInit`, `RunAicpuIndOpChannelInitV2`, `RunAicpuDfxOpInfoInitV2`, `RunAicpuThreadSupplementNotify` | HCOMM communication framework core library | |
| 334 | | `libscatter_aicpu_kernel.so` | `HcclLaunchAicpuKernel` | AICPU collective communication operator kernel library | | 331 | | `libscatter_aicpu_kernel.so` | `HcclLaunchAicpuKernel` | AICPU collective communication operator kernel library | |
| 335 | | `libslog.so` | (logging library dependency) | CANN secure logging library | | 332 | | `libslog.so` | (logging library dependency) | CANN secure logging library | |
| 336 | | `libc_sec.so` | (security library dependency) | CANN security function library | | 333 | | `libc_sec.so` | (security library dependency) | CANN security function library | |
| 337 | 334 | ||
| 338 | **Dependency Risks**: | 335 | **Dependency Risks**: |
| 339 | - | ||
| 340 | 1. **SO Name Changes**: If the HCOMM side renames or splits the above SOs (for example, renaming `libccl_kernel.so`), the `dlopen` call in the tool will fail. This causes all AICPU kernels to become non-executable | 336 | 1. **SO Name Changes**: If the HCOMM side renames or splits the above SOs (for example, renaming `libccl_kernel.so`), the `dlopen` call in the tool will fail. This causes all AICPU kernels to become non-executable |
| 341 | 2. **Operator Kernel SO Name Ambiguity**: The operator kernel SO name is fixed as `libscatter_aicpu_kernel.so` regardless of the operator type (AllReduce, AllGather, ReduceScatter, and so on). The "scatter" in the name suggests applicability only to the Scatter operator, but the library actually hosts kernels for all collective communication operators. If the HCOMM side later splits SOs by operator type (for example, `liballreduce_aicpu_kernel.so`), the tool will fail to adapt | 337 | 2. **Operator Kernel SO Name Ambiguity**: The operator kernel SO name is fixed as `libscatter_aicpu_kernel.so` regardless of the operator type (AllReduce, AllGather, ReduceScatter, and so on). The "scatter" in the name suggests applicability only to the Scatter operator, but the library actually hosts kernels for all collective communication operators. If the HCOMM side later splits SOs by operator type (for example, `liballreduce_aicpu_kernel.so`), the tool will fail to adapt |
| 342 | 3. **User-Defined Operator Scenarios**: The HCCL business layer does not strictly define SO names for user-defined operators. Users may compile private SOs (for example, `libcustom_alltoall_kernel.so`). The tool currently recognizes only the fixed SO names listed above. The tool cannot load or simulate private SOs for user-defined operators | 338 | 3. **User-Defined Operator Scenarios**: The HCCL business layer does not strictly define SO names for user-defined operators. Users may compile private SOs (for example, `libcustom_alltoall_kernel.so`). The tool currently recognizes only the fixed SO names listed above. The tool cannot load or simulate private SOs for user-defined operators |
| @@ -355,3 +351,5 @@ The `InitKernelFuncHandle` function hardcodes the following SO library names to | |||
| 355 | | **AIV Operator Ops** | 8 header files (6 from HCOMM + 2 from HCCL) | proxy/aiv_kernel module only (AIV operator simulation) | | 351 | | **AIV Operator Ops** | 8 header files (6 from HCOMM + 2 from HCCL) | proxy/aiv_kernel module only (AIV operator simulation) | |
| 356 | | **AscendC Interfaces** | 6 stub header files | proxy/aiv_kernel module only (AIV operator simulation) | | 352 | | **AscendC Interfaces** | 6 stub header files | proxy/aiv_kernel module only (AIV operator simulation) | |
| 357 | | **AICPU Backup** | 8 local structures + 6 kernel functions + 4 SO library names | device_arm module only (AICPU mode host-device interaction) | | 353 | | **AICPU Backup** | 8 local structures + 6 kernel functions + 4 SO library names | device_arm module only (AICPU mode host-device interaction) | |
| 354 | + | ||
| 355 | + | ||
| @@ -21,8 +21,8 @@ HVRM Insight V3 当前主要支持 `DAGView` 任务图查看。本文档围绕 D | |||
| 21 | ... | 21 | ... |
| 22 | "setting": { // Checker插件配置项 | 22 | "setting": { // Checker插件配置项 |
| 23 | ... | 23 | ... |
| 24 | - "enable_insight_dump": true, // 是否启用可视化数据输出（默认为关闭状态） | 24 | + "enable_insight_dump": true, // 是否输出 Checker V3 Insight 数据（默认为关闭状态） |
| 25 | - "enable_memory_snapshot_dump": false // 是否启用可视化内存快照数据输出（默认关闭，仅支持老Checker，需要先开启可视化数据输出"enable_insight_dump"） | 25 | + "enable_memory_snapshot_dump": false // 是否输出内存快照数据（暂不支持，默认关闭，需要先开启 Insight 数据输出） |
| 26 | } | 26 | } |
| 27 | } | 27 | } |
| 28 | ``` | 28 | ``` |
| @@ -55,34 +55,34 @@ Insight V3 当前前端源码位于： | |||
| 55 | 55 | ||
| 56 | 推荐步骤如下： | 56 | 推荐步骤如下： |
| 57 | 57 | ||
| 58 | -1. 进入前端目录。 | 58 | +1. 进入前端目录： |
| 59 | 59 | ||
| 60 | - ```bash | 60 | +```bash |
| 61 | - cd {hccl_vm目录}/src/plugin/insight/frontend_v3 | 61 | +cd {hccl_vm目录}/src/plugin/insight/frontend_v3 |
| 62 | - ``` | 62 | +``` |
| 63 | 63 | ||
| 64 | -2. 安装前端依赖。 | 64 | +2. 安装前端依赖： |
| 65 | 65 | ||
| 66 | - ```bash | 66 | +```bash |
| 67 | - npm install | 67 | +npm install |
| 68 | - ``` | 68 | +``` |
| 69 | 69 | ||
| 70 | -3. 编译前端产物。 | 70 | +3. 编译前端产物： |
| 71 | 71 | ||
| 72 | - ```bash | 72 | +```bash |
| 73 | - npm run build | 73 | +npm run build |
| 74 | - ``` | 74 | +``` |
| 75 | 75 | ||
| 76 | - 前端编译完成后，会生成 `src/plugin/insight/dist/` 目录。 | 76 | +前端编译完成后，会生成 `src/plugin/insight/dist/` 目录。 |
| 77 | 77 | ||
| 78 | -4. 回到 HCCL_VM 根目录，执行 `build.sh` 编译并安装 hccl-vm。 | 78 | +4. 回到 HCCL_VM 根目录，执行 `build.sh` 编译并安装 hccl-vm： |
| 79 | 79 | ||
| 80 | - ```bash | 80 | +```bash |
| 81 | - cd {HCCL_VM目录} | 81 | +cd {HCCL_VM目录} |
| 82 | - bash build.sh --package-path <ASCEND_CANN_PATH> --hcomm-path <HCOMM_CODE_PATH> | 82 | +bash build.sh --package-path <ASCEND_CANN_PATH> --hcomm-path <HCOMM_CODE_PATH> |
| 83 | - ``` | 83 | +``` |
| 84 | 84 | ||
| 85 | - 如需打包安装目录，可附加 `--pkg`。若当前构建场景要求 AICPU / AIV / FULL 模式，可按项目常规流程追加 `--aicpu`、`--aiv` 或 `--full` 参数。 | 85 | +如需打包安装目录，可附加 `--pkg`。若当前构建场景要求 AICPU / AIV / FULL 模式，可按项目常规流程追加 `--aicpu`、`--aiv` 或 `--full` 参数。 |
| 86 | 86 | ||
| 87 | --- | 87 | --- |
| 88 | 88 | ||
| @@ -21,8 +21,8 @@ To configure Checker data output, confirm that the Checker configuration file ha | |||
| 21 | ... | 21 | ... |
| 22 | "setting": { // Checker plugin configuration items | 22 | "setting": { // Checker plugin configuration items |
| 23 | ... | 23 | ... |
| 24 | - "enable_insight_dump": true, // Enable or disable visualization data output (disabled by default) | 24 | + "enable_insight_dump": true, // Whether to output Checker V3 Insight data (disabled by default) |
| 25 | - "enable_memory_snapshot_dump": false // Enable or disable visualization memory snapshot data output (disabled by default, only supported by legacy Checker, requires enabling visualization data output "enable_insight_dump" first) | 25 | + "enable_memory_snapshot_dump": false // Whether to output memory snapshot data (not yet supported, disabled by default, requires enabling Insight data output first) |
| 26 | } | 26 | } |
| 27 | } | 27 | } |
| 28 | ``` | 28 | ``` |
| @@ -55,34 +55,34 @@ Note: Compiling the Insight visualization frontend requires `Node.js` and `npm`. | |||
| 55 | 55 | ||
| 56 | The recommended steps are as follows: | 56 | The recommended steps are as follows: |
| 57 | 57 | ||
| 58 | -1. Navigate to the frontend directory. | 58 | +1. Navigate to the frontend directory: |
| 59 | 59 | ||
| 60 | - ```bash | 60 | +```bash |
| 61 | - cd {hccl_vm directory}/src/plugin/insight/frontend_v3 | 61 | +cd {hccl_vm directory}/src/plugin/insight/frontend_v3 |
| 62 | - ``` | 62 | +``` |
| 63 | 63 | ||
| 64 | -2. Install frontend dependencies. | 64 | +2. Install frontend dependencies: |
| 65 | 65 | ||
| 66 | - ```bash | 66 | +```bash |
| 67 | - npm install | 67 | +npm install |
| 68 | - ``` | 68 | +``` |
| 69 | 69 | ||
| 70 | -3. Build the frontend output. | 70 | +3. Build the frontend output: |
| 71 | 71 | ||
| 72 | - ```bash | 72 | +```bash |
| 73 | - npm run build | 73 | +npm run build |
| 74 | - ``` | 74 | +``` |
| 75 | 75 | ||
| 76 | - After the frontend compilation completes, the system generates the `src/plugin/insight/dist/` directory. | 76 | +After the frontend compilation completes, the system generates the `src/plugin/insight/dist/` directory. |
| 77 | 77 | ||
| 78 | -4. Return to the HCCL_VM root directory, and run `build.sh` to compile and install hccl-vm. | 78 | +4. Return to the HCCL_VM root directory, and run `build.sh` to compile and install hccl-vm: |
| 79 | 79 | ||
| 80 | - ```bash | 80 | +```bash |
| 81 | - cd {HCCL_VM directory} | 81 | +cd {HCCL_VM directory} |
| 82 | - bash build.sh --package-path <ASCEND_CANN_PATH> --hcomm-path <HCOMM_CODE_PATH> | 82 | +bash build.sh --package-path <ASCEND_CANN_PATH> --hcomm-path <HCOMM_CODE_PATH> |
| 83 | - ``` | 83 | +``` |
| 84 | 84 | ||
| 85 | - To package the installation directory, append `--pkg`. If the current build scenario requires AICPU / AIV / FULL mode, append the `--aicpu`, `--aiv`, or `--full` parameter following the standard project procedure. | 85 | +To package the installation directory, append `--pkg`. If the current build scenario requires AICPU / AIV / FULL mode, append the `--aicpu`, `--aiv`, or `--full` parameter following the standard project procedure. |
| 86 | 86 | ||
| 87 | --- | 87 | --- |
| 88 | 88 | ||
| @@ -1,226 +0,0 @@ | |||
| 1 | -# HVRM Insight 使用指南 | ||
| 2 | - | ||
| 3 | -## 1. 概述 | ||
| 4 | - | ||
| 5 | -HVRM Insight 是一个分布式算子可视化分析工具，用于对 HCCL 集合通信算子执行过程中的图结构、内存快照和校验报错信息进行统一展示和联动分析。工具包含三个页面： | ||
| 6 | - | ||
| 7 | -| 页面 | 标签 | 用途 | | ||
| 8 | -|------|------|------| | ||
| 9 | -| **Dashboard** | 总览 | 浏览数据集、选择 Rank、查看统计信息，是使用工具的起点 | | ||
| 10 | -| **MemView** | 关联 | 内存时间线与 DAG 任务图联动展示，是排查问题的核心分析页面 | | ||
| 11 | -| **Analytic** | 报错 | 集中展示校验错误，支持一键跳转到问题现场 | | ||
| 12 | - | ||
| 13 | ---- | ||
| 14 | - | ||
| 15 | -## 2. 安装与启动 | ||
| 16 | - | ||
| 17 | -### 2.1 运行前准备 | ||
| 18 | - | ||
| 19 | -HVRM Insight 依赖 Checker 插件输出的数据文件。使用前，请确保已完整运行过 Checker，并在 Checker 的配置文件中开启以下两个数据输出开关： | ||
| 20 | - | ||
| 21 | -Checker配置文件默认位于`/path/to/hccl_vm_install/plugin/checker/manifest.json`。 | ||
| 22 | - | ||
| 23 | -```json | ||
| 24 | -"setting": { | ||
| 25 | - "enable_insight_dump": true, | ||
| 26 | - "enable_memory_snapshot_dump": true | ||
| 27 | -} | ||
| 28 | -``` | ||
| 29 | - | ||
| 30 | -- **`enable_insight_dump`**：开启后，Checker 运行时会输出 Insight 所需的图结构和校验结果文件。 | ||
| 31 | -- **`enable_memory_snapshot_dump`**：开启后，Checker 运行时会输出各 Rank 的内存快照文件。 | ||
| 32 | - | ||
| 33 | -这两个开关默认为 `false`。如未开启，Insight 工具将无法读取到完整的分析数据。 | ||
| 34 | - | ||
| 35 | -### 2.2 数据目录结构 | ||
| 36 | - | ||
| 37 | -工具从 `data/` 目录读取数据集，每个数据集是一个独立的子目录： | ||
| 38 | - | ||
| 39 | -```text | ||
| 40 | -data/ | ||
| 41 | -└── <dataset_name>/ | ||
| 42 | - ├── manifest.json # 数据集元信息（必须） | ||
| 43 | - ├── graph/ # 图结构文件（DAG） | ||
| 44 | - ├── memory/ # 内存快照文件 | ||
| 45 | - └── validation/ | ||
| 46 | - └── issues.msgpack # 校验报错记录（可选） | ||
| 47 | -``` | ||
| 48 | - | ||
| 49 | -缺少可选目录时，对应页面会显示空状态。 | ||
| 50 | - | ||
| 51 | -### 2.3 方式一：通过 hccl-vm 插件（推荐） | ||
| 52 | - | ||
| 53 | -```bash | ||
| 54 | -# 安装并启动后端服务 | ||
| 55 | -hccl-vm plugin install insight | ||
| 56 | -# 安装成功后，终端会输出访问地址（默认 `http://localhost:8000`），在浏览器中打开即可使用。 | ||
| 57 | - | ||
| 58 | -# 再次查看访问地址 | ||
| 59 | -hccl-vm plugin run @insight | ||
| 60 | - | ||
| 61 | -# 卸载插件（会同时停止后端服务） | ||
| 62 | -hccl-vm plugin uninstall @insight | ||
| 63 | -``` | ||
| 64 | - | ||
| 65 | -### 2.4 方式二：本地直接启动 | ||
| 66 | - | ||
| 67 | -```bash | ||
| 68 | -# 进入 insight 插件目录 | ||
| 69 | -python3 serve.py | ||
| 70 | -``` | ||
| 71 | - | ||
| 72 | -后端默认监听 `http://localhost:8000`，同时提供前端页面和数据 API。 | ||
| 73 | - | ||
| 74 | -### 2.5 验证启动成功 | ||
| 75 | - | ||
| 76 | -浏览器打开地址后，如果能看到 Dashboard 页面的数据集列表，说明启动成功。如果页面空白，请检查 `data/` 目录是否存在且包含 `manifest.json` 文件。 | ||
| 77 | - | ||
| 78 | ---- | ||
| 79 | - | ||
| 80 | -## 3. 场景演示 | ||
| 81 | - | ||
| 82 | -### 3.1 场景一：选择数据集并查看概览（Dashboard） | ||
| 83 | - | ||
| 84 | -浏览所有数据集，点击选择目标数据集，查看算子信息和统计数据，然后进入关联分析页面。 | ||
| 85 | - | ||
| 86 | - | ||
| 87 | - | ||
| 88 | -### 3.2 场景二：联动分析内存与任务图（MemView） | ||
| 89 | - | ||
| 90 | -通过内存时间线和 DAG 任务图的联动，定位某个 Step 的内存操作、查看节点详情、追溯数据来源。 | ||
| 91 | - | ||
| 92 | - | ||
| 93 | - | ||
| 94 | -### 3.3 场景三：查看报错并跳转到问题现场（Analytic） | ||
| 95 | - | ||
| 96 | -在 Analytic 页面或 MemView 左侧报错列表中查看错误详情，然后一键跳转到关联的 DAG 节点或内存上下文。 | ||
| 97 | - | ||
| 98 | - | ||
| 99 | - | ||
| 100 | ---- | ||
| 101 | - | ||
| 102 | -## 4.页面详解 | ||
| 103 | - | ||
| 104 | -### 4.1 Dashboard 页面 | ||
| 105 | - | ||
| 106 | -Dashboard 是使用工具的起点，页面从左到右分为三个区域： | ||
| 107 | - | ||
| 108 | -- **左侧**：算子摘要和 Rank 树。Rank 树支持勾选 / 取消勾选，选择结果会同步到所有页面。 | ||
| 109 | -- **中间**：数据集列表。点击某一行选中数据集，左侧和右侧同步刷新。 | ||
| 110 | -- **右侧**：选中数据集的详情面板，展示统计信息和跳转按钮。 | ||
| 111 | - | ||
| 112 | - | ||
| 113 | - | ||
| 114 | -选中数据集后，右侧展示详细统计： | ||
| 115 | - | ||
| 116 | - | ||
| 117 | - | ||
| 118 | -详情面板底部提供两个跳转入口： | ||
| 119 | - | ||
| 120 | -- **`进入关联视图`**：跳转到 MemView，带入当前数据集和 Rank 选择。 | ||
| 121 | -- **`查看诊断`**：跳转到 Analytic，查看该数据集的所有校验错误。 | ||
| 122 | - | ||
| 123 | -### 4.2 MemView 页面 | ||
| 124 | - | ||
| 125 | -MemView 将内存时间线和 DAG 任务图放在同一视图中联动展示，是排查问题的核心页面。 | ||
| 126 | - | ||
| 127 | - | ||
| 128 | - | ||
| 129 | -**页面布局：** | ||
| 130 | - | ||
| 131 | -- **左侧**：Rank 选择树、报错列表（Issues）、搜索面板。 | ||
| 132 | -- **上半区**：内存时间线 —— 按 Step 展示各 Rank 的 buffer 状态演化。 | ||
| 133 | -- **下半区**：DAG 任务图 —— 按 Rank / Stream 泳道展示任务节点和依赖关系。 | ||
| 134 | -- **右侧**：详情栏 —— 展示当前选中 Step 或节点的详细信息。 | ||
| 135 | - | ||
| 136 | - | ||
| 137 | - | ||
| 138 | -#### 4.3 核心联动机制：Step、Rank、Task 三者联动 | ||
| 139 | - | ||
| 140 | -- 在时间线上选中一个 Step → DAG 同步高亮对应节点。 | ||
| 141 | -- 在 DAG 中点击一个节点 → 右侧切到节点详情。 | ||
| 142 | -- 从右侧点击跳转按钮 → 时间线同步定位到对应 Step。 | ||
| 143 | - | ||
| 144 | -**右侧详情栏**有两种模式，根据是否选中 DAG 节点自动切换： | ||
| 145 | - | ||
| 146 | -- **Step 详情模式**：展示当前 Step 的内存操作列表（Task Memory Ops）和 Buffer 切片详情（Buffer Layout）。Buffer Layout 中的每个切片可展开查看数据来源语义，支持点击跳转。 | ||
| 147 | -- **节点详情模式**：展示节点属性、父/子节点关系、跨阶段节点映射、以及该节点关联的内存操作。 | ||
| 148 | - | ||
| 149 | -**其他功能：** | ||
| 150 | - | ||
| 151 | -- **底部播放器**：支持按 Step 顺序浏览，可拖拽滑块快速跳转，播放速度为 1 秒 / Step。 | ||
| 152 | -- **Focus 视图**：选中 Step 后 DAG 自动进入局部视图，只显示当前 Step 相关节点及其上下文，适合节点密集时缩小范围。 | ||
| 153 | -- **子图浏览**：节点包含子图时，可在右侧点击 `打开子图` 进入查看更细粒度的任务结构。 | ||
| 154 | -- **Stage 切换**：通过顶部下拉框切换图阶段（如 `input_graph` / `input_task_queues`） | ||
| 155 | -- **搜索**：支持按 taskId / taskType / notifyId 搜索 DAG 节点，结果点击即定位。 | ||
| 156 | - | ||
| 157 | -### 4.4 Analytic 页面 | ||
| 158 | - | ||
| 159 | -Analytic 集中展示校验过程中发现的所有错误，页面分为三个区域： | ||
| 160 | - | ||
| 161 | - | ||
| 162 | - | ||
| 163 | -- **左侧**：数据集与 Rank 选择。 | ||
| 164 | -- **中间**：报错列表，每条错误展示标题、严重程度标签（颜色区分）、关键字段摘要和原始错误码。 | ||
| 165 | -- **右侧**：选中错误的详情面板，包括错误基本信息、涉及的 Rank、关联节点信息、Slice 详情（精确定位数据不一致的内存位置）、补充信息和原始 JSON。 | ||
| 166 | - | ||
| 167 | -详情面板顶部提供跳转按钮： | ||
| 168 | - | ||
| 169 | -- **`查看 memView`**：跳转到 MemView，定位到错误关联的内存 Step。 | ||
| 170 | -- **`查看 dagView`**：跳转到 MemView，直接定位到错误关联的 DAG 节点。 | ||
| 171 | - | ||
| 172 | -跳转时自动带入数据集、Rank 和节点位置信息，到达 MemView 后无需重新选择即可开始排查。 | ||
| 173 | - | ||
| 174 | ---- | ||
| 175 | - | ||
| 176 | -## 5. 常见使用流程 | ||
| 177 | - | ||
| 178 | -### 5.1 从 Dashboard 开始分析 | ||
| 179 | - | ||
| 180 | -1. 在数据集列表中点击目标数据集。 | ||
| 181 | -2. 在左侧 Rank 树中勾选需要查看的 Rank。 | ||
| 182 | -3. 在右侧确认数据集的文件数量和错误统计。 | ||
| 183 | -4. 点击 `进入关联视图` 进入 MemView。 | ||
| 184 | - | ||
| 185 | -### 5.2 在 MemView 中定位内存问题 | ||
| 186 | - | ||
| 187 | -1. 在时间线上点击目标 Step（或使用底部播放器跳转） | ||
| 188 | -2. 在右侧 `Task Memory Ops` 查看该 Step 的内存操作。 | ||
| 189 | -3. 在 `Buffer Layout` 中选择 Rank 和 Buffer Type，展开 slice 查看数据来源。 | ||
| 190 | -4. 点击 `定位 Task` 或语义卡片跳转到对应 DAG 节点。 | ||
| 191 | - | ||
| 192 | -### 5.3 从 DAG 节点反查内存 | ||
| 193 | - | ||
| 194 | -1. 在 DAG 中点击目标节点。 | ||
| 195 | -2. 在右侧查看 `节点 Memory Ops`。 | ||
| 196 | -3. 点击 `跳转到 Step` 回到时间线，或点击父子节点继续沿依赖链排查。 | ||
| 197 | - | ||
| 198 | -### 5.4 从报错快速定位 | ||
| 199 | - | ||
| 200 | -1. 在 Analytic 页面或 MemView 左侧报错列表中点击一条错误。 | ||
| 201 | -2. 查看详情中的关联节点和 Slice 信息。 | ||
| 202 | -3. 点击 `查看 memView` 或 `查看 dagView` 跳转到问题现场。 | ||
| 203 | - | ||
| 204 | ---- | ||
| 205 | - | ||
| 206 | -## 6. 使用技巧 | ||
| 207 | - | ||
| 208 | -- 先在 Dashboard 确认数据集和 Rank 范围，再进入 MemView 分析。 | ||
| 209 | -- 看内存问题时，优先用 `Task Memory Ops` 和 `Buffer Layout` 配合排查。 | ||
| 210 | -- 看节点依赖关系时，优先在 DAG 中选中节点，再用右侧父子关系和节点映射跳转。 | ||
| 211 | -- DAG 节点太密集时，先用 Focus 视图缩小范围，再逐步展开。 | ||
| 212 | -- MemView 左侧的报错列表可以在不离开当前页面的情况下快速浏览并定位错误节点。 | ||
| 213 | - | ||
| 214 | ---- | ||
| 215 | - | ||
| 216 | -## 7. 术语说明 | ||
| 217 | - | ||
| 218 | -| 术语 | 说明 | | ||
| 219 | -|------|------| | ||
| 220 | -| **Rank** | 单个运算单元（如 NPU），每个 Rank 独立执行计算 | | ||
| 221 | -| **DAG** | 有向无环图，描述任务节点之间的依赖关系 | | ||
| 222 | -| **Step** | 时间线中的一个时间步，对应一次内存状态快照 | | ||
| 223 | -| **Buffer** | 内存中的一块数据区域，用于存放计算或通信的数据 | | ||
| 224 | -| **Stage** | 图的一个阶段，如 `input_graph` 或 `input_task_queues` | | ||
| 225 | -| **Stream** | 任务执行流，同一个 Stream 内的任务按顺序执行 | | ||
| 226 | -| **Issue** | 校验过程中发现的错误或异常记录 | | ||
| @@ -1,226 +0,0 @@ | |||
| 1 | -# HVRM Insight User Guide | ||
| 2 | - | ||
| 3 | -## 1. Overview | ||
| 4 | - | ||
| 5 | -HVRM Insight is a distributed operator visualization and analysis tool. The tool displays graph structures, memory snapshots, and validation error information from HCCL collective communication operator execution in a unified view. The tool includes three pages: | ||
| 6 | - | ||
| 7 | -| Page | Tab | Purpose | | ||
| 8 | -|------|------|------| | ||
| 9 | -| **Dashboard** | Overview | Browse datasets, select ranks, and view statistics. This page is the starting point of the tool. | | ||
| 10 | -| **MemView** | Correlation | Display the memory timeline and DAG task graph together. This page is the core analysis page for troubleshooting. | | ||
| 11 | -| **Analytic** | Error | Display validation errors in a centralized view. Support one-click navigation to the problem context. | | ||
| 12 | - | ||
| 13 | ---- | ||
| 14 | - | ||
| 15 | -## 2. Installation and Startup | ||
| 16 | - | ||
| 17 | -### 2.1 Prerequisites | ||
| 18 | - | ||
| 19 | -HVRM Insight depends on data files generated by the Checker plugin. Before use, make sure the Checker has completed execution. Enable the following two data output switches in the Checker configuration file: | ||
| 20 | - | ||
| 21 | -The Checker configuration file is located at `/path/to/hccl_vm_install/plugin/checker/manifest.json` by default. | ||
| 22 | - | ||
| 23 | -```json | ||
| 24 | -"setting": { | ||
| 25 | - "enable_insight_dump": true, | ||
| 26 | - "enable_memory_snapshot_dump": true | ||
| 27 | -} | ||
| 28 | -``` | ||
| 29 | - | ||
| 30 | -- **`enable_insight_dump`**: When enabled, the Checker outputs graph structure and validation result files required by Insight during execution. | ||
| 31 | -- **`enable_memory_snapshot_dump`**: When enabled, the Checker outputs memory snapshot files for each rank during execution. | ||
| 32 | - | ||
| 33 | -Both switches default to `false`. If not enabled, the Insight tool cannot read complete analysis data. | ||
| 34 | - | ||
| 35 | -### 2.2 Data Directory Structure | ||
| 36 | - | ||
| 37 | -The tool reads datasets from the `data/` directory. Each dataset is a separate subdirectory: | ||
| 38 | - | ||
| 39 | -```text | ||
| 40 | -data/ | ||
| 41 | -└── <dataset_name>/ | ||
| 42 | - ├── manifest.json # Dataset metadata (required) | ||
| 43 | - ├── graph/ # Graph structure files (DAG) | ||
| 44 | - ├── memory/ # Memory snapshot files | ||
| 45 | - └── validation/ | ||
| 46 | - └── issues.msgpack # Validation error records (optional) | ||
| 47 | -``` | ||
| 48 | - | ||
| 49 | -When an optional directory is missing, the corresponding page displays an empty state. | ||
| 50 | - | ||
| 51 | -### 2.3 Method 1: Through hccl-vm Plugin (Recommended) | ||
| 52 | - | ||
| 53 | -```bash | ||
| 54 | -# Install and start the backend service | ||
| 55 | -hccl-vm plugin install insight | ||
| 56 | -# After successful installation, the terminal outputs the access URL (default `http://localhost:8000`). Open it in a browser to use the tool. | ||
| 57 | - | ||
| 58 | -# View the access URL again | ||
| 59 | -hccl-vm plugin run @insight | ||
| 60 | - | ||
| 61 | -# Uninstall the plugin (this also stops the backend service) | ||
| 62 | -hccl-vm plugin uninstall @insight | ||
| 63 | -``` | ||
| 64 | - | ||
| 65 | -### 2.4 Method 2: Local Direct Startup | ||
| 66 | - | ||
| 67 | -```bash | ||
| 68 | -# Enter the insight plugin directory | ||
| 69 | -python3 serve.py | ||
| 70 | -``` | ||
| 71 | - | ||
| 72 | -The backend listens on `http://localhost:8000` by default. The backend provides both the frontend page and the data API. | ||
| 73 | - | ||
| 74 | -### 2.5 Verify Successful Startup | ||
| 75 | - | ||
| 76 | -Open the URL in a browser. If the dataset list on the Dashboard page is visible, the startup is successful. If the page is blank, check whether the `data/` directory exists and contains a `manifest.json` file. | ||
| 77 | - | ||
| 78 | ---- | ||
| 79 | - | ||
| 80 | -## 3. Scenario Demonstrations | ||
| 81 | - | ||
| 82 | -### 3.1 Scenario 1: Select a Dataset and View the Overview (Dashboard) | ||
| 83 | - | ||
| 84 | -Browse all datasets. Click to select a target dataset. View operator information and statistics. Then navigate to the correlation analysis page. | ||
| 85 | - | ||
| 86 | - | ||
| 87 | - | ||
| 88 | -### 3.2 Scenario 2: Correlate Memory and Task Graph Analysis (MemView) | ||
| 89 | - | ||
| 90 | -Use the memory timeline and DAG task graph correlation to locate memory operations for a specific step. View node details and trace data sources. | ||
| 91 | - | ||
| 92 | - | ||
| 93 | - | ||
| 94 | -### 3.3 Scenario 3: View Errors and Navigate to the Problem Context (Analytic) | ||
| 95 | - | ||
| 96 | -View error details on the Analytic page or in the MemView left-side error list. Then use one-click navigation to jump to the associated DAG node or memory context. | ||
| 97 | - | ||
| 98 | - | ||
| 99 | - | ||
| 100 | ---- | ||
| 101 | - | ||
| 102 | -## 4. Page Details | ||
| 103 | - | ||
| 104 | -### 4.1 Dashboard Page | ||
| 105 | - | ||
| 106 | -The Dashboard is the starting point of the tool. The page is divided into three areas from left to right: | ||
| 107 | - | ||
| 108 | -- **Left side**: Operator summary and rank tree. The rank tree supports check and uncheck operations. The selection result synchronizes across all pages. | ||
| 109 | -- **Center**: Dataset list. Click a row to select a dataset. The left side and right side refresh simultaneously. | ||
| 110 | -- **Right side**: Detail panel for the selected dataset. The panel displays statistics and navigation buttons. | ||
| 111 | - | ||
| 112 | - | ||
| 113 | - | ||
| 114 | -After selecting a dataset, the right side displays detailed statistics: | ||
| 115 | - | ||
| 116 | - | ||
| 117 | - | ||
| 118 | -The bottom of the detail panel provides two navigation entries: | ||
| 119 | - | ||
| 120 | -- **`Enter Correlation View`**: Navigate to MemView with the current dataset and rank selection. | ||
| 121 | -- **`View Diagnostics`**: Navigate to Analytic to view all validation errors for the dataset. | ||
| 122 | - | ||
| 123 | -### 4.2 MemView Page | ||
| 124 | - | ||
| 125 | -MemView places the memory timeline and DAG task graph in the same view for correlated display. This page is the core page for troubleshooting. | ||
| 126 | - | ||
| 127 | - | ||
| 128 | - | ||
| 129 | -**Page layout:** | ||
| 130 | - | ||
| 131 | -- **Left side**: Rank selection tree, error list (Issues), and search panel. | ||
| 132 | -- **Upper area**: Memory timeline — display buffer status evolution for each rank by step. | ||
| 133 | -- **Lower area**: DAG task graph — display task nodes and dependency relationships by rank/stream lanes. | ||
| 134 | -- **Right side**: Detail panel — display detailed information for the currently selected step or node. | ||
| 135 | - | ||
| 136 | - | ||
| 137 | - | ||
| 138 | -#### 4.3 Core Correlation Mechanism: Step, Rank, and Task Correlation | ||
| 139 | - | ||
| 140 | -- Select a step on the timeline → the DAG highlights the corresponding node. | ||
| 141 | -- Click a node in the DAG → the right side switches to node details. | ||
| 142 | -- Click a navigation button on the right side → the timeline locates the corresponding step. | ||
| 143 | - | ||
| 144 | -The **right-side detail panel** has two modes. The panel switches automatically based on whether a DAG node is selected: | ||
| 145 | - | ||
| 146 | -- **Step detail mode**: Display the memory operation list (Task Memory Ops) and buffer slice details (Buffer Layout) for the current step. Each slice in the Buffer Layout can be expanded to view data source semantics. Click to navigate. | ||
| 147 | -- **Node detail mode**: Display node attributes, parent/child node relationships, cross-stage node mappings, and memory operations associated with the node. | ||
| 148 | - | ||
| 149 | -**Other features:** | ||
| 150 | - | ||
| 151 | -- **Bottom player**: Support step-by-step browsing. Drag the slider for quick navigation. The playback speed is 1 second per step. | ||
| 152 | -- **Focus view**: After selecting a step, the DAG automatically enters a local view. The view displays only nodes related to the current step and their context. This feature is suitable for narrowing the scope when nodes are dense. | ||
| 153 | -- **Subgraph browsing**: When a node contains a subgraph, click `Open Subgraph` on the right side to view finer-grained task structures. | ||
| 154 | -- **Stage switching**: Switch graph stages through the top dropdown box (such as `input_graph` / `input_task_queues`). | ||
| 155 | -- **Search**: Search DAG nodes by taskId, taskType, or notifyId. Click a result to navigate to the node. | ||
| 156 | - | ||
| 157 | -### 4.4 Analytic Page | ||
| 158 | - | ||
| 159 | -The Analytic page displays all errors found during validation in a centralized view. The page is divided into three areas: | ||
| 160 | - | ||
| 161 | - | ||
| 162 | - | ||
| 163 | -- **Left side**: Dataset and rank selection. | ||
| 164 | -- **Center**: Error list. Each error displays a title, severity label (color-coded), key field summary, and original error code. | ||
| 165 | -- **Right side**: Detail panel for the selected error. The panel includes basic error information, involved ranks, associated node information, slice details (for precise location of inconsistent memory positions), supplementary information, and original JSON. | ||
| 166 | - | ||
| 167 | -The top of the detail panel provides navigation buttons: | ||
| 168 | - | ||
| 169 | -- **`View memView`**: Navigate to MemView and locate the memory step associated with the error. | ||
| 170 | -- **`View dagView`**: Navigate to MemView and locate the DAG node associated with the error directly. | ||
| 171 | - | ||
| 172 | -The navigation automatically carries the dataset, rank, and node position information. After arriving at MemView, no re-selection is needed to start troubleshooting. | ||
| 173 | - | ||
| 174 | ---- | ||
| 175 | - | ||
| 176 | -## 5. Common Usage Workflows | ||
| 177 | - | ||
| 178 | -### 5.1 Start Analysis from the Dashboard | ||
| 179 | - | ||
| 180 | -1. Click the target dataset in the dataset list. | ||
| 181 | -2. Check the ranks to view in the left-side rank tree. | ||
| 182 | -3. Confirm the file count and error statistics of the dataset on the right side. | ||
| 183 | -4. Click `Enter Correlation View` to enter MemView. | ||
| 184 | - | ||
| 185 | -### 5.2 Locate Memory Issues in MemView | ||
| 186 | - | ||
| 187 | -1. Click the target step on the timeline (or use the bottom player to navigate). | ||
| 188 | -2. View the memory operations for the step in `Task Memory Ops` on the right side. | ||
| 189 | -3. Select a rank and buffer type in `Buffer Layout`. Expand a slice to view the data source. | ||
| 190 | -4. Click `Locate Task` or a semantic card to navigate to the corresponding DAG node. | ||
| 191 | - | ||
| 192 | -### 5.3 Trace Memory from a DAG Node | ||
| 193 | - | ||
| 194 | -1. Click the target node in the DAG. | ||
| 195 | -2. View `Node Memory Ops` on the right side. | ||
| 196 | -3. Click `Navigate to Step` to return to the timeline, or click a parent/child node to continue tracing along the dependency chain. | ||
| 197 | - | ||
| 198 | -### 5.4 Quick Location from Errors | ||
| 199 | - | ||
| 200 | -1. Click an error in the Analytic page or in the MemView left-side error list. | ||
| 201 | -2. View the associated nodes and slice information in the detail panel. | ||
| 202 | -3. Click `View memView` or `View dagView` to navigate to the problem context. | ||
| 203 | - | ||
| 204 | ---- | ||
| 205 | - | ||
| 206 | -## 6. Usage Tips | ||
| 207 | - | ||
| 208 | -- Confirm the dataset and rank scope on the Dashboard first. Then enter MemView for analysis. | ||
| 209 | -- When investigating memory issues, use `Task Memory Ops` and `Buffer Layout` together for troubleshooting first. | ||
| 210 | -- When investigating node dependencies, select a node in the DAG first. Then use the parent/child relationships and node mappings on the right side to navigate. | ||
| 211 | -- When DAG nodes are too dense, use the Focus view to narrow the scope first. Then expand gradually. | ||
| 212 | -- The error list on the left side of MemView allows quick browsing and locating of error nodes without leaving the current page. | ||
| 213 | - | ||
| 214 | ---- | ||
| 215 | - | ||
| 216 | -## 7. Terminology | ||
| 217 | - | ||
| 218 | -| Term | Description | | ||
| 219 | -|------|------| | ||
| 220 | -| **Rank** | A single computing unit (such as an NPU). Each rank executes computations independently. | | ||
| 221 | -| **DAG** | A directed acyclic graph. The graph describes dependency relationships between task nodes. | | ||
| 222 | -| **Step** | A time step on the timeline. Each step corresponds to a memory state snapshot. | | ||
| 223 | -| **Buffer** | A data area in memory. The buffer stores data for computation or communication. | | ||
| 224 | -| **Stage** | A stage of the graph, such as `input_graph` or `input_task_queues`. | | ||
| 225 | -| **Stream** | A task execution flow. Tasks within the same stream execute in order. | | ||
| 226 | -| **Issue** | An error or exception record found during validation. | | ||
| @@ -0,0 +1,423 @@ | |||
| 1 | +# CCU Executor Debug 工具设计文档 — 可行性审视与优化报告 | ||
| 2 | + | ||
| 3 | +基于 `src/plugin/solver/virtual_runtime/ccu_executor/` 代码与 `runner_debug/debug_tool_design_doc.md` 设计方案的全面审视。 | ||
| 4 | + | ||
| 5 | +--- | ||
| 6 | + | ||
| 7 | +## 一、可行性问题（必须修正） | ||
| 8 | + | ||
| 9 | +### P0-1: CKE Wait 流程与插桩设计不匹配 | ||
| 10 | + | ||
| 11 | +**严重程度**: P0（方案可行性错误） | ||
| 12 | + | ||
| 13 | +**问题描述**: | ||
| 14 | + | ||
| 15 | +文档 4.1 节的插桩伪代码假设 `waitCKE_` 标志在 `ExecuteInstr()` 返回之前就能检测，并在之后计算 `snapshotBefore → snapshotAfter` 的 delta。但实际代码流程： | ||
| 16 | + | ||
| 17 | +``` | ||
| 18 | +WaitCkeProcess() → CKE 不满足 → ccuSimulator_->SetWaitCKEFlag(true) → return (不调用 Process()) | ||
| 19 | +→ UpdateLoopStatus() 返回 false → ExecuteInstr() 返回 false → Execute() 返回 false | ||
| 20 | +``` | ||
| 21 | + | ||
| 22 | +关键：CKE 不满足时，`Run()` 内部的 `WaitCkeProcess` 发现 CKE 不满足就设置 waitCKE 标志后直接 return，**不调用 Process()，不修改任何资源**。因此： | ||
| 23 | +- 执行前快照 = 执行后快照（delta 为空） | ||
| 24 | +- 不应该捕获 snapshotAfter 和计算 delta | ||
| 25 | + | ||
| 26 | +**代码依据**: | ||
| 27 | + | ||
| 28 | +`CcuExecutorBase.cc` 第 162-172 行： | ||
| 29 | +```cpp | ||
| 30 | +if ((waitCKE & waitCKEMask) == waitCKEMask) { | ||
| 31 | + Process(ccuResMgr); // CKE 满足 → 执行 Process | ||
| 32 | +} else { | ||
| 33 | + ccuSimulator_->SetWaitCKEFlag(true); // CKE 不满足 → 设置标志，不执行 Process | ||
| 34 | + return; | ||
| 35 | +} | ||
| 36 | +``` | ||
| 37 | + | ||
| 38 | +**修正建议**: | ||
| 39 | + | ||
| 40 | +waitCKE 不满足时，只记录关键自旋字段，不计算 delta，不创建完整 trace entry： | ||
| 41 | +```cpp | ||
| 42 | +// waitCKE 时只记录自旋信息 | ||
| 43 | +CcuTraceCollector::RecordWaitSpin(rankId_, dieId_, curInstrId, | ||
| 44 | + waitCKEId, waitCKEMask, actualCKEValue); | ||
| 45 | +// 不捕获 snapshotAfter，不计算 delta | ||
| 46 | +// 不创建完整 CcuTraceEntry | ||
| 47 | +return false; | ||
| 48 | +``` | ||
| 49 | + | ||
| 50 | +直到 CKE 通过后才创建完整 entry（合并之前的自旋信息）。 | ||
| 51 | + | ||
| 52 | +--- | ||
| 53 | + | ||
| 54 | +### P0-2: globalSeqId/execRound 的维护位置应在 SequentialExecutor | ||
| 55 | + | ||
| 56 | +**严重程度**: P0（方案可行性错误） | ||
| 57 | + | ||
| 58 | +**问题描述**: | ||
| 59 | + | ||
| 60 | +文档假设 `CcuTraceCollector` 在 `CcuSimulator::ExecuteInstr()` 内部维护 `globalSeqId/execRound/currentSqeTaskId`。但 `CcuSimulator` 只能看到本 CCU 的执行，无法感知全局调度轮次和 CCU 切换。 | ||
| 61 | + | ||
| 62 | +`CcuSimulator::ExecuteInstr()` 在单个 CCU 内执行，不知道： | ||
| 63 | +- 当前外层 `while(HasTask())` 的第几轮（`execRound`） | ||
| 64 | +- 全局执行序号（`globalSeqId`） | ||
| 65 | +- 当前执行的是哪个 SQE 任务（`sqeTaskId`） | ||
| 66 | + | ||
| 67 | +这些信息只有在 `SequentialExecutor::Execute()` 和 `TaskCcuGraph()` 的上层调用链中才能获取。 | ||
| 68 | + | ||
| 69 | +**代码依据**: | ||
| 70 | + | ||
| 71 | +`hccl_task_sequential_execute.cc` 第 51-72 行 — 全局调度循环： | ||
| 72 | +```cpp | ||
| 73 | +while (HasTask()) { | ||
| 74 | + uint32_t rankId = 0; | ||
| 75 | + for (auto& rankTasks : allRankTaskQueues_) { // 遍历所有 rank | ||
| 76 | + for (auto& streamTasks : rankTasks) { // 遍历每个 stream | ||
| 77 | + while (!streamTasks.empty()) { | ||
| 78 | + auto task = streamTasks.front(); | ||
| 79 | + auto ret = ExecuteOneTask(task); // ← globalSeqId 在此递增 | ||
| 80 | + if (ret == HCCL_SIM_VRT_HOLD_CMD) break; | ||
| 81 | + streamTasks.pop(); | ||
| 82 | + } | ||
| 83 | + } | ||
| 84 | + } | ||
| 85 | +} | ||
| 86 | +``` | ||
| 87 | + | ||
| 88 | +`hccl_task_thread.cc` 第 135-148 行 — TaskCcuGraph： | ||
| 89 | +```cpp | ||
| 90 | +auto simulator = ccuResMgr.InitSimulator(rankId, dieId, instrStartId, endInstrId, instCnt); | ||
| 91 | +// sqeTaskId 在此可知（从 task 参数获取） | ||
| 92 | +if (simulator->Execute() == false) { | ||
| 93 | + return HCCL_SIM_VRT_HOLD_CMD; | ||
| 94 | +} | ||
| 95 | +``` | ||
| 96 | + | ||
| 97 | +**修正建议**: | ||
| 98 | + | ||
| 99 | +全局上下文由 `SequentialExecutor` 管理，通过参数传递： | ||
| 100 | +```cpp | ||
| 101 | +// SequentialExecutor::Execute() 中 | ||
| 102 | +while (HasTask()) { | ||
| 103 | + uint32_t execRound = 0; | ||
| 104 | + for (rankTasks : allRankTaskQueues_) { | ||
| 105 | + for (streamTasks : rankTasks) { | ||
| 106 | + while (!streamTasks.empty()) { | ||
| 107 | + CcuTraceCollector::BeginGlobalStep(execRound); | ||
| 108 | + // globalSeqId 在 ExecuteOneTask 内递增 | ||
| 109 | + auto ret = ExecuteOneTask(task); | ||
| 110 | + ... | ||
| 111 | + } | ||
| 112 | + } | ||
| 113 | + } | ||
| 114 | + execRound++; | ||
| 115 | +} | ||
| 116 | +``` | ||
| 117 | + | ||
| 118 | +`TaskCcuGraph()` 中将 sqeTaskId 传递给 `CcuSimulator`： | ||
| 119 | +```cpp | ||
| 120 | +uint32_t sqeTaskId = CcuTraceCollector::RegisterSqeTask(rankId, dieId, task); | ||
| 121 | +CcuSimulator::SetCurrentSqeTaskId(sqeTaskId); // Simulator 内部使用 | ||
| 122 | +``` | ||
| 123 | + | ||
| 124 | +--- | ||
| 125 | + | ||
| 126 | +### P0-3: 全量 Snapshot 采集性能开销过大 | ||
| 127 | + | ||
| 128 | +**严重程度**: P0（性能瓶颈） | ||
| 129 | + | ||
| 130 | +**问题描述**: | ||
| 131 | + | ||
| 132 | +文档 4.1 中每条指令前后各做一次 `CaptureResourceSnapshot()`（全量读取所有资源），开销极大： | ||
| 133 | + | ||
| 134 | +- 单个 snapshot：4096×8(XN) + 4096×8(GSA) + 1024×2(CKE) ≈ 67KB | ||
| 135 | +- 每条指令前后各一次 ≈ 134KB | ||
| 136 | +- 10000 条指令 ≈ 1.34GB（仅 snapshot 数据） | ||
| 137 | + | ||
| 138 | +文档 4.5 提到了增量采集优化，但 4.1 的伪代码仍然是全量 snapshot + diff 计算的流程，与优化方案矛盾。 | ||
| 139 | + | ||
| 140 | +**业界做法参考**: | ||
| 141 | + | ||
| 142 | +- **rr debugger**: 检查点（checkpoint）+ 增量记录模式，只在关键节点做全量快照 | ||
| 143 | +- **Chrome DevTools Timeline**: 事件流格式，每个 event 只包含变化的 args | ||
| 144 | +- **Perfetto (Android trace)**: protobuf 二进制增量格式 | ||
| 145 | + | ||
| 146 | +**修正建议**: | ||
| 147 | + | ||
| 148 | +改为 ResourceManager 拦截变更 + 初始快照 + 定期 checkpoint： | ||
| 149 | + | ||
| 150 | +1. **初始快照**：Run 开始时做一次全量（存储在 `CcuIdentity.initialSnapshot` 中） | ||
| 151 | +2. **变更拦截**：在 `CcuResourceManager::UpdateXnValue/UpdateGsaValue/UpdateCkeValue` 等方法中直接拦截变更，记录 delta： | ||
| 152 | +```cpp | ||
| 153 | +void CcuResourceManager::UpdateXnValue(int rankId, int dieId, uint16_t xnId, uint64_t value) { | ||
| 154 | + uint64_t oldValue = GetXnValue(rankId, dieId, xnId); | ||
| 155 | + if (oldValue != value) { | ||
| 156 | + CcuTraceCollector::RecordXnDelta(rankId, dieId, xnId, oldValue, value); | ||
| 157 | + } | ||
| 158 | + xnData_[rankId][dieId][xnId] = value; | ||
| 159 | +} | ||
| 160 | +``` | ||
| 161 | +3. **定期 checkpoint**：每 N 条指令做一次全量快照（N 可配置，默认 1000），用于快速恢复中间状态 | ||
| 162 | +4. **恢复任意点状态**：从最近的 checkpoint + 累加后续 delta | ||
| 163 | + | ||
| 164 | +--- | ||
| 165 | + | ||
| 166 | +## 二、设计优化点（建议改进） | ||
| 167 | + | ||
| 168 | +### P1-1: Loop 内偏移公式不完整 | ||
| 169 | + | ||
| 170 | +**严重程度**: P1（信息缺失） | ||
| 171 | + | ||
| 172 | +**问题描述**: | ||
| 173 | + | ||
| 174 | +代码中 `UpdateAddress()` 的偏移计算比文档中 `CcuExecutionContext.gsaOffset` 更复杂： | ||
| 175 | +```cpp | ||
| 176 | +uint64_t CcuExecutorBase::UpdateAddress(uint64_t addr, uint16_t addrExpandCoef) { | ||
| 177 | + return addr + ((ccuSimulator_->GetLoopExtendNum() * ccuSimulator_->GetGSAOffset()) << addrExpandCoef) | ||
| 178 | + + ((ccuSimulator_->GetCurLoopCnt() * ccuSimulator_->GetLoopIterStepGSA()) << addrExpandCoef); | ||
| 179 | +} | ||
| 180 | +``` | ||
| 181 | + | ||
| 182 | +涉及 5 个参数：`addr`（原始地址）、`addrExpandCoef`（扩展系数）、`loopExtendNum`、`gsaOffset`、`curLoopCnt`、`loopIterStepGSA`。 | ||
| 183 | + | ||
| 184 | +当前 `CcuExecutionContext` 只有一个简单的 `gsaOffset` 字段，无法完整恢复 Loop 内 GSA 地址偏移的计算过程。 | ||
| 185 | + | ||
| 186 | +**修正建议**: | ||
| 187 | + | ||
| 188 | +`CcuExecutionContext` 增加完整的偏移参数： | ||
| 189 | +```cpp | ||
| 190 | +struct CcuExecutionContext { | ||
| 191 | + bool inLoop; | ||
| 192 | + uint16_t loopRound; // 当前迭代轮次 | ||
| 193 | + uint16_t loopExtendIndex; // 展开索引 | ||
| 194 | + | ||
| 195 | + // Loop 内偏移参数（仅在 inLoop=true 时有效） | ||
| 196 | + uint64_t gsaAddrOffset; // = extendIndex * gsaOffset + curLoopCnt * iterStepGSA（计算后的最终偏移） | ||
| 197 | + uint64_t gsaOffset; // GSA 基础偏移系数（GetGSAOffset） | ||
| 198 | + uint64_t iterStepGSA; // 每轮迭代的 GSA 步长（GetLoopIterStepGSA） | ||
| 199 | + uint32_t curLoopCnt; // 当前迭代计数（GetCurLoopCnt） | ||
| 200 | + uint32_t loopExtendNum; // 展开数（GetLoopExtendNum） | ||
| 201 | + uint16_t addrExpandCoef; // 地址扩展系数 | ||
| 202 | + | ||
| 203 | + uint16_t msOffset; // MS ID 偏移（GetLoopMsOffset） | ||
| 204 | + uint16_t ckeOffset; // CKE ID 偏移（GetLoopCKEOffset） | ||
| 205 | + uint16_t xnIdOffset; // XN ID 偏移（GetLoopXnIdOffset） | ||
| 206 | +}; | ||
| 207 | +``` | ||
| 208 | + | ||
| 209 | +--- | ||
| 210 | + | ||
| 211 | +### P1-2: 跨 CCU 变更应在 ResourceManager 中拦截 | ||
| 212 | + | ||
| 213 | +**严重程度**: P1（采集精度） | ||
| 214 | + | ||
| 215 | +**问题描述**: | ||
| 216 | + | ||
| 217 | +文档 4.1 中跨 CCU 变更的采集点在 `CcuSimulator::ExecuteInstr()` 中事后调用 `CaptureCrossCcuChanges()`。但实际跨 CCU 操作发生在 Executor 的 `Process()` 方法中（如 `SyncCkeExecutor::Process()` 调用 `SetRmtCKESignal()`）。 | ||
| 218 | + | ||
| 219 | +事后采集的方式需要对比远端 CCU 的前后状态，这意味着需要在指令执行前后各做一次远端 CCU 的快照——开销大且可能遗漏（远端 CCU 可能被其他代码同时修改）。 | ||
| 220 | + | ||
| 221 | +**修正建议**: | ||
| 222 | + | ||
| 223 | +在 `CcuResourceManager::UpdateCkeValue()` 中直接拦截跨 CCU 变更： | ||
| 224 | +```cpp | ||
| 225 | +void CcuResourceManager::UpdateCkeValue(int rankId, int dieId, uint16_t ckeId, uint16_t value) { | ||
| 226 | + uint16_t oldValue = GetCkeValue(rankId, dieId, ckeId); | ||
| 227 | + // 检测是否为跨 CCU 操作 | ||
| 228 | + auto [execRank, execDie] = CcuTraceCollector::GetCurrentExecutingCcu(); | ||
| 229 | + if (rankId != execRank || dieId != execDie) { | ||
| 230 | + CcuTraceCollector::RecordCrossCcuCkeChange( | ||
| 231 | + rankId, dieId, ckeId, oldValue, value, execRank, execDie); | ||
| 232 | + } | ||
| 233 | + ckeData_[rankId][dieId][ckeId] = value; | ||
| 234 | +} | ||
| 235 | +``` | ||
| 236 | + | ||
| 237 | +同理适用于 `UpdateXnValue()`（TransXnToRmt 等操作）、`UpdateGsaValue()`、`TransMSToMS()` 等。 | ||
| 238 | + | ||
| 239 | +--- | ||
| 240 | + | ||
| 241 | +### P1-3: CcuInstrTraceDetail 多态过度设计 | ||
| 242 | + | ||
| 243 | +**严重程度**: P1（架构优化） | ||
| 244 | + | ||
| 245 | +**问题描述**: | ||
| 246 | + | ||
| 247 | +当前设计了 **13 个** `CcuInstrTraceDetail` 子类（Load/Arith/Trans/Sync/Loop/LoopGroup/Cke/Jump/Wait/Fence/Reduce/...），但存在以下问题： | ||
| 248 | + | ||
| 249 | +1. **与公共层重复**：`CcuTransTraceDetail` 的 `waitCKEId/setCKEId` 与 `CcuTraceEntry.waitInfo` 重复；`CcuCkeTraceDetail` 的 `ckeId/ckeValueBefore/ckeValueAfter` 与 `resourceDelta.ckeChanges` 重复 | ||
| 250 | +2. **子类数量膨胀**：代码中有 30+ 种指令类型（V1 约 20 种 + V2 约 10+ 种），每种都需要一个子类 | ||
| 251 | +3. **前端渲染复杂**：需要根据 typeName 分发不同渲染模板 | ||
| 252 | +4. **新增指令成本高**：每新增一种指令类型都需要新增一个子类结构体 + CollectTraceDetail 实现 | ||
| 253 | + | ||
| 254 | +**业界做法参考**: | ||
| 255 | + | ||
| 256 | +- **Chrome DevTools Trace Event Format**: 扁平的 `args` 字段（key-value map），不使用多态继承 | ||
| 257 | +- **LLVM Execution Trace**: 扁平的"附加信息"字段 | ||
| 258 | +- **GDB MI (Machine Interface)**: key-value 属性列表 | ||
| 259 | + | ||
| 260 | +**修正建议**: | ||
| 261 | + | ||
| 262 | +将 `CcuInstrTraceDetail` 简化为扁平结构： | ||
| 263 | +```cpp | ||
| 264 | +struct CcuInstrTraceDetail { | ||
| 265 | + std::string typeName; // 如 "TransLocMemToLocMem" | ||
| 266 | + std::map<std::string, std::string> args; // 指令特有参数的 key-value 表 | ||
| 267 | + // 示例: | ||
| 268 | + // Trans 类: {"srcAddr": "0x7f0000", "dstAddr": "0x7f1000", "length": "4KB", "channelId": "0"} | ||
| 269 | + // Reduce 类: {"op": "Add", "msList": "[0,1,2,3]", "dataType": "FP32", "castEn": "0"} | ||
| 270 | + // CKE 类: {"ckeOp": "Set", "ckeId": "3", "ckeMask": "0x0001", "isRemote": "true"} | ||
| 271 | +}; | ||
| 272 | +``` | ||
| 273 | + | ||
| 274 | +优点： | ||
| 275 | +- 不需要为每种指令定义子类 | ||
| 276 | +- 新增指令时只需在 `CollectTraceDetail()` 中填充 args | ||
| 277 | +- 前端渲染逻辑统一（key-value 表格） | ||
| 278 | +- `Describe()` 的输出可直接作为 args 来源 | ||
| 279 | + | ||
| 280 | +--- | ||
| 281 | + | ||
| 282 | +### P1-4: JSON 序列化格式与大数据量的矛盾 | ||
| 283 | + | ||
| 284 | +**严重程度**: P1（性能优化） | ||
| 285 | + | ||
| 286 | +**问题描述**: | ||
| 287 | + | ||
| 288 | +文档 3.4 采用 JSON 作为序列化格式，但 JSON 的弱点： | ||
| 289 | +- 每个 uint64_t 的 XN/GSA 值需要字符串表示（`"0x7f3a0000"`），8 字节变 ~16 字节 | ||
| 290 | +- 大量冗余的 key 名称重复（`"xnChanges"` 在每条 entry 中重复出现） | ||
| 291 | +- 10000 条指令的 JSON 文件可达 50MB+ | ||
| 292 | +- 前端加载和解析大 JSON 文件耗时严重 | ||
| 293 | + | ||
| 294 | +**业界做法参考**: | ||
| 295 | + | ||
| 296 | +| 工具 | 格式 | 特点 | | ||
| 297 | +|------|------|------| | ||
| 298 | +| Chrome DevTools | JSON 流式（每行一个 event） | 支持逐行解析 | | ||
| 299 | +| Perfetto (Android) | protobuf 二进制 | 前端 WASM 解析，体积小 | | ||
| 300 | +| rr debugger | 自定义二进制 | 高效记录/回放 | | ||
| 301 | +| Intel VTune | 自定义二进制 + SQLite | 按需查询 | | ||
| 302 | + | ||
| 303 | +**修正建议**: | ||
| 304 | + | ||
| 305 | +分层序列化策略： | ||
| 306 | +- **采集端**：msgpack/flatbuffers 二进制格式，体积小 5-10 倍，写入速度快 | ||
| 307 | +- **前端加载**：通过 Worker 将二进制解码为前端索引结构 | ||
| 308 | +- **导出/调试**：可选导出为 JSON（便于人类阅读和调试） | ||
| 309 | +- JSON Schema 中 `metadata.rankId/dieId` 应删除（改为 `rankSize/diePerRank`，与全局模型一致） | ||
| 310 | + | ||
| 311 | +--- | ||
| 312 | + | ||
| 313 | +## 三、设计优化点（建议改进，优先级较低） | ||
| 314 | + | ||
| 315 | +### P2-1: Describe() 可作为 MVP 的 trace 信息来源 | ||
| 316 | + | ||
| 317 | +**严重程度**: P2（落地策略） | ||
| 318 | + | ||
| 319 | +**问题描述**: | ||
| 320 | + | ||
| 321 | +代码中每个 Executor 都实现了 `Describe()` 方法，返回格式化字符串如： | ||
| 322 | +``` | ||
| 323 | +"[Simulation Execute] Wait CKE[3:0001], Sync LocCKE[5:00ff] To rmtCKE[8:00ff] | ||
| 324 | + Use Channel[0], Set CKE[10:0001], clearType[1]" | ||
| 325 | +``` | ||
| 326 | + | ||
| 327 | +这些字符串已经包含了指令的关键参数信息。当前设计要求每个 Executor 新增 `CollectTraceDetail()` 虚方法并实现，工作量较大（30+ 种指令类型）。 | ||
| 328 | + | ||
| 329 | +**修正建议**: | ||
| 330 | + | ||
| 331 | +分阶段落地策略： | ||
| 332 | + | ||
| 333 | +**第一阶段（MVP）**: | ||
| 334 | +- 只采集 `Describe()` 输出 + resource delta | ||
| 335 | +- `instrDescribe` 字段直接使用 `executor->Describe()` 的返回值 | ||
| 336 | +- 不实现 `CollectTraceDetail()`，detail 字段为空或仅含 typeName | ||
| 337 | + | ||
| 338 | +**第二阶段（增强）**: | ||
| 339 | +- 为高频/关键指令类型实现 `CollectTraceDetail()`（如 Trans、Reduce、SyncCke） | ||
| 340 | +- detail 采用扁平 key-value map（见 P1-3） | ||
| 341 | +- 逐步覆盖所有指令类型 | ||
| 342 | + | ||
| 343 | +这样可以快速落地 MVP，降低初始开发成本。 | ||
| 344 | + | ||
| 345 | +--- | ||
| 346 | + | ||
| 347 | +### P2-2: Notify 等非 CCU 任务不在 trace 中 | ||
| 348 | + | ||
| 349 | +**严重程度**: P2（信息缺失） | ||
| 350 | + | ||
| 351 | +**问题描述**: | ||
| 352 | + | ||
| 353 | +`SequentialExecutor::Execute()` 中除了 `CCU_GRAPH` 外，还有 `NOTIFY_RECORD/NOTIFY_WAIT/REDUCE/MEM_CPY/AIV_GRAPH` 等任务类型。`NotifyWait` 也会返回 `HOLD_CMD`，与 CCU 的 waitCKE 有类似的阻塞行为。 | ||
| 354 | + | ||
| 355 | +当前 trace 只覆盖 `CCU_GRAPH` 类型任务。但 Notify 的 Record/Wait 会影响执行流程： | ||
| 356 | +- `NotifyWait` 阻塞会导致外层循环跳过该 stream | ||
| 357 | +- `NotifyRecord` 解除阻塞后，下一轮该 stream 的 CCU 任务才能继续执行 | ||
| 358 | + | ||
| 359 | +如果不在 trace 中体现，前端回放时会出现"CCU 指令突然跳到下一轮"的现象，无法解释中间发生了什么。 | ||
| 360 | + | ||
| 361 | +**修正建议**: | ||
| 362 | + | ||
| 363 | +在 `globalEntries[]` 中增加非 CCU 任务类型的简化记录： | ||
| 364 | +```cpp | ||
| 365 | +struct CcuTraceNonCcuEntry { | ||
| 366 | + uint32_t globalSeqId; | ||
| 367 | + uint32_t execRound; | ||
| 368 | + int32_t rankId; | ||
| 369 | + HccLTaskMetaType taskType; // NOTIFY_RECORD, NOTIFY_WAIT, REDUCE, MEM_CPY... | ||
| 370 | + HcclVmResult execResult; // SUCCESS 或 HOLD_CMD | ||
| 371 | + std::string description; // 任务描述（便于前端展示） | ||
| 372 | +}; | ||
| 373 | +``` | ||
| 374 | + | ||
| 375 | +前端在 CCU 切换提示时可显示："Round 1: Rank1 NotifyWait 阻塞，跳过"。 | ||
| 376 | + | ||
| 377 | +--- | ||
| 378 | + | ||
| 379 | +### P2-3: JSON Schema 中 metadata 模型不一致 | ||
| 380 | + | ||
| 381 | +**严重程度**: P2（模型一致性） | ||
| 382 | + | ||
| 383 | +**问题描述**: | ||
| 384 | + | ||
| 385 | +文档 3.4 的 JSON Schema 示例中 `metadata` 仍包含旧设计的 `rankId`/`dieId` 字段（per-CCU 模型），但 3.3.1 的 `CcuRunMetadata` 已改为 `rankSize`/`diePerRank`（全局模型）。两处不一致。 | ||
| 386 | + | ||
| 387 | +此外，JSON Schema 示例中使用了 `"seqId"` 而非新设计的 `"globalSeqId"`，缺少 `"execRound"`/`"sqeTaskId"`/`"ccuRegistry"`/`"sqeTaskRegistry"` 等新字段。 | ||
| 388 | + | ||
| 389 | +**修正建议**: | ||
| 390 | + | ||
| 391 | +更新 JSON Schema 示例，与 3.3.1-3.3.2 的数据结构定义保持一致： | ||
| 392 | +- 删除 `metadata.rankId`/`metadata.dieId` | ||
| 393 | +- 增加 `metadata.rankSize`/`metadata.diePerRank`/`metadata.totalExecRounds` | ||
| 394 | +- 增加 `ccuRegistry[]` 和 `sqeTaskRegistry[]` | ||
| 395 | +- entry 中 `seqId` 改为 `globalSeqId` | ||
| 396 | +- 增加 `execRound`/`sqeTaskId`/`rankId`/`dieId` 字段 | ||
| 397 | + | ||
| 398 | +--- | ||
| 399 | + | ||
| 400 | +## 四、汇总表 | ||
| 401 | + | ||
| 402 | +| 优先级 | 编号 | 问题 | 类型 | 修正要点 | | ||
| 403 | +|--------|------|------|------|---------| | ||
| 404 | +| P0 | P0-1 | CKE Wait 时不应计算 delta | 可行性错误 | waitCKE 时只记录关键字段，CKE 通过后才创建完整 entry | | ||
| 405 | +| P0 | P0-2 | globalSeqId/execRound 维护位置错误 | 可行性错误 | 全局上下文由 SequentialExecutor 管理，传参给 Simulator | | ||
| 406 | +| P0 | P0-3 | 全量 snapshot 性能瓶颈 | 性能瓶颈 | ResourceManager 拦截变更 + 初始快照 + 定期 checkpoint | | ||
| 407 | +| P1 | P1-1 | Loop 偏移公式不完整 | 信息缺失 | 补充 iterStepGSA/curLoopCnt/extendNum/addrExpandCoef | | ||
| 408 | +| P1 | P1-2 | 跨 CCU 变更采集时机 | 采集精度 | ResourceManager 内部拦截跨 CCU 变更 | | ||
| 409 | +| P1 | P1-3 | Detail 多态过度设计 | 架构优化 | 简化为扁平 key-value map | | ||
| 410 | +| P1 | P1-4 | JSON 格式不适合大数据量 | 性能优化 | 采集端 msgpack/flatbuffers，前端 WASM 解析 | | ||
| 411 | +| P2 | P2-1 | Describe() 可作为 MVP 来源 | 落地策略 | 第一阶段只做 Describe() + delta | | ||
| 412 | +| P2 | P2-2 | Notify 等非 CCU 任务缺失 | 信息缺失 | 增加简化记录 | | ||
| 413 | +| P2 | P2-3 | JSON Schema 模型不一致 | 模型一致性 | 与 3.3.1-3.3.2 保持一致 | | ||
| 414 | + | ||
| 415 | +--- | ||
| 416 | + | ||
| 417 | +## 五、修正优先级建议 | ||
| 418 | + | ||
| 419 | +**第一步（必须）**: 修正 P0-1、P0-2、P0-3，否则方案无法落地 | ||
| 420 | + | ||
| 421 | +**第二步（建议）**: 修正 P1-1、P1-2、P1-3，提高方案质量和实用性 | ||
| 422 | + | ||
| 423 | +**第三步（可选）**: 修正 P1-4、P2-1、P2-2、P2-3，完善细节和用户体验 | ||
| @@ -0,0 +1,423 @@ | |||
| 1 | +# CCU Executor Debug Tool Design Document — Feasibility Review and Optimization Report | ||
| 2 | + | ||
| 3 | +A comprehensive review based on the code in `src/plugin/solver/virtual_runtime/ccu_executor/` and the design proposal in `runner_debug/debug_tool_design_doc.md`. | ||
| 4 | + | ||
| 5 | +--- | ||
| 6 | + | ||
| 7 | +## I. Feasibility Issues (Must Fix) | ||
| 8 | + | ||
| 9 | +### P0-1: CKE Wait Flow Does Not Match Instrumentation Design | ||
| 10 | + | ||
| 11 | +**Severity**: P0 (Approach feasibility error) | ||
| 12 | + | ||
| 13 | +**Problem Description**: | ||
| 14 | + | ||
| 15 | +The instrumentation pseudocode in Section 4.1 assumes that the `waitCKE_` flag can be detected before `ExecuteInstr()` returns, and the delta from `snapshotBefore → snapshotAfter` is computed afterward. However, the actual code flow is: | ||
| 16 | + | ||
| 17 | +``` | ||
| 18 | +WaitCkeProcess() → CKE not met → ccuSimulator_->SetWaitCKEFlag(true) → return (Process() not called) | ||
| 19 | +→ UpdateLoopStatus() returns false → ExecuteInstr() returns false → Execute() returns false | ||
| 20 | +``` | ||
| 21 | + | ||
| 22 | +Key point: When CKE is not met, `WaitCkeProcess` inside `Run()` sets the waitCKE flag and returns directly, **without calling Process() and without modifying any resources**. Therefore: | ||
| 23 | +- Pre-execution snapshot = post-execution snapshot (delta is empty) | ||
| 24 | +- snapshotAfter should not be captured and delta should not be computed | ||
| 25 | + | ||
| 26 | +**Code Evidence**: | ||
| 27 | + | ||
| 28 | +`CcuExecutorBase.cc` lines 162-172: | ||
| 29 | +```cpp | ||
| 30 | +if ((waitCKE & waitCKEMask) == waitCKEMask) { | ||
| 31 | + Process(ccuResMgr); // CKE met → execute Process | ||
| 32 | +} else { | ||
| 33 | + ccuSimulator_->SetWaitCKEFlag(true); // CKE not met → set flag, do not execute Process | ||
| 34 | + return; | ||
| 35 | +} | ||
| 36 | +``` | ||
| 37 | + | ||
| 38 | +**Fix Suggestion**: | ||
| 39 | + | ||
| 40 | +When waitCKE is not met, only record key spin fields; do not compute delta or create a full trace entry: | ||
| 41 | +```cpp | ||
| 42 | +// During waitCKE, only record spin information | ||
| 43 | +CcuTraceCollector::RecordWaitSpin(rankId_, dieId_, curInstrId, | ||
| 44 | + waitCKEId, waitCKEMask, actualCKEValue); | ||
| 45 | +// Do not capture snapshotAfter, do not compute delta | ||
| 46 | +// Do not create full CcuTraceEntry | ||
| 47 | +return false; | ||
| 48 | +``` | ||
| 49 | + | ||
| 50 | +Only create a full entry after CKE passes (merging previous spin information). | ||
| 51 | + | ||
| 52 | +--- | ||
| 53 | + | ||
| 54 | +### P0-2: globalSeqId/execRound Maintenance Should Be in SequentialExecutor | ||
| 55 | + | ||
| 56 | +**Severity**: P0 (Approach feasibility error) | ||
| 57 | + | ||
| 58 | +**Problem Description**: | ||
| 59 | + | ||
| 60 | +The document assumes `CcuTraceCollector` maintains `globalSeqId/execRound/currentSqeTaskId` inside `CcuSimulator::ExecuteInstr()`. However, `CcuSimulator` can only see execution within its own CCU and cannot perceive global scheduling rounds or CCU switching. | ||
| 61 | + | ||
| 62 | +`CcuSimulator::ExecuteInstr()` executes within a single CCU and does not know: | ||
| 63 | +- Which round of the outer `while(HasTask())` it is (`execRound`) | ||
| 64 | +- The global execution sequence number (`globalSeqId`) | ||
| 65 | +- Which SQE task is currently being executed (`sqeTaskId`) | ||
| 66 | + | ||
| 67 | +This information is only available in the upper call chain of `SequentialExecutor::Execute()` and `TaskCcuGraph()`. | ||
| 68 | + | ||
| 69 | +**Code Evidence**: | ||
| 70 | + | ||
| 71 | +`hccl_task_sequential_execute.cc` lines 51-72 — Global scheduling loop: | ||
| 72 | +```cpp | ||
| 73 | +while (HasTask()) { | ||
| 74 | + uint32_t rankId = 0; | ||
| 75 | + for (auto& rankTasks : allRankTaskQueues_) { // Iterate all ranks | ||
| 76 | + for (auto& streamTasks : rankTasks) { // Iterate each stream | ||
| 77 | + while (!streamTasks.empty()) { | ||
| 78 | + auto task = streamTasks.front(); | ||
| 79 | + auto ret = ExecuteOneTask(task); // ← globalSeqId incremented here | ||
| 80 | + if (ret == HCCL_SIM_VRT_HOLD_CMD) break; | ||
| 81 | + streamTasks.pop(); | ||
| 82 | + } | ||
| 83 | + } | ||
| 84 | + } | ||
| 85 | +} | ||
| 86 | +``` | ||
| 87 | + | ||
| 88 | +`hccl_task_thread.cc` lines 135-148 — TaskCcuGraph: | ||
| 89 | +```cpp | ||
| 90 | +auto simulator = ccuResMgr.InitSimulator(rankId, dieId, instrStartId, endInstrId, instCnt); | ||
| 91 | +// sqeTaskId is known here (obtained from task parameters) | ||
| 92 | +if (simulator->Execute() == false) { | ||
| 93 | + return HCCL_SIM_VRT_HOLD_CMD; | ||
| 94 | +} | ||
| 95 | +``` | ||
| 96 | + | ||
| 97 | +**Fix Suggestion**: | ||
| 98 | + | ||
| 99 | +Global context is managed by `SequentialExecutor` and passed via parameters: | ||
| 100 | +```cpp | ||
| 101 | +// In SequentialExecutor::Execute() | ||
| 102 | +while (HasTask()) { | ||
| 103 | + uint32_t execRound = 0; | ||
| 104 | + for (rankTasks : allRankTaskQueues_) { | ||
| 105 | + for (streamTasks : rankTasks) { | ||
| 106 | + while (!streamTasks.empty()) { | ||
| 107 | + CcuTraceCollector::BeginGlobalStep(execRound); | ||
| 108 | + // globalSeqId incremented inside ExecuteOneTask | ||
| 109 | + auto ret = ExecuteOneTask(task); | ||
| 110 | + ... | ||
| 111 | + } | ||
| 112 | + } | ||
| 113 | + } | ||
| 114 | + execRound++; | ||
| 115 | +} | ||
| 116 | +``` | ||
| 117 | + | ||
| 118 | +Pass sqeTaskId to `CcuSimulator` in `TaskCcuGraph()`: | ||
| 119 | +```cpp | ||
| 120 | +uint32_t sqeTaskId = CcuTraceCollector::RegisterSqeTask(rankId, dieId, task); | ||
| 121 | +CcuSimulator::SetCurrentSqeTaskId(sqeTaskId); // Used internally by Simulator | ||
| 122 | +``` | ||
| 123 | + | ||
| 124 | +--- | ||
| 125 | + | ||
| 126 | +### P0-3: Full Snapshot Collection Has Excessive Performance Overhead | ||
| 127 | + | ||
| 128 | +**Severity**: P0 (Performance bottleneck) | ||
| 129 | + | ||
| 130 | +**Problem Description**: | ||
| 131 | + | ||
| 132 | +In Section 4.1, a `CaptureResourceSnapshot()` (full read of all resources) is performed before and after each instruction, with significant overhead: | ||
| 133 | + | ||
| 134 | +- Single snapshot: 4096×8(XN) + 4096×8(GSA) + 1024×2(CKE) ≈ 67KB | ||
| 135 | +- Before and after each instruction ≈ 134KB | ||
| 136 | +- 10000 instructions ≈ 1.34GB (snapshot data only) | ||
| 137 | + | ||
| 138 | +Section 4.5 mentions incremental collection optimization, but the pseudocode in 4.1 still follows the full snapshot + diff computation flow, which contradicts the optimization approach. | ||
| 139 | + | ||
| 140 | +**Industry Reference**: | ||
| 141 | + | ||
| 142 | +- **rr debugger**: Checkpoint + incremental recording model; full snapshots only at key nodes | ||
| 143 | +- **Chrome DevTools Timeline**: Event stream format; each event contains only changed args | ||
| 144 | +- **Perfetto (Android trace)**: Protobuf binary incremental format | ||
| 145 | + | ||
| 146 | +**Fix Suggestion**: | ||
| 147 | + | ||
| 148 | +Change to ResourceManager change interception + initial snapshot + periodic checkpoints: | ||
| 149 | + | ||
| 150 | +1. **Initial Snapshot**: Full snapshot at run start (stored in `CcuIdentity.initialSnapshot`) | ||
| 151 | +2. **Change Interception**: Intercept changes directly in `CcuResourceManager::UpdateXnValue/UpdateGsaValue/UpdateCkeValue` etc., recording delta: | ||
| 152 | +```cpp | ||
| 153 | +void CcuResourceManager::UpdateXnValue(int rankId, int dieId, uint16_t xnId, uint64_t value) { | ||
| 154 | + uint64_t oldValue = GetXnValue(rankId, dieId, xnId); | ||
| 155 | + if (oldValue != value) { | ||
| 156 | + CcuTraceCollector::RecordXnDelta(rankId, dieId, xnId, oldValue, value); | ||
| 157 | + } | ||
| 158 | + xnData_[rankId][dieId][xnId] = value; | ||
| 159 | +} | ||
| 160 | +``` | ||
| 161 | +3. **Periodic Checkpoint**: Full snapshot every N instructions (N configurable, default 1000) for quick intermediate state recovery | ||
| 162 | +4. **Restore Arbitrary Point State**: From nearest checkpoint + accumulate subsequent deltas | ||
| 163 | + | ||
| 164 | +--- | ||
| 165 | + | ||
| 166 | +## II. Design Optimization Points (Recommended Improvements) | ||
| 167 | + | ||
| 168 | +### P1-1: Loop Offset Formula Is Incomplete | ||
| 169 | + | ||
| 170 | +**Severity**: P1 (Information gap) | ||
| 171 | + | ||
| 172 | +**Problem Description**: | ||
| 173 | + | ||
| 174 | +The offset calculation in `UpdateAddress()` in the code is more complex than the `CcuExecutionContext.gsaOffset` in the document: | ||
| 175 | +```cpp | ||
| 176 | +uint64_t CcuExecutorBase::UpdateAddress(uint64_t addr, uint16_t addrExpandCoef) { | ||
| 177 | + return addr + ((ccuSimulator_->GetLoopExtendNum() * ccuSimulator_->GetGSAOffset()) << addrExpandCoef) | ||
| 178 | + + ((ccuSimulator_->GetCurLoopCnt() * ccuSimulator_->GetLoopIterStepGSA()) << addrExpandCoef); | ||
| 179 | +} | ||
| 180 | +``` | ||
| 181 | + | ||
| 182 | +This involves 5 parameters: `addr` (original address), `addrExpandCoef` (expansion coefficient), `loopExtendNum`, `gsaOffset`, `curLoopCnt`, `loopIterStepGSA`. | ||
| 183 | + | ||
| 184 | +The current `CcuExecutionContext` has only a simple `gsaOffset` field and cannot fully reproduce the Loop GSA address offset calculation process. | ||
| 185 | + | ||
| 186 | +**Fix Suggestion**: | ||
| 187 | + | ||
| 188 | +Add complete offset parameters to `CcuExecutionContext`: | ||
| 189 | +```cpp | ||
| 190 | +struct CcuExecutionContext { | ||
| 191 | + bool inLoop; | ||
| 192 | + uint16_t loopRound; // Current iteration round | ||
| 193 | + uint16_t loopExtendIndex; // Expand index | ||
| 194 | + | ||
| 195 | + // Loop offset parameters (valid only when inLoop=true) | ||
| 196 | + uint64_t gsaAddrOffset; // = extendIndex * gsaOffset + curLoopCnt * iterStepGSA (computed final offset) | ||
| 197 | + uint64_t gsaOffset; // GSA base offset coefficient (GetGSAOffset) | ||
| 198 | + uint64_t iterStepGSA; // GSA step per iteration round (GetLoopIterStepGSA) | ||
| 199 | + uint32_t curLoopCnt; // Current iteration count (GetCurLoopCnt) | ||
| 200 | + uint32_t loopExtendNum; // Expand count (GetLoopExtendNum) | ||
| 201 | + uint16_t addrExpandCoef; // Address expansion coefficient | ||
| 202 | + | ||
| 203 | + uint16_t msOffset; // MS ID offset (GetLoopMsOffset) | ||
| 204 | + uint16_t ckeOffset; // CKE ID offset (GetLoopCKEOffset) | ||
| 205 | + uint16_t xnIdOffset; // XN ID offset (GetLoopXnIdOffset) | ||
| 206 | +}; | ||
| 207 | +``` | ||
| 208 | + | ||
| 209 | +--- | ||
| 210 | + | ||
| 211 | +### P1-2: Cross-CCU Changes Should Be Intercepted in ResourceManager | ||
| 212 | + | ||
| 213 | +**Severity**: P1 (Collection accuracy) | ||
| 214 | + | ||
| 215 | +**Problem Description**: | ||
| 216 | + | ||
| 217 | +In Section 4.1, the cross-CCU change collection point is a post-hoc call to `CaptureCrossCcuChanges()` in `CcuSimulator::ExecuteInstr()`. However, cross-CCU operations actually occur in the Executor's `Process()` method (e.g., `SyncCkeExecutor::Process()` calls `SetRmtCKESignal()`). | ||
| 218 | + | ||
| 219 | +The post-hoc collection approach requires comparing the remote CCU's before/after state, meaning a remote CCU snapshot would be needed before and after instruction execution — high overhead and potentially missing changes (the remote CCU could be simultaneously modified by other code). | ||
| 220 | + | ||
| 221 | +**Fix Suggestion**: | ||
| 222 | + | ||
| 223 | +Intercept cross-CCU changes directly in `CcuResourceManager::UpdateCkeValue()`: | ||
| 224 | +```cpp | ||
| 225 | +void CcuResourceManager::UpdateCkeValue(int rankId, int dieId, uint16_t ckeId, uint16_t value) { | ||
| 226 | + uint16_t oldValue = GetCkeValue(rankId, dieId, ckeId); | ||
| 227 | + // Detect whether this is a cross-CCU operation | ||
| 228 | + auto [execRank, execDie] = CcuTraceCollector::GetCurrentExecutingCcu(); | ||
| 229 | + if (rankId != execRank || dieId != execDie) { | ||
| 230 | + CcuTraceCollector::RecordCrossCcuCkeChange( | ||
| 231 | + rankId, dieId, ckeId, oldValue, value, execRank, execDie); | ||
| 232 | + } | ||
| 233 | + ckeData_[rankId][dieId][ckeId] = value; | ||
| 234 | +} | ||
| 235 | +``` | ||
| 236 | + | ||
| 237 | +Similarly applicable to `UpdateXnValue()` (TransXnToRmt operations), `UpdateGsaValue()`, `TransMSToMS()`, etc. | ||
| 238 | + | ||
| 239 | +--- | ||
| 240 | + | ||
| 241 | +### P1-3: CcuInstrTraceDetail Polymorphism Is Over-engineered | ||
| 242 | + | ||
| 243 | +**Severity**: P1 (Architecture optimization) | ||
| 244 | + | ||
| 245 | +**Problem Description**: | ||
| 246 | + | ||
| 247 | +The current design has **13** `CcuInstrTraceDetail` subclasses (Load/Arith/Trans/Sync/Loop/LoopGroup/Cke/Jump/Wait/Fence/Reduce/...), but there are the following issues: | ||
| 248 | + | ||
| 249 | +1. **Duplication with common layer**: `CcuTransTraceDetail`'s `waitCKEId/setCKEId` duplicates `CcuTraceEntry.waitInfo`; `CcuCkeTraceDetail`'s `ckeId/ckeValueBefore/ckeValueAfter` duplicates `resourceDelta.ckeChanges` | ||
| 250 | +2. **Subclass proliferation**: The code has 30+ instruction types (~20 in V1 + ~10+ in V2); each requires a subclass | ||
| 251 | +3. **Complex frontend rendering**: Requires dispatching different rendering templates based on typeName | ||
| 252 | +4. **High cost of new instructions**: Each new instruction type requires a new subclass struct + CollectTraceDetail implementation | ||
| 253 | + | ||
| 254 | +**Industry Reference**: | ||
| 255 | + | ||
| 256 | +- **Chrome DevTools Trace Event Format**: Flat `args` field (key-value map), no polymorphic inheritance | ||
| 257 | +- **LLVM Execution Trace**: Flat "additional info" field | ||
| 258 | +- **GDB MI (Machine Interface)**: Key-value property list | ||
| 259 | + | ||
| 260 | +**Fix Suggestion**: | ||
| 261 | + | ||
| 262 | +Simplify `CcuInstrTraceDetail` to a flat structure: | ||
| 263 | +```cpp | ||
| 264 | +struct CcuInstrTraceDetail { | ||
| 265 | + std::string typeName; // e.g., "TransLocMemToLocMem" | ||
| 266 | + std::map<std::string, std::string> args; // Instruction-specific parameter key-value table | ||
| 267 | + // Examples: | ||
| 268 | + // Trans type: {"srcAddr": "0x7f0000", "dstAddr": "0x7f1000", "length": "4KB", "channelId": "0"} | ||
| 269 | + // Reduce type: {"op": "Add", "msList": "[0,1,2,3]", "dataType": "FP32", "castEn": "0"} | ||
| 270 | + // CKE type: {"ckeOp": "Set", "ckeId": "3", "ckeMask": "0x0001", "isRemote": "true"} | ||
| 271 | +}; | ||
| 272 | +``` | ||
| 273 | + | ||
| 274 | +Advantages: | ||
| 275 | +- No need to define a subclass for each instruction type | ||
| 276 | +- New instructions only need to populate args in `CollectTraceDetail()` | ||
| 277 | +- Unified frontend rendering logic (key-value table) | ||
| 278 | +- `Describe()` output can be used directly as args source | ||
| 279 | + | ||
| 280 | +--- | ||
| 281 | + | ||
| 282 | +### P1-4: JSON Serialization Format vs Large Data Volume | ||
| 283 | + | ||
| 284 | +**Severity**: P1 (Performance optimization) | ||
| 285 | + | ||
| 286 | +**Problem Description**: | ||
| 287 | + | ||
| 288 | +Section 3.4 adopts JSON as the serialization format, but JSON has weaknesses: | ||
| 289 | +- Each uint64_t XN/GSA value requires string representation (`"0x7f3a0000"`), 8 bytes becomes ~16 bytes | ||
| 290 | +- Large amounts of redundant key name repetition (`"xnChanges"` repeated in every entry) | ||
| 291 | +- 10000 instructions' JSON file can reach 50MB+ | ||
| 292 | +- Frontend loading and parsing of large JSON files is seriously slow | ||
| 293 | + | ||
| 294 | +**Industry Reference**: | ||
| 295 | + | ||
| 296 | +| Tool | Format | Characteristics | | ||
| 297 | +|------|------|------| | ||
| 298 | +| Chrome DevTools | JSON streaming (one event per line) | Supports line-by-line parsing | | ||
| 299 | +| Perfetto (Android) | Protobuf binary | Frontend WASM parsing, small size | | ||
| 300 | +| rr debugger | Custom binary | Efficient recording/playback | | ||
| 301 | +| Intel VTune | Custom binary + SQLite | On-demand querying | | ||
| 302 | + | ||
| 303 | +**Fix Suggestion**: | ||
| 304 | + | ||
| 305 | +Layered serialization strategy: | ||
| 306 | +- **Collection side**: msgpack/flatbuffers binary format, 5-10x smaller, faster writes | ||
| 307 | +- **Frontend loading**: Decode binary into frontend index structures via Worker | ||
| 308 | +- **Export/debug**: Optional export to JSON (for human readability and debugging) | ||
| 309 | +- JSON Schema `metadata.rankId/dieId` should be removed (changed to `rankSize/diePerRank`, consistent with global model) | ||
| 310 | + | ||
| 311 | +--- | ||
| 312 | + | ||
| 313 | +## III. Design Optimization Points (Recommended Improvements, Lower Priority) | ||
| 314 | + | ||
| 315 | +### P2-1: Describe() Can Serve as MVP Trace Information Source | ||
| 316 | + | ||
| 317 | +**Severity**: P2 (Implementation strategy) | ||
| 318 | + | ||
| 319 | +**Problem Description**: | ||
| 320 | + | ||
| 321 | +Every Executor in the code implements `Describe()`, returning formatted strings like: | ||
| 322 | +``` | ||
| 323 | +"[Simulation Execute] Wait CKE[3:0001], Sync LocCKE[5:00ff] To rmtCKE[8:00ff] | ||
| 324 | + Use Channel[0], Set CKE[10:0001], clearType[1]" | ||
| 325 | +``` | ||
| 326 | + | ||
| 327 | +These strings already contain key instruction parameter information. The current design requires each Executor to add a `CollectTraceDetail()` virtual method and implement it, which is a significant effort (30+ instruction types). | ||
| 328 | + | ||
| 329 | +**Fix Suggestion**: | ||
| 330 | + | ||
| 331 | +Phased implementation strategy: | ||
| 332 | + | ||
| 333 | +**Phase 1 (MVP)**: | ||
| 334 | +- Collect only `Describe()` output + resource delta | ||
| 335 | +- `instrDescribe` field directly uses `executor->Describe()` return value | ||
| 336 | +- Do not implement `CollectTraceDetail()`; detail field is empty or contains only typeName | ||
| 337 | + | ||
| 338 | +**Phase 2 (Enhancement)**: | ||
| 339 | +- Implement `CollectTraceDetail()` for high-frequency/critical instruction types (e.g., Trans, Reduce, SyncCke) | ||
| 340 | +- Detail uses flat key-value map (see P1-3) | ||
| 341 | +- Progressively cover all instruction types | ||
| 342 | + | ||
| 343 | +This enables rapid MVP delivery, reducing initial development cost. | ||
| 344 | + | ||
| 345 | +--- | ||
| 346 | + | ||
| 347 | +### P2-2: Notify and Other Non-CCU Tasks Not in Trace | ||
| 348 | + | ||
| 349 | +**Severity**: P2 (Information gap) | ||
| 350 | + | ||
| 351 | +**Problem Description**: | ||
| 352 | + | ||
| 353 | +`SequentialExecutor::Execute()` includes task types beyond `CCU_GRAPH`, such as `NOTIFY_RECORD/NOTIFY_WAIT/REDUCE/MEM_CPY/AIV_GRAPH`. `NotifyWait` can also return `HOLD_CMD`, with similar blocking behavior to CCU's waitCKE. | ||
| 354 | + | ||
| 355 | +The current trace only covers `CCU_GRAPH` type tasks. But Notify's Record/Wait affects execution flow: | ||
| 356 | +- `NotifyWait` blocking causes the outer loop to skip that stream | ||
| 357 | +- After `NotifyRecord` releases the block, the next round allows that stream's CCU task to continue | ||
| 358 | + | ||
| 359 | +If not reflected in trace, the frontend replay will show "CCU instructions suddenly jumping to the next round" with no explanation of what happened in between. | ||
| 360 | + | ||
| 361 | +**Fix Suggestion**: | ||
| 362 | + | ||
| 363 | +Add simplified records for non-CCU task types in `globalEntries[]`: | ||
| 364 | +```cpp | ||
| 365 | +struct CcuTraceNonCcuEntry { | ||
| 366 | + uint32_t globalSeqId; | ||
| 367 | + uint32_t execRound; | ||
| 368 | + int32_t rankId; | ||
| 369 | + HccLTaskMetaType taskType; // NOTIFY_RECORD, NOTIFY_WAIT, REDUCE, MEM_CPY... | ||
| 370 | + HcclVmResult execResult; // SUCCESS or HOLD_CMD | ||
| 371 | + std::string description; // Task description (for frontend display) | ||
| 372 | +}; | ||
| 373 | +``` | ||
| 374 | + | ||
| 375 | +During CCU switch notifications, the frontend can display: "Round 1: Rank1 NotifyWait blocked, skipped". | ||
| 376 | + | ||
| 377 | +--- | ||
| 378 | + | ||
| 379 | +### P2-3: JSON Schema Metadata Model Inconsistency | ||
| 380 | + | ||
| 381 | +**Severity**: P2 (Model consistency) | ||
| 382 | + | ||
| 383 | +**Problem Description**: | ||
| 384 | + | ||
| 385 | +The JSON Schema example in Section 3.4 still includes `rankId`/`dieId` fields in `metadata` (per-CCU model), but Section 3.3.1's `CcuRunMetadata` has already been changed to `rankSize`/`diePerRank` (global model). The two are inconsistent. | ||
| 386 | + | ||
| 387 | +Additionally, the JSON Schema example uses `"seqId"` instead of the new design's `"globalSeqId"`, and is missing `"execRound"`/`"sqeTaskId"`/`"ccuRegistry"`/`"sqeTaskRegistry"` and other new fields. | ||
| 388 | + | ||
| 389 | +**Fix Suggestion**: | ||
| 390 | + | ||
| 391 | +Update the JSON Schema example to be consistent with the data structure definitions in 3.3.1-3.3.2: | ||
| 392 | +- Remove `metadata.rankId`/`metadata.dieId` | ||
| 393 | +- Add `metadata.rankSize`/`metadata.diePerRank`/`metadata.totalExecRounds` | ||
| 394 | +- Add `ccuRegistry[]` and `sqeTaskRegistry[]` | ||
| 395 | +- Change `seqId` to `globalSeqId` in entries | ||
| 396 | +- Add `execRound`/`sqeTaskId`/`rankId`/`dieId` fields | ||
| 397 | + | ||
| 398 | +--- | ||
| 399 | + | ||
| 400 | +## IV. Summary Table | ||
| 401 | + | ||
| 402 | +| Priority | ID | Issue | Type | Fix Summary | | ||
| 403 | +|--------|------|------|------|---------| | ||
| 404 | +| P0 | P0-1 | Delta should not be computed during CKE Wait | Feasibility error | Record only key fields during waitCKE; create full entry only after CKE passes | | ||
| 405 | +| P0 | P0-2 | globalSeqId/execRound maintenance location is wrong | Feasibility error | Global context managed by SequentialExecutor, passed to Simulator via parameters | | ||
| 406 | +| P0 | P0-3 | Full snapshot performance bottleneck | Performance bottleneck | ResourceManager change interception + initial snapshot + periodic checkpoints | | ||
| 407 | +| P1 | P1-1 | Loop offset formula incomplete | Information gap | Add iterStepGSA/curLoopCnt/extendNum/addrExpandCoef | | ||
| 408 | +| P1 | P1-2 | Cross-CCU change collection timing | Collection accuracy | Intercept cross-CCU changes inside ResourceManager | | ||
| 409 | +| P1 | P1-3 | Detail polymorphism over-engineered | Architecture optimization | Simplify to flat key-value map | | ||
| 410 | +| P1 | P1-4 | JSON format unsuitable for large data | Performance optimization | Collection-side msgpack/flatbuffers, frontend WASM parsing | | ||
| 411 | +| P2 | P2-1 | Describe() can serve as MVP source | Implementation strategy | Phase 1: only Describe() + delta | | ||
| 412 | +| P2 | P2-2 | Notify and other non-CCU tasks missing | Information gap | Add simplified records | | ||
| 413 | +| P2 | P2-3 | JSON Schema model inconsistency | Model consistency | Align with 3.3.1-3.3.2 | | ||
| 414 | + | ||
| 415 | +--- | ||
| 416 | + | ||
| 417 | +## V. Fix Priority Recommendations | ||
| 418 | + | ||
| 419 | +**Step 1 (Required)**: Fix P0-1, P0-2, P0-3; otherwise the approach cannot be implemented | ||
| 420 | + | ||
| 421 | +**Step 2 (Recommended)**: Fix P1-1, P1-2, P1-3; improve approach quality and practicality | ||
| 422 | + | ||
| 423 | +**Step 3 (Optional)**: Fix P1-4, P2-1, P2-2, P2-3; refine details and user experience | ||
| @@ -786,7 +786,7 @@ erDiagram | |||
| 786 | EventTask { | 786 | EventTask { |
| 787 | typ task-id FK | 787 | typ task-id FK |
| 788 | typ event-id FK | 788 | typ event-id FK |
| 789 | - typ excute-time | 789 | + typ execute-time |
| 790 | typ finish-time | 790 | typ finish-time |
| 791 | typ first-capture-taskid FK | 791 | typ first-capture-taskid FK |
| 792 | } | 792 | } |
| @@ -797,23 +797,23 @@ erDiagram | |||
| 797 | 797 | ||
| 798 | EventSyncTask { | 798 | EventSyncTask { |
| 799 | typ event-id FK | 799 | typ event-id FK |
| 800 | - typ excute-time | 800 | + typ execute-time |
| 801 | typ finish-time | 801 | typ finish-time |
| 802 | typ op-timeout-s | 802 | typ op-timeout-s |
| 803 | } | 803 | } |
| 804 | EventRecordTask { | 804 | EventRecordTask { |
| 805 | typ event-id FK | 805 | typ event-id FK |
| 806 | - typ excute-time | 806 | + typ execute-time |
| 807 | typ finish-time | 807 | typ finish-time |
| 808 | } | 808 | } |
| 809 | EventWaitTask { | 809 | EventWaitTask { |
| 810 | typ event-id FK | 810 | typ event-id FK |
| 811 | - typ excute-time | 811 | + typ execute-time |
| 812 | typ finish-time | 812 | typ finish-time |
| 813 | } | 813 | } |
| 814 | EventTimeTask { | 814 | EventTimeTask { |
| 815 | typ event-id FK | 815 | typ event-id FK |
| 816 | - typ excute-time | 816 | + typ execute-time |
| 817 | } | 817 | } |
| 818 | EventTraceTask { | 818 | EventTraceTask { |
| 819 | typ event-id FK | 819 | typ event-id FK |
| @@ -562,7 +562,7 @@ graph RL | |||
| 562 | 562 | ||
| 563 | ### 3.3 Asynchronous / Synchronous Execution Modeling | 563 | ### 3.3 Asynchronous / Synchronous Execution Modeling |
| 564 | 564 | ||
| 565 | -#### 3.3.1 [Notify Resource Management](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850alpha001/appdevg/acldevg/aclcppdevg_000524.html) | 565 | +#### 3.3.1 [Notify Resource Management](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/850alpha001/appdevg/acldevg/aclcppdevg_000524.html) |
| 566 | 566 | ||
| 567 | ```mermaid | 567 | ```mermaid |
| 568 | erDiagram | 568 | erDiagram |
| @@ -601,7 +601,7 @@ erDiagram | |||
| 601 | } | 601 | } |
| 602 | ``` | 602 | ``` |
| 603 | 603 | ||
| 604 | -##### Key Relationship Description ([Notify Resource Management](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850alpha001/appdevg/acldevg/aclcppdevg_000524.html)) | 604 | +##### Key Relationship Description ([Notify Resource Management](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/850alpha001/appdevg/acldevg/aclcppdevg_000524.html)) |
| 605 | 605 | ||
| 606 | **Notify and Device hardware constraints**: | 606 | **Notify and Device hardware constraints**: |
| 607 | 607 | ||
| @@ -622,7 +622,7 @@ erDiagram | |||
| 622 | - `rtWaitAndResetNotify` waits for the Notify to reach Ready state and resets it. | 622 | - `rtWaitAndResetNotify` waits for the Notify to reach Ready state and resets it. |
| 623 | - Notify is used for inter-Stream synchronization and cross-process synchronization. | 623 | - Notify is used for inter-Stream synchronization and cross-process synchronization. |
| 624 | 624 | ||
| 625 | -##### API Mapping Table ([Notify Resource Management](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850alpha001/appdevg/acldevg/aclcppdevg_000524.html)) | 625 | +##### API Mapping Table ([Notify Resource Management](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/850alpha001/appdevg/acldevg/aclcppdevg_000524.html)) |
| 626 | 626 | ||
| 627 | | Entity | Key Management Interfaces | | 627 | | Entity | Key Management Interfaces | |
| 628 | | -------------------------- | --------------------------------------------------- | | 628 | | -------------------------- | --------------------------------------------------- | |
| @@ -786,7 +786,7 @@ erDiagram | |||
| 786 | EventTask { | 786 | EventTask { |
| 787 | typ task-id FK | 787 | typ task-id FK |
| 788 | typ event-id FK | 788 | typ event-id FK |
| 789 | - typ excute-time | 789 | + typ execute-time |
| 790 | typ finish-time | 790 | typ finish-time |
| 791 | typ first-capture-taskid FK | 791 | typ first-capture-taskid FK |
| 792 | } | 792 | } |
| @@ -797,23 +797,23 @@ erDiagram | |||
| 797 | 797 | ||
| 798 | EventSyncTask { | 798 | EventSyncTask { |
| 799 | typ event-id FK | 799 | typ event-id FK |
| 800 | - typ excute-time | 800 | + typ execute-time |
| 801 | typ finish-time | 801 | typ finish-time |
| 802 | typ op-timeout-s | 802 | typ op-timeout-s |
| 803 | } | 803 | } |
| 804 | EventRecordTask { | 804 | EventRecordTask { |
| 805 | typ event-id FK | 805 | typ event-id FK |
| 806 | - typ excute-time | 806 | + typ execute-time |
| 807 | typ finish-time | 807 | typ finish-time |
| 808 | } | 808 | } |
| 809 | EventWaitTask { | 809 | EventWaitTask { |
| 810 | typ event-id FK | 810 | typ event-id FK |
| 811 | - typ excute-time | 811 | + typ execute-time |
| 812 | typ finish-time | 812 | typ finish-time |
| 813 | } | 813 | } |
| 814 | EventTimeTask { | 814 | EventTimeTask { |
| 815 | typ event-id FK | 815 | typ event-id FK |
| 816 | - typ excute-time | 816 | + typ execute-time |
| 817 | } | 817 | } |
| 818 | EventTraceTask { | 818 | EventTraceTask { |
| 819 | typ event-id FK | 819 | typ event-id FK |
Rtest/hccl_vm/src/plugin/checker/src/framework/task_graph_generator/ccu_task_transform_instruct_common.h→test/hccl_vm/include/runnerdb/db_sim_communicator.h+15-23
| @@ -8,32 +8,24 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | -#ifndef HCCLV2_CCU_TRANSFORM_TASK_COMMON_H | 11 | +#ifndef DB_SIM_COMMUNICATOR_H |
| 12 | -#define HCCLV2_CCU_TRANSFORM_TASK_COMMON_H | 12 | +#define DB_SIM_COMMUNICATOR_H |
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | -#include <hccl_types.h> | 15 | +#include <string> |
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | 16 | ||
| 22 | 17 | ||
| 23 | -#include "base.h" | 18 | +namespace sim { |
| 24 | -#include "ccu_instr_info.h" | 19 | +struct CommunicatorMemberInfo { |
| 25 | -#include "ccu_microcode_v1.h" | 20 | + uint64_t memberId; |
| 26 | -#include "data_slice.h" | 21 | + uint64_t deviceId; |
| 27 | -#include "data_type.h" | 22 | + uint32_t rankId; |
| 28 | -#include "log.h" | 23 | +}; |
| 29 | - | ||
| 30 | - | ||
| 31 | - | ||
| 32 | - | ||
| 33 | 24 | ||
| 34 | -using namespace hcomm; | 25 | +bool GetCommunicatorMembers(uint64_t commId, std::vector<CommunicatorMemberInfo>& members); |
| 35 | -namespace HcclSim { | 26 | +// 根据通信域成员 ID 查询其所属通信域名称。 |
| 36 | -std::string ParseMSList(const CcuRep::CcuInstr* instr); | 27 | +bool GetCommunicatorName(uint64_t commId, std::string& commName); |
| 37 | -} | 28 | +bool GetCommunicatorIdentity(uint64_t commId, std::string& commName, uint64_t& commHash); |
| 29 | +} // namespace sim | ||
| 38 | 30 | ||
| 39 | -#endif | 31 | +#endif // DB_SIM_COMMUNICATOR_H |
| @@ -19,9 +19,15 @@ | |||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | namespace sim { | 21 | namespace sim { |
| 22 | +extern uint32_t g_currOpDetailId; | ||
| 23 | + | ||
| 22 | int InitOpDataDb(); | 24 | int InitOpDataDb(); |
| 23 | int SetDbConfig(DBConfig& config); | 25 | int SetDbConfig(DBConfig& config); |
| 24 | 26 | ||
| 27 | +void AddSyncStreamIter(uint64_t streamId); | ||
| 28 | +void RemoveSyncStreamIter(uint64_t streamId); | ||
| 29 | +bool NextStreamSyncIdx(uint64_t streamId, uint64_t& syncIdx); | ||
| 30 | + | ||
| 25 | int InsertOpDetail(OpDetailTab& rec); | 31 | int InsertOpDetail(OpDetailTab& rec); |
| 26 | int InsertOpMem(OpMemInfoTab& rec); | 32 | int InsertOpMem(OpMemInfoTab& rec); |
| 27 | int InsertOpDetailAndMem(OpDetailTab& detail, OpMemInfoTab& mem); | 33 | int InsertOpDetailAndMem(OpDetailTab& detail, OpMemInfoTab& mem); |
| @@ -31,10 +37,12 @@ int InsertOpTask(OpTaskTab& rec, bool isDevice = false); | |||
| 31 | int InsertSyncRecord(SyncRecordTab& rec); | 37 | int InsertSyncRecord(SyncRecordTab& rec); |
| 32 | int InsertCcuInstrRes(CcuInstrResTab& rec); | 38 | int InsertCcuInstrRes(CcuInstrResTab& rec); |
| 33 | int InsertCcuInstr(CcuInstrTab& rec); | 39 | int InsertCcuInstr(CcuInstrTab& rec); |
| 40 | +int InsertHalfRTT(HalfRTTTab& rec); | ||
| 34 | 41 | ||
| 35 | int UpdateAndInsertByCcuId( | 42 | int UpdateAndInsertByCcuId( |
| 36 | - uint64_t& ccuId, uint32_t deviceId, uint32_t rankId, uint32_t dieId, uint32_t instrCount, uint32_t instrOffset, | 43 | + uint64_t& ccuId, uint32_t deviceId, uint32_t dieId, uint32_t startId, uint32_t instrCount, uint32_t instrOffset, |
| 37 | - uint32_t instrInfoSize, const void* instrInfo); | 44 | + uint32_t instrInfoSize, const void* instrInfo, std::vector<uint8_t>* mergedInstrSpace = nullptr, |
| 45 | + uint32_t* totalInstrCount = nullptr); | ||
| 38 | int UpdateSyncRecordStatus(std::vector<SyncRecordTab>& syncRecord); | 46 | int UpdateSyncRecordStatus(std::vector<SyncRecordTab>& syncRecord); |
| 39 | int UpdateOpMemCclBuffer(uint64_t cclAddr, uint64_t cclSize); | 47 | int UpdateOpMemCclBuffer(uint64_t cclAddr, uint64_t cclSize); |
| 40 | int UpdateOpExpansionMode(uint8_t mode); | 48 | int UpdateOpExpansionMode(uint8_t mode); |
| @@ -44,10 +52,20 @@ int QueryCcuChannelAll(std::vector<CcuChannelTab>& out); | |||
| 44 | int QueryJettyMapAll(std::vector<JettyMapTab>& out); | 52 | int QueryJettyMapAll(std::vector<JettyMapTab>& out); |
| 45 | int QuerySyncRecordAll(std::vector<SyncRecordTab>& out); | 53 | int QuerySyncRecordAll(std::vector<SyncRecordTab>& out); |
| 46 | int QueryCcuInstrResAll(std::vector<CcuInstrResTab>& out); | 54 | int QueryCcuInstrResAll(std::vector<CcuInstrResTab>& out); |
| 47 | -int QueryNewestOpDeatailIdByPid(uint64_t pid, uint32_t& OpDetailId); | 55 | +int QueryCcuResourceMetaCount(uint32_t& instrLoadCnt, uint32_t& channelCnt); |
| 48 | -int QueryCurrentOpMemInfoByRank(uint32_t rankId, OpMemInfoTab& out); | 56 | +int QueryOpDetailIdentity(uint32_t opDetailId, uint64_t& commId, uint32_t& rankId, uint32_t& deviceId); |
| 57 | +int QueryCurrentOpMemInfo(uint64_t commId, uint32_t deviceId, OpMemInfoTab& out); | ||
| 49 | int QuerySyncRecordByStatus(uint8_t status, std::vector<SyncRecordTab>& out); | 58 | int QuerySyncRecordByStatus(uint8_t status, std::vector<SyncRecordTab>& out); |
| 59 | +int QueryOpExecutionIndexEntries(std::vector<OpExecutionIndexEntry>& out); | ||
| 50 | int QueryCompositeOpDetailBySyncIter(uint32_t syncIter, std::map<uint32_t, std::vector<CompositeOpDetail>>& detail); | 60 | int QueryCompositeOpDetailBySyncIter(uint32_t syncIter, std::map<uint32_t, std::vector<CompositeOpDetail>>& detail); |
| 61 | +int QueryHalfRTTAll(std::vector<HalfRTTTab>& out); | ||
| 62 | +int QueryOpTaskTabNames(std::vector<std::string>& names); | ||
| 63 | +int QueryCompositeOpDetailByOpIter( | ||
| 64 | + const std::string& commName, uint64_t commHash, uint32_t opIter, std::vector<CompositeOpDetail>& details); | ||
| 65 | +int QueryAllOpTasks(std::vector<OpTaskTab>& tasks, bool filterDone = false); | ||
| 66 | +int QueryOpTasksByStreamId(uint64_t streamId, std::vector<OpTaskTab>& tasks); | ||
| 67 | +int FinishOpTask(const OpTaskTab& task); | ||
| 68 | +int QueryHalfRTTAll(std::vector<HalfRTTTab>& out); | ||
| 51 | } // namespace sim | 69 | } // namespace sim |
| 52 | 70 | ||
| 53 | 71 | ||
| @@ -11,15 +11,58 @@ | |||
| 11 | 11 | ||
| 12 | 12 | ||
| 13 | 13 | ||
| 14 | - | ||
| 15 | - | ||
| 16 | 14 | ||
| 17 | -#include "sim_models.h" | 15 | +#include "db_sim_communicator.h" |
| 18 | 16 | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +extern thread_local uint64_t g_cur_device_key; | ||
| 25 | +// 当前线程正在执行算子的本地 Communicator 表行 id; 它不是 HcclComm 句柄, | ||
| 26 | +// 也不是整个通信域共用的 id. | ||
| 27 | +extern thread_local uint64_t g_cur_comm_key; | ||
| 19 | 28 | ||
| 20 | namespace sim { | 29 | namespace sim { |
| 30 | +// 通信域server信息: 由TopoMetaConfig表项指向的topo_meta配置解析生成, | ||
| 31 | +// 按rankId分配顺序排列 | ||
| 32 | +struct CommConfigServer { | ||
| 33 | + uint64_t serverKey{0}; // Server表主键 | ||
| 34 | + uint32_t podId{0}; // 超级节点id | ||
| 35 | + uint32_t serverId{0}; // server id | ||
| 36 | + uint32_t devNum{0}; // 通信域内该server的device数 | ||
| 37 | + uint32_t rankOffset{0}; // 该server首个rankId(前面server的device累计数) | ||
| 38 | + std::vector<PhyDeviceId> phyIds; // server内按rank顺序的物理device id | ||
| 39 | +}; | ||
| 40 | + | ||
| 41 | +// 进程级通信域配置缓存(TopoMetaConfig指向的配置解析结果) | ||
| 42 | +struct CommConfigData { | ||
| 43 | + std::string fileName; // topo_meta yaml文件名; "ranktable"表示ranktable.json模式 | ||
| 44 | + std::vector<CommConfigServer> servers; // 按(podId, serverId)升序, 与rankId分配顺序一致 | ||
| 45 | +}; | ||
| 46 | +} // namespace sim | ||
| 47 | + | ||
| 48 | +// 进程级通信域配置缓存: mock-comm执行时刷新, 其余进程首次使用时加载; | ||
| 49 | +// 仅经sim::接口访问 | ||
| 50 | +extern sim::CommConfigData g_comm_config_data; | ||
| 51 | + | ||
| 52 | +namespace sim { | ||
| 53 | +// 仅供仍需"查找或复用"通信域成员的旧调用方使用; 主/子通信域创建路径均直接插入本 | ||
| 54 | +// rank 的表行. | ||
| 55 | +bool GetOrInsertCommunicator( | ||
| 56 | + const char* commName, uint32_t rankSize, uint32_t rankId, uint64_t deviceId, uint64_t commHash, uint64_t& commId); | ||
| 57 | +bool WaitCommunicatorReady(const char* commName, uint64_t commHash, uint32_t rankSize); | ||
| 58 | +// level 1 专用 | ||
| 59 | +bool WaitCommunicatorDestroyReady(uint64_t commId); | ||
| 60 | +// level 0 专用: 避免快卡提前销毁, 标记本 rank | ||
| 61 | +// 已调用销毁接口并等待同通信域全部成员到达后再返回 | ||
| 62 | +bool MarkAndSyncCommunicatorDestroy(uint64_t commId); | ||
| 63 | + | ||
| 21 | aclError GetDeviceByLogicId(uint32_t deviceId, sim::Device& device); | 64 | aclError GetDeviceByLogicId(uint32_t deviceId, sim::Device& device); |
| 22 | -aclError GetDeviceByRankId(uint32_t rankId, sim::Device& device); | 65 | +aclError GetDeviceByCommRank(uint64_t commId, uint32_t rankId, sim::Device& device); |
| 23 | aclError GetDeviceByPhysicalId(uint32_t deviceId, sim::Device& device); | 66 | aclError GetDeviceByPhysicalId(uint32_t deviceId, sim::Device& device); |
| 24 | aclError UpdateDeviceLogicId(uint64_t serverKey, uint32_t phyDevId, uint32_t logicDevId, uint32_t userId); | 67 | aclError UpdateDeviceLogicId(uint64_t serverKey, uint32_t phyDevId, uint32_t logicDevId, uint32_t userId); |
| 25 | aclError UpdateSuperDeviceId(uint32_t logicDevId, uint32_t superDeviceId); | 68 | aclError UpdateSuperDeviceId(uint32_t logicDevId, uint32_t superDeviceId); |
| @@ -28,14 +71,29 @@ aclError GetCcuResourceByCcu(uint64_t ccuKey, sim::CcuResource& ccuRes); | |||
| 28 | aclError GetContextByDevId(uint32_t deviceId, sim::Context& context); | 71 | aclError GetContextByDevId(uint32_t deviceId, sim::Context& context); |
| 29 | aclError GetPortByName(uint64_t serverKey, uint32_t phyDevId, const std::string& name, sim::Port& port); | 72 | aclError GetPortByName(uint64_t serverKey, uint32_t phyDevId, const std::string& name, sim::Port& port); |
| 30 | aclError GetEndPointByIpAddr(const std::string& ip, sim::EndPoint& endPoint); | 73 | aclError GetEndPointByIpAddr(const std::string& ip, sim::EndPoint& endPoint); |
| 74 | +aclError GetEndPointByEid(const IpAddress& addr, sim::EndPoint& endPoint); | ||
| 31 | aclError GetPortById(uint64_t portId, sim::Port& port); | 75 | aclError GetPortById(uint64_t portId, sim::Port& port); |
| 32 | 76 | ||
| 33 | uint32_t GetAICpuCount(uint64_t deviceId); | 77 | uint32_t GetAICpuCount(uint64_t deviceId); |
| 34 | uint32_t GetAICoreCount(uint64_t deviceId); | 78 | uint32_t GetAICoreCount(uint64_t deviceId); |
| 35 | uint32_t GetVectorCoreCount(uint64_t deviceId); | 79 | uint32_t GetVectorCoreCount(uint64_t deviceId); |
| 36 | -int GetRankIdByDeviceId(uint32_t deviceId); | 80 | +std::string GetHardwareTypeByDevice(uint64_t deviceId); |
| 81 | +bool GetCommRankByDeviceId(uint64_t commId, uint32_t deviceId, uint32_t& rankId); | ||
| 37 | bool ResetAllDeviceLogicId(); | 82 | bool ResetAllDeviceLogicId(); |
| 38 | bool GetRankIdByMPI(uint32_t& rankId, uint64_t& serverId); | 83 | bool GetRankIdByMPI(uint32_t& rankId, uint64_t& serverId); |
| 39 | uint64_t GetCurServerId(); | 84 | uint64_t GetCurServerId(); |
| 85 | + | ||
| 86 | +// 公共解析函数: 解析topo_meta | ||
| 87 | +// yaml(或ranktable模式下的data/ranktable.json)生成通信域server有序表. | ||
| 88 | +// topoMeta非空时直接使用调用方已解析的结果, 避免重复读文件 | ||
| 89 | +bool ParseCommConfigData(const std::string& fileName, const TopoMeta* topoMeta, CommConfigData& data); | ||
| 90 | +// mock-comm每次执行时调用: 覆盖TopoMetaConfig表项并刷新本进程g_comm_config_data | ||
| 91 | +bool RefreshTopoMetaConfig(const std::string& fileName, const TopoMeta* topoMeta = nullptr); | ||
| 92 | +// 通信域重置时清空进程缓存, 下次使用时按TopoMetaConfig表项重新加载 | ||
| 93 | +void ResetCommConfigData(); | ||
| 94 | +// rankId → 所属server的serverKey | ||
| 95 | +bool GetServerKeyByRankId(uint32_t rankId, uint64_t& serverKey); | ||
| 96 | +// serverKey + 物理device id → 通信域内device序号(与Device.logic_id一致) | ||
| 97 | +aclError GetDevIndexByPhyId(uint64_t serverKey, uint32_t phyId, uint32_t& devIndex); | ||
| 40 | } // namespace sim | 98 | } // namespace sim |
| 41 | 99 | ||
| @@ -13,15 +13,15 @@ | |||
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | - | ||
| 17 | 16 | ||
| 17 | + | ||
| 18 | 18 | ||
| 19 | namespace sim { | 19 | namespace sim { |
| 20 | bool GetCurrRunnerTls(uint64_t serverKey, Runner& runner); | 20 | bool GetCurrRunnerTls(uint64_t serverKey, Runner& runner); |
| 21 | bool SetCurrCtxTls(uint64_t ctx); | 21 | bool SetCurrCtxTls(uint64_t ctx); |
| 22 | -uint64_t GetCurrRankId(); | 22 | +uint64_t GetCurrDeviceId(); |
| 23 | uint64_t GetCurrDeviceKey(); | 23 | uint64_t GetCurrDeviceKey(); |
| 24 | -uint64_t GetRankIdByCtxId(uint64_t ctxId); | 24 | +uint64_t GetDeviceIdByCtxId(uint64_t ctxId); |
| 25 | 25 | ||
| 26 | void SetLastStreamIdTls(uint64_t streamId); | 26 | void SetLastStreamIdTls(uint64_t streamId); |
| 27 | void SetLastTaskIdTls(uint64_t taskId); | 27 | void SetLastTaskIdTls(uint64_t taskId); |
| @@ -32,7 +32,6 @@ uint64_t GetLastTaskIdTls(); | |||
| 32 | void SetTsDevice(int tsId); | 32 | void SetTsDevice(int tsId); |
| 33 | uint32_t GetRankSize(); | 33 | uint32_t GetRankSize(); |
| 34 | uint32_t GetHostSize(); | 34 | uint32_t GetHostSize(); |
| 35 | -uint32_t GetCurrentStreamId(uint64_t streamKey); | ||
| 36 | uint64_t GetServerKeyById(uint32_t superPodIdx, uint32_t serverIdx); | 35 | uint64_t GetServerKeyById(uint32_t superPodIdx, uint32_t serverIdx); |
| 37 | } // namespace sim | 36 | } // namespace sim |
| 38 | 37 | ||
| @@ -41,6 +41,8 @@ private: | |||
| 41 | sim::SqliteTable<sim::Server> m_serverTbl; | 41 | sim::SqliteTable<sim::Server> m_serverTbl; |
| 42 | sim::SqliteTable<sim::Host> m_hostTbl; | 42 | sim::SqliteTable<sim::Host> m_hostTbl; |
| 43 | sim::SqliteTable<sim::Runner> m_runnerTbl; | 43 | sim::SqliteTable<sim::Runner> m_runnerTbl; |
| 44 | + sim::SqliteTable<sim::Communicator> m_communicatorTbl; | ||
| 45 | + sim::SqliteTable<sim::CommunicatorDestroySync> m_communicatorDestroySyncTbl; | ||
| 44 | sim::SqliteTable<sim::Device> m_deviceTbl; | 46 | sim::SqliteTable<sim::Device> m_deviceTbl; |
| 45 | sim::SqliteTable<sim::DeviceStatus> m_deviceStatusTbl; | 47 | sim::SqliteTable<sim::DeviceStatus> m_deviceStatusTbl; |
| 46 | sim::SqliteTable<sim::Context> m_contextTbl; | 48 | sim::SqliteTable<sim::Context> m_contextTbl; |
| @@ -75,7 +77,6 @@ private: | |||
| 75 | sim::SqliteTable<sim::RaMR> m_raMRTbl; | 77 | sim::SqliteTable<sim::RaMR> m_raMRTbl; |
| 76 | sim::SqliteTable<sim::MemoryLayout> m_memoryLayoutTbl; | 78 | sim::SqliteTable<sim::MemoryLayout> m_memoryLayoutTbl; |
| 77 | sim::SqliteTable<sim::SimModelData> m_simModelDataTbl; | 79 | sim::SqliteTable<sim::SimModelData> m_simModelDataTbl; |
| 78 | - sim::SqliteTable<sim::Rank> m_rankTbl; | ||
| 79 | sim::SqliteTable<sim::IpcNotify> m_ipcNotifyTbl; | 80 | sim::SqliteTable<sim::IpcNotify> m_ipcNotifyTbl; |
| 80 | sim::SqliteTable<sim::IpcNotifyVistorList> m_ipcNotifyVistorListTbl; | 81 | sim::SqliteTable<sim::IpcNotifyVistorList> m_ipcNotifyVistorListTbl; |
| 81 | sim::SqliteTable<sim::NotifyRecordTask> m_notifyRecordTaskTbl; | 82 | sim::SqliteTable<sim::NotifyRecordTask> m_notifyRecordTaskTbl; |
| @@ -91,6 +92,17 @@ private: | |||
| 91 | sim::SqliteTable<sim::RaCr> m_raCrTbl; | 92 | sim::SqliteTable<sim::RaCr> m_raCrTbl; |
| 92 | sim::SqliteTable<sim::RaTlv> m_raTlvTbl; | 93 | sim::SqliteTable<sim::RaTlv> m_raTlvTbl; |
| 93 | sim::SqliteTable<sim::RunModeConfig> m_runModeConfigTbl; | 94 | sim::SqliteTable<sim::RunModeConfig> m_runModeConfigTbl; |
| 95 | + sim::SqliteTable<sim::HcclBuffer> m_hcclBufferTbl; | ||
| 96 | + sim::SqliteTable<sim::HcclThread> m_hcclThreadTbl; | ||
| 97 | + sim::SqliteTable<sim::HcclChannel> m_hcclChannelTbl; | ||
| 98 | + sim::SqliteTable<sim::HcommEndpoint> m_hcommEndpointTbl; | ||
| 99 | + sim::SqliteTable<sim::HcommMemReg> m_hcommMemRegTbl; | ||
| 100 | + sim::SqliteTable<sim::HcclEngineCtx> m_hcclEngineCtxTbl; | ||
| 101 | + sim::SqliteTable<sim::HcclMem> m_hcclMemTbl; | ||
| 102 | + sim::SqliteTable<sim::TopoMetaConfig> m_topoMetaConfigTbl; | ||
| 103 | + sim::SqliteTable<sim::Plugin> m_pluginTbl; | ||
| 104 | + sim::SqliteTable<sim::DpuPendingNotify> m_dpuPendingNotifyTbl; | ||
| 105 | + sim::SqliteTable<sim::DpuDeviceInfo> m_dpuDeviceInfoTbl; | ||
| 94 | 106 | ||
| 95 | std::mutex m_lazyMutex; | 107 | std::mutex m_lazyMutex; |
| 96 | std::unordered_map<std::type_index, sim::TableBase*> m_tableMap; | 108 | std::unordered_map<std::type_index, sim::TableBase*> m_tableMap; |
| @@ -109,6 +121,8 @@ public: | |||
| 109 | : m_serverTbl(m_db.GetDb(), "Server"), | 121 | : m_serverTbl(m_db.GetDb(), "Server"), |
| 110 | m_hostTbl(m_db.GetDb(), "Host"), | 122 | m_hostTbl(m_db.GetDb(), "Host"), |
| 111 | m_runnerTbl(m_db.GetDb(), "Runner"), | 123 | m_runnerTbl(m_db.GetDb(), "Runner"), |
| 124 | + m_communicatorTbl(m_db.GetDb(), "Communicator"), | ||
| 125 | + m_communicatorDestroySyncTbl(m_db.GetDb(), "CommunicatorDestroySync"), | ||
| 112 | m_deviceTbl(m_db.GetDb(), "Device"), | 126 | m_deviceTbl(m_db.GetDb(), "Device"), |
| 113 | m_deviceStatusTbl(m_db.GetDb(), "DeviceStatus"), | 127 | m_deviceStatusTbl(m_db.GetDb(), "DeviceStatus"), |
| 114 | m_contextTbl(m_db.GetDb(), "Context"), | 128 | m_contextTbl(m_db.GetDb(), "Context"), |
| @@ -143,7 +157,6 @@ public: | |||
| 143 | m_raMRTbl(m_db.GetDb(), "RaMR"), | 157 | m_raMRTbl(m_db.GetDb(), "RaMR"), |
| 144 | m_memoryLayoutTbl(m_db.GetDb(), "MemoryLayout"), | 158 | m_memoryLayoutTbl(m_db.GetDb(), "MemoryLayout"), |
| 145 | m_simModelDataTbl(m_db.GetDb(), "SimModelData"), | 159 | m_simModelDataTbl(m_db.GetDb(), "SimModelData"), |
| 146 | - m_rankTbl(m_db.GetDb(), "Rank"), | ||
| 147 | m_ipcNotifyTbl(m_db.GetDb(), "IpcNotify"), | 160 | m_ipcNotifyTbl(m_db.GetDb(), "IpcNotify"), |
| 148 | m_ipcNotifyVistorListTbl(m_db.GetDb(), "IpcNotifyVistorList"), | 161 | m_ipcNotifyVistorListTbl(m_db.GetDb(), "IpcNotifyVistorList"), |
| 149 | m_notifyRecordTaskTbl(m_db.GetDb(), "NotifyRecordTask"), | 162 | m_notifyRecordTaskTbl(m_db.GetDb(), "NotifyRecordTask"), |
| @@ -158,11 +171,24 @@ public: | |||
| 158 | m_raJfcTbl(m_db.GetDb(), "RaJfc"), | 171 | m_raJfcTbl(m_db.GetDb(), "RaJfc"), |
| 159 | m_raCrTbl(m_db.GetDb(), "RaCr"), | 172 | m_raCrTbl(m_db.GetDb(), "RaCr"), |
| 160 | m_raTlvTbl(m_db.GetDb(), "RaTlv"), | 173 | m_raTlvTbl(m_db.GetDb(), "RaTlv"), |
| 161 | - m_runModeConfigTbl(m_db.GetDb(), "RunModeConfig") | 174 | + m_runModeConfigTbl(m_db.GetDb(), "RunModeConfig"), |
| 175 | + m_hcclBufferTbl(m_db.GetDb(), "HcclBuffer"), | ||
| 176 | + m_hcclThreadTbl(m_db.GetDb(), "HcclThread"), | ||
| 177 | + m_hcclChannelTbl(m_db.GetDb(), "HcclChannel"), | ||
| 178 | + m_hcommEndpointTbl(m_db.GetDb(), "HcommEndpoint"), | ||
| 179 | + m_hcommMemRegTbl(m_db.GetDb(), "HcommMemReg"), | ||
| 180 | + m_hcclEngineCtxTbl(m_db.GetDb(), "HcclEngineCtx"), | ||
| 181 | + m_hcclMemTbl(m_db.GetDb(), "HcclMem"), | ||
| 182 | + m_topoMetaConfigTbl(m_db.GetDb(), "TopoMetaConfig"), | ||
| 183 | + m_pluginTbl(m_db.GetDb(), "Plugin"), | ||
| 184 | + m_dpuPendingNotifyTbl(m_db.GetDb(), "DpuPendingNotify"), | ||
| 185 | + m_dpuDeviceInfoTbl(m_db.GetDb(), "DpuDeviceInfo") | ||
| 162 | { | 186 | { |
| 163 | RegisterTable(m_serverTbl, "Server"); | 187 | RegisterTable(m_serverTbl, "Server"); |
| 164 | RegisterTable(m_hostTbl, "Host"); | 188 | RegisterTable(m_hostTbl, "Host"); |
| 165 | RegisterTable(m_runnerTbl, "Runner"); | 189 | RegisterTable(m_runnerTbl, "Runner"); |
| 190 | + RegisterTable(m_communicatorTbl, "Communicator"); | ||
| 191 | + RegisterTable(m_communicatorDestroySyncTbl, "CommunicatorDestroySync"); | ||
| 166 | RegisterTable(m_deviceTbl, "Device"); | 192 | RegisterTable(m_deviceTbl, "Device"); |
| 167 | RegisterTable(m_deviceStatusTbl, "DeviceStatus"); | 193 | RegisterTable(m_deviceStatusTbl, "DeviceStatus"); |
| 168 | RegisterTable(m_contextTbl, "Context"); | 194 | RegisterTable(m_contextTbl, "Context"); |
| @@ -197,7 +223,6 @@ public: | |||
| 197 | RegisterTable(m_raMRTbl, "RaMR"); | 223 | RegisterTable(m_raMRTbl, "RaMR"); |
| 198 | RegisterTable(m_memoryLayoutTbl, "MemoryLayout"); | 224 | RegisterTable(m_memoryLayoutTbl, "MemoryLayout"); |
| 199 | RegisterTable(m_simModelDataTbl, "SimModelData"); | 225 | RegisterTable(m_simModelDataTbl, "SimModelData"); |
| 200 | - RegisterTable(m_rankTbl, "Rank"); | ||
| 201 | RegisterTable(m_ipcNotifyTbl, "IpcNotify"); | 226 | RegisterTable(m_ipcNotifyTbl, "IpcNotify"); |
| 202 | RegisterTable(m_ipcNotifyVistorListTbl, "IpcNotifyVistorList"); | 227 | RegisterTable(m_ipcNotifyVistorListTbl, "IpcNotifyVistorList"); |
| 203 | RegisterTable(m_notifyRecordTaskTbl, "NotifyRecordTask"); | 228 | RegisterTable(m_notifyRecordTaskTbl, "NotifyRecordTask"); |
| @@ -213,6 +238,17 @@ public: | |||
| 213 | RegisterTable(m_raCrTbl, "RaCr"); | 238 | RegisterTable(m_raCrTbl, "RaCr"); |
| 214 | RegisterTable(m_raTlvTbl, "RaTlv"); | 239 | RegisterTable(m_raTlvTbl, "RaTlv"); |
| 215 | RegisterTable(m_runModeConfigTbl, "RunModeConfig"); | 240 | RegisterTable(m_runModeConfigTbl, "RunModeConfig"); |
| 241 | + RegisterTable(m_hcclBufferTbl, "HcclBuffer"); | ||
| 242 | + RegisterTable(m_hcclThreadTbl, "HcclThread"); | ||
| 243 | + RegisterTable(m_hcclChannelTbl, "HcclChannel"); | ||
| 244 | + RegisterTable(m_hcommEndpointTbl, "HcommEndpoint"); | ||
| 245 | + RegisterTable(m_hcommMemRegTbl, "HcommMemReg"); | ||
| 246 | + RegisterTable(m_hcclEngineCtxTbl, "HcclEngineCtx"); | ||
| 247 | + RegisterTable(m_hcclMemTbl, "HcclMem"); | ||
| 248 | + RegisterTable(m_topoMetaConfigTbl, "TopoMetaConfig"); | ||
| 249 | + RegisterTable(m_pluginTbl, "Plugin"); | ||
| 250 | + RegisterTable(m_dpuPendingNotifyTbl, "DpuPendingNotify"); | ||
| 251 | + RegisterTable(m_dpuDeviceInfoTbl, "DpuDeviceInfo"); | ||
| 216 | } | 252 | } |
| 217 | 253 | ||
| 218 | SimRunnerSqliteDB(const SimRunnerSqliteDB&) = delete; | 254 | SimRunnerSqliteDB(const SimRunnerSqliteDB&) = delete; |
| @@ -241,7 +277,10 @@ public: | |||
| 241 | return *static_cast<sim::SqliteTable<T>*>(it->second); | 277 | return *static_cast<sim::SqliteTable<T>*>(it->second); |
| 242 | } | 278 | } |
| 243 | std::string name = std::string("Unregistered_") + typeid(T).name(); | 279 | std::string name = std::string("Unregistered_") + typeid(T).name(); |
| 244 | - HCCL_VM_WARN("[SimRunnerSqliteDB::GetTable] Lazily registering unregistered type: {}", name.c_str()); | 280 | + HCCL_VM_WARN( |
| 281 | + "[SimRunnerSqliteDB::GetTable] Lazily registering " | ||
| 282 | + "unregistered type: {}", | ||
| 283 | + name.c_str()); | ||
| 245 | auto table = std::make_unique<sim::SqliteTable<T>>(m_db.GetDb(), name); | 284 | auto table = std::make_unique<sim::SqliteTable<T>>(m_db.GetDb(), name); |
| 246 | auto* rawPtr = table.get(); | 285 | auto* rawPtr = table.get(); |
| 247 | m_lazyTables.push_back(std::move(table)); | 286 | m_lazyTables.push_back(std::move(table)); |
| @@ -294,7 +333,50 @@ public: | |||
| 294 | 333 | ||
| 295 | std::vector<std::string> GetAllTableName() const { return m_tableNames; } | 334 | std::vector<std::string> GetAllTableName() const { return m_tableNames; } |
| 296 | 335 | ||
| 336 | + // 获取底层数据库连接对象(供批量写事务SqliteWriteTransaction使用) | ||
| 337 | + sim::SqliteDatabase& GetDatabase() { return m_db; } | ||
| 338 | + | ||
| 297 | void ClearAll() { m_db.ClearAllTables(); } | 339 | void ClearAll() { m_db.ClearAllTables(); } |
| 298 | }; | 340 | }; |
| 299 | 341 | ||
| 342 | +namespace sim { | ||
| 343 | +// RAII批量写事务守卫: 构造时在RunnerDB所在SQLite连接上开启事务, 作用域内的 | ||
| 344 | +// RunnerDB::Add/Update并入该事务统一提交, 消除逐条独立事务的提交开销; | ||
| 345 | +// 显式调用Commit()提交; 未提交即离开作用域(异常/提前返回)时自动回滚。 | ||
| 346 | +// 注意: 事务期间应保证无其他线程并发写库。 | ||
| 347 | +class SqliteWriteTransaction { | ||
| 348 | +public: | ||
| 349 | + SqliteWriteTransaction() : db_(SimRunnerSqliteDB::Instance().GetDatabase()), began_(false), committed_(false) | ||
| 350 | + { | ||
| 351 | + began_ = db_.BeginTransaction(); | ||
| 352 | + } | ||
| 353 | + | ||
| 354 | + ~SqliteWriteTransaction() | ||
| 355 | + { | ||
| 356 | + if (began_ && !committed_) { | ||
| 357 | + db_.RollbackTransaction(); | ||
| 358 | + } | ||
| 359 | + } | ||
| 360 | + | ||
| 361 | + SqliteWriteTransaction(const SqliteWriteTransaction&) = delete; | ||
| 362 | + SqliteWriteTransaction& operator=(const SqliteWriteTransaction&) = delete; | ||
| 363 | + SqliteWriteTransaction(SqliteWriteTransaction&&) = delete; | ||
| 364 | + SqliteWriteTransaction& operator=(SqliteWriteTransaction&&) = delete; | ||
| 365 | + | ||
| 366 | + bool Commit() | ||
| 367 | + { | ||
| 368 | + if (!began_ || committed_) { | ||
| 369 | + return began_ && committed_; | ||
| 370 | + } | ||
| 371 | + committed_ = db_.CommitTransaction(); | ||
| 372 | + return committed_; | ||
| 373 | + } | ||
| 374 | + | ||
| 375 | +private: | ||
| 376 | + SqliteDatabase& db_; | ||
| 377 | + bool began_; | ||
| 378 | + bool committed_; | ||
| 379 | +}; | ||
| 380 | +} // namespace sim | ||
| 381 | + | ||
| 300 | 382 | ||
Rtest/hccl_vm/src/plugin/runner/aiv_executor/aiv_graph_executor_mgr.h→test/hccl_vm/src/common/sim_dpu_kernel_lib_mgr.h+27-17
Rtest/hccl_vm/src/device_arm/aicpu_args_stub.h→test/hccl_vm/src/device_arm/common/aicpu_args_stub.h+0-0
Rtest/hccl_vm/src/device_arm/hccl_device_pub.cc→test/hccl_vm/src/device_arm/common/hccl_device_pub.cc+172-78
Rtest/hccl_vm/src/device_arm/hccl_device_pub.h→test/hccl_vm/src/device_arm/common/hccl_device_pub.h+40-4
Rtest/hccl_vm/src/device_arm/hccl_task_collection.cc→test/hccl_vm/src/device_arm/common/hccl_task_collection.cc+15-14
Rtest/hccl_vm/src/device_arm/hccl_task_collection.h→test/hccl_vm/src/device_arm/common/hccl_task_collection.h+1-1
Rtest/hccl_vm/src/device_arm/sim_aicpu_pipe_handler.cc→test/hccl_vm/src/device_arm/common/sim_aicpu_pipe_handler.cc+74-34
Rtest/hccl_vm/src/device_arm/sim_aicpu_pipe_handler.h→test/hccl_vm/src/device_arm/common/sim_aicpu_pipe_handler.h+6-4
Rtest/hccl_vm/src/device_arm/sim_kernel_lib_mgr.cc→test/hccl_vm/src/device_arm/common/sim_kernel_lib_mgr.cc+4-4
Rtest/hccl_vm/src/device_arm/sim_kernel_lib_mgr.h→test/hccl_vm/src/device_arm/common/sim_kernel_lib_mgr.h+2-2
Rtest/hccl_vm/src/plugin/checker/src/framework/task_graph_generator/ccu_task_transform_instruct_common.cc→test/hccl_vm/src/device_arm/level1/comm_domain_device_stub.cc+36-18
Rtest/hccl_vm/src/plugin/checker/src/framework/singletask_check/CMakeLists.txt→test/hccl_vm/src/device_arm/level2/CMakeLists.txt+2-3
Rtest/hccl_vm/src/device_arm/proxy/CMakeLists.txt→test/hccl_vm/src/device_arm/level2/proxy/CMakeLists.txt+5-2
Rtest/hccl_vm/src/device_arm/proxy/aicpu_stub.cc→test/hccl_vm/src/device_arm/level2/proxy/aicpu_stub.cc+0-0
Rtest/hccl_vm/src/device_arm/proxy/aprofiling_stub.cc→test/hccl_vm/src/device_arm/level2/proxy/aprofiling_stub.cc+0-0
Rtest/hccl_vm/src/device_arm/proxy/ascend_hal_stub.cc→test/hccl_vm/src/device_arm/level2/proxy/ascend_hal_stub.cc+5-3
Rtest/hccl_vm/src/device_arm/proxy/depends_stub.cc→test/hccl_vm/src/device_arm/level2/proxy/depends_stub.cc+0-0
Rtest/hccl_vm/src/device_arm/proxy/device_sqe_parse_stub.cc→test/hccl_vm/src/device_arm/level2/proxy/device_sqe_parse_stub.cc+303-87
Rtest/hccl_vm/src/device_arm/proxy/device_sqe_parse_stub.h→test/hccl_vm/src/device_arm/level2/proxy/device_sqe_parse_stub.h+1-1
Rtest/hccl_vm/src/plugin/checker/src/utils/dump/dump_memory_timeline.h→test/hccl_vm/src/device_arm/level2/proxy/hcomm_api_stub.cc+36-34
Rtest/hccl_vm/src/device_arm/proxy/slog_stub.cc→test/hccl_vm/src/device_arm/level2/proxy/slog_stub.cc+0-0
Rtest/hccl_vm/src/device_arm/proxy/sqe_v82_stub.h→test/hccl_vm/src/device_arm/level2/proxy/sqe_v82_stub.h+2-1
Rtest/hccl_vm/src/device_arm/proxy/udma_data_struct_stub.h→test/hccl_vm/src/device_arm/level2/proxy/udma_data_struct_stub.h+0-0
Rtest/hccl_vm/src/plugin/checker/src/utils/dump/dump_graph.h→test/hccl_vm/src/plugin/checker/src/framework/composite_op_grouping.h+14-12
Rtest/hccl_vm/src/plugin/checker/src/utils/task_utils.h→test/hccl_vm/src/plugin/checker/src/framework/task_graph_generator_v3/ccu_graph_generator_v3/ccu_convert_config_generator_v3.h+14-16
Rtest/hccl_vm/src/plugin/checker/src/framework/task_graph_generator/type_conversion.cc→test/hccl_vm/src/plugin/checker/src/utils/type_conversion.cc+0-0
Rtest/hccl_vm/src/plugin/checker/src/framework/task_graph_generator/type_conversion.h→test/hccl_vm/src/plugin/checker/src/utils/type_conversion.h+7-1
Rtest/hccl_vm/src/plugin/checker/src/utils/dump/msgpack_writer.h→test/hccl_vm/src/proxy/ibv_dpu_stub.h+33-29
Rtest/hccl_vm/src/plugin/checker/src/framework/mem_conflict_check/CMakeLists.txt→test/hccl_vm/src/proxy/level0/CMakeLists.txt+24-8
Rtest/hccl_vm/src/plugin/runner/aiv_executor/aiv_graph_executor_mgr.cc→test/hccl_vm/src/proxy/level0/ccu_fallback_stub.cc+18-12
Rtest/hccl_vm/src/proxy/aclrt_context_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_context_stub.cc+2-3
Rtest/hccl_vm/src/proxy/aclrt_device_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_device_stub.cc+97-47
Rtest/hccl_vm/src/proxy/aclrt_exception_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_exception_stub.cc+2-9
Rtest/hccl_vm/src/proxy/aclrt_exec_control.cc→test/hccl_vm/src/proxy/level2/aclrt_exec_control.cc+1-1
Rtest/hccl_vm/src/proxy/aclrt_kernel_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_kernel_stub.cc+329-542
Rtest/hccl_vm/src/proxy/aclrt_memory_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_memory_stub.cc+149-80
Rtest/hccl_vm/src/proxy/aclrt_notify_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_notify_stub.cc+51-35
Rtest/hccl_vm/src/proxy/aclrt_runtime_config.cc→test/hccl_vm/src/proxy/level2/aclrt_runtime_config.cc+4-4
Rtest/hccl_vm/src/proxy/aclrt_stream_stub.cc→test/hccl_vm/src/proxy/level2/aclrt_stream_stub.cc+84-54
Rtest/hccl_vm/src/proxy/aiv_kernel/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/CMakeLists.txt+13-1
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/ai_core_stub.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/ai_core_stub.cc+111-14
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/ai_core_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/ai_core_stub.h+37-9
Rtest/hccl_vm/test/plugin/checker/framework/task_graph_generator/ccu_task_transform_stub.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/aiv_db_runtime.h+11-10
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/aiv_mode_stub_base.h→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/aiv_mode_stub_base.h+20-4
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/aiv_task.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/aiv_task.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/aiv_task.h→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/aiv_task.h+44-58
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/aiv_task_json.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/aiv_task_json.cc+46-11
Rtest/hccl_vm/src/proxy/aiv_kernel/aiv_mode_stub/aiv_task_json.h→test/hccl_vm/src/proxy/level2/aiv_kernel/aiv_mode_stub/aiv_task_json.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_base_stub.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_base_stub.cc+4-4
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_base_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_base_stub.h+25-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_copy_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_copy_stub.h+25-15
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_math_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_math_stub.h+1-1
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_memory_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_memory_stub.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_sync_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_sync_stub.h+9-3
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/ascendc_utils_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/ascendc_utils_stub.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/global_tensor_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/global_tensor_stub.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/kernel_common.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/kernel_common.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/kernel_operator.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/kernel_operator.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/local_tensor_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/local_tensor_stub.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/t_buf_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/t_buf_stub.h+2-2
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/t_pipe.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/t_pipe.h+5-8
Rtest/hccl_vm/src/proxy/aiv_kernel/ascendc_stub/t_que_bind_stub.h→test/hccl_vm/src/proxy/level2/aiv_kernel/ascendc_stub/t_que_bind_stub.h+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/CMakeLists.txt+1-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/aiv_communication_base_v2.h→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/aiv_communication_base_v2.h+80-81
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/aiv_model_init.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/aiv_model_init.cc+154-7
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/aiv_model_init.h→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/aiv_model_init.h+1-1
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/all_gather/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/all_gather/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/all_gather/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/all_gather/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/all_reduce/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/all_reduce/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/all_reduce/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/all_reduce/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/all_to_all_v/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/all_to_all_v/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/all_to_all_v/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/all_to_all_v/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/broadcast/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/broadcast/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/broadcast/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/broadcast/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/recv/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/recv/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/recv/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/recv/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/reduce/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/reduce/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/reduce/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/reduce/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/reduce_scatter/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/reduce_scatter/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/reduce_scatter/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/reduce_scatter/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/scatter/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/scatter/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/scatter/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/scatter/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/send/CMakeLists.txt→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/send/CMakeLists.txt+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/send/aiv_communication_v2.cc→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/send/aiv_communication_v2.cc+0-0
Rtest/hccl_vm/src/proxy/aiv_kernel/hccl_op_stub/sync_interface.h→test/hccl_vm/src/proxy/level2/aiv_kernel/hccl_op_stub/sync_interface.h+0-0
Rtest/hccl_vm/src/plugin/checker/src/framework/mem_conflict_check/mem_conflict_check_utils.h→test/hccl_vm/src/proxy/level2/ibv_dpu_stub.h+33-18
Rtest/hccl_vm/src/plugin/checker/src/framework/task_graph_generator/CMakeLists.txt→test/hccl_vm/test/topo/generate_golden.sh+18-18