已合并
test目录新增hccl_vm工具 #2925
lijuntao26创建于 6月22日
test目录新增hccl_vm工具 #2925
已合并
共 744 个文件变更+175389-0
| @@ -0,0 +1,19 @@ | |||
| 1 | +**/.idea/ | ||
| 2 | +**/.vscode/ | ||
| 3 | +**/cmake-build-debug/ | ||
| 4 | +**/cmake-build-release/ | ||
| 5 | +**/output/ | ||
| 6 | +**/build/ | ||
| 7 | +**/build_test/ | ||
| 8 | +**/tmp/ | ||
| 9 | +**/CMakeFiles/ | ||
| 10 | +**/_deps/ | ||
| 11 | +**/hccl_vm_install/ | ||
| 12 | +**/lib64/ | ||
| 13 | +**/hccl_device_install/ | ||
| 14 | +**/build_device/ | ||
| 15 | +**/node_modules/ | ||
| 16 | +**/dist/ | ||
| 17 | +coverage_report/ | ||
| 18 | +ut_logs/ | ||
| 19 | +third_party/ | ||
| @@ -0,0 +1,147 @@ | |||
| 1 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 2 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 3 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 4 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 5 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 6 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 7 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 8 | + | ||
| 9 | +cmake_minimum_required(VERSION 3.10) | ||
| 10 | +project(hccl_vm VERSION 1.0.0) | ||
| 11 | +#set(CMAKE_VERBOSE_MAKEFILE true) | ||
| 12 | + | ||
| 13 | +set(CMAKE_CXX_STANDARD 17) | ||
| 14 | +set(CMAKE_CXX_STANDARD_REQUIRED True) | ||
| 15 | +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") | ||
| 16 | +set(CMAKE_POSITION_INDEPENDENT_CODE ON) | ||
| 17 | + | ||
| 18 | +set(OUTPUT_DIR ${CMAKE_BINARY_DIR}/output) | ||
| 19 | +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OUTPUT_DIR}) | ||
| 20 | +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${OUTPUT_DIR}) | ||
| 21 | +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT_DIR}) | ||
| 22 | + | ||
| 23 | +option(ENABLE_COVERAGE "Enable gcov/lcov coverage instrumentation for all targets" OFF) | ||
| 24 | +if(ENABLE_COVERAGE) | ||
| 25 | + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -fprofile-arcs -ftest-coverage") | ||
| 26 | + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") | ||
| 27 | + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage") | ||
| 28 | + message(STATUS "Coverage instrumentation enabled: --coverage (all targets)") | ||
| 29 | +endif() | ||
| 30 | + | ||
| 31 | +option(BUILD_DEVICE_ARM "Build device_arm (ARM cross-compile)" OFF) | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +set(HCOMM_VM_ROOT_PATH ${CMAKE_CURRENT_SOURCE_DIR}) | ||
| 35 | + | ||
| 36 | +if(NOT DEFINED ASCEND_CANN_PACKAGE_PATH) | ||
| 37 | + if(NOT DEFINED ENV{ASCEND_TOOLKIT_HOME}) | ||
| 38 | + message(FATAL_ERROR "Error: ASCEND_TOOLKIT_HOME environment variable is not set. Please set it to the Ascend Toolkit installation path.") | ||
| 39 | + endif() | ||
| 40 | + set(ASCEND_CANN_PACKAGE_PATH $ENV{ASCEND_TOOLKIT_HOME}) | ||
| 41 | +endif() | ||
| 42 | +message(STATUS "ASCEND_CANN_PACKAGE_PATH: ${ASCEND_CANN_PACKAGE_PATH}") | ||
| 43 | + | ||
| 44 | +if(BUILD_DEVICE_ARM) | ||
| 45 | + if(NOT DEFINED HCCL_CODE_ROOT) | ||
| 46 | + if(NOT DEFINED ENV{HCCL_CODE_HOME}) | ||
| 47 | + message(FATAL_ERROR "Error: BUILD_DEVICE_ARM is ON, but HCCL_CODE_HOME environment variable is not set. Please set it to the HCCL source path.") | ||
| 48 | + endif() | ||
| 49 | + set(HCCL_CODE_ROOT $ENV{HCCL_CODE_HOME}) | ||
| 50 | + endif() | ||
| 51 | + message(STATUS "HCCL_CODE_ROOT: ${HCCL_CODE_ROOT}") | ||
| 52 | +endif() | ||
| 53 | + | ||
| 54 | +if(NOT DEFINED HCOMM_CODE_ROOT) | ||
| 55 | + if(NOT DEFINED ENV{HCOMM_CODE_HOME}) | ||
| 56 | + message(FATAL_ERROR "Error: HCOMM_CODE_HOME environment variable is not set. Please set it to the Ascend Toolkit installation path.") | ||
| 57 | + endif() | ||
| 58 | + set(HCOMM_CODE_ROOT $ENV{HCOMM_CODE_HOME}) | ||
| 59 | +endif() | ||
| 60 | +message(STATUS "HCOMM_CODE_ROOT: ${HCOMM_CODE_ROOT}") | ||
| 61 | + | ||
| 62 | +set(HCCL_VM_INCLUDE ${CMAKE_SOURCE_DIR}/include) | ||
| 63 | +set(HCCL_VM_SRC ${CMAKE_SOURCE_DIR}/src) | ||
| 64 | +set(HCCL_VM_INSTALL ${CMAKE_SOURCE_DIR}/hccl_vm_install) | ||
| 65 | +set(THRID_PARTY_DIR ${CMAKE_SOURCE_DIR}/third_party) | ||
| 66 | +message(STATUS "HCOMM_CODE_ROOT: ${HCOMM_CODE_ROOT}") | ||
| 67 | + | ||
| 68 | +set(HCOMM_CODE_PLATFORM ${HCOMM_CODE_ROOT}/src/platform) | ||
| 69 | + | ||
| 70 | +# 使用一个自定义的 Cache 变量，默认值设为 OFF 或空 | ||
| 71 | +set(HCCL_VM_PROJECT_PREFIX_SET OFF CACHE INTERNAL "Check if install prefix is set") | ||
| 72 | + | ||
| 73 | +if(NOT HCCL_VM_PROJECT_PREFIX_SET) | ||
| 74 | + # 只要这个逻辑没跑成功过，就设置一次 | ||
| 75 | + set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/hccl_vm_install" CACHE PATH "Default install prefix" FORCE) | ||
| 76 | + set(HCCL_VM_PROJECT_PREFIX_SET ON CACHE INTERNAL "Check if install prefix is set") | ||
| 77 | + message(STATUS "First-time setup: Installation prefix locked to: ${CMAKE_INSTALL_PREFIX}") | ||
| 78 | +endif() | ||
| 79 | + | ||
| 80 | +set(CMAKE_INSTALL_RPATH "\$ORIGIN") | ||
| 81 | + | ||
| 82 | +include(cmake/third_party/yaml-cpp.cmake) | ||
| 83 | +include(cmake/third_party/nlohmann_json.cmake) | ||
| 84 | +include(cmake/third_party/cli11.cmake) | ||
| 85 | +include(cmake/third_party/spdlog.cmake) | ||
| 86 | +include(cmake/third_party/sqlite3.cmake) | ||
| 87 | + | ||
| 88 | +function(config_hcclvm_install) | ||
| 89 | + install(DIRECTORY DESTINATION "data") | ||
| 90 | + install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/asset/" | ||
| 91 | + DESTINATION "." | ||
| 92 | + # 文件的权限 (755) | ||
| 93 | + FILE_PERMISSIONS | ||
| 94 | + OWNER_READ OWNER_WRITE OWNER_EXECUTE | ||
| 95 | + GROUP_READ GROUP_EXECUTE | ||
| 96 | + WORLD_READ WORLD_EXECUTE | ||
| 97 | + # 目录的权限 (755) | ||
| 98 | + DIRECTORY_PERMISSIONS | ||
| 99 | + OWNER_READ OWNER_WRITE OWNER_EXECUTE | ||
| 100 | + GROUP_READ GROUP_EXECUTE | ||
| 101 | + WORLD_READ WORLD_EXECUTE | ||
| 102 | + ) | ||
| 103 | +endfunction() | ||
| 104 | + | ||
| 105 | +# 编译HCCL-VM | ||
| 106 | +if (NOT BUILD_DEVICE_ARM) | ||
| 107 | + config_hcclvm_install() | ||
| 108 | + add_subdirectory(src) | ||
| 109 | +endif() | ||
| 110 | + | ||
| 111 | +set(CPACK_PACKAGE_NAME "hccl_vm") | ||
| 112 | +set(CPACK_GENERATOR "TGZ") | ||
| 113 | +include(CPack) | ||
| 114 | + | ||
| 115 | +# 编译DEVICE-ARM-交叉编译版本 | ||
| 116 | +if(BUILD_DEVICE_ARM) | ||
| 117 | + message(STATUS "BUILD_DEVICE_ARM start") | ||
| 118 | + set(DEVICE_ARM_BUILD_DIR ${CMAKE_SOURCE_DIR}/build_device) | ||
| 119 | + set(DEVICE_ARM_INSTALL_DIR ${CMAKE_SOURCE_DIR}/hccl_vm_install/device) | ||
| 120 | + include(ExternalProject) | ||
| 121 | + ExternalProject_Add(device_arm_external | ||
| 122 | + SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/device_arm | ||
| 123 | + CMAKE_ARGS | ||
| 124 | + -DCMAKE_INSTALL_PREFIX=${DEVICE_ARM_INSTALL_DIR} | ||
| 125 | + -DHCCL_VM_SRC=${HCCL_VM_SRC} | ||
| 126 | + -DTHRID_PARTY_DIR=${THRID_PARTY_DIR} | ||
| 127 | + -DHCCL_VM_INCLUDE=${HCCL_VM_INCLUDE} | ||
| 128 | + -DASCEND_CANN_PACKAGE_PATH=${ASCEND_CANN_PACKAGE_PATH} | ||
| 129 | + -DHCOMM_CODE_ROOT=${HCOMM_CODE_ROOT} | ||
| 130 | + -DHCCL_CODE_ROOT=${HCCL_CODE_ROOT} | ||
| 131 | + | ||
| 132 | + BINARY_DIR ${DEVICE_ARM_BUILD_DIR} | ||
| 133 | + INSTALL_DIR ${DEVICE_ARM_INSTALL_DIR} | ||
| 134 | + BUILD_ALWAYS ON | ||
| 135 | + ) | ||
| 136 | + | ||
| 137 | + message(STATUS "device_arm cross-compilation enabled") | ||
| 138 | + message(STATUS " Build dir: ${DEVICE_ARM_BUILD_DIR}") | ||
| 139 | + message(STATUS " Install dir: ${DEVICE_ARM_INSTALL_DIR}") | ||
| 140 | +endif() | ||
| 141 | + | ||
| 142 | + | ||
| 143 | +option(BUILD_TESTS "Build unit tests" OFF) | ||
| 144 | +if(BUILD_TESTS) | ||
| 145 | + include(cmake/third_party/gtest.cmake) | ||
| 146 | + add_subdirectory(test) | ||
| 147 | +endif() | ||
| @@ -0,0 +1,745 @@ | |||
| 1 | +# HCCL-VM 指导手册 | ||
| 2 | + | ||
| 3 | +## 1. 概述 | ||
| 4 | + | ||
| 5 | +HCCL-VM是面向华为昇腾NPU卡的高性能集合通信的虚拟执行环境，该工具旨在无真实昇腾硬件的条件下，实现HCCL集合通信算子的开发和功能验证。 | ||
| 6 | + | ||
| 7 | +## 2. 前置依赖 | ||
| 8 | + | ||
| 9 | +| | | | ||
| 10 | +| -------- | ------------- | | ||
| 11 | +| 系统架构 | x86_64 Ubuntu22.04及以上 | | ||
| 12 | +| 规格约束 | Ascend950，其余参照[约束详情](#45-工具规格约束) | | ||
| 13 | + | ||
| 14 | +### 2.1 CANN包安装 | ||
| 15 | + | ||
| 16 | +安装最新版本CANN Toolkit开发套件包和CANN ops算子包 [下载链接](https://ascend.devcloud.huaweicloud.com/artifactory/cann-run-mirror/software/master/) | ||
| 17 | + | ||
| 18 | +```bash | ||
| 19 | +./Ascend-cann-toolkit_9.1.0_linux-x86_64.run --install --install-path=/home/Ascend | ||
| 20 | +./Ascend-cann-950-ops_9.1.0_linux-x86_64.run --install --install-path=/home/Ascend | ||
| 21 | +``` | ||
| 22 | + | ||
| 23 | +### 2.2 hccl_test编译 | ||
| 24 | + | ||
| 25 | +hccl_test是昇腾官方提供的HCCL性能测试工具，详见[HCCL性能测试工具](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/910beta1/devaids/hccltool/HCCLpertest_16_0001.html)，HCCL-VM支持在虚拟环境中运行hccl_test用例。请先参照[hccl_test用例构建](#42-hccl-test用例构建)章节进行用例二进制程序的编译。 | ||
| 26 | + | ||
| 27 | +备注：可选，未来支持Pytorch用例 | ||
| 28 | + | ||
| 29 | +--- | ||
| 30 | + | ||
| 31 | +## 3. 快速上手 | ||
| 32 | + | ||
| 33 | +### 3.1 工具构建&安装 | ||
| 34 | + | ||
| 35 | +```bash | ||
| 36 | +# 1. 创建工作目录 | ||
| 37 | +mkdir -p /home/workspace | ||
| 38 | +cd /home/workspace | ||
| 39 | + | ||
| 40 | +# 2. 下载依赖源码 | ||
| 41 | +git clone https://gitcode.com/cann/hccl.git | ||
| 42 | +git clone https://gitcode.com/cann/hcomm.git | ||
| 43 | +git clone https://gitcode.com/zhupc158/CheckerL2.git | ||
| 44 | + | ||
| 45 | +# 3. 安装第三方依赖 | ||
| 46 | +sudo apt-get update | ||
| 47 | +sudo apt install build-essential cmake libsqlite3-dev rdma-core libibverbs-dev pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu qemu-user-static binfmt-support | ||
| 48 | + | ||
| 49 | +# 4. 编译HCCL-VM工具 | ||
| 50 | +source /home/Ascend/cann/set_env.sh | ||
| 51 | +export HCCL_CODE_HOME=/home/workspace/hccl | ||
| 52 | +export HCOMM_CODE_HOME=/home/workspace/hcomm | ||
| 53 | +./build.sh --full | ||
| 54 | +``` | ||
| 55 | + | ||
| 56 | +### 3.2 使用示例 | ||
| 57 | + | ||
| 58 | +#### 3.2.1 环境配置 | ||
| 59 | + | ||
| 60 | +请参照[hccl_rootinfo文件内容](#47-hccl_rootinfojson文件)，创建并配置hccl_rootinfo.json文件， | ||
| 61 | + | ||
| 62 | +#### 3.2.2 CCU模式 | ||
| 63 | + | ||
| 64 | +1. 环境变量配置 | ||
| 65 | + | ||
| 66 | +```bash | ||
| 67 | +# 进入工具安装目录 | ||
| 68 | +cd /home/workspace/CheckerL2/hccl_vm_install | ||
| 69 | +source /home/Ascend/cann/set_env.sh | ||
| 70 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | ||
| 71 | +export RANK_TABLE_FILE=$(pwd)/ranktable.json | ||
| 72 | +export HCCL_OP_EXPANSION_MODE="CCU_SCHED" | ||
| 73 | +``` | ||
| 74 | + | ||
| 75 | +2. 执行 | ||
| 76 | + | ||
| 77 | +```bash | ||
| 78 | +# 进入工具安装目录 | ||
| 79 | +cd /home/workspace/CheckerL2/hccl_vm_install | ||
| 80 | + | ||
| 81 | +# 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 | ||
| 82 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml | ||
| 83 | + | ||
| 84 | +# 如需启用runner插件（可选） | ||
| 85 | +(hvm)$> hccl-vm plugin install @runner | ||
| 86 | + | ||
| 87 | +# 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | ||
| 88 | +(hvm)$> hccl-vm mock-comm 112 | ||
| 89 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 90 | + | ||
| 91 | +# 执行checker校验 | ||
| 92 | +(hvm)$> hccl-vm plugin run @checker | ||
| 93 | + | ||
| 94 | +# 退出工具终端 | ||
| 95 | +(hvm)$> exit | ||
| 96 | +``` | ||
| 97 | + | ||
| 98 | +3. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) [Checker结果查看](#492-checker插件结果) | ||
| 99 | + | ||
| 100 | +#### 3.2.3 AICPU模式 | ||
| 101 | + | ||
| 102 | +AICPU展开模式需要将算法展开步骤放到设备侧执行，因此hccl-vm工具需要将HCCL的设备侧的符号编译并模拟执行。由于设备侧符号是ARM架构的，因此在X86环境上编译时需要借助交叉编译器，运行时需要借助QEMU实现AICPU模式的模拟运行 | ||
| 103 | + | ||
| 104 | +设备侧符号使用hccl和hcomm的源码编译，为了保证Host与Device通信协议正确，需要同时编译Host侧的安装包并进行替换安装 | ||
| 105 | + | ||
| 106 | +1. HCCL设备侧符号编译、安装、拷贝等 | ||
| 107 | + | ||
| 108 | +```bash | ||
| 109 | +cd /home/workspace/CheckerL2 | ||
| 110 | +bash ./build_pkg.sh | ||
| 111 | +``` | ||
| 112 | + | ||
| 113 | +2. 环境变量配置 | ||
| 114 | + | ||
| 115 | +```bash | ||
| 116 | +# 进入工具安装目录 | ||
| 117 | +cd /home/workspace/CheckerL2/hccl_vm_install | ||
| 118 | +source /home/Ascend/cann/set_env.sh | ||
| 119 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | ||
| 120 | +export RANK_TABLE_FILE=$(pwd)/ranktable.json | ||
| 121 | +export HCCL_OP_EXPANSION_MODE="AI_CPU" | ||
| 122 | +``` | ||
| 123 | + | ||
| 124 | +3. 执行 | ||
| 125 | + | ||
| 126 | +```bash | ||
| 127 | +# 进入工具安装目录 | ||
| 128 | +cd /home/workspace/CheckerL2/hccl_vm_install | ||
| 129 | + | ||
| 130 | +# 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 | ||
| 131 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml | ||
| 132 | + | ||
| 133 | +# 如需启用runner插件（可选） | ||
| 134 | +(hvm)$> hccl-vm plugin install @runner | ||
| 135 | + | ||
| 136 | +# 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | ||
| 137 | +(hvm)$> hccl-vm mock-comm 112 | ||
| 138 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 139 | + | ||
| 140 | +# 执行checker校验 | ||
| 141 | +(hvm)$> hccl-vm plugin run @checker | ||
| 142 | + | ||
| 143 | +# 退出工具终端 | ||
| 144 | +(hvm)$> exit | ||
| 145 | +``` | ||
| 146 | + | ||
| 147 | +4. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) [Checker结果查看](#492-checker插件结果) | ||
| 148 | + | ||
| 149 | +#### 3.2.4 AIV模式 | ||
| 150 | + | ||
| 151 | +**当前仅支持Runner插件** | ||
| 152 | + | ||
| 153 | +1. 环境变量配置 | ||
| 154 | + | ||
| 155 | +```bash | ||
| 156 | +# 进入工具安装目录 | ||
| 157 | +cd /home/workspace/CheckerL2/hccl_vm_install | ||
| 158 | +source /home/Ascend/cann/set_env.sh | ||
| 159 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | ||
| 160 | +export RANK_TABLE_FILE=$(pwd)/ranktable.json | ||
| 161 | +export HCCL_OP_EXPANSION_MODE="AIV" | ||
| 162 | +``` | ||
| 163 | + | ||
| 164 | +2. 执行 | ||
| 165 | + | ||
| 166 | +```bash | ||
| 167 | +# 进入工具安装目录 | ||
| 168 | +cd /home/workspace/CheckerL2/hccl_vm_install | ||
| 169 | + | ||
| 170 | +# 选择昇腾集群拓扑配置文件，启动工具，初始化集群环境，进入工具命令行 | ||
| 171 | +./hccl-vm start ascend950_cluster_32_server_normal.yaml | ||
| 172 | + | ||
| 173 | +# 如需启用runner插件（可选） | ||
| 174 | +(hvm)$> hccl-vm plugin install @runner | ||
| 175 | + | ||
| 176 | +# 选择本次算子执行的通信域配置文件（在1个超节点1个Server1个NPU的集群环境运行hccl_test用例） | ||
| 177 | +(hvm)$> hccl-vm mock-comm 112 | ||
| 178 | +(hvm)$> mpirun --allow-run-as-root --oversubscribe -np 2 ${ASCEND_HOME_PATH}/tools/hccl_test/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 > log.txt | ||
| 179 | + | ||
| 180 | +# 退出工具终端 | ||
| 181 | +(hvm)$> exit | ||
| 182 | +``` | ||
| 183 | + | ||
| 184 | +3. 验证hccl_test用例运行结果 [Runner结果查看](#491-runner插件结果) | ||
| 185 | + | ||
| 186 | +### 3.3 Pytorch用例示例 | ||
| 187 | + | ||
| 188 | +暂不支持 | ||
| 189 | + | ||
| 190 | +### 3.4 hccl代码修改验证示例 | ||
| 191 | + | ||
| 192 | +若您修改了CANN的算子包代码，如新增了算法类型，为保证您的修改生效，需要按照如下步骤操作执行。build_pkg.sh脚本帮助用户执行编包、装包、拷贝Device侧依赖符号，执行前需设置环境变量: | ||
| 193 | + | ||
| 194 | +```bash | ||
| 195 | +# 假设您的CANN安装目录为：/home/Ascend | ||
| 196 | +source /home/Ascend/cann/set_env.sh | ||
| 197 | +# 配置hccl代码仓路径 | ||
| 198 | +export HCCL_CODE_HOME=/home/workspace/hccl | ||
| 199 | +# 配置hcomm代码仓路径 | ||
| 200 | +export HCOMM_CODE_HOME=/home/workspace/hcomm | ||
| 201 | +``` | ||
| 202 | + | ||
| 203 | +1. 若您修改了CANN hccl仓代码，请执行bash build_pkg.sh --install hccl | ||
| 204 | +2. 若您修改了CANN hcomm仓代码，请执行bash build_pkg.sh --install hcomm | ||
| 205 | +3. 参考[使用示例](#32-使用示例)步骤，重新运行用例。 | ||
| 206 | + | ||
| 207 | +--- | ||
| 208 | + | ||
| 209 | +## 4. 详细指导 | ||
| 210 | + | ||
| 211 | +### 4.1 工具环境变量配置 | ||
| 212 | + | ||
| 213 | +**HCCL-VM环境变量说明**： | ||
| 214 | + | ||
| 215 | +| 环境变量 | 用途 | 示例 | | ||
| 216 | +| ------------------------- | --------------------------------------------------------------------------------------------------------- | --------------------- | | ||
| 217 | +| `HCCL_CODE_HOME` | HCCL-VM编译指定HCCL源码路径。默认未配置。 | `export HCCL_CODE_HOME=/home/workspace/hccl` | | ||
| 218 | +| `HCOMM_CODE_HOME` | HCCL-VM编译指定HCOMM源码路径。默认未配置。 | `export HCOMM_CODE_HOME=/home/workspace/hcomm` | | ||
| 219 | +| `HCCLVM_ENABLE_DUMP_DATA` | 使能Runner插件dump input\&output数据。若使能，则在测试用例执行过程中，会将每个算子的input\&output数据dump到all\_rank\_input\_output.txt文件中 | `export HCCLVM_ENABLE_DUMP_DATA=1 使能，export HCCLVM_ENABLE_DUMP_DATA=0 禁用` | | ||
| 220 | + | ||
| 221 | +### 4.2 HCCL-Test用例构建 | ||
| 222 | + | ||
| 223 | +hccl_test用例源码在CANN包安装目录下，支持OpenMPI和MPICH两种环境编译、运行，运行时差异详见[OpenMPI和MPICH环境用例执行差异](#48-openmpi和mpich环境用例运行差异)，本用例指导中以OpenMPI环境为例。 | ||
| 224 | + | ||
| 225 | +#### 4.2.1 OpenMPI环境编译 | ||
| 226 | + | ||
| 227 | +1. 安装OpenMPI | ||
| 228 | + | ||
| 229 | +```bash | ||
| 230 | +sudo apt-get update | ||
| 231 | +sudo apt install openmpi-bin libopenmpi-dev | ||
| 232 | +``` | ||
| 233 | + | ||
| 234 | +2. 编译hccl_test | ||
| 235 | + | ||
| 236 | +```bash | ||
| 237 | +# 修改CANN安装目录权限 | ||
| 238 | +chmod -R 755 /home/Ascend | ||
| 239 | + | ||
| 240 | +# 进入hccl_test用例源码目录 | ||
| 241 | +cd /home/Ascend/cann/tools/hccl_test | ||
| 242 | + | ||
| 243 | +# 设置CANN环境变量 | ||
| 244 | +source /home/Ascend/cann/set_env.sh | ||
| 245 | + | ||
| 246 | +# 临时修改Makefile脚本 | ||
| 247 | +if ! grep -q '\-lmpi_cxx' Makefile; then | ||
| 248 | + sed -i 's/-lmpi/-lmpi -lmpi_cxx/g' Makefile | ||
| 249 | +fi | ||
| 250 | + | ||
| 251 | +# 编译hccl_test用例 | ||
| 252 | +MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi make ASCEND_DIR=${ASCEND_HOME_PATH} | ||
| 253 | +``` | ||
| 254 | + | ||
| 255 | +#### 4.2.2 MPICH环境编译 | ||
| 256 | + | ||
| 257 | +假设mpich的路径为: `/usr/lib/mpich` | ||
| 258 | + | ||
| 259 | +```bash | ||
| 260 | +# 进入hccl_test用例源码目录 | ||
| 261 | +cd /home/Ascend/cann/tools/hccl_test | ||
| 262 | + | ||
| 263 | +# 设置CANN环境变量 | ||
| 264 | +source /home/Ascend/cann/set_env.sh | ||
| 265 | + | ||
| 266 | +# 配置环境变量 | ||
| 267 | +export LD_LIBRARY_PATH=/usr/lib/mpich/lib/:${ASCEND_HOME_PATH}/lib64/:${ASCEND_HOME_PATH}/x86_64-linux/devlib:$LD_LIBRARY_PATH | ||
| 268 | + | ||
| 269 | +# 编译hccl_test用例 | ||
| 270 | +make MPI_HOME=/usr/lib/mpich/ ASCEND_DIR=${ASCEND_HOME_PATH} | ||
| 271 | +``` | ||
| 272 | + | ||
| 273 | +### 4.3 昇腾集群拓扑配置文件说明 | ||
| 274 | + | ||
| 275 | +#### 4.3.1 Server/Pod拓扑配置文件说明 | ||
| 276 | + | ||
| 277 | +昇腾集群拓扑是由一个或多个Server/Pod形态的子拓扑，按照CLOS分层网络规则组合而成的。因此用户在生成集群拓扑前，需要确认每个Server/Pod的拓扑类型。 | ||
| 278 | +用户既可以选择HCCL-VM工具提供的预定义拓扑类型，也可以根据配置文件格式要求，自定义Server/Pod的拓扑类型。 | ||
| 279 | + | ||
| 280 | +描述一个Server/Pod的拓扑网络关系，主要包括以下方面： | ||
| 281 | + | ||
| 282 | + - **端口配置表**: 描述一个NPU卡的物理端口配置信息，如NPU与NPU直连端口（P2P），NPU出框端口（P2NET）等。 | ||
| 283 | + - **链路配置表**: 描述一个Server/Pod内的所有NPU卡之间的连接关系，如全连接（Full Mesh）等。 | ||
| 284 | + - **PortBound**: 描述一个NPU卡的某些端口的绑定关系，即多个端口绑定成一个PortGroup。 | ||
| 285 | + | ||
| 286 | +```yaml | ||
| 287 | +type: "server_intra_links" | ||
| 288 | +name: "ascend950_links_topo_demo" | ||
| 289 | +description: "ascend950芯片普通拓扑连接关系描述文件" | ||
| 290 | + | ||
| 291 | +soc_version: "Ascend950" | ||
| 292 | +device_num: 16 | ||
| 293 | + | ||
| 294 | +device_ports_allocate_map: | ||
| 295 | + # port分配表: 0: 不使用, 1: device直连, 2: device连交换机, 3: d2h端口 | ||
| 296 | + # portId: 0 1 2 3 4 5 6 7 8 | ||
| 297 | + - {die_id: 0, pin_map: [1, 1, 1, 0, 2, 2, 2, 2, 3]} # die0 | ||
| 298 | + - {die_id: 1, pin_map: [0, 0, 0, 0, 0, 0, 0, 0, 0]} # die1 | ||
| 299 | + | ||
| 300 | +# port_group: 描述哪些port合并为一个portGroup，相同portGroup的port对于IP地址相同 | ||
| 301 | +port_group: | ||
| 302 | + - {layer: 0, ports: ["0/4", "0/5", "0/6", "0/7"]} | ||
| 303 | + | ||
| 304 | +links: | ||
| 305 | + # ── 描述法：每行8个device组成全互联 ── | ||
| 306 | + - link_mode: "fullmesh" | ||
| 307 | + connections: | ||
| 308 | + # 如下示例表示：die0的device 0, 1, 2, 3都通过die0的port进行全连接 | ||
| 309 | + - {die_id: 0, devices_range: [0, 3]} | ||
| 310 | + - {die_id: 0, devices_range: [4, 7]} | ||
| 311 | + - {die_id: 0, devices_range: [8, 11]} | ||
| 312 | + - {die_id: 0, devices_range: [12, 15]} | ||
| 313 | + | ||
| 314 | + #- link_mode: "enum" | ||
| 315 | + # device_to_device_links: | ||
| 316 | + # 如下示例表示：device 0和1都通过die0的port，分别与device 1, 3, 5, 7的die1的port进行连接 | ||
| 317 | + # 即：device0与device1, device3, device5, device7进行连接，device1与device3, device5, device7进行连接 | ||
| 318 | + # - {src_die_id: 1, src_local_id_range: [0, 2], dst_die_id: 1, dst_local_id_range: [1, 3, 5, 7]} | ||
| 319 | + | ||
| 320 | + - link_mode: "enum" | ||
| 321 | + device_to_switch_links: | ||
| 322 | + # 如下示例表示：device0到device15都通过die0的port连接到交换机。结合portGroup可知，device0到device15都通过portGroup[0/4, 0/5, 0/6, 0/7]连接到交换机。 | ||
| 323 | + - {die_id: 0, devices_range: [0, 15]} | ||
| 324 | + | ||
| 325 | +``` | ||
| 326 | + | ||
| 327 | +**字段说明**： | ||
| 328 | + | ||
| 329 | + - **soc_version**: 芯片型号，如 `Ascend950`。 | ||
| 330 | + - **device_num**: 设备总数，根据芯片型号和拓扑类型确定。 | ||
| 331 | + - **device_ports_allocate_map**: 端口分配表，描述每个die的端口配置。1表示device直连端口，2表示device连交换机端口，3表示d2h端口。 | ||
| 332 | + - **port_group**: 描述哪些port合并为一个portGroup，相同portGroup的port对应IP地址相同。没有配置的port，则默认每个port为一个portGroup。 | ||
| 333 | + - **links**: 链路配置表，描述Server/Pod内的所有NPU卡之间，以及NPU与交换机之间的连接关系。 | ||
| 334 | + - **NPU直连关系**: 工具提供了两种方式配置NPU直连关系： | ||
| 335 | + - **link_mode == "fullmesh"**: 表示所有Device基于一个Die的Port进行全连接。后续有新增典型的连接方式，可以新增link_mode类型，如"ring"。 | ||
| 336 | + - **link_mode == "enum"**: 枚举法。当Server/Pod内的NPU连接方式比较复杂时，可以通过枚举所有的链路关系来描述。 | ||
| 337 | + - **NPU与交换机连接关系**: 用户可以通过枚举法配置NPU与交换机之间的连接关系。 | ||
| 338 | + - **device_to_device_links**: 描述NPU与NPU之间的连接关系。 | ||
| 339 | + - **device_to_switch_links**: 描述NPU与交换机之间的连接关系。 | ||
| 340 | + | ||
| 341 | +#### 4.3.2 集群拓扑配置文件说明 | ||
| 342 | + | ||
| 343 | +昇腾集群网络是由一个或多个Server/Pod形态的子拓扑，按照CLOS分层网络规则组合而成的。用户可根据集群规模和需求，选择不同的Server/Pod拓扑类型。 | ||
| 344 | + | ||
| 345 | +用户可按照如下配置文件格式，自定义集群拓扑配置文件： | ||
| 346 | + | ||
| 347 | +```yaml | ||
| 348 | +name: "ascend950_cluster_32_server_normal" | ||
| 349 | +description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | ||
| 350 | + | ||
| 351 | +# 超节点总数量 | ||
| 352 | +super_node_num: 4 | ||
| 353 | +# sever/pod总数量 | ||
| 354 | +server_num: 32 | ||
| 355 | +server_list: | ||
| 356 | + # 0-7 server: 均采用ascend950_server_topo_normal拓扑类型 | ||
| 357 | + - {super_pod_id: 0, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 358 | + - {super_pod_id: 1, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 359 | + - {super_pod_id: 2, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 360 | + - {super_pod_id: 3, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 361 | + | ||
| 362 | +``` | ||
| 363 | + | ||
| 364 | +上述配置文件描述了一个包含4个超节点，32个Server，共128个NPU卡的集群拓扑。其中，每个Server/Pod采用ascend950_server_topo_normal拓扑类型。 | ||
| 365 | + | ||
| 366 | +**字段说明**： | ||
| 367 | + | ||
| 368 | + - **super_node_num**: 超节点总数量。 | ||
| 369 | + - **server_num**: sever/pod总数量。 | ||
| 370 | + - **server_list**: 每个Server/Pod的配置信息，包括超节点ID、设备ID范围、芯片型号、Server/Pod拓扑配置文件路径。 | ||
| 371 | + | ||
| 372 | +#### 4.3.3 通信域配置文件说明 | ||
| 373 | + | ||
| 374 | +用户在昇腾集群环境中，需要根据待执行算子所需的通信域不同，选择不同的通信域配置文件。 | ||
| 375 | + | ||
| 376 | +工具提供了hccl-vm mock-comm命令读取和配置算子通信域配置文件。通信域配置文件格式为yaml，路径为hccl_vm_install/cluster_model/topo_meta。若目录中没有对应的通信域配置文件，则用户需要先创建一个。 | ||
| 377 | + | ||
| 378 | +hccl-vm工具支持非对称拓扑通信域配置。如下所示： | ||
| 379 | + | ||
| 380 | +```yaml | ||
| 381 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 382 | +meta: | ||
| 383 | + podNum: 1 # 总的超节点数 | ||
| 384 | + serNum: 2 # 总的server数 | ||
| 385 | + rankNum: 6 # 总的rank数 | ||
| 386 | + | ||
| 387 | +# 2. 详细拓扑结构 | ||
| 388 | +topology: | ||
| 389 | + - podId: 0 | ||
| 390 | + servers: | ||
| 391 | + - serId: 0 | ||
| 392 | + # 每个server实际跑的rank的local id | ||
| 393 | + ranks: [0, 2] | ||
| 394 | + - serId: 1 | ||
| 395 | + # 每个server实际跑的rank的local id | ||
| 396 | + ranks: [1, 3, 5, 7] | ||
| 397 | +``` | ||
| 398 | + | ||
| 399 | +**注意事项**： | ||
| 400 | + | ||
| 401 | +- 配置通信域时，工具会根据指定的通信域配置编号重新生成topo.json和ranktable.json文件。 | ||
| 402 | +- 上述通信域配置yaml文件中，ranks字段表示每个server实际跑的rank的local id（即device的物理ID）列表。 | ||
| 403 | + | ||
| 404 | +#### 4.3.4 topo和ranktable.json文件说明 | ||
| 405 | + | ||
| 406 | +`topo.json` 和 `ranktable.json` 文件不需要手动创建，工具会根据以下信息自动生成： | ||
| 407 | + | ||
| 408 | +- **拓扑配置编号**：用户在启动时指定的编号（如 112、113 等） | ||
| 409 | +- **芯片类型**：根据运行环境自动识别的芯片类型 | ||
| 410 | + | ||
| 411 | +虽然配置文件由工具自动生成，但了解其结构有助于理解拓扑配置。 | ||
| 412 | + | ||
| 413 | +**topo.json 结构**： | ||
| 414 | + | ||
| 415 | +`topo.json` 描述了一个 server 内所有设备的连接关系： | ||
| 416 | + | ||
| 417 | +```json | ||
| 418 | +{ | ||
| 419 | + "server": { | ||
| 420 | + "device_count": 8, | ||
| 421 | + "groups": [ | ||
| 422 | + { | ||
| 423 | + "group_id": 0, | ||
| 424 | + "device_start": 0, | ||
| 425 | + "device_count": 8, | ||
| 426 | + "topo_layout": "1D" | ||
| 427 | + } | ||
| 428 | + ] | ||
| 429 | + }, | ||
| 430 | + "ports": [ | ||
| 431 | + { | ||
| 432 | + "ccu": "die0", | ||
| 433 | + "port_pattern": "0/{0-6}", | ||
| 434 | + "protocol": "HCCS", | ||
| 435 | + "func_id": 2, | ||
| 436 | + "usage": "peer2peer", | ||
| 437 | + "ip_binding": "independent" | ||
| 438 | + }, | ||
| 439 | + { | ||
| 440 | + "ccu": "die0", | ||
| 441 | + "port_pattern": "0/7,0/8", | ||
| 442 | + "protocol": "ROCE", | ||
| 443 | + "func_id": 3, | ||
| 444 | + "usage": "peer2net", | ||
| 445 | + "ip_binding": "independent" | ||
| 446 | + } | ||
| 447 | + ], | ||
| 448 | + "links": [ | ||
| 449 | + { | ||
| 450 | + "net_layer": 0, | ||
| 451 | + "link_type": "PEER2PEER", | ||
| 452 | + "topo_type": "1DMESH", | ||
| 453 | + "ccu": "die0", | ||
| 454 | + "port_pattern": "0/{0-6}", | ||
| 455 | + "connect_pattern": "full_mesh", | ||
| 456 | + "group_id": 0 | ||
| 457 | + }, | ||
| 458 | + { | ||
| 459 | + "net_layer": 1, | ||
| 460 | + "link_type": "PEER2NET", | ||
| 461 | + "topo_type": "CLOS", | ||
| 462 | + "ccu": "die0", | ||
| 463 | + "port_pattern": "0/7,0/8", | ||
| 464 | + "connect_pattern": "all_to_net", | ||
| 465 | + "group_id": 0 | ||
| 466 | + } | ||
| 467 | + ] | ||
| 468 | +} | ||
| 469 | +``` | ||
| 470 | + | ||
| 471 | +**字段说明**： | ||
| 472 | + | ||
| 473 | +- `server.device_count`：设备总数 | ||
| 474 | +- `server.groups`：设备分组信息 | ||
| 475 | +- `ports`：端口配置 | ||
| 476 | + - `usage`：端口用途（`peer2peer` 表示设备间连接，`peer2net` 表示与外部连接） | ||
| 477 | +- `links`：链路配置 | ||
| 478 | + - `link_type`：链路类型（`PEER2PEER` 或 `PEER2NET`） | ||
| 479 | + - `topo_type`：拓扑类型（`1DMESH`、`CLOS` 等） | ||
| 480 | + | ||
| 481 | +**ranktable.json 结构**： | ||
| 482 | + | ||
| 483 | +`ranktable.json` 描述了本次运行使用的设备和 IP 映射： | ||
| 484 | + | ||
| 485 | +```json | ||
| 486 | +{ | ||
| 487 | + "version": "1.0", | ||
| 488 | + "server_count": 1, | ||
| 489 | + "device_count": 8, | ||
| 490 | + "server_list": [ | ||
| 491 | + { | ||
| 492 | + "server_id": 0, | ||
| 493 | + "device_id": 0, | ||
| 494 | + "device_ip": "192.168.1.10", | ||
| 495 | + "port": "2222" | ||
| 496 | + } | ||
| 497 | + ] | ||
| 498 | +} | ||
| 499 | +``` | ||
| 500 | + | ||
| 501 | +**字段说明**： | ||
| 502 | + | ||
| 503 | +- `server_count`：服务器数量 | ||
| 504 | +- `device_count`：设备总数 | ||
| 505 | +- `server_list`：服务器和设备列表 | ||
| 506 | + - `device_ip`：设备 IP 地址 | ||
| 507 | + - `port`：设备端口号 | ||
| 508 | + | ||
| 509 | +### 4.4 hccl\_config.sh文件说明 | ||
| 510 | + | ||
| 511 | +hccl\_config.sh文件中包含了HCCL\_Test用例运行所需的环境变量配置。其中的环境变量与Hccl\_Test用例在真机环境中的环境变量一致。 | ||
| 512 | +用户需要根据自己的用例和需求，修改hccl\_config.sh脚本，配置HCCL用例运行环境变量。 | ||
| 513 | + | ||
| 514 | +```bash | ||
| 515 | +#!/bin/bash | ||
| 516 | +# hccl_config.sh - HCCL 环境变量配置 | ||
| 517 | + | ||
| 518 | +remove_files_by_prefix() { | ||
| 519 | + if [ "$#" -ne 1 ]; then | ||
| 520 | + echo "Usage: remove_files_by_prefix <prefix>" >&2 | ||
| 521 | + return 2 | ||
| 522 | + fi | ||
| 523 | + | ||
| 524 | + local prefix="$1" | ||
| 525 | + if [ -z "$prefix" ]; then | ||
| 526 | + return 0 | ||
| 527 | + fi | ||
| 528 | + | ||
| 529 | + shopt -s nullglob | ||
| 530 | + local any_deleted=0 | ||
| 531 | + for f in "${prefix}"*; do | ||
| 532 | + if [ -f "$f" ]; then | ||
| 533 | + rm -f -- "$f" && any_deleted=1 | ||
| 534 | + fi | ||
| 535 | + done | ||
| 536 | + shopt -u nullglob | ||
| 537 | + | ||
| 538 | + # 无论是否删除了文件，均返回 0，确保脚本继续执行 | ||
| 539 | + return 0 | ||
| 540 | +} | ||
| 541 | + | ||
| 542 | +# 清理执行目录的冗余文件 | ||
| 543 | +remove_files_by_prefix "sqe_info_rank_" | ||
| 544 | +remove_files_by_prefix "mc_instr_info_rank_" | ||
| 545 | +rm -f "all_rank_input_output.txt" | ||
| 546 | + | ||
| 547 | +# 设置CANN环境变量 | ||
| 548 | +source /home/Ascend/cann/set_env.sh | ||
| 549 | + | ||
| 550 | +# 配置LD_LIBRARY_PATH | ||
| 551 | +export LD_LIBRARY_PATH=$ASCEND_HOME_PATH/lib64:$ASCEND_HOME_PATH/devlib:$LD_LIBRARY_PATH | ||
| 552 | + | ||
| 553 | +# 设置 ranktable.json文件路径 | ||
| 554 | +export RANK_TABLE_FILE=$(pwd)/ranktable.json | ||
| 555 | + | ||
| 556 | +# 设置日志级别 | ||
| 557 | +export ASCEND_GLOBAL_LOG_LEVEL=1 | ||
| 558 | + | ||
| 559 | +# 设置日志打屏输出 | ||
| 560 | +export ASCEND_SLOG_PRINT_TO_STDOUT=1 | ||
| 561 | + | ||
| 562 | +# 设置 HCCL 运行模式（CCU、AI_CPU、AIV等展开模式） | ||
| 563 | +export HCCL_OP_EXPANSION_MODE="CCU_SCHED" | ||
| 564 | +# 或设置 HCCL 运行参数(AI_CPU展开模式) AI_CPU模式环境变量与其他模式不可同时设置 | ||
| 565 | +# export HCCL_OP_EXPANSION_MODE="AI_CPU" | ||
| 566 | +# 或设置 HCCL 运行参数(AIV展开模式) AIV模式环境变量与其他模式不可同时设置 | ||
| 567 | +# export HCCL_OP_EXPANSION_MODE="AIV" | ||
| 568 | + | ||
| 569 | +echo "HCCL-VM environment configured successfully!" | ||
| 570 | + | ||
| 571 | +``` | ||
| 572 | + | ||
| 573 | +### 4.5 工具规格约束 | ||
| 574 | + | ||
| 575 | +**支持算子类型**： | ||
| 576 | + | ||
| 577 | +工具支持算子类型包含：allgather/allreduce/alltoall/reduce/reduce\_scatter/scatter/alltoallv. | ||
| 578 | + | ||
| 579 | +**支持数据类型**： | ||
| 580 | + | ||
| 581 | +HCCL-VM工具支持数据类型包含：int8/int16/int32/fp16/fp32/uint8/uint16/uint32/bfp16/hif8/fp8e4m3/fp8e5m2/fp8e8m0. | ||
| 582 | + | ||
| 583 | +HCCL-VM Runner插件支持数据类如下： | ||
| 584 | + | ||
| 585 | +| ReduceOp | DataType | | ||
| 586 | +| -------- | ---------------------------------- | | ||
| 587 | +| `ADD` | `int8/int16/int32/uint8` | | ||
| 588 | +| `MIN` | `int8/int16/int32/uint8` | | ||
| 589 | +| `MAX` | `int8/int16/int32/uint8` | | ||
| 590 | + | ||
| 591 | +**硬件规格**： | ||
| 592 | + | ||
| 593 | +目前本工具仅适配支持Ascend950芯片。单server内最大支持8张卡；超过8张卡，需要跨server运行。 | ||
| 594 | + | ||
| 595 | +### 4.6 HCCL-VM插件功能 | ||
| 596 | + | ||
| 597 | +#### 4.6.1 Runner插件 | ||
| 598 | + | ||
| 599 | +Runner插件，即模拟执行Hccl业务编排生成的任务，输出output数据。 | ||
| 600 | +模拟运行器插件在hccl\_test用例执行过程中默认**关闭**。hccl\_test用例调用算子接口后，通过aclrtSynchronizeStream接口等待算子任务执行完成。模拟运行器工具会等待所有rank都处于等待状态时，开启模拟执行所有rank的task。执行完成后，通知各个rank的用例进行继续执行。 | ||
| 601 | +用例执行完成后，用户可以通过执行目录下的"all\_rank\_input\_output.txt"文件，查看每个rank的input buffer和output buffer数据。此功能默认不开启，用户可以在用例执行前通过对应命令开启。 | ||
| 602 | + | ||
| 603 | +**安装与卸载**： | ||
| 604 | + | ||
| 605 | +Runner插件支持通过 `hccl-vm plugin install/uninstall` 命令进行安装和卸载。需在进入hccl-vm工具命令行后、执行算例前安装runner插件，后续的执行才会运行runner。 | ||
| 606 | + | ||
| 607 | +```bash | ||
| 608 | +# 安装runner插件 | ||
| 609 | +(hvm)$> hccl-vm plugin install @runner | ||
| 610 | + | ||
| 611 | +# 卸载runner插件 | ||
| 612 | +(hvm)$> hccl-vm plugin uninstall @runner | ||
| 613 | +``` | ||
| 614 | + | ||
| 615 | +#### 4.6.2 Checker插件 | ||
| 616 | + | ||
| 617 | +Checker插件，即算法分析器插件：功能是将hccl生成的所有task形成一个DAG图，并且通过分析DAG图，判断是否存在内存冲突；通过模拟执行DAG图，判断是否存在语义错误等问题。 | ||
| 618 | +算法分析器插件，是由用户自行通过命令启动执行。 | ||
| 619 | + | ||
| 620 | +Checker插件正处于新旧交替阶段，Checker V3为原Checker的重构版，主要提高了校验性能，在默认情况下将会运行新Checker（Checker V3），可以通过修改Checker的`manifest.json`文件中的配置参数进行调整。 | ||
| 621 | + | ||
| 622 | +```bash | ||
| 623 | +# 配置文件位于 /pathto/hccl_vm_install/plugin/validate/checker/manifest.json | ||
| 624 | + | ||
| 625 | +{ | ||
| 626 | + "name": "checker", // Checker插件名 | ||
| 627 | + "version": "1.0.0", // Checker插件版本 | ||
| 628 | + "entry": "./checker", // Checker插件启动指令 | ||
| 629 | + "dependency": { | ||
| 630 | + "min_core_version": "1.0.0" | ||
| 631 | + }, | ||
| 632 | + "setting": { // Checker插件配置项 | ||
| 633 | + "enable_new_checker": true, // 是否启用新Checker（Checker V3，默认开启） | ||
| 634 | + "enable_old_checker": false, // 是否启用老Checker（默认关闭） | ||
| 635 | + "enable_insight_dump": false, // 是否启用可视化数据输出（默认关闭，仅支持老Checker） | ||
| 636 | + "enable_memory_snapshot_dump": false // 是否启用可视化内存快照数据输出（默认关闭，仅支持老Checker，需要先开启可视化数据输出"enable_insight_dump"） | ||
| 637 | + } | ||
| 638 | +} | ||
| 639 | +``` | ||
| 640 | + | ||
| 641 | +### 4.7 hccl_rootinfo.json文件 | ||
| 642 | + | ||
| 643 | +目前工具初始化通信域使用的是ranktable.json文件，因此hccl_rootinfo.json文件的作用仅限于提供topo.json文件的路径。 | ||
| 644 | +若/etc路径下没有hccl_rootinfo.json文件，则用户需自行创建该文件，内容如下： | ||
| 645 | + | ||
| 646 | +```json | ||
| 647 | +{ | ||
| 648 | + "version": "2.0", | ||
| 649 | + "topo_file_path": "/home/myuser/workspace/CheckerL2/hccl_vm_install/topo.json" | ||
| 650 | +} | ||
| 651 | +``` | ||
| 652 | + | ||
| 653 | +### 4.8 OpenMPI和MPICH环境用例运行差异 | ||
| 654 | + | ||
| 655 | +运行hccl_test用例前，用户可以通过which命令判断当前环境使用的是哪个mpirun。 | ||
| 656 | + | ||
| 657 | +#### 4.8.1 环境变量配置差异 | ||
| 658 | + | ||
| 659 | +环境中一般默认配置的是OpenMPI，若用户使用OpenMPI运行用例，一般不需要额外配置环境变量。 | ||
| 660 | +若用户使用MPICH环境运行用例，需要按照如下方式配置环境变量： | ||
| 661 | + | ||
| 662 | +```bash | ||
| 663 | +# 配置mpich环境变量 | ||
| 664 | +export LD_LIBRARY_PATH=/usr/lib/mpich/lib/:${ASCEND_HOME_PATH}/lib64/:${ASCEND_HOME_PATH}/x86_64-linux/devlib:$LD_LIBRARY_PATH | ||
| 665 | +export PATH=/usr/lib/mpich/bin:$PATH | ||
| 666 | +``` | ||
| 667 | + | ||
| 668 | +#### 4.8.2 mpirun命令参数差异 | ||
| 669 | + | ||
| 670 | +OpenMPI环境，用户按照如下命令运行hccl_test用例： | ||
| 671 | + | ||
| 672 | +```bash | ||
| 673 | +export HCCL_TEST_PATH=/home/myuser/workspace/Ascend/cann/tools/hccl_test | ||
| 674 | +mpirun --allow-run-as-root --oversubscribe -np 2 ${HCCL_TEST_PATH}/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 | ||
| 675 | +``` | ||
| 676 | + | ||
| 677 | +**参数说明**： | ||
| 678 | + | ||
| 679 | + - --allow-run-as-root：OpenMPI专属参数，允许以root用户运行MPI进程，用于在没有root权限的环境中运行。 | ||
| 680 | + - --oversubscribe：OpenMPI专属参数，放开CPU槽位(slot)限制，允许单节点启动[进程数>CPU逻辑核数]，也就是超额超配跑进程。 | ||
| 681 | + - -np 2：指定进程数为2，与节点数一致。 | ||
| 682 | + | ||
| 683 | +MPICH环境，用户按照如下命令运行hccl_test用例： | ||
| 684 | + | ||
| 685 | +```bash | ||
| 686 | +export HCCL_TEST_PATH=/home/myuser/workspace/Ascend/cann/tools/hccl_test | ||
| 687 | +mpirun -np 2 ${HCCL_TEST_PATH}/bin/reduce_scatter_test -b 64 -e 64 -d int32 -o sum -w 0 -n 1 -c 1 | ||
| 688 | +``` | ||
| 689 | + | ||
| 690 | +**参数说明**： | ||
| 691 | + | ||
| 692 | + - -np 2：指定进程数为2，与节点数一致。 | ||
| 693 | + | ||
| 694 | +### 4.9 结果查看 | ||
| 695 | + | ||
| 696 | +#### 4.9.1 Runner插件结果 | ||
| 697 | + | ||
| 698 | +如果在hccl-vm终端执行hccl-vm plugin install @runner安装插件后，算子流程执行完成后会自动触发runner插件的执行，最终结果依赖hccl_test进行校验，用户需关注在重定向的日志文件中是否存在[error]级别日志和最终校验结果： | ||
| 699 | + | ||
| 700 | +```bash | ||
| 701 | +data_size(Bytes): | aveg_time(us): | alg_bandwidth(GB/s): | check_result: | ||
| 702 | +64 | 1000.00 | 0.00006 | success | ||
| 703 | +``` | ||
| 704 | + | ||
| 705 | +#### 4.9.2 Checker插件结果 | ||
| 706 | + | ||
| 707 | +在hccl-vm终端内执行hccl-vm plugin run @checker后，Checker校验流程及结果会打印在终端内，用于需关注是否存在[error]级别日志和最终校验结果： | ||
| 708 | + | ||
| 709 | +```bash | ||
| 710 | +[info][PID:144373][TID:144880][main.cc][RunChecker] [RunChecker] op[0] Checker Success. | ||
| 711 | +``` | ||
| 712 | + | ||
| 713 | +--- | ||
| 714 | + | ||
| 715 | +## 5 附录 | ||
| 716 | + | ||
| 717 | +### 开源第三方软件依赖 | ||
| 718 | + | ||
| 719 | +编译本项目时，依赖的第三方开源软件列表如下，离线编译场景可下载并重命名软件包后放置在本项目内的third_party目录下 | ||
| 720 | + | ||
| 721 | +| 开源软件 | 版本 |下载地址 | | ||
| 722 | +| ------------ | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | | ||
| 723 | +| CLI11 | 2.2.0 | [cli11-2.2.0.tar.gz](https://raw.gitcode.com/src-openeuler/cli11/blobs/58c912141164a5c0f0139bfa91343fefe151d525/cli11-2.2.0.tar.gz) | | ||
| 724 | +| json | 3.11.3 | [include.zip](https://gitcode.com/cann-src-third-party/json/releases/download/v3.11.3/include.zip) | | ||
| 725 | +| spdlog | 1.11.0 | [spdlog-v1.11.0.tar.gz](https://raw.gitcode.com/src-openeuler/spdlog/blobs/c2dfb1aca26c607393665c836155613ff283de66/v1.11.0.tar.gz) | | ||
| 726 | +| yaml-cpp | 0.8.0 | [yaml-cpp-0.8.0.tar.gz](https://raw.gitcode.com/src-openeuler/yaml-cpp/blobs/d1ead4fff417073b9cdbf98b8b55eb0efc00b0ba/yaml-cpp-0.8.0.tar.gz) | | ||
| 727 | +| sqlite | 3.51.0 | [sqlite-amalgamation-3510300.zip](https://www.sqlite.org/2026/sqlite-amalgamation-3510300.zip) | | ||
| 728 | +| googletest | 1.14.0 | [googletest-1.14.0.tar.gz](https://gitcode.com/cann-src-third-party/googletest/releases/download/v1.14.0/googletest-1.14.0.tar.gz) | | ||
| 729 | + | ||
| 730 | +### 术语表 | ||
| 731 | + | ||
| 732 | +| 术语 | 说明 | | ||
| 733 | +| -------- | ------------------------------------------------------- | | ||
| 734 | +| HCCL | Huawei Collective Communication Library，华为集合通信库 | | ||
| 735 | +| NPU | Neural Processing Unit，神经网络处理器 | | ||
| 736 | +| CANN | Compute Architecture for Neural Networks，华为昇腾 AI 处理器软件栈 | | ||
| 737 | +| MPI | Message Passing Interface，消息传递接口 | | ||
| 738 | +| CCU | Collective Communication Unit，集合通信单元 | | ||
| 739 | +| Topology | 拓扑，设备连接关系 | | ||
| 740 | +| Rank | 进程编号，在 MPI 中的标识 | | ||
| 741 | + | ||
| 742 | +--- | ||
| 743 | + | ||
| 744 | +**文档版本**：v1.0 | ||
| 745 | +**最后更新**：2026-06-22 | ||
| @@ -0,0 +1,252 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + | ||
| 10 | +# 统一分配EID/IP地址 | ||
| 11 | +# 功能： | ||
| 12 | +# 1. 遍历output_temp目录下所有rootinfo相关json文件 | ||
| 13 | +# 2. 根据IP地址分配规则，为每个server的portGroup分配IP地址 | ||
| 14 | +# 3. 将IP地址转换为EID字符串，修改rootinfo文件中的addr字段 | ||
| 15 | +# | ||
| 16 | +# IP地址分配规则 (AA.BB.CC.DD): | ||
| 17 | +# AA = 192 + superpod_index (超节点段，从192开始) | ||
| 18 | +# BB = server_index + 1 (服务器段，从1开始，排除192.0.0.0~192.0.0.24段) | ||
| 19 | +# CC:DD = device port计数器 (设备段，从0.1开始) | ||
| 20 | +# Host IP: AA.BB.0.0 (每个server固定一个host IP，仅供host使用) | ||
| 21 | +# Device port IPs: AA.BB.0.1, AA.BB.0.2, ... (依次递增，d2h端口也从此段分配) | ||
| 22 | +# | ||
| 23 | +# IP→EID转换规则 (等价于C++ IpToEidString): | ||
| 24 | +# IPv4地址(4字节)放入16字节EID的末尾(网络字节序)，前12字节为0 | ||
| 25 | +# 输出格式: "0x" + 32位十六进制字符串 | ||
| 26 | +# 示例: 192.0.0.1 → 0x000000000000000000000000c0000001 | ||
| 27 | + | ||
| 28 | +set -e | ||
| 29 | + | ||
| 30 | +RED='\033[0;31m' | ||
| 31 | +GREEN='\033[0;32m' | ||
| 32 | +YELLOW='\033[1;33m' | ||
| 33 | +BLUE='\033[0;34m' | ||
| 34 | +NC='\033[0m' | ||
| 35 | + | ||
| 36 | +error_exit() { | ||
| 37 | + echo -e "${RED}[错误] $1${NC}" >&2 | ||
| 38 | + exit 1 | ||
| 39 | +} | ||
| 40 | + | ||
| 41 | +success_msg() { | ||
| 42 | + echo -e "${GREEN}[成功] $1${NC}" | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +info_msg() { | ||
| 46 | + echo -e "${BLUE}[信息] $1${NC}" | ||
| 47 | +} | ||
| 48 | + | ||
| 49 | +warn_msg() { | ||
| 50 | + echo -e "${YELLOW}[警告] $1${NC}" | ||
| 51 | +} | ||
| 52 | + | ||
| 53 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | ||
| 54 | +BASE_DIR="${SCRIPT_DIR}" | ||
| 55 | +OUTPUT_DIR="${BASE_DIR}/cluster_model/network/cluster" | ||
| 56 | + | ||
| 57 | +show_usage() { | ||
| 58 | + cat << EOF | ||
| 59 | +用法: $(basename "$0") [OPTIONS] | ||
| 60 | + | ||
| 61 | +选项: | ||
| 62 | + -d, --dir DIR 指定集群输出目录 (默认: ${OUTPUT_DIR}) | ||
| 63 | + -h, --help 显示帮助信息 | ||
| 64 | + | ||
| 65 | +说明: | ||
| 66 | + 遍历output_temp目录下所有rootinfo JSON文件，按照IP地址分配规则 | ||
| 67 | + 统一分配IP地址，转换为EID字符串后写入addr字段。 | ||
| 68 | + | ||
| 69 | + IP地址分配规则 (AA.BB.CC.DD): | ||
| 70 | + AA = 192 + superpod_index (超节点段) | ||
| 71 | + BB = server_index + 1 (服务器段，从1开始，排除192.0.0.0~192.0.0.24段) | ||
| 72 | + CC:DD = device port计数器 (设备段，从0.1开始) | ||
| 73 | + Host IP: AA.BB.0.0 (每个server固定一个host IP，仅供host使用) | ||
| 74 | + Device port IPs: AA.BB.0.1, AA.BB.0.2, ... (依次递增，d2h端口也从此段分配) | ||
| 75 | + | ||
| 76 | + IP→EID转换: | ||
| 77 | + IPv4地址放入16字节EID末尾(网络字节序)，前12字节为0 | ||
| 78 | + 示例: 192.0.0.1 → 0x000000000000000000000000c0000001 | ||
| 79 | + | ||
| 80 | +示例: | ||
| 81 | + $(basename "$0") | ||
| 82 | + $(basename "$0") -d /tmp/my_output | ||
| 83 | + | ||
| 84 | +EOF | ||
| 85 | +} | ||
| 86 | + | ||
| 87 | +while [[ $# -gt 0 ]]; do | ||
| 88 | + case $1 in | ||
| 89 | + -h|--help) | ||
| 90 | + show_usage | ||
| 91 | + exit 0 | ||
| 92 | + ;; | ||
| 93 | + -d|--dir) | ||
| 94 | + OUTPUT_DIR="$2" | ||
| 95 | + shift 2 | ||
| 96 | + ;; | ||
| 97 | + *) | ||
| 98 | + error_exit "未知参数: $1\n$(show_usage)" | ||
| 99 | + ;; | ||
| 100 | + esac | ||
| 101 | +done | ||
| 102 | + | ||
| 103 | +if [[ ! -d "$OUTPUT_DIR" ]]; then | ||
| 104 | + error_exit "输出目录不存在: $OUTPUT_DIR\n请先运行 generate_cluster_topo.sh 生成集群拓扑目录" | ||
| 105 | +fi | ||
| 106 | + | ||
| 107 | +echo "==========================================" | ||
| 108 | +echo " 昇腾集群EID统一分配工具" | ||
| 109 | +echo "==========================================" | ||
| 110 | +echo "" | ||
| 111 | +echo "输出目录: $OUTPUT_DIR" | ||
| 112 | +echo "" | ||
| 113 | + | ||
| 114 | +export OUTPUT_DIR | ||
| 115 | + | ||
| 116 | +python3 << 'PYTHON_SCRIPT' | ||
| 117 | +import json | ||
| 118 | +import os | ||
| 119 | +import re | ||
| 120 | +import sys | ||
| 121 | +import struct | ||
| 122 | +import socket | ||
| 123 | + | ||
| 124 | +OUTPUT_DIR = os.environ.get('OUTPUT_DIR', '') | ||
| 125 | +if not OUTPUT_DIR: | ||
| 126 | + print("[错误] OUTPUT_DIR 环境变量未设置") | ||
| 127 | + sys.exit(1) | ||
| 128 | + | ||
| 129 | +URMA_EID_LEN = 16 | ||
| 130 | + | ||
| 131 | +def ip_to_eid_string(ip, family=socket.AF_INET): | ||
| 132 | + """ | ||
| 133 | + 将IP地址转换为EID字符串。 | ||
| 134 | + 等价于C++函数: | ||
| 135 | + std::string IpToEidString(const std::string& ip, int family = AF_INET) | ||
| 136 | + | ||
| 137 | + 转换逻辑: | ||
| 138 | + 1. 将IPv4地址解析为4字节(网络字节序) | ||
| 139 | + 2. 构造16字节EID: 前12字节为0，后4字节为IPv4地址 | ||
| 140 | + 3. 输出 "0x" + 32位十六进制字符串 | ||
| 141 | + | ||
| 142 | + 示例: | ||
| 143 | + ip_to_eid_string("192.0.0.1") -> "0x000000000000000000000000c0000001" | ||
| 144 | + ip_to_eid_string("192.0.0.16") -> "0x000000000000000000000000c0000010" | ||
| 145 | + ip_to_eid_string("193.3.0.5") -> "0x000000000000000000000000c1030005" | ||
| 146 | + """ | ||
| 147 | + if family == socket.AF_INET: | ||
| 148 | + ip_bytes = socket.inet_pton(socket.AF_INET, ip) | ||
| 149 | + eid = bytearray(URMA_EID_LEN) | ||
| 150 | + eid[12:16] = ip_bytes | ||
| 151 | + elif family == socket.AF_INET6: | ||
| 152 | + ip_bytes = socket.inet_pton(socket.AF_INET6, ip) | ||
| 153 | + eid = bytearray(URMA_EID_LEN) | ||
| 154 | + eid[0:16] = ip_bytes | ||
| 155 | + else: | ||
| 156 | + raise ValueError(f"不支持的地址族: {family}") | ||
| 157 | + | ||
| 158 | + hex_str = ''.join(f'{b:02x}' for b in eid) | ||
| 159 | + return f'0x{hex_str}' | ||
| 160 | + | ||
| 161 | +def find_rootinfo_files(output_dir): | ||
| 162 | + rootinfo_files = [] | ||
| 163 | + for root, dirs, files in os.walk(output_dir): | ||
| 164 | + for f in files: | ||
| 165 | + if 'rootinfo' in f and f.endswith('.json'): | ||
| 166 | + rootinfo_files.append(os.path.join(root, f)) | ||
| 167 | + return rootinfo_files | ||
| 168 | + | ||
| 169 | +def extract_indices(filepath): | ||
| 170 | + match = re.search(r'superpod(\d+)/server(\d+)', filepath) | ||
| 171 | + if match: | ||
| 172 | + return (int(match.group(1)), int(match.group(2))) | ||
| 173 | + return (-1, -1) | ||
| 174 | + | ||
| 175 | +def allocate_ip(aa, bb, counter): | ||
| 176 | + cc = (counter >> 8) & 0xFF | ||
| 177 | + dd = counter & 0xFF | ||
| 178 | + return f"{aa}.{bb}.{cc}.{dd}" | ||
| 179 | + | ||
| 180 | +rootinfo_files = find_rootinfo_files(OUTPUT_DIR) | ||
| 181 | + | ||
| 182 | +if not rootinfo_files: | ||
| 183 | + print("[警告] 未找到任何rootinfo JSON文件") | ||
| 184 | + sys.exit(0) | ||
| 185 | + | ||
| 186 | +rootinfo_files.sort(key=extract_indices) | ||
| 187 | + | ||
| 188 | +print(f"[信息] 找到 {len(rootinfo_files)} 个rootinfo文件") | ||
| 189 | +print(f"[信息] IP→EID转换规则: IPv4地址放入16字节EID末尾(网络字节序)") | ||
| 190 | +print("") | ||
| 191 | + | ||
| 192 | +total_addrs = 0 | ||
| 193 | + | ||
| 194 | +for filepath in rootinfo_files: | ||
| 195 | + sp_idx, srv_idx = extract_indices(filepath) | ||
| 196 | + | ||
| 197 | + if sp_idx < 0 or srv_idx < 0: | ||
| 198 | + print(f" [警告] 无法从路径提取索引，跳过: {filepath}") | ||
| 199 | + continue | ||
| 200 | + | ||
| 201 | + aa = 192 + sp_idx | ||
| 202 | + bb = srv_idx + 1 | ||
| 203 | + | ||
| 204 | + host_ip = f"{aa}.{bb}.0.0" | ||
| 205 | + host_eid = ip_to_eid_string(host_ip) | ||
| 206 | + | ||
| 207 | + with open(filepath, 'r') as f: | ||
| 208 | + data = json.load(f) | ||
| 209 | + | ||
| 210 | + DEVICE_PORT_START = 1 | ||
| 211 | + port_counter = DEVICE_PORT_START | ||
| 212 | + addr_count = 0 | ||
| 213 | + first_device_eid = None | ||
| 214 | + last_device_eid = None | ||
| 215 | + | ||
| 216 | + for rank in data.get('rank_list', []): | ||
| 217 | + for level in rank.get('level_list', []): | ||
| 218 | + for addr_entry in level.get('rank_addr_list', []): | ||
| 219 | + ip_addr = allocate_ip(aa, bb, port_counter) | ||
| 220 | + eid_str = ip_to_eid_string(ip_addr) | ||
| 221 | + addr_entry['addr'] = eid_str | ||
| 222 | + if first_device_eid is None: | ||
| 223 | + first_device_eid = eid_str | ||
| 224 | + last_device_eid = eid_str | ||
| 225 | + port_counter += 1 | ||
| 226 | + addr_count += 1 | ||
| 227 | + | ||
| 228 | + with open(filepath, 'w') as f: | ||
| 229 | + json.dump(data, f, indent=4, ensure_ascii=False) | ||
| 230 | + | ||
| 231 | + print(f" [成功] superpod{sp_idx}/server{srv_idx}: " | ||
| 232 | + f"分配 {addr_count} 个EID") | ||
| 233 | + print(f" Host IP: {host_ip} → EID: {host_eid}") | ||
| 234 | + if first_device_eid: | ||
| 235 | + last_cc = ((port_counter - 1) >> 8) & 0xFF | ||
| 236 | + last_dd = (port_counter - 1) & 0xFF | ||
| 237 | + print(f" Device IP: {aa}.{bb}.0.1 ~ {aa}.{bb}.{last_cc}.{last_dd}") | ||
| 238 | + print(f" Device EID: {first_device_eid} ~ {last_device_eid}") | ||
| 239 | + | ||
| 240 | + total_addrs += addr_count | ||
| 241 | + | ||
| 242 | +print("") | ||
| 243 | +print(f"[信息] 共处理 {len(rootinfo_files)} 个文件，分配 {total_addrs} 个EID") | ||
| 244 | + | ||
| 245 | +PYTHON_SCRIPT | ||
| 246 | + | ||
| 247 | +echo "" | ||
| 248 | +echo "==========================================" | ||
| 249 | +echo " 分配完成" | ||
| 250 | +echo "==========================================" | ||
| 251 | +echo "" | ||
| 252 | +success_msg "EID统一分配完成！" | ||
| @@ -0,0 +1,13 @@ | |||
| 1 | +name: "ascend950_cluster_32_server_normal" | ||
| 2 | +description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | ||
| 3 | + | ||
| 4 | +# 超节点总数量 | ||
| 5 | +super_node_num: 4 | ||
| 6 | +# sever/pod总数量 | ||
| 7 | +server_num: 128 | ||
| 8 | +server_list: | ||
| 9 | + # 0-31 server: 均采用ascend950_server_topo_normal拓扑类型 | ||
| 10 | + - {super_pod_id: 0, id_range: [0, 31], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 11 | + - {super_pod_id: 1, id_range: [0, 31], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 12 | + - {super_pod_id: 2, id_range: [0, 31], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 13 | + - {super_pod_id: 3, id_range: [0, 31], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| @@ -0,0 +1,11 @@ | |||
| 1 | +name: "ascend950_cluster_32_server_normal" | ||
| 2 | +description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | ||
| 3 | + | ||
| 4 | +# 超节点总数量 | ||
| 5 | +super_node_num: 2 | ||
| 6 | +# sever/pod总数量 | ||
| 7 | +server_num: 16 | ||
| 8 | +server_list: | ||
| 9 | + # 0-7 server: 均采用ascend950_server_topo_normal拓扑类型 | ||
| 10 | + - {super_pod_id: 0, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 11 | + - {super_pod_id: 1, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| @@ -0,0 +1,13 @@ | |||
| 1 | +name: "ascend950_cluster_32_server_normal" | ||
| 2 | +description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | ||
| 3 | + | ||
| 4 | +# 超节点总数量 | ||
| 5 | +super_node_num: 4 | ||
| 6 | +# sever/pod总数量 | ||
| 7 | +server_num: 32 | ||
| 8 | +server_list: | ||
| 9 | + # 0-7 server: 均采用ascend950_server_topo_normal拓扑类型 | ||
| 10 | + - {super_pod_id: 0, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 11 | + - {super_pod_id: 1, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 12 | + - {super_pod_id: 2, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 13 | + - {super_pod_id: 3, id_range: [0, 7], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| @@ -0,0 +1,11 @@ | |||
| 1 | +name: "ascend950_cluster_4_server_1dpod64" | ||
| 2 | +description: "昇腾950 1DPod64集群组网：4个超级节点，4个服务器每个超级节点1个服务器" | ||
| 3 | + | ||
| 4 | +# 超节点总数量 | ||
| 5 | +super_node_num: 2 | ||
| 6 | +# sever/pod总数量 | ||
| 7 | +server_num: 4 | ||
| 8 | +server_list: | ||
| 9 | + # 0-1 server: 均采用ascend950_server_topo_1dpod64拓扑类型 | ||
| 10 | + - {super_pod_id: 0, id_range: [0, 1], soc_version: "Ascend950", server_topo: "ascend950_server_topo_1dpod64.yaml"} | ||
| 11 | + - {super_pod_id: 1, id_range: [0, 1], soc_version: "Ascend950", server_topo: "ascend950_server_topo_1dpod64.yaml"} | ||
| @@ -0,0 +1,10 @@ | |||
| 1 | +name: "ascend950_cluster_4_server_competition" | ||
| 2 | +description: "昇腾950 competition组网：32个超级节点，每个超级节点1个服务器" | ||
| 3 | + | ||
| 4 | +# 超节点总数量 | ||
| 5 | +super_node_num: 1 | ||
| 6 | +# sever/pod总数量 | ||
| 7 | +server_num: 4 | ||
| 8 | +server_list: | ||
| 9 | + # 0-3 server: 均采用ascend950_server_topo_competition拓扑类型 | ||
| 10 | + - {super_pod_id: 0, id_range: [0, 3], soc_version: "Ascend950", server_topo: "ascend950_server_topo_competition.yaml"} | ||
| @@ -0,0 +1,11 @@ | |||
| 1 | +name: "ascend950_cluster_32_server_normal" | ||
| 2 | +description: "昇腾950 normal组网：32个超级节点，每个超级节点1个服务器" | ||
| 3 | + | ||
| 4 | +# 超节点总数量 | ||
| 5 | +super_node_num: 2 | ||
| 6 | +# sever/pod总数量 | ||
| 7 | +server_num: 4 | ||
| 8 | +server_list: | ||
| 9 | + # 0-1 server: 均采用ascend950_server_topo_normal拓扑类型 | ||
| 10 | + - {super_pod_id: 0, id_range: [0, 1], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| 11 | + - {super_pod_id: 1, id_range: [0, 1], soc_version: "Ascend950", server_topo: "ascend950_server_topo_normal.yaml"} | ||
| @@ -0,0 +1,78 @@ | |||
| 1 | +type: "server_intra_links" | ||
| 2 | +name: "ascend950_links_topo_1dpod64" | ||
| 3 | +description: "ascend950芯片1dpod64类型拓扑连接关系描述文件" | ||
| 4 | + | ||
| 5 | +soc_version: "Ascend950" | ||
| 6 | +device_num: 64 | ||
| 7 | + | ||
| 8 | +device_group: | ||
| 9 | + - {group_id: 1, device_list: [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59]} | ||
| 10 | + - {group_id: 2, device_list: [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31, 36, 37, 38, 39, 44, 45, 46, 47, 52, 53, 54, 55, 60, 61, 62, 63]} | ||
| 11 | + | ||
| 12 | +device_ports_allocate_map: | ||
| 13 | + # port分配表: 0: 不使用, 1: device直连, 2: device连交换机, 3: d2h端口 | ||
| 14 | + # portId: 0 1 2 3 4 5 6 7 8 | ||
| 15 | + # 端口分配表1: 适用于device group 1 | ||
| 16 | + - group_id: 1 | ||
| 17 | + - {die_id: 0, pin_map: [1, 2, 2, 1, 1, 1, 1, 1 , 1]} # die0 | ||
| 18 | + - {die_id: 1, pin_map: [2, 2, 2, 2, 0, 2, 2, 0, 3]} # die1 | ||
| 19 | + # 端口分配表2: 适用于device group 2 | ||
| 20 | + - group_id: 2 | ||
| 21 | + # portId: 0 1 2 3 4 5 6 7 8 | ||
| 22 | + - {die_id: 0, pin_map: [2, 2, 2, 2, 2, 2, 0, 0, 3]} # die0 | ||
| 23 | + - {die_id: 1, pin_map: [1, 2, 2, 1, 1, 1, 1, 1, 1]} # die1 | ||
| 24 | + | ||
| 25 | +# port_list: topo.json文件中各netlayer的port列表 | ||
| 26 | +port_list: | ||
| 27 | + - group_id: 1 | ||
| 28 | + - {layer: 0, ports: ["0/1", "0/2", "1/0", "1/1", "1/2", "1/3", "1/5", "1/6"]} | ||
| 29 | + - {layer: 1, ports: ["0/1", "0/2", "1/0", "1/1", "1/2", "1/3", "1/5", "1/6"]} | ||
| 30 | + - {layer: 2, ports: ["0/1", "0/2", "1/1", "1/2"]} | ||
| 31 | + - {layer: 3, ports: ["d2h"]} | ||
| 32 | + - group_id: 2 | ||
| 33 | + - {layer: 0, ports: ["1/1", "1/2", "0/0", "0/1", "0/2", "0/3", "0/4", "0/5"]} | ||
| 34 | + - {layer: 1, ports: ["1/1", "1/2", "0/0", "0/1", "0/2", "0/3", "0/4", "0/5"]} | ||
| 35 | + - {layer: 2, ports: ["0/1", "0/2", "1/1", "1/2"]} | ||
| 36 | + - {layer: 3, ports: ["d2h"]} | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +# port_group: rootinfo.json文件中的portGroup，相同portGroup的port对于IP地址相同 | ||
| 40 | +port_group: | ||
| 41 | + - group_id: 1 | ||
| 42 | + - {layer: 1, ports: ["0/1", "0/2"]} | ||
| 43 | + - {layer: 1, ports: ["1/0", "1/1", "1/2", "1/3", "1/5", "1/6"]} | ||
| 44 | + - group_id: 2 | ||
| 45 | + - {layer: 1, ports: ["1/1", "1/2"]} | ||
| 46 | + - {layer: 1, ports: ["0/0", "0/1", "0/2", "0/3", "0/4", "0/5"]} | ||
| 47 | + | ||
| 48 | +links: | ||
| 49 | + # ── 描述法：每行8个device组成全互联 ── | ||
| 50 | + - link_mode: "fullmesh" | ||
| 51 | + connections: | ||
| 52 | + - {die_id: 0, devices: [0, 1, 2, 3]} | ||
| 53 | + - {die_id: 1, devices: [4, 5, 6, 7]} | ||
| 54 | + - {die_id: 0, devices: [8, 9, 10, 11]} | ||
| 55 | + - {die_id: 1, devices: [12, 13, 14, 15]} | ||
| 56 | + - {die_id: 0, devices: [16, 17, 18, 19]} | ||
| 57 | + - {die_id: 1, devices: [20, 21, 22, 23]} | ||
| 58 | + - {die_id: 0, devices: [24, 25, 26, 27]} | ||
| 59 | + - {die_id: 1, devices: [28, 29, 30, 31]} | ||
| 60 | + - {die_id: 0, devices: [32, 33, 34, 35]} | ||
| 61 | + - {die_id: 1, devices: [36, 37, 38, 39]} | ||
| 62 | + - {die_id: 0, devices: [40, 41, 42, 43]} | ||
| 63 | + - {die_id: 1, devices: [44, 45, 46, 47]} | ||
| 64 | + - {die_id: 0, devices: [48, 49, 50, 51]} | ||
| 65 | + - {die_id: 1, devices: [52, 53, 54, 55]} | ||
| 66 | + - {die_id: 0, devices: [56, 57, 58, 59]} | ||
| 67 | + - {die_id: 1, devices: [60, 61, 62, 63]} | ||
| 68 | + | ||
| 69 | + - link_mode: "enum" | ||
| 70 | + device_to_device_links: | ||
| 71 | + - {src_die_id: 0, src_local_id: [0, 1, 2, 3], dst_die_id: 1, dst_local_id: [4, 5, 6, 7]} | ||
| 72 | + - {src_die_id: 0, src_local_id: [8, 9, 10, 11], dst_die_id: 1, dst_local_id: [12, 13, 14, 15]} | ||
| 73 | + - {src_die_id: 0, src_local_id: [16, 17, 18, 19], dst_die_id: 1, dst_local_id: [20, 21, 22, 23]} | ||
| 74 | + - {src_die_id: 0, src_local_id: [24, 25, 26, 27], dst_die_id: 1, dst_local_id: [28, 29, 30, 31]} | ||
| 75 | + - {src_die_id: 0, src_local_id: [32, 33, 34, 35], dst_die_id: 1, dst_local_id: [36, 37, 38, 39]} | ||
| 76 | + - {src_die_id: 0, src_local_id: [40, 41, 42, 43], dst_die_id: 1, dst_local_id: [44, 45, 46, 47]} | ||
| 77 | + - {src_die_id: 0, src_local_id: [48, 49, 50, 51], dst_die_id: 1, dst_local_id: [52, 53, 54, 55]} | ||
| 78 | + - {src_die_id: 0, src_local_id: [56, 57, 58, 59], dst_die_id: 1, dst_local_id: [60, 61, 62, 63]} | ||
| @@ -0,0 +1,32 @@ | |||
| 1 | +type: "server_intra_links" | ||
| 2 | +name: "ascend950_links_topo_competition" | ||
| 3 | +description: "ascend950芯片competition类型拓扑连接关系描述文件" | ||
| 4 | + | ||
| 5 | +soc_version: "Ascend950" | ||
| 6 | +device_num: 8 | ||
| 7 | + | ||
| 8 | +device_ports_allocate_map: | ||
| 9 | + # port分配表: 0: 不使用, 1: device直连, 2: device连交换机, 3: d2h端口 | ||
| 10 | + # portId: 0 1 2 3 4 5 6 7 8 | ||
| 11 | + - {die_id: 0, pin_map: [0, 1, 1, 1, 2, 2, 2, 2, 0]} # die0 | ||
| 12 | + - {die_id: 1, pin_map: [1, 1, 1, 1, 0, 2, 2, 0, 0]} # die1 | ||
| 13 | + | ||
| 14 | +port_group: | ||
| 15 | + - {layer: 1, ports: ["0/4", "0/5", "0/6", "0/7", "1/5", "1/6"]} | ||
| 16 | + | ||
| 17 | +links: | ||
| 18 | + - link_mode: "enum" | ||
| 19 | + device_to_device_links: | ||
| 20 | + - {src_die_id: 1, src_local_id_range: [0], dst_die_id: 1, dst_local_id_range: [1, 3, 5, 7]} | ||
| 21 | + - {src_die_id: 0, src_local_id_range: [0], dst_die_id: 1, dst_local_id_range: [2, 4, 6]} | ||
| 22 | + - {src_die_id: 0, src_local_id_range: [1], dst_die_id: 0, dst_local_id_range: [2, 4, 6]} | ||
| 23 | + - {src_die_id: 1, src_local_id_range: [1], dst_die_id: 0, dst_local_id_range: [3, 5, 7]} | ||
| 24 | + - {src_die_id: 1, src_local_id_range: [2], dst_die_id: 1, dst_local_id_range: [3, 5, 7]} | ||
| 25 | + - {src_die_id: 0, src_local_id_range: [2], dst_die_id: 1, dst_local_id_range: [4, 6]} | ||
| 26 | + - {src_die_id: 0, src_local_id_range: [3], dst_die_id: 0, dst_local_id_range: [4, 6]} | ||
| 27 | + - {src_die_id: 1, src_local_id_range: [3], dst_die_id: 0, dst_local_id_range: [5, 7]} | ||
| 28 | + - {src_die_id: 1, src_local_id_range: [4], dst_die_id: 1, dst_local_id_range: [5, 7]} | ||
| 29 | + - {src_die_id: 0, src_local_id_range: [4], dst_die_id: 1, dst_local_id_range: [6]} | ||
| 30 | + - {src_die_id: 0, src_local_id_range: [5], dst_die_id: 0, dst_local_id_range: [6]} | ||
| 31 | + - {src_die_id: 1, src_local_id_range: [5], dst_die_id: 0, dst_local_id_range: [7]} | ||
| 32 | + - {src_die_id: 1, src_local_id_range: [6], dst_die_id: 1, dst_local_id_range: [7]} | ||
| @@ -0,0 +1,29 @@ | |||
| 1 | +type: "server_intra_links" | ||
| 2 | +name: "ascend950_links_topo_normal" | ||
| 3 | +description: "ascend950芯片NORMAL类型拓扑连接关系描述文件" | ||
| 4 | + | ||
| 5 | +soc_version: "Ascend950" | ||
| 6 | +device_num: 8 | ||
| 7 | +hardware_type: "910D-2D-Fullmsh_64_plus_1" | ||
| 8 | + | ||
| 9 | +device_ports_allocate_map: | ||
| 10 | + # port分配表: 0: 不使用, 1: device直连, 2: device连交换机, 3: d2h端口 | ||
| 11 | + # portId: 0 1 2 3 4 5 6 7 8 | ||
| 12 | + - {die_id: 0, pin_map: [2, 2, 2, 2, 2, 2, 2, 2, 3]} # die0 | ||
| 13 | + - {die_id: 1, pin_map: [1, 1, 1, 1, 1, 1, 1, 0, 0]} # die1 | ||
| 14 | + | ||
| 15 | +# port_group: 描述哪些port合并为一个portGroup，相同portGroup的port对于IP地址相同 | ||
| 16 | +port_group: | ||
| 17 | + - {layer: 1, ports: ["0/0", "0/1", "0/2", "0/3", "0/4", "0/5", "0/6", "0/7"]} | ||
| 18 | + - {layer: 2, ports: ["0/7"]} | ||
| 19 | + - {layer: 3, ports: ["d2h"]} | ||
| 20 | + | ||
| 21 | +links: | ||
| 22 | + # ── 描述法：每行8个device组成全互联 ── | ||
| 23 | + - link_mode: "fullmesh" | ||
| 24 | + connections: | ||
| 25 | + - {die_id: 1, devices_range: [0, 7]} | ||
| 26 | + | ||
| 27 | + - link_mode: "enum" | ||
| 28 | + device_to_switch_links: | ||
| 29 | + - {die_id: 0, devices_range: [0, 7]} | ||
| @@ -0,0 +1,12 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 1 | ||
| 5 | + rankNum: 2 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1] | ||
| @@ -0,0 +1,12 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 1 | ||
| 5 | + rankNum: 4 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1, 2, 3] | ||
| @@ -0,0 +1,12 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 1 | ||
| 5 | + rankNum: 8 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1, 2, 3, 4, 5, 6, 7] | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 2 | ||
| 5 | + rankNum: 2 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0] | ||
| 13 | + - serId: 1 | ||
| 14 | + ranks: [0] | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 2 | ||
| 5 | + rankNum: 4 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1] | ||
| 13 | + - serId: 1 | ||
| 14 | + ranks: [0, 1] | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 2 | ||
| 5 | + rankNum: 8 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1, 2, 3] | ||
| 13 | + - serId: 1 | ||
| 14 | + ranks: [0, 1, 2, 3] | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 2 | ||
| 5 | + rankNum: 16 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1, 2, 3, 4, 5, 6, 7] | ||
| 13 | + - serId: 1 | ||
| 14 | + ranks: [0, 1, 2, 3, 4, 5, 6, 7] | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 2 | ||
| 5 | + rankNum: 6 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1] | ||
| 13 | + - serId: 1 | ||
| 14 | + ranks: [0, 1, 2, 3] | ||
| @@ -0,0 +1,18 @@ | |||
| 1 | +# 1. 全局统计信息 podNum, serNum, rankNum 都小于 1024 | ||
| 2 | +meta: | ||
| 3 | + podNum: 1 | ||
| 4 | + serNum: 4 | ||
| 5 | + rankNum: 16 | ||
| 6 | + | ||
| 7 | +# 2. 详细拓扑结构 | ||
| 8 | +topology: | ||
| 9 | + - podId: 0 | ||
| 10 | + servers: | ||
| 11 | + - serId: 0 | ||
| 12 | + ranks: [0, 1, 2, 3] | ||
| 13 | + - serId: 1 | ||
| 14 | + ranks: [0, 1, 2, 3] | ||
| 15 | + - serId: 2 | ||
| 16 | + ranks: [0, 1, 2, 3] | ||
| 17 | + - serId: 3 | ||
| 18 | + ranks: [0, 1, 2, 3] | ||
| @@ -0,0 +1,523 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + | ||
| 10 | +# 根据集群拓扑配置文件，生成集群组网目录结构 | ||
| 11 | +# 功能： | ||
| 12 | +# a. 解析集群YAML文件，在输出目录下创建所有超节点组网目录 | ||
| 13 | +# b. 支持两种集群配置格式： | ||
| 14 | +# - 新格式(扁平)：集群YAML直接包含server_list，每个server通过super_pod_id归属超节点 | ||
| 15 | +# - 旧格式(三层)：集群YAML通过super_node_list引用超节点YAML，超节点YAML中定义server_list | ||
| 16 | +# c. 支持两种方式获取server拓扑文件： | ||
| 17 | +# - 如果server_topo是yaml文件，调用generate_topo_rootinfo.sh生成topo.json和rootinfo.json | ||
| 18 | +# - 如果server_topo是目录名，从topo_bank/server_topo_bank目录拷贝 | ||
| 19 | +# d. 支持id_range扩展：集群配置中super_node_list的id_range，server_list的id_range | ||
| 20 | +# e. 调用allocate_eid.sh脚本，统一分配server和device ports的IP地址/EID | ||
| 21 | +# f. 生成servers_info.json文件，记录所有server的host IP和soc_version信息 | ||
| 22 | + | ||
| 23 | +set -e | ||
| 24 | + | ||
| 25 | +RED='\033[0;31m' | ||
| 26 | +GREEN='\033[0;32m' | ||
| 27 | +YELLOW='\033[1;33m' | ||
| 28 | +BLUE='\033[0;34m' | ||
| 29 | +NC='\033[0m' | ||
| 30 | + | ||
| 31 | +error_exit() { | ||
| 32 | + echo -e "${RED}[错误] $1${NC}" >&2 | ||
| 33 | + exit 1 | ||
| 34 | +} | ||
| 35 | + | ||
| 36 | +success_msg() { | ||
| 37 | + echo -e "${GREEN}[成功] $1${NC}" | ||
| 38 | +} | ||
| 39 | + | ||
| 40 | +info_msg() { | ||
| 41 | + echo -e "${BLUE}[信息] $1${NC}" | ||
| 42 | +} | ||
| 43 | + | ||
| 44 | +warn_msg() { | ||
| 45 | + echo -e "${YELLOW}[警告] $1${NC}" | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | ||
| 49 | +BASE_DIR="${SCRIPT_DIR}" | ||
| 50 | + | ||
| 51 | +CLUSTER_CONFIG_DIR="${BASE_DIR}/cluster_model/config/cluster" | ||
| 52 | +SUPERPOD_CONFIG_DIR="${BASE_DIR}/cluster_model/config/super_pod" | ||
| 53 | +SERVER_CONFIG_DIR="${BASE_DIR}/cluster_model/config/server_or_pod" | ||
| 54 | +SERVER_TOPO_BANK_DIR="${BASE_DIR}/topo_bank/server_topo_bank" | ||
| 55 | +GENERATE_TOPO_SCRIPT="${BASE_DIR}/generate_server_topo.sh" | ||
| 56 | +ALLOCATE_EID_SCRIPT="${BASE_DIR}/allocate_eid.sh" | ||
| 57 | +DEFAULT_OUTPUT_DIR="${BASE_DIR}/cluster_model/network/cluster" | ||
| 58 | + | ||
| 59 | +resolve_path() { | ||
| 60 | + local input="$1" | ||
| 61 | + local default_dir="$2" | ||
| 62 | + if [[ "$input" = /* ]]; then | ||
| 63 | + if [[ ! -f "$input" ]]; then | ||
| 64 | + error_exit "文件不存在: $input" | ||
| 65 | + fi | ||
| 66 | + realpath "$input" | ||
| 67 | + else | ||
| 68 | + local full_path="${default_dir}/${input}" | ||
| 69 | + if [[ ! -f "$full_path" ]]; then | ||
| 70 | + error_exit "文件不存在: $full_path (默认路径: ${default_dir})" | ||
| 71 | + fi | ||
| 72 | + realpath "$full_path" | ||
| 73 | + fi | ||
| 74 | +} | ||
| 75 | + | ||
| 76 | +show_usage() { | ||
| 77 | + cat << EOF | ||
| 78 | +用法: $(basename "$0") [OPTIONS] <集群配置文件> | ||
| 79 | + | ||
| 80 | +参数: | ||
| 81 | + 集群配置文件 配置文件路径或文件名 | ||
| 82 | + 绝对路径: 直接使用 | ||
| 83 | + 文件名/相对路径: 在 ${CLUSTER_CONFIG_DIR} 下查找 | ||
| 84 | + | ||
| 85 | +选项: | ||
| 86 | + -o, --output DIR 指定输出目录 (默认: ${DEFAULT_OUTPUT_DIR}) | ||
| 87 | + -h, --help 显示帮助信息 | ||
| 88 | + | ||
| 89 | +示例: | ||
| 90 | + $(basename "$0") ascend950_cluster_4_server_normal.yaml | ||
| 91 | + $(basename "$0") /absolute/path/to/cluster_config.yaml | ||
| 92 | + $(basename "$0") -o /tmp/my_output ascend950_cluster_4_server_normal.yaml | ||
| 93 | + | ||
| 94 | +说明: | ||
| 95 | + 1. 解析集群YAML文件，自动检测配置格式（新格式/旧格式） | ||
| 96 | + 2. 新格式：server_list直接在集群YAML中，通过super_pod_id归属超节点 | ||
| 97 | + 3. 旧格式：通过super_node_list引用超节点YAML，超节点YAML中定义server_list | ||
| 98 | + 4. 在输出目录下为每个超节点创建目录 | ||
| 99 | + 5. 在超节点目录下为每个server创建目录 | ||
| 100 | + 6. 根据server_topo配置生成或拷贝topo.json和rootinfo.json | ||
| 101 | + 7. 调用allocate_eid.sh统一分配IP/EID地址 | ||
| 102 | + 8. 生成servers_info.json文件 | ||
| 103 | + | ||
| 104 | +EOF | ||
| 105 | +} | ||
| 106 | + | ||
| 107 | +CLUSTER_YAML_INPUT="" | ||
| 108 | +OUTPUT_DIR="" | ||
| 109 | + | ||
| 110 | +while [[ $# -gt 0 ]]; do | ||
| 111 | + case $1 in | ||
| 112 | + -h|--help) | ||
| 113 | + show_usage | ||
| 114 | + exit 0 | ||
| 115 | + ;; | ||
| 116 | + -o|--output) | ||
| 117 | + OUTPUT_DIR="$2" | ||
| 118 | + shift 2 | ||
| 119 | + ;; | ||
| 120 | + *) | ||
| 121 | + CLUSTER_YAML_INPUT="$1" | ||
| 122 | + shift | ||
| 123 | + ;; | ||
| 124 | + esac | ||
| 125 | +done | ||
| 126 | + | ||
| 127 | +if [[ -z "$CLUSTER_YAML_INPUT" ]]; then | ||
| 128 | + error_exit "请指定集群配置文件\n$(show_usage)" | ||
| 129 | +fi | ||
| 130 | + | ||
| 131 | +CLUSTER_YAML_PATH="$(resolve_path "$CLUSTER_YAML_INPUT" "$CLUSTER_CONFIG_DIR")" | ||
| 132 | + | ||
| 133 | +if [[ -z "$OUTPUT_DIR" ]]; then | ||
| 134 | + OUTPUT_DIR="${DEFAULT_OUTPUT_DIR}" | ||
| 135 | +fi | ||
| 136 | + | ||
| 137 | +echo "==========================================" | ||
| 138 | +echo " 昇腾集群组网拓扑生成工具" | ||
| 139 | +echo "==========================================" | ||
| 140 | +echo "" | ||
| 141 | +echo "集群配置文件: $CLUSTER_YAML_PATH" | ||
| 142 | +echo "输出根目录: $OUTPUT_DIR" | ||
| 143 | +echo "" | ||
| 144 | + | ||
| 145 | +CLUSTER_BASENAME=$(basename "$CLUSTER_YAML_PATH" .yaml) | ||
| 146 | + | ||
| 147 | +OUTPUT_DIR="${OUTPUT_DIR}/${CLUSTER_BASENAME}" | ||
| 148 | + | ||
| 149 | +echo "集群输出目录: $OUTPUT_DIR" | ||
| 150 | +echo "" | ||
| 151 | + | ||
| 152 | +rm -rf "$OUTPUT_DIR" | ||
| 153 | +mkdir -p "$OUTPUT_DIR" | ||
| 154 | + | ||
| 155 | +export CLUSTER_YAML_PATH | ||
| 156 | +export OUTPUT_DIR | ||
| 157 | +export CLUSTER_CONFIG_DIR | ||
| 158 | +export SUPERPOD_CONFIG_DIR | ||
| 159 | +export SERVER_CONFIG_DIR | ||
| 160 | +export SERVER_TOPO_BANK_DIR | ||
| 161 | +export GENERATE_TOPO_SCRIPT | ||
| 162 | +export ALLOCATE_EID_SCRIPT | ||
| 163 | + | ||
| 164 | +python3 << 'PYTHON_SCRIPT' | ||
| 165 | +import json | ||
| 166 | +import os | ||
| 167 | +import re | ||
| 168 | +import shutil | ||
| 169 | +import subprocess | ||
| 170 | +import sys | ||
| 171 | +import socket | ||
| 172 | + | ||
| 173 | +try: | ||
| 174 | + import yaml | ||
| 175 | +except ImportError: | ||
| 176 | + print("[错误] 需要PyYAML库，请安装: pip install pyyaml") | ||
| 177 | + sys.exit(1) | ||
| 178 | + | ||
| 179 | +CLUSTER_YAML_PATH = os.environ.get('CLUSTER_YAML_PATH', '') | ||
| 180 | +OUTPUT_DIR = os.environ.get('OUTPUT_DIR', '') | ||
| 181 | +SUPERPOD_CONFIG_DIR = os.environ.get('SUPERPOD_CONFIG_DIR', '') | ||
| 182 | +SERVER_CONFIG_DIR = os.environ.get('SERVER_CONFIG_DIR', '') | ||
| 183 | +SERVER_TOPO_BANK_DIR = os.environ.get('SERVER_TOPO_BANK_DIR', '') | ||
| 184 | +GENERATE_TOPO_SCRIPT = os.environ.get('GENERATE_TOPO_SCRIPT', '') | ||
| 185 | +ALLOCATE_EID_SCRIPT = os.environ.get('ALLOCATE_EID_SCRIPT', '') | ||
| 186 | + | ||
| 187 | +URMA_EID_LEN = 16 | ||
| 188 | + | ||
| 189 | +def ip_to_eid_string(ip, family=socket.AF_INET): | ||
| 190 | + if family == socket.AF_INET: | ||
| 191 | + ip_bytes = socket.inet_pton(socket.AF_INET, ip) | ||
| 192 | + eid = bytearray(URMA_EID_LEN) | ||
| 193 | + eid[12:16] = ip_bytes | ||
| 194 | + elif family == socket.AF_INET6: | ||
| 195 | + ip_bytes = socket.inet_pton(socket.AF_INET6, ip) | ||
| 196 | + eid = bytearray(URMA_EID_LEN) | ||
| 197 | + eid[0:16] = ip_bytes | ||
| 198 | + else: | ||
| 199 | + raise ValueError(f"不支持的地址族: {family}") | ||
| 200 | + hex_str = ''.join(f'{b:02x}' for b in eid) | ||
| 201 | + return f'0x{hex_str}' | ||
| 202 | + | ||
| 203 | +def expand_id_range(entry): | ||
| 204 | + ids = [] | ||
| 205 | + if 'id' in entry: | ||
| 206 | + raw = entry['id'] | ||
| 207 | + if isinstance(raw, list): | ||
| 208 | + ids.extend([int(x) for x in raw]) | ||
| 209 | + else: | ||
| 210 | + ids.append(int(raw)) | ||
| 211 | + if 'id_range' in entry: | ||
| 212 | + rng = entry['id_range'] | ||
| 213 | + if isinstance(rng, list) and len(rng) == 2: | ||
| 214 | + ids.extend(list(range(int(rng[0]), int(rng[1]) + 1))) | ||
| 215 | + elif isinstance(rng, list): | ||
| 216 | + ids.extend([int(x) for x in rng]) | ||
| 217 | + return ids | ||
| 218 | + | ||
| 219 | +def find_superpod_yaml(topo_name): | ||
| 220 | + if not topo_name: | ||
| 221 | + return None | ||
| 222 | + if os.path.isfile(topo_name): | ||
| 223 | + return topo_name | ||
| 224 | + candidates = [ | ||
| 225 | + os.path.join(SUPERPOD_CONFIG_DIR, topo_name), | ||
| 226 | + os.path.join(SUPERPOD_CONFIG_DIR, topo_name if topo_name.endswith('.yaml') else topo_name + '.yaml'), | ||
| 227 | + ] | ||
| 228 | + for c in candidates: | ||
| 229 | + if os.path.isfile(c): | ||
| 230 | + return c | ||
| 231 | + return None | ||
| 232 | + | ||
| 233 | +def find_server_topo(topo_name): | ||
| 234 | + if not topo_name: | ||
| 235 | + return None, None | ||
| 236 | + yaml_candidates = [ | ||
| 237 | + os.path.join(SERVER_CONFIG_DIR, topo_name), | ||
| 238 | + os.path.join(SERVER_CONFIG_DIR, topo_name if topo_name.endswith('.yaml') else topo_name + '.yaml'), | ||
| 239 | + ] | ||
| 240 | + for c in yaml_candidates: | ||
| 241 | + if os.path.isfile(c): | ||
| 242 | + return 'yaml', c | ||
| 243 | + dir_candidates = [ | ||
| 244 | + os.path.join(SERVER_TOPO_BANK_DIR, topo_name), | ||
| 245 | + ] | ||
| 246 | + for c in dir_candidates: | ||
| 247 | + if os.path.isdir(c): | ||
| 248 | + return 'bank', c | ||
| 249 | + return None, None | ||
| 250 | + | ||
| 251 | +def generate_server_topo(yaml_path, output_dir): | ||
| 252 | + os.makedirs(output_dir, exist_ok=True) | ||
| 253 | + cmd = ['bash', GENERATE_TOPO_SCRIPT, yaml_path, '-o', output_dir, '-b'] | ||
| 254 | + result = subprocess.run(cmd, capture_output=True, text=True) | ||
| 255 | + if result.returncode != 0: | ||
| 256 | + print(f" [错误] 生成server拓扑失败: {result.stderr}") | ||
| 257 | + return False | ||
| 258 | + return True | ||
| 259 | + | ||
| 260 | +def copy_server_topo_from_bank(bank_dir, output_dir): | ||
| 261 | + os.makedirs(output_dir, exist_ok=True) | ||
| 262 | + copied_files = [] | ||
| 263 | + for f in os.listdir(bank_dir): | ||
| 264 | + src = os.path.join(bank_dir, f) | ||
| 265 | + if os.path.isfile(src): | ||
| 266 | + shutil.copy2(src, output_dir) | ||
| 267 | + copied_files.append(f) | ||
| 268 | + return copied_files | ||
| 269 | + | ||
| 270 | +# ============================================================ | ||
| 271 | +# Step A: 解析集群YAML文件，自动检测格式 | ||
| 272 | +# ============================================================ | ||
| 273 | +print("[信息] Step A: 解析集群配置文件...") | ||
| 274 | + | ||
| 275 | +with open(CLUSTER_YAML_PATH, 'r') as f: | ||
| 276 | + cluster_config = yaml.safe_load(f) | ||
| 277 | + | ||
| 278 | +cluster_name = cluster_config.get('name', 'unknown') | ||
| 279 | +super_node_num = cluster_config.get('super_node_num', 0) | ||
| 280 | +server_list_raw = cluster_config.get('server_list', []) | ||
| 281 | +super_node_list_raw = cluster_config.get('super_node_list', []) | ||
| 282 | + | ||
| 283 | +is_new_format = bool(server_list_raw and not super_node_list_raw) | ||
| 284 | + | ||
| 285 | +print(f" 集群名称: {cluster_name}") | ||
| 286 | +print(f" 超节点数量: {super_node_num}") | ||
| 287 | +print(f" 配置格式: {'新格式(扁平)' if is_new_format else '旧格式(三层)'}") | ||
| 288 | + | ||
| 289 | +if super_node_num == 0 and not is_new_format: | ||
| 290 | + print("[错误] 集群配置文件中未定义超节点 (super_node_num=0)") | ||
| 291 | + sys.exit(1) | ||
| 292 | + | ||
| 293 | +# ============================================================ | ||
| 294 | +# Step B: 构建超节点→server映射，创建目录结构 | ||
| 295 | +# ============================================================ | ||
| 296 | +# sp_servers: {sp_id: [{'id': srv_id, 'server_topo': ..., 'soc_version': ...}, ...]} | ||
| 297 | +sp_servers = {} | ||
| 298 | + | ||
| 299 | +if is_new_format: | ||
| 300 | + # 新格式：server_list直接在集群YAML中，通过super_pod_id归属超节点 | ||
| 301 | + server_num = cluster_config.get('server_num', 0) | ||
| 302 | + print(f" Server数量: {server_num}") | ||
| 303 | + | ||
| 304 | + for entry in server_list_raw: | ||
| 305 | + sp_id = entry.get('super_pod_id', 0) | ||
| 306 | + ids = expand_id_range(entry) | ||
| 307 | + srv_topo = entry.get('server_topo', '') | ||
| 308 | + soc_version = entry.get('soc_version', '') | ||
| 309 | + for sid in ids: | ||
| 310 | + if sp_id not in sp_servers: | ||
| 311 | + sp_servers[sp_id] = [] | ||
| 312 | + sp_servers[sp_id].append({ | ||
| 313 | + 'id': sid, | ||
| 314 | + 'server_topo': srv_topo, | ||
| 315 | + 'soc_version': soc_version | ||
| 316 | + }) | ||
| 317 | + | ||
| 318 | + actual_server_count = sum(len(v) for v in sp_servers.values()) | ||
| 319 | + actual_super_node_count = len(sp_servers) | ||
| 320 | + print(f" 展开后: {actual_super_node_count} 个超节点, {actual_server_count} 个server") | ||
| 321 | + | ||
| 322 | + if server_num > 0 and actual_server_count != server_num: | ||
| 323 | + print(f"[错误] server_num({server_num})与server_list展开后的实际server数量({actual_server_count})不一致") | ||
| 324 | + sys.exit(1) | ||
| 325 | + | ||
| 326 | + if super_node_num > 0 and actual_super_node_count != super_node_num: | ||
| 327 | + print(f"[错误] super_node_num({super_node_num})与server_list中实际超节点数量({actual_super_node_count})不一致") | ||
| 328 | + sys.exit(1) | ||
| 329 | +else: | ||
| 330 | + # 旧格式：通过super_node_list引用超节点YAML | ||
| 331 | + expanded_super_nodes = [] | ||
| 332 | + for entry in super_node_list_raw: | ||
| 333 | + ids = expand_id_range(entry) | ||
| 334 | + topo = entry.get('superpod_topo', '') | ||
| 335 | + for sid in ids: | ||
| 336 | + expanded_super_nodes.append({'id': sid, 'superpod_topo': topo}) | ||
| 337 | + | ||
| 338 | + print(f" 展开后超节点数: {len(expanded_super_nodes)}") | ||
| 339 | + | ||
| 340 | + if super_node_num > 0 and len(expanded_super_nodes) != super_node_num: | ||
| 341 | + print(f"[错误] super_node_num({super_node_num})与super_node_list展开后的实际超节点数量({len(expanded_super_nodes)})不一致") | ||
| 342 | + sys.exit(1) | ||
| 343 | + | ||
| 344 | + for sp_entry in expanded_super_nodes: | ||
| 345 | + sp_id = sp_entry['id'] | ||
| 346 | + sp_topo_name = sp_entry['superpod_topo'] | ||
| 347 | + | ||
| 348 | + sp_yaml_path = find_superpod_yaml(sp_topo_name) | ||
| 349 | + if not sp_yaml_path: | ||
| 350 | + print(f" [错误] 超节点拓扑配置文件不存在: {sp_topo_name}") | ||
| 351 | + continue | ||
| 352 | + | ||
| 353 | + with open(sp_yaml_path, 'r') as f: | ||
| 354 | + sp_config = yaml.safe_load(f) | ||
| 355 | + | ||
| 356 | + sp_name = sp_config.get('name', 'unknown') | ||
| 357 | + sp_form_type = sp_config.get('form_type', 'server') | ||
| 358 | + server_num = sp_config.get('server_num', 0) | ||
| 359 | + sp_server_list_raw = sp_config.get('server_list', []) | ||
| 360 | + | ||
| 361 | + print(f" [信息] 超节点{sp_id}: {sp_name}, 形态: {sp_form_type}, Server数量: {server_num}") | ||
| 362 | + | ||
| 363 | + expanded_servers = [] | ||
| 364 | + for srv_entry in sp_server_list_raw: | ||
| 365 | + ids = expand_id_range(srv_entry) | ||
| 366 | + srv_topo = srv_entry.get('server_topo', '') | ||
| 367 | + soc_version = srv_entry.get('soc_version', '') | ||
| 368 | + for sid in ids: | ||
| 369 | + expanded_servers.append({ | ||
| 370 | + 'id': sid, | ||
| 371 | + 'server_topo': srv_topo, | ||
| 372 | + 'soc_version': soc_version | ||
| 373 | + }) | ||
| 374 | + | ||
| 375 | + sp_servers[sp_id] = expanded_servers | ||
| 376 | + print(f" [信息] 展开后Server数: {len(expanded_servers)}") | ||
| 377 | + | ||
| 378 | + if server_num > 0 and len(expanded_servers) != server_num: | ||
| 379 | + print(f"[错误] 超节点{sp_id}的server_num({server_num})与server_list展开后的实际server数量({len(expanded_servers)})不一致") | ||
| 380 | + sys.exit(1) | ||
| 381 | + | ||
| 382 | +print("") | ||
| 383 | + | ||
| 384 | +# ============================================================ | ||
| 385 | +# Step C: 创建目录结构，生成/拷贝server拓扑文件 | ||
| 386 | +# ============================================================ | ||
| 387 | +for sp_id in sorted(sp_servers.keys()): | ||
| 388 | + sp_dir = os.path.join(OUTPUT_DIR, f'superpod{sp_id}') | ||
| 389 | + os.makedirs(sp_dir, exist_ok=True) | ||
| 390 | + print(f"[成功] 创建超节点目录: superpod{sp_id}") | ||
| 391 | + | ||
| 392 | + for srv in sp_servers[sp_id]: | ||
| 393 | + srv_id = srv['id'] | ||
| 394 | + srv_topo_name = srv['server_topo'] | ||
| 395 | + srv_soc_version = srv['soc_version'] | ||
| 396 | + | ||
| 397 | + srv_dir = os.path.join(sp_dir, f'server{srv_id}') | ||
| 398 | + | ||
| 399 | + topo_type, topo_path = find_server_topo(srv_topo_name) | ||
| 400 | + | ||
| 401 | + if topo_type == 'yaml': | ||
| 402 | + if generate_server_topo(topo_path, srv_dir): | ||
| 403 | + print(f" [成功] server{srv_id}: 生成拓扑 ({srv_topo_name})") | ||
| 404 | + else: | ||
| 405 | + print(f" [错误] server{srv_id}: 生成拓扑失败 ({srv_topo_name})") | ||
| 406 | + elif topo_type == 'bank': | ||
| 407 | + copied = copy_server_topo_from_bank(topo_path, srv_dir) | ||
| 408 | + print(f" [成功] server{srv_id}: 拷贝 {len(copied)} 个文件 ({srv_topo_name})") | ||
| 409 | + else: | ||
| 410 | + print(f" [错误] server{srv_id}: 找不到拓扑配置 ({srv_topo_name})") | ||
| 411 | + | ||
| 412 | + print("") | ||
| 413 | + | ||
| 414 | +# ============================================================ | ||
| 415 | +# Step D: 调用allocate_eid.sh分配EID/IP地址 | ||
| 416 | +# ============================================================ | ||
| 417 | +print("[信息] Step D: 调用allocate_eid.sh分配EID/IP地址...") | ||
| 418 | + | ||
| 419 | +if os.path.isfile(ALLOCATE_EID_SCRIPT): | ||
| 420 | + result = subprocess.run(['bash', ALLOCATE_EID_SCRIPT, '-d', OUTPUT_DIR], capture_output=True, text=True) | ||
| 421 | + if result.returncode != 0: | ||
| 422 | + print(f" [警告] EID分配失败: {result.stderr}") | ||
| 423 | + else: | ||
| 424 | + print(result.stdout) | ||
| 425 | +else: | ||
| 426 | + print(f" [警告] allocate_eid.sh脚本不存在: {ALLOCATE_EID_SCRIPT}") | ||
| 427 | + | ||
| 428 | +# ============================================================ | ||
| 429 | +# Step E: 生成servers_info.json | ||
| 430 | +# ============================================================ | ||
| 431 | +print("[信息] Step E: 生成servers_info.json...") | ||
| 432 | + | ||
| 433 | +def extract_soc_version(filename): | ||
| 434 | + match = re.match(r'(ascend\d+)', filename) | ||
| 435 | + if match: | ||
| 436 | + return match.group(1) | ||
| 437 | + return 'unknown' | ||
| 438 | + | ||
| 439 | +server_ip_list = [] | ||
| 440 | + | ||
| 441 | +for sp_id in sorted(sp_servers.keys()): | ||
| 442 | + sp_idx = sp_id | ||
| 443 | + for srv in sp_servers[sp_id]: | ||
| 444 | + srv_idx = srv['id'] | ||
| 445 | + srv_soc_version = srv['soc_version'] | ||
| 446 | + | ||
| 447 | + if not srv_soc_version: | ||
| 448 | + srv_dir = os.path.join(OUTPUT_DIR, f'superpod{sp_id}', f'server{srv_idx}') | ||
| 449 | + soc_version = 'unknown' | ||
| 450 | + if os.path.isdir(srv_dir): | ||
| 451 | + for f in os.listdir(srv_dir): | ||
| 452 | + if 'rootinfo' in f and f.endswith('.json'): | ||
| 453 | + soc_version = extract_soc_version(f) | ||
| 454 | + break | ||
| 455 | + if soc_version == 'unknown': | ||
| 456 | + topo_json_path = os.path.join(srv_dir, 'topo.json') | ||
| 457 | + if os.path.isfile(topo_json_path): | ||
| 458 | + try: | ||
| 459 | + with open(topo_json_path, 'r') as tf: | ||
| 460 | + topo_data = json.load(tf) | ||
| 461 | + soc_version = topo_data.get('soc_version', 'unknown') | ||
| 462 | + except Exception: | ||
| 463 | + pass | ||
| 464 | + srv_soc_version = soc_version | ||
| 465 | + | ||
| 466 | + aa = 192 + sp_idx | ||
| 467 | + bb = srv_idx + 1 | ||
| 468 | + host_ip = f"{aa}.{bb}.0.0" | ||
| 469 | + host_eid = ip_to_eid_string(host_ip) | ||
| 470 | + | ||
| 471 | + server_ip_list.append({ | ||
| 472 | + "soc_version": srv_soc_version, | ||
| 473 | + "addr": host_eid | ||
| 474 | + }) | ||
| 475 | + | ||
| 476 | +servers_info = { | ||
| 477 | + "version": "2.0", | ||
| 478 | + "server_count": len(server_ip_list), | ||
| 479 | + "server_ip_list": server_ip_list | ||
| 480 | +} | ||
| 481 | + | ||
| 482 | +output_path = os.path.join(OUTPUT_DIR, 'servers_info.json') | ||
| 483 | +with open(output_path, 'w') as f: | ||
| 484 | + json.dump(servers_info, f, indent=4, ensure_ascii=False) | ||
| 485 | + | ||
| 486 | +print(f" [成功] 生成servers_info.json: {output_path}") | ||
| 487 | +print(f" 服务器数量: {len(server_ip_list)}") | ||
| 488 | +for i, entry in enumerate(server_ip_list): | ||
| 489 | + print(f" [{i}] {entry['soc_version']}: {entry['addr']}") | ||
| 490 | + | ||
| 491 | +PYTHON_SCRIPT | ||
| 492 | + | ||
| 493 | +echo "" | ||
| 494 | +echo "==========================================" | ||
| 495 | +echo " 生成完成" | ||
| 496 | +echo "==========================================" | ||
| 497 | +echo "" | ||
| 498 | +echo "输出目录: $OUTPUT_DIR" | ||
| 499 | +echo "" | ||
| 500 | + | ||
| 501 | +TOTAL_SUPERPODS=$(find "$OUTPUT_DIR" -maxdepth 1 -type d -name "superpod*" | wc -l | tr -d ' ') | ||
| 502 | +TOTAL_SERVERS=$(find "$OUTPUT_DIR" -mindepth 2 -maxdepth 2 -type d -name "server*" | wc -l | tr -d ' ') | ||
| 503 | +TOTAL_FILES=$(find "$OUTPUT_DIR" -type f | wc -l | tr -d ' ') | ||
| 504 | + | ||
| 505 | +echo "超节点目录数: $TOTAL_SUPERPODS" | ||
| 506 | +echo "Server目录数: $TOTAL_SERVERS" | ||
| 507 | +echo "文件总数: $TOTAL_FILES" | ||
| 508 | + | ||
| 509 | +if [[ -f "${OUTPUT_DIR}/servers_info.json" ]]; then | ||
| 510 | + echo "servers_info.json: 已生成" | ||
| 511 | +fi | ||
| 512 | + | ||
| 513 | +echo "" | ||
| 514 | + | ||
| 515 | +echo "目录结构:" | ||
| 516 | +if command -v tree &>/dev/null; then | ||
| 517 | + tree -L 3 "$OUTPUT_DIR" 2>/dev/null || find "$OUTPUT_DIR" -type f | head -30 | ||
| 518 | +else | ||
| 519 | + find "$OUTPUT_DIR" -type f | head -30 | ||
| 520 | +fi | ||
| 521 | + | ||
| 522 | +echo "" | ||
| 523 | +success_msg "集群拓扑目录生成完成！" | ||