Star844
1308
代码
代码
Issues1424
Pull Requests2265
流水线
项目讨论
Wiki
分析
Star844
1308
已合并
feat(libtorch_npu): add HCCL test case for libtorch_npu #31741
kuhn7创建于 3月12日
feat(libtorch_npu): add HCCL test case for libtorch_npu #31741
已合并
kuhn7创建于 3月12日
从kuhn7/pytorchv2.8.0_libtorch_ut_for_hccl合入到Ascend/pytorchv2.8.0
共 8 个文件变更+305-9
Mbuild_libtorch_npu.py+3-2
@@ -228,9 +228,10 @@ def copy_hpp():
228 "torch_npu/csrc/libs/*.h",228 "torch_npu/csrc/libs/*.h",
229 "torch_npu/csrc/inductor/**/*.h",229 "torch_npu/csrc/inductor/**/*.h",
230 "torch_npu/csrc/distributed/*.h",230 "torch_npu/csrc/distributed/*.h",
231- "torch_npu/csrc/distributed/*/*.h",
232 "torch_npu/csrc/distributed/*.hpp",231 "torch_npu/csrc/distributed/*.hpp",
233- "torch_npu/csrc/distributed/*/*.hpp",232+ "third_party/acl/inc/*/*.h",
233+ "third_party/acl/inc/*/*/*.h",
234+ "third_party/hccl/inc/*/*.h",
234 ]235 ]
235 glob_header_files = []236 glob_header_files = []
236 for regex_pattern in header_files:237 for regex_pattern in header_files:
Aexamples/libtorch_hccl/CMakeLists.txt+76-0
@@ -0,0 +1,76 @@
1+cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
2+ 
3+project(HCCLAllreduceExample CXX C)
4+ 
5+set(CMAKE_CXX_STANDARD 17)
6+set(CMAKE_C_STANDARD 11)
7+set(CMAKE_CXX_EXTENSIONS OFF)
8+ 
9+# 定义构建类型
10+IF(CMAKE_BUILD_TYPE MATCHES Debug)
11+ message("Debug构建")
12+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_DEBUG")
13+ELSEIF(CMAKE_BUILD_TYPE MATCHES RelWithDebInfo)
14+ message("RelWithDebInfo构建")
15+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
16+ELSE()
17+ message("Release构建")
18+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
19+ENDIF()
20+ 
21+SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")
22+SET(CMAKE_CXX_FLAGS_RELEASE "-O2")
23+SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
24+ 
25+# 自动查找libtorch路径
26+# 优先使用环境变量，如果未设置则自动从Python获取
27+if(DEFINED ENV{TORCH_INSTALL_DIR})
28+ set(torch_path "$ENV{TORCH_INSTALL_DIR}")
29+else()
30+ execute_process(
31+ COMMAND ${PYTHON_EXECUTABLE} -c "import torch; import os; print(os.path.dirname(torch.__file__))"
32+ OUTPUT_VARIABLE torch_path
33+ OUTPUT_STRIP_TRAILING_WHITESPACE
34+ ERROR_QUIET
35+ )
36+endif()
37+ 
38+if(NOT EXISTS ${torch_path})
39+ message(FATAL_ERROR "未找到libtorch安装目录，请设置环境变量TORCH_INSTALL_DIR或确保Python环境中已安装torch")
40+endif()
41+message(STATUS "libtorch路径: ${torch_path}")
42+include_directories(${torch_path}/include)
43+# c10d/ProcessGroup.hpp
44+include_directories(${torch_path}/include/torch/csrc/distributed)
45+# torch/torch.h
46+include_directories(${torch_path}/include/torch/csrc/api/include)
47+link_directories(${torch_path}/lib)
48+ 
49+ 
50+# 使用环境变量查找libtorch_npu路径
51+if(DEFINED ENV{TORCH_NPU_INSTALL_DIR})
52+ set(torch_npu_path "$ENV{TORCH_NPU_INSTALL_DIR}")
53+endif()
54+ 
55+ 
56+if(NOT EXISTS ${torch_npu_path})
57+ message(WARNING "未找到libtorch_npu安装目录，请设置环境变量TORCH_NPU_INSTALL_DIR")
58+else()
59+ message(STATUS "libtorch_npu路径: ${torch_npu_path}")
60+ include_directories(${torch_npu_path}/include)
61+ link_directories(${torch_npu_path}/lib)
62+endif()
63+ 
64+# arm环境构建依赖torch.libs/*.so库，需要手动加载
65+# link_directories(<python path>/site-packages/torch.libs)
66+ 
67+# 添加可执行文件
68+add_executable(example_allreduce_hccl allreduce_hccl.cpp)
69+ 
70+# 链接库
71+target_link_libraries(example_allreduce_hccl
72+ -ltorch
73+ -ltorch_cpu
74+ -lc10
75+ -ltorch_npu
76+)
Aexamples/libtorch_hccl/README_HCCL.md+85-0
@@ -0,0 +1,85 @@
1+# HCCL AllReduce 示例
2+ 
3+这是一个独立的HCCL集合通信测试用例，用于验证NPU设备上的ProcessGroupHCCL功能。
4+ 
5+## 文件说明
6+ 
7+- `allreduce_hccl.cpp` - HCCL allreduce示例代码
8+- `CMakeLists.txt` - 独立的CMake构建文件
9+- `libtorch_hccl.sh` - 运行脚本
10+ 
11+## 构建
12+ 
13+### 1. 环境准备
14+ 
15+CMakeLists.txt会自动从Python环境检测libtorch的安装路径。
16+ 
17+**自动检测**（推荐）:
18+- 确保Python环境中已安装torch
19+- CMake会自动从Python获取路径
20+ 
21+**手动指定**（可选）:
22+```bash
23+# 如果自动检测失败，可以手动设置环境变量
24+export TORCH_INSTALL_DIR=/path/to/torch # 通常在 site-packages/torch
25+export TORCH_NPU_INSTALL_DIR=/path/to/torch_npu
26+```
27+ 
28+### 2. 构建示例
29+ 
30+```bash
31+cd examples/libtorch_hccl
32+mkdir build && cd build
33+cmake ..
34+make
35+```
36+ 
37+## 运行
38+ 
39+### 方法:
40+ 
41+```bash
42+cd examples/libtorch_hccl
43+chmod +x libtorch_hccl.sh
44+./libtorch_hccl.sh
45+```
46+ 
47+自定义进程数:
48+```bash
49+NUM_PROCESSES=4 ./libtorch_hccl.sh
50+```
51+ 
52+## 预期输出
53+ 
54+```
55+启动HCCL allreduce示例: rank=0, size=2
56+NPU设备 0 初始化完成
57+在NPU设备 0 上创建了 10 个张量
58+已提交 10 个allreduce操作
59+所有操作已完成!
60+张量 0 第一个元素: 2.0
61+张量 1 第一个元素: 2.0
62+张量 2 第一个元素: 2.0
63+HCCL allreduce示例运行成功!
64+```
65+ 
66+值 `2.0` 是2个进程对全1张量进行allreduce求和的结果 (1+1=2)。
67+ 
68+## 代码说明
69+ 
70+示例演示了以下关键步骤：
71+ 
72+1. **NPU设备初始化**: 使用 `torch_npu::init_npu("npu:0")` 初始化设备
73+2. **创建ProcessGroupHCCL**: 用于NPU集合通信
74+3. **创建张量**: 在NPU设备上创建张量
75+4. **执行allreduce**: 对张量进行全局归约操作
76+5. **验证结果**: 检查归约结果是否正确
77+6. **资源清理**: 调用 `torch_npu::finalize_npu()` 释放资源
78+ 
79+## 注意事项
80+ 
81+- 确保NPU设备可用且CANN已正确安装
82+- 使用NPU设备结束后必须调用 `torch_npu::finalize_npu()` 释放资源
83+- 所有进程需要同时启动才能正常工作
84+- libtorch通常安装在Python的site-packages/torch目录下
85+- libtorch_npu通常使用build_libtorch_npu.py脚本构建安装
Aexamples/libtorch_hccl/allreduce_hccl.cpp+87-0
@@ -0,0 +1,87 @@
1+#include <iostream>
2+#include <cstdlib>
3+#include <string>
4+#include <c10/util/irange.h>
5+#include <torch/csrc/distributed/c10d/FileStore.hpp>
6+#include <torch_npu/csrc/distributed/ProcessGroupHCCL.hpp>
7+#include <torch/torch.h>
8+#include "torch_npu/torch_npu.h"
9+ 
10+using namespace c10d;
11+using namespace c10d_npu;
12+ 
13+const int g_rank = []() {
14+ const char* env = std::getenv("RANK");
15+ return env ? std::stoi(std::string(env)) : 0;
16+}();
17+ 
18+const int g_size = []() {
19+ const char* env = std::getenv("SIZE");
20+ return env ? std::stoi(std::string(env)) : 1;
21+}();
22+ 
23+int main(int argc, char** argv)
24+{
25+ int rank = g_rank;
26+ int size = g_size;
27+
28+ std::cout << "启动 HCCL allreduce 示例: rank=" << rank << ", size=" << size << std::endl;
29+
30+ // 初始化NPU设备 - 使用npu字符串格式
31+ std::string device_str = "npu:" + std::to_string(rank);
32+ torch_npu::init_npu(device_str);
33+ std::cout << "NPU设备 " << rank << " 初始化完成" << std::endl;
34+
35+ // 创建FileStore用于进程间通信协调
36+ auto store = c10::make_intrusive<FileStore>("/tmp/c10d_hccl_example", size);
37+
38+ // 创建ProcessGroupHCCL选项
39+ auto options = ProcessGroupHCCL::Options::create();
40+
41+ // 创建ProcessGroupHCCL实例
42+ auto pg = c10::make_intrusive<ProcessGroupHCCL>(store, rank, size, options);
43+
44+ // 通过传NPU字符串构造NPU设备
45+ auto device = at::Device(device_str);
46+
47+ // 创建10个张量用于测试
48+ const auto ntensors = 10;
49+ std::vector<at::Tensor> tensors;
50+
51+ for (const auto i : c10::irange(ntensors)) {
52+ // 在NPU设备上创建全1张量
53+ auto x = at::ones({1000, 16 * (i + 1)}, at::TensorOptions(device).dtype(at::kFloat));
54+ tensors.push_back(x);
55+ }
56+
57+ std::cout << "在NPU设备 " << rank << " 上创建了 " << ntensors << " 个张量" << std::endl;
58+
59+ // 提交所有allreduce操作
60+ std::vector<c10::intrusive_ptr<Work>> pending;
61+ for (const auto i : c10::irange(ntensors)) {
62+ std::vector<at::Tensor> tmp = {tensors[i]};
63+ pending.push_back(pg->allreduce(tmp));
64+ }
65+
66+ std::cout << "已提交 " << ntensors << " 个allreduce操作" << std::endl;
67+
68+ // 等待所有操作完成
69+ for (auto& work : pending) {
70+ work->wait();
71+ }
72+
73+ std::cout << "所有操作已完成!" << std::endl;
74+
75+ // 验证结果 - 打印前3个张量的第一个元素
76+ for (const auto i : c10::irange(std::min(ntensors, 3))) {
77+ auto cpu_tensor = tensors[i].to(at::kCPU);
78+ std::cout << "张量 " << i << " 第一个元素: " << cpu_tensor.data_ptr<float>()[0] << std::endl;
79+ }
80+
81+ std::cout << "HCCL allreduce示例运行成功!" << std::endl;
82+
83+ // 使用NPU设备结束需进行反初始化
84+ torch_npu::finalize_npu();
85+
86+ return 0;
87+}
Aexamples/libtorch_hccl/libtorch_hccl.sh+44-0
@@ -0,0 +1,44 @@
1+#!/bin/bash
2+ 
3+set -e
4+ 
5+# 默认使用2个进程
6+NUM_PROCESSES=${NUM_PROCESSES:-2}
7+ 
8+# 获取脚本所在目录
9+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
10+ 
11+# 检查二进制文件是否存在
12+if [ ! -f "${SCRIPT_DIR}/build/example_allreduce_hccl" ]; then
13+ echo "错误: 未找到example_allreduce_hccl二进制文件"
14+ echo "请先构建项目:"
15+ echo " cd ${SCRIPT_DIR}"
16+ echo " mkdir build && cd build"
17+ echo " cmake .."
18+ echo " make"
19+ exit 1
20+fi
21+ 
22+echo "运行HCCL allreduce示例，进程数: ${NUM_PROCESSES}"
23+echo "二进制文件: ${SCRIPT_DIR}/build/example_allreduce_hccl"
24+ 
25+# 清理之前的临时文件
26+rm -f /tmp/c10d_hccl_example
27+ 
28+# 启动多个进程
29+pids=()
30+for ((rank=0; rank<NUM_PROCESSES; rank++)); do
31+ echo "启动进程 ${rank}"
32+ RANK=${rank} SIZE=${NUM_PROCESSES} \
33+ "${SCRIPT_DIR}/build/example_allreduce_hccl" &
34+ pids+=($!)
35+done
36+ 
37+# 等待所有进程完成
38+for pid in "${pids[@]}"; do
39+ wait ${pid}
40+done
41+ 
42+echo "所有进程已完成!"
43+# 清理临时文件
44+rm -f /tmp/c10d_hccl_example
Mtorch_npu/csrc/distributed/ProcessGroupHCCL.cpp+0-4
@@ -57,9 +57,7 @@
57#ifndef BUILD_LIBTORCH57#ifndef BUILD_LIBTORCH
58#include "torch_npu/csrc/toolkit/profiler/common/utils.h"58#include "torch_npu/csrc/toolkit/profiler/common/utils.h"
59#include "torch_npu/csrc/profiler/npu_profiler.h"59#include "torch_npu/csrc/profiler/npu_profiler.h"
60-#endif
61 60 
62-#ifndef BUILD_LIBTORCH
63namespace py = pybind11;61namespace py = pybind11;
64using namespace py::literals;62using namespace py::literals;
65#endif63#endif
@@ -3455,11 +3453,9 @@ std::string ProcessGroupHCCL::getMstxHcclMsg(
3455 {HCCL_DATA_TYPE_BFP16, "bfp16"}3453 {HCCL_DATA_TYPE_BFP16, "bfp16"}
3456 };3454 };
3457 static std::map<HcclComm, std::string> commNames;3455 static std::map<HcclComm, std::string> commNames;
3458-#ifndef BUILD_LIBTORCH
3459 if (!torch_npu::profiler::mstxEnable()) {3456 if (!torch_npu::profiler::mstxEnable()) {
3460 return "";3457 return "";
3461 }3458 }
3462-#endif
3463 std::unordered_map<std::string, std::string> msgDict;3459 std::unordered_map<std::string, std::string> msgDict;
3464 msgDict["opName"] = opName;3460 msgDict["opName"] = opName;
3465 auto nameIter = commNames.find(comm);3461 auto nameIter = commNames.find(comm);
Mtorch_npu/csrc/distributed/ProcessGroupHCCL.hpp+6-1
@@ -6,6 +6,9 @@
6#include <variant>6#include <variant>
7#include <future>7#include <future>
8#include <atomic>8#include <atomic>
9+#include <sys/stat.h>
10+#include <unistd.h>
11+ 
9#include <c10d/ProcessGroup.hpp>12#include <c10d/ProcessGroup.hpp>
10#include <c10d/Store.hpp>13#include <c10d/Store.hpp>
11#include <c10d/Utils.hpp>14#include <c10d/Utils.hpp>
@@ -14,7 +17,7 @@
14#include "third_party/hccl/inc/hccl/hccl.h"17#include "third_party/hccl/inc/hccl/hccl.h"
15#include "torch_npu/csrc/core/npu/interface/HcclInterface.h"18#include "torch_npu/csrc/core/npu/interface/HcclInterface.h"
16#include "torch_npu/csrc/distributed/HCCLUtils.hpp"19#include "torch_npu/csrc/distributed/HCCLUtils.hpp"
17-#include "torch_npu/csrc/npu/Event.h"20+#include "torch_npu/csrc/core/npu/NPUEvent.h"
AtlasAccount
AtlasAccountAtlasAccount3月12日
代码结构与可维护性: 将头文件"torch_npu/csrc/npu/Event.h"更改为"torch_npu/csrc/core/npu/NPUEvent.h"。虽然这是路径修正，但需要确保新路径下的头文件内容与原有头文件兼容，特别是NPUEvent类的接口是否一致。如果接口有变化，可能影响代码中多处使用NPUEvent的地方，例如hcclStartEvents_和hcclEndEvents_成员的类型定义。
问题类型: 代码结构与可维护性
文件路径: torch_npu/csrc/distributed/ProcessGroupHCCL.hpp
行号: 20
问题代码:
#include "torch_npu/csrc/core/npu/NPUEvent.h"
修改建议:
1. 确认新头文件中的NPUEvent类与原有头文件中的Event类接口一致。2. 如果接口有差异，需要同步修改所有使用NPUEvent的代码，例如WorkHCCL类中的hcclStartEvents_和hcclEndEvents_成员类型。3. 建议在修改后运行完整的编译和测试，确保没有因头文件变更引入的编译错误或行为变化。
---
此评论由代码审查工具自动生成
likedislike
18 21 
19 22 
20namespace c10d_npu {23namespace c10d_npu {
@@ -104,6 +107,7 @@ struct ProcessGroupStatus {
104 size_t lastStartedNumelOut;107 size_t lastStartedNumelOut;
105};108};
106 109 
110+#ifndef BUILD_LIBTORCH
107struct DumpPipe {111struct DumpPipe {
108 DumpPipe(int rank)112 DumpPipe(int rank)
109 {113 {
@@ -148,6 +152,7 @@ struct DumpPipe {
148private:152private:
149 int fd_ = -1;153 int fd_ = -1;
150};154};
155+#endif
151 156 
152// A shelf for stashing tensors between op call and `work.wait()`.157// A shelf for stashing tensors between op call and `work.wait()`.
153// Used in case of async ops.158// Used in case of async ops.
Mtorch_npu/csrc/distributed/TraceUtils.h+4-2
@@ -392,11 +392,13 @@ DEFINE_CONSTANT(started_state, "started")
392 // Current pg_status is not in FR.392 // Current pg_status is not in FR.
393 all_pg_status_[pg_id] = std::move(pg_status);393 all_pg_status_[pg_id] = std::move(pg_status);
394 }394 }
395- std::lock_guard<std::mutex> guard(mutex_);
396- 
397#ifndef BUILD_LIBTORCH395#ifndef BUILD_LIBTORCH
398 auto traceback =396 auto traceback =
399 torch::CapturedTraceback::gather(true, true, capture_cpp_stack_);397 torch::CapturedTraceback::gather(true, true, capture_cpp_stack_);
398+#endif
399+ std::lock_guard<std::mutex> guard(mutex_);
400+ 
401+#ifndef BUILD_LIBTORCH
400 auto te = Entry{402 auto te = Entry{
401 id_,403 id_,
402 pg_id,404 pg_id,