Star857
1352
代码
代码
Issues1684
Pull Requests2229
项目讨论
Wiki
分析
Star857
1352
已合并
libtorch_npu.so support hccl api #31394
kuhn7创建于 3月4日
libtorch_npu.so support hccl api #31394
已合并
kuhn7创建于 3月4日
从kuhn7/pytorchv2.7.1_hccl_libtorch_v2合入到Ascend/pytorchv2.7.1
共 6 个文件变更+132-8
MCMakeLists.txt+3-3
@@ -231,9 +231,9 @@ set(CORE_SRCS)
231set(FRAMEWORK_SRCS)231set(FRAMEWORK_SRCS)
232set(LOGGING_SRCS)232set(LOGGING_SRCS)
233set(INDUCTOR_SRCS)233set(INDUCTOR_SRCS)
234+set(DIST_SRCS)
234 235 
235if (NOT DEFINED BUILD_LIBTORCH)236if (NOT DEFINED BUILD_LIBTORCH)
236- set(DIST_SRCS)
237 set(FLOP_SRCS)237 set(FLOP_SRCS)
238 set(NPU_SRCS)238 set(NPU_SRCS)
239 set(PROF_SRCS)239 set(PROF_SRCS)
@@ -254,9 +254,9 @@ add_subdirectory(${TORCHNPU_ROOT}/flopcount)
254add_subdirectory(${TORCHNPU_ROOT}/logging)254add_subdirectory(${TORCHNPU_ROOT}/logging)
255add_subdirectory(${TORCHNPU_ROOT}/custom_dtype)255add_subdirectory(${TORCHNPU_ROOT}/custom_dtype)
256add_subdirectory(${TORCHNPU_ROOT}/inductor)256add_subdirectory(${TORCHNPU_ROOT}/inductor)
257+add_subdirectory(${TORCHNPU_ROOT}/distributed)
257 258 
258if (NOT DEFINED BUILD_LIBTORCH)259if (NOT DEFINED BUILD_LIBTORCH)
259- add_subdirectory(${TORCHNPU_ROOT}/distributed)
260 add_subdirectory(${TORCHNPU_ROOT}/npu)260 add_subdirectory(${TORCHNPU_ROOT}/npu)
261 add_subdirectory(${TORCHNPU_ROOT}/profiler)261 add_subdirectory(${TORCHNPU_ROOT}/profiler)
262 add_subdirectory(${TORCHNPU_ROOT}/ipc)262 add_subdirectory(${TORCHNPU_ROOT}/ipc)
@@ -282,7 +282,7 @@ if (DEFINED BUILD_TENSORPIPE)
282endif()282endif()
283 283 
284if (DEFINED BUILD_LIBTORCH)284if (DEFINED BUILD_LIBTORCH)
285- set(CPP_SRCS ${ATEN_SRCS} ${INDUCTOR_SRCS} ${CORE_SRCS} ${OPS_PLUGIN_SRCS} ${FLOP_SRCS} ${CUS_DTYPE_SRCS} ${FRAMEWORK_SRCS} ${LOGGING_SRCS} ${NPU_CPP_LIBS_SRCS})285+ set(CPP_SRCS ${ATEN_SRCS} ${INDUCTOR_SRCS} ${CORE_SRCS} ${OPS_PLUGIN_SRCS} ${DIST_SRCS} ${FLOP_SRCS} ${CUS_DTYPE_SRCS} ${FRAMEWORK_SRCS} ${LOGGING_SRCS} ${NPU_CPP_LIBS_SRCS} )
286else()286else()
287# Compile code with pybind11287# Compile code with pybind11
288 set(CPP_SRCS ${ATEN_SRCS} ${INDUCTOR_SRCS} ${CORE_SRCS} ${OPS_PLUGIN_SRCS} ${DIST_SRCS} ${FLOP_SRCS} ${CUS_DTYPE_SRCS} ${LOGGING_SRCS} ${FRAMEWORK_SRCS} ${NPU_SRCS} ${PROF_SRCS} ${IPC_SRCS} ${UTILS_SRCS} ${SAN_SRCS} ${AFD_SRCS})288 set(CPP_SRCS ${ATEN_SRCS} ${INDUCTOR_SRCS} ${CORE_SRCS} ${OPS_PLUGIN_SRCS} ${DIST_SRCS} ${FLOP_SRCS} ${CUS_DTYPE_SRCS} ${LOGGING_SRCS} ${FRAMEWORK_SRCS} ${NPU_SRCS} ${PROF_SRCS} ${IPC_SRCS} ${UTILS_SRCS} ${SAN_SRCS} ${AFD_SRCS})
Mbuild_libtorch_npu.py+4-0
@@ -227,6 +227,10 @@ def copy_hpp():
227 "torch_npu/csrc/framework/*/*/*.h",227 "torch_npu/csrc/framework/*/*/*.h",
228 "torch_npu/csrc/libs/*.h",228 "torch_npu/csrc/libs/*.h",
229 "torch_npu/csrc/inductor/**/*.h",229 "torch_npu/csrc/inductor/**/*.h",
230+ "torch_npu/csrc/distributed/*.h",
231+ "torch_npu/csrc/distributed/*/*.h",
H
Hhuangyunlong3月5日

如果rpc/symm_mem暂不提供，对应的头文件也不需要

likedislike
K
kuhn7
3月5日 评论：
232+ "torch_npu/csrc/distributed/*.hpp",
233+ "torch_npu/csrc/distributed/*/*.hpp",
230 ]234 ]
231 glob_header_files = []235 glob_header_files = []
232 for regex_pattern in header_files:236 for regex_pattern in header_files:
Mtorch_npu/csrc/distributed/CMakeLists.txt+7-1
@@ -1,4 +1,10 @@
1-FILE(GLOB _DIST_SRCS *.cpp rpc/*.cpp symm_mem/*.cpp)1+if (DEFINED BUILD_LIBTORCH)
2+ FILE(GLOB _DIST_SRCS *.cpp)
3+ # Exclude Python binding files when building libtorch
4+ list(REMOVE_ITEM _DIST_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/Init.cpp")
5+else()
6+ FILE(GLOB _DIST_SRCS *.cpp rpc/*.cpp symm_mem/*.cpp)
7+endif()
2 8 
3LIST(APPEND DIST_SRCS ${_DIST_SRCS})9LIST(APPEND DIST_SRCS ${_DIST_SRCS})
4 10 
Mtorch_npu/csrc/distributed/ProcessGroupHCCL.cpp+87-2
@@ -10,9 +10,11 @@
10#include <cstdlib>10#include <cstdlib>
11#include <linux/limits.h>11#include <linux/limits.h>
12 12 
13+#ifndef BUILD_LIBTORCH
13#include <pybind11/pybind11.h>14#include <pybind11/pybind11.h>
14#include <pybind11/eval.h>15#include <pybind11/eval.h>
15#include <pybind11/embed.h>16#include <pybind11/embed.h>
17+#endif
16 18 
17#include <c10/util/Optional.h>19#include <c10/util/Optional.h>
18#include <c10/util/irange.h>20#include <c10/util/irange.h>
@@ -46,16 +48,21 @@
46#include "torch_npu/csrc/distributed/HCCLUtils.hpp"48#include "torch_npu/csrc/distributed/HCCLUtils.hpp"
47#include "torch_npu/csrc/distributed/HcclCompile.h"49#include "torch_npu/csrc/distributed/HcclCompile.h"
48#include "torch_npu/csrc/distributed/TraceUtils.h"50#include "torch_npu/csrc/distributed/TraceUtils.h"
49-#include "torch_npu/csrc/toolkit/profiler/common/utils.h"
50#include "torch_npu/csrc/framework/OpHook.h"51#include "torch_npu/csrc/framework/OpHook.h"
51#include "torch_npu/csrc/framework/FormatHelper.h"52#include "torch_npu/csrc/framework/FormatHelper.h"
52#include "torch_npu/csrc/framework/utils/OpPreparation.h"53#include "torch_npu/csrc/framework/utils/OpPreparation.h"
53-#include "torch_npu/csrc/profiler/npu_profiler.h"
54#include "torch_npu/csrc/logging/LogContext.h"54#include "torch_npu/csrc/logging/LogContext.h"
55#include "torch_npu/csrc/distributed/ProcessGroupHCCL.hpp"55#include "torch_npu/csrc/distributed/ProcessGroupHCCL.hpp"
56 56 
57+#ifndef BUILD_LIBTORCH
58+#include "torch_npu/csrc/toolkit/profiler/common/utils.h"
59+#include "torch_npu/csrc/profiler/npu_profiler.h"
60+#endif
61+ 
62+#ifndef BUILD_LIBTORCH
57namespace py = pybind11;63namespace py = pybind11;
H
Hhuangyunlong3月5日

这个和上面的头文件在一起的话，可以考虑用一个宏

likedislike
K
kuhn7
3月5日 评论：
58using namespace py::literals;64using namespace py::literals;
65+#endif
59 66 
60namespace c10d_npu {67namespace c10d_npu {
61namespace {68namespace {
@@ -1376,6 +1383,7 @@ ProcessGroupHCCL::~ProcessGroupHCCL()
1376 TORCH_NPU_HCCL_LOGI("process group destroyed, group id is %s.", options_->group_id.c_str());1383 TORCH_NPU_HCCL_LOGI("process group destroyed, group id is %s.", options_->group_id.c_str());
1377}1384}
1378 1385 
1386+#ifndef BUILD_LIBTORCH
1379std::future<bool> ProcessGroupHCCL::launchAsyncPythonTracebackDump()1387std::future<bool> ProcessGroupHCCL::launchAsyncPythonTracebackDump()
1380{1388{
1381 std::promise<bool> resultPromise;1389 std::promise<bool> resultPromise;
@@ -1485,6 +1493,7 @@ void ProcessGroupHCCL::dumpTraceAndResetStatus()
1485 heartbeat_++;1493 heartbeat_++;
1486 shouldDump_.store(false);1494 shouldDump_.store(false);
1487}1495}
1496+#endif
1488 1497 
1489void ProcessGroupHCCL::terminateProcess(std::string errMsg)1498void ProcessGroupHCCL::terminateProcess(std::string errMsg)
1490{1499{
@@ -1501,6 +1510,7 @@ int computeDeltaMS(
1501 .count();1510 .count();
1502}1511}
1503 1512 
1513+#ifndef BUILD_LIBTORCH
1504void ProcessGroupHCCL::heartbeatMonitor()1514void ProcessGroupHCCL::heartbeatMonitor()
1505{1515{
1506 uint64_t heartBeatCounter = 0ULL;1516 uint64_t heartBeatCounter = 0ULL;
@@ -1734,6 +1744,7 @@ void ProcessGroupHCCL::heartbeatMonitor()
1734 }1744 }
1735 }1745 }
1736}1746}
1747+#endif
1737 1748 
1738ProcessGroupHCCL::Watchdog::Watchdog(ProcessGroupHCCL *pg)1749ProcessGroupHCCL::Watchdog::Watchdog(ProcessGroupHCCL *pg)
1739{1750{
@@ -1768,9 +1779,11 @@ void ProcessGroupHCCL::Watchdog::run()
1768 c10_npu::SetThreadType(c10_npu::ThreadType::WATCHDOG_THREAD);1779 c10_npu::SetThreadType(c10_npu::ThreadType::WATCHDOG_THREAD);
1769 try {1780 try {
1770 LOG(INFO) << "[Rank " << rank_ << "] HCCL watchdog thread started!";1781 LOG(INFO) << "[Rank " << rank_ << "] HCCL watchdog thread started!";
1782+#ifndef BUILD_LIBTORCH
1771 if (ProcessGroupHCCL::monitorThreadEnabled_.load()) {1783 if (ProcessGroupHCCL::monitorThreadEnabled_.load()) {
1772 pg_->hcclHeartbeatMonitorThread_ = std::thread(&ProcessGroupHCCL::heartbeatMonitor, pg_);1784 pg_->hcclHeartbeatMonitorThread_ = std::thread(&ProcessGroupHCCL::heartbeatMonitor, pg_);
1773 }1785 }
1786+#endif
1774 runLoop();1787 runLoop();
1775 LOG(INFO) << "[Rank " << rank_ << "] HCCL watchdog thread terminated normally";1788 LOG(INFO) << "[Rank " << rank_ << "] HCCL watchdog thread terminated normally";
1776 } catch (std::exception& e) {1789 } catch (std::exception& e) {
@@ -2650,10 +2663,20 @@ bool ProcessGroupHCCL::createHCCLCommEx(
2650 hcclid = (std::hash<string>{}(devicesKey));2663 hcclid = (std::hash<string>{}(devicesKey));
2651 std::string p2pName = "group" + options_->group_id + "_p2p_" + std::to_string(lowRank) + "_" + std::to_string(highRank);2664 std::string p2pName = "group" + options_->group_id + "_p2p_" + std::to_string(lowRank) + "_" + std::to_string(highRank);
2652 if (strlen(commConfig->hcclCommName) > 0) {2665 if (strlen(commConfig->hcclCommName) > 0) {
2666+#ifndef BUILD_LIBTORCH
2653 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(commConfig->hcclCommName, p2pName.c_str(), COMM_NAME_MAX_LENGTH);2667 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(commConfig->hcclCommName, p2pName.c_str(), COMM_NAME_MAX_LENGTH);
2668+#else
2669+ strncpy(commConfig->hcclCommName, p2pName.c_str(), COMM_NAME_MAX_LENGTH - 1);
2670+ commConfig->hcclCommName[COMM_NAME_MAX_LENGTH - 1] = '\0';
H
Hhuangyunlong3月5日

这里建议用安全拷贝函数

likedislike
K
kuhn7
3月5日 评论：
2671+#endif
2654 }2672 }
2655 if (strlen(commConfig->hcclUdi) > 0) {2673 if (strlen(commConfig->hcclUdi) > 0) {
2674+#ifndef BUILD_LIBTORCH
2656 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(commConfig->hcclUdi, p2pName.c_str(), UDI_MAX_LENGTH);2675 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(commConfig->hcclUdi, p2pName.c_str(), UDI_MAX_LENGTH);
2676+#else
2677+ strncpy(commConfig->hcclUdi, p2pName.c_str(), COMM_NAME_MAX_LENGTH - 1);
2678+ commConfig->hcclUdi[COMM_NAME_MAX_LENGTH - 1] = '\0';
2679+#endif
2657 }2680 }
2658 subComm = HCCLComm::createSubHcclComm(globalHcclComm, numRanks, p2pRanks.data(), hcclid, rank, commConfig);2681 subComm = HCCLComm::createSubHcclComm(globalHcclComm, numRanks, p2pRanks.data(), hcclid, rank, commConfig);
2659 } else {2682 } else {
@@ -3349,8 +3372,13 @@ std::string ProcessGroupHCCL::getHcclCommName(int rankid, bool init_comm)
3349 hcclCommName = devHCCLCommNameMap_[key];3372 hcclCommName = devHCCLCommNameMap_[key];
3350 }3373 }
3351 if (!hcclCommName.empty()) {3374 if (!hcclCommName.empty()) {
3375+#ifndef BUILD_LIBTORCH
3352 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(config.hcclCommName, hcclCommName.c_str(),3376 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(config.hcclCommName, hcclCommName.c_str(),
3353 COMM_NAME_MAX_LENGTH);3377 COMM_NAME_MAX_LENGTH);
3378+#else
3379+ strncpy(config.hcclCommName, hcclCommName.c_str(), COMM_NAME_MAX_LENGTH - 1);
3380+ config.hcclCommName[COMM_NAME_MAX_LENGTH - 1] = '\0';
3381+#endif
3354 }3382 }
3355 std::vector <std::shared_ptr<HCCLComm>> hcclComms = getHCCLComm(key, devices, HcclCommType::DEFAULT, &config);3383 std::vector <std::shared_ptr<HCCLComm>> hcclComms = getHCCLComm(key, devices, HcclCommType::DEFAULT, &config);
3356 3384 
@@ -3389,6 +3417,7 @@ std::string mapToJson(const std::unordered_map<std::string, std::string>& map)
3389 return ss.str();3417 return ss.str();
3390}3418}
3391 3419 
3420+#ifndef BUILD_LIBTORCH
3392std::string ProcessGroupHCCL::getMstxHcclMsg(3421std::string ProcessGroupHCCL::getMstxHcclMsg(
3393 const std::string &opName, uint64_t dataCnt, HcclDataType dataType, HcclComm comm, int64_t streamId,3422 const std::string &opName, uint64_t dataCnt, HcclDataType dataType, HcclComm comm, int64_t streamId,
3394 int srcRank, int dstRank)3423 int srcRank, int dstRank)
@@ -3408,9 +3437,11 @@ std::string ProcessGroupHCCL::getMstxHcclMsg(
3408 {HCCL_DATA_TYPE_BFP16, "bfp16"}3437 {HCCL_DATA_TYPE_BFP16, "bfp16"}
3409 };3438 };
3410 static std::map<HcclComm, std::string> commNames;3439 static std::map<HcclComm, std::string> commNames;
3440+#ifndef BUILD_LIBTORCH
3411 if (!torch_npu::profiler::mstxEnable()) {3441 if (!torch_npu::profiler::mstxEnable()) {
3412 return "";3442 return "";
3413 }3443 }
3444+#endif
H
Hhuangyunlong3月5日

如果整个函数隔离了，这个感觉有点冗余了

likedislike
K
kuhn7
3月5日 评论：
3414 std::unordered_map<std::string, std::string> msgDict;3445 std::unordered_map<std::string, std::string> msgDict;
3415 msgDict["opName"] = opName;3446 msgDict["opName"] = opName;
3416 auto nameIter = commNames.find(comm);3447 auto nameIter = commNames.find(comm);
@@ -3439,6 +3470,7 @@ std::string ProcessGroupHCCL::getMstxHcclMsg(
3439 msgDict["streamId"] = std::to_string(streamId);3470 msgDict["streamId"] = std::to_string(streamId);
3440 return mapToJson(msgDict);3471 return mapToJson(msgDict);
3441}3472}
3473+#endif
3442 3474 
3443void ProcessGroupHCCL::silenceCheck(at::Tensor &input, c10d::OpType opType)3475void ProcessGroupHCCL::silenceCheck(at::Tensor &input, c10d::OpType opType)
3444{3476{
@@ -3480,7 +3512,12 @@ HcclCommConfig ProcessGroupHCCL::createHcclCommConfigWithOptions()
3480 if (isHcclFeatureSupported(HcclCommConfigCapability::HCCL_COMM_CONFIG_COMM_NAME)) {3512 if (isHcclFeatureSupported(HcclCommConfigCapability::HCCL_COMM_CONFIG_COMM_NAME)) {
3481 // Update group name in hccl comm config when this capability is supported.3513 // Update group name in hccl comm config when this capability is supported.
3482 std::string groupName = getGroupName();3514 std::string groupName = getGroupName();
3515+#ifndef BUILD_LIBTORCH
3483 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(config.hcclCommName, groupName.c_str(), COMM_NAME_MAX_LENGTH);3516 torch_npu::toolkit::profiler::Utils::safe_strcpy_s(config.hcclCommName, groupName.c_str(), COMM_NAME_MAX_LENGTH);
3517+#else
3518+ strncpy(config.hcclCommName, groupName.c_str(), COMM_NAME_MAX_LENGTH - 1);
3519+ config.hcclCommName[COMM_NAME_MAX_LENGTH - 1] = '\0';
3520+#endif
3484 }3521 }
3485 3522 
3486 if (options_->hccl_config.empty()) {3523 if (options_->hccl_config.empty()) {
@@ -4379,9 +4416,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allreduce(
4379 auto numel = getNumelForHCCL(input);4416 auto numel = getNumelForHCCL(input);
4380 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);4417 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);
4381 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {4418 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {
4419+#ifndef BUILD_LIBTORCH
4382 torch_npu::profiler::MstxRange range(4420 torch_npu::profiler::MstxRange range(
4383 getMstxHcclMsg("HcclAllreduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),4421 getMstxHcclMsg("HcclAllreduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
4384 torch_npu::profiler::DOMAIN_COMMUNICATION);4422 torch_npu::profiler::DOMAIN_COMMUNICATION);
4423+#endif
4385 if (c10_npu::is_core_control_enabled) {4424 if (c10_npu::is_core_control_enabled) {
4386 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4425 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4387 }4426 }
@@ -4468,9 +4507,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::batch_isend_irecv(
4468 remote_rank_list[i]4507 remote_rank_list[i]
4469 };4508 };
4470 }4509 }
4510+#ifndef BUILD_LIBTORCH
4471 torch_npu::profiler::MstxRange range(4511 torch_npu::profiler::MstxRange range(
4472 getMstxHcclMsg("HcclBatchSendRecv", sendRecvInfo[0].count, sendRecvInfo[0].dataType, comm, stream.id(), -1, -1),4512 getMstxHcclMsg("HcclBatchSendRecv", sendRecvInfo[0].count, sendRecvInfo[0].dataType, comm, stream.id(), -1, -1),
4473 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);4513 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
4514+#endif
4474 if (c10_npu::is_core_control_enabled) {4515 if (c10_npu::is_core_control_enabled) {
4475 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4516 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4476 }4517 }
@@ -4518,9 +4559,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::broadcast(
4518 auto numel = getNumelForHCCL(input);4559 auto numel = getNumelForHCCL(input);
4519 auto hcclType = getHcclDataType(input.scalar_type());4560 auto hcclType = getHcclDataType(input.scalar_type());
4520 auto hccl_call = [inputDataPtr, numel, hcclType, root, comm, stream, is_dispatched]() -> int {4561 auto hccl_call = [inputDataPtr, numel, hcclType, root, comm, stream, is_dispatched]() -> int {
4562+#ifndef BUILD_LIBTORCH
4521 torch_npu::profiler::MstxRange range(4563 torch_npu::profiler::MstxRange range(
4522 getMstxHcclMsg("HcclBroadcast", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),4564 getMstxHcclMsg("HcclBroadcast", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
4523 torch_npu::profiler::DOMAIN_COMMUNICATION);4565 torch_npu::profiler::DOMAIN_COMMUNICATION);
4566+#endif
4524 if (c10_npu::is_core_control_enabled) {4567 if (c10_npu::is_core_control_enabled) {
4525 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4568 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4526 }4569 }
@@ -4569,9 +4612,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allreduce_coalesced(
4569 auto numel = getNumelForHCCL(input);4612 auto numel = getNumelForHCCL(input);
4570 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);4613 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);
4571 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {4614 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {
4615+#ifndef BUILD_LIBTORCH
4572 torch_npu::profiler::MstxRange range(4616 torch_npu::profiler::MstxRange range(
4573 getMstxHcclMsg("HcclAllreduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),4617 getMstxHcclMsg("HcclAllreduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
4574 torch_npu::profiler::DOMAIN_COMMUNICATION);4618 torch_npu::profiler::DOMAIN_COMMUNICATION);
4619+#endif
4575 if (c10_npu::is_core_control_enabled) {4620 if (c10_npu::is_core_control_enabled) {
4576 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4621 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4577 }4622 }
@@ -4646,9 +4691,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce(
4646 auto numel = getNumelForHCCL(input);4691 auto numel = getNumelForHCCL(input);
4647 auto reduceOp = getHcclReduceOp(opts.reduceOp, input);4692 auto reduceOp = getHcclReduceOp(opts.reduceOp, input);
4648 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, reduceOp, rank, comm, stream, is_dispatched]() -> int {4693 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, reduceOp, rank, comm, stream, is_dispatched]() -> int {
4694+#ifndef BUILD_LIBTORCH
4649 torch_npu::profiler::MstxRange range(4695 torch_npu::profiler::MstxRange range(
4650 getMstxHcclMsg("HcclReduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),4696 getMstxHcclMsg("HcclReduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
4651 torch_npu::profiler::DOMAIN_COMMUNICATION);4697 torch_npu::profiler::DOMAIN_COMMUNICATION);
4698+#endif
4652 if (c10_npu::is_core_control_enabled) {4699 if (c10_npu::is_core_control_enabled) {
4653 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4700 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4654 }4701 }
@@ -4719,9 +4766,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::_reduce_oop(
4719 auto numel = getNumelForHCCL(input);4766 auto numel = getNumelForHCCL(input);
4720 auto reduceOp = getHcclReduceOp(opts.reduceOp, input);4767 auto reduceOp = getHcclReduceOp(opts.reduceOp, input);
4721 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, reduceOp, rank, comm, stream, is_dispatched]() -> int {4768 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, reduceOp, rank, comm, stream, is_dispatched]() -> int {
4769+#ifndef BUILD_LIBTORCH
4722 torch_npu::profiler::MstxRange range(4770 torch_npu::profiler::MstxRange range(
4723 getMstxHcclMsg("HcclReduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),4771 getMstxHcclMsg("HcclReduce", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
4724 torch_npu::profiler::DOMAIN_COMMUNICATION);4772 torch_npu::profiler::DOMAIN_COMMUNICATION);
4773+#endif
4725 if (c10_npu::is_core_control_enabled) {4774 if (c10_npu::is_core_control_enabled) {
4726 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4775 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4727 }4776 }
@@ -4854,9 +4903,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::_reduce_scatter_base_uneven(
4854 comm,4903 comm,
4855 stream,4904 stream,
4856 is_dispatched]() -> int {4905 is_dispatched]() -> int {
4906+#ifndef BUILD_LIBTORCH
4857 torch_npu::profiler::MstxRange range(4907 torch_npu::profiler::MstxRange range(
4858 getMstxHcclMsg("HcclReduceScatterV", numel, hcclType, comm, stream.id(), -1, -1),4908 getMstxHcclMsg("HcclReduceScatterV", numel, hcclType, comm, stream.id(), -1, -1),
4859 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);4909 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
4910+#endif
4860 if (c10_npu::is_core_control_enabled) {4911 if (c10_npu::is_core_control_enabled) {
4861 c10_npu::UseStreamResInCurrentThread(stream.stream(false));4912 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4862 }4913 }
@@ -4952,9 +5003,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::_allgather_base_uneven(
4952 comm,5003 comm,
4953 stream,5004 stream,
4954 is_dispatched]() -> int {5005 is_dispatched]() -> int {
5006+#ifndef BUILD_LIBTORCH
4955 torch_npu::profiler::MstxRange range(5007 torch_npu::profiler::MstxRange range(
4956 getMstxHcclMsg("HcclAllGatherV", numel, hcclType, comm, stream.id(), -1, -1),5008 getMstxHcclMsg("HcclAllGatherV", numel, hcclType, comm, stream.id(), -1, -1),
4957 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);5009 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
5010+#endif
4958 if (c10_npu::is_core_control_enabled) {5011 if (c10_npu::is_core_control_enabled) {
4959 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5012 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
4960 }5013 }
@@ -5026,9 +5079,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allgather(
5026 auto numel = getNumelForHCCL(input);5079 auto numel = getNumelForHCCL(input);
5027 auto hcclType = getHcclDataType(input.scalar_type());5080 auto hcclType = getHcclDataType(input.scalar_type());
5028 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {5081 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {
5082+#ifndef BUILD_LIBTORCH
5029 torch_npu::profiler::MstxRange range(5083 torch_npu::profiler::MstxRange range(
5030 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5084 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5031 torch_npu::profiler::DOMAIN_COMMUNICATION);5085 torch_npu::profiler::DOMAIN_COMMUNICATION);
5086+#endif
5032 if (c10_npu::is_core_control_enabled) {5087 if (c10_npu::is_core_control_enabled) {
5033 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5088 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5034 }5089 }
@@ -5106,9 +5161,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allgather(
5106 comm,5161 comm,
5107 stream,5162 stream,
5108 is_dispatched]() -> int {5163 is_dispatched]() -> int {
5164+#ifndef BUILD_LIBTORCH
5109 torch_npu::profiler::MstxRange range(5165 torch_npu::profiler::MstxRange range(
5110 getMstxHcclMsg("HcclAllGatherV", numel, hcclType, comm, stream.id(), -1, -1),5166 getMstxHcclMsg("HcclAllGatherV", numel, hcclType, comm, stream.id(), -1, -1),
5111 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);5167 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
5168+#endif
5112 if (c10_npu::is_core_control_enabled) {5169 if (c10_npu::is_core_control_enabled) {
5113 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5170 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5114 }5171 }
@@ -5184,9 +5241,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allgather(
5184 auto inputDataPtr = input.data_ptr();5241 auto inputDataPtr = input.data_ptr();
5185 auto numel = getNumelForHCCL(input);5242 auto numel = getNumelForHCCL(input);
5186 auto hcclType = getHcclDataType(input.scalar_type());5243 auto hcclType = getHcclDataType(input.scalar_type());
5244+#ifndef BUILD_LIBTORCH
5187 torch_npu::profiler::MstxRange range(5245 torch_npu::profiler::MstxRange range(
5188 getMstxHcclMsg("HcclBroadcast", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5246 getMstxHcclMsg("HcclBroadcast", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5189 torch_npu::profiler::DOMAIN_COMMUNICATION);5247 torch_npu::profiler::DOMAIN_COMMUNICATION);
5248+#endif
5190 auto hccl_result = HcclBroadcast(inputDataPtr, numel, hcclType, root, comm, stream.stream());5249 auto hccl_result = HcclBroadcast(inputDataPtr, numel, hcclType, root, comm, stream.stream());
5191 *is_dispatched = true;5250 *is_dispatched = true;
5192 return hccl_result;5251 return hccl_result;
@@ -5223,9 +5282,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allgather_into_tensor_coalesced
5223 auto numel = getNumelForHCCL(input);5282 auto numel = getNumelForHCCL(input);
5224 auto hcclType = getHcclDataType(input.scalar_type());5283 auto hcclType = getHcclDataType(input.scalar_type());
5225 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {5284 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {
5285+#ifndef BUILD_LIBTORCH
5226 torch_npu::profiler::MstxRange range(5286 torch_npu::profiler::MstxRange range(
5227 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5287 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5228 torch_npu::profiler::DOMAIN_COMMUNICATION);5288 torch_npu::profiler::DOMAIN_COMMUNICATION);
5289+#endif
5229 if (c10_npu::is_core_control_enabled) {5290 if (c10_npu::is_core_control_enabled) {
5230 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5291 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5231 }5292 }
@@ -5268,9 +5329,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allgather_togather(
5268 auto numel = getNumelForHCCL(input);5329 auto numel = getNumelForHCCL(input);
5269 auto hcclType = getHcclDataType(input.scalar_type());5330 auto hcclType = getHcclDataType(input.scalar_type());
5270 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {5331 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {
5332+#ifndef BUILD_LIBTORCH
5271 torch_npu::profiler::MstxRange range(5333 torch_npu::profiler::MstxRange range(
5272 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5334 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5273 torch_npu::profiler::DOMAIN_COMMUNICATION);5335 torch_npu::profiler::DOMAIN_COMMUNICATION);
5336+#endif
5274 if (c10_npu::is_core_control_enabled) {5337 if (c10_npu::is_core_control_enabled) {
5275 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5338 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5276 }5339 }
@@ -5321,9 +5384,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::_allgather_base(
5321 auto numel = getNumelForHCCL(input);5384 auto numel = getNumelForHCCL(input);
5322 auto hcclType = getHcclDataType(input.scalar_type());5385 auto hcclType = getHcclDataType(input.scalar_type());
5323 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {5386 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, comm, stream, is_dispatched]() -> int {
5387+#ifndef BUILD_LIBTORCH
5324 torch_npu::profiler::MstxRange range(5388 torch_npu::profiler::MstxRange range(
5325 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5389 getMstxHcclMsg("HcclAllGather", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5326 torch_npu::profiler::DOMAIN_COMMUNICATION);5390 torch_npu::profiler::DOMAIN_COMMUNICATION);
5391+#endif
5327 if (c10_npu::is_core_control_enabled) {5392 if (c10_npu::is_core_control_enabled) {
5328 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5393 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5329 }5394 }
@@ -5371,9 +5436,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce_scatter(
5371 auto numel = getNumelForHCCL(output);5436 auto numel = getNumelForHCCL(output);
5372 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);5437 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);
5373 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {5438 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {
5439+#ifndef BUILD_LIBTORCH
5374 torch_npu::profiler::MstxRange range(5440 torch_npu::profiler::MstxRange range(
5375 getMstxHcclMsg("HcclReduceScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5441 getMstxHcclMsg("HcclReduceScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5376 torch_npu::profiler::DOMAIN_COMMUNICATION);5442 torch_npu::profiler::DOMAIN_COMMUNICATION);
5443+#endif
5377 if (c10_npu::is_core_control_enabled) {5444 if (c10_npu::is_core_control_enabled) {
5378 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5445 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5379 }5446 }
@@ -5470,9 +5537,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce_scatter(
5470 comm,5537 comm,
5471 stream,5538 stream,
5472 is_dispatched]() -> int {5539 is_dispatched]() -> int {
5540+#ifndef BUILD_LIBTORCH
5473 torch_npu::profiler::MstxRange range(5541 torch_npu::profiler::MstxRange range(
5474 getMstxHcclMsg("HcclReduceScatterV", numel, hcclType, comm, stream.id(), -1, -1),5542 getMstxHcclMsg("HcclReduceScatterV", numel, hcclType, comm, stream.id(), -1, -1),
5475 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);5543 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
5544+#endif
5476 if (c10_npu::is_core_control_enabled) {5545 if (c10_npu::is_core_control_enabled) {
5477 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5546 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5478 }5547 }
@@ -5597,9 +5666,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::_reduce_scatter_base(
5597 auto numel = getNumelForHCCL(output);5666 auto numel = getNumelForHCCL(output);
5598 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);5667 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);
5599 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {5668 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {
5669+#ifndef BUILD_LIBTORCH
5600 torch_npu::profiler::MstxRange range(5670 torch_npu::profiler::MstxRange range(
5601 getMstxHcclMsg("HcclReduceScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5671 getMstxHcclMsg("HcclReduceScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5602 torch_npu::profiler::DOMAIN_COMMUNICATION);5672 torch_npu::profiler::DOMAIN_COMMUNICATION);
5673+#endif
5603 if (c10_npu::is_core_control_enabled) {5674 if (c10_npu::is_core_control_enabled) {
5604 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5675 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5605 }5676 }
@@ -5658,9 +5729,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce_scatter_tensor_coalesced
5658 auto numel = getNumelForHCCL(output);5729 auto numel = getNumelForHCCL(output);
5659 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);5730 auto hcclReduceOp = getHcclReduceOp(opts.reduceOp, input);
5660 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {5731 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, hcclReduceOp, comm, stream, is_dispatched]() -> int {
5732+#ifndef BUILD_LIBTORCH
5661 torch_npu::profiler::MstxRange range(5733 torch_npu::profiler::MstxRange range(
5662 getMstxHcclMsg("HcclReduceScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5734 getMstxHcclMsg("HcclReduceScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5663 torch_npu::profiler::DOMAIN_COMMUNICATION);5735 torch_npu::profiler::DOMAIN_COMMUNICATION);
5736+#endif
5664 if (c10_npu::is_core_control_enabled) {5737 if (c10_npu::is_core_control_enabled) {
5665 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5738 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5666 }5739 }
@@ -5903,9 +5976,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::scatter(
5903 auto numel = getNumelForHCCL(output);5976 auto numel = getNumelForHCCL(output);
5904 auto hcclType = getHcclDataType(input.scalar_type());5977 auto hcclType = getHcclDataType(input.scalar_type());
5905 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, root, comm, stream, is_dispatched]() -> int {5978 auto hccl_call = [inputDataPtr, outputDataPtr, numel, hcclType, root, comm, stream, is_dispatched]() -> int {
5979+#ifndef BUILD_LIBTORCH
5906 torch_npu::profiler::MstxRange range(5980 torch_npu::profiler::MstxRange range(
5907 getMstxHcclMsg("HcclScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),5981 getMstxHcclMsg("HcclScatter", numel, hcclType, comm, stream.id(), -1, -1), stream.stream(false),
5908 torch_npu::profiler::DOMAIN_COMMUNICATION);5982 torch_npu::profiler::DOMAIN_COMMUNICATION);
5983+#endif
5909 if (c10_npu::is_core_control_enabled) {5984 if (c10_npu::is_core_control_enabled) {
5910 c10_npu::UseStreamResInCurrentThread(stream.stream(false));5985 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5911 }5986 }
@@ -5964,9 +6039,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::send(std::vector<at::Tensor>& t
5964 auto numel = getNumelForHCCL(input);6039 auto numel = getNumelForHCCL(input);
5965 auto hcclType = getHcclDataType(input.scalar_type());6040 auto hcclType = getHcclDataType(input.scalar_type());
5966 auto hccl_call = [inputDataPtr, numel, hcclType, dst_rank, comm, stream, is_dispatched]() -> int {6041 auto hccl_call = [inputDataPtr, numel, hcclType, dst_rank, comm, stream, is_dispatched]() -> int {
6042+#ifndef BUILD_LIBTORCH
5967 torch_npu::profiler::MstxRange range(6043 torch_npu::profiler::MstxRange range(
5968 getMstxHcclMsg("HcclSend", numel, hcclType, comm, stream.id(), -1, dst_rank), stream.stream(false),6044 getMstxHcclMsg("HcclSend", numel, hcclType, comm, stream.id(), -1, dst_rank), stream.stream(false),
5969 torch_npu::profiler::DOMAIN_COMMUNICATION);6045 torch_npu::profiler::DOMAIN_COMMUNICATION);
6046+#endif
5970 if (c10_npu::is_core_control_enabled) {6047 if (c10_npu::is_core_control_enabled) {
5971 c10_npu::UseStreamResInCurrentThread(stream.stream(false));6048 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
5972 }6049 }
@@ -5999,9 +6076,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::recv(std::vector<at::Tensor>& t
5999 auto numel = getNumelForHCCL(output);6076 auto numel = getNumelForHCCL(output);
6000 auto hcclType = getHcclDataType(output.scalar_type());6077 auto hcclType = getHcclDataType(output.scalar_type());
6001 auto hccl_call = [outputDataPtr, numel, hcclType, src_rank, comm, stream, is_dispatched]() -> int {6078 auto hccl_call = [outputDataPtr, numel, hcclType, src_rank, comm, stream, is_dispatched]() -> int {
6079+#ifndef BUILD_LIBTORCH
6002 torch_npu::profiler::MstxRange range(6080 torch_npu::profiler::MstxRange range(
6003 getMstxHcclMsg("HcclRecv", numel, hcclType, comm, stream.id(), src_rank, -1), stream.stream(false),6081 getMstxHcclMsg("HcclRecv", numel, hcclType, comm, stream.id(), src_rank, -1), stream.stream(false),
6004 torch_npu::profiler::DOMAIN_COMMUNICATION);6082 torch_npu::profiler::DOMAIN_COMMUNICATION);
6083+#endif
6005 if (c10_npu::is_core_control_enabled) {6084 if (c10_npu::is_core_control_enabled) {
6006 c10_npu::UseStreamResInCurrentThread(stream.stream(false));6085 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
6007 }6086 }
@@ -6092,9 +6171,11 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::alltoall_base(
6092 comm,6171 comm,
6093 stream,6172 stream,
6094 is_dispatched]() -> int {6173 is_dispatched]() -> int {
6174+#ifndef BUILD_LIBTORCH
6095 torch_npu::profiler::MstxRange range(6175 torch_npu::profiler::MstxRange range(
6096 getMstxHcclMsg("HcclAlltoAll", input_counts, inputhcclDataType, comm, stream.id(), -1, -1),6176 getMstxHcclMsg("HcclAlltoAll", input_counts, inputhcclDataType, comm, stream.id(), -1, -1),
6097 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);6177 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
6178+#endif
6098 if (c10_npu::is_core_control_enabled) {6179 if (c10_npu::is_core_control_enabled) {
6099 c10_npu::UseStreamResInCurrentThread(stream.stream(false));6180 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
6100 }6181 }
@@ -6192,10 +6273,12 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::alltoall_base(
6192 comm,6273 comm,
6193 stream,6274 stream,
6194 is_dispatched]() -> int {6275 is_dispatched]() -> int {
6276+#ifndef BUILD_LIBTORCH
6195 torch_npu::profiler::MstxRange range(6277 torch_npu::profiler::MstxRange range(
6196 getMstxHcclMsg("HcclAlltoAllV", static_cast<uint64_t>(inputCounts.size()),6278 getMstxHcclMsg("HcclAlltoAllV", static_cast<uint64_t>(inputCounts.size()),
6197 inputhcclDataType, comm, stream.id(), -1, -1),6279 inputhcclDataType, comm, stream.id(), -1, -1),
6198 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);6280 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
6281+#endif
6199 if (c10_npu::is_core_control_enabled) {6282 if (c10_npu::is_core_control_enabled) {
6200 c10_npu::UseStreamResInCurrentThread(stream.stream(false));6283 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
6201 }6284 }
@@ -6337,10 +6420,12 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::alltoall(
6337 comm,6420 comm,
6338 stream,6421 stream,
6339 is_dispatched]() -> int {6422 is_dispatched]() -> int {
6423+#ifndef BUILD_LIBTORCH
6340 torch_npu::profiler::MstxRange range(6424 torch_npu::profiler::MstxRange range(
6341 getMstxHcclMsg("HcclAlltoAllV", static_cast<uint64_t>(input_counts.size()),6425 getMstxHcclMsg("HcclAlltoAllV", static_cast<uint64_t>(input_counts.size()),
6342 inputhcclDataType, comm, stream.id(), -1, -1),6426 inputhcclDataType, comm, stream.id(), -1, -1),
6343 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);6427 stream.stream(false), torch_npu::profiler::DOMAIN_COMMUNICATION);
6428+#endif
6344 if (c10_npu::is_core_control_enabled) {6429 if (c10_npu::is_core_control_enabled) {
6345 c10_npu::UseStreamResInCurrentThread(stream.stream(false));6430 c10_npu::UseStreamResInCurrentThread(stream.stream(false));
6346 }6431 }
Mtorch_npu/csrc/distributed/ProcessGroupHCCL.hpp+2-0
@@ -797,6 +797,7 @@ protected:
797 // In the timeout case and we will dump debug info such as the NCCL flight797 // In the timeout case and we will dump debug info such as the NCCL flight
798 // recorder to storage. Down the road, if we have more complicated or blocking798 // recorder to storage. Down the road, if we have more complicated or blocking
799 // operations, we might need to use a side thread to do it.799 // operations, we might need to use a side thread to do it.
800+#ifndef BUILD_LIBTORCH
800 bool dumpDebuggingInfo();801 bool dumpDebuggingInfo();
801 void dumpTraceAndResetStatus();802 void dumpTraceAndResetStatus();
802 bool dumpPythonTraceback();803 bool dumpPythonTraceback();
@@ -807,6 +808,7 @@ protected:
807 // so that when we get stuck in some HCCL/CANN calls,808 // so that when we get stuck in some HCCL/CANN calls,
808 // we can dump the debugging information and abort the process.809 // we can dump the debugging information and abort the process.
809 virtual void heartbeatMonitor();810 virtual void heartbeatMonitor();
811+#endif
810 812
811 // Instance of the watchdog thread.813 // Instance of the watchdog thread.
812 std::unique_ptr<Watchdog> watchdog_;814 std::unique_ptr<Watchdog> watchdog_;
Mtorch_npu/csrc/distributed/TraceUtils.h+29-2
@@ -7,7 +7,10 @@
7#include <torch/csrc/distributed/c10d/Types.hpp>7#include <torch/csrc/distributed/c10d/Types.hpp>
8#include <torch/csrc/distributed/c10d/Utils.hpp>8#include <torch/csrc/distributed/c10d/Utils.hpp>
9#include <torch/csrc/jit/serialization/pickler.h>9#include <torch/csrc/jit/serialization/pickler.h>
10+ 
11+#ifndef BUILD_LIBTORCH
10#include <torch/csrc/profiler/combined_traceback.h>12#include <torch/csrc/profiler/combined_traceback.h>
13+#endif
11 14 
12#include "torch_npu/csrc/core/npu/NPUEvent.h"15#include "torch_npu/csrc/core/npu/NPUEvent.h"
13#include "torch_npu/csrc/distributed/HCCLUtils.hpp"16#include "torch_npu/csrc/distributed/HCCLUtils.hpp"
@@ -232,6 +235,7 @@ DEFINE_CONSTANT(started_state, "started")
232 return std::string(result.begin(), result.end());235 return std::string(result.begin(), result.end());
233 }236 }
234 237 
238+#ifndef BUILD_LIBTORCH
235 inline std::string get_python_cpp_trace()239 inline std::string get_python_cpp_trace()
236 {240 {
237 // usage:241 // usage:
@@ -254,6 +258,7 @@ DEFINE_CONSTANT(started_state, "started")
254 }258 }
255 return oss.str();259 return oss.str();
256 }260 }
261+#endif
257 262 
258 inline c10::Dict<c10::IValue, c10::IValue> new_dict()263 inline c10::Dict<c10::IValue, c10::IValue> new_dict()
259 {264 {
@@ -313,7 +318,9 @@ DEFINE_CONSTANT(started_state, "started")
313 size_t op_id_;318 size_t op_id_;
314 std::string profiling_name_;319 std::string profiling_name_;
315 320 
321+#ifndef BUILD_LIBTORCH
316 std::shared_ptr<torch::CapturedTraceback> traceback_;322 std::shared_ptr<torch::CapturedTraceback> traceback_;
323+#endif
317 // we borrow pointers to start_ and end_ so we can query the state324 // we borrow pointers to start_ and end_ so we can query the state
318 // on reporting. However, once the event is completed, the call325 // on reporting. However, once the event is completed, the call
319 // to `complete` will clear these.326 // to `complete` will clear these.
@@ -385,10 +392,11 @@ DEFINE_CONSTANT(started_state, "started")
385 // Current pg_status is not in FR.392 // Current pg_status is not in FR.
386 all_pg_status_[pg_id] = std::move(pg_status);393 all_pg_status_[pg_id] = std::move(pg_status);
387 }394 }
388- auto traceback =
389- torch::CapturedTraceback::gather(true, true, capture_cpp_stack_);
390 std::lock_guard<std::mutex> guard(mutex_);395 std::lock_guard<std::mutex> guard(mutex_);
391 396 
397+#ifndef BUILD_LIBTORCH
H
Hhuangyunlong3月5日

这里尽量不要改变guard的范围

likedislike
K
kuhn7
3月5日 评论：
398+ auto traceback =
399+ torch::CapturedTraceback::gather(true, true, capture_cpp_stack_);
392 auto te = Entry{400 auto te = Entry{
393 id_,401 id_,
394 pg_id,402 pg_id,
@@ -403,6 +411,21 @@ DEFINE_CONSTANT(started_state, "started")
403 c10::getTime(),411 c10::getTime(),
404 timeout_ms.count(),412 timeout_ms.count(),
405 isP2P};413 isP2P};
414+#else
415+ auto te = Entry{
416+ id_,
417+ pg_id,
418+ pg_name,
419+ collective_seq_id,
420+ p2p_seq_id,
421+ op_id,
422+ std::move(profiling_name),
423+ std::move(start),
424+ std::move(end),
425+ c10::getTime(),
426+ timeout_ms.count(),
427+ isP2P};
428+#endif
406 429 
407 for (const auto &input : inputs) {430 for (const auto &input : inputs) {
408 c10::IntArrayRef sizes = input.sizes();431 c10::IntArrayRef sizes = input.sizes();
@@ -538,6 +561,7 @@ DEFINE_CONSTANT(started_state, "started")
538 {561 {
539 auto entries = new_list();562 auto entries = new_list();
540 auto result = dump_entries();563 auto result = dump_entries();
564+#ifndef BUILD_LIBTORCH
541 std::vector<torch::CapturedTraceback *> tracebacks;565 std::vector<torch::CapturedTraceback *> tracebacks;
542 torch::SymbolizedTracebacks stracebacks;566 torch::SymbolizedTracebacks stracebacks;
543 std::vector<c10::IValue> all_frames;567 std::vector<c10::IValue> all_frames;
@@ -554,6 +578,7 @@ DEFINE_CONSTANT(started_state, "started")
554 all_frames.emplace_back(std::move(d));578 all_frames.emplace_back(std::move(d));
555 }579 }
556 }580 }
581+#endif
557 for (auto i : c10::irange(result.size())) {582 for (auto i : c10::irange(result.size())) {
558 auto dict = new_dict();583 auto dict = new_dict();
559 auto &e = result.at(i);584 auto &e = result.at(i);
@@ -562,6 +587,7 @@ DEFINE_CONSTANT(started_state, "started")
562 continue;587 continue;
563 }588 }
564 589 
590+#ifndef BUILD_LIBTORCH
AtlasAccount
AtlasAccountAtlasAccount3月4日
条件编译范围不匹配: 在 `getCollectiveTrace` 函数中，第565-580行的堆栈跟踪处理逻辑被 `#ifndef BUILD_LIBTORCH` 保护。然而，第591-597行使用 `includeStacktraces` 参数在 `#ifndef BUILD_LIBTORCH` 块内部访问 `stracebacks` 和 `tb` 变量。如果 `BUILD_LIBTORCH` 被定义，`stracebacks` 和 `tb` 变量将不存在，但第591行的 `if (includeStacktraces)` 检查仍然会执行，导致编译错误，因为 `stracebacks` 未定义。此外，第598行的 `#endif` 位置可能过早，因为第591-597行的代码也依赖于 `BUILD_LIBTORCH` 未定义的条件。
问题类型: 条件编译范围不匹配
文件路径: torch_npu/csrc/distributed/TraceUtils.h
行号: 590
问题代码:
#ifndef BUILD_LIBTORCH
            if (includeStacktraces) {
                auto &tb = stracebacks.tracebacks.at(i);
                auto frames = new_list();
                for (int64_t frame : tb) {
                    frames.push_back(all_frames.at(frame));
                }
                dict.insert(frames_key, frames);
            }
#endif
修改建议:
1. 将第591-597行的整个 `if (includeStacktraces)` 块也包含在 `#ifndef BUILD_LIBTORCH` 条件编译内，确保当 `BUILD_LIBTORCH` 定义时，这段代码不会被编译。
2. 或者，在函数开头，如果 `BUILD_LIBTORCH` 被定义且 `includeStacktraces` 为 true，可以提前返回或记录警告，因为堆栈跟踪功能不可用。
3. 调整 `#endif` 的位置，使其覆盖从第565行到第597行的所有依赖 `torch::CapturedTraceback` 的代码。
---
此评论由代码审查工具自动生成
likedislike
565 if (includeStacktraces) {591 if (includeStacktraces) {
566 auto &tb = stracebacks.tracebacks.at(i);592 auto &tb = stracebacks.tracebacks.at(i);
567 auto frames = new_list();593 auto frames = new_list();
@@ -570,6 +596,7 @@ DEFINE_CONSTANT(started_state, "started")
570 }596 }
571 dict.insert(frames_key, frames);597 dict.insert(frames_key, frames);
572 }598 }
599+#endif
573 600 
574 dict.insert(record_id_key, int64_t(e.id_));601 dict.insert(record_id_key, int64_t(e.id_));
575 dict.insert(pg_id_key, int64_t(e.pg_id_));602 dict.insert(pg_id_key, int64_t(e.pg_id_));