已关闭
[WIP] modify error log #44441
wanglijun55创建于 8月12日关闭于 9月20日
[WIP] modify error log #44441
已关闭
共 4 个文件变更+1102-3
| @@ -397,4 +397,34 @@ on the device."}, | |||
| 397 | Rectify the fault based on the error information in the ascend log."}, | 397 | Rectify the fault based on the error information in the ascend log."}, |
| 398 | }; /* aclError code */ | 398 | }; /* aclError code */ |
| 399 | }; | 399 | }; |
| 400 | + | ||
| 401 | +class HcclErrorCode { | ||
| 402 | +public: | ||
| 403 | + std::unordered_map<int, std::string> error_code_map = { | ||
| 404 | + {1, "parameter error"}, | ||
| 405 | + {2, "empty pointer"}, | ||
| 406 | + {3, "memory error"}, | ||
| 407 | + {4, "internal error"}, | ||
| 408 | + {5, "not support feature"}, | ||
| 409 | + {6, "not found specific resource"}, | ||
| 410 | + {7, "resource unavailable"}, | ||
| 411 | + {8, "call system interface error"}, | ||
| 412 | + {9, "timeout"}, | ||
| 413 | + {10, "open file fail"}, | ||
| 414 | + {11, "tcp connect fail"}, | ||
| 415 | + {12, "roce connect fail"}, | ||
| 416 | + {13, "tcp transfer fail"}, | ||
| 417 | + {14, "roce transfer fail"}, | ||
| 418 | + {15, "call runtime api fail"}, | ||
| 419 | + {16, "call driver api fail"}, | ||
| 420 | + {17, "call profiling api fail"}, | ||
| 421 | + {18, "call cce api fail"}, | ||
| 422 | + {19, "call network api fail"}, | ||
| 423 | + {20, "try again"}, | ||
| 424 | + {21, "error cqe"}, | ||
| 425 | + {22, "error communicator suspending"}, | ||
| 426 | + {23, "retry constraint"}, | ||
| 427 | + {24, "out of memory"}, | ||
| 428 | + }; /* hcclError code */ | ||
| 429 | +}; | ||
| 400 | } /* c10_npu::acl */ | 430 | } /* c10_npu::acl */ |
| @@ -127,6 +127,8 @@ inline const char* getErrorFunction(const char* /* msg */, const char* args) | |||
| 127 | } \ | 127 | } \ |
| 128 | std::string device_error_msg = c10_npu::getDeviceErrorMessage(error_code); \ | 128 | std::string device_error_msg = c10_npu::getDeviceErrorMessage(error_code); \ |
| 129 | if (!device_error_msg.empty()) { \ | 129 | if (!device_error_msg.empty()) { \ |
| 130 | + static c10_npu::acl::AclErrorCode err_map; \ | ||
| 131 | + static c10_npu::acl::HcclErrorCode hccl_err_map; \ | ||
| 130 | TORCH_CHECK( \ | 132 | TORCH_CHECK( \ |
| 131 | false, \ | 133 | false, \ |
| 132 | __func__, \ | 134 | __func__, \ |
| @@ -136,6 +138,10 @@ inline const char* getErrorFunction(const char* /* msg */, const char* args) | |||
| 136 | __LINE__, \ | 138 | __LINE__, \ |
| 137 | " NPU function error: ", device_error_msg, \ | 139 | " NPU function error: ", device_error_msg, \ |
| 138 | ", error code is ", error_code, \ | 140 | ", error code is ", error_code, \ |
| 141 | + (err_map.error_code_map.find(error_code) != err_map.error_code_map.end() ? \ | ||
| 142 | + "\n[Error]: " + err_map.error_code_map[error_code] : \ | ||
| 143 | + (hccl_err_map.error_code_map.find(error_code) != hccl_err_map.error_code_map.end() ? \ | ||
| 144 | + "\n[Error]: " + hccl_err_map.error_code_map[error_code] : ".")), \ | ||
| 139 | PTA_ERROR(ErrCode::ACL)); \ | 145 | PTA_ERROR(ErrCode::ACL)); \ |
| 140 | } \ | 146 | } \ |
| 141 | 147 | ||
| @@ -19,11 +19,14 @@ | |||
| 19 | auto Error = err_code; \ | 19 | auto Error = err_code; \ |
| 20 | if ((Error) != HCCL_SUCCESS) { \ | 20 | if ((Error) != HCCL_SUCCESS) { \ |
| 21 | CHECK_AND_THROW_ERROR_WITH_SPECIFIC_MESSAGE(Error); \ | 21 | CHECK_AND_THROW_ERROR_WITH_SPECIFIC_MESSAGE(Error); \ |
| 22 | + static c10_npu::acl::HcclErrorCode hccl_err_map; \ | ||
| 22 | if (c10_npu::option::OptionsManager::IsCompactErrorOutput()) { \ | 23 | if (c10_npu::option::OptionsManager::IsCompactErrorOutput()) { \ |
| 23 | std::ostringstream oss; \ | 24 | std::ostringstream oss; \ |
| 24 | oss << " HCCL function error: " << getErrorFunction(#err_code, ##__VA_ARGS__) \ | 25 | oss << " HCCL function error: " << getErrorFunction(#err_code, ##__VA_ARGS__) \ |
| 25 | - << ", error code is " << Error << " " \ | 26 | + << ", error code is " << Error \ |
| 26 | - << DIST_ERROR(ErrCode::HCCL) + ".\n"; \ | 27 | + << (hccl_err_map.error_code_map.find(static_cast<int>(Error)) != hccl_err_map.error_code_map.end() ? \ |
| 28 | + " " + hccl_err_map.error_code_map[static_cast<int>(Error)] : "") \ | ||
| 29 | + << " " << DIST_ERROR(ErrCode::HCCL) + ".\n"; \ | ||
| 27 | std::string err_msg = oss.str(); \ | 30 | std::string err_msg = oss.str(); \ |
| 28 | ASCEND_LOGE("%s", err_msg.c_str()); \ | 31 | ASCEND_LOGE("%s", err_msg.c_str()); \ |
| 29 | std::string errmsg(c10_npu::c10_npu_get_error_message()); \ | 32 | std::string errmsg(c10_npu::c10_npu_get_error_message()); \ |
| @@ -33,7 +36,10 @@ | |||
| 33 | } else { \ | 36 | } else { \ |
| 34 | auto retmsg = std::string(__func__) + ":" + __FILE__ + ":" + std::to_string(__LINE__) + \ | 37 | auto retmsg = std::string(__func__) + ":" + __FILE__ + ":" + std::to_string(__LINE__) + \ |
| 35 | " HCCL function error: " + getErrorFunction(#err_code, ##__VA_ARGS__) + \ | 38 | " HCCL function error: " + getErrorFunction(#err_code, ##__VA_ARGS__) + \ |
| 36 | - ", error code is " + std::to_string(Error) + " " + DIST_ERROR(ErrCode::HCCL) + ".\n" + \ | 39 | + ", error code is " + std::to_string(Error) + \ |
| 40 | + (hccl_err_map.error_code_map.find(static_cast<int>(Error)) != hccl_err_map.error_code_map.end() ? \ | ||
| 41 | + " " + hccl_err_map.error_code_map[static_cast<int>(Error)] : "") + \ | ||
| 42 | + " " + DIST_ERROR(ErrCode::HCCL) + ".\n" + \ | ||
| 37 | c10_npu::c10_npu_get_error_message(); \ | 43 | c10_npu::c10_npu_get_error_message(); \ |
| 38 | if (c10_npu::isCannOOM(retmsg)) { \ | 44 | if (c10_npu::isCannOOM(retmsg)) { \ |
| 39 | if (c10_npu::option::OptionsManager::IsOomSnapshotEnable()) { \ | 45 | if (c10_npu::option::OptionsManager::IsOomSnapshotEnable()) { \ |