已合并
PTA Error code #10047
AtomGit-Bot创建于 2024年3月7日
PTA Error code #10047
已合并
从refs/pull/10047/head合入到master
共 6 个文件变更+39-13
| @@ -0,0 +1,22 @@ | |||
| 1 | +import torch | ||
| 2 | +import torch.nn as nn | ||
| 3 | + | ||
| 4 | +import torch_npu | ||
| 5 | +from torch_npu.testing.testcase import TestCase, run_tests | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +class TestErrorCode(TestCase): | ||
| 9 | + | ||
| 10 | + def test_set_per_process_memory_fraction(self): | ||
| 11 | + with self.assertRaisesRegex(TypeError, "ERR00002 PTA invalid type"): | ||
| 12 | + torch_npu.npu.set_per_process_memory_fraction(1) | ||
| 13 | + | ||
| 14 | + def test_div(self): | ||
| 15 | + x1 = torch.tensor(1).npu() | ||
| 16 | + x2 = torch.tensor(1).npu() | ||
| 17 | + with self.assertRaisesRegex(RuntimeError, "ERR01001 OPS invalid parameter"): | ||
| 18 | + torch.div(x1, x2, rounding_mode="test") | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +if __name__ == "__main__": | ||
| 22 | + run_tests() | ||
| @@ -4,6 +4,7 @@ from torch.nn.functional import layer_norm as origin_layernorm | |||
| 4 | from torch.nn.functional import embedding as origin_embedding | 4 | from torch.nn.functional import embedding as origin_embedding |
| 5 | 5 | ||
| 6 | import torch_npu | 6 | import torch_npu |
| 7 | +from torch_npu.utils.error_code import ErrCode, pta_error | ||
| 7 | from .silent_fault_data import SilentFaultData | 8 | from .silent_fault_data import SilentFaultData |
| 8 | 9 | ||
| 9 | 10 | ||
| @@ -32,7 +33,7 @@ class SilentFaultDetector: | |||
| 32 | 33 | ||
| 33 | def set_asd_loss_scale(self, loss_scale=1.0): | 34 | def set_asd_loss_scale(self, loss_scale=1.0): |
| 34 | if loss_scale == 0: | 35 | if loss_scale == 0: |
| 35 | - raise ValueError("loss scale cannot be 0.") | 36 | + raise ValueError("loss scale cannot be 0." + pta_error(ErrCode.VALUE)) |
| 36 | self.set_loss_scale_flag = True | 37 | self.set_loss_scale_flag = True |
| 37 | self.loss_scale = loss_scale | 38 | self.loss_scale = loss_scale |
| 38 | 39 | ||
| @@ -92,7 +93,7 @@ def patch_embedding(input_embedding, weight, padding_idx, max_norm, | |||
| 92 | def asd_patch(): | 93 | def asd_patch(): |
| 93 | env_value = os.getenv("NPU_ASD_ENABLE", "0") | 94 | env_value = os.getenv("NPU_ASD_ENABLE", "0") |
| 94 | if env_value not in ["0", "1"]: | 95 | if env_value not in ["0", "1"]: |
| 95 | - raise ValueError("NPU_ASD_ENABLE should be 0 or 1!") | 96 | + raise ValueError("NPU_ASD_ENABLE should be 0 or 1!" + pta_error(ErrCode.VALUE)) |
| 96 | 97 | ||
| 97 | if int(env_value): | 98 | if int(env_value): |
| 98 | torch.nn.functional.layer_norm = patch_layernorm | 99 | torch.nn.functional.layer_norm = patch_layernorm |
| @@ -81,7 +81,7 @@ at::Tensor NPUNativeFunctions::scalar_tensor(const c10::Scalar& s, c10::optional | |||
| 81 | auto result = at::native::empty_cpu({}, dtype, layout, c10::make_optional(c10::Device(at::kCPU)), pin_memory); | 81 | auto result = at::native::empty_cpu({}, dtype, layout, c10::make_optional(c10::Device(at::kCPU)), pin_memory); |
| 82 | at::native::fill_(result, s); | 82 | at::native::fill_(result, s); |
| 83 | if (device.has_value()) { | 83 | if (device.has_value()) { |
| 84 | - AT_ASSERT(device.value().type() == c10::DeviceType::PrivateUse1); | 84 | + AT_ASSERT(device.value().type() == c10::DeviceType::PrivateUse1, OPS_ERROR(ErrCode::TYPE)); |
| 85 | return result.to(device.value()); | 85 | return result.to(device.value()); |
| 86 | } | 86 | } |
| 87 | return result.to(at::device(c10::DeviceType::PrivateUse1)); | 87 | return result.to(at::device(c10::DeviceType::PrivateUse1)); |
| @@ -506,7 +506,7 @@ aclError AclGetCannAttributeList(const aclCannAttr **cannAttrList, size_t *num) | |||
| 506 | if (func == nullptr) { | 506 | if (func == nullptr) { |
| 507 | func = (AclGetCannAttributeList) GET_FUNC(aclGetCannAttributeList); | 507 | func = (AclGetCannAttributeList) GET_FUNC(aclGetCannAttributeList); |
| 508 | } | 508 | } |
| 509 | - TORCH_CHECK(func, "Failed to find function ", "aclGetCannAttributeList"); | 509 | + TORCH_CHECK(func, "Failed to find function ", "aclGetCannAttributeList", PTA_ERROR(ErrCode::NOT_FOUND)); |
| 510 | return func(cannAttrList, num); | 510 | return func(cannAttrList, num); |
| 511 | } | 511 | } |
| 512 | 512 | ||
| @@ -517,7 +517,7 @@ aclError AclGetCannAttribute(aclCannAttr cannAttr, int32_t *value) | |||
| 517 | if (func == nullptr) { | 517 | if (func == nullptr) { |
| 518 | func = (AclGetCannAttribute) GET_FUNC(aclGetCannAttribute); | 518 | func = (AclGetCannAttribute) GET_FUNC(aclGetCannAttribute); |
| 519 | } | 519 | } |
| 520 | - TORCH_CHECK(func, "Failed to find function ", "aclGetCannAttribute"); | 520 | + TORCH_CHECK(func, "Failed to find function ", "aclGetCannAttribute", PTA_ERROR(ErrCode::NOT_FOUND)); |
| 521 | return func(cannAttr, value); | 521 | return func(cannAttr, value); |
| 522 | } | 522 | } |
| 523 | 523 | ||
| @@ -528,7 +528,7 @@ aclError AclGetDeviceCapability(uint32_t deviceId, aclDeviceInfo deviceInfo, int | |||
| 528 | if (func == nullptr) { | 528 | if (func == nullptr) { |
| 529 | func = (AclGetDeviceCapability) GET_FUNC(aclGetDeviceCapability); | 529 | func = (AclGetDeviceCapability) GET_FUNC(aclGetDeviceCapability); |
| 530 | } | 530 | } |
| 531 | - TORCH_CHECK(func, "Failed to find function ", "aclGetDeviceCapability"); | 531 | + TORCH_CHECK(func, "Failed to find function ", "aclGetDeviceCapability", PTA_ERROR(ErrCode::NOT_FOUND)); |
| 532 | return func(deviceId, deviceInfo, value); | 532 | return func(deviceId, deviceInfo, value); |
| 533 | } | 533 | } |
| 534 | 534 | ||
| @@ -889,12 +889,12 @@ void ProcessGroupHCCL::recordDataVol(std::string opName, const std::string dataV | |||
| 889 | std::stringstream fileName; | 889 | std::stringstream fileName; |
| 890 | std::string commName = getHcclCommNameWithoutInit(currRank, hcclComms); | 890 | std::string commName = getHcclCommNameWithoutInit(currRank, hcclComms); |
| 891 | auto master_addr = getenv("MASTER_ADDR"); | 891 | auto master_addr = getenv("MASTER_ADDR"); |
| 892 | - TORCH_CHECK(master_addr != nullptr, "Unable to fetch master IP addr, environment variable is null."); | 892 | + TORCH_CHECK(master_addr != nullptr, "Unable to fetch master IP addr, environment variable is null.", DIST_ERROR(ErrCode::NOT_FOUND)); |
| 893 | fileName << master_addr << "_" << commName << "_" << std::to_string(currRank) << ".log"; | 893 | fileName << master_addr << "_" << commName << "_" << std::to_string(currRank) << ".log"; |
| 894 | try { | 894 | try { |
| 895 | outfile.open(c10::str(getenv("NSLB_CP"), "/", fileName.str()), std::ios::app); | 895 | outfile.open(c10::str(getenv("NSLB_CP"), "/", fileName.str()), std::ios::app); |
| 896 | } catch (std::exception& e) { | 896 | } catch (std::exception& e) { |
| 897 | - throw std::runtime_error("Open shared directory failed. Please check whether input path is valid."); | 897 | + throw std::runtime_error("Open shared directory failed. Please check whether input path is valid." + DIST_ERROR(ErrCode::NOT_FOUND)); |
| 898 | } | 898 | } |
| 899 | std::transform(opName.begin(), opName.end(), opName.begin(), ::tolower); | 899 | std::transform(opName.begin(), opName.end(), opName.begin(), ::tolower); |
| 900 | outfile << opName << " " << dataVol << " " << std::to_string(currRank) << "\n"; | 900 | outfile << opName << " " << dataVol << " " << std::to_string(currRank) << "\n"; |
| @@ -1165,7 +1165,8 @@ int64_t ProcessGroupHCCL::getHcclComm(int rankid) | |||
| 1165 | std::vector<at::Device> devices = {device}; | 1165 | std::vector<at::Device> devices = {device}; |
| 1166 | const auto key = getKeyFromDevices(devices); | 1166 | const auto key = getKeyFromDevices(devices); |
| 1167 | auto& hcclComms = getHCCLComm(key, devices); | 1167 | auto& hcclComms = getHCCLComm(key, devices); |
| 1168 | - TORCH_CHECK(hcclComms.size() == 1, "expect hcclComms.size() = 1, but hcclComms.size() = ", hcclComms.size(), DIST_ERROR(ErrCode::VALUE)); | 1168 | + TORCH_CHECK(hcclComms.size() == 1, "expect hcclComms.size() = 1, but hcclComms.size() = ", |
| 1169 | + hcclComms.size(), DIST_ERROR(ErrCode::VALUE)); | ||
| 1169 | auto ret_hcom = hcclComms[0]->getHcclComm(); | 1170 | auto ret_hcom = hcclComms[0]->getHcclComm(); |
| 1170 | int64_t hccl_comm = static_cast<int64_t>(reinterpret_cast<intptr_t>(ret_hcom)); | 1171 | int64_t hccl_comm = static_cast<int64_t>(reinterpret_cast<intptr_t>(ret_hcom)); |
| 1171 | return hccl_comm; | 1172 | return hccl_comm; |
| @@ -1188,7 +1189,7 @@ std::string ProcessGroupHCCL::getHcclCommName(int rankid) { | |||
| 1188 | std::string ProcessGroupHCCL::getHcclCommNameWithoutInit(int rankid, std::vector<std::shared_ptr<HCCLComm>>& hcclComms) | 1189 | std::string ProcessGroupHCCL::getHcclCommNameWithoutInit(int rankid, std::vector<std::shared_ptr<HCCLComm>>& hcclComms) |
| 1189 | { | 1190 | { |
| 1190 | TORCH_CHECK(hcclComms.size() == 1, "expect hcclComms.size() = 1, but hcclComms.size() = ", | 1191 | TORCH_CHECK(hcclComms.size() == 1, "expect hcclComms.size() = 1, but hcclComms.size() = ", |
| 1191 | - hcclComms.size()); | 1192 | + hcclComms.size(), DIST_ERROR(ErrCode::VALUE)); |
| 1192 | HcclComm ret_hcom = hcclComms[0]->getHcclComm(); | 1193 | HcclComm ret_hcom = hcclComms[0]->getHcclComm(); |
| 1193 | char commName[MAX_GROUP_NAME_LEN]; | 1194 | char commName[MAX_GROUP_NAME_LEN]; |
| 1194 | HCCL_CHECK_ERROR(at_npu::hccl::HcclGetCommNameFace(ret_hcom, commName)); | 1195 | HCCL_CHECK_ERROR(at_npu::hccl::HcclGetCommNameFace(ret_hcom, commName)); |
| @@ -1239,7 +1240,7 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::collective( | |||
| 1239 | dataVol += tensor.storage().nbytes(); | 1240 | dataVol += tensor.storage().nbytes(); |
| 1240 | } | 1241 | } |
| 1241 | char* global_rank = getenv("RANK"); | 1242 | char* global_rank = getenv("RANK"); |
| 1242 | - TORCH_CHECK(global_rank != nullptr, "Unable to fetch global rank for NSLB."); | 1243 | + TORCH_CHECK(global_rank != nullptr, "Unable to fetch global rank for NSLB.", DIST_ERROR(ErrCode::NOT_FOUND)); |
| 1243 | recordDataVol(opTypeToString(opType), std::to_string(dataVol), atoi(global_rank), hcclComms); | 1244 | recordDataVol(opTypeToString(opType), std::to_string(dataVol), atoi(global_rank), hcclComms); |
| 1244 | } | 1245 | } |
| 1245 | if (seq_ >= nslb_num) { | 1246 | if (seq_ >= nslb_num) { |
| @@ -11,6 +11,8 @@ from torch.distributed.optim import ( | |||
| 11 | _FunctionalSGD | 11 | _FunctionalSGD |
| 12 | ) | 12 | ) |
| 13 | 13 | ||
| 14 | +from torch_npu.utils.error_code import ErrCode, dist_error | ||
| 15 | + | ||
| 14 | functional_optim_map = { | 16 | functional_optim_map = { |
| 15 | "Adagrad": _FunctionalAdagrad, | 17 | "Adagrad": _FunctionalAdagrad, |
| 16 | "Adam": _FunctionalAdam, | 18 | "Adam": _FunctionalAdam, |
| @@ -33,7 +35,7 @@ def _get_optimizer_constructor(self, optimizer_class): | |||
| 33 | if optimizer_class in functional_optims: | 35 | if optimizer_class in functional_optims: |
| 34 | raise ValueError( | 36 | raise ValueError( |
| 35 | f"Passing in a functional optimizer {optimizer_class_name} " | 37 | f"Passing in a functional optimizer {optimizer_class_name} " |
| 36 | - "when `overlap_with_ddp=False`" | 38 | + "when `overlap_with_ddp=False`" + dist_error(ErrCode.VALUE) |
| 37 | ) | 39 | ) |
| 38 | else: | 40 | else: |
| 39 | return optimizer_class | 41 | return optimizer_class |
| @@ -53,5 +55,5 @@ def _get_optimizer_constructor(self, optimizer_class): | |||
| 53 | raise ValueError( | 55 | raise ValueError( |
| 54 | "Using `ddp_with_overlap=True` requires using a " | 56 | "Using `ddp_with_overlap=True` requires using a " |
| 55 | "functional optimizer, but there is no supported functional " | 57 | "functional optimizer, but there is no supported functional " |
| 56 | - f"optimizer equivalent for {optimizer_class_name}" | 58 | + f"optimizer equivalent for {optimizer_class_name}" + dist_error(ErrCode.VALUE) |
| 57 | ) | 59 | ) |