已合并
Update test_torch.py #27368
haiyan8创建于 2025年12月2日
Update test_torch.py #27368
已合并
共 2 个文件变更+432-170
| @@ -39,33 +39,32 @@ from torch.testing._internal.common_optimizers import ( | |||
| 39 | optim_db, optims, _get_optim_inputs_including_global_cliquey_kwargs) | 39 | optim_db, optims, _get_optim_inputs_including_global_cliquey_kwargs) |
| 40 | 40 | ||
| 41 | from torch.testing._internal.common_utils import ( # type: ignore[attr-defined] | 41 | from torch.testing._internal.common_utils import ( # type: ignore[attr-defined] |
| 42 | - TEST_WITH_TORCHINDUCTOR, TestCase, TEST_WITH_ROCM, run_tests, IS_JETSON, | 42 | + MI300_ARCH, TEST_WITH_TORCHINDUCTOR, TEST_WITH_ROCM, run_tests, IS_JETSON, |
| 43 | - IS_WINDOWS, IS_FILESYSTEM_UTF8_ENCODING, NO_MULTIPROCESSING_SPAWN, | 43 | + IS_FILESYSTEM_UTF8_ENCODING, |
| 44 | - IS_SANDCASTLE, IS_FBCODE, IS_REMOTE_GPU, skipIfTorchInductor, load_tests, slowTest, slowTestIf, | 44 | + IS_SANDCASTLE, IS_FBCODE, IS_REMOTE_GPU, skipIfRocmArch, skipIfTorchInductor, load_tests, slowTest, slowTestIf, |
| 45 | - TEST_WITH_CROSSREF, skipIfTorchDynamo, skipRocmIfTorchInductor, set_default_dtype, | 45 | + skipIfCrossRef, TEST_WITH_CROSSREF, skipIfTorchDynamo, skipRocmIfTorchInductor, set_default_dtype, |
| 46 | skipCUDAMemoryLeakCheckIf, BytesIOContext, | 46 | skipCUDAMemoryLeakCheckIf, BytesIOContext, |
| 47 | skipIfRocm, skipIfNoSciPy, TemporaryFileName, TemporaryDirectoryName, | 47 | skipIfRocm, skipIfNoSciPy, TemporaryFileName, TemporaryDirectoryName, |
| 48 | wrapDeterministicFlagAPITest, DeterministicGuard, CudaSyncGuard, | 48 | wrapDeterministicFlagAPITest, DeterministicGuard, CudaSyncGuard, |
| 49 | - skipIfNotRegistered, bytes_to_scalar, parametrize, skipIfMps, noncontiguous_like, | 49 | + bytes_to_scalar, parametrize, skipIfMPS, noncontiguous_like, |
| 50 | - AlwaysWarnTypedStorageRemoval, TEST_WITH_TORCHDYNAMO, xfailIfTorchDynamo) | 50 | + AlwaysWarnTypedStorageRemoval, TEST_WITH_TORCHDYNAMO, xfailIfTorchDynamo, set_warn_always_context) |
| 51 | from torch.testing._internal.common_device_type import ( | 51 | from torch.testing._internal.common_device_type import ( |
| 52 | expectedFailureMeta, | 52 | expectedFailureMeta, |
| 53 | expectedFailureXLA, | 53 | expectedFailureXLA, |
| 54 | instantiate_device_type_tests, | 54 | instantiate_device_type_tests, |
| 55 | onlyPRIVATEUSE1, onlyCPU, | 55 | onlyPRIVATEUSE1, onlyCPU, |
| 56 | dtypes, dtypesIfPRIVATEUSE1, dtypesIfCPU, deviceCountAtLeast, | 56 | dtypes, dtypesIfPRIVATEUSE1, dtypesIfCPU, deviceCountAtLeast, |
| 57 | - skipMeta, | 57 | + skipMeta, PYTORCH_CUDA_MEMCHECK, largeTensorTest, onlyNativeDeviceTypes, skipCUDAIfNotRocm, |
| 58 | - PYTORCH_CUDA_MEMCHECK, largeTensorTest, onlyNativeDeviceTypes, | ||
| 59 | get_all_device_types, skipXLA) | 58 | get_all_device_types, skipXLA) |
| 60 | import torch.backends.quantized | 59 | import torch.backends.quantized |
| 61 | import torch.testing._internal.data | 60 | import torch.testing._internal.data |
| 62 | from torch.testing._internal.common_cuda import ( | 61 | from torch.testing._internal.common_cuda import ( |
| 63 | - tf32_on_and_off, tf32_is_not_fp32, TEST_CUDNN, | 62 | + tf32_on_and_off, TEST_CUDNN, |
| 64 | _create_scaling_case, _create_scaling_models_optimizers) | 63 | _create_scaling_case, _create_scaling_models_optimizers) |
| 65 | from torch.testing._internal.common_dtype import ( | 64 | from torch.testing._internal.common_dtype import ( |
| 66 | floating_types_and, get_all_math_dtypes, all_types_and_complex_and, complex_types, | 65 | floating_types_and, get_all_math_dtypes, all_types_and_complex_and, complex_types, |
| 67 | all_types_and, floating_types, floating_and_complex_types, integral_types_and, | 66 | all_types_and, floating_types, floating_and_complex_types, integral_types_and, |
| 68 | - get_all_qint_dtypes, | 67 | + get_all_qint_dtypes, all_types_complex_float8_and, |
| 69 | ) | 68 | ) |
| 70 | 69 | ||
| 71 | TEST_MULTINPU = torch.npu.is_available() and torch.npu.device_count() >= 2 | 70 | TEST_MULTINPU = torch.npu.is_available() and torch.npu.device_count() >= 2 |
| @@ -82,8 +81,6 @@ assert torch.get_default_dtype() is torch.float32 | |||
| 82 | # sharding on sandcastle. This line silences flake warnings | 81 | # sharding on sandcastle. This line silences flake warnings |
| 83 | load_tests = load_tests | 82 | load_tests = load_tests |
| 84 | 83 | ||
| 85 | -AMPERE_OR_ROCM = TEST_WITH_ROCM or tf32_is_not_fp32() | ||
| 86 | - | ||
| 87 | DEVICE_NAME = torch_npu.npu.get_device_name(0) | 84 | DEVICE_NAME = torch_npu.npu.get_device_name(0) |
| 88 | 85 | ||
| 89 | device_is_910A = False | 86 | device_is_910A = False |
| @@ -185,6 +182,18 @@ class TestTorchDeviceType(TestCase): | |||
| 185 | scalar = bytes_to_scalar(bytes_list, dtype, device) | 182 | scalar = bytes_to_scalar(bytes_list, dtype, device) |
| 186 | self.assertEqual(scalar.storage().untyped().tolist(), bytes_list) | 183 | self.assertEqual(scalar.storage().untyped().tolist(), bytes_list) |
| 187 | 184 | ||
| 185 | + # For testing in64 support in upsample_nearest3d | ||
| 186 | + | ||
| 187 | + | ||
| 188 | + | ||
| 189 | + | ||
| 190 | + def test_int64_upsample3d(self, device, dtype): | ||
| 191 | + x = torch.ones((1, 256, 16, 720, 1280), dtype=dtype, device=device) | ||
| 192 | + try: | ||
| 193 | + torch.nn.functional.interpolate(x, scale_factor=2, mode='nearest') | ||
| 194 | + except Exception as e: | ||
| 195 | + self.fail(f"Unexpected exception raised: {e}") | ||
| 196 | + | ||
| 188 | 197 | ||
| 189 | torch.bool, torch.float32, torch.complex64, torch.float64, | 198 | torch.bool, torch.float32, torch.complex64, torch.float64, |
| 190 | torch.complex128, torch.uint16, torch.uint32, torch.uint64) | 199 | torch.complex128, torch.uint16, torch.uint32, torch.uint64) |
| @@ -247,6 +256,19 @@ class TestTorchDeviceType(TestCase): | |||
| 247 | s[2:7] = 1 | 256 | s[2:7] = 1 |
| 248 | self.assertEqual(s, storage_type(n_list)) | 257 | self.assertEqual(s, storage_type(n_list)) |
| 249 | 258 | ||
| 259 | + | ||
| 260 | + | ||
| 261 | + | ||
| 262 | + "RelWithAssert" in torch.__config__.show(), | ||
| 263 | + "failing in debug build, see github.com/pytorch/pytorch/pull/156731 for example", | ||
| 264 | + ) | ||
| 265 | + def test_storage_use_count(self, device): | ||
| 266 | + a = torch.randn(10, device=device) | ||
| 267 | + prev_cf = torch._C._storage_Use_Count(a.untyped_storage()._cdata) | ||
| 268 | + self.assertEqual(prev_cf, 1) | ||
| 269 | + b = a.view(2, 5) | ||
| 270 | + self.assertEqual(torch._C._storage_Use_Count(b.untyped_storage()._cdata), prev_cf + 1) | ||
| 271 | + | ||
| 250 | 272 | ||
| 251 | 273 | ||
| 252 | 274 | ||
| @@ -664,8 +686,8 @@ class TestTorchDeviceType(TestCase): | |||
| 664 | self.assertEqual((1,), torch.masked_select(zero_d_bool, one_d_bool).shape) | 686 | self.assertEqual((1,), torch.masked_select(zero_d_bool, one_d_bool).shape) |
| 665 | self.assertEqual((1,), torch.masked_select(one_d_bool, zero_d_bool).shape) | 687 | self.assertEqual((1,), torch.masked_select(one_d_bool, zero_d_bool).shape) |
| 666 | 688 | ||
| 667 | - zero_d_uint8 = torch.tensor(1, dtype=torch.uint8, device=device) | 689 | + torch.tensor(1, dtype=torch.uint8, device=device) |
| 668 | - one_d_uint8 = torch.tensor([1], dtype=torch.uint8, device=device) | 690 | + torch.tensor([1], dtype=torch.uint8, device=device) |
| 669 | 691 | ||
| 670 | # mode | 692 | # mode |
| 671 | self.assertEqual([(), ()], [x.shape for x in torch.mode(zero_d, dim=0, keepdim=True)]) | 693 | self.assertEqual([(), ()], [x.shape for x in torch.mode(zero_d, dim=0, keepdim=True)]) |
| @@ -968,10 +990,10 @@ class TestTorchDeviceType(TestCase): | |||
| 968 | def test_dtypetensor_warnings(self, device): | 990 | def test_dtypetensor_warnings(self, device): |
| 969 | msg = 'The torch_npu.npu.*DtypeTensor constructors are no longer recommended' | 991 | msg = 'The torch_npu.npu.*DtypeTensor constructors are no longer recommended' |
| 970 | with self.assertWarnsOnceRegex(UserWarning, msg): | 992 | with self.assertWarnsOnceRegex(UserWarning, msg): |
| 971 | - t = torch_npu.npu.FloatTensor([0]) | 993 | + torch_npu.npu.FloatTensor([0]) |
| 972 | 994 | ||
| 973 | with self.assertWarnsOnceRegex(UserWarning, msg): | 995 | with self.assertWarnsOnceRegex(UserWarning, msg): |
| 974 | - t = torch_npu.npu.DoubleTensor([0]) | 996 | + torch_npu.npu.DoubleTensor([0]) |
| 975 | 997 | ||
| 976 | def test_set_default_tensor_type_warnings(self, device): | 998 | def test_set_default_tensor_type_warnings(self, device): |
| 977 | msg = '.*is deprecated as of PyTorch 2.1, please use torch.set_default_dtype().*' | 999 | msg = '.*is deprecated as of PyTorch 2.1, please use torch.set_default_dtype().*' |
| @@ -1021,7 +1043,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1021 | stride=2, padding=2, output_padding=1).to(device) | 1043 | stride=2, padding=2, output_padding=1).to(device) |
| 1022 | 1044 | ||
| 1023 | x = torch.rand([1, 64, 8, 128, 172]).to(device) | 1045 | x = torch.rand([1, 64, 8, 128, 172]).to(device) |
| 1024 | - y = conv(x) | 1046 | + conv(x) |
| 1025 | 1047 | ||
| 1026 | def test_is_set_to(self, device): | 1048 | def test_is_set_to(self, device): |
| 1027 | t1 = torch.empty(3, 4, 9, 10, device=device) | 1049 | t1 = torch.empty(3, 4, 9, 10, device=device) |
| @@ -1053,7 +1075,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1053 | self.assertFalse(t1.is_set_to(t2)) | 1075 | self.assertFalse(t1.is_set_to(t2)) |
| 1054 | self.assertFalse(t2.is_set_to(t1)) | 1076 | self.assertFalse(t2.is_set_to(t1)) |
| 1055 | 1077 | ||
| 1056 | - @skipIfMps | 1078 | + @skipIfMPS |
| 1057 | 1079 | ||
| 1058 | 1080 | ||
| 1059 | "fn", | 1081 | "fn", |
| @@ -1305,7 +1327,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1305 | max_val = torch.iinfo(dtype).max | 1327 | max_val = torch.iinfo(dtype).max |
| 1306 | self.assertTrue(res.eq(max_val).all()) | 1328 | self.assertTrue(res.eq(max_val).all()) |
| 1307 | 1329 | ||
| 1308 | - @skipIfMps | 1330 | + @skipIfMPS |
| 1309 | 1331 | ||
| 1310 | def test_nondeterministic_alert_AvgPool3d(self, device): | 1332 | def test_nondeterministic_alert_AvgPool3d(self, device): |
| 1311 | module = torch.nn.AvgPool3d(3) | 1333 | module = torch.nn.AvgPool3d(3) |
| @@ -1318,7 +1340,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1318 | 'avg_pool3d_backward_npu', | 1340 | 'avg_pool3d_backward_npu', |
| 1319 | torch.device(device).type == 'npu') | 1341 | torch.device(device).type == 'npu') |
| 1320 | 1342 | ||
| 1321 | - @skipIfMps | 1343 | + @skipIfMPS |
| 1322 | 1344 | ||
| 1323 | def test_nondeterministic_alert_AdaptiveAvgPool2d(self, device): | 1345 | def test_nondeterministic_alert_AdaptiveAvgPool2d(self, device): |
| 1324 | module = torch.nn.AdaptiveAvgPool2d(3) | 1346 | module = torch.nn.AdaptiveAvgPool2d(3) |
| @@ -1331,7 +1353,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1331 | 'adaptive_avg_pool2d_backward_npu', | 1353 | 'adaptive_avg_pool2d_backward_npu', |
| 1332 | torch.device(device).type == 'npu') | 1354 | torch.device(device).type == 'npu') |
| 1333 | 1355 | ||
| 1334 | - @skipIfMps | 1356 | + @skipIfMPS |
| 1335 | 1357 | ||
| 1336 | def test_nondeterministic_alert_AdaptiveAvgPool3d(self, device): | 1358 | def test_nondeterministic_alert_AdaptiveAvgPool3d(self, device): |
| 1337 | module = torch.nn.AdaptiveAvgPool3d(3) | 1359 | module = torch.nn.AdaptiveAvgPool3d(3) |
| @@ -1344,7 +1366,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1344 | 'adaptive_avg_pool3d_backward_npu', | 1366 | 'adaptive_avg_pool3d_backward_npu', |
| 1345 | torch.device(device).type == 'npu') | 1367 | torch.device(device).type == 'npu') |
| 1346 | 1368 | ||
| 1347 | - @skipIfMps | 1369 | + @skipIfMPS |
| 1348 | 1370 | ||
| 1349 | def test_nondeterministic_alert_MaxPool3d(self, device): | 1371 | def test_nondeterministic_alert_MaxPool3d(self, device): |
| 1350 | module = torch.nn.MaxPool3d(3) | 1372 | module = torch.nn.MaxPool3d(3) |
| @@ -1357,7 +1379,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1357 | 'max_pool3d_with_indices_backward_npu', | 1379 | 'max_pool3d_with_indices_backward_npu', |
| 1358 | torch.device(device).type == 'npu') | 1380 | torch.device(device).type == 'npu') |
| 1359 | 1381 | ||
| 1360 | - @skipIfMps | 1382 | + @skipIfMPS |
| 1361 | 1383 | ||
| 1362 | def test_nondeterministic_alert_AdaptiveMaxPool2d(self, device): | 1384 | def test_nondeterministic_alert_AdaptiveMaxPool2d(self, device): |
| 1363 | module = torch.nn.AdaptiveMaxPool2d(3) | 1385 | module = torch.nn.AdaptiveMaxPool2d(3) |
| @@ -1370,7 +1392,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1370 | 'adaptive_max_pool2d_backward_npu', | 1392 | 'adaptive_max_pool2d_backward_npu', |
| 1371 | torch.device(device).type == 'npu') | 1393 | torch.device(device).type == 'npu') |
| 1372 | 1394 | ||
| 1373 | - @skipIfMps | 1395 | + @skipIfMPS |
| 1374 | 1396 | ||
| 1375 | def test_nondeterministic_alert_FractionalMaxPool2d(self, device): | 1397 | def test_nondeterministic_alert_FractionalMaxPool2d(self, device): |
| 1376 | module = torch.nn.FractionalMaxPool2d(2, output_ratio=0.5) | 1398 | module = torch.nn.FractionalMaxPool2d(2, output_ratio=0.5) |
| @@ -1383,7 +1405,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1383 | 'fractional_max_pool2d_backward_npu', | 1405 | 'fractional_max_pool2d_backward_npu', |
| 1384 | torch.device(device).type == 'npu') | 1406 | torch.device(device).type == 'npu') |
| 1385 | 1407 | ||
| 1386 | - @skipIfMps | 1408 | + @skipIfMPS |
| 1387 | 1409 | ||
| 1388 | def test_nondeterministic_alert_FractionalMaxPool3d(self, device): | 1410 | def test_nondeterministic_alert_FractionalMaxPool3d(self, device): |
| 1389 | module = torch.nn.FractionalMaxPool3d(2, output_ratio=0.5) | 1411 | module = torch.nn.FractionalMaxPool3d(2, output_ratio=0.5) |
| @@ -1438,7 +1460,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1438 | lambda: module(input_, indices), | 1460 | lambda: module(input_, indices), |
| 1439 | 'max_unpooling3d_forward_out') | 1461 | 'max_unpooling3d_forward_out') |
| 1440 | 1462 | ||
| 1441 | - @skipIfMps | 1463 | + @skipIfMPS |
| 1442 | 1464 | ||
| 1443 | def test_nondeterministic_alert_interpolate_linear(self, device): | 1465 | def test_nondeterministic_alert_interpolate_linear(self, device): |
| 1444 | input_ = torch.randn(1, 2, 4, device=device, requires_grad=True) | 1466 | input_ = torch.randn(1, 2, 4, device=device, requires_grad=True) |
| @@ -1518,7 +1540,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1518 | self.assertEqual(grad, input_.grad, atol=0, rtol=0) | 1540 | self.assertEqual(grad, input_.grad, atol=0, rtol=0) |
| 1519 | input_.grad = None | 1541 | input_.grad = None |
| 1520 | 1542 | ||
| 1521 | - @skipIfMps | 1543 | + @skipIfMPS |
| 1522 | 1544 | ||
| 1523 | def test_nondeterministic_alert_interpolate_bicubic(self, device): | 1545 | def test_nondeterministic_alert_interpolate_bicubic(self, device): |
| 1524 | input_ = torch.randn(1, 2, 4, 4, device=device, requires_grad=True) | 1546 | input_ = torch.randn(1, 2, 4, 4, device=device, requires_grad=True) |
| @@ -1534,7 +1556,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1534 | 'upsample_bicubic2d_backward_out_npu', | 1556 | 'upsample_bicubic2d_backward_out_npu', |
| 1535 | torch.device(device).type == 'npu') | 1557 | torch.device(device).type == 'npu') |
| 1536 | 1558 | ||
| 1537 | - @skipIfMps | 1559 | + @skipIfMPS |
| 1538 | 1560 | ||
| 1539 | def test_nondeterministic_alert_interpolate_trilinear(self, device): | 1561 | def test_nondeterministic_alert_interpolate_trilinear(self, device): |
| 1540 | input_ = torch.randn(1, 2, 4, 4, 4, device=device, requires_grad=True) | 1562 | input_ = torch.randn(1, 2, 4, 4, 4, device=device, requires_grad=True) |
| @@ -1550,7 +1572,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1550 | 'upsample_trilinear3d_backward_out_npu', | 1572 | 'upsample_trilinear3d_backward_out_npu', |
| 1551 | torch.device(device).type == 'npu') | 1573 | torch.device(device).type == 'npu') |
| 1552 | 1574 | ||
| 1553 | - @skipIfMps | 1575 | + @skipIfMPS |
| 1554 | 1576 | ||
| 1555 | def test_nondeterministic_alert_ReflectionPad1d(self, device): | 1577 | def test_nondeterministic_alert_ReflectionPad1d(self, device): |
| 1556 | module = torch.nn.ReflectionPad1d((1, 2)) | 1578 | module = torch.nn.ReflectionPad1d((1, 2)) |
| @@ -1563,19 +1585,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1563 | 'reflection_pad1d_backward_out_npu', | 1585 | 'reflection_pad1d_backward_out_npu', |
| 1564 | torch.device(device).type == 'npu') | 1586 | torch.device(device).type == 'npu') |
| 1565 | 1587 | ||
| 1566 | - @skipIfTorchInductor("pytorch issues 113707") | 1588 | + @skipIfMPS |
| 1567 | - def test_nondeterministic_alert_ReflectionPad2d(self, device): | ||
| 1568 | - module = torch.nn.ReflectionPad2d((1, 2, 3, 4)) | ||
| 1569 | - input_ = torch.randn(2, 3, 8, 8, device=device, requires_grad=True) | ||
| 1570 | - res = module(input_) | ||
| 1571 | - grad = torch.ones_like(res) | ||
| 1572 | - | ||
| 1573 | - self.check_nondeterministic_alert( | ||
| 1574 | - lambda: res.backward(grad, retain_graph=True), | ||
| 1575 | - 'reflection_pad2d_backward_npu', | ||
| 1576 | - torch.device(device).type == 'npu') | ||
| 1577 | - | ||
| 1578 | - | ||
| 1579 | 1589 | ||
| 1580 | def test_nondeterministic_alert_ReflectionPad3d(self, device): | 1590 | def test_nondeterministic_alert_ReflectionPad3d(self, device): |
| 1581 | module = torch.nn.ReflectionPad3d((1, 2, 3, 4, 5, 6)) | 1591 | module = torch.nn.ReflectionPad3d((1, 2, 3, 4, 5, 6)) |
| @@ -1588,7 +1598,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1588 | 'reflection_pad3d_backward_out_npu', | 1598 | 'reflection_pad3d_backward_out_npu', |
| 1589 | torch.device(device).type == 'npu') | 1599 | torch.device(device).type == 'npu') |
| 1590 | 1600 | ||
| 1591 | - @skipIfMps | 1601 | + @skipIfMPS |
| 1592 | 1602 | ||
| 1593 | def test_nondeterministic_alert_ReplicationPad1d(self, device): | 1603 | def test_nondeterministic_alert_ReplicationPad1d(self, device): |
| 1594 | module = torch.nn.ReplicationPad1d((1, 2)) | 1604 | module = torch.nn.ReplicationPad1d((1, 2)) |
| @@ -1627,7 +1637,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1627 | 'replication_pad2d_backward_npu', | 1637 | 'replication_pad2d_backward_npu', |
| 1628 | False) | 1638 | False) |
| 1629 | 1639 | ||
| 1630 | - @skipIfMps | 1640 | + @skipIfMPS |
| 1631 | 1641 | ||
| 1632 | def test_nondeterministic_alert_ReplicationPad3d(self, device): | 1642 | def test_nondeterministic_alert_ReplicationPad3d(self, device): |
| 1633 | module = torch.nn.ReplicationPad3d((1, 2, 3, 4, 5, 6)) | 1643 | module = torch.nn.ReplicationPad3d((1, 2, 3, 4, 5, 6)) |
| @@ -1686,10 +1696,12 @@ class TestTorchDeviceType(TestCase): | |||
| 1686 | def test_deterministic_cumsum(self, device): | 1696 | def test_deterministic_cumsum(self, device): |
| 1687 | test_cases = [ | 1697 | test_cases = [ |
| 1688 | # size, dim | 1698 | # size, dim |
| 1689 | - [(2, 3, 4), 0], | 1699 | + [(1025,), 0], |
| 1690 | - [(2, 3, 4), 1], | 1700 | + [(8193,), 0], |
| 1691 | - [(2, 3, 4), 2], | 1701 | + [(8191,), 0], |
| 1692 | - [(1000, 10, 2), 0], | 1702 | + [(128256,), 0], |
| 1703 | + [(1282560,), 0], | ||
| 1704 | + [(12825600,), 0], | ||
| 1693 | ] | 1705 | ] |
| 1694 | for size, dim in test_cases: | 1706 | for size, dim in test_cases: |
| 1695 | input_ = 100 * torch.randn(*size, device=device) | 1707 | input_ = 100 * torch.randn(*size, device=device) |
| @@ -1701,18 +1713,32 @@ class TestTorchDeviceType(TestCase): | |||
| 1701 | 1713 | ||
| 1702 | res_cpu = input_.cpu().cumsum(dim) | 1714 | res_cpu = input_.cpu().cumsum(dim) |
| 1703 | self.assertEqual(res0, res_cpu, atol=1e-3, rtol=1e-2) | 1715 | self.assertEqual(res0, res_cpu, atol=1e-3, rtol=1e-2) |
| 1716 | + num_sm = 132 | ||
| 1717 | + elems_per_cta = 256 * 16 | ||
| 1718 | + N = num_sm * elems_per_cta | ||
| 1719 | + input = torch.rand(N, dtype=torch.complex128, device=device) | ||
| 1720 | + with DeterministicGuard(True): | ||
| 1721 | + res0 = input.cumsum(dim) | ||
| 1722 | + for _ in range(3): | ||
| 1723 | + res1 = input.cumsum(dim) | ||
| 1724 | + self.assertEqual(res0, res1, atol=0, rtol=0) | ||
| 1704 | 1725 | ||
| 1705 | - @dtypes(*all_types_and_complex_and(torch.bool)) | 1726 | + res_cpu = input.cpu().cumsum(dim) |
| 1706 | - @skipIfTorchInductor("pytorch issues 113707") | 1727 | + self.assertEqual(res0, res_cpu, atol=1e-3, rtol=1e-2) |
| 1707 | - def test_nondeterministic_alert_cumsum(self, device, dtype): | ||
| 1708 | - input_ = make_tensor((10,), dtype=dtype, device=device, low=-9, high=9) | ||
| 1709 | - should_alert = False | ||
| 1710 | 1728 | ||
| 1711 | - for op_call in [torch.Tensor.cumsum, torch.cumsum]: | 1729 | + @onlyPRIVATEUSE1 |
| 1712 | - self.check_nondeterministic_alert( | 1730 | + @largeTensorTest('49GB') |
| 1713 | - lambda: op_call(input_, 0), | 1731 | + def test_cumsum_64bit_indexing(self, device): |
| 1714 | - 'cumsum_npu_kernel', | 1732 | + b = torch.ones(2 * 4096 * 8, 100000, dtype=torch.float, device='npu') |
| 1715 | - should_alert) | 1733 | + b /= 100000 |
| 1734 | + d = b.cumsum(dim=-1) | ||
| 1735 | + chunk = 2**30 // b.shape[-1] | ||
| 1736 | + for i in range(0, b.shape[0], chunk): | ||
| 1737 | + end = min(i + chunk, b.shape[0]) | ||
| 1738 | + b[i:end, :].cumsum_(dim=-1) | ||
| 1739 | + # cheat a bit to avoid OOM | ||
| 1740 | + self.assertEqual(b[0, :], d[0, :], atol=3e-5, rtol=3e-5) | ||
| 1741 | + self.assertEqual(b[-1, :], d[-1, :], atol=3e-5, rtol=3e-5) | ||
| 1716 | 1742 | ||
| 1717 | # expected a non-determinitic error, but it was not raised | 1743 | # expected a non-determinitic error, but it was not raised |
| 1718 | 1744 | ||
| @@ -1741,16 +1767,18 @@ class TestTorchDeviceType(TestCase): | |||
| 1741 | 'put_', | 1767 | 'put_', |
| 1742 | torch.device(device).type == 'npu') | 1768 | torch.device(device).type == 'npu') |
| 1743 | 1769 | ||
| 1744 | - @skipIfMps | 1770 | + @dtypes(torch.float32) |
| 1745 | - def test_nondeterministic_alert_histc(self, device): | 1771 | + @dtypesIfPRIVATEUSE1(torch.float32, torch.int32) |
| 1746 | - a = torch.tensor([], device=device) | 1772 | + @skipIfMPS |
| 1773 | + def test_nondeterministic_alert_histc(self, device, dtype): | ||
| 1774 | + a = torch.tensor([], device=device, dtype=dtype) | ||
| 1747 | for op_call in [torch.histc, torch.Tensor.histc]: | 1775 | for op_call in [torch.histc, torch.Tensor.histc]: |
| 1748 | self.check_nondeterministic_alert( | 1776 | self.check_nondeterministic_alert( |
| 1749 | lambda: op_call(a, min=0, max=3), | 1777 | lambda: op_call(a, min=0, max=3), |
| 1750 | - '_histc_npu', | 1778 | + '_histc_npu with floating point input', |
| 1751 | - torch.device(device).type == 'npu') | 1779 | + torch.device(device).type == 'npu' and dtype.is_floating_point) |
| 1752 | 1780 | ||
| 1753 | - @skipIfMps | 1781 | + @skipIfMPS |
| 1754 | def test_nondeterministic_alert_bincount(self, device): | 1782 | def test_nondeterministic_alert_bincount(self, device): |
| 1755 | a = torch.tensor([], device=device, dtype=torch.long) | 1783 | a = torch.tensor([], device=device, dtype=torch.long) |
| 1756 | weights = torch.tensor([], device=device) | 1784 | weights = torch.tensor([], device=device) |
| @@ -1792,7 +1820,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1792 | 'kthvalue NPU', | 1820 | 'kthvalue NPU', |
| 1793 | torch.device(device).type == 'npu') | 1821 | torch.device(device).type == 'npu') |
| 1794 | 1822 | ||
| 1795 | - @skipIfMps | 1823 | + @skipIfMPS |
| 1796 | 1824 | ||
| 1797 | def test_nondeterministic_alert_grid_sample_2d(self, device): | 1825 | def test_nondeterministic_alert_grid_sample_2d(self, device): |
| 1798 | input_ = torch.empty(1, 1, 2, 2, device=device, requires_grad=True) | 1826 | input_ = torch.empty(1, 1, 2, 2, device=device, requires_grad=True) |
| @@ -1805,7 +1833,7 @@ class TestTorchDeviceType(TestCase): | |||
| 1805 | 'grid_sampler_2d_backward_npu', | 1833 | 'grid_sampler_2d_backward_npu', |
| 1806 | torch.device(device).type == 'npu') | 1834 | torch.device(device).type == 'npu') |
| 1807 | 1835 | ||
| 1808 | - @skipIfMps | 1836 | + @skipIfMPS |
| 1809 | 1837 | ||
| 1810 | def test_nondeterministic_alert_grid_sample_3d(self, device): | 1838 | def test_nondeterministic_alert_grid_sample_3d(self, device): |
| 1811 | input_ = torch.empty(1, 1, 2, 2, 2, device=device, requires_grad=True) | 1839 | input_ = torch.empty(1, 1, 2, 2, 2, device=device, requires_grad=True) |
| @@ -2042,20 +2070,20 @@ class TestTorchDeviceType(TestCase): | |||
| 2042 | _sync_raises_helper(f, level) | 2070 | _sync_raises_helper(f, level) |
| 2043 | 2071 | ||
| 2044 | 2072 | ||
| 2045 | - @skipIfMps | 2073 | + @skipIfMPS |
| 2046 | def test_log_normal(self, device, dtype): | 2074 | def test_log_normal(self, device, dtype): |
| 2047 | a = torch.tensor([10], dtype=dtype, device=device).log_normal_() | 2075 | a = torch.tensor([10], dtype=dtype, device=device).log_normal_() |
| 2048 | self.assertEqual(a.dtype, dtype) | 2076 | self.assertEqual(a.dtype, dtype) |
| 2049 | self.assertEqual(a.size(), torch.Size([1])) | 2077 | self.assertEqual(a.size(), torch.Size([1])) |
| 2050 | 2078 | ||
| 2051 | 2079 | ||
| 2052 | - @skipIfMps | 2080 | + @skipIfMPS |
| 2053 | def test_geometric(self, device, dtype): | 2081 | def test_geometric(self, device, dtype): |
| 2054 | a = torch.tensor([10], dtype=dtype, device=device).geometric_(0.5) | 2082 | a = torch.tensor([10], dtype=dtype, device=device).geometric_(0.5) |
| 2055 | self.assertEqual(a.dtype, dtype) | 2083 | self.assertEqual(a.dtype, dtype) |
| 2056 | self.assertEqual(a.size(), torch.Size([1])) | 2084 | self.assertEqual(a.size(), torch.Size([1])) |
| 2057 | 2085 | ||
| 2058 | - @skipIfMps | 2086 | + @skipIfMPS |
| 2059 | def test_repeat_interleave(self, device): | 2087 | def test_repeat_interleave(self, device): |
| 2060 | y = torch.tensor([[1, 2], [3, 4]], device=device) | 2088 | y = torch.tensor([[1, 2], [3, 4]], device=device) |
| 2061 | # exercise single argument function signature | 2089 | # exercise single argument function signature |
| @@ -2146,7 +2174,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2146 | self.assertEqual(num_zeros, 0) | 2174 | self.assertEqual(num_zeros, 0) |
| 2147 | 2175 | ||
| 2148 | 2176 | ||
| 2149 | - @skipIfMps | 2177 | + @skipIfMPS |
| 2150 | def test_exponential(self, device, dtype): | 2178 | def test_exponential(self, device, dtype): |
| 2151 | a = torch.tensor([10], dtype=dtype, device=device).exponential_(0.5) | 2179 | a = torch.tensor([10], dtype=dtype, device=device).exponential_(0.5) |
| 2152 | self.assertEqual(a.dtype, dtype) | 2180 | self.assertEqual(a.dtype, dtype) |
| @@ -2236,7 +2264,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2236 | res = stats.kstest(t.cpu().to(torch.double), 'norm', args=(mean, std)) | 2264 | res = stats.kstest(t.cpu().to(torch.double), 'norm', args=(mean, std)) |
| 2237 | self.assertTrue(res.statistic < 0.1) | 2265 | self.assertTrue(res.statistic < 0.1) |
| 2238 | 2266 | ||
| 2239 | - @skipIfMps | 2267 | + @skipIfMPS |
| 2240 | 2268 | ||
| 2241 | 2269 | ||
| 2242 | 2270 | ||
| @@ -2252,7 +2280,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2252 | else: | 2280 | else: |
| 2253 | self.assertTrue(res.statistic < 0.1) | 2281 | self.assertTrue(res.statistic < 0.1) |
| 2254 | 2282 | ||
| 2255 | - @skipIfMps | 2283 | + @skipIfMPS |
| 2256 | 2284 | ||
| 2257 | 2285 | ||
| 2258 | def test_exponential_kstest(self, device, dtype): | 2286 | def test_exponential_kstest(self, device, dtype): |
| @@ -2263,7 +2291,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2263 | res = stats.kstest(t.cpu().to(torch.double), 'expon', args=(0, 1 / lambd,)) | 2291 | res = stats.kstest(t.cpu().to(torch.double), 'expon', args=(0, 1 / lambd,)) |
| 2264 | self.assertTrue(res.statistic < 0.1) | 2292 | self.assertTrue(res.statistic < 0.1) |
| 2265 | 2293 | ||
| 2266 | - @skipIfMps | 2294 | + @skipIfMPS |
| 2267 | 2295 | ||
| 2268 | 2296 | ||
| 2269 | 2297 | ||
| @@ -2300,7 +2328,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2300 | with self.assertRaises(RuntimeError): | 2328 | with self.assertRaises(RuntimeError): |
| 2301 | torch.empty((1,), device=device, dtype=dtype).cauchy_(0.0, 0.0) | 2329 | torch.empty((1,), device=device, dtype=dtype).cauchy_(0.0, 0.0) |
| 2302 | 2330 | ||
| 2303 | - @skipIfMps | 2331 | + @skipIfMPS |
| 2304 | 2332 | ||
| 2305 | 2333 | ||
| 2306 | 2334 | ||
| @@ -2365,7 +2393,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2365 | return torch.empty(r1, r2, device=x.device) | 2393 | return torch.empty(r1, r2, device=x.device) |
| 2366 | return torch.norm(x[..., None, :] - y[..., None, :, :], p=p, dim=-1) | 2394 | return torch.norm(x[..., None, :] - y[..., None, :, :], p=p, dim=-1) |
| 2367 | 2395 | ||
| 2368 | - @skipIfMps | 2396 | + @skipIfMPS |
| 2369 | def test_cdist_norm(self, device): | 2397 | def test_cdist_norm(self, device): |
| 2370 | for r1 in [3, 4, 5, 6]: | 2398 | for r1 in [3, 4, 5, 6]: |
| 2371 | for m in [2, 3, 4, 10]: | 2399 | for m in [2, 3, 4, 10]: |
| @@ -2383,7 +2411,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2383 | expected = self._brute_cdist(x, y, p=p) | 2411 | expected = self._brute_cdist(x, y, p=p) |
| 2384 | self.assertEqual(expected, actual) | 2412 | self.assertEqual(expected, actual) |
| 2385 | 2413 | ||
| 2386 | - @skipIfMps | 2414 | + @skipIfMPS |
| 2387 | def test_cdist_norm_batch(self, device): | 2415 | def test_cdist_norm_batch(self, device): |
| 2388 | for r1 in [3, 4, 5, 6]: | 2416 | for r1 in [3, 4, 5, 6]: |
| 2389 | for m in [2, 3, 4, 10]: | 2417 | for m in [2, 3, 4, 10]: |
| @@ -2425,7 +2453,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2425 | self.assertEqual(y1.grad, y2.grad, rtol=0, atol=0.001) | 2453 | self.assertEqual(y1.grad, y2.grad, rtol=0, atol=0.001) |
| 2426 | 2454 | ||
| 2427 | 2455 | ||
| 2428 | - @bf32_on_and_off(0.005) | 2456 | + @bf32_on_and_off(0.08) |
| 2429 | def test_cdist_large(self, device): | 2457 | def test_cdist_large(self, device): |
| 2430 | for cm in ['use_mm_for_euclid_dist_if_necessary', 'use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: | 2458 | for cm in ['use_mm_for_euclid_dist_if_necessary', 'use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: |
| 2431 | x = torch.randn(1000, 10, device=device) | 2459 | x = torch.randn(1000, 10, device=device) |
| @@ -2436,7 +2464,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2436 | 2464 | ||
| 2437 | 2465 | ||
| 2438 | 2466 | ||
| 2439 | - @bf32_on_and_off(0.01) | 2467 | + @bf32_on_and_off(0.08) |
| 2440 | def test_cdist_large_batch(self, device): | 2468 | def test_cdist_large_batch(self, device): |
| 2441 | for cm in ['use_mm_for_euclid_dist_if_necessary', 'use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: | 2469 | for cm in ['use_mm_for_euclid_dist_if_necessary', 'use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: |
| 2442 | x = torch.randn(4, 3, 1000, 10, device=device) | 2470 | x = torch.randn(4, 3, 1000, 10, device=device) |
| @@ -2446,7 +2474,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2446 | self.assertEqual(expected, actual) | 2474 | self.assertEqual(expected, actual) |
| 2447 | 2475 | ||
| 2448 | 2476 | ||
| 2449 | - @bf32_on_and_off(0.005) | 2477 | + @bf32_on_and_off(0.04) |
| 2450 | def test_cdist_non_contiguous(self, device): | 2478 | def test_cdist_non_contiguous(self, device): |
| 2451 | for cm in ['use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: | 2479 | for cm in ['use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: |
| 2452 | x = torch.randn(5, 7, device=device).mT | 2480 | x = torch.randn(5, 7, device=device).mT |
| @@ -2474,7 +2502,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2474 | self.assertEqual(expected, actual) | 2502 | self.assertEqual(expected, actual) |
| 2475 | 2503 | ||
| 2476 | 2504 | ||
| 2477 | - @bf32_on_and_off(0.005) | 2505 | + @bf32_on_and_off(0.04) |
| 2478 | def test_cdist_non_contiguous_batch(self, device): | 2506 | def test_cdist_non_contiguous_batch(self, device): |
| 2479 | for cm in ['use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: | 2507 | for cm in ['use_mm_for_euclid_dist', 'donot_use_mm_for_euclid_dist']: |
| 2480 | x = torch.randn(4, 3, 2, 5, 7, device=device).mT | 2508 | x = torch.randn(4, 3, 2, 5, 7, device=device).mT |
| @@ -2522,11 +2550,11 @@ class TestTorchDeviceType(TestCase): | |||
| 2522 | _test_euclidean_large_cdist((2000, 5)) | 2550 | _test_euclidean_large_cdist((2000, 5)) |
| 2523 | 2551 | ||
| 2524 | # Ensure that cdist backward with p<1 does not produce NaNs | 2552 | # Ensure that cdist backward with p<1 does not produce NaNs |
| 2525 | - @skipIfMps | 2553 | + @skipIfMPS |
| 2526 | def test_cdist_grad_p_lt_1_no_nan(self, device): | 2554 | def test_cdist_grad_p_lt_1_no_nan(self, device): |
| 2527 | for p in [0.99, 0.7, 0.5, 0.1, 0.01]: | 2555 | for p in [0.99, 0.7, 0.5, 0.1, 0.01]: |
| 2528 | x = torch.randn(1, 2, device=device) | 2556 | x = torch.randn(1, 2, device=device) |
| 2529 | - y = x.clone().detach() + torch.tensor([[1., 0.]], device=device) | 2557 | + y = x.detach().clone() + torch.tensor([[1., 0.]], device=device) |
| 2530 | x.requires_grad = True | 2558 | x.requires_grad = True |
| 2531 | y.requires_grad = True | 2559 | y.requires_grad = True |
| 2532 | result = torch.cdist(x, y, p=p) | 2560 | result = torch.cdist(x, y, p=p) |
| @@ -2542,7 +2570,6 @@ class TestTorchDeviceType(TestCase): | |||
| 2542 | x = torch.randn(sizex, device=device, dtype=torch.float) | 2570 | x = torch.randn(sizex, device=device, dtype=torch.float) |
| 2543 | dist_grad = torch.randn((1, 27, 27), device=device, dtype=torch.float) | 2571 | dist_grad = torch.randn((1, 27, 27), device=device, dtype=torch.float) |
| 2544 | y = x.clone() | 2572 | y = x.clone() |
| 2545 | - eps = 1e-6 | ||
| 2546 | x.requires_grad = True | 2573 | x.requires_grad = True |
| 2547 | d = torch.cdist(x, y) | 2574 | d = torch.cdist(x, y) |
| 2548 | d.backward(dist_grad) | 2575 | d.backward(dist_grad) |
| @@ -2550,7 +2577,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2550 | # values such as nan or inf | 2577 | # values such as nan or inf |
| 2551 | assert torch.isfinite(x.grad).all() | 2578 | assert torch.isfinite(x.grad).all() |
| 2552 | 2579 | ||
| 2553 | - @skipIfMps | 2580 | + @skipIfMPS |
| 2554 | def test_cumsum(self, device): | 2581 | def test_cumsum(self, device): |
| 2555 | x = torch.rand(100, 100, device=device) | 2582 | x = torch.rand(100, 100, device=device) |
| 2556 | res1 = torch.cumsum(x, 1) | 2583 | res1 = torch.cumsum(x, 1) |
| @@ -2601,7 +2628,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2601 | # Check that output maintained correct shape | 2628 | # Check that output maintained correct shape |
| 2602 | self.assertEqual(raw_tensor.shape, raw_tensor.grad.shape) | 2629 | self.assertEqual(raw_tensor.shape, raw_tensor.grad.shape) |
| 2603 | 2630 | ||
| 2604 | - @skipIfMps | 2631 | + @skipIfMPS |
| 2605 | def test_cumprod(self, device): | 2632 | def test_cumprod(self, device): |
| 2606 | x = torch.rand(100, 100, device=device) | 2633 | x = torch.rand(100, 100, device=device) |
| 2607 | res1 = torch.cumprod(x, 1) | 2634 | res1 = torch.cumprod(x, 1) |
| @@ -2653,7 +2680,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2653 | # Check that output maintained correct shape | 2680 | # Check that output maintained correct shape |
| 2654 | self.assertEqual(raw_tensor.shape, raw_tensor.grad.shape) | 2681 | self.assertEqual(raw_tensor.shape, raw_tensor.grad.shape) |
| 2655 | 2682 | ||
| 2656 | - @skipIfMps | 2683 | + @skipIfMPS |
| 2657 | def test_cummax_cummin(self, device): | 2684 | def test_cummax_cummin(self, device): |
| 2658 | def test_ops(op, string_of_function_name, expected_output1, expected_output2): | 2685 | def test_ops(op, string_of_function_name, expected_output1, expected_output2): |
| 2659 | x = torch.rand(100, 100, device=device) | 2686 | x = torch.rand(100, 100, device=device) |
| @@ -2688,6 +2715,14 @@ class TestTorchDeviceType(TestCase): | |||
| 2688 | 'expected scalar_type Float but found Short'): | 2715 | 'expected scalar_type Float but found Short'): |
| 2689 | op(t, 0, out=(values, indices)) | 2716 | op(t, 0, out=(values, indices)) |
| 2690 | 2717 | ||
| 2718 | + # Range-check 0-d tensors | ||
| 2719 | + x = torch.rand([]) | ||
| 2720 | + dim = 100 | ||
| 2721 | + with self.assertRaisesRegex( | ||
| 2722 | + IndexError, | ||
| 2723 | + 'Expected reduction dim -1 or 0 for scalar but got 100'): | ||
| 2724 | + op(x, dim) | ||
| 2725 | + | ||
| 2691 | # Check that op over a zero length dimension doesn't crash on backprop. | 2726 | # Check that op over a zero length dimension doesn't crash on backprop. |
| 2692 | # Also check that op over other dimensions in a tensor with a zero-length | 2727 | # Also check that op over other dimensions in a tensor with a zero-length |
| 2693 | # dimension also works | 2728 | # dimension also works |
| @@ -2720,7 +2755,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2720 | [0, 0, 0], | 2755 | [0, 0, 0], |
| 2721 | [0, 0, 0]]), expected_out) | 2756 | [0, 0, 0]]), expected_out) |
| 2722 | 2757 | ||
| 2723 | - @skipIfMps | 2758 | + @skipIfMPS |
| 2724 | 2759 | ||
| 2725 | def test_logcumsumexp(self, device): | 2760 | def test_logcumsumexp(self, device): |
| 2726 | def logcumsumexp(a, axis): | 2761 | def logcumsumexp(a, axis): |
| @@ -2849,7 +2884,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2849 | 2884 | ||
| 2850 | # if the given input arg is not a list, it returns a list of single element: [arg] | 2885 | # if the given input arg is not a list, it returns a list of single element: [arg] |
| 2851 | def _wrap_to_list(self, input_array): | 2886 | def _wrap_to_list(self, input_array): |
| 2852 | - return input_array if isinstance(input_array, list) else [input_array] | 2887 | + return list(input_array) if isinstance(input_array, (list, tuple)) else [input_array] |
| 2853 | 2888 | ||
| 2854 | # To ensure inf, -inf, and nan values do not cause divergence between Numpy and PyTorch. | 2889 | # To ensure inf, -inf, and nan values do not cause divergence between Numpy and PyTorch. |
| 2855 | # There are two types of possible divergence: | 2890 | # There are two types of possible divergence: |
| @@ -2987,7 +3022,7 @@ class TestTorchDeviceType(TestCase): | |||
| 2987 | # Result is given just as real number and all the imaginary parts to be equal to zero. | 3022 | # Result is given just as real number and all the imaginary parts to be equal to zero. |
| 2988 | self.assertEqual(expected[i].imag, torch.zeros(actual[i].shape), exact_dtype=False) | 3023 | self.assertEqual(expected[i].imag, torch.zeros(actual[i].shape), exact_dtype=False) |
| 2989 | else: | 3024 | else: |
| 2990 | - actual, expected = self._inf_nan_preprocess(list(actual), expected) | 3025 | + actual, expected = self._inf_nan_preprocess(list(actual), list(expected)) |
| 2991 | self.assertEqual(actual, expected, equal_nan=True, exact_dtype=False) | 3026 | self.assertEqual(actual, expected, equal_nan=True, exact_dtype=False) |
| 2992 | 3027 | ||
| 2993 | 3028 | ||
| @@ -3051,7 +3086,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3051 | self._test_large_cum_fn_helper(x, lambda x: torch.cumprod(x, 0)) | 3086 | self._test_large_cum_fn_helper(x, lambda x: torch.cumprod(x, 0)) |
| 3052 | 3087 | ||
| 3053 | 3088 | ||
| 3054 | - @skipIfMps | 3089 | + @skipIfMPS |
| 3055 | def test_discontiguous_out_cumsum(self, device): | 3090 | def test_discontiguous_out_cumsum(self, device): |
| 3056 | x = torch.randn(4, 8, device=device) | 3091 | x = torch.randn(4, 8, device=device) |
| 3057 | y = torch.empty(4, 16, device=device)[:, ::2] | 3092 | y = torch.empty(4, 16, device=device)[:, ::2] |
| @@ -3075,14 +3110,14 @@ class TestTorchDeviceType(TestCase): | |||
| 3075 | self.assertEqual(out_val, expected_val, atol=0, rtol=0) | 3110 | self.assertEqual(out_val, expected_val, atol=0, rtol=0) |
| 3076 | self.assertEqual(out_ind, expected_ind, atol=0, rtol=0) | 3111 | self.assertEqual(out_ind, expected_ind, atol=0, rtol=0) |
| 3077 | 3112 | ||
| 3078 | - @skipIfMps | 3113 | + @skipIfMPS |
| 3079 | def test_cummax_discontiguous(self, device): | 3114 | def test_cummax_discontiguous(self, device): |
| 3080 | x = torch.tensor([[0, 1, 2, 3, 2, 1], [4, 5, 6, 5, 6, 7]], device=device, dtype=torch.float).t().contiguous().t() | 3115 | x = torch.tensor([[0, 1, 2, 3, 2, 1], [4, 5, 6, 5, 6, 7]], device=device, dtype=torch.float).t().contiguous().t() |
| 3081 | expected_val = torch.tensor([[0, 1, 2, 3, 3, 3], [4, 5, 6, 6, 6, 7]], device=device, dtype=torch.float) | 3116 | expected_val = torch.tensor([[0, 1, 2, 3, 3, 3], [4, 5, 6, 6, 6, 7]], device=device, dtype=torch.float) |
| 3082 | expected_ind = torch.tensor([[0, 1, 2, 3, 3, 3], [0, 1, 2, 2, 4, 5]], device=device, dtype=torch.long) | 3117 | expected_ind = torch.tensor([[0, 1, 2, 3, 3, 3], [0, 1, 2, 2, 4, 5]], device=device, dtype=torch.long) |
| 3083 | self._test_cumminmax_helper(x, torch.cummax, expected_val, expected_ind) | 3118 | self._test_cumminmax_helper(x, torch.cummax, expected_val, expected_ind) |
| 3084 | 3119 | ||
| 3085 | - @skipIfMps | 3120 | + @skipIfMPS |
| 3086 | def test_cummin_discontiguous(self, device): | 3121 | def test_cummin_discontiguous(self, device): |
| 3087 | x = torch.tensor([[3, 2, 1, 0, 1, 2], [7, 6, 5, 4, 5, 2]], device=device, dtype=torch.float).t().contiguous().t() | 3122 | x = torch.tensor([[3, 2, 1, 0, 1, 2], [7, 6, 5, 4, 5, 2]], device=device, dtype=torch.float).t().contiguous().t() |
| 3088 | expected_val = torch.tensor([[3, 2, 1, 0, 0, 0], [7, 6, 5, 4, 4, 2]], device=device, dtype=torch.float) | 3123 | expected_val = torch.tensor([[3, 2, 1, 0, 0, 0], [7, 6, 5, 4, 4, 2]], device=device, dtype=torch.float) |
| @@ -3095,25 +3130,6 @@ class TestTorchDeviceType(TestCase): | |||
| 3095 | x[1] = True | 3130 | x[1] = True |
| 3096 | self.assertEqual(x, torch.tensor([False, True], dtype=torch.bool, device=device)) | 3131 | self.assertEqual(x, torch.tensor([False, True], dtype=torch.bool, device=device)) |
| 3097 | 3132 | ||
| 3098 | - def test_unfold_all_devices_and_dtypes(self, device): | ||
| 3099 | - for dt in all_types_and_complex_and(torch.half, torch.bool, torch.bfloat16): | ||
| 3100 | - | ||
| 3101 | - if dt == torch.bool: | ||
| 3102 | - x = torch.empty((0, 1, 3, 0), dtype=dt, device=device) | ||
| 3103 | - self.assertEqual((0, 1, 1, 0, 3), x.unfold(2, 3, 2).shape) | ||
| 3104 | - else: | ||
| 3105 | - x = torch.empty((0, 1, 3, 0), dtype=dt, device=device) | ||
| 3106 | - self.assertEqual((0, 1, 1, 0, 3), x.unfold(2, 3, 2).shape) | ||
| 3107 | - | ||
| 3108 | - def test_unfold_scalars(self, device): | ||
| 3109 | - x = torch.tensor(0.5, device=device) | ||
| 3110 | - # unfold on a 0-dimensional tensor should always return a 1-d dimensional | ||
| 3111 | - # tensor of shape [size] (i.e., the second parameter to unfold) | ||
| 3112 | - | ||
| 3113 | - self.assertEqual(torch.empty(0, device=device), x.unfold(0, 0, 1)) | ||
| 3114 | - self.assertEqual(torch.empty(0, device=device), x.unfold(0, 0, 2)) | ||
| 3115 | - self.assertEqual(torch.tensor([0.5], device=device), x.unfold(0, 1, 1)) | ||
| 3116 | - | ||
| 3117 | def test_copy_all_dtypes_and_devices(self, device): | 3133 | def test_copy_all_dtypes_and_devices(self, device): |
| 3118 | from copy import copy | 3134 | from copy import copy |
| 3119 | if not device_is_910A: | 3135 | if not device_is_910A: |
| @@ -3122,7 +3138,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3122 | dtypes_ = all_types_and(torch.half, torch.bool) | 3138 | dtypes_ = all_types_and(torch.half, torch.bool) |
| 3123 | for dt in dtypes_: | 3139 | for dt in dtypes_: |
| 3124 | x = torch.tensor([1, 2, 3, 4], dtype=dt, device=device) | 3140 | x = torch.tensor([1, 2, 3, 4], dtype=dt, device=device) |
| 3125 | - x_clone = x.clone() | 3141 | + _x_clone = x.clone() |
| 3126 | y = copy(x) | 3142 | y = copy(x) |
| 3127 | y.fill_(1) | 3143 | y.fill_(1) |
| 3128 | # copy is a shallow copy, only copies the tensor view, | 3144 | # copy is a shallow copy, only copies the tensor view, |
| @@ -3238,9 +3254,10 @@ class TestTorchDeviceType(TestCase): | |||
| 3238 | # should retain permutation after densification | 3254 | # should retain permutation after densification |
| 3239 | self.assertTrue(y.stride() == (1, 4)) | 3255 | self.assertTrue(y.stride() == (1, 4)) |
| 3240 | 3256 | ||
| 3257 | + | ||
| 3241 | 3258 | ||
| 3242 | 3259 | ||
| 3243 | - def test_addcmul(self, device, dtype): | 3260 | + def test_addcmul(self, device, dtype, use_cpu_scalar): |
| 3244 | # Returns floating or integral scalar corresponding to dtype | 3261 | # Returns floating or integral scalar corresponding to dtype |
| 3245 | def _number(floating, integer, dtype): | 3262 | def _number(floating, integer, dtype): |
| 3246 | if dtype in [torch.half, torch.float, torch.double, torch.bfloat16]: | 3263 | if dtype in [torch.half, torch.float, torch.double, torch.bfloat16]: |
| @@ -3260,7 +3277,10 @@ class TestTorchDeviceType(TestCase): | |||
| 3260 | 3277 | ||
| 3261 | a = rand_tensor((2, 2), dtype=dtype, device=device) | 3278 | a = rand_tensor((2, 2), dtype=dtype, device=device) |
| 3262 | b = rand_tensor((2, 2), dtype=dtype, device=device) | 3279 | b = rand_tensor((2, 2), dtype=dtype, device=device) |
| 3263 | - c = rand_tensor((2, 2), dtype=dtype, device=device) | 3280 | + if use_cpu_scalar: |
| 3281 | + c = rand_tensor([], device="cpu", dtype=dtype) | ||
| 3282 | + else: | ||
| 3283 | + c = rand_tensor((2, 2), dtype=dtype, device=device) | ||
| 3264 | 3284 | ||
| 3265 | alpha = _number(0.5, 3, dtype) | 3285 | alpha = _number(0.5, 3, dtype) |
| 3266 | 3286 | ||
| @@ -3279,6 +3299,21 @@ class TestTorchDeviceType(TestCase): | |||
| 3279 | c = torch.tensor([2.0], device=device, dtype=dtype) | 3299 | c = torch.tensor([2.0], device=device, dtype=dtype) |
| 3280 | out = torch.addcmul(a, b, c, value=-1) | 3300 | out = torch.addcmul(a, b, c, value=-1) |
| 3281 | self.assertTrue(not (out.isnan() or out.isinf())) | 3301 | self.assertTrue(not (out.isnan() or out.isinf())) |
| 3302 | + | ||
| 3303 | + | ||
| 3304 | + def test_addcmul_cuda_errors_with_cpu_scalars(self, device): | ||
| 3305 | + # Logic is dtype agnostic, so dtype isn't tested | ||
| 3306 | + alpha = 0.5 | ||
| 3307 | + | ||
| 3308 | + a = torch.rand((2, 2), device=device) | ||
| 3309 | + b = torch.rand((2, 2), device=device) | ||
| 3310 | + c = torch.rand((2, 2), device=device) | ||
| 3311 | + scalar = torch.rand([], device="cpu") | ||
| 3312 | + | ||
| 3313 | + with self.assertRaisesRegex(RuntimeError, r'CPU Scalar support for tensor1 argument'): | ||
| 3314 | + torch.addcmul(a, scalar, c, value=alpha) | ||
| 3315 | + with self.assertRaisesRegex(RuntimeError, r'CPU Scalar support for self argument'): | ||
| 3316 | + torch.addcmul(scalar, b, c, value=alpha) | ||
| 3282 | 3317 | ||
| 3283 | def test_narrow_empty(self, device): | 3318 | def test_narrow_empty(self, device): |
| 3284 | x = torch.randn(2, 3, 4, device=device) | 3319 | x = torch.randn(2, 3, 4, device=device) |
| @@ -3415,7 +3450,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3415 | 3450 | ||
| 3416 | def _prepare_data_for_index_copy_and_add_deterministic( | 3451 | def _prepare_data_for_index_copy_and_add_deterministic( |
| 3417 | self, dim: int, device: torch.device | 3452 | self, dim: int, device: torch.device |
| 3418 | - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | 3453 | + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: |
| 3419 | assert (dim >= 0 and dim < 3) | 3454 | assert (dim >= 0 and dim < 3) |
| 3420 | a = [5, 4, 3] | 3455 | a = [5, 4, 3] |
| 3421 | a[dim] = 2000 | 3456 | a[dim] = 2000 |
| @@ -3434,7 +3469,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3434 | with DeterministicGuard(True): | 3469 | with DeterministicGuard(True): |
| 3435 | y0 = torch.index_copy(x, dim, index, src) | 3470 | y0 = torch.index_copy(x, dim, index, src) |
| 3436 | 3471 | ||
| 3437 | - x0 = x.clone().detach() | 3472 | + x0 = x.detach().clone() |
| 3438 | index_list = index.tolist() | 3473 | index_list = index.tolist() |
| 3439 | for i in range(len(index_list)): | 3474 | for i in range(len(index_list)): |
| 3440 | if dim == 0: | 3475 | if dim == 0: |
| @@ -3484,7 +3519,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3484 | 3519 | ||
| 3485 | 3520 | ||
| 3486 | else all_types_and(torch.half, torch.bool))) | 3521 | else all_types_and(torch.half, torch.bool))) |
| 3487 | - @skipIfMps | 3522 | + @skipIfMPS |
| 3488 | def test_index_fill(self, device, dtype): | 3523 | def test_index_fill(self, device, dtype): |
| 3489 | x = torch.tensor([[1, 2], [4, 5]], dtype=dtype, device=device) | 3524 | x = torch.tensor([[1, 2], [4, 5]], dtype=dtype, device=device) |
| 3490 | index = torch.tensor([0], device=device) | 3525 | index = torch.tensor([0], device=device) |
| @@ -3511,11 +3546,12 @@ class TestTorchDeviceType(TestCase): | |||
| 3511 | return make_tensor(size_arg, dtype=dtype, device=device, low=None, high=None, noncontiguous=not contig) | 3546 | return make_tensor(size_arg, dtype=dtype, device=device, low=None, high=None, noncontiguous=not contig) |
| 3512 | 3547 | ||
| 3513 | def ref_index_select(src, dim, idx): | 3548 | def ref_index_select(src, dim, idx): |
| 3514 | - # bfloat16 is just used on GPU, so it's not supported on numpy | 3549 | + # some types not supported on numpy |
| 3515 | - if dtype == torch.bfloat16: | 3550 | + not_np_dtypes = (torch.bfloat16, torch.float8_e5m2, torch.float8_e5m2fnuz, torch.float8_e4m3fn, torch.float8_e4m3fnuz) |
| 3551 | + if dtype in not_np_dtypes: | ||
| 3516 | src = src.float() | 3552 | src = src.float() |
| 3517 | out = torch.from_numpy(np.take(src.cpu().numpy(), idx.cpu().numpy(), axis=dim)) | 3553 | out = torch.from_numpy(np.take(src.cpu().numpy(), idx.cpu().numpy(), axis=dim)) |
| 3518 | - if dtype == torch.bfloat16: | 3554 | + if dtype in not_np_dtypes: |
| 3519 | out = out.to(device=device, dtype=dtype) | 3555 | out = out.to(device=device, dtype=dtype) |
| 3520 | return out | 3556 | return out |
| 3521 | 3557 | ||
| @@ -3671,7 +3707,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3671 | out = orig.put(idx, source, accumulate=True) | 3707 | out = orig.put(idx, source, accumulate=True) |
| 3672 | self.assertEqual(out, orig + source.sum(), rtol=rtol, atol=atol) | 3708 | self.assertEqual(out, orig + source.sum(), rtol=rtol, atol=atol) |
| 3673 | 3709 | ||
| 3674 | - @skipIfMps | 3710 | + @skipIfMPS |
| 3675 | def test_take_empty(self, device): | 3711 | def test_take_empty(self, device): |
| 3676 | for input_shape in [(0,), (0, 1, 2, 0), (1, 2, 3)]: | 3712 | for input_shape in [(0,), (0, 1, 2, 0), (1, 2, 3)]: |
| 3677 | for indices_shape in [(0,), (0, 1, 2, 0)]: | 3713 | for indices_shape in [(0,), (0, 1, 2, 0)]: |
| @@ -3876,7 +3912,7 @@ class TestTorchDeviceType(TestCase): | |||
| 3876 | src = torch.empty((0,), dtype=dt, device=device) | 3912 | src = torch.empty((0,), dtype=dt, device=device) |
| 3877 | dest.masked_scatter_(mask, src) | 3913 | dest.masked_scatter_(mask, src) |
| 3878 | 3914 | ||
| 3879 | - @skipIfMps | 3915 | + @skipIfMPS |
| 3880 | def test_masked_scatter_bool_tensor(self, device): | 3916 | def test_masked_scatter_bool_tensor(self, device): |
| 3881 | src = torch.tensor([True, True, True], device=device) | 3917 | src = torch.tensor([True, True, True], device=device) |
| 3882 | dst = torch.tensor([False, False, False], device=device) | 3918 | dst = torch.tensor([False, False, False], device=device) |
| @@ -3901,10 +3937,6 @@ class TestTorchDeviceType(TestCase): | |||
| 3901 | 3937 | ||
| 3902 | else all_types_and(torch.half, torch.bool))) | 3938 | else all_types_and(torch.half, torch.bool))) |
| 3903 | def test_masked_select(self, device, dtype): | 3939 | def test_masked_select(self, device, dtype): |
| 3904 | - if device == 'cpu': | ||
| 3905 | - warn = 'masked_select received a mask with dtype torch.uint8,' | ||
| 3906 | - else: | ||
| 3907 | - warn = 'indexing with dtype torch.uint8 is now deprecated, pl' | ||
| 3908 | for maskType in integral_types_and(torch.bool): | 3940 | for maskType in integral_types_and(torch.bool): |
| 3909 | num_src = 10 | 3941 | num_src = 10 |
| 3910 | src = torch.tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=dtype, device=device) | 3942 | src = torch.tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=dtype, device=device) |
| @@ -4360,6 +4392,26 @@ class TestTorchDeviceType(TestCase): | |||
| 4360 | with self.assertRaisesRegex(RuntimeError, 'unsupported operation'): | 4392 | with self.assertRaisesRegex(RuntimeError, 'unsupported operation'): |
| 4361 | ind.index_add_(0, ind.clone(), ind) | 4393 | ind.index_add_(0, ind.clone(), ind) |
| 4362 | 4394 | ||
| 4395 | + | ||
| 4396 | +# This UT throws an OOM error on CUDA | ||
| 4397 | + def test_index_add_large_inputs(self, device): | ||
| 4398 | + D = 6144 | ||
| 4399 | + x = torch.zeros([16384, D], device=device, dtype=torch.bfloat16) | ||
| 4400 | + index = torch.randint(0, 16384, (1, 32, 16384), device=device, dtype=torch.int64) | ||
| 4401 | + output = torch.ones([1, 32, 16384, D], device=device, dtype=torch.bfloat16) # Use random values for test | ||
| 4402 | + | ||
| 4403 | + x_before = x.clone() | ||
| 4404 | + # Manually update x_before to generate expected values | ||
| 4405 | + for batch in range(output.shape[1]): # Loop over batch size (32 in this case) | ||
| 4406 | + for idx in range(output.shape[2]): # Loop over index | ||
| 4407 | + idx_val = index[0, batch, idx].item() | ||
| 4408 | + x_before[idx_val] += output[0, batch, idx] | ||
| 4409 | + | ||
| 4410 | + # Run index_add to get actual values | ||
| 4411 | + x.index_add_(0, index.view(-1), output.view(-1, D)) | ||
| 4412 | + | ||
| 4413 | + self.assertEqual(x_before, x) | ||
| 4414 | + | ||
| 4363 | # (but have to extend ErrorInputs to handle inplace-only errors!) | 4415 | # (but have to extend ErrorInputs to handle inplace-only errors!) |
| 4364 | 4416 | ||
| 4365 | def test_index_copy_mem_overlap(self, device): | 4417 | def test_index_copy_mem_overlap(self, device): |
| @@ -4381,9 +4433,7 @@ class TestTorchDeviceType(TestCase): | |||
| 4381 | 4433 | ||
| 4382 | def test_index_fill_mem_overlap(self, device): | 4434 | def test_index_fill_mem_overlap(self, device): |
| 4383 | x = torch.rand((1,), device=device).expand((6,)) | 4435 | x = torch.rand((1,), device=device).expand((6,)) |
| 4384 | - y = torch.rand((6,), device=device) | ||
| 4385 | ind = torch.tensor([2, 1, 0], device=device) | 4436 | ind = torch.tensor([2, 1, 0], device=device) |
| 4386 | - value = torch.rand((3,), device=device) | ||
| 4387 | 4437 | ||
| 4388 | with self.assertWarnsRegex(UserWarning, "index_fill_ on expanded tensors"): | 4438 | with self.assertWarnsRegex(UserWarning, "index_fill_ on expanded tensors"): |
| 4389 | x.index_fill_(0, ind, 1.0) | 4439 | x.index_fill_(0, ind, 1.0) |
| @@ -4705,7 +4755,7 @@ class TestTorchDeviceType(TestCase): | |||
| 4705 | result = ambiguous * 5 | 4755 | result = ambiguous * 5 |
| 4706 | self.assertEqual(ambiguous.stride(), result.stride()) | 4756 | self.assertEqual(ambiguous.stride(), result.stride()) |
| 4707 | 4757 | ||
| 4708 | - @skipIfMps | 4758 | + @skipIfMPS |
| 4709 | def test_memory_format_empty_like(self, device): | 4759 | def test_memory_format_empty_like(self, device): |
| 4710 | def test_helper(x, memory_format): | 4760 | def test_helper(x, memory_format): |
| 4711 | xc = x.contiguous(memory_format=memory_format) | 4761 | xc = x.contiguous(memory_format=memory_format) |
| @@ -4732,7 +4782,7 @@ class TestTorchDeviceType(TestCase): | |||
| 4732 | 4782 | ||
| 4733 | sparse = x.to_sparse() | 4783 | sparse = x.to_sparse() |
| 4734 | with self.assertRaises(RuntimeError): | 4784 | with self.assertRaises(RuntimeError): |
| 4735 | - z = torch.empty_like(sparse, memory_format=torch.preserve_format) | 4785 | + torch.empty_like(sparse, memory_format=torch.preserve_format) |
| 4736 | 4786 | ||
| 4737 | test_helper(torch.randn(4, 3, 8, 8, device=device), torch.channels_last) | 4787 | test_helper(torch.randn(4, 3, 8, 8, device=device), torch.channels_last) |
| 4738 | test_helper(torch.randn(4, 3, 8, 8, 8, device=device), torch.channels_last_3d) | 4788 | test_helper(torch.randn(4, 3, 8, 8, 8, device=device), torch.channels_last_3d) |
| @@ -4992,10 +5042,20 @@ class TestTorchDeviceType(TestCase): | |||
| 4992 | 5042 | ||
| 4993 | 5043 | ||
| 4994 | 5044 | ||
| 4995 | - def test_storage_all_devices(self, devices): | 5045 | + @parametrize("non_blocking", (True, False)) |
| 5046 | + def test_storage_all_devices(self, devices, non_blocking): | ||
| 4996 | for device in devices: | 5047 | for device in devices: |
| 4997 | - t = torch.tensor((), device=device) | 5048 | + t = torch.randn(6, device=device) |
| 4998 | self.assertEqual(t.dtype, t.storage().dtype) | 5049 | self.assertEqual(t.dtype, t.storage().dtype) |
| 5050 | + s = t.untyped_storage() | ||
| 5051 | + s_cpu = s.to(device='cpu', non_blocking=non_blocking) | ||
| 5052 | + if non_blocking: | ||
| 5053 | + torch.cuda.synchronize() | ||
| 5054 | + self.assertTrue(s_cpu.is_pinned()) | ||
| 5055 | + else: | ||
| 5056 | + self.assertFalse(s_cpu.is_pinned()) | ||
| 5057 | + t_cpu = torch.empty(()).set_(s_cpu) | ||
| 5058 | + self.assertEqual(t.cpu(), t_cpu) | ||
| 4999 | 5059 | ||
| 5000 | # Note [lazy_clone_ tests with inductor enabled] | 5060 | # Note [lazy_clone_ tests with inductor enabled] |
| 5001 | # These `lazy_clone_` tests are written in a way that makes them pass in | 5061 | # These `lazy_clone_` tests are written in a way that makes them pass in |
| @@ -5127,7 +5187,7 @@ class TestTorchDeviceType(TestCase): | |||
| 5127 | def test_lazy_clone_binary_op_no_materialize(self, device, dtype): | 5187 | def test_lazy_clone_binary_op_no_materialize(self, device, dtype): |
| 5128 | t = torch.tensor([[0, 1], [2, 3]], device=device, dtype=dtype) | 5188 | t = torch.tensor([[0, 1], [2, 3]], device=device, dtype=dtype) |
| 5129 | clone = t._lazy_clone() | 5189 | clone = t._lazy_clone() |
| 5130 | - res = t + clone | 5190 | + t + clone |
| 5131 | self.assertTrue(torch._C._is_cow_tensor(t)) | 5191 | self.assertTrue(torch._C._is_cow_tensor(t)) |
| 5132 | self.assertTrue(torch._C._is_cow_tensor(clone)) | 5192 | self.assertTrue(torch._C._is_cow_tensor(clone)) |
| 5133 | 5193 | ||
| @@ -5173,7 +5233,7 @@ class TestTorchDeviceType(TestCase): | |||
| 5173 | run(10, 2, False, True) | 5233 | run(10, 2, False, True) |
| 5174 | run(10, 2, True, True) | 5234 | run(10, 2, True, True) |
| 5175 | 5235 | ||
| 5176 | - @skipIfMps | 5236 | + @skipIfMPS |
| 5177 | 5237 | ||
| 5178 | 5238 | ||
| 5179 | def test_multinomial(self, device, dtype): | 5239 | def test_multinomial(self, device, dtype): |
| @@ -5253,7 +5313,6 @@ class TestTorchDeviceType(TestCase): | |||
| 5253 | sample_indices = torch.multinomial(prob_dist, n_sample, True) | 5313 | sample_indices = torch.multinomial(prob_dist, n_sample, True) |
| 5254 | for sample_index in sample_indices: | 5314 | for sample_index in sample_indices: |
| 5255 | self.assertNotEqual(sample_index, zero_prob_idx, msg="sampled an index with zero probability") | 5315 | self.assertNotEqual(sample_index, zero_prob_idx, msg="sampled an index with zero probability") |
| 5256 | - s_dim = sample_indices.dim() | ||
| 5257 | self.assertEqual(sample_indices.dim(), 1, msg="wrong number of dimensions") | 5316 | self.assertEqual(sample_indices.dim(), 1, msg="wrong number of dimensions") |
| 5258 | self.assertEqual(prob_dist.dim(), 1, msg="wrong number of prob_dist dimensions") | 5317 | self.assertEqual(prob_dist.dim(), 1, msg="wrong number of prob_dist dimensions") |
| 5259 | self.assertEqual(sample_indices.size(0), n_sample, msg="wrong number of samples") | 5318 | self.assertEqual(sample_indices.size(0), n_sample, msg="wrong number of samples") |
| @@ -5835,13 +5894,7 @@ class TestTorchDeviceType(TestCase): | |||
| 5835 | # Make sure that the parameters become nonsense when scaled gradients are finite | 5894 | # Make sure that the parameters become nonsense when scaled gradients are finite |
| 5836 | # but they get invalidated before `optimizer.step`, after `GradScaler.unscale_` | 5895 | # but they get invalidated before `optimizer.step`, after `GradScaler.unscale_` |
| 5837 | 5896 | ||
| 5838 | - @onlyNativeDeviceTypes | 5897 | + def _test_params_invalidated_with_grads_invalidated_between_unscale_and_step(self, device, dtype, optim_info): |
| 5839 | - | ||
| 5840 | - [optim for optim in optim_db if optim.optim_cls in [torch.optim.AdamW, torch.optim.Adam, torch.optim.SGD]], | ||
| 5841 | - dtypes=[torch.float32] | ||
| 5842 | - ) | ||
| 5843 | - | ||
| 5844 | - def test_params_invalidated_with_grads_invalidated_between_unscale_and_step(self, device, dtype, optim_info): | ||
| 5845 | optimizer_ctor = optim_info.optim_cls | 5898 | optimizer_ctor = optim_info.optim_cls |
| 5846 | all_optim_inputs = _get_optim_inputs_including_global_cliquey_kwargs( | 5899 | all_optim_inputs = _get_optim_inputs_including_global_cliquey_kwargs( |
| 5847 | device, dtype, optim_info, skip=("differentiable",)) | 5900 | device, dtype, optim_info, skip=("differentiable",)) |
| @@ -5869,6 +5922,23 @@ class TestTorchDeviceType(TestCase): | |||
| 5869 | 5922 | ||
| 5870 | self.assertTrue(all((p.isnan().any() or p.isinf().any()) for p in model.parameters())) | 5923 | self.assertTrue(all((p.isnan().any() or p.isinf().any()) for p in model.parameters())) |
| 5871 | 5924 | ||
| 5925 | + | ||
| 5926 | + | ||
| 5927 | + [optim for optim in optim_db if optim.optim_cls in [torch.optim.AdamW, torch.optim.Adam, torch.optim.SGD]], | ||
| 5928 | + dtypes=[torch.float32] | ||
| 5929 | + ) | ||
| 5930 | + def test_params_invalidated_with_grads_invalidated_between_unscale_and_step(self, device, dtype, optim_info): | ||
| 5931 | + self._test_params_invalidated_with_grads_invalidated_between_unscale_and_step(device, dtype, optim_info) | ||
| 5932 | + | ||
| 5933 | + | ||
| 5934 | + | ||
| 5935 | + [optim for optim in optim_db if optim.optim_cls in [torch.optim.AdamW, torch.optim.Adam, torch.optim.SGD]], | ||
| 5936 | + dtypes=[torch.float32] | ||
| 5937 | + ) | ||
| 5938 | + | ||
| 5939 | + def test_params_invalidated_with_grads_invalidated_and_graph_partition(self, device, dtype, optim_info): | ||
| 5940 | + self._test_params_invalidated_with_grads_invalidated_between_unscale_and_step(device, dtype, optim_info) | ||
| 5941 | + | ||
| 5872 | 5942 | ||
| 5873 | def test_grad_scale_will_not_overflow(self, device): | 5943 | def test_grad_scale_will_not_overflow(self, device): |
| 5874 | device = torch.device(device) | 5944 | device = torch.device(device) |
| @@ -5938,7 +6008,6 @@ class TestTorchDeviceType(TestCase): | |||
| 5938 | self._run_scaling_case(device.type, run, unskipped=3, skipped=1) | 6008 | self._run_scaling_case(device.type, run, unskipped=3, skipped=1) |
| 5939 | 6009 | ||
| 5940 | 6010 | ||
| 5941 | - | ||
| 5942 | 6011 | ||
| 5943 | def test_grad_scaling_penalty(self, device): | 6012 | def test_grad_scaling_penalty(self, device): |
| 5944 | device = torch.device(device) | 6013 | device = torch.device(device) |
| @@ -6282,15 +6351,29 @@ class TestTorchDeviceType(TestCase): | |||
| 6282 | atol = 1e-2 | 6351 | atol = 1e-2 |
| 6283 | self.assertEqual(src, dst.copy_(t), rtol=rtol, atol=atol) | 6352 | self.assertEqual(src, dst.copy_(t), rtol=rtol, atol=atol) |
| 6284 | 6353 | ||
| 6285 | - @dtypes(*(all_types_and_complex_and(torch.bool, torch.half, torch.bfloat16, torch.complex32, | 6354 | + @dtypes(*(all_types_complex_float8_and(torch.bool, torch.half, torch.bfloat16, torch.complex32, |
| 6286 | torch.uint16, torch.uint32, torch.uint64) | 6355 | torch.uint16, torch.uint32, torch.uint64) |
| 6287 | if not device_is_910A else all_types_and(torch.bool, torch.half))) | 6356 | if not device_is_910A else all_types_and(torch.bool, torch.half))) |
| 6288 | def test_item(self, device, dtype): | 6357 | def test_item(self, device, dtype): |
| 6289 | - if torch.device(device).type == 'xla' and dtype in [torch.uint16, torch.uint32, torch.uint64]: | 6358 | + xla_unsupported_dtypes = [ |
| 6290 | - self.skipTest('uint16,32,64 not implemented on XLA') | 6359 | + torch.uint16, |
| 6360 | + torch.uint32, | ||
| 6361 | + torch.uint64, | ||
| 6362 | + torch.float8_e4m3fn, | ||
| 6363 | + torch.float8_e5m2, | ||
| 6364 | + torch.float8_e4m3fnuz, | ||
| 6365 | + torch.float8_e5m2fnuz, | ||
| 6366 | + ] | ||
| 6367 | + if torch.device(device).type == 'xla' and dtype in xla_unsupported_dtypes: | ||
| 6368 | + self.skipTest('uint16,32,64,float8 not implemented on XLA') | ||
| 6291 | t = torch.ones((), device=device, dtype=dtype) | 6369 | t = torch.ones((), device=device, dtype=dtype) |
| 6292 | self.assertEqual(1, t.item()) | 6370 | self.assertEqual(1, t.item()) |
| 6293 | 6371 | ||
| 6372 | + def test__local_scalar_dense_with_empty_tensor(self, device): | ||
| 6373 | + input = torch.randn(0, device=device) | ||
| 6374 | + with self.assertRaisesRegex(RuntimeError, "Empty tensor not supported"): | ||
| 6375 | + torch.ops.aten._local_scalar_dense(input) | ||
| 6376 | + | ||
| 6294 | 6377 | ||
| 6295 | def test_masked_scatter_inplace_noncontiguous(self, device): | 6378 | def test_masked_scatter_inplace_noncontiguous(self, device): |
| 6296 | t = torch.zeros(5, 2, dtype=torch.long, device=device) | 6379 | t = torch.zeros(5, 2, dtype=torch.long, device=device) |
| @@ -6444,9 +6527,9 @@ class TestDevicePrecision(TestCase): | |||
| 6444 | def test(x: torch.Tensor, ia: torch.Tensor, ib: torch.Tensor) -> None: | 6527 | def test(x: torch.Tensor, ia: torch.Tensor, ib: torch.Tensor) -> None: |
| 6445 | # test getitem | 6528 | # test getitem |
| 6446 | with self.assertRaisesRegex(RuntimeError, fr"indices should be either .* \({x.device}\)"): | 6529 | with self.assertRaisesRegex(RuntimeError, fr"indices should be either .* \({x.device}\)"): |
| 6447 | - value = x[:, ia, None, ib, 0] | 6530 | + x[:, ia, None, ib, 0] |
| 6448 | with self.assertRaisesRegex(RuntimeError, fr"indices should be either .* \({x.device}\)"): | 6531 | with self.assertRaisesRegex(RuntimeError, fr"indices should be either .* \({x.device}\)"): |
| 6449 | - value = x[ib] | 6532 | + x[ib] |
| 6450 | 6533 | ||
| 6451 | cpu = torch.device('cpu') | 6534 | cpu = torch.device('cpu') |
| 6452 | for device in devices: | 6535 | for device in devices: |
| @@ -6872,7 +6955,7 @@ class TestTorch(TestCase): | |||
| 6872 | torch.tensor([1]).unflatten(0, []) | 6955 | torch.tensor([1]).unflatten(0, []) |
| 6873 | with self.assertRaisesRegex(RuntimeError, r"Provided sizes \[2, 2\] don't multiply up to the size of dim 0 \(1\)"): | 6956 | with self.assertRaisesRegex(RuntimeError, r"Provided sizes \[2, 2\] don't multiply up to the size of dim 0 \(1\)"): |
| 6874 | torch.tensor([1]).unflatten(0, [2, 2]) | 6957 | torch.tensor([1]).unflatten(0, [2, 2]) |
| 6875 | - with self.assertRaisesRegex(IndexError, r"Dimension specified as 0 but tensor has no dimensions"): | 6958 | + with self.assertRaisesRegex(RuntimeError, r".*Dimension specified as 0 but tensor has no dimensions"): |
| 6876 | torch.tensor(1).unflatten(0, [0]) | 6959 | torch.tensor(1).unflatten(0, [0]) |
| 6877 | with self.assertRaisesRegex(RuntimeError, r"only one dimension can be inferred"): | 6960 | with self.assertRaisesRegex(RuntimeError, r"only one dimension can be inferred"): |
| 6878 | torch.randn(5, 10).unflatten(1, (-1, -1)) | 6961 | torch.randn(5, 10).unflatten(1, (-1, -1)) |
| @@ -6984,6 +7067,18 @@ class TestTorch(TestCase): | |||
| 6984 | f_cpu = torch.randn((2, 3), dtype=torch.float32) | 7067 | f_cpu = torch.randn((2, 3), dtype=torch.float32) |
| 6985 | d_cpu = torch.randn((2, 3), dtype=torch.float64) | 7068 | d_cpu = torch.randn((2, 3), dtype=torch.float64) |
| 6986 | 7069 | ||
| 7070 | + storage_offset = 0x41414141 | ||
| 7071 | + with self.assertRaisesRegex(RuntimeError, "out of bounds for storage of size"): | ||
| 7072 | + t = torch.randn(1) | ||
| 7073 | + t.set_(t.untyped_storage(), storage_offset, t.size()) | ||
| 7074 | + | ||
| 7075 | + # if size changes, set_ will resize the storage inplace | ||
| 7076 | + t = torch.randn(1) | ||
| 7077 | + size = torch.Size([2, 3]) | ||
| 7078 | + t.set_(t.untyped_storage(), storage_offset, size) | ||
| 7079 | + self.assertEqual(t.storage_offset(), storage_offset) | ||
| 7080 | + self.assertEqual(t.untyped_storage().nbytes(), (storage_offset + size[0] * size[1]) * 4) | ||
| 7081 | + | ||
| 6987 | # change dtype | 7082 | # change dtype |
| 6988 | self.assertRaises(RuntimeError, lambda: f_cpu.set_(d_cpu.storage())) | 7083 | self.assertRaises(RuntimeError, lambda: f_cpu.set_(d_cpu.storage())) |
| 6989 | self.assertRaises(RuntimeError, | 7084 | self.assertRaises(RuntimeError, |
| @@ -7438,10 +7533,10 @@ class TestTorch(TestCase): | |||
| 7438 | torch.mean(sample, dim=0), torch.full((d,), 0.5), atol=2, rtol=2 | 7533 | torch.mean(sample, dim=0), torch.full((d,), 0.5), atol=2, rtol=2 |
| 7439 | ) | 7534 | ) |
| 7440 | torch.testing.assert_close( | 7535 | torch.testing.assert_close( |
| 7441 | - np.percentile(sample, 25, axis=0), np.repeat(0.25, d), atol=2, rtol=2 | 7536 | + np.percentile(sample, 25, axis=0).astype(np.float64), np.repeat(0.25, d), atol=2, rtol=2 |
| 7442 | ) | 7537 | ) |
| 7443 | torch.testing.assert_close( | 7538 | torch.testing.assert_close( |
| 7444 | - np.percentile(sample, 75, axis=0), np.repeat(0.75, d), atol=2, rtol=2 | 7539 | + np.percentile(sample, 75, axis=0).astype(np.float64), np.repeat(0.75, d), atol=2, rtol=2 |
| 7445 | ) | 7540 | ) |
| 7446 | 7541 | ||
| 7447 | 7542 | ||
| @@ -8372,17 +8467,96 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 8372 | lambda: res.map2_(y, z, lambda a, b, c: a + b * c)) | 8467 | lambda: res.map2_(y, z, lambda a, b, c: a + b * c)) |
| 8373 | 8468 | ||
| 8374 | def test_Size(self): | 8469 | def test_Size(self): |
| 8375 | - x = torch.Size([1, 2, 3]) | 8470 | + # expects iterable of int, not Tensor |
| 8376 | - self.assertIsInstance(x, tuple) | ||
| 8377 | - self.assertEqual(x[0], 1) | ||
| 8378 | - self.assertEqual(x[1], 2) | ||
| 8379 | - self.assertEqual(x[2], 3) | ||
| 8380 | - self.assertEqual(len(x), 3) | ||
| 8381 | self.assertRaises(TypeError, lambda: torch.Size(torch.ones(3))) | 8471 | self.assertRaises(TypeError, lambda: torch.Size(torch.ones(3))) |
| 8472 | + # initialization | ||
| 8473 | + empty_size = torch.Size([]) | ||
| 8474 | + size = torch.Size([1, 2, 3]) | ||
| 8475 | + self.assertIsInstance(empty_size, tuple) | ||
| 8476 | + self.assertIsInstance(size, tuple) | ||
| 8477 | + # value check __len__ | ||
| 8478 | + self.assertEqual(len(empty_size), 0) | ||
| 8479 | + self.assertEqual(len(size), 3) | ||
| 8480 | + # type check __getitem__[int] | ||
| 8481 | + self.assertIsInstance(size[0], int) | ||
| 8482 | + self.assertIsInstance(size[1], int) | ||
| 8483 | + self.assertIsInstance(size[2], int) | ||
| 8484 | + # value check __getitem__[int] | ||
| 8485 | + self.assertEqual(size[0], 1) | ||
| 8486 | + self.assertEqual(size[1], 2) | ||
| 8487 | + self.assertEqual(size[2], 3) | ||
| 8488 | + # type check __getitem__[slice] | ||
| 8489 | + self.assertIsInstance(size[:], torch.Size) | ||
| 8490 | + self.assertIsInstance(size[:-1], torch.Size) | ||
| 8491 | + self.assertIsInstance(size[0:0], torch.Size) | ||
| 8492 | + # value check __getitem__[slice] | ||
| 8493 | + self.assertEqual(size[:], (1, 2, 3)) | ||
| 8494 | + self.assertEqual(size[:-1], (1, 2)) | ||
| 8495 | + self.assertEqual(size[0:0], ()) | ||
| 8496 | + # type check __add__ | ||
| 8497 | + self.assertIsInstance(empty_size + (), torch.Size) | ||
| 8498 | + self.assertIsInstance(size + (), torch.Size) | ||
| 8499 | + self.assertIsInstance(size + (4, 5), torch.Size) | ||
| 8500 | + self.assertIsInstance(size + size, torch.Size) | ||
| 8501 | + # value check __add__ | ||
| 8502 | + self.assertEqual(empty_size + (), ()) | ||
| 8503 | + self.assertEqual(size + (), (1, 2, 3)) | ||
| 8504 | + self.assertEqual(size + (4, 5), (1, 2, 3, 4, 5)) | ||
| 8505 | + self.assertEqual(size + size, (1, 2, 3, 1, 2, 3)) | ||
| 8506 | + # type check __radd__ | ||
| 8507 | + self.assertIsInstance(() + empty_size, torch.Size) | ||
| 8508 | + self.assertIsInstance((4, 5) + size, torch.Size) | ||
| 8509 | + # value check __radd__ | ||
| 8510 | + self.assertEqual(() + size, (1, 2, 3)) | ||
| 8511 | + self.assertEqual((4, 5) + size, (4, 5, 1, 2, 3)) | ||
| 8512 | + # type check __mul__ | ||
| 8513 | + self.assertIsInstance(empty_size * 0, torch.Size) | ||
| 8514 | + self.assertIsInstance(size * 0, torch.Size) | ||
| 8515 | + self.assertIsInstance(size * 1, torch.Size) | ||
| 8516 | + self.assertIsInstance(size * 2, torch.Size) | ||
| 8517 | + # value check __mul__ | ||
| 8518 | + self.assertEqual(empty_size * 0, ()) | ||
| 8519 | + self.assertEqual(size * 0, ()) | ||
| 8520 | + self.assertEqual(size * 1, (1, 2, 3)) | ||
| 8521 | + self.assertEqual(size * 2, (1, 2, 3, 1, 2, 3)) | ||
| 8522 | + # type check __rmul__ | ||
| 8523 | + self.assertIsInstance(0 * empty_size, torch.Size) | ||
| 8524 | + self.assertIsInstance(0 * size, torch.Size) | ||
| 8525 | + self.assertIsInstance(1 * size, torch.Size) | ||
| 8526 | + self.assertIsInstance(2 * size, torch.Size) | ||
| 8527 | + # value check __rmul__ | ||
| 8528 | + self.assertEqual(0 * empty_size, ()) | ||
| 8529 | + self.assertEqual(0 * size, ()) | ||
| 8530 | + self.assertEqual(1 * size, (1, 2, 3)) | ||
| 8531 | + self.assertEqual(2 * size, (1, 2, 3, 1, 2, 3)) | ||
| 8382 | 8532 | ||
| 8383 | - self.assertIsInstance(x * 2, torch.Size) | 8533 | + def test_Size_concat_non_tuple_sequence(self): |
| 8384 | - self.assertIsInstance(x[:-1], torch.Size) | 8534 | + # check that TypeError get's raised on adding non-tuple sequences. |
| 8385 | - self.assertIsInstance(x + x, torch.Size) | 8535 | + from collections.abc import Sequence |
| 8536 | + | ||
| 8537 | + class DummySequence(Sequence): | ||
| 8538 | + vals = list(range(5)) | ||
| 8539 | + def __len__(self): return len(self.vals) | ||
| 8540 | + def __getitem__(self, i): return self.vals[i] | ||
| 8541 | + def __iter__(self): return iter(self.vals) | ||
| 8542 | + | ||
| 8543 | + size = torch.Size([1, 2, 3]) | ||
| 8544 | + seq = DummySequence() | ||
| 8545 | + msg = r"can only concatenate tuple \(not \w+\) to torch.Size" | ||
| 8546 | + self.assertRaisesRegex(TypeError, msg, lambda: size + seq) | ||
| 8547 | + msg = r"unsupported operand type" | ||
| 8548 | + self.assertRaisesRegex(TypeError, msg, lambda: seq + size) | ||
| 8549 | + | ||
| 8550 | + def test_Size_concat_wildcard(self): | ||
| 8551 | + # check that 3rd party classes can support addition with torch.Size | ||
| 8552 | + class Wildcard: | ||
| 8553 | + def __add__(self, other): return 42 | ||
| 8554 | + def __radd__(self, other): return 42 | ||
| 8555 | + | ||
| 8556 | + size = torch.Size([1, 2, 3]) | ||
| 8557 | + wildcard = Wildcard() | ||
| 8558 | + self.assertEqual(wildcard + size, 42) | ||
| 8559 | + self.assertEqual(size + wildcard, 42) | ||
| 8386 | 8560 | ||
| 8387 | def test_Size_scalar(self): | 8561 | def test_Size_scalar(self): |
| 8388 | three = torch.tensor(3) | 8562 | three = torch.tensor(3) |
| @@ -8519,7 +8693,7 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 8519 | self.assertEqual(torch._debug_has_internal_overlap(c), OVERLAP_TOO_HARD) | 8693 | self.assertEqual(torch._debug_has_internal_overlap(c), OVERLAP_TOO_HARD) |
| 8520 | 8694 | ||
| 8521 | def test_allow_tensor_metadata_change(self): | 8695 | def test_allow_tensor_metadata_change(self): |
| 8522 | - a = torch.ones(2, 3) | 8696 | + torch.ones(2, 3) |
| 8523 | # Metadata changes are allowed on view tensors that are created from detach(). | 8697 | # Metadata changes are allowed on view tensors that are created from detach(). |
| 8524 | 8698 | ||
| 8525 | def test_memory_format(self): | 8699 | def test_memory_format(self): |
| @@ -8551,11 +8725,13 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 8551 | test_helper((3, 3), (3, 3, 3, 3), torch.channels_last) | 8725 | test_helper((3, 3), (3, 3, 3, 3), torch.channels_last) |
| 8552 | test_helper((3, 3, 3), (3, 3, 3, 3, 3), torch.channels_last_3d) | 8726 | test_helper((3, 3, 3), (3, 3, 3, 3, 3), torch.channels_last_3d) |
| 8553 | 8727 | ||
| 8728 | + | ||
| 8554 | def test_dim_order(self): | 8729 | def test_dim_order(self): |
| 8555 | shape = (2, 3, 5, 7) | 8730 | shape = (2, 3, 5, 7) |
| 8556 | 8731 | ||
| 8557 | t = torch.empty(shape) | 8732 | t = torch.empty(shape) |
| 8558 | self.assertSequenceEqual(t.dim_order(), (0, 1, 2, 3), seq_type=tuple) | 8733 | self.assertSequenceEqual(t.dim_order(), (0, 1, 2, 3), seq_type=tuple) |
| 8734 | + self.assertSequenceEqual(t.dim_order(ambiguity_check=True), (0, 1, 2, 3), seq_type=tuple) | ||
| 8559 | # transpose doesn't really change the underlying physical memory | 8735 | # transpose doesn't really change the underlying physical memory |
| 8560 | # so expecting dim_order change to reflect that (like strides) | 8736 | # so expecting dim_order change to reflect that (like strides) |
| 8561 | self.assertSequenceEqual(t.transpose(0, 1).dim_order(), (1, 0, 2, 3)) | 8737 | self.assertSequenceEqual(t.transpose(0, 1).dim_order(), (1, 0, 2, 3)) |
| @@ -8571,15 +8747,41 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 8571 | dim_order, torch.empty_permuted(shape, dim_order).dim_order() | 8747 | dim_order, torch.empty_permuted(shape, dim_order).dim_order() |
| 8572 | ) | 8748 | ) |
| 8573 | 8749 | ||
| 8574 | - for shape in [(2, 2, 2, 2), (2, 1, 2, 2), (2, 2, 1, 2), (2, 2, 2, 1), (2, 2, 1, 1), (2, 1, 1, 2)]: | 8750 | + target_shapes = [[2, 2, 1, 2], [1, 2, 2, 2], [2, 2, 2, 1], [1, 2, 2, 1], [1, 2, 1, 2]] |
| 8751 | + | ||
| 8752 | + for shape in target_shapes: | ||
| 8575 | for memory_format in (torch.contiguous_format, torch.channels_last): | 8753 | for memory_format in (torch.contiguous_format, torch.channels_last): |
| 8576 | t = torch.empty(shape).to(memory_format=memory_format) | 8754 | t = torch.empty(shape).to(memory_format=memory_format) |
| 8755 | + with self.assertRaises(RuntimeError): | ||
| 8756 | + t.dim_order(ambiguity_check=True) | ||
| 8577 | if memory_format == torch.contiguous_format: | 8757 | if memory_format == torch.contiguous_format: |
| 8578 | dim_order_target = list(range(len(shape))) | 8758 | dim_order_target = list(range(len(shape))) |
| 8579 | elif memory_format == torch.channels_last: | 8759 | elif memory_format == torch.channels_last: |
| 8580 | dim_order_target = [0, *list(range(2, len(shape))), 1] | 8760 | dim_order_target = [0, *list(range(2, len(shape))), 1] |
| 8581 | 8761 | ||
| 8582 | - self.assertSequenceEqual(dim_order_target, t.dim_order()) | 8762 | + self.assertSequenceEqual( |
| 8763 | + dim_order_target, t.dim_order(ambiguity_check=[torch.contiguous_format, torch.channels_last]) | ||
| 8764 | + ) | ||
| 8765 | + | ||
| 8766 | + ambiguous_shapes = [[2, 1, 2, 2], [2, 2, 1, 1], [1, 2, 1, 1], [2, 1, 1, 2], [2, 1, 2, 1], | ||
| 8767 | + [1, 1, 1, 2], [1, 1, 2, 2], [1, 1, 1, 1], [2, 1, 1, 1], [1, 1, 2, 1]] | ||
| 8768 | + | ||
| 8769 | + for shape in ambiguous_shapes: | ||
| 8770 | + for memory_format in (torch.contiguous_format, torch.channels_last): | ||
| 8771 | + t = torch.empty(shape).to(memory_format=memory_format) | ||
| 8772 | + with self.assertRaises(RuntimeError): | ||
| 8773 | + t.dim_order(ambiguity_check=True) | ||
| 8774 | + t.dim_order(ambiguity_check=[torch.contiguous_format, torch.channels_last]) | ||
| 8775 | + | ||
| 8776 | + with self.assertRaises(TypeError): | ||
| 8777 | + torch.empty((1, 2, 3, 4)).dim_order(ambiguity_check="ILLEGAL_STR") | ||
| 8778 | + | ||
| 8779 | + # sparse tensor does not support dim order | ||
| 8780 | + with self.assertRaises(AttributeError): | ||
| 8781 | + indices = torch.tensor([[0, 1, 2], [0, 1, 2]]) # (row, column) indices | ||
| 8782 | + values = torch.tensor([1.0, 2.0, 3.0]) # values at those indices | ||
| 8783 | + sparse_tensor = torch.sparse_coo_tensor(indices, values, size=(3, 3)) | ||
| 8784 | + sparse_tensor.dim_order() | ||
| 8583 | 8785 | ||
| 8584 | def test_subclass_tensors(self): | 8786 | def test_subclass_tensors(self): |
| 8585 | # raise an error when trying to subclass FloatTensor | 8787 | # raise an error when trying to subclass FloatTensor |
| @@ -8778,7 +8980,6 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 8778 | self.assertEqual(y.size(), x.size()) | 8980 | self.assertEqual(y.size(), x.size()) |
| 8779 | 8981 | ||
| 8780 | def test_normal_shape(self): | 8982 | def test_normal_shape(self): |
| 8781 | - warned = False | ||
| 8782 | for device in get_all_device_types(): | 8983 | for device in get_all_device_types(): |
| 8783 | tensor1 = torch.rand(1, device=device) | 8984 | tensor1 = torch.rand(1, device=device) |
| 8784 | tensor4 = torch.rand(4, device=device) | 8985 | tensor4 = torch.rand(4, device=device) |
| @@ -8904,6 +9105,12 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 8904 | qy = qyraw.permute(0, 3, 2, 1) | 9105 | qy = qyraw.permute(0, 3, 2, 1) |
| 8905 | test_memory_layout(qx, qy, 0.1, 5, torch.ops.quantized.add(qx, qy, 0.1, 5)) | 9106 | test_memory_layout(qx, qy, 0.1, 5, torch.ops.quantized.add(qx, qy, 0.1, 5)) |
| 8906 | 9107 | ||
| 9108 | + def test_conj_physical_meta_stride(self): | ||
| 9109 | + a = torch.zeros((5, 3, 6), dtype=torch.complex128, device='meta') | ||
| 9110 | + b = torch._fft_c2c(a, [1], 1, True) | ||
| 9111 | + c = torch.conj_physical(b) | ||
| 9112 | + self.assertEqual(b.stride(), c.stride()) | ||
| 9113 | + | ||
| 8907 | # Tests to make sure we still handle .data properly until it is removed | 9114 | # Tests to make sure we still handle .data properly until it is removed |
| 8908 | def test_dot_data_use(self): | 9115 | def test_dot_data_use(self): |
| 8909 | # .data allows to change the Tensors types inplace, check that we still | 9116 | # .data allows to change the Tensors types inplace, check that we still |
| @@ -9350,9 +9557,6 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 9350 | self.assertIn('Unhandled exception caught in c10/util/AbortHandler.h', output) | 9557 | self.assertIn('Unhandled exception caught in c10/util/AbortHandler.h', output) |
| 9351 | 9558 | ||
| 9352 | 9559 | ||
| 9353 | - | ||
| 9354 | - don't support multiprocessing with spawn start method") | ||
| 9355 | - | ||
| 9356 | def test_multinomial_invalid_probs(self): | 9560 | def test_multinomial_invalid_probs(self): |
| 9357 | def _spawn_method(self, method, arg): | 9561 | def _spawn_method(self, method, arg): |
| 9358 | try: | 9562 | try: |
| @@ -9896,6 +10100,30 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 9896 | del s | 10100 | del s |
| 9897 | self.assertEqual(MyStorage.finalized_count, 1) | 10101 | self.assertEqual(MyStorage.finalized_count, 1) |
| 9898 | self.assertTrue(m[0]) | 10102 | self.assertTrue(m[0]) |
| 10103 | + | ||
| 10104 | + def test_tensor_ressurecting_clear(self): | ||
| 10105 | + # Regression test for github.com/pytorch/pytorch/issues/136358 | ||
| 10106 | + # A Tensor with custom __dict__ | ||
| 10107 | + # Autograd here is for the c++ reference later | ||
| 10108 | + t = torch.rand(2, requires_grad=True).clone() | ||
| 10109 | + t.foo = 2 | ||
| 10110 | + | ||
| 10111 | + # that is part of a cycle | ||
| 10112 | + l = [] | ||
| 10113 | + l.append(l) | ||
| 10114 | + l.append(t) | ||
| 10115 | + | ||
| 10116 | + # Keep the Tensor alive from c++ | ||
| 10117 | + # Using autograd graph here (any other mean would work) | ||
| 10118 | + t2 = t ** 2 | ||
| 10119 | + self.assertIs(t2.grad_fn._saved_self, t) | ||
| 10120 | + | ||
| 10121 | + # Clear all python references and trigger the gc | ||
| 10122 | + del t, l | ||
| 10123 | + gc.collect() | ||
| 10124 | + | ||
| 10125 | + # We used to loose the dict! | ||
| 10126 | + self.assertTrue(hasattr(t2.grad_fn._saved_self, "foo")) | ||
| 9899 | 10127 | ||
| 9900 | def test_tensor_slot_dealloc(self): | 10128 | def test_tensor_slot_dealloc(self): |
| 9901 | 10129 | ||
| @@ -9985,7 +10213,6 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 9985 | 10213 | ||
| 9986 | 10214 | ||
| 9987 | def test_tensor_weakref_dealloc(self): | 10215 | def test_tensor_weakref_dealloc(self): |
| 9988 | - | ||
| 9989 | x = torch.empty(2) | 10216 | x = torch.empty(2) |
| 9990 | m = [False] | 10217 | m = [False] |
| 9991 | 10218 | ||
| @@ -10114,6 +10341,7 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 10114 | m2[0] = True | 10341 | m2[0] = True |
| 10115 | 10342 | ||
| 10116 | x = SlotTensor1(torch.empty(2)) | 10343 | x = SlotTensor1(torch.empty(2)) |
| 10344 | + x_ref = weakref.ref(x) | ||
| 10117 | y = SlotTensor2(torch.empty(2)) | 10345 | y = SlotTensor2(torch.empty(2)) |
| 10118 | 10346 | ||
| 10119 | x.slot1 = y | 10347 | x.slot1 = y |
| @@ -10128,6 +10356,11 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 10128 | gc.collect() | 10356 | gc.collect() |
| 10129 | self.assertTrue(m1[0]) | 10357 | self.assertTrue(m1[0]) |
| 10130 | self.assertTrue(m2[0]) | 10358 | self.assertTrue(m2[0]) |
| 10359 | + self.assertIsNone(x_ref()) | ||
| 10360 | + | ||
| 10361 | + # At this point, we know the finalizer ran and the weakref | ||
| 10362 | + # was cleared. But is the object really gone? | ||
| 10363 | + self.assertFalse(any(isinstance(o, SlotTensor1) for o in gc.get_objects())) | ||
| 10131 | 10364 | ||
| 10132 | def test_storage_cycle_via_slots(self): | 10365 | def test_storage_cycle_via_slots(self): |
| 10133 | m1 = [False] | 10366 | m1 = [False] |
| @@ -10487,7 +10720,6 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 10487 | with self.assertRaisesRegex(RuntimeError, "has weakref"): | 10720 | with self.assertRaisesRegex(RuntimeError, "has weakref"): |
| 10488 | torch.utils.swap_tensors(t1, t2) | 10721 | torch.utils.swap_tensors(t1, t2) |
| 10489 | 10722 | ||
| 10490 | - | ||
| 10491 | 10723 | ||
| 10492 | def test_swap_fail_slots(self): | 10724 | def test_swap_fail_slots(self): |
| 10493 | class MyTwoTensor(TwoTensor): | 10725 | class MyTwoTensor(TwoTensor): |
| @@ -10531,6 +10763,36 @@ tensor([[[1.+1.j, 1.+1.j, 1.+1.j, ..., 1.+1.j, 1.+1.j, 1.+1.j], | |||
| 10531 | t6.c = "cat" | 10763 | t6.c = "cat" |
| 10532 | t7.d = "dog" | 10764 | t7.d = "dog" |
| 10533 | self._checked_swap(t6, t7) | 10765 | self._checked_swap(t6, t7) |
| 10766 | + | ||
| 10767 | + | ||
| 10768 | + def test_bf16_supported_on_cpu(self): | ||
| 10769 | + self.assertFalse(torch.npu.is_bf16_supported()) | ||
| 10770 | + | ||
| 10771 | + def test_tensor_with_grad_to_scalar_warning(self) -> None: | ||
| 10772 | + with warnings.catch_warnings(record=True) as w: | ||
| 10773 | + with set_warn_always_context(True): | ||
| 10774 | + warnings.simplefilter("always") | ||
| 10775 | + | ||
| 10776 | + x = torch.tensor(2.0, requires_grad=True) | ||
| 10777 | + math.pow(x, 3) # calling this results in a warning | ||
| 10778 | + | ||
| 10779 | + self.assertEqual(len(w), 1) | ||
| 10780 | + self.assertTrue(issubclass(w[0].category, UserWarning)) | ||
| 10781 | + self.assertIn( | ||
| 10782 | + "Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.", | ||
| 10783 | + str(w[0].message) | ||
| 10784 | + ) | ||
| 10785 | + | ||
| 10786 | + def test_tensor_item_no_warning(self): | ||
| 10787 | + with (warnings.catch_warnings(record=True) as w, | ||
| 10788 | + set_warn_always_context(True)): | ||
| 10789 | + warnings.simplefilter("always") | ||
| 10790 | + | ||
| 10791 | + x = torch.tensor(2.0, requires_grad=True) | ||
| 10792 | + max(x, 3) # No warning | ||
| 10793 | + x.item() # No warning | ||
| 10794 | + | ||
| 10795 | + self.assertEqual(len(w), 0) | ||
| 10534 | 10796 | ||
| 10535 | 10797 | ||
| 10536 | # The following block extends TestTorch with negative dim wrapping tests | 10798 | # The following block extends TestTorch with negative dim wrapping tests |
| @@ -10540,7 +10802,6 @@ INPLACE_METHOD = 2 | |||
| 10540 | FUNCTIONAL = 4 | 10802 | FUNCTIONAL = 4 |
| 10541 | DIM_ARG: None = None | 10803 | DIM_ARG: None = None |
| 10542 | 10804 | ||
| 10543 | - | ||
| 10544 | def make_neg_dim_test(name, tensor_arg, arg_constr, types_, extra_dim=0): | 10805 | def make_neg_dim_test(name, tensor_arg, arg_constr, types_, extra_dim=0): |
| 10545 | def neg_dim_test(self): | 10806 | def neg_dim_test(self): |
| 10546 | if isinstance(tensor_arg, list): | 10807 | if isinstance(tensor_arg, list): |
| @@ -16,6 +16,7 @@ | |||
| 16 | "test_nn_module_tests (__main__.TestComplexity)": ["", [""]], | 16 | "test_nn_module_tests (__main__.TestComplexity)": ["", [""]], |
| 17 | "test_not_import_sympy (main.TestImports)": ["", [""]], | 17 | "test_not_import_sympy (main.TestImports)": ["", [""]], |
| 18 | "test_no_warning_on_import (main.TestImports)": ["", [""]], | 18 | "test_no_warning_on_import (main.TestImports)": ["", [""]], |
| 19 | + "test_NPU_vitals_NPU_only_npu (__main__.TestVitalSignsNpuPRIVATEUSE1)": ["", [""]], | ||
| 19 | "test_output_match_nn_functional_batch_norm_cpu_float32 (__main__.TestOnnxModelOutputConsistency_opset18CPU)": ["", [""]], | 20 | "test_output_match_nn_functional_batch_norm_cpu_float32 (__main__.TestOnnxModelOutputConsistency_opset18CPU)": ["", [""]], |
| 20 | "test_dispatch_overload_fall_back_default_raise_diagnostic_warning (__main__.TestFxToOnnx)": ["", [""]], | 21 | "test_dispatch_overload_fall_back_default_raise_diagnostic_warning (__main__.TestFxToOnnx)": ["", [""]], |
| 21 | "test_fake_autocast_index_add_npu_float32 (__main__.TestFakeTensorPRIVATEUSE1)": ["", [""]], | 22 | "test_fake_autocast_index_add_npu_float32 (__main__.TestFakeTensorPRIVATEUSE1)": ["", [""]], |