已合并
fix test_autograd to avoid calling policy_fn during recompute #44661
SCh_zx创建于 9 天前
fix test_autograd to avoid calling policy_fn during recompute #44661
已合并
共 2 个文件变更+40-26
| @@ -41,7 +41,7 @@ from torch.testing import make_tensor | |||
| 41 | from torch.testing._internal.common_utils import ( | 41 | from torch.testing._internal.common_utils import ( |
| 42 | TestCase, run_tests, skipIfNoLapack, slowTest, IS_WINDOWS, IS_MACOS, | 42 | TestCase, run_tests, skipIfNoLapack, slowTest, IS_WINDOWS, IS_MACOS, |
| 43 | disable_gc, gradcheck, gradgradcheck, parametrize, | 43 | disable_gc, gradcheck, gradgradcheck, parametrize, |
| 44 | - instantiate_parametrized_tests, skipIfMps, set_warn_always_context, TEST_PRIVATEUSE1, | 44 | + instantiate_parametrized_tests, skipIfMPS, set_warn_always_context, TEST_PRIVATEUSE1, |
| 45 | skipIfTorchDynamo, xfailIfTorchDynamo) | 45 | skipIfTorchDynamo, xfailIfTorchDynamo) |
| 46 | from torch.autograd import Variable, Function, detect_anomaly, kineto_available, _calculate_shape | 46 | from torch.autograd import Variable, Function, detect_anomaly, kineto_available, _calculate_shape |
| 47 | from torch.autograd.function import InplaceFunction | 47 | from torch.autograd.function import InplaceFunction |
| @@ -4321,11 +4321,18 @@ SinBackward0, MulBackward0, torch::autograd::AccumulateGrad | |||
| 4321 | # version counter doesn't change inside of the context manager | 4321 | # version counter doesn't change inside of the context manager |
| 4322 | self.assertEqual(2, x._version) | 4322 | self.assertEqual(2, x._version) |
| 4323 | 4323 | ||
| 4324 | - torch._C._autograd._unsafe_set_version_counter(x, 0) | 4324 | + torch._C._autograd._unsafe_set_version_counter((x,), (0,)) |
| 4325 | self.assertEqual(0, x._version) | 4325 | self.assertEqual(0, x._version) |
| 4326 | with self.assertRaisesRegex(RuntimeError, "Cannot set"): | 4326 | with self.assertRaisesRegex(RuntimeError, "Cannot set"): |
| 4327 | - torch._C._autograd._unsafe_set_version_counter(x, -1) | 4327 | + torch._C._autograd._unsafe_set_version_counter((x,), (-1,)) |
| 4328 | 4328 | ||
| 4329 | + y = torch.ones(2, requires_grad=True).clone() | ||
| 4330 | + with torch.autograd._unsafe_preserve_version_counter((x, y)): | ||
| 4331 | + x.mul_(2) | ||
| 4332 | + y.mul_(3) | ||
| 4333 | + # version counter doesn't change inside of the context manager | ||
| 4334 | + self.assertEqual(0, x._version) | ||
| 4335 | + self.assertEqual(0, y._version) | ||
| 4329 | 4336 | ||
| 4330 | def test_current_node(self): | 4337 | def test_current_node(self): |
| 4331 | pr = [] | 4338 | pr = [] |
| @@ -8160,11 +8167,12 @@ for shape in [(1,), ()]: | |||
| 8160 | 8167 | ||
| 8161 | def get_ref(input_requires_grad, nb_hooks): | 8168 | def get_ref(input_requires_grad, nb_hooks): |
| 8162 | t = torch.randn(10, requires_grad=input_requires_grad) | 8169 | t = torch.randn(10, requires_grad=input_requires_grad) |
| 8163 | - a = torch.tensor(1., requires_grad=True) | 8170 | + a = torch.tensor(1.0, requires_grad=True) |
| 8164 | 8171 | ||
| 8165 | class Test(nn.Module): | 8172 | class Test(nn.Module): |
| 8166 | def forward(self, x): | 8173 | def forward(self, x): |
| 8167 | - return x ** 2 * a ** 2 | 8174 | + return x**2 * a**2 |
| 8175 | + | ||
| 8168 | mod = Test() | 8176 | mod = Test() |
| 8169 | 8177 | ||
| 8170 | for _ in range(nb_hooks): | 8178 | for _ in range(nb_hooks): |
| @@ -8180,8 +8188,14 @@ for shape in [(1,), ()]: | |||
| 8180 | with set_warn_always_context(True): | 8188 | with set_warn_always_context(True): |
| 8181 | with warnings.catch_warnings(record=True) as w: | 8189 | with warnings.catch_warnings(record=True) as w: |
| 8182 | tmp.exp().sum().backward(create_graph=True) | 8190 | tmp.exp().sum().backward(create_graph=True) |
| 8183 | - self.assertTrue(len(w) == 1) | 8191 | + self.assertTrue(w) |
| 8184 | - self.assertTrue("Using backward() with create_graph=True" in str(w[0].message)) | 8192 | + found = 0 |
| 8193 | + for warning in w: | ||
| 8194 | + if "Using backward() with create_graph=True" in str( | ||
| 8195 | + warning.message | ||
| 8196 | + ): | ||
| 8197 | + found += 1 | ||
| 8198 | + self.assertEqual(found, 1) | ||
| 8185 | 8199 | ||
| 8186 | # Remove the backward + create_graph=True cycle | 8200 | # Remove the backward + create_graph=True cycle |
| 8187 | a.grad = None | 8201 | a.grad = None |
| @@ -10119,7 +10133,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10119 | with self.assertRaisesRegex(RuntimeError, "Double backward is unsupported for"): | 10133 | with self.assertRaisesRegex(RuntimeError, "Double backward is unsupported for"): |
| 10120 | gradgradcheck(fn, (input_, 0, idx, src, 'prod')) | 10134 | gradgradcheck(fn, (input_, 0, idx, src, 'prod')) |
| 10121 | 10135 | ||
| 10122 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10136 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10123 | def test_parameter_resize(self, device): | 10137 | def test_parameter_resize(self, device): |
| 10124 | asd = torch.nn.Parameter(torch.ones(16, dtype=torch.double, device=device)) | 10138 | asd = torch.nn.Parameter(torch.ones(16, dtype=torch.double, device=device)) |
| 10125 | 10139 | ||
| @@ -10131,7 +10145,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10131 | m = torch.cat((asd, asd)) | 10145 | m = torch.cat((asd, asd)) |
| 10132 | m.sum().backward() | 10146 | m.sum().backward() |
| 10133 | 10147 | ||
| 10134 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10148 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10135 | 10149 | ||
| 10136 | def test_sparse_ctor_getter_backward(self, device, dtype): | 10150 | def test_sparse_ctor_getter_backward(self, device, dtype): |
| 10137 | # See NOTE [ Sparse: autograd and API ] on the expected behavior of this test | 10151 | # See NOTE [ Sparse: autograd and API ] on the expected behavior of this test |
| @@ -10166,7 +10180,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10166 | _test(sparse_size + dense_size, len(sparse_size), nnz, device) | 10180 | _test(sparse_size + dense_size, len(sparse_size), nnz, device) |
| 10167 | 10181 | ||
| 10168 | 10182 | ||
| 10169 | - @skipIfMps | 10183 | + @skipIfMPS |
| 10170 | 10184 | ||
| 10171 | def test_sparse_backward(self, device, dtype): | 10185 | def test_sparse_backward(self, device, dtype): |
| 10172 | class FixedGradientFunction(Function): | 10186 | class FixedGradientFunction(Function): |
| @@ -10209,7 +10223,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10209 | (fn.apply(x, sparse_grad1) + fn.apply(x, sparse_grad2)).sum().abs().backward() | 10223 | (fn.apply(x, sparse_grad1) + fn.apply(x, sparse_grad2)).sum().abs().backward() |
| 10210 | self.assertEqual(x.grad, sparse_grad1 + sparse_grad2) | 10224 | self.assertEqual(x.grad, sparse_grad1 + sparse_grad2) |
| 10211 | 10225 | ||
| 10212 | - @skipIfMps | 10226 | + @skipIfMPS |
| 10213 | def test_sparse_mask_autograd(self, device): | 10227 | def test_sparse_mask_autograd(self, device): |
| 10214 | tensor = torch.randn(3, requires_grad=True, device=device) | 10228 | tensor = torch.randn(3, requires_grad=True, device=device) |
| 10215 | mask = torch.ones(3, device=device) | 10229 | mask = torch.ones(3, device=device) |
| @@ -10219,7 +10233,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10219 | converted.sum().backward() | 10233 | converted.sum().backward() |
| 10220 | self.assertEqual(tensor.grad, mask.to_dense()) | 10234 | self.assertEqual(tensor.grad, mask.to_dense()) |
| 10221 | 10235 | ||
| 10222 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10236 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10223 | def test_pyscalar_conversions(self, device): | 10237 | def test_pyscalar_conversions(self, device): |
| 10224 | def _test_pyscalar_conversions(t, integral_conv): | 10238 | def _test_pyscalar_conversions(t, integral_conv): |
| 10225 | # integral -> integral | 10239 | # integral -> integral |
| @@ -10378,7 +10392,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10378 | 10392 | ||
| 10379 | self.assertEqual(before, after) | 10393 | self.assertEqual(before, after) |
| 10380 | 10394 | ||
| 10381 | - @skipIfMps # the test doesn't work on MPS | 10395 | + @skipIfMPS # the test doesn't work on MPS |
| 10382 | def test_where_functional(self, device): | 10396 | def test_where_functional(self, device): |
| 10383 | x = torch.randn(5, 5, dtype=torch.double, device=device, requires_grad=True) | 10397 | x = torch.randn(5, 5, dtype=torch.double, device=device, requires_grad=True) |
| 10384 | y = torch.randn(5, 5, dtype=torch.double, device=device, requires_grad=True) | 10398 | y = torch.randn(5, 5, dtype=torch.double, device=device, requires_grad=True) |
| @@ -10395,7 +10409,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10395 | gradcheck(where, [cond, x, y], raise_exception=True) | 10409 | gradcheck(where, [cond, x, y], raise_exception=True) |
| 10396 | gradgradcheck(where, [cond, x, y], [torch.randn(5, 5, 5, device=device)]) | 10410 | gradgradcheck(where, [cond, x, y], [torch.randn(5, 5, 5, device=device)]) |
| 10397 | 10411 | ||
| 10398 | - @skipIfMps # the test doesn't work on MPS | 10412 | + @skipIfMPS # the test doesn't work on MPS |
| 10399 | def test_where_scalar(self, device): | 10413 | def test_where_scalar(self, device): |
| 10400 | x = torch.randn(5, 5, dtype=torch.double, device=device, requires_grad=True) | 10414 | x = torch.randn(5, 5, dtype=torch.double, device=device, requires_grad=True) |
| 10401 | scalar = 4. | 10415 | scalar = 4. |
| @@ -10472,7 +10486,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10472 | with emit_itt(): | 10486 | with emit_itt(): |
| 10473 | a.add(1.0) | 10487 | a.add(1.0) |
| 10474 | 10488 | ||
| 10475 | - @skipIfMps # the test doesn't work as randn is not supported with type long | 10489 | + @skipIfMPS # the test doesn't work as randn is not supported with type long |
| 10476 | 10490 | ||
| 10477 | def test_grad_assignment(self, devices): | 10491 | def test_grad_assignment(self, devices): |
| 10478 | x = torch.randn(5, 5, device=devices[0]) | 10492 | x = torch.randn(5, 5, device=devices[0]) |
| @@ -10695,7 +10709,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10695 | x.sum().backward() | 10709 | x.sum().backward() |
| 10696 | self.assertEqual(root.grad.tolist(), [[1, 2], [1, 1]]) | 10710 | self.assertEqual(root.grad.tolist(), [[1, 2], [1, 1]]) |
| 10697 | 10711 | ||
| 10698 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10712 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10699 | def test_inplace_on_view_then_no_grad(self, device): | 10713 | def test_inplace_on_view_then_no_grad(self, device): |
| 10700 | # Perform an in-place operation on a view of a non-leaf variable. | 10714 | # Perform an in-place operation on a view of a non-leaf variable. |
| 10701 | a = torch.ones(3, 1, dtype=torch.double, device=device, requires_grad=True) | 10715 | a = torch.ones(3, 1, dtype=torch.double, device=device, requires_grad=True) |
| @@ -10709,7 +10723,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10709 | 10723 | ||
| 10710 | c.sum().backward() | 10724 | c.sum().backward() |
| 10711 | 10725 | ||
| 10712 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10726 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10713 | def test_inplace_on_view_gradcheck(self, device): | 10727 | def test_inplace_on_view_gradcheck(self, device): |
| 10714 | # gradcheck modifications to views | 10728 | # gradcheck modifications to views |
| 10715 | a = torch.randn(4, 4, dtype=torch.double, device=device, requires_grad=True) | 10729 | a = torch.randn(4, 4, dtype=torch.double, device=device, requires_grad=True) |
| @@ -10732,7 +10746,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10732 | with self.assertRaises(RuntimeError): | 10746 | with self.assertRaises(RuntimeError): |
| 10733 | v1[0].mul_(2) | 10747 | v1[0].mul_(2) |
| 10734 | 10748 | ||
| 10735 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10749 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10736 | def test_inplace_on_view_of_multiple_output_view(self, device): | 10750 | def test_inplace_on_view_of_multiple_output_view(self, device): |
| 10737 | a = torch.rand(10, dtype=torch.double, device=device, requires_grad=True).clone() | 10751 | a = torch.rand(10, dtype=torch.double, device=device, requires_grad=True).clone() |
| 10738 | b = a.unbind(0) | 10752 | b = a.unbind(0) |
| @@ -10740,7 +10754,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10740 | with self.assertRaises(RuntimeError): | 10754 | with self.assertRaises(RuntimeError): |
| 10741 | c.mul_(2) | 10755 | c.mul_(2) |
| 10742 | 10756 | ||
| 10743 | - @skipIfMps # MPS backend doesn't support double types | 10757 | + @skipIfMPS # MPS backend doesn't support double types |
| 10744 | def test_inplace_multiple_output_view_of_view(self, device): | 10758 | def test_inplace_multiple_output_view_of_view(self, device): |
| 10745 | a = torch.rand(10, dtype=torch.double, device=device, requires_grad=True).clone() | 10759 | a = torch.rand(10, dtype=torch.double, device=device, requires_grad=True).clone() |
| 10746 | b = a.view_as(a) | 10760 | b = a.view_as(a) |
| @@ -10748,7 +10762,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10748 | with self.assertRaises(RuntimeError): | 10762 | with self.assertRaises(RuntimeError): |
| 10749 | c[0].mul_(2) | 10763 | c[0].mul_(2) |
| 10750 | 10764 | ||
| 10751 | - @skipIfMps # MPS backend doesn't support double types | 10765 | + @skipIfMPS # MPS backend doesn't support double types |
| 10752 | def test_inplace_on_view_makes_base_require_grad(self, device): | 10766 | def test_inplace_on_view_makes_base_require_grad(self, device): |
| 10753 | # in-place modification to view makes base require grad | 10767 | # in-place modification to view makes base require grad |
| 10754 | a = torch.randn(4, 4, dtype=torch.double, device=device, requires_grad=False) | 10768 | a = torch.randn(4, 4, dtype=torch.double, device=device, requires_grad=False) |
| @@ -10774,7 +10788,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10774 | self.assertEqual(b.grad.tolist(), [5]) | 10788 | self.assertEqual(b.grad.tolist(), [5]) |
| 10775 | self.assertIsNone(a.grad) | 10789 | self.assertIsNone(a.grad) |
| 10776 | 10790 | ||
| 10777 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10791 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10778 | def test_inplace_on_view_modify_base(self, device): | 10792 | def test_inplace_on_view_modify_base(self, device): |
| 10779 | # Test that an in-place operation on a base that forced it to require | 10793 | # Test that an in-place operation on a base that forced it to require |
| 10780 | # grad also forces any previous views to require grad and backprop | 10794 | # grad also forces any previous views to require grad and backprop |
| @@ -10793,7 +10807,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10793 | gradcheck(fn, [r]) | 10807 | gradcheck(fn, [r]) |
| 10794 | gradgradcheck(fn, [r]) | 10808 | gradgradcheck(fn, [r]) |
| 10795 | 10809 | ||
| 10796 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10810 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10797 | def test_inplace_on_view_python(self, device): | 10811 | def test_inplace_on_view_python(self, device): |
| 10798 | # in-place modifications of Python-autograd created view | 10812 | # in-place modifications of Python-autograd created view |
| 10799 | a = torch.randn(4, 4, dtype=torch.double, device=device, requires_grad=True) | 10813 | a = torch.randn(4, 4, dtype=torch.double, device=device, requires_grad=True) |
| @@ -10871,7 +10885,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10871 | self.assertIsNone(b.grad) | 10885 | self.assertIsNone(b.grad) |
| 10872 | self.assertEqual(a.grad.item(), 2) | 10886 | self.assertEqual(a.grad.item(), 2) |
| 10873 | 10887 | ||
| 10874 | - @skipIfMps # the test doesn't work on MPS as double types are not supported | 10888 | + @skipIfMPS # the test doesn't work on MPS as double types are not supported |
| 10875 | def test_mv_grad_stride_0(self, device): | 10889 | def test_mv_grad_stride_0(self, device): |
| 10876 | mat = torch.randn(2, 2, dtype=torch.double, device=device) | 10890 | mat = torch.randn(2, 2, dtype=torch.double, device=device) |
| 10877 | vec = torch.randn(1, dtype=torch.double, device=device).requires_grad_(True) | 10891 | vec = torch.randn(1, dtype=torch.double, device=device).requires_grad_(True) |
| @@ -10925,7 +10939,7 @@ class TestAutogradDeviceType(TestCase): | |||
| 10925 | (c * d).sum().backward() | 10939 | (c * d).sum().backward() |
| 10926 | self.assertEqual(c.grad.stride(), (2, 1)) | 10940 | self.assertEqual(c.grad.stride(), (2, 1)) |
| 10927 | 10941 | ||
| 10928 | - @skipIfMps | 10942 | + @skipIfMPS |
| 10929 | def test_copy_r_to_c(self, device): | 10943 | def test_copy_r_to_c(self, device): |
| 10930 | out_c = torch.empty(3, 2, dtype=torch.cdouble, device=device) | 10944 | out_c = torch.empty(3, 2, dtype=torch.cdouble, device=device) |
| 10931 | inp_r = torch.randn(3, 2, dtype=torch.double, device=device, | 10945 | inp_r = torch.randn(3, 2, dtype=torch.double, device=device, |
| @@ -170,11 +170,11 @@ | |||
| 170 | "test_sparse_backward_npu_complex128 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 170 | "test_sparse_backward_npu_complex128 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 171 | "test_sparse_backward_npu_float64 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 171 | "test_sparse_backward_npu_float64 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 172 | "test_sparse_ctor_getter_backward_npu_complex128 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 172 | "test_sparse_ctor_getter_backward_npu_complex128 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 173 | - "test_sparse_ctor_getter_backward_privateuse1_float64 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 173 | + "test_sparse_ctor_getter_backward_npu_float64 (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 174 | "test_sparse_mask_autograd_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 174 | "test_sparse_mask_autograd_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 175 | "test_strided_leaf_grad_layout_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 175 | "test_strided_leaf_grad_layout_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 176 | "test_to_r_to_c_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 176 | "test_to_r_to_c_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 177 | - "test_view_copy_privateuse1 (__main__.TestAutogradMultipleDispatchPRIVATEUSE1)": ["", [""]], | 177 | + "test_view_copy_npu (__main__.TestAutogradMultipleDispatchPRIVATEUSE1)": ["", [""]], |
| 178 | "test_where_functional_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 178 | "test_where_functional_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 179 | "test_where_scalar_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], | 179 | "test_where_scalar_npu (__main__.TestAutogradDeviceTypePRIVATEUSE1)": ["", [""]], |
| 180 | "test_view_clone_view_inplace (__main__.TestCrossRefFunctionalization)": ["", [""]], | 180 | "test_view_clone_view_inplace (__main__.TestCrossRefFunctionalization)": ["", [""]], |