已合并
test(utils): adapt torch.utils.checkpoint.checkpoint, torch.utils.checkpoint.checkpoint_sequential and torch.overrides.is_tensor_like on NPU #39482
yulin520创建于 6月27日
test(utils): adapt torch.utils.checkpoint.checkpoint, torch.utils.checkpoint.checkpoint_sequential and torch.overrides.is_tensor_like on NPU #39482
已合并
共 2 个文件变更+102-26
| @@ -1,32 +1,82 @@ | |||
| 1 | diff --git a/test/dynamo/test_misc.py b/test/dynamo/test_misc.py | 1 | diff --git a/test/dynamo/test_misc.py b/test/dynamo/test_misc.py |
| 2 | -index 7541bd3..6b39b3d 100644 | 2 | +index 7541bd3..6a8f84b 100644 |
| 3 | --- a/test/dynamo/test_misc.py | 3 | --- a/test/dynamo/test_misc.py |
| 4 | +++ b/test/dynamo/test_misc.py | 4 | +++ b/test/dynamo/test_misc.py |
| 5 | -@@ -0,0 +1,3 @@ | 5 | +@@ -30,6 +30,8 @@ from unittest.mock import patch |
| 6 | + import numpy as np | ||
| 7 | + | ||
| 8 | + import torch | ||
| 6 | +import torch_npu | 9 | +import torch_npu |
| 7 | +from torch_npu.contrib import transfer_to_npu | 10 | +from torch_npu.contrib import transfer_to_npu |
| 8 | -+ | 11 | + import torch._dynamo.testing |
| 9 | -@@ -10277 +10280 @@ def ___make_guard_fn(): | 12 | + import torch._inductor.test_case |
| 13 | + import torch.onnx.operators | ||
| 14 | +@@ -4160,9 +4162,9 @@ utils_device.CURRENT_DEVICE == None""".split( | ||
| 15 | + def f(x): | ||
| 16 | + if torch.overrides.is_tensor_like(x): | ||
| 17 | + return (x * 2,) | ||
| 18 | +- return (torch.ones(10) + x,) | ||
| 19 | ++ return (torch.ones(10).npu() + x,) | ||
| 20 | + | ||
| 21 | +- x = torch.randn(10) | ||
| 22 | ++ x = torch.randn(10).npu() | ||
| 23 | + ref0 = f(x) | ||
| 24 | + ref1 = f(4) | ||
| 25 | + opt_f = torch.compile(f, backend=cnts, fullgraph=True) | ||
| 26 | +@@ -10274,7 +10276,7 @@ def ___make_guard_fn(): | ||
| 27 | + | ||
| 28 | + # Call create_symbolic_sizes_strides_storage_offset on both of them. | ||
| 29 | + r = main.create_symbolic_sizes_strides_storage_offset( | ||
| 10 | - torch.randn(3, 2), ConstantSource("x") | 30 | - torch.randn(3, 2), ConstantSource("x") |
| 11 | + torch.randn(3, 2).npu(), ConstantSource("x") | 31 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 12 | -@@ -10331 +10334 @@ ShapeEnv not equal: field values don't match: | 32 | + ) |
| 33 | + | ||
| 34 | + # Create a guard: size[0] == 3 (call evaluate_expr) | ||
| 35 | +@@ -10328,7 +10330,7 @@ ShapeEnv not equal: field values don't match: | ||
| 36 | + def test_shape_env_equal_create_symbolic_sizes_strides_storage_offset(self): | ||
| 37 | + main, other = ShapeEnv(), ShapeEnv() | ||
| 38 | + main.create_symbolic_sizes_strides_storage_offset( | ||
| 13 | - torch.randn(3, 2), ConstantSource("x") | 39 | - torch.randn(3, 2), ConstantSource("x") |
| 14 | + torch.randn(3, 2).npu(), ConstantSource("x") | 40 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 15 | -@@ -10398 +10401 @@ ShapeEnv not equal: field values don't match: | 41 | + ) |
| 42 | + self.assertExpectedRaisesInline( | ||
| 43 | + NotEqualError, | ||
| 44 | +@@ -10395,10 +10397,10 @@ ShapeEnv not equal: field values don't match: | ||
| 45 | + | ||
| 46 | + # Call create_symbolic_sizes_strides_storage_offset on both of them. | ||
| 47 | + r = main.create_symbolic_sizes_strides_storage_offset( | ||
| 16 | - torch.randn(3, 2), ConstantSource("x") | 48 | - torch.randn(3, 2), ConstantSource("x") |
| 17 | + torch.randn(3, 2).npu(), ConstantSource("x") | 49 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 18 | -@@ -10401 +10404 @@ ShapeEnv not equal: field values don't match: | 50 | + ) |
| 51 | + other.create_symbolic_sizes_strides_storage_offset( | ||
| 19 | - torch.randn(3, 2), ConstantSource("x") | 52 | - torch.randn(3, 2), ConstantSource("x") |
| 20 | + torch.randn(3, 2).npu(), ConstantSource("x") | 53 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 21 | -@@ -10438 +10441 @@ ShapeEnv not equal: field values don't match: | 54 | + ) |
| 55 | + | ||
| 56 | + # Create a guard: size[0] % 3 == 0 (only in the main ShapeEnv) | ||
| 57 | +@@ -10435,10 +10437,10 @@ ShapeEnv not equal: field values don't match: | ||
| 58 | + | ||
| 59 | + # Call create_symbolic_sizes_strides_storage_offset on both of them. | ||
| 60 | + r = main.create_symbolic_sizes_strides_storage_offset( | ||
| 22 | - torch.randn(3, 2), ConstantSource("x") | 61 | - torch.randn(3, 2), ConstantSource("x") |
| 23 | + torch.randn(3, 2).npu(), ConstantSource("x") | 62 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 24 | -@@ -10441 +10444 @@ ShapeEnv not equal: field values don't match: | 63 | + ) |
| 64 | + other.create_symbolic_sizes_strides_storage_offset( | ||
| 25 | - torch.randn(3, 2), ConstantSource("x") | 65 | - torch.randn(3, 2), ConstantSource("x") |
| 26 | + torch.randn(3, 2).npu(), ConstantSource("x") | 66 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 27 | -@@ -10481 +10484 @@ ShapeEnv not equal: field values don't match: | 67 | + ) |
| 68 | + | ||
| 69 | + # Create a guard: size[0] == 3 (only in the main ShapeEnv) | ||
| 70 | +@@ -10478,10 +10480,10 @@ ShapeEnv not equal: field values don't match: | ||
| 71 | + | ||
| 72 | + # Call create_symbolic_sizes_strides_storage_offset on both of them. | ||
| 73 | + r = main.create_symbolic_sizes_strides_storage_offset( | ||
| 28 | - torch.randn(3, 2), ConstantSource("x") | 74 | - torch.randn(3, 2), ConstantSource("x") |
| 29 | + torch.randn(3, 2).npu(), ConstantSource("x") | 75 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 30 | -@@ -10484 +10487 @@ ShapeEnv not equal: field values don't match: | 76 | + ) |
| 77 | + other.create_symbolic_sizes_strides_storage_offset( | ||
| 31 | - torch.randn(3, 2), ConstantSource("x") | 78 | - torch.randn(3, 2), ConstantSource("x") |
| 32 | + torch.randn(3, 2).npu(), ConstantSource("x") | 79 | + torch.randn(3, 2).npu(), ConstantSource("x") |
| 80 | + ) | ||
| 81 | + | ||
| 82 | + # Create a guard: size[0] >= 3 (only in the main ShapeEnv) | ||
| @@ -1,16 +1,42 @@ | |||
| 1 | diff --git a/test/test_autograd.py b/test/test_autograd.py | 1 | diff --git a/test/test_autograd.py b/test/test_autograd.py |
| 2 | -index 4aeeb87..aa38cdf 100644 | 2 | +index 4aeeb87..5ac1beb 100644 |
| 3 | --- a/test/test_autograd.py | 3 | --- a/test/test_autograd.py |
| 4 | +++ b/test/test_autograd.py | 4 | +++ b/test/test_autograd.py |
| 5 | -@@ -28,6 +28,7 @@ from operator import mul | 5 | +@@ -28,6 +28,8 @@ from operator import mul |
| 6 | from typing import TYPE_CHECKING | 6 | from typing import TYPE_CHECKING |
| 7 | 7 | ||
| 8 | import torch | 8 | import torch |
| 9 | +import torch_npu | 9 | +import torch_npu |
| 10 | ++from torch_npu.contrib import transfer_to_npu | ||
| 10 | import torch.autograd._functions | 11 | import torch.autograd._functions |
| 11 | import torch.autograd.forward_ad as fwAD | 12 | import torch.autograd.forward_ad as fwAD |
| 12 | from torch import inf, nan, nn | 13 | from torch import inf, nan, nn |
| 13 | -@@ -7184,7 +7185,7 @@ for shape in [(1,), ()]: | 14 | +@@ -7142,11 +7144,11 @@ for shape in [(1,), ()]: |
| 15 | + checkpoint(lambda x: x, a) | ||
| 16 | + | ||
| 17 | + def test_checkpoint_sequential_warns_if_use_reentrant_not_passed_explcitly(self): | ||
| 18 | +- a = torch.randn(3, requires_grad=True) | ||
| 19 | ++ a = torch.randn(3, device="npu", requires_grad=True) | ||
| 20 | + modules_list = [ | ||
| 21 | +- torch.nn.Linear(3, 3), | ||
| 22 | +- torch.nn.Linear(3, 3), | ||
| 23 | +- torch.nn.Linear(3, 3), | ||
| 24 | ++ torch.nn.Linear(3, 3).npu(), | ||
| 25 | ++ torch.nn.Linear(3, 3).npu(), | ||
| 26 | ++ torch.nn.Linear(3, 3).npu(), | ||
| 27 | + ] | ||
| 28 | + | ||
| 29 | + # Passing explicitly should not warn | ||
| 30 | +@@ -7170,7 +7172,7 @@ for shape in [(1,), ()]: | ||
| 31 | + return x.sin().exp() | ||
| 32 | + | ||
| 33 | + def save_2_tensors_alt(x): | ||
| 34 | +- return x.sin() * torch.tensor([1.0, 2.0]) | ||
| 35 | ++ return x.sin() * torch.tensor([1., 2.], device=x.device) | ||
| 36 | + | ||
| 37 | + def get_non_det_fn(orig_fn, recompute_fn): | ||
| 38 | + counter = [0] | ||
| 39 | +@@ -7184,7 +7186,7 @@ for shape in [(1,), ()]: | ||
| 14 | 40 | ||
| 15 | return fn | 41 | return fn |
| 16 | 42 | ||
| @@ -19,7 +45,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 19 | 45 | ||
| 20 | # Save fewer tensors during recompute | 46 | # Save fewer tensors during recompute |
| 21 | fn = get_non_det_fn(orig_fn=save_3_tensors, recompute_fn=save_2_tensors) | 47 | fn = get_non_det_fn(orig_fn=save_3_tensors, recompute_fn=save_2_tensors) |
| 22 | -@@ -7333,7 +7334,7 @@ for shape in [(1,), ()]: | 48 | +@@ -7333,7 +7335,7 @@ for shape in [(1,), ()]: |
| 23 | self.assertEqual(checkpoint_param.grad, param.grad) | 49 | self.assertEqual(checkpoint_param.grad, param.grad) |
| 24 | 50 | ||
| 25 | def test_checkpoint_valid_reset_on_error(self): | 51 | def test_checkpoint_valid_reset_on_error(self): |
| @@ -28,7 +54,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 28 | 54 | ||
| 29 | with self.assertRaisesRegex( | 55 | with self.assertRaisesRegex( |
| 30 | Exception, "torch.utils.checkpoint is incompatible" | 56 | Exception, "torch.utils.checkpoint is incompatible" |
| 31 | -@@ -13754,8 +13755,8 @@ class TestSelectiveActivationCheckpoint(TestCase): | 57 | +@@ -13754,8 +13756,8 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 32 | def fn(x, y): | 58 | def fn(x, y): |
| 33 | return x.view(-1), y.sin().cos() | 59 | return x.view(-1), y.sin().cos() |
| 34 | 60 | ||
| @@ -39,7 +65,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 39 | 65 | ||
| 40 | context_fn = functools.partial( | 66 | context_fn = functools.partial( |
| 41 | create_selective_checkpoint_contexts, | 67 | create_selective_checkpoint_contexts, |
| 42 | -@@ -13771,7 +13772,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 68 | +@@ -13771,7 +13773,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 43 | # | 69 | # |
| 44 | # 1. size/strides are dispatched upon | 70 | # 1. size/strides are dispatched upon |
| 45 | # 2. our policy saves sizes | 71 | # 2. our policy saves sizes |
| @@ -48,7 +74,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 48 | 74 | ||
| 49 | class CustomSizeDynamicShapesTensor(torch.Tensor): | 75 | class CustomSizeDynamicShapesTensor(torch.Tensor): |
| 50 | @staticmethod | 76 | @staticmethod |
| 51 | -@@ -13831,7 +13832,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 77 | +@@ -13831,7 +13833,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 52 | self.assertEqual(view_out.grad_fn._saved_self_sym_sizes, [12]) | 78 | self.assertEqual(view_out.grad_fn._saved_self_sym_sizes, [12]) |
| 53 | return view_out.exp() | 79 | return view_out.exp() |
| 54 | 80 | ||
| @@ -57,7 +83,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 57 | x_wrapper = CustomSizeDynamicShapesTensor(x) | 83 | x_wrapper = CustomSizeDynamicShapesTensor(x) |
| 58 | context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) | 84 | context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) |
| 59 | out = checkpoint(fn, x_wrapper, use_reentrant=False, context_fn=context_fn) | 85 | out = checkpoint(fn, x_wrapper, use_reentrant=False, context_fn=context_fn) |
| 60 | -@@ -13880,7 +13881,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 86 | +@@ -13880,7 +13882,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 61 | def fn(x): | 87 | def fn(x): |
| 62 | return x.sin().sin().sin() | 88 | return x.sin().sin().sin() |
| 63 | 89 | ||
| @@ -66,7 +92,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 66 | context_fn = functools.partial( | 92 | context_fn = functools.partial( |
| 67 | create_selective_checkpoint_contexts, | 93 | create_selective_checkpoint_contexts, |
| 68 | Policy(), | 94 | Policy(), |
| 69 | -@@ -13932,7 +13933,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 95 | +@@ -13932,7 +13934,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 70 | with disable_gc(): | 96 | with disable_gc(): |
| 71 | # Case 1: If graph goes away without backward, make sure there's no reference cycle | 97 | # Case 1: If graph goes away without backward, make sure there's no reference cycle |
| 72 | # keeping storage alive. | 98 | # keeping storage alive. |
| @@ -75,7 +101,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 75 | context_fn = functools.partial( | 101 | context_fn = functools.partial( |
| 76 | create_selective_checkpoint_contexts, policy_fn | 102 | create_selective_checkpoint_contexts, policy_fn |
| 77 | ) | 103 | ) |
| 78 | -@@ -13942,7 +13943,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 104 | +@@ -13942,7 +13944,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 79 | self.assertIsNone(ref()) | 105 | self.assertIsNone(ref()) |
| 80 | 106 | ||
| 81 | # Case 2: After backward, even if retain_graph=True, the storage should go away | 107 | # Case 2: After backward, even if retain_graph=True, the storage should go away |
| @@ -84,7 +110,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 84 | context_fn = functools.partial( | 110 | context_fn = functools.partial( |
| 85 | create_selective_checkpoint_contexts, policy_fn | 111 | create_selective_checkpoint_contexts, policy_fn |
| 86 | ) | 112 | ) |
| 87 | -@@ -13964,7 +13965,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 113 | +@@ -13964,7 +13966,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 88 | def fn(x): | 114 | def fn(x): |
| 89 | return x.sin().mul_(2).cos().exp() | 115 | return x.sin().mul_(2).cos().exp() |
| 90 | 116 | ||
| @@ -93,7 +119,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 93 | context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) | 119 | context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) |
| 94 | out = checkpoint(fn, x, use_reentrant=False, context_fn=context_fn) | 120 | out = checkpoint(fn, x, use_reentrant=False, context_fn=context_fn) |
| 95 | 121 | ||
| 96 | -@@ -13972,7 +13973,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 122 | +@@ -13972,7 +13974,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 97 | with self.assertRaisesRegex(RuntimeError, "has been mutated"): | 123 | with self.assertRaisesRegex(RuntimeError, "has been mutated"): |
| 98 | out.sum().backward() | 124 | out.sum().backward() |
| 99 | 125 | ||
| @@ -102,7 +128,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 102 | context_fn = functools.partial( | 128 | context_fn = functools.partial( |
| 103 | create_selective_checkpoint_contexts, | 129 | create_selective_checkpoint_contexts, |
| 104 | policy_fn, | 130 | policy_fn, |
| 105 | -@@ -14000,7 +14001,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 131 | +@@ -14000,7 +14002,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 106 | a, b = torch.var_mean(x) | 132 | a, b = torch.var_mean(x) |
| 107 | return a * b | 133 | return a * b |
| 108 | 134 | ||
| @@ -111,7 +137,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 111 | context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) | 137 | context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) |
| 112 | out = checkpoint(fn, x, use_reentrant=False, context_fn=context_fn) | 138 | out = checkpoint(fn, x, use_reentrant=False, context_fn=context_fn) |
| 113 | x_grad = torch.autograd.grad(out.sum(), (x,)) | 139 | x_grad = torch.autograd.grad(out.sum(), (x,)) |
| 114 | -@@ -14031,14 +14032,14 @@ class TestSelectiveActivationCheckpoint(TestCase): | 140 | +@@ -14031,14 +14033,14 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 115 | "mylib::sin_with_extra", backward, setup_context=setup_context | 141 | "mylib::sin_with_extra", backward, setup_context=setup_context |
| 116 | ) | 142 | ) |
| 117 | 143 | ||
| @@ -128,7 +154,7 @@ index 4aeeb87..aa38cdf 100644 | |||
| 128 | context_fn = functools.partial( | 154 | context_fn = functools.partial( |
| 129 | create_selective_checkpoint_contexts, ops_list | 155 | create_selective_checkpoint_contexts, ops_list |
| 130 | ) | 156 | ) |
| 131 | -@@ -14062,7 +14063,7 @@ class TestSelectiveActivationCheckpoint(TestCase): | 157 | +@@ -14062,7 +14064,7 @@ class TestSelectiveActivationCheckpoint(TestCase): |
| 132 | def fn(x): | 158 | def fn(x): |
| 133 | return x.sin().cos().exp() | 159 | return x.sin().cos().exp() |
| 134 | 160 | ||