在A5环境上输入数据类型是int类型时调用tensor.div_加rounding_mode=trunc；在A5环境跳过ND->NC1HWC0格式的转换。-pytorch-AtomGit

已合并

在A5环境上输入数据类型是int类型时调用tensor.div_加rounding_mode=trunc；在A5环境跳过ND->NC1HWC0格式的转换。 #41495

xiaoqi-zhou创建于 7月13日从xiaoqi-zhou/pytorch_ascend： npu_format_cast_v2.12.0-26.1.0 合入到 Ascend/pytorch： v2.12.0-26.1.0

在A5环境上输入数据类型是int类型时调用tensor.div_加rounding_mode=trunc；在A5环境跳过ND->NC1HWC0格式的转换。 #41495

已合并

xiaoqi-zhou创建于 7月13日

从xiaoqi-zhou/pytorch_ascend npu_format_cast_v2.12.0-26.1.0合入到Ascend/pytorch v2.12.0-26.1.0

共 6 个文件变更+47-31

Mtest/distributed/test_allgather.py+5-4

	@@ -1,16 +1,15 @@
1	-import unittest
2	import os	1	import os
3	from random import randint	2	from random import randint
4	-
5	import numpy as np	3	import numpy as np
		4	+
6	import torch	5	import torch
7	import torch.distributed as dist	6	import torch.distributed as dist
8	import torch.multiprocessing as mp	7	import torch.multiprocessing as mp
9	-import torch_npu
10		8
		9	+import torch_npu
11	import torch_npu.distributed	10	import torch_npu.distributed
12	from torch_npu.testing.testcase import TestCase, run_tests	11	from torch_npu.testing.testcase import TestCase, run_tests
13	-from torch_npu.testing.common_utils import create_common_tensor	12	+from torch_npu.testing.common_utils import create_common_tensor, SupportedDevices
14	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU	13	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU
15		14
16		15
	@@ -155,6 +154,7 @@ class HcclAllGatherTest(HcclAllGatherTestBase):
155	pg.barrier()	154	pg.barrier()
156	p2c.get()	155	p2c.get()
157		156
		157	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
158	@skipIfUnsupportMultiNPU(2)	158	@skipIfUnsupportMultiNPU(2)
159	def test_all_gather_dist(self):	159	def test_all_gather_dist(self):
160	ranks = [2]	160	ranks = [2]
	@@ -174,6 +174,7 @@ class HcclAllGatherTest(HcclAllGatherTestBase):
174	self._test_multiprocess(HcclAllGatherTest._test_all_gather,	174	self._test_multiprocess(HcclAllGatherTest._test_all_gather,
175	HcclAllGatherTest._init_dist_hccl, expected, input1, world_size)	175	HcclAllGatherTest._init_dist_hccl, expected, input1, world_size)
176		176
		177	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
177	@skipIfUnsupportMultiNPU(2)	178	@skipIfUnsupportMultiNPU(2)
178	def test_all_gather_dist_different_shape(self):	179	def test_all_gather_dist_different_shape(self):
179	ranks = [2]	180	ranks = [2]

Mtest/distributed/test_allgather_base.py+4-8

	@@ -1,15 +1,10 @@
1	-import unittest
2	-import os
3	-from random import randint
4	-
5	import numpy as np	1	import numpy as np
		2	+
6	import torch	3	import torch
7	import torch.distributed as dist	4	import torch.distributed as dist
8	-import torch.multiprocessing as mp
9	-import torch_npu
10		5
11	-from torch_npu.testing.testcase import TestCase, run_tests	6	+from torch_npu.testing.testcase import run_tests
12	-from torch_npu.testing.common_utils import create_common_tensor	7	+from torch_npu.testing.common_utils import create_common_tensor, SupportedDevices
13	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU	8	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU
14		9
15	from test_allgather import HcclAllGatherTestBase	10	from test_allgather import HcclAllGatherTestBase
	@@ -29,6 +24,7 @@ class HcclAllGatherBaseTest(HcclAllGatherTestBase):
29	pg.barrier()	24	pg.barrier()
30	p2c.get()	25	p2c.get()
31		26
		27	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
32	@skipIfUnsupportMultiNPU(2)	28	@skipIfUnsupportMultiNPU(2)
33	def test_all_gather_base_dist(self):	29	def test_all_gather_base_dist(self):
34	ranks = [2]	30	ranks = [2]

Mtest/distributed/test_reduce.py+4-4

	@@ -1,14 +1,13 @@
1	-import unittest
2	import os	1	import os
3	-
4	import numpy as np	2	import numpy as np
		3	+
5	import torch	4	import torch
6	import torch.distributed as dist	5	import torch.distributed as dist
7	import torch.multiprocessing as mp	6	import torch.multiprocessing as mp
8	-import torch_npu
9		7
		8	+import torch_npu
10	from torch_npu.testing.testcase import TestCase, run_tests	9	from torch_npu.testing.testcase import TestCase, run_tests
11	-from torch_npu.testing.common_utils import create_common_tensor	10	+from torch_npu.testing.common_utils import create_common_tensor, SupportedDevices
12	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU	11	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU
13		12
14		13
	@@ -73,6 +72,7 @@ class HcclReduceTest(TestCase):
73		72
74	return expected	73	return expected
75		74
		75	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
76	@skipIfUnsupportMultiNPU(2)	76	@skipIfUnsupportMultiNPU(2)
77	def test_reduce_dist(self):	77	def test_reduce_dist(self):
78	ranks = [2, 4, 8]	78	ranks = [2, 4, 8]

Mtest/distributed/test_reduce_scatter.py+5-4

	@@ -1,15 +1,14 @@
1	-import unittest
2	import os	1	import os
3	from random import randint	2	from random import randint
4	-
5	import numpy as np	3	import numpy as np
		4	+
6	import torch	5	import torch
7	import torch.distributed as dist	6	import torch.distributed as dist
8	import torch.multiprocessing as mp	7	import torch.multiprocessing as mp
9	-import torch_npu
10		8
		9	+import torch_npu
11	from torch_npu.testing.testcase import TestCase, run_tests	10	from torch_npu.testing.testcase import TestCase, run_tests
12	-from torch_npu.testing.common_utils import create_common_tensor	11	+from torch_npu.testing.common_utils import create_common_tensor, SupportedDevices
13	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU	12	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU
14		13
15		14
	@@ -114,6 +113,7 @@ class HcclReduceScatterTest(HcclReduceScatterTestBase):
114	with test_case.assertRaisesRegex(RuntimeError, error_expect):	113	with test_case.assertRaisesRegex(RuntimeError, error_expect):
115	pg.reduce_scatter(output, input_list_npu)	114	pg.reduce_scatter(output, input_list_npu)
116		115
		116	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
117	@skipIfUnsupportMultiNPU(2)	117	@skipIfUnsupportMultiNPU(2)
118	def test_reduce_scatter(self):	118	def test_reduce_scatter(self):
119	ranks = [2]	119	ranks = [2]
	@@ -135,6 +135,7 @@ class HcclReduceScatterTest(HcclReduceScatterTestBase):
135	self._test_multiprocess(HcclReduceScatterTest._test_reduce_scatter,	135	self._test_multiprocess(HcclReduceScatterTest._test_reduce_scatter,
136	HcclReduceScatterTest._init_dist_hccl, expected, input_list, world_size)	136	HcclReduceScatterTest._init_dist_hccl, expected, input_list, world_size)
137		137
		138	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
138	@skipIfUnsupportMultiNPU(2)	139	@skipIfUnsupportMultiNPU(2)
139	def test_reduce_scatter_with_different_shape(self):	140	def test_reduce_scatter_with_different_shape(self):
140	ranks = [2]	141	ranks = [2]

Mtest/distributed/test_reduce_scatter_base.py+5-7

	@@ -1,14 +1,11 @@
1	-import unittest
2	-import os
3	-
4	import numpy as np	1	import numpy as np
		2	+
5	import torch	3	import torch
6	import torch.distributed as dist	4	import torch.distributed as dist
7	-import torch.multiprocessing as mp
8	-import torch_npu
9		5
10	-from torch_npu.testing.testcase import TestCase, run_tests	6	+import torch_npu
11	-from torch_npu.testing.common_utils import create_common_tensor	7	+from torch_npu.testing.testcase import run_tests
		8	+from torch_npu.testing.common_utils import create_common_tensor, SupportedDevices
12	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU	9	from torch_npu.testing.common_distributed import skipIfUnsupportMultiNPU
13		10
14	from test_reduce_scatter import HcclReduceScatterTestBase	11	from test_reduce_scatter import HcclReduceScatterTestBase
	@@ -28,6 +25,7 @@ class HcclReduceScatterBaseTest(HcclReduceScatterTestBase):
28	pg.barrier()	25	pg.barrier()
29	p2c.get()	26	p2c.get()
30		27
		28	+ @SupportedDevices(['Ascend910A', 'Ascend910B', 'Ascend910_93'])
31	@skipIfUnsupportMultiNPU(2)	29	@skipIfUnsupportMultiNPU(2)
32	def test_reduce_scatter_base(self):	30	def test_reduce_scatter_base(self):
33	ranks = [2]	31	ranks = [2]

Mtorch_npu/csrc/distributed/ProcessGroupHCCL.cpp+24-4

	@@ -4754,8 +4754,13 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::allreduce(
4754	[&](std::vector<c10_npu::NPUStream>& hcclStreams, c10::intrusive_ptr<ProcessGroupHCCL::WorkHCCL>&) {	4754	[&](std::vector<c10_npu::NPUStream>& hcclStreams, c10::intrusive_ptr<ProcessGroupHCCL::WorkHCCL>&) {
4755	if (opts.reduceOp == c10d::ReduceOp::AVG) {	4755	if (opts.reduceOp == c10d::ReduceOp::AVG) {
4756	c10_npu::NPUStreamGuard guard(hcclStreams[0]);	4756	c10_npu::NPUStreamGuard guard(hcclStreams[0]);
		4757	+ bool is_atlas_a5 = c10_npu::GetSocVersion() >= c10_npu::SocVersion::Ascend950;
4757	for (auto& tensor : tensors_cp) {	4758	for (auto& tensor : tensors_cp) {
4758	- tensor.div_(getSize());	4759	+ if (is_atlas_a5 && at::isIntegralType(tensor.scalar_type(), /includeBool=/false)) {
		4760	+ tensor.div_(getSize(), "trunc");
		4761	+ } else {
		4762	+ tensor.div_(getSize());
		4763	+ }
4759	}	4764	}
4760	}	4765	}
4761	if (tensors_cp[0].scalar_type() != tensors[0].scalar_type()) {	4766	if (tensors_cp[0].scalar_type() != tensors[0].scalar_type()) {
	@@ -5092,8 +5097,13 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce(
5092	[&](std::vector<c10_npu::NPUStream>& hcclStreams, c10::intrusive_ptr<ProcessGroupHCCL::WorkHCCL>&) {	5097	[&](std::vector<c10_npu::NPUStream>& hcclStreams, c10::intrusive_ptr<ProcessGroupHCCL::WorkHCCL>&) {
5093	if (opts.reduceOp == c10d::ReduceOp::AVG) {	5098	if (opts.reduceOp == c10d::ReduceOp::AVG) {
5094	c10_npu::NPUStreamGuard guard(hcclStreams[0]);	5099	c10_npu::NPUStreamGuard guard(hcclStreams[0]);
		5100	+ bool is_atlas_a5 = c10_npu::GetSocVersion() >= c10_npu::SocVersion::Ascend950;
5095	for (auto& tensor : tensors_cp) {	5101	for (auto& tensor : tensors_cp) {
5096	- tensor.div_(getSize());	5102	+ if (is_atlas_a5 && at::isIntegralType(tensor.scalar_type(), /includeBool=/false)) {
		5103	+ tensor.div_(getSize(), "trunc");
		5104	+ } else {
		5105	+ tensor.div_(getSize());
		5106	+ }
5097	}	5107	}
5098	}	5108	}
5099	if (tensors_cp[0].scalar_type() != tensors[0].scalar_type()) {	5109	if (tensors_cp[0].scalar_type() != tensors[0].scalar_type()) {
	@@ -5920,8 +5930,13 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce_scatter(
5920	[&](std::vector<c10_npu::NPUStream>& hcclStreams, c10::intrusive_ptr<ProcessGroupHCCL::WorkHCCL>&) {	5930	[&](std::vector<c10_npu::NPUStream>& hcclStreams, c10::intrusive_ptr<ProcessGroupHCCL::WorkHCCL>&) {
5921	if (opts.reduceOp == c10d::ReduceOp::AVG) {	5931	if (opts.reduceOp == c10d::ReduceOp::AVG) {
5922	c10_npu::NPUStreamGuard guard(hcclStreams[0]);	5932	c10_npu::NPUStreamGuard guard(hcclStreams[0]);
		5933	+ bool is_atlas_a5 = c10_npu::GetSocVersion() >= c10_npu::SocVersion::Ascend950;
5923	for (auto& tensor : outputTensors) {	5934	for (auto& tensor : outputTensors) {
5924	- tensor.div_(getSize());	5935	+ if (is_atlas_a5 && at::isIntegralType(tensor.scalar_type(), /includeBool=/false)) {
		5936	+ tensor.div_(getSize(), "trunc");
		5937	+ } else {
		5938	+ tensor.div_(getSize());
		5939	+ }
5925	}	5940	}
5926	}	5941	}
5927	},	5942	},
	@@ -6027,8 +6042,13 @@ c10::intrusive_ptr<c10d::Work> ProcessGroupHCCL::reduce_scatter(
6027	}	6042	}
6028	if (opts.reduceOp == c10d::ReduceOp::AVG) {	6043	if (opts.reduceOp == c10d::ReduceOp::AVG) {
6029	c10_npu::NPUStreamGuard guard(hcclStreams[0]);	6044	c10_npu::NPUStreamGuard guard(hcclStreams[0]);
		6045	+ bool is_atlas_a5 = c10_npu::GetSocVersion() >= c10_npu::SocVersion::Ascend950;
6030	for (auto& tensor : outputTensors) {	6046	for (auto& tensor : outputTensors) {
6031	- tensor.div_(getSize());	6047	+ if (is_atlas_a5 && at::isIntegralType(tensor.scalar_type(), /includeBool=/false)) {
		6048	+ tensor.div_(getSize(), "trunc");
		6049	+ } else {
		6050	+ tensor.div_(getSize());
		6051	+ }
6032	}	6052	}
6033	}	6053	}
6034	},	6054	},