已合并
[mindspore][patch][master]del patch #3001
王一博创建于 2025年11月15日
[mindspore][patch][master]del patch #3001
已合并
共 3 个文件变更+0-136
| @@ -1,17 +0,0 @@ | |||
| 1 | -# Copyright (c) 2025, Huawei Technologies Co., Ltd. All rights reserved. | ||
| 2 | -import torch | ||
| 3 | -from torch.autograd import Variable | ||
| 4 | -from torch.nn.parameter import Parameter | ||
| 5 | -from megatron.core.transformer.module import conversion_helper | ||
| 6 | - | ||
| 7 | - | ||
| 8 | -def fp32_to_float16(val, float16_convertor): | ||
| 9 | - def half_conversion(val): | ||
| 10 | - val_typecheck = val | ||
| 11 | - if isinstance(val_typecheck, (Parameter, Variable)): | ||
| 12 | - val_typecheck = val.data | ||
| 13 | - if val_typecheck is not None and val_typecheck.dtype == torch.float32: | ||
| 14 | - val = float16_convertor(val) | ||
| 15 | - return val | ||
| 16 | - | ||
| 17 | - return conversion_helper(val, half_conversion) | ||
| @@ -1,62 +0,0 @@ | |||
| 1 | -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. | ||
| 2 | -# Copyright (c) 2025, Huawei Technologies Co., Ltd. All rights reserved. | ||
| 3 | -# | ||
| 4 | -# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 5 | -# you may not use this file except in compliance with the License. | ||
| 6 | -# You may obtain a copy of the License at | ||
| 7 | -# | ||
| 8 | -# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 9 | -# | ||
| 10 | -# Unless required by applicable law or agreed to in writing, software | ||
| 11 | -# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 12 | -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 13 | -# See the License for the specific language governing permissions and | ||
| 14 | -# limitations under the License. | ||
| 15 | - | ||
| 16 | -import torch | ||
| 17 | -from torch.autograd import Variable | ||
| 18 | -from torch.nn.parameter import Parameter | ||
| 19 | - | ||
| 20 | - | ||
| 21 | -_FLOAT_TYPES = (torch.FloatTensor, torch.cuda.FloatTensor) | ||
| 22 | -_HALF_TYPES = (torch.HalfTensor, torch.cuda.HalfTensor) | ||
| 23 | -_BF16_TYPES = (torch.BFloat16Tensor, torch.cuda.BFloat16Tensor) | ||
| 24 | - | ||
| 25 | - | ||
| 26 | -def conversion_helper_ms(val, conversion): | ||
| 27 | - """ | ||
| 28 | - Apply conversion to val. Recursively apply conversion if `val` | ||
| 29 | - #is a nested tuple/list structure. | ||
| 30 | - """ | ||
| 31 | - if not isinstance(val, (tuple, list)): | ||
| 32 | - return conversion(val) | ||
| 33 | - rtn = [conversion_helper_ms(v, conversion) for v in val] | ||
| 34 | - if isinstance(val, tuple): | ||
| 35 | - rtn = tuple(rtn) | ||
| 36 | - return rtn | ||
| 37 | - | ||
| 38 | - | ||
| 39 | -def fp32_to_float16(val, float16_convertor): | ||
| 40 | - """Convert fp32 `val` to fp16/bf16""" | ||
| 41 | - def half_conversion(val): | ||
| 42 | - val_typecheck = val | ||
| 43 | - if isinstance(val_typecheck, (Parameter, Variable)): | ||
| 44 | - val_typecheck = val.data | ||
| 45 | - if isinstance(val_typecheck, _FLOAT_TYPES): | ||
| 46 | - if val is not None and val_typecheck.dtype == torch.float32: | ||
| 47 | - val = float16_convertor(val) | ||
| 48 | - return val | ||
| 49 | - return conversion_helper_ms(val, half_conversion) | ||
| 50 | - | ||
| 51 | - | ||
| 52 | -def float16_to_fp32(val): | ||
| 53 | - """Convert fp16/bf16 `val` to fp32""" | ||
| 54 | - def float_conversion(val): | ||
| 55 | - val_typecheck = val | ||
| 56 | - if isinstance(val_typecheck, (Parameter, Variable)): | ||
| 57 | - val_typecheck = val.data | ||
| 58 | - if isinstance(val_typecheck, (_BF16_TYPES, _HALF_TYPES)): | ||
| 59 | - if val_typecheck.dtype in (torch.float16, torch.bfloat16): | ||
| 60 | - val = val.float() | ||
| 61 | - return val | ||
| 62 | - return conversion_helper_ms(val, float_conversion) | ||
| @@ -1,57 +0,0 @@ | |||
| 1 | -# Copyright (c) 2025, Huawei Technologies Co., Ltd. All rights reserved. | ||
| 2 | - | ||
| 3 | -import torch | ||
| 4 | - | ||
| 5 | -from megatron.core import mpu | ||
| 6 | - | ||
| 7 | -import torch_npu | ||
| 8 | - | ||
| 9 | - | ||
| 10 | -def matmul_all_reduce(self, input1, input2, output, bias=None): | ||
| 11 | - device = torch.distributed.get_rank() | ||
| 12 | - tp_size = mpu.get_tensor_model_parallel_world_size() | ||
| 13 | - comm_domain = str(device // tp_size) | ||
| 14 | - rank = device % tp_size | ||
| 15 | - torch_npu.matmul_all_reduce(input1, input2, bias, output, rank, tp_size, comm_domain) | ||
| 16 | - return output | ||
| 17 | - | ||
| 18 | - | ||
| 19 | -def all_gather_matmul(self, input1, input2, output, bias=None): | ||
| 20 | - device = torch.distributed.get_rank() | ||
| 21 | - tp_size = mpu.get_tensor_model_parallel_world_size() | ||
| 22 | - comm_domain = str(device // tp_size) | ||
| 23 | - rank = device % tp_size | ||
| 24 | - torch_npu.all_gather_matmul(input1, input2, bias, output, rank, tp_size, comm_domain) | ||
| 25 | - return output | ||
| 26 | - | ||
| 27 | - | ||
| 28 | -def all_gather_matmul_v2(self, input1, input2, output, comm_output, bias=None): | ||
| 29 | - device = torch.distributed.get_rank() | ||
| 30 | - if self.comm_config is None: | ||
| 31 | - tp_size = mpu.get_tensor_model_parallel_world_size() | ||
| 32 | - comm_domain = str(device // tp_size) | ||
| 33 | - rank = device % tp_size | ||
| 34 | - else: | ||
| 35 | - tp_size = self.comm_config.rank_size | ||
| 36 | - comm_domain = self.comm_config.comm_domain | ||
| 37 | - rank = self.comm_config.rank_id | ||
| 38 | - torch_npu.all_gather_matmul_v2(input1, input2, bias, output, comm_output, rank, tp_size, comm_domain) | ||
| 39 | - return output, comm_output | ||
| 40 | - | ||
| 41 | - | ||
| 42 | -def matmul_reduce_scatter(self, input1, input2, output, bias=None): | ||
| 43 | - device = torch.distributed.get_rank() | ||
| 44 | - tp_size = mpu.get_tensor_model_parallel_world_size() | ||
| 45 | - comm_domain = str(device // tp_size) | ||
| 46 | - rank = device % tp_size | ||
| 47 | - torch_npu.matmul_reduce_scatter(input1, input2, bias, output, rank, tp_size, comm_domain) | ||
| 48 | - return output | ||
| 49 | - | ||
| 50 | - | ||
| 51 | -def pure_matmul(self, input1, input2, output, bias=None): | ||
| 52 | - device = torch.distributed.get_rank() | ||
| 53 | - tp_size = mpu.get_tensor_model_parallel_world_size() | ||
| 54 | - comm_domain = str(device // tp_size) | ||
| 55 | - rank = device % tp_size | ||
| 56 | - torch_npu.pure_matmul(input1, input2, bias, output, rank, tp_size, comm_domain) | ||
| 57 | - return output | ||