update:mindspeed版本升级至core080 2025-02-22-MindSpeed-LLM-AtomGit

Mdocs/features/install_guide.md+1-1

	@@ -44,7 +44,7 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh # source ascend-toolkit环
44	# 安装MindSpeed加速库	44	# 安装MindSpeed加速库
45	git clone https://gitee.com/ascend/MindSpeed.git	45	git clone https://gitee.com/ascend/MindSpeed.git
46	cd MindSpeed	46	cd MindSpeed
47	-git checkout 9b3ad3fd928 # checkout commit from MindSpeed core_r0.8.0 in 2024.12.25	47	+git checkout 0b832e42 # checkout commit from MindSpeed core_r0.8.0 in 2025.02.22
48	pip install -r requirements.txt	48	pip install -r requirements.txt
49	pip3 install -e .	49	pip3 install -e .
50	cd ..	50	cd ..

Mmindspeed_llm/core/transformer/dot_product_attention.py+53-4

	@@ -15,7 +15,9 @@ from megatron.core.transformer.dot_product_attention import DotProductAttention
15	from megatron.core.transformer.enums import AttnMaskType	15	from megatron.core.transformer.enums import AttnMaskType
16	from megatron.core.transformer.utils import attention_mask_func	16	from megatron.core.transformer.utils import attention_mask_func
17	from megatron.core.utils import divide	17	from megatron.core.utils import divide
		18	+from megatron.core.packed_seq_params import PackedSeqParams
18	from mindspeed.core.context_parallel.ring_context_parallel import ringattn_context_parallel	19	from mindspeed.core.context_parallel.ring_context_parallel import ringattn_context_parallel
		20	+from mindspeed.core.context_parallel.ulysses_context_parallel import ulyssesattn_context_parallel
19	from mindspeed.core.parallel_state import (get_context_parallel_group_for_hybrid_ring,	21	from mindspeed.core.parallel_state import (get_context_parallel_group_for_hybrid_ring,
20	get_context_parallel_for_hybrid_ring_world_size,	22	get_context_parallel_for_hybrid_ring_world_size,
21	get_context_parallel_for_hybrid_ring_rank,	23	get_context_parallel_for_hybrid_ring_rank,
	@@ -29,6 +31,8 @@ from mindspeed.core.context_parallel.adaptive_context_parallel import adaptive_a
29	from mindspeed.core.context_parallel.utils import get_scheduling_info	31	from mindspeed.core.context_parallel.utils import get_scheduling_info
30	from mindspeed.model.transformer import get_attention_mask	32	from mindspeed.model.transformer import get_attention_mask
31	from mindspeed.utils import get_actual_seq_len	33	from mindspeed.utils import get_actual_seq_len
		34	+from mindspeed.core.context_parallel.context_parallel_kv_cache import get_cache_policy
		35	+from mindspeed.utils import get_actual_seq_len, compute_qkv_index, get_position_ids
32		36
33	from mindspeed_llm.core.models.common.embeddings.rotary_pos_embedding import yarn_get_mscale	37	from mindspeed_llm.core.models.common.embeddings.rotary_pos_embedding import yarn_get_mscale
34	from mindspeed_llm.tasks.models.common.alibi import Alibi	38	from mindspeed_llm.tasks.models.common.alibi import Alibi
	@@ -39,9 +43,20 @@ except ImportError:
39	rearrange = None	43	rearrange = None
40		44
41		45
42	-def do_ring_context_parallel(q, k, v, head_num, softmax_scale, attn_mask, dropout_p=0., pse=None, pse_type=None):	46	+def do_ring_context_parallel(q, k, v, head_num, softmax_scale, attn_mask, dropout_p=0., pse=None, pse_type=None, packed_seq_params=None):
43	args = get_args()	47	args = get_args()
44	actual_seq_len = get_actual_seq_len()	48	actual_seq_len = get_actual_seq_len()
		49	+ if args.shape_order == "TND":
		50	+ packed_seq_params = PackedSeqParams(
		51	+ cu_seqlens_q=torch.tensor(actual_seq_len, dtype=torch.int64, device=torch.cuda.current_device()),
		52	+ cu_seqlens_kv=torch.tensor(actual_seq_len, dtype=torch.int64, device=torch.cuda.current_device())
		53	+ )
		54	+
		55	+ q_index, kv_index = compute_qkv_index(torch.tensor(actual_seq_len, dtype=torch.int64, device=torch.cuda.current_device()).clone().tolist())
		56	+ packed_seq_params.q_index = q_index
		57	+ packed_seq_params.kv_index = kv_index
		58	+ packed_seq_params.position_ids = get_position_ids()
		59	+
45	in_hybrid_mode = get_context_parallel_group_for_hybrid_ring(check_initialized=False) is not None	60	in_hybrid_mode = get_context_parallel_group_for_hybrid_ring(check_initialized=False) is not None
46	if in_hybrid_mode:	61	if in_hybrid_mode:
47	cp_group = get_context_parallel_group_for_hybrid_ring()	62	cp_group = get_context_parallel_group_for_hybrid_ring()
	@@ -74,7 +89,7 @@ def do_ring_context_parallel(q, k, v, head_num, softmax_scale, attn_mask, dropou
74	cp_para['cp_group_for_intra_window_send_recv_overlap'] = get_ring_group_for_intra_window_send_recv_overlap()	89	cp_para['cp_group_for_intra_window_send_recv_overlap'] = get_ring_group_for_intra_window_send_recv_overlap()
75		90
76	output = ringattn_context_parallel(q, k, v, head_num, cp_para, softmax_scale, attn_mask, dropout_p,	91	output = ringattn_context_parallel(q, k, v, head_num, cp_para, softmax_scale, attn_mask, dropout_p,
77	- actual_seq_len, actual_seq_len)	92	+ packed_seq_params)
78	else:	93	else:
79	cp_para['scheduling_info'] = get_scheduling_info()	94	cp_para['scheduling_info'] = get_scheduling_info()
80	output = adaptive_attn_context_parallel(q, k, v, head_num, cp_para, softmax_scale, attn_mask,	95	output = adaptive_attn_context_parallel(q, k, v, head_num, cp_para, softmax_scale, attn_mask,
	@@ -82,6 +97,39 @@ def do_ring_context_parallel(q, k, v, head_num, softmax_scale, attn_mask, dropou
82	return output	97	return output
83		98
84		99
		100	+def do_ulyssesattn_context_parallel(self,
		101	+ query: Tensor,
		102	+ key: Tensor,
		103	+ value: Tensor,
		104	+ attention_mask,
		105	+ attn_mask_type,
		106	+ packed_seq_params):
		107	+ args = get_args()
		108	+
		109	+ sparse_mode = args.sparse_mode
		110	+ if attn_mask_type == AttnMaskType.no_mask:
		111	+ sparse_mode = 0 # default mask
		112	+
		113	+ scale = 1.0 / math.sqrt(
		114	+ self.hidden_size_per_attention_head) if self.scale_mask_softmax.scale is None else self.softmax_scale
		115	+
		116	+ self.ulysses_comm_para['cache_policy'] = get_cache_policy(
		117	+ self.layer_number, args.context_parallel_kv_cache_policy, args.context_parallel_cache_interval
		118	+ )
		119	+ self.ulysses_comm_para['use_ulysses_allgather_kv'] = args.use_ulysses_allgather_kv
		120	+ attn_para = dict()
		121	+ attn_para['packed_seq_params'] = packed_seq_params
		122	+ attn_para['attention_mask'] = attention_mask
		123	+ attn_para['scale'] = scale
		124	+ attn_para['pre_tokens'] = args.pre_tockens
		125	+ attn_para['next_tokens'] = args.next_tockens
		126	+ attn_para['keep_prob'] = 1 - self.attention_dropout.p
		127	+ attn_para['sparse_mode'] = sparse_mode
		128	+ output = ulyssesattn_context_parallel(query, key, value, attn_para, self.ulysses_comm_para)
		129	+
		130	+ return output
		131	+
		132	+
85	def dot_product_attention_init(	133	def dot_product_attention_init(
86	self,	134	self,
87	config: TransformerConfig,	135	config: TransformerConfig,
	@@ -209,7 +257,8 @@ def dot_product_attention_forward_wrapper(fn):
209	if args.use_flash_attn and args.tp_2d:	257	if args.use_flash_attn and args.tp_2d:
210	from mindspeed.core.transformer.dot_product_attention import dot_product_attention_forward	258	from mindspeed.core.transformer.dot_product_attention import dot_product_attention_forward
211	return dot_product_attention_forward(self, query, key, value, attention_mask, attn_mask_type, packed_seq_params)	259	return dot_product_attention_forward(self, query, key, value, attention_mask, attn_mask_type, packed_seq_params)
212	-	260	+ if self.config.context_parallel_size > 1 and args.context_parallel_algo == "ulysses_cp_algo":
		261	+ return do_ulyssesattn_context_parallel(self, query, key, value, attention_mask, attn_mask_type, packed_seq_params)
213	# ===================================	262	# ===================================
214	# Raw attention scores. [b, n/p, s, s]	263	# Raw attention scores. [b, n/p, s, s]
215	# ===================================	264	# ===================================
	@@ -365,7 +414,7 @@ def flash_attention_forward(
365	query, key, value = [rearrange(x, 's b h d -> s b (h d)') for x in [query, key, value]]	414	query, key, value = [rearrange(x, 's b h d -> s b (h d)') for x in [query, key, value]]
366	return do_ring_context_parallel(	415	return do_ring_context_parallel(
367	query, key, value, head_num=n_head, softmax_scale=scale, attn_mask=attention_mask, pse=self.pse,	416	query, key, value, head_num=n_head, softmax_scale=scale, attn_mask=attention_mask, pse=self.pse,
368	- pse_type=self.pse_type)	417	+ pse_type=self.pse_type, packed_seq_params=packed_seq_params)
369		418
370	if args.shape_order == "TND": # varlen FA	419	if args.shape_order == "TND": # varlen FA
371	query, key, value = [rearrange(x, 's b h d -> (s b) h d') for x in [query, key, value]]	420	query, key, value = [rearrange(x, 's b h d -> (s b) h d') for x in [query, key, value]]

Mmindspeed_llm/tasks/megatron_adaptor.py+5-0

	@@ -20,6 +20,7 @@ import argparse
20	import torch	20	import torch
21	import tensordict	21	import tensordict
22	from torch_npu.contrib import transfer_to_npu	22	from torch_npu.contrib import transfer_to_npu
		23	+from mindspeed.features_manager import FEATURES_LIST
23		24
24		25
25	def dummy_jit(fn):	26	def dummy_jit(fn):
	@@ -90,6 +91,10 @@ class MegatronAdaptation:
90	group.add_argument('--adaptive-recompute-device-swap', type=bool, default=False)	91	group.add_argument('--adaptive-recompute-device-swap', type=bool, default=False)
91	group.add_argument('--swap-attention', action='store_true', default=False)	92	group.add_argument('--swap-attention', action='store_true', default=False)
92	group.add_argument('--memory-fragmentation', type=bool, default=False)	93	group.add_argument('--memory-fragmentation', type=bool, default=False)
		94	+ group.add_argument('--layerzero', action='store_true', default=False)
		95	+
		96	+ for feature in FEATURES_LIST:
		97	+ feature.default_patches = False
93	return parser	98	return parser
94		99
95	def _get_dummy_args():	100	def _get_dummy_args():

Mmindspeed_llm/training/arguments.py+7-0

	@@ -1242,6 +1242,13 @@ def _add_dummy_args(args):
1242	args.moe_zero_memory = 'disable'	1242	args.moe_zero_memory = 'disable'
1243	args.moe_zero_memory_num_layers = None	1243	args.moe_zero_memory_num_layers = None
1244	args.attention_mask_type = args.cp_attention_mask_type	1244	args.attention_mask_type = args.cp_attention_mask_type
		1245	+ args.hccl_group_buffer_adaptive = False
		1246	+ args.moe_bmm_mc2 = False
		1247	+ args.moe_hierarchical_alltoallv = False
		1248	+ args.moe_experts_pipeline_degree = 0
		1249	+ args.context_parallel_kv_cache_policy = None
		1250	+ args.context_parallel_cache_interval = 0
		1251	+ args.use_ulysses_allgather_kv = False
1245		1252
1246		1253
1247	def _validate_noop_layer(args):	1254	def _validate_noop_layer(args):

Mtests/pipeline/context_parallel/test_adaptive_context_parallel.py+2-0

	@@ -86,6 +86,7 @@ def run_adaptive_cp(cp_size, bs, seq_len, dtype, cp_args):
86	args.use_nd_matmul = False	86	args.use_nd_matmul = False
87	args.ampipe_degree = 0	87	args.ampipe_degree = 0
88	args.use_cp_send_recv_overlap = True	88	args.use_cp_send_recv_overlap = True
		89	+ args.hccl_group_buffer_adaptive = False
		JW J jzh62292025年2月26日 pipeline中的没有引入dummy_args这个函数，所以这些参量并没有生效。该问题将在下一次commiter时提出进行整改。 W wx_4e191bb7af2025年2月26日这个自适应buffer不是默认就是关闭的吗？ args.hccl_group_buffer_adaptive参数还未适配需要在ut 用例和 dummy args函数中同步屏蔽
89	set_args(args)	90	set_args(args)
90	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)	91	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)
91	initialize_model_parallel_nest(context_parallel_size=cp_size)	92	initialize_model_parallel_nest(context_parallel_size=cp_size)
	@@ -191,6 +192,7 @@ def run_hybrid_adaptive_cp(cp_size, bs, seq_len, dtype, cp_args):
191	args.tp_x = 1	192	args.tp_x = 1
192	args.tp_y = 1	193	args.tp_y = 1
193	args.use_nd_matmul = False	194	args.use_nd_matmul = False
		195	+ args.hccl_group_buffer_adaptive = False
194	args.ampipe_degree = 0	196	args.ampipe_degree = 0
195	set_args(args)	197	set_args(args)
196	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)	198	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)

Mtests/pipeline/context_parallel/test_hybrid_context_parallel.py+1-0

	@@ -72,6 +72,7 @@ def run_hybridattn_cp(test_args, cp_size, u_size, cp_args):
72	args.tp_y = 1	72	args.tp_y = 1
73	args.use_nd_matmul = False	73	args.use_nd_matmul = False
74	args.ampipe_degree = 0	74	args.ampipe_degree = 0
		75	+ args.hccl_group_buffer_adaptive = False
75	set_args(args)	76	set_args(args)
76	# clear global attention mask set by last test case	77	# clear global attention mask set by last test case
77	set_attention_mask(None)	78	set_attention_mask(None)

Mtests/pipeline/context_parallel/test_ringattn_context_parallel.py+1-0

	@@ -55,6 +55,7 @@ def run_ringattn_cp(cp_size, bs, seq_len, dtype, cp_args):
55	args.tp_y = 1	55	args.tp_y = 1
56	args.use_nd_matmul = False	56	args.use_nd_matmul = False
57	args.ampipe_degree = 0	57	args.ampipe_degree = 0
		58	+ args.hccl_group_buffer_adaptive = False
58	set_args(args)	59	set_args(args)
59	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)	60	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)
60	initialize_model_parallel_nest(context_parallel_size=cp_size)	61	initialize_model_parallel_nest(context_parallel_size=cp_size)

Mtests/pipeline/context_parallel/test_ulysses_context_parallel.py+1-0

	@@ -145,6 +145,7 @@ class TestUlyssesAttention(DistributedTest):
145	args.tp_y = 1	145	args.tp_y = 1
146	args.use_nd_matmul = False	146	args.use_nd_matmul = False
147	args.ampipe_degree = 0	147	args.ampipe_degree = 0
		148	+ args.hccl_group_buffer_adaptive = False
148	set_args(args)	149	set_args(args)
149	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)	150	initialize_model_parallel_nest = initialize_model_parallel_decorator(initialize_model_parallel)
150	initialize_model_parallel_nest(context_parallel_size=self.world_size)	151	initialize_model_parallel_nest(context_parallel_size=self.world_size)

Mtests/pipeline/model_module/test_attention.py+1-0

	@@ -59,6 +59,7 @@ def run_attention_module(test_args, use_mcore, use_cp, cp_size, u_size, use_alib
59	args.tp_y = 1	59	args.tp_y = 1
60	args.use_nd_matmul = False	60	args.use_nd_matmul = False
61	args.ampipe_degree = 0	61	args.ampipe_degree = 0
		62	+ args.hccl_group_buffer_adaptive = False
62		63
63	if use_alibi:	64	if use_alibi:
64	args.position_embedding_type = 'alibi'	65	args.position_embedding_type = 'alibi'

Mtests/pipeline/model_module/test_rotary_pos_embedding.py+1-0

	@@ -36,6 +36,7 @@ class TestRotaryPosEmbedding:
36	test_name_space.tp_2d = False	36	test_name_space.tp_2d = False
37	test_name_space.tp_x = 1	37	test_name_space.tp_x = 1
38	test_name_space.tp_y = 1	38	test_name_space.tp_y = 1
		39	+ test_name_space.dynamic_factor = 1.0
39	return test_name_space	40	return test_name_space
40	# set up name space function	41	# set up name space function
41	import mindspeed_llm	42	import mindspeed_llm

Mtests/pipeline/pipe_run.sh+2-2

	@@ -9,8 +9,8 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
9	git clone -b core_r0.8.0 https://gitee.com/ascend/MindSpeed.git	9	git clone -b core_r0.8.0 https://gitee.com/ascend/MindSpeed.git
10	cd MindSpeed	10	cd MindSpeed
11		11
12	-# checkout commit from MindSpeed core_r0.8.0 in 2024.12.25	12	+# checkout commit from MindSpeed core_r0.8.0 in 2025.02.22
13	-git checkout 9b3ad3fd928	13	+git checkout 0b832e42
14	pip install -r requirements.txt	14	pip install -r requirements.txt
15	pip3 install -e .	15	pip3 install -e .
16	cd ..	16	cd ..