update the ci framework-MindSpeed-LLM-AtomGit

Mci/access_control_test.py+6-1

	@@ -34,6 +34,10 @@ def is_txt(file):
34	return file.endswith(".txt")	34	return file.endswith(".txt")
35		35
36		36
		37	+def is_json(file):
		38	+ return file.endswith(".json")
		39	+
		40	+
37	def is_owners(file):	41	def is_owners(file):
38	return file.startswith("OWNERS")	42	return file.startswith("OWNERS")
39		43
	@@ -72,7 +76,8 @@ def choose_skip_ci(raw_txt_file):
72	is_license,	76	is_license,
73	is_no_suffix,	77	is_no_suffix,
74	is_poc,	78	is_poc,
75	- is_0day	79	+ is_0day,
		80	+ is_json
76	]	81	]
77		82
78	return skip_ci(file_list, skip_conds)	83	return skip_ci(file_list, skip_conds)

Mtests/st/baseline_results/deepseek_v3_mcore_tp1_pp2_ep4.json+17-0

	@@ -16,6 +16,23 @@
16	6.782255E+00,	16	6.782255E+00,
17	6.936630E+00	17	6.936630E+00
18	],	18	],
		19	+ "time info": [
		20	+ 198136.5,
		21	+ 2825.0,
		22	+ 2810.4,
		23	+ 2820.2,
		24	+ 2818.4,
		25	+ 2817.1,
		26	+ 2812.5,
		27	+ 2818.4,
		28	+ 2821.2,
		29	+ 2827.7,
		30	+ 2822.8,
		31	+ 2820.3,
		32	+ 2824.8,
		33	+ 2830.1,
		34	+ 2818.3
		35	+ ],
19	"throughput": [	36	"throughput": [
20	35.0,	37	35.0,
21	129.3,	38	129.3,

Mtests/test_tools/acquire_json.py+8-5

	@@ -23,7 +23,7 @@ def transfer_logs_as_json(log_file, output_json_file):
23	"""	23	"""
24		24
25	log_pattern = re.compile(	25	log_pattern = re.compile(
26	- r"throughput per GPU \(TFLOP/s/GPU\):\s+([0-9.]+)\s+\\|.?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . actor/pg_loss : ([0-9.]+)"	26	+ r"elapsed time per iteration \(ms\):\s+([0-9.]+)\s+\\| throughput per GPU \(TFLOP/s/GPU\):\s+([0-9.]+)\s+\\| .?lm loss:\s+([0-9.]+E[+-][0-9]+) \| . actor/pg_loss : ([0-9.]+)"
27	)	27	)
28		28
29	if "trl_ppo" in log_file:	29	if "trl_ppo" in log_file:
	@@ -37,8 +37,10 @@ def transfer_logs_as_json(log_file, output_json_file):
37		37
38	data = {	38	data = {
39	"lm loss": [],	39	"lm loss": [],
		40	+ "time info": [],
40	"throughput": [],	41	"throughput": [],
41	- "memo info": []	42	+ "memo info": [],
		43	+
42	}	44	}
43	with open(log_file, "r") as f:	45	with open(log_file, "r") as f:
44	log_content = f.read()	46	log_content = f.read()
	@@ -46,9 +48,10 @@ def transfer_logs_as_json(log_file, output_json_file):
46	memory_matches = memory_pattern.findall(log_content)	48	memory_matches = memory_pattern.findall(log_content)
47		49
48	if log_matches:	50	if log_matches:
49	- if log_matches[0][1] != "":	51	+ if log_matches[0][1][2] != "":
50	- data["lm loss"] = [float(match[1]) for match in log_matches]	52	+ data["lm loss"] = [float(match[2]) for match in log_matches]
51	- data["throughput"] = [float(match[0]) for match in log_matches]	53	+ data["time info"] = [float(match[0]) for match in log_matches]
		54	+ data["throughput"] = [float(match[1]) for match in log_matches]
52	else:	55	else:
53	data["lm loss"] = [float(match[2]) for match in log_matches]	56	data["lm loss"] = [float(match[2]) for match in log_matches]
54		57

Mtests/test_tools/test_ci_st.py+39-5

	@@ -1,3 +1,4 @@
		1	+import warnings
1	import pytest	2	import pytest
2	from mindspeed_llm import megatron_adaptor	3	from mindspeed_llm import megatron_adaptor
3	from tests.test_tools.acquire_json import transfer_logs_as_json, read_json	4	from tests.test_tools.acquire_json import transfer_logs_as_json, read_json
	@@ -5,6 +6,7 @@ from tests.test_tools.acquire_json import transfer_logs_as_json, read_json
5	MEMO_INFO = "memo info"	6	MEMO_INFO = "memo info"
6	THROUGHPUT = "throughput"	7	THROUGHPUT = "throughput"
7	LOSS = "lm loss"	8	LOSS = "lm loss"
		9	+TIME_INFO = "time info"
8		10
9	WARM_UP = 5	11	WARM_UP = 5
10		12
	@@ -14,12 +16,14 @@ class TestMargin:
14	loss = 0.02	16	loss = 0.02
15	memory = 0.1	17	memory = 0.1
16	throughput = 0.05	18	throughput = 0.05
		19	+ time = 0.05
17		20
18	@classmethod	21	@classmethod
19	def refresh_margin_from_json(cls, json_obj):	22	def refresh_margin_from_json(cls, json_obj):
20	cls.loss = json_obj.get(LOSS + cls._MARGIN_NAME, cls.loss)	23	cls.loss = json_obj.get(LOSS + cls._MARGIN_NAME, cls.loss)
21	cls.memory = json_obj.get(MEMO_INFO + cls._MARGIN_NAME, cls.memory)	24	cls.memory = json_obj.get(MEMO_INFO + cls._MARGIN_NAME, cls.memory)
22	cls.throughput = json_obj.get(THROUGHPUT + cls._MARGIN_NAME, cls.throughput)	25	cls.throughput = json_obj.get(THROUGHPUT + cls._MARGIN_NAME, cls.throughput)
		26	+ cls.time = json_obj.get(TIME_INFO + cls._MARGIN_NAME, cls.time)
23		27
24		28
25	class TestCIST:	29	class TestCIST:
	@@ -27,6 +31,7 @@ class TestCIST:
27	margin_loss = 0.02	31	margin_loss = 0.02
28	margin_throughput_percent = 0.05	32	margin_throughput_percent = 0.05
29	margin_memory_percent = 0.1	33	margin_memory_percent = 0.1
		34	+ margin_time_percent = 0.05
30		35
31	def _get_baseline(self, baseline_json):	36	def _get_baseline(self, baseline_json):
32	# acquire expected results	37	# acquire expected results
	@@ -46,14 +51,26 @@ class TestCIST:
46	test_obj: the object we want to test compare.	51	test_obj: the object we want to test compare.
47	test_type: deterministic or approximate, default is None.	52	test_type: deterministic or approximate, default is None.
48		53
49	- Here we temperally test `lm loss`, 'throughput' and `allocated memory`	54	+ Here we temperally test `lm loss`, 'throughput' , `allocated memory` and `elapsed time per iteration`
50	"""	55	"""
51	- comparison_selection = {	56	+ comparison_base = {
52	LOSS: self._compare_lm_loss,	57	LOSS: self._compare_lm_loss,
53	- THROUGHPUT: self._compare_throughput,	58	+ MEMO_INFO: self._compare_memory,
54	- MEMO_INFO: self._compare_memory
55	}	59	}
56		60
		61	+ comparison_throughput = {
		62	+ THROUGHPUT: self._compare_throughput,
		63	+ }
		64	+
		65	+ comparison_time = {
		66	+ TIME_INFO: self._compare_time,
		67	+ }
		68	+
		69	+ if "time info" in self.expected:
		70	+ comparison_selection = {comparison_base, comparison_time}
		71	+ else:
		72	+ comparison_selection = {comparison_base, comparison_throughput}
		73	+
57	if test_obj in comparison_selection:	74	if test_obj in comparison_selection:
58	expected_list = self.expected[test_obj]	75	expected_list = self.expected[test_obj]
59	if not expected_list:	76	if not expected_list:
	@@ -73,7 +90,7 @@ class TestCIST:
73	compare_func = comparison_selection[test_obj]	90	compare_func = comparison_selection[test_obj]
74	compare_func(expected_list, actual_list)	91	compare_func(expected_list, actual_list)
75	else:	92	else:
76	- raise ValueError(f"Unsupported test object: {test_obj}")	93	+ warnings.warn(f"The metric of {test_obj} is not selected and will be skipped.", RuntimeWarning, stacklevel=2)
77		94
78	def _compare_lm_loss(self, expected_list, actual_list):	95	def _compare_lm_loss(self, expected_list, actual_list):
79	# Because "deterministic computation" affects the throughput, so we just test	96	# Because "deterministic computation" affects the throughput, so we just test
	@@ -94,6 +111,18 @@ class TestCIST:
94	assert actual_avg_throughput >= expected_avg_throughput or \	111	assert actual_avg_throughput >= expected_avg_throughput or \
95	abs(actual_avg_throughput - expected_avg_throughput) / expected_avg_throughput <= TestMargin.throughput, \	112	abs(actual_avg_throughput - expected_avg_throughput) / expected_avg_throughput <= TestMargin.throughput, \
96	f"The actual avg throughput {actual_avg_throughput} degradate expected avg throughput {expected_avg_throughput}"	113	f"The actual avg throughput {actual_avg_throughput} degradate expected avg throughput {expected_avg_throughput}"
		114	+
		115	+ def _compare_time(self, expected_list, actual_list):
		116	+ try:
		117	+ expected_avg_time = sum(expected_list[WARM_UP:]) / (len(expected_list) - WARM_UP)
		118	+ actual_avg_time = sum(actual_list[WARM_UP:]) / (len(actual_list) - WARM_UP)
		119	+ except:
		120	+ raise ZeroDivisionError
		121	+
		122	+ assert actual_avg_time <= expected_avg_time or \
		123	+ abs(actual_avg_time - expected_avg_time) / expected_avg_time <= TestMargin.time, \
		124	+ f"The actual avg time ({actual_avg_time} ms) was {exceed_percentage:.1f}% slower than the expected ({expected_avg_time} ms)."
		125	+
97		126
98	def _compare_memory(self, expected_list, actual_list):	127	def _compare_memory(self, expected_list, actual_list):
99	for i, (expected_val, actual_val) in enumerate(zip(expected_list, actual_list)):	128	for i, (expected_val, actual_val) in enumerate(zip(expected_list, actual_list)):
	@@ -115,6 +144,11 @@ class TestCIST:
115	self._get_baseline(baseline_json)	144	self._get_baseline(baseline_json)
116	self._get_actual(generate_log, generate_json)	145	self._get_actual(generate_log, generate_json)
117	self._test_helper("throughput")	146	self._test_helper("throughput")
		147	+
		148	+ def test_time(self, baseline_json, generate_log, generate_json):
		149	+ self._get_baseline(baseline_json)
		150	+ self._get_actual(generate_log, generate_json)
		151	+ self._test_helper("time info")
118		152
119	def test_allocated_memory(self, baseline_json, generate_log, generate_json):	153	def test_allocated_memory(self, baseline_json, generate_log, generate_json):
120	self._get_baseline(baseline_json)	154	self._get_baseline(baseline_json)