feat: 重构 CI 配置解析，生成按算子独立的构建命令-ops-math-AtomGit

已合并

feat: 重构 CI 配置解析，生成按算子独立的构建命令 #1989

liulinxiang创建于 3月30日从liulinxiang/ops-math： feat/resolve-config-pipeline 合入到 cann/ops-math： master

feat: 重构 CI 配置解析，生成按算子独立的构建命令 #1989

已合并

liulinxiang创建于 3月30日

从liulinxiang/ops-math feat/resolve-config-pipeline合入到cann/ops-math master

共 4 个文件变更+569-308

Mbuild.sh+12-71

	@@ -948,16 +948,14 @@ checkopts() {
948		948
949	check_param	949	check_param
950	set_create_libs	950	set_create_libs
951	- parse_changed_files
952	set_ut_mode	951	set_ut_mode
953	check_group_compile_config	952	check_group_compile_config
		953	+ if [[ "$CI_MODE" == "TRUE" ]]; then
		954	+ run_ci_mode
		955	+ fi
954	}	956	}
955		957
956	-parse_changed_files() {	958	+run_ci_mode() {
957	- if [[ -z "$CHANGED_FILES" ]]; then
958	- return
959	- fi
960	-
961	if [[ "$CHANGED_FILES" != /* ]]; then	959	if [[ "$CHANGED_FILES" != /* ]]; then
962	CHANGED_FILES=$PWD/$CHANGED_FILES	960	CHANGED_FILES=$PWD/$CHANGED_FILES
963	fi	961	fi
	@@ -968,73 +966,16 @@ parse_changed_files() {
968	cat $CHANGED_FILES	966	cat $CHANGED_FILES
969	echo $dotted_line	967	echo $dotted_line
970		968
971	- COMPILED_OPS=$(python3 scripts/ci/parse_changed_ops.py $CHANGED_FILES "$ENABLE_EXPERIMENTAL")	969	+ # 直接调用 gen_ci_cmd.py 生成命令并执行
972	- echo "related ops "$COMPILED_OPS	970	+ local resolve_cmd="python3 scripts/ci/gen_ci_cmd.py -f $CHANGED_FILES --exec --experimental=${ENABLE_EXPERIMENTAL} --pkg=${ENABLE_PACKAGE} --run_example=${ENABLE_RUN_EXAMPLE}"
973	-	971	+ if [[ -n "$CANN_3RD_LIB_PATH" && "$CANN_3RD_LIB_PATH" != "${BASE_PATH}/third_party" ]]; then
974	- local DEFAULT_OP_SET=FALSE	972	+ resolve_cmd="$resolve_cmd --cann_3rd_lib_path=$CANN_3RD_LIB_PATH"
975	- if [[ -z $COMPILED_OPS ]]; then
976	- if [[ "$ENABLE_EXPERIMENTAL" == "TRUE" ]]; then
977	- COMPILED_OPS='acos'
978	- else
979	- COMPILED_OPS='is_finite'
980	- fi
981	- DEFAULT_OP_SET=TRUE
982	- echo "No ops changed found, set op $COMPILED_OPS as default."
983	fi	973	fi
		974	+ $resolve_cmd
		975	+ local ret=$?
984		976
985	- if [[ "$ENABLE_PACKAGE" == "TRUE" ]]; then	977	+ # 执行完成后退出，返回 gen_ci_cmd.py 的退出码
986	- return	978	+ exit $ret
987	- fi
988	-
989	- local script_ret=$(python3 scripts/ci/parse_changed_files.py $CHANGED_FILES "$ENABLE_EXPERIMENTAL")
990	- IFS='&&' read -r related_ut soc_info <<<"$script_ret"
991	- echo "related ut "$related_ut
992	- echo "related soc_info "$soc_info
993	-
994	- COMPUTE_UNIT=$soc_info
995	-
996	- if [[ "$related_ut" == "set()" ]]; then
997	- # 默认算子时，固定触发 op_api UT 和 op_kernel UT
998	- if [[ "$DEFAULT_OP_SET" == "TRUE" ]]; then
999	- echo "Default op $COMPILED_OPS set, trigger op_api UT and op_kernel UT"
1000	- OP_API_UT=TRUE
1001	- OP_KERNEL_UT=TRUE
1002	- ENABLE_CUSTOM=TRUE
1003	- ENABLE_TEST=TRUE
1004	- COMPUTE_UNIT="ascend910b"
1005	- return
1006	- fi
1007	-
1008	- ENABLE_TEST=FALSE
1009	- echo "no ut matched! no need to run!"
1010	- echo "---------------- CANN build finished ----------------"
1011	- return
1012	- else
1013	- ENABLE_TEST=TRUE
1014	- fi
1015	-
1016	- if [[ "$related_ut" =~ "ALL_UT" ]]; then
1017	- echo "ALL UT is triggered!"
1018	- return
1019	- fi
1020	- if [[ ("$related_ut" =~ "OP_HOST_UT" \|\| "$related_ut" =~ "OP_GRAPH_UT") && "$OP_HOST" == "TRUE" ]]; then
1021	- echo "OP_HOST_UT is triggered!"
1022	- OP_HOST_UT=TRUE
1023	- OP_KERNEL_UT=TRUE
1024	- OP_KERNEL=TRUE
1025	- OP_GRAPH=TRUE
1026	- ENABLE_CUSTOM=TRUE
1027	- fi
1028	- if [[ "$related_ut" =~ "OP_API_UT" && "$OP_API" == "TRUE" ]]; then
1029	- echo "OP_API_UT is triggered!"
1030	- OP_API_UT=TRUE
1031	- ENABLE_CUSTOM=TRUE
1032	- fi
1033	- if [[ "$related_ut" =~ "OP_KERNEL_UT" && "$OP_KERNEL" == "TRUE" ]]; then
1034	- echo "OP_KERNEL_UT is triggered!"
1035	- OP_KERNEL_UT=TRUE
1036	- ENABLE_CUSTOM=TRUE
1037	- fi
1038	}	979	}
1039		980
1040	custom_cmake_args() {	981	custom_cmake_args() {

Ascripts/ci/gen_ci_cmd.py+557-0

@@ -0,0 +1,557 @@
	1	+#!/usr/bin/env python3
	2	+# -- coding: utf-8 --
	3	+# ----------------------------------------------------------------------------
	4	+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
	5	+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
	6	+# CANN Open Software License Agreement Version 2.0 (the "License").
	7	+# Please refer to the License for details. You may not use this file except in compliance with the License.
	8	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
	9	+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
	10	+# See LICENSE in the root of the software repository for the full text of the License.
	11	+# ----------------------------------------------------------------------------
	12	+"""
	13	+根据 CI 变更文件生成构建命令
	14	+
	15	+用法:
	16	+ # UT 测试模式（默认）
	17	+ python3 gen_ci_cmd.py -f changes.txt # 打印命令
	18	+ python3 gen_ci_cmd.py -f changes.txt --exec # 直接执行
	19	+ python3 gen_ci_cmd.py -f changes.txt --experimental=TRUE # 指定 experimental 算子构建
	20	+
	21	+ # CI 出包模式
	22	+ python3 gen_ci_cmd.py -f changes.txt --pkg=TRUE # 生成出包命令
	23	+ python3 gen_ci_cmd.py -f changes.txt --pkg=TRUE --experimental=TRUE # 出包 experimental 算子
	24	+
	25	+ # 示例运行模式
	26	+ python3 gen_ci_cmd.py -f changes.txt --run_example=TRUE # 每个算子运行示例
	27	+
	28	+CI用法
	29	+ # UT
	30	+ ## experimental
	31	+ bash build.sh -f pr_filelist.txt --experimental -u --cann_3rd_lib_path=/home/jenkins/opensource -j16
	32	+ ## 非experimental
	33	+ bash build.sh -u -f pr_filelist.txt --cann_3rd_lib_path=/home/jenkins/opensource -j16
	34	+
	35	+ # 编译
	36	+ ## experimental
	37	+ bash build.sh -f pr_filelist.txt --experimental --cann_3rd_lib_path=/home/jenkins/opensource -j16
	38	+ ## 非experimental
	39	+ bash build.sh --pkg --jit --cann_3rd_lib_path=/home/jenkins/opensource -j16
	40	+
	41	+ #examples
	42	+ ## experimental
	43	+ bash build.sh -f pr_filelist.txt --experimental --run_example
	44	+ ## 非experimental
	45	+ bash build.sh -f pr_filelist.txt --run_example
	46	+
	47	+"""
	48	+import argparse
	49	+import os
	50	+import re
	51	+import subprocess
	52	+import sys
	53	+
	54	+# ============================================================
	55	+# 配置常量
	56	+# ============================================================
	57	+DEFAULT_SOC = 'ascend910b'
	58	+DEFAULT_EXP_OP = 'acos'
	59	+DEFAULT_NORMAL_OP = 'is_finite'
	60	+DEFAULT_UTS = {'ophost', 'opapi', 'opkernel', 'opgraph'}
	61	+
	62	+# 规则定义：pattern 匹配路径，提取对应信息
	63	+RULES = [
	64	+ # experimental 算子
	65	+ {'pattern': r'experimental/(?:math\|conversion\|random)/([^/]+)', 'type': 'exp_ops'},
	66	+ # 普通算子
	67	+ {'pattern': r'^(?:math\|conversion\|random)/([^/]+)', 'type': 'ops'},
	68	+]
	69	+
	70	+# 默认SOC（用于pkg和run_example命令的SOC过滤）
	71	+DEFAULT_FILTER_SOC = 'ascend910b'
	72	+
	73	+
	74	+def check_op_supports_soc(op_name, soc, is_experimental=False):
	75	+ """检查算子是否支持指定的SOC
	76	+
	77	+ 检查方法：查找算子的 op_host/*def.cpp 文件，
	78	+ 检查是否包含 AddConfig("{soc}") 字符串
	79	+
	80	+ Args:
	81	+ op_name: 算子名称
	82	+ soc: SOC名称，如 'ascend910b'
	83	+ is_experimental: 是否为 experimental 算子
	84	+
	85	+ Returns:
	86	+ bool: 是否支持该SOC
	87	+ """
	88	+ prefixes = ['math/', 'conversion/', 'random/']
	89	+ if is_experimental:
	90	+ prefixes = ['experimental/math/', 'experimental/conversion/', 'experimental/random/']
	91	+
	92	+ for prefix in prefixes:
	93	+ op_host_dir = os.path.join(prefix, op_name, 'op_host')
	94	+ if not os.path.isdir(op_host_dir):
	95	+ continue
	96	+
	97	+ for filename in os.listdir(op_host_dir):
	98	+ if filename.endswith('_def.cpp'):
	99	+ def_file = os.path.join(op_host_dir, filename)
	100	+ try:
	101	+ with open(def_file, 'r', encoding='utf-8') as f:
	102	+ content = f.read()
	103	+ if f'AddConfig("{soc}"' in content:
	104	+ return True
	105	+ except (IOError, OSError) as e:
	106	+ # 文件读取失败，继续检查下一个文件
	107	+ continue
	108	+
	109	+ return False
	110	+
	111	+
	112	+def filter_ops_by_soc_support(ops, soc, is_experimental=False):
	113	+ """过滤出支持指定SOC的算子
	114	+
	115	+ Args:
	116	+ ops: 算子集合
	117	+ soc: 目标SOC
	118	+ is_experimental: 是否为 experimental 算子
	119	+
	120	+ Returns:
	121	+ set: 支持该SOC的算子集合
	122	+ """
	123	+ supported_ops = set()
	124	+ for op in ops:
	125	+ if check_op_supports_soc(op, soc, is_experimental):
	126	+ supported_ops.add(op)
	127	+ return supported_ops
	128	+
	129	+
	130	+def read_file_lines(filepath):
	131	+ """读取文件并返回非空、非注释行列表
	132	+
	133	+ Args:
	134	+ filepath: 文件路径
	135	+
	136	+ Returns:
	137	+ list: 行列表，文件不存在返回空列表
	138	+ """
	139	+ try:
	140	+ with open(filepath, 'r', encoding='utf-8') as f:
	141	+ return [line.strip() for line in f if line.strip() and not line.strip().startswith('#')]
	142	+ except (FileNotFoundError, PermissionError, UnicodeDecodeError):
	143	+ return []
	144	+
	145	+
	146	+def parse_changed_files(filepath):
	147	+ """解析变更文件，提取算子
	148	+
	149	+ Returns:
	150	+ dict: {'exp_ops': set(), 'normal_ops': set()}，文件不存在返回空集合字典
	151	+ """
	152	+ exp_ops, normal_ops = set(), set()
	153	+
	154	+ files = read_file_lines(filepath)
	155	+
	156	+ for file_path in files:
	157	+ if file_path.endswith('.md'):
	158	+ continue
	159	+
	160	+ for rule in RULES:
	161	+ m = re.search(rule['pattern'], file_path)
	162	+ if not m:
	163	+ continue
	164	+
	165	+ rule_type = rule['type']
	166	+ if rule_type == 'exp_ops':
	167	+ exp_ops.add(m.group(1))
	168	+ elif rule_type == 'ops':
	169	+ normal_ops.add(m.group(1))
	170	+
	171	+ return {'exp_ops': exp_ops, 'normal_ops': normal_ops}
	172	+
	173	+
	174	+def get_op_ut_types(op_name, files, is_experimental):
	175	+ """获取单个算子的 UT 类型和 SOC
	176	+
	177	+ Returns:
	178	+ tuple: (uts, socs)
	179	+ - uts: UT 类型集合，如 {'ophost', 'opapi'}
	180	+ - socs: 该算子检测到的 SOC 集合（通过 arch35 目录），如 {'ascend950'}
	181	+ """
	182	+ uts = set()
	183	+ socs = set()
	184	+
	185	+ # 算子可能的路径前缀
	186	+ prefixes = ['math/', 'conversion/', 'random/']
	187	+ if is_experimental:
	188	+ prefixes = ['experimental/math/', 'experimental/conversion/', 'experimental/random/']
	189	+
	190	+ for f in files:
	191	+ # 检查文件是否属于该算子
	192	+ matched = False
	193	+ for prefix in prefixes:
	194	+ if f.startswith(f'{prefix}{op_name}/'):
	195	+ matched = True
	196	+ break
	197	+
	198	+ if matched:
	199	+ # 检测 UT 类型
	200	+ if '/op_host/' in f or '/tests/ut/op_host/' in f:
	201	+ uts.add('ophost')
	202	+ if '/op_api/' in f or '/tests/ut/op_api/' in f:
	203	+ uts.add('opapi')
	204	+ if '/op_kernel/' in f or '/tests/ut/op_kernel/' in f:
	205	+ uts.add('opkernel')
	206	+ if '/op_graph/' in f or '/tests/ut/op_graph/' in f:
	207	+ uts.add('opgraph')
	208	+
	209	+ # 检测该算子的 SOC（通过 arch35 目录）
	210	+ if '/arch35/' in f:
	211	+ socs.add('ascend950')
	212	+
	213	+ return uts, socs
	214	+
	215	+
	216	+def make_command(op, uts, soc, cann_3rd_lib_path=None, is_experimental=False):
	217	+ """生成单个算子的构建命令"""
	218	+ cmd_parts = ['bash build.sh', '-u', '-j16']
	219	+
	220	+ if is_experimental:
	221	+ cmd_parts.append('--experimental')
	222	+
	223	+ cmd_parts.append(f"--ops={op}")
	224	+
	225	+ # 使用检测到的 UT 类型，无则使用默认全部
	226	+ for ut in sorted(uts if uts else DEFAULT_UTS):
	227	+ cmd_parts.append(f"--{ut}")
	228	+
	229	+ cmd_parts.append(f"--soc={soc}")
	230	+
	231	+ if cann_3rd_lib_path:
	232	+ cmd_parts.append(f"--cann_3rd_lib_path={cann_3rd_lib_path}")
	233	+
	234	+ return ' '.join(cmd_parts)
	235	+
	236	+
	237	+def make_merged_command(merged_ops, ut_type, soc, cann_3rd_lib_path=None, is_experimental=False):
	238	+ """生成合并后的构建命令（多个算子合并到一条命令）
	239	+ 合并策略：按 (SOC, UT类型) 分组合并算子，减少命令数量
	240	+
	241	+ 合并规则：
	242	+ - opapi/opgraph: 只跑默认 SOC (ascend910b)，所有算子合并为一条命令
	243	+ - ophost/opkernel: 按 SOC 分组（有 arch35 变更时跑多个 SOC），同一 SOC 的算子合并
	244	+ """
	245	+ cmd_parts = ['bash build.sh', '-u', '-j16']
	246	+
	247	+ if is_experimental:
	248	+ cmd_parts.append('--experimental')
	249	+
	250	+ cmd_parts.append(f"--ops={merged_ops}")
	251	+ cmd_parts.append(f"--{ut_type}")
	252	+ cmd_parts.append(f"--soc={soc}")
	253	+
	254	+ if cann_3rd_lib_path:
	255	+ cmd_parts.append(f"--cann_3rd_lib_path={cann_3rd_lib_path}")
	256	+
	257	+ return ' '.join(cmd_parts)
	258	+
	259	+
	260	+def make_run_example_command(op_name, mode, is_experimental=False):
	261	+ """生成算子示例运行命令
	262	+
	263	+ 命令格式：bash build.sh [--experimental] --run_example $opname $mode cust [--vendor_name=experimental]
	264	+
	265	+ Args:
	266	+ op_name: 算子名称
	267	+ mode: 运行模式，'eager' 或 'graph'
	268	+ is_experimental: 是否为 experimental 算子
	269	+ """
	270	+ if is_experimental:
	271	+ cmd_parts = [
	272	+ 'bash build.sh', '--experimental', '--run_example',
	273	+ op_name, mode, 'cust', '--vendor_name=experimental'
	274	+ ]
	275	+ else:
	276	+ cmd_parts = [
	277	+ 'bash build.sh', '--run_example', op_name, mode, 'cust'
	278	+ ]
	279	+ return ' '.join(cmd_parts)
	280	+
	281	+
	282	+def build_ut_commands(filepath, experimental=False, cann_3rd_lib_path=None):
	283	+ """构建 UT 命令列表
	284	+
	285	+ 按算子维度检测 SOC（arch35），每个算子只跑它实际支持的 SOC。
	286	+
	287	+ Args:
	288	+ filepath: 变更文件路径
	289	+ experimental: 是否为 experimental 算子
	290	+ cann_3rd_lib_path: 第三方库路径
	291	+ """
	292	+ parsed = parse_changed_files(filepath)
	293	+ files = read_file_lines(filepath)
	294	+
	295	+ exp_ops = parsed['exp_ops']
	296	+ normal_ops = parsed['normal_ops']
	297	+
	298	+ # 根据参数确定跑哪类算子
	299	+ ops = exp_ops if experimental else normal_ops
	300	+ default_op = DEFAULT_EXP_OP if experimental else DEFAULT_NORMAL_OP
	301	+
	302	+ # 无变更，用默认命令
	303	+ if not ops:
	304	+ return [make_command(default_op, DEFAULT_UTS, DEFAULT_SOC, cann_3rd_lib_path, experimental)]
	305	+
	306	+ # 以下为 UT 命令生成逻辑
	307	+ # 分组收集：{(soc, ut_type): set(op_names)}
	308	+ # 按算子维度检测 SOC，每个算子只跑它实际支持的 SOC
	309	+ groups = {}
	310	+
	311	+ for op in ops:
	312	+ # 按算子维度获取 UT 类型和 SOC
	313	+ uts, op_socs = get_op_ut_types(op, files, experimental)
	314	+
	315	+ # 该算子的 SOC 列表：默认 + 该算子检测到的 arch35
	316	+ op_socs_to_run = {DEFAULT_SOC} \| op_socs
	317	+
	318	+ # opapi/opgraph 不区分 SOC，只跑默认 SOC，合并所有算子
	319	+ if 'opapi' in uts:
	320	+ key = (DEFAULT_SOC, 'opapi')
	321	+ groups.setdefault(key, set()).add(op)
	322	+ if 'opgraph' in uts:
	323	+ key = (DEFAULT_SOC, 'opgraph')
	324	+ groups.setdefault(key, set()).add(op)
	325	+
	326	+ # ophost/opkernel 按该算子检测到的 SOC 分组
	327	+ host_kernel_uts = uts - {'opapi', 'opgraph'}
	328	+ for ut in host_kernel_uts:
	329	+ for soc in op_socs_to_run:
	330	+ key = (soc, ut)
	331	+ groups.setdefault(key, set()).add(op)
	332	+
	333	+ # 生成合并后的命令
	334	+ commands = []
	335	+ for (soc, ut_type), op_names in sorted(groups.items()):
	336	+ # 将算子列表合并为逗号分隔的字符串
	337	+ merged_ops = ','.join(sorted(op_names))
	338	+ cmd = make_merged_command(merged_ops, ut_type, soc, cann_3rd_lib_path, experimental)
	339	+ commands.append(cmd)
	340	+
	341	+ return list(dict.fromkeys(commands)) # 去重保序
	342	+
	343	+
	344	+def check_op_examples(op_name, is_experimental):
	345	+ """检查算子 examples 目录下的测试文件类型
	346	+
	347	+ Args:
	348	+ op_name: 算子名称
	349	+ is_experimental: 是否为 experimental 算子
	350	+
	351	+ Returns:
	352	+ dict: {'has_eager': bool, 'has_graph': bool}
	353	+ """
	354	+ # 算子可能的路径前缀
	355	+ prefixes = ['math/', 'conversion/', 'random/']
	356	+ if is_experimental:
	357	+ prefixes = ['experimental/math/', 'experimental/conversion/', 'experimental/random/']
	358	+
	359	+ result = {'has_eager': False, 'has_graph': False}
	360	+
	361	+ for prefix in prefixes:
	362	+ examples_dir = os.path.join(prefix, op_name, 'examples')
	363	+ if not os.path.isdir(examples_dir):
	364	+ continue
	365	+
	366	+ # 检查目录下的文件
	367	+ try:
	368	+ for filename in os.listdir(examples_dir):
	369	+ if filename.startswith('test_aclnn') and filename.endswith('.cpp'):
	370	+ result['has_eager'] = True
	371	+ if filename.startswith('test_geir') and filename.endswith('.cpp'):
	372	+ result['has_graph'] = True
	373	+ except OSError:
	374	+ continue
	375	+
	376	+ return result
	377	+
	378	+
	379	+def build_example_commands(filepath, experimental=False):
	380	+ """构建 run_example 命令列表
	381	+
	382	+ 根据算子 examples 目录下的测试文件类型生成命令：
	383	+ - 存在 test_aclnn*.cpp → 生成 eager 命令
	384	+ - 存在 test_geir*.cpp → 生成 graph 命令
	385	+
	386	+ Args:
	387	+ filepath: 变更文件路径
	388	+ experimental: 是否为 experimental 算子
	389	+
	390	+ 示例：
	391	+ bash build.sh --run_example add_n eager cust
	392	+ bash build.sh --run_example add_n graph cust
	393	+ """
	394	+ parsed = parse_changed_files(filepath)
	395	+
	396	+ exp_ops = parsed['exp_ops']
	397	+ normal_ops = parsed['normal_ops']
	398	+
	399	+ # 根据参数确定跑哪类算子
	400	+ ops = exp_ops if experimental else normal_ops
	401	+ default_op = DEFAULT_EXP_OP if experimental else DEFAULT_NORMAL_OP
	402	+
	403	+ # 过滤不支持 DEFAULT_FILTER_SOC 的算子
	404	+ ops = filter_ops_by_soc_support(ops, DEFAULT_FILTER_SOC, experimental)
	405	+
	406	+ # 如果没有检测到算子或过滤后为空，使用默认算子
	407	+ if not ops:
	408	+ ops = {default_op}
	409	+
	410	+ # 生成 run_example 命令
	411	+ commands = []
	412	+ for op in sorted(ops):
	413	+ # 检查算子的 examples 目录
	414	+ example_check = check_op_examples(op, experimental)
	415	+
	416	+ # 根据存在的测试文件类型生成对应命令
	417	+ if example_check['has_eager']:
	418	+ commands.append(make_run_example_command(op, 'eager', experimental))
	419	+ # TODO: 后续放开 graph 命令生成
	420	+ # if example_check['has_graph']:
	421	+ # commands.append(make_run_example_command(op, 'graph', experimental))
	422	+
	423	+ return commands
	424	+
	425	+
	426	+def make_package_command(merged_ops, cann_3rd_lib_path=None, is_experimental=False):
	427	+ """生成出包构建命令
	428	+
	429	+ 命令格式：bash build.sh --pkg -j16 --ops=op1,op2 --vendor_name=experimental/custom
	430	+
	431	+ 生成的包名：
	432	+ experimental=FALSE: cann-ops-math-custom_linux-x86_64.run
	433	+ experimental=TRUE: cann-ops-math-experimental_linux-x86_64.run
	434	+
	435	+ Args:
	436	+ merged_ops: 合并后的算子列表字符串，如 "op1,op2,op3"
	437	+ cann_3rd_lib_path: 可选的第三方库路径
	438	+ is_experimental: 是否为 experimental 算子
	439	+ """
	440	+ cmd_parts = ['bash build.sh', '--pkg', '-j16']
	441	+
	442	+ if is_experimental:
	443	+ cmd_parts.append('--experimental')
	444	+ cmd_parts.append('--vendor_name=experimental')
	445	+ else:
	446	+ cmd_parts.append('--vendor_name=custom')
	447	+
	448	+ cmd_parts.append(f"--ops={merged_ops}")
	449	+
	450	+ if cann_3rd_lib_path:
	451	+ cmd_parts.append(f"--cann_3rd_lib_path={cann_3rd_lib_path}")
	452	+
	453	+ return ' '.join(cmd_parts)
	454	+
	455	+
	456	+def build_package_commands(filepath, experimental=False, cann_3rd_lib_path=None):
	457	+ """构建出包命令列表
	458	+
	459	+ 出包模式特点：
	460	+ - 不区分 UT 类型和 SOC
	461	+ - 所有算子合并到一条命令
	462	+
	463	+ Args:
	464	+ filepath: 变更文件路径
	465	+ experimental: 是否为 experimental 算子
	466	+ cann_3rd_lib_path: 第三方库路径
	467	+
	468	+ 示例：
	469	+ bash build.sh --pkg -j16 --ops=op1,op2,op3
	470	+ """
	471	+ parsed = parse_changed_files(filepath)
	472	+
	473	+ exp_ops = parsed['exp_ops']
	474	+ normal_ops = parsed['normal_ops']
	475	+
	476	+ # 根据参数确定跑哪类算子
	477	+ ops = exp_ops if experimental else normal_ops
	478	+ default_op = DEFAULT_EXP_OP if experimental else DEFAULT_NORMAL_OP
	479	+
	480	+ # 过滤不支持 DEFAULT_FILTER_SOC 的算子
	481	+ ops = filter_ops_by_soc_support(ops, DEFAULT_FILTER_SOC, experimental)
	482	+
	483	+ # 无变更或过滤后为空，用默认命令
	484	+ if not ops:
	485	+ return [make_package_command(default_op, cann_3rd_lib_path, experimental)]
	486	+
	487	+ # 所有算子合并到一条命令
	488	+ merged_ops = ','.join(sorted(ops))
	489	+ cmd = make_package_command(merged_ops, cann_3rd_lib_path, experimental)
	490	+
	491	+ return [cmd]
	492	+
	493	+
	494	+def print_commands(commands):
	495	+ """打印生成的命令列表"""
	496	+ print(f"生成 {len(commands)} 条命令:", flush=True)
	497	+ for cmd in commands:
	498	+ print(f" {cmd}", flush=True)
	499	+ print(flush=True)
	500	+
	501	+
	502	+def execute_commands(commands, mode='ut'):
	503	+ """执行命令列表并根据模式打印结果
	504	+
	505	+ Args:
	506	+ commands: 命令列表
	507	+ mode: 执行模式，可选 'ut', 'pkg', 'example'
	508	+
	509	+ Returns:
	510	+ int: 0 表示全部成功，1 表示有命令失败
	511	+ """
	512	+ for cmd in commands:
	513	+ print(f"执行: {cmd}", flush=True)
	514	+ result = subprocess.run(cmd, shell=True)
	515	+ if result.returncode != 0:
	516	+ print(f"run {mode} fail: {cmd} (返回码: {result.returncode})", flush=True)
	517	+ return 1
	518	+ print(f"命令成功: {cmd}", flush=True)
	519	+ return 0
	520	+
	521	+
	522	+def main():
	523	+ parser = argparse.ArgumentParser(description='根据 CI 变更文件生成构建命令')
	524	+ parser.add_argument('-f', '--file', required=True, help='变更文件列表必选参数')
	525	+ parser.add_argument('--exec', action='store_true', help='直接执行生成的命令可选参数')
	526	+ parser.add_argument('--experimental', choices=['TRUE', 'FALSE'], default='FALSE',
	527	+ help='可选参数默认FALSE TRUE表示跑experimental目录下的用例检测不到算子跑默认experimental目录下的acos算子 '
	528	+ 'FALSE表示跑基本算子的用例检查不到算子跑math目录下的is_finite 是否指定 experimental 算子构建 (TRUE/FALSE)')
	529	+ parser.add_argument('--pkg', choices=['TRUE', 'FALSE'], default='FALSE',
	530	+ help='是否生成出包命令 (TRUE/FALSE) 可选参数默认是FALSE TRUE表示对涉及变更的算子打自定义算子包 '
	531	+ 'FALSE 不打包只跑UT')
	532	+ parser.add_argument('--run_example', choices=['TRUE', 'FALSE'], default='FALSE',
	533	+ help='是否生成 run_example 命令 (TRUE/FALSE) 可选参数默认FALSE TRUE表示每个算子运行示例')
	534	+ parser.add_argument('--cann_3rd_lib_path', help='可选参数 CANN third party lib path')
	535	+ args = parser.parse_args()
	536	+
	537	+ # 根据模式选择不同的命令生成函数
	538	+ if args.run_example == 'TRUE':
	539	+ commands = build_example_commands(args.file, args.experimental == 'TRUE')
	540	+ mode = 'example'
	541	+ elif args.pkg == 'TRUE':
	542	+ commands = build_package_commands(args.file, args.experimental == 'TRUE', args.cann_3rd_lib_path)
	543	+ mode = 'pkg'
	544	+ else:
	545	+ commands = build_ut_commands(args.file, args.experimental == 'TRUE', args.cann_3rd_lib_path)
	546	+ mode = 'ut'
	547	+
	548	+ # 打印生成的命令
	549	+ print_commands(commands)
	550	+
	551	+ # 执行命令
	552	+ if args.exec:
	553	+ sys.exit(execute_commands(commands, mode))
	554	+
	555	+
	556	+if __name__ == '__main__':
	557	+ main()

Dscripts/ci/parse_changed_files.py+0-145

	@@ -1,145 +0,0 @@
1	-# Copyright (c) 2025 Huawei Technologies Co., Ltd.
2	-# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
3	-# CANN Open Software License Agreement Version 2.0 (the "License").
4	-# Please refer to the License for details. You may not use this file except in compliance with the License.
5	-# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
6	-# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
7	-# See LICENSE in the root of the software repository for the full text of the License.
8	-
9	-import os
10	-import sys
11	-import re
12	-import logging
13	-
14	-OP_API_UT = "OP_API_UT"
15	-OP_HOST_UT = "OP_HOST_UT"
16	-OP_GRAPH_UT = "OP_GRAPH_UT"
17	-OP_KERNEL_UT = "OP_KERNEL_UT"
18	-ALL_UT = "ALL_UT"
19	-
20	-NEW_OPS_PATH = [
21	- "math",
22	- "conversion",
23	- "random"
24	- # 添加更多算子路径
25	-]
26	-NEW_EXPERIMENTAL_OPS_PATH = [
27	- "experimental/math",
28	- "experimental/conversion",
29	- "experimental/random"
30	- # 添加更多算子路径
31	-]
32	-COMM_FILES = [
33	- "tests",
34	- "common"
35	- # 添加更多算子路径
36	-]
37	-SOC_MAPPING = {
38	- "arch35": "ascend950"
39	-}
40	-
41	-
42	-class FileChangeInfo:
43	- def __init__(self, op_api_changed_files=None, op_host_changed_files=None, op_graph_changed_files=None,
44	- op_kernel_changed_files=None, comm_changed_files=None, soc_info=None):
45	- self.op_api_changed_files = [] if op_api_changed_files is None else op_api_changed_files
46	- self.op_host_changed_files = [] if op_host_changed_files is None else op_host_changed_files
47	- self.op_graph_changed_files = [] if op_graph_changed_files is None else op_graph_changed_files
48	- self.op_kernel_changed_files = [] if op_kernel_changed_files is None else op_kernel_changed_files
49	- self.comm_changed_files = [] if comm_changed_files is None else comm_changed_files
50	- self.soc_info = set() if soc_info is None else soc_info
51	-
52	-
53	-def get_file_change_info_from_ci(changed_file_info_from_ci, ops_path):
54	- """
55	- get file change info from ci, ci will write `git diff > /or_filelist.txt`
56	- :param changed_file_info_from_ci: git diff result file from ci
57	- :return: None or FileChangeInf
58	- """
59	- or_file_path = os.path.realpath(changed_file_info_from_ci)
60	- if not os.path.exists(or_file_path):
61	- logging.error("[ERROR] change file is not exist, can not get file change info in this pull request.")
62	- return None
63	- with open(or_file_path) as or_f:
64	- lines = or_f.readlines()
65	- op_api_changed_files = []
66	- op_host_changed_files = []
67	- op_graph_changed_files = []
68	- op_kernel_changed_files = []
69	- comm_changed_files = []
70	- soc_info = set()
71	- host_pattern = re.compile(rf"({'\|'.join(ops_path)})/./op_host/.\.(cc\|cpp\|h)$")
72	- api_pattern = re.compile(rf"({'\|'.join(ops_path)})/./op_api/.\.(cc\|cpp\|h)$")
73	- kernel_pattern = re.compile(rf"({'\|'.join(ops_path)})/./op_kernel/.\.(cc\|cpp\|h)$")
74	- graph_pattern = re.compile(rf"({'\|'.join(ops_path)})/./op_graph/.\.(cc\|cpp\|h)$")
75	- host_test_pattern = re.compile(rf"({'\|'.join(ops_path)})/./tests/ut/op_host/.\.(cc\|cpp\|txt)$")
76	- api_test_pattern = re.compile(rf"({'\|'.join(ops_path)})/./tests/ut/op_api/.\.(cc\|cpp\|txt\|py)$")
77	- graph_test_pattern = re.compile(rf"({'\|'.join(ops_path)})/./tests/ut/op_graph/.\.(cc\|cpp\|txt)$")
78	- kernel_test_pattern = re.compile(rf"({'\|'.join(ops_path)})/./tests/ut/op_kernel/.\.(cc\|cpp\|txt)$")
79	- comm_files_pattern = re.compile(rf"^({'\|'.join(COMM_FILES)})")
80	- soc_pattern = re.compile(rf"({'\|'.join(re.escape(key) for key in SOC_MAPPING)})")
81	-
82	- for line in lines:
83	- line = line.strip()
84	- ext = os.path.splitext(line)[-1].lower()
85	- if ext in (".md",):
86	- continue
87	- if not os.path.exists(line):
88	- continue
89	- if api_pattern.match(line) or api_test_pattern.match(line):
90	- op_api_changed_files.append(line)
91	- elif host_pattern.match(line) or host_test_pattern.match(line):
92	- op_host_changed_files.append(line)
93	- elif kernel_pattern.match(line) or kernel_test_pattern.match(line):
94	- op_kernel_changed_files.append(line)
95	- elif graph_pattern.match(line) or graph_test_pattern.match(line):
96	- op_graph_changed_files.append(line)
97	- elif comm_files_pattern.match(line):
98	- comm_changed_files.append(line)
99	- soc_match = soc_pattern.search(line)
100	- if soc_match:
101	- matched_key = soc_match.group(1)
102	- soc_info.add(SOC_MAPPING[matched_key])
103	-
104	- return FileChangeInfo(op_host_changed_files=op_host_changed_files,
105	- op_api_changed_files=op_api_changed_files,
106	- op_graph_changed_files=op_graph_changed_files,
107	- op_kernel_changed_files=op_kernel_changed_files,
108	- comm_changed_files=comm_changed_files,
109	- soc_info=soc_info)
110	-
111	-
112	-def get_change_relate_ut_dir_list(changed_file_info_from_ci, is_experimental):
113	- if is_experimental == "TRUE":
114	- ops_path = NEW_EXPERIMENTAL_OPS_PATH
115	- else:
116	- ops_path = NEW_OPS_PATH
117	- file_change_info = get_file_change_info_from_ci(changed_file_info_from_ci, ops_path)
118	- if not file_change_info:
119	- logging.info("[INFO] not found file change info, run all c++.")
120	- return None
121	-
122	- def _get_relate_ut_list_by_file_change():
123	- relate_ut = set()
124	- if len(file_change_info.op_host_changed_files) > 0:
125	- relate_ut.add(OP_HOST_UT)
126	- if len(file_change_info.op_api_changed_files) > 0:
127	- relate_ut.add(OP_API_UT)
128	- if len(file_change_info.op_graph_changed_files) > 0:
129	- relate_ut.add(OP_GRAPH_UT)
130	- if len(file_change_info.op_kernel_changed_files) > 0:
131	- relate_ut.add(OP_KERNEL_UT)
132	- if len(file_change_info.comm_changed_files) > 0:
133	- relate_ut.add(ALL_UT)
134	- return relate_ut
135	-
136	- try:
137	- relate_uts = _get_relate_ut_list_by_file_change()
138	- except BaseException as e:
139	- logging.error(e.args)
140	- return None
141	- return f'{str(relate_uts)}&{",".join(file_change_info.soc_info)}'
142	-
143	-
144	-if __name__ == '__main__':
145	- print(get_change_relate_ut_dir_list(sys.argv[1], sys.argv[2]))

Dscripts/ci/parse_changed_ops.py+0-92

	@@ -1,92 +0,0 @@
1	-# Copyright (c) 2025 Huawei Technologies Co., Ltd.
2	-# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
3	-# CANN Open Software License Agreement Version 2.0 (the "License").
4	-# Please refer to the License for details. You may not use this file except in compliance with the License.
5	-# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
6	-# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
7	-# See LICENSE in the root of the software repository for the full text of the License.
8	-
9	-import os
10	-import sys
11	-import re
12	-import logging
13	-
14	-NEW_OPS_PATH = [
15	- "math",
16	- "conversion",
17	- "random"
18	- # 添加更多算子路径
19	-]
20	-
21	-
22	-class OperatorChangeInfo:
23	- def __init__(self, changed_operators=None, operator_file_map=None):
24	- self.changed_operators = [] if changed_operators is None else changed_operators
25	- self.operator_file_map = {} if operator_file_map is None else operator_file_map
26	-
27	-
28	-def extract_operator_name(file_path, is_experimental):
29	- clean_path = file_path.lstrip('/')
30	- path_parts = clean_path.split('/')
31	- default_name = ''
32	- operator_name = ''
33	- domain = ''
34	- if is_experimental == "TRUE":
35	- if len(path_parts) >= 3:
36	- domain = path_parts[1]
37	- operator_name = path_parts[2]
38	- if operator_name == "common" or not os.path.exists(f'experimental/{domain}/{operator_name}'):
39	- return default_name
40	- else:
41	- if len(path_parts) >= 2:
42	- domain = path_parts[0]
43	- operator_name = path_parts[1]
44	- if operator_name == "common" or not os.path.exists(f'{domain}/{operator_name}'):
45	- return default_name
46	- if domain in NEW_OPS_PATH:
47	- return operator_name
48	- return default_name
49	-
50	-
51	-def get_operator_info_from_ci(changed_file_info_from_ci, is_experimental):
52	- """
53	- get operator change info from ci, ci will write `git diff > /or_filelist.txt`
54	- :param changed_file_info_from_ci: git diff result file from ci
55	- :return: None or OperatorChangeInf
56	- """
57	- or_file_path = os.path.realpath(changed_file_info_from_ci)
58	- if not os.path.exists(or_file_path):
59	- logging.error("[ERROR] change file is not exist, can not get file change info in this pull request.")
60	- return None
61	- with open(or_file_path) as or_f:
62	- lines = or_f.readlines()
63	- changed_operators = set()
64	- operator_file_map = {}
65	-
66	- for line in lines:
67	- line = line.strip()
68	- ext = os.path.splitext(line)[-1].lower()
69	- if ext in (".md",):
70	- continue
71	- operator_name = extract_operator_name(line, is_experimental)
72	- if not operator_name:
73	- continue
74	- changed_operators.add(operator_name)
75	- if operator_name not in operator_file_map:
76	- operator_file_map[operator_name] = []
77	- operator_file_map[operator_name].append(line)
78	-
79	- return OperatorChangeInfo(changed_operators=list(changed_operators), operator_file_map=operator_file_map)
80	-
81	-
82	-def get_change_ops_list(changed_file_info_from_ci, is_experimental):
83	- ops_change_info = get_operator_info_from_ci(changed_file_info_from_ci, is_experimental)
84	- if not ops_change_info:
85	- logging.info("[INFO] not found ops change info, run all c++.")
86	- return None
87	-
88	- return ";".join(ops_change_info.changed_operators)
89	-
90	-
91	-if __name__ == '__main__':
92	- print(get_change_ops_list(sys.argv[1], sys.argv[2]))