已合并
feat(ci): 出包命令过滤无 op_kernel 目录的算子 #4896
songkai111创建于 8月20日
feat(ci): 出包命令过滤无 op_kernel 目录的算子 #4896
已合并
共 1 个文件变更+208-101
| @@ -45,6 +45,7 @@ CI用法 | |||
| 45 | bash build.sh -f pr_filelist.txt --run_example | 45 | bash build.sh -f pr_filelist.txt --run_example |
| 46 | 46 | ||
| 47 | """ | 47 | """ |
| 48 | + | ||
| 48 | import argparse | 49 | import argparse |
| 49 | import os | 50 | import os |
| 50 | import re | 51 | import re |
| @@ -54,21 +55,21 @@ import sys | |||
| 54 | # ============================================================ | 55 | # ============================================================ |
| 55 | # 配置常量 | 56 | # 配置常量 |
| 56 | # ============================================================ | 57 | # ============================================================ |
| 57 | -DEFAULT_SOC = 'ascend910b' | 58 | +DEFAULT_SOC = "ascend910b" |
| 58 | -DEFAULT_EXP_OP = 'acos' | 59 | +DEFAULT_EXP_OP = "acos" |
| 59 | -DEFAULT_NORMAL_OP = 'is_finite' | 60 | +DEFAULT_NORMAL_OP = "is_finite" |
| 60 | -DEFAULT_UTS = {'ophost', 'opapi', 'opkernel', 'opgraph'} | 61 | +DEFAULT_UTS = {"ophost", "opapi", "opkernel", "opgraph"} |
| 61 | 62 | ||
| 62 | # 规则定义：pattern 匹配路径，提取对应信息 | 63 | # 规则定义：pattern 匹配路径，提取对应信息 |
| 63 | RULES = [ | 64 | RULES = [ |
| 64 | # experimental 算子 | 65 | # experimental 算子 |
| 65 | - {'pattern': r'experimental/(?:math|conversion|random)/([^/]+)', 'type': 'exp_ops'}, | 66 | + {"pattern": r"experimental/(?:math|conversion|random)/([^/]+)", "type": "exp_ops"}, |
| 66 | # 普通算子 | 67 | # 普通算子 |
| 67 | - {'pattern': r'^(?:math|conversion|random)/([^/]+)', 'type': 'ops'}, | 68 | + {"pattern": r"^(?:math|conversion|random)/([^/]+)", "type": "ops"}, |
| 68 | ] | 69 | ] |
| 69 | 70 | ||
| 70 | # 默认SOC（用于pkg和run_example命令的SOC过滤） | 71 | # 默认SOC（用于pkg和run_example命令的SOC过滤） |
| 71 | -DEFAULT_FILTER_SOC = 'ascend910b' | 72 | +DEFAULT_FILTER_SOC = "ascend910b" |
| 72 | 73 | ||
| 73 | 74 | ||
| 74 | def check_op_supports_soc(op_name, soc, is_experimental=False): | 75 | def check_op_supports_soc(op_name, soc, is_experimental=False): |
| @@ -85,24 +86,28 @@ def check_op_supports_soc(op_name, soc, is_experimental=False): | |||
| 85 | Returns: | 86 | Returns: |
| 86 | bool: 是否支持该SOC | 87 | bool: 是否支持该SOC |
| 87 | """ | 88 | """ |
| 88 | - prefixes = ['math/', 'conversion/', 'random/'] | 89 | + prefixes = ["math/", "conversion/", "random/"] |
| 89 | if is_experimental: | 90 | if is_experimental: |
| 90 | - prefixes = ['experimental/math/', 'experimental/conversion/', 'experimental/random/'] | 91 | + prefixes = [ |
| 92 | + "experimental/math/", | ||
| 93 | + "experimental/conversion/", | ||
| 94 | + "experimental/random/", | ||
| 95 | + ] | ||
| 91 | 96 | ||
| 92 | for prefix in prefixes: | 97 | for prefix in prefixes: |
| 93 | - op_host_dir = os.path.join(prefix, op_name, 'op_host') | 98 | + op_host_dir = os.path.join(prefix, op_name, "op_host") |
| 94 | if not os.path.isdir(op_host_dir): | 99 | if not os.path.isdir(op_host_dir): |
| 95 | continue | 100 | continue |
| 96 | 101 | ||
| 97 | for filename in os.listdir(op_host_dir): | 102 | for filename in os.listdir(op_host_dir): |
| 98 | - if filename.endswith('_def.cpp'): | 103 | + if filename.endswith("_def.cpp"): |
| 99 | def_file = os.path.join(op_host_dir, filename) | 104 | def_file = os.path.join(op_host_dir, filename) |
| 100 | try: | 105 | try: |
| 101 | - with open(def_file, 'r', encoding='utf-8') as f: | 106 | + with open(def_file, "r", encoding="utf-8") as f: |
| 102 | content = f.read() | 107 | content = f.read() |
| 103 | if f'AddConfig("{soc}"' in content: | 108 | if f'AddConfig("{soc}"' in content: |
| 104 | return True | 109 | return True |
| 105 | - except (IOError, OSError) as e: | 110 | + except (IOError, OSError): |
| 106 | # 文件读取失败，继续检查下一个文件 | 111 | # 文件读取失败，继续检查下一个文件 |
| 107 | continue | 112 | continue |
| 108 | 113 | ||
| @@ -127,6 +132,47 @@ def filter_ops_by_soc_support(ops, soc, is_experimental=False): | |||
| 127 | return supported_ops | 132 | return supported_ops |
| 128 | 133 | ||
| 129 | 134 | ||
| 135 | +def check_op_has_op_kernel(op_name, is_experimental=False): | ||
| 136 | + """检查算子是否存在 op_kernel 目录 | ||
| 137 | + | ||
| 138 | + 出包模式需要生成 kernel 二进制，没有 op_kernel 目录的算子无法出包。 | ||
| 139 | + | ||
| 140 | + Args: | ||
| 141 | + op_name: 算子名称 | ||
| 142 | + is_experimental: 是否为 experimental 算子 | ||
| 143 | + | ||
| 144 | + Returns: | ||
| 145 | + bool: 是否存在 op_kernel 目录 | ||
| 146 | + """ | ||
| 147 | + prefixes = ["math/", "conversion/", "random/"] | ||
| 148 | + if is_experimental: | ||
| 149 | + prefixes = [ | ||
| 150 | + "experimental/math/", | ||
| 151 | + "experimental/conversion/", | ||
| 152 | + "experimental/random/", | ||
| 153 | + ] | ||
| 154 | + | ||
| 155 | + for prefix in prefixes: | ||
| 156 | + op_kernel_dir = os.path.join(prefix, op_name, "op_kernel") | ||
| 157 | + if os.path.isdir(op_kernel_dir): | ||
| 158 | + return True | ||
| 159 | + | ||
| 160 | + return False | ||
| 161 | + | ||
| 162 | + | ||
| 163 | +def filter_ops_by_op_kernel(ops, is_experimental=False): | ||
| 164 | + """过滤出存在 op_kernel 目录的算子 | ||
| 165 | + | ||
| 166 | + Args: | ||
| 167 | + ops: 算子集合 | ||
| 168 | + is_experimental: 是否为 experimental 算子 | ||
| 169 | + | ||
| 170 | + Returns: | ||
| 171 | + set: 存在 op_kernel 目录的算子集合 | ||
| 172 | + """ | ||
| 173 | + return {op for op in ops if check_op_has_op_kernel(op, is_experimental)} | ||
| 174 | + | ||
| 175 | + | ||
| 130 | def read_file_lines(filepath): | 176 | def read_file_lines(filepath): |
| 131 | """读取文件并返回非空、非注释行列表 | 177 | """读取文件并返回非空、非注释行列表 |
| 132 | 178 | ||
| @@ -137,8 +183,12 @@ def read_file_lines(filepath): | |||
| 137 | list: 行列表，文件不存在返回空列表 | 183 | list: 行列表，文件不存在返回空列表 |
| 138 | """ | 184 | """ |
| 139 | try: | 185 | try: |
| 140 | - with open(filepath, 'r', encoding='utf-8') as f: | 186 | + with open(filepath, "r", encoding="utf-8") as f: |
| 141 | - return [line.strip() for line in f if line.strip() and not line.strip().startswith('#')] | 187 | + return [ |
| 188 | + line.strip() | ||
| 189 | + for line in f | ||
| 190 | + if line.strip() and not line.strip().startswith("#") | ||
| 191 | + ] | ||
| 142 | except (FileNotFoundError, PermissionError, UnicodeDecodeError): | 192 | except (FileNotFoundError, PermissionError, UnicodeDecodeError): |
| 143 | return [] | 193 | return [] |
| 144 | 194 | ||
| @@ -154,23 +204,23 @@ def parse_changed_files(filepath): | |||
| 154 | files = read_file_lines(filepath) | 204 | files = read_file_lines(filepath) |
| 155 | 205 | ||
| 156 | for file_path in files: | 206 | for file_path in files: |
| 157 | - if file_path.endswith('.md'): | 207 | + if file_path.endswith(".md"): |
| 158 | continue | 208 | continue |
| 159 | if not os.path.exists(file_path): | 209 | if not os.path.exists(file_path): |
| 160 | continue | 210 | continue |
| 161 | 211 | ||
| 162 | for rule in RULES: | 212 | for rule in RULES: |
| 163 | - m = re.search(rule['pattern'], file_path) | 213 | + m = re.search(rule["pattern"], file_path) |
| 164 | if not m: | 214 | if not m: |
| 165 | continue | 215 | continue |
| 166 | 216 | ||
| 167 | - rule_type = rule['type'] | 217 | + rule_type = rule["type"] |
| 168 | - if rule_type == 'exp_ops': | 218 | + if rule_type == "exp_ops": |
| 169 | exp_ops.add(m.group(1)) | 219 | exp_ops.add(m.group(1)) |
| 170 | - elif rule_type == 'ops': | 220 | + elif rule_type == "ops": |
| 171 | normal_ops.add(m.group(1)) | 221 | normal_ops.add(m.group(1)) |
| 172 | 222 | ||
| 173 | - return {'exp_ops': exp_ops, 'normal_ops': normal_ops} | 223 | + return {"exp_ops": exp_ops, "normal_ops": normal_ops} |
| 174 | 224 | ||
| 175 | 225 | ||
| 176 | def get_op_ut_types(op_name, files, is_experimental): | 226 | def get_op_ut_types(op_name, files, is_experimental): |
| @@ -185,42 +235,46 @@ def get_op_ut_types(op_name, files, is_experimental): | |||
| 185 | socs = set() | 235 | socs = set() |
| 186 | 236 | ||
| 187 | # 算子可能的路径前缀 | 237 | # 算子可能的路径前缀 |
| 188 | - prefixes = ['math/', 'conversion/', 'random/'] | 238 | + prefixes = ["math/", "conversion/", "random/"] |
| 189 | if is_experimental: | 239 | if is_experimental: |
| 190 | - prefixes = ['experimental/math/', 'experimental/conversion/', 'experimental/random/'] | 240 | + prefixes = [ |
| 241 | + "experimental/math/", | ||
| 242 | + "experimental/conversion/", | ||
| 243 | + "experimental/random/", | ||
| 244 | + ] | ||
| 191 | 245 | ||
| 192 | for f in files: | 246 | for f in files: |
| 193 | # 检查文件是否属于该算子 | 247 | # 检查文件是否属于该算子 |
| 194 | matched = False | 248 | matched = False |
| 195 | for prefix in prefixes: | 249 | for prefix in prefixes: |
| 196 | - if f.startswith(f'{prefix}{op_name}/'): | 250 | + if f.startswith(f"{prefix}{op_name}/"): |
| 197 | matched = True | 251 | matched = True |
| 198 | break | 252 | break |
| 199 | 253 | ||
| 200 | if matched: | 254 | if matched: |
| 201 | # 检测 UT 类型 | 255 | # 检测 UT 类型 |
| 202 | - if '/op_host/' in f or '/tests/ut/op_host/' in f: | 256 | + if "/op_host/" in f or "/tests/ut/op_host/" in f: |
| 203 | - uts.add('ophost') | 257 | + uts.add("ophost") |
| 204 | - if '/op_api/' in f or '/tests/ut/op_api/' in f: | 258 | + if "/op_api/" in f or "/tests/ut/op_api/" in f: |
| 205 | - uts.add('opapi') | 259 | + uts.add("opapi") |
| 206 | - if '/op_kernel/' in f or '/tests/ut/op_kernel/' in f: | 260 | + if "/op_kernel/" in f or "/tests/ut/op_kernel/" in f: |
| 207 | - uts.add('opkernel') | 261 | + uts.add("opkernel") |
| 208 | - if '/op_graph/' in f or '/tests/ut/op_graph/' in f: | 262 | + if "/op_graph/" in f or "/tests/ut/op_graph/" in f: |
| 209 | - uts.add('opgraph') | 263 | + uts.add("opgraph") |
| 210 | 264 | ||
| 211 | # 检测该算子的 SOC（通过 arch35 目录） | 265 | # 检测该算子的 SOC（通过 arch35 目录） |
| 212 | - if '/arch35/' in f: | 266 | + if "/arch35/" in f: |
| 213 | - socs.add('ascend950') | 267 | + socs.add("ascend950") |
| 214 | 268 | ||
| 215 | return uts, socs | 269 | return uts, socs |
| 216 | 270 | ||
| 217 | 271 | ||
| 218 | def make_command(op, uts, soc, cann_3rd_lib_path=None, is_experimental=False): | 272 | def make_command(op, uts, soc, cann_3rd_lib_path=None, is_experimental=False): |
| 219 | """生成单个算子的构建命令""" | 273 | """生成单个算子的构建命令""" |
| 220 | - cmd_parts = ['bash', 'build.sh', '-u', '-j16'] | 274 | + cmd_parts = ["bash", "build.sh", "-u", "-j16"] |
| 221 | 275 | ||
| 222 | if is_experimental: | 276 | if is_experimental: |
| 223 | - cmd_parts.append('--experimental') | 277 | + cmd_parts.append("--experimental") |
| 224 | 278 | ||
| 225 | cmd_parts.append(f"--ops={op}") | 279 | cmd_parts.append(f"--ops={op}") |
| 226 | 280 | ||
| @@ -236,7 +290,9 @@ def make_command(op, uts, soc, cann_3rd_lib_path=None, is_experimental=False): | |||
| 236 | return cmd_parts | 290 | return cmd_parts |
| 237 | 291 | ||
| 238 | 292 | ||
| 239 | -def make_merged_command(merged_ops, ut_type, soc, cann_3rd_lib_path=None, is_experimental=False): | 293 | +def make_merged_command( |
| 294 | + merged_ops, ut_type, soc, cann_3rd_lib_path=None, is_experimental=False | ||
| 295 | +): | ||
| 240 | """生成合并后的构建命令（多个算子合并到一条命令） | 296 | """生成合并后的构建命令（多个算子合并到一条命令） |
| 241 | 合并策略：按 (SOC, UT类型) 分组合并算子，减少命令数量 | 297 | 合并策略：按 (SOC, UT类型) 分组合并算子，减少命令数量 |
| 242 | 298 | ||
| @@ -244,10 +300,10 @@ def make_merged_command(merged_ops, ut_type, soc, cann_3rd_lib_path=None, is_exp | |||
| 244 | - opapi/opgraph: 只跑默认 SOC (ascend910b)，所有算子合并为一条命令 | 300 | - opapi/opgraph: 只跑默认 SOC (ascend910b)，所有算子合并为一条命令 |
| 245 | - ophost/opkernel: 按 SOC 分组（有 arch35 变更时跑多个 SOC），同一 SOC 的算子合并 | 301 | - ophost/opkernel: 按 SOC 分组（有 arch35 变更时跑多个 SOC），同一 SOC 的算子合并 |
| 246 | """ | 302 | """ |
| 247 | - cmd_parts = ['bash', 'build.sh', '-u', '-j16'] | 303 | + cmd_parts = ["bash", "build.sh", "-u", "-j16"] |
| 248 | 304 | ||
| 249 | if is_experimental: | 305 | if is_experimental: |
| 250 | - cmd_parts.append('--experimental') | 306 | + cmd_parts.append("--experimental") |
| 251 | 307 | ||
| 252 | cmd_parts.append(f"--ops={merged_ops}") | 308 | cmd_parts.append(f"--ops={merged_ops}") |
| 253 | cmd_parts.append(f"--{ut_type}") | 309 | cmd_parts.append(f"--{ut_type}") |
| @@ -271,13 +327,17 @@ def make_run_example_command(op_name, mode, is_experimental=False): | |||
| 271 | """ | 327 | """ |
| 272 | if is_experimental: | 328 | if is_experimental: |
| 273 | cmd_parts = [ | 329 | cmd_parts = [ |
| 274 | - 'bash', 'build.sh', '--experimental', '--run_example', | 330 | + "bash", |
| 275 | - op_name, mode, 'cust', '--vendor_name=experimental' | 331 | + "build.sh", |
| 332 | + "--experimental", | ||
| 333 | + "--run_example", | ||
| 334 | + op_name, | ||
| 335 | + mode, | ||
| 336 | + "cust", | ||
| 337 | + "--vendor_name=experimental", | ||
| 276 | ] | 338 | ] |
| 277 | else: | 339 | else: |
| 278 | - cmd_parts = [ | 340 | + cmd_parts = ["bash", "build.sh", "--run_example", op_name, mode, "cust"] |
| 279 | - 'bash', 'build.sh', '--run_example', op_name, mode, 'cust' | ||
| 280 | - ] | ||
| 281 | return cmd_parts | 341 | return cmd_parts |
| 282 | 342 | ||
| 283 | 343 | ||
| @@ -294,8 +354,8 @@ def build_ut_commands(filepath, experimental=False, cann_3rd_lib_path=None): | |||
| 294 | parsed = parse_changed_files(filepath) | 354 | parsed = parse_changed_files(filepath) |
| 295 | files = read_file_lines(filepath) | 355 | files = read_file_lines(filepath) |
| 296 | 356 | ||
| 297 | - exp_ops = parsed['exp_ops'] | 357 | + exp_ops = parsed["exp_ops"] |
| 298 | - normal_ops = parsed['normal_ops'] | 358 | + normal_ops = parsed["normal_ops"] |
| 299 | 359 | ||
| 300 | # 根据参数确定跑哪类算子 | 360 | # 根据参数确定跑哪类算子 |
| 301 | ops = exp_ops if experimental else normal_ops | 361 | ops = exp_ops if experimental else normal_ops |
| @@ -303,7 +363,11 @@ def build_ut_commands(filepath, experimental=False, cann_3rd_lib_path=None): | |||
| 303 | 363 | ||
| 304 | # 无变更，用默认命令 | 364 | # 无变更，用默认命令 |
| 305 | if not ops: | 365 | if not ops: |
| 306 | - return [make_command(default_op, DEFAULT_UTS, DEFAULT_SOC, cann_3rd_lib_path, experimental)] | 366 | + return [ |
| 367 | + make_command( | ||
| 368 | + default_op, DEFAULT_UTS, DEFAULT_SOC, cann_3rd_lib_path, experimental | ||
| 369 | + ) | ||
| 370 | + ] | ||
| 307 | 371 | ||
| 308 | # 以下为 UT 命令生成逻辑 | 372 | # 以下为 UT 命令生成逻辑 |
| 309 | # 分组收集：{(soc, ut_type): set(op_names)} | 373 | # 分组收集：{(soc, ut_type): set(op_names)} |
| @@ -318,15 +382,15 @@ def build_ut_commands(filepath, experimental=False, cann_3rd_lib_path=None): | |||
| 318 | op_socs_to_run = {DEFAULT_SOC} | op_socs | 382 | op_socs_to_run = {DEFAULT_SOC} | op_socs |
| 319 | 383 | ||
| 320 | # opapi/opgraph 不区分 SOC，只跑默认 SOC，合并所有算子 | 384 | # opapi/opgraph 不区分 SOC，只跑默认 SOC，合并所有算子 |
| 321 | - if 'opapi' in uts: | 385 | + if "opapi" in uts: |
| 322 | - key = (DEFAULT_SOC, 'opapi') | 386 | + key = (DEFAULT_SOC, "opapi") |
| 323 | groups.setdefault(key, set()).add(op) | 387 | groups.setdefault(key, set()).add(op) |
| 324 | - if 'opgraph' in uts: | 388 | + if "opgraph" in uts: |
| 325 | - key = (DEFAULT_SOC, 'opgraph') | 389 | + key = (DEFAULT_SOC, "opgraph") |
| 326 | groups.setdefault(key, set()).add(op) | 390 | groups.setdefault(key, set()).add(op) |
| 327 | 391 | ||
| 328 | # ophost/opkernel 按该算子检测到的 SOC 分组 | 392 | # ophost/opkernel 按该算子检测到的 SOC 分组 |
| 329 | - host_kernel_uts = uts - {'opapi', 'opgraph'} | 393 | + host_kernel_uts = uts - {"opapi", "opgraph"} |
| 330 | for ut in host_kernel_uts: | 394 | for ut in host_kernel_uts: |
| 331 | for soc in op_socs_to_run: | 395 | for soc in op_socs_to_run: |
| 332 | key = (soc, ut) | 396 | key = (soc, ut) |
| @@ -336,8 +400,10 @@ def build_ut_commands(filepath, experimental=False, cann_3rd_lib_path=None): | |||
| 336 | commands = [] | 400 | commands = [] |
| 337 | for (soc, ut_type), op_names in sorted(groups.items()): | 401 | for (soc, ut_type), op_names in sorted(groups.items()): |
| 338 | # 将算子列表合并为逗号分隔的字符串 | 402 | # 将算子列表合并为逗号分隔的字符串 |
| 339 | - merged_ops = ','.join(sorted(op_names)) | 403 | + merged_ops = ",".join(sorted(op_names)) |
| 340 | - cmd = make_merged_command(merged_ops, ut_type, soc, cann_3rd_lib_path, experimental) | 404 | + cmd = make_merged_command( |
| 405 | + merged_ops, ut_type, soc, cann_3rd_lib_path, experimental | ||
| 406 | + ) | ||
| 341 | commands.append(cmd) | 407 | commands.append(cmd) |
| 342 | 408 | ||
| 343 | # 去重保序（基于 tuple 去重，保留 list 类型） | 409 | # 去重保序（基于 tuple 去重，保留 list 类型） |
| @@ -362,24 +428,28 @@ def check_op_examples(op_name, is_experimental): | |||
| 362 | dict: {'has_eager': bool, 'has_graph': bool} | 428 | dict: {'has_eager': bool, 'has_graph': bool} |
| 363 | """ | 429 | """ |
| 364 | # 算子可能的路径前缀 | 430 | # 算子可能的路径前缀 |
| 365 | - prefixes = ['math/', 'conversion/', 'random/'] | 431 | + prefixes = ["math/", "conversion/", "random/"] |
| 366 | if is_experimental: | 432 | if is_experimental: |
| 367 | - prefixes = ['experimental/math/', 'experimental/conversion/', 'experimental/random/'] | 433 | + prefixes = [ |
| 434 | + "experimental/math/", | ||
| 435 | + "experimental/conversion/", | ||
| 436 | + "experimental/random/", | ||
| 437 | + ] | ||
| 368 | 438 | ||
| 369 | - result = {'has_eager': False, 'has_graph': False} | 439 | + result = {"has_eager": False, "has_graph": False} |
| 370 | 440 | ||
| 371 | for prefix in prefixes: | 441 | for prefix in prefixes: |
| 372 | - examples_dir = os.path.join(prefix, op_name, 'examples') | 442 | + examples_dir = os.path.join(prefix, op_name, "examples") |
| 373 | if not os.path.isdir(examples_dir): | 443 | if not os.path.isdir(examples_dir): |
| 374 | continue | 444 | continue |
| 375 | 445 | ||
| 376 | # 检查目录下的文件 | 446 | # 检查目录下的文件 |
| 377 | try: | 447 | try: |
| 378 | for filename in os.listdir(examples_dir): | 448 | for filename in os.listdir(examples_dir): |
| 379 | - if filename.startswith('test_aclnn') and filename.endswith('.cpp'): | 449 | + if filename.startswith("test_aclnn") and filename.endswith(".cpp"): |
| 380 | - result['has_eager'] = True | 450 | + result["has_eager"] = True |
| 381 | - if filename.startswith('test_geir') and filename.endswith('.cpp'): | 451 | + if filename.startswith("test_geir") and filename.endswith(".cpp"): |
| 382 | - result['has_graph'] = True | 452 | + result["has_graph"] = True |
| 383 | except OSError: | 453 | except OSError: |
| 384 | continue | 454 | continue |
| 385 | 455 | ||
| @@ -403,8 +473,8 @@ def build_example_commands(filepath, experimental=False): | |||
| 403 | """ | 473 | """ |
| 404 | parsed = parse_changed_files(filepath) | 474 | parsed = parse_changed_files(filepath) |
| 405 | 475 | ||
| 406 | - exp_ops = parsed['exp_ops'] | 476 | + exp_ops = parsed["exp_ops"] |
| 407 | - normal_ops = parsed['normal_ops'] | 477 | + normal_ops = parsed["normal_ops"] |
| 408 | 478 | ||
| 409 | # 根据参数确定跑哪类算子 | 479 | # 根据参数确定跑哪类算子 |
| 410 | ops = exp_ops if experimental else normal_ops | 480 | ops = exp_ops if experimental else normal_ops |
| @@ -424,8 +494,8 @@ def build_example_commands(filepath, experimental=False): | |||
| 424 | example_check = check_op_examples(op, experimental) | 494 | example_check = check_op_examples(op, experimental) |
| 425 | 495 | ||
| 426 | # 根据存在的测试文件类型生成对应命令 | 496 | # 根据存在的测试文件类型生成对应命令 |
| 427 | - if example_check['has_eager']: | 497 | + if example_check["has_eager"]: |
| 428 | - commands.append(make_run_example_command(op, 'eager', experimental)) | 498 | + commands.append(make_run_example_command(op, "eager", experimental)) |
| 429 | 499 | ||
| 430 | return commands | 500 | return commands |
| 431 | 501 | ||
| @@ -444,13 +514,13 @@ def make_package_command(merged_ops, cann_3rd_lib_path=None, is_experimental=Fal | |||
| 444 | cann_3rd_lib_path: 可选的第三方库路径 | 514 | cann_3rd_lib_path: 可选的第三方库路径 |
| 445 | is_experimental: 是否为 experimental 算子 | 515 | is_experimental: 是否为 experimental 算子 |
| 446 | """ | 516 | """ |
| 447 | - cmd_parts = ['bash', 'build.sh', '--pkg', '-j16'] | 517 | + cmd_parts = ["bash", "build.sh", "--pkg", "-j16"] |
| 448 | 518 | ||
| 449 | if is_experimental: | 519 | if is_experimental: |
| 450 | - cmd_parts.append('--experimental') | 520 | + cmd_parts.append("--experimental") |
| 451 | - cmd_parts.append('--vendor_name=experimental') | 521 | + cmd_parts.append("--vendor_name=experimental") |
| 452 | else: | 522 | else: |
| 453 | - cmd_parts.append('--vendor_name=custom') | 523 | + cmd_parts.append("--vendor_name=custom") |
| 454 | 524 | ||
| 455 | cmd_parts.append(f"--ops={merged_ops}") | 525 | cmd_parts.append(f"--ops={merged_ops}") |
| 456 | 526 | ||
| @@ -466,6 +536,7 @@ def build_package_commands(filepath, experimental=False, cann_3rd_lib_path=None) | |||
| 466 | 出包模式特点： | 536 | 出包模式特点： |
| 467 | - 不区分 UT 类型 和 SOC | 537 | - 不区分 UT 类型 和 SOC |
| 468 | - 所有算子合并到一条命令 | 538 | - 所有算子合并到一条命令 |
| 539 | + - 仅保留支持默认SOC且存在 op_kernel 目录的算子（出包需要 kernel 二进制） | ||
| 469 | 540 | ||
| 470 | Args: | 541 | Args: |
| 471 | filepath: 变更文件路径 | 542 | filepath: 变更文件路径 |
| @@ -477,8 +548,8 @@ def build_package_commands(filepath, experimental=False, cann_3rd_lib_path=None) | |||
| 477 | """ | 548 | """ |
| 478 | parsed = parse_changed_files(filepath) | 549 | parsed = parse_changed_files(filepath) |
| 479 | 550 | ||
| 480 | - exp_ops = parsed['exp_ops'] | 551 | + exp_ops = parsed["exp_ops"] |
| 481 | - normal_ops = parsed['normal_ops'] | 552 | + normal_ops = parsed["normal_ops"] |
| 482 | 553 | ||
| 483 | # 根据参数确定跑哪类算子 | 554 | # 根据参数确定跑哪类算子 |
| 484 | ops = exp_ops if experimental else normal_ops | 555 | ops = exp_ops if experimental else normal_ops |
| @@ -487,12 +558,22 @@ def build_package_commands(filepath, experimental=False, cann_3rd_lib_path=None) | |||
| 487 | # 过滤不支持 DEFAULT_FILTER_SOC 的算子 | 558 | # 过滤不支持 DEFAULT_FILTER_SOC 的算子 |
| 488 | ops = filter_ops_by_soc_support(ops, DEFAULT_FILTER_SOC, experimental) | 559 | ops = filter_ops_by_soc_support(ops, DEFAULT_FILTER_SOC, experimental) |
| 489 | 560 | ||
| 561 | + # 过滤没有 op_kernel 目录的算子（出包需要 kernel 二进制） | ||
| 562 | + ops_with_kernel = filter_ops_by_op_kernel(ops, experimental) | ||
| 563 | + dropped_no_kernel = ops - ops_with_kernel | ||
| 564 | + if dropped_no_kernel: | ||
| 565 | + print( | ||
| 566 | + f"警告: 以下算子无 op_kernel 目录，出包命令中丢弃: {','.join(sorted(dropped_no_kernel))}", | ||
| 567 | + flush=True, | ||
| 568 | + ) | ||
| 569 | + ops = ops_with_kernel | ||
| 570 | + | ||
| 490 | # 无变更或过滤后为空，用默认命令 | 571 | # 无变更或过滤后为空，用默认命令 |
| 491 | if not ops: | 572 | if not ops: |
| 492 | return [make_package_command(default_op, cann_3rd_lib_path, experimental)] | 573 | return [make_package_command(default_op, cann_3rd_lib_path, experimental)] |
| 493 | 574 | ||
| 494 | # 所有算子合并到一条命令 | 575 | # 所有算子合并到一条命令 |
| 495 | - merged_ops = ','.join(sorted(ops)) | 576 | + merged_ops = ",".join(sorted(ops)) |
| 496 | cmd = make_package_command(merged_ops, cann_3rd_lib_path, experimental) | 577 | cmd = make_package_command(merged_ops, cann_3rd_lib_path, experimental) |
| 497 | 578 | ||
| 498 | return [cmd] | 579 | return [cmd] |
| @@ -506,7 +587,7 @@ def print_commands(commands): | |||
| 506 | print(flush=True) | 587 | print(flush=True) |
| 507 | 588 | ||
| 508 | 589 | ||
| 509 | -def execute_commands(commands, mode='ut'): | 590 | +def execute_commands(commands, mode="ut"): |
| 510 | """执行命令列表并根据模式打印结果 | 591 | """执行命令列表并根据模式打印结果 |
| 511 | 592 | ||
| 512 | Args: | 593 | Args: |
| @@ -520,46 +601,72 @@ def execute_commands(commands, mode='ut'): | |||
| 520 | print(f"执行: {' '.join(cmd)}", flush=True) | 601 | print(f"执行: {' '.join(cmd)}", flush=True) |
| 521 | result = subprocess.run(cmd) | 602 | result = subprocess.run(cmd) |
| 522 | if result.returncode != 0: | 603 | if result.returncode != 0: |
| 523 | - print(f"run {mode} fail: {' '.join(cmd)} (返回码: {result.returncode})", flush=True) | 604 | + print( |
| 605 | + f"run {mode} fail: {' '.join(cmd)} (返回码: {result.returncode})", | ||
| 606 | + flush=True, | ||
| 607 | + ) | ||
| 524 | return 1 | 608 | return 1 |
| 525 | print(f"命令成功: {' '.join(cmd)}", flush=True) | 609 | print(f"命令成功: {' '.join(cmd)}", flush=True) |
| 526 | return 0 | 610 | return 0 |
| 527 | 611 | ||
| 528 | 612 | ||
| 529 | def main(): | 613 | def main(): |
| 530 | - parser = argparse.ArgumentParser(description='根据 CI 变更文件生成构建命令') | 614 | + parser = argparse.ArgumentParser(description="根据 CI 变更文件生成构建命令") |
| 531 | - parser.add_argument('-f', '--file', required=True, help='变更文件列表 必选参数') | 615 | + parser.add_argument("-f", "--file", required=True, help="变更文件列表 必选参数") |
| 532 | - parser.add_argument('--exec', action='store_true', help='直接执行生成的命令 可选参数') | 616 | + parser.add_argument( |
| 533 | - parser.add_argument('--experimental', choices=['TRUE', 'FALSE'], default='FALSE', | 617 | + "--exec", action="store_true", help="直接执行生成的命令 可选参数" |
| 534 | - help='可选参数 默认FALSE TRUE表示跑experimental目录下的用例 检测不到算子跑默认experimental目录下的acos算子 ' | 618 | + ) |
| 535 | - 'FALSE表示跑基本算子的用例 检查不到算子跑math目录下的is_finite 是否指定 experimental 算子构建 (TRUE/FALSE)') | 619 | + parser.add_argument( |
| 536 | - parser.add_argument('--pkg', choices=['TRUE', 'FALSE'], default='FALSE', | 620 | + "--experimental", |
| 537 | - help='是否生成出包命令 (TRUE/FALSE) 可选参数 默认是FALSE TRUE表示对涉及变更的算子打自定义算子包 ' | 621 | + choices=["TRUE", "FALSE"], |
| 538 | - 'FALSE 不打包 只跑UT') | 622 | + default="FALSE", |
| 539 | - parser.add_argument('--run_example', choices=['TRUE', 'FALSE'], default='FALSE', | 623 | + help="可选参数 默认FALSE TRUE表示跑experimental目录下的用例 检测不到算子跑默认experimental目录下的acos算子 " |
| 540 | - help='是否生成 run_example 命令 (TRUE/FALSE) 可选参数 默认FALSE TRUE表示每个算子运行示例') | 624 | + "FALSE表示跑基本算子的用例 检查不到算子跑math目录下的is_finite 是否指定 experimental 算子构建 (TRUE/FALSE)", |
| 541 | - parser.add_argument('--cann_3rd_lib_path', help='可选参数 CANN third party lib path') | 625 | + ) |
| 542 | - parser.add_argument('--list_ops', action='store_true', | 626 | + parser.add_argument( |
| 543 | - help='仅输出变更涉及的算子名列表(逗号分隔)，不生成命令') | 627 | + "--pkg", |
| 628 | + choices=["TRUE", "FALSE"], | ||
| 629 | + default="FALSE", | ||
| 630 | + help="是否生成出包命令 (TRUE/FALSE) 可选参数 默认是FALSE TRUE表示对涉及变更的算子打自定义算子包 " | ||
| 631 | + "FALSE 不打包 只跑UT", | ||
| 632 | + ) | ||
| 633 | + parser.add_argument( | ||
| 634 | + "--run_example", | ||
| 635 | + choices=["TRUE", "FALSE"], | ||
| 636 | + default="FALSE", | ||
| 637 | + help="是否生成 run_example 命令 (TRUE/FALSE) 可选参数 默认FALSE TRUE表示每个算子运行示例", | ||
| 638 | + ) | ||
| 639 | + parser.add_argument( | ||
| 640 | + "--cann_3rd_lib_path", help="可选参数 CANN third party lib path" | ||
| 641 | + ) | ||
| 642 | + parser.add_argument( | ||
| 643 | + "--list_ops", | ||
| 644 | + action="store_true", | ||
| 645 | + help="仅输出变更涉及的算子名列表(逗号分隔)，不生成命令", | ||
| 646 | + ) | ||
| 544 | args = parser.parse_args() | 647 | args = parser.parse_args() |
| 545 | 648 | ||
| 546 | if args.list_ops: | 649 | if args.list_ops: |
| 547 | parsed = parse_changed_files(args.file) | 650 | parsed = parse_changed_files(args.file) |
| 548 | - ops = parsed['exp_ops'] if args.experimental == 'TRUE' else parsed['normal_ops'] | 651 | + ops = parsed["exp_ops"] if args.experimental == "TRUE" else parsed["normal_ops"] |
| 549 | if ops: | 652 | if ops: |
| 550 | - print(','.join(sorted(ops))) | 653 | + print(",".join(sorted(ops))) |
| 551 | return | 654 | return |
| 552 | 655 | ||
| 553 | # 根据模式选择不同的命令生成函数 | 656 | # 根据模式选择不同的命令生成函数 |
| 554 | - if args.run_example == 'TRUE': | 657 | + if args.run_example == "TRUE": |
| 555 | - commands = build_example_commands(args.file, args.experimental == 'TRUE') | 658 | + commands = build_example_commands(args.file, args.experimental == "TRUE") |
| 556 | - mode = 'example' | 659 | + mode = "example" |
| 557 | - elif args.pkg == 'TRUE': | 660 | + elif args.pkg == "TRUE": |
| 558 | - commands = build_package_commands(args.file, args.experimental == 'TRUE', args.cann_3rd_lib_path) | 661 | + commands = build_package_commands( |
| 559 | - mode = 'pkg' | 662 | + args.file, args.experimental == "TRUE", args.cann_3rd_lib_path |
| 663 | + ) | ||
| 664 | + mode = "pkg" | ||
| 560 | else: | 665 | else: |
| 561 | - commands = build_ut_commands(args.file, args.experimental == 'TRUE', args.cann_3rd_lib_path) | 666 | + commands = build_ut_commands( |
| 562 | - mode = 'ut' | 667 | + args.file, args.experimental == "TRUE", args.cann_3rd_lib_path |
| 668 | + ) | ||
| 669 | + mode = "ut" | ||
| 563 | 670 | ||
| 564 | # 打印生成的命令 | 671 | # 打印生成的命令 |
| 565 | print_commands(commands) | 672 | print_commands(commands) |
| @@ -569,5 +676,5 @@ def main(): | |||
| 569 | sys.exit(execute_commands(commands, mode)) | 676 | sys.exit(execute_commands(commands, mode)) |
| 570 | 677 | ||
| 571 | 678 | ||
| 572 | -if __name__ == '__main__': | 679 | +if __name__ == "__main__": |
| 573 | - main() | 680 | + main() |