add op compile time group scripts-ops-math-AtomGit

已合并

add op compile time group scripts #919

liulinxiang创建于 1月26日从liulinxiang/ops-math： dev 合入到 cann/ops-math： master

add op compile time group scripts #919

已合并

liulinxiang创建于 1月26日

从liulinxiang/ops-math dev合入到cann/ops-math master

共 2 个文件变更+238-24

Mscripts/ci/ascend910_95/ops_math_operator_list.yaml+24-24

	@@ -13,16 +13,15 @@
13	operator_group_1:	13	operator_group_1:
14	- abs	14	- abs
15	- add	15	- add
16	- - add_n
17	- addcdiv	16	- addcdiv
18	- addcmul	17	- addcmul
		18	+ - add_n
19	- addr	19	- addr
20	- adjacent_difference	20	- adjacent_difference
21	- arg_max_v2	21	- arg_max_v2
22	- arg_max_with_value	22	- arg_max_with_value
23	- arg_min	23	- arg_min
24	- arg_min_with_value	24	- arg_min_with_value
25	- - assign
26	- assign_add	25	- assign_add
27	- assign_sub	26	- assign_sub
28	- axpy	27	- axpy
	@@ -30,16 +29,16 @@ operator_group_1:
30	- bias_add	29	- bias_add
31	- bias_add_grad	30	- bias_add_grad
32	- bincount	31	- bincount
33	- - broadcast_to
34	- bitwise_and	32	- bitwise_and
35	- bitwise_or	33	- bitwise_or
		34	+ - cast
		35	+ - cdist
36	- ceil	36	- ceil
37	- cos	37	- cos
38	- - cast
39	- cumsum	38	- cumsum
		39	+ - diag_flat
		40	+ - diag_v2
40	- diag_part	41	- diag_part
41	- - tensor_move
42	-operator_group_2:
43	- div	42	- div
44	- div_no_nan	43	- div_no_nan
45	- dot	44	- dot
	@@ -50,6 +49,7 @@ operator_group_2:
50	- floor	49	- floor
51	- floor_div	50	- floor_div
52	- floor_mod	51	- floor_mod
		52	+ - fused_mul_add_n
53	- greater	53	- greater
54	- greater_equal	54	- greater_equal
55	- grouped_bias_add_grad	55	- grouped_bias_add_grad
	@@ -61,13 +61,13 @@ operator_group_2:
61	- is_nan	61	- is_nan
62	- is_neg_inf	62	- is_neg_inf
63	- is_pos_inf	63	- is_pos_inf
		64	+operator_group_2:
64	- lerp	65	- lerp
65	- less	66	- less
66	- less_equal	67	- less_equal
67	- lin_space	68	- lin_space
68	- - log
69	- log1p	69	- log1p
70	-operator_group_3:	70	+ - log
71	- logical_and	71	- logical_and
72	- logical_not	72	- logical_not
73	- logical_or	73	- logical_or
	@@ -87,17 +87,18 @@ operator_group_3:
87	- range	87	- range
88	- real_div	88	- real_div
89	- reciprocal	89	- reciprocal
90	- - reduce_max
91	- - reduce_std_v2
92	- - reduce_var
93	- - reduce_min
94	- reduce_all	90	- reduce_all
95	- reduce_any	91	- reduce_any
96	- reduce_log_sum_exp	92	- reduce_log_sum_exp
		93	+operator_group_3:
		94	+ - reduce_max
97	- reduce_mean	95	- reduce_mean
		96	+ - reduce_min
98	- reduce_prod	97	- reduce_prod
99	- - reduce_sum	98	+ - reduce_std_v2
100	operator_group_4:	99	operator_group_4:
		100	+ - reduce_sum
		101	+ - reduce_var
101	- rfft1_d	102	- rfft1_d
102	- round	103	- round
103	- rsqrt	104	- rsqrt
	@@ -112,21 +113,20 @@ operator_group_4:
112	- sqrt	113	- sqrt
113	- sqrt_grad	114	- sqrt_grad
114	- square	115	- square
115	- - square_sum_v1
116	- squared_difference	116	- squared_difference
		117	+ - square_sum_v1
117	- sub	118	- sub
118	- tanh	119	- tanh
119	- tanh_grad	120	- tanh_grad
120	- - transform_bias_rescale_qkv
121	- - as_strided
122	- tensor_equal	121	- tensor_equal
123	- - cdist	122	+ - top_k_v2
124	- - clip_by_value
125	- - fused_mul_add_n
126	- - space_to_depth
127	- - diag_flat
128	- - diag_v2
129	operator_group_5:	123	operator_group_5:
		124	+ - transform_bias_rescale_qkv
		125	+ - trunc
		126	+ - assign
		127	+ - as_strided
		128	+ - broadcast_to
		129	+ - clip_by_value
130	- clip_by_value_v2	130	- clip_by_value_v2
131	- concat	131	- concat
132	- concat_d	132	- concat_d
	@@ -145,16 +145,16 @@ operator_group_5:
145	- reflection_pad3d_grad	145	- reflection_pad3d_grad
146	- roll	146	- roll
147	- slice	147	- slice
		148	+ - space_to_depth
148	- split	149	- split
149	- split_v	150	- split_v
150	- strided_slice	151	- strided_slice
151	- strided_slice_assign_v2	152	- strided_slice_assign_v2
152	- strided_slice_grad	153	- strided_slice_grad
		154	+ - tensor_move
153	- transpose	155	- transpose
154	- tril	156	- tril
155	- triu	157	- triu
156	- - trunc
157	- - top_k_v2
158	- unpack	158	- unpack
159	- zeros_like	159	- zeros_like
160	- drop_out_do_mask	160	- drop_out_do_mask

Ascripts/ci/op_compile_time_grouping.sh+214-0

@@ -0,0 +1,214 @@
	1	+#!/bin/bash
	2	+# ----------------------------------------------------------------------------
	3	+# Copyright (c) 2025 Huawei Technologies Co., Ltd.
	4	+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
	5	+# CANN Open Software License Agreement Version 2.0 (the "License").
	6	+# Please refer to the License for details. You may not use this file except in compliance with the License.
	7	+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
	8	+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
	9	+# See LICENSE in the root of the software repository for the full text of the License.
	10	+# ----------------------------------------------------------------------------
	11	+
	12	+set -e
	13	+
	14	+SCRIPT_NAME="op_compile_time_grouping"
	15	+
	16	+LOG_FILE=""
	17	+CODE_PATH=""
	18	+
	19	+TMP_FILES=(
	20	+ result_math.csv
	21	+ result_conversion.csv
	22	+ result_random.csv
	23	+ result0.csv
	24	+ result1.csv
	25	+)
	26	+
	27	+# =========================================
	28	+# Help
	29	+# =========================================
	30	+show_help() {
	31	+cat << EOF
	32	+${SCRIPT_NAME}
	33	+
	34	+Description:
	35	+ Analyze compile logs, extract each operator's maximum compile time,
	36	+ and group operators into 5 groups based on total time proportion.
	37	+ Generates result.csv and op_compile_group.yaml.
	38	+
	39	+Usage:
	40	+ ${SCRIPT_NAME}.sh -l <log_file> -c <code_path>
	41	+
	42	+Options:
	43	+ -l, --log Compile log file (required)
	44	+ -c, --code Operator source root path (required)
	45	+ -h, --help Show this help message
	46	+
	47	+Example:
	48	+ ./${SCRIPT_NAME}.sh -l build.log -c /workspace/op_code
	49	+EOF
	50	+}
	51	+
	52	+# =========================================
	53	+# Argument parsing
	54	+# =========================================
	55	+parse_args() {
	56	+ while [[ $# -gt 0 ]]; do
	57	+ case "$1" in
	58	+ -l\|--log)
	59	+ LOG_FILE="$2"
	60	+ shift 2
	61	+ ;;
	62	+ -c\|--code)
	63	+ CODE_PATH="$2"
	64	+ shift 2
	65	+ ;;
	66	+ -h\|--help)
	67	+ show_help
	68	+ exit 0
	69	+ ;;
	70	+ *)
	71	+ echo "Unknown option: $1"
	72	+ show_help
	73	+ exit 1
	74	+ ;;
	75	+ esac
	76	+ done
	77	+
	78	+ if [[ -z "$LOG_FILE" \|\| -z "$CODE_PATH" ]]; then
	79	+ echo "ERROR: --log and --code are required"
	80	+ show_help
	81	+ exit 1
	82	+ fi
	83	+
	84	+ [[ -f "$LOG_FILE" ]] \|\| { echo "ERROR: log file not found: $LOG_FILE"; exit 1; }
	85	+ [[ -d "$CODE_PATH" ]] \|\| { echo "ERROR: code path not found: $CODE_PATH"; exit 1; }
	86	+}
	87	+
	88	+cleanup() {
	89	+ for f in "${TMP_FILES[@]}"; do
	90	+ [[ -f "$f" ]] && rm -f "$f"
	91	+ done
	92	+}
	93	+trap cleanup EXIT
	94	+
	95	+generate_yaml() {
	96	+ yaml_file="op_compile_group.yaml"
	97	+ echo "# Generated from result.csv" > "$yaml_file"
	98	+
	99	+ for g in {1..5}; do
	100	+ echo "operator_group_${g}:" >> "$yaml_file"
	101	+ awk -F',' -v grp="$g" '$3==grp {print " - "$1}' result.csv >> "$yaml_file"
	102	+ done
	103	+
	104	+ echo "Generate $yaml_file successfully"
	105	+}
	106	+
	107	+main() {
	108	+
	109	+math_path="$CODE_PATH/math"
	110	+conversion_path="$CODE_PATH/conversion"
	111	+random_path="$CODE_PATH/random"
	112	+
	113	+declare -A op_time_max
	114	+
	115	+while read -r op time; do
	116	+ prev=${op_time_max[$op]:-0}
	117	+ max=$(awk -v a="$prev" -v b="$time" 'BEGIN{print (a>=b)?a:b}')
	118	+ op_time_max["$op"]=$max
	119	+done < <(
	120	+ grep -- '--exe_time=' "$LOG_FILE" \| \
	121	+ awk '
	122	+ {
	123	+ op=""
	124	+ time=""
	125	+
	126	+ for (i=1; i<=NF; i++) {
	127	+ if ($i ~ /--main_func=/) {
	128	+ split($i,a,"=")
	129	+ op=a[2]
	130	+ }
	131	+ else if ($i ~ /--exe_time=/) {
	132	+ split($i,b,"=")
	133	+ time=b[2]
	134	+ }
	135	+ }
	136	+
	137	+ if (op!="" && time!="")
	138	+ print op, time
	139	+ }'
	140	+)
	141	+
	142	+# safety check
	143	+if [[ ${#op_time_max[@]} -eq 0 ]]; then
	144	+ echo "ERROR: no operators parsed from log"
	145	+ exit 1
	146	+fi
	147	+
	148	+total_time=0
	149	+
	150	+for op in "${!op_time_max[@]}"; do
	151	+ val="${op_time_max[$op]}"
	152	+ base_op="${op%_apt}"
	153	+
	154	+ if [[ -d "$math_path/$base_op" ]]; then
	155	+ echo "$base_op,$val" >> result_math.csv
	156	+ elif [[ -d "$conversion_path/$base_op" ]]; then
	157	+ echo "$base_op,$val" >> result_conversion.csv
	158	+ elif [[ -d "$random_path/$base_op" ]]; then
	159	+ echo "$base_op,$val" >> result_random.csv
	160	+ else
	161	+ echo "ERROR: operator not found in any category: $base_op"
	162	+ exit 1
	163	+ fi
	164	+
	165	+ total_time=$(awk -v a="$total_time" -v b="$val" 'BEGIN{print a+b}')
	166	+done
	167	+
	168	+echo "TOTAL_TIME=$total_time"
	169	+
	170	+{
	171	+ [[ -f result_math.csv ]] && sort result_math.csv
	172	+ [[ -f result_conversion.csv ]] && sort result_conversion.csv
	173	+ [[ -f result_random.csv ]] && sort result_random.csv
	174	+} > result0.csv
	175	+
	176	+echo "op_name,exe_time_sum" > result1.csv
	177	+cat result0.csv >> result1.csv
	178	+
	179	+awk -F',' '
	180	+function assign_group(n, i,sum,threshold,group) {
	181	+ threshold = total / 5
	182	+ sum = 0
	183	+ group = 1
	184	+ for (i=2;i<=n;i++) {
	185	+ sum += time[i]
	186	+ if (sum > group * threshold && group < 5)
	187	+ group++
	188	+ gid[i]=group
	189	+ }
	190	+}
	191	+
	192	+NR==1 { header=$0; next }
	193	+
	194	+{
	195	+ op[NR]=$1
	196	+ time[NR]=$2
	197	+ total+=$2
	198	+}
	199	+
	200	+END{
	201	+ assign_group(NR)
	202	+ print header ",group"
	203	+ for(i=2;i<=NR;i++)
	204	+ print op[i] "," time[i] "," gid[i]
	205	+}
	206	+' result1.csv > result.csv
	207	+
	208	+echo "Generate result.csv successfully"
	209	+
	210	+generate_yaml
	211	+}
	212	+
	213	+parse_args "$@"
	214	+main