fix(build): 跳过仅有信息库无 op_kernel 的算子的 kernel 编译-ops-math-AtomGit

已合并

fix(build): 跳过仅有信息库无 op_kernel 的算子的 kernel 编译 #4900

songkai111创建于 16 天前从songkai111/ops-math_6421： fix-skip-kernel-for-defonly-op 合入到 cann/ops-math： master

fix(build): 跳过仅有信息库无 op_kernel 的算子的 kernel 编译 #4900

已合并

songkai111创建于 16 天前

从songkai111/ops-math_6421 fix-skip-kernel-for-defonly-op合入到cann/ops-math master

共 3 个文件变更+198-47

Mcmake/gen_ops_info.cmake+7-0

	@@ -562,6 +562,13 @@ function(gen_ops_info_and_python)
562	foreach(OP_DIR ${COMPILED_OP_DIRS})	562	foreach(OP_DIR ${COMPILED_OP_DIRS})
563	get_filename_component(op_name ${OP_DIR} NAME)	563	get_filename_component(op_name ${OP_DIR} NAME)
564		564
		565	+ # 仅定义了信息库、没有 kernel 实现的算子(如 bitcast),无需也无法编译 kernel 二进制,
		566	+ # 直接跳过 kernel 编译链路,与 get_op_type_and_validate 中 op_kernel 目录判断保持一致。
		567	+ if(NOT EXISTS "${OP_DIR}/op_kernel")
		568	+ message(STATUS "[INFO] On [${compute_unit}], [${op_name}] has no op_kernel directory, skip kernel binary compile.")
		569	+ continue()
		570	+ endif()
		571	+
565	set(op_type)	572	set(op_type)
566	get_op_type_from_op_name("${op_name}" op_type)	573	get_op_type_from_op_name("${op_name}" op_type)
567	if(NOT op_type)	574	if(NOT op_type)

Mscripts/kernel/binary_script/gen_opcinfo_from_opinfo.py+19-3

	@@ -13,6 +13,7 @@
13	"""	13	"""
14	gen_opcinfo_from_opinfo.py	14	gen_opcinfo_from_opinfo.py
15	"""	15	"""
		16	+
16	import sys	17	import sys
17	import os	18	import os
18	import json	19	import json
	@@ -74,6 +75,15 @@ def get_res_from_file(json_file):
74	op_file = convert_to_snake(op_type)	75	op_file = convert_to_snake(op_type)
75	else:	76	else:
76	op_file = op_file.get("value")	77	op_file = op_file.get("value")
		78	+ # opFile.value=Null 表示算子只有信息库定义、没有 kernel 实现(如 bitcast),
		79	+ # 无需进入 kernel 二进制编译清单,否则会生成 dynamic/Null.py 导致编译失败。
		80	+ if op_file is None or str(op_file).strip().lower() in ("null", "none"):
		81	+ print(
		82	+ "[INFO] op {} has no kernel implementation (opFile={}), skip kernel compile.".format(
		83	+ op_type, op_file
		84	+ )
		85	+ )
		86	+ continue
77	if op_interface is None:	87	if op_interface is None:
78	op_interface = convert_to_snake(op_type)	88	op_interface = convert_to_snake(op_type)
79	else:	89	else:
	@@ -84,7 +94,7 @@ def get_res_from_file(json_file):
84	return output_res	94	return output_res
85		95
86		96
87	-if __name__ == '__main__':	97	+if __name__ == "__main__":
88	args = sys.argv	98	args = sys.argv
89		99
90	in_file_path_list = list()	100	in_file_path_list = list()
	@@ -97,7 +107,7 @@ if __name__ == '__main__':
97	out_csv_file = arg	107	out_csv_file = arg
98		108
99	res = dict()	109	res = dict()
100	- wr_header = ['op_type', 'file_name', 'file_func']	110	+ wr_header = ["op_type", "file_name", "file_func"]
101	wr_data = list()	111	wr_data = list()
102	for ops_json_file in in_file_path_list:	112	for ops_json_file in in_file_path_list:
103	_res = get_res_from_file(ops_json_file)	113	_res = get_res_from_file(ops_json_file)
	@@ -106,7 +116,13 @@ if __name__ == '__main__':
106	res[_op_type] = _op_value	116	res[_op_type] = _op_value
107	file_name = _op_value[0]	117	file_name = _op_value[0]
108	func_name = _op_value[1]	118	func_name = _op_value[1]
109	- wr_data.append({wr_header[0]: _op_type, wr_header[1]: file_name, wr_header[2]: func_name})	119	+ wr_data.append(
		120	+ {
		121	+ wr_header[0]: _op_type,
		122	+ wr_header[1]: file_name,
		123	+ wr_header[2]: func_name,
		124	+ }
		125	+ )
110	continue	126	continue
111	global_op_value = res.get(_op_type)	127	global_op_value = res.get(_op_type)
112	if global_op_value != _op_value:	128	if global_op_value != _op_value:

Mscripts/util/parse_ini_to_json.py+172-44

	@@ -12,7 +12,7 @@
12	# Copyright 2020-2021 Huawei Technologies Co., Ltd.	12	# Copyright 2020-2021 Huawei Technologies Co., Ltd.
13	# Licensed under the Apache License, Version 2.0 (the "License");	13	# Licensed under the Apache License, Version 2.0 (the "License");
14	# You may obtain a copy of the License at	14	# You may obtain a copy of the License at
15	-#	15	+#
16	# http://www.apache.org/licenses/LICENSE-2.0	16	# http://www.apache.org/licenses/LICENSE-2.0
17	# ----------------------------------------------------------------------------	17	# ----------------------------------------------------------------------------
18		18
	@@ -26,25 +26,120 @@ import stat
26	import sys	26	import sys
27		27
28		28
29	-ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt",	29	+ATTR_TYPE_LIST = [
30	- "type", "listType", "tensor", "listTensor"]	30	+ "int",
		31	+ "float",
		32	+ "bool",
		33	+ "str",
		34	+ "listInt",
		35	+ "listFloat",
		36	+ "listBool",
		37	+ "listStr",
		38	+ "listListInt",
		39	+ "type",
		40	+ "listType",
		41	+ "tensor",
		42	+ "listTensor",
		43	+]
31	ATTR_PARAMTYPE_LIST = ["optional", "required"]	44	ATTR_PARAMTYPE_LIST = ["optional", "required"]
32	-BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp",	45	+BOOL_FLAG_KEY = [
33	- "needCheckSupport", "enableVectorCore"]	46	+ "dynamicFormat",
		47	+ "dynamicShapeSupport",
		48	+ "dynamicRankSupport",
		49	+ "precision_reduce",
		50	+ "heavyOp",
		51	+ "needCheckSupport",
		52	+ "enableVectorCore",
		53	+]
34	BOOL_LIST = ["true", "false"]	54	BOOL_LIST = ["true", "false"]
35	-DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool",	55	+DTYPE_LIST = [
36	- "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex32", "complex64",	56	+ "float16",
37	- "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref",	57	+ "float",
38	- "int4", "bfloat16", "uint1", "hifloat8", "float8_e4m3fn", "float8_e5m2", "float8_e8m0", "float4_e2m1",	58	+ "float32",
39	- "float4_e1m2", "int2"]	59	+ "int8",
40	-FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV",	60	+ "int16",
41	- "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04",	61	+ "int32",
42	- "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT",	62	+ "uint8",
43	- "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE",	63	+ "uint16",
44	- "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ",	64	+ "uint32",
45	- "FRACTAL_NZ", "FRACTAL_NZ_C0_2", "FRACTAL_NZ_C0_4", "FRACTAL_NZ_C0_16", "FRACTAL_NZ_C0_32", "NCDHW",	65	+ "bool",
46	- "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM",	66	+ "int64",
47	- "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"]	67	+ "uint64",
		68	+ "qint8",
		69	+ "qint16",
		70	+ "qint32",
		71	+ "quint8",
		72	+ "quint16",
		73	+ "double",
		74	+ "complex32",
		75	+ "complex64",
		76	+ "complex128",
		77	+ "string",
		78	+ "resource",
		79	+ "dual",
		80	+ "dual_sub_int8",
		81	+ "dual_sub_uint8",
		82	+ "string_ref",
		83	+ "int4",
		84	+ "bfloat16",
		85	+ "uint1",
		86	+ "hifloat8",
		87	+ "float8_e4m3fn",
		88	+ "float8_e5m2",
		89	+ "float8_e8m0",
		90	+ "float4_e2m1",
		91	+ "float4_e1m2",
		92	+ "int2",
		93	+]
		94	+FORMAT_LIST = [
		95	+ "NCHW",
		96	+ "NHWC",
		97	+ "ND",
		98	+ "NC1HWC0",
		99	+ "FRACTAL_Z",
		100	+ "NC1C0HWPAD",
		101	+ "NHWC1C0",
		102	+ "FSR_NCHW",
		103	+ "FRACTAL_DECONV",
		104	+ "C1HWNC0",
		105	+ "FRACTAL_DECONV_TRANSPOSE",
		106	+ "FRACTAL_DECONV_SP_STRIDE_TRANS",
		107	+ "NC1HWC0_C04",
		108	+ "FRACTAL_Z_C04",
		109	+ "CHWN",
		110	+ "FRACTAL_DECONV_SP_STRIDE8_TRANS",
		111	+ "HWCN",
		112	+ "NC1KHKWHWC0",
		113	+ "BN_WEIGHT",
		114	+ "FILTER_HWCK",
		115	+ "HASHTABLE_LOOKUP_LOOKUPS",
		116	+ "HASHTABLE_LOOKUP_KEYS",
		117	+ "HASHTABLE_LOOKUP_VALUE",
		118	+ "HASHTABLE_LOOKUP_OUTPUT",
		119	+ "HASHTABLE_LOOKUP_HITS",
		120	+ "C1HWNCoC0",
		121	+ "MD",
		122	+ "NDHWC",
		123	+ "FRACTAL_ZZ",
		124	+ "FRACTAL_NZ",
		125	+ "FRACTAL_NZ_C0_2",
		126	+ "FRACTAL_NZ_C0_4",
		127	+ "FRACTAL_NZ_C0_8",
		128	+ "FRACTAL_NZ_C0_16",
		129	+ "FRACTAL_NZ_C0_32",
		130	+ "NCDHW",
		131	+ "DHWCN",
		132	+ "NDC1HWC0",
		133	+ "FRACTAL_Z_3D",
		134	+ "CN",
		135	+ "NC",
		136	+ "DHWNC",
		137	+ "FRACTAL_Z_3D_TRANSPOSE",
		138	+ "FRACTAL_ZN_LSTM",
		139	+ "FRACTAL_ZN_RNN",
		140	+ "FRACTAL_Z_G",
		141	+ "NULL",
		142	+]
48		143
49		144
50	def parse_ini_files(ini_files):	145	def parse_ini_files(ini_files):
	@@ -69,8 +164,11 @@ def check_file_size(input_file):
69	except OSError as os_error:	164	except OSError as os_error:
70	print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error)))	165	print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error)))
71	raise OSError from os_error	166	raise OSError from os_error
72	- if file_size > 1010241024:	167	+ if file_size > 10 * 1024 * 1024:
73	- print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file)	168	+ print(
		169	+ "[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait."
		170	+ % input_file
		171	+ )
74		172
75		173
76	def parse_ini_to_obj(ini_file, tbe_ops_info):	174	def parse_ini_to_obj(ini_file, tbe_ops_info):
	@@ -98,16 +196,17 @@ def parse_ini_to_obj(ini_file, tbe_ops_info):
98	tbe_ops_info[op_name] = op_dict	196	tbe_ops_info[op_name] = op_dict
99	find_op_type = True	197	find_op_type = True
100	elif "=" in line:	198	elif "=" in line:
101	- key1 = line[:line.index("=")]	199	+ key1 = line[: line.index("=")]
102	- key2 = line[line.index("=")+1:]	200	+ key2 = line[line.index("=") + 1 :]
103	key1_0, key1_1 = key1.split(".")	201	key1_0, key1_1 = key1.split(".")
104	if key1_0 == "kernelSrc":	202	if key1_0 == "kernelSrc":
105	continue	203	continue
106	if key1_0 not in op_dict:	204	if key1_0 not in op_dict:
107	op_dict[key1_0] = {}	205	op_dict[key1_0] = {}
108	if key1_1 in op_dict.get(key1_0):	206	if key1_1 in op_dict.get(key1_0):
109	- raise RuntimeError("Op:" + op_name + " " + key1_0 + " " +	207	+ raise RuntimeError(
110	- key1_1 + " is repeated!")	208	+ "Op:" + op_name + " " + key1_0 + " " + key1_1 + " is repeated!"
		209	+ )
111	dic_key = op_dict.get(key1_0)	210	dic_key = op_dict.get(key1_0)
112	dic_key[key1_1] = key2	211	dic_key[key1_1] = key2
113	else:	212	else:
	@@ -179,7 +278,10 @@ def check_attr(op_dict, is_valid):
179	is_valid = check_attr_dict(attr_dict, is_valid, attr)	278	is_valid = check_attr_dict(attr_dict, is_valid, attr)
180	else:	279	else:
181	is_valid = False	280	is_valid = False
182	- print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str))	281	+ print(
		282	+ "%s is required in .ini file, when attr.list is %s!"
		283	+ % (attr, attr_list_str)
		284	+ )
183	return is_valid	285	return is_valid
184		286
185		287
	@@ -228,7 +330,11 @@ def check_type_format(op_info, is_valid, op_info_key):
228	if op_info_dtype_num > 0 and op_info_format_num > 0:	330	if op_info_dtype_num > 0 and op_info_format_num > 0:
229	if op_info_dtype_num != op_info_format_num:	331	if op_info_dtype_num != op_info_format_num:
230	is_valid = False	332	is_valid = False
231	- print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key))	333	+ print(
		334	+ "The number of {0}.dtype not match the number of {0}.format.".format(
		335	+ op_info_key
		336	+ )
		337	+ )
232	return is_valid	338	return is_valid
233		339
234		340
	@@ -254,14 +360,25 @@ def check_op_info(tbe_ops):
254	if required_op_input_info_key not in op_input_info:	360	if required_op_input_info_key not in op_input_info:
255	missing_keys.append(required_op_input_info_key)	361	missing_keys.append(required_op_input_info_key)
256	if len(missing_keys) > 0:	362	if len(missing_keys) > 0:
257	- print("op: " + op_key + " " + op_info_key + " missing: " +	363	+ print(
258	- ",".join(missing_keys))	364	+ "op: "
		365	+ + op_key
		366	+ + " "
		367	+ + op_info_key
		368	+ + " missing: "
		369	+ + ",".join(missing_keys)
		370	+ )
259	is_valid = False	371	is_valid = False
260	else:	372	else:
261	- if not op_input_info["paramType"] in param_type_valid_value:	373	+ if op_input_info["paramType"] not in param_type_valid_value:
262	- print("op: " + op_key + " " + op_info_key + \	374	+ print(
263	- " paramType not valid, valid key:[dynamic, "	375	+ "op: "
264	- "optional, required]")	376	+ + op_key
		377	+ + " "
		378	+ + op_info_key
		379	+ + " paramType not valid, valid key:[dynamic, "
		380	+ "optional, required]"
		381	+ )
265	is_valid = False	382	is_valid = False
266	is_valid = check_type_format(op_input_info, is_valid, op_info_key)	383	is_valid = check_type_format(op_input_info, is_valid, op_info_key)
267	if op_info_key.startswith("output"):	384	if op_info_key.startswith("output"):
	@@ -271,14 +388,25 @@ def check_op_info(tbe_ops):
271	if required_op_input_info_key not in op_input_info:	388	if required_op_input_info_key not in op_input_info:
272	missing_keys.append(required_op_input_info_key)	389	missing_keys.append(required_op_input_info_key)
273	if len(missing_keys) > 0:	390	if len(missing_keys) > 0:
274	- print("op: " + op_key + " " + op_info_key + " missing: " +	391	+ print(
275	- ",".join(missing_keys))	392	+ "op: "
		393	+ + op_key
		394	+ + " "
		395	+ + op_info_key
		396	+ + " missing: "
		397	+ + ",".join(missing_keys)
		398	+ )
276	is_valid = False	399	is_valid = False
277	else:	400	else:
278	- if not op_input_info["paramType"] in param_type_valid_value:	401	+ if op_input_info["paramType"] not in param_type_valid_value:
279	- print("op: " + op_key + " " + op_info_key +	402	+ print(
280	- " paramType not valid, valid key:[dynamic, "	403	+ "op: "
281	- "optional, required]")	404	+ + op_key
		405	+ + " "
		406	+ + op_info_key
		407	+ + " paramType not valid, valid key:[dynamic, "
		408	+ "optional, required]"
		409	+ )
282	is_valid = False	410	is_valid = False
283	is_valid = check_type_format(op_input_info, is_valid, op_info_key)	411	is_valid = check_type_format(op_input_info, is_valid, op_info_key)
284	is_valid = check_attr(op_dict, is_valid)	412	is_valid = check_attr(op_dict, is_valid)
	@@ -299,12 +427,12 @@ def write_json_file(tbe_ops_info, json_file_path):
299	json_file_real_path = os.path.realpath(json_file_path)	427	json_file_real_path = os.path.realpath(json_file_path)
300	wr_flag = os.O_WRONLY \| os.O_CREAT	428	wr_flag = os.O_WRONLY \| os.O_CREAT
301	wr_mode = stat.S_IWUSR \| stat.S_IRUSR	429	wr_mode = stat.S_IWUSR \| stat.S_IRUSR
302	- with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path:	430	+ with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), "w") as file_path:
303	# The owner have all rights£¬group only have read rights	431	# The owner have all rights£¬group only have read rights
304	- os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP	432	+ os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + stat.S_IRUSR)
305	- + stat.S_IRUSR)	433	+ json.dump(
306	- json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4,	434	+ tbe_ops_info, file_path, sort_keys=True, indent=4, separators=(",", ":")
307	- separators=(',', ':'))	435	+ )
308	print("Compile op info cfg successfully.")	436	print("Compile op info cfg successfully.")
309		437
310		438
	@@ -325,7 +453,7 @@ def parse_ini_to_json(ini_file_paths, outfile_path):
325	return True	453	return True
326		454
327		455
328	-if __name__ == '__main__':	456	+if __name__ == "__main__":
329	args = sys.argv	457	args = sys.argv
330		458
331	OUTPUT_FILE_PATH = "tbe_ops_info.json"	459	OUTPUT_FILE_PATH = "tbe_ops_info.json"