Star887
1788
代码
代码
Issues236
Pull Requests322
Actions
项目讨论
Wiki
分析
Star887
1788
已合并
matmul support ascend350 #5116
丛吉钰创建于 5月21日
matmul support ascend350 #5116
已合并
丛吉钰创建于 5月21日
从cong-jiyu/ops-nnmaster合入到cann/ops-nnmaster
共 35 个文件变更+23111-13
Mmatmul/batch_mat_mul_v3/op_host/batch_mat_mul_v3_def.cpp+1-0
@@ -152,6 +152,7 @@ public:
152 ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});152 ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
153 aicConfig.PrecisionReduceFlag(true);153 aicConfig.PrecisionReduceFlag(true);
154 this->AICore().AddConfig("ascend950", aicConfig);154 this->AICore().AddConfig("ascend950", aicConfig);
155+ this->AICore().AddConfig("ascend350", aicConfig);
155 this->AICore().AddConfig("ascend910_55", aicConfig);156 this->AICore().AddConfig("ascend910_55", aicConfig);
156 157 
157 OpAICoreConfig config_kirin = GetKirinCoreConfig();158 OpAICoreConfig config_kirin = GetKirinCoreConfig();
Amatmul/batch_mat_mul_v3/op_host/config/ascend350/batch_mat_mul_v3_binary.json+1206-0
@@ -0,0 +1,1206 @@
1+{
2+ "op_type": "BatchMatMulV3",
3+ "optional_input_mode": "gen_placeholder",
4+ "op_list": [
5+ {
6+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_FP16_FP16_FP16_FP16",
7+ "simplified_key": "diy,2/2/2/2/1/1/1/1",
8+ "inputs": [
9+ {
10+ "name": "x1",
11+ "index": 0,
12+ "dtype": "float16",
13+ "format": "ND",
14+ "paramType": "required",
15+ "shape": [
16+ -2
17+ ]
18+ },
19+ {
20+ "name": "x2",
21+ "index": 1,
22+ "dtype": "float16",
23+ "format": "ND",
24+ "paramType": "required",
25+ "shape": [
26+ -2
27+ ]
28+ },
29+ {
30+ "name": "bias",
31+ "index": 2,
32+ "dtype": "float16",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "offset_w",
41+ "index": 3,
42+ "dtype": "int8",
43+ "format": "ND",
44+ "paramType": "optional",
45+ "shape": [
46+ -2
47+ ]
48+ }
49+ ],
50+ "outputs": [
51+ {
52+ "name": "y",
53+ "index": 0,
54+ "dtype": "float16",
55+ "format": "ND",
56+ "paramType": "required",
57+ "shape": [
58+ -2
59+ ]
60+ }
61+ ],
62+ "attrs": [
63+ {
64+ "name": "adj_x1",
65+ "dtype": "bool",
66+ "value": false
67+ },
68+ {
69+ "name": "adj_x2",
70+ "dtype": "bool",
71+ "value": false
72+ },
73+ {
74+ "name": "offset_x",
75+ "dtype": "int",
76+ "value": 0
77+ },
78+ {
79+ "name": "enable_hf32",
80+ "dtype": "bool",
81+ "value": false
82+ }
83+ ]
84+ },
85+ {
86+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_FP16_FP16_FP32_FP16",
87+ "simplified_key": "diy,2/2/2/2/1/1/0/1",
88+ "inputs": [
89+ {
90+ "name": "x1",
91+ "index": 0,
92+ "dtype": "float16",
93+ "format": "ND",
94+ "paramType": "required",
95+ "shape": [
96+ -2
97+ ]
98+ },
99+ {
100+ "name": "x2",
101+ "index": 1,
102+ "dtype": "float16",
103+ "format": "ND",
104+ "paramType": "required",
105+ "shape": [
106+ -2
107+ ]
108+ },
109+ {
110+ "name": "bias",
111+ "index": 2,
112+ "dtype": "float32",
113+ "format": "ND",
114+ "paramType": "optional",
115+ "shape": [
116+ -2
117+ ]
118+ },
119+ {
120+ "name": "offset_w",
121+ "index": 3,
122+ "dtype": "int8",
123+ "format": "ND",
124+ "paramType": "optional",
125+ "shape": [
126+ -2
127+ ]
128+ }
129+ ],
130+ "outputs": [
131+ {
132+ "name": "y",
133+ "index": 0,
134+ "dtype": "float16",
135+ "format": "ND",
136+ "paramType": "required",
137+ "shape": [
138+ -2
139+ ]
140+ }
141+ ],
142+ "attrs": [
143+ {
144+ "name": "adj_x1",
145+ "dtype": "bool",
146+ "value": true
147+ },
148+ {
149+ "name": "adj_x2",
150+ "dtype": "bool",
151+ "value": true
152+ },
153+ {
154+ "name": "offset_x",
155+ "dtype": "int",
156+ "value": 0
157+ },
158+ {
159+ "name": "enable_hf32",
160+ "dtype": "bool",
161+ "value": false
162+ }
163+ ]
164+ },
165+ {
166+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_BF16_BF16_BF16_BF16",
167+ "simplified_key": "diy,2/2/2/2/27/27/27/27",
168+ "inputs": [
169+ {
170+ "name": "x1",
171+ "index": 0,
172+ "dtype": "bfloat16",
173+ "format": "ND",
174+ "paramType": "required",
175+ "shape": [
176+ -2
177+ ]
178+ },
179+ {
180+ "name": "x2",
181+ "index": 1,
182+ "dtype": "bfloat16",
183+ "format": "ND",
184+ "paramType": "required",
185+ "shape": [
186+ -2
187+ ]
188+ },
189+ {
190+ "name": "bias",
191+ "index": 2,
192+ "dtype": "bfloat16",
193+ "format": "ND",
194+ "paramType": "optional",
195+ "shape": [
196+ -2
197+ ]
198+ },
199+ {
200+ "name": "offset_w",
201+ "index": 3,
202+ "dtype": "int8",
203+ "format": "ND",
204+ "paramType": "optional",
205+ "shape": [
206+ -2
207+ ]
208+ }
209+ ],
210+ "outputs": [
211+ {
212+ "name": "y",
213+ "index": 0,
214+ "dtype": "bfloat16",
215+ "format": "ND",
216+ "paramType": "required",
217+ "shape": [
218+ -2
219+ ]
220+ }
221+ ],
222+ "attrs": [
223+ {
224+ "name": "adj_x1",
225+ "dtype": "bool",
226+ "value": false
227+ },
228+ {
229+ "name": "adj_x2",
230+ "dtype": "bool",
231+ "value": false
232+ },
233+ {
234+ "name": "offset_x",
235+ "dtype": "int",
236+ "value": 0
237+ },
238+ {
239+ "name": "enable_hf32",
240+ "dtype": "bool",
241+ "value": false
242+ }
243+ ]
244+ },
245+ {
246+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_FP32_FP32_FP32_FP32",
247+ "simplified_key": "diy,2/2/2/2/0/0/0/0",
248+ "inputs": [
249+ {
250+ "name": "x1",
251+ "index": 0,
252+ "dtype": "float32",
253+ "format": "ND",
254+ "paramType": "required",
255+ "shape": [
256+ -2
257+ ]
258+ },
259+ {
260+ "name": "x2",
261+ "index": 1,
262+ "dtype": "float32",
263+ "format": "ND",
264+ "paramType": "required",
265+ "shape": [
266+ -2
267+ ]
268+ },
269+ {
270+ "name": "bias",
271+ "index": 2,
272+ "dtype": "float32",
273+ "format": "ND",
274+ "paramType": "optional",
275+ "shape": [
276+ -2
277+ ]
278+ },
279+ {
280+ "name": "offset_w",
281+ "index": 3,
282+ "dtype": "int8",
283+ "format": "ND",
284+ "paramType": "optional",
285+ "shape": [
286+ -2
287+ ]
288+ }
289+ ],
290+ "outputs": [
291+ {
292+ "name": "y",
293+ "index": 0,
294+ "dtype": "float32",
295+ "format": "ND",
296+ "paramType": "required",
297+ "shape": [
298+ -2
299+ ]
300+ }
301+ ],
302+ "attrs": [
303+ {
304+ "name": "adj_x1",
305+ "dtype": "bool",
306+ "value": false
307+ },
308+ {
309+ "name": "adj_x2",
310+ "dtype": "bool",
311+ "value": false
312+ },
313+ {
314+ "name": "offset_x",
315+ "dtype": "int",
316+ "value": 0
317+ },
318+ {
319+ "name": "enable_hf32",
320+ "dtype": "bool",
321+ "value": false
322+ }
323+ ]
324+ },
325+ {
326+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_BF16_BF16_FP32_BF16",
327+ "simplified_key": "diy,2/2/2/2/27/27/0/27",
328+ "inputs": [
329+ {
330+ "name": "x1",
331+ "index": 0,
332+ "dtype": "bfloat16",
333+ "format": "ND",
334+ "paramType": "required",
335+ "shape": [
336+ -2
337+ ]
338+ },
339+ {
340+ "name": "x2",
341+ "index": 1,
342+ "dtype": "bfloat16",
343+ "format": "ND",
344+ "paramType": "required",
345+ "shape": [
346+ -2
347+ ]
348+ },
349+ {
350+ "name": "bias",
351+ "index": 2,
352+ "dtype": "float32",
353+ "format": "ND",
354+ "paramType": "optional",
355+ "shape": [
356+ -2
357+ ]
358+ },
359+ {
360+ "name": "offset_w",
361+ "index": 3,
362+ "dtype": "int8",
363+ "format": "ND",
364+ "paramType": "optional",
365+ "shape": [
366+ -2
367+ ]
368+ }
369+ ],
370+ "outputs": [
371+ {
372+ "name": "y",
373+ "index": 0,
374+ "dtype": "bfloat16",
375+ "format": "ND",
376+ "paramType": "required",
377+ "shape": [
378+ -2
379+ ]
380+ }
381+ ],
382+ "attrs": [
383+ {
384+ "name": "adj_x1",
385+ "dtype": "bool",
386+ "value": false
387+ },
388+ {
389+ "name": "adj_x2",
390+ "dtype": "bool",
391+ "value": false
392+ },
393+ {
394+ "name": "offset_x",
395+ "dtype": "int",
396+ "value": 0
397+ },
398+ {
399+ "name": "enable_hf32",
400+ "dtype": "bool",
401+ "value": false
402+ }
403+ ]
404+ },
405+ {
406+ "bin_filename": "BatchMatMulV3_ND_NZ_ND_ND_FP16_FP16_FP16_FP16",
407+ "simplified_key": "diy,2/29/2/2/1/1/1/1",
408+ "inputs": [
409+ {
410+ "name": "x1",
411+ "index": 0,
412+ "dtype": "float16",
413+ "format": "ND",
414+ "paramType": "required",
415+ "shape": [
416+ -2
417+ ]
418+ },
419+ {
420+ "name": "x2",
421+ "index": 1,
422+ "dtype": "float16",
423+ "format": "FRACTAL_NZ",
424+ "paramType": "required",
425+ "shape": [
426+ -2
427+ ]
428+ },
429+ {
430+ "name": "bias",
431+ "index": 2,
432+ "dtype": "float16",
433+ "format": "ND",
434+ "paramType": "optional",
435+ "shape": [
436+ -2
437+ ]
438+ },
439+ {
440+ "name": "offset_w",
441+ "index": 3,
442+ "dtype": "int8",
443+ "format": "ND",
444+ "paramType": "optional",
445+ "shape": [
446+ -2
447+ ]
448+ }
449+ ],
450+ "outputs": [
451+ {
452+ "name": "y",
453+ "index": 0,
454+ "dtype": "float16",
455+ "format": "ND",
456+ "paramType": "required",
457+ "shape": [
458+ -2
459+ ]
460+ }
461+ ],
462+ "attrs": [
463+ {
464+ "name": "adj_x1",
465+ "dtype": "bool",
466+ "value": false
467+ },
468+ {
469+ "name": "adj_x2",
470+ "dtype": "bool",
471+ "value": false
472+ },
473+ {
474+ "name": "offset_x",
475+ "dtype": "int",
476+ "value": 0
477+ },
478+ {
479+ "name": "enable_hf32",
480+ "dtype": "bool",
481+ "value": false
482+ }
483+ ]
484+ },
485+ {
486+ "bin_filename": "BatchMatMulV3_ND_NZ_ND_ND_BF16_BF16_BF16_BF16",
487+ "simplified_key": "diy,2/29/2/2/27/27/27/27",
488+ "inputs": [
489+ {
490+ "name": "x1",
491+ "index": 0,
492+ "dtype": "bfloat16",
493+ "format": "ND",
494+ "paramType": "required",
495+ "shape": [
496+ -2
497+ ]
498+ },
499+ {
500+ "name": "x2",
501+ "index": 1,
502+ "dtype": "bfloat16",
503+ "format": "FRACTAL_NZ",
504+ "paramType": "required",
505+ "shape": [
506+ -2
507+ ]
508+ },
509+ {
510+ "name": "bias",
511+ "index": 2,
512+ "dtype": "bfloat16",
513+ "format": "ND",
514+ "paramType": "optional",
515+ "shape": [
516+ -2
517+ ]
518+ },
519+ {
520+ "name": "offset_w",
521+ "index": 3,
522+ "dtype": "int8",
523+ "format": "ND",
524+ "paramType": "optional",
525+ "shape": [
526+ -2
527+ ]
528+ }
529+ ],
530+ "outputs": [
531+ {
532+ "name": "y",
533+ "index": 0,
534+ "dtype": "bfloat16",
535+ "format": "ND",
536+ "paramType": "required",
537+ "shape": [
538+ -2
539+ ]
540+ }
541+ ],
542+ "attrs": [
543+ {
544+ "name": "adj_x1",
545+ "dtype": "bool",
546+ "value": false
547+ },
548+ {
549+ "name": "adj_x2",
550+ "dtype": "bool",
551+ "value": false
552+ },
553+ {
554+ "name": "offset_x",
555+ "dtype": "int",
556+ "value": 0
557+ },
558+ {
559+ "name": "enable_hf32",
560+ "dtype": "bool",
561+ "value": false
562+ }
563+ ]
564+ },
565+ {
566+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_FP16_FP16_FP16_FP32",
567+ "simplified_key": "diy,2/2/2/2/1/1/1/0",
568+ "inputs": [
569+ {
570+ "name": "x1",
571+ "index": 0,
572+ "dtype": "float16",
573+ "format": "ND",
574+ "paramType": "required",
575+ "shape": [
576+ -2
577+ ]
578+ },
579+ {
580+ "name": "x2",
581+ "index": 1,
582+ "dtype": "float16",
583+ "format": "ND",
584+ "paramType": "required",
585+ "shape": [
586+ -2
587+ ]
588+ },
589+ {
590+ "name": "bias",
591+ "index": 2,
592+ "dtype": "float16",
593+ "format": "ND",
594+ "paramType": "optional",
595+ "shape": [
596+ -2
597+ ]
598+ },
599+ {
600+ "name": "offset_w",
601+ "index": 3,
602+ "dtype": "int8",
603+ "format": "ND",
604+ "paramType": "optional",
605+ "shape": [
606+ -2
607+ ]
608+ }
609+ ],
610+ "outputs": [
611+ {
612+ "name": "y",
613+ "index": 0,
614+ "dtype": "float32",
615+ "format": "ND",
616+ "paramType": "required",
617+ "shape": [
618+ -2
619+ ]
620+ }
621+ ],
622+ "attrs": [
623+ {
624+ "name": "adj_x1",
625+ "dtype": "bool",
626+ "value": false
627+ },
628+ {
629+ "name": "adj_x2",
630+ "dtype": "bool",
631+ "value": false
632+ },
633+ {
634+ "name": "offset_x",
635+ "dtype": "int",
636+ "value": 0
637+ },
638+ {
639+ "name": "enable_hf32",
640+ "dtype": "bool",
641+ "value": false
642+ }
643+ ]
644+ },
645+ {
646+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_FP16_FP16_FP32_FP32",
647+ "simplified_key": "diy,2/2/2/2/1/1/0/0",
648+ "inputs": [
649+ {
650+ "name": "x1",
651+ "index": 0,
652+ "dtype": "float16",
653+ "format": "ND",
654+ "paramType": "required",
655+ "shape": [
656+ -2
657+ ]
658+ },
659+ {
660+ "name": "x2",
661+ "index": 1,
662+ "dtype": "float16",
663+ "format": "ND",
664+ "paramType": "required",
665+ "shape": [
666+ -2
667+ ]
668+ },
669+ {
670+ "name": "bias",
671+ "index": 2,
672+ "dtype": "float32",
673+ "format": "ND",
674+ "paramType": "optional",
675+ "shape": [
676+ -2
677+ ]
678+ },
679+ {
680+ "name": "offset_w",
681+ "index": 3,
682+ "dtype": "int8",
683+ "format": "ND",
684+ "paramType": "optional",
685+ "shape": [
686+ -2
687+ ]
688+ }
689+ ],
690+ "outputs": [
691+ {
692+ "name": "y",
693+ "index": 0,
694+ "dtype": "float32",
695+ "format": "ND",
696+ "paramType": "required",
697+ "shape": [
698+ -2
699+ ]
700+ }
701+ ],
702+ "attrs": [
703+ {
704+ "name": "adj_x1",
705+ "dtype": "bool",
706+ "value": false
707+ },
708+ {
709+ "name": "adj_x2",
710+ "dtype": "bool",
711+ "value": false
712+ },
713+ {
714+ "name": "offset_x",
715+ "dtype": "int",
716+ "value": 0
717+ },
718+ {
719+ "name": "enable_hf32",
720+ "dtype": "bool",
721+ "value": false
722+ }
723+ ]
724+ },
725+ {
726+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_BF16_BF16_BF16_FP32",
727+ "simplified_key": "diy,2/2/2/2/27/27/27/0",
728+ "inputs": [
729+ {
730+ "name": "x1",
731+ "index": 0,
732+ "dtype": "bfloat16",
733+ "format": "ND",
734+ "paramType": "required",
735+ "shape": [
736+ -2
737+ ]
738+ },
739+ {
740+ "name": "x2",
741+ "index": 1,
742+ "dtype": "bfloat16",
743+ "format": "ND",
744+ "paramType": "required",
745+ "shape": [
746+ -2
747+ ]
748+ },
749+ {
750+ "name": "bias",
751+ "index": 2,
752+ "dtype": "bfloat16",
753+ "format": "ND",
754+ "paramType": "optional",
755+ "shape": [
756+ -2
757+ ]
758+ },
759+ {
760+ "name": "offset_w",
761+ "index": 3,
762+ "dtype": "int8",
763+ "format": "ND",
764+ "paramType": "optional",
765+ "shape": [
766+ -2
767+ ]
768+ }
769+ ],
770+ "outputs": [
771+ {
772+ "name": "y",
773+ "index": 0,
774+ "dtype": "float32",
775+ "format": "ND",
776+ "paramType": "required",
777+ "shape": [
778+ -2
779+ ]
780+ }
781+ ],
782+ "attrs": [
783+ {
784+ "name": "adj_x1",
785+ "dtype": "bool",
786+ "value": false
787+ },
788+ {
789+ "name": "adj_x2",
790+ "dtype": "bool",
791+ "value": false
792+ },
793+ {
794+ "name": "offset_x",
795+ "dtype": "int",
796+ "value": 0
797+ },
798+ {
799+ "name": "enable_hf32",
800+ "dtype": "bool",
801+ "value": false
802+ }
803+ ]
804+ },
805+ {
806+ "bin_filename": "BatchMatMulV3_ND_ND_ND_ND_BF16_BF16_FP32_FP32",
807+ "simplified_key": "diy,2/2/2/2/27/27/0/0",
808+ "inputs": [
809+ {
810+ "name": "x1",
811+ "index": 0,
812+ "dtype": "bfloat16",
813+ "format": "ND",
814+ "paramType": "required",
815+ "shape": [
816+ -2
817+ ]
818+ },
819+ {
820+ "name": "x2",
821+ "index": 1,
822+ "dtype": "bfloat16",
823+ "format": "ND",
824+ "paramType": "required",
825+ "shape": [
826+ -2
827+ ]
828+ },
829+ {
830+ "name": "bias",
831+ "index": 2,
832+ "dtype": "float32",
833+ "format": "ND",
834+ "paramType": "optional",
835+ "shape": [
836+ -2
837+ ]
838+ },
839+ {
840+ "name": "offset_w",
841+ "index": 3,
842+ "dtype": "int8",
843+ "format": "ND",
844+ "paramType": "optional",
845+ "shape": [
846+ -2
847+ ]
848+ }
849+ ],
850+ "outputs": [
851+ {
852+ "name": "y",
853+ "index": 0,
854+ "dtype": "float32",
855+ "format": "ND",
856+ "paramType": "required",
857+ "shape": [
858+ -2
859+ ]
860+ }
861+ ],
862+ "attrs": [
863+ {
864+ "name": "adj_x1",
865+ "dtype": "bool",
866+ "value": false
867+ },
868+ {
869+ "name": "adj_x2",
870+ "dtype": "bool",
871+ "value": false
872+ },
873+ {
874+ "name": "offset_x",
875+ "dtype": "int",
876+ "value": 0
877+ },
878+ {
879+ "name": "enable_hf32",
880+ "dtype": "bool",
881+ "value": false
882+ }
883+ ]
884+ },
885+ {
886+ "bin_filename": "BatchMatMulV3_ND_NZ_ND_ND_FP16_FP16_FP16_FP32",
887+ "simplified_key": "diy,2/29/2/2/1/1/1/0",
888+ "inputs": [
889+ {
890+ "name": "x1",
891+ "index": 0,
892+ "dtype": "float16",
893+ "format": "ND",
894+ "paramType": "required",
895+ "shape": [
896+ -2
897+ ]
898+ },
899+ {
900+ "name": "x2",
901+ "index": 1,
902+ "dtype": "float16",
903+ "format": "FRACTAL_NZ",
904+ "paramType": "required",
905+ "shape": [
906+ -2
907+ ]
908+ },
909+ {
910+ "name": "bias",
911+ "index": 2,
912+ "dtype": "float16",
913+ "format": "ND",
914+ "paramType": "optional",
915+ "shape": [
916+ -2
917+ ]
918+ },
919+ {
920+ "name": "offset_w",
921+ "index": 3,
922+ "dtype": "int8",
923+ "format": "ND",
924+ "paramType": "optional",
925+ "shape": [
926+ -2
927+ ]
928+ }
929+ ],
930+ "outputs": [
931+ {
932+ "name": "y",
933+ "index": 0,
934+ "dtype": "float32",
935+ "format": "ND",
936+ "paramType": "required",
937+ "shape": [
938+ -2
939+ ]
940+ }
941+ ],
942+ "attrs": [
943+ {
944+ "name": "adj_x1",
945+ "dtype": "bool",
946+ "value": false
947+ },
948+ {
949+ "name": "adj_x2",
950+ "dtype": "bool",
951+ "value": false
952+ },
953+ {
954+ "name": "offset_x",
955+ "dtype": "int",
956+ "value": 0
957+ },
958+ {
959+ "name": "enable_hf32",
960+ "dtype": "bool",
961+ "value": false
962+ }
963+ ]
964+ },
965+ {
966+ "bin_filename": "BatchMatMulV3_ND_NZ_ND_ND_FP16_FP16_FP32_FP32",
967+ "simplified_key": "diy,2/29/2/2/1/1/0/0",
968+ "inputs": [
969+ {
970+ "name": "x1",
971+ "index": 0,
972+ "dtype": "float16",
973+ "format": "ND",
974+ "paramType": "required",
975+ "shape": [
976+ -2
977+ ]
978+ },
979+ {
980+ "name": "x2",
981+ "index": 1,
982+ "dtype": "float16",
983+ "format": "FRACTAL_NZ",
984+ "paramType": "required",
985+ "shape": [
986+ -2
987+ ]
988+ },
989+ {
990+ "name": "bias",
991+ "index": 2,
992+ "dtype": "float32",
993+ "format": "ND",
994+ "paramType": "optional",
995+ "shape": [
996+ -2
997+ ]
998+ },
999+ {
1000+ "name": "offset_w",
1001+ "index": 3,
1002+ "dtype": "int8",
1003+ "format": "ND",
1004+ "paramType": "optional",
1005+ "shape": [
1006+ -2
1007+ ]
1008+ }
1009+ ],
1010+ "outputs": [
1011+ {
1012+ "name": "y",
1013+ "index": 0,
1014+ "dtype": "float32",
1015+ "format": "ND",
1016+ "paramType": "required",
1017+ "shape": [
1018+ -2
1019+ ]
1020+ }
1021+ ],
1022+ "attrs": [
1023+ {
1024+ "name": "adj_x1",
1025+ "dtype": "bool",
1026+ "value": false
1027+ },
1028+ {
1029+ "name": "adj_x2",
1030+ "dtype": "bool",
1031+ "value": false
1032+ },
1033+ {
1034+ "name": "offset_x",
1035+ "dtype": "int",
1036+ "value": 0
1037+ },
1038+ {
1039+ "name": "enable_hf32",
1040+ "dtype": "bool",
1041+ "value": false
1042+ }
1043+ ]
1044+ },
1045+ {
1046+ "bin_filename": "BatchMatMulV3_ND_NZ_ND_ND_BF16_BF16_BF16_FP32",
1047+ "simplified_key": "diy,2/29/2/2/27/27/27/0",
1048+ "inputs": [
1049+ {
1050+ "name": "x1",
1051+ "index": 0,
1052+ "dtype": "bfloat16",
1053+ "format": "ND",
1054+ "paramType": "required",
1055+ "shape": [
1056+ -2
1057+ ]
1058+ },
1059+ {
1060+ "name": "x2",
1061+ "index": 1,
1062+ "dtype": "bfloat16",
1063+ "format": "FRACTAL_NZ",
1064+ "paramType": "required",
1065+ "shape": [
1066+ -2
1067+ ]
1068+ },
1069+ {
1070+ "name": "bias",
1071+ "index": 2,
1072+ "dtype": "bfloat16",
1073+ "format": "ND",
1074+ "paramType": "optional",
1075+ "shape": [
1076+ -2
1077+ ]
1078+ },
1079+ {
1080+ "name": "offset_w",
1081+ "index": 3,
1082+ "dtype": "int8",
1083+ "format": "ND",
1084+ "paramType": "optional",
1085+ "shape": [
1086+ -2
1087+ ]
1088+ }
1089+ ],
1090+ "outputs": [
1091+ {
1092+ "name": "y",
1093+ "index": 0,
1094+ "dtype": "float32",
1095+ "format": "ND",
1096+ "paramType": "required",
1097+ "shape": [
1098+ -2
1099+ ]
1100+ }
1101+ ],
1102+ "attrs": [
1103+ {
1104+ "name": "adj_x1",
1105+ "dtype": "bool",
1106+ "value": false
1107+ },
1108+ {
1109+ "name": "adj_x2",
1110+ "dtype": "bool",
1111+ "value": false
1112+ },
1113+ {
1114+ "name": "offset_x",
1115+ "dtype": "int",
1116+ "value": 0
1117+ },
1118+ {
1119+ "name": "enable_hf32",
1120+ "dtype": "bool",
1121+ "value": false
1122+ }
1123+ ]
1124+ },
1125+ {
1126+ "bin_filename": "BatchMatMulV3_ND_NZ_ND_ND_BF16_BF16_FP32_FP32",
1127+ "simplified_key": "diy,2/29/2/2/27/27/0/0",
1128+ "inputs": [
1129+ {
1130+ "name": "x1",
1131+ "index": 0,
1132+ "dtype": "bfloat16",
1133+ "format": "ND",
1134+ "paramType": "required",
1135+ "shape": [
1136+ -2
1137+ ]
1138+ },
1139+ {
1140+ "name": "x2",
1141+ "index": 1,
1142+ "dtype": "bfloat16",
1143+ "format": "FRACTAL_NZ",
1144+ "paramType": "required",
1145+ "shape": [
1146+ -2
1147+ ]
1148+ },
1149+ {
1150+ "name": "bias",
1151+ "index": 2,
1152+ "dtype": "float32",
1153+ "format": "ND",
1154+ "paramType": "optional",
1155+ "shape": [
1156+ -2
1157+ ]
1158+ },
1159+ {
1160+ "name": "offset_w",
1161+ "index": 3,
1162+ "dtype": "int8",
1163+ "format": "ND",
1164+ "paramType": "optional",
1165+ "shape": [
1166+ -2
1167+ ]
1168+ }
1169+ ],
1170+ "outputs": [
1171+ {
1172+ "name": "y",
1173+ "index": 0,
1174+ "dtype": "float32",
1175+ "format": "ND",
1176+ "paramType": "required",
1177+ "shape": [
1178+ -2
1179+ ]
1180+ }
1181+ ],
1182+ "attrs": [
1183+ {
1184+ "name": "adj_x1",
1185+ "dtype": "bool",
1186+ "value": false
1187+ },
1188+ {
1189+ "name": "adj_x2",
1190+ "dtype": "bool",
1191+ "value": false
1192+ },
1193+ {
1194+ "name": "offset_x",
1195+ "dtype": "int",
1196+ "value": 0
1197+ },
1198+ {
1199+ "name": "enable_hf32",
1200+ "dtype": "bool",
1201+ "value": false
1202+ }
1203+ ]
1204+ }
1205+ ]
1206+}
Mmatmul/batch_mat_mul_v3/op_kernel/CMakeLists.txt+1-1
@@ -21,7 +21,7 @@ add_kernel_sources(
21 21 
22add_kernel_sources(22add_kernel_sources(
23 KERNEL_SRC arch35/batch_mat_mul_v3.cpp23 KERNEL_SRC arch35/batch_mat_mul_v3.cpp
24- COMPUTE_UNITS ascend95024+ COMPUTE_UNITS ascend950 ascend350
25 SIMPLIFIED_KEY None25 SIMPLIFIED_KEY None
26 AUTO_SYNC false26 AUTO_SYNC false
27 OPTIONS "--cce-no-dcache-flush"27 OPTIONS "--cce-no-dcache-flush"
Mmatmul/dual_level_quant_batch_matmul/op_host/dual_level_quant_batch_matmul_def.cpp+1-0
@@ -75,6 +75,7 @@ public:
75 .PrecisionReduceFlag(true)75 .PrecisionReduceFlag(true)
76 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");76 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
77 this->AICore().AddConfig("ascend950", config950);77 this->AICore().AddConfig("ascend950", config950);
78+ this->AICore().AddConfig("ascend350", config950);
78 }79 }
79 80 
80 static constexpr int64_t LEVEL0_GROUP_SIZE = 512L;81 static constexpr int64_t LEVEL0_GROUP_SIZE = 512L;
Mmatmul/dual_level_quant_batch_matmul/op_kernel/CMakeLists.txt+1-1
@@ -10,5 +10,5 @@
10 10 
11add_kernel_sources(11add_kernel_sources(
12 KERNEL_SRC arch35/dual_level_quant_batch_matmul.cpp12 KERNEL_SRC arch35/dual_level_quant_batch_matmul.cpp
13- COMPUTE_UNITS ascend95013+ COMPUTE_UNITS ascend950 ascend350
14)14)
Amatmul/fused_mat_mul/op_host/config/ascend350/fused_mat_mul_binary.json+728-0
@@ -0,0 +1,728 @@
1+{
2+ "op_type": "FusedMatMul",
3+ "optional_input_mode": "gen_placeholder",
4+ "op_list": [
5+ {
6+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP16_FP16_FP16",
7+ "simplified_key": "diy,2/2/2/2/2/1/1/1/1/1",
8+ "inputs": [
9+ {
10+ "name": "x1",
11+ "index": 0,
12+ "dtype": "float16",
13+ "format": "ND",
14+ "paramType": "required",
15+ "shape": [
16+ -2
17+ ]
18+ },
19+ {
20+ "name": "x2",
21+ "index": 1,
22+ "dtype": "float16",
23+ "format": "ND",
24+ "paramType": "required",
25+ "shape": [
26+ -2
27+ ]
28+ },
29+ {
30+ "name": "bias",
31+ "index": 2,
32+ "dtype": "float16",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "x3",
41+ "index": 3,
42+ "dtype": "float16",
43+ "format": "ND",
44+ "paramType": "optional",
45+ "shape": [
46+ -2
47+ ]
48+ }
49+ ],
50+ "outputs": [
51+ {
52+ "name": "y",
53+ "index": 0,
54+ "dtype": "float16",
55+ "format": "ND",
56+ "paramType": "required",
57+ "shape": [
58+ -2
59+ ]
60+ }
61+ ],
62+ "attrs": [
63+ {
64+ "name": "transpose_x1",
65+ "dtype": "bool",
66+ "value": false
67+ },
68+ {
69+ "name": "transpose_x2",
70+ "dtype": "bool",
71+ "value": false
72+ },
73+ {
74+ "name": "enable_hf32",
75+ "dtype": "bool",
76+ "value": false
77+ },
78+ {
79+ "name": "fused_op_type",
80+ "dtype": "string",
81+ "value": ""
82+ }
83+ ]
84+ },
85+ {
86+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP32_FP16_FP16",
87+ "simplified_key": "diy,2/2/2/2/2/1/1/0/1/1",
88+ "inputs": [
89+ {
90+ "name": "x1",
91+ "index": 0,
92+ "dtype": "float16",
93+ "format": "ND",
94+ "paramType": "required",
95+ "shape": [
96+ -2
97+ ]
98+ },
99+ {
100+ "name": "x2",
101+ "index": 1,
102+ "dtype": "float16",
103+ "format": "ND",
104+ "paramType": "required",
105+ "shape": [
106+ -2
107+ ]
108+ },
109+ {
110+ "name": "bias",
111+ "index": 2,
112+ "dtype": "float32",
113+ "format": "ND",
114+ "paramType": "optional",
115+ "shape": [
116+ -2
117+ ]
118+ },
119+ {
120+ "name": "x3",
121+ "index": 3,
122+ "dtype": "float16",
123+ "format": "ND",
124+ "paramType": "optional",
125+ "shape": [
126+ -2
127+ ]
128+ }
129+ ],
130+ "outputs": [
131+ {
132+ "name": "y",
133+ "index": 0,
134+ "dtype": "float16",
135+ "format": "ND",
136+ "paramType": "required",
137+ "shape": [
138+ -2
139+ ]
140+ }
141+ ],
142+ "attrs": [
143+ {
144+ "name": "transpose_x1",
145+ "dtype": "bool",
146+ "value": false
147+ },
148+ {
149+ "name": "transpose_x2",
150+ "dtype": "bool",
151+ "value": false
152+ },
153+ {
154+ "name": "enable_hf32",
155+ "dtype": "bool",
156+ "value": false
157+ },
158+ {
159+ "name": "fused_op_type",
160+ "dtype": "string",
161+ "value": ""
162+ }
163+ ]
164+ },
165+ {
166+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_BF16_BF16_BF16_BF16_BF16",
167+ "simplified_key": "diy,2/2/2/2/2/27/27/27/27/27",
168+ "inputs": [
169+ {
170+ "name": "x1",
171+ "index": 0,
172+ "dtype": "bfloat16",
173+ "format": "ND",
174+ "paramType": "required",
175+ "shape": [
176+ -2
177+ ]
178+ },
179+ {
180+ "name": "x2",
181+ "index": 1,
182+ "dtype": "bfloat16",
183+ "format": "ND",
184+ "paramType": "required",
185+ "shape": [
186+ -2
187+ ]
188+ },
189+ {
190+ "name": "bias",
191+ "index": 2,
192+ "dtype": "bfloat16",
193+ "format": "ND",
194+ "paramType": "optional",
195+ "shape": [
196+ -2
197+ ]
198+ },
199+ {
200+ "name": "x3",
201+ "index": 3,
202+ "dtype": "bfloat16",
203+ "format": "ND",
204+ "paramType": "optional",
205+ "shape": [
206+ -2
207+ ]
208+ }
209+ ],
210+ "outputs": [
211+ {
212+ "name": "y",
213+ "index": 0,
214+ "dtype": "bfloat16",
215+ "format": "ND",
216+ "paramType": "required",
217+ "shape": [
218+ -2
219+ ]
220+ }
221+ ],
222+ "attrs": [
223+ {
224+ "name": "transpose_x1",
225+ "dtype": "bool",
226+ "value": false
227+ },
228+ {
229+ "name": "transpose_x2",
230+ "dtype": "bool",
231+ "value": false
232+ },
233+ {
234+ "name": "enable_hf32",
235+ "dtype": "bool",
236+ "value": false
237+ },
238+ {
239+ "name": "fused_op_type",
240+ "dtype": "string",
241+ "value": ""
242+ }
243+ 
244+ ]
245+ },
246+ {
247+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_BF16_BF16_FP32_BF16_BF16",
248+ "simplified_key": "diy,2/2/2/2/2/27/27/0/27/27",
249+ "inputs": [
250+ {
251+ "name": "x1",
252+ "index": 0,
253+ "dtype": "bfloat16",
254+ "format": "ND",
255+ "paramType": "required",
256+ "shape": [
257+ -2
258+ ]
259+ },
260+ {
261+ "name": "x2",
262+ "index": 1,
263+ "dtype": "bfloat16",
264+ "format": "ND",
265+ "paramType": "required",
266+ "shape": [
267+ -2
268+ ]
269+ },
270+ {
271+ "name": "bias",
272+ "index": 2,
273+ "dtype": "float32",
274+ "format": "ND",
275+ "paramType": "optional",
276+ "shape": [
277+ -2
278+ ]
279+ },
280+ {
281+ "name": "x3",
282+ "index": 3,
283+ "dtype": "bfloat16",
284+ "format": "ND",
285+ "paramType": "optional",
286+ "shape": [
287+ -2
288+ ]
289+ }
290+ ],
291+ "outputs": [
292+ {
293+ "name": "y",
294+ "index": 0,
295+ "dtype": "bfloat16",
296+ "format": "ND",
297+ "paramType": "required",
298+ "shape": [
299+ -2
300+ ]
301+ }
302+ ],
303+ "attrs": [
304+ {
305+ "name": "transpose_x1",
306+ "dtype": "bool",
307+ "value": false
308+ },
309+ {
310+ "name": "transpose_x2",
311+ "dtype": "bool",
312+ "value": false
313+ },
314+ {
315+ "name": "enable_hf32",
316+ "dtype": "bool",
317+ "value": false
318+ },
319+ {
320+ "name": "fused_op_type",
321+ "dtype": "string",
322+ "value": ""
323+ }
324+ 
325+ ]
326+ },
327+ {
328+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_FP32_FP32_FP32_FP32_FP32",
329+ "simplified_key": "diy,2/2/2/2/2/0/0/0/0/0",
330+ "inputs": [
331+ {
332+ "name": "x1",
333+ "index": 0,
334+ "dtype": "float32",
335+ "format": "ND",
336+ "paramType": "required",
337+ "shape": [
338+ -2
339+ ]
340+ },
341+ {
342+ "name": "x2",
343+ "index": 1,
344+ "dtype": "float32",
345+ "format": "ND",
346+ "paramType": "required",
347+ "shape": [
348+ -2
349+ ]
350+ },
351+ {
352+ "name": "bias",
353+ "index": 2,
354+ "dtype": "float32",
355+ "format": "ND",
356+ "paramType": "optional",
357+ "shape": [
358+ -2
359+ ]
360+ },
361+ {
362+ "name": "x3",
363+ "index": 3,
364+ "dtype": "float32",
365+ "format": "ND",
366+ "paramType": "optional",
367+ "shape": [
368+ -2
369+ ]
370+ }
371+ ],
372+ "outputs": [
373+ {
374+ "name": "y",
375+ "index": 0,
376+ "dtype": "float32",
377+ "format": "ND",
378+ "paramType": "required",
379+ "shape": [
380+ -2
381+ ]
382+ }
383+ ],
384+ "attrs": [
385+ {
386+ "name": "transpose_x1",
387+ "dtype": "bool",
388+ "value": false
389+ },
390+ {
391+ "name": "transpose_x2",
392+ "dtype": "bool",
393+ "value": false
394+ },
395+ {
396+ "name": "enable_hf32",
397+ "dtype": "bool",
398+ "value": false
399+ },
400+ {
401+ "name": "fused_op_type",
402+ "dtype": "string",
403+ "value": ""
404+ }
405+ ]
406+ },
407+ {
408+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP16_FP16_FP32",
409+ "simplified_key": "diy,2/2/2/2/2/1/1/1/1/0",
410+ "inputs": [
411+ {
412+ "name": "x1",
413+ "index": 0,
414+ "dtype": "float16",
415+ "format": "ND",
416+ "paramType": "required",
417+ "shape": [
418+ -2
419+ ]
420+ },
421+ {
422+ "name": "x2",
423+ "index": 1,
424+ "dtype": "float16",
425+ "format": "ND",
426+ "paramType": "required",
427+ "shape": [
428+ -2
429+ ]
430+ },
431+ {
432+ "name": "bias",
433+ "index": 2,
434+ "dtype": "float16",
435+ "format": "ND",
436+ "paramType": "optional",
437+ "shape": [
438+ -2
439+ ]
440+ },
441+ {
442+ "name": "x3",
443+ "index": 3,
444+ "dtype": "float16",
445+ "format": "ND",
446+ "paramType": "optional",
447+ "shape": [
448+ -2
449+ ]
450+ }
451+ ],
452+ "outputs": [
453+ {
454+ "name": "y",
455+ "index": 0,
456+ "dtype": "float32",
457+ "format": "ND",
458+ "paramType": "required",
459+ "shape": [
460+ -2
461+ ]
462+ }
463+ ],
464+ "attrs": [
465+ {
466+ "name": "transpose_x1",
467+ "dtype": "bool",
468+ "value": false
469+ },
470+ {
471+ "name": "transpose_x2",
472+ "dtype": "bool",
473+ "value": false
474+ },
475+ {
476+ "name": "enable_hf32",
477+ "dtype": "bool",
478+ "value": false
479+ },
480+ {
481+ "name": "fused_op_type",
482+ "dtype": "string",
483+ "value": "16cast32"
484+ }
485+ ]
486+ },
487+ {
488+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP32_FP16_FP32",
489+ "simplified_key": "diy,2/2/2/2/2/1/1/0/1/0",
490+ "inputs": [
491+ {
492+ "name": "x1",
493+ "index": 0,
494+ "dtype": "float16",
495+ "format": "ND",
496+ "paramType": "required",
497+ "shape": [
498+ -2
499+ ]
500+ },
501+ {
502+ "name": "x2",
503+ "index": 1,
504+ "dtype": "float16",
505+ "format": "ND",
506+ "paramType": "required",
507+ "shape": [
508+ -2
509+ ]
510+ },
511+ {
512+ "name": "bias",
513+ "index": 2,
514+ "dtype": "float32",
515+ "format": "ND",
516+ "paramType": "optional",
517+ "shape": [
518+ -2
519+ ]
520+ },
521+ {
522+ "name": "x3",
523+ "index": 3,
524+ "dtype": "float16",
525+ "format": "ND",
526+ "paramType": "optional",
527+ "shape": [
528+ -2
529+ ]
530+ }
531+ ],
532+ "outputs": [
533+ {
534+ "name": "y",
535+ "index": 0,
536+ "dtype": "float32",
537+ "format": "ND",
538+ "paramType": "required",
539+ "shape": [
540+ -2
541+ ]
542+ }
543+ ],
544+ "attrs": [
545+ {
546+ "name": "transpose_x1",
547+ "dtype": "bool",
548+ "value": false
549+ },
550+ {
551+ "name": "transpose_x2",
552+ "dtype": "bool",
553+ "value": false
554+ },
555+ {
556+ "name": "enable_hf32",
557+ "dtype": "bool",
558+ "value": false
559+ },
560+ {
561+ "name": "fused_op_type",
562+ "dtype": "string",
563+ "value": "16cast32"
564+ }
565+ ]
566+ },
567+ {
568+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_BF16_BF16_BF16_BF16_FP32",
569+ "simplified_key": "diy,2/2/2/2/2/27/27/27/27/0",
570+ "inputs": [
571+ {
572+ "name": "x1",
573+ "index": 0,
574+ "dtype": "bfloat16",
575+ "format": "ND",
576+ "paramType": "required",
577+ "shape": [
578+ -2
579+ ]
580+ },
581+ {
582+ "name": "x2",
583+ "index": 1,
584+ "dtype": "bfloat16",
585+ "format": "ND",
586+ "paramType": "required",
587+ "shape": [
588+ -2
589+ ]
590+ },
591+ {
592+ "name": "bias",
593+ "index": 2,
594+ "dtype": "bfloat16",
595+ "format": "ND",
596+ "paramType": "optional",
597+ "shape": [
598+ -2
599+ ]
600+ },
601+ {
602+ "name": "x3",
603+ "index": 3,
604+ "dtype": "bfloat16",
605+ "format": "ND",
606+ "paramType": "optional",
607+ "shape": [
608+ -2
609+ ]
610+ }
611+ ],
612+ "outputs": [
613+ {
614+ "name": "y",
615+ "index": 0,
616+ "dtype": "float32",
617+ "format": "ND",
618+ "paramType": "required",
619+ "shape": [
620+ -2
621+ ]
622+ }
623+ ],
624+ "attrs": [
625+ {
626+ "name": "transpose_x1",
627+ "dtype": "bool",
628+ "value": false
629+ },
630+ {
631+ "name": "transpose_x2",
632+ "dtype": "bool",
633+ "value": false
634+ },
635+ {
636+ "name": "enable_hf32",
637+ "dtype": "bool",
638+ "value": false
639+ },
640+ {
641+ "name": "fused_op_type",
642+ "dtype": "string",
643+ "value": "16cast32"
644+ }
645+ ]
646+ },
647+ {
648+ "bin_filename": "FusedMatMul_ND_ND_ND_ND_ND_BF16_BF16_FP32_BF16_FP32",
649+ "simplified_key": "diy,2/2/2/2/2/27/27/0/27/0",
650+ "inputs": [
651+ {
652+ "name": "x1",
653+ "index": 0,
654+ "dtype": "bfloat16",
655+ "format": "ND",
656+ "paramType": "required",
657+ "shape": [
658+ -2
659+ ]
660+ },
661+ {
662+ "name": "x2",
663+ "index": 1,
664+ "dtype": "bfloat16",
665+ "format": "ND",
666+ "paramType": "required",
667+ "shape": [
668+ -2
669+ ]
670+ },
671+ {
672+ "name": "bias",
673+ "index": 2,
674+ "dtype": "float32",
675+ "format": "ND",
676+ "paramType": "optional",
677+ "shape": [
678+ -2
679+ ]
680+ },
681+ {
682+ "name": "x3",
683+ "index": 3,
684+ "dtype": "bfloat16",
685+ "format": "ND",
686+ "paramType": "optional",
687+ "shape": [
688+ -2
689+ ]
690+ }
691+ ],
692+ "outputs": [
693+ {
694+ "name": "y",
695+ "index": 0,
696+ "dtype": "float32",
697+ "format": "ND",
698+ "paramType": "required",
699+ "shape": [
700+ -2
701+ ]
702+ }
703+ ],
704+ "attrs": [
705+ {
706+ "name": "transpose_x1",
707+ "dtype": "bool",
708+ "value": false
709+ },
710+ {
711+ "name": "transpose_x2",
712+ "dtype": "bool",
713+ "value": false
714+ },
715+ {
716+ "name": "enable_hf32",
717+ "dtype": "bool",
718+ "value": false
719+ },
720+ {
721+ "name": "fused_op_type",
722+ "dtype": "string",
723+ "value": "16cast32"
724+ }
725+ ]
726+ }
727+ ]
728+}
Mmatmul/fused_mat_mul/op_host/fused_mat_mul_def.cpp+1-0
@@ -66,6 +66,7 @@
66 .PrecisionReduceFlag(true)66 .PrecisionReduceFlag(true)
67 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");67 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
68 this->AICore().AddConfig("ascend950", aicConfig);68 this->AICore().AddConfig("ascend950", aicConfig);
69+ this->AICore().AddConfig("ascend350", aicConfig);
69 }70 }
70};71};
71 72 
Mmatmul/fused_mat_mul/op_kernel/CMakeLists.txt+1-1
@@ -9,7 +9,7 @@
9# ----------------------------------------------------------------------------9# ----------------------------------------------------------------------------
10 10 
11add_kernel_sources(11add_kernel_sources(
12- COMPUTE_UNITS ascend95012+ COMPUTE_UNITS ascend950 ascend350
13 AUTO_SYNC false13 AUTO_SYNC false
14 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"14 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"
15)15)
Mmatmul/fused_quant_mat_mul/op_kernel/CMakeLists.txt+1-1
@@ -10,5 +10,5 @@
10 10 
11add_kernel_sources(11add_kernel_sources(
12 KERNEL_SRC arch35/fused_quant_mat_mul.cpp12 KERNEL_SRC arch35/fused_quant_mat_mul.cpp
13- COMPUTE_UNITS ascend950 mc62cm12a13+ COMPUTE_UNITS mc62cm12a
14)14)
Amatmul/gemm_v3/op_host/config/ascend350/gemm_v3_binary.json+234-0
@@ -0,0 +1,234 @@
1+{
2+ "op_type": "GemmV3",
3+ "optional_input_mode": "gen_placeholder",
4+ "op_list": [
5+ {
6+ "bin_filename": "GemmV3_ND_ND_ND_ND_FP16_FP16_FP32_FP32",
7+ "simplified_key": "diy,2/2/2/2/1/1/0/0",
8+ "inputs": [
9+ {
10+ "name": "a",
11+ "index": 0,
12+ "dtype": "float16",
13+ "format": "ND",
14+ "paramType": "required",
15+ "shape": [
16+ -2
17+ ]
18+ },
19+ {
20+ "name": "b",
21+ "index": 1,
22+ "dtype": "float16",
23+ "format": "ND",
24+ "paramType": "required",
25+ "shape": [
26+ -2
27+ ]
28+ },
29+ {
30+ "name": "c",
31+ "index": 2,
32+ "dtype": "float32",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ]
38+ }
39+ ],
40+ "outputs": [
41+ {
42+ "name": "y",
43+ "index": 0,
44+ "dtype": "float32",
45+ "format": "ND",
46+ "paramType": "required",
47+ "shape": [
48+ -2
49+ ]
50+ }
51+ ],
52+ "attrs": [
53+ {
54+ "name": "alpha",
55+ "dtype": "float",
56+ "value": 1.0
57+ },
58+ {
59+ "name": "beta",
60+ "dtype": "float",
61+ "value": 1.0
62+ 
63+ },
64+ {
65+ "name": "transpose_a",
66+ "dtype": "bool",
67+ "value": false
68+ },
69+ {
70+ "name": "transpose_b",
71+ "dtype": "bool",
72+ "value": false
73+ },
74+ {
75+ "name": "enable_hf32",
76+ "dtype": "bool",
77+ "value": false
78+ }
79+ ]
80+ },
81+ {
82+ "bin_filename": "GemmV3_ND_ND_ND_ND_BF16_BF16_FP32_FP32",
83+ "simplified_key": "diy,2/2/2/2/27/27/0/0",
84+ "inputs": [
85+ {
86+ "name": "a",
87+ "index": 0,
88+ "dtype": "bfloat16",
89+ "format": "ND",
90+ "paramType": "required",
91+ "shape": [
92+ -2
93+ ]
94+ },
95+ {
96+ "name": "b",
97+ "index": 1,
98+ "dtype": "bfloat16",
99+ "format": "ND",
100+ "paramType": "required",
101+ "shape": [
102+ -2
103+ ]
104+ },
105+ {
106+ "name": "c",
107+ "index": 2,
108+ "dtype": "float32",
109+ "format": "ND",
110+ "paramType": "optional",
111+ "shape": [
112+ -2
113+ ]
114+ }
115+ ],
116+ "outputs": [
117+ {
118+ "name": "y",
119+ "index": 0,
120+ "dtype": "float32",
121+ "format": "ND",
122+ "paramType": "required",
123+ "shape": [
124+ -2
125+ ]
126+ }
127+ ],
128+ "attrs": [
129+ {
130+ "name": "alpha",
131+ "dtype": "float",
132+ "value": 1.0
133+ },
134+ {
135+ "name": "beta",
136+ "dtype": "float",
137+ "value": 1.0
138+ 
139+ },
140+ {
141+ "name": "transpose_a",
142+ "dtype": "bool",
143+ "value": false
144+ },
145+ {
146+ "name": "transpose_b",
147+ "dtype": "bool",
148+ "value": false
149+ },
150+ {
151+ "name": "enable_hf32",
152+ "dtype": "bool",
153+ "value": false
154+ }
155+ ]
156+ },
157+ {
158+ "bin_filename": "GemmV3_ND_ND_ND_ND_FP32_FP32_FP32_FP32",
159+ "simplified_key": "diy,2/2/2/2/0/0/0/0",
160+ "inputs": [
161+ {
162+ "name": "a",
163+ "index": 0,
164+ "dtype": "float32",
165+ "format": "ND",
166+ "paramType": "required",
167+ "shape": [
168+ -2
169+ ]
170+ },
171+ {
172+ "name": "b",
173+ "index": 1,
174+ "dtype": "float32",
175+ "format": "ND",
176+ "paramType": "required",
177+ "shape": [
178+ -2
179+ ]
180+ },
181+ {
182+ "name": "c",
183+ "index": 2,
184+ "dtype": "float32",
185+ "format": "ND",
186+ "paramType": "optional",
187+ "shape": [
188+ -2
189+ ]
190+ }
191+ ],
192+ "outputs": [
193+ {
194+ "name": "y",
195+ "index": 0,
196+ "dtype": "float32",
197+ "format": "ND",
198+ "paramType": "required",
199+ "shape": [
200+ -2
201+ ]
202+ }
203+ ],
204+ "attrs": [
205+ {
206+ "name": "alpha",
207+ "dtype": "float",
208+ "value": 1.0
209+ },
210+ {
211+ "name": "beta",
212+ "dtype": "float",
213+ "value": 1.0
214+ 
215+ },
216+ {
217+ "name": "transpose_a",
218+ "dtype": "bool",
219+ "value": false
220+ },
221+ {
222+ "name": "transpose_b",
223+ "dtype": "bool",
224+ "value": false
225+ },
226+ {
227+ "name": "enable_hf32",
228+ "dtype": "bool",
229+ "value": false
230+ }
231+ ]
232+ }
233+ ]
234+}
Mmatmul/gemm_v3/op_host/gemm_v3_def.cpp+1-0
@@ -75,6 +75,7 @@ public:
75 .NeedCheckSupportFlag(false)75 .NeedCheckSupportFlag(false)
76 .ExtendCfgInfo("softsync.flag", "true");76 .ExtendCfgInfo("softsync.flag", "true");
77 this->AICore().AddConfig("ascend950", aicConfig_95);77 this->AICore().AddConfig("ascend950", aicConfig_95);
78+ this->AICore().AddConfig("ascend350", aicConfig_95);
78 }79 }
79};80};
80 81 
Mmatmul/gemm_v3/op_kernel/CMakeLists.txt+1-1
@@ -15,7 +15,7 @@ add_kernel_sources(
15 15 
16add_kernel_sources(16add_kernel_sources(
17 KERNEL_SRC arch35/gemm_v3.cpp17 KERNEL_SRC arch35/gemm_v3.cpp
18- COMPUTE_UNITS ascend95018+ COMPUTE_UNITS ascend950 ascend350
19 AUTO_SYNC false19 AUTO_SYNC false
20 OPTIONS "--cce-no-dcache-flush"20 OPTIONS "--cce-no-dcache-flush"
21)21)
Amatmul/mat_mul_v3/op_host/config/ascend350/mat_mul_v3_binary.json+1446-0
@@ -0,0 +1,1446 @@
1+{
2+ "op_type": "MatMulV3",
3+ "optional_input_mode": "gen_placeholder",
4+ "op_list": [
5+ {
6+ "bin_filename": "MatMulV3_ND_ND_ND_ND_FP16_FP16_FP16_FP16",
7+ "simplified_key": "diy,2/2/2/2/1/1/1/1",
8+ "inputs": [
9+ {
10+ "name": "x1",
11+ "index": 0,
12+ "dtype": "float16",
13+ "format": "ND",
14+ "paramType": "required",
15+ "shape": [
16+ -2
17+ ]
18+ },
19+ {
20+ "name": "x2",
21+ "index": 1,
22+ "dtype": "float16",
23+ "format": "ND",
24+ "paramType": "required",
25+ "shape": [
26+ -2
27+ ]
28+ },
29+ {
30+ "name": "bias",
31+ "index": 2,
32+ "dtype": "float16",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "offset_w",
41+ "index": 3,
42+ "dtype": "int8",
43+ "format": "ND",
44+ "paramType": "optional",
45+ "shape": [
46+ -2
47+ ]
48+ }
49+ ],
50+ "outputs": [
51+ {
52+ "name": "y",
53+ "index": 0,
54+ "dtype": "float16",
55+ "format": "ND",
56+ "paramType": "required",
57+ "shape": [
58+ -2
59+ ]
60+ }
61+ ],
62+ "attrs": [
63+ {
64+ "name": "transpose_x1",
65+ "dtype": "bool",
66+ "value": false
67+ },
68+ {
69+ "name": "transpose_x2",
70+ "dtype": "bool",
71+ "value": false
72+ },
73+ {
74+ "name": "offset_x",
75+ "dtype": "int",
76+ "value": 0
77+ },
78+ {
79+ "name": "opImplMode",
80+ "dtype": "int",
81+ "value": 1
82+ }
83+ ]
84+ },
85+ {
86+ "bin_filename": "MatMulV3_ND_ND_ND_ND_FP16_FP16_FP32_FP16",
87+ "simplified_key": "diy,2/2/2/2/1/1/0/1",
88+ "inputs": [
89+ {
90+ "name": "x1",
91+ "index": 0,
92+ "dtype": "float16",
93+ "format": "ND",
94+ "paramType": "required",
95+ "shape": [
96+ -2
97+ ]
98+ },
99+ {
100+ "name": "x2",
101+ "index": 1,
102+ "dtype": "float16",
103+ "format": "ND",
104+ "paramType": "required",
105+ "shape": [
106+ -2
107+ ]
108+ },
109+ {
110+ "name": "bias",
111+ "index": 2,
112+ "dtype": "float32",
113+ "format": "ND",
114+ "paramType": "optional",
115+ "shape": [
116+ -2
117+ ]
118+ },
119+ {
120+ "name": "offset_w",
121+ "index": 3,
122+ "dtype": "int8",
123+ "format": "ND",
124+ "paramType": "optional",
125+ "shape": [
126+ -2
127+ ]
128+ }
129+ ],
130+ "outputs": [
131+ {
132+ "name": "y",
133+ "index": 0,
134+ "dtype": "float16",
135+ "format": "ND",
136+ "paramType": "required",
137+ "shape": [
138+ -2
139+ ]
140+ }
141+ ],
142+ "attrs": [
143+ {
144+ "name": "transpose_x1",
145+ "dtype": "bool",
146+ "value": false
147+ },
148+ {
149+ "name": "transpose_x2",
150+ "dtype": "bool",
151+ "value": false
152+ },
153+ {
154+ "name": "offset_x",
155+ "dtype": "int",
156+ "value": 0
157+ },
158+ {
159+ "name": "opImplMode",
160+ "dtype": "int",
161+ "value": 1
162+ }
163+ ]
164+ },
165+ {
166+ "bin_filename": "MatMulV3_ND_ND_ND_ND_BF16_BF16_BF16_BF16",
167+ "simplified_key": "diy,2/2/2/2/27/27/27/27",
168+ "inputs": [
169+ {
170+ "name": "x1",
171+ "index": 0,
172+ "dtype": "bfloat16",
173+ "format": "ND",
174+ "paramType": "required",
175+ "shape": [
176+ -2
177+ ]
178+ },
179+ {
180+ "name": "x2",
181+ "index": 1,
182+ "dtype": "bfloat16",
183+ "format": "ND",
184+ "paramType": "required",
185+ "shape": [
186+ -2
187+ ]
188+ },
189+ {
190+ "name": "bias",
191+ "index": 2,
192+ "dtype": "bfloat16",
193+ "format": "ND",
194+ "paramType": "optional",
195+ "shape": [
196+ -2
197+ ]
198+ },
199+ {
200+ "name": "offset_w",
201+ "index": 3,
202+ "dtype": "int8",
203+ "format": "ND",
204+ "paramType": "optional",
205+ "shape": [
206+ -2
207+ ]
208+ }
209+ ],
210+ "outputs": [
211+ {
212+ "name": "y",
213+ "index": 0,
214+ "dtype": "bfloat16",
215+ "format": "ND",
216+ "paramType": "required",
217+ "shape": [
218+ -2
219+ ]
220+ }
221+ ],
222+ "attrs": [
223+ {
224+ "name": "transpose_x1",
225+ "dtype": "bool",
226+ "value": false
227+ },
228+ {
229+ "name": "transpose_x2",
230+ "dtype": "bool",
231+ "value": false
232+ },
233+ {
234+ "name": "offset_x",
235+ "dtype": "int",
236+ "value": 0
237+ },
238+ {
239+ "name": "opImplMode",
240+ "dtype": "int",
241+ "value": 1
242+ }
243+ ]
244+ },
245+ {
246+ "bin_filename": "MatMulV3_ND_ND_ND_ND_BF16_BF16_FP32_BF16",
247+ "simplified_key": "diy,2/2/2/2/27/27/0/27",
248+ "inputs": [
249+ {
250+ "name": "x1",
251+ "index": 0,
252+ "dtype": "bfloat16",
253+ "format": "ND",
254+ "paramType": "required",
255+ "shape": [
256+ -2
257+ ]
258+ },
259+ {
260+ "name": "x2",
261+ "index": 1,
262+ "dtype": "bfloat16",
263+ "format": "ND",
264+ "paramType": "required",
265+ "shape": [
266+ -2
267+ ]
268+ },
269+ {
270+ "name": "bias",
271+ "index": 2,
272+ "dtype": "float32",
273+ "format": "ND",
274+ "paramType": "optional",
275+ "shape": [
276+ -2
277+ ]
278+ },
279+ {
280+ "name": "offset_w",
281+ "index": 3,
282+ "dtype": "int8",
283+ "format": "ND",
284+ "paramType": "optional",
285+ "shape": [
286+ -2
287+ ]
288+ }
289+ ],
290+ "outputs": [
291+ {
292+ "name": "y",
293+ "index": 0,
294+ "dtype": "bfloat16",
295+ "format": "ND",
296+ "paramType": "required",
297+ "shape": [
298+ -2
299+ ]
300+ }
301+ ],
302+ "attrs": [
303+ {
304+ "name": "transpose_x1",
305+ "dtype": "bool",
306+ "value": false
307+ },
308+ {
309+ "name": "transpose_x2",
310+ "dtype": "bool",
311+ "value": false
312+ },
313+ {
314+ "name": "offset_x",
315+ "dtype": "int",
316+ "value": 0
317+ },
318+ {
319+ "name": "opImplMode",
320+ "dtype": "int",
321+ "value": 1
322+ }
323+ ]
324+ },
325+ {
326+ "bin_filename": "MatMulV3_ND_ND_ND_ND_FP32_FP32_FP32_FP32",
327+ "simplified_key": "diy,2/2/2/2/0/0/0/0",
328+ "inputs": [
329+ {
330+ "name": "x1",
331+ "index": 0,
332+ "dtype": "float32",
333+ "format": "ND",
334+ "paramType": "required",
335+ "shape": [
336+ -2
337+ ]
338+ },
339+ {
340+ "name": "x2",
341+ "index": 1,
342+ "dtype": "float32",
343+ "format": "ND",
344+ "paramType": "required",
345+ "shape": [
346+ -2
347+ ]
348+ },
349+ {
350+ "name": "bias",
351+ "index": 2,
352+ "dtype": "float32",
353+ "format": "ND",
354+ "paramType": "optional",
355+ "shape": [
356+ -2
357+ ]
358+ },
359+ {
360+ "name": "offset_w",
361+ "index": 3,
362+ "dtype": "int8",
363+ "format": "ND",
364+ "paramType": "optional",
365+ "shape": [
366+ -2
367+ ]
368+ }
369+ ],
370+ "outputs": [
371+ {
372+ "name": "y",
373+ "index": 0,
374+ "dtype": "float32",
375+ "format": "ND",
376+ "paramType": "required",
377+ "shape": [
378+ -2
379+ ]
380+ }
381+ ],
382+ "attrs": [
383+ {
384+ "name": "transpose_x1",
385+ "dtype": "bool",
386+ "value": false
387+ },
388+ {
389+ "name": "transpose_x2",
390+ "dtype": "bool",
391+ "value": true
392+ },
393+ {
394+ "name": "offset_x",
395+ "dtype": "int",
396+ "value": 0
397+ },
398+ {
399+ "name": "opImplMode",
400+ "dtype": "int",
401+ "value": 1
402+ }
403+ ]
404+ },
405+ {
406+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_FP16_FP16_FP16_FP16",
407+ "simplified_key": "diy,2/29/2/2/1/1/1/1",
408+ "inputs": [
409+ {
410+ "name": "x1",
411+ "index": 0,
412+ "dtype": "float16",
413+ "format": "ND",
414+ "paramType": "required",
415+ "shape": [
416+ -2
417+ ]
418+ },
419+ {
420+ "name": "x2",
421+ "index": 1,
422+ "dtype": "float16",
423+ "format": "FRACTAL_NZ",
424+ "paramType": "required",
425+ "shape": [
426+ -2
427+ ]
428+ },
429+ {
430+ "name": "bias",
431+ "index": 2,
432+ "dtype": "float16",
433+ "format": "ND",
434+ "paramType": "optional",
435+ "shape": [
436+ -2
437+ ]
438+ },
439+ {
440+ "name": "offset_w",
441+ "index": 3,
442+ "dtype": "int8",
443+ "format": "ND",
444+ "paramType": "optional",
445+ "shape": [
446+ -2
447+ ]
448+ }
449+ ],
450+ "outputs": [
451+ {
452+ "name": "y",
453+ "index": 0,
454+ "dtype": "float16",
455+ "format": "ND",
456+ "paramType": "required",
457+ "shape": [
458+ -2
459+ ]
460+ }
461+ ],
462+ "attrs": [
463+ {
464+ "name": "transpose_x1",
465+ "dtype": "bool",
466+ "value": false
467+ },
468+ {
469+ "name": "transpose_x2",
470+ "dtype": "bool",
471+ "value": false
472+ },
473+ {
474+ "name": "offset_x",
475+ "dtype": "int",
476+ "value": 0
477+ },
478+ {
479+ "name": "opImplMode",
480+ "dtype": "int",
481+ "value": 1
482+ }
483+ ]
484+ },
485+ {
486+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_FP16_FP16_FP32_FP16",
487+ "simplified_key": "diy,2/29/2/2/1/1/0/1",
488+ "inputs": [
489+ {
490+ "name": "x1",
491+ "index": 0,
492+ "dtype": "float16",
493+ "format": "ND",
494+ "paramType": "required",
495+ "shape": [
496+ -2
497+ ]
498+ },
499+ {
500+ "name": "x2",
501+ "index": 1,
502+ "dtype": "float16",
503+ "format": "FRACTAL_NZ",
504+ "paramType": "required",
505+ "shape": [
506+ -2
507+ ]
508+ },
509+ {
510+ "name": "bias",
511+ "index": 2,
512+ "dtype": "float32",
513+ "format": "ND",
514+ "paramType": "optional",
515+ "shape": [
516+ -2
517+ ]
518+ },
519+ {
520+ "name": "offset_w",
521+ "index": 3,
522+ "dtype": "int8",
523+ "format": "ND",
524+ "paramType": "optional",
525+ "shape": [
526+ -2
527+ ]
528+ }
529+ ],
530+ "outputs": [
531+ {
532+ "name": "y",
533+ "index": 0,
534+ "dtype": "float16",
535+ "format": "ND",
536+ "paramType": "required",
537+ "shape": [
538+ -2
539+ ]
540+ }
541+ ],
542+ "attrs": [
543+ {
544+ "name": "transpose_x1",
545+ "dtype": "bool",
546+ "value": false
547+ },
548+ {
549+ "name": "transpose_x2",
550+ "dtype": "bool",
551+ "value": false
552+ },
553+ {
554+ "name": "offset_x",
555+ "dtype": "int",
556+ "value": 0
557+ },
558+ {
559+ "name": "opImplMode",
560+ "dtype": "int",
561+ "value": 1
562+ }
563+ ]
564+ },
565+ {
566+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_BF16_BF16_BF16_BF16",
567+ "simplified_key": "diy,2/29/2/2/27/27/27/27",
568+ "inputs": [
569+ {
570+ "name": "x1",
571+ "index": 0,
572+ "dtype": "bfloat16",
573+ "format": "ND",
574+ "paramType": "required",
575+ "shape": [
576+ -2
577+ ]
578+ },
579+ {
580+ "name": "x2",
581+ "index": 1,
582+ "dtype": "bfloat16",
583+ "format": "FRACTAL_NZ",
584+ "paramType": "required",
585+ "shape": [
586+ -2
587+ ]
588+ },
589+ {
590+ "name": "bias",
591+ "index": 2,
592+ "dtype": "bfloat16",
593+ "format": "ND",
594+ "paramType": "optional",
595+ "shape": [
596+ -2
597+ ]
598+ },
599+ {
600+ "name": "offset_w",
601+ "index": 3,
602+ "dtype": "int8",
603+ "format": "ND",
604+ "paramType": "optional",
605+ "shape": [
606+ -2
607+ ]
608+ }
609+ ],
610+ "outputs": [
611+ {
612+ "name": "y",
613+ "index": 0,
614+ "dtype": "bfloat16",
615+ "format": "ND",
616+ "paramType": "required",
617+ "shape": [
618+ -2
619+ ]
620+ }
621+ ],
622+ "attrs": [
623+ {
624+ "name": "transpose_x1",
625+ "dtype": "bool",
626+ "value": false
627+ },
628+ {
629+ "name": "transpose_x2",
630+ "dtype": "bool",
631+ "value": false
632+ },
633+ {
634+ "name": "offset_x",
635+ "dtype": "int",
636+ "value": 0
637+ },
638+ {
639+ "name": "opImplMode",
640+ "dtype": "int",
641+ "value": 1
642+ }
643+ ]
644+ },
645+ {
646+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_BF16_BF16_FP32_BF16",
647+ "simplified_key": "diy,2/29/2/2/27/27/0/27",
648+ "inputs": [
649+ {
650+ "name": "x1",
651+ "index": 0,
652+ "dtype": "bfloat16",
653+ "format": "ND",
654+ "paramType": "required",
655+ "shape": [
656+ -2
657+ ]
658+ },
659+ {
660+ "name": "x2",
661+ "index": 1,
662+ "dtype": "bfloat16",
663+ "format": "FRACTAL_NZ",
664+ "paramType": "required",
665+ "shape": [
666+ -2
667+ ]
668+ },
669+ {
670+ "name": "bias",
671+ "index": 2,
672+ "dtype": "float32",
673+ "format": "ND",
674+ "paramType": "optional",
675+ "shape": [
676+ -2
677+ ]
678+ },
679+ {
680+ "name": "offset_w",
681+ "index": 3,
682+ "dtype": "int8",
683+ "format": "ND",
684+ "paramType": "optional",
685+ "shape": [
686+ -2
687+ ]
688+ }
689+ ],
690+ "outputs": [
691+ {
692+ "name": "y",
693+ "index": 0,
694+ "dtype": "bfloat16",
695+ "format": "ND",
696+ "paramType": "required",
697+ "shape": [
698+ -2
699+ ]
700+ }
701+ ],
702+ "attrs": [
703+ {
704+ "name": "transpose_x1",
705+ "dtype": "bool",
706+ "value": false
707+ },
708+ {
709+ "name": "transpose_x2",
710+ "dtype": "bool",
711+ "value": false
712+ },
713+ {
714+ "name": "offset_x",
715+ "dtype": "int",
716+ "value": 0
717+ },
718+ {
719+ "name": "opImplMode",
720+ "dtype": "int",
721+ "value": 1
722+ }
723+ ]
724+ },
725+ {
726+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_FP32_FP32_FP32_FP32",
727+ "simplified_key": "diy,2/29/2/2/0/0/0/0",
728+ "inputs": [
729+ {
730+ "name": "x1",
731+ "index": 0,
732+ "dtype": "float32",
733+ "format": "ND",
734+ "paramType": "required",
735+ "shape": [
736+ -2
737+ ]
738+ },
739+ {
740+ "name": "x2",
741+ "index": 1,
742+ "dtype": "float32",
743+ "format": "FRACTAL_NZ",
744+ "paramType": "required",
745+ "shape": [
746+ -2
747+ ]
748+ },
749+ {
750+ "name": "bias",
751+ "index": 2,
752+ "dtype": "float32",
753+ "format": "ND",
754+ "paramType": "optional",
755+ "shape": [
756+ -2
757+ ]
758+ },
759+ {
760+ "name": "offset_w",
761+ "index": 3,
762+ "dtype": "int8",
763+ "format": "ND",
764+ "paramType": "optional",
765+ "shape": [
766+ -2
767+ ]
768+ }
769+ ],
770+ "outputs": [
771+ {
772+ "name": "y",
773+ "index": 0,
774+ "dtype": "float32",
775+ "format": "ND",
776+ "paramType": "required",
777+ "shape": [
778+ -2
779+ ]
780+ }
781+ ],
782+ "attrs": [
783+ {
784+ "name": "transpose_x1",
785+ "dtype": "bool",
786+ "value": false
787+ },
788+ {
789+ "name": "transpose_x2",
790+ "dtype": "bool",
791+ "value": true
792+ },
793+ {
794+ "name": "offset_x",
795+ "dtype": "int",
796+ "value": 0
797+ },
798+ {
799+ "name": "opImplMode",
800+ "dtype": "int",
801+ "value": 1
802+ }
803+ ]
804+ },
805+ {
806+ "bin_filename": "MatMulV3_ND_ND_ND_ND_FP16_FP16_FP16_FP32",
807+ "simplified_key": "diy,2/2/2/2/1/1/1/0",
808+ "inputs": [
809+ {
810+ "name": "x1",
811+ "index": 0,
812+ "dtype": "float16",
813+ "format": "ND",
814+ "paramType": "required",
815+ "shape": [
816+ -2
817+ ]
818+ },
819+ {
820+ "name": "x2",
821+ "index": 1,
822+ "dtype": "float16",
823+ "format": "ND",
824+ "paramType": "required",
825+ "shape": [
826+ -2
827+ ]
828+ },
829+ {
830+ "name": "bias",
831+ "index": 2,
832+ "dtype": "float16",
833+ "format": "ND",
834+ "paramType": "optional",
835+ "shape": [
836+ -2
837+ ]
838+ },
839+ {
840+ "name": "offset_w",
841+ "index": 3,
842+ "dtype": "int8",
843+ "format": "ND",
844+ "paramType": "optional",
845+ "shape": [
846+ -2
847+ ]
848+ }
849+ ],
850+ "outputs": [
851+ {
852+ "name": "y",
853+ "index": 0,
854+ "dtype": "float32",
855+ "format": "ND",
856+ "paramType": "required",
857+ "shape": [
858+ -2
859+ ]
860+ }
861+ ],
862+ "attrs": [
863+ {
864+ "name": "transpose_x1",
865+ "dtype": "bool",
866+ "value": false
867+ },
868+ {
869+ "name": "transpose_x2",
870+ "dtype": "bool",
871+ "value": false
872+ },
873+ {
874+ "name": "offset_x",
875+ "dtype": "int",
876+ "value": 0
877+ },
878+ {
879+ "name": "opImplMode",
880+ "dtype": "int",
881+ "value": 1
882+ }
883+ ]
884+ },
885+ {
886+ "bin_filename": "MatMulV3_ND_ND_ND_ND_FP16_FP16_FP32_FP32",
887+ "simplified_key": "diy,2/2/2/2/1/1/0/0",
888+ "inputs": [
889+ {
890+ "name": "x1",
891+ "index": 0,
892+ "dtype": "float16",
893+ "format": "ND",
894+ "paramType": "required",
895+ "shape": [
896+ -2
897+ ]
898+ },
899+ {
900+ "name": "x2",
901+ "index": 1,
902+ "dtype": "float16",
903+ "format": "ND",
904+ "paramType": "required",
905+ "shape": [
906+ -2
907+ ]
908+ },
909+ {
910+ "name": "bias",
911+ "index": 2,
912+ "dtype": "float32",
913+ "format": "ND",
914+ "paramType": "optional",
915+ "shape": [
916+ -2
917+ ]
918+ },
919+ {
920+ "name": "offset_w",
921+ "index": 3,
922+ "dtype": "int8",
923+ "format": "ND",
924+ "paramType": "optional",
925+ "shape": [
926+ -2
927+ ]
928+ }
929+ ],
930+ "outputs": [
931+ {
932+ "name": "y",
933+ "index": 0,
934+ "dtype": "float32",
935+ "format": "ND",
936+ "paramType": "required",
937+ "shape": [
938+ -2
939+ ]
940+ }
941+ ],
942+ "attrs": [
943+ {
944+ "name": "transpose_x1",
945+ "dtype": "bool",
946+ "value": false
947+ },
948+ {
949+ "name": "transpose_x2",
950+ "dtype": "bool",
951+ "value": false
952+ },
953+ {
954+ "name": "offset_x",
955+ "dtype": "int",
956+ "value": 0
957+ },
958+ {
959+ "name": "opImplMode",
960+ "dtype": "int",
961+ "value": 1
962+ }
963+ ]
964+ },
965+ {
966+ "bin_filename": "MatMulV3_ND_ND_ND_ND_BF16_BF16_BF16_FP32",
967+ "simplified_key": "diy,2/2/2/2/27/27/27/0",
968+ "inputs": [
969+ {
970+ "name": "x1",
971+ "index": 0,
972+ "dtype": "bfloat16",
973+ "format": "ND",
974+ "paramType": "required",
975+ "shape": [
976+ -2
977+ ]
978+ },
979+ {
980+ "name": "x2",
981+ "index": 1,
982+ "dtype": "bfloat16",
983+ "format": "ND",
984+ "paramType": "required",
985+ "shape": [
986+ -2
987+ ]
988+ },
989+ {
990+ "name": "bias",
991+ "index": 2,
992+ "dtype": "bfloat16",
993+ "format": "ND",
994+ "paramType": "optional",
995+ "shape": [
996+ -2
997+ ]
998+ },
999+ {
1000+ "name": "offset_w",
1001+ "index": 3,
1002+ "dtype": "int8",
1003+ "format": "ND",
1004+ "paramType": "optional",
1005+ "shape": [
1006+ -2
1007+ ]
1008+ }
1009+ ],
1010+ "outputs": [
1011+ {
1012+ "name": "y",
1013+ "index": 0,
1014+ "dtype": "float32",
1015+ "format": "ND",
1016+ "paramType": "required",
1017+ "shape": [
1018+ -2
1019+ ]
1020+ }
1021+ ],
1022+ "attrs": [
1023+ {
1024+ "name": "transpose_x1",
1025+ "dtype": "bool",
1026+ "value": false
1027+ },
1028+ {
1029+ "name": "transpose_x2",
1030+ "dtype": "bool",
1031+ "value": false
1032+ },
1033+ {
1034+ "name": "offset_x",
1035+ "dtype": "int",
1036+ "value": 0
1037+ },
1038+ {
1039+ "name": "opImplMode",
1040+ "dtype": "int",
1041+ "value": 1
1042+ }
1043+ ]
1044+ },
1045+ {
1046+ "bin_filename": "MatMulV3_ND_ND_ND_ND_BF16_BF16_FP32_FP32",
1047+ "simplified_key": "diy,2/2/2/2/27/27/0/0",
1048+ "inputs": [
1049+ {
1050+ "name": "x1",
1051+ "index": 0,
1052+ "dtype": "bfloat16",
1053+ "format": "ND",
1054+ "paramType": "required",
1055+ "shape": [
1056+ -2
1057+ ]
1058+ },
1059+ {
1060+ "name": "x2",
1061+ "index": 1,
1062+ "dtype": "bfloat16",
1063+ "format": "ND",
1064+ "paramType": "required",
1065+ "shape": [
1066+ -2
1067+ ]
1068+ },
1069+ {
1070+ "name": "bias",
1071+ "index": 2,
1072+ "dtype": "float32",
1073+ "format": "ND",
1074+ "paramType": "optional",
1075+ "shape": [
1076+ -2
1077+ ]
1078+ },
1079+ {
1080+ "name": "offset_w",
1081+ "index": 3,
1082+ "dtype": "int8",
1083+ "format": "ND",
1084+ "paramType": "optional",
1085+ "shape": [
1086+ -2
1087+ ]
1088+ }
1089+ ],
1090+ "outputs": [
1091+ {
1092+ "name": "y",
1093+ "index": 0,
1094+ "dtype": "float32",
1095+ "format": "ND",
1096+ "paramType": "required",
1097+ "shape": [
1098+ -2
1099+ ]
1100+ }
1101+ ],
1102+ "attrs": [
1103+ {
1104+ "name": "transpose_x1",
1105+ "dtype": "bool",
1106+ "value": false
1107+ },
1108+ {
1109+ "name": "transpose_x2",
1110+ "dtype": "bool",
1111+ "value": false
1112+ },
1113+ {
1114+ "name": "offset_x",
1115+ "dtype": "int",
1116+ "value": 0
1117+ },
1118+ {
1119+ "name": "opImplMode",
1120+ "dtype": "int",
1121+ "value": 1
1122+ }
1123+ ]
1124+ },
1125+ {
1126+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_FP16_FP16_FP16_FP32",
1127+ "simplified_key": "diy,2/29/2/2/1/1/1/0",
1128+ "inputs": [
1129+ {
1130+ "name": "x1",
1131+ "index": 0,
1132+ "dtype": "float16",
1133+ "format": "ND",
1134+ "paramType": "required",
1135+ "shape": [
1136+ -2
1137+ ]
1138+ },
1139+ {
1140+ "name": "x2",
1141+ "index": 1,
1142+ "dtype": "float16",
1143+ "format": "FRACTAL_NZ",
1144+ "paramType": "required",
1145+ "shape": [
1146+ -2
1147+ ]
1148+ },
1149+ {
1150+ "name": "bias",
1151+ "index": 2,
1152+ "dtype": "float16",
1153+ "format": "ND",
1154+ "paramType": "optional",
1155+ "shape": [
1156+ -2
1157+ ]
1158+ },
1159+ {
1160+ "name": "offset_w",
1161+ "index": 3,
1162+ "dtype": "int8",
1163+ "format": "ND",
1164+ "paramType": "optional",
1165+ "shape": [
1166+ -2
1167+ ]
1168+ }
1169+ ],
1170+ "outputs": [
1171+ {
1172+ "name": "y",
1173+ "index": 0,
1174+ "dtype": "float32",
1175+ "format": "ND",
1176+ "paramType": "required",
1177+ "shape": [
1178+ -2
1179+ ]
1180+ }
1181+ ],
1182+ "attrs": [
1183+ {
1184+ "name": "transpose_x1",
1185+ "dtype": "bool",
1186+ "value": false
1187+ },
1188+ {
1189+ "name": "transpose_x2",
1190+ "dtype": "bool",
1191+ "value": false
1192+ },
1193+ {
1194+ "name": "offset_x",
1195+ "dtype": "int",
1196+ "value": 0
1197+ },
1198+ {
1199+ "name": "opImplMode",
1200+ "dtype": "int",
1201+ "value": 1
1202+ }
1203+ ]
1204+ },
1205+ {
1206+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_FP16_FP16_FP32_FP32",
1207+ "simplified_key": "diy,2/29/2/2/1/1/0/0",
1208+ "inputs": [
1209+ {
1210+ "name": "x1",
1211+ "index": 0,
1212+ "dtype": "float16",
1213+ "format": "ND",
1214+ "paramType": "required",
1215+ "shape": [
1216+ -2
1217+ ]
1218+ },
1219+ {
1220+ "name": "x2",
1221+ "index": 1,
1222+ "dtype": "float16",
1223+ "format": "FRACTAL_NZ",
1224+ "paramType": "required",
1225+ "shape": [
1226+ -2
1227+ ]
1228+ },
1229+ {
1230+ "name": "bias",
1231+ "index": 2,
1232+ "dtype": "float32",
1233+ "format": "ND",
1234+ "paramType": "optional",
1235+ "shape": [
1236+ -2
1237+ ]
1238+ },
1239+ {
1240+ "name": "offset_w",
1241+ "index": 3,
1242+ "dtype": "int8",
1243+ "format": "ND",
1244+ "paramType": "optional",
1245+ "shape": [
1246+ -2
1247+ ]
1248+ }
1249+ ],
1250+ "outputs": [
1251+ {
1252+ "name": "y",
1253+ "index": 0,
1254+ "dtype": "float32",
1255+ "format": "ND",
1256+ "paramType": "required",
1257+ "shape": [
1258+ -2
1259+ ]
1260+ }
1261+ ],
1262+ "attrs": [
1263+ {
1264+ "name": "transpose_x1",
1265+ "dtype": "bool",
1266+ "value": false
1267+ },
1268+ {
1269+ "name": "transpose_x2",
1270+ "dtype": "bool",
1271+ "value": false
1272+ },
1273+ {
1274+ "name": "offset_x",
1275+ "dtype": "int",
1276+ "value": 0
1277+ },
1278+ {
1279+ "name": "opImplMode",
1280+ "dtype": "int",
1281+ "value": 1
1282+ }
1283+ ]
1284+ },
1285+ {
1286+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_BF16_BF16_BF16_FP32",
1287+ "simplified_key": "diy,2/29/2/2/27/27/27/0",
1288+ "inputs": [
1289+ {
1290+ "name": "x1",
1291+ "index": 0,
1292+ "dtype": "bfloat16",
1293+ "format": "ND",
1294+ "paramType": "required",
1295+ "shape": [
1296+ -2
1297+ ]
1298+ },
1299+ {
1300+ "name": "x2",
1301+ "index": 1,
1302+ "dtype": "bfloat16",
1303+ "format": "FRACTAL_NZ",
1304+ "paramType": "required",
1305+ "shape": [
1306+ -2
1307+ ]
1308+ },
1309+ {
1310+ "name": "bias",
1311+ "index": 2,
1312+ "dtype": "bfloat16",
1313+ "format": "ND",
1314+ "paramType": "optional",
1315+ "shape": [
1316+ -2
1317+ ]
1318+ },
1319+ {
1320+ "name": "offset_w",
1321+ "index": 3,
1322+ "dtype": "int8",
1323+ "format": "ND",
1324+ "paramType": "optional",
1325+ "shape": [
1326+ -2
1327+ ]
1328+ }
1329+ ],
1330+ "outputs": [
1331+ {
1332+ "name": "y",
1333+ "index": 0,
1334+ "dtype": "float32",
1335+ "format": "ND",
1336+ "paramType": "required",
1337+ "shape": [
1338+ -2
1339+ ]
1340+ }
1341+ ],
1342+ "attrs": [
1343+ {
1344+ "name": "transpose_x1",
1345+ "dtype": "bool",
1346+ "value": false
1347+ },
1348+ {
1349+ "name": "transpose_x2",
1350+ "dtype": "bool",
1351+ "value": false
1352+ },
1353+ {
1354+ "name": "offset_x",
1355+ "dtype": "int",
1356+ "value": 0
1357+ },
1358+ {
1359+ "name": "opImplMode",
1360+ "dtype": "int",
1361+ "value": 1
1362+ }
1363+ ]
1364+ },
1365+ {
1366+ "bin_filename": "MatMulV3_ND_NZ_ND_ND_BF16_BF16_FP32_FP32",
1367+ "simplified_key": "diy,2/29/2/2/27/27/0/0",
1368+ "inputs": [
1369+ {
1370+ "name": "x1",
1371+ "index": 0,
1372+ "dtype": "bfloat16",
1373+ "format": "ND",
1374+ "paramType": "required",
1375+ "shape": [
1376+ -2
1377+ ]
1378+ },
1379+ {
1380+ "name": "x2",
1381+ "index": 1,
1382+ "dtype": "bfloat16",
1383+ "format": "FRACTAL_NZ",
1384+ "paramType": "required",
1385+ "shape": [
1386+ -2
1387+ ]
1388+ },
1389+ {
1390+ "name": "bias",
1391+ "index": 2,
1392+ "dtype": "float32",
1393+ "format": "ND",
1394+ "paramType": "optional",
1395+ "shape": [
1396+ -2
1397+ ]
1398+ },
1399+ {
1400+ "name": "offset_w",
1401+ "index": 3,
1402+ "dtype": "int8",
1403+ "format": "ND",
1404+ "paramType": "optional",
1405+ "shape": [
1406+ -2
1407+ ]
1408+ }
1409+ ],
1410+ "outputs": [
1411+ {
1412+ "name": "y",
1413+ "index": 0,
1414+ "dtype": "float32",
1415+ "format": "ND",
1416+ "paramType": "required",
1417+ "shape": [
1418+ -2
1419+ ]
1420+ }
1421+ ],
1422+ "attrs": [
1423+ {
1424+ "name": "transpose_x1",
1425+ "dtype": "bool",
1426+ "value": false
1427+ },
1428+ {
1429+ "name": "transpose_x2",
1430+ "dtype": "bool",
1431+ "value": false
1432+ },
1433+ {
1434+ "name": "offset_x",
1435+ "dtype": "int",
1436+ "value": 0
1437+ },
1438+ {
1439+ "name": "opImplMode",
1440+ "dtype": "int",
1441+ "value": 1
1442+ }
1443+ ]
1444+ }
1445+ ]
1446+}
Mmatmul/mat_mul_v3/op_host/mat_mul_v3_def.cpp+1-0
@@ -173,6 +173,7 @@ public:
173 aicConfig.PrecisionReduceFlag(true)173 aicConfig.PrecisionReduceFlag(true)
174 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");174 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
175 this->AICore().AddConfig("ascend950", aicConfig);175 this->AICore().AddConfig("ascend950", aicConfig);
176+ this->AICore().AddConfig("ascend350", aicConfig);
176 177 
177 OpAICoreConfig config_kirin = GetKirinCoreConfig();178 OpAICoreConfig config_kirin = GetKirinCoreConfig();
178 this->AICore().AddConfig("kirinx90", config_kirin);179 this->AICore().AddConfig("kirinx90", config_kirin);
Mmatmul/mat_mul_v3/op_kernel/CMakeLists.txt+1-1
@@ -26,7 +26,7 @@ add_kernel_sources(
26 26 
27add_kernel_sources(27add_kernel_sources(
28 KERNEL_SRC arch35/mat_mul_v3.cpp28 KERNEL_SRC arch35/mat_mul_v3.cpp
29- COMPUTE_UNITS ascend95029+ COMPUTE_UNITS ascend950 ascend350
30 SIMPLIFIED_KEY None30 SIMPLIFIED_KEY None
31 AUTO_SYNC false31 AUTO_SYNC false
32 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"32 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"
Mmatmul/quant_batch_matmul_inplace_add/op_host/quant_batch_matmul_inplace_add_def.cpp+1-0
@@ -61,6 +61,7 @@ public:
61 .PrecisionReduceFlag(true)61 .PrecisionReduceFlag(true)
62 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");62 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
63 this->AICore().AddConfig("ascend950", config950);63 this->AICore().AddConfig("ascend950", config950);
64+ this->AICore().AddConfig("ascend350", config950);
64 }65 }
65};66};
66 67 
Mmatmul/quant_batch_matmul_inplace_add/op_kernel/CMakeLists.txt+1-1
@@ -10,6 +10,6 @@
10 10 
11add_kernel_sources(11add_kernel_sources(
12 KERNEL_SRC arch35/quant_batch_matmul_inplace_add.cpp12 KERNEL_SRC arch35/quant_batch_matmul_inplace_add.cpp
13- COMPUTE_UNITS ascend95013+ COMPUTE_UNITS ascend950 ascend350
14 AUTO_SYNC false14 AUTO_SYNC false
15)15)
Mmatmul/quant_batch_matmul_v3/op_host/quant_batch_matmul_v3_def.cpp+1-0
@@ -1080,6 +1080,7 @@ public:
1080 .PrecisionReduceFlag(true)1080 .PrecisionReduceFlag(true)
1081 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");1081 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
1082 this->AICore().AddConfig("ascend950", config950);1082 this->AICore().AddConfig("ascend950", config950);
1083+ this->AICore().AddConfig("ascend350", config950);
1083 1084 
1084 OpAICoreConfig config_kirin = GetKirinCoreConfig();1085 OpAICoreConfig config_kirin = GetKirinCoreConfig();
1085 this->AICore().AddConfig("kirinx90", config_kirin);1086 this->AICore().AddConfig("kirinx90", config_kirin);
Mmatmul/quant_batch_matmul_v3/op_kernel/CMakeLists.txt+1-1
@@ -14,7 +14,7 @@ add_kernel_sources(
14 14 
15add_kernel_sources(15add_kernel_sources(
16 KERNEL_SRC arch35/quant_batch_matmul_v3.cpp16 KERNEL_SRC arch35/quant_batch_matmul_v3.cpp
17- COMPUTE_UNITS ascend95017+ COMPUTE_UNITS ascend950 ascend350
18 AUTO_SYNC false18 AUTO_SYNC false
19 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"19 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"
20)20)
Amatmul/quant_batch_matmul_v4/op_host/config/ascend350/quant_batch_matmul_v4_binary.json+1400-0
@@ -0,0 +1,1400 @@
1+{
2+ "op_type": "QuantBatchMatmulV4",
3+ "op_list": [
4+ {
5+ "bin_filename": "QuantBatchMatmulV4_ND_ND_float8_e4m3fn_float4_e2m1_fp16",
6+ "inputs": [
7+ {
8+ "name": "x1",
9+ "index": 0,
10+ "dtype": "float8_e4m3fn",
11+ "format": "ND",
12+ "paramType": "required",
13+ "shape": [
14+ -2
15+ ],
16+ "format_match_mode": "FormatDefault"
17+ },
18+ {
19+ "name": "x2",
20+ "index": 1,
21+ "dtype": "float4_e2m1",
22+ "format": "ND",
23+ "paramType": "required",
24+ "shape": [
25+ -2
26+ ],
27+ "format_match_mode": "FormatDefault"
28+ },
29+ {
30+ "name": "bias",
31+ "index": 2,
32+ "dtype": "bfloat16",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ],
38+ "format_match_mode": "FormatDefault"
39+ },
40+ {
41+ "name": "x1_scale",
42+ "index": 3,
43+ "dtype": "bfloat16",
44+ "format": "ND",
45+ "paramType": "optional",
46+ "shape": [
47+ -2
48+ ],
49+ "format_match_mode": "FormatDefault"
50+ },
51+ {
52+ "name": "x2_scale",
53+ "index": 4,
54+ "dtype": "bfloat16",
55+ "format": "ND",
56+ "paramType": "optional",
57+ "shape": [
58+ -2
59+ ],
60+ "format_match_mode": "FormatDefault"
61+ },
62+ {
63+ "name": "y_scale",
64+ "index": 5,
65+ "dtype": "uint64",
66+ "format": "ND",
67+ "paramType": "optional",
68+ "shape": [
69+ -2
70+ ],
71+ "format_match_mode": "FormatDefault"
72+ },
73+ {
74+ "name": "x1_offset",
75+ "index": 6,
76+ "dtype": "bfloat16",
77+ "format": "ND",
78+ "paramType": "optional",
79+ "shape": [
80+ -2
81+ ],
82+ "format_match_mode": "FormatDefault"
83+ },
84+ {
85+ "name": "x2_offset",
86+ "index": 7,
87+ "dtype": "bfloat16",
88+ "format": "ND",
89+ "paramType": "optional",
90+ "shape": [
91+ -2
92+ ],
93+ "format_match_mode": "FormatDefault"
94+ },
95+ {
96+ "name": "y_offset",
97+ "index": 8,
98+ "dtype": "bfloat16",
99+ "format": "ND",
100+ "paramType": "optional",
101+ "shape": [
102+ -2
103+ ],
104+ "format_match_mode": "FormatDefault"
105+ },
106+ {
107+ "name": "x2_table",
108+ "index": 9,
109+ "dtype": "int8",
110+ "format": "ND",
111+ "paramType": "optional",
112+ "shape": [
113+ -2
114+ ],
115+ "format_match_mode": "FormatDefault"
116+ }
117+ ],
118+ "outputs": [
119+ {
120+ "name": "y",
121+ "index": 0,
122+ "dtype": "bfloat16",
123+ "format": "ND",
124+ "paramType": "required",
125+ "shape": [
126+ -2
127+ ],
128+ "format_match_mode": "FormatDefault"
129+ }
130+ ],
131+ "attrs": [
132+ {
133+ "name": "dtype",
134+ "dtype": "int",
135+ "value": -1
136+ },
137+ {
138+ "name": "compute_type",
139+ "dtype": "int",
140+ "value": -1
141+ },
142+ {
143+ "name": "transpose_x1",
144+ "dtype": "bool",
145+ "value": false
146+ },
147+ {
148+ "name": "transpose_x2",
149+ "dtype": "bool",
150+ "value": false
151+ },
152+ {
153+ "name": "group_size",
154+ "dtype": "int",
155+ "value": -1
156+ }
157+ ]
158+ },
159+ {
160+ "bin_filename": "QuantBatchMatmulV4_ND_NZ_float8_e4m3fn_float4_e2m1_fp16",
161+ "inputs": [
162+ {
163+ "name": "x1",
164+ "index": 0,
165+ "dtype": "float8_e4m3fn",
166+ "format": "ND",
167+ "paramType": "required",
168+ "shape": [
169+ -2
170+ ],
171+ "format_match_mode": "FormatDefault"
172+ },
173+ {
174+ "name": "x2",
175+ "index": 1,
176+ "dtype": "float4_e2m1",
177+ "format": "FRACTAL_NZ",
178+ "paramType": "required",
179+ "shape": [
180+ -2
181+ ],
182+ "format_match_mode": "FormatDefault"
183+ },
184+ {
185+ "name": "bias",
186+ "index": 2,
187+ "dtype": "bfloat16",
188+ "format": "ND",
189+ "paramType": "optional",
190+ "shape": [
191+ -2
192+ ],
193+ "format_match_mode": "FormatDefault"
194+ },
195+ {
196+ "name": "x1_scale",
197+ "index": 3,
198+ "dtype": "bfloat16",
199+ "format": "ND",
200+ "paramType": "optional",
201+ "shape": [
202+ -2
203+ ],
204+ "format_match_mode": "FormatDefault"
205+ },
206+ {
207+ "name": "x2_scale",
208+ "index": 4,
209+ "dtype": "bfloat16",
210+ "format": "ND",
211+ "paramType": "optional",
212+ "shape": [
213+ -2
214+ ],
215+ "format_match_mode": "FormatDefault"
216+ },
217+ {
218+ "name": "y_scale",
219+ "index": 5,
220+ "dtype": "uint64",
221+ "format": "ND",
222+ "paramType": "optional",
223+ "shape": [
224+ -2
225+ ],
226+ "format_match_mode": "FormatDefault"
227+ },
228+ {
229+ "name": "x1_offset",
230+ "index": 6,
231+ "dtype": "bfloat16",
232+ "format": "ND",
233+ "paramType": "optional",
234+ "shape": [
235+ -2
236+ ],
237+ "format_match_mode": "FormatDefault"
238+ },
239+ {
240+ "name": "x2_offset",
241+ "index": 7,
242+ "dtype": "bfloat16",
243+ "format": "ND",
244+ "paramType": "optional",
245+ "shape": [
246+ -2
247+ ],
248+ "format_match_mode": "FormatDefault"
249+ },
250+ {
251+ "name": "y_offset",
252+ "index": 8,
253+ "dtype": "bfloat16",
254+ "format": "ND",
255+ "paramType": "optional",
256+ "shape": [
257+ -2
258+ ],
259+ "format_match_mode": "FormatDefault"
260+ },
261+ {
262+ "name": "x2_table",
263+ "index": 9,
264+ "dtype": "int8",
265+ "format": "ND",
266+ "paramType": "optional",
267+ "shape": [
268+ -2
269+ ],
270+ "format_match_mode": "FormatDefault"
271+ }
272+ ],
273+ "outputs": [
274+ {
275+ "name": "y",
276+ "index": 0,
277+ "dtype": "bfloat16",
278+ "format": "ND",
279+ "paramType": "required",
280+ "shape": [
281+ -2
282+ ],
283+ "format_match_mode": "FormatDefault"
284+ }
285+ ],
286+ "attrs": [
287+ {
288+ "name": "dtype",
289+ "dtype": "int",
290+ "value": -1
291+ },
292+ {
293+ "name": "compute_type",
294+ "dtype": "int",
295+ "value": -1
296+ },
297+ {
298+ "name": "transpose_x1",
299+ "dtype": "bool",
300+ "value": false
301+ },
302+ {
303+ "name": "transpose_x2",
304+ "dtype": "bool",
305+ "value": false
306+ },
307+ {
308+ "name": "group_size",
309+ "dtype": "int",
310+ "value": -1
311+ }
312+ ]
313+ },
314+ {
315+ "bin_filename": "QuantBatchMatmulV4_ND_ND_int8_int8_bf16",
316+ "inputs": [
317+ {
318+ "name": "x1",
319+ "index": 0,
320+ "dtype": "int8",
321+ "format": "ND",
322+ "paramType": "required",
323+ "shape": [
324+ -2
325+ ],
326+ "format_match_mode": "FormatDefault"
327+ },
328+ {
329+ "name": "x2",
330+ "index": 1,
331+ "dtype": "int8",
332+ "format": "ND",
333+ "paramType": "required",
334+ "shape": [
335+ -2
336+ ],
337+ "format_match_mode": "FormatDefault"
338+ },
339+ {
340+ "name": "bias",
341+ "index": 2,
342+ "dtype": "float32",
343+ "format": "ND",
344+ "paramType": "optional",
345+ "shape": [
346+ -2
347+ ],
348+ "format_match_mode": "FormatDefault"
349+ },
350+ {
351+ "name": "x1_scale",
352+ "index": 3,
353+ "dtype": "float32",
354+ "format": "ND",
355+ "paramType": "optional",
356+ "shape": [
357+ -2
358+ ],
359+ "format_match_mode": "FormatDefault"
360+ },
361+ {
362+ "name": "x2_scale",
363+ "index": 4,
364+ "dtype": "float32",
365+ "format": "ND",
366+ "paramType": "optional",
367+ "shape": [
368+ -2
369+ ],
370+ "format_match_mode": "FormatDefault"
371+ },
372+ {
373+ "name": "y_scale",
374+ "index": 5,
375+ "dtype": "uint64",
376+ "format": "ND",
377+ "paramType": "optional",
378+ "shape": [
379+ -2
380+ ],
381+ "format_match_mode": "FormatDefault"
382+ },
383+ {
384+ "name": "x1_offset",
385+ "index": 6,
386+ "dtype": "bfloat16",
387+ "format": "ND",
388+ "paramType": "optional",
389+ "shape": [
390+ -2
391+ ],
392+ "format_match_mode": "FormatDefault"
393+ },
394+ {
395+ "name": "x2_offset",
396+ "index": 7,
397+ "dtype": "bfloat16",
398+ "format": "ND",
399+ "paramType": "optional",
400+ "shape": [
401+ -2
402+ ],
403+ "format_match_mode": "FormatDefault"
404+ },
405+ {
406+ "name": "y_offset",
407+ "index": 8,
408+ "dtype": "bfloat16",
409+ "format": "ND",
410+ "paramType": "optional",
411+ "shape": [
412+ -2
413+ ],
414+ "format_match_mode": "FormatDefault"
415+ },
416+ {
417+ "name": "x2_table",
418+ "index": 9,
419+ "dtype": "int8",
420+ "format": "ND",
421+ "paramType": "optional",
422+ "shape": [
423+ -2
424+ ],
425+ "format_match_mode": "FormatDefault"
426+ }
427+ ],
428+ "outputs": [
429+ {
430+ "name": "y",
431+ "index": 0,
432+ "dtype": "bfloat16",
433+ "format": "ND",
434+ "paramType": "required",
435+ "shape": [
436+ -2
437+ ],
438+ "format_match_mode": "FormatDefault"
439+ }
440+ ],
441+ "attrs": [
442+ {
443+ "name": "dtype",
444+ "dtype": "int",
445+ "value": -1
446+ },
447+ {
448+ "name": "compute_type",
449+ "dtype": "int",
450+ "value": -1
451+ },
452+ {
453+ "name": "transpose_x1",
454+ "dtype": "bool",
455+ "value": false
456+ },
457+ {
458+ "name": "transpose_x2",
459+ "dtype": "bool",
460+ "value": false
461+ },
462+ {
463+ "name": "group_size",
464+ "dtype": "int",
465+ "value": -1
466+ }
467+ ]
468+ },
469+ {
470+ "bin_filename": "QuantBatchMatmulV4_ND_ND_int4_int4_bf16",
471+ "inputs": [
472+ {
473+ "name": "x1",
474+ "index": 0,
475+ "dtype": "int4",
476+ "format": "ND",
477+ "paramType": "required",
478+ "shape": [
479+ -2
480+ ],
481+ "format_match_mode": "FormatDefault"
482+ },
483+ {
484+ "name": "x2",
485+ "index": 1,
486+ "dtype": "int4",
487+ "format": "ND",
488+ "paramType": "required",
489+ "shape": [
490+ -2
491+ ],
492+ "format_match_mode": "FormatDefault"
493+ },
494+ {
495+ "name": "bias",
496+ "index": 2,
497+ "dtype": "bfloat16",
498+ "format": "ND",
499+ "paramType": "optional",
500+ "shape": [
501+ -2
502+ ],
503+ "format_match_mode": "FormatDefault"
504+ },
505+ {
506+ "name": "x1_scale",
507+ "index": 3,
508+ "dtype": "float32",
509+ "format": "ND",
510+ "paramType": "optional",
511+ "shape": [
512+ -2
513+ ],
514+ "format_match_mode": "FormatDefault"
515+ },
516+ {
517+ "name": "x2_scale",
518+ "index": 4,
519+ "dtype": "float32",
520+ "format": "ND",
521+ "paramType": "optional",
522+ "shape": [
523+ -2
524+ ],
525+ "format_match_mode": "FormatDefault"
526+ },
527+ {
528+ "name": "y_scale",
529+ "index": 5,
530+ "dtype": "uint64",
531+ "format": "ND",
532+ "paramType": "optional",
533+ "shape": [
534+ -2
535+ ],
536+ "format_match_mode": "FormatDefault"
537+ },
538+ {
539+ "name": "x1_offset",
540+ "index": 6,
541+ "dtype": "bfloat16",
542+ "format": "ND",
543+ "paramType": "optional",
544+ "shape": [
545+ -2
546+ ],
547+ "format_match_mode": "FormatDefault"
548+ },
549+ {
550+ "name": "x2_offset",
551+ "index": 7,
552+ "dtype": "float16",
553+ "format": "ND",
554+ "paramType": "optional",
555+ "shape": [
556+ -2
557+ ],
558+ "format_match_mode": "FormatDefault"
559+ },
560+ {
561+ "name": "y_offset",
562+ "index": 8,
563+ "dtype": "float32",
564+ "format": "ND",
565+ "paramType": "optional",
566+ "shape": [
567+ -2
568+ ],
569+ "format_match_mode": "FormatDefault"
570+ },
571+ {
572+ "name": "x2_table",
573+ "index": 9,
574+ "dtype": "int8",
575+ "format": "ND",
576+ "paramType": "optional",
577+ "shape": [
578+ -2
579+ ],
580+ "format_match_mode": "FormatDefault"
581+ }
582+ ],
583+ "outputs": [
584+ {
585+ "name": "y",
586+ "index": 0,
587+ "dtype": "bfloat16",
588+ "format": "ND",
589+ "paramType": "required",
590+ "shape": [
591+ -2
592+ ],
593+ "format_match_mode": "FormatDefault"
594+ }
595+ ],
596+ "attrs": [
597+ {
598+ "name": "dtype",
599+ "dtype": "int",
600+ "value": -1
601+ },
602+ {
603+ "name": "compute_type",
604+ "dtype": "int",
605+ "value": -1
606+ },
607+ {
608+ "name": "transpose_x1",
609+ "dtype": "bool",
610+ "value": false
611+ },
612+ {
613+ "name": "transpose_x2",
614+ "dtype": "bool",
615+ "value": true
616+ },
617+ {
618+ "name": "group_size",
619+ "dtype": "int",
620+ "value": -1
621+ }
622+ ]
623+ },
624+ {
625+ "bin_filename": "QuantBatchMatmulV4_ND_ND_int4_int4_fp16",
626+ "inputs": [
627+ {
628+ "name": "x1",
629+ "index": 0,
630+ "dtype": "int4",
631+ "format": "ND",
632+ "paramType": "required",
633+ "shape": [
634+ -2
635+ ],
636+ "format_match_mode": "FormatDefault"
637+ },
638+ {
639+ "name": "x2",
640+ "index": 1,
641+ "dtype": "int4",
642+ "format": "ND",
643+ "paramType": "required",
644+ "shape": [
645+ -2
646+ ],
647+ "format_match_mode": "FormatDefault"
648+ },
649+ {
650+ "name": "bias",
651+ "index": 2,
652+ "dtype": "bfloat16",
653+ "format": "ND",
654+ "paramType": "optional",
655+ "shape": [
656+ -2
657+ ],
658+ "format_match_mode": "FormatDefault"
659+ },
660+ {
661+ "name": "x1_scale",
662+ "index": 3,
663+ "dtype": "float32",
664+ "format": "ND",
665+ "paramType": "optional",
666+ "shape": [
667+ -2
668+ ],
669+ "format_match_mode": "FormatDefault"
670+ },
671+ {
672+ "name": "x2_scale",
673+ "index": 4,
674+ "dtype": "float32",
675+ "format": "ND",
676+ "paramType": "optional",
677+ "shape": [
678+ -2
679+ ],
680+ "format_match_mode": "FormatDefault"
681+ },
682+ {
683+ "name": "y_scale",
684+ "index": 5,
685+ "dtype": "uint64",
686+ "format": "ND",
687+ "paramType": "optional",
688+ "shape": [
689+ -2
690+ ],
691+ "format_match_mode": "FormatDefault"
692+ },
693+ {
694+ "name": "x1_offset",
695+ "index": 6,
696+ "dtype": "bfloat16",
697+ "format": "ND",
698+ "paramType": "optional",
699+ "shape": [
700+ -2
701+ ],
702+ "format_match_mode": "FormatDefault"
703+ },
704+ {
705+ "name": "x2_offset",
706+ "index": 7,
707+ "dtype": "float16",
708+ "format": "ND",
709+ "paramType": "optional",
710+ "shape": [
711+ -2
712+ ],
713+ "format_match_mode": "FormatDefault"
714+ },
715+ {
716+ "name": "y_offset",
717+ "index": 8,
718+ "dtype": "float32",
719+ "format": "ND",
720+ "paramType": "optional",
721+ "shape": [
722+ -2
723+ ],
724+ "format_match_mode": "FormatDefault"
725+ },
726+ {
727+ "name": "x2_table",
728+ "index": 9,
729+ "dtype": "int8",
730+ "format": "ND",
731+ "paramType": "optional",
732+ "shape": [
733+ -2
734+ ],
735+ "format_match_mode": "FormatDefault"
736+ }
737+ ],
738+ "outputs": [
739+ {
740+ "name": "y",
741+ "index": 0,
742+ "dtype": "float16",
743+ "format": "ND",
744+ "paramType": "required",
745+ "shape": [
746+ -2
747+ ],
748+ "format_match_mode": "FormatDefault"
749+ }
750+ ],
751+ "attrs": [
752+ {
753+ "name": "dtype",
754+ "dtype": "int",
755+ "value": -1
756+ },
757+ {
758+ "name": "compute_type",
759+ "dtype": "int",
760+ "value": -1
761+ },
762+ {
763+ "name": "transpose_x1",
764+ "dtype": "bool",
765+ "value": false
766+ },
767+ {
768+ "name": "transpose_x2",
769+ "dtype": "bool",
770+ "value": false
771+ },
772+ {
773+ "name": "group_size",
774+ "dtype": "int",
775+ "value": -1
776+ }
777+ ]
778+ },
779+ {
780+ "bin_filename": "QuantBatchMatmulV4_ND_ND_float8_e4m3fn_float4_e2m1_fp16_out_fp16",
781+ "inputs": [
782+ {
783+ "name": "x1",
784+ "index": 0,
785+ "dtype": "float8_e4m3fn",
786+ "format": "ND",
787+ "paramType": "required",
788+ "shape": [
789+ -2
790+ ],
791+ "format_match_mode": "FormatDefault"
792+ },
793+ {
794+ "name": "x2",
795+ "index": 1,
796+ "dtype": "float4_e2m1",
797+ "format": "ND",
798+ "paramType": "required",
799+ "shape": [
800+ -2
801+ ],
802+ "format_match_mode": "FormatDefault"
803+ },
804+ {
805+ "name": "bias",
806+ "index": 2,
807+ "dtype": "bfloat16",
808+ "format": "ND",
809+ "paramType": "optional",
810+ "shape": [
811+ -2
812+ ],
813+ "format_match_mode": "FormatDefault"
814+ },
815+ {
816+ "name": "x1_scale",
817+ "index": 3,
818+ "dtype": "bfloat16",
819+ "format": "ND",
820+ "paramType": "optional",
821+ "shape": [
822+ -2
823+ ],
824+ "format_match_mode": "FormatDefault"
825+ },
826+ {
827+ "name": "x2_scale",
828+ "index": 4,
829+ "dtype": "float16",
830+ "format": "ND",
831+ "paramType": "optional",
832+ "shape": [
833+ -2
834+ ],
835+ "format_match_mode": "FormatDefault"
836+ },
837+ {
838+ "name": "y_scale",
839+ "index": 5,
840+ "dtype": "uint64",
841+ "format": "ND",
842+ "paramType": "optional",
843+ "shape": [
844+ -2
845+ ],
846+ "format_match_mode": "FormatDefault"
847+ },
848+ {
849+ "name": "x1_offset",
850+ "index": 6,
851+ "dtype": "float16",
852+ "format": "ND",
853+ "paramType": "optional",
854+ "shape": [
855+ -2
856+ ],
857+ "format_match_mode": "FormatDefault"
858+ },
859+ {
860+ "name": "x2_offset",
861+ "index": 7,
862+ "dtype": "float16",
863+ "format": "ND",
864+ "paramType": "optional",
865+ "shape": [
866+ -2
867+ ],
868+ "format_match_mode": "FormatDefault"
869+ },
870+ {
871+ "name": "y_offset",
872+ "index": 8,
873+ "dtype": "float16",
874+ "format": "ND",
875+ "paramType": "optional",
876+ "shape": [
877+ -2
878+ ],
879+ "format_match_mode": "FormatDefault"
880+ },
881+ {
882+ "name": "x2_table",
883+ "index": 9,
884+ "dtype": "int8",
885+ "format": "ND",
886+ "paramType": "optional",
887+ "shape": [
888+ -2
889+ ],
890+ "format_match_mode": "FormatDefault"
891+ }
892+ ],
893+ "outputs": [
894+ {
895+ "name": "y",
896+ "index": 0,
897+ "dtype": "float16",
898+ "format": "ND",
899+ "paramType": "required",
900+ "shape": [
901+ -2
902+ ],
903+ "format_match_mode": "FormatDefault"
904+ }
905+ ],
906+ "attrs": [
907+ {
908+ "name": "dtype",
909+ "dtype": "int",
910+ "value": -1
911+ },
912+ {
913+ "name": "compute_type",
914+ "dtype": "int",
915+ "value": -1
916+ },
917+ {
918+ "name": "transpose_x1",
919+ "dtype": "bool",
920+ "value": false
921+ },
922+ {
923+ "name": "transpose_x2",
924+ "dtype": "bool",
925+ "value": false
926+ },
927+ {
928+ "name": "group_size",
929+ "dtype": "int",
930+ "value": -1
931+ }
932+ ]
933+ },
934+ {
935+ "bin_filename": "QuantBatchMatmulV4_ND_NZ_float8_e4m3fn_float4_e2m1_fp16_out_fp16",
936+ "inputs": [
937+ {
938+ "name": "x1",
939+ "index": 0,
940+ "dtype": "float8_e4m3fn",
941+ "format": "ND",
942+ "paramType": "required",
943+ "shape": [
944+ -2
945+ ],
946+ "format_match_mode": "FormatDefault"
947+ },
948+ {
949+ "name": "x2",
950+ "index": 1,
951+ "dtype": "float4_e2m1",
952+ "format": "FRACTAL_NZ",
953+ "paramType": "required",
954+ "shape": [
955+ -2
956+ ],
957+ "format_match_mode": "FormatDefault"
958+ },
959+ {
960+ "name": "bias",
961+ "index": 2,
962+ "dtype": "float32",
963+ "format": "ND",
964+ "paramType": "optional",
965+ "shape": [
966+ -2
967+ ],
968+ "format_match_mode": "FormatDefault"
969+ },
970+ {
971+ "name": "x1_scale",
972+ "index": 3,
973+ "dtype": "float32",
974+ "format": "ND",
975+ "paramType": "optional",
976+ "shape": [
977+ -2
978+ ],
979+ "format_match_mode": "FormatDefault"
980+ },
981+ {
982+ "name": "x2_scale",
983+ "index": 4,
984+ "dtype": "float16",
985+ "format": "ND",
986+ "paramType": "optional",
987+ "shape": [
988+ -2
989+ ],
990+ "format_match_mode": "FormatDefault"
991+ },
992+ {
993+ "name": "y_scale",
994+ "index": 5,
995+ "dtype": "uint64",
996+ "format": "ND",
997+ "paramType": "optional",
998+ "shape": [
999+ -2
1000+ ],
1001+ "format_match_mode": "FormatDefault"
1002+ },
1003+ {
1004+ "name": "x1_offset",
1005+ "index": 6,
1006+ "dtype": "float16",
1007+ "format": "ND",
1008+ "paramType": "optional",
1009+ "shape": [
1010+ -2
1011+ ],
1012+ "format_match_mode": "FormatDefault"
1013+ },
1014+ {
1015+ "name": "x2_offset",
1016+ "index": 7,
1017+ "dtype": "float16",
1018+ "format": "ND",
1019+ "paramType": "optional",
1020+ "shape": [
1021+ -2
1022+ ],
1023+ "format_match_mode": "FormatDefault"
1024+ },
1025+ {
1026+ "name": "y_offset",
1027+ "index": 8,
1028+ "dtype": "float16",
1029+ "format": "ND",
1030+ "paramType": "optional",
1031+ "shape": [
1032+ -2
1033+ ],
1034+ "format_match_mode": "FormatDefault"
1035+ },
1036+ {
1037+ "name": "x2_table",
1038+ "index": 9,
1039+ "dtype": "int8",
1040+ "format": "ND",
1041+ "paramType": "optional",
1042+ "shape": [
1043+ -2
1044+ ],
1045+ "format_match_mode": "FormatDefault"
1046+ }
1047+ ],
1048+ "outputs": [
1049+ {
1050+ "name": "y",
1051+ "index": 0,
1052+ "dtype": "float16",
1053+ "format": "ND",
1054+ "paramType": "required",
1055+ "shape": [
1056+ -2
1057+ ],
1058+ "format_match_mode": "FormatDefault"
1059+ }
1060+ ],
1061+ "attrs": [
1062+ {
1063+ "name": "dtype",
1064+ "dtype": "int",
1065+ "value": -1
1066+ },
1067+ {
1068+ "name": "compute_type",
1069+ "dtype": "int",
1070+ "value": -1
1071+ },
1072+ {
1073+ "name": "transpose_x1",
1074+ "dtype": "bool",
1075+ "value": false
1076+ },
1077+ {
1078+ "name": "transpose_x2",
1079+ "dtype": "bool",
1080+ "value": false
1081+ },
1082+ {
1083+ "name": "group_size",
1084+ "dtype": "int",
1085+ "value": -1
1086+ }
1087+ ]
1088+ },
1089+ {
1090+ "bin_filename": "QuantBatchMatmulV4_ND_NZ_MX_float8_e4m3fn_float4_e2m1_fp16_out_fp16",
1091+ "inputs": [
1092+ {
1093+ "name": "x1",
1094+ "index": 0,
1095+ "dtype": "float8_e4m3fn",
1096+ "format": "ND",
1097+ "paramType": "required",
1098+ "shape": [
1099+ -2
1100+ ],
1101+ "format_match_mode": "FormatDefault"
1102+ },
1103+ {
1104+ "name": "x2",
1105+ "index": 1,
1106+ "dtype": "float4_e2m1",
1107+ "format": "FRACTAL_NZ",
1108+ "paramType": "required",
1109+ "shape": [
1110+ -2
1111+ ],
1112+ "format_match_mode": "FormatDefault"
1113+ },
1114+ {
1115+ "name": "bias",
1116+ "index": 2,
1117+ "dtype": "float16",
1118+ "format": "ND",
1119+ "paramType": "optional",
1120+ "shape": [
1121+ -2
1122+ ],
1123+ "format_match_mode": "FormatDefault"
1124+ },
1125+ {
1126+ "name": "x1_scale",
1127+ "index": 3,
1128+ "dtype": "float8_e8m0",
1129+ "format": "ND",
1130+ "paramType": "optional",
1131+ "shape": [
1132+ -2
1133+ ],
1134+ "format_match_mode": "FormatDefault"
1135+ },
1136+ {
1137+ "name": "x2_scale",
1138+ "index": 4,
1139+ "dtype": "float8_e8m0",
1140+ "format": "ND",
1141+ "paramType": "optional",
1142+ "shape": [
1143+ -2
1144+ ],
1145+ "format_match_mode": "FormatDefault"
1146+ },
1147+ {
1148+ "name": "y_scale",
1149+ "index": 5,
1150+ "dtype": "uint64",
1151+ "format": "ND",
1152+ "paramType": "optional",
1153+ "shape": [
1154+ -2
1155+ ],
1156+ "format_match_mode": "FormatDefault"
1157+ },
1158+ {
1159+ "name": "x1_offset",
1160+ "index": 6,
1161+ "dtype": "float16",
1162+ "format": "ND",
1163+ "paramType": "optional",
1164+ "shape": [
1165+ -2
1166+ ],
1167+ "format_match_mode": "FormatDefault"
1168+ },
1169+ {
1170+ "name": "x2_offset",
1171+ "index": 7,
1172+ "dtype": "float16",
1173+ "format": "ND",
1174+ "paramType": "optional",
1175+ "shape": [
1176+ -2
1177+ ],
1178+ "format_match_mode": "FormatDefault"
1179+ },
1180+ {
1181+ "name": "y_offset",
1182+ "index": 8,
1183+ "dtype": "float16",
1184+ "format": "ND",
1185+ "paramType": "optional",
1186+ "shape": [
1187+ -2
1188+ ],
1189+ "format_match_mode": "FormatDefault"
1190+ },
1191+ {
1192+ "name": "x2_table",
1193+ "index": 9,
1194+ "dtype": "int8",
1195+ "format": "ND",
1196+ "paramType": "optional",
1197+ "shape": [
1198+ -2
1199+ ],
1200+ "format_match_mode": "FormatDefault"
1201+ }
1202+ ],
1203+ "outputs": [
1204+ {
1205+ "name": "y",
1206+ "index": 0,
1207+ "dtype": "float16",
1208+ "format": "ND",
1209+ "paramType": "required",
1210+ "shape": [
1211+ -2
1212+ ],
1213+ "format_match_mode": "FormatDefault"
1214+ }
1215+ ],
1216+ "attrs": [
1217+ {
1218+ "name": "dtype",
1219+ "dtype": "int",
1220+ "value": -1
1221+ },
1222+ {
1223+ "name": "compute_type",
1224+ "dtype": "int",
1225+ "value": -1
1226+ },
1227+ {
1228+ "name": "transpose_x1",
1229+ "dtype": "bool",
1230+ "value": false
1231+ },
1232+ {
1233+ "name": "transpose_x2",
1234+ "dtype": "bool",
1235+ "value": false
1236+ },
1237+ {
1238+ "name": "group_size",
1239+ "dtype": "int",
1240+ "value": -1
1241+ }
1242+ ]
1243+ },
1244+ {
1245+ "bin_filename": "QuantBatchMatmulV4_ND_ND_MX_float8_e4m3fn_float4_e2m1_fp16_out_fp16",
1246+ "inputs": [
1247+ {
1248+ "name": "x1",
1249+ "index": 0,
1250+ "dtype": "float8_e4m3fn",
1251+ "format": "ND",
1252+ "paramType": "required",
1253+ "shape": [
1254+ -2
1255+ ],
1256+ "format_match_mode": "FormatDefault"
1257+ },
1258+ {
1259+ "name": "x2",
1260+ "index": 1,
1261+ "dtype": "float4_e2m1",
1262+ "format": "ND",
1263+ "paramType": "required",
1264+ "shape": [
1265+ -2
1266+ ],
1267+ "format_match_mode": "FormatDefault"
1268+ },
1269+ {
1270+ "name": "bias",
1271+ "index": 2,
1272+ "dtype": "float16",
1273+ "format": "ND",
1274+ "paramType": "optional",
1275+ "shape": [
1276+ -2
1277+ ],
1278+ "format_match_mode": "FormatDefault"
1279+ },
1280+ {
1281+ "name": "x1_scale",
1282+ "index": 3,
1283+ "dtype": "float8_e8m0",
1284+ "format": "ND",
1285+ "paramType": "optional",
1286+ "shape": [
1287+ -2
1288+ ],
1289+ "format_match_mode": "FormatDefault"
1290+ },
1291+ {
1292+ "name": "x2_scale",
1293+ "index": 4,
1294+ "dtype": "float8_e8m0",
1295+ "format": "ND",
1296+ "paramType": "optional",
1297+ "shape": [
1298+ -2
1299+ ],
1300+ "format_match_mode": "FormatDefault"
1301+ },
1302+ {
1303+ "name": "y_scale",
1304+ "index": 5,
1305+ "dtype": "uint64",
1306+ "format": "ND",
1307+ "paramType": "optional",
1308+ "shape": [
1309+ -2
1310+ ],
1311+ "format_match_mode": "FormatDefault"
1312+ },
1313+ {
1314+ "name": "x1_offset",
1315+ "index": 6,
1316+ "dtype": "float16",
1317+ "format": "ND",
1318+ "paramType": "optional",
1319+ "shape": [
1320+ -2
1321+ ],
1322+ "format_match_mode": "FormatDefault"
1323+ },
1324+ {
1325+ "name": "x2_offset",
1326+ "index": 7,
1327+ "dtype": "float16",
1328+ "format": "ND",
1329+ "paramType": "optional",
1330+ "shape": [
1331+ -2
1332+ ],
1333+ "format_match_mode": "FormatDefault"
1334+ },
1335+ {
1336+ "name": "y_offset",
1337+ "index": 8,
1338+ "dtype": "float16",
1339+ "format": "ND",
1340+ "paramType": "optional",
1341+ "shape": [
1342+ -2
1343+ ],
1344+ "format_match_mode": "FormatDefault"
1345+ },
1346+ {
1347+ "name": "x2_table",
1348+ "index": 9,
1349+ "dtype": "int8",
1350+ "format": "ND",
1351+ "paramType": "optional",
1352+ "shape": [
1353+ -2
1354+ ],
1355+ "format_match_mode": "FormatDefault"
1356+ }
1357+ ],
1358+ "outputs": [
1359+ {
1360+ "name": "y",
1361+ "index": 0,
1362+ "dtype": "float16",
1363+ "format": "ND",
1364+ "paramType": "required",
1365+ "shape": [
1366+ -2
1367+ ],
1368+ "format_match_mode": "FormatDefault"
1369+ }
1370+ ],
1371+ "attrs": [
1372+ {
1373+ "name": "dtype",
1374+ "dtype": "int",
1375+ "value": -1
1376+ },
1377+ {
1378+ "name": "compute_type",
1379+ "dtype": "int",
1380+ "value": -1
1381+ },
1382+ {
1383+ "name": "transpose_x1",
1384+ "dtype": "bool",
1385+ "value": false
1386+ },
1387+ {
1388+ "name": "transpose_x2",
1389+ "dtype": "bool",
1390+ "value": false
1391+ },
1392+ {
1393+ "name": "group_size",
1394+ "dtype": "int",
1395+ "value": -1
1396+ }
1397+ ]
1398+ }
1399+ ]
1400+}
Mmatmul/quant_batch_matmul_v4/op_host/quant_batch_matmul_v4_def.cpp+1-0
@@ -989,6 +989,7 @@ public:
989 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");989 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
990 990 
991 this->AICore().AddConfig("ascend950", aicore_config);991 this->AICore().AddConfig("ascend950", aicore_config);
992+ this->AICore().AddConfig("ascend350", aicore_config);
992 993 
993 OpAICoreConfig config_910BC;994 OpAICoreConfig config_910BC;
994 config_910BC.Input("x1")995 config_910BC.Input("x1")
Mmatmul/quant_batch_matmul_v4/op_kernel/CMakeLists.txt+1-1
@@ -14,7 +14,7 @@ add_kernel_sources(
14 14 
15add_kernel_sources(15add_kernel_sources(
16 KERNEL_SRC arch35/quant_batch_matmul_v4.cpp16 KERNEL_SRC arch35/quant_batch_matmul_v4.cpp
17- COMPUTE_UNITS ascend95017+ COMPUTE_UNITS ascend950 ascend350
18 AUTO_SYNC false18 AUTO_SYNC false
19 OPTIONS "--cce-no-dcache-flush"19 OPTIONS "--cce-no-dcache-flush"
20)20)
Amatmul/sparse_tensor_dense_mat_mul/op_host/config/ascend350/sparse_tensor_dense_mat_mul_binary.json+419-0
@@ -0,0 +1,419 @@
1+{
2+ "op_type": "SparseTensorDenseMatMul",
3+ "op_list": [
4+ {
5+ "bin_filename": "SparseTensorDenseMatMul_int32_float16",
6+ "inputs": [
7+ {
8+ "name": "x1_indices",
9+ "index": 0,
10+ "dtype": "int32",
11+ "format": "ND",
12+ "paramType": "required",
13+ "shape": [
14+ -2
15+ ]
16+ },
17+ {
18+ "name": "x1_values",
19+ "index": 1,
20+ "dtype": "float16",
21+ "format": "ND",
22+ "paramType": "required",
23+ "shape": [
24+ -2
25+ ]
26+ },
27+ {
28+ "name": "x1_shape",
29+ "index": 2,
30+ "dtype": "int64",
31+ "format": "ND",
32+ "paramType": "required",
33+ "shape": [
34+ -2
35+ ]
36+ },
37+ {
38+ "name": "x2",
39+ "index": 3,
40+ "dtype": "float16",
41+ "format": "ND",
42+ "paramType": "required",
43+ "shape": [
44+ -2
45+ ]
46+ }
47+ ],
48+ "outputs": [
49+ {
50+ "name": "y",
51+ "index": 0,
52+ "dtype": "float16",
53+ "format": "ND",
54+ "paramType": "required",
55+ "shape": [
56+ -2
57+ ]
58+ }
59+ ],
60+ "attrs": [
61+ {
62+ "name": "adjoint_a",
63+ "dtype": "bool",
64+ "value": false
65+ },
66+ {
67+ "name": "adjoint_b",
68+ "dtype": "bool",
69+ "value": false
70+ }
71+ ]
72+ },
73+ {
74+ "bin_filename": "SparseTensorDenseMatMul_int32_float32",
75+ "inputs": [
76+ {
77+ "name": "x1_indices",
78+ "index": 0,
79+ "dtype": "int32",
80+ "format": "ND",
81+ "paramType": "required",
82+ "shape": [
83+ -2
84+ ]
85+ },
86+ {
87+ "name": "x1_values",
88+ "index": 1,
89+ "dtype": "float32",
90+ "format": "ND",
91+ "paramType": "required",
92+ "shape": [
93+ -2
94+ ]
95+ },
96+ {
97+ "name": "x1_shape",
98+ "index": 2,
99+ "dtype": "int64",
100+ "format": "ND",
101+ "paramType": "required",
102+ "shape": [
103+ -2
104+ ]
105+ },
106+ {
107+ "name": "x2",
108+ "index": 3,
109+ "dtype": "float32",
110+ "format": "ND",
111+ "paramType": "required",
112+ "shape": [
113+ -2
114+ ]
115+ }
116+ ],
117+ "outputs": [
118+ {
119+ "name": "y",
120+ "index": 0,
121+ "dtype": "float32",
122+ "format": "ND",
123+ "paramType": "required",
124+ "shape": [
125+ -2
126+ ]
127+ }
128+ ],
129+ "attrs": [
130+ {
131+ "name": "adjoint_a",
132+ "dtype": "bool",
133+ "value": false
134+ },
135+ {
136+ "name": "adjoint_b",
137+ "dtype": "bool",
138+ "value": false
139+ }
140+ ]
141+ },
142+ {
143+ "bin_filename": "SparseTensorDenseMatMul_int32_int32",
144+ "inputs": [
145+ {
146+ "name": "x1_indices",
147+ "index": 0,
148+ "dtype": "int32",
149+ "format": "ND",
150+ "paramType": "required",
151+ "shape": [
152+ -2
153+ ]
154+ },
155+ {
156+ "name": "x1_values",
157+ "index": 1,
158+ "dtype": "int32",
159+ "format": "ND",
160+ "paramType": "required",
161+ "shape": [
162+ -2
163+ ]
164+ },
165+ {
166+ "name": "x1_shape",
167+ "index": 2,
168+ "dtype": "int64",
169+ "format": "ND",
170+ "paramType": "required",
171+ "shape": [
172+ -2
173+ ]
174+ },
175+ {
176+ "name": "x2",
177+ "index": 3,
178+ "dtype": "int32",
179+ "format": "ND",
180+ "paramType": "required",
181+ "shape": [
182+ -2
183+ ]
184+ }
185+ ],
186+ "outputs": [
187+ {
188+ "name": "y",
189+ "index": 0,
190+ "dtype": "int32",
191+ "format": "ND",
192+ "paramType": "required",
193+ "shape": [
194+ -2
195+ ]
196+ }
197+ ],
198+ "attrs": [
199+ {
200+ "name": "adjoint_a",
201+ "dtype": "bool",
202+ "value": false
203+ },
204+ {
205+ "name": "adjoint_b",
206+ "dtype": "bool",
207+ "value": false
208+ }
209+ ]
210+ },
211+ {
212+ "bin_filename": "SparseTensorDenseMatMul_int64_float16",
213+ "inputs": [
214+ {
215+ "name": "x1_indices",
216+ "index": 0,
217+ "dtype": "int64",
218+ "format": "ND",
219+ "paramType": "required",
220+ "shape": [
221+ -2
222+ ]
223+ },
224+ {
225+ "name": "x1_values",
226+ "index": 1,
227+ "dtype": "float16",
228+ "format": "ND",
229+ "paramType": "required",
230+ "shape": [
231+ -2
232+ ]
233+ },
234+ {
235+ "name": "x1_shape",
236+ "index": 2,
237+ "dtype": "int64",
238+ "format": "ND",
239+ "paramType": "required",
240+ "shape": [
241+ -2
242+ ]
243+ },
244+ {
245+ "name": "x2",
246+ "index": 3,
247+ "dtype": "float16",
248+ "format": "ND",
249+ "paramType": "required",
250+ "shape": [
251+ -2
252+ ]
253+ }
254+ ],
255+ "outputs": [
256+ {
257+ "name": "y",
258+ "index": 0,
259+ "dtype": "float16",
260+ "format": "ND",
261+ "paramType": "required",
262+ "shape": [
263+ -2
264+ ]
265+ }
266+ ],
267+ "attrs": [
268+ {
269+ "name": "adjoint_a",
270+ "dtype": "bool",
271+ "value": false
272+ },
273+ {
274+ "name": "adjoint_b",
275+ "dtype": "bool",
276+ "value": false
277+ }
278+ ]
279+ },
280+ {
281+ "bin_filename": "SparseTensorDenseMatMul_int64_float32",
282+ "inputs": [
283+ {
284+ "name": "x1_indices",
285+ "index": 0,
286+ "dtype": "int64",
287+ "format": "ND",
288+ "paramType": "required",
289+ "shape": [
290+ -2
291+ ]
292+ },
293+ {
294+ "name": "x1_values",
295+ "index": 1,
296+ "dtype": "float32",
297+ "format": "ND",
298+ "paramType": "required",
299+ "shape": [
300+ -2
301+ ]
302+ },
303+ {
304+ "name": "x1_shape",
305+ "index": 2,
306+ "dtype": "int64",
307+ "format": "ND",
308+ "paramType": "required",
309+ "shape": [
310+ -2
311+ ]
312+ },
313+ {
314+ "name": "x2",
315+ "index": 3,
316+ "dtype": "float32",
317+ "format": "ND",
318+ "paramType": "required",
319+ "shape": [
320+ -2
321+ ]
322+ }
323+ ],
324+ "outputs": [
325+ {
326+ "name": "y",
327+ "index": 0,
328+ "dtype": "float32",
329+ "format": "ND",
330+ "paramType": "required",
331+ "shape": [
332+ -2
333+ ]
334+ }
335+ ],
336+ "attrs": [
337+ {
338+ "name": "adjoint_a",
339+ "dtype": "bool",
340+ "value": false
341+ },
342+ {
343+ "name": "adjoint_b",
344+ "dtype": "bool",
345+ "value": false
346+ }
347+ ]
348+ },
349+ {
350+ "bin_filename": "SparseTensorDenseMatMul_int64_int32",
351+ "inputs": [
352+ {
353+ "name": "x1_indices",
354+ "index": 0,
355+ "dtype": "int64",
356+ "format": "ND",
357+ "paramType": "required",
358+ "shape": [
359+ -2
360+ ]
361+ },
362+ {
363+ "name": "x1_values",
364+ "index": 1,
365+ "dtype": "int32",
366+ "format": "ND",
367+ "paramType": "required",
368+ "shape": [
369+ -2
370+ ]
371+ },
372+ {
373+ "name": "x1_shape",
374+ "index": 2,
375+ "dtype": "int64",
376+ "format": "ND",
377+ "paramType": "required",
378+ "shape": [
379+ -2
380+ ]
381+ },
382+ {
383+ "name": "x2",
384+ "index": 3,
385+ "dtype": "int32",
386+ "format": "ND",
387+ "paramType": "required",
388+ "shape": [
389+ -2
390+ ]
391+ }
392+ ],
393+ "outputs": [
394+ {
395+ "name": "y",
396+ "index": 0,
397+ "dtype": "int32",
398+ "format": "ND",
399+ "paramType": "required",
400+ "shape": [
401+ -2
402+ ]
403+ }
404+ ],
405+ "attrs": [
406+ {
407+ "name": "adjoint_a",
408+ "dtype": "bool",
409+ "value": false
410+ },
411+ {
412+ "name": "adjoint_b",
413+ "dtype": "bool",
414+ "value": false
415+ }
416+ ]
417+ }
418+ ]
419+}
Mmatmul/sparse_tensor_dense_mat_mul/op_host/sparse_tensor_dense_mat_mul_def.cpp+1-0
@@ -76,6 +76,7 @@ public:
76 .DynamicShapeSupportFlag(true)76 .DynamicShapeSupportFlag(true)
77 .NeedCheckSupportFlag(true);77 .NeedCheckSupportFlag(true);
78 this->AICore().AddConfig("ascend950", config);78 this->AICore().AddConfig("ascend950", config);
79+ this->AICore().AddConfig("ascend350", config);
79 }80 }
80};81};
81 82 
Mmatmul/sparse_tensor_dense_mat_mul/op_kernel/CMakeLists.txt+1-1
@@ -14,7 +14,7 @@ add_kernel_sources(
14 14 
15add_kernel_sources(15add_kernel_sources(
16 KERNEL_SRC arch35/sparse_tensor_dense_mat_mul.cpp16 KERNEL_SRC arch35/sparse_tensor_dense_mat_mul.cpp
17- COMPUTE_UNITS ascend95017+ COMPUTE_UNITS ascend950 ascend350
18 AUTO_SYNC false18 AUTO_SYNC false
19 OPTIONS "--cce-no-dcache-flush"19 OPTIONS "--cce-no-dcache-flush"
20)20)
Amatmul/transpose_batch_mat_mul/op_host/config/ascend350/transpose_batch_mat_mul_binary.json+771-0
@@ -0,0 +1,771 @@
1+{
2+ "op_type": "TransposeBatchMatMul",
3+ "optional_input_mode": "gen_placeholder",
4+ "op_list": [
5+ {
6+ "bin_filename": "TransposeBatchMatMul_ND_NZ_ND_ND_ND_FP16_FP16_FP16_UINT64_INT8",
7+ "simplified_key": "diy,2/29/2/2/2/1/1/1/10/2",
8+ "inputs": [
9+ {
10+ "name": "x1",
11+ "index": 0,
12+ "dtype": "float16",
13+ "format": "ND",
14+ "paramType": "required",
15+ "shape": [
16+ -2
17+ ]
18+ },
19+ {
20+ "name": "x2",
21+ "index": 1,
22+ "dtype": "float16",
23+ "format": "FRACTAL_NZ",
24+ "paramType": "required",
25+ "shape": [
26+ -2
27+ ]
28+ },
29+ {
30+ "name": "bias",
31+ "index": 2,
32+ "dtype": "float16",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "scale",
41+ "index": 3,
42+ "dtype": "uint64",
43+ "format": "ND",
44+ "paramType": "optional",
45+ "shape": [
46+ -2
47+ ]
48+ }
49+ ],
50+ "outputs": [
51+ {
52+ "name": "y",
53+ "index": 0,
54+ "dtype": "int8",
55+ "format": "ND",
56+ "paramType": "required",
57+ "shape": [
58+ -2
59+ ]
60+ }
61+ ],
62+ "attrs": [
63+ {
64+ "name": "perm_x1",
65+ "dtype": "list_int",
66+ "value": [1,0,2]
67+ },
68+ {
69+ "name": "perm_x2",
70+ "dtype": "list_int",
71+ "value": [0,1,2]
72+ },
73+ {
74+ "name": "perm_y",
75+ "dtype": "list_int",
76+ "value": [1,0,2]
77+ },
78+ {
79+ "name": "enable_hf32",
80+ "dtype": "bool",
81+ "value": false
82+ },
83+ {
84+ "name": "batch_split_factor",
85+ "dtype": "int",
86+ "value": 1
87+ }
88+ ]
89+ },
90+ {
91+ "bin_filename": "TransposeBatchMatMul_ND_NZ_ND_ND_ND_FP16_FP16_FP16_INT64_INT8",
92+ "simplified_key": "diy,2/29/2/2/2/1/1/1/9/2",
93+ "inputs": [
94+ {
95+ "name": "x1",
96+ "index": 0,
97+ "dtype": "float16",
98+ "format": "ND",
99+ "paramType": "required",
100+ "shape": [
101+ -2
102+ ]
103+ },
104+ {
105+ "name": "x2",
106+ "index": 1,
107+ "dtype": "float16",
108+ "format": "FRACTAL_NZ",
109+ "paramType": "required",
110+ "shape": [
111+ -2
112+ ]
113+ },
114+ {
115+ "name": "bias",
116+ "index": 2,
117+ "dtype": "float16",
118+ "format": "ND",
119+ "paramType": "optional",
120+ "shape": [
121+ -2
122+ ]
123+ },
124+ {
125+ "name": "scale",
126+ "index": 3,
127+ "dtype": "int64",
128+ "format": "ND",
129+ "paramType": "optional",
130+ "shape": [
131+ -2
132+ ]
133+ }
134+ ],
135+ "outputs": [
136+ {
137+ "name": "y",
138+ "index": 0,
139+ "dtype": "int8",
140+ "format": "ND",
141+ "paramType": "required",
142+ "shape": [
143+ -2
144+ ]
145+ }
146+ ],
147+ "attrs": [
148+ {
149+ "name": "perm_x1",
150+ "dtype": "list_int",
151+ "value": [1,0,2]
152+ },
153+ {
154+ "name": "perm_x2",
155+ "dtype": "list_int",
156+ "value": [0,1,2]
157+ },
158+ {
159+ "name": "perm_y",
160+ "dtype": "list_int",
161+ "value": [1,0,2]
162+ },
163+ {
164+ "name": "enable_hf32",
165+ "dtype": "bool",
166+ "value": false
167+ },
168+ {
169+ "name": "batch_split_factor",
170+ "dtype": "int",
171+ "value": 1
172+ }
173+ ]
174+ },
175+ {
176+ "bin_filename": "TransposeBatchMatMul_ND_NZ_ND_ND_ND_FP16_FP16_FP16_INT64_FP16",
177+ "simplified_key": "diy,2/29/2/2/2/1/1/1/9/1",
178+ "inputs": [
179+ {
180+ "name": "x1",
181+ "index": 0,
182+ "dtype": "float16",
183+ "format": "ND",
184+ "paramType": "required",
185+ "shape": [
186+ -2
187+ ]
188+ },
189+ {
190+ "name": "x2",
191+ "index": 1,
192+ "dtype": "float16",
193+ "format": "FRACTAL_NZ",
194+ "paramType": "required",
195+ "shape": [
196+ -2
197+ ]
198+ },
199+ {
200+ "name": "bias",
201+ "index": 2,
202+ "dtype": "float16",
203+ "format": "ND",
204+ "paramType": "optional",
205+ "shape": [
206+ -2
207+ ]
208+ },
209+ {
210+ "name": "scale",
211+ "index": 3,
212+ "dtype": "int64",
213+ "format": "ND",
214+ "paramType": "optional",
215+ "shape": [
216+ -2
217+ ]
218+ }
219+ ],
220+ "outputs": [
221+ {
222+ "name": "y",
223+ "index": 0,
224+ "dtype": "float16",
225+ "format": "ND",
226+ "paramType": "required",
227+ "shape": [
228+ -2
229+ ]
230+ }
231+ ],
232+ "attrs": [
233+ {
234+ "name": "perm_x1",
235+ "dtype": "list_int",
236+ "value": [1,0,2]
237+ },
238+ {
239+ "name": "perm_x2",
240+ "dtype": "list_int",
241+ "value": [0,1,2]
242+ },
243+ {
244+ "name": "perm_y",
245+ "dtype": "list_int",
246+ "value": [1,0,2]
247+ },
248+ {
249+ "name": "enable_hf32",
250+ "dtype": "bool",
251+ "value": false
252+ },
253+ {
254+ "name": "batch_split_factor",
255+ "dtype": "int",
256+ "value": 1
257+ }
258+ ]
259+ },
260+ {
261+ "bin_filename": "TransposeBatchMatMul_ND_NZ_ND_ND_ND_BF16_BF16_BF16_INT64_BF16",
262+ "simplified_key": "diy,2/29/2/2/2/27/27/27/9/27",
263+ "inputs": [
264+ {
265+ "name": "x1",
266+ "index": 0,
267+ "dtype": "bfloat16",
268+ "format": "ND",
269+ "paramType": "required",
270+ "shape": [
271+ -2
272+ ]
273+ },
274+ {
275+ "name": "x2",
276+ "index": 1,
277+ "dtype": "bfloat16",
278+ "format": "FRACTAL_NZ",
279+ "paramType": "required",
280+ "shape": [
281+ -2
282+ ]
283+ },
284+ {
285+ "name": "bias",
286+ "index": 2,
287+ "dtype": "bfloat16",
288+ "format": "ND",
289+ "paramType": "optional",
290+ "shape": [
291+ -2
292+ ]
293+ },
294+ {
295+ "name": "scale",
296+ "index": 3,
297+ "dtype": "int64",
298+ "format": "ND",
299+ "paramType": "optional",
300+ "shape": [
301+ -2
302+ ]
303+ }
304+ ],
305+ "outputs": [
306+ {
307+ "name": "y",
308+ "index": 0,
309+ "dtype": "bfloat16",
310+ "format": "ND",
311+ "paramType": "required",
312+ "shape": [
313+ -2
314+ ]
315+ }
316+ ],
317+ "attrs": [
318+ {
319+ "name": "perm_x1",
320+ "dtype": "list_int",
321+ "value": [1,0,2]
322+ },
323+ {
324+ "name": "perm_x2",
325+ "dtype": "list_int",
326+ "value": [0,1,2]
327+ },
328+ {
329+ "name": "perm_y",
330+ "dtype": "list_int",
331+ "value": [1,0,2]
332+ },
333+ {
334+ "name": "enable_hf32",
335+ "dtype": "bool",
336+ "value": false
337+ },
338+ {
339+ "name": "batch_split_factor",
340+ "dtype": "int",
341+ "value": 1
342+ }
343+ ]
344+ },
345+ {
346+ "bin_filename": "TransposeBatchMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP16_INT64_FP16",
347+ "simplified_key": "diy,2/2/2/2/2/1/1/1/9/1",
348+ "inputs": [
349+ {
350+ "name": "x1",
351+ "index": 0,
352+ "dtype": "float16",
353+ "format": "ND",
354+ "paramType": "required",
355+ "shape": [
356+ -2
357+ ]
358+ },
359+ {
360+ "name": "x2",
361+ "index": 1,
362+ "dtype": "float16",
363+ "format": "ND",
364+ "paramType": "required",
365+ "shape": [
366+ -2
367+ ]
368+ },
369+ {
370+ "name": "bias",
371+ "index": 2,
372+ "dtype": "float16",
373+ "format": "ND",
374+ "paramType": "optional",
375+ "shape": [
376+ -2
377+ ]
378+ },
379+ {
380+ "name": "scale",
381+ "index": 3,
382+ "dtype": "int64",
383+ "format": "ND",
384+ "paramType": "optional",
385+ "shape": [
386+ -2
387+ ]
388+ }
389+ ],
390+ "outputs": [
391+ {
392+ "name": "y",
393+ "index": 0,
394+ "dtype": "float16",
395+ "format": "ND",
396+ "paramType": "required",
397+ "shape": [
398+ -2
399+ ]
400+ }
401+ ],
402+ "attrs": [
403+ {
404+ "name": "perm_x1",
405+ "dtype": "list_int",
406+ "value": [1,0,2]
407+ },
408+ {
409+ "name": "perm_x2",
410+ "dtype": "list_int",
411+ "value": [0,1,2]
412+ },
413+ {
414+ "name": "perm_y",
415+ "dtype": "list_int",
416+ "value": [1,0,2]
417+ },
418+ {
419+ "name": "enable_hf32",
420+ "dtype": "bool",
421+ "value": false
422+ },
423+ {
424+ "name": "batch_split_factor",
425+ "dtype": "int",
426+ "value": 1
427+ }
428+ ]
429+ },
430+ {
431+ "bin_filename": "TransposeBatchMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP16_UINT64_INT8",
432+ "simplified_key": "diy,2/2/2/2/2/1/1/1/10/2",
433+ "inputs": [
434+ {
435+ "name": "x1",
436+ "index": 0,
437+ "dtype": "float16",
438+ "format": "ND",
439+ "paramType": "required",
440+ "shape": [
441+ -2
442+ ]
443+ },
444+ {
445+ "name": "x2",
446+ "index": 1,
447+ "dtype": "float16",
448+ "format": "ND",
449+ "paramType": "required",
450+ "shape": [
451+ -2
452+ ]
453+ },
454+ {
455+ "name": "bias",
456+ "index": 2,
457+ "dtype": "float16",
458+ "format": "ND",
459+ "paramType": "optional",
460+ "shape": [
461+ -2
462+ ]
463+ },
464+ {
465+ "name": "scale",
466+ "index": 3,
467+ "dtype": "uint64",
468+ "format": "ND",
469+ "paramType": "optional",
470+ "shape": [
471+ -2
472+ ]
473+ }
474+ ],
475+ "outputs": [
476+ {
477+ "name": "y",
478+ "index": 0,
479+ "dtype": "int8",
480+ "format": "ND",
481+ "paramType": "required",
482+ "shape": [
483+ -2
484+ ]
485+ }
486+ ],
487+ "attrs": [
488+ {
489+ "name": "perm_x1",
490+ "dtype": "list_int",
491+ "value": [1,0,2]
492+ },
493+ {
494+ "name": "perm_x2",
495+ "dtype": "list_int",
496+ "value": [0,1,2]
497+ },
498+ {
499+ "name": "perm_y",
500+ "dtype": "list_int",
501+ "value": [1,0,2]
502+ },
503+ {
504+ "name": "enable_hf32",
505+ "dtype": "bool",
506+ "value": false
507+ },
508+ {
509+ "name": "batch_split_factor",
510+ "dtype": "int",
511+ "value": 1
512+ }
513+ ]
514+ },
515+ {
516+ "bin_filename": "TransposeBatchMatMul_ND_ND_ND_ND_ND_FP16_FP16_FP16_INT64_INT8",
517+ "simplified_key": "diy,2/2/2/2/2/1/1/1/9/2",
518+ "inputs": [
519+ {
520+ "name": "x1",
521+ "index": 0,
522+ "dtype": "float16",
523+ "format": "ND",
524+ "paramType": "required",
525+ "shape": [
526+ -2
527+ ]
528+ },
529+ {
530+ "name": "x2",
531+ "index": 1,
532+ "dtype": "float16",
533+ "format": "ND",
534+ "paramType": "required",
535+ "shape": [
536+ -2
537+ ]
538+ },
539+ {
540+ "name": "bias",
541+ "index": 2,
542+ "dtype": "float16",
543+ "format": "ND",
544+ "paramType": "optional",
545+ "shape": [
546+ -2
547+ ]
548+ },
549+ {
550+ "name": "scale",
551+ "index": 3,
552+ "dtype": "int64",
553+ "format": "ND",
554+ "paramType": "optional",
555+ "shape": [
556+ -2
557+ ]
558+ }
559+ ],
560+ "outputs": [
561+ {
562+ "name": "y",
563+ "index": 0,
564+ "dtype": "int8",
565+ "format": "ND",
566+ "paramType": "required",
567+ "shape": [
568+ -2
569+ ]
570+ }
571+ ],
572+ "attrs": [
573+ {
574+ "name": "perm_x1",
575+ "dtype": "list_int",
576+ "value": [1,0,2]
577+ },
578+ {
579+ "name": "perm_x2",
580+ "dtype": "list_int",
581+ "value": [0,1,2]
582+ },
583+ {
584+ "name": "perm_y",
585+ "dtype": "list_int",
586+ "value": [1,0,2]
587+ },
588+ {
589+ "name": "enable_hf32",
590+ "dtype": "bool",
591+ "value": false
592+ },
593+ {
594+ "name": "batch_split_factor",
595+ "dtype": "int",
596+ "value": 1
597+ }
598+ ]
599+ },
600+ {
601+ "bin_filename": "TransposeBatchMatMul_ND_ND_ND_ND_ND_BF16_BF16_BF16_INT64_BF16",
602+ "simplified_key": "diy,2/2/2/2/2/27/27/27/9/27",
603+ "inputs": [
604+ {
605+ "name": "x1",
606+ "index": 0,
607+ "dtype": "bfloat16",
608+ "format": "ND",
609+ "paramType": "required",
610+ "shape": [
611+ -2
612+ ]
613+ },
614+ {
615+ "name": "x2",
616+ "index": 1,
617+ "dtype": "bfloat16",
618+ "format": "ND",
619+ "paramType": "required",
620+ "shape": [
621+ -2
622+ ]
623+ },
624+ {
625+ "name": "bias",
626+ "index": 2,
627+ "dtype": "bfloat16",
628+ "format": "ND",
629+ "paramType": "optional",
630+ "shape": [
631+ -2
632+ ]
633+ },
634+ {
635+ "name": "scale",
636+ "index": 3,
637+ "dtype": "int64",
638+ "format": "ND",
639+ "paramType": "optional",
640+ "shape": [
641+ -2
642+ ]
643+ }
644+ ],
645+ "outputs": [
646+ {
647+ "name": "y",
648+ "index": 0,
649+ "dtype": "bfloat16",
650+ "format": "ND",
651+ "paramType": "required",
652+ "shape": [
653+ -2
654+ ]
655+ }
656+ ],
657+ "attrs": [
658+ {
659+ "name": "perm_x1",
660+ "dtype": "list_int",
661+ "value": [1,0,2]
662+ },
663+ {
664+ "name": "perm_x2",
665+ "dtype": "list_int",
666+ "value": [0,1,2]
667+ },
668+ {
669+ "name": "perm_y",
670+ "dtype": "list_int",
671+ "value": [1,0,2]
672+ },
673+ {
674+ "name": "enable_hf32",
675+ "dtype": "bool",
676+ "value": false
677+ },
678+ {
679+ "name": "batch_split_factor",
680+ "dtype": "int",
681+ "value": 1
682+ }
683+ ]
684+ },
685+ {
686+ "bin_filename": "TransposeBatchMatMul_ND_ND_ND_ND_ND_FP32_FP32_FP32_INT64_FP32",
687+ "simplified_key": "diy,2/2/2/2/2/0/0/0/9/0",
688+ "inputs": [
689+ {
690+ "name": "x1",
691+ "index": 0,
692+ "dtype": "float32",
693+ "format": "ND",
694+ "paramType": "required",
695+ "shape": [
696+ -2
697+ ]
698+ },
699+ {
700+ "name": "x2",
701+ "index": 1,
702+ "dtype": "float32",
703+ "format": "ND",
704+ "paramType": "required",
705+ "shape": [
706+ -2
707+ ]
708+ },
709+ {
710+ "name": "bias",
711+ "index": 2,
712+ "dtype": "float32",
713+ "format": "ND",
714+ "paramType": "optional",
715+ "shape": [
716+ -2
717+ ]
718+ },
719+ {
720+ "name": "scale",
721+ "index": 3,
722+ "dtype": "int64",
723+ "format": "ND",
724+ "paramType": "optional",
725+ "shape": [
726+ -2
727+ ]
728+ }
729+ ],
730+ "outputs": [
731+ {
732+ "name": "y",
733+ "index": 0,
734+ "dtype": "float32",
735+ "format": "ND",
736+ "paramType": "required",
737+ "shape": [
738+ -2
739+ ]
740+ }
741+ ],
742+ "attrs": [
743+ {
744+ "name": "perm_x1",
745+ "dtype": "list_int",
746+ "value": [1,0,2]
747+ },
748+ {
749+ "name": "perm_x2",
750+ "dtype": "list_int",
751+ "value": [0,1,2]
752+ },
753+ {
754+ "name": "perm_y",
755+ "dtype": "list_int",
756+ "value": [1,0,2]
757+ },
758+ {
759+ "name": "enable_hf32",
760+ "dtype": "bool",
761+ "value": false
762+ },
763+ {
764+ "name": "batch_split_factor",
765+ "dtype": "int",
766+ "value": 1
767+ }
768+ ]
769+ }
770+ ]
771+}
Mmatmul/transpose_batch_mat_mul/op_host/transpose_batch_mat_mul_def.cpp+1-0
@@ -95,6 +95,7 @@ public:
95 .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});95 .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
96 aicConfig.ExtendCfgInfo("opFile.value", "transpose_batch_mat_mul");96 aicConfig.ExtendCfgInfo("opFile.value", "transpose_batch_mat_mul");
97 this->AICore().AddConfig("ascend950", aicConfig);97 this->AICore().AddConfig("ascend950", aicConfig);
98+ this->AICore().AddConfig("ascend350", aicConfig);
98 this->AICore().AddConfig("ascend910_55", aicConfig);99 this->AICore().AddConfig("ascend910_55", aicConfig);
99 100 
100 OpAICoreConfig config_kirin = GetKirinCoreConfig();101 OpAICoreConfig config_kirin = GetKirinCoreConfig();
Mmatmul/transpose_batch_mat_mul/op_kernel/CMakeLists.txt+1-1
@@ -14,7 +14,7 @@ add_kernel_sources(
14 14 
15add_kernel_sources(15add_kernel_sources(
16 KERNEL_SRC arch35/transpose_batch_mat_mul.cpp16 KERNEL_SRC arch35/transpose_batch_mat_mul.cpp
17- COMPUTE_UNITS ascend95017+ COMPUTE_UNITS ascend950 ascend350
18 AUTO_SYNC false18 AUTO_SYNC false
19 OPTIONS "--cce-no-dcache-flush"19 OPTIONS "--cce-no-dcache-flush"
20)20)
Amatmul/transpose_quant_batch_mat_mul/op_host/config/ascend350/transpose_quant_batch_mat_mul_binary.json+1806-0
@@ -0,0 +1,1806 @@
1+{
2+ "op_type": "TransposeQuantBatchMatMul",
3+ "optional_input_mode": "gen_placeholder",
4+ "op_list": [
5+ {
6+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E5M2_FP16_FP32_FP32_FP16",
7+ "simplified_key": "diy,2/2/2/2/2/2/35/35/1/0/0/1",
8+ "inputs": [
9+ {
10+ "name": "x1",
11+ "index": 0,
12+ "dtype": "float8_e5m2",
13+ "format": "ND",
14+ "paramType": "required",
15+ "shape": [
16+ -2
17+ ]
18+ },
19+ {
20+ "name": "x2",
21+ "index": 1,
22+ "dtype": "float8_e5m2",
23+ "format": "ND",
24+ "paramType": "required",
25+ "shape": [
26+ -2
27+ ]
28+ },
29+ {
30+ "name": "bias",
31+ "index": 2,
32+ "dtype": "float16",
33+ "format": "ND",
34+ "paramType": "optional",
35+ "shape": [
36+ -2
37+ ]
38+ },
39+ {
40+ "name": "x1_scale",
41+ "index": 3,
42+ "dtype": "float32",
43+ "format": "ND",
44+ "paramType": "optional",
45+ "shape": [
46+ -2
47+ ]
48+ },
49+ {
50+ "name": "x2_scale",
51+ "index": 4,
52+ "dtype": "float32",
53+ "format": "ND",
54+ "paramType": "optional",
55+ "shape": [
56+ -2
57+ ]
58+ }
59+ ],
60+ "outputs": [
61+ {
62+ "name": "y",
63+ "index": 0,
64+ "dtype": "float16",
65+ "format": "ND",
66+ "paramType": "required",
67+ "shape": [
68+ -2
69+ ]
70+ }
71+ ],
72+ "attrs": [
73+ {
74+ "name": "dtype",
75+ "dtype": "int",
76+ "value": 1
77+ },
78+ {
79+ "name": "group_size",
80+ "dtype": "int",
81+ "value": 0
82+ },
83+ {
84+ "name": "perm_x1",
85+ "dtype": "list_int",
86+ "value": [1,0,2]
87+ },
88+ {
89+ "name": "perm_x2",
90+ "dtype": "list_int",
91+ "value": [0,1,2]
92+ },
93+ {
94+ "name": "perm_y",
95+ "dtype": "list_int",
96+ "value": [1,0,2]
97+ },
98+ {
99+ "name": "batch_split_factor",
100+ "dtype": "int",
101+ "value": 1
102+ }
103+ ]
104+ },
105+ {
106+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E5M2_BF16_FP32_FP32_BF16",
107+ "simplified_key": "diy,2/2/2/2/2/2/35/35/27/0/0/27",
108+ "inputs": [
109+ {
110+ "name": "x1",
111+ "index": 0,
112+ "dtype": "float8_e5m2",
113+ "format": "ND",
114+ "paramType": "required",
115+ "shape": [
116+ -2
117+ ]
118+ },
119+ {
120+ "name": "x2",
121+ "index": 1,
122+ "dtype": "float8_e5m2",
123+ "format": "ND",
124+ "paramType": "required",
125+ "shape": [
126+ -2
127+ ]
128+ },
129+ {
130+ "name": "bias",
131+ "index": 2,
132+ "dtype": "bfloat16",
133+ "format": "ND",
134+ "paramType": "optional",
135+ "shape": [
136+ -2
137+ ]
138+ },
139+ {
140+ "name": "x1_scale",
141+ "index": 3,
142+ "dtype": "float32",
143+ "format": "ND",
144+ "paramType": "optional",
145+ "shape": [
146+ -2
147+ ]
148+ },
149+ {
150+ "name": "x2_scale",
151+ "index": 4,
152+ "dtype": "float32",
153+ "format": "ND",
154+ "paramType": "optional",
155+ "shape": [
156+ -2
157+ ]
158+ }
159+ ],
160+ "outputs": [
161+ {
162+ "name": "y",
163+ "index": 0,
164+ "dtype": "bfloat16",
165+ "format": "ND",
166+ "paramType": "required",
167+ "shape": [
168+ -2
169+ ]
170+ }
171+ ],
172+ "attrs": [
173+ {
174+ "name": "dtype",
175+ "dtype": "int",
176+ "value": 27
177+ },
178+ {
179+ "name": "group_size",
180+ "dtype": "int",
181+ "value": 0
182+ },
183+ {
184+ "name": "perm_x1",
185+ "dtype": "list_int",
186+ "value": [1,0,2]
187+ },
188+ {
189+ "name": "perm_x2",
190+ "dtype": "list_int",
191+ "value": [0,1,2]
192+ },
193+ {
194+ "name": "perm_y",
195+ "dtype": "list_int",
196+ "value": [1,0,2]
197+ },
198+ {
199+ "name": "batch_split_factor",
200+ "dtype": "int",
201+ "value": 1
202+ }
203+ ]
204+ },
205+ {
206+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E5M2_FP32_FP32_FP32_FP16",
207+ "simplified_key": "diy,2/2/2/2/2/2/35/35/0/0/0/1",
208+ "inputs": [
209+ {
210+ "name": "x1",
211+ "index": 0,
212+ "dtype": "float8_e5m2",
213+ "format": "ND",
214+ "paramType": "required",
215+ "shape": [
216+ -2
217+ ]
218+ },
219+ {
220+ "name": "x2",
221+ "index": 1,
222+ "dtype": "float8_e5m2",
223+ "format": "ND",
224+ "paramType": "required",
225+ "shape": [
226+ -2
227+ ]
228+ },
229+ {
230+ "name": "bias",
231+ "index": 2,
232+ "dtype": "float32",
233+ "format": "ND",
234+ "paramType": "optional",
235+ "shape": [
236+ -2
237+ ]
238+ },
239+ {
240+ "name": "x1_scale",
241+ "index": 3,
242+ "dtype": "float32",
243+ "format": "ND",
244+ "paramType": "optional",
245+ "shape": [
246+ -2
247+ ]
248+ },
249+ {
250+ "name": "x2_scale",
251+ "index": 4,
252+ "dtype": "float32",
253+ "format": "ND",
254+ "paramType": "optional",
255+ "shape": [
256+ -2
257+ ]
258+ }
259+ ],
260+ "outputs": [
261+ {
262+ "name": "y",
263+ "index": 0,
264+ "dtype": "float16",
265+ "format": "ND",
266+ "paramType": "required",
267+ "shape": [
268+ -2
269+ ]
270+ }
271+ ],
272+ "attrs": [
273+ {
274+ "name": "dtype",
275+ "dtype": "int",
276+ "value": 1
277+ },
278+ {
279+ "name": "group_size",
280+ "dtype": "int",
281+ "value": 0
282+ },
283+ {
284+ "name": "perm_x1",
285+ "dtype": "list_int",
286+ "value": [1,0,2]
287+ },
288+ {
289+ "name": "perm_x2",
290+ "dtype": "list_int",
291+ "value": [0,1,2]
292+ },
293+ {
294+ "name": "perm_y",
295+ "dtype": "list_int",
296+ "value": [1,0,2]
297+ },
298+ {
299+ "name": "batch_split_factor",
300+ "dtype": "int",
301+ "value": 1
302+ }
303+ ]
304+ },
305+ {
306+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E5M2_FP32_FP32_FP32_BF16",
307+ "simplified_key": "diy,2/2/2/2/2/2/35/35/0/0/0/27",
308+ "inputs": [
309+ {
310+ "name": "x1",
311+ "index": 0,
312+ "dtype": "float8_e5m2",
313+ "format": "ND",
314+ "paramType": "required",
315+ "shape": [
316+ -2
317+ ]
318+ },
319+ {
320+ "name": "x2",
321+ "index": 1,
322+ "dtype": "float8_e5m2",
323+ "format": "ND",
324+ "paramType": "required",
325+ "shape": [
326+ -2
327+ ]
328+ },
329+ {
330+ "name": "bias",
331+ "index": 2,
332+ "dtype": "float32",
333+ "format": "ND",
334+ "paramType": "optional",
335+ "shape": [
336+ -2
337+ ]
338+ },
339+ {
340+ "name": "x1_scale",
341+ "index": 3,
342+ "dtype": "float32",
343+ "format": "ND",
344+ "paramType": "optional",
345+ "shape": [
346+ -2
347+ ]
348+ },
349+ {
350+ "name": "x2_scale",
351+ "index": 4,
352+ "dtype": "float32",
353+ "format": "ND",
354+ "paramType": "optional",
355+ "shape": [
356+ -2
357+ ]
358+ }
359+ ],
360+ "outputs": [
361+ {
362+ "name": "y",
363+ "index": 0,
364+ "dtype": "bfloat16",
365+ "format": "ND",
366+ "paramType": "required",
367+ "shape": [
368+ -2
369+ ]
370+ }
371+ ],
372+ "attrs": [
373+ {
374+ "name": "dtype",
375+ "dtype": "int",
376+ "value": 27
377+ },
378+ {
379+ "name": "group_size",
380+ "dtype": "int",
381+ "value": 0
382+ },
383+ {
384+ "name": "perm_x1",
385+ "dtype": "list_int",
386+ "value": [1,0,2]
387+ },
388+ {
389+ "name": "perm_x2",
390+ "dtype": "list_int",
391+ "value": [0,1,2]
392+ },
393+ {
394+ "name": "perm_y",
395+ "dtype": "list_int",
396+ "value": [1,0,2]
397+ },
398+ {
399+ "name": "batch_split_factor",
400+ "dtype": "int",
401+ "value": 1
402+ }
403+ ]
404+ },
405+ {
406+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E4M3FN_FP16_FP32_FP32_FP16",
407+ "simplified_key": "diy,2/2/2/2/2/2/36/36/1/0/0/1",
408+ "inputs": [
409+ {
410+ "name": "x1",
411+ "index": 0,
412+ "dtype": "float8_e4m3fn",
413+ "format": "ND",
414+ "paramType": "required",
415+ "shape": [
416+ -2
417+ ]
418+ },
419+ {
420+ "name": "x2",
421+ "index": 1,
422+ "dtype": "float8_e4m3fn",
423+ "format": "ND",
424+ "paramType": "required",
425+ "shape": [
426+ -2
427+ ]
428+ },
429+ {
430+ "name": "bias",
431+ "index": 2,
432+ "dtype": "float16",
433+ "format": "ND",
434+ "paramType": "optional",
435+ "shape": [
436+ -2
437+ ]
438+ },
439+ {
440+ "name": "x1_scale",
441+ "index": 3,
442+ "dtype": "float32",
443+ "format": "ND",
444+ "paramType": "optional",
445+ "shape": [
446+ -2
447+ ]
448+ },
449+ {
450+ "name": "x2_scale",
451+ "index": 4,
452+ "dtype": "float32",
453+ "format": "ND",
454+ "paramType": "optional",
455+ "shape": [
456+ -2
457+ ]
458+ }
459+ ],
460+ "outputs": [
461+ {
462+ "name": "y",
463+ "index": 0,
464+ "dtype": "float16",
465+ "format": "ND",
466+ "paramType": "required",
467+ "shape": [
468+ -2
469+ ]
470+ }
471+ ],
472+ "attrs": [
473+ {
474+ "name": "dtype",
475+ "dtype": "int",
476+ "value": 1
477+ },
478+ {
479+ "name": "group_size",
480+ "dtype": "int",
481+ "value": 0
482+ },
483+ {
484+ "name": "perm_x1",
485+ "dtype": "list_int",
486+ "value": [1,0,2]
487+ },
488+ {
489+ "name": "perm_x2",
490+ "dtype": "list_int",
491+ "value": [0,1,2]
492+ },
493+ {
494+ "name": "perm_y",
495+ "dtype": "list_int",
496+ "value": [1,0,2]
497+ },
498+ {
499+ "name": "batch_split_factor",
500+ "dtype": "int",
501+ "value": 1
502+ }
503+ ]
504+ },
505+ {
506+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E4M3FN_BF16_FP32_FP32_BF16",
507+ "simplified_key": "diy,2/2/2/2/2/2/36/36/27/0/0/27",
508+ "inputs": [
509+ {
510+ "name": "x1",
511+ "index": 0,
512+ "dtype": "float8_e4m3fn",
513+ "format": "ND",
514+ "paramType": "required",
515+ "shape": [
516+ -2
517+ ]
518+ },
519+ {
520+ "name": "x2",
521+ "index": 1,
522+ "dtype": "float8_e4m3fn",
523+ "format": "ND",
524+ "paramType": "required",
525+ "shape": [
526+ -2
527+ ]
528+ },
529+ {
530+ "name": "bias",
531+ "index": 2,
532+ "dtype": "bfloat16",
533+ "format": "ND",
534+ "paramType": "optional",
535+ "shape": [
536+ -2
537+ ]
538+ },
539+ {
540+ "name": "x1_scale",
541+ "index": 3,
542+ "dtype": "float32",
543+ "format": "ND",
544+ "paramType": "optional",
545+ "shape": [
546+ -2
547+ ]
548+ },
549+ {
550+ "name": "x2_scale",
551+ "index": 4,
552+ "dtype": "float32",
553+ "format": "ND",
554+ "paramType": "optional",
555+ "shape": [
556+ -2
557+ ]
558+ }
559+ ],
560+ "outputs": [
561+ {
562+ "name": "y",
563+ "index": 0,
564+ "dtype": "bfloat16",
565+ "format": "ND",
566+ "paramType": "required",
567+ "shape": [
568+ -2
569+ ]
570+ }
571+ ],
572+ "attrs": [
573+ {
574+ "name": "dtype",
575+ "dtype": "int",
576+ "value": 27
577+ },
578+ {
579+ "name": "group_size",
580+ "dtype": "int",
581+ "value": 0
582+ },
583+ {
584+ "name": "perm_x1",
585+ "dtype": "list_int",
586+ "value": [1,0,2]
587+ },
588+ {
589+ "name": "perm_x2",
590+ "dtype": "list_int",
591+ "value": [0,1,2]
592+ },
593+ {
594+ "name": "perm_y",
595+ "dtype": "list_int",
596+ "value": [1,0,2]
597+ },
598+ {
599+ "name": "batch_split_factor",
600+ "dtype": "int",
601+ "value": 1
602+ }
603+ ]
604+ },
605+ {
606+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E4M3FN_FP32_FP32_FP32_FP16",
607+ "simplified_key": "diy,2/2/2/2/2/2/36/36/0/0/0/1",
608+ "inputs": [
609+ {
610+ "name": "x1",
611+ "index": 0,
612+ "dtype": "float8_e4m3fn",
613+ "format": "ND",
614+ "paramType": "required",
615+ "shape": [
616+ -2
617+ ]
618+ },
619+ {
620+ "name": "x2",
621+ "index": 1,
622+ "dtype": "float8_e4m3fn",
623+ "format": "ND",
624+ "paramType": "required",
625+ "shape": [
626+ -2
627+ ]
628+ },
629+ {
630+ "name": "bias",
631+ "index": 2,
632+ "dtype": "float32",
633+ "format": "ND",
634+ "paramType": "optional",
635+ "shape": [
636+ -2
637+ ]
638+ },
639+ {
640+ "name": "x1_scale",
641+ "index": 3,
642+ "dtype": "float32",
643+ "format": "ND",
644+ "paramType": "optional",
645+ "shape": [
646+ -2
647+ ]
648+ },
649+ {
650+ "name": "x2_scale",
651+ "index": 4,
652+ "dtype": "float32",
653+ "format": "ND",
654+ "paramType": "optional",
655+ "shape": [
656+ -2
657+ ]
658+ }
659+ ],
660+ "outputs": [
661+ {
662+ "name": "y",
663+ "index": 0,
664+ "dtype": "float16",
665+ "format": "ND",
666+ "paramType": "required",
667+ "shape": [
668+ -2
669+ ]
670+ }
671+ ],
672+ "attrs": [
673+ {
674+ "name": "dtype",
675+ "dtype": "int",
676+ "value": 1
677+ },
678+ {
679+ "name": "group_size",
680+ "dtype": "int",
681+ "value": 0
682+ },
683+ {
684+ "name": "perm_x1",
685+ "dtype": "list_int",
686+ "value": [1,0,2]
687+ },
688+ {
689+ "name": "perm_x2",
690+ "dtype": "list_int",
691+ "value": [0,1,2]
692+ },
693+ {
694+ "name": "perm_y",
695+ "dtype": "list_int",
696+ "value": [1,0,2]
697+ },
698+ {
699+ "name": "batch_split_factor",
700+ "dtype": "int",
701+ "value": 1
702+ }
703+ ]
704+ },
705+ {
706+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E4M3FN_FP32_FP32_FP32_BF16",
707+ "simplified_key": "diy,2/2/2/2/2/2/36/36/0/0/0/27",
708+ "inputs": [
709+ {
710+ "name": "x1",
711+ "index": 0,
712+ "dtype": "float8_e4m3fn",
713+ "format": "ND",
714+ "paramType": "required",
715+ "shape": [
716+ -2
717+ ]
718+ },
719+ {
720+ "name": "x2",
721+ "index": 1,
722+ "dtype": "float8_e4m3fn",
723+ "format": "ND",
724+ "paramType": "required",
725+ "shape": [
726+ -2
727+ ]
728+ },
729+ {
730+ "name": "bias",
731+ "index": 2,
732+ "dtype": "float32",
733+ "format": "ND",
734+ "paramType": "optional",
735+ "shape": [
736+ -2
737+ ]
738+ },
739+ {
740+ "name": "x1_scale",
741+ "index": 3,
742+ "dtype": "float32",
743+ "format": "ND",
744+ "paramType": "optional",
745+ "shape": [
746+ -2
747+ ]
748+ },
749+ {
750+ "name": "x2_scale",
751+ "index": 4,
752+ "dtype": "float32",
753+ "format": "ND",
754+ "paramType": "optional",
755+ "shape": [
756+ -2
757+ ]
758+ }
759+ ],
760+ "outputs": [
761+ {
762+ "name": "y",
763+ "index": 0,
764+ "dtype": "bfloat16",
765+ "format": "ND",
766+ "paramType": "required",
767+ "shape": [
768+ -2
769+ ]
770+ }
771+ ],
772+ "attrs": [
773+ {
774+ "name": "dtype",
775+ "dtype": "int",
776+ "value": 27
777+ },
778+ {
779+ "name": "group_size",
780+ "dtype": "int",
781+ "value": 0
782+ },
783+ {
784+ "name": "perm_x1",
785+ "dtype": "list_int",
786+ "value": [1,0,2]
787+ },
788+ {
789+ "name": "perm_x2",
790+ "dtype": "list_int",
791+ "value": [0,1,2]
792+ },
793+ {
794+ "name": "perm_y",
795+ "dtype": "list_int",
796+ "value": [1,0,2]
797+ },
798+ {
799+ "name": "batch_split_factor",
800+ "dtype": "int",
801+ "value": 1
802+ }
803+ ]
804+ },
805+ {
806+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E5M2_FP16_FP32_FP32_FP16",
807+ "simplified_key": "diy,2/2/2/2/2/2/36/35/1/0/0/1",
808+ "inputs": [
809+ {
810+ "name": "x1",
811+ "index": 0,
812+ "dtype": "float8_e4m3fn",
813+ "format": "ND",
814+ "paramType": "required",
815+ "shape": [
816+ -2
817+ ]
818+ },
819+ {
820+ "name": "x2",
821+ "index": 1,
822+ "dtype": "float8_e5m2",
823+ "format": "ND",
824+ "paramType": "required",
825+ "shape": [
826+ -2
827+ ]
828+ },
829+ {
830+ "name": "bias",
831+ "index": 2,
832+ "dtype": "float16",
833+ "format": "ND",
834+ "paramType": "optional",
835+ "shape": [
836+ -2
837+ ]
838+ },
839+ {
840+ "name": "x1_scale",
841+ "index": 3,
842+ "dtype": "float32",
843+ "format": "ND",
844+ "paramType": "optional",
845+ "shape": [
846+ -2
847+ ]
848+ },
849+ {
850+ "name": "x2_scale",
851+ "index": 4,
852+ "dtype": "float32",
853+ "format": "ND",
854+ "paramType": "optional",
855+ "shape": [
856+ -2
857+ ]
858+ }
859+ ],
860+ "outputs": [
861+ {
862+ "name": "y",
863+ "index": 0,
864+ "dtype": "float16",
865+ "format": "ND",
866+ "paramType": "required",
867+ "shape": [
868+ -2
869+ ]
870+ }
871+ ],
872+ "attrs": [
873+ {
874+ "name": "dtype",
875+ "dtype": "int",
876+ "value": 1
877+ },
878+ {
879+ "name": "group_size",
880+ "dtype": "int",
881+ "value": 0
882+ },
883+ {
884+ "name": "perm_x1",
885+ "dtype": "list_int",
886+ "value": [1,0,2]
887+ },
888+ {
889+ "name": "perm_x2",
890+ "dtype": "list_int",
891+ "value": [0,1,2]
892+ },
893+ {
894+ "name": "perm_y",
895+ "dtype": "list_int",
896+ "value": [1,0,2]
897+ },
898+ {
899+ "name": "batch_split_factor",
900+ "dtype": "int",
901+ "value": 1
902+ }
903+ ]
904+ },
905+ {
906+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E5M2_BF16_FP32_FP32_BF16",
907+ "simplified_key": "diy,2/2/2/2/2/2/36/35/27/0/0/27",
908+ "inputs": [
909+ {
910+ "name": "x1",
911+ "index": 0,
912+ "dtype": "float8_e4m3fn",
913+ "format": "ND",
914+ "paramType": "required",
915+ "shape": [
916+ -2
917+ ]
918+ },
919+ {
920+ "name": "x2",
921+ "index": 1,
922+ "dtype": "float8_e5m2",
923+ "format": "ND",
924+ "paramType": "required",
925+ "shape": [
926+ -2
927+ ]
928+ },
929+ {
930+ "name": "bias",
931+ "index": 2,
932+ "dtype": "bfloat16",
933+ "format": "ND",
934+ "paramType": "optional",
935+ "shape": [
936+ -2
937+ ]
938+ },
939+ {
940+ "name": "x1_scale",
941+ "index": 3,
942+ "dtype": "float32",
943+ "format": "ND",
944+ "paramType": "optional",
945+ "shape": [
946+ -2
947+ ]
948+ },
949+ {
950+ "name": "x2_scale",
951+ "index": 4,
952+ "dtype": "float32",
953+ "format": "ND",
954+ "paramType": "optional",
955+ "shape": [
956+ -2
957+ ]
958+ }
959+ ],
960+ "outputs": [
961+ {
962+ "name": "y",
963+ "index": 0,
964+ "dtype": "bfloat16",
965+ "format": "ND",
966+ "paramType": "required",
967+ "shape": [
968+ -2
969+ ]
970+ }
971+ ],
972+ "attrs": [
973+ {
974+ "name": "dtype",
975+ "dtype": "int",
976+ "value": 27
977+ },
978+ {
979+ "name": "group_size",
980+ "dtype": "int",
981+ "value": 0
982+ },
983+ {
984+ "name": "perm_x1",
985+ "dtype": "list_int",
986+ "value": [1,0,2]
987+ },
988+ {
989+ "name": "perm_x2",
990+ "dtype": "list_int",
991+ "value": [0,1,2]
992+ },
993+ {
994+ "name": "perm_y",
995+ "dtype": "list_int",
996+ "value": [1,0,2]
997+ },
998+ {
999+ "name": "batch_split_factor",
1000+ "dtype": "int",
1001+ "value": 1
1002+ }
1003+ ]
1004+ },
1005+ {
1006+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E5M2_FP32_FP32_FP32_FP16",
1007+ "simplified_key": "diy,2/2/2/2/2/2/36/35/0/0/0/1",
1008+ "inputs": [
1009+ {
1010+ "name": "x1",
1011+ "index": 0,
1012+ "dtype": "float8_e4m3fn",
1013+ "format": "ND",
1014+ "paramType": "required",
1015+ "shape": [
1016+ -2
1017+ ]
1018+ },
1019+ {
1020+ "name": "x2",
1021+ "index": 1,
1022+ "dtype": "float8_e5m2",
1023+ "format": "ND",
1024+ "paramType": "required",
1025+ "shape": [
1026+ -2
1027+ ]
1028+ },
1029+ {
1030+ "name": "bias",
1031+ "index": 2,
1032+ "dtype": "float32",
1033+ "format": "ND",
1034+ "paramType": "optional",
1035+ "shape": [
1036+ -2
1037+ ]
1038+ },
1039+ {
1040+ "name": "x1_scale",
1041+ "index": 3,
1042+ "dtype": "float32",
1043+ "format": "ND",
1044+ "paramType": "optional",
1045+ "shape": [
1046+ -2
1047+ ]
1048+ },
1049+ {
1050+ "name": "x2_scale",
1051+ "index": 4,
1052+ "dtype": "float32",
1053+ "format": "ND",
1054+ "paramType": "optional",
1055+ "shape": [
1056+ -2
1057+ ]
1058+ }
1059+ ],
1060+ "outputs": [
1061+ {
1062+ "name": "y",
1063+ "index": 0,
1064+ "dtype": "float16",
1065+ "format": "ND",
1066+ "paramType": "required",
1067+ "shape": [
1068+ -2
1069+ ]
1070+ }
1071+ ],
1072+ "attrs": [
1073+ {
1074+ "name": "dtype",
1075+ "dtype": "int",
1076+ "value": 1
1077+ },
1078+ {
1079+ "name": "group_size",
1080+ "dtype": "int",
1081+ "value": 0
1082+ },
1083+ {
1084+ "name": "perm_x1",
1085+ "dtype": "list_int",
1086+ "value": [1,0,2]
1087+ },
1088+ {
1089+ "name": "perm_x2",
1090+ "dtype": "list_int",
1091+ "value": [0,1,2]
1092+ },
1093+ {
1094+ "name": "perm_y",
1095+ "dtype": "list_int",
1096+ "value": [1,0,2]
1097+ },
1098+ {
1099+ "name": "batch_split_factor",
1100+ "dtype": "int",
1101+ "value": 1
1102+ }
1103+ ]
1104+ },
1105+ {
1106+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E5M2_FP32_FP32_FP32_BF16",
1107+ "simplified_key": "diy,2/2/2/2/2/2/36/35/0/0/0/27",
1108+ "inputs": [
1109+ {
1110+ "name": "x1",
1111+ "index": 0,
1112+ "dtype": "float8_e4m3fn",
1113+ "format": "ND",
1114+ "paramType": "required",
1115+ "shape": [
1116+ -2
1117+ ]
1118+ },
1119+ {
1120+ "name": "x2",
1121+ "index": 1,
1122+ "dtype": "float8_e5m2",
1123+ "format": "ND",
1124+ "paramType": "required",
1125+ "shape": [
1126+ -2
1127+ ]
1128+ },
1129+ {
1130+ "name": "bias",
1131+ "index": 2,
1132+ "dtype": "float32",
1133+ "format": "ND",
1134+ "paramType": "optional",
1135+ "shape": [
1136+ -2
1137+ ]
1138+ },
1139+ {
1140+ "name": "x1_scale",
1141+ "index": 3,
1142+ "dtype": "float32",
1143+ "format": "ND",
1144+ "paramType": "optional",
1145+ "shape": [
1146+ -2
1147+ ]
1148+ },
1149+ {
1150+ "name": "x2_scale",
1151+ "index": 4,
1152+ "dtype": "float32",
1153+ "format": "ND",
1154+ "paramType": "optional",
1155+ "shape": [
1156+ -2
1157+ ]
1158+ }
1159+ ],
1160+ "outputs": [
1161+ {
1162+ "name": "y",
1163+ "index": 0,
1164+ "dtype": "bfloat16",
1165+ "format": "ND",
1166+ "paramType": "required",
1167+ "shape": [
1168+ -2
1169+ ]
1170+ }
1171+ ],
1172+ "attrs": [
1173+ {
1174+ "name": "dtype",
1175+ "dtype": "int",
1176+ "value": 27
1177+ },
1178+ {
1179+ "name": "group_size",
1180+ "dtype": "int",
1181+ "value": 0
1182+ },
1183+ {
1184+ "name": "perm_x1",
1185+ "dtype": "list_int",
1186+ "value": [1,0,2]
1187+ },
1188+ {
1189+ "name": "perm_x2",
1190+ "dtype": "list_int",
1191+ "value": [0,1,2]
1192+ },
1193+ {
1194+ "name": "perm_y",
1195+ "dtype": "list_int",
1196+ "value": [1,0,2]
1197+ },
1198+ {
1199+ "name": "batch_split_factor",
1200+ "dtype": "int",
1201+ "value": 1
1202+ }
1203+ ]
1204+ },
1205+ {
1206+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E4M3FN_FP16_FP32_FP32_FP16",
1207+ "simplified_key": "diy,2/2/2/2/2/2/35/36/1/0/0/1",
1208+ "inputs": [
1209+ {
1210+ "name": "x1",
1211+ "index": 0,
1212+ "dtype": "float8_e5m2",
1213+ "format": "ND",
1214+ "paramType": "required",
1215+ "shape": [
1216+ -2
1217+ ]
1218+ },
1219+ {
1220+ "name": "x2",
1221+ "index": 1,
1222+ "dtype": "float8_e4m3fn",
1223+ "format": "ND",
1224+ "paramType": "required",
1225+ "shape": [
1226+ -2
1227+ ]
1228+ },
1229+ {
1230+ "name": "bias",
1231+ "index": 2,
1232+ "dtype": "float16",
1233+ "format": "ND",
1234+ "paramType": "optional",
1235+ "shape": [
1236+ -2
1237+ ]
1238+ },
1239+ {
1240+ "name": "x1_scale",
1241+ "index": 3,
1242+ "dtype": "float32",
1243+ "format": "ND",
1244+ "paramType": "optional",
1245+ "shape": [
1246+ -2
1247+ ]
1248+ },
1249+ {
1250+ "name": "x2_scale",
1251+ "index": 4,
1252+ "dtype": "float32",
1253+ "format": "ND",
1254+ "paramType": "optional",
1255+ "shape": [
1256+ -2
1257+ ]
1258+ }
1259+ ],
1260+ "outputs": [
1261+ {
1262+ "name": "y",
1263+ "index": 0,
1264+ "dtype": "float16",
1265+ "format": "ND",
1266+ "paramType": "required",
1267+ "shape": [
1268+ -2
1269+ ]
1270+ }
1271+ ],
1272+ "attrs": [
1273+ {
1274+ "name": "dtype",
1275+ "dtype": "int",
1276+ "value": 1
1277+ },
1278+ {
1279+ "name": "group_size",
1280+ "dtype": "int",
1281+ "value": 0
1282+ },
1283+ {
1284+ "name": "perm_x1",
1285+ "dtype": "list_int",
1286+ "value": [1,0,2]
1287+ },
1288+ {
1289+ "name": "perm_x2",
1290+ "dtype": "list_int",
1291+ "value": [0,1,2]
1292+ },
1293+ {
1294+ "name": "perm_y",
1295+ "dtype": "list_int",
1296+ "value": [1,0,2]
1297+ },
1298+ {
1299+ "name": "batch_split_factor",
1300+ "dtype": "int",
1301+ "value": 1
1302+ }
1303+ ]
1304+ },
1305+ {
1306+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E4M3FN_BF16_FP32_FP32_BF16",
1307+ "simplified_key": "diy,2/2/2/2/2/2/35/36/27/0/0/27",
1308+ "inputs": [
1309+ {
1310+ "name": "x1",
1311+ "index": 0,
1312+ "dtype": "float8_e5m2",
1313+ "format": "ND",
1314+ "paramType": "required",
1315+ "shape": [
1316+ -2
1317+ ]
1318+ },
1319+ {
1320+ "name": "x2",
1321+ "index": 1,
1322+ "dtype": "float8_e4m3fn",
1323+ "format": "ND",
1324+ "paramType": "required",
1325+ "shape": [
1326+ -2
1327+ ]
1328+ },
1329+ {
1330+ "name": "bias",
1331+ "index": 2,
1332+ "dtype": "bfloat16",
1333+ "format": "ND",
1334+ "paramType": "optional",
1335+ "shape": [
1336+ -2
1337+ ]
1338+ },
1339+ {
1340+ "name": "x1_scale",
1341+ "index": 3,
1342+ "dtype": "float32",
1343+ "format": "ND",
1344+ "paramType": "optional",
1345+ "shape": [
1346+ -2
1347+ ]
1348+ },
1349+ {
1350+ "name": "x2_scale",
1351+ "index": 4,
1352+ "dtype": "float32",
1353+ "format": "ND",
1354+ "paramType": "optional",
1355+ "shape": [
1356+ -2
1357+ ]
1358+ }
1359+ ],
1360+ "outputs": [
1361+ {
1362+ "name": "y",
1363+ "index": 0,
1364+ "dtype": "bfloat16",
1365+ "format": "ND",
1366+ "paramType": "required",
1367+ "shape": [
1368+ -2
1369+ ]
1370+ }
1371+ ],
1372+ "attrs": [
1373+ {
1374+ "name": "dtype",
1375+ "dtype": "int",
1376+ "value": 27
1377+ },
1378+ {
1379+ "name": "group_size",
1380+ "dtype": "int",
1381+ "value": 0
1382+ },
1383+ {
1384+ "name": "perm_x1",
1385+ "dtype": "list_int",
1386+ "value": [1,0,2]
1387+ },
1388+ {
1389+ "name": "perm_x2",
1390+ "dtype": "list_int",
1391+ "value": [0,1,2]
1392+ },
1393+ {
1394+ "name": "perm_y",
1395+ "dtype": "list_int",
1396+ "value": [1,0,2]
1397+ },
1398+ {
1399+ "name": "batch_split_factor",
1400+ "dtype": "int",
1401+ "value": 1
1402+ }
1403+ ]
1404+ },
1405+ {
1406+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E4M3FN_FP32_FP32_FP32_FP16",
1407+ "simplified_key": "diy,2/2/2/2/2/2/35/36/0/0/0/1",
1408+ "inputs": [
1409+ {
1410+ "name": "x1",
1411+ "index": 0,
1412+ "dtype": "float8_e5m2",
1413+ "format": "ND",
1414+ "paramType": "required",
1415+ "shape": [
1416+ -2
1417+ ]
1418+ },
1419+ {
1420+ "name": "x2",
1421+ "index": 1,
1422+ "dtype": "float8_e4m3fn",
1423+ "format": "ND",
1424+ "paramType": "required",
1425+ "shape": [
1426+ -2
1427+ ]
1428+ },
1429+ {
1430+ "name": "bias",
1431+ "index": 2,
1432+ "dtype": "float32",
1433+ "format": "ND",
1434+ "paramType": "optional",
1435+ "shape": [
1436+ -2
1437+ ]
1438+ },
1439+ {
1440+ "name": "x1_scale",
1441+ "index": 3,
1442+ "dtype": "float32",
1443+ "format": "ND",
1444+ "paramType": "optional",
1445+ "shape": [
1446+ -2
1447+ ]
1448+ },
1449+ {
1450+ "name": "x2_scale",
1451+ "index": 4,
1452+ "dtype": "float32",
1453+ "format": "ND",
1454+ "paramType": "optional",
1455+ "shape": [
1456+ -2
1457+ ]
1458+ }
1459+ ],
1460+ "outputs": [
1461+ {
1462+ "name": "y",
1463+ "index": 0,
1464+ "dtype": "float16",
1465+ "format": "ND",
1466+ "paramType": "required",
1467+ "shape": [
1468+ -2
1469+ ]
1470+ }
1471+ ],
1472+ "attrs": [
1473+ {
1474+ "name": "dtype",
1475+ "dtype": "int",
1476+ "value": 1
1477+ },
1478+ {
1479+ "name": "group_size",
1480+ "dtype": "int",
1481+ "value": 0
1482+ },
1483+ {
1484+ "name": "perm_x1",
1485+ "dtype": "list_int",
1486+ "value": [1,0,2]
1487+ },
1488+ {
1489+ "name": "perm_x2",
1490+ "dtype": "list_int",
1491+ "value": [0,1,2]
1492+ },
1493+ {
1494+ "name": "perm_y",
1495+ "dtype": "list_int",
1496+ "value": [1,0,2]
1497+ },
1498+ {
1499+ "name": "batch_split_factor",
1500+ "dtype": "int",
1501+ "value": 1
1502+ }
1503+ ]
1504+ },
1505+ {
1506+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E5M2_FP8_E4M3FN_FP32_FP32_FP32_BF16",
1507+ "simplified_key": "diy,2/2/2/2/2/2/35/36/0/0/0/27",
1508+ "inputs": [
1509+ {
1510+ "name": "x1",
1511+ "index": 0,
1512+ "dtype": "float8_e5m2",
1513+ "format": "ND",
1514+ "paramType": "required",
1515+ "shape": [
1516+ -2
1517+ ]
1518+ },
1519+ {
1520+ "name": "x2",
1521+ "index": 1,
1522+ "dtype": "float8_e4m3fn",
1523+ "format": "ND",
1524+ "paramType": "required",
1525+ "shape": [
1526+ -2
1527+ ]
1528+ },
1529+ {
1530+ "name": "bias",
1531+ "index": 2,
1532+ "dtype": "float32",
1533+ "format": "ND",
1534+ "paramType": "optional",
1535+ "shape": [
1536+ -2
1537+ ]
1538+ },
1539+ {
1540+ "name": "x1_scale",
1541+ "index": 3,
1542+ "dtype": "float32",
1543+ "format": "ND",
1544+ "paramType": "optional",
1545+ "shape": [
1546+ -2
1547+ ]
1548+ },
1549+ {
1550+ "name": "x2_scale",
1551+ "index": 4,
1552+ "dtype": "float32",
1553+ "format": "ND",
1554+ "paramType": "optional",
1555+ "shape": [
1556+ -2
1557+ ]
1558+ }
1559+ ],
1560+ "outputs": [
1561+ {
1562+ "name": "y",
1563+ "index": 0,
1564+ "dtype": "bfloat16",
1565+ "format": "ND",
1566+ "paramType": "required",
1567+ "shape": [
1568+ -2
1569+ ]
1570+ }
1571+ ],
1572+ "attrs": [
1573+ {
1574+ "name": "dtype",
1575+ "dtype": "int",
1576+ "value": 27
1577+ },
1578+ {
1579+ "name": "group_size",
1580+ "dtype": "int",
1581+ "value": 0
1582+ },
1583+ {
1584+ "name": "perm_x1",
1585+ "dtype": "list_int",
1586+ "value": [1,0,2]
1587+ },
1588+ {
1589+ "name": "perm_x2",
1590+ "dtype": "list_int",
1591+ "value": [0,1,2]
1592+ },
1593+ {
1594+ "name": "perm_y",
1595+ "dtype": "list_int",
1596+ "value": [1,0,2]
1597+ },
1598+ {
1599+ "name": "batch_split_factor",
1600+ "dtype": "int",
1601+ "value": 1
1602+ }
1603+ ]
1604+ },
1605+ {
1606+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E4M3FN_FP32_FP8_E8M0_FP8_E8M0_FP16",
1607+ "simplified_key": "diy,2/2/2/2/2/2/36/36/0/37/37/1",
1608+ "inputs": [
1609+ {
1610+ "name": "x1",
1611+ "index": 0,
1612+ "dtype": "float8_e4m3fn",
1613+ "format": "ND",
1614+ "paramType": "required",
1615+ "shape": [
1616+ -2
1617+ ]
1618+ },
1619+ {
1620+ "name": "x2",
1621+ "index": 1,
1622+ "dtype": "float8_e4m3fn",
1623+ "format": "ND",
1624+ "paramType": "required",
1625+ "shape": [
1626+ -2
1627+ ]
1628+ },
1629+ {
1630+ "name": "bias",
1631+ "index": 2,
1632+ "dtype": "float32",
1633+ "format": "ND",
1634+ "paramType": "optional",
1635+ "shape": [
1636+ -2
1637+ ]
1638+ },
1639+ {
1640+ "name": "x1_scale",
1641+ "index": 3,
1642+ "dtype": "float8_e8m0",
1643+ "format": "ND",
1644+ "paramType": "optional",
1645+ "shape": [
1646+ -2
1647+ ]
1648+ },
1649+ {
1650+ "name": "x2_scale",
1651+ "index": 4,
1652+ "dtype": "float8_e8m0",
1653+ "format": "ND",
1654+ "paramType": "optional",
1655+ "shape": [
1656+ -2
1657+ ]
1658+ }
1659+ ],
1660+ "outputs": [
1661+ {
1662+ "name": "y",
1663+ "index": 0,
1664+ "dtype": "float16",
1665+ "format": "ND",
1666+ "paramType": "required",
1667+ "shape": [
1668+ -2
1669+ ]
1670+ }
1671+ ],
1672+ "attrs": [
1673+ {
1674+ "name": "dtype",
1675+ "dtype": "int",
1676+ "value": 1
1677+ },
1678+ {
1679+ "name": "group_size",
1680+ "dtype": "int",
1681+ "value": 32
1682+ },
1683+ {
1684+ "name": "perm_x1",
1685+ "dtype": "list_int",
1686+ "value": [1,0,2]
1687+ },
1688+ {
1689+ "name": "perm_x2",
1690+ "dtype": "list_int",
1691+ "value": [0,1,2]
1692+ },
1693+ {
1694+ "name": "perm_y",
1695+ "dtype": "list_int",
1696+ "value": [1,0,2]
1697+ },
1698+ {
1699+ "name": "batch_split_factor",
1700+ "dtype": "int",
1701+ "value": 1
1702+ }
1703+ ]
1704+ },
1705+ {
1706+ "bin_filename": "TransposeQuantBatchMatMul_ND_ND_ND_ND_ND_ND_FP8_E4M3FN_FP8_E4M3FN_FP32_FP8_E8M0_FP8_E8M0_BF16",
1707+ "simplified_key": "diy,2/2/2/2/2/2/36/36/0/37/37/27",
1708+ "inputs": [
1709+ {
1710+ "name": "x1",
1711+ "index": 0,
1712+ "dtype": "float8_e4m3fn",
1713+ "format": "ND",
1714+ "paramType": "required",
1715+ "shape": [
1716+ -2
1717+ ]
1718+ },
1719+ {
1720+ "name": "x2",
1721+ "index": 1,
1722+ "dtype": "float8_e4m3fn",
1723+ "format": "ND",
1724+ "paramType": "required",
1725+ "shape": [
1726+ -2
1727+ ]
1728+ },
1729+ {
1730+ "name": "bias",
1731+ "index": 2,
1732+ "dtype": "float32",
1733+ "format": "ND",
1734+ "paramType": "optional",
1735+ "shape": [
1736+ -2
1737+ ]
1738+ },
1739+ {
1740+ "name": "x1_scale",
1741+ "index": 3,
1742+ "dtype": "float8_e8m0",
1743+ "format": "ND",
1744+ "paramType": "optional",
1745+ "shape": [
1746+ -2
1747+ ]
1748+ },
1749+ {
1750+ "name": "x2_scale",
1751+ "index": 4,
1752+ "dtype": "float8_e8m0",
1753+ "format": "ND",
1754+ "paramType": "optional",
1755+ "shape": [
1756+ -2
1757+ ]
1758+ }
1759+ ],
1760+ "outputs": [
1761+ {
1762+ "name": "y",
1763+ "index": 0,
1764+ "dtype": "bfloat16",
1765+ "format": "ND",
1766+ "paramType": "required",
1767+ "shape": [
1768+ -2
1769+ ]
1770+ }
1771+ ],
1772+ "attrs": [
1773+ {
1774+ "name": "dtype",
1775+ "dtype": "int",
1776+ "value": 27
1777+ },
1778+ {
1779+ "name": "group_size",
1780+ "dtype": "int",
1781+ "value": 32
1782+ },
1783+ {
1784+ "name": "perm_x1",
1785+ "dtype": "list_int",
1786+ "value": [1,0,2]
1787+ },
1788+ {
1789+ "name": "perm_x2",
1790+ "dtype": "list_int",
1791+ "value": [0,1,2]
1792+ },
1793+ {
1794+ "name": "perm_y",
1795+ "dtype": "list_int",
1796+ "value": [1,0,2]
1797+ },
1798+ {
1799+ "name": "batch_split_factor",
1800+ "dtype": "int",
1801+ "value": 1
1802+ }
1803+ ]
1804+ }
1805+ ]
1806+}
Mmatmul/transpose_quant_batch_mat_mul/op_host/transpose_quant_batch_mat_mul_def.cpp+1-0
@@ -68,6 +68,7 @@ public:
68 .NeedCheckSupportFlag(false)68 .NeedCheckSupportFlag(false)
69 .ExtendCfgInfo("softsync.flag", "true");69 .ExtendCfgInfo("softsync.flag", "true");
70 this->AICore().AddConfig("ascend950", aicore_config);70 this->AICore().AddConfig("ascend950", aicore_config);
71+ this->AICore().AddConfig("ascend350", aicore_config);
71 }72 }
72};73};
73 74 
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/CMakeLists.txt+1-1
@@ -14,7 +14,7 @@ add_kernel_sources(
14 14 
15add_kernel_sources(15add_kernel_sources(
16 KERNEL_SRC arch35/transpose_quant_batch_mat_mul.cpp16 KERNEL_SRC arch35/transpose_quant_batch_mat_mul.cpp
17- COMPUTE_UNITS ascend95017+ COMPUTE_UNITS ascend950 ascend350
18 AUTO_SYNC false18 AUTO_SYNC false
19 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"19 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"
20)20)
Amatmul/weight_quant_batch_matmul_v2/op_host/config/ascend350/weight_quant_batch_matmul_v2_binary.json+2445-0
@@ -0,0 +1,2445 @@
1+{
2+ "op_type": "WeightQuantBatchMatmulV2",
3+ "op_list": [
4+ {
5+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_int8_weight_nd_fp16",
6+ "inputs": [
7+ {
8+ "name": "x",
9+ "index": 0,
10+ "dtype": "float16",
11+ "format": "ND",
12+ "paramType": "required",
13+ "shape": [
14+ -2
15+ ]
16+ },
17+ {
18+ "name": "weight",
19+ "index": 1,
20+ "dtype": "int8",
21+ "format": "ND",
22+ "paramType": "required",
23+ "shape": [
24+ -2
25+ ]
26+ },
27+ {
28+ "name": "antiquant_scale",
29+ "index": 2,
30+ "dtype": "float16",
31+ "format": "ND",
32+ "paramType": "required",
33+ "shape": [
34+ -2
35+ ]
36+ },
37+ {
38+ "name": "antiquant_offset",
39+ "index": 3,
40+ "dtype": "float16",
41+ "format": "ND",
42+ "paramType": "optional",
43+ "shape": [
44+ -2
45+ ]
46+ },
47+ {
48+ "name": "quant_scale",
49+ "index": 4,
50+ "dtype": "uint64",
51+ "format": "ND",
52+ "paramType": "optional",
53+ "shape": [
54+ -2
55+ ]
56+ },
57+ {
58+ "name": "quant_offset",
59+ "index": 5,
60+ "dtype": "float32",
61+ "format": "ND",
62+ "paramType": "optional",
63+ "shape": [
64+ -2
65+ ]
66+ },
67+ {
68+ "name": "bias",
69+ "index": 6,
70+ "dtype": "float16",
71+ "format": "ND",
72+ "paramType": "optional",
73+ "shape": [
74+ -2
75+ ]
76+ }
77+ ],
78+ "outputs": [
79+ {
80+ "name": "y",
81+ "index": 0,
82+ "dtype": "float16",
83+ "format": "ND",
84+ "paramType": "required",
85+ "shape": [
86+ -2
87+ ]
88+ }
89+ ],
90+ "attrs": [
91+ {
92+ "name": "transpose_x",
93+ "dtype": "bool",
94+ "value": null
95+ },
96+ {
97+ "name": "transpose_weight",
98+ "dtype": "bool",
99+ "value": null
100+ },
101+ {
102+ "name": "antiquant_group_size",
103+ "dtype": "int",
104+ "value": 0
105+ },
106+ {
107+ "name": "dtype",
108+ "dtype": "int",
109+ "value": -1
110+ },
111+ {
112+ "name": "inner_precise",
113+ "dtype": "int",
114+ "value": 0
115+ }
116+ ]
117+ },
118+ {
119+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_int8_weight_nz_fp16",
120+ "inputs": [
121+ {
122+ "name": "x",
123+ "index": 0,
124+ "dtype": "float16",
125+ "format": "ND",
126+ "paramType": "required",
127+ "shape": [
128+ -2
129+ ]
130+ },
131+ {
132+ "name": "weight",
133+ "index": 1,
134+ "dtype": "int8",
135+ "format": "FRACTAL_NZ",
136+ "paramType": "required",
137+ "shape": [
138+ -2
139+ ]
140+ },
141+ {
142+ "name": "antiquant_scale",
143+ "index": 2,
144+ "dtype": "float16",
145+ "format": "ND",
146+ "paramType": "required",
147+ "shape": [
148+ -2
149+ ]
150+ },
151+ {
152+ "name": "antiquant_offset",
153+ "index": 3,
154+ "dtype": "float16",
155+ "format": "ND",
156+ "paramType": "optional",
157+ "shape": [
158+ -2
159+ ]
160+ },
161+ {
162+ "name": "quant_scale",
163+ "index": 4,
164+ "dtype": "uint64",
165+ "format": "ND",
166+ "paramType": "optional",
167+ "shape": [
168+ -2
169+ ]
170+ },
171+ {
172+ "name": "quant_offset",
173+ "index": 5,
174+ "dtype": "float32",
175+ "format": "ND",
176+ "paramType": "optional",
177+ "shape": [
178+ -2
179+ ]
180+ },
181+ {
182+ "name": "bias",
183+ "index": 6,
184+ "dtype": "float16",
185+ "format": "ND",
186+ "paramType": "optional",
187+ "shape": [
188+ -2
189+ ]
190+ }
191+ ],
192+ "outputs": [
193+ {
194+ "name": "y",
195+ "index": 0,
196+ "dtype": "float16",
197+ "format": "ND",
198+ "paramType": "required",
199+ "shape": [
200+ -2
201+ ]
202+ }
203+ ],
204+ "attrs": [
205+ {
206+ "name": "transpose_x",
207+ "dtype": "bool",
208+ "value": null
209+ },
210+ {
211+ "name": "transpose_weight",
212+ "dtype": "bool",
213+ "value": null
214+ },
215+ {
216+ "name": "antiquant_group_size",
217+ "dtype": "int",
218+ "value": 0
219+ },
220+ {
221+ "name": "dtype",
222+ "dtype": "int",
223+ "value": -1
224+ },
225+ {
226+ "name": "inner_precise",
227+ "dtype": "int",
228+ "value": 0
229+ }
230+ ]
231+ },
232+ {
233+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_int4_weight_nz_fp16",
234+ "inputs": [
235+ {
236+ "name": "x",
237+ "index": 0,
238+ "dtype": "float16",
239+ "dtypeForBinQuery": [
240+ "float16",
241+ "float16"
242+ ],
243+ "format": "ND",
244+ "formatForBinQuery": [
245+ "ND",
246+ "ND"
247+ ],
248+ "paramType": "required",
249+ "shape": [
250+ -2
251+ ]
252+ },
253+ {
254+ "name": "weight",
255+ "index": 1,
256+ "dtype": "int4",
257+ "dtypeForBinQuery": [
258+ "int4",
259+ "int32"
260+ ],
261+ "format": "FRACTAL_NZ",
262+ "formatForBinQuery": [
263+ "FRACTAL_NZ",
264+ "FRACTAL_NZ_C0_2"
265+ ],
266+ "paramType": "required",
267+ "shape": [
268+ -2
269+ ]
270+ },
271+ {
272+ "name": "antiquant_scale",
273+ "index": 2,
274+ "dtype": "float16",
275+ "dtypeForBinQuery": [
276+ "float16",
277+ "float16"
278+ ],
279+ "format": "ND",
280+ "formatForBinQuery": [
281+ "ND",
282+ "ND"
283+ ],
284+ "paramType": "required",
285+ "shape": [
286+ -2
287+ ]
288+ },
289+ {
290+ "name": "antiquant_offset",
291+ "index": 3,
292+ "dtype": "float16",
293+ "dtypeForBinQuery": [
294+ "float16",
295+ "float16"
296+ ],
297+ "format": "ND",
298+ "formatForBinQuery": [
299+ "ND",
300+ "ND"
301+ ],
302+ "paramType": "optional",
303+ "shape": [
304+ -2
305+ ]
306+ },
307+ {
308+ "name": "quant_scale",
309+ "index": 4,
310+ "dtype": "uint64",
311+ "dtypeForBinQuery": [
312+ "uint64",
313+ "uint64"
314+ ],
315+ "format": "ND",
316+ "formatForBinQuery": [
317+ "ND",
318+ "ND"
319+ ],
320+ "paramType": "optional",
321+ "shape": [
322+ -2
323+ ]
324+ },
325+ {
326+ "name": "quant_offset",
327+ "index": 5,
328+ "dtype": "float32",
329+ "dtypeForBinQuery": [
330+ "float32",
331+ "float32"
332+ ],
333+ "format": "ND",
334+ "formatForBinQuery": [
335+ "ND",
336+ "ND"
337+ ],
338+ "paramType": "optional",
339+ "shape": [
340+ -2
341+ ]
342+ },
343+ {
344+ "name": "bias",
345+ "index": 6,
346+ "dtype": "float16",
347+ "dtypeForBinQuery": [
348+ "float16",
349+ "float16"
350+ ],
351+ "format": "ND",
352+ "formatForBinQuery": [
353+ "ND",
354+ "ND"
355+ ],
356+ "paramType": "optional",
357+ "shape": [
358+ -2
359+ ]
360+ }
361+ ],
362+ "outputs": [
363+ {
364+ "name": "y",
365+ "index": 0,
366+ "dtype": "float16",
367+ "dtypeForBinQuery": [
368+ "float16",
369+ "float16"
370+ ],
371+ "format": "ND",
372+ "formatForBinQuery": [
373+ "ND",
374+ "ND"
375+ ],
376+ "paramType": "required",
377+ "shape": [
378+ -2
379+ ]
380+ }
381+ ],
382+ "attrs": [
383+ {
384+ "name": "transpose_x",
385+ "dtype": "bool",
386+ "value": null
387+ },
388+ {
389+ "name": "transpose_weight",
390+ "dtype": "bool",
391+ "value": null
392+ },
393+ {
394+ "name": "antiquant_group_size",
395+ "dtype": "int",
396+ "value": 0
397+ },
398+ {
399+ "name": "dtype",
400+ "dtype": "int",
401+ "value": -1
402+ },
403+ {
404+ "name": "inner_precise",
405+ "dtype": "int",
406+ "value": 0
407+ }
408+ ]
409+ },
410+ {
411+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_int4_weight_nz_bf16",
412+ "inputs": [
413+ {
414+ "name": "x",
415+ "index": 0,
416+ "dtype": "bfloat16",
417+ "dtypeForBinQuery": [
418+ "bfloat16",
419+ "bfloat16"
420+ ],
421+ "format": "ND",
422+ "formatForBinQuery": [
423+ "ND",
424+ "ND"
425+ ],
426+ "paramType": "required",
427+ "shape": [
428+ -2
429+ ]
430+ },
431+ {
432+ "name": "weight",
433+ "index": 1,
434+ "dtype": "int4",
435+ "dtypeForBinQuery": [
436+ "int4",
437+ "int32"
438+ ],
439+ "format": "FRACTAL_NZ",
440+ "formatForBinQuery": [
441+ "FRACTAL_NZ",
442+ "FRACTAL_NZ_C0_2"
443+ ],
444+ "paramType": "required",
445+ "shape": [
446+ -2
447+ ]
448+ },
449+ {
450+ "name": "antiquant_scale",
451+ "index": 2,
452+ "dtype": "bfloat16",
453+ "dtypeForBinQuery": [
454+ "bfloat16",
455+ "bfloat16"
456+ ],
457+ "format": "ND",
458+ "formatForBinQuery": [
459+ "ND",
460+ "ND"
461+ ],
462+ "paramType": "required",
463+ "shape": [
464+ -2
465+ ]
466+ },
467+ {
468+ "name": "antiquant_offset",
469+ "index": 3,
470+ "dtype": "bfloat16",
471+ "dtypeForBinQuery": [
472+ "bfloat16",
473+ "bfloat16"
474+ ],
475+ "format": "ND",
476+ "formatForBinQuery": [
477+ "ND",
478+ "ND"
479+ ],
480+ "paramType": "optional",
481+ "shape": [
482+ -2
483+ ]
484+ },
485+ {
486+ "name": "quant_scale",
487+ "index": 4,
488+ "dtype": "uint64",
489+ "dtypeForBinQuery": [
490+ "uint64",
491+ "uint64"
492+ ],
493+ "format": "ND",
494+ "formatForBinQuery": [
495+ "ND",
496+ "ND"
497+ ],
498+ "paramType": "optional",
499+ "shape": [
500+ -2
501+ ]
502+ },
503+ {
504+ "name": "quant_offset",
505+ "index": 5,
506+ "dtype": "float32",
507+ "dtypeForBinQuery": [
508+ "float32",
509+ "float32"
510+ ],
511+ "format": "ND",
512+ "formatForBinQuery": [
513+ "ND",
514+ "ND"
515+ ],
516+ "paramType": "optional",
517+ "shape": [
518+ -2
519+ ]
520+ },
521+ {
522+ "name": "bias",
523+ "index": 6,
524+ "dtype": "bfloat16",
525+ "dtypeForBinQuery": [
526+ "bfloat16",
527+ "bfloat16"
528+ ],
529+ "format": "ND",
530+ "formatForBinQuery": [
531+ "ND",
532+ "ND"
533+ ],
534+ "paramType": "optional",
535+ "shape": [
536+ -2
537+ ]
538+ }
539+ ],
540+ "outputs": [
541+ {
542+ "name": "y",
543+ "index": 0,
544+ "dtype": "bfloat16",
545+ "dtypeForBinQuery": [
546+ "bfloat16",
547+ "bfloat16"
548+ ],
549+ "format": "ND",
550+ "formatForBinQuery": [
551+ "ND",
552+ "ND"
553+ ],
554+ "paramType": "required",
555+ "shape": [
556+ -2
557+ ]
558+ }
559+ ],
560+ "attrs": [
561+ {
562+ "name": "transpose_x",
563+ "dtype": "bool",
564+ "value": null
565+ },
566+ {
567+ "name": "transpose_weight",
568+ "dtype": "bool",
569+ "value": null
570+ },
571+ {
572+ "name": "antiquant_group_size",
573+ "dtype": "int",
574+ "value": 0
575+ },
576+ {
577+ "name": "dtype",
578+ "dtype": "int",
579+ "value": -1
580+ },
581+ {
582+ "name": "inner_precise",
583+ "dtype": "int",
584+ "value": 0
585+ }
586+ ]
587+ },
588+ {
589+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_int8_weight_nd_bf16",
590+ "inputs": [
591+ {
592+ "name": "x",
593+ "index": 0,
594+ "dtype": "bfloat16",
595+ "format": "ND",
596+ "paramType": "required",
597+ "shape": [
598+ -2
599+ ]
600+ },
601+ {
602+ "name": "weight",
603+ "index": 1,
604+ "dtype": "int8",
605+ "format": "ND",
606+ "paramType": "required",
607+ "shape": [
608+ -2
609+ ]
610+ },
611+ {
612+ "name": "antiquant_scale",
613+ "index": 2,
614+ "dtype": "bfloat16",
615+ "format": "ND",
616+ "paramType": "required",
617+ "shape": [
618+ -2
619+ ]
620+ },
621+ {
622+ "name": "antiquant_offset",
623+ "index": 3,
624+ "dtype": "bfloat16",
625+ "format": "ND",
626+ "paramType": "optional",
627+ "shape": [
628+ -2
629+ ]
630+ },
631+ {
632+ "name": "quant_scale",
633+ "index": 4,
634+ "dtype": "uint64",
635+ "format": "ND",
636+ "paramType": "optional",
637+ "shape": [
638+ -2
639+ ]
640+ },
641+ {
642+ "name": "quant_offset",
643+ "index": 5,
644+ "dtype": "float32",
645+ "format": "ND",
646+ "paramType": "optional",
647+ "shape": [
648+ -2
649+ ]
650+ },
651+ {
652+ "name": "bias",
653+ "index": 6,
654+ "dtype": "bfloat16",
655+ "format": "ND",
656+ "paramType": "optional",
657+ "shape": [
658+ -2
659+ ]
660+ }
661+ ],
662+ "outputs": [
663+ {
664+ "name": "y",
665+ "index": 0,
666+ "dtype": "bfloat16",
667+ "format": "ND",
668+ "paramType": "required",
669+ "shape": [
670+ -2
671+ ]
672+ }
673+ ],
674+ "attrs": [
675+ {
676+ "name": "transpose_x",
677+ "dtype": "bool",
678+ "value": null
679+ },
680+ {
681+ "name": "transpose_weight",
682+ "dtype": "bool",
683+ "value": null
684+ },
685+ {
686+ "name": "antiquant_group_size",
687+ "dtype": "int",
688+ "value": 0
689+ },
690+ {
691+ "name": "dtype",
692+ "dtype": "int",
693+ "value": -1
694+ },
695+ {
696+ "name": "inner_precise",
697+ "dtype": "int",
698+ "value": 0
699+ }
700+ ]
701+ },
702+ {
703+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_int8_weight_nz_bf16",
704+ "inputs": [
705+ {
706+ "name": "x",
707+ "index": 0,
708+ "dtype": "bfloat16",
709+ "format": "ND",
710+ "paramType": "required",
711+ "shape": [
712+ -2
713+ ]
714+ },
715+ {
716+ "name": "weight",
717+ "index": 1,
718+ "dtype": "int8",
719+ "format": "FRACTAL_NZ",
720+ "paramType": "required",
721+ "shape": [
722+ -2
723+ ]
724+ },
725+ {
726+ "name": "antiquant_scale",
727+ "index": 2,
728+ "dtype": "bfloat16",
729+ "format": "ND",
730+ "paramType": "required",
731+ "shape": [
732+ -2
733+ ]
734+ },
735+ {
736+ "name": "antiquant_offset",
737+ "index": 3,
738+ "dtype": "bfloat16",
739+ "format": "ND",
740+ "paramType": "optional",
741+ "shape": [
742+ -2
743+ ]
744+ },
745+ {
746+ "name": "quant_scale",
747+ "index": 4,
748+ "dtype": "uint64",
749+ "format": "ND",
750+ "paramType": "optional",
751+ "shape": [
752+ -2
753+ ]
754+ },
755+ {
756+ "name": "quant_offset",
757+ "index": 5,
758+ "dtype": "float32",
759+ "format": "ND",
760+ "paramType": "optional",
761+ "shape": [
762+ -2
763+ ]
764+ },
765+ {
766+ "name": "bias",
767+ "index": 6,
768+ "dtype": "float32",
769+ "format": "ND",
770+ "paramType": "optional",
771+ "shape": [
772+ -2
773+ ]
774+ }
775+ ],
776+ "outputs": [
777+ {
778+ "name": "y",
779+ "index": 0,
780+ "dtype": "bfloat16",
781+ "format": "ND",
782+ "paramType": "required",
783+ "shape": [
784+ -2
785+ ]
786+ }
787+ ],
788+ "attrs": [
789+ {
790+ "name": "transpose_x",
791+ "dtype": "bool",
792+ "value": null
793+ },
794+ {
795+ "name": "transpose_weight",
796+ "dtype": "bool",
797+ "value": null
798+ },
799+ {
800+ "name": "antiquant_group_size",
801+ "dtype": "int",
802+ "value": 0
803+ },
804+ {
805+ "name": "dtype",
806+ "dtype": "int",
807+ "value": -1
808+ },
809+ {
810+ "name": "inner_precise",
811+ "dtype": "int",
812+ "value": 0
813+ }
814+ ]
815+ },
816+ {
817+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_int4_weight_nd_fp16",
818+ "inputs": [
819+ {
820+ "name": "x",
821+ "index": 0,
822+ "dtype": "float16",
823+ "format": "ND",
824+ "paramType": "required",
825+ "shape": [
826+ -2
827+ ]
828+ },
829+ {
830+ "name": "weight",
831+ "index": 1,
832+ "dtype": "int4",
833+ "format": "ND",
834+ "paramType": "required",
835+ "shape": [
836+ -2
837+ ]
838+ },
839+ {
840+ "name": "antiquant_scale",
841+ "index": 2,
842+ "dtype": "float16",
843+ "format": "ND",
844+ "paramType": "required",
845+ "shape": [
846+ -2
847+ ]
848+ },
849+ {
850+ "name": "antiquant_offset",
851+ "index": 3,
852+ "dtype": "float16",
853+ "format": "ND",
854+ "paramType": "optional",
855+ "shape": [
856+ -2
857+ ]
858+ },
859+ {
860+ "name": "quant_scale",
861+ "index": 4,
862+ "dtype": "uint64",
863+ "format": "ND",
864+ "paramType": "optional",
865+ "shape": [
866+ -2
867+ ]
868+ },
869+ {
870+ "name": "quant_offset",
871+ "index": 5,
872+ "dtype": "float32",
873+ "format": "ND",
874+ "paramType": "optional",
875+ "shape": [
876+ -2
877+ ]
878+ },
879+ {
880+ "name": "bias",
881+ "index": 6,
882+ "dtype": "float16",
883+ "format": "ND",
884+ "paramType": "optional",
885+ "shape": [
886+ -2
887+ ]
888+ }
889+ ],
890+ "outputs": [
891+ {
892+ "name": "y",
893+ "index": 0,
894+ "dtype": "float16",
895+ "format": "ND",
896+ "paramType": "required",
897+ "shape": [
898+ -2
899+ ]
900+ }
901+ ],
902+ "attrs": [
903+ {
904+ "name": "transpose_x",
905+ "dtype": "bool",
906+ "value": null
907+ },
908+ {
909+ "name": "transpose_weight",
910+ "dtype": "bool",
911+ "value": null
912+ },
913+ {
914+ "name": "antiquant_group_size",
915+ "dtype": "int",
916+ "value": 0
917+ },
918+ {
919+ "name": "dtype",
920+ "dtype": "int",
921+ "value": -1
922+ },
923+ {
924+ "name": "inner_precise",
925+ "dtype": "int",
926+ "value": 0
927+ }
928+ ]
929+ },
930+ {
931+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_int4_weight_nd_bf16",
932+ "inputs": [
933+ {
934+ "name": "x",
935+ "index": 0,
936+ "dtype": "bfloat16",
937+ "format": "ND",
938+ "paramType": "required",
939+ "shape": [
940+ -2
941+ ]
942+ },
943+ {
944+ "name": "weight",
945+ "index": 1,
946+ "dtype": "int4",
947+ "format": "ND",
948+ "paramType": "required",
949+ "shape": [
950+ -2
951+ ]
952+ },
953+ {
954+ "name": "antiquant_scale",
955+ "index": 2,
956+ "dtype": "bfloat16",
957+ "format": "ND",
958+ "paramType": "required",
959+ "shape": [
960+ -2
961+ ]
962+ },
963+ {
964+ "name": "antiquant_offset",
965+ "index": 3,
966+ "dtype": "bfloat16",
967+ "format": "ND",
968+ "paramType": "optional",
969+ "shape": [
970+ -2
971+ ]
972+ },
973+ {
974+ "name": "quant_scale",
975+ "index": 4,
976+ "dtype": "uint64",
977+ "format": "ND",
978+ "paramType": "optional",
979+ "shape": [
980+ -2
981+ ]
982+ },
983+ {
984+ "name": "quant_offset",
985+ "index": 5,
986+ "dtype": "float32",
987+ "format": "ND",
988+ "paramType": "optional",
989+ "shape": [
990+ -2
991+ ]
992+ },
993+ {
994+ "name": "bias",
995+ "index": 6,
996+ "dtype": "bfloat16",
997+ "format": "ND",
998+ "paramType": "optional",
999+ "shape": [
1000+ -2
1001+ ]
1002+ }
1003+ ],
1004+ "outputs": [
1005+ {
1006+ "name": "y",
1007+ "index": 0,
1008+ "dtype": "bfloat16",
1009+ "format": "ND",
1010+ "paramType": "required",
1011+ "shape": [
1012+ -2
1013+ ]
1014+ }
1015+ ],
1016+ "attrs": [
1017+ {
1018+ "name": "transpose_x",
1019+ "dtype": "bool",
1020+ "value": null
1021+ },
1022+ {
1023+ "name": "transpose_weight",
1024+ "dtype": "bool",
1025+ "value": null
1026+ },
1027+ {
1028+ "name": "antiquant_group_size",
1029+ "dtype": "int",
1030+ "value": 0
1031+ },
1032+ {
1033+ "name": "dtype",
1034+ "dtype": "int",
1035+ "value": -1
1036+ },
1037+ {
1038+ "name": "inner_precise",
1039+ "dtype": "int",
1040+ "value": 0
1041+ }
1042+ ]
1043+ },
1044+ {
1045+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_float8_e4m3fn_weight_nd_fp16",
1046+ "inputs": [
1047+ {
1048+ "name": "x",
1049+ "index": 0,
1050+ "dtype": "float16",
1051+ "format": "ND",
1052+ "paramType": "required",
1053+ "shape": [
1054+ -2
1055+ ]
1056+ },
1057+ {
1058+ "name": "weight",
1059+ "index": 1,
1060+ "dtype": "float8_e4m3fn",
1061+ "format": "ND",
1062+ "paramType": "required",
1063+ "shape": [
1064+ -2
1065+ ]
1066+ },
1067+ {
1068+ "name": "antiquant_scale",
1069+ "index": 2,
1070+ "dtype": "float16",
1071+ "format": "ND",
1072+ "paramType": "required",
1073+ "shape": [
1074+ -2
1075+ ]
1076+ },
1077+ {
1078+ "name": "antiquant_offset",
1079+ "index": 3,
1080+ "dtype": "float16",
1081+ "format": "ND",
1082+ "paramType": "optional",
1083+ "shape": [
1084+ -2
1085+ ]
1086+ },
1087+ {
1088+ "name": "quant_scale",
1089+ "index": 4,
1090+ "dtype": "uint64",
1091+ "format": "ND",
1092+ "paramType": "optional",
1093+ "shape": [
1094+ -2
1095+ ]
1096+ },
1097+ {
1098+ "name": "quant_offset",
1099+ "index": 5,
1100+ "dtype": "float32",
1101+ "format": "ND",
1102+ "paramType": "optional",
1103+ "shape": [
1104+ -2
1105+ ]
1106+ },
1107+ {
1108+ "name": "bias",
1109+ "index": 6,
1110+ "dtype": "float16",
1111+ "format": "ND",
1112+ "paramType": "optional",
1113+ "shape": [
1114+ -2
1115+ ]
1116+ }
1117+ ],
1118+ "outputs": [
1119+ {
1120+ "name": "y",
1121+ "index": 0,
1122+ "dtype": "float16",
1123+ "format": "ND",
1124+ "paramType": "required",
1125+ "shape": [
1126+ -2
1127+ ]
1128+ }
1129+ ],
1130+ "attrs": [
1131+ {
1132+ "name": "transpose_x",
1133+ "dtype": "bool",
1134+ "value": null
1135+ },
1136+ {
1137+ "name": "transpose_weight",
1138+ "dtype": "bool",
1139+ "value": null
1140+ },
1141+ {
1142+ "name": "antiquant_group_size",
1143+ "dtype": "int",
1144+ "value": 0
1145+ },
1146+ {
1147+ "name": "dtype",
1148+ "dtype": "int",
1149+ "value": -1
1150+ },
1151+ {
1152+ "name": "inner_precise",
1153+ "dtype": "int",
1154+ "value": 0
1155+ }
1156+ ]
1157+ },
1158+ {
1159+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_float8_e4m3fn_weight_nd_bf16",
1160+ "inputs": [
1161+ {
1162+ "name": "x",
1163+ "index": 0,
1164+ "dtype": "bfloat16",
1165+ "format": "ND",
1166+ "paramType": "required",
1167+ "shape": [
1168+ -2
1169+ ]
1170+ },
1171+ {
1172+ "name": "weight",
1173+ "index": 1,
1174+ "dtype": "float8_e4m3fn",
1175+ "format": "ND",
1176+ "paramType": "required",
1177+ "shape": [
1178+ -2
1179+ ]
1180+ },
1181+ {
1182+ "name": "antiquant_scale",
1183+ "index": 2,
1184+ "dtype": "bfloat16",
1185+ "format": "ND",
1186+ "paramType": "required",
1187+ "shape": [
1188+ -2
1189+ ]
1190+ },
1191+ {
1192+ "name": "antiquant_offset",
1193+ "index": 3,
1194+ "dtype": "bfloat16",
1195+ "format": "ND",
1196+ "paramType": "optional",
1197+ "shape": [
1198+ -2
1199+ ]
1200+ },
1201+ {
1202+ "name": "quant_scale",
1203+ "index": 4,
1204+ "dtype": "uint64",
1205+ "format": "ND",
1206+ "paramType": "optional",
1207+ "shape": [
1208+ -2
1209+ ]
1210+ },
1211+ {
1212+ "name": "quant_offset",
1213+ "index": 5,
1214+ "dtype": "float32",
1215+ "format": "ND",
1216+ "paramType": "optional",
1217+ "shape": [
1218+ -2
1219+ ]
1220+ },
1221+ {
1222+ "name": "bias",
1223+ "index": 6,
1224+ "dtype": "bfloat16",
1225+ "format": "ND",
1226+ "paramType": "optional",
1227+ "shape": [
1228+ -2
1229+ ]
1230+ }
1231+ ],
1232+ "outputs": [
1233+ {
1234+ "name": "y",
1235+ "index": 0,
1236+ "dtype": "bfloat16",
1237+ "format": "ND",
1238+ "paramType": "required",
1239+ "shape": [
1240+ -2
1241+ ]
1242+ }
1243+ ],
1244+ "attrs": [
1245+ {
1246+ "name": "transpose_x",
1247+ "dtype": "bool",
1248+ "value": null
1249+ },
1250+ {
1251+ "name": "transpose_weight",
1252+ "dtype": "bool",
1253+ "value": null
1254+ },
1255+ {
1256+ "name": "antiquant_group_size",
1257+ "dtype": "int",
1258+ "value": 0
1259+ },
1260+ {
1261+ "name": "dtype",
1262+ "dtype": "int",
1263+ "value": -1
1264+ },
1265+ {
1266+ "name": "inner_precise",
1267+ "dtype": "int",
1268+ "value": 0
1269+ }
1270+ ]
1271+ },
1272+ {
1273+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_hifloat8_weight_nd_fp16",
1274+ "inputs": [
1275+ {
1276+ "name": "x",
1277+ "index": 0,
1278+ "dtype": "float16",
1279+ "format": "ND",
1280+ "paramType": "required",
1281+ "shape": [
1282+ -2
1283+ ]
1284+ },
1285+ {
1286+ "name": "weight",
1287+ "index": 1,
1288+ "dtype": "hifloat8",
1289+ "format": "ND",
1290+ "paramType": "required",
1291+ "shape": [
1292+ -2
1293+ ]
1294+ },
1295+ {
1296+ "name": "antiquant_scale",
1297+ "index": 2,
1298+ "dtype": "float16",
1299+ "format": "ND",
1300+ "paramType": "required",
1301+ "shape": [
1302+ -2
1303+ ]
1304+ },
1305+ {
1306+ "name": "antiquant_offset",
1307+ "index": 3,
1308+ "dtype": "float16",
1309+ "format": "ND",
1310+ "paramType": "optional",
1311+ "shape": [
1312+ -2
1313+ ]
1314+ },
1315+ {
1316+ "name": "quant_scale",
1317+ "index": 4,
1318+ "dtype": "uint64",
1319+ "format": "ND",
1320+ "paramType": "optional",
1321+ "shape": [
1322+ -2
1323+ ]
1324+ },
1325+ {
1326+ "name": "quant_offset",
1327+ "index": 5,
1328+ "dtype": "float32",
1329+ "format": "ND",
1330+ "paramType": "optional",
1331+ "shape": [
1332+ -2
1333+ ]
1334+ },
1335+ {
1336+ "name": "bias",
1337+ "index": 6,
1338+ "dtype": "float16",
1339+ "format": "ND",
1340+ "paramType": "optional",
1341+ "shape": [
1342+ -2
1343+ ]
1344+ }
1345+ ],
1346+ "outputs": [
1347+ {
1348+ "name": "y",
1349+ "index": 0,
1350+ "dtype": "float16",
1351+ "format": "ND",
1352+ "paramType": "required",
1353+ "shape": [
1354+ -2
1355+ ]
1356+ }
1357+ ],
1358+ "attrs": [
1359+ {
1360+ "name": "transpose_x",
1361+ "dtype": "bool",
1362+ "value": null
1363+ },
1364+ {
1365+ "name": "transpose_weight",
1366+ "dtype": "bool",
1367+ "value": null
1368+ },
1369+ {
1370+ "name": "antiquant_group_size",
1371+ "dtype": "int",
1372+ "value": 0
1373+ },
1374+ {
1375+ "name": "dtype",
1376+ "dtype": "int",
1377+ "value": -1
1378+ },
1379+ {
1380+ "name": "inner_precise",
1381+ "dtype": "int",
1382+ "value": 0
1383+ }
1384+ ]
1385+ },
1386+ {
1387+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_hifloat8_weight_nd_bf16",
1388+ "inputs": [
1389+ {
1390+ "name": "x",
1391+ "index": 0,
1392+ "dtype": "bfloat16",
1393+ "format": "ND",
1394+ "paramType": "required",
1395+ "shape": [
1396+ -2
1397+ ]
1398+ },
1399+ {
1400+ "name": "weight",
1401+ "index": 1,
1402+ "dtype": "hifloat8",
1403+ "format": "ND",
1404+ "paramType": "required",
1405+ "shape": [
1406+ -2
1407+ ]
1408+ },
1409+ {
1410+ "name": "antiquant_scale",
1411+ "index": 2,
1412+ "dtype": "bfloat16",
1413+ "format": "ND",
1414+ "paramType": "required",
1415+ "shape": [
1416+ -2
1417+ ]
1418+ },
1419+ {
1420+ "name": "antiquant_offset",
1421+ "index": 3,
1422+ "dtype": "bfloat16",
1423+ "format": "ND",
1424+ "paramType": "optional",
1425+ "shape": [
1426+ -2
1427+ ]
1428+ },
1429+ {
1430+ "name": "quant_scale",
1431+ "index": 4,
1432+ "dtype": "uint64",
1433+ "format": "ND",
1434+ "paramType": "optional",
1435+ "shape": [
1436+ -2
1437+ ]
1438+ },
1439+ {
1440+ "name": "quant_offset",
1441+ "index": 5,
1442+ "dtype": "float32",
1443+ "format": "ND",
1444+ "paramType": "optional",
1445+ "shape": [
1446+ -2
1447+ ]
1448+ },
1449+ {
1450+ "name": "bias",
1451+ "index": 6,
1452+ "dtype": "bfloat16",
1453+ "format": "ND",
1454+ "paramType": "optional",
1455+ "shape": [
1456+ -2
1457+ ]
1458+ }
1459+ ],
1460+ "outputs": [
1461+ {
1462+ "name": "y",
1463+ "index": 0,
1464+ "dtype": "bfloat16",
1465+ "format": "ND",
1466+ "paramType": "required",
1467+ "shape": [
1468+ -2
1469+ ]
1470+ }
1471+ ],
1472+ "attrs": [
1473+ {
1474+ "name": "transpose_x",
1475+ "dtype": "bool",
1476+ "value": null
1477+ },
1478+ {
1479+ "name": "transpose_weight",
1480+ "dtype": "bool",
1481+ "value": null
1482+ },
1483+ {
1484+ "name": "antiquant_group_size",
1485+ "dtype": "int",
1486+ "value": 0
1487+ },
1488+ {
1489+ "name": "dtype",
1490+ "dtype": "int",
1491+ "value": -1
1492+ },
1493+ {
1494+ "name": "inner_precise",
1495+ "dtype": "int",
1496+ "value": 0
1497+ }
1498+ ]
1499+ },
1500+ {
1501+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_float4_e2m1_weight_nd_mx_fp16",
1502+ "inputs": [
1503+ {
1504+ "name": "x",
1505+ "index": 0,
1506+ "dtype": "float16",
1507+ "format": "ND",
1508+ "paramType": "required",
1509+ "shape": [
1510+ -2
1511+ ]
1512+ },
1513+ {
1514+ "name": "weight",
1515+ "index": 1,
1516+ "dtype": "float4_e2m1",
1517+ "format": "ND",
1518+ "paramType": "required",
1519+ "shape": [
1520+ -2
1521+ ]
1522+ },
1523+ {
1524+ "name": "antiquant_scale",
1525+ "index": 2,
1526+ "dtype": "float8_e8m0",
1527+ "format": "ND",
1528+ "paramType": "required",
1529+ "shape": [
1530+ -2
1531+ ]
1532+ },
1533+ {
1534+ "name": "antiquant_offset",
1535+ "index": 3,
1536+ "dtype": "float16",
1537+ "format": "ND",
1538+ "paramType": "optional",
1539+ "shape": [
1540+ -2
1541+ ]
1542+ },
1543+ {
1544+ "name": "quant_scale",
1545+ "index": 4,
1546+ "dtype": "uint64",
1547+ "format": "ND",
1548+ "paramType": "optional",
1549+ "shape": [
1550+ -2
1551+ ]
1552+ },
1553+ {
1554+ "name": "quant_offset",
1555+ "index": 5,
1556+ "dtype": "float32",
1557+ "format": "ND",
1558+ "paramType": "optional",
1559+ "shape": [
1560+ -2
1561+ ]
1562+ },
1563+ {
1564+ "name": "bias",
1565+ "index": 6,
1566+ "dtype": "float16",
1567+ "format": "ND",
1568+ "paramType": "optional",
1569+ "shape": [
1570+ -2
1571+ ]
1572+ }
1573+ ],
1574+ "outputs": [
1575+ {
1576+ "name": "y",
1577+ "index": 0,
1578+ "dtype": "float16",
1579+ "format": "ND",
1580+ "paramType": "required",
1581+ "shape": [
1582+ -2
1583+ ]
1584+ }
1585+ ],
1586+ "attrs": [
1587+ {
1588+ "name": "transpose_x",
1589+ "dtype": "bool",
1590+ "value": null
1591+ },
1592+ {
1593+ "name": "transpose_weight",
1594+ "dtype": "bool",
1595+ "value": null
1596+ },
1597+ {
1598+ "name": "antiquant_group_size",
1599+ "dtype": "int",
1600+ "value": 0
1601+ },
1602+ {
1603+ "name": "dtype",
1604+ "dtype": "int",
1605+ "value": -1
1606+ },
1607+ {
1608+ "name": "inner_precise",
1609+ "dtype": "int",
1610+ "value": 0
1611+ }
1612+ ]
1613+ },
1614+ {
1615+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_float4_e2m1_weight_nd_mx_bf16",
1616+ "inputs": [
1617+ {
1618+ "name": "x",
1619+ "index": 0,
1620+ "dtype": "bfloat16",
1621+ "format": "ND",
1622+ "paramType": "required",
1623+ "shape": [
1624+ -2
1625+ ]
1626+ },
1627+ {
1628+ "name": "weight",
1629+ "index": 1,
1630+ "dtype": "float4_e2m1",
1631+ "format": "ND",
1632+ "paramType": "required",
1633+ "shape": [
1634+ -2
1635+ ]
1636+ },
1637+ {
1638+ "name": "antiquant_scale",
1639+ "index": 2,
1640+ "dtype": "float8_e8m0",
1641+ "format": "ND",
1642+ "paramType": "required",
1643+ "shape": [
1644+ -2
1645+ ]
1646+ },
1647+ {
1648+ "name": "antiquant_offset",
1649+ "index": 3,
1650+ "dtype": "bfloat16",
1651+ "format": "ND",
1652+ "paramType": "optional",
1653+ "shape": [
1654+ -2
1655+ ]
1656+ },
1657+ {
1658+ "name": "quant_scale",
1659+ "index": 4,
1660+ "dtype": "uint64",
1661+ "format": "ND",
1662+ "paramType": "optional",
1663+ "shape": [
1664+ -2
1665+ ]
1666+ },
1667+ {
1668+ "name": "quant_offset",
1669+ "index": 5,
1670+ "dtype": "float32",
1671+ "format": "ND",
1672+ "paramType": "optional",
1673+ "shape": [
1674+ -2
1675+ ]
1676+ },
1677+ {
1678+ "name": "bias",
1679+ "index": 6,
1680+ "dtype": "bfloat16",
1681+ "format": "ND",
1682+ "paramType": "optional",
1683+ "shape": [
1684+ -2
1685+ ]
1686+ }
1687+ ],
1688+ "outputs": [
1689+ {
1690+ "name": "y",
1691+ "index": 0,
1692+ "dtype": "bfloat16",
1693+ "format": "ND",
1694+ "paramType": "required",
1695+ "shape": [
1696+ -2
1697+ ]
1698+ }
1699+ ],
1700+ "attrs": [
1701+ {
1702+ "name": "transpose_x",
1703+ "dtype": "bool",
1704+ "value": null
1705+ },
1706+ {
1707+ "name": "transpose_weight",
1708+ "dtype": "bool",
1709+ "value": null
1710+ },
1711+ {
1712+ "name": "antiquant_group_size",
1713+ "dtype": "int",
1714+ "value": 0
1715+ },
1716+ {
1717+ "name": "dtype",
1718+ "dtype": "int",
1719+ "value": -1
1720+ },
1721+ {
1722+ "name": "inner_precise",
1723+ "dtype": "int",
1724+ "value": 0
1725+ }
1726+ ]
1727+ },
1728+ {
1729+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_float4_e2m1_weight_nz_mx_fp16",
1730+ "inputs": [
1731+ {
1732+ "name": "x",
1733+ "index": 0,
1734+ "dtype": "float16",
1735+ "dtypeForBinQuery": [
1736+ "float16",
1737+ "float16"
1738+ ],
1739+ "format": "ND",
1740+ "formatForBinQuery": [
1741+ "ND",
1742+ "ND"
1743+ ],
1744+ "paramType": "required",
1745+ "shape": [
1746+ -2
1747+ ]
1748+ },
1749+ {
1750+ "name": "weight",
1751+ "index": 1,
1752+ "dtype": "float4_e2m1",
1753+ "dtypeForBinQuery": [
1754+ "float4_e2m1",
1755+ "float32"
1756+ ],
1757+ "format": "FRACTAL_NZ",
1758+ "formatForBinQuery": [
1759+ "FRACTAL_NZ",
1760+ "FRACTAL_NZ_C0_2"
1761+ ],
1762+ "paramType": "required",
1763+ "shape": [
1764+ -2
1765+ ]
1766+ },
1767+ {
1768+ "name": "antiquant_scale",
1769+ "index": 2,
1770+ "dtype": "float8_e8m0",
1771+ "dtypeForBinQuery": [
1772+ "float8_e8m0",
1773+ "float8_e8m0"
1774+ ],
1775+ "format": "ND",
1776+ "formatForBinQuery": [
1777+ "ND",
1778+ "ND"
1779+ ],
1780+ "paramType": "required",
1781+ "shape": [
1782+ -2
1783+ ]
1784+ },
1785+ {
1786+ "name": "antiquant_offset",
1787+ "index": 3,
1788+ "dtype": "float16",
1789+ "dtypeForBinQuery": [
1790+ "float16",
1791+ "float16"
1792+ ],
1793+ "format": "ND",
1794+ "formatForBinQuery": [
1795+ "ND",
1796+ "ND"
1797+ ],
1798+ "paramType": "optional",
1799+ "shape": [
1800+ -2
1801+ ]
1802+ },
1803+ {
1804+ "name": "quant_scale",
1805+ "index": 4,
1806+ "dtype": "uint64",
1807+ "dtypeForBinQuery": [
1808+ "uint64",
1809+ "uint64"
1810+ ],
1811+ "format": "ND",
1812+ "formatForBinQuery": [
1813+ "ND",
1814+ "ND"
1815+ ],
1816+ "paramType": "optional",
1817+ "shape": [
1818+ -2
1819+ ]
1820+ },
1821+ {
1822+ "name": "quant_offset",
1823+ "index": 5,
1824+ "dtype": "float32",
1825+ "dtypeForBinQuery": [
1826+ "float32",
1827+ "float32"
1828+ ],
1829+ "format": "ND",
1830+ "formatForBinQuery": [
1831+ "ND",
1832+ "ND"
1833+ ],
1834+ "paramType": "optional",
1835+ "shape": [
1836+ -2
1837+ ]
1838+ },
1839+ {
1840+ "name": "bias",
1841+ "index": 6,
1842+ "dtype": "float16",
1843+ "dtypeForBinQuery": [
1844+ "float16",
1845+ "float16"
1846+ ],
1847+ "format": "ND",
1848+ "formatForBinQuery": [
1849+ "ND",
1850+ "ND"
1851+ ],
1852+ "paramType": "optional",
1853+ "shape": [
1854+ -2
1855+ ]
1856+ }
1857+ ],
1858+ "outputs": [
1859+ {
1860+ "name": "y",
1861+ "index": 0,
1862+ "dtype": "float16",
1863+ "dtypeForBinQuery": [
1864+ "float16",
1865+ "float16"
1866+ ],
1867+ "format": "ND",
1868+ "formatForBinQuery": [
1869+ "ND",
1870+ "ND"
1871+ ],
1872+ "paramType": "required",
1873+ "shape": [
1874+ -2
1875+ ]
1876+ }
1877+ ],
1878+ "attrs": [
1879+ {
1880+ "name": "transpose_x",
1881+ "dtype": "bool",
1882+ "value": null
1883+ },
1884+ {
1885+ "name": "transpose_weight",
1886+ "dtype": "bool",
1887+ "value": null
1888+ },
1889+ {
1890+ "name": "antiquant_group_size",
1891+ "dtype": "int",
1892+ "value": 0
1893+ },
1894+ {
1895+ "name": "dtype",
1896+ "dtype": "int",
1897+ "value": -1
1898+ },
1899+ {
1900+ "name": "inner_precise",
1901+ "dtype": "int",
1902+ "value": 0
1903+ }
1904+ ]
1905+ },
1906+ {
1907+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_float4_e2m1_weight_nz_mx_bf16",
1908+ "inputs": [
1909+ {
1910+ "name": "x",
1911+ "index": 0,
1912+ "dtype": "bfloat16",
1913+ "dtypeForBinQuery": [
1914+ "bfloat16",
1915+ "bfloat16"
1916+ ],
1917+ "format": "ND",
1918+ "formatForBinQuery": [
1919+ "ND",
1920+ "ND"
1921+ ],
1922+ "paramType": "required",
1923+ "shape": [
1924+ -2
1925+ ]
1926+ },
1927+ {
1928+ "name": "weight",
1929+ "index": 1,
1930+ "dtype": "float4_e2m1",
1931+ "dtypeForBinQuery": [
1932+ "float4_e2m1",
1933+ "float32"
1934+ ],
1935+ "format": "FRACTAL_NZ",
1936+ "formatForBinQuery": [
1937+ "FRACTAL_NZ",
1938+ "FRACTAL_NZ_C0_2"
1939+ ],
1940+ "paramType": "required",
1941+ "shape": [
1942+ -2
1943+ ]
1944+ },
1945+ {
1946+ "name": "antiquant_scale",
1947+ "index": 2,
1948+ "dtype": "float8_e8m0",
1949+ "dtypeForBinQuery": [
1950+ "float8_e8m0",
1951+ "float8_e8m0"
1952+ ],
1953+ "format": "ND",
1954+ "formatForBinQuery": [
1955+ "ND",
1956+ "ND"
1957+ ],
1958+ "paramType": "required",
1959+ "shape": [
1960+ -2
1961+ ]
1962+ },
1963+ {
1964+ "name": "antiquant_offset",
1965+ "index": 3,
1966+ "dtype": "bfloat16",
1967+ "dtypeForBinQuery": [
1968+ "bfloat16",
1969+ "bfloat16"
1970+ ],
1971+ "format": "ND",
1972+ "formatForBinQuery": [
1973+ "ND",
1974+ "ND"
1975+ ],
1976+ "paramType": "optional",
1977+ "shape": [
1978+ -2
1979+ ]
1980+ },
1981+ {
1982+ "name": "quant_scale",
1983+ "index": 4,
1984+ "dtype": "uint64",
1985+ "dtypeForBinQuery": [
1986+ "uint64",
1987+ "uint64"
1988+ ],
1989+ "format": "ND",
1990+ "formatForBinQuery": [
1991+ "ND",
1992+ "ND"
1993+ ],
1994+ "paramType": "optional",
1995+ "shape": [
1996+ -2
1997+ ]
1998+ },
1999+ {
2000+ "name": "quant_offset",
2001+ "index": 5,
2002+ "dtype": "float32",
2003+ "dtypeForBinQuery": [
2004+ "float32",
2005+ "float32"
2006+ ],
2007+ "format": "ND",
2008+ "formatForBinQuery": [
2009+ "ND",
2010+ "ND"
2011+ ],
2012+ "paramType": "optional",
2013+ "shape": [
2014+ -2
2015+ ]
2016+ },
2017+ {
2018+ "name": "bias",
2019+ "index": 6,
2020+ "dtype": "bfloat16",
2021+ "dtypeForBinQuery": [
2022+ "bfloat16",
2023+ "bfloat16"
2024+ ],
2025+ "format": "ND",
2026+ "formatForBinQuery": [
2027+ "ND",
2028+ "ND"
2029+ ],
2030+ "paramType": "optional",
2031+ "shape": [
2032+ -2
2033+ ]
2034+ }
2035+ ],
2036+ "outputs": [
2037+ {
2038+ "name": "y",
2039+ "index": 0,
2040+ "dtype": "bfloat16",
2041+ "dtypeForBinQuery": [
2042+ "bfloat16",
2043+ "bfloat16"
2044+ ],
2045+ "format": "ND",
2046+ "formatForBinQuery": [
2047+ "ND",
2048+ "ND"
2049+ ],
2050+ "paramType": "required",
2051+ "shape": [
2052+ -2
2053+ ]
2054+ }
2055+ ],
2056+ "attrs": [
2057+ {
2058+ "name": "transpose_x",
2059+ "dtype": "bool",
2060+ "value": null
2061+ },
2062+ {
2063+ "name": "transpose_weight",
2064+ "dtype": "bool",
2065+ "value": null
2066+ },
2067+ {
2068+ "name": "antiquant_group_size",
2069+ "dtype": "int",
2070+ "value": 0
2071+ },
2072+ {
2073+ "name": "dtype",
2074+ "dtype": "int",
2075+ "value": -1
2076+ },
2077+ {
2078+ "name": "inner_precise",
2079+ "dtype": "int",
2080+ "value": 0
2081+ }
2082+ ]
2083+ },
2084+ {
2085+ "bin_filename": "WeightQuantBatchMatmulV2_fp16_float4_e2m1_weight_nz_fp16",
2086+ "inputs": [
2087+ {
2088+ "name": "x",
2089+ "index": 0,
2090+ "dtype": "float16",
2091+ "dtypeForBinQuery": [
2092+ "float16",
2093+ "float16"
2094+ ],
2095+ "format": "ND",
2096+ "formatForBinQuery": [
2097+ "ND",
2098+ "ND"
2099+ ],
2100+ "paramType": "required",
2101+ "shape": [
2102+ -2
2103+ ]
2104+ },
2105+ {
2106+ "name": "weight",
2107+ "index": 1,
2108+ "dtype": "float4_e2m1",
2109+ "dtypeForBinQuery": [
2110+ "float4_e2m1",
2111+ "float32"
2112+ ],
2113+ "format": "FRACTAL_NZ",
2114+ "formatForBinQuery": [
2115+ "FRACTAL_NZ",
2116+ "FRACTAL_NZ_C0_2"
2117+ ],
2118+ "paramType": "required",
2119+ "shape": [
2120+ -2
2121+ ]
2122+ },
2123+ {
2124+ "name": "antiquant_scale",
2125+ "index": 2,
2126+ "dtype": "float16",
2127+ "dtypeForBinQuery": [
2128+ "float16",
2129+ "float16"
2130+ ],
2131+ "format": "ND",
2132+ "formatForBinQuery": [
2133+ "ND",
2134+ "ND"
2135+ ],
2136+ "paramType": "required",
2137+ "shape": [
2138+ -2
2139+ ]
2140+ },
2141+ {
2142+ "name": "antiquant_offset",
2143+ "index": 3,
2144+ "dtype": "float16",
2145+ "dtypeForBinQuery": [
2146+ "float16",
2147+ "float16"
2148+ ],
2149+ "format": "ND",
2150+ "formatForBinQuery": [
2151+ "ND",
2152+ "ND"
2153+ ],
2154+ "paramType": "optional",
2155+ "shape": [
2156+ -2
2157+ ]
2158+ },
2159+ {
2160+ "name": "quant_scale",
2161+ "index": 4,
2162+ "dtype": "uint64",
2163+ "dtypeForBinQuery": [
2164+ "uint64",
2165+ "uint64"
2166+ ],
2167+ "format": "ND",
2168+ "formatForBinQuery": [
2169+ "ND",
2170+ "ND"
2171+ ],
2172+ "paramType": "optional",
2173+ "shape": [
2174+ -2
2175+ ]
2176+ },
2177+ {
2178+ "name": "quant_offset",
2179+ "index": 5,
2180+ "dtype": "float32",
2181+ "dtypeForBinQuery": [
2182+ "float32",
2183+ "float32"
2184+ ],
2185+ "format": "ND",
2186+ "formatForBinQuery": [
2187+ "ND",
2188+ "ND"
2189+ ],
2190+ "paramType": "optional",
2191+ "shape": [
2192+ -2
2193+ ]
2194+ },
2195+ {
2196+ "name": "bias",
2197+ "index": 6,
2198+ "dtype": "float16",
2199+ "dtypeForBinQuery": [
2200+ "float16",
2201+ "float16"
2202+ ],
2203+ "format": "ND",
2204+ "formatForBinQuery": [
2205+ "ND",
2206+ "ND"
2207+ ],
2208+ "paramType": "optional",
2209+ "shape": [
2210+ -2
2211+ ]
2212+ }
2213+ ],
2214+ "outputs": [
2215+ {
2216+ "name": "y",
2217+ "index": 0,
2218+ "dtype": "float16",
2219+ "dtypeForBinQuery": [
2220+ "float16",
2221+ "float16"
2222+ ],
2223+ "dtypeForBinQuery":[
2224+ "float16",
2225+ "float16"
2226+ ],
2227+ "format": "ND",
2228+ "formatForBinQuery": [
2229+ "ND",
2230+ "ND"
2231+ ],
2232+ "paramType": "required",
2233+ "shape": [
2234+ -2
2235+ ]
2236+ }
2237+ ],
2238+ "attrs": [
2239+ {
2240+ "name": "transpose_x",
2241+ "dtype": "bool",
2242+ "value": null
2243+ },
2244+ {
2245+ "name": "transpose_weight",
2246+ "dtype": "bool",
2247+ "value": null
2248+ },
2249+ {
2250+ "name": "antiquant_group_size",
2251+ "dtype": "int",
2252+ "value": 0
2253+ },
2254+ {
2255+ "name": "dtype",
2256+ "dtype": "int",
2257+ "value": -1
2258+ },
2259+ {
2260+ "name": "inner_precise",
2261+ "dtype": "int",
2262+ "value": 0
2263+ }
2264+ ]
2265+ },
2266+ {
2267+ "bin_filename": "WeightQuantBatchMatmulV2_bf16_float4_e2m1_weight_nz_bf16",
2268+ "inputs": [
2269+ {
2270+ "name": "x",
2271+ "index": 0,
2272+ "dtype": "bfloat16",
2273+ "dtypeForBinQuery": [
2274+ "bfloat16",
2275+ "bfloat16"
2276+ ],
2277+ "format": "ND",
2278+ "formatForBinQuery": [
2279+ "ND",
2280+ "ND"
2281+ ],
2282+ "paramType": "required",
2283+ "shape": [
2284+ -2
2285+ ]
2286+ },
2287+ {
2288+ "name": "weight",
2289+ "index": 1,
2290+ "dtype": "float4_e2m1",
2291+ "dtypeForBinQuery": [
2292+ "float4_e2m1",
2293+ "float32"
2294+ ],
2295+ "format": "FRACTAL_NZ",
2296+ "formatForBinQuery": [
2297+ "FRACTAL_NZ",
2298+ "FRACTAL_NZ_C0_2"
2299+ ],
2300+ "paramType": "required",
2301+ "shape": [
2302+ -2
2303+ ]
2304+ },
2305+ {
2306+ "name": "antiquant_scale",
2307+ "index": 2,
2308+ "dtype": "bfloat16",
2309+ "dtypeForBinQuery": [
2310+ "bfloat16",
2311+ "bfloat16"
2312+ ],
2313+ "format": "ND",
2314+ "formatForBinQuery": [
2315+ "ND",
2316+ "ND"
2317+ ],
2318+ "paramType": "required",
2319+ "shape": [
2320+ -2
2321+ ]
2322+ },
2323+ {
2324+ "name": "antiquant_offset",
2325+ "index": 3,
2326+ "dtype": "bfloat16",
2327+ "dtypeForBinQuery": [
2328+ "bfloat16",
2329+ "bfloat16"
2330+ ],
2331+ "format": "ND",
2332+ "formatForBinQuery": [
2333+ "ND",
2334+ "ND"
2335+ ],
2336+ "paramType": "optional",
2337+ "shape": [
2338+ -2
2339+ ]
2340+ },
2341+ {
2342+ "name": "quant_scale",
2343+ "index": 4,
2344+ "dtype": "uint64",
2345+ "dtypeForBinQuery": [
2346+ "uint64",
2347+ "uint64"
2348+ ],
2349+ "format": "ND",
2350+ "formatForBinQuery": [
2351+ "ND",
2352+ "ND"
2353+ ],
2354+ "paramType": "optional",
2355+ "shape": [
2356+ -2
2357+ ]
2358+ },
2359+ {
2360+ "name": "quant_offset",
2361+ "index": 5,
2362+ "dtype": "float32",
2363+ "dtypeForBinQuery": [
2364+ "float32",
2365+ "float32"
2366+ ],
2367+ "format": "ND",
2368+ "formatForBinQuery": [
2369+ "ND",
2370+ "ND"
2371+ ],
2372+ "paramType": "optional",
2373+ "shape": [
2374+ -2
2375+ ]
2376+ },
2377+ {
2378+ "name": "bias",
2379+ "index": 6,
2380+ "dtype": "bfloat16",
2381+ "dtypeForBinQuery": [
2382+ "bfloat16",
2383+ "bfloat16"
2384+ ],
2385+ "format": "ND",
2386+ "formatForBinQuery": [
2387+ "ND",
2388+ "ND"
2389+ ],
2390+ "paramType": "optional",
2391+ "shape": [
2392+ -2
2393+ ]
2394+ }
2395+ ],
2396+ "outputs": [
2397+ {
2398+ "name": "y",
2399+ "index": 0,
2400+ "dtype": "bfloat16",
2401+ "dtypeForBinQuery": [
2402+ "bfloat16",
2403+ "bfloat16"
2404+ ],
2405+ "format": "ND",
2406+ "formatForBinQuery": [
2407+ "ND",
2408+ "ND"
2409+ ],
2410+ "paramType": "required",
2411+ "shape": [
2412+ -2
2413+ ]
2414+ }
2415+ ],
2416+ "attrs": [
2417+ {
2418+ "name": "transpose_x",
2419+ "dtype": "bool",
2420+ "value": null
2421+ },
2422+ {
2423+ "name": "transpose_weight",
2424+ "dtype": "bool",
2425+ "value": null
2426+ },
2427+ {
2428+ "name": "antiquant_group_size",
2429+ "dtype": "int",
2430+ "value": 0
2431+ },
2432+ {
2433+ "name": "dtype",
2434+ "dtype": "int",
2435+ "value": -1
2436+ },
2437+ {
2438+ "name": "inner_precise",
2439+ "dtype": "int",
2440+ "value": 0
2441+ }
2442+ ]
2443+ }
2444+ ]
2445+}
Mmatmul/weight_quant_batch_matmul_v2/op_host/weight_quant_batch_matmul_v2_def.cpp+1-0
@@ -527,6 +527,7 @@ public:
527 .PrecisionReduceFlag(true)527 .PrecisionReduceFlag(true)
528 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");528 .ExtendCfgInfo("aclnnSupport.value", "support_aclnn");
529 this->AICore().AddConfig("ascend950", aicore_config_950);529 this->AICore().AddConfig("ascend950", aicore_config_950);
530+ this->AICore().AddConfig("ascend350", aicore_config_950);
530 }531 }
531};532};
532 533 
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/CMakeLists.txt+1-1
@@ -14,7 +14,7 @@ add_kernel_sources(
14 14 
15add_kernel_sources(15add_kernel_sources(
16 KERNEL_SRC arch35/weight_quant_batch_matmul_v2.cpp16 KERNEL_SRC arch35/weight_quant_batch_matmul_v2.cpp
17- COMPUTE_UNITS ascend95017+ COMPUTE_UNITS ascend950 ascend350
18 AUTO_SYNC false18 AUTO_SYNC false
19 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"19 OPTIONS "--cce-no-dcache-flush" "-DENABLE_CV_COMM_VIA_SSBUF=true"
20)20)