Star1111
2705
代码
代码
Issues204
Pull Requests369
Actions
项目讨论
Wiki
分析
Star1111
2705
已关闭
完成第二部分修改，提交代码 #10842
chen-wei43创建于 5 天前关闭于 5 天前
完成第二部分修改，提交代码 #10842
已关闭
chen-wei43创建于 5 天前关闭于 5 天前
从weixin_54263091/ops-transformermaster合入到cann/ops-transformermaster
共 28 个文件变更+1585-897
Mgmm/grouped_matmul/op_api/aclnn_grouped_matmul.cpp+233-163
@@ -1017,6 +1017,73 @@ static int64_t GetPergroupSize(const gmm::GroupedMatmulParams &gmmParams, size_t
1017 return pergroupSize;1017 return pergroupSize;
1018}1018}
1019 1019 
1020+static aclnnStatus CheckAntiQuantPergroupSize(const gmm::GroupedMatmulParams &gmmParams, const char *opName,
1021+ const char *scenario)
1022+{
1023+ // check perGroupNum
1024+ auto antiquantScale0Shape = (*gmmParams.antiquantScaleOptional)[0]->GetViewShape();
1025+ size_t antiquantScale0DimNum = antiquantScale0Shape.GetDimNum();
1026+ auto w0Shape = (*gmmParams.weight)[0]->GetViewShape();
1027+ size_t w0DimNum = w0Shape.GetDimNum();
1028+ int64_t pergroupSize = GetPergroupSize(gmmParams, w0DimNum, w0Shape, antiquantScale0Shape);
1029+ CHECK_COND(!gmmParams.transposeWeight || pergroupSize % 2 == 0, ACLNN_ERR_PARAM_INVALID, // 2: a factor
1030+ "In op [%s], when A16W4 per-group antiquant and weight is transposed, [%s] is not supported, got "
1031+ "[%ld]. Constraint:[pergroup size should be even].",
1032+ opName, "pergroupSize", pergroupSize);
1033+ for (size_t i = 0; i < gmmParams.antiquantScaleOptional->Size(); ++i) {
1034+ auto antiquantScaleShape = (*gmmParams.antiquantScaleOptional)[i]->GetViewShape();
1035+ size_t antiquantScaleDimNum = antiquantScaleShape.GetDimNum();
1036+ CHECK_COND(
1037+ antiquantScaleDimNum == antiquantScale0DimNum, ACLNN_ERR_PARAM_INVALID,
1038+ "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantScale[%zu] dim "
1039+ "num %zu should equal antiquantScale[0] dim num %zu when %s].",
1040+ opName, "antiquantScale", i, antiquantScaleDimNum, antiquantScale0DimNum, scenario);
1041+ auto wShape = (*gmmParams.weight)[i]->GetViewShape();
1042+ int64_t pergroupSizeOfScale = GetPergroupSize(gmmParams, w0DimNum, wShape, antiquantScaleShape);
1043+ CHECK_COND(pergroupSizeOfScale == pergroupSize, ACLNN_ERR_PARAM_INVALID,
1044+ "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantScale[%zu] "
1045+ "pergroup size %ld should be %ld when %s].",
1046+ opName, "antiquantScale, weight", i, pergroupSizeOfScale, pergroupSize, scenario);
1047+ if (gmmParams.antiquantOffsetOptional != nullptr) {
1048+ auto antiquantOffsetShape = (*gmmParams.antiquantOffsetOptional)[i]->GetViewShape();
1049+ size_t antiquantOffsetDimNum = antiquantOffsetShape.GetDimNum();
1050+ CHECK_COND(
1051+ antiquantScale0DimNum == antiquantOffsetDimNum, ACLNN_ERR_PARAM_INVALID,
1052+ "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantOffset[%zu] dim "
1053+ "num %zu should equal antiquantScale[0] dim num %zu when %s].",
1054+ opName, "antiquantOffset, antiquantScale", i, antiquantOffsetDimNum, antiquantScale0DimNum,
1055+ scenario);
1056+ int64_t pergroupSizeOfOffset = GetPergroupSize(gmmParams, w0DimNum, wShape, antiquantOffsetShape);
1057+ CHECK_COND(
1058+ pergroupSizeOfOffset == pergroupSize, ACLNN_ERR_PARAM_INVALID,
1059+ "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantOffset[%zu] "
1060+ "pergroup size %ld should be %ld when %s].",
1061+ opName, "antiquantOffset, weight", i, pergroupSizeOfOffset, pergroupSize, scenario);
1062+ }
1063+ }
1064+ return ACLNN_SUCCESS;
1065+}
1066+ 
1067+static aclnnStatus CheckAntiQuantDtypeAndEmpty(const gmm::GroupedMatmulParams &gmmParams, const char *opName,
1068+ const char *scenario)
1069+{
1070+ CHECK_COND(CheckTensorListDataType(gmmParams.antiquantScaleOptional, gmmParams.xDtype) == ACLNN_SUCCESS,
1071+ ACLNN_ERR_PARAM_INVALID,
1072+ "In op [%s], the data type of [%s] is not supported. Constraint:[antiquantScale dtype should be %s "
1073+ "when %s].",
1074+ opName, "antiquantScale", gmm::dTypeToString(gmmParams.xDtype).c_str(), scenario);
1075+ if (gmmParams.antiquantOffsetOptional != nullptr) {
1076+ CHECK_COND(CheckTensorListDataType(gmmParams.antiquantOffsetOptional, gmmParams.xDtype) == ACLNN_SUCCESS,
1077+ ACLNN_ERR_PARAM_INVALID,
1078+ "In op [%s], the data type of [%s] is not supported. Constraint:[antiquantOffset dtype should be %s "
1079+ "when %s].",
1080+ opName, "antiquantOffset", gmm::dTypeToString(gmmParams.xDtype).c_str(), scenario);
1081+ }
1082+ CHECK_COND(IsGmmQuantEmpty(gmmParams) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,
1083+ "In op [%s], when %s, [%s] must be empty.", opName, scenario, "quant inputs");
1084+ return ACLNN_SUCCESS;
1085+}
1086+ 
1020static aclnnStatus CheckGroupedMatmulAntiQuant(const gmm::GroupedMatmulParams &gmmParams, const char *opName)1087static aclnnStatus CheckGroupedMatmulAntiQuant(const gmm::GroupedMatmulParams &gmmParams, const char *opName)
1021{1088{
1022 DataType weightDtype = (*gmmParams.weight)[0]->GetDataType();1089 DataType weightDtype = (*gmmParams.weight)[0]->GetDataType();
@@ -1044,64 +1111,10 @@ static aclnnStatus CheckGroupedMatmulAntiQuant(const gmm::GroupedMatmulParams &g
1044 ACLNN_ERR_PARAM_INVALID, "In op [%s], when %s, [%s] tensor list is invalid.", opName, scenario,1111 ACLNN_ERR_PARAM_INVALID, "In op [%s], when %s, [%s] tensor list is invalid.", opName, scenario,
1045 "antiquantOffset");1112 "antiquantOffset");
1046 }1113 }
1047- // check perGroupNum
1048 if (isAntiquantInt4) {1114 if (isAntiquantInt4) {
1049- auto antiquantScale0Shape = (*gmmParams.antiquantScaleOptional)[0]->GetViewShape();1115+ CHECK_RET(CheckAntiQuantPergroupSize(gmmParams, opName, scenario) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1050- size_t antiquantScale0DimNum = antiquantScale0Shape.GetDimNum();
1051- auto w0Shape = (*gmmParams.weight)[0]->GetViewShape();
1052- size_t w0DimNum = w0Shape.GetDimNum();
1053- int64_t pergroupSize = GetPergroupSize(gmmParams, w0DimNum, w0Shape, antiquantScale0Shape);
1054- CHECK_COND(!gmmParams.transposeWeight || pergroupSize % 2 == 0, ACLNN_ERR_PARAM_INVALID, // 2: a factor
1055- "In op [%s], when A16W4 per-group antiquant and weight is transposed, [%s] is not supported, got "
1056- "[%ld]. Constraint:[pergroup size should be even].",
1057- opName, "pergroupSize", pergroupSize);
1058- for (size_t i = 0; i < gmmParams.antiquantScaleOptional->Size(); ++i) {
1059- auto antiquantScaleShape = (*gmmParams.antiquantScaleOptional)[i]->GetViewShape();
1060- size_t antiquantScaleDimNum = antiquantScaleShape.GetDimNum();
1061- CHECK_COND(
1062- antiquantScaleDimNum == antiquantScale0DimNum, ACLNN_ERR_PARAM_INVALID,
1063- "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantScale[%zu] dim "
1064- "num %zu should equal antiquantScale[0] dim num %zu when %s].",
1065- opName, "antiquantScale", i, antiquantScaleDimNum, antiquantScale0DimNum, scenario);
1066- auto wShape = (*gmmParams.weight)[i]->GetViewShape();
1067- int64_t pergroupSizeOfScale = GetPergroupSize(gmmParams, w0DimNum, wShape, antiquantScaleShape);
1068- CHECK_COND(pergroupSizeOfScale == pergroupSize, ACLNN_ERR_PARAM_INVALID,
1069- "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantScale[%zu] "
1070- "pergroup size %ld should be %ld when %s].",
1071- opName, "antiquantScale, weight", i, pergroupSizeOfScale, pergroupSize, scenario);
1072- if (gmmParams.antiquantOffsetOptional != nullptr) {
1073- auto antiquantOffsetShape = (*gmmParams.antiquantOffsetOptional)[i]->GetViewShape();
1074- size_t antiquantOffsetDimNum = antiquantOffsetShape.GetDimNum();
1075- CHECK_COND(
1076- antiquantScale0DimNum == antiquantOffsetDimNum, ACLNN_ERR_PARAM_INVALID,
1077- "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantOffset[%zu] dim "
1078- "num %zu should equal antiquantScale[0] dim num %zu when %s].",
1079- opName, "antiquantOffset, antiquantScale", i, antiquantOffsetDimNum, antiquantScale0DimNum,
1080- scenario);
1081- int64_t pergroupSizeOfOffset = GetPergroupSize(gmmParams, w0DimNum, wShape, antiquantOffsetShape);
1082- CHECK_COND(
1083- pergroupSizeOfOffset == pergroupSize, ACLNN_ERR_PARAM_INVALID,
1084- "In op [%s], the tensor shapes of [%s...] are mismatched, the reason is: [antiquantOffset[%zu] "
1085- "pergroup size %ld should be %ld when %s].",
1086- opName, "antiquantOffset, weight", i, pergroupSizeOfOffset, pergroupSize, scenario);
1087- }
1088- }
1089 }1116 }
1090- CHECK_COND(CheckTensorListDataType(gmmParams.antiquantScaleOptional, gmmParams.xDtype) == ACLNN_SUCCESS,1117+ return CheckAntiQuantDtypeAndEmpty(gmmParams, opName, scenario);
1091- ACLNN_ERR_PARAM_INVALID,
1092- "In op [%s], the data type of [%s] is not supported. Constraint:[antiquantScale dtype should be %s "
1093- "when %s].",
1094- opName, "antiquantScale", gmm::dTypeToString(gmmParams.xDtype).c_str(), scenario);
1095- if (gmmParams.antiquantOffsetOptional != nullptr) {
1096- CHECK_COND(CheckTensorListDataType(gmmParams.antiquantOffsetOptional, gmmParams.xDtype) == ACLNN_SUCCESS,
1097- ACLNN_ERR_PARAM_INVALID,
1098- "In op [%s], the data type of [%s] is not supported. Constraint:[antiquantOffset dtype should be %s "
1099- "when %s].",
1100- opName, "antiquantOffset", gmm::dTypeToString(gmmParams.xDtype).c_str(), scenario);
1101- }
1102- CHECK_COND(IsGmmQuantEmpty(gmmParams) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,
1103- "In op [%s], when %s, [%s] must be empty.", opName, scenario, "quant inputs");
1104- return ACLNN_SUCCESS;
1105}1118}
1106static aclnnStatus Check310PlatformForFunction(const gmm::GroupedMatmulParams &gmmParams, const DataType &weightDtype,1119static aclnnStatus Check310PlatformForFunction(const gmm::GroupedMatmulParams &gmmParams, const DataType &weightDtype,
1107 bool isNoActivation, const char *opName)1120 bool isNoActivation, const char *opName)
@@ -1754,31 +1767,31 @@ static aclnnStatus SplitMSingleXSingleWeightSingleY(const gmm::GroupedMatmulPara
1754 return ACLNN_SUCCESS;1767 return ACLNN_SUCCESS;
1755}1768}
1756 1769 
1757-static aclnnStatus SplitMSingleXSeparatedWeightSingleY(const gmm::GroupedMatmulParams &gmmParams, const char *opName)1770+static aclnnStatus CheckSplitMDimNumAndFormat(const gmm::GroupedMatmulParams &gmmParams, const char *opName,
1771+ const std::string &errorMessage, const std::string &scenario)
1758{1772{
1759- size_t weightSize = gmmParams.weight->Size();
1760- static const std::vector<std::string> TENSOR_WEIGHT_X{"Weight", "x", "true"};
1761- static const std::vector<std::string> TENSOR_X_Y{"x", "y", "false"};
1762- static const std::vector<std::string> TENSOR_WEIGHT_Y{"Weight", "y", "true"};
1763- std::string errorMessage =
1764- gmmParams.apiVersion != gmm::GMMApiVersion::V2 ? "split axis is M" : "groupType == 0(split-M)";
1765- CHECK_COND(gmmParams.splitItem == X_SEPARATED || gmmParams.splitItem == NO_SEPARATED, ACLNN_ERR_PARAM_INVALID,
1766- "In op [%s], when %s and y is not separated, [%s] is invalid, got [%ld]. Constraint:[splitItem should "
1767- "be 2 or 3].",
1768- opName, errorMessage.c_str(), "splitItem", gmmParams.splitItem);
1769 // check dim1773 // check dim
1770 CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.x, gmm::MIN_FM_DIM, "x") == ACLNN_SUCCESS,1774 CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.x, gmm::MIN_FM_DIM, "x") == ACLNN_SUCCESS,
1771 ACLNN_ERR_PARAM_INVALID,1775 ACLNN_ERR_PARAM_INVALID,
1772- "In op [%s], when %s with single x, separated weight and single y, [%s] dim num or format is invalid.",1776+ "In op [%s], when %s %s, [%s] dim num or format is invalid.",
1773- opName, errorMessage.c_str(), "x");1777+ opName, errorMessage.c_str(), scenario.c_str(), "x");
1774 CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.weight, SEPARATED_WEIGHT_DIM, "weight") == ACLNN_SUCCESS,1778 CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.weight, SEPARATED_WEIGHT_DIM, "weight") == ACLNN_SUCCESS,
1775 ACLNN_ERR_PARAM_INVALID,1779 ACLNN_ERR_PARAM_INVALID,
1776- "In op [%s], when %s with single x, separated weight and single y, [%s] dim num or format is invalid.",1780+ "In op [%s], when %s %s, [%s] dim num or format is invalid.",
1777- opName, errorMessage.c_str(), "weight");1781+ opName, errorMessage.c_str(), scenario.c_str(), "weight");
1778 CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.y, gmm::MIN_FM_DIM, "y") == ACLNN_SUCCESS,1782 CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.y, gmm::MIN_FM_DIM, "y") == ACLNN_SUCCESS,
1779 ACLNN_ERR_PARAM_INVALID,1783 ACLNN_ERR_PARAM_INVALID,
1780- "In op [%s], when %s with single x, separated weight and single y, [%s] dim num or format is invalid.",1784+ "In op [%s], when %s %s, [%s] dim num or format is invalid.",
1781- opName, errorMessage.c_str(), "y");1785+ opName, errorMessage.c_str(), scenario.c_str(), "y");
1786+ return ACLNN_SUCCESS;
1787+}
1788+ 
1789+static aclnnStatus CheckSplitMShape(const gmm::GroupedMatmulParams &gmmParams, const char *opName,
1790+ const std::string &errorMessage, size_t weightSize)
1791+{
1792+ static const std::vector<std::string> TENSOR_WEIGHT_X{"Weight", "x", "true"};
1793+ static const std::vector<std::string> TENSOR_X_Y{"x", "y", "false"};
1794+ static const std::vector<std::string> TENSOR_WEIGHT_Y{"Weight", "y", "true"};
1782 // check shape, x(m,k), weight(k,n), y(m,n)1795 // check shape, x(m,k), weight(k,n), y(m,n)
1783 int64_t innerAxisDimId = 1; // x always is not transposed, check K axis1796 int64_t innerAxisDimId = 1; // x always is not transposed, check K axis
1784 CHECK_COND(CheckShapeDiffLengthTensorList(gmmParams.weight, gmmParams.x, {0, 1}, innerAxisDimId, TENSOR_WEIGHT_X) ==1797 CHECK_COND(CheckShapeDiffLengthTensorList(gmmParams.weight, gmmParams.x, {0, 1}, innerAxisDimId, TENSOR_WEIGHT_X) ==
@@ -1806,6 +1819,22 @@ static aclnnStatus SplitMSingleXSeparatedWeightSingleY(const gmm::GroupedMatmulP
1806 ACLNN_ERR_PARAM_INVALID,1819 ACLNN_ERR_PARAM_INVALID,
1807 "In op [%s], when %s with single x, separated weight and single y, weight inner axis check failed.",1820 "In op [%s], when %s with single x, separated weight and single y, weight inner axis check failed.",
1808 opName, errorMessage.c_str());1821 opName, errorMessage.c_str());
1822+ return ACLNN_SUCCESS;
1823+}
1824+ 
1825+static aclnnStatus SplitMSingleXSeparatedWeightSingleY(const gmm::GroupedMatmulParams &gmmParams, const char *opName)
1826+{
1827+ size_t weightSize = gmmParams.weight->Size();
1828+ std::string errorMessage =
1829+ gmmParams.apiVersion != gmm::GMMApiVersion::V2 ? "split axis is M" : "groupType == 0(split-M)";
1830+ CHECK_COND(gmmParams.splitItem == X_SEPARATED || gmmParams.splitItem == NO_SEPARATED, ACLNN_ERR_PARAM_INVALID,
1831+ "In op [%s], when %s and y is not separated, [%s] is invalid, got [%ld]. Constraint:[splitItem should "
1832+ "be 2 or 3].",
1833+ opName, errorMessage.c_str(), "splitItem", gmmParams.splitItem);
1834+ CHECK_RET(CheckSplitMDimNumAndFormat(gmmParams, opName, errorMessage, "with single x, separated weight and single y") ==
1835+ ACLNN_SUCCESS,
1836+ ACLNN_ERR_PARAM_INVALID);
1837+ CHECK_RET(CheckSplitMShape(gmmParams, opName, errorMessage, weightSize) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1809 // check groupList1838 // check groupList
1810 CHECK_COND(CheckGroupListSplitM(gmmParams, true, false, false, weightSize, opName) == ACLNN_SUCCESS,1839 CHECK_COND(CheckGroupListSplitM(gmmParams, true, false, false, weightSize, opName) == ACLNN_SUCCESS,
1811 ACLNN_ERR_PARAM_INVALID,1840 ACLNN_ERR_PARAM_INVALID,
@@ -1882,39 +1911,12 @@ static aclnnStatus SplitMSingleXSeparatedWeightSeparatedY(const gmm::GroupedMatm
1882 return ACLNN_SUCCESS;1911 return ACLNN_SUCCESS;
1883}1912}
1884 1913 
1885-static aclnnStatus SplitMSeparatedXSeparatedWeightSingleY(const gmm::GroupedMatmulParams &gmmParams, const char *opName)1914+static aclnnStatus CheckSeparatedXShape(const gmm::GroupedMatmulParams &gmmParams, const char *opName,
1915+ const std::string &errorMessage, size_t weightSize)
1886{1916{
1887- size_t xSize = gmmParams.x->Size();
1888- size_t weightSize = gmmParams.weight->Size();
1889 static const std::vector<std::string> TENSOR_WEIGHT_X{"Weight", "x", "true"};1917 static const std::vector<std::string> TENSOR_WEIGHT_X{"Weight", "x", "true"};
1890 static const std::vector<std::string> TENSOR_X_Y{"x", "y", "false"};1918 static const std::vector<std::string> TENSOR_X_Y{"x", "y", "false"};
1891 static const std::vector<std::string> TENSOR_WEIGHT_Y{"Weight", "y", "true"};1919 static const std::vector<std::string> TENSOR_WEIGHT_Y{"Weight", "y", "true"};
1892- std::string errorMessage =
1893- gmmParams.apiVersion != gmm::GMMApiVersion::V2 ? "split axis is M" : "groupType == 0(split-M)";
1894- CHECK_COND(gmmParams.splitItem == X_SEPARATED || gmmParams.splitItem == NO_SEPARATED, ACLNN_ERR_PARAM_INVALID,
1895- "In op [%s], when %s and y is not separated, [%s] is invalid, got [%ld]. Constraint:[splitItem should "
1896- "be 2 or 3].",
1897- opName, errorMessage.c_str(), "splitItem", gmmParams.splitItem);
1898- CHECK_COND(xSize == weightSize, ACLNN_ERR_PARAM_INVALID,
1899- "In op [%s], when %s with separated x, separated weight and single y, tensor list lengths are "
1900- "mismatched, got [x size %zu, weight size %zu].",
1901- opName, errorMessage.c_str(), xSize, weightSize);
1902- // check dim
1903- CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.x, gmm::MIN_FM_DIM, "x") == ACLNN_SUCCESS,
1904- ACLNN_ERR_PARAM_INVALID,
1905- "In op [%s], when %s with separated x, separated weight and single y, [%s] dim num or format is "
1906- "invalid.",
1907- opName, errorMessage.c_str(), "x");
1908- CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.weight, SEPARATED_WEIGHT_DIM, "weight") == ACLNN_SUCCESS,
1909- ACLNN_ERR_PARAM_INVALID,
1910- "In op [%s], when %s with separated x, separated weight and single y, [%s] dim num or format is "
1911- "invalid.",
1912- opName, errorMessage.c_str(), "weight");
1913- CHECK_COND(CheckDimNumAndFormat(gmmParams, gmmParams.y, gmm::MIN_FM_DIM, "y") == ACLNN_SUCCESS,
1914- ACLNN_ERR_PARAM_INVALID,
1915- "In op [%s], when %s with separated x, separated weight and single y, [%s] dim num or format is "
1916- "invalid.",
1917- opName, errorMessage.c_str(), "y");
1918 // check shape, x(m,k), weight(k,n), y(m,n)1920 // check shape, x(m,k), weight(k,n), y(m,n)
1919 int64_t innerAxisDimId = 0; // 0: the index of weight's K axis. x always is not transposed, check K axis1921 int64_t innerAxisDimId = 0; // 0: the index of weight's K axis. x always is not transposed, check K axis
1920 CHECK_COND(CheckShapeSameLengthTensorList(gmmParams.weight, gmmParams.x, {0, 1}, innerAxisDimId, TENSOR_WEIGHT_X) ==1922 CHECK_COND(CheckShapeSameLengthTensorList(gmmParams.weight, gmmParams.x, {0, 1}, innerAxisDimId, TENSOR_WEIGHT_X) ==
@@ -1942,6 +1944,28 @@ static aclnnStatus SplitMSeparatedXSeparatedWeightSingleY(const gmm::GroupedMatm
1942 ACLNN_ERR_PARAM_INVALID,1944 ACLNN_ERR_PARAM_INVALID,
1943 "In op [%s], when %s with separated x, separated weight and single y, weight inner axis check failed.",1945 "In op [%s], when %s with separated x, separated weight and single y, weight inner axis check failed.",
1944 opName, errorMessage.c_str());1946 opName, errorMessage.c_str());
1947+ return ACLNN_SUCCESS;
1948+}
1949+ 
1950+static aclnnStatus SplitMSeparatedXSeparatedWeightSingleY(const gmm::GroupedMatmulParams &gmmParams, const char *opName)
1951+{
1952+ size_t xSize = gmmParams.x->Size();
1953+ size_t weightSize = gmmParams.weight->Size();
1954+ std::string errorMessage =
1955+ gmmParams.apiVersion != gmm::GMMApiVersion::V2 ? "split axis is M" : "groupType == 0(split-M)";
1956+ CHECK_COND(gmmParams.splitItem == X_SEPARATED || gmmParams.splitItem == NO_SEPARATED, ACLNN_ERR_PARAM_INVALID,
1957+ "In op [%s], when %s and y is not separated, [%s] is invalid, got [%ld]. Constraint:[splitItem should "
1958+ "be 2 or 3].",
1959+ opName, errorMessage.c_str(), "splitItem", gmmParams.splitItem);
1960+ CHECK_COND(xSize == weightSize, ACLNN_ERR_PARAM_INVALID,
1961+ "In op [%s], when %s with separated x, separated weight and single y, tensor list lengths are "
1962+ "mismatched, got [x size %zu, weight size %zu].",
1963+ opName, errorMessage.c_str(), xSize, weightSize);
1964+ CHECK_RET(CheckSplitMDimNumAndFormat(gmmParams, opName, errorMessage,
1965+ "with separated x, separated weight and single y") == ACLNN_SUCCESS,
1966+ ACLNN_ERR_PARAM_INVALID);
1967+ CHECK_RET(CheckSeparatedXShape(gmmParams, opName, errorMessage, weightSize) == ACLNN_SUCCESS,
1968+ ACLNN_ERR_PARAM_INVALID);
1945 // check groupList1969 // check groupList
1946 CHECK_COND(CheckGroupListSplitM(gmmParams, false, true, false, xSize, opName) == ACLNN_SUCCESS,1970 CHECK_COND(CheckGroupListSplitM(gmmParams, false, true, false, xSize, opName) == ACLNN_SUCCESS,
1947 ACLNN_ERR_PARAM_INVALID,1971 ACLNN_ERR_PARAM_INVALID,
@@ -2084,13 +2108,8 @@ static aclnnStatus CheckCaseSplitK(const gmm::GroupedMatmulParams &gmmParams, co
2084 return ACLNN_ERR_PARAM_INVALID;2108 return ACLNN_ERR_PARAM_INVALID;
2085}2109}
2086 2110 
2087-static aclnnStatus CheckCaseNoSplit(const gmm::GroupedMatmulParams &gmmParams, const char *opName)2111+static aclnnStatus CheckNoSplitGroupNum(const gmm::GroupedMatmulParams &gmmParams, const char *opName)
2088{2112{
2089- // When groupType is -1, splitItem mast be 0/1.
2090- CHECK_COND(gmmParams.splitItem == X_Y_SEPARATED || gmmParams.splitItem == Y_SEPARATED, ACLNN_ERR_PARAM_INVALID,
2091- "In op [%s], when groupType == -1(no split) and y is separated, [%s] is invalid, got [%ld]. "
2092- "Constraint:[splitItem should be 0 or 1].",
2093- opName, "splitItem", gmmParams.splitItem);
2094 // 校验group num2113 // 校验group num
2095 size_t xSize = gmmParams.x->Size();2114 size_t xSize = gmmParams.x->Size();
2096 size_t ySize = gmmParams.y->Size();2115 size_t ySize = gmmParams.y->Size();
@@ -2103,11 +2122,13 @@ static aclnnStatus CheckCaseNoSplit(const gmm::GroupedMatmulParams &gmmParams, c
2103 "In op [%s], when groupType == -1(no split) and x/weight are separated, tensor list lengths are "2122 "In op [%s], when groupType == -1(no split) and x/weight are separated, tensor list lengths are "
2104 "mismatched, got [x size %zu, weight size %zu].",2123 "mismatched, got [x size %zu, weight size %zu].",
2105 opName, xSize, weightSize);2124 opName, xSize, weightSize);
2106- // check dim2125+ return ACLNN_SUCCESS;
2107- CHECK_COND(CheckDimNumAndGroupListNoSplitAndFormat(gmmParams, opName) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,2126+}
2108- "In op [%s], when groupType == -1(no split), tensor list dim num, format or [%s] check failed.", opName,2127+ 
2109- "groupList");2128+static aclnnStatus CheckNoSplitShape(const gmm::GroupedMatmulParams &gmmParams, const char *opName)
2129+{
2110 // check shape2130 // check shape
2131+ size_t xSize = gmmParams.x->Size();
2111 for (size_t i = 0; i < xSize; i++) {2132 for (size_t i = 0; i < xSize; i++) {
2112 size_t xDimNum = (*gmmParams.x)[i]->GetViewShape().GetDimNum();2133 size_t xDimNum = (*gmmParams.x)[i]->GetViewShape().GetDimNum();
2113 // 2: Indicates validation up to the second last dimension, x and y must be equal in every dimension except the2134 // 2: Indicates validation up to the second last dimension, x and y must be equal in every dimension except the
@@ -2161,6 +2182,21 @@ static aclnnStatus CheckCaseNoSplit(const gmm::GroupedMatmulParams &gmmParams, c
2161 return ACLNN_SUCCESS;2182 return ACLNN_SUCCESS;
2162}2183}
2163 2184 
2185+static aclnnStatus CheckCaseNoSplit(const gmm::GroupedMatmulParams &gmmParams, const char *opName)
2186+{
2187+ // When groupType is -1, splitItem mast be 0/1.
2188+ CHECK_COND(gmmParams.splitItem == X_Y_SEPARATED || gmmParams.splitItem == Y_SEPARATED, ACLNN_ERR_PARAM_INVALID,
2189+ "In op [%s], when groupType == -1(no split) and y is separated, [%s] is invalid, got [%ld]. "
2190+ "Constraint:[splitItem should be 0 or 1].",
2191+ opName, "splitItem", gmmParams.splitItem);
2192+ CHECK_RET(CheckNoSplitGroupNum(gmmParams, opName) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
2193+ // check dim
2194+ CHECK_COND(CheckDimNumAndGroupListNoSplitAndFormat(gmmParams, opName) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,
2195+ "In op [%s], when groupType == -1(no split), tensor list dim num, format or [%s] check failed.", opName,
2196+ "groupList");
2197+ return CheckNoSplitShape(gmmParams, opName);
2198+}
2199+ 
2164static bool IsMultiTensorWeight(const gmm::GroupedMatmulParams &gmmParams)2200static bool IsMultiTensorWeight(const gmm::GroupedMatmulParams &gmmParams)
2165{2201{
2166 return gmmParams.weight != nullptr && gmmParams.weight->Size() > 0UL && (*gmmParams.weight)[0] != nullptr &&2202 return gmmParams.weight != nullptr && gmmParams.weight->Size() > 0UL && (*gmmParams.weight)[0] != nullptr &&
@@ -2923,12 +2959,8 @@ static aclnnStatus SetStorageShape(gmm::GroupedMatmulParams &params, const std::
2923 return ACLNN_SUCCESS;2959 return ACLNN_SUCCESS;
2924}2960}
2925 2961 
2926-static aclnnStatus GetGMMResultByL0Api(gmm::GroupedMatmulParams &params, uint64_t *workspaceSize,2962+static aclnnStatus PrepareParamsForL0(gmm::GroupedMatmulParams &params, aclOpExecutor *executorPtr)
2927- aclOpExecutor **executor, const char *opName)
2928{2963{
2929- auto uniqueExecutor = CREATE_EXECUTOR(); // fixed writen style, create OpExecutor
2930- aclOpExecutor *executorPtr = uniqueExecutor.get();
2931- CHECK_RET(executorPtr != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR);
2932 if (params.xDtype != DataType::DT_INT4) { // A4W4 has no bias2964 if (params.xDtype != DataType::DT_INT4) { // A4W4 has no bias
2933 CHECK_COND(gmm::BIAS_DTYPE.find(params.xDtype) != gmm::BIAS_DTYPE.cend(), ACLNN_ERR_PARAM_INVALID,2965 CHECK_COND(gmm::BIAS_DTYPE.find(params.xDtype) != gmm::BIAS_DTYPE.cend(), ACLNN_ERR_PARAM_INVALID,
2934 "GMM: Cannot find bias dtype match with xDtype[%s]", gmm::dTypeToString(params.xDtype).c_str());2966 "GMM: Cannot find bias dtype match with xDtype[%s]", gmm::dTypeToString(params.xDtype).c_str());
@@ -2938,18 +2970,11 @@ static aclnnStatus GetGMMResultByL0Api(gmm::GroupedMatmulParams &params, uint64_
2938 CHECK_RET(SetTransposedTensorListContiguous(params, executorPtr) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR);2970 CHECK_RET(SetTransposedTensorListContiguous(params, executorPtr) == ACLNN_SUCCESS, ACLNN_ERR_INNER_NULLPTR);
2939 CHECK_COND(ParamsDataContiguous(params, executorPtr) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,2971 CHECK_COND(ParamsDataContiguous(params, executorPtr) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,
2940 "ParamsDataContiguous failed.");2972 "ParamsDataContiguous failed.");
2941- if (params.groupType == gmm::SPLIT_K) {2973+ return ACLNN_SUCCESS;
2942- if (CheckZeroShapeSplitK(params, workspaceSize, opName) != ACLNN_SUCCESS) {2974+}
2943- uniqueExecutor.ReleaseTo(executor);
2944- return ACLNN_SUCCESS;
2945- }
2946- } else {
2947- if (CheckZeroShape(params, workspaceSize) != ACLNN_SUCCESS) {
2948- uniqueExecutor.ReleaseTo(executor);
2949- return ACLNN_SUCCESS;
2950- }
2951- }
2952 2975 
2976+static aclnnStatus PrepareWeightNz(gmm::GroupedMatmulParams &params, const char *opName, aclOpExecutor *executorPtr)
2977+{
2953 std::vector<op::Shape> nzShapes;2978 std::vector<op::Shape> nzShapes;
2954 for (uint64_t idx = 0; idx < (*params.weight).Size(); idx++) {2979 for (uint64_t idx = 0; idx < (*params.weight).Size(); idx++) {
2955 nzShapes.push_back((*params.weight)[idx]->GetStorageShape());2980 nzShapes.push_back((*params.weight)[idx]->GetStorageShape());
@@ -2972,11 +2997,12 @@ static aclnnStatus GetGMMResultByL0Api(gmm::GroupedMatmulParams &params, uint64_
2972 }2997 }
2973 CHECK_COND(TransWeightToNz(params, executorPtr) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,2998 CHECK_COND(TransWeightToNz(params, executorPtr) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID,
2974 "TransWeightToNz failed.");2999 "TransWeightToNz failed.");
2975- if (params.groupListOptional != nullptr) {3000+ return ACLNN_SUCCESS;
2976- params.groupTensorOptional =3001+}
2977- uniqueExecutor->ConvertToTensor(params.groupListOptional, op::ToOpDataType(ACL_INT64));3002+ 
2978- CHECK_RET(params.groupTensorOptional != nullptr, ACLNN_ERR_INNER_NULLPTR);3003+static aclnnStatus ExecuteL0GroupedMatmul(gmm::GroupedMatmulParams &params, aclOpExecutor *executorPtr,
2979- }3004+ const char *opName)
3005+{
2980 auto perTokenScaleOptional =3006 auto perTokenScaleOptional =
2981 (*params.perTokenScaleOptional)[0]->IsEmpty() ? nullptr : (*params.perTokenScaleOptional)[0];3007 (*params.perTokenScaleOptional)[0]->IsEmpty() ? nullptr : (*params.perTokenScaleOptional)[0];
2982 // Invoke l0 operator GroupedMatmul for calculation.3008 // Invoke l0 operator GroupedMatmul for calculation.
@@ -2994,6 +3020,38 @@ static aclnnStatus GetGMMResultByL0Api(gmm::GroupedMatmulParams &params, uint64_
2994 auto viewCopyResult = l0op::ViewCopy((*result)[i], (*params.y)[i], executorPtr);3020 auto viewCopyResult = l0op::ViewCopy((*result)[i], (*params.y)[i], executorPtr);
2995 CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR);3021 CHECK_RET(viewCopyResult != nullptr, ACLNN_ERR_INNER_NULLPTR);
2996 }3022 }
3023+ return ACLNN_SUCCESS;
3024+}
3025+ 
3026+static aclnnStatus GetGMMResultByL0Api(gmm::GroupedMatmulParams &params, uint64_t *workspaceSize,
3027+ aclOpExecutor **executor, const char *opName)
3028+{
3029+ auto uniqueExecutor = CREATE_EXECUTOR(); // fixed writen style, create OpExecutor
3030+ aclOpExecutor *executorPtr = uniqueExecutor.get();
3031+ CHECK_RET(executorPtr != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR);
3032+ aclnnStatus ret = PrepareParamsForL0(params, executorPtr);
3033+ CHECK_RET(ret == ACLNN_SUCCESS, ret);
3034+ if (params.groupType == gmm::SPLIT_K) {
3035+ if (CheckZeroShapeSplitK(params, workspaceSize, opName) != ACLNN_SUCCESS) {
3036+ uniqueExecutor.ReleaseTo(executor);
3037+ return ACLNN_SUCCESS;
3038+ }
3039+ } else {
3040+ if (CheckZeroShape(params, workspaceSize) != ACLNN_SUCCESS) {
3041+ uniqueExecutor.ReleaseTo(executor);
3042+ return ACLNN_SUCCESS;
3043+ }
3044+ }
3045+ 
3046+ ret = PrepareWeightNz(params, opName, executorPtr);
3047+ CHECK_RET(ret == ACLNN_SUCCESS, ret);
3048+ if (params.groupListOptional != nullptr) {
3049+ params.groupTensorOptional =
3050+ uniqueExecutor->ConvertToTensor(params.groupListOptional, op::ToOpDataType(ACL_INT64));
3051+ CHECK_RET(params.groupTensorOptional != nullptr, ACLNN_ERR_INNER_NULLPTR);
3052+ }
3053+ ret = ExecuteL0GroupedMatmul(params, executorPtr, opName);
3054+ CHECK_RET(ret == ACLNN_SUCCESS, ret);
2997 // Standard syntax, get the size of workspace needed during computation.3055 // Standard syntax, get the size of workspace needed during computation.
2998 *workspaceSize = uniqueExecutor->GetWorkspaceSize();3056 *workspaceSize = uniqueExecutor->GetWorkspaceSize();
2999 uniqueExecutor.ReleaseTo(executor);3057 uniqueExecutor.ReleaseTo(executor);
@@ -3063,6 +3121,22 @@ static aclnnStatus CheckEmptyTensor(const aclTensorList *x, const aclTensorList
3063 return ACLNN_SUCCESS;3121 return ACLNN_SUCCESS;
3064}3122}
3065 3123 
3124+static aclnnStatus PrepareGmmParams(gmm::GroupedMatmulParams &gmmParams, const aclTensorList *x,
3125+ const aclTensorList *y, int64_t splitItem, const char *opName)
3126+{
3127+ if (gmmParams.scaleOptional != nullptr) {
3128+ for (size_t i = 0; i < gmmParams.scaleOptional->Size(); i++) {
3129+ if ((*gmmParams.scaleOptional)[i]->GetDataType() == DataType::DT_INT64) {
3130+ (void)const_cast<aclTensor *>((*gmmParams.scaleOptional)[i])->SetDataType(op::DataType::DT_UINT64);
3131+ }
3132+ }
3133+ }
3134+ ResetEmptyTensor(gmmParams); // make empty tensor/tensorList nullptr
3135+ CHECK_RET(CheckParam(gmmParams, opName) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
3136+ gmmParams.splitItem = CorrectSplitItem(x, y, splitItem);
3137+ return ACLNN_SUCCESS;
3138+}
3139+ 
3066static aclnnStatus aclnnGroupedMatmulGetWorkspaceSizeCommon(3140static aclnnStatus aclnnGroupedMatmulGetWorkspaceSizeCommon(
3067 const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional,3141 const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional,
3068 const aclTensorList *scaleOptional, const aclTensorList *offsetOptional,3142 const aclTensorList *scaleOptional, const aclTensorList *offsetOptional,
@@ -3117,16 +3191,7 @@ static aclnnStatus aclnnGroupedMatmulGetWorkspaceSizeCommon(
3117 activationFeatureOutOptional,3191 activationFeatureOutOptional,
3118 dynQuantScaleOutOptional,3192 dynQuantScaleOutOptional,
3119 xDtype};3193 xDtype};
3120- if (gmmParams.scaleOptional != nullptr) {3194+ CHECK_RET(PrepareGmmParams(gmmParams, x, y, splitItem, opName) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
3121- for (size_t i = 0; i < gmmParams.scaleOptional->Size(); i++) {
3122- if ((*gmmParams.scaleOptional)[i]->GetDataType() == DataType::DT_INT64) {
3123- (void)const_cast<aclTensor *>((*gmmParams.scaleOptional)[i])->SetDataType(op::DataType::DT_UINT64);
3124- }
3125- }
3126- }
3127- ResetEmptyTensor(gmmParams); // make empty tensor/tensorList nullptr
3128- CHECK_RET(CheckParam(gmmParams, opName) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
3129- gmmParams.splitItem = CorrectSplitItem(x, y, splitItem);
3130 3195 
3131 aclnnStatus ret = GetGMMResultByL0Api(gmmParams, workspaceSize, executor, opName);3196 aclnnStatus ret = GetGMMResultByL0Api(gmmParams, workspaceSize, executor, opName);
3132 3197 
@@ -3177,27 +3242,8 @@ aclnnStatus CheckCommonParam(const aclTensorList *x, const aclTensorList *weight
3177 return ACLNN_SUCCESS;3242 return ACLNN_SUCCESS;
3178}3243}
3179 3244 
3180-aclnnStatus aclnnGroupedMatmulWeightNzGetWorkspaceSize(3245+static void UnpackWeightQuantInputs(const aclTensorList *x, const aclTensorList *weight)
3181- const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional,
3182- const aclTensorList *scaleOptional, const aclTensorList *offsetOptional,
3183- const aclTensorList *antiquantScaleOptional, const aclTensorList *antiquantOffsetOptional,
3184- const aclTensorList *perTokenScaleOptional, const aclTensor *groupListOptional,
3185- const aclTensorList *activationInputOptional, const aclTensorList *activationQuantScaleOptional,
3186- const aclTensorList *activationQuantOffsetOptional, int64_t splitItem, int64_t groupType, int64_t groupListType,
3187- int64_t actType, aclIntArray *tuningConfigOptional, int64_t quantGroupSize, aclTensorList *out,
3188- aclTensorList *activationFeatureOutOptional, aclTensorList *dynQuantScaleOutOptional, uint64_t *workspaceSize,
3189- aclOpExecutor **executor)
3190{3246{
3191- const char *opName = "grouped_matmul";
3192- CHECK_COND(CheckNotNull(x, weight, out) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_NULLPTR,
3193- "In op [%s], required inputs must not be nullptr.", opName);
3194- // Standard syntax, Check parameters.
3195- L2_DFX_PHASE_1(aclnnGroupedMatmulWeightNz,
3196- DFX_IN(x, weight, biasOptional, scaleOptional, offsetOptional, antiquantScaleOptional,
3197- antiquantOffsetOptional, perTokenScaleOptional, activationInputOptional,
3198- activationQuantScaleOptional, activationQuantOffsetOptional, groupListOptional, splitItem,
3199- groupType, groupListType, actType, tuningConfigOptional),
3200- DFX_OUT(out, activationFeatureOutOptional, dynQuantScaleOutOptional));
3201 if ((*weight)[0]->GetDataType() == DataType::DT_INT32) {3247 if ((*weight)[0]->GetDataType() == DataType::DT_INT32) {
3202 // convert weight from int32 to int43248 // convert weight from int32 to int4
3203 UnpackB32ToB4(weight, "weight");3249 UnpackB32ToB4(weight, "weight");
@@ -3223,6 +3269,30 @@ aclnnStatus aclnnGroupedMatmulWeightNzGetWorkspaceSize(
3223 UnpackB32ToB4(x, "x", true);3269 UnpackB32ToB4(x, "x", true);
3224 }3270 }
3225 }3271 }
3272+}
3273+ 
3274+aclnnStatus aclnnGroupedMatmulWeightNzGetWorkspaceSize(
3275+ const aclTensorList *x, const aclTensorList *weight, const aclTensorList *biasOptional,
3276+ const aclTensorList *scaleOptional, const aclTensorList *offsetOptional,
3277+ const aclTensorList *antiquantScaleOptional, const aclTensorList *antiquantOffsetOptional,
3278+ const aclTensorList *perTokenScaleOptional, const aclTensor *groupListOptional,
3279+ const aclTensorList *activationInputOptional, const aclTensorList *activationQuantScaleOptional,
3280+ const aclTensorList *activationQuantOffsetOptional, int64_t splitItem, int64_t groupType, int64_t groupListType,
3281+ int64_t actType, aclIntArray *tuningConfigOptional, int64_t quantGroupSize, aclTensorList *out,
3282+ aclTensorList *activationFeatureOutOptional, aclTensorList *dynQuantScaleOutOptional, uint64_t *workspaceSize,
3283+ aclOpExecutor **executor)
3284+{
3285+ const char *opName = "grouped_matmul";
3286+ CHECK_COND(CheckNotNull(x, weight, out) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_NULLPTR,
3287+ "In op [%s], required inputs must not be nullptr.", opName);
3288+ // Standard syntax, Check parameters.
3289+ L2_DFX_PHASE_1(aclnnGroupedMatmulWeightNz,
3290+ DFX_IN(x, weight, biasOptional, scaleOptional, offsetOptional, antiquantScaleOptional,
3291+ antiquantOffsetOptional, perTokenScaleOptional, activationInputOptional,
3292+ activationQuantScaleOptional, activationQuantOffsetOptional, groupListOptional, splitItem,
3293+ groupType, groupListType, actType, tuningConfigOptional),
3294+ DFX_OUT(out, activationFeatureOutOptional, dynQuantScaleOutOptional));
3295+ UnpackWeightQuantInputs(x, weight);
3226 if (IsS8S4PseudoQuantWeightNz(x, weight, scaleOptional)) {3296 if (IsS8S4PseudoQuantWeightNz(x, weight, scaleOptional)) {
3227 bool hasOffset = offsetOptional != nullptr && offsetOptional->Size() == 1 && (*offsetOptional)[0] != nullptr;3297 bool hasOffset = offsetOptional != nullptr && offsetOptional->Size() == 1 && (*offsetOptional)[0] != nullptr;
3228 if (hasOffset) {3298 if (hasOffset) {
Mgmm/grouped_matmul/op_api/grouped_matmul_950_checker.cpp+335-189
@@ -259,8 +259,7 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckQuantCasesFormat() const
259}259}
260 260 
261template <typename T>261template <typename T>
262-aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightStorageShape(const aclTensor *weightTensor,262+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightNzStorageDim(const aclTensor *weightTensor) const
263- int64_t kDimValue, int64_t nDimValue) const
264{263{
265 auto weightStorage = weightTensor->GetStorageShape();264 auto weightStorage = weightTensor->GetStorageShape();
266 auto weightStorageShapeDim = weightStorage.GetDimNum();265 auto weightStorageShapeDim = weightStorage.GetDimNum();
@@ -273,12 +272,14 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightStorageShape(const a
273 IsWeightNzMultiTensorLayout() ? std::to_string(WEIGHTNZ_MULTI_TENSOR_STORAGE_DIM) + " or " +272 IsWeightNzMultiTensorLayout() ? std::to_string(WEIGHTNZ_MULTI_TENSOR_STORAGE_DIM) + " or " +
274 std::to_string(QUANT_WEIGHTNZ_STORAGE_DIM) :273 std::to_string(QUANT_WEIGHTNZ_STORAGE_DIM) :
275 std::to_string(QUANT_WEIGHTNZ_STORAGE_DIM)));274 std::to_string(QUANT_WEIGHTNZ_STORAGE_DIM)));
275+ return ACLNN_SUCCESS;
276+}
276 277 
277- auto weightStorageLastFourthDim = weightStorage.GetDim(weightStorageShapeDim - LAST_FOURTH_DIM_INDEX);278+template <typename T>
278- auto weightStorageLastThirdDim = weightStorage.GetDim(weightStorageShapeDim - LAST_THIRD_DIM_INDEX);279+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightNzC0(const aclTensor *weightTensor,
279- auto weightStorageLastSecondDim = weightStorage.GetDim(weightStorageShapeDim - LAST_SECOND_DIM_INDEX);280+ int64_t weightStorageLastDim,
280- auto weightStorageLastDim = weightStorage.GetDim(weightStorageShapeDim - LAST_FIRST_DIM_INDEX);281+ int64_t &cubeBlockSizeK) const
281- 282+{
282 const bool isInt4 = (gmmParams_.xDtype == DataType::DT_INT4);283 const bool isInt4 = (gmmParams_.xDtype == DataType::DT_INT4);
283 const bool isMxfp4 =284 const bool isMxfp4 =
284 (gmmParams_.xDtype == DataType::DT_FLOAT4_E2M1 || gmmParams_.xDtype == DataType::DT_FLOAT4_E1M2);285 (gmmParams_.xDtype == DataType::DT_FLOAT4_E2M1 || gmmParams_.xDtype == DataType::DT_FLOAT4_E1M2);
@@ -286,17 +287,20 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightStorageShape(const a
286 // INT4 C0 is 32. Keep accepting the historical C0=64 layout as well; the selected287 // INT4 C0 is 32. Keep accepting the historical C0=64 layout as well; the selected
287 // C0 is used below to validate the two outer NZ dimensions consistently.288 // C0 is used below to validate the two outer NZ dimensions consistently.
288 const bool validInt4C0 = weightStorageLastDim == CUBE_BLOCK_SIZE_32 || weightStorageLastDim == CUBE_BLOCK_SIZE_64;289 const bool validInt4C0 = weightStorageLastDim == CUBE_BLOCK_SIZE_32 || weightStorageLastDim == CUBE_BLOCK_SIZE_64;
289- const int64_t cubeBlockSizeK = isInt4 ? weightStorageLastDim : (isMxfp4 ? CUBE_BLOCK_SIZE_64 : CUBE_BLOCK_SIZE_32);290+ cubeBlockSizeK = isInt4 ? weightStorageLastDim : (isMxfp4 ? CUBE_BLOCK_SIZE_64 : CUBE_BLOCK_SIZE_32);
290 GMM_CHECK_REPORT(isInt4 ? validInt4C0 : weightStorageLastDim == cubeBlockSizeK,291 GMM_CHECK_REPORT(isInt4 ? validInt4C0 : weightStorageLastDim == cubeBlockSizeK,
291 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(292 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
292 GetAclnnOpName(), weightName_.c_str(), StorageShapeToString(weightTensor),293 GetAclnnOpName(), weightName_.c_str(), StorageShapeToString(weightTensor),
293 "when the format of weight is FRACTAL_NZ, storage shape last dim of weight must be equal to "294 "when the format of weight is FRACTAL_NZ, storage shape last dim of weight must be equal to "
294 "32 for fp8 dtype, 32 or 64 for int4 dtype, or 64 for mxfp4 dtype"));295 "32 for fp8 dtype, 32 or 64 for int4 dtype, or 64 for mxfp4 dtype"));
295- GMM_CHECK_REPORT(296+ return ACLNN_SUCCESS;
296- weightStorageLastSecondDim == CUBE_BLOCK_SIZE_16,297+}
297- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(298+ 
298- GetAclnnOpName(), weightName_.c_str(), StorageShapeToString(weightTensor),299+template <typename T>
299- "when the format of weight is FRACTAL_NZ, storage shape last second dim of weight must be equal to 16"));300+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightNzOuterDims(
301+ const aclTensor *weightTensor, int64_t kDimValue, int64_t nDimValue, int64_t cubeBlockSizeK,
302+ int64_t weightStorageLastFourthDim, int64_t weightStorageLastThirdDim) const
303+{
300 if (gmmParams_.transposeWeight) {304 if (gmmParams_.transposeWeight) {
301 GMM_CHECK_REPORT(305 GMM_CHECK_REPORT(
302 weightStorageLastFourthDim == (kDimValue + cubeBlockSizeK - 1) / cubeBlockSizeK,306 weightStorageLastFourthDim == (kDimValue + cubeBlockSizeK - 1) / cubeBlockSizeK,
@@ -331,6 +335,31 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightStorageShape(const a
331 return ACLNN_SUCCESS;335 return ACLNN_SUCCESS;
332}336}
333 337 
338+template <typename T>
339+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightStorageShape(const aclTensor *weightTensor,
340+ int64_t kDimValue, int64_t nDimValue) const
341+{
342+ auto weightStorage = weightTensor->GetStorageShape();
343+ auto weightStorageShapeDim = weightStorage.GetDimNum();
344+ CHECK_RET(CheckWeightNzStorageDim(weightTensor) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
345+ 
346+ auto weightStorageLastFourthDim = weightStorage.GetDim(weightStorageShapeDim - LAST_FOURTH_DIM_INDEX);
347+ auto weightStorageLastThirdDim = weightStorage.GetDim(weightStorageShapeDim - LAST_THIRD_DIM_INDEX);
348+ auto weightStorageLastSecondDim = weightStorage.GetDim(weightStorageShapeDim - LAST_SECOND_DIM_INDEX);
349+ auto weightStorageLastDim = weightStorage.GetDim(weightStorageShapeDim - LAST_FIRST_DIM_INDEX);
350+ 
351+ int64_t cubeBlockSizeK = CUBE_BLOCK_SIZE_32;
352+ CHECK_RET(CheckWeightNzC0(weightTensor, weightStorageLastDim, cubeBlockSizeK) == ACLNN_SUCCESS,
353+ ACLNN_ERR_PARAM_INVALID);
354+ GMM_CHECK_REPORT(
355+ weightStorageLastSecondDim == CUBE_BLOCK_SIZE_16,
356+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
357+ GetAclnnOpName(), weightName_.c_str(), StorageShapeToString(weightTensor),
358+ "when the format of weight is FRACTAL_NZ, storage shape last second dim of weight must be equal to 16"));
359+ return CheckWeightNzOuterDims(weightTensor, kDimValue, nDimValue, cubeBlockSizeK, weightStorageLastFourthDim,
360+ weightStorageLastThirdDim);
361+}
362+ 
334template <typename T>363template <typename T>
335aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightNzSpecialParams() const364aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckWeightNzSpecialParams() const
336{365{
@@ -496,6 +525,62 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulMxDtype() con
496 return ACLNN_SUCCESS;525 return ACLNN_SUCCESS;
497}526}
498 527 
528+template <typename T>
529+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckPerGroupWeightDim(size_t weightDimNumber) const
530+{
531+ if (gmmParams_.groupType == SPLIT_M) {
532+ const size_t expectedWeightDim = IsWeightNzMultiTensorLayout() ? MIN_FM_DIM : SPLIT_M_SINGLE_WEIGHT_DIM;
533+ GMM_CHECK_REPORT(
534+ weightDimNumber == expectedWeightDim,
535+ OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), weightName_.c_str(), std::to_string(weightDimNumber),
536+ std::to_string(expectedWeightDim)));
537+ } else if (gmmParams_.groupType == SPLIT_K) {
538+ GMM_CHECK_REPORT(
539+ weightDimNumber == SPLIT_K_SINGLE_WEIGHT_DIM,
540+ OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), weightName_.c_str(), std::to_string(weightDimNumber),
541+ std::to_string(SPLIT_K_SINGLE_WEIGHT_DIM)));
542+ }
543+ return ACLNN_SUCCESS;
544+}
545+ 
546+template <typename T>
547+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckPerGroupScaleDim(const TensorIndexInfo &tensorIndex,
548+ size_t scaleDimNumber,
549+ size_t perTokenDimNumber, size_t xDimNumber,
550+ size_t weightDimNumber) const
551+{
552+ if (gmmParams_.groupType == SPLIT_M) {
553+ DataType scaleDtype = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetDataType();
554+ if (scaleDtype == DataType::DT_FLOAT8_E8M0) {
555+ const size_t expectedScaleDim =
556+ IsWeightNzMultiTensorLayout() ? MX_SPLIT_K_SCALE_DIM : MX_SPLIT_M_SCALE_DIM;
557+ GMM_CHECK_REPORT(
558+ scaleDimNumber == expectedScaleDim,
559+ OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), scaleName_.c_str(), std::to_string(scaleDimNumber),
560+ std::to_string(expectedScaleDim)));
561+ GMM_CHECK_REPORT(perTokenDimNumber == MX_SPLIT_M_PER_TOKEN_SCALE_DIM,
562+ OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), perTokenScaleName_.c_str(),
563+ std::to_string(perTokenDimNumber),
564+ std::to_string(MX_SPLIT_M_PER_TOKEN_SCALE_DIM)));
565+ } else {
566+ GMM_CHECK_REPORT(scaleDimNumber == weightDimNumber,
567+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
568+ GetAclnnOpName(), scaleName_.c_str(), std::to_string(scaleDimNumber),
569+ "when split m and in G-B quant mode, the shape dim of scale must be equal to the "
570+ "shape dim of weight [" +
571+ std::to_string(weightDimNumber) + "]"));
572+ GMM_CHECK_REPORT(
573+ perTokenDimNumber == xDimNumber,
574+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
575+ GetAclnnOpName(), perTokenScaleName_.c_str(), std::to_string(perTokenDimNumber),
576+ "when split m and in G-B quant mode, the shape dim of perTokenScale must be equal to "
577+ "the shape dim of x [" +
578+ std::to_string(xDimNumber) + "]"));
579+ }
580+ }
581+ return ACLNN_SUCCESS;
582+}
583+ 
499template <typename T>584template <typename T>
500aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulPerGroupDim() const585aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulPerGroupDim() const
501{586{
@@ -510,47 +595,10 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulPerGroupDim()
510 GMM_CHECK_REPORT(xDimNumber == MIN_FM_DIM,595 GMM_CHECK_REPORT(xDimNumber == MIN_FM_DIM,
511 OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), xName_.c_str(), std::to_string(xDimNumber),596 OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), xName_.c_str(), std::to_string(xDimNumber),
512 std::to_string(MIN_FM_DIM)));597 std::to_string(MIN_FM_DIM)));
513- if (gmmParams_.groupType == SPLIT_M) {598+ CHECK_RET(CheckPerGroupWeightDim(weightDimNumber) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
514- const size_t expectedWeightDim = IsWeightNzMultiTensorLayout() ? MIN_FM_DIM : SPLIT_M_SINGLE_WEIGHT_DIM;599+ CHECK_RET(CheckPerGroupScaleDim(tensorIndex, scaleDimNumber, perTokenDimNumber, xDimNumber, weightDimNumber) ==
515- GMM_CHECK_REPORT(600+ ACLNN_SUCCESS,
516- weightDimNumber == expectedWeightDim,601+ ACLNN_ERR_PARAM_INVALID);
517- OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), weightName_.c_str(), std::to_string(weightDimNumber),
518- std::to_string(expectedWeightDim)));
519- } else if (gmmParams_.groupType == SPLIT_K) {
520- GMM_CHECK_REPORT(
521- weightDimNumber == SPLIT_K_SINGLE_WEIGHT_DIM,
522- OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), weightName_.c_str(), std::to_string(weightDimNumber),
523- std::to_string(SPLIT_K_SINGLE_WEIGHT_DIM)));
524- }
525- if (gmmParams_.groupType == SPLIT_M) {
526- DataType scaleDtype = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetDataType();
527- if (scaleDtype == DataType::DT_FLOAT8_E8M0) {
528- const size_t expectedScaleDim =
529- IsWeightNzMultiTensorLayout() ? MX_SPLIT_K_SCALE_DIM : MX_SPLIT_M_SCALE_DIM;
530- GMM_CHECK_REPORT(
531- scaleDimNumber == expectedScaleDim,
532- OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), scaleName_.c_str(), std::to_string(scaleDimNumber),
533- std::to_string(expectedScaleDim)));
534- GMM_CHECK_REPORT(perTokenDimNumber == MX_SPLIT_M_PER_TOKEN_SCALE_DIM,
535- OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), perTokenScaleName_.c_str(),
536- std::to_string(perTokenDimNumber),
537- std::to_string(MX_SPLIT_M_PER_TOKEN_SCALE_DIM)));
538- } else {
539- GMM_CHECK_REPORT(scaleDimNumber == weightDimNumber,
540- OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
541- GetAclnnOpName(), scaleName_.c_str(), std::to_string(scaleDimNumber),
542- "when split m and in G-B quant mode, the shape dim of scale must be equal to the "
543- "shape dim of weight [" +
544- std::to_string(weightDimNumber) + "]"));
545- GMM_CHECK_REPORT(
546- perTokenDimNumber == xDimNumber,
547- OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(
548- GetAclnnOpName(), perTokenScaleName_.c_str(), std::to_string(perTokenDimNumber),
549- "when split m and in G-B quant mode, the shape dim of perTokenScale must be equal to "
550- "the shape dim of x [" +
551- std::to_string(xDimNumber) + "]"));
552- }
553- }
554 }602 }
555 return ACLNN_SUCCESS;603 return ACLNN_SUCCESS;
556}604}
@@ -587,102 +635,127 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxBiasInputShape(const Ten
587}635}
588 636 
589template <typename T>637template <typename T>
590-aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxTypeMCaseInputShape(const TensorDimInfo &dimInfo,638+typename AclnnGroupedMatmulDAV3510Checker<T>::MxTypeMDims
591- size_t index) const639+AclnnGroupedMatmulDAV3510Checker<T>::ExtractMxTypeMDims(const TensorDimInfo &dimInfo, size_t index) const
592{640{
593 const auto tensorIndex = GetTensorIndexInfo(index);641 const auto tensorIndex = GetTensorIndexInfo(index);
594 auto weightNIndex = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDimNum() - 1;642 auto weightNIndex = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDimNum() - 1;
595- size_t scaleDimNum = dimInfo.scaleDimNum;643+ MxTypeMDims dims{};
596- size_t pertokenScaleDimNum = dimInfo.pertokenScaleDimNum;644+ dims.xMDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(0);
597- int64_t groupNum = dimInfo.groupNum;645+ dims.xKDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(1);
598- auto xMDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(0);646+ dims.pertokenMDimValue =
599- auto xKDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(1);
600- auto pertokenMDimValue =
601 GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)->GetViewShape().GetDim(0);647 GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)->GetViewShape().GetDim(0);
602- auto pertokenScaleKDimValue =648+ dims.pertokenScaleKDimValue =
603 GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)->GetViewShape().GetDim(1);649 GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)->GetViewShape().GetDim(1);
604- auto pertokenScaleLastDimValue = GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)650+ dims.pertokenScaleLastDimValue = GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)
605 ->GetViewShape()651 ->GetViewShape()
606- .GetDim(pertokenScaleDimNum - 1);652+ .GetDim(dimInfo.pertokenScaleDimNum - 1);
607- auto weightNDimValue = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDim(weightNIndex);653+ dims.weightNDimValue = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDim(weightNIndex);
608- auto inferedScaleKDimValue = (xKDimValue + MXFP_DIVISOR_SIZE - 1) / MXFP_DIVISOR_SIZE;654+ dims.inferedScaleKDimValue = (dims.xKDimValue + MXFP_DIVISOR_SIZE - 1) / MXFP_DIVISOR_SIZE;
609- auto scaleLastDimValue =655+ dims.scaleLastDimValue =
610- GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(scaleDimNum - 1);656+ GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(dimInfo.scaleDimNum - 1);
611- CHECK_RET(CheckMxBiasInputShape(dimInfo, index) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);657+ return dims;
612- GMM_CHECK_REPORT(658+}
613- xMDimValue == pertokenMDimValue,659+ 
614- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(660+template <typename T>
615- GetAclnnOpName(), xName_.c_str(), ViewShapeToString(GetInputTensor(gmmParams_.x, tensorIndex.x)),661+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxTypeMScaleShape(const MxTypeMDims &dims,
616- "axis M of x must be equal to axis M of perTokenScale [" + std::to_string(pertokenMDimValue) + "]"));662+ const TensorDimInfo &tensorIndex) const
663+{
664+ auto weightNIndex = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDimNum() - 1;
617 if (IsWeightNzMultiTensorLayout()) {665 if (IsWeightNzMultiTensorLayout()) {
618 auto scaleFirstDimValue = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(0);666 auto scaleFirstDimValue = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(0);
619 auto scaleSecondDimValue =667 auto scaleSecondDimValue =
620 GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(1);668 GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(1);
621- const bool isNkLayout = scaleFirstDimValue == weightNDimValue && scaleSecondDimValue == inferedScaleKDimValue;669+ const bool isNkLayout = scaleFirstDimValue == dims.weightNDimValue &&
622- const bool isKnLayout = scaleFirstDimValue == inferedScaleKDimValue && scaleSecondDimValue == weightNDimValue;670+ scaleSecondDimValue == dims.inferedScaleKDimValue;
671+ const bool isKnLayout = scaleFirstDimValue == dims.inferedScaleKDimValue &&
672+ scaleSecondDimValue == dims.weightNDimValue;
623 const bool isBroadcastLayout = scaleFirstDimValue == 1L || scaleSecondDimValue == 1L;673 const bool isBroadcastLayout = scaleFirstDimValue == 1L || scaleSecondDimValue == 1L;
624 GMM_CHECK_REPORT(isNkLayout || isKnLayout || isBroadcastLayout,674 GMM_CHECK_REPORT(isNkLayout || isKnLayout || isBroadcastLayout,
625 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(675 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
626 GetAclnnOpName(), scaleName_.c_str(),676 GetAclnnOpName(), scaleName_.c_str(),
627 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),677 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),
628 "the first two axes of scale must match axis N of weight and ceil(k/" +678 "the first two axes of scale must match axis N of weight and ceil(k/" +
629- std::to_string(MXFP_DIVISOR_SIZE) + ") [" + std::to_string(weightNDimValue) + ", " +679+ std::to_string(MXFP_DIVISOR_SIZE) + ") [" + std::to_string(dims.weightNDimValue) +
630- std::to_string(inferedScaleKDimValue) + "]"));680+ ", " + std::to_string(dims.inferedScaleKDimValue) + "]"));
631 } else {681 } else {
632 auto scaleNDimValue =682 auto scaleNDimValue =
633 GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(weightNIndex);683 GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(weightNIndex);
634 auto scaleKDimValue = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(1);684 auto scaleKDimValue = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(1);
635 GMM_CHECK_REPORT(685 GMM_CHECK_REPORT(
636- weightNDimValue == scaleNDimValue,686+ dims.weightNDimValue == scaleNDimValue,
637 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(687 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
638 GetAclnnOpName(), weightName_.c_str(),688 GetAclnnOpName(), weightName_.c_str(),
639 ViewShapeToString(GetInputTensor(gmmParams_.weight, tensorIndex.weight)),689 ViewShapeToString(GetInputTensor(gmmParams_.weight, tensorIndex.weight)),
640 "axis N of weight must be equal to axis N of scale [" + std::to_string(scaleNDimValue) + "]"));690 "axis N of weight must be equal to axis N of scale [" + std::to_string(scaleNDimValue) + "]"));
641 auto scaleGDimValue = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(0);691 auto scaleGDimValue = GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)->GetViewShape().GetDim(0);
642 GMM_CHECK_REPORT(692 GMM_CHECK_REPORT(
643- scaleGDimValue == groupNum,693+ scaleGDimValue == dims.groupNum,
644 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(694 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
645 GetAclnnOpName(), scaleName_.c_str(),695 GetAclnnOpName(), scaleName_.c_str(),
646 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),696 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),
647- "axis group of scale must be equal to axis 0 of groupTensor [" + std::to_string(groupNum) + "]"));697+ "axis group of scale must be equal to axis 0 of groupTensor [" +
648- GMM_CHECK_REPORT(scaleKDimValue == inferedScaleKDimValue,698+ std::to_string(dims.groupNum) + "]"));
699+ GMM_CHECK_REPORT(scaleKDimValue == dims.inferedScaleKDimValue,
649 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(700 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
650 GetAclnnOpName(), scaleName_.c_str(),701 GetAclnnOpName(), scaleName_.c_str(),
651 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),702 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),
652 "axis K of scale must be equal to ceil(k/" + std::to_string(MXFP_DIVISOR_SIZE) + ") [" +703 "axis K of scale must be equal to ceil(k/" + std::to_string(MXFP_DIVISOR_SIZE) + ") [" +
653- std::to_string(inferedScaleKDimValue) + "]"));704+ std::to_string(dims.inferedScaleKDimValue) + "]"));
654 }705 }
655- GMM_CHECK_REPORT(pertokenScaleKDimValue == inferedScaleKDimValue,706+ return ACLNN_SUCCESS;
707+}
708+ 
709+template <typename T>
710+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxTypeMPerTokenAndScaleLastDim(
711+ const MxTypeMDims &dims, const TensorDimInfo &tensorIndex) const
712+{
713+ GMM_CHECK_REPORT(dims.pertokenScaleKDimValue == dims.inferedScaleKDimValue,
656 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(714 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
657 GetAclnnOpName(), perTokenScaleName_.c_str(),715 GetAclnnOpName(), perTokenScaleName_.c_str(),
658 ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)),716 ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)),
659 "axis K of perTokenScale must be equal to ceil(k/" + std::to_string(MXFP_DIVISOR_SIZE) +717 "axis K of perTokenScale must be equal to ceil(k/" + std::to_string(MXFP_DIVISOR_SIZE) +
660- ") [" + std::to_string(inferedScaleKDimValue) + "]"));718+ ") [" + std::to_string(dims.inferedScaleKDimValue) + "]"));
661- GMM_CHECK_REPORT(pertokenScaleLastDimValue == MXFP_MULTI_BASE_SIZE,719+ GMM_CHECK_REPORT(dims.pertokenScaleLastDimValue == MXFP_MULTI_BASE_SIZE,
662 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(720 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
663 GetAclnnOpName(), perTokenScaleName_.c_str(),721 GetAclnnOpName(), perTokenScaleName_.c_str(),
664 ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)),722 ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, tensorIndex.perTokenScale)),
665- "current perTokenScale last axis is " + std::to_string(pertokenScaleLastDimValue) +723+ "current perTokenScale last axis is " + std::to_string(dims.pertokenScaleLastDimValue) +
666 ", expected " + std::to_string(MXFP_MULTI_BASE_SIZE)));724 ", expected " + std::to_string(MXFP_MULTI_BASE_SIZE)));
667- GMM_CHECK_REPORT(scaleLastDimValue == MXFP_MULTI_BASE_SIZE,725+ GMM_CHECK_REPORT(dims.scaleLastDimValue == MXFP_MULTI_BASE_SIZE,
668 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(726 OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
669 GetAclnnOpName(), scaleName_.c_str(),727 GetAclnnOpName(), scaleName_.c_str(),
670 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),728 ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, tensorIndex.scale)),
671- "current scale last axis is " + std::to_string(scaleLastDimValue) + ", expected " +729+ "current scale last axis is " + std::to_string(dims.scaleLastDimValue) + ", expected " +
672 std::to_string(MXFP_MULTI_BASE_SIZE)));730 std::to_string(MXFP_MULTI_BASE_SIZE)));
673 return ACLNN_SUCCESS;731 return ACLNN_SUCCESS;
674}732}
675 733 
676template <typename T>734template <typename T>
677-aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxFp8TypeKCaseInputShape(const TensorDimInfo &dimInfo,735+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxTypeMCaseInputShape(const TensorDimInfo &dimInfo,
678- size_t index) const736+ size_t index) const
679{737{
680 const auto tensorIndex = GetTensorIndexInfo(index);738 const auto tensorIndex = GetTensorIndexInfo(index);
739+ auto dims = ExtractMxTypeMDims(dimInfo, index);
740+ dims.groupNum = dimInfo.groupNum;
741+ CHECK_RET(CheckMxBiasInputShape(dimInfo, index) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
742+ GMM_CHECK_REPORT(
743+ dims.xMDimValue == dims.pertokenMDimValue,
744+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
745+ GetAclnnOpName(), xName_.c_str(), ViewShapeToString(GetInputTensor(gmmParams_.x, tensorIndex.x)),
746+ "axis M of x must be equal to axis M of perTokenScale [" + std::to_string(dims.pertokenMDimValue) + "]"));
747+ CHECK_RET(CheckMxTypeMScaleShape(dims, tensorIndex) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
748+ CHECK_RET(CheckMxTypeMPerTokenAndScaleLastDim(dims, tensorIndex) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
749+ return ACLNN_SUCCESS;
750+}
751+ 
752+template <typename T>
753+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxSplitKDimNum(const TensorDimInfo &dimInfo) const
754+{
681 size_t xDimNum = dimInfo.xDimNum;755 size_t xDimNum = dimInfo.xDimNum;
682 size_t weightDimNum = dimInfo.weightDimNum;756 size_t weightDimNum = dimInfo.weightDimNum;
683 size_t scaleDimNum = dimInfo.scaleDimNum;757 size_t scaleDimNum = dimInfo.scaleDimNum;
684 size_t pertokenScaleDimNum = dimInfo.pertokenScaleDimNum;758 size_t pertokenScaleDimNum = dimInfo.pertokenScaleDimNum;
685- int64_t groupNum = dimInfo.groupNum;
686 // split k, x is (m,k), weight is (k,n), scale is (k//64+g, n, 2), pertoken is (m, k//64+g, 2)759 // split k, x is (m,k), weight is (k,n), scale is (k//64+g, n, 2), pertoken is (m, k//64+g, 2)
687 GMM_CHECK_REPORT(xDimNum == MX_SPLIT_K_SINGLE_X_DIM,760 GMM_CHECK_REPORT(xDimNum == MX_SPLIT_K_SINGLE_X_DIM,
688 OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), xName_.c_str(), std::to_string(xDimNum),761 OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), xName_.c_str(), std::to_string(xDimNum),
@@ -697,6 +770,13 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxFp8TypeKCaseInputShape(c
697 pertokenScaleDimNum == MX_SPLIT_K_PER_TOKEN_SCALE_DIM,770 pertokenScaleDimNum == MX_SPLIT_K_PER_TOKEN_SCALE_DIM,
698 OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), perTokenScaleName_.c_str(), std::to_string(pertokenScaleDimNum),771 OP_LOGE_FOR_INVALID_SHAPEDIM(GetAclnnOpName(), perTokenScaleName_.c_str(), std::to_string(pertokenScaleDimNum),
699 std::to_string(MX_SPLIT_K_PER_TOKEN_SCALE_DIM)));772 std::to_string(MX_SPLIT_K_PER_TOKEN_SCALE_DIM)));
773+ return ACLNN_SUCCESS;
774+}
775+ 
776+template <typename T>
777+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxSplitKDimValue(const TensorIndexInfo &tensorIndex,
778+ int64_t groupNum) const
779+{
700 auto xMDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(0);780 auto xMDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(0);
701 auto xKDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(1);781 auto xKDimValue = GetInputTensor(gmmParams_.x, tensorIndex.x)->GetViewShape().GetDim(1);
702 auto weightNDimValue = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDim(1);782 auto weightNDimValue = GetInputTensor(gmmParams_.weight, tensorIndex.weight)->GetViewShape().GetDim(1);
@@ -749,6 +829,15 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxFp8TypeKCaseInputShape(c
749 return ACLNN_SUCCESS;829 return ACLNN_SUCCESS;
750}830}
751 831 
832+template <typename T>
833+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckMxFp8TypeKCaseInputShape(const TensorDimInfo &dimInfo,
834+ size_t index) const
835+{
836+ const auto tensorIndex = GetTensorIndexInfo(index);
837+ CHECK_RET(CheckMxSplitKDimNum(dimInfo) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
838+ return CheckMxSplitKDimValue(tensorIndex, dimInfo.groupNum);
839+}
840+ 
752template <typename T>841template <typename T>
753aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulMxShape() const842aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulMxShape() const
754{843{
@@ -970,6 +1059,65 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckNonPerGroupQuantDim() cons
970 return ACLNN_SUCCESS;1059 return ACLNN_SUCCESS;
971}1060}
972 1061 
1062+template <typename T>
1063+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckSplitMPerTokenShape(size_t perTokenDimNumber,
1064+ int64_t perTokenFirstDim, int64_t xMDim,
1065+ int64_t groupNum) const
1066+{
1067+ if (perTokenDimNumber == 1) {
1068+ GMM_CHECK_REPORT(
1069+ perTokenFirstDim == xMDim || perTokenFirstDim == groupNum,
1070+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1071+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1072+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1073+ "in non-pergroup quantification mode and when groupType is 0 (split M) and "
1074+ "perTokenScale has 1 dim, axis 0 of perTokenScale must be equal to axis M of x [" +
1075+ std::to_string(xMDim) + "] or axis 0 of groupTensor [" + std::to_string(groupNum) + "]"));
1076+ } else {
1077+ GMM_CHECK_REPORT(perTokenFirstDim == groupNum,
1078+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1079+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1080+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1081+ "in non-pergroup quantification mode and when groupType is 0 (split M) and "
1082+ "perTokenScale has 2 dims, axis 0 of perTokenScale must be equal to axis 0 of "
1083+ "groupTensor [" +
1084+ std::to_string(groupNum) + "]"));
1085+ auto perTokenSecondDim = GetInputTensor(gmmParams_.perTokenScaleOptional)->GetViewShape().GetDim(1);
1086+ GMM_CHECK_REPORT(perTokenSecondDim == 1,
1087+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1088+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1089+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1090+ "in non-pergroup quantification mode and when groupType is 0 (split M) and "
1091+ "perTokenScale has 2 dims, axis 1 of perTokenScale must be equal to 1"));
1092+ }
1093+ return ACLNN_SUCCESS;
1094+}
1095+ 
1096+template <typename T>
1097+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckSplitKPerTokenShape(size_t perTokenDimNumber,
1098+ int64_t perTokenFirstDim, int64_t xMDim,
1099+ int64_t groupNum) const
1100+{
1101+ GMM_CHECK_REPORT(perTokenFirstDim == groupNum,
1102+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1103+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1104+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1105+ "in non-pergroup quantification mode and when groupType is 2 (split K), axis 0 of "
1106+ "perTokenScale must be equal to axis 0 of groupTensor [" +
1107+ std::to_string(groupNum) + "]"));
1108+ if (perTokenDimNumber > 1) {
1109+ auto perTokenSecondDim = GetInputTensor(gmmParams_.perTokenScaleOptional)->GetViewShape().GetDim(1);
1110+ GMM_CHECK_REPORT(perTokenSecondDim == xMDim || perTokenSecondDim == 1,
1111+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1112+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1113+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1114+ "in non-pergroup quantification mode and when groupType is 2 (split K) and "
1115+ "perTokenScale has 2 dims, axis 1 of perTokenScale must be equal to axis M of x [" +
1116+ std::to_string(xMDim) + "] or 1"));
1117+ }
1118+ return ACLNN_SUCCESS;
1119+}
1120+ 
973template <typename T>1121template <typename T>
974aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckNonPerGroupQuantPertokenShape() const1122aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckNonPerGroupQuantPertokenShape() const
975{1123{
@@ -978,50 +1126,9 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckNonPerGroupQuantPertokenSh
978 auto xMDim = GetInputTensor(gmmParams_.x)->GetViewShape().GetDim(0);1126 auto xMDim = GetInputTensor(gmmParams_.x)->GetViewShape().GetDim(0);
979 auto groupNum = gmmParams_.groupTensorOptional->GetViewShape().GetDim(0);1127 auto groupNum = gmmParams_.groupTensorOptional->GetViewShape().GetDim(0);
980 if (gmmParams_.groupType == SPLIT_M) {1128 if (gmmParams_.groupType == SPLIT_M) {
981- if (perTokenDimNumber == 1) {1129+ return CheckSplitMPerTokenShape(perTokenDimNumber, perTokenFirstDim, xMDim, groupNum);
982- GMM_CHECK_REPORT(
983- perTokenFirstDim == xMDim || perTokenFirstDim == groupNum,
984- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
985- GetAclnnOpName(), perTokenScaleName_.c_str(),
986- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
987- "in non-pergroup quantification mode and when groupType is 0 (split M) and "
988- "perTokenScale has 1 dim, axis 0 of perTokenScale must be equal to axis M of x [" +
989- std::to_string(xMDim) + "] or axis 0 of groupTensor [" + std::to_string(groupNum) + "]"));
990- } else {
991- GMM_CHECK_REPORT(perTokenFirstDim == groupNum,
992- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
993- GetAclnnOpName(), perTokenScaleName_.c_str(),
994- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
995- "in non-pergroup quantification mode and when groupType is 0 (split M) and "
996- "perTokenScale has 2 dims, axis 0 of perTokenScale must be equal to axis 0 of "
997- "groupTensor [" +
998- std::to_string(groupNum) + "]"));
999- auto perTokenSecondDim = GetInputTensor(gmmParams_.perTokenScaleOptional)->GetViewShape().GetDim(1);
1000- GMM_CHECK_REPORT(perTokenSecondDim == 1,
1001- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1002- GetAclnnOpName(), perTokenScaleName_.c_str(),
1003- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1004- "in non-pergroup quantification mode and when groupType is 0 (split M) and "
1005- "perTokenScale has 2 dims, axis 1 of perTokenScale must be equal to 1"));
1006- }
1007 } else if (gmmParams_.groupType == SPLIT_K) {1130 } else if (gmmParams_.groupType == SPLIT_K) {
1008- GMM_CHECK_REPORT(perTokenFirstDim == groupNum,1131+ return CheckSplitKPerTokenShape(perTokenDimNumber, perTokenFirstDim, xMDim, groupNum);
1009- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1010- GetAclnnOpName(), perTokenScaleName_.c_str(),
1011- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1012- "in non-pergroup quantification mode and when groupType is 2 (split K), axis 0 of "
1013- "perTokenScale must be equal to axis 0 of groupTensor [" +
1014- std::to_string(groupNum) + "]"));
1015- if (perTokenDimNumber > 1) {
1016- auto perTokenSecondDim = GetInputTensor(gmmParams_.perTokenScaleOptional)->GetViewShape().GetDim(1);
1017- GMM_CHECK_REPORT(perTokenSecondDim == xMDim || perTokenSecondDim == 1,
1018- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1019- GetAclnnOpName(), perTokenScaleName_.c_str(),
1020- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional)),
1021- "in non-pergroup quantification mode and when groupType is 2 (split K) and "
1022- "perTokenScale has 2 dims, axis 1 of perTokenScale must be equal to axis M of x [" +
1023- std::to_string(xMDim) + "] or 1"));
1024- }
1025 }1132 }
1026 return ACLNN_SUCCESS;1133 return ACLNN_SUCCESS;
1027}1134}
@@ -1229,6 +1336,62 @@ bool AclnnGroupedMatmulDAV3510Checker<T>::IsPerTileQuantMode() const
1229 return isPerTileQuantMode;1336 return isPerTileQuantMode;
1230}1337}
1231 1338 
1339+template <typename T>
1340+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckPerTileMNShape(size_t i, int64_t xMDim, int64_t perTokenMDim,
1341+ int64_t weightNDim, int64_t scaleNDim) const
1342+{
1343+ GMM_CHECK_REPORT(
1344+ xMDim == perTokenMDim,
1345+ OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(
1346+ GetAclnnOpName(), "x and perTokenScale",
1347+ xName_ + "=" + ViewShapeToString(GetInputTensor(gmmParams_.x, i)) + ", " + perTokenScaleName_ + "=" +
1348+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, i)),
1349+ "when quantification mode is G-B quantification, the M value in x and perTokenScale should "
1350+ "be consistent"));
1351+ GMM_CHECK_REPORT(
1352+ scaleNDim == (weightNDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE,
1353+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1354+ GetAclnnOpName(), scaleName_.c_str(), ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, i)),
1355+ "when quantification mode is G-B quantification, axis N of scale must be equal to axis N of "
1356+ "weight divided by 128 [" +
1357+ std::to_string((weightNDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE) + "]"));
1358+ return ACLNN_SUCCESS;
1359+}
1360+ 
1361+template <typename T>
1362+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckPerTileKShape(size_t i, int64_t weightKDim, int64_t scaleKDim,
1363+ int64_t perTokenKDim) const
1364+{
1365+ if (gmmParams_.groupType == SPLIT_M) {
1366+ int64_t expectScaleKValue = (weightKDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE;
1367+ GMM_CHECK_REPORT(
1368+ perTokenKDim == scaleKDim && scaleKDim == expectScaleKValue,
1369+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1370+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1371+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, i)),
1372+ "when quantification mode is G-B quantification and groupType is 0 (split M), axis K of "
1373+ "perTokenScale must be equal to axis K of scale [" +
1374+ std::to_string(scaleKDim) +
1375+ "], and its value must be equal to axis K of weight divided by 128, rounded up to the "
1376+ "next integer [" +
1377+ std::to_string(expectScaleKValue) + "]"));
1378+ } else {
1379+ int64_t expectScaleKValue =
1380+ (weightKDim / PERTILE_GROUP_SIZE) + gmmParams_.groupTensorOptional->GetViewShape().GetDim(0);
1381+ GMM_CHECK_REPORT(
1382+ perTokenKDim == scaleKDim && scaleKDim == expectScaleKValue,
1383+ OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1384+ GetAclnnOpName(), perTokenScaleName_.c_str(),
1385+ ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, i)),
1386+ "when quantification mode is G-B quantification and groupType is 2 (split K), axis K of "
1387+ "perTokenScale must be equal to axis K of scale [" +
1388+ std::to_string(scaleKDim) +
1389+ "], and its value must be equal to axis K of weight divided by 128, plus groupSize [" +
1390+ std::to_string(expectScaleKValue) + "]"));
1391+ }
1392+ return ACLNN_SUCCESS;
1393+}
1394+ 
1232template <typename T>1395template <typename T>
1233aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulPerTileShape() const1396aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulPerTileShape() const
1234{1397{
@@ -1244,48 +1407,10 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulPerTileShape(
1244 auto perTokenKDim = GetInputTensor(gmmParams_.perTokenScaleOptional, i)->GetViewShape().GetDim(xKIndex);1407 auto perTokenKDim = GetInputTensor(gmmParams_.perTokenScaleOptional, i)->GetViewShape().GetDim(xKIndex);
1245 auto scaleNDim = GetInputTensor(gmmParams_.scaleOptional, i)->GetViewShape().GetDim(weightNIndex);1408 auto scaleNDim = GetInputTensor(gmmParams_.scaleOptional, i)->GetViewShape().GetDim(weightNIndex);
1246 auto perTokenMDim = GetInputTensor(gmmParams_.perTokenScaleOptional, i)->GetViewShape().GetDim(xMIndex);1409 auto perTokenMDim = GetInputTensor(gmmParams_.perTokenScaleOptional, i)->GetViewShape().GetDim(xMIndex);
1247- GMM_CHECK_REPORT(1410+ CHECK_RET(CheckPerTileMNShape(i, xMDim, perTokenMDim, weightNDim, scaleNDim) == ACLNN_SUCCESS,
1248- xMDim == perTokenMDim,1411+ ACLNN_ERR_PARAM_INVALID);
1249- OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(1412+ CHECK_RET(CheckPerTileKShape(i, weightKDim, scaleKDim, perTokenKDim) == ACLNN_SUCCESS,
1250- GetAclnnOpName(), "x and perTokenScale",1413+ ACLNN_ERR_PARAM_INVALID);
1251- xName_ + "=" + ViewShapeToString(GetInputTensor(gmmParams_.x, i)) + ", " + perTokenScaleName_ + "=" +
1252- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, i)),
1253- "when quantification mode is G-B quantification, the M value in x and perTokenScale should "
1254- "be consistent"));
1255- GMM_CHECK_REPORT(
1256- scaleNDim == (weightNDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE,
1257- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1258- GetAclnnOpName(), scaleName_.c_str(), ViewShapeToString(GetInputTensor(gmmParams_.scaleOptional, i)),
1259- "when quantification mode is G-B quantification, axis N of scale must be equal to axis N of "
1260- "weight divided by 128 [" +
1261- std::to_string((weightNDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE) + "]"));
1262- if (gmmParams_.groupType == SPLIT_M) {
1263- int64_t expectScaleKValue = (weightKDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE;
1264- GMM_CHECK_REPORT(
1265- perTokenKDim == scaleKDim && scaleKDim == expectScaleKValue,
1266- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1267- GetAclnnOpName(), perTokenScaleName_.c_str(),
1268- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, i)),
1269- "when quantification mode is G-B quantification and groupType is 0 (split M), axis K of "
1270- "perTokenScale must be equal to axis K of scale [" +
1271- std::to_string(scaleKDim) +
1272- "], and its value must be equal to axis K of weight divided by 128, rounded up to the "
1273- "next integer [" +
1274- std::to_string(expectScaleKValue) + "]"));
1275- } else {
1276- int64_t expectScaleKValue =
1277- (weightKDim / PERTILE_GROUP_SIZE) + gmmParams_.groupTensorOptional->GetViewShape().GetDim(0);
1278- GMM_CHECK_REPORT(
1279- perTokenKDim == scaleKDim && scaleKDim == expectScaleKValue,
1280- OP_LOGE_FOR_INVALID_SHAPE_WITH_REASON(
1281- GetAclnnOpName(), perTokenScaleName_.c_str(),
1282- ViewShapeToString(GetInputTensor(gmmParams_.perTokenScaleOptional, i)),
1283- "when quantification mode is G-B quantification and groupType is 2 (split K), axis K of "
1284- "perTokenScale must be equal to axis K of scale [" +
1285- std::to_string(scaleKDim) +
1286- "], and its value must be equal to axis K of weight divided by 128, plus groupSize [" +
1287- std::to_string(expectScaleKValue) + "]"));
1288- }
1289 }1414 }
1290 return ACLNN_SUCCESS;1415 return ACLNN_SUCCESS;
1291}1416}
@@ -1479,11 +1604,8 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckInputParamsForV3Version()
1479}1604}
1480 1605 
1481template <typename T>1606template <typename T>
1482-aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulDAV3510() const1607+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckBasicQuantParams(DataType yDtype) const
1483{1608{
1484- DataType xDtype = gmmParams_.xDtype;
1485- DataType weightDtype = GetInputTensor(gmmParams_.weight)->GetDataType();
1486- DataType yDtype = GetInputTensor(gmmParams_.y)->GetDataType();
1487 if (yDtype != DataType::DT_INT32) {1609 if (yDtype != DataType::DT_INT32) {
1488 GMM_CHECK_REPORT(1610 GMM_CHECK_REPORT(
1489 gmmParams_.scaleOptional != nullptr,1611 gmmParams_.scaleOptional != nullptr,
@@ -1500,6 +1622,12 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulDAV3510() con
1500 gmmParams_.groupType != SPLIT_N,1622 gmmParams_.groupType != SPLIT_N,
1501 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(GetAclnnOpName(), "groupType", std::to_string(gmmParams_.groupType),1623 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(GetAclnnOpName(), "groupType", std::to_string(gmmParams_.groupType),
1502 "the value of groupType must be in 0 or 2"));1624 "the value of groupType must be in 0 or 2"));
1625+ return ACLNN_SUCCESS;
1626+}
1627+ 
1628+template <typename T>
1629+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckQuantShapeAndFormat() const
1630+{
1503 const size_t xTensorNum = GetInputTensorSize(gmmParams_.x);1631 const size_t xTensorNum = GetInputTensorSize(gmmParams_.x);
1504 const size_t weightTensorNum = GetInputTensorSize(gmmParams_.weight);1632 const size_t weightTensorNum = GetInputTensorSize(gmmParams_.weight);
1505 const size_t yTensorNum = GetInputTensorSize(gmmParams_.y);1633 const size_t yTensorNum = GetInputTensorSize(gmmParams_.y);
@@ -1515,14 +1643,13 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulDAV3510() con
1515 CHECK_RET(CheckWeightNzSpecialParams() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);1643 CHECK_RET(CheckWeightNzSpecialParams() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1516 }1644 }
1517 CHECK_RET(CheckGeneralQuantShape() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);1645 CHECK_RET(CheckGeneralQuantShape() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1646+ return ACLNN_SUCCESS;
1647+}
1518 1648 
1519- if (gmmParams_.apiVersion == gmm::GMMApiVersion::V3) {1649+template <typename T>
1520- CHECK_RET(CheckInputParamsForV3Version() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);1650+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckQuantParamsByDtype(DataType xDtype, DataType weightDtype,
1521- }1651+ DataType yDtype, DataType scaleDtype) const
1522- DataType scaleDtype = DataType::DT_UINT64;1652+{
1523- if (gmmParams_.scaleOptional != nullptr) {
1524- scaleDtype = GetInputTensor(gmmParams_.scaleOptional)->GetDataType();
1525- }
1526 if (xDtype == DataType::DT_INT8 && weightDtype == DataType::DT_INT8) {1653 if (xDtype == DataType::DT_INT8 && weightDtype == DataType::DT_INT8) {
1527 return CheckInt8QuantParams();1654 return CheckInt8QuantParams();
1528 } else if (xDtype == DataType::DT_HIFLOAT8 && weightDtype == DataType::DT_HIFLOAT8) {1655 } else if (xDtype == DataType::DT_HIFLOAT8 && weightDtype == DataType::DT_HIFLOAT8) {
@@ -1572,3 +1699,22 @@ aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulDAV3510() con
1572 }1699 }
1573 return ACLNN_SUCCESS;1700 return ACLNN_SUCCESS;
1574}1701}
1702+ 
1703+template <typename T>
1704+aclnnStatus AclnnGroupedMatmulDAV3510Checker<T>::CheckGroupedMatmulDAV3510() const
1705+{
1706+ DataType xDtype = gmmParams_.xDtype;
1707+ DataType weightDtype = GetInputTensor(gmmParams_.weight)->GetDataType();
1708+ DataType yDtype = GetInputTensor(gmmParams_.y)->GetDataType();
1709+ CHECK_RET(CheckBasicQuantParams(yDtype) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1710+ CHECK_RET(CheckQuantShapeAndFormat() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1711+ 
1712+ if (gmmParams_.apiVersion == gmm::GMMApiVersion::V3) {
1713+ CHECK_RET(CheckInputParamsForV3Version() == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
1714+ }
1715+ DataType scaleDtype = DataType::DT_UINT64;
1716+ if (gmmParams_.scaleOptional != nullptr) {
1717+ scaleDtype = GetInputTensor(gmmParams_.scaleOptional)->GetDataType();
1718+ }
1719+ return CheckQuantParamsByDtype(xDtype, weightDtype, yDtype, scaleDtype);
1720+}
Mgmm/grouped_matmul/op_api/grouped_matmul_950_checker.h+36-0
@@ -52,19 +52,39 @@ private:
52 aclnnStatus CheckWeightNzTensorShape(const aclTensor *weightTensor, const aclTensor *firstWeightTensor,52 aclnnStatus CheckWeightNzTensorShape(const aclTensor *weightTensor, const aclTensor *firstWeightTensor,
53 size_t index, int64_t &firstKDimValue, int64_t &firstNDimValue) const;53 size_t index, int64_t &firstKDimValue, int64_t &firstNDimValue) const;
54 aclnnStatus CheckWeightStorageShape(const aclTensor *weightTensor, int64_t kDimValue, int64_t nDimValue) const;54 aclnnStatus CheckWeightStorageShape(const aclTensor *weightTensor, int64_t kDimValue, int64_t nDimValue) const;
55+ aclnnStatus CheckWeightNzStorageDim(const aclTensor *weightTensor) const;
56+ aclnnStatus CheckWeightNzC0(const aclTensor *weightTensor, int64_t weightStorageLastDim,
57+ int64_t &cubeBlockSizeK) const;
58+ aclnnStatus CheckWeightNzOuterDims(const aclTensor *weightTensor, int64_t kDimValue, int64_t nDimValue,
59+ int64_t cubeBlockSizeK, int64_t weightStorageLastFourthDim,
60+ int64_t weightStorageLastThirdDim) const;
61+ aclnnStatus CheckBasicQuantParams(DataType yDtype) const;
62+ aclnnStatus CheckQuantShapeAndFormat() const;
63+ aclnnStatus CheckQuantParamsByDtype(DataType xDtype, DataType weightDtype, DataType yDtype,
64+ DataType scaleDtype) const;
55 65 
56 aclnnStatus CheckGroupedMatmulMxDtype() const;66 aclnnStatus CheckGroupedMatmulMxDtype() const;
57 aclnnStatus CheckGroupedMatmulPerGroupDim() const;67 aclnnStatus CheckGroupedMatmulPerGroupDim() const;
68+ aclnnStatus CheckPerGroupWeightDim(size_t weightDimNumber) const;
69+ aclnnStatus CheckPerGroupScaleDim(const TensorIndexInfo &tensorIndex, size_t scaleDimNumber,
70+ size_t perTokenDimNumber, size_t xDimNumber, size_t weightDimNumber) const;
58 aclnnStatus CheckGroupedMatmulMxShape() const;71 aclnnStatus CheckGroupedMatmulMxShape() const;
59 aclnnStatus CheckGroupedMatmulMxScaleTranspose() const;72 aclnnStatus CheckGroupedMatmulMxScaleTranspose() const;
60 aclnnStatus CheckGroupedMatmulPerTile() const;73 aclnnStatus CheckGroupedMatmulPerTile() const;
61 aclnnStatus CheckGroupedMatmulPerTileShape() const;74 aclnnStatus CheckGroupedMatmulPerTileShape() const;
75+ aclnnStatus CheckPerTileMNShape(size_t i, int64_t xMDim, int64_t perTokenMDim, int64_t weightNDim,
76+ int64_t scaleNDim) const;
77+ aclnnStatus CheckPerTileKShape(size_t i, int64_t weightKDim, int64_t scaleKDim, int64_t perTokenKDim) const;
62 aclnnStatus CheckGroupedMatmulMxfp8() const;78 aclnnStatus CheckGroupedMatmulMxfp8() const;
63 aclnnStatus CheckGroupedMatmulMxfp4() const;79 aclnnStatus CheckGroupedMatmulMxfp4() const;
64 aclnnStatus CheckGroupedMatmulFp4MxDimValue() const;80 aclnnStatus CheckGroupedMatmulFp4MxDimValue() const;
65 81 
66 aclnnStatus CheckNonPerGroupQuantDim() const;82 aclnnStatus CheckNonPerGroupQuantDim() const;
67 aclnnStatus CheckNonPerGroupQuantPertokenShape() const;83 aclnnStatus CheckNonPerGroupQuantPertokenShape() const;
84+ aclnnStatus CheckSplitMPerTokenShape(size_t perTokenDimNumber, int64_t perTokenFirstDim, int64_t xMDim,
85+ int64_t groupNum) const;
86+ aclnnStatus CheckSplitKPerTokenShape(size_t perTokenDimNumber, int64_t perTokenFirstDim, int64_t xMDim,
87+ int64_t groupNum) const;
68 aclnnStatus CheckNonPerGroupQuantShape() const;88 aclnnStatus CheckNonPerGroupQuantShape() const;
69 aclnnStatus CheckInt8QuantDtype() const;89 aclnnStatus CheckInt8QuantDtype() const;
70 aclnnStatus CheckInt8QuantParams() const;90 aclnnStatus CheckInt8QuantParams() const;
@@ -82,7 +102,23 @@ private:
82 bool IsWeightNzMultiTensorLayout() const;102 bool IsWeightNzMultiTensorLayout() const;
83 TensorIndexInfo GetTensorIndexInfo(size_t index = 0) const;103 TensorIndexInfo GetTensorIndexInfo(size_t index = 0) const;
84 aclnnStatus CheckMxFp8TypeKCaseInputShape(const TensorDimInfo &dimInfo, size_t index) const;104 aclnnStatus CheckMxFp8TypeKCaseInputShape(const TensorDimInfo &dimInfo, size_t index) const;
105+ aclnnStatus CheckMxSplitKDimNum(const TensorDimInfo &dimInfo) const;
106+ aclnnStatus CheckMxSplitKDimValue(const TensorIndexInfo &tensorIndex, int64_t groupNum) const;
107+ struct MxTypeMDims {
108+ int64_t xMDimValue;
109+ int64_t xKDimValue;
110+ int64_t pertokenMDimValue;
111+ int64_t pertokenScaleKDimValue;
112+ int64_t pertokenScaleLastDimValue;
113+ int64_t weightNDimValue;
114+ int64_t inferedScaleKDimValue;
115+ int64_t scaleLastDimValue;
116+ };
85 aclnnStatus CheckMxTypeMCaseInputShape(const TensorDimInfo &dimInfo, size_t index) const;117 aclnnStatus CheckMxTypeMCaseInputShape(const TensorDimInfo &dimInfo, size_t index) const;
118+ MxTypeMDims ExtractMxTypeMDims(const TensorDimInfo &dimInfo, size_t index) const;
119+ aclnnStatus CheckMxTypeMScaleShape(const MxTypeMDims &dims, const TensorDimInfo &tensorIndex) const;
120+ aclnnStatus CheckMxTypeMPerTokenAndScaleLastDim(const MxTypeMDims &dims,
121+ const TensorDimInfo &tensorIndex) const;
86 aclnnStatus CheckMxBiasInputShape(const TensorDimInfo &dimInfo, size_t index) const;122 aclnnStatus CheckMxBiasInputShape(const TensorDimInfo &dimInfo, size_t index) const;
87 bool LastTwoDimValueIsOne(const aclTensor *tensor) const;123 bool LastTwoDimValueIsOne(const aclTensor *tensor) const;
88 bool IsSpecialperTileScene(int64_t groupNum, int64_t weightNDim, int64_t weightKDim, int64_t xMDim,124 bool IsSpecialperTileScene(int64_t groupNum, int64_t weightNDim, int64_t weightKDim, int64_t xMDim,
Mgmm/grouped_matmul/op_api/grouped_matmul_weight_quant_950_checker.cpp+109-59
@@ -282,6 +282,81 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckTensorDtype(const
282 return ACLNN_SUCCESS;282 return ACLNN_SUCCESS;
283}283}
284 284 
285+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::GetExpectedDimNum(
286+ const AclnnGroupedMatmulWeightQuantDAV3510Checker &checker, const std::string &tensorType, size_t tensorDimNum,
287+ size_t &expectedDimNum)
288+{
289+ if ((checker.IsA16MxFp4NZ() || checker.IsS8S4NZ()) && tensorType.find("antiquant") != std::string::npos) {
290+ expectedDimNum = 3; // Mx / PerGroup量化，仅支持antiquantSacle/antiquantOffset维度为3
291+ } else if (checker.IsMxA8W4NZ()) {
292+ if (tensorType.find("antiquant") != std::string::npos) {
293+ expectedDimNum =
294+ checker.IsMultiTensorWeight() ? MX_MULTI_ANTIQUANT_SCALE_DIM : MX_SINGLE_ANTIQUANT_SCALE_DIM;
295+ } else if (tensorType.find("token") != std::string::npos) {
296+ expectedDimNum = 3; // MxA8W4场景，perTokenScale维度为3
297+ } else if (tensorType.find("bias") != std::string::npos) {
298+ expectedDimNum = checker.IsMultiTensorWeight() ? MX_MULTI_BIAS_DIM : MX_SINGLE_BIAS_DIM;
299+ }
300+ } else if (checker.IsA16W4() && tensorType.find("antiquant") != std::string::npos) {
301+ size_t perchannelDim =
302+ checker.gmmParams_.groupType == SPLIT_M ? 2 : 1; // 单单单场景默认维度为2，多多多场景默认维度为1
303+ size_t pergroupDim = perchannelDim + 1;
304+ if (tensorDimNum != perchannelDim && tensorDimNum != pergroupDim) {
305+ std::string reason = "When x_dtype-weight_dtype is fp16/bf16-int4, Dim must be [" +
306+ std::to_string(perchannelDim) + "] (perchannel) or [" + std::to_string(pergroupDim) +
307+ "] (pergroup), but now is [" + std::to_string(tensorDimNum) + "]";
308+ OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(checker.GetAclnnName(), tensorType,
309+ std::to_string(tensorDimNum), reason);
310+ return ACLNN_ERR_PARAM_INVALID;
311+ }
312+ expectedDimNum = tensorDimNum;
313+ }
314+ return ACLNN_SUCCESS;
315+}
316+ 
317+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckBatchSizeConsistency(
318+ const AclnnGroupedMatmulWeightQuantDAV3510Checker &checker, const op::Shape &tensorShape,
319+ const op::Shape &weightShape, const std::string &tensorType)
320+{
321+ if (checker.gmmParams_.groupType == SPLIT_M && !checker.IsMultiTensorWeight()) {
322+ uint64_t groupNum = weightShape.GetDim(0);
323+ uint64_t batchSize = tensorShape.GetDim(0);
324+ if (unlikely(batchSize != groupNum)) {
325+ std::string incorrectValue = std::to_string(batchSize);
326+ std::string reason = "batch size[" + incorrectValue + "] should be equal with groupList length[" +
327+ std::to_string(groupNum) + "]";
328+ OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(checker.GetAclnnName(), tensorType, incorrectValue, reason);
329+ return ACLNN_ERR_PARAM_INVALID;
330+ }
331+ }
332+ return ACLNN_SUCCESS;
333+}
334+ 
335+// Check tensor’s Ndim must match weight’s Ndim.
336+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckNDimConsistency(
337+ const AclnnGroupedMatmulWeightQuantDAV3510Checker &checker, const op::Shape &tensorShape,
338+ const op::Shape &weightShape, size_t tensorDimNum, const std::string &tensorType)
339+{
340+ uint64_t weightNDimIdx =
341+ checker.IsS8S4SpecialWeightFormat() ? S8S4_WEIGHT_K_DIM : weightShape.GetDimNum() - 1;
342+ int64_t weightNDimValue = weightShape.GetDim(weightNDimIdx);
343+ int64_t tensorNDimValue;
344+ if (checker.IsMxA8W4NZ() && tensorType.find("antiquant") != std::string::npos) { // viewShape,所以是-2
345+ tensorNDimValue = tensorShape.GetDim(tensorDimNum - 2);
346+ } else {
347+ tensorNDimValue = tensorShape.GetDim(tensorDimNum - 1);
348+ }
349+ if (unlikely(tensorNDimValue != weightNDimValue)) {
350+ std::string incorrectValues = std::to_string(tensorNDimValue) + ", " + std::to_string(weightNDimValue);
351+ std::string reason = "NDim[" + std::to_string(tensorNDimValue) + "] of " + tensorType +
352+ " must be equal to NDim[" + std::to_string(weightNDimValue) + "] of weight";
353+ OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(checker.GetAclnnName(), tensorType + ", weight", incorrectValues,
354+ reason);
355+ return ACLNN_ERR_PARAM_INVALID;
356+ }
357+ return ACLNN_SUCCESS;
358+}
359+ 
285aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckTensorShape(const aclTensorList *tensorList, size_t idx,360aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckTensorShape(const aclTensorList *tensorList, size_t idx,
286 const std::string &tensorType) const361 const std::string &tensorType) const
287{362{
@@ -292,28 +367,8 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckTensorShape(const
292 size_t tensorDimNum = tensorShape.GetDimNum();367 size_t tensorDimNum = tensorShape.GetDimNum();
293 size_t expectedDimNum = gmmParams_.groupType == SPLIT_M ? 2 : 1; // 单单单场景默认维度为2，多多多场景默认维度为1368 size_t expectedDimNum = gmmParams_.groupType == SPLIT_M ? 2 : 1; // 单单单场景默认维度为2，多多多场景默认维度为1
294 369 
295- if ((IsA16MxFp4NZ() || IsS8S4NZ()) && tensorType.find("antiquant") != std::string::npos) {370+ CHECK_RET(GetExpectedDimNum(*this, tensorType, tensorDimNum, expectedDimNum) == ACLNN_SUCCESS,
296- expectedDimNum = 3; // Mx / PerGroup量化，仅支持antiquantSacle/antiquantOffset维度为3371+ ACLNN_ERR_PARAM_INVALID);
297- } else if (IsMxA8W4NZ()) {
298- if (tensorType.find("antiquant") != std::string::npos) {
299- expectedDimNum = IsMultiTensorWeight() ? MX_MULTI_ANTIQUANT_SCALE_DIM : MX_SINGLE_ANTIQUANT_SCALE_DIM;
300- } else if (tensorType.find("token") != std::string::npos) {
301- expectedDimNum = 3; // MxA8W4场景，perTokenScale维度为3
302- } else if (tensorType.find("bias") != std::string::npos) {
303- expectedDimNum = IsMultiTensorWeight() ? MX_MULTI_BIAS_DIM : MX_SINGLE_BIAS_DIM;
304- }
305- } else if (IsA16W4() && tensorType.find("antiquant") != std::string::npos) {
306- size_t perchannelDim = gmmParams_.groupType == SPLIT_M ? 2 : 1; // 单单单场景默认维度为2，多多多场景默认维度为1
307- size_t pergroupDim = perchannelDim + 1;
308- if (tensorDimNum != perchannelDim && tensorDimNum != pergroupDim) {
309- std::string reason = "When x_dtype-weight_dtype is fp16/bf16-int4, Dim must be [" +
310- std::to_string(perchannelDim) + "] (perchannel) or [" + std::to_string(pergroupDim) +
311- "] (pergroup), but now is [" + std::to_string(tensorDimNum) + "]";
312- OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(GetAclnnName(), tensorType, std::to_string(tensorDimNum), reason);
313- return ACLNN_ERR_PARAM_INVALID;
314- }
315- expectedDimNum = tensorDimNum;
316- }
317 372 
318 if (unlikely(tensorDimNum != expectedDimNum)) {373 if (unlikely(tensorDimNum != expectedDimNum)) {
319 std::string incorrectDim = std::to_string(tensorDimNum);374 std::string incorrectDim = std::to_string(tensorDimNum);
@@ -322,35 +377,11 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckTensorShape(const
322 return ACLNN_ERR_PARAM_INVALID;377 return ACLNN_ERR_PARAM_INVALID;
323 }378 }
324 379 
325- if (gmmParams_.groupType == SPLIT_M && !IsMultiTensorWeight()) {380+ CHECK_RET(CheckBatchSizeConsistency(*this, tensorShape, wShape, tensorType) == ACLNN_SUCCESS,
326- uint64_t groupNum = wShape.GetDim(0);381+ ACLNN_ERR_PARAM_INVALID);
327- uint64_t batchSize = tensorShape.GetDim(0);
328- if (unlikely(batchSize != groupNum)) {
329- std::string incorrectValue = std::to_string(batchSize);
330- std::string reason = "batch size[" + incorrectValue + "] should be equal with groupList length[" +
331- std::to_string(groupNum) + "]";
332- OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(GetAclnnName(), tensorType, incorrectValue, reason);
333- return ACLNN_ERR_PARAM_INVALID;
334- }
335- }
336 382 
337- // Check tensor’s Ndim must match weight’s Ndim.383+ CHECK_RET(CheckNDimConsistency(*this, tensorShape, wShape, tensorDimNum, tensorType) == ACLNN_SUCCESS,
338- uint64_t weightNDimIdx =384+ ACLNN_ERR_PARAM_INVALID);
339- IsS8S4SpecialWeightFormat() ? S8S4_WEIGHT_K_DIM : wShape.GetDimNum() - 1;
340- int64_t weightNDimValue = wShape.GetDim(weightNDimIdx);
341- int64_t tensorNDimValue;
342- if (IsMxA8W4NZ() && tensorType.find("antiquant") != std::string::npos) { // viewShape,所以是-2
343- tensorNDimValue = tensorShape.GetDim(tensorDimNum - 2);
344- } else {
345- tensorNDimValue = tensorShape.GetDim(tensorDimNum - 1);
346- }
347- if (unlikely(tensorNDimValue != weightNDimValue)) {
348- std::string incorrectValues = std::to_string(tensorNDimValue) + ", " + std::to_string(weightNDimValue);
349- std::string reason = "NDim[" + std::to_string(tensorNDimValue) + "] of " + tensorType +
350- " must be equal to NDim[" + std::to_string(weightNDimValue) + "] of weight";
351- OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(GetAclnnName(), tensorType + ", weight", incorrectValues, reason);
352- return ACLNN_ERR_PARAM_INVALID;
353- }
354 385 
355 return ACLNN_SUCCESS;386 return ACLNN_SUCCESS;
356}387}
@@ -430,8 +461,7 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckQuantParams() cons
430 return ACLNN_SUCCESS;461 return ACLNN_SUCCESS;
431}462}
432 463 
433-aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckDimNumAndFormat(size_t xIdx, size_t yIdx,464+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckInputFormat(size_t xIdx, size_t yIdx) const
434- size_t wIdx) const
435{465{
436 if (IsS8S4PseudoQuant()) {466 if (IsS8S4PseudoQuant()) {
437 CHECK_COND(ge::GetPrimaryFormat((*gmmParams_.x)[xIdx]->GetStorageFormat()) == op::Format::FORMAT_ND &&467 CHECK_COND(ge::GetPrimaryFormat((*gmmParams_.x)[xIdx]->GetStorageFormat()) == op::Format::FORMAT_ND &&
@@ -448,7 +478,11 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckDimNumAndFormat(si
448 op::ToString((*gmmParams_.y)[yIdx]->GetStorageFormat()).GetString(), "ND");478 op::ToString((*gmmParams_.y)[yIdx]->GetStorageFormat()).GetString(), "ND");
449 return ACLNN_ERR_PARAM_INVALID;479 return ACLNN_ERR_PARAM_INVALID;
450 }480 }
481+ return ACLNN_SUCCESS;
482+}
451 483 
484+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckWeightFormat(size_t wIdx) const
485+{
452 if (IsS8S4PseudoQuant()) {486 if (IsS8S4PseudoQuant()) {
453 const auto weightFormat = static_cast<op::Format>(487 const auto weightFormat = static_cast<op::Format>(
454 ge::GetPrimaryFormat((*gmmParams_.weight)[wIdx]->GetStorageFormat()));488 ge::GetPrimaryFormat((*gmmParams_.weight)[wIdx]->GetStorageFormat()));
@@ -478,12 +512,13 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckDimNumAndFormat(si
478 return ACLNN_ERR_PARAM_INVALID;512 return ACLNN_ERR_PARAM_INVALID;
479 }513 }
480 }514 }
515+ return ACLNN_SUCCESS;
516+}
481 517 
482- // check dimNum518+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckDimNumByGroupType(size_t xIdx, size_t wIdx,
483- size_t xDimNum = (*gmmParams_.x)[xIdx]->GetViewShape().GetDimNum();519+ size_t xDimNum,
484- size_t weightDimNum = (*gmmParams_.weight)[wIdx]->GetViewShape().GetDimNum();520+ size_t weightDimNum) const
485- size_t yDimNum = (*gmmParams_.y)[yIdx]->GetViewShape().GetDimNum();521+{
486- 
487 if (gmmParams_.groupType == NO_SPLIT) {522 if (gmmParams_.groupType == NO_SPLIT) {
488 if (IsA16W4Pergroup(wIdx)) {523 if (IsA16W4Pergroup(wIdx)) {
489 if (unlikely(xDimNum != MIN_FM_DIM)) {524 if (unlikely(xDimNum != MIN_FM_DIM)) {
@@ -529,6 +564,21 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckDimNumAndFormat(si
529 }564 }
530 }565 }
531 }566 }
567+ return ACLNN_SUCCESS;
568+}
569+ 
570+aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckDimNumAndFormat(size_t xIdx, size_t yIdx,
571+ size_t wIdx) const
572+{
573+ CHECK_RET(CheckInputFormat(xIdx, yIdx) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
574+ CHECK_RET(CheckWeightFormat(wIdx) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
575+ 
576+ // check dimNum
577+ size_t xDimNum = (*gmmParams_.x)[xIdx]->GetViewShape().GetDimNum();
578+ size_t weightDimNum = (*gmmParams_.weight)[wIdx]->GetViewShape().GetDimNum();
579+ size_t yDimNum = (*gmmParams_.y)[yIdx]->GetViewShape().GetDimNum();
580+ 
581+ CHECK_RET(CheckDimNumByGroupType(xIdx, wIdx, xDimNum, weightDimNum) == ACLNN_SUCCESS, ACLNN_ERR_PARAM_INVALID);
532 582 
533 if (unlikely(xDimNum != yDimNum)) {583 if (unlikely(xDimNum != yDimNum)) {
534 std::string incorrectValues = std::to_string(xDimNum) + ", " + std::to_string(yDimNum);584 std::string incorrectValues = std::to_string(xDimNum) + ", " + std::to_string(yDimNum);
@@ -967,7 +1017,7 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckS8S4NZScaleShape()
967 return ACLNN_ERR_PARAM_INVALID;1017 return ACLNN_ERR_PARAM_INVALID;
968 }1018 }
969 auto scaleShape = (*gmmParams_.scaleOptional)[0]->GetViewShape();1019 auto scaleShape = (*gmmParams_.scaleOptional)[0]->GetViewShape();
970- if (unlikely(scaleShape.GetDimNum() != 2)) {1020+ if (unlikely(scaleShape.GetDimNum() != 2)) { //S8S4 NZ 模式下 scale tensor 必须是 2 维
971 OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(GetAclnnName(), "scale", std::to_string(scaleShape.GetDimNum()),1021 OP_LOGE_FOR_INVALID_SHAPEDIM_WITH_REASON(GetAclnnName(), "scale", std::to_string(scaleShape.GetDimNum()),
972 "The shape dim of scale must be 2");1022 "The shape dim of scale must be 2");
973 return ACLNN_ERR_PARAM_INVALID;1023 return ACLNN_ERR_PARAM_INVALID;
@@ -977,7 +1027,7 @@ aclnnStatus AclnnGroupedMatmulWeightQuantDAV3510Checker::CheckS8S4NZScaleShape()
977 "The dim0 of scale must be equal to g");1027 "The dim0 of scale must be equal to g");
978 return ACLNN_ERR_PARAM_INVALID;1028 return ACLNN_ERR_PARAM_INVALID;
979 }1029 }
980- if (unlikely(scaleShape.GetDim(1) != weightShape.GetDim(2))) {1030+ if (unlikely(scaleShape.GetDim(1) != weightShape.GetDim(2))) { //校验 S8S4 NZ 格式下 scale tensor 的 N 维度是否和 weight tensor 的 N 维度一致
981 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(GetAclnnName(), "scale", std::to_string(scaleShape.GetDim(1)),1031 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(GetAclnnName(), "scale", std::to_string(scaleShape.GetDim(1)),
982 "The dim1 of scale must be equal to n");1032 "The dim1 of scale must be equal to n");
983 return ACLNN_ERR_PARAM_INVALID;1033 return ACLNN_ERR_PARAM_INVALID;
Mgmm/grouped_matmul/op_api/grouped_matmul_weight_quant_950_checker.h+13-1
@@ -38,9 +38,21 @@ private:
38 aclnnStatus CheckTensorDtype(const aclTensorList *tensorList, const DataType &tensorDtype, size_t idx,38 aclnnStatus CheckTensorDtype(const aclTensorList *tensorList, const DataType &tensorDtype, size_t idx,
39 const std::string &tensorType) const;39 const std::string &tensorType) const;
40 aclnnStatus CheckTensorShape(const aclTensorList *tensorList, size_t idx, const std::string &tensorType) const;40 aclnnStatus CheckTensorShape(const aclTensorList *tensorList, size_t idx, const std::string &tensorType) const;
41+ static aclnnStatus GetExpectedDimNum(const AclnnGroupedMatmulWeightQuantDAV3510Checker &checker,
42+ const std::string &tensorType, size_t tensorDimNum,
43+ size_t &expectedDimNum);
44+ static aclnnStatus CheckBatchSizeConsistency(const AclnnGroupedMatmulWeightQuantDAV3510Checker &checker,
45+ const op::Shape &tensorShape, const op::Shape &weightShape,
46+ const std::string &tensorType);
47+ static aclnnStatus CheckNDimConsistency(const AclnnGroupedMatmulWeightQuantDAV3510Checker &checker,
48+ const op::Shape &tensorShape, const op::Shape &weightShape,
49+ size_t tensorDimNum, const std::string &tensorType);
41 50 
42 aclnnStatus CheckWeightInnerAxisEven(size_t idx) const;51 aclnnStatus CheckWeightInnerAxisEven(size_t idx) const;
43 aclnnStatus CheckDimNumAndFormat(size_t xIdx, size_t yIdx, size_t wIdx) const;52 aclnnStatus CheckDimNumAndFormat(size_t xIdx, size_t yIdx, size_t wIdx) const;
53+ aclnnStatus CheckInputFormat(size_t xIdx, size_t yIdx) const;
54+ aclnnStatus CheckWeightFormat(size_t wIdx) const;
55+ aclnnStatus CheckDimNumByGroupType(size_t xIdx, size_t wIdx, size_t xDimNum, size_t weightDimNum) const;
44 aclnnStatus CheckTransposeStatus() const;56 aclnnStatus CheckTransposeStatus() const;
45 aclnnStatus CheckDimValue(size_t xIdx, size_t yIdx, size_t wIdx) const;57 aclnnStatus CheckDimValue(size_t xIdx, size_t yIdx, size_t wIdx) const;
46 aclnnStatus CheckDimMatching(size_t xIdx, size_t yIdx, size_t wIdx) const;58 aclnnStatus CheckDimMatching(size_t xIdx, size_t yIdx, size_t wIdx) const;
@@ -73,7 +85,7 @@ private:
73 bool IsS8S4AsymmetricQuant() const;85 bool IsS8S4AsymmetricQuant() const;
74 bool IsA16W4() const;86 bool IsA16W4() const;
75 bool IsMultiTensorWeight() const;87 bool IsMultiTensorWeight() const;
76- bool IsA16W4Pergroup(const size_t xIdx) const;88+ bool IsA16W4Pergroup(const size_t idx) const;
77 89 
78 std::string GetDataFlowString() const;90 std::string GetDataFlowString() const;
79 const char *GetAclnnName() const;91 const char *GetAclnnName() const;
Mgmm/grouped_matmul/op_graph/fallback_grouped_matmul.cpp+98-55
@@ -283,49 +283,117 @@ static bool IsPerTileQuantMode(const OpExecuteContext* host_api_ctx, bool transp
283 return (scaleNDim == (weightNDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE && isKdimValid);283 return (scaleNDim == (weightNDim + PERTILE_GROUP_SIZE - 1) / PERTILE_GROUP_SIZE && isKdimValid);
284}284}
285 285 
286+struct GmmExecuteAttrs {
287+ const int64_t* splitItemGe;
288+ const bool* isWeightTransposedPtr;
289+ const bool* isXTransposed;
290+ const int64_t* groupTypeGe;
291+ const int64_t* groupListTypeGe;
292+ const int64_t* actTypeGe;
293+ const gert::TypedContinuousVector<int64_t>* tuningConfigGe;
294+};
295+ 
296+static graphStatus GetGmmAttrs(const OpExecuteContext* host_api_ctx, GmmExecuteAttrs& out)
297+{
298+ auto attrs = host_api_ctx->GetAttrs();
299+ OP_CHECK_IF(attrs == nullptr, OP_LOGE("aclnnfallback", "attrs is null"), return GRAPH_FAILED);
300+ out.splitItemGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_SPLIT_ITEM);
301+ OP_CHECK_IF(out.splitItemGe == nullptr, OP_LOGE("aclnnfallback", "splitItemGe is null"), return GRAPH_FAILED);
302+ out.isWeightTransposedPtr = attrs->GetAttrPointer<bool>(INDEX_GMM_ATTR_TRANSPOSE_WEIGHT);
303+ OP_CHECK_IF(out.isWeightTransposedPtr == nullptr, OP_LOGE("aclnnfallback", "isWeightTransposedPtr is null"),
304+ return GRAPH_FAILED);
305+ out.isXTransposed = attrs->GetAttrPointer<bool>(INDEX_GMM_ATTR_TRANSPOSE_X);
306+ OP_CHECK_IF(out.isXTransposed == nullptr, OP_LOGE("aclnnfallback", "isXTransposed is null"), return GRAPH_FAILED);
307+ out.groupTypeGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_GROUP_TYPE);
308+ OP_CHECK_IF(out.groupTypeGe == nullptr, OP_LOGE("aclnnfallback", "groupTypeGe is null"), return GRAPH_FAILED);
309+ out.groupListTypeGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_GROUP_LIST_TYPE);
310+ OP_CHECK_IF(out.groupListTypeGe == nullptr, OP_LOGE("aclnnfallback", "groupListTypeGe is null"), return GRAPH_FAILED);
311+ out.actTypeGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_ACT_TYPE);
312+ OP_CHECK_IF(out.actTypeGe == nullptr, OP_LOGE("aclnnfallback", "actTypeGe is null"), return GRAPH_FAILED);
313+ out.tuningConfigGe = attrs->GetListInt(INDEX_GMM_ATTR_TUNING_CONFIG);
314+ OP_CHECK_IF(out.tuningConfigGe == nullptr, OP_LOGE("aclnnfallback", "tuningConfigGe is null"), return GRAPH_FAILED);
315+ return GRAPH_SUCCESS;
316+}
317+ 
318+static graphStatus PreparePerTokenScaleList(const OpExecuteContext* host_api_ctx,
319+ std::vector<const aclTensor*>& perTokenScaleVec,
320+ bool isXTransposed, bool isPerTile)
321+{
322+ auto perTokenScaleTensor = host_api_ctx->GetOptionalInputTensor(INDEX_GMM_INPUT_PER_TOKEN_SCALE);
323+ auto perTokenScale = ConvertType(perTokenScaleTensor);
324+ if (perTokenScale == nullptr) {
325+ std::vector<int64_t> shape{0};
326+ static const auto aclCreateTensor = GET_OP_API_FUNC(aclCreateTensor);
327+ OP_CHECK_IF(aclCreateTensor == nullptr, OP_LOGE("aclnnfallback", "aclCreateTensor nullptr"), return GRAPH_FAILED);
328+ perTokenScale = aclCreateTensor(shape.data(), shape.size(), aclDataType::ACL_FLOAT, shape.data(),
329+ 0, aclFormat::ACL_FORMAT_ND, shape.data(), shape.size(), nullptr);
330+ OP_CHECK_IF(perTokenScale == nullptr, OP_LOGE("aclnnfallback", "perTokenScale nullptr"), return GRAPH_FAILED);
331+ }
332+ if (perTokenScaleTensor != nullptr && (perTokenScaleTensor->GetDataType() == ge::DataType::DT_FLOAT8_E8M0 ||
333+ isPerTile)) {
334+ PrepareAclTensorVector(host_api_ctx, perTokenScaleVec, INDEX_GMM_INPUT_PER_TOKEN_SCALE, isXTransposed, false);
335+ } else {
336+ perTokenScaleVec.push_back(perTokenScale);
337+ }
338+ return GRAPH_SUCCESS;
339+}
340+ 
341+static graphStatus ExecuteGmmApi(
342+ aclTensorList* aclTensorListX, aclTensorList* aclTensorListWeight,
343+ std::vector<const gert::Tensor*>& geTensorVectorBias, aclTensorList* aclTensorListScale,
344+ std::vector<const gert::Tensor*>& geTensorVectorOffset,
345+ std::vector<const gert::Tensor*>& geTensorVectorAntiquantScale,
346+ std::vector<const gert::Tensor*>& geTensorVectorAntiquantOffset,
347+ aclTensorList* aclTensorListPerTokenScale, const gert::Tensor* groupListTensor,
348+ int64_t splitItem, int64_t groupType, int64_t groupListType, int64_t actType,
349+ std::vector<int64_t>& tuningConfig, std::vector<const gert::Tensor*>& geTensorVectorY)
350+{
351+ aclTensorList* activationInputOptional = nullptr;
352+ aclTensorList* activationQuantScaleOptional = nullptr;
353+ aclTensorList* activationQuantOffsetOptional = nullptr;
354+ aclTensorList* actFeatureOutOptional = nullptr;
355+ aclTensorList* dynQuantScaleOutOptional = nullptr;
356+ auto api_ret_gmm = EXEC_OPAPI_CMD(aclnnGroupedMatmulV5, aclTensorListX, aclTensorListWeight, geTensorVectorBias,
357+ aclTensorListScale, geTensorVectorOffset, geTensorVectorAntiquantScale,
358+ geTensorVectorAntiquantOffset, aclTensorListPerTokenScale, groupListTensor,
359+ activationInputOptional, activationQuantScaleOptional, activationQuantOffsetOptional,
360+ splitItem, groupType, groupListType, actType, tuningConfig,
361+ geTensorVectorY, actFeatureOutOptional, dynQuantScaleOutOptional);
362+ OP_CHECK_IF(api_ret_gmm != GRAPH_SUCCESS, OP_LOGE("aclnnfallback", "api_ret failed:%u", api_ret_gmm),
363+ return GRAPH_FAILED);
364+ return GRAPH_SUCCESS;
365+}
366+ 
286static graphStatus GroupedMatmulExecuteFunc(OpExecuteContext* host_api_ctx)367static graphStatus GroupedMatmulExecuteFunc(OpExecuteContext* host_api_ctx)
287{368{
288 OP_CHECK_IF(host_api_ctx == nullptr, OP_LOGE("aclnnfallback", "host_api_ctx is null"), return GRAPH_FAILED);369 OP_CHECK_IF(host_api_ctx == nullptr, OP_LOGE("aclnnfallback", "host_api_ctx is null"), return GRAPH_FAILED);
289 370 
290- auto attrs = host_api_ctx->GetAttrs();371+ GmmExecuteAttrs attrs{};
291- OP_CHECK_IF(attrs == nullptr, OP_LOGE("aclnnfallback", "attrs is null"), return GRAPH_FAILED);372+ auto attrRet = GetGmmAttrs(host_api_ctx, attrs);
292- const int64_t* splitItemGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_SPLIT_ITEM);373+ OP_CHECK_IF(attrRet != GRAPH_SUCCESS, return attrRet);
293- OP_CHECK_IF(splitItemGe == nullptr, OP_LOGE("aclnnfallback", "splitItemGe is null"), return GRAPH_FAILED);
294- const bool* isWeightTransposedPtr = attrs->GetAttrPointer<bool>(INDEX_GMM_ATTR_TRANSPOSE_WEIGHT);
295- OP_CHECK_IF(isWeightTransposedPtr == nullptr, OP_LOGE("aclnnfallback", "isWeightTransposedPtr is null"),
296- return GRAPH_FAILED);
297- const bool* isXTransposed = attrs->GetAttrPointer<bool>(INDEX_GMM_ATTR_TRANSPOSE_X);
298- OP_CHECK_IF(isXTransposed == nullptr, OP_LOGE("aclnnfallback", "isXTransposed is null"), return GRAPH_FAILED);
299- const int64_t* groupTypeGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_GROUP_TYPE);
300- OP_CHECK_IF(groupTypeGe == nullptr, OP_LOGE("aclnnfallback", "groupTypeGe is null"), return GRAPH_FAILED);
301- const int64_t* groupListTypeGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_GROUP_LIST_TYPE);
302- OP_CHECK_IF(groupListTypeGe == nullptr, OP_LOGE("aclnnfallback", "groupListTypeGe is null"), return GRAPH_FAILED);
303- const int64_t* actTypeGe = attrs->GetAttrPointer<int64_t>(INDEX_GMM_ATTR_ACT_TYPE);
304- OP_CHECK_IF(actTypeGe == nullptr, OP_LOGE("aclnnfallback", "actTypeGe is null"), return GRAPH_FAILED);
305- const auto tuningConfigGe = attrs->GetListInt(INDEX_GMM_ATTR_TUNING_CONFIG);
306- OP_CHECK_IF(tuningConfigGe == nullptr, OP_LOGE("aclnnfallback", "tuningConfigGe is null"), return GRAPH_FAILED);
307 374 
308 static const auto aclCreateTensorList = GET_OP_API_FUNC(aclCreateTensorList);375 static const auto aclCreateTensorList = GET_OP_API_FUNC(aclCreateTensorList);
309 OP_CHECK_IF(aclCreateTensorList == nullptr,376 OP_CHECK_IF(aclCreateTensorList == nullptr,
310 OP_LOGE("aclnnfallback", "Get opapi func aclCreateTensorList failed"), return GRAPH_FAILED);377 OP_LOGE("aclnnfallback", "Get opapi func aclCreateTensorList failed"), return GRAPH_FAILED);
311 378 
312 std::vector<const aclTensor*> aclTensorVectorX;379 std::vector<const aclTensor*> aclTensorVectorX;
313- PrepareAclTensorVector(host_api_ctx, aclTensorVectorX, INDEX_GMM_INPUT_X, *isXTransposed, false);380+ PrepareAclTensorVector(host_api_ctx, aclTensorVectorX, INDEX_GMM_INPUT_X, *attrs.isXTransposed, false);
314 auto aclTensorListX = aclCreateTensorList(aclTensorVectorX.data(), aclTensorVectorX.size());381 auto aclTensorListX = aclCreateTensorList(aclTensorVectorX.data(), aclTensorVectorX.size());
315 382 
316 std::vector<const aclTensor*> aclTensorVectorWeight;383 std::vector<const aclTensor*> aclTensorVectorWeight;
317- PrepareAclTensorVector(host_api_ctx, aclTensorVectorWeight, INDEX_GMM_INPUT_WEIGHT, *isWeightTransposedPtr, true);384+ PrepareAclTensorVector(host_api_ctx, aclTensorVectorWeight, INDEX_GMM_INPUT_WEIGHT, *attrs.isWeightTransposedPtr, true);
318 size_t numGeWeight = aclTensorVectorWeight.size();385 size_t numGeWeight = aclTensorVectorWeight.size();
319 auto aclTensorListWeight = aclCreateTensorList(aclTensorVectorWeight.data(), aclTensorVectorWeight.size());386 auto aclTensorListWeight = aclCreateTensorList(aclTensorVectorWeight.data(), aclTensorVectorWeight.size());
320 387 
321 std::vector<const gert::Tensor*> geTensorVectorBias;388 std::vector<const gert::Tensor*> geTensorVectorBias;
322 PrepareGeTensorVector(host_api_ctx, geTensorVectorBias, INDEX_GMM_INPUT_BIAS);389 PrepareGeTensorVector(host_api_ctx, geTensorVectorBias, INDEX_GMM_INPUT_BIAS);
390+ 
323 bool isScaleTransposed = false;391 bool isScaleTransposed = false;
324 const bool* isScaleTransposedPtr = &isScaleTransposed;392 const bool* isScaleTransposedPtr = &isScaleTransposed;
325 auto scaleTensor = host_api_ctx->GetDynamicInputTensor(INDEX_GMM_INPUT_SCALE, 0);393 auto scaleTensor = host_api_ctx->GetDynamicInputTensor(INDEX_GMM_INPUT_SCALE, 0);
326- bool isPerTile = IsPerTileQuantMode(host_api_ctx, *isXTransposed, *isWeightTransposedPtr, *groupTypeGe);394+ bool isPerTile = IsPerTileQuantMode(host_api_ctx, *attrs.isXTransposed, *attrs.isWeightTransposedPtr, *attrs.groupTypeGe);
327 if (scaleTensor != nullptr && (scaleTensor->GetDataType() == ge::DataType::DT_FLOAT8_E8M0 || isPerTile)) {395 if (scaleTensor != nullptr && (scaleTensor->GetDataType() == ge::DataType::DT_FLOAT8_E8M0 || isPerTile)) {
328- isScaleTransposedPtr = isWeightTransposedPtr;396+ isScaleTransposedPtr = attrs.isWeightTransposedPtr;
329 }397 }
330 std::vector<const aclTensor*> aclTensorVectorScale;398 std::vector<const aclTensor*> aclTensorVectorScale;
331 PrepareAclTensorVector(host_api_ctx, aclTensorVectorScale, INDEX_GMM_INPUT_SCALE, *isScaleTransposedPtr, false);399 PrepareAclTensorVector(host_api_ctx, aclTensorVectorScale, INDEX_GMM_INPUT_SCALE, *isScaleTransposedPtr, false);
@@ -342,49 +410,24 @@ static graphStatus GroupedMatmulExecuteFunc(OpExecuteContext* host_api_ctx)
342 410 
343 auto groupListTensor = host_api_ctx->GetOptionalInputTensor(INDEX_GMM_INPUT_GROUP_LIST);411 auto groupListTensor = host_api_ctx->GetOptionalInputTensor(INDEX_GMM_INPUT_GROUP_LIST);
344 412 
345- auto perTokenScaleTensor = host_api_ctx->GetOptionalInputTensor(INDEX_GMM_INPUT_PER_TOKEN_SCALE);
346 std::vector<const aclTensor*> geTensorVectorPerTokenScale;413 std::vector<const aclTensor*> geTensorVectorPerTokenScale;
347- auto perTokenScale = ConvertType(perTokenScaleTensor);414+ auto ptsRet = PreparePerTokenScaleList(host_api_ctx, geTensorVectorPerTokenScale, *attrs.isXTransposed, isPerTile);
348- if (perTokenScale == nullptr) {415+ OP_CHECK_IF(ptsRet != GRAPH_SUCCESS, return ptsRet);
349- std::vector<int64_t> shape{0};
350- static const auto aclCreateTensor = GET_OP_API_FUNC(aclCreateTensor);
351- OP_CHECK_IF(aclCreateTensor == nullptr, OP_LOGE("aclnnfallback", "aclCreateTensor nullptr"), return GRAPH_FAILED);
352- perTokenScale = aclCreateTensor(shape.data(), shape.size(), aclDataType::ACL_FLOAT, shape.data(),
353- 0, aclFormat::ACL_FORMAT_ND, shape.data(), shape.size(), nullptr);
354- OP_CHECK_IF(perTokenScale == nullptr, OP_LOGE("aclnnfallback", "perTokenScale nullptr"), return GRAPH_FAILED);
355- }
356- if (perTokenScaleTensor != nullptr && (perTokenScaleTensor->GetDataType() == ge::DataType::DT_FLOAT8_E8M0 ||
357- isPerTile)) {
358- PrepareAclTensorVector(host_api_ctx, geTensorVectorPerTokenScale, INDEX_GMM_INPUT_PER_TOKEN_SCALE,
359- *isXTransposed, false);
360- } else {
361- geTensorVectorPerTokenScale.push_back(perTokenScale);
362- }
363 auto aclTensorListPerTokenScale = aclCreateTensorList(geTensorVectorPerTokenScale.data(),416 auto aclTensorListPerTokenScale = aclCreateTensorList(geTensorVectorPerTokenScale.data(),
364 geTensorVectorPerTokenScale.size());417 geTensorVectorPerTokenScale.size());
365 418 
366 std::vector<const gert::Tensor*> geTensorVectorY;419 std::vector<const gert::Tensor*> geTensorVectorY;
367- PrepareOutputTensorVector(host_api_ctx, geTensorVectorY, INDEX_GMM_OUTPUT_Y, numGeWeight, *splitItemGe);420+ PrepareOutputTensorVector(host_api_ctx, geTensorVectorY, INDEX_GMM_OUTPUT_Y, numGeWeight, *attrs.splitItemGe);
368 421 
369- aclTensorList* activationInputOptional = nullptr;
370- aclTensorList* activationQuantScaleOptional = nullptr;
371- aclTensorList* activationQuantOffsetOptional = nullptr;
372- aclTensorList* actFeatureOutOptional = nullptr;
373- aclTensorList* dynQuantScaleOutOptional = nullptr;
374 std::vector<int64_t> tuningConfig;422 std::vector<int64_t> tuningConfig;
375 tuningConfig.reserve(1);423 tuningConfig.reserve(1);
376- tuningConfig.push_back(tuningConfigGe->GetData()[0]);424+ tuningConfig.push_back(attrs.tuningConfigGe->GetData()[0]);
377 425 
378- // execute opapi426+ return ExecuteGmmApi(aclTensorListX, aclTensorListWeight, geTensorVectorBias, aclTensorListScale,
379- auto api_ret_gmm = EXEC_OPAPI_CMD(aclnnGroupedMatmulV5, aclTensorListX, aclTensorListWeight, geTensorVectorBias,427+ geTensorVectorOffset, geTensorVectorAntiquantScale, geTensorVectorAntiquantOffset,
380- aclTensorListScale, geTensorVectorOffset, geTensorVectorAntiquantScale,428+ aclTensorListPerTokenScale, groupListTensor,
381- geTensorVectorAntiquantOffset, aclTensorListPerTokenScale, groupListTensor,429+ *attrs.splitItemGe, *attrs.groupTypeGe, *attrs.groupListTypeGe, *attrs.actTypeGe,
382- activationInputOptional, activationQuantScaleOptional, activationQuantOffsetOptional,430+ tuningConfig, geTensorVectorY);
383- *splitItemGe, *groupTypeGe, *groupListTypeGe, *actTypeGe, tuningConfig,
384- geTensorVectorY, actFeatureOutOptional, dynQuantScaleOutOptional);
385- OP_CHECK_IF(api_ret_gmm != GRAPH_SUCCESS, OP_LOGE("aclnnfallback", "api_ret failed:%u", api_ret_gmm),
386- return GRAPH_FAILED);
387- return GRAPH_SUCCESS;
388}431}
389 432 
390IMPL_OP(GroupedMatmul).OpExecuteFunc(GroupedMatmulExecuteFunc);433IMPL_OP(GroupedMatmul).OpExecuteFunc(GroupedMatmulExecuteFunc);
Mgmm/grouped_matmul/op_graph/fusion_pass/grouped_matmul_transpose_fusion_pass.cpp+42-15
@@ -27,6 +27,7 @@
27#include "log/log.h"27#include "log/log.h"
28#include "platform/platform_info.h"28#include "platform/platform_info.h"
29#include "version/ge-compiler_version.h"29#include "version/ge-compiler_version.h"
30+#include "securec.h"
30 31 
31using namespace ge;32using namespace ge;
32using namespace ge::fusion;33using namespace ge::fusion;
@@ -165,11 +166,17 @@ bool GetConstScalar(const GNode& node, int32_t inputIndex, int64_t& value)
165 return false;166 return false;
166 }167 }
167 if (desc.GetDataType() == DT_INT32 && tensor.GetSize() >= sizeof(int32_t)) {168 if (desc.GetDataType() == DT_INT32 && tensor.GetSize() >= sizeof(int32_t)) {
168- value = static_cast<int64_t>(*reinterpret_cast<const int32_t*>(tensor.GetData()));169+ int32_t int32Value = 0;
170+ if (memcpy_s(&int32Value, sizeof(int32Value), tensor.GetData(), sizeof(int32_t)) != EOK) {
171+ return false;
172+ }
173+ value = static_cast<int64_t>(int32Value);
169 return true;174 return true;
170 }175 }
171 if (desc.GetDataType() == DT_INT64 && tensor.GetSize() >= sizeof(int64_t)) {176 if (desc.GetDataType() == DT_INT64 && tensor.GetSize() >= sizeof(int64_t)) {
172- value = *reinterpret_cast<const int64_t*>(tensor.GetData());177+ if (memcpy_s(&value, sizeof(value), tensor.GetData(), sizeof(int64_t)) != EOK) {
178+ return false;
179+ }
173 return true;180 return true;
174 }181 }
175 return false;182 return false;
@@ -298,6 +305,24 @@ bool IsMxWeightQuantMode(const GNode& groupedMatmulNode)
298 antiquantScaleDesc.GetDataType() == DT_FLOAT8_E8M0;305 antiquantScaleDesc.GetDataType() == DT_FLOAT8_E8M0;
299}306}
300 307 
308+bool GetTransposeAxesForScale(bool isMxScale, bool isMxPertokenScale, std::size_t dimNum,
309+ std::size_t &firstTransposeAxis, std::size_t &secondTransposeAxis)
310+{
311+ firstTransposeAxis = dimNum - kMiniShapeLen;
312+ secondTransposeAxis = dimNum - 1;
313+ if (isMxScale) {
314+ if (dimNum < kPermScale.size()) {
315+ return false;
316+ }
317+ firstTransposeAxis = dimNum - 3; //倒数索引，定位N轴
318+ secondTransposeAxis = dimNum - 2; //倒数索引，定位K/gs
319+ } else if (isMxPertokenScale) {
320+ firstTransposeAxis = 0;
321+ secondTransposeAxis = 1;
322+ }
323+ return true;
324+}
325+ 
301bool IsReshapeTransForScale(326bool IsReshapeTransForScale(
302 int32_t index, const GNodePtr& nodePerInput, bool isMxQuantMode, bool allowFullyDynamicMxFp4Scale)327 int32_t index, const GNodePtr& nodePerInput, bool isMxQuantMode, bool allowFullyDynamicMxFp4Scale)
303{328{
@@ -318,20 +343,14 @@ bool IsReshapeTransForScale(
318 return false;343 return false;
319 }344 }
320 345 
321- std::size_t firstTransposeAxis = expectedOutputDims.size() - kMiniShapeLen;
322- std::size_t secondTransposeAxis = expectedOutputDims.size() - 1;
323 const bool isMxScale =346 const bool isMxScale =
324 index == kScaleIndex && (isMxQuantMode || inputDesc.GetDataType() == DT_FLOAT8_E8M0);347 index == kScaleIndex && (isMxQuantMode || inputDesc.GetDataType() == DT_FLOAT8_E8M0);
325 const bool isMxPertokenScale = index == kPertokenScaleIndex && inputDesc.GetDataType() == DT_FLOAT8_E8M0;348 const bool isMxPertokenScale = index == kPertokenScaleIndex && inputDesc.GetDataType() == DT_FLOAT8_E8M0;
326- if (isMxScale) {349+ std::size_t firstTransposeAxis = 0;
327- if (expectedOutputDims.size() < kPermScale.size()) {350+ std::size_t secondTransposeAxis = 0;
328- return false;351+ if (!GetTransposeAxesForScale(isMxScale, isMxPertokenScale, expectedOutputDims.size(), firstTransposeAxis,
329- }352+ secondTransposeAxis)) {
330- firstTransposeAxis = expectedOutputDims.size() - 3;353+ return false;
331- secondTransposeAxis = expectedOutputDims.size() - 2;
332- } else if (isMxPertokenScale) {
333- firstTransposeAxis = 0;
334- secondTransposeAxis = 1;
335 }354 }
336 355 
337 if (expectedOutputDims[firstTransposeAxis] != 1 && expectedOutputDims[secondTransposeAxis] != 1) {356 if (expectedOutputDims[firstTransposeAxis] != 1 && expectedOutputDims[secondTransposeAxis] != 1) {
@@ -385,12 +404,20 @@ bool GetTransposePerm(const GNodePtr& transposeNode, std::vector<int64_t>& perm)
385 if (permDtype == DT_INT32) {404 if (permDtype == DT_INT32) {
386 size = permTensor.GetSize() / sizeof(int32_t);405 size = permTensor.GetSize() / sizeof(int32_t);
387 for (std::size_t i = 0; i < size; ++i) {406 for (std::size_t i = 0; i < size; ++i) {
388- perm.emplace_back(static_cast<int64_t>(*(reinterpret_cast<const int32_t*>(constDataPtr) + i)));407+ int32_t val32 = 0;
408+ if (memcpy_s(&val32, sizeof(val32), constDataPtr + i * sizeof(int32_t), sizeof(int32_t)) != EOK) {
409+ return false;
410+ }
411+ perm.emplace_back(val32);
389 }412 }
390 } else if (permDtype == DT_INT64) {413 } else if (permDtype == DT_INT64) {
391 size = permTensor.GetSize() / sizeof(int64_t);414 size = permTensor.GetSize() / sizeof(int64_t);
392 for (std::size_t i = 0; i < size; ++i) {415 for (std::size_t i = 0; i < size; ++i) {
393- perm.emplace_back(*(reinterpret_cast<const int64_t*>(constDataPtr) + i));416+ int64_t val64 = 0;
417+ if (memcpy_s(&val64, sizeof(val64), constDataPtr + i * sizeof(int64_t), sizeof(int64_t)) != EOK) {
418+ return false;
419+ }
420+ perm.emplace_back(val64);
394 }421 }
395 } else {422 } else {
396 OP_LOGW(kPassName, "Transpose perm dtype must be int32 or int64.");423 OP_LOGW(kPassName, "Transpose perm dtype must be int32 or int64.");
Mgmm/grouped_matmul/op_host/grouped_matmul_infershape.cpp+60-36
@@ -254,9 +254,9 @@ static ge::graphStatus GetAttrsValue(T context, GMMAttrs &gmmAttrs)
254 gmmAttrs.outputDtype = *dtypePtr;254 gmmAttrs.outputDtype = *dtypePtr;
255 OP_LOGI(context->GetNodeName(), "Attr dtype = %ld", gmmAttrs.outputDtype);255 OP_LOGI(context->GetNodeName(), "Attr dtype = %ld", gmmAttrs.outputDtype);
256 256 
257- const auto tuningConfigPtr = attrs->GetAttrPointer<gert::ContinuousVector>(GMM_INDEX_ATTR_TUNING_CONFIG);257+ const auto tuningConfigPtr = attrs->GetAttrPointer<gert::TypedContinuousVector<int64_t>>(GMM_INDEX_ATTR_TUNING_CONFIG);
258 gmmAttrs.tuningConfig = (tuningConfigPtr != nullptr && tuningConfigPtr->GetSize() > 0) ?258 gmmAttrs.tuningConfig = (tuningConfigPtr != nullptr && tuningConfigPtr->GetSize() > 0) ?
259- (reinterpret_cast<const int64_t *>(tuningConfigPtr->GetData()))[0] : 0;259+ tuningConfigPtr->GetData()[0] : 0;
260 OP_LOGI(context->GetNodeName(), "Attr tuningConfig = %ld", gmmAttrs.tuningConfig);260 OP_LOGI(context->GetNodeName(), "Attr tuningConfig = %ld", gmmAttrs.tuningConfig);
261 261 
262 const int64_t *groupTypePtr = attrs->GetAttrPointer<int64_t>(GMM_INDEX_ATTR_GROUP_TYPE);262 const int64_t *groupTypePtr = attrs->GetAttrPointer<int64_t>(GMM_INDEX_ATTR_GROUP_TYPE);
@@ -394,11 +394,11 @@ static bool IsS8S4SpecialWeightFormat(const gert::InferShapeContext* context)
394 if (attrs == nullptr) {394 if (attrs == nullptr) {
395 return false;395 return false;
396 }396 }
397- const auto tuningConfigPtr = attrs->GetAttrPointer<gert::ContinuousVector>(GMM_INDEX_ATTR_TUNING_CONFIG);397+ const auto tuningConfigPtr = attrs->GetAttrPointer<gert::TypedContinuousVector<int64_t>>(GMM_INDEX_ATTR_TUNING_CONFIG);
398 if (tuningConfigPtr == nullptr || tuningConfigPtr->GetSize() <= 1) {398 if (tuningConfigPtr == nullptr || tuningConfigPtr->GetSize() <= 1) {
399 return false;399 return false;
400 }400 }
401- const auto tuningConfig = reinterpret_cast<const int64_t *>(tuningConfigPtr->GetData());401+ const auto tuningConfig = tuningConfigPtr->GetData();
402 return tuningConfig[1] == 1;402 return tuningConfig[1] == 1;
403}403}
404 404 
@@ -1176,7 +1176,7 @@ static ge::graphStatus SplitMSingleXSingleWeightSingleY(gert::InferShapeContext*
1176 OP_CHECK_IF(CheckShapeSameLengthTensorList(context, {1, kAxisOfWeight}, innerAxisDimId, tenorXAndWeight, paramsInfo.numX) != GRAPH_SUCCESS,1176 OP_CHECK_IF(CheckShapeSameLengthTensorList(context, {1, kAxisOfWeight}, innerAxisDimId, tenorXAndWeight, paramsInfo.numX) != GRAPH_SUCCESS,
1177 OP_LOGE(context->GetNodeName(), "k dim value of x and weight is not matched."),1177 OP_LOGE(context->GetNodeName(), "k dim value of x and weight is not matched."),
1178 return GRAPH_FAILED);1178 return GRAPH_FAILED);
1179- innerAxisDimId = specialWeightFormat ? static_cast<int64_t>(weightAxis.n) : (!transposeWeight ? 2 : -1);1179+ innerAxisDimId = specialWeightFormat ? static_cast<int64_t>(weightAxis.n) : (!transposeWeight ? 2 : -1); //转置 weight 的内轴 K 位于第 3 维（索引 2）
1180 OP_CHECK_IF(CheckInnerAxisOfTensorList(context, GMM_INDEX_IN_WEIGHT, innerAxisDimId, paramsInfo.numWeight) != GRAPH_SUCCESS,1180 OP_CHECK_IF(CheckInnerAxisOfTensorList(context, GMM_INDEX_IN_WEIGHT, innerAxisDimId, paramsInfo.numWeight) != GRAPH_SUCCESS,
1181 OP_LOGE(context->GetNodeName(), "inner axis size of weight is larger than %ld!", GMM_MAX_INNER_AXIS),1181 OP_LOGE(context->GetNodeName(), "inner axis size of weight is larger than %ld!", GMM_MAX_INNER_AXIS),
1182 return GRAPH_FAILED);1182 return GRAPH_FAILED);
@@ -1528,51 +1528,60 @@ static graphStatus IsDavidQuantGMMByShape(T context) {
1528 return (GetSizeByDataType(xDtype) == 1 && GetSizeByDataType(weightDtype) == 1) ? GRAPH_SUCCESS : GRAPH_FAILED;1528 return (GetSizeByDataType(xDtype) == 1 && GetSizeByDataType(weightDtype) == 1) ? GRAPH_SUCCESS : GRAPH_FAILED;
1529}1529}
1530 1530 
1531-static ge::graphStatus InferShape4GroupedMatmul(gert::InferShapeContext* context) {1531+static ge::graphStatus TryDavidInferShape(gert::InferShapeContext* context)
1532- OP_CHECK_NULL_WITH_CONTEXT(context, context);1532+{
1533 fe::PlatformInfo platformInfo;1533 fe::PlatformInfo platformInfo;
1534 fe::OptionalInfo optionalInfo;1534 fe::OptionalInfo optionalInfo;
1535 auto ret = fe::PlatformInfoManager::Instance().GetPlatformInfoWithOutSocVersion(platformInfo, optionalInfo);1535 auto ret = fe::PlatformInfoManager::Instance().GetPlatformInfoWithOutSocVersion(platformInfo, optionalInfo);
1536- if (ret == GRAPH_SUCCESS && GmmDavidSupportSoc.count(platformInfo.str_info.short_soc_version) > 0 &&1536+ if (ret != GRAPH_SUCCESS || GmmDavidSupportSoc.count(platformInfo.str_info.short_soc_version) == 0 ||
1537- !IsS8S4PseudoQuant(context)) {1537+ IsS8S4PseudoQuant(context)) {
1538- if (IsDavidQuantGMMByShape(context) == GRAPH_SUCCESS) {1538+ return GRAPH_FAILED; // not handled by David path
1539- OP_CHECK_IF(InferShape4DavidQuantGMM(context) != GRAPH_SUCCESS,
1540- OP_LOGE(context->GetNodeName(), "Check params failed"), return GRAPH_FAILED);
1541- return GRAPH_SUCCESS;
1542- } else if (IsDavidWeightQuantGMMByShape(context) == GRAPH_SUCCESS) {
1543- OP_CHECK_IF(InferShape4DavidWeightQuantGMM(context) != GRAPH_SUCCESS,
1544- OP_LOGE(context->GetNodeName(), "Check params failed"), return GRAPH_FAILED);
1545- return GRAPH_SUCCESS;
1546- }
1547 }1539 }
1548- GMMAttrs gmmAttrs{GMM_X_Y_SEPARATED, 0, GMM_NO_SPLIT, false, false, 0, 0};1540+ if (IsDavidQuantGMMByShape(context) == GRAPH_SUCCESS) {
1549- OP_CHECK_IF(GetAttrsValue(context, gmmAttrs) != GRAPH_SUCCESS || CheckAttrs(context, gmmAttrs) != GRAPH_SUCCESS,1541+ OP_CHECK_IF(InferShape4DavidQuantGMM(context) != GRAPH_SUCCESS,
1550- OP_LOGE(context->GetNodeName(), "Failed to get attrs."), return GRAPH_FAILED);1542+ OP_LOGE(context->GetNodeName(), "Check params failed"), return GRAPH_FAILED);
1543+ return GRAPH_SUCCESS;
1544+ } else if (IsDavidWeightQuantGMMByShape(context) == GRAPH_SUCCESS) {
1545+ OP_CHECK_IF(InferShape4DavidWeightQuantGMM(context) != GRAPH_SUCCESS,
1546+ OP_LOGE(context->GetNodeName(), "Check params failed"), return GRAPH_FAILED);
1547+ return GRAPH_SUCCESS;
1548+ }
1549+ return GRAPH_FAILED; // not a David quant case
1550+}
1551 1551 
1552- size_t numX = 0; // init numX1552+static ge::graphStatus ParseAttrsAndCountInputs(gert::InferShapeContext* context, GMMAttrs& gmmAttrs,
1553- size_t numWeight = 0; // init numWeight1553+ size_t& numX, size_t& numWeight, int64_t& lenGroupList)
1554- int64_t lenGroupList = 0; // init lenGroupList1554+{
1555- size_t numY = context->GetComputeNodeOutputNum();1555+ numX = 0;
1556- if (GetNumOfInputs(context, numX, numWeight, lenGroupList) == GRAPH_SUCCESS) { // check input shape value inside1556+ numWeight = 0;
1557- GMMParamsInfo paramsInfo{numX, numWeight, numY, lenGroupList, 0, 0, 0, 0, 0, PlatformID::UNKNOWN};1557+ lenGroupList = 0;
1558- OP_CHECK_IF(GetGroupSize(context, paramsInfo) != GRAPH_SUCCESS,1558+ if (GetNumOfInputs(context, numX, numWeight, lenGroupList) != GRAPH_SUCCESS) {
1559- OP_LOGE(context->GetNodeName(), "check groupNum failed"), return GRAPH_FAILED);1559+ OP_CHECK_IF(CheckDimNum(context, numX, GMM_MIN_FM_DIM, "x") != GRAPH_SUCCESS,
1560- OP_CHECK_IF(CheckFunctionParamsForShape(context, gmmAttrs, paramsInfo) != GRAPH_SUCCESS,
1561- OP_LOGE(context->GetNodeName(), "CheckFunctionParamsForShape failed."), return GRAPH_FAILED);
1562- OP_CHECK_IF(CheckParamDifferentGroupType(context, gmmAttrs, paramsInfo) != GRAPH_SUCCESS,
1563- OP_LOGE(context->GetNodeName(), "CheckParamDifferentGroupType failed."), return GRAPH_FAILED);
1564- } else {
1565- OP_CHECK_IF(CheckDimNum(context, numX, GMM_MIN_FM_DIM, "x") != GRAPH_SUCCESS, // check dim number of tensors
1566 OP_LOGE(context->GetNodeName(), "Dim num of tensor in tensorList x is invalid."),1560 OP_LOGE(context->GetNodeName(), "Dim num of tensor in tensorList x is invalid."),
1567 return GRAPH_FAILED);1561 return GRAPH_FAILED);
1562+ return GRAPH_SUCCESS;
1568 }1563 }
1564+ size_t numY = context->GetComputeNodeOutputNum();
1565+ GMMParamsInfo paramsInfo{numX, numWeight, numY, lenGroupList, 0, 0, 0, 0, 0, PlatformID::UNKNOWN};
1566+ OP_CHECK_IF(GetGroupSize(context, paramsInfo) != GRAPH_SUCCESS,
1567+ OP_LOGE(context->GetNodeName(), "check groupNum failed"), return GRAPH_FAILED);
1568+ OP_CHECK_IF(CheckFunctionParamsForShape(context, gmmAttrs, paramsInfo) != GRAPH_SUCCESS,
1569+ OP_LOGE(context->GetNodeName(), "CheckFunctionParamsForShape failed."), return GRAPH_FAILED);
1570+ OP_CHECK_IF(CheckParamDifferentGroupType(context, gmmAttrs, paramsInfo) != GRAPH_SUCCESS,
1571+ OP_LOGE(context->GetNodeName(), "CheckParamDifferentGroupType failed."), return GRAPH_FAILED);
1572+ return GRAPH_SUCCESS;
1573+}
1569 1574 
1575+static ge::graphStatus ComputeAndSetOutputShape(gert::InferShapeContext* context, GMMAttrs& gmmAttrs,
1576+ size_t numX, size_t numWeight, int64_t lenGroupList)
1577+{
1570 const gert::Shape* x0Shape = context->GetDynamicInputShape(GMM_INDEX_IN_X, 0);1578 const gert::Shape* x0Shape = context->GetDynamicInputShape(GMM_INDEX_IN_X, 0);
1571 OP_CHECK_NULL_WITH_CONTEXT(context, x0Shape);1579 OP_CHECK_NULL_WITH_CONTEXT(context, x0Shape);
1572 size_t xDimNum = x0Shape->GetDimNum();1580 size_t xDimNum = x0Shape->GetDimNum();
1573 const gert::Shape* w0Shape = context->GetDynamicInputShape(GMM_INDEX_IN_WEIGHT, 0);1581 const gert::Shape* w0Shape = context->GetDynamicInputShape(GMM_INDEX_IN_WEIGHT, 0);
1574 OP_CHECK_NULL_WITH_CONTEXT(context, w0Shape);1582 OP_CHECK_NULL_WITH_CONTEXT(context, w0Shape);
1575 size_t weightDimNum = w0Shape->GetDimNum();1583 size_t weightDimNum = w0Shape->GetDimNum();
1584+ size_t numY = context->GetComputeNodeOutputNum();
1576 bool isSingleX = (numX == 1UL) && (gmmAttrs.groupType != GMM_NO_SPLIT);1585 bool isSingleX = (numX == 1UL) && (gmmAttrs.groupType != GMM_NO_SPLIT);
1577 bool isSingleY = (numY == 1UL) && (gmmAttrs.groupType != GMM_NO_SPLIT);1586 bool isSingleY = (numY == 1UL) && (gmmAttrs.groupType != GMM_NO_SPLIT);
1578 size_t xDimM = gmmAttrs.transposeX ? xDimNum - 1UL : xDimNum - 2UL;1587 size_t xDimM = gmmAttrs.transposeX ? xDimNum - 1UL : xDimNum - 2UL;
@@ -1588,10 +1597,25 @@ static ge::graphStatus InferShape4GroupedMatmul(gert::InferShapeContext* context
1588 outputParams.numWeight = numWeight;1597 outputParams.numWeight = numWeight;
1589 OP_CHECK_IF(GMMSetOutputShape(context, gmmAttrs, outputParams, x0Shape, w0Shape) != GRAPH_SUCCESS,1598 OP_CHECK_IF(GMMSetOutputShape(context, gmmAttrs, outputParams, x0Shape, w0Shape) != GRAPH_SUCCESS,
1590 OP_LOGE(context->GetNodeName(), "GMMSetOutputShape failed"), return GRAPH_FAILED);1599 OP_LOGE(context->GetNodeName(), "GMMSetOutputShape failed"), return GRAPH_FAILED);
1591- 
1592 return GRAPH_SUCCESS;1600 return GRAPH_SUCCESS;
1593}1601}
1594 1602 
1603+static ge::graphStatus InferShape4GroupedMatmul(gert::InferShapeContext* context) {
1604+ OP_CHECK_NULL_WITH_CONTEXT(context, context);
1605+ if (TryDavidInferShape(context) == GRAPH_SUCCESS) {
1606+ return GRAPH_SUCCESS;
1607+ }
1608+ GMMAttrs gmmAttrs{GMM_X_Y_SEPARATED, 0, GMM_NO_SPLIT, false, false, 0, 0};
1609+ OP_CHECK_IF(GetAttrsValue(context, gmmAttrs) != GRAPH_SUCCESS || CheckAttrs(context, gmmAttrs) != GRAPH_SUCCESS,
1610+ OP_LOGE(context->GetNodeName(), "Failed to get attrs."), return GRAPH_FAILED);
1611+ size_t numX = 0;
1612+ size_t numWeight = 0;
1613+ int64_t lenGroupList = 0;
1614+ OP_CHECK_IF(ParseAttrsAndCountInputs(context, gmmAttrs, numX, numWeight, lenGroupList) != GRAPH_SUCCESS,
1615+ return GRAPH_FAILED);
1616+ return ComputeAndSetOutputShape(context, gmmAttrs, numX, numWeight, lenGroupList);
1617+}
1618+ 
1595// =========================================================================================1619// =========================================================================================
1596// =========================================================================================1620// =========================================================================================
1597static graphStatus CheckTensorListDataType(const gert::InferDataTypeContext* context, uint32_t index,1621static graphStatus CheckTensorListDataType(const gert::InferDataTypeContext* context, uint32_t index,
Mgmm/grouped_matmul/op_host/grouped_matmul_infershape_weight_quant_checker.cpp+121-86
@@ -329,6 +329,7 @@ ge::graphStatus GroupedMatmulWeightQuantChecker::CheckTensorNDimMultiScenario(co
329 auto antiquantScaleShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_SCALE, index);329 auto antiquantScaleShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_SCALE, index);
330 OP_CHECK_NULL_WITH_CONTEXT(context, antiquantScaleShape);330 OP_CHECK_NULL_WITH_CONTEXT(context, antiquantScaleShape);
331 int64_t antiquantScaleNDim;331 int64_t antiquantScaleNDim;
332+ //返回维度数量
332 if (IsA16W4(xDtype_, weightDtype_) && antiquantScaleShape->GetDimNum() == 2) {333 if (IsA16W4(xDtype_, weightDtype_) && antiquantScaleShape->GetDimNum() == 2) {
333 antiquantScaleNDim = antiquantScaleShape->GetDim(antiquantScaleShape->GetDimNum() - 1);334 antiquantScaleNDim = antiquantScaleShape->GetDim(antiquantScaleShape->GetDimNum() - 1);
334 } else {335 } else {
@@ -356,6 +357,7 @@ ge::graphStatus GroupedMatmulWeightQuantChecker::CheckTensorNDimMultiScenario(co
356 auto antiquantOffsetShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_OFFSET, index);357 auto antiquantOffsetShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_OFFSET, index);
357 OP_CHECK_NULL_WITH_CONTEXT(context, antiquantOffsetShape);358 OP_CHECK_NULL_WITH_CONTEXT(context, antiquantOffsetShape);
358 int64_t antiquantOffsetNDim;359 int64_t antiquantOffsetNDim;
360+ //返回维度数量
359 if (IsA16W4(xDtype_, weightDtype_) && antiquantOffsetShape->GetDimNum() == 2) {361 if (IsA16W4(xDtype_, weightDtype_) && antiquantOffsetShape->GetDimNum() == 2) {
360 antiquantOffsetNDim = antiquantOffsetShape->GetDim(antiquantOffsetShape->GetDimNum() - 1);362 antiquantOffsetNDim = antiquantOffsetShape->GetDim(antiquantOffsetShape->GetDimNum() - 1);
361 } else {363 } else {
@@ -370,26 +372,12 @@ ge::graphStatus GroupedMatmulWeightQuantChecker::CheckTensorNDimMultiScenario(co
370 return ge::GRAPH_SUCCESS;372 return ge::GRAPH_SUCCESS;
371}373}
372 374 
373-ge::graphStatus GroupedMatmulWeightQuantChecker::CheckCaseMultiScenario(const gert::InferShapeContext *context,375+ge::graphStatus GroupedMatmulWeightQuantChecker::CheckPerTensorShapeMultiScenario(const gert::InferShapeContext *context,
374- const GMMAttrs &gmmAttrs,376+ size_t wKDimIdx, size_t wNDimIdx,
375- const GMMInputParamsInfo &paramsInputInfo) const377+ size_t xSize) const
376{378{
377- const size_t &xSize = paramsInputInfo.numX;
378- // check group size
379- OP_CHECK_IF(CheckXWeightYGroupSizeMultiScenario(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
380- OP_LOGE(context->GetNodeName(), "The size of X, Y and weight are not equal."), return ge::GRAPH_FAILED);
381- OP_CHECK_IF(CheckTensorListSizeMultiScenario(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
382- OP_LOGE(context->GetNodeName(), "CheckTensorListSizeMultiScenario failed."), return ge::GRAPH_FAILED);
383- // check dimension
384- OP_CHECK_IF(CheckDimNumNoSplit(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
385- OP_LOGE(context->GetNodeName(), "Dim num of tensor in tensor lists or grouplist is invalid."),
386- return ge::GRAPH_FAILED);
387- 
388 auto xShape = context->GetDynamicInputShape(GMM_INDEX_IN_X, 0);379 auto xShape = context->GetDynamicInputShape(GMM_INDEX_IN_X, 0);
389 auto wShape = context->GetDynamicInputShape(GMM_INDEX_IN_WEIGHT, 0);380 auto wShape = context->GetDynamicInputShape(GMM_INDEX_IN_WEIGHT, 0);
390- size_t wKDimIdx = gmmAttrs.transposeWeight ? 1UL : 0UL;
391- size_t wNDimIdx = gmmAttrs.transposeWeight ? 0UL : 1UL;
392- 
393 int64_t weightKDimValue = wShape->GetDim(wKDimIdx);381 int64_t weightKDimValue = wShape->GetDim(wKDimIdx);
394 int64_t weightNDimValue = wShape->GetDim(wNDimIdx);382 int64_t weightNDimValue = wShape->GetDim(wNDimIdx);
395 383 
@@ -429,28 +417,60 @@ ge::graphStatus GroupedMatmulWeightQuantChecker::CheckCaseMultiScenario(const ge
429 OP_LOGE(context->GetNodeName(), "CheckTensorNDimMultiScenario is failed."),417 OP_LOGE(context->GetNodeName(), "CheckTensorNDimMultiScenario is failed."),
430 return ge::GRAPH_FAILED);418 return ge::GRAPH_FAILED);
431 }419 }
420+ return ge::GRAPH_SUCCESS;
421+}
422+ 
423+ge::graphStatus GroupedMatmulWeightQuantChecker::CheckA16W4PergroupMultiScenario(const gert::InferShapeContext *context,
424+ size_t wKDimIdx, size_t xSize) const
425+{
426+ for (size_t i = 0; i < xSize; i++) {
427+ auto antiquantScaleShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_SCALE, i);
428+ if (antiquantScaleShape == nullptr || antiquantScaleShape->GetDimNum() <= 1) {
429+ continue;
430+ }
431+ auto wShape_i = context->GetDynamicInputShape(GMM_INDEX_IN_WEIGHT, i);
432+ OP_CHECK_NULL_WITH_CONTEXT(context, wShape_i);
433+ int64_t kSize = wShape_i->GetDim(wKDimIdx);
434+ int64_t groupNum = antiquantScaleShape->GetDim(0);
435+ OP_CHECK_IF(groupNum <= 0, OP_LOGE(context->GetNodeName(), "GroupNum must be greater than 0."),
436+ return ge::GRAPH_FAILED);
437+ OP_CHECK_IF(kSize % groupNum != 0,
438+ OP_LOGE(context->GetNodeName(), "GroupNum must be multiple of the k axis of weight."),
439+ return ge::GRAPH_FAILED);
440+ int64_t groupSize = kSize / groupNum;
441+ OP_CHECK_IF(
442+ groupSize != 32 && groupSize != 64 && groupSize != 128 && groupSize != 256,
443+ OP_LOGE(context->GetNodeName(),
444+ "groupSize must be 32/64/128/256 on Ascend 950PR, but current groupSize is (%ld).", groupSize),
445+ return ge::GRAPH_FAILED);
446+ }
447+ return ge::GRAPH_SUCCESS;
448+}
449+ 
450+ge::graphStatus GroupedMatmulWeightQuantChecker::CheckCaseMultiScenario(const gert::InferShapeContext *context,
451+ const GMMAttrs &gmmAttrs,
452+ const GMMInputParamsInfo &paramsInputInfo) const
453+{
454+ const size_t &xSize = paramsInputInfo.numX;
455+ // check group size
456+ OP_CHECK_IF(CheckXWeightYGroupSizeMultiScenario(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
457+ OP_LOGE(context->GetNodeName(), "The size of X, Y and weight are not equal."), return ge::GRAPH_FAILED);
458+ OP_CHECK_IF(CheckTensorListSizeMultiScenario(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
459+ OP_LOGE(context->GetNodeName(), "CheckTensorListSizeMultiScenario failed."), return ge::GRAPH_FAILED);
460+ // check dimension
461+ OP_CHECK_IF(CheckDimNumNoSplit(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
462+ OP_LOGE(context->GetNodeName(), "Dim num of tensor in tensor lists or grouplist is invalid."),
463+ return ge::GRAPH_FAILED);
464+ 
465+ size_t wKDimIdx = gmmAttrs.transposeWeight ? 1UL : 0UL;
466+ size_t wNDimIdx = gmmAttrs.transposeWeight ? 0UL : 1UL;
467+ if (CheckPerTensorShapeMultiScenario(context, wKDimIdx, wNDimIdx, xSize) != ge::GRAPH_SUCCESS) {
468+ return ge::GRAPH_FAILED;
469+ }
432 470 
433 if (IsA16W4(xDtype_, weightDtype_)) {471 if (IsA16W4(xDtype_, weightDtype_)) {
434- for (size_t i = 0; i < xSize; i++) {472+ if (CheckA16W4PergroupMultiScenario(context, wKDimIdx, xSize) != ge::GRAPH_SUCCESS) {
435- auto antiquantScaleShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_SCALE, i);473+ return ge::GRAPH_FAILED;
436- if (antiquantScaleShape == nullptr || antiquantScaleShape->GetDimNum() <= 1) {
437- continue;
438- }
439- auto wShape_i = context->GetDynamicInputShape(GMM_INDEX_IN_WEIGHT, i);
440- OP_CHECK_NULL_WITH_CONTEXT(context, wShape_i);
441- int64_t kSize = wShape_i->GetDim(wKDimIdx);
442- int64_t groupNum = antiquantScaleShape->GetDim(0);
443- OP_CHECK_IF(groupNum <= 0, OP_LOGE(context->GetNodeName(), "GroupNum must be greater than 0."),
444- return ge::GRAPH_FAILED);
445- OP_CHECK_IF(kSize % groupNum != 0,
446- OP_LOGE(context->GetNodeName(), "GroupNum must be multiple of the k axis of weight."),
447- return ge::GRAPH_FAILED);
448- int64_t groupSize = kSize / groupNum;
449- OP_CHECK_IF(
450- groupSize != 32 && groupSize != 64 && groupSize != 128 && groupSize != 256,
451- OP_LOGE(context->GetNodeName(),
452- "groupSize must be 32/64/128/256 on Ascend 950PR, but current groupSize is (%ld).", groupSize),
453- return ge::GRAPH_FAILED);
454 }474 }
455 }475 }
456 476 
@@ -630,6 +650,7 @@ ge::graphStatus GroupedMatmulWeightQuantChecker::CheckShapeForTensorListAtIndex(
630 const GMMAttrs &gmmAttrs,650 const GMMAttrs &gmmAttrs,
631 size_t tensorIdx) const651 size_t tensorIdx) const
632{652{
653+ (void)gmmAttrs; // 保留传参以保持接口一致性，当前逻辑暂未使用
633 auto tensorShape = context->GetDynamicInputShape(gmm_index, tensorIdx);654 auto tensorShape = context->GetDynamicInputShape(gmm_index, tensorIdx);
634 if (IsNonEmpty(tensorShape)) {655 if (IsNonEmpty(tensorShape)) {
635 size_t tensorDimNum = tensorShape->GetDimNum();656 size_t tensorDimNum = tensorShape->GetDimNum();
@@ -832,60 +853,74 @@ ge::graphStatus GroupedMatmulWeightQuantChecker::CheckTensorListSizeMultiScenari
832 return ge::GRAPH_SUCCESS;853 return ge::GRAPH_SUCCESS;
833}854}
834 855 
856+ge::graphStatus GroupedMatmulWeightQuantChecker::CheckShapeValidNoSplit(const gert::InferShapeContext *context,
857+ const GMMAttrs &gmmAttrs)
858+{
859+ // 多多多场景校验
860+ GMMInputParamsInfo paramsInputInfo{0, 0, 0, 0, 0, 0, 0};
861+ OP_CHECK_IF(GetNumOfInputs(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
862+ OP_LOGE(context->GetNodeName(), "GetNumOfInputs failed."), return ge::GRAPH_FAILED);
863+ hasBias_ = paramsInputInfo.numBias > 0;
864+ if (paramsInputInfo.numBias == 1) {
865+ auto biasShape = context->GetDynamicInputShape(GMM_INDEX_IN_BIAS, 0);
866+ hasBias_ = biasShape->GetShapeSize() != 0;
867+ }
868+ hasAntiquantOffset_ = paramsInputInfo.numAntiquantOffset > 0;
869+ if (paramsInputInfo.numAntiquantOffset == 1) {
870+ auto antiQuantOffsetShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_OFFSET, 0);
871+ hasAntiquantOffset_ = antiQuantOffsetShape->GetShapeSize() != 0;
872+ }
873+ OP_CHECK_IF(CheckCaseMultiScenario(context, gmmAttrs, paramsInputInfo) != ge::GRAPH_SUCCESS,
874+ OP_LOGE(context->GetNodeName(), "CheckCaseMultiScenario failed."), return ge::GRAPH_FAILED);
875+ return ge::GRAPH_SUCCESS;
876+}
877+ 
878+ge::graphStatus GroupedMatmulWeightQuantChecker::CheckShapeValidSplitM(const gert::InferShapeContext *context,
879+ const GMMAttrs &gmmAttrs)
880+{
881+ auto groupListShape = context->GetOptionalInputShape(GMM_INDEX_IN_GROUP_LIST);
882+ OP_CHECK_NULL_WITH_CONTEXT(context, groupListShape);
883+ OP_CHECK_IF(CheckShapeForGrouplist(context, groupListShape) != ge::GRAPH_SUCCESS,
884+ OP_LOGE(context->GetNodeName(), "CheckShapeForGrouplist failed."), return ge::GRAPH_FAILED);
885+ groupNum_ = groupListShape->GetDim(0);
886+ OP_CHECK_IF(CheckShapeForXAndWeight(context) != ge::GRAPH_SUCCESS,
887+ OP_LOGE(context->GetNodeName(), "CheckShapeForXAndWeight failed."), return ge::GRAPH_FAILED);
888+ if (isMultiTensorWeight_) {
889+ for (size_t i = 0; i < numWeight_; i++) {
890+ OP_CHECK_IF(CheckShapeForTensorListAtIndex(context, GMM_INDEX_IN_BIAS, "bias", gmmAttrs, i) !=
891+ ge::GRAPH_SUCCESS,
892+ OP_LOGE(context->GetNodeName(), "CheckShapeForBias at index [%zu] failed.", i),
893+ return ge::GRAPH_FAILED);
894+ OP_CHECK_IF(CheckShapeForTensorListAtIndex(context, GMM_INDEX_IN_ANTIQUANT_SCALE, "antiquantScale",
895+ gmmAttrs, i) != ge::GRAPH_SUCCESS,
896+ OP_LOGE(context->GetNodeName(), "CheckShapeForAntiquantScale at index [%zu] failed.", i),
897+ return ge::GRAPH_FAILED);
898+ OP_CHECK_IF(CheckShapeForTensorListAtIndex(context, GMM_INDEX_IN_ANTIQUANT_OFFSET, "antiquantOffset",
899+ gmmAttrs, i) != ge::GRAPH_SUCCESS,
900+ OP_LOGE(context->GetNodeName(), "CheckShapeForAntiquantOffset failed."),
901+ return ge::GRAPH_FAILED);
902+ OP_CHECK_IF(CheckGroupSize(context, gmmAttrs, i) != ge::GRAPH_SUCCESS,
903+ OP_LOGE(context->GetNodeName(), "CheckGroupSize failed."), return ge::GRAPH_FAILED);
904+ }
905+ } else {
906+ OP_CHECK_IF(CheckShapeForTensorList(context, GMM_INDEX_IN_BIAS, "bias", gmmAttrs) != ge::GRAPH_SUCCESS,
907+ OP_LOGE(context->GetNodeName(), "CheckShapeForBias failed."), return ge::GRAPH_FAILED);
908+ OP_CHECK_IF(CheckShapeForWeightQuantParam(context, gmmAttrs) != ge::GRAPH_SUCCESS,
909+ OP_LOGE(context->GetNodeName(), "CheckShapeForWeightQuantParam failed."),
910+ return ge::GRAPH_FAILED);
911+ OP_CHECK_IF(CheckGroupSize(context, gmmAttrs, 0) != ge::GRAPH_SUCCESS,
912+ OP_LOGE(context->GetNodeName(), "CheckGroupSize failed."), return ge::GRAPH_FAILED);
913+ }
914+ return ge::GRAPH_SUCCESS;
915+}
916+ 
835ge::graphStatus GroupedMatmulWeightQuantChecker::CheckShapeValid(const gert::InferShapeContext *context,917ge::graphStatus GroupedMatmulWeightQuantChecker::CheckShapeValid(const gert::InferShapeContext *context,
836 const GMMAttrs &gmmAttrs)918 const GMMAttrs &gmmAttrs)
837{919{
838 if (gmmAttrs.groupType == GMM_NO_SPLIT) {920 if (gmmAttrs.groupType == GMM_NO_SPLIT) {
839- // 多多多场景校验921+ return CheckShapeValidNoSplit(context, gmmAttrs);
840- GMMInputParamsInfo paramsInputInfo{0, 0, 0, 0, 0, 0, 0};
841- OP_CHECK_IF(GetNumOfInputs(context, paramsInputInfo) != ge::GRAPH_SUCCESS,
842- OP_LOGE(context->GetNodeName(), "GetNumOfInputs failed."), return ge::GRAPH_FAILED);
843- hasBias_ = paramsInputInfo.numBias > 0;
844- if (paramsInputInfo.numBias == 1) {
845- auto biasShape = context->GetDynamicInputShape(GMM_INDEX_IN_BIAS, 0);
846- hasBias_ = biasShape->GetShapeSize() != 0;
847- }
848- hasAntiquantOffset_ = paramsInputInfo.numAntiquantOffset > 0;
849- if (paramsInputInfo.numAntiquantOffset == 1) {
850- auto antiQuantOffsetShape = context->GetDynamicInputShape(GMM_INDEX_IN_ANTIQUANT_OFFSET, 0);
851- hasAntiquantOffset_ = antiQuantOffsetShape->GetShapeSize() != 0;
852- }
853- OP_CHECK_IF(CheckCaseMultiScenario(context, gmmAttrs, paramsInputInfo) != ge::GRAPH_SUCCESS,
854- OP_LOGE(context->GetNodeName(), "CheckCaseMultiScenario failed."), return ge::GRAPH_FAILED);
855 } else {922 } else {
856- auto groupListShape = context->GetOptionalInputShape(GMM_INDEX_IN_GROUP_LIST);923+ return CheckShapeValidSplitM(context, gmmAttrs);
857- OP_CHECK_NULL_WITH_CONTEXT(context, groupListShape);
858- OP_CHECK_IF(CheckShapeForGrouplist(context, groupListShape) != ge::GRAPH_SUCCESS,
859- OP_LOGE(context->GetNodeName(), "CheckShapeForGrouplist failed."), return ge::GRAPH_FAILED);
860- groupNum_ = groupListShape->GetDim(0);
861- OP_CHECK_IF(CheckShapeForXAndWeight(context) != ge::GRAPH_SUCCESS,
862- OP_LOGE(context->GetNodeName(), "CheckShapeForXAndWeight failed."), return ge::GRAPH_FAILED);
863- if (isMultiTensorWeight_) {
864- for (size_t i = 0; i < numWeight_; i++) {
865- OP_CHECK_IF(CheckShapeForTensorListAtIndex(context, GMM_INDEX_IN_BIAS, "bias", gmmAttrs, i) !=
866- ge::GRAPH_SUCCESS,
867- OP_LOGE(context->GetNodeName(), "CheckShapeForBias at index [%zu] failed.", i),
868- return ge::GRAPH_FAILED);
869- OP_CHECK_IF(CheckShapeForTensorListAtIndex(context, GMM_INDEX_IN_ANTIQUANT_SCALE, "antiquantScale",
870- gmmAttrs, i) != ge::GRAPH_SUCCESS,
871- OP_LOGE(context->GetNodeName(), "CheckShapeForAntiquantScale at index [%zu] failed.", i),
872- return ge::GRAPH_FAILED);
873- OP_CHECK_IF(CheckShapeForTensorListAtIndex(context, GMM_INDEX_IN_ANTIQUANT_OFFSET, "antiquantOffset",
874- gmmAttrs, i) != ge::GRAPH_SUCCESS,
875- OP_LOGE(context->GetNodeName(), "CheckShapeForAntiquantOffset failed."),
876- return ge::GRAPH_FAILED);
877- OP_CHECK_IF(CheckGroupSize(context, gmmAttrs, i) != ge::GRAPH_SUCCESS,
878- OP_LOGE(context->GetNodeName(), "CheckGroupSize failed."), return ge::GRAPH_FAILED);
879- }
880- } else {
881- OP_CHECK_IF(CheckShapeForTensorList(context, GMM_INDEX_IN_BIAS, "bias", gmmAttrs) != ge::GRAPH_SUCCESS,
882- OP_LOGE(context->GetNodeName(), "CheckShapeForBias failed."), return ge::GRAPH_FAILED);
883- OP_CHECK_IF(CheckShapeForWeightQuantParam(context, gmmAttrs) != ge::GRAPH_SUCCESS,
884- OP_LOGE(context->GetNodeName(), "CheckShapeForWeightQuantParam failed."),
885- return ge::GRAPH_FAILED);
886- OP_CHECK_IF(CheckGroupSize(context, gmmAttrs, 0) != ge::GRAPH_SUCCESS,
887- OP_LOGE(context->GetNodeName(), "CheckGroupSize failed."), return ge::GRAPH_FAILED);
888- }
889 }924 }
890 return ge::GRAPH_SUCCESS;925 return ge::GRAPH_SUCCESS;
891}926}
Mgmm/grouped_matmul/op_host/grouped_matmul_infershape_weight_quant_checker.h+6-0
@@ -52,6 +52,10 @@ private:
52 const int64_t weightNDimValue, const size_t index) const;52 const int64_t weightNDimValue, const size_t index) const;
53 ge::graphStatus CheckCaseMultiScenario(const gert::InferShapeContext *context, const GMMAttrs &gmmAttrs,53 ge::graphStatus CheckCaseMultiScenario(const gert::InferShapeContext *context, const GMMAttrs &gmmAttrs,
54 const GMMInputParamsInfo &paramsInputInfo) const;54 const GMMInputParamsInfo &paramsInputInfo) const;
55+ ge::graphStatus CheckPerTensorShapeMultiScenario(const gert::InferShapeContext *context, size_t wKDimIdx,
56+ size_t wNDimIdx, size_t xSize) const;
57+ ge::graphStatus CheckA16W4PergroupMultiScenario(const gert::InferShapeContext *context, size_t wKDimIdx,
58+ size_t xSize) const;
55 ge::graphStatus CheckShapeForTensorList(const gert::InferShapeContext *context, size_t gmm_index,59 ge::graphStatus CheckShapeForTensorList(const gert::InferShapeContext *context, size_t gmm_index,
56 const std::string &tensorType, const GMMAttrs &gmmAttrs) const;60 const std::string &tensorType, const GMMAttrs &gmmAttrs) const;
57 ge::graphStatus CheckShapeForTensorListAtIndex(const gert::InferShapeContext *context, size_t gmm_index,61 ge::graphStatus CheckShapeForTensorListAtIndex(const gert::InferShapeContext *context, size_t gmm_index,
@@ -62,6 +66,8 @@ private:
62 ge::graphStatus CheckTensorListSizeMultiScenario(const gert::InferShapeContext *context,66 ge::graphStatus CheckTensorListSizeMultiScenario(const gert::InferShapeContext *context,
63 const GMMInputParamsInfo &paramsInputInfo) const;67 const GMMInputParamsInfo &paramsInputInfo) const;
64 ge::graphStatus CheckShapeValid(const gert::InferShapeContext *context, const GMMAttrs &gmmAttrs);68 ge::graphStatus CheckShapeValid(const gert::InferShapeContext *context, const GMMAttrs &gmmAttrs);
69+ ge::graphStatus CheckShapeValidNoSplit(const gert::InferShapeContext *context, const GMMAttrs &gmmAttrs);
70+ ge::graphStatus CheckShapeValidSplitM(const gert::InferShapeContext *context, const GMMAttrs &gmmAttrs);
65 71 
66 ge::graphStatus CheckShapeForWeightQuantParam(const gert::InferShapeContext *context,72 ge::graphStatus CheckShapeForWeightQuantParam(const gert::InferShapeContext *context,
67 const GMMAttrs &gmmAttrs) const;73 const GMMAttrs &gmmAttrs) const;
Mgmm/grouped_matmul/op_host/op_tiling/arch35/grouped_quant_basic_api_matmul_tiling.cpp+48-26
@@ -558,11 +558,13 @@ ge::graphStatus GroupedS4S4IntQuantTiling::GetShapeAttrsInfo()
558 OP_CHECK_NULL_WITH_CONTEXT(context_, xShapePtr);558 OP_CHECK_NULL_WITH_CONTEXT(context_, xShapePtr);
559 OP_CHECK_NULL_WITH_CONTEXT(context_, wShapePtr);559 OP_CHECK_NULL_WITH_CONTEXT(context_, wShapePtr);
560 const auto &weightStorageShape = wShapePtr->GetStorageShape();560 const auto &weightStorageShape = wShapePtr->GetStorageShape();
561- weightNzC032_ = inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ && weightStorageShape.GetDimNum() > 0U &&561+ weightNzC032_ = inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ &&
562- weightStorageShape.GetDim(weightStorageShape.GetDimNum() - 1U) == 32;562+ weightStorageShape.GetDimNum() > 0U &&
563- OP_CHECK_IF(!SetMKN(xShapePtr->GetOriginShape(), wShapePtr->GetOriginShape()),563+ weightStorageShape.GetDim(weightStorageShape.GetDimNum() - 1U) == INT8_NZ_C0;
564- OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 SetMKN failed."), return ge::GRAPH_FAILED);564+ OP_CHECK_IF(!SetMKNForS4S4(xShapePtr->GetOriginShape(), wShapePtr->GetOriginShape()),
565- OP_CHECK_IF(!SetGroupNum(GROUPLIST_INDEX),565+ OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 SetMKN failed."),
566+ return ge::GRAPH_FAILED);
567+ OP_CHECK_IF(!SetGroupNumForS4S4(GROUPLIST_INDEX),
566 OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 SetGroupNum failed."),568 OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 SetGroupNum failed."),
567 return ge::GRAPH_FAILED);569 return ge::GRAPH_FAILED);
568 OP_CHECK_IF(!AnalyzeS4S4(), OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 AnalyzeS4S4 failed."),570 OP_CHECK_IF(!AnalyzeS4S4(), OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 AnalyzeS4S4 failed."),
@@ -573,7 +575,7 @@ ge::graphStatus GroupedS4S4IntQuantTiling::GetShapeAttrsInfo()
573 return ge::GRAPH_SUCCESS;575 return ge::GRAPH_SUCCESS;
574}576}
575 577 
576-bool GroupedS4S4IntQuantTiling::SetMKN(const gert::Shape &xShape, const gert::Shape &wShape)578+bool GroupedS4S4IntQuantTiling::SetMKNForS4S4(const gert::Shape &xShape, const gert::Shape &wShape)
577{579{
578 uint32_t xDimNum = static_cast<uint32_t>(xShape.GetDimNum());580 uint32_t xDimNum = static_cast<uint32_t>(xShape.GetDimNum());
579 uint32_t wDimNum = static_cast<uint32_t>(wShape.GetDimNum());581 uint32_t wDimNum = static_cast<uint32_t>(wShape.GetDimNum());
@@ -589,7 +591,7 @@ bool GroupedS4S4IntQuantTiling::SetMKN(const gert::Shape &xShape, const gert::Sh
589 return true;591 return true;
590}592}
591 593 
592-bool GroupedS4S4IntQuantTiling::SetGroupNum(uint32_t groupListIndex)594+bool GroupedS4S4IntQuantTiling::SetGroupNumForS4S4(uint32_t groupListIndex)
593{595{
594 OP_CHECK_IF(!GroupedQmmBasicApiTiling::SetGroupNum(groupListIndex),596 OP_CHECK_IF(!GroupedQmmBasicApiTiling::SetGroupNum(groupListIndex),
595 OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 base SetGroupNum failed."), return false);597 OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "S4S4 base SetGroupNum failed."), return false);
@@ -597,7 +599,7 @@ bool GroupedS4S4IntQuantTiling::SetGroupNum(uint32_t groupListIndex)
597 return true;599 return true;
598}600}
599 601 
600-bool GroupedS4S4IntQuantTiling::SetMKNList()602+bool GroupedS4S4IntQuantTiling::SetMKNListForS4S4()
601{603{
602 // Base's mList_/kList_/nList_ is private (inaccessible to derived).604 // Base's mList_/kList_/nList_ is private (inaccessible to derived).
603 // Write to our own shadow arrays (mimic base GroupedQmmTiling::SetMKNList).605 // Write to our own shadow arrays (mimic base GroupedQmmTiling::SetMKNList).
@@ -641,7 +643,7 @@ bool GroupedS4S4IntQuantTiling::AnalyzeS4S4()
641 return true;643 return true;
642}644}
643 645 
644-bool GroupedS4S4IntQuantTiling::CheckS4S4Params()646+bool GroupedS4S4IntQuantTiling::CheckS4S4Params() const
645{647{
646 OP_CHECK_IF(inputParams_.aDtype != ge::DT_INT4 || inputParams_.bDtype != ge::DT_INT4,648 OP_CHECK_IF(inputParams_.aDtype != ge::DT_INT4 || inputParams_.bDtype != ge::DT_INT4,
647 OP_LOGE(context_->GetNodeName(), "S4S4: x/weight must be INT4."), return false);649 OP_LOGE(context_->GetNodeName(), "S4S4: x/weight must be INT4."), return false);
@@ -706,7 +708,8 @@ ge::graphStatus GroupedS4S4IntQuantTiling::DoOpTiling()
706 OP_CHECK_IF(CalUbDivideS4S4() != ge::GRAPH_SUCCESS,708 OP_CHECK_IF(CalUbDivideS4S4() != ge::GRAPH_SUCCESS,
707 OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "CalUbDivideS4S4 failed."),709 OPS_REPORT_VECTOR_INNER_ERR(context_->GetNodeName(), "CalUbDivideS4S4 failed."),
708 return ge::GRAPH_FAILED);710 return ge::GRAPH_FAILED);
709- OP_CHECK_IF(!SetMKNList(), OP_LOGE(context_->GetNodeName(), "SetMKNList failed."), return ge::GRAPH_FAILED);711+ OP_CHECK_IF(!SetMKNListForS4S4(), OP_LOGE(context_->GetNodeName(), "SetMKNList failed."),
712+ return ge::GRAPH_FAILED);
710 s4s4Tiling_.singleN = FindBestSingleN();713 s4s4Tiling_.singleN = FindBestSingleN();
711 return ge::GRAPH_SUCCESS;714 return ge::GRAPH_SUCCESS;
712}715}
@@ -750,7 +753,7 @@ ge::graphStatus GroupedS4S4IntQuantTiling::CalUbDivideS4S4()
750 return ge::GRAPH_SUCCESS;753 return ge::GRAPH_SUCCESS;
751}754}
752 755 
753-void GroupedS4S4IntQuantTiling::InitCommonL1TilingFields()756+void GroupedS4S4IntQuantTiling::InitS4S4CommonL1TilingFields()
754{757{
755 s4s4Tiling_.stepM = 1UL;758 s4s4Tiling_.stepM = 1UL;
756 s4s4Tiling_.stepN = 1UL;759 s4s4Tiling_.stepN = 1UL;
@@ -764,7 +767,7 @@ void GroupedS4S4IntQuantTiling::InitCommonL1TilingFields()
764 1U;767 1U;
765}768}
766 769 
767-ge::graphStatus GroupedS4S4IntQuantTiling::CalcLeftL1Size(uint64_t &leftL1Size) const770+ge::graphStatus GroupedS4S4IntQuantTiling::CalcS4S4LeftL1Size(uint64_t &leftL1Size) const
768{771{
769 leftL1Size = aicoreParams_.l1Size;772 leftL1Size = aicoreParams_.l1Size;
770 return ge::GRAPH_SUCCESS;773 return ge::GRAPH_SUCCESS;
@@ -772,13 +775,11 @@ ge::graphStatus GroupedS4S4IntQuantTiling::CalcLeftL1Size(uint64_t &leftL1Size)
772 775 
773ge::graphStatus GroupedS4S4IntQuantTiling::CalL1Tiling()776ge::graphStatus GroupedS4S4IntQuantTiling::CalL1Tiling()
774{777{
775- InitCommonL1TilingFields();778+ InitS4S4CommonL1TilingFields();
776- if (inputParams_.kSize == 0UL) {779+ if (inputParams_.kSize == 0UL) { return ge::GRAPH_SUCCESS; }
777- return ge::GRAPH_SUCCESS;
778- }
779 uint64_t leftL1Size = 0UL;780 uint64_t leftL1Size = 0UL;
780- OP_CHECK_IF(CalcLeftL1Size(leftL1Size) != ge::GRAPH_SUCCESS,781+ OP_CHECK_IF(CalcS4S4LeftL1Size(leftL1Size) != ge::GRAPH_SUCCESS,
781- OP_LOGE(context_->GetNodeName(), "CalcLeftL1Size failed."), return ge::GRAPH_FAILED);782+ OP_LOGE(context_->GetNodeName(), "CalcS4S4LeftL1Size failed."), return ge::GRAPH_FAILED);
782 return CalL1Depth(leftL1Size);783 return CalL1Depth(leftL1Size);
783}784}
784 785 
@@ -801,16 +802,18 @@ ge::graphStatus GroupedS4S4IntQuantTiling::CalL1Depth(uint64_t leftL1Size)
801 constexpr uint64_t elemBytes = 1UL;802 constexpr uint64_t elemBytes = 1UL;
802 constexpr uint64_t scaleElemBytes = sizeof(uint64_t);803 constexpr uint64_t scaleElemBytes = sizeof(uint64_t);
803 auto AlignUpL = [](uint64_t v, uint64_t a) -> uint64_t { return (v + a - 1UL) / a * a; };804 auto AlignUpL = [](uint64_t v, uint64_t a) -> uint64_t { return (v + a - 1UL) / a * a; };
804- auto GetAL1Bytes = [&](uint64_t kAL1) -> uint64_t {805+ auto GetAL1Bytes = [this](uint64_t kAL1) -> uint64_t {
805 return AlignUpL(s4s4Tiling_.baseM, CUBE_BLOCK) * AlignUpL(kAL1, INT8_C0) * elemBytes;806 return AlignUpL(s4s4Tiling_.baseM, CUBE_BLOCK) * AlignUpL(kAL1, INT8_C0) * elemBytes;
806 };807 };
807- auto GetBL1Bytes = [&](uint64_t kBL1) -> uint64_t {808+ auto GetBL1Bytes = [this](uint64_t kBL1) -> uint64_t {
808 return AlignUpL(kBL1, INT8_C0) * AlignUpL(s4s4Tiling_.baseN, CUBE_BLOCK) * elemBytes;809 return AlignUpL(kBL1, INT8_C0) * AlignUpL(s4s4Tiling_.baseN, CUBE_BLOCK) * elemBytes;
809 };810 };
810- auto GetScaleL1Bytes = [&]() -> uint64_t { return AlignUpL(s4s4Tiling_.baseN * scaleElemBytes, DATA_BLOCK_BYTES); };811+ auto GetScaleL1Bytes = [this]() -> uint64_t {
811- auto GetL1StageBytes = [&](uint64_t kAL1, uint64_t kBL1) -> uint64_t {812+ return AlignUpL(s4s4Tiling_.baseN * scaleElemBytes, DATA_BLOCK_BYTES);
812- return AlignUpL(GetAL1Bytes(kAL1), DATA_BLOCK_BYTES) + AlignUpL(GetBL1Bytes(kBL1), DATA_BLOCK_BYTES) +813+ };
813- GetScaleL1Bytes();814+ auto GetL1StageBytes = [this](uint64_t kAL1, uint64_t kBL1) -> uint64_t {
815+ return AlignUpL(GetAL1Bytes(kAL1), DATA_BLOCK_BYTES) +
816+ AlignUpL(GetBL1Bytes(kBL1), DATA_BLOCK_BYTES) + GetScaleL1Bytes();
814 };817 };
815 818 
816 const uint64_t halfL1 = leftL1Size / L1_BUFFER_NUM;819 const uint64_t halfL1 = leftL1Size / L1_BUFFER_NUM;
@@ -891,7 +894,7 @@ uint64_t GroupedS4S4IntQuantTiling::GetTilingKey() const
891 return GET_TPL_TILING_KEY(static_cast<uint64_t>(inputParams_.transB ? 1 : 0), 0UL, S4S4_KERNEL_TYPE_MIX);894 return GET_TPL_TILING_KEY(static_cast<uint64_t>(inputParams_.transB ? 1 : 0), 0UL, S4S4_KERNEL_TYPE_MIX);
892}895}
893 896 
894-ge::graphStatus GroupedS4S4IntQuantTiling::PostTiling()897+void GroupedS4S4IntQuantTiling::SetS4S4Params()
895{898{
896 using namespace GroupedMatmulTilingData;899 using namespace GroupedMatmulTilingData;
897 auto &p = tilingData_.gmmS4S4Params;900 auto &p = tilingData_.gmmS4S4Params;
@@ -913,7 +916,11 @@ ge::graphStatus GroupedS4S4IntQuantTiling::PostTiling()
913 p.groupListType = s4s4Tiling_.groupListType;916 p.groupListType = s4s4Tiling_.groupListType;
914 p.reserved = ((inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ) ? S4S4_WEIGHT_NZ_FLAG : 0ULL) |917 p.reserved = ((inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ) ? S4S4_WEIGHT_NZ_FLAG : 0ULL) |
915 (inputParams_.transB ? S4S4_TRANSPOSE_WEIGHT_FLAG : 0ULL) |918 (inputParams_.transB ? S4S4_TRANSPOSE_WEIGHT_FLAG : 0ULL) |
916- (weightNzC032_ ? S4S4_WEIGHT_NZ_C0_32_FLAG : 0ULL);919+ (weightNzC032_ ? S4S4_WEIGHT_NZ_C0_32_FLAG : 0ULL);
920+}
921+ 
922+ge::graphStatus GroupedS4S4IntQuantTiling::SetArrayParams()
923+{
917 errno_t retM = memcpy_s(tilingData_.gmmArray.mList, sizeof(tilingData_.gmmArray.mList), mList_, sizeof(mList_));924 errno_t retM = memcpy_s(tilingData_.gmmArray.mList, sizeof(tilingData_.gmmArray.mList), mList_, sizeof(mList_));
918 OP_CHECK_IF(retM != EOK, OP_LOGE(context_->GetNodeName(), "memcpy_s mList failed, ret=%d", retM),925 OP_CHECK_IF(retM != EOK, OP_LOGE(context_->GetNodeName(), "memcpy_s mList failed, ret=%d", retM),
919 return ge::GRAPH_FAILED);926 return ge::GRAPH_FAILED);
@@ -923,6 +930,11 @@ ge::graphStatus GroupedS4S4IntQuantTiling::PostTiling()
923 errno_t retN = memcpy_s(tilingData_.gmmArray.nList, sizeof(tilingData_.gmmArray.nList), nList_, sizeof(nList_));930 errno_t retN = memcpy_s(tilingData_.gmmArray.nList, sizeof(tilingData_.gmmArray.nList), nList_, sizeof(nList_));
924 OP_CHECK_IF(retN != EOK, OP_LOGE(context_->GetNodeName(), "memcpy_s nList failed, ret=%d", retN),931 OP_CHECK_IF(retN != EOK, OP_LOGE(context_->GetNodeName(), "memcpy_s nList failed, ret=%d", retN),
925 return ge::GRAPH_FAILED);932 return ge::GRAPH_FAILED);
933+ return ge::GRAPH_SUCCESS;
934+}
935+ 
936+void GroupedS4S4IntQuantTiling::SetMmTilingData()
937+{
926 // mmTilingData(TCubeTiling)938 // mmTilingData(TCubeTiling)
927 auto &mm = tilingData_.mmTilingData;939 auto &mm = tilingData_.mmTilingData;
928 mm.M = inputParams_.mSize;940 mm.M = inputParams_.mSize;
@@ -947,6 +959,16 @@ ge::graphStatus GroupedS4S4IntQuantTiling::PostTiling()
947 mm.dbL0A = DB_SIZE;959 mm.dbL0A = DB_SIZE;
948 mm.dbL0B = DB_SIZE;960 mm.dbL0B = DB_SIZE;
949 mm.dbL0C = s4s4Tiling_.dbL0c;961 mm.dbL0C = s4s4Tiling_.dbL0c;
962+}
963+ 
964+ge::graphStatus GroupedS4S4IntQuantTiling::PostTiling()
965+{
966+ SetS4S4Params();
967+ if (SetArrayParams() != ge::GRAPH_SUCCESS) {
968+ return ge::GRAPH_FAILED;
969+ }
970+ SetMmTilingData();
971+ auto &p = tilingData_.gmmS4S4Params;
950 OP_LOGI(context_->GetNodeName(),972 OP_LOGI(context_->GetNodeName(),
951 "S4S4 tiling: baseM=%u baseN=%u baseK=%u ubCalSize=%u ubRestBytes=%u quantGroupNum=%u isPerToken=%u "973 "S4S4 tiling: baseM=%u baseN=%u baseK=%u ubCalSize=%u ubRestBytes=%u quantGroupNum=%u isPerToken=%u "
952 "depthA1=%lu depthB1=%lu",974 "depthA1=%lu depthB1=%lu",
Mgmm/grouped_matmul/op_host/op_tiling/arch35/grouped_quant_basic_api_matmul_tiling.h+10-7
@@ -118,20 +118,23 @@ protected:
118 ge::graphStatus PostTiling() override;118 ge::graphStatus PostTiling() override;
119 virtual void Reset();119 virtual void Reset();
120 120 
121- // S4S4-specific (hide base's same-named; call base then sync s4s4Tiling_)121+ // S4S4-specific (call base then sync s4s4Tiling_)
122- bool SetMKN(const gert::Shape &xShape, const gert::Shape &wShape);122+ bool SetMKNForS4S4(const gert::Shape &xShape, const gert::Shape &wShape);
123- bool SetGroupNum(uint32_t groupListIndex);123+ bool SetGroupNumForS4S4(uint32_t groupListIndex);
124- bool SetMKNList();124+ bool SetMKNListForS4S4();
125 bool AnalyzeS4S4();125 bool AnalyzeS4S4();
126- bool CheckS4S4Params();126+ bool CheckS4S4Params() const;
127 void CalBasicBlockS4S4();127 void CalBasicBlockS4S4();
128 ge::graphStatus CalUbDivideS4S4();128 ge::graphStatus CalUbDivideS4S4();
129 ge::graphStatus CalWorkspaceS4S4();129 ge::graphStatus CalWorkspaceS4S4();
130 uint32_t FindBestSingleN() const;130 uint32_t FindBestSingleN() const;
131- void InitCommonL1TilingFields();131+ void InitS4S4CommonL1TilingFields();
132- ge::graphStatus CalcLeftL1Size(uint64_t &leftL1Size) const;132+ ge::graphStatus CalcS4S4LeftL1Size(uint64_t &leftL1Size) const;
133 ge::graphStatus CalL1Tiling();133 ge::graphStatus CalL1Tiling();
134 ge::graphStatus CalL1Depth(uint64_t leftL1Size);134 ge::graphStatus CalL1Depth(uint64_t leftL1Size);
135+ void SetS4S4Params();
136+ ge::graphStatus SetArrayParams();
137+ void SetMmTilingData();
135 138 
136private:139private:
137 struct S4S4BasicTiling {140 struct S4S4BasicTiling {
Mgmm/grouped_matmul/op_host/op_tiling/arch35/grouped_weight_quant_batch_matmul_tiling.cpp+114-54
@@ -717,7 +717,7 @@ bool GroupedWeightQuantBatchMatmulTiling::CheckGroupList(const gert::TilingConte
717 return true;717 return true;
718}718}
719 719 
720-bool GroupedWeightQuantBatchMatmulTiling::AnalyzeAttr(const gert::TilingContext *context)720+bool GroupedWeightQuantBatchMatmulTiling::AnalyzeCompileInfoAndAttrs(const gert::TilingContext *context)
721{721{
722 auto compileInfoPtr = context->GetCompileInfo<GMMCompileInfo>();722 auto compileInfoPtr = context->GetCompileInfo<GMMCompileInfo>();
723 OP_CHECK_IF(compileInfoPtr == nullptr, OP_LOGE(context->GetNodeName(), "compileInfoPtr is nullptr."), return false);723 OP_CHECK_IF(compileInfoPtr == nullptr, OP_LOGE(context->GetNodeName(), "compileInfoPtr is nullptr."), return false);
@@ -738,7 +738,11 @@ bool GroupedWeightQuantBatchMatmulTiling::AnalyzeAttr(const gert::TilingContext
738 groupListType_ = groupListTypePtr != nullptr ? *groupListTypePtr : 0;738 groupListType_ = groupListTypePtr != nullptr ? *groupListTypePtr : 0;
739 isSingleX_ = (groupType_ != static_cast<int64_t>(GroupType::NO_SPLIT) &&739 isSingleX_ = (groupType_ != static_cast<int64_t>(GroupType::NO_SPLIT) &&
740 context->GetDynamicInputTensor(X_IDX, 1) == nullptr);740 context->GetDynamicInputTensor(X_IDX, 1) == nullptr);
741+ return true;
742+}
741 743 
744+bool GroupedWeightQuantBatchMatmulTiling::AnalyzeTensorLayout(const gert::TilingContext *context)
745+{
742 // 2: when x is multi-tensor, y is single-tensor; 3: when x is single-tensor, y is single-tensor746 // 2: when x is multi-tensor, y is single-tensor; 3: when x is single-tensor, y is single-tensor
743 isSingleY_ = (splitItem_ == 2 || splitItem_ == 3);747 isSingleY_ = (splitItem_ == 2 || splitItem_ == 3);
744 GetNumOfInputs(context);748 GetNumOfInputs(context);
@@ -751,7 +755,11 @@ bool GroupedWeightQuantBatchMatmulTiling::AnalyzeAttr(const gert::TilingContext
751 context->GetDynamicInputTensor(WEIGHT_IDX, 0) != nullptr &&755 context->GetDynamicInputTensor(WEIGHT_IDX, 0) != nullptr &&
752 context->GetDynamicInputTensor(WEIGHT_IDX, 0)->GetStorageShape().GetDimNum() == singleWeightDim);756 context->GetDynamicInputTensor(WEIGHT_IDX, 0)->GetStorageShape().GetDimNum() == singleWeightDim);
753 isSingleMultiSingle_ = IsMxA8W4() && isSingleX_ && !isSingleWeight_ && isSingleY_;757 isSingleMultiSingle_ = IsMxA8W4() && isSingleX_ && !isSingleWeight_ && isSingleY_;
758+ return true;
759+}
754 760 
761+bool GroupedWeightQuantBatchMatmulTiling::CheckAllParams(const gert::TilingContext *context)
762+{
755 // 参数校验763 // 参数校验
756 OP_CHECK_IF(!CheckCoreNum(context), OP_LOGE(context->GetNodeName(), "Invalid core number ratio"), return false);764 OP_CHECK_IF(!CheckCoreNum(context), OP_LOGE(context->GetNodeName(), "Invalid core number ratio"), return false);
757 OP_CHECK_IF(!CheckUnsupportDataFlow(),765 OP_CHECK_IF(!CheckUnsupportDataFlow(),
@@ -770,6 +778,22 @@ bool GroupedWeightQuantBatchMatmulTiling::AnalyzeAttr(const gert::TilingContext
770 OP_CHECK_IF(!CheckEveryTensor(context), OP_LOGE(context->GetNodeName(), "CheckEveryTensor failed."),778 OP_CHECK_IF(!CheckEveryTensor(context), OP_LOGE(context->GetNodeName(), "CheckEveryTensor failed."),
771 return false);779 return false);
772 }780 }
781+ return true;
782+}
783+ 
784+bool GroupedWeightQuantBatchMatmulTiling::AnalyzeAttr(const gert::TilingContext *context)
785+{
786+ if (!AnalyzeCompileInfoAndAttrs(context)) {
787+ return false;
788+ }
789+ 
790+ if (!AnalyzeTensorLayout(context)) {
791+ return false;
792+ }
793+ 
794+ if (!CheckAllParams(context)) {
795+ return false;
796+ }
773 797 
774 // 参数设置798 // 参数设置
775 OP_CHECK_IF(!SetShapeList(context), OP_LOGE(context->GetNodeName(), "SetShapeList failed."), return false);799 OP_CHECK_IF(!SetShapeList(context), OP_LOGE(context->GetNodeName(), "SetShapeList failed."), return false);
@@ -1265,8 +1289,8 @@ bool GroupedWeightQuantBatchMatmulTiling::CheckEmptyTensorSingleXMultiWeightSing
1265 int64_t n = transB_ ? wShape.GetDim(wDimNum - 2) : wShape.GetDim(wDimNum - 1);1289 int64_t n = transB_ ? wShape.GetDim(wDimNum - 2) : wShape.GetDim(wDimNum - 1);
1266 if (weightNzFlag_) {1290 if (weightNzFlag_) {
1267 // 非转置NZ排布(N1, K1, K0, N0), 转置NZ排布(K1, N1, N0, K0)1291 // 非转置NZ排布(N1, K1, K0, N0), 转置NZ排布(K1, N1, N0, K0)
1268- n = transB_ ? wShape.GetDim(wDimNum - 3) * wShape.GetDim(wDimNum - 2) :1292+ n = transB_ ? wShape.GetDim(wDimNum - 3) * wShape.GetDim(wDimNum - 2) : //wDimNum = 4，代表4维NZ，数字各自代表倒数第几维
1269- wShape.GetDim(wDimNum - 4) * wShape.GetDim(wDimNum - 1);1293+ wShape.GetDim(wDimNum - 4) * wShape.GetDim(wDimNum - 1); //wDimNum = 4，代表4维NZ，数字各自代表倒数第几维
1270 }1294 }
1271 // k不能单独为01295 // k不能单独为0
1272 OP_CHECK_IF(1296 OP_CHECK_IF(
@@ -1344,8 +1368,7 @@ bool GroupedWeightQuantBatchMatmulTiling::SetShapeListSplitMSingleXSingleWeightS
1344 // -3含义：转置N1索引；-2含义：转置N0索引1368 // -3含义：转置N1索引；-2含义：转置N0索引
1345 nSize_ = transB_ ? wShape.GetDim(wDimNum - 3) * wShape.GetDim(wDimNum - 2)1369 nSize_ = transB_ ? wShape.GetDim(wDimNum - 3) * wShape.GetDim(wDimNum - 2)
1346 // -4含义：非转置N1索引；-1含义：非转置N0索引1370 // -4含义：非转置N1索引；-1含义：非转置N0索引
1347- :1371+ :wShape.GetDim(wDimNum - 4) * wShape.GetDim(wDimNum - 1);
1348- wShape.GetDim(wDimNum - 4) * wShape.GetDim(wDimNum - 1);
1349 const gert::StorageShape *yShapePtr = context->GetOutputShape(0);1372 const gert::StorageShape *yShapePtr = context->GetOutputShape(0);
1350 OP_CHECK_IF(yShapePtr == nullptr, OP_LOGE(context->GetNodeName(), "yShapePtr is nullptr."), return false);1373 OP_CHECK_IF(yShapePtr == nullptr, OP_LOGE(context->GetNodeName(), "yShapePtr is nullptr."), return false);
1351 const gert::Shape &yShape = yShapePtr->GetOriginShape();1374 const gert::Shape &yShape = yShapePtr->GetOriginShape();
@@ -1415,8 +1438,7 @@ bool GroupedWeightQuantBatchMatmulTiling::SetShapeListSplitMSingleXMultiWeightSi
1415 // -3含义：转置N1索引；-2含义：转置N0索引1438 // -3含义：转置N1索引；-2含义：转置N0索引
1416 nSize = transB_ ? wShape.GetDim(wDimNum - 3) * wShape.GetDim(wDimNum - 2)1439 nSize = transB_ ? wShape.GetDim(wDimNum - 3) * wShape.GetDim(wDimNum - 2)
1417 // -4含义：非转置N1索引；-1含义：非转置N0索引1440 // -4含义：非转置N1索引；-1含义：非转置N0索引
1418- :1441+ :wShape.GetDim(wDimNum - 4) * wShape.GetDim(wDimNum - 1);
1419- wShape.GetDim(wDimNum - 4) * wShape.GetDim(wDimNum - 1);
1420 }1442 }
1421 if (isFp4PackType && !transB_) {1443 if (isFp4PackType && !transB_) {
1422 // 一个float32/int32表示8个fp4/int4，设置为正确shape；kSize来自x，不需要考虑转置场景k轴扩大1444 // 一个float32/int32表示8个fp4/int4，设置为正确shape；kSize来自x，不需要考虑转置场景k轴扩大
@@ -1550,7 +1572,7 @@ bool GroupedWeightQuantBatchMatmulTiling::DeriveGroupSizeSingle(const gert::Tili
1550 } else {1572 } else {
1551 // (g, K, N) format: K axis index depends on transB1573 // (g, K, N) format: K axis index depends on transB
1552 groupNum = transB_ ? antiquantScaleShape.GetDim(antiquantScaleDimNum - 1) :1574 groupNum = transB_ ? antiquantScaleShape.GetDim(antiquantScaleDimNum - 1) :
1553- antiquantScaleShape.GetDim(antiquantScaleDimNum - 2);1575+ antiquantScaleShape.GetDim(antiquantScaleDimNum - 2); //- 2代表倒数第二个维度，目的是在是否转置时取得N维度
1554 }1576 }
1555 } else if (antiquantScaleDimNum == ANTIQUANT_SCALE_DIM_NUM) {1577 } else if (antiquantScaleDimNum == ANTIQUANT_SCALE_DIM_NUM) {
1556 // antiquantScaleShape: (g,n,k/64,2) or (g,k/64,n,2)1578 // antiquantScaleShape: (g,n,k/64,2) or (g,k/64,n,2)
@@ -1927,7 +1949,7 @@ ge::graphStatus GroupedS8S4BasicApiTiling::GetShapeAttrsInfo()
1927 return GroupedQmmTiling::GetShapeAttrsInfo();1949 return GroupedQmmTiling::GetShapeAttrsInfo();
1928}1950}
1929 1951 
1930-bool GroupedS8S4BasicApiTiling::AnalyzeDtype()1952+bool GroupedS8S4BasicApiTiling::AnalyzeInputDtypes()
1931{1953{
1932 auto xDesc = context_->GetDynamicInputDesc(X_INDEX, 0);1954 auto xDesc = context_->GetDynamicInputDesc(X_INDEX, 0);
1933 auto weightDesc = context_->GetDynamicInputDesc(WEIGHT_INDEX, 0);1955 auto weightDesc = context_->GetDynamicInputDesc(WEIGHT_INDEX, 0);
@@ -1953,7 +1975,11 @@ bool GroupedS8S4BasicApiTiling::AnalyzeDtype()
1953 "S8S4 BasicApi expects weight format ND, NCL, NCHW or FRACTAL_NZ."),1975 "S8S4 BasicApi expects weight format ND, NCL, NCHW or FRACTAL_NZ."),
1954 return false);1976 return false);
1955 inputParams_.bFormat = weightFormat == ge::FORMAT_FRACTAL_NZ ? ge::FORMAT_FRACTAL_NZ : ge::FORMAT_ND;1977 inputParams_.bFormat = weightFormat == ge::FORMAT_FRACTAL_NZ ? ge::FORMAT_FRACTAL_NZ : ge::FORMAT_ND;
1978+ return true;
1979+}
1956 1980 
1981+bool GroupedS8S4BasicApiTiling::CheckDtypeAndFormat()
1982+{
1957 OP_CHECK_IF(inputParams_.aDtype != ge::DT_INT8 ||1983 OP_CHECK_IF(inputParams_.aDtype != ge::DT_INT8 ||
1958 (inputParams_.bDtype != ge::DT_INT4 && inputParams_.bDtype != ge::DT_INT32),1984 (inputParams_.bDtype != ge::DT_INT4 && inputParams_.bDtype != ge::DT_INT32),
1959 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects x INT8 and weight INT4/packed INT32."),1985 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects x INT8 and weight INT4/packed INT32."),
@@ -1967,6 +1993,11 @@ bool GroupedS8S4BasicApiTiling::AnalyzeDtype()
1967 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects out BF16 or FLOAT16."), return false);1993 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects out BF16 or FLOAT16."), return false);
1968 OP_CHECK_IF(inputParams_.aFormat != ge::FORMAT_ND,1994 OP_CHECK_IF(inputParams_.aFormat != ge::FORMAT_ND,
1969 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects x in ND format."), return false);1995 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects x in ND format."), return false);
1996+ return true;
1997+}
1998+ 
1999+bool GroupedS8S4BasicApiTiling::AnalyzeBiasAndOffset()
2000+{
1970 auto biasShape = context_->GetDynamicInputShape(BIAS_INDEX, 0);2001 auto biasShape = context_->GetDynamicInputShape(BIAS_INDEX, 0);
1971 inputParams_.hasBias = biasShape != nullptr && biasShape->GetStorageShape().GetShapeSize() != 0;2002 inputParams_.hasBias = biasShape != nullptr && biasShape->GetStorageShape().GetShapeSize() != 0;
1972 auto biasDesc = context_->GetDynamicInputDesc(BIAS_INDEX, 0);2003 auto biasDesc = context_->GetDynamicInputDesc(BIAS_INDEX, 0);
@@ -1990,6 +2021,21 @@ bool GroupedS8S4BasicApiTiling::AnalyzeDtype()
1990 return true;2021 return true;
1991}2022}
1992 2023 
2024+bool GroupedS8S4BasicApiTiling::AnalyzeDtype()
2025+{
2026+ if (!AnalyzeInputDtypes()) {
2027+ return false;
2028+ }
2029+ 
2030+ if (!CheckDtypeAndFormat()) {
2031+ return false;
2032+ }
2033+ if (!AnalyzeBiasAndOffset()) {
2034+ return false;
2035+ }
2036+ return true;
2037+}
2038+ 
1993bool GroupedS8S4BasicApiTiling::AnalyzeAttrs()2039bool GroupedS8S4BasicApiTiling::AnalyzeAttrs()
1994{2040{
1995 auto attrs = context_->GetAttrs();2041 auto attrs = context_->GetAttrs();
@@ -2000,7 +2046,7 @@ bool GroupedS8S4BasicApiTiling::AnalyzeAttrs()
2000 const int64_t *groupTypePtr = attrs->GetAttrPointer<int64_t>(ATTR_INDEX_GROUPTYPE);2046 const int64_t *groupTypePtr = attrs->GetAttrPointer<int64_t>(ATTR_INDEX_GROUPTYPE);
2001 const int64_t *groupListTypePtr = attrs->GetAttrPointer<int64_t>(ATTR_INDEX_GROUP_LIST_TYPE);2047 const int64_t *groupListTypePtr = attrs->GetAttrPointer<int64_t>(ATTR_INDEX_GROUP_LIST_TYPE);
2002 const int64_t *actTypePtr = attrs->GetAttrPointer<int64_t>(ATTR_INDEX_ACT_TYPE);2048 const int64_t *actTypePtr = attrs->GetAttrPointer<int64_t>(ATTR_INDEX_ACT_TYPE);
2003- const auto tuningConfigPtr = attrs->GetAttrPointer<gert::ContinuousVector>(ATTR_INDEX_TUNING_CONFIG);2049+ const auto tuningConfigPtr = attrs->GetAttrPointer<gert::TypedContinuousVector<int64_t>>(ATTR_INDEX_TUNING_CONFIG);
2004 2050 
2005 inputParams_.splitItem = splitItemPtr != nullptr ? static_cast<int8_t>(*splitItemPtr) : inputParams_.splitItem;2051 inputParams_.splitItem = splitItemPtr != nullptr ? static_cast<int8_t>(*splitItemPtr) : inputParams_.splitItem;
2006 inputParams_.transB = transposeWeightPtr != nullptr ? *transposeWeightPtr : false;2052 inputParams_.transB = transposeWeightPtr != nullptr ? *transposeWeightPtr : false;
@@ -2015,7 +2061,7 @@ bool GroupedS8S4BasicApiTiling::AnalyzeAttrs()
2015 OP_CHECK_IF(inputParams_.splitItem != X_SEPARATED && inputParams_.splitItem != NO_SEPARATED,2061 OP_CHECK_IF(inputParams_.splitItem != X_SEPARATED && inputParams_.splitItem != NO_SEPARATED,
2016 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects splitItem to be 2 or 3."), return false);2062 OP_LOGE(context_->GetNodeName(), "S8S4 BasicApi expects splitItem to be 2 or 3."), return false);
2017 if (tuningConfigPtr != nullptr && tuningConfigPtr->GetSize() > 0) {2063 if (tuningConfigPtr != nullptr && tuningConfigPtr->GetSize() > 0) {
2018- const auto tuningConfig = reinterpret_cast<const int64_t *>(tuningConfigPtr->GetData());2064+ const auto tuningConfig = tuningConfigPtr->GetData();
2019 OP_CHECK_IF(tuningConfig[TUNING_CONFIG_EXPECTED_TOKEN_INDEX] < 0 ||2065 OP_CHECK_IF(tuningConfig[TUNING_CONFIG_EXPECTED_TOKEN_INDEX] < 0 ||
2020 static_cast<uint64_t>(tuningConfig[TUNING_CONFIG_EXPECTED_TOKEN_INDEX]) > UINT32_MAX,2066 static_cast<uint64_t>(tuningConfig[TUNING_CONFIG_EXPECTED_TOKEN_INDEX]) > UINT32_MAX,
2021 OP_LOGE(context_->GetNodeName(), "tuningConfig[0] must be in [0, UINT32_MAX]."), return false);2067 OP_LOGE(context_->GetNodeName(), "tuningConfig[0] must be in [0, UINT32_MAX]."), return false);
@@ -2150,41 +2196,46 @@ bool GroupedS8S4BasicApiTiling::CheckAntiquantInputsEmpty() const
2150 return true;2196 return true;
2151}2197}
2152 2198 
2199+bool GroupedS8S4BasicApiTiling::CheckWeightNzStorageShape(const gert::Shape &storageShape)
2200+{
2201+ // torch_npu constructs the normal A5 KN-NZ path as follows:
2202+ // INT8 NZ: [E, ceil(N/32), ceil(K/16), 16, 32]
2203+ // packed INT32: [E, ceil(N/32), ceil(K/16), 16, 4]
2204+ // The special ENK-NZ path uses the same C0 and swaps the logical K/N
2205+ // block axes: [E, ceil(K/32), ceil(N/16), 16, 32/4].
2206+ // ACLNN reinterprets the INT32 carrier as INT4 and restores the last
2207+ // storage dimension from 4 to 32 before Host tiling. Keep support for
2208+ // a direct INT32 Host invocation as well, but reject all other physical
2209+ // layouts instead of accepting a same-sized, differently blocked NZ.
2210+ const uint64_t expectedC0 = weightPackedInt32_ ? 4UL : 32UL;
2211+ const uint64_t expectedOuter0 = specialWeightFormat_ ?
2212+ CeilDiv(inputParams_.kSize, NZ_SPECIAL_K_ALIGN) :
2213+ CeilDiv(inputParams_.nSize, NZ_BASE_N_ALIGN);
2214+ const uint64_t expectedOuter1 = specialWeightFormat_ ?
2215+ CeilDiv(inputParams_.nSize, NZ_SPECIAL_N_ALIGN) :
2216+ CeilDiv(inputParams_.kSize, NZ_BASE_K_ALIGN);
2217+ const bool shapeMatches = storageShape.GetDimNum() == 5 &&
2218+ static_cast<uint64_t>(storageShape.GetDim(0)) == inputParams_.groupNum &&
2219+ static_cast<uint64_t>(storageShape.GetDim(1)) == expectedOuter0 &&
2220+ static_cast<uint64_t>(storageShape.GetDim(2)) == expectedOuter1 &&
2221+ static_cast<uint64_t>(storageShape.GetDim(3)) == NZ_BASE_K_ALIGN &&
2222+ static_cast<uint64_t>(storageShape.GetDim(4)) == expectedC0;
2223+ OP_CHECK_IF(!shapeMatches,
2224+ OP_LOGE(context_->GetNodeName(),
2225+ "A5 INT4 %s weight storage shape must be "
2226+ "[E,%s,%s,16,%lu].",
2227+ specialWeightFormat_ ? "special ENK-NZ" : "normal KN-NZ",
2228+ specialWeightFormat_ ? "ceil(K/32)" : "ceil(N/32)",
2229+ specialWeightFormat_ ? "ceil(N/16)" : "ceil(K/16)", expectedC0),
2230+ return false);
2231+ return true;
2232+}
2233+ 
2153bool GroupedS8S4BasicApiTiling::CheckWeightStorageShape(const gert::StorageShape &weightShape)2234bool GroupedS8S4BasicApiTiling::CheckWeightStorageShape(const gert::StorageShape &weightShape)
2154{2235{
2155 const gert::Shape &storageShape = weightShape.GetStorageShape();2236 const gert::Shape &storageShape = weightShape.GetStorageShape();
2156 if (inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ) {2237 if (inputParams_.bFormat == ge::FORMAT_FRACTAL_NZ) {
2157- // torch_npu constructs the normal A5 KN-NZ path as follows:2238+ return CheckWeightNzStorageShape(storageShape);
2158- // INT8 NZ: [E, ceil(N/32), ceil(K/16), 16, 32]
2159- // packed INT32: [E, ceil(N/32), ceil(K/16), 16, 4]
2160- // The special ENK-NZ path uses the same C0 and swaps the logical K/N
2161- // block axes: [E, ceil(K/32), ceil(N/16), 16, 32/4].
2162- // ACLNN reinterprets the INT32 carrier as INT4 and restores the last
2163- // storage dimension from 4 to 32 before Host tiling. Keep support for
2164- // a direct INT32 Host invocation as well, but reject all other physical
2165- // layouts instead of accepting a same-sized, differently blocked NZ.
2166- const uint64_t expectedC0 = weightPackedInt32_ ? 4UL : 32UL;
2167- const uint64_t expectedOuter0 = specialWeightFormat_ ?
2168- CeilDiv(inputParams_.kSize, NZ_SPECIAL_K_ALIGN) :
2169- CeilDiv(inputParams_.nSize, NZ_BASE_N_ALIGN);
2170- const uint64_t expectedOuter1 = specialWeightFormat_ ?
2171- CeilDiv(inputParams_.nSize, NZ_SPECIAL_N_ALIGN) :
2172- CeilDiv(inputParams_.kSize, NZ_BASE_K_ALIGN);
2173- const bool shapeMatches = storageShape.GetDimNum() == 5 &&
2174- static_cast<uint64_t>(storageShape.GetDim(0)) == inputParams_.groupNum &&
2175- static_cast<uint64_t>(storageShape.GetDim(1)) == expectedOuter0 &&
2176- static_cast<uint64_t>(storageShape.GetDim(2)) == expectedOuter1 &&
2177- static_cast<uint64_t>(storageShape.GetDim(3)) == NZ_BASE_K_ALIGN &&
2178- static_cast<uint64_t>(storageShape.GetDim(4)) == expectedC0;
2179- OP_CHECK_IF(!shapeMatches,
2180- OP_LOGE(context_->GetNodeName(),
2181- "A5 INT4 %s weight storage shape must be "
2182- "[E,%s,%s,16,%lu].",
2183- specialWeightFormat_ ? "special ENK-NZ" : "normal KN-NZ",
2184- specialWeightFormat_ ? "ceil(K/32)" : "ceil(N/32)",
2185- specialWeightFormat_ ? "ceil(N/16)" : "ceil(K/16)", expectedC0),
2186- return false);
2187- return true;
2188 }2239 }
2189 2240 
2190 uint64_t scalarSlotNum = inputParams_.groupNum * inputParams_.kSize * inputParams_.nSize;2241 uint64_t scalarSlotNum = inputParams_.groupNum * inputParams_.kSize * inputParams_.nSize;
@@ -2384,18 +2435,8 @@ void GroupedS8S4BasicApiTiling::SetBasicBlock()
2384 basicTiling_.baseK = S8S4_BASE_K;2435 basicTiling_.baseK = S8S4_BASE_K;
2385}2436}
2386 2437 
2387-ge::graphStatus GroupedS8S4BasicApiTiling::DoLibApiTiling()2438+void GroupedS8S4BasicApiTiling::SetMmTilingData(uint64_t kL1)
2388{2439{
2389- SetBasicBlock();
2390- uint64_t kL1 = dequantMode_ == DequantMode::SYMMETRIC_PER_GROUP ?
2391- S8S4_PER_GROUP_K_L1 :
2392- S8S4_PER_CHANNEL_K_L1;
2393- const uint64_t alignedK = CeilAlign(inputParams_.kSize, static_cast<uint64_t>(QUANT_GROUP_SIZE));
2394- kL1 = std::min(kL1, alignedK);
2395- OP_CHECK_IF(ValidateFixedTileResources(kL1) != ge::GRAPH_SUCCESS,
2396- OP_LOGE(context_->GetNodeName(), "S8S4 fixed tile resource validation failed."),
2397- return ge::GRAPH_FAILED);
2398- 
2399 tilingData_.mmTilingData.m = static_cast<uint32_t>(inputParams_.mSize);2440 tilingData_.mmTilingData.m = static_cast<uint32_t>(inputParams_.mSize);
2400 tilingData_.mmTilingData.n = static_cast<uint32_t>(inputParams_.nSize);2441 tilingData_.mmTilingData.n = static_cast<uint32_t>(inputParams_.nSize);
2401 tilingData_.mmTilingData.k = static_cast<uint32_t>(inputParams_.kSize);2442 tilingData_.mmTilingData.k = static_cast<uint32_t>(inputParams_.kSize);
@@ -2412,7 +2453,10 @@ ge::graphStatus GroupedS8S4BasicApiTiling::DoLibApiTiling()
2412 const uint64_t l0cTileBytes = basicTiling_.baseM * basicTiling_.baseN * sizeof(int32_t);2453 const uint64_t l0cTileBytes = basicTiling_.baseM * basicTiling_.baseN * sizeof(int32_t);
2413 tilingData_.mmTilingData.dbL0C =2454 tilingData_.mmTilingData.dbL0C =
2414 l0cTileBytes * 2UL <= aicoreParams_.l0cSize ? 2U : 1U;2455 l0cTileBytes * 2UL <= aicoreParams_.l0cSize ? 2U : 1U;
2456+}
2415 2457 
2458+ge::graphStatus GroupedS8S4BasicApiTiling::ComputeWorkspaceLayout()
2459+{
2416 {2460 {
2417 constexpr uint64_t WORKSPACE_ALIGN = 512UL;2461 constexpr uint64_t WORKSPACE_ALIGN = 512UL;
2418 constexpr uint64_t NZ_K_ALIGN = 16UL;2462 constexpr uint64_t NZ_K_ALIGN = 16UL;
@@ -2432,7 +2476,7 @@ ge::graphStatus GroupedS8S4BasicApiTiling::DoLibApiTiling()
2432 result = lhs + rhs;2476 result = lhs + rhs;
2433 return true;2477 return true;
2434 };2478 };
2435- auto checkedAlign = [&](uint64_t value, uint64_t &result) {2479+ auto checkedAlign = [](uint64_t value, uint64_t &result) {
2436 if (value > UINT64_MAX_VALUE - (WORKSPACE_ALIGN - 1UL)) {2480 if (value > UINT64_MAX_VALUE - (WORKSPACE_ALIGN - 1UL)) {
2437 return false;2481 return false;
2438 }2482 }
@@ -2495,6 +2539,22 @@ ge::graphStatus GroupedS8S4BasicApiTiling::DoLibApiTiling()
2495 return ge::GRAPH_SUCCESS;2539 return ge::GRAPH_SUCCESS;
2496}2540}
2497 2541 
2542+ge::graphStatus GroupedS8S4BasicApiTiling::DoLibApiTiling()
2543+{
2544+ SetBasicBlock();
2545+ uint64_t kL1 = dequantMode_ == DequantMode::SYMMETRIC_PER_GROUP ?
2546+ S8S4_PER_GROUP_K_L1 :
2547+ S8S4_PER_CHANNEL_K_L1;
2548+ const uint64_t alignedK = CeilAlign(inputParams_.kSize, static_cast<uint64_t>(QUANT_GROUP_SIZE));
2549+ kL1 = std::min(kL1, alignedK);
2550+ OP_CHECK_IF(ValidateFixedTileResources(kL1) != ge::GRAPH_SUCCESS,
2551+ OP_LOGE(context_->GetNodeName(), "S8S4 fixed tile resource validation failed."),
2552+ return ge::GRAPH_FAILED);
2553+ 
2554+ SetMmTilingData(kL1);
2555+ return ComputeWorkspaceLayout();
2556+}
2557+ 
2498ge::graphStatus GroupedS8S4BasicApiTiling::PostTiling()2558ge::graphStatus GroupedS8S4BasicApiTiling::PostTiling()
2499{2559{
2500 // Weight preprocessing synchronizes all vector cores, so all participating cores must be resident.2560 // Weight preprocessing synchronizes all vector cores, so all participating cores must be resident.
Mgmm/grouped_matmul/op_host/op_tiling/arch35/grouped_weight_quant_batch_matmul_tiling.h+9-0
@@ -270,10 +270,16 @@ private:
270 bool CheckPerTokenScaleShape() const;270 bool CheckPerTokenScaleShape() const;
271 bool CheckAntiquantInputsEmpty() const;271 bool CheckAntiquantInputsEmpty() const;
272 bool CheckWeightStorageShape(const gert::StorageShape &weightShape);272 bool CheckWeightStorageShape(const gert::StorageShape &weightShape);
273+ bool CheckWeightNzStorageShape(const gert::Shape &storageShape);
273 bool CheckScaleAndOffsetShapes();274 bool CheckScaleAndOffsetShapes();
274 bool SetLogicalMKN(const gert::Shape &xShape, const gert::Shape &wShape);275 bool SetLogicalMKN(const gert::Shape &xShape, const gert::Shape &wShape);
275 void SetBasicBlock();276 void SetBasicBlock();
276 ge::graphStatus ValidateFixedTileResources(uint64_t kL1) const;277 ge::graphStatus ValidateFixedTileResources(uint64_t kL1) const;
278+ void SetMmTilingData(uint64_t kL1);
279+ ge::graphStatus ComputeWorkspaceLayout();
280+ bool AnalyzeInputDtypes();
281+ bool CheckDtypeAndFormat();
282+ bool AnalyzeBiasAndOffset();
277 283 
278 static constexpr uint32_t QUANT_GROUP_SIZE = 256U;284 static constexpr uint32_t QUANT_GROUP_SIZE = 256U;
279 static constexpr uint32_t S8S4_BASE_M = 256U;285 static constexpr uint32_t S8S4_BASE_M = 256U;
@@ -343,6 +349,9 @@ protected:
343 bool CheckEveryTensorSingleXMultiWeightSingleY(const gert::TilingContext *context) const;349 bool CheckEveryTensorSingleXMultiWeightSingleY(const gert::TilingContext *context) const;
344 bool CheckGroupList(const gert::TilingContext *context) const;350 bool CheckGroupList(const gert::TilingContext *context) const;
345 bool AnalyzeAttr(const gert::TilingContext *context);351 bool AnalyzeAttr(const gert::TilingContext *context);
352+ bool AnalyzeCompileInfoAndAttrs(const gert::TilingContext *context);
353+ bool AnalyzeTensorLayout(const gert::TilingContext *context);
354+ bool CheckAllParams(const gert::TilingContext *context);
346 bool AnalyzeInput(const gert::TilingContext *context);355 bool AnalyzeInput(const gert::TilingContext *context);
347 bool CalcResplitTiling(const gert::TilingContext *context);356 bool CalcResplitTiling(const gert::TilingContext *context);
348 bool SetBaseTiling();357 bool SetBaseTiling();
Mgmm/grouped_matmul_finalize_routing/op_api/aclnn_grouped_matmul_finalize_routing.cpp+55-40
@@ -1358,6 +1358,53 @@ aclnnStatus aclnnGroupedMatmulFinalizeRouting(void *workspace, uint64_t workspac
1358 return CommonOpExecutorRun(workspace, workspaceSize, executor, stream);1358 return CommonOpExecutorRun(workspace, workspaceSize, executor, stream);
1359}1359}
1360 1360 
1361+static aclnnStatus ValidateV2Inputs(const aclTensor *x1, const aclTensor *x2,
1362+ const aclTensor *scaleOptional, const aclTensor *biasOptional,
1363+ const aclTensor *pertokenScaleOptional, const aclTensor *groupListOptional,
1364+ const aclTensor *logitOptional, const aclTensor *rowIndexOptional,
1365+ const aclTensor *antiquantScaleOptional, const aclTensor *antiquantOffsetOptional,
1366+ int64_t dtype, bool transposeX1, bool transposeX2)
1367+{
1368+ auto scene = x1 != nullptr && x2 != nullptr && scaleOptional != nullptr && pertokenScaleOptional != nullptr &&
1369+ groupListOptional != nullptr && logitOptional != nullptr && rowIndexOptional != nullptr &&
1370+ biasOptional != nullptr && antiquantScaleOptional == nullptr && antiquantOffsetOptional == nullptr;
1371+ if (!scene) {
1372+ OP_LOGE(ACLNN_ERR_PARAM_NULLPTR, "GroupedMatmulFinalizeRoutingV2 weightNd do not support input nullptr.");
1373+ return ACLNN_ERR_PARAM_NULLPTR;
1374+ }
1375+ int64_t viewDimNum = x2->GetViewShape().GetDimNum();
1376+ if (dtype != 0) {
1377+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "GroupedMatmulFinalizeRoutingV2 weightNd dtype must be 0, but is %lld.",
1378+ dtype);
1379+ return ACLNN_ERR_PARAM_INVALID;
1380+ } else if (viewDimNum < MIN_DIM_NUM_ND) {
1381+ OP_LOGE(ACLNN_ERR_PARAM_INVALID,
1382+ "GroupedMatmulFinalizeRoutingV2 weightNd x2's view dimNum should greater than 1, but is %lld.",
1383+ viewDimNum);
1384+ return ACLNN_ERR_PARAM_INVALID;
1385+ } else if (!(transposeX1 == false && transposeX2 == false)) {
1386+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "GroupedMatmulFinalizeRoutingV2 weightNd transpose should be false");
1387+ return ACLNN_ERR_PARAM_INVALID;
1388+ }
1389+ return ACLNN_SUCCESS;
1390+}
1391+ 
1392+static aclnnStatus UnpackWeightInt32ToInt4(aclTensor *&weight, const char *opName)
1393+{
1394+ if (weight->GetDataType() != DataType::DT_INT32) {
1395+ OP_LOGE(ACLNN_ERR_PARAM_INVALID,
1396+ "%s weightNd weight type should be DT_INT32, but now is %s",
1397+ opName, op::ToString(weight->GetDataType()).GetString());
1398+ return ACLNN_ERR_PARAM_INVALID;
1399+ }
1400+ op::Shape weightShape = weight->GetViewShape();
1401+ auto viewShapeDim = weightShape.GetDimNum();
1402+ weightShape[viewShapeDim - 1] = weightShape[viewShapeDim - 1] * PER_INT4_IN_U32;
1403+ weight->SetViewShape(weightShape);
1404+ weight->SetDataType(DataType::DT_INT4);
1405+ return ACLNN_SUCCESS;
1406+}
1407+ 
1361aclnnStatus aclnnGroupedMatmulFinalizeRoutingV2GetWorkspaceSize(1408aclnnStatus aclnnGroupedMatmulFinalizeRoutingV2GetWorkspaceSize(
1362 const aclTensor *x1, aclTensor *x2, const aclTensor *scaleOptional, const aclTensor *biasOptional,1409 const aclTensor *x1, aclTensor *x2, const aclTensor *scaleOptional, const aclTensor *biasOptional,
1363 const aclTensor *offsetOptional, const aclTensor *antiquantScaleOptional, const aclTensor *antiquantOffsetOptional,1410 const aclTensor *offsetOptional, const aclTensor *antiquantScaleOptional, const aclTensor *antiquantOffsetOptional,
@@ -1373,44 +1420,12 @@ aclnnStatus aclnnGroupedMatmulFinalizeRoutingV2GetWorkspaceSize(
1373 sharedInputOptional, logitOptional, rowIndexOptional, dtype, sharedInputWeight,1420 sharedInputOptional, logitOptional, rowIndexOptional, dtype, sharedInputWeight,
1374 sharedInputOffset, transposeX1, transposeX2, groupListType),1421 sharedInputOffset, transposeX1, transposeX2, groupListType),
1375 DFX_OUT(out));1422 DFX_OUT(out));
1376- auto scene = x1 != nullptr && x2 != nullptr && scaleOptional != nullptr && pertokenScaleOptional != nullptr &&1423+ auto ret0 = ValidateV2Inputs(x1, x2, scaleOptional, biasOptional, pertokenScaleOptional, groupListOptional,
1377- groupListOptional != nullptr && logitOptional != nullptr && rowIndexOptional != nullptr &&1424+ logitOptional, rowIndexOptional, antiquantScaleOptional, antiquantOffsetOptional,
1378- biasOptional != nullptr && antiquantScaleOptional == nullptr && antiquantOffsetOptional == nullptr;1425+ dtype, transposeX1, transposeX2);
1379- if (!scene) {1426+ CHECK_RET(ret0 == ACLNN_SUCCESS, ret0);
1380- OP_LOGE(ACLNN_ERR_PARAM_NULLPTR, "GroupedMatmulFinalizeRoutingV2 weightNd do not support input nullptr.");1427+ auto ret1 = UnpackWeightInt32ToInt4(x2, "GroupedMatmulFinalizeRoutingV2");
1381- return ACLNN_ERR_PARAM_NULLPTR;1428+ CHECK_RET(ret1 == ACLNN_SUCCESS, ret1);
1382- }
1383- 
1384- int64_t viewDimNum = x2->GetViewShape().GetDimNum();
1385- if (dtype != 0) {
1386- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "GroupedMatmulFinalizeRoutingV2 weightNd dtype must be 0, but is %lld.",
1387- dtype);
1388- return ACLNN_ERR_PARAM_INVALID;
1389- } else if (viewDimNum < MIN_DIM_NUM_ND) {
1390- OP_LOGE(ACLNN_ERR_PARAM_INVALID,
1391- "GroupedMatmulFinalizeRoutingV2 weightNd x2's view dimNum should greater than 1, but is %lld.",
1392- viewDimNum);
1393- return ACLNN_ERR_PARAM_INVALID;
1394- } else if (!(transposeX1 == false && transposeX2 == false)) {
1395- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "GroupedMatmulFinalizeRoutingV2 weightNd transpose should be false");
1396- return ACLNN_ERR_PARAM_INVALID;
1397- }
1398- 
1399- // unpack int32 to int4
1400- auto tmpWeightV2 = x2;
1401- if (tmpWeightV2->GetDataType() == DataType::DT_INT32) {
1402- op::Shape weightShapeV2 = tmpWeightV2->GetViewShape();
1403- auto viewShapeDimV2 = weightShapeV2.GetDimNum();
1404- weightShapeV2[viewShapeDimV2 - 1] = weightShapeV2[viewShapeDimV2 - 1] * PER_INT4_IN_U32;
1405- tmpWeightV2->SetViewShape(weightShapeV2);
1406- tmpWeightV2->SetDataType(DataType::DT_INT4);
1407- } else {
1408- OP_LOGE(ACLNN_ERR_PARAM_INVALID,
1409- "GroupedMatmulFinalizeRoutingV2 weightNd weight type should be DT_INT32, but now is %s",
1410- op::ToString(tmpWeightV2->GetDataType()).GetString());
1411- return ACLNN_ERR_PARAM_INVALID;
1412- }
1413- 
1414 auto uniqueExecutor = CREATE_EXECUTOR();1429 auto uniqueExecutor = CREATE_EXECUTOR();
1415 CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR);1430 CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR);
1416 const aclIntArray *unusedTuningConfig = nullptr;1431 const aclIntArray *unusedTuningConfig = nullptr;
@@ -1427,8 +1442,8 @@ aclnnStatus aclnnGroupedMatmulFinalizeRoutingV2GetWorkspaceSize(
1427 .SetNumbers(sharedInputWeight, sharedInputOffset, groupListType)1442 .SetNumbers(sharedInputWeight, sharedInputOffset, groupListType)
1428 .SetTranspose(transposeX1, transposeX2)1443 .SetTranspose(transposeX1, transposeX2)
1429 .Build();1444 .Build();
1430- auto ret1 = aclnnGroupedMatmulFinalizeRoutingGetWorkspaceSizeCommonProcess(params, uniqueExecutor.get());1445+ auto ret = aclnnGroupedMatmulFinalizeRoutingGetWorkspaceSizeCommonProcess(params, uniqueExecutor.get());
1431- CHECK_RET(ret1 == ACLNN_SUCCESS, ret1);1446+ CHECK_RET(ret == ACLNN_SUCCESS, ret);
1432 *workspaceSize = uniqueExecutor->GetWorkspaceSize();1447 *workspaceSize = uniqueExecutor->GetWorkspaceSize();
1433 uniqueExecutor.ReleaseTo(executor);1448 uniqueExecutor.ReleaseTo(executor);
1434 return ACLNN_SUCCESS;1449 return ACLNN_SUCCESS;
Mgmm/grouped_matmul_swiglu_quant_v2/op_api/aclnn_grouped_matmul_swiglu_quant_v2.cpp+2-2
@@ -200,7 +200,7 @@ static aclnnStatus ProcessSingleWeightNz(const aclTensorList *weight)
200 auto w = (*weight)[0];200 auto w = (*weight)[0];
201 auto storgeShape = w->GetStorageShape();201 auto storgeShape = w->GetStorageShape();
202 auto viewShape = w->GetViewShape();202 auto viewShape = w->GetViewShape();
203- aclTensor *weightNZ = const_cast<aclTensor *>(w);203+ aclTensor *weightNZ = const_cast<aclTensor *>(w); // NOLINT: SetStorageFormat/SetViewFormat are non-const but do not alter tensor data
204 std::ostringstream gotShape;204 std::ostringstream gotShape;
205 gotShape << op::ToString(storgeShape).GetString() << " with dim num " << storgeShape.GetDimNum();205 gotShape << op::ToString(storgeShape).GetString() << " with dim num " << storgeShape.GetDimNum();
206 std::string gotShapeStr = gotShape.str();206 std::string gotShapeStr = gotShape.str();
@@ -227,7 +227,7 @@ static aclnnStatus ProcessMultiWeightNz(const aclTensorList *weight)
227 CHECK_RET(w != nullptr, ACLNN_ERR_PARAM_NULLPTR);227 CHECK_RET(w != nullptr, ACLNN_ERR_PARAM_NULLPTR);
228 auto storgeShape = w->GetStorageShape();228 auto storgeShape = w->GetStorageShape();
229 auto viewShape = w->GetViewShape();229 auto viewShape = w->GetViewShape();
230- aclTensor *weightNZ = const_cast<aclTensor *>(w);230+ aclTensor *weightNZ = const_cast<aclTensor *>(w); // NOLINT: SetStorageFormat/SetViewFormat are non-const but do not alter tensor data
231 std::ostringstream gotShape;231 std::ostringstream gotShape;
232 gotShape << op::ToString(storgeShape).GetString() << " with dim num " << storgeShape.GetDimNum();232 gotShape << op::ToString(storgeShape).GetString() << " with dim num " << storgeShape.GetDimNum();
233 std::string gotShapeStr = gotShape.str();233 std::string gotShapeStr = gotShape.str();
Mgmm/grouped_matmul_swiglu_quant_v2/op_api/gmm_dsq_base.h+61-30文件内容可能有违规信息
Mgmm/grouped_matmul_swiglu_quant_v2/op_api/grouped_matmul_swiglu_quant_v2_utils.h+38-15
@@ -555,7 +555,7 @@ protected:
555 shape.SetScalar();555 shape.SetScalar();
556 shape.AppendDim(viewShape.GetDim(1));556 shape.AppendDim(viewShape.GetDim(1));
557 shape.AppendDim(viewShape.GetDim(0));557 shape.AppendDim(viewShape.GetDim(0));
558- shape.AppendDim(viewShape.GetDim(2));558+ shape.AppendDim(viewShape.GetDim(2)); //第三维不交换
559 aclTensor *tensor = executor->CreateView(inputTensor, shape, inputTensor->GetViewOffset());559 aclTensor *tensor = executor->CreateView(inputTensor, shape, inputTensor->GetViewOffset());
560 tensor->SetStorageFormat(inputTensor->GetStorageFormat());560 tensor->SetStorageFormat(inputTensor->GetStorageFormat());
561 tensor->SetStorageShape(storageShape);561 tensor->SetStorageShape(storageShape);
@@ -966,7 +966,7 @@ protected:
966 // 1：检查K是否为偶数966 // 1：检查K是否为偶数
967 int64_t kModValue = kValue % MXFP4_K_CONSTRAINT;967 int64_t kModValue = kValue % MXFP4_K_CONSTRAINT;
968 // 2：检查N是否为偶数968 // 2：检查N是否为偶数
969- int64_t nModValue = nValue % MXFP4_N_CONSTRAINT;969+ //int64_t nModValue = nValue % MXFP4_N_CONSTRAINT;
970 if (kModValue != 0) {970 if (kModValue != 0) {
971 std::string gotStr = BuildLogValue("K", kValue);971 std::string gotStr = BuildLogValue("K", kValue);
972 OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(972 OP_LOGE_FOR_INVALID_SHAPES_WITH_REASON(
@@ -1039,7 +1039,7 @@ protected:
1039 1039 
1040 bool CheckMxA8W4InputShape()1040 bool CheckMxA8W4InputShape()
1041 {1041 {
1042- int64_t m = gmmDsqParams_.x->GetViewShape().GetDim(0);1042+ //int64_t m = gmmDsqParams_.x->GetViewShape().GetDim(0);
1043 int64_t k = gmmDsqParams_.x->GetViewShape().GetDim(1);1043 int64_t k = gmmDsqParams_.x->GetViewShape().GetDim(1);
1044 // 多tensor下shape:(n,k), 单tensor下shape:(e, n, k)1044 // 多tensor下shape:(n,k), 单tensor下shape:(e, n, k)
1045 int64_t n = IsMultiTensorWeight() ? ((*gmmDsqParams_.weight)[0])->GetViewShape().GetDim(0) :1045 int64_t n = IsMultiTensorWeight() ? ((*gmmDsqParams_.weight)[0])->GetViewShape().GetDim(0) :
@@ -1291,19 +1291,9 @@ protected:
1291 return true;1291 return true;
1292 }1292 }
1293 1293 
1294- bool CheckDtypeValid() override1294+ bool CheckNdE1M2Restriction(DataType xDtype, DataType weightDtype, const aclTensor *weight,
1295+ const aclTensor *output)
1295 {1296 {
1296- DataType xDtype = gmmDsqParams_.x->GetDataType();
1297- DataType weightDtype = ((*gmmDsqParams_.weight)[0])->GetDataType();
1298- const aclTensor *x = gmmDsqParams_.x;
1299- const aclTensor *weight = (*gmmDsqParams_.weight)[0];
1300- const aclTensor *xScale = gmmDsqParams_.xScale;
1301- const aclTensor *groupList = gmmDsqParams_.groupList;
1302- const aclTensor *output = gmmDsqParams_.output;
1303- const aclTensor *outputScale = gmmDsqParams_.outputScale;
1304- if (!CheckWeightFormatConsistency()) {
1305- return false;
1306- }
1307 if (weight->GetStorageFormat() == ge::FORMAT_ND &&1297 if (weight->GetStorageFormat() == ge::FORMAT_ND &&
1308 (weightDtype == DataType::DT_FLOAT4_E1M2 || xDtype == DataType::DT_FLOAT4_E1M2 ||1298 (weightDtype == DataType::DT_FLOAT4_E1M2 || xDtype == DataType::DT_FLOAT4_E1M2 ||
1309 output->GetDataType() == DataType::DT_FLOAT4_E1M2)) {1299 output->GetDataType() == DataType::DT_FLOAT4_E1M2)) {
@@ -1314,6 +1304,11 @@ protected:
1314 "when the format of weight is ND, the dtypes of weight, x and output can not be DT_FLOAT4_E1M2");1304 "when the format of weight is ND, the dtypes of weight, x and output can not be DT_FLOAT4_E1M2");
1315 return false;1305 return false;
1316 }1306 }
1307+ return true;
1308+ }
1309+ 
1310+ bool CheckXAndWeightDtypeSupport(DataType xDtype, DataType weightDtype, const aclTensor *weight)
1311+ {
1317 const auto &xDtypeSupportListMxfp4 = GetXSupportListMxfp4(weight);1312 const auto &xDtypeSupportListMxfp4 = GetXSupportListMxfp4(weight);
1318 const auto &weightDtypeSupportListMxfp4 = GetWeightSupportListMxfp4(weight);1313 const auto &weightDtypeSupportListMxfp4 = GetWeightSupportListMxfp4(weight);
1319 const char *dtypeSupportList = IsMxfp4WeightNzFormat(weight) ?1314 const char *dtypeSupportList = IsMxfp4WeightNzFormat(weight) ?
@@ -1343,6 +1338,12 @@ protected:
1343 dtypeSupportList);1338 dtypeSupportList);
1344 return false;1339 return false;
1345 }1340 }
1341+ return true;
1342+ }
1343+ 
1344+ bool CheckQuantModeDtype(DataType xDtype, DataType weightDtype, const aclTensor *x, const aclTensor *xScale,
1345+ const aclTensor *groupList, const aclTensor *output, const aclTensor *outputScale)
1346+ {
1346 if (gmmDsqParams_.quantMode == QUNAT_MODE_MX &&1347 if (gmmDsqParams_.quantMode == QUNAT_MODE_MX &&
1347 (xDtype == DataType::DT_FLOAT8_E4M3FN || xDtype == DataType::DT_FLOAT8_E5M2) &&1348 (xDtype == DataType::DT_FLOAT8_E4M3FN || xDtype == DataType::DT_FLOAT8_E5M2) &&
1348 (weightDtype == DataType::DT_FLOAT8_E4M3FN || weightDtype == DataType::DT_FLOAT8_E5M2)) {1349 (weightDtype == DataType::DT_FLOAT8_E4M3FN || weightDtype == DataType::DT_FLOAT8_E5M2)) {
@@ -1372,6 +1373,28 @@ protected:
1372 return true;1373 return true;
1373 }1374 }
1374 1375 
1376+ bool CheckDtypeValid() override
1377+ {
1378+ DataType xDtype = gmmDsqParams_.x->GetDataType();
1379+ DataType weightDtype = ((*gmmDsqParams_.weight)[0])->GetDataType();
1380+ const aclTensor *x = gmmDsqParams_.x;
1381+ const aclTensor *weight = (*gmmDsqParams_.weight)[0];
1382+ const aclTensor *xScale = gmmDsqParams_.xScale;
1383+ const aclTensor *groupList = gmmDsqParams_.groupList;
1384+ const aclTensor *output = gmmDsqParams_.output;
1385+ const aclTensor *outputScale = gmmDsqParams_.outputScale;
1386+ if (!CheckWeightFormatConsistency()) {
1387+ return false;
1388+ }
1389+ if (!CheckNdE1M2Restriction(xDtype, weightDtype, weight, output)) {
1390+ return false;
1391+ }
1392+ if (!CheckXAndWeightDtypeSupport(xDtype, weightDtype, weight)) {
1393+ return false;
1394+ }
1395+ return CheckQuantModeDtype(xDtype, weightDtype, x, xScale, groupList, output, outputScale);
1396+ }
1397+ 
1375 bool CheckFormat() override1398 bool CheckFormat() override
1376 {1399 {
1377 size_t wLength = gmmDsqParams_.weight->Size();1400 size_t wLength = gmmDsqParams_.weight->Size();
Mgmm/grouped_matmul_swiglu_quant_v2/op_host/op_tiling/arch35/grouped_matmul_swiglu_quant_v2_basic_api_tiling.cpp+2-2
@@ -258,14 +258,14 @@ bool GroupedMatmulSwigluQuantV2BasicApiTiling950::CheckTensorApiScaleShapes() co
258 258 
259 const int64_t kBlocks = static_cast<int64_t>(GroupedMatmul::CeilDiv(inputParams_.kSize, MX_SCALE_K_ALIGN));259 const int64_t kBlocks = static_cast<int64_t>(GroupedMatmul::CeilDiv(inputParams_.kSize, MX_SCALE_K_ALIGN));
260 if (xScaleShape.GetDim(0) != static_cast<int64_t>(inputParams_.mSize) || xScaleShape.GetDim(1) != kBlocks ||260 if (xScaleShape.GetDim(0) != static_cast<int64_t>(inputParams_.mSize) || xScaleShape.GetDim(1) != kBlocks ||
261- xScaleShape.GetDim(2) != SHAPE_DIM_TWO) {261+ xScaleShape.GetDim(2) != SHAPE_DIM_TWO) { //xScale 的最后一维（索引 2，即第 3 维）也必须等于 2
262 return false;262 return false;
263 }263 }
264 const int64_t weightScaleN = inputParams_.transB ? weightScaleShape.GetDim(1) : weightScaleShape.GetDim(2);264 const int64_t weightScaleN = inputParams_.transB ? weightScaleShape.GetDim(1) : weightScaleShape.GetDim(2);
265 const int64_t weightScaleK = inputParams_.transB ? weightScaleShape.GetDim(2) : weightScaleShape.GetDim(1);265 const int64_t weightScaleK = inputParams_.transB ? weightScaleShape.GetDim(2) : weightScaleShape.GetDim(1);
266 return weightScaleShape.GetDim(0) == static_cast<int64_t>(inputParams_.groupNum) &&266 return weightScaleShape.GetDim(0) == static_cast<int64_t>(inputParams_.groupNum) &&
267 weightScaleN == static_cast<int64_t>(inputParams_.nSize) && weightScaleK == kBlocks &&267 weightScaleN == static_cast<int64_t>(inputParams_.nSize) && weightScaleK == kBlocks &&
268- weightScaleShape.GetDim(3) == SHAPE_DIM_TWO;268+ weightScaleShape.GetDim(3) == SHAPE_DIM_TWO; //4 维 weightScale 的最后一维索引，该维固定为 2
269}269}
270 270 
271bool GroupedMatmulSwigluQuantV2BasicApiTiling950::IsCapable()271bool GroupedMatmulSwigluQuantV2BasicApiTiling950::IsCapable()
Mgmm/grouped_matmul_swiglu_quant_v2/op_host/op_tiling/arch35/grouped_matmul_swiglu_quant_v2_basic_tiling.cpp+129-76
@@ -509,50 +509,119 @@ bool GroupedMatmulSwigluQuantV2Tiling950::CheckDims(const gert::Shape &xShape, c
509 509 
510 return true;510 return true;
511}511}
512+bool GroupedMatmulSwigluQuantV2Tiling950::GetInputShapes(const gert::Shape *&xShape, const gert::Shape *&wShape,
513+ const gert::Shape *&wScaleShape)
514+{
515+ auto xStorageShape = context_->GetInputShape(X_INDEX);
516+ OP_CHECK_IF(
517+ xStorageShape == nullptr,
518+ OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "x", "nullptr", "xStorageShape cannot be nullptr"),
519+ return false);
520+ xShape = &xStorageShape->GetOriginShape();
521+ auto wStorageShape = context_->GetDynamicInputShape(WEIGHT_INDEX, 0);
522+ OP_CHECK_IF(wStorageShape == nullptr,
523+ OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "weight", "nullptr",
524+ "wStorageShape cannot be nullptr"),
525+ return false);
526+ wShape = &wStorageShape->GetOriginShape();
527+ auto scaleStorageShape = context_->GetDynamicInputShape(SCALE_INDEX, 0);
528+ OP_CHECK_IF(scaleStorageShape == nullptr,
529+ OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "scale", "nullptr",
530+ "scaleStorageShape cannot be nullptr"),
531+ return false);
532+ wScaleShape = &scaleStorageShape->GetOriginShape();
533+ return true;
534+}
535+ 
536+bool GroupedMatmulSwigluQuantV2Tiling950::GetAndCheckXScaleShape(const gert::Shape *&xScaleShape)
537+{
538+ auto x1ScaleStorageShape = context_->GetInputShape(PER_TOKEN_SCALE_INDEX);
539+ OP_CHECK_IF(x1ScaleStorageShape == nullptr,
540+ OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "x_scale", "nullptr",
541+ "xScaleStorageShape cannot be nullptr"),
542+ return false);
543+ xScaleShape = &x1ScaleStorageShape->GetOriginShape();
544+ auto xScaleDimNum = xScaleShape->GetDimNum();
545+ OP_CHECK_IF(xScaleDimNum != MX_X_SCALE_DIM,
546+ OP_LOGE_FOR_INVALID_SHAPEDIM(inputParams_.opType, "x_scale", std::to_string(xScaleDimNum), "3"),
547+ return false);
548+ return true;
549+}
550+ 
551+bool GroupedMatmulSwigluQuantV2Tiling950::CheckMxPerGroupShape(const gert::Shape &xScaleShape,
552+ const gert::Shape &wScaleShape, size_t weightCount,
553+ bool isMultiWeightNz)
554+{
555+ if (isMultiWeightNz) {
556+ auto expectedKDimValue = GroupedMatmul::CeilDiv(inputParams_.kSize, MXFP_BASEK_FACTOR);
557+ uint64_t expectedWeightDim0 = inputParams_.transB ? inputParams_.nSize : inputParams_.kSize;
558+ uint64_t expectedWeightDim1 = inputParams_.transB ? inputParams_.kSize : inputParams_.nSize;
559+ uint64_t expectedScaleDim0 = inputParams_.transB ? inputParams_.nSize : expectedKDimValue;
560+ uint64_t expectedScaleDim1 = inputParams_.transB ? expectedKDimValue : inputParams_.nSize;
561+ for (size_t i = 0; i < weightCount; ++i) {
562+ auto curWStorageShape = context_->GetDynamicInputShape(WEIGHT_INDEX, i);
563+ auto curScaleStorageShape = context_->GetDynamicInputShape(SCALE_INDEX, i);
564+ OP_CHECK_IF(curWStorageShape == nullptr || curScaleStorageShape == nullptr,
565+ OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "weight/weight_scale", "nullptr",
566+ "dynamic weight and weightScale cannot be nullptr"),
567+ return false);
568+ const gert::Shape &curWShape = curWStorageShape->GetOriginShape();
569+ const gert::Shape &curScaleShape = curScaleStorageShape->GetOriginShape();
570+ OP_CHECK_IF(curWShape.GetDimNum() != MIN_WEIGHT_ORIGIN_SHAPE_DIM ||
571+ static_cast<uint64_t>(curWShape.GetDim(0)) != expectedWeightDim0 ||
572+ static_cast<uint64_t>(curWShape.GetDim(1)) != expectedWeightDim1,
573+ OP_LOGE_FOR_INVALID_SHAPE(inputParams_.opType, "weight", ShapeToString(curWShape),
574+ ShapeDimsToString(expectedWeightDim0, expectedWeightDim1)),
575+ return false);
576+ OP_CHECK_IF(curScaleShape.GetDimNum() != MX_MULTI_WEIGHT_SCALE_DIM ||
577+ static_cast<uint64_t>(curScaleShape.GetDim(0)) != expectedScaleDim0 ||
578+ static_cast<uint64_t>(curScaleShape.GetDim(1)) != expectedScaleDim1 ||
579+ static_cast<uint64_t>(curScaleShape.GetDim(2)) != MXFP_MULTI_BASE_SIZE,
580+ OP_LOGE_FOR_INVALID_SHAPE(
581+ inputParams_.opType, "weight_scale", ShapeToString(curScaleShape),
582+ ShapeDimsToString(expectedScaleDim0, expectedScaleDim1, MXFP_MULTI_BASE_SIZE)),
583+ return false);
584+ }
585+ OP_CHECK_IF(static_cast<uint64_t>(xScaleShape.GetDim(0)) != inputParams_.mSize ||
586+ static_cast<uint64_t>(xScaleShape.GetDim(1)) != expectedKDimValue ||
587+ static_cast<uint64_t>(xScaleShape.GetDim(2)) != MXFP_MULTI_BASE_SIZE,
588+ OP_LOGE_FOR_INVALID_SHAPE(
589+ inputParams_.opType, "x_scale", ShapeToString(xScaleShape),
590+ ShapeDimsToString(inputParams_.mSize, expectedKDimValue, MXFP_MULTI_BASE_SIZE)),
591+ return false);
592+ } else {
593+ OP_CHECK_IF(!CheckQuantParamsForMXTypeM(xScaleShape, wScaleShape),
594+ OP_LOGE(inputParams_.opName, "CheckShapeForMxQuant failed."), return false);
595+ }
596+ return true;
597+}
598+ 
512bool GroupedMatmulSwigluQuantV2Tiling950::AnalyzeInputs()599bool GroupedMatmulSwigluQuantV2Tiling950::AnalyzeInputs()
513{600{
514 OP_CHECK_IF(!CheckCoreNum(), OP_LOGE(inputParams_.opName, "CheckCoreNum failed."), return false);601 OP_CHECK_IF(!CheckCoreNum(), OP_LOGE(inputParams_.opName, "CheckCoreNum failed."), return false);
515 if (inputParams_.aQuantMode == optiling::QuantMode::PERTOKEN_MODE) {602 if (inputParams_.aQuantMode == optiling::QuantMode::PERTOKEN_MODE) {
516 return AnalyzeInputsPertoken();603 return AnalyzeInputsPertoken();
517 }604 }
518- auto xStorageShape = context_->GetInputShape(X_INDEX);605+ const gert::Shape *xShape = nullptr;
519- OP_CHECK_IF(606+ const gert::Shape *wShape = nullptr;
520- xStorageShape == nullptr,607+ const gert::Shape *wScaleShape = nullptr;
521- OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "x", "nullptr", "xStorageShape cannot be nullptr"),608+ if (!GetInputShapes(xShape, wShape, wScaleShape)) {
522- return false);609+ return false;
523- const gert::Shape &xShape = xStorageShape->GetOriginShape();610+ }
524- auto wStorageShape = context_->GetDynamicInputShape(WEIGHT_INDEX, 0);611+ auto scaleDimNum = wScaleShape->GetDimNum();
525- OP_CHECK_IF(wStorageShape == nullptr,
526- OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "weight", "nullptr",
527- "wStorageShape cannot be nullptr"),
528- return false);
529- const gert::Shape &wShape = wStorageShape->GetOriginShape();
530- auto scaleStorageShape = context_->GetDynamicInputShape(SCALE_INDEX, 0);
531- OP_CHECK_IF(scaleStorageShape == nullptr,
532- OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "scale", "nullptr",
533- "scaleStorageShape cannot be nullptr"),
534- return false);
535- const gert::Shape &wScaleShape = scaleStorageShape->GetOriginShape();
536- auto scaleDimNum = wScaleShape.GetDimNum();
537 size_t weightCount = GetDynamicInputCount(context_, WEIGHT_INDEX);612 size_t weightCount = GetDynamicInputCount(context_, WEIGHT_INDEX);
538 size_t scaleCount = GetDynamicInputCount(context_, SCALE_INDEX);613 size_t scaleCount = GetDynamicInputCount(context_, SCALE_INDEX);
539- bool isMultiWeightNz = IsMxWeightNzMultiTensor(wShape);614+ bool isMultiWeightNz = IsMxWeightNzMultiTensor(*wShape);
540 isMxWeightNzMultiTensor_ = isMultiWeightNz;615 isMxWeightNzMultiTensor_ = isMultiWeightNz;
541 size_t expectedScaleDim = isMultiWeightNz ? MX_MULTI_WEIGHT_SCALE_DIM : MX_WEIGHT_SCALE_DIM;616 size_t expectedScaleDim = isMultiWeightNz ? MX_MULTI_WEIGHT_SCALE_DIM : MX_WEIGHT_SCALE_DIM;
542 OP_CHECK_IF(scaleDimNum != expectedScaleDim,617 OP_CHECK_IF(scaleDimNum != expectedScaleDim,
543 OP_LOGE_FOR_INVALID_SHAPEDIM(inputParams_.opType, "weight_scale", std::to_string(scaleDimNum),618 OP_LOGE_FOR_INVALID_SHAPEDIM(inputParams_.opType, "weight_scale", std::to_string(scaleDimNum),
544 std::to_string(expectedScaleDim)),619 std::to_string(expectedScaleDim)),
545 return false);620 return false);
546- auto x1ScaleStorageShape = context_->GetInputShape(PER_TOKEN_SCALE_INDEX);621+ const gert::Shape *xScaleShape = nullptr;
547- OP_CHECK_IF(x1ScaleStorageShape == nullptr,622+ if (!GetAndCheckXScaleShape(xScaleShape)) {
548- OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "x_scale", "nullptr",623+ return false;
549- "xScaleStorageShape cannot be nullptr"),624+ }
550- return false);
551- const gert::Shape &xScaleShape = x1ScaleStorageShape->GetOriginShape();
552- auto xScaleDimNum = xScaleShape.GetDimNum();
553- OP_CHECK_IF(xScaleDimNum != MX_X_SCALE_DIM,
554- OP_LOGE_FOR_INVALID_SHAPEDIM(inputParams_.opType, "x_scale", std::to_string(xScaleDimNum), "3"),
555- return false);
556 OP_CHECK_IF(!SetGroupNum(GROUPLIST_INDEX), OP_LOGE(inputParams_.opName, "SetGroupNum failed."), return false);625 OP_CHECK_IF(!SetGroupNum(GROUPLIST_INDEX), OP_LOGE(inputParams_.opName, "SetGroupNum failed."), return false);
557 OP_CHECK_IF(isMultiWeightNz && weightCount != static_cast<size_t>(inputParams_.groupNum),626 OP_CHECK_IF(isMultiWeightNz && weightCount != static_cast<size_t>(inputParams_.groupNum),
558 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "weight", std::to_string(weightCount),627 OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "weight", std::to_string(weightCount),
@@ -563,49 +632,11 @@ bool GroupedMatmulSwigluQuantV2Tiling950::AnalyzeInputs()
563 "weightScale tensor list size must equal "632 "weightScale tensor list size must equal "
564 "weight tensor list size"),633 "weight tensor list size"),
565 return false);634 return false);
566- OP_CHECK_IF(!SetMKN(xShape, wShape), OP_LOGE(inputParams_.opName, "SetMKN failed."), return false);635+ OP_CHECK_IF(!SetMKN(*xShape, *wShape), OP_LOGE(inputParams_.opName, "SetMKN failed."), return false);
567- OP_CHECK_IF(!CheckDims(xShape, wShape), OP_LOGE(inputParams_.opName, "CheckDims failed."), return false);636+ OP_CHECK_IF(!CheckDims(*xShape, *wShape), OP_LOGE(inputParams_.opName, "CheckDims failed."), return false);
568 if (inputParams_.bQuantMode == optiling::QuantMode::MX_PERGROUP_MODE) {637 if (inputParams_.bQuantMode == optiling::QuantMode::MX_PERGROUP_MODE) {
569- if (isMultiWeightNz) {638+ if (!CheckMxPerGroupShape(*xScaleShape, wScaleShape, weightCount, isMultiWeightNz)) {
570- auto expectedKDimValue = GroupedMatmul::CeilDiv(inputParams_.kSize, MXFP_BASEK_FACTOR);639+ return false;
571- uint64_t expectedWeightDim0 = inputParams_.transB ? inputParams_.nSize : inputParams_.kSize;
572- uint64_t expectedWeightDim1 = inputParams_.transB ? inputParams_.kSize : inputParams_.nSize;
573- uint64_t expectedScaleDim0 = inputParams_.transB ? inputParams_.nSize : expectedKDimValue;
574- uint64_t expectedScaleDim1 = inputParams_.transB ? expectedKDimValue : inputParams_.nSize;
575- for (size_t i = 0; i < weightCount; ++i) {
576- auto curWStorageShape = context_->GetDynamicInputShape(WEIGHT_INDEX, i);
577- auto curScaleStorageShape = context_->GetDynamicInputShape(SCALE_INDEX, i);
578- OP_CHECK_IF(curWStorageShape == nullptr || curScaleStorageShape == nullptr,
579- OP_LOGE_FOR_INVALID_VALUE_WITH_REASON(inputParams_.opType, "weight/weight_scale", "nullptr",
580- "dynamic weight and weightScale cannot be nullptr"),
581- return false);
582- const gert::Shape &curWShape = curWStorageShape->GetOriginShape();
583- const gert::Shape &curScaleShape = curScaleStorageShape->GetOriginShape();
584- OP_CHECK_IF(curWShape.GetDimNum() != MIN_WEIGHT_ORIGIN_SHAPE_DIM ||
585- static_cast<uint64_t>(curWShape.GetDim(0)) != expectedWeightDim0 ||
586- static_cast<uint64_t>(curWShape.GetDim(1)) != expectedWeightDim1,
587- OP_LOGE_FOR_INVALID_SHAPE(inputParams_.opType, "weight", ShapeToString(curWShape),
588- ShapeDimsToString(expectedWeightDim0, expectedWeightDim1)),
589- return false);
590- OP_CHECK_IF(curScaleShape.GetDimNum() != MX_MULTI_WEIGHT_SCALE_DIM ||
591- static_cast<uint64_t>(curScaleShape.GetDim(0)) != expectedScaleDim0 ||
592- static_cast<uint64_t>(curScaleShape.GetDim(1)) != expectedScaleDim1 ||
593- static_cast<uint64_t>(curScaleShape.GetDim(2)) != MXFP_MULTI_BASE_SIZE,
594- OP_LOGE_FOR_INVALID_SHAPE(
595- inputParams_.opType, "weight_scale", ShapeToString(curScaleShape),
596- ShapeDimsToString(expectedScaleDim0, expectedScaleDim1, MXFP_MULTI_BASE_SIZE)),
597- return false);
598- }
599- OP_CHECK_IF(static_cast<uint64_t>(xScaleShape.GetDim(0)) != inputParams_.mSize ||
600- static_cast<uint64_t>(xScaleShape.GetDim(1)) != expectedKDimValue ||
601- static_cast<uint64_t>(xScaleShape.GetDim(2)) != MXFP_MULTI_BASE_SIZE,
602- OP_LOGE_FOR_INVALID_SHAPE(
603- inputParams_.opType, "x_scale", ShapeToString(xScaleShape),
604- ShapeDimsToString(inputParams_.mSize, expectedKDimValue, MXFP_MULTI_BASE_SIZE)),
605- return false);
606- } else {
607- OP_CHECK_IF(!CheckQuantParamsForMXTypeM(xScaleShape, wScaleShape),
608- OP_LOGE(inputParams_.opName, "CheckShapeForMxQuant failed."), return false);
609 }640 }
610 }641 }
611 return true;642 return true;
@@ -799,14 +830,16 @@ void GroupedMatmulSwigluQuantV2Tiling950::ModifyWeightNzDepthForUnalign(uint64_t
799 }830 }
800}831}
801 832 
802-ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalWeightNzScaleFactors()833+ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalBaseSizesAndScaleInit(uint64_t &baseScaleASize,
834+ uint64_t &baseScaleBSize,
835+ uint32_t &scaleInit)
803{836{
804 uint64_t baseASize = GetSizeWithDataType(basicTiling_.baseM * basicTiling_.baseK, inputParams_.aDtype);837 uint64_t baseASize = GetSizeWithDataType(basicTiling_.baseM * basicTiling_.baseK, inputParams_.aDtype);
805 uint64_t baseBSize = GetSizeWithDataType(basicTiling_.baseN * basicTiling_.baseK, inputParams_.bDtype);838 uint64_t baseBSize = GetSizeWithDataType(basicTiling_.baseN * basicTiling_.baseK, inputParams_.bDtype);
806- uint64_t baseScaleASize =839+ baseScaleASize =
807 GetSizeWithDataType(GroupedMatmul::CeilDiv(basicTiling_.baseK, MX_GROUP_SIZE) * basicTiling_.baseM,840 GetSizeWithDataType(GroupedMatmul::CeilDiv(basicTiling_.baseK, MX_GROUP_SIZE) * basicTiling_.baseM,
808 inputParams_.perTokenScaleDtype);841 inputParams_.perTokenScaleDtype);
809- uint64_t baseScaleBSize = GetSizeWithDataType(842+ baseScaleBSize = GetSizeWithDataType(
810 GroupedMatmul::CeilDiv(basicTiling_.baseK, MX_GROUP_SIZE) * basicTiling_.baseN, inputParams_.scaleDtype);843 GroupedMatmul::CeilDiv(basicTiling_.baseK, MX_GROUP_SIZE) * basicTiling_.baseN, inputParams_.scaleDtype);
811 OP_CHECK_IF(baseScaleASize == 0 || baseScaleBSize == 0,844 OP_CHECK_IF(baseScaleASize == 0 || baseScaleBSize == 0,
812 OP_LOGE(context_->GetNodeName(),845 OP_LOGE(context_->GetNodeName(),
@@ -819,7 +852,7 @@ ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalWeightNzScaleFactors()
819 uint64_t baseBiasSize = inputParams_.hasBias ? basicTiling_.baseN * biasDtypeSize : 0;852 uint64_t baseBiasSize = inputParams_.hasBias ? basicTiling_.baseN * biasDtypeSize : 0;
820 uint64_t leftL1Size =853 uint64_t leftL1Size =
821 aicoreParams_.l1Size - (basicTiling_.depthA1 * baseASize + basicTiling_.depthB1 * baseBSize + baseBiasSize);854 aicoreParams_.l1Size - (basicTiling_.depthA1 * baseASize + basicTiling_.depthB1 * baseBSize + baseBiasSize);
822- uint32_t scaleInit = static_cast<uint32_t>(855+ scaleInit = static_cast<uint32_t>(
823 leftL1Size / (std::max(basicTiling_.depthA1, basicTiling_.depthB1) * (baseScaleASize + baseScaleBSize)));856 leftL1Size / (std::max(basicTiling_.depthA1, basicTiling_.depthB1) * (baseScaleASize + baseScaleBSize)));
824 OP_CHECK_IF(857 OP_CHECK_IF(
825 scaleInit == 0,858 scaleInit == 0,
@@ -827,7 +860,12 @@ ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalWeightNzScaleFactors()
827 "When m(%lu)/n(%lu)/k(%lu)/groupNum(%lu) in mx quant mode, scaleFactor should not be equal to 0.",860 "When m(%lu)/n(%lu)/k(%lu)/groupNum(%lu) in mx quant mode, scaleFactor should not be equal to 0.",
828 inputParams_.mSize, inputParams_.nSize, inputParams_.kSize, inputParams_.groupNum),861 inputParams_.mSize, inputParams_.nSize, inputParams_.kSize, inputParams_.groupNum),
829 return ge::GRAPH_FAILED);862 return ge::GRAPH_FAILED);
863+ return ge::GRAPH_SUCCESS;
864+}
830 865 
866+ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalScaleFactors(uint64_t baseScaleASize, uint64_t baseScaleBSize,
867+ uint32_t scaleInit)
868+{
831 uint32_t scaleFactorAMax =869 uint32_t scaleFactorAMax =
832 std::min(static_cast<uint32_t>(MTE2_MIN_LOAD_SIZE_V120 / baseScaleASize), SCALER_FACTOR_MAX);870 std::min(static_cast<uint32_t>(MTE2_MIN_LOAD_SIZE_V120 / baseScaleASize), SCALER_FACTOR_MAX);
833 uint32_t scaleFactorBMax =871 uint32_t scaleFactorBMax =
@@ -857,6 +895,21 @@ ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalWeightNzScaleFactors()
857 basicTiling_.scaleFactorB = scaleInit;895 basicTiling_.scaleFactorB = scaleInit;
858 }896 }
859 }897 }
898+ return ge::GRAPH_SUCCESS;
899+}
900+ 
901+ge::graphStatus GroupedMatmulSwigluQuantV2Tiling950::CalWeightNzScaleFactors()
902+{
903+ uint64_t baseScaleASize = 0;
904+ uint64_t baseScaleBSize = 0;
905+ uint32_t scaleInit = 0;
906+ if (CalBaseSizesAndScaleInit(baseScaleASize, baseScaleBSize, scaleInit) != ge::GRAPH_SUCCESS) {
907+ return ge::GRAPH_FAILED;
908+ }
909+ 
910+ if (CalScaleFactors(baseScaleASize, baseScaleBSize, scaleInit) != ge::GRAPH_SUCCESS) {
911+ return ge::GRAPH_FAILED;
912+ }
860 OP_CHECK_IF(basicTiling_.scaleFactorA < SCALER_FACTOR_MIN || basicTiling_.scaleFactorA > SCALER_FACTOR_MAX ||913 OP_CHECK_IF(basicTiling_.scaleFactorA < SCALER_FACTOR_MIN || basicTiling_.scaleFactorA > SCALER_FACTOR_MAX ||
861 basicTiling_.scaleFactorB < SCALER_FACTOR_MIN || basicTiling_.scaleFactorB > SCALER_FACTOR_MAX,914 basicTiling_.scaleFactorB < SCALER_FACTOR_MIN || basicTiling_.scaleFactorB > SCALER_FACTOR_MAX,
862 OP_LOGE(context_->GetNodeName(),915 OP_LOGE(context_->GetNodeName(),
Mgmm/grouped_matmul_swiglu_quant_v2/op_host/op_tiling/arch35/grouped_matmul_swiglu_quant_v2_basic_tiling.h+6-0
@@ -64,6 +64,10 @@ private:
64 bool AnalyzeAttrs() override;64 bool AnalyzeAttrs() override;
65 bool AnalyzeDtype() override;65 bool AnalyzeDtype() override;
66 bool AnalyzeInputs() override;66 bool AnalyzeInputs() override;
67+ bool GetInputShapes(const gert::Shape *&xShape, const gert::Shape *&wShape, const gert::Shape *&wScaleShape);
68+ bool GetAndCheckXScaleShape(const gert::Shape *&xScaleShape);
69+ bool CheckMxPerGroupShape(const gert::Shape &xScaleShape, const gert::Shape &wScaleShape, size_t weightCount,
70+ bool isMultiWeightNz);
67 int64_t LogQuantParams();71 int64_t LogQuantParams();
68 bool SetQuantModeForGMMSwigluQuant(const gert::Shape &wScaleShape, const gert::Shape &xScaleShape);72 bool SetQuantModeForGMMSwigluQuant(const gert::Shape &wScaleShape, const gert::Shape &xScaleShape);
69 bool CheckShapeForMxQuant(const gert::Shape &x1ScaleShape, const gert::Shape &x2ScaleShape);73 bool CheckShapeForMxQuant(const gert::Shape &x1ScaleShape, const gert::Shape &x2ScaleShape);
@@ -84,6 +88,8 @@ private:
84 void ModifyWeightNzDepthForUnalign(uint64_t leftL1Size, uint64_t baseASize, uint64_t baseBSize,88 void ModifyWeightNzDepthForUnalign(uint64_t leftL1Size, uint64_t baseASize, uint64_t baseBSize,
85 uint64_t baseScaleABSize);89 uint64_t baseScaleABSize);
86 ge::graphStatus CalWeightNzScaleFactors();90 ge::graphStatus CalWeightNzScaleFactors();
91+ ge::graphStatus CalBaseSizesAndScaleInit(uint64_t &baseScaleASize, uint64_t &baseScaleBSize, uint32_t &scaleInit);
92+ ge::graphStatus CalScaleFactors(uint64_t baseScaleASize, uint64_t baseScaleBSize, uint32_t scaleInit);
87 // add for pertoken quant mode93 // add for pertoken quant mode
88 bool AnalyzeAttrsPertoken();94 bool AnalyzeAttrsPertoken();
89 bool IsB8(ge::DataType dtype);95 bool IsB8(ge::DataType dtype);
Mgmm/grouped_matmul_swiglu_quant_v2/op_kernel/arch35/grouped_matmul_swiglu_quant_v2_tensor_api_tiling_data.h+0-2
@@ -16,8 +16,6 @@
16#ifndef GROUPED_MATMUL_SWIGLU_QUANT_V2_TENSOR_API_TILING_DATA_H16#ifndef GROUPED_MATMUL_SWIGLU_QUANT_V2_TENSOR_API_TILING_DATA_H
17#define GROUPED_MATMUL_SWIGLU_QUANT_V2_TENSOR_API_TILING_DATA_H17#define GROUPED_MATMUL_SWIGLU_QUANT_V2_TENSOR_API_TILING_DATA_H
18 18 
19-#include "kernel_tiling/kernel_tiling.h"
20- 
21#ifndef __CCE_AICORE__19#ifndef __CCE_AICORE__
22#include <cstdint>20#include <cstdint>
23#endif21#endif
Mgmm/quant_grouped_matmul_inplace_add/op_api/aclnn_quant_grouped_matmul_inplace_add.cpp+1-1
@@ -278,7 +278,7 @@ static aclnnStatus SetTransViewShape(const aclTensor *&inputTensor, aclOpExecuto
278{278{
279 op::Shape viewShape = inputTensor->GetViewShape();279 op::Shape viewShape = inputTensor->GetViewShape();
280 uint32_t viewShapeDimsNum = viewShape.GetDimNum();280 uint32_t viewShapeDimsNum = viewShape.GetDimNum();
281- CHECK_RET(viewShapeDimsNum >= 2, ACLNN_ERR_PARAM_INVALID);281+ CHECK_RET(viewShapeDimsNum >= 2, ACLNN_ERR_PARAM_INVALID); //矩阵转置操作要求 tensor 至少要有 2 个维度
282 op::Shape shape;282 op::Shape shape;
283 shape.SetScalar();283 shape.SetScalar();
284 // 2: the second last dimension; in for-loops, it indicates dimensions before the second last remain unchanged.284 // 2: the second last dimension; in for-loops, it indicates dimensions before the second last remain unchanged.
Mgmm/quant_grouped_matmul_inplace_add/op_api/quant_grouped_matmul_inplace_add_950_checker.cpp+1-1
@@ -168,7 +168,7 @@ aclnnStatus AclnnQuantGroupedMatmulInplaceAddDAV3510Checker<T>::CheckHif8QuantPa
168{168{
169 auto groupNum = gmmParams_.groupTensorOptional->GetViewShape().GetDim(0);169 auto groupNum = gmmParams_.groupTensorOptional->GetViewShape().GetDim(0);
170 for (size_t i = 0; i < GetInputTensorSize(gmmParams_.weight); ++i) {170 for (size_t i = 0; i < GetInputTensorSize(gmmParams_.weight); ++i) {
171- auto weightDimNumber = GetInputTensor(gmmParams_.weight, i)->GetViewShape().GetDimNum();171+ //auto weightDimNumber = GetInputTensor(gmmParams_.weight, i)->GetViewShape().GetDimNum();
172 auto scaleDimNumber = GetInputTensor(gmmParams_.scaleOptional, i)->GetViewShape().GetDimNum();172 auto scaleDimNumber = GetInputTensor(gmmParams_.scaleOptional, i)->GetViewShape().GetDimNum();
173 auto perTokenDimNumber = GetInputTensor(gmmParams_.perTokenScaleOptional, i)->GetViewShape().GetDimNum();173 auto perTokenDimNumber = GetInputTensor(gmmParams_.perTokenScaleOptional, i)->GetViewShape().GetDimNum();
174 const auto *perTokenScaleTensor = GetInputTensor(gmmParams_.perTokenScaleOptional, i);174 const auto *perTokenScaleTensor = GetInputTensor(gmmParams_.perTokenScaleOptional, i);
Mgmm/quant_grouped_matmul_inplace_add/op_graph/fallback_quant_grouped_matmul_inplace_add.cpp+52-34
@@ -32,6 +32,53 @@ constexpr size_t INDEX_OUTPUT_Y = 0;
32constexpr size_t INDEX_ATTR_GROUP_LIST_TYPE = 0;32constexpr size_t INDEX_ATTR_GROUP_LIST_TYPE = 0;
33constexpr size_t DIM_TWO = 2;33constexpr size_t DIM_TWO = 2;
34 34 
35+static bool BuildViewShapeAndStrides(const gert::Tensor *geTensor, std::vector<int64_t> &viewShape,
36+ std::vector<int64_t> &strides)
37+{
38+ auto origin_shape = geTensor->GetOriginShape();
39+ for (size_t i = 0; i < origin_shape.GetDimNum(); ++i) {
40+ viewShape.push_back(origin_shape.GetDim(i));
41+ }
42+ strides.assign(viewShape.size(), 1);
43+ // Compute the strides of contiguous tensor
44+ OP_CHECK_IF(viewShape.size() < DIM_TWO,
45+ OP_LOGE("QuantGroupedMatmulInplaceAdd aclnnfallback",
46+ "The dim num of viewshape should be greater than or equal to 2, but the actual is %zu.",
47+ viewShape.size()),
48+ return false);
49+ for (int64_t i = viewShape.size() - 2; i >= 0; i--) {
50+ strides[i] = viewShape[i + 1] * strides[i + 1];
51+ }
52+ return true;
53+}
54+ 
55+static bool ApplyTranspose(size_t index, bool enableTranspose, ge::DataType dataType_ge,
56+ std::vector<int64_t> &viewShape, std::vector<int64_t> &strides)
57+{
58+ if (index == INDEX_INPUT_SCALE1 && dataType_ge == ge::DataType::DT_FLOAT8_E8M0 && enableTranspose) {
59+ OP_CHECK_IF(viewShape.size() < 3, OP_LOGE("aclnnfallback", "Mx tpyek wrong pertokenscale size"), return false);
60+ auto swap = viewShape[0];
61+ viewShape[0] = viewShape[1];
62+ viewShape[1] = swap;
63+ strides[0] = 2; // 2 in shape(k//64 + g, M, 2)
64+ strides[1] = viewShape[0] * 2; // since last dim is contiguous 2
65+ strides[2] = 1; // last axis of stride with index 2 has velue 1
66+ } else if (enableTranspose) { // when tensor is transposed, last two dims in strides and viewShape should swap
67+ // dimM the second-to-last dim， dimN the last dim
68+ auto dimM = viewShape.size() - 2;
69+ auto dimN = viewShape.size() - 1;
70+ if (viewShape[dimM] != 1 && viewShape[dimN] != 0) {
71+ auto swap = strides[dimN];
72+ strides[dimN] = strides[dimM];
73+ strides[dimM] = swap;
74+ swap = viewShape[dimN];
75+ viewShape[dimN] = viewShape[dimM];
76+ viewShape[dimM] = swap;
77+ }
78+ }
79+ return true;
80+}
81+ 
35static inline aclTensor *GeTensor2AclTensor(const gert::Tensor *geTensor, bool enableTranspose, size_t index)82static inline aclTensor *GeTensor2AclTensor(const gert::Tensor *geTensor, bool enableTranspose, size_t index)
36{83{
37 if (geTensor == nullptr) {84 if (geTensor == nullptr) {
@@ -56,43 +103,14 @@ static inline aclTensor *GeTensor2AclTensor(const gert::Tensor *geTensor, bool e
56 } else {103 } else {
57 dataType = ToAclDataType(dataType_ge);104 dataType = ToAclDataType(dataType_ge);
58 }105 }
59- auto origin_shape = geTensor->GetOriginShape();
60 std::vector<int64_t> viewShape;106 std::vector<int64_t> viewShape;
61- for (size_t i = 0; i < origin_shape.GetDimNum(); ++i) {107+ std::vector<int64_t> strides;
62- viewShape.push_back(origin_shape.GetDim(i));108+ if (!BuildViewShapeAndStrides(geTensor, viewShape, strides)) {
63- }109+ return nullptr;
64- std::vector<int64_t> strides(viewShape.size(), 1);
65- // Compute the strides of contiguous tensor
66- OP_CHECK_IF(viewShape.size() < DIM_TWO,
67- OP_LOGE("QuantGroupedMatmulInplaceAdd aclnnfallback",
68- "The dim num of viewshape should be greater than or equal to 2, but the actual is %zu.",
69- viewShape.size()),
70- return nullptr);
71- for (int64_t i = viewShape.size() - 2; i >= 0; i--) {
72- strides[i] = viewShape[i + 1] * strides[i + 1];
73 }110 }
74 111 
75- if (index == INDEX_INPUT_SCALE1 && dataType_ge == ge::DataType::DT_FLOAT8_E8M0 && enableTranspose) {112+ if (!ApplyTranspose(index, enableTranspose, dataType_ge, viewShape, strides)) {
76- OP_CHECK_IF(viewShape.size() < 3, OP_LOGE("aclnnfallback", "Mx tpyek wrong pertokenscale size"),113+ return nullptr;
77- return nullptr);
78- auto swap = viewShape[0];
79- viewShape[0] = viewShape[1];
80- viewShape[1] = swap;
81- strides[0] = 2; // 2 in shape(k//64 + g, M, 2)
82- strides[1] = viewShape[0] * 2; // since last dim is contiguous 2
83- strides[2] = 1; // last axis of stride with index 2 has velue 1
84- } else if (enableTranspose) { // when tensor is transposed, last two dims in strides and viewShape should swap
85- // dimM the second-to-last dim， dimN the last dim
86- auto dimM = viewShape.size() - 2;
87- auto dimN = viewShape.size() - 1;
88- if (viewShape[dimM] != 1 && viewShape[dimN] != 0) {
89- auto swap = strides[dimN];
90- strides[dimN] = strides[dimM];
91- strides[dimM] = swap;
92- swap = viewShape[dimN];
93- viewShape[dimN] = viewShape[dimM];
94- viewShape[dimM] = swap;
95- }
96 }114 }
97 auto aclFormat = aclFormat::ACL_FORMAT_ND;115 auto aclFormat = aclFormat::ACL_FORMAT_ND;
98 aclTensor *out = aclCreateTensor(viewShape.data(), viewShape.size(), dataType, strides.data(), 0, aclFormat,116 aclTensor *out = aclCreateTensor(viewShape.data(), viewShape.size(), dataType, strides.data(), 0, aclFormat,
Mgmm/quant_grouped_matmul_inplace_add/op_host/quant_grouped_matmul_inplace_add_host_utils.h+1-1
@@ -16,7 +16,7 @@
16#ifndef QUANT_GROUPED_MATMUL_INPLACE_ADD_UTILS_H16#ifndef QUANT_GROUPED_MATMUL_INPLACE_ADD_UTILS_H
17#define QUANT_GROUPED_MATMUL_INPLACE_ADD_UTILS_H17#define QUANT_GROUPED_MATMUL_INPLACE_ADD_UTILS_H
18 18 
19-#include <map>19+#include <cstdint>
20 20 
21namespace QuantGroupedMatmulInplaceAdd {21namespace QuantGroupedMatmulInplaceAdd {
22constexpr uint32_t X_INDEX = 0;22constexpr uint32_t X_INDEX = 0;