Star138
245
代码
代码
Issues86
Pull Requests179
项目讨论
Wiki
分析
Star138
245
已合并
[新需求]: 模型日落 #46
CXY-Katrina创建于 2025年12月25日
[新需求]: 模型日落 #46
已合并
CXY-Katrina创建于 2025年12月25日
从Katrina-CXY/MindIE-LLM_opensourcedev合入到Ascend/MindIE-LLMdev
共 18 个文件变更+11-1847
Dexamples/atb_models/atb_framework/models/aquila/7b/layer/decoder_layer.cpp+0-24
@@ -1,24 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12-#include "decoder_layer.h"
13- 
14-namespace atb_speed {
15-namespace aquila_7b {
16- 
17-DecoderLayer::DecoderLayer(
18- const atb_speed::base::LayerParam &param) : atb_speed::base::DecoderLayer<atb::infer::RmsNormParam>(
19- static_cast<atb_speed::base::LayerParam>(param))
20-{
21-}
22- 
23-} // namespace aquila_7b
24-} // namespace atb_speed
Dexamples/atb_models/atb_framework/models/aquila/7b/layer/decoder_layer.h+0-28
@@ -1,28 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#ifndef ATB_SPEED_MODELS_AQUILA_7B_PA_QUANT_LAYER_H
14-#define ATB_SPEED_MODELS_AQUILA_7B_PA_QUANT_LAYER_H
15- 
16-#include "models/base/layer/decoder_layer.h"
17- 
18-namespace atb_speed {
19-namespace aquila_7b {
20- 
21-class DecoderLayer : public atb_speed::base::DecoderLayer<atb::infer::RmsNormParam> {
22-public:
23- explicit DecoderLayer(const atb_speed::base::LayerParam &param);
24-};
25- 
26-} // namespace aquila_7b
27-} // namespace atb_speed
28-#endif
Dexamples/atb_models/atb_framework/models/aquila/7b/model/decoder_model.cpp+0-31
@@ -1,31 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12-#include "decoder_model.h"
13- 
14-namespace atb_speed {
15-namespace aquila_7b {
16- 
17-DecoderModel::DecoderModel(const std::string &param) : atb_speed::base::DecoderModel(param)
18-{
19-}
20- 
21-atb::Status DecoderModel::CreateLayerOperation(atb::Operation **op, uint32_t layerId)
22-{
23- atb_speed::base::LayerParam layerParam;
24- atb_speed::base::DecoderModel::SetLayerParam(layerParam, layerId);
25- DecoderLayer decoderLayer(layerParam);
26- CHECK_OPERATION_STATUS_RETURN(decoderLayer.BuildGraph(op));
27- return atb::NO_ERROR;
28-}
29- 
30-} // namespace aquila_7b
31-} // namespace atb_speed
Dexamples/atb_models/atb_framework/models/aquila/7b/model/decoder_model.h+0-34
@@ -1,34 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#ifndef ATB_SPEED_MODELS_AQUILA_7B_PA_QUANT_MODEL_H
14-#define ATB_SPEED_MODELS_AQUILA_7B_PA_QUANT_MODEL_H
15- 
16-#include "atb_speed/utils/model_factory.h"
17-#include "models/aquila/7b/layer/decoder_layer.h"
18-#include "models/base/model/decoder_model.h"
19- 
20-namespace atb_speed {
21-namespace aquila_7b {
22- 
23-class DecoderModel : public atb_speed::base::DecoderModel {
24-public:
25- explicit DecoderModel(const std::string &param);
26- 
27-protected:
28- atb::Status CreateLayerOperation(atb::Operation **op, uint32_t layerId) override;
29-};
30- 
31-REGISTER_MODEL(aquila_7b, DecoderModel);
32-} // namespace aquila_7b
33-} // namespace atb_speed
34-#endif
Dexamples/atb_models/atb_framework/models/gptneox/20b/layer/decoder_layer.cpp+0-536
@@ -1,536 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024-2025. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12-#include "decoder_layer.h"
13-#include "atb_speed/utils/operation_util.h"
14- 
15-namespace atb_speed {
16-namespace gptneox_20b {
17-static const uint64_t IN_TENSOR_COUNT = 22;
18-static const uint64_t OUT_TENSOR_COUNT = 1;
19-static const uint64_t INTERMEDIATE_TENSOR_COUNT = 14;
20-static const uint64_t LAYER_NORM_AXIS_NUM = 1;
21- 
22-int64_t AddInputNormNode(atb::Node &inputLayerNormNode, const PALayerParam &param)
23-{
24- // norm [n_tokens, hidden_size]
25- atb::infer::LayerNormParam layerNormParam;
26- layerNormParam.layerType = atb::infer::LayerNormParam::LayerNormType::LAYER_NORM_NORM;
27- layerNormParam.normParam.epsilon = param.layerNormEps;
28- layerNormParam.normParam.beginNormAxis = LAYER_NORM_AXIS_NUM;
29- layerNormParam.normParam.beginParamsAxis = LAYER_NORM_AXIS_NUM;
30- 
31- CREATE_OPERATION(layerNormParam, &inputLayerNormNode.operation);
32- inputLayerNormNode.inTensorIds = {IN_HIDDENSTATES, IN_INPUTLAYERNORMWEIGTH, IN_INPUTLAYERNORMBIAS};
33- inputLayerNormNode.outTensorIds = {INTERMEDIATE_INPUTLAYERNORMOUT};
34- return atb::NO_ERROR;
35-}
36- 
37-int64_t AddqkvLinearNodeNodePa(atb::Node &qkvLinearNode)
38-{
39- // qkv [n_tokens, hidden_size] to [n_tokens, 3 * hidden_size]
40- atb::infer::LinearParam linearParam;
41- CREATE_OPERATION(linearParam, &qkvLinearNode.operation);
42- qkvLinearNode.inTensorIds = {INTERMEDIATE_INPUTLAYERNORMOUT, IN_QKVWEIGHT, IN_QKVBIAS};
43- qkvLinearNode.outTensorIds = {INTERMEDIATE_MIXEDQKVLINEAROUT};
44- return atb::NO_ERROR;
45-}
46- 
47-int64_t AddPositionEmbeddingNode(atb::Node &positionEmbeddingNode, const PALayerParam &param)
48-{
49- // rope [n_tokens, hidden_size] to 3 * [n_tokens, hidden_size]
50- atb_speed::gptneox_20b::PositionEmbeddingPAParam positionEmbeddingPAParam;
51- positionEmbeddingPAParam.headNum = param.headNum;
52- positionEmbeddingPAParam.dk = param.dk;
53- positionEmbeddingPAParam.rotaryPct = param.rotaryPct;
54- CHECK_OPERATION_STATUS_RETURN(atb_speed::gptneox_20b::PositionEmbeddingPAOperation(
55- positionEmbeddingPAParam, &positionEmbeddingNode.operation));
56- positionEmbeddingNode.inTensorIds = {INTERMEDIATE_MIXEDQKVLINEAROUT, IN_COSEMBED, IN_SINEMBED, IN_INPUT_LENGTHS};
57- positionEmbeddingNode.outTensorIds = {INTERMEDIATE_QUERYEMBED, INTERMEDIATE_KEYEMBED, INTERMEDIATE_VALUE};
58- return atb::NO_ERROR;
59-}
60- 
61-int64_t AddReshapeAndCacheNode(atb::Node &reshapeAndCacheNode)
62-{
63- // self attention
64- atb::infer::ReshapeAndCacheParam reshapeCacheParm;
65- CREATE_OPERATION(reshapeCacheParm, &reshapeAndCacheNode.operation);
66- reshapeAndCacheNode.inTensorIds = {INTERMEDIATE_KEYEMBED, INTERMEDIATE_VALUE, IN_CACHEK, IN_CACHEV, IN_SLOTS};
67- reshapeAndCacheNode.outTensorIds = {IN_CACHEK, IN_CACHEV};
68- return atb::NO_ERROR;
69-}
70- 
71-int64_t AddMul0Node(atb::Node &mul0Node, const PALayerParam &param)
72-{
73- atb::infer::ElewiseParam mul0Param;
74- mul0Param.elewiseType = atb::infer::ElewiseParam::ElewiseType::ELEWISE_MULS;
75- mul0Param.mulsParam.varAttr = param.qScale;
76- CreateOperation(mul0Param, &mul0Node.operation);
77- mul0Node.inTensorIds = {INTERMEDIATE_QUERYEMBED};
78- mul0Node.outTensorIds = {INTERMEDIATE_QUERYEMBED_SCALED};
79- mul0Node.inTensorReshapeFuncs.resize(mul0Node.inTensorIds.size());
80- return atb::NO_ERROR;
81-}
82- 
83-int64_t AddAttentionNode(atb::Node &attentionNode, const PALayerParam &param)
84-{
85- if (param.isPrefill) {
86- atb::infer::SelfAttentionParam faEnParam;
87- faEnParam.headNum = param.headNum;
88- faEnParam.qkScale = param.qkScale;
89- faEnParam.kvHeadNum = param.headNum;
90- faEnParam.calcType = atb::infer::SelfAttentionParam::PA_ENCODER;
91- faEnParam.maskType = atb::infer::SelfAttentionParam::MaskType::MASK_TYPE_NORM;
92- faEnParam.isTriuMask = param.isPrefill ? 1 : 0;
93- CREATE_OPERATION(faEnParam, &attentionNode.operation);
94- attentionNode.inTensorIds = {INTERMEDIATE_QUERYEMBED_SCALED, INTERMEDIATE_KEYEMBED, INTERMEDIATE_VALUE,
95- IN_ATTENTIONMASK, IN_INPUT_LENGTHS};
96- attentionNode.outTensorIds = {INTERMEDIATE_SELFATTNOUT};
97- } else {
98- atb::infer::PagedAttentionParam paDeParam;
99- paDeParam.headNum = param.headNum;
100- paDeParam.qkScale = param.qkScale;
Dexamples/atb_models/atb_framework/models/gptneox/20b/layer/decoder_layer.h+0-94
@@ -1,94 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#ifndef ATB_SPEED_MODELS_GPTNEOX_20B_PA_LAYER_H
14-#define ATB_SPEED_MODELS_GPTNEOX_20B_PA_LAYER_H
15- 
16-#include <atb/atb_infer.h>
17-#include <nlohmann/json.hpp>
18- 
19-#include "atb_speed/log.h"
20- 
21-namespace atb_speed {
22-namespace gptneox_20b {
23-struct PALayerParam {
24- float layerNormEps = 0;
25- int headNum = 0;
26- int dk = 0;
27- float rotaryPct = 0.0;
28- float qScale = 1.0;
29- float qkScale = 1.0;
30- bool transposedWeight = true;
31- std::string model = "gptneox_20b";
32- bool isPrefill = false;
33- int rank = 0;
34- int rankSize = 1;
35- std::string backend = "hccl";
36-};
37- 
38-enum LayerPATensorId : int {
39- IN_HIDDENSTATES = 0,
40- IN_INPUTLAYERNORMWEIGTH, // weights
41- IN_INPUTLAYERNORMBIAS,
42- IN_POSTATTNLAYERNORMWEIGHT,
43- IN_POSTATTNLAYERNORMBIAS,
44- IN_QKVWEIGHT,
45- IN_QKVBIAS,
46- IN_SELFOUTLINEARWEIGHT,
47- IN_SELFOUTLINEARBIAS,
48- IN_FFNLINEARWEIGHT,
49- IN_FFNLINEARBIAS,
50- IN_FFNOUTLINEARWEIGHT,
51- IN_FFNOUTLINEARBIAS,
52- IN_POSITIONIDS, // inputs
53- IN_COSEMBED,
54- IN_SINEMBED,
55- IN_ATTENTIONMASK,
56- IN_CACHEK, // kvcache
57- IN_CACHEV,
58- IN_BLOCK_TABLES,
59- IN_SLOTS,
60- IN_INPUT_LENGTHS,
61- 
62- OUT_GPTNEOXLAYEROUT,
63- 
64- INTERMEDIATE_INPUTLAYERNORMOUT,
65- INTERMEDIATE_MIXEDQKVLINEAROUT,
66- INTERMEDIATE_QUERYEMBED,
67- INTERMEDIATE_KEYEMBED,
68- INTERMEDIATE_VALUE,
69- INTERMEDIATE_QUERYEMBED_SCALED,
70- INTERMEDIATE_SELFATTNOUT,
71- INTERMEDIATE_SELFATTNLINEAROUT,
72- INTERMEDIATE_POSTATTNLAYERNORMOUT,
73- INTERMEDIATE_FFNLINEAROUT,
74- INTERMEDIATE_FFNACTOUT,
75- INTERMEDIATE_FFNOUTLINEAROUT,
76- INTERMEDIATE_ATTNMLPADDOUT,
77- INTERMEDIATE_ATTNMLP_ALLREDUCEOUT
78-};
79- 
80-struct PositionEmbeddingPAParam {
81- int32_t headNum = 0;
82- int32_t dk = 0;
83- float rotaryPct = 0.25;
84-};
85- 
86-atb::Status PositionEmbeddingPAOperation(const PositionEmbeddingPAParam &param, atb::Operation **operation);
87- 
88-atb::Status PALayer(const PALayerParam &param, atb::Operation **operation);
89- 
90-atb::Operation *CreatePALayer(const nlohmann::json &paramJson);
91-} // namespace gptneox_20b
92-} // namespace atb_speed
93- 
94-#endif // ATB_SPEED_MODELS_GPTNEOX_20B_PA_LAYER_H
Dexamples/atb_models/atb_framework/models/gptneox/20b/model/pa_model.cpp+0-287
@@ -1,287 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024-2025. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12-#include "pa_model.h"
13- 
14-#include "atb/atb_infer.h"
15-#include "nlohmann/json.hpp"
16- 
17-#include "atb_speed/log.h"
18-#include "atb_speed/utils/model_factory.h"
19-#include "models/gptneox/20b/layer/decoder_layer.h"
20-#include "operations/fusion/lmhead/lmhead.h"
21- 
22-namespace atb_speed {
23-namespace gptneox_20b {
24- 
25-REGISTER_MODEL(gptneox_20b, PAModel);
26- 
27-const int WEIGHT_COUNT_PER_LAYER = 12;
28-const int WORDEMBEDDINGNODE_WEIGHT_COUNT = 1;
29-const int FINALNORMNODE_WEIGHT_COUNT = 2;
30-const int OUT_LM_HEAD_WEIGHT_COUNT = 1;
31-const int OPERATION_COUNT_BEFORE_LAYER = 1;
32-const int INTERMEDIATETENSOR_COUNT_BEFORE_LAYER = 1;
33-const int OPERATION_COUNT_AFTER_LAYER = 2;
34-const int OUT_TENSOR_DIM_NUM = 3;
35-const int LAYER_NORM_AXIS_NUM = 1;
36- 
37-void PAModel::Param::FromString(const std::string &param)
38-{
39- nlohmann::json paramJson;
40- try {
41- paramJson = nlohmann::json::parse(param);
42- } catch (const std::exception &e) {
43- std::stringstream ss;
44- ss << "Parameter parsing failed. Please verify the format of the parameters. Error:" << e.what() << std::endl;
45- ATB_SPEED_LOG_ERROR("Parameter parsing failed. Please verify the format of the parameters. Error:" << e.what());
46- throw std::runtime_error(ss.str());
47- }
48- layerNormEps = paramJson["layerNormEps"].get<float>();
49- headNum = CheckPositive(paramJson["headNum"].get<int>());
50- dk = CheckPositive(paramJson["dk"].get<int>());
51- layerNum = CheckNumHiddenLayersValid(paramJson["layerNum"].get<int>());
52- rotaryPct = paramJson["rotaryPct"].get<float>();
53- if (paramJson.contains("isPrefill")) {
54- isPrefill = paramJson["isPrefill"].get<bool>();
55- }
56- if (paramJson.contains("rank")) {
57- rank = paramJson["rank"].get<int>();
58- }
59- if (paramJson.contains("rankSize")) {
60- rankSize = CheckPositive(paramJson["rankSize"].get<int>());
61- }
62- if (paramJson.contains("qScale")) {
63- qScale = paramJson["qScale"].get<float>();
64- }
65- if (paramJson.contains("qkScale")) {
66- qkScale = paramJson["qkScale"].get<float>();
67- }
68- if (paramJson.contains("backend")) {
69- backend = paramJson["backend"];
70- }
71- 
72- ATB_SPEED_LOG_DEBUG("GptNeox20BModel param layerNormEps:"
73- << layerNormEps << ", headNum:" << headNum << ", dk:" << dk << ", layerNum:" << layerNum
74- << ", rotaryPct:" << rotaryPct << ", backend: " << backend);
75-}
76- 
77-PAModel::PAModel(const std::string &param) : Model("GptNeoX_20B_MODEL", param)
78-{
79- param_.FromString(param);
80- modelName_ += param_.isPrefill ? "_Prefill" : "_Decoder";
81-}
82- 
83-PAModel::~PAModel() {}
84- 
85-uint32_t PAModel::GetInputNum() const { return graph_.inTensors.size(); }
86- 
87-uint32_t PAModel::GetOutputNum() const { return graph_.outTensors.size(); }
88- 
89-atb::Status PAModel::InferShape(const std::vector<atb::TensorDesc> &inTensorDescs,
90- std::vector<atb::TensorDesc> &outTensorDescs)
91-{
92- if (outTensorDescs.size() != GetOutputNum()) {
93- return atb::ERROR_INVALID_GRAPH;
94- }
95- 
96- const int64_t outTensorLastDimSize = graph_.weightTensors.at(graph_.weightTensors.size() - 1).desc.shape.dims[0];
97- outTensorDescs.at(0) = graph_.weightTensors.at(0).desc;
98- auto outDimNum = inTensorDescs.at(0).shape.dimNum + 1;
99- for (uint i = 0; i < outDimNum - 1; i++) {
100- outTensorDescs.at(0).shape.dims[i] = inTensorDescs.at(0).shape.dims[i];
101- }
102- CHECK_TENSORDESC_DIMNUM_VALID(outDimNum);
103- outTensorDescs.at(0).shape.dims[outDimNum - 1] = CheckIntMulOverFlow(outTensorLastDimSize, param_.rankSize);
104- 
105- // change first dim
106- if (param_.isPrefill) {
107- outTensorDescs.at(0).shape.dims[0] = inTensorDescs.at(IN_TENSOR_LOGTIS_INDICES).shape.dims[0];
108- }
109- 
110- return atb::NO_ERROR;
111-}
112- 
113-int64_t PAModel::AddWordEmbedding()
114-{
115- atb::Operation *op = nullptr;
116- 
117- // wordEmbeddingNode
118- atb_speed::Model::Node wordEmbeddingNode;
119- atb::infer::GatherParam wordEmbeddingParam;
120- CREATE_OPERATION(wordEmbeddingParam, &op);
121- wordEmbeddingNode.operation.reset(op);
122- wordEmbeddingNode.inTensors = {&graph_.weightTensors.at(0), &graph_.inTensors.at(IN_TENSOR_INPUTIDS)};
123- wordEmbeddingNode.outTensors = {&graph_.internalTensors.at(INTERNAL_TENSOR_ID)};
124- graph_.nodes.push_back(wordEmbeddingNode);
125- 
126- return atb::NO_ERROR;
127-}
128- 
129-int64_t PAModel::AddLayer()
130-{
131- atb::Operation *op = nullptr;
132- for (int layerId = 0; layerId < param_.layerNum; ++layerId) {
133- atb_speed::Model::Node layerNode;
134- atb_speed::gptneox_20b::PALayerParam opParam;
135- opParam.layerNormEps = param_.layerNormEps;
136- opParam.headNum = param_.headNum;
137- opParam.dk = param_.dk;
138- opParam.rotaryPct = param_.rotaryPct;
139- opParam.model = "gptneox_20b";
140- opParam.isPrefill = param_.isPrefill;
141- opParam.qScale = param_.qScale;
142- opParam.qkScale = param_.qkScale;
143- opParam.rank = param_.rank;
144- opParam.rankSize = param_.rankSize;
145- opParam.backend = param_.backend;
146- CHECK_OPERATION_STATUS_RETURN(atb_speed::gptneox_20b::PALayer(opParam, &op));
147- layerNode.operation.reset(op);
148- layerNode.inTensors.resize(layerNode.operation->GetInputNum());
149- 
150- uint32_t inTensorId = 0;
151- layerNode.inTensors.at(inTensorId++) = &graph_.internalTensors.at(INTERNAL_TENSOR_ID);
152- for (size_t weightTensorId = 0; weightTensorId < WEIGHT_COUNT_PER_LAYER; ++weightTensorId) {
153- layerNode.inTensors.at(inTensorId++) = &graph_.weightTensors.at(
154- layerId * WEIGHT_COUNT_PER_LAYER + weightTensorId + WORDEMBEDDINGNODE_WEIGHT_COUNT);
155- }
156- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_POSITIONID); // positionIdTensor
157- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_COSEMBED); // cosEmbed
158- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_SINEMBED); // sinEmbed
159- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_ATTENTIONMASK); // attentionMaskTensor
160- layerNode.inTensors.at(inTensorId++) = &graph_.kCacheTensors.at(layerId);
161- layerNode.inTensors.at(inTensorId++) = &graph_.vCacheTensors.at(layerId);
162- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_BLOCK_TABLES);
163- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_SLOTS);
164- layerNode.inTensors.at(inTensorId++) = &graph_.inTensors.at(IN_TENSOR_INPUT_LENGTHS);
165- layerNode.outTensors = {&graph_.internalTensors.at(INTERNAL_TENSOR_ID)};
166- graph_.nodes.push_back(layerNode);
167- }
168- return atb::NO_ERROR;
169-}
170- 
171-int64_t PAModel::AddFinalNorm()
172-{
173- atb::Operation *op = nullptr;
174- 
175- atb_speed::Model::Node finalNormNode;
176- atb::infer::LayerNormParam finalNormParam;
177- finalNormParam.layerType = atb::infer::LayerNormParam::LayerNormType::LAYER_NORM_NORM;
178- finalNormParam.normParam.epsilon = param_.layerNormEps;
179- finalNormParam.normParam.beginNormAxis = LAYER_NORM_AXIS_NUM;
180- finalNormParam.normParam.beginParamsAxis = LAYER_NORM_AXIS_NUM;
181- CREATE_OPERATION(finalNormParam, &op);
182- finalNormNode.operation.reset(op);
183- const uint32_t finalLayerNormWeightTensorId =
184- graph_.weightTensors.size() - FINALNORMNODE_WEIGHT_COUNT - OUT_LM_HEAD_WEIGHT_COUNT;
185- const uint32_t finalLayerNormBiasTensorId =
186- graph_.weightTensors.size() - (FINALNORMNODE_WEIGHT_COUNT - 1) - OUT_LM_HEAD_WEIGHT_COUNT;
187- finalNormNode.inTensors = {&graph_.internalTensors.at(INTERNAL_TENSOR_ID),
188- &graph_.weightTensors.at(finalLayerNormWeightTensorId),
189- &graph_.weightTensors.at(finalLayerNormBiasTensorId)};
190- finalNormNode.outTensors = {&graph_.internalTensors.at(INTERNAL_TENSOR_ID)};
191- 
192- graph_.nodes.push_back(finalNormNode);
193- 
194- return atb::NO_ERROR;
195-}
196- 
197-int64_t PAModel::AddLmhead()
198-{
199- atb::Operation *op = nullptr;
200- 
201- atb_speed::Model::Node lmHeadNode;
202- atb_speed::common::LmHeadParam lmHeadParam;
203- lmHeadParam.unpadInputs = true;
204- lmHeadParam.gatherAhead = param_.isPrefill;
205- lmHeadParam.hiddenSizePerAttentionHead = param_.dk;
206- 
207- lmHeadParam.linearParallelParam.unpadInputs = true;
208- lmHeadParam.linearParallelParam.parallelType = atb_speed::common::COLUMN_PARALLEL;
209- lmHeadParam.linearParallelParam.tensorParallelInfo = {
210- param_.rank, param_.rankSize, param_.backend
211- };
212- CHECK_OPERATION_STATUS_RETURN(LmHead(lmHeadParam, &op));
213- lmHeadNode.operation.reset(op);
214- const uint64_t finalLinearWeightTensorId = graph_.weightTensors.size() - OUT_LM_HEAD_WEIGHT_COUNT;
215- lmHeadNode.inTensors = {
216- &graph_.internalTensors.at(INTERNAL_TENSOR_ID),
217- &graph_.weightTensors.at(finalLinearWeightTensorId),
218- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER),
219- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER),
220- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER),
221- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER),
222- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER),
223- param_.isPrefill ? \
224- &graph_.inTensors.at(IN_TENSOR_LOGTIS_INDICES) : \
225- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER),
226- &graph_.inTensors.at(IN_TENSOR_PLACE_HOLDER)
227- };
228- lmHeadNode.outTensors = {&graph_.outTensors.at(OUT_TENSOR_ID)};
229- graph_.nodes.push_back(lmHeadNode);
230- 
231- return atb::NO_ERROR;
232-}
233- 
234-int64_t PAModel::BuildGraph()
235-{
236- const int weightTensorSize = WORDEMBEDDINGNODE_WEIGHT_COUNT + WEIGHT_COUNT_PER_LAYER * param_.layerNum +
237- FINALNORMNODE_WEIGHT_COUNT + OUT_LM_HEAD_WEIGHT_COUNT;
238- graph_.weightTensors.resize(weightTensorSize);
239- graph_.kCacheTensors.resize(param_.layerNum);
240- graph_.vCacheTensors.resize(param_.layerNum);
241- graph_.inTensors.resize(IN_TENSOR_MAX);
242- graph_.outTensors.resize(OUT_TENSOR_MAX);
243- graph_.internalTensors.resize(INTERNAL_TENSOR_MAX);
244- 
245- CHECK_OPERATION_STATUS_RETURN(AddWordEmbedding());
246- CHECK_OPERATION_STATUS_RETURN(AddLayer());
247- CHECK_OPERATION_STATUS_RETURN(AddFinalNorm());
248- CHECK_OPERATION_STATUS_RETURN(AddLmhead());
249- return atb::NO_ERROR;
250-}
251- 
252-atb::Status PAModel::ParseParam(const std::string &param)
253-{
254- CHECK_PARAM_LT(param.size(), MAX_PARAM_STRING_LENGTH);
255- nlohmann::json paramJson;
256- try {
257- paramJson = nlohmann::json::parse(param);
258- } catch (const std::exception &e) {
259- std::stringstream ss;
260- ss << "Parameter parsing failed. Please verify the format of the parameters. Error:" << e.what() << std::endl;
261- ATB_SPEED_LOG_ERROR("Parameter parsing failed. Please verify the format of the parameters. Error:" << e.what());
262- throw std::runtime_error(ss.str());
263- }
264- seqLen_.clear();
265- for (auto item : paramJson["seqLen"]) {
266- int seqLen = item.get<int>();
267- CHECK_PARAM_LT(seqLen, MAX_PARAM_VALUE);
268- seqLen_.push_back(seqLen);
269- }
270- ATB_SPEED_LOG_DEBUG("PAModel ParseParam seqLen: " << seqLen_.capacity());
271- return atb::NO_ERROR;
272-}
273- 
274-atb::Status PAModel::BindParamHostTensor(uint32_t nodeId)
275-{
276- if (nodeId < OPERATION_COUNT_BEFORE_LAYER) {
277- return atb::NO_ERROR;
278- } else if (nodeId >= static_cast<uint32_t>(OPERATION_COUNT_BEFORE_LAYER + param_.layerNum)) {
279- return atb::NO_ERROR;
280- }
281- auto &node = graph_.nodes.at(nodeId);
282- const uint32_t seqLenTensorId = LayerPATensorId::IN_INPUT_LENGTHS;
283- node.variantPack.inTensors.at(seqLenTensorId).hostData = seqLen_.data();
284- return atb::NO_ERROR;
285-}
286-} // namespace gptneox_20b
287-} // namespace atb_speed
Dexamples/atb_models/atb_framework/models/gptneox/20b/model/pa_model.h+0-90
@@ -1,90 +0,0 @@
1-/*
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024-2025. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#ifndef ATB_SPEED_MODELS_GPTNEOX_20B_PA_MODEL_H
14-#define ATB_SPEED_MODELS_GPTNEOX_20B_PA_MODEL_H
15- 
16-#include "atb_speed/base/model.h"
17- 
18-namespace atb_speed {
19-namespace gptneox_20b {
20- 
21-enum InTensorId : uint32_t {
22- IN_TENSOR_INPUTIDS = 0,
23- IN_TENSOR_POSITIONID,
24- IN_TENSOR_COSEMBED,
25- IN_TENSOR_SINEMBED,
26- IN_TENSOR_ATTENTIONMASK,
27- IN_TENSOR_BLOCK_TABLES,
28- IN_TENSOR_SLOTS,
29- IN_TENSOR_INPUT_LENGTHS,
30- IN_TENSOR_LOGTIS_INDICES,
31- IN_TENSOR_PLACE_HOLDER,
32- IN_TENSOR_MAX
33-};
34- 
35-enum InternalTensorId : uint32_t {
36- INTERNAL_TENSOR_ID = 0,
37- INTERNAL_TENSOR_MAX
38-};
39- 
40-enum OutTensorId : uint32_t {
41- OUT_TENSOR_ID = 0,
42- OUT_TENSOR_MAX,
43-};
44- 
45-class PAModel : public Model {
46-public:
47- struct Param {
48- int headNum = 0;
49- int dk = 0;
50- int layerNum = 0;
51- float layerNormEps = 0;
52- float rotaryPct = 0.0;
53- int rank = 0;
54- int rankSize = 1;
55- bool isPrefill = false;
56- float qScale = 1.0;
57- float qkScale = 1.0;
58- std::string backend = "hccl";
59- 
60- void FromString(const std::string &param);
61- };
62- 
63- explicit PAModel(const std::string &param);
64- 
65- ~PAModel() override;
66- 
67- uint32_t GetInputNum() const override;
68- 
69- uint32_t GetOutputNum() const override;
70- 
71- atb::Status InferShape(const std::vector<atb::TensorDesc> &inTensorDescs,
72- std::vector<atb::TensorDesc> &outTensorDescs) override;
73- 
74-private:
75- int64_t BuildGraph() override;
76- int64_t AddWordEmbedding();
77- int64_t AddLayer();
78- int64_t AddFinalNorm();
79- int64_t AddLmhead();
80- atb::Status ParseParam(const std::string &param) override;
81- atb::Status BindParamHostTensor(uint32_t nodeId) override;
82- 
83-private:
84- Param param_;
85- std::vector<int32_t> seqLen_;
86-};
87- 
88-} // namespace gptneox_20b
89-} // namespace atb_speed
90-#endif // ATB_SPEED_MODELS_GPTNEOX_20B_PA_MODEL_H
Mexamples/atb_models/examples/models/qwen/README.md+11-53
@@ -72,7 +72,6 @@
72## 推理须知：72## 推理须知：
73- Qwen模型权重所在路径中的config.json文件需添加字段`torch_dtype`，例如`"torch_dtype": "float16"`73- Qwen模型权重所在路径中的config.json文件需添加字段`torch_dtype`，例如`"torch_dtype": "float16"`
74- 执行量化推理时，须在量化权重所在路径的config.json文件中添加字段`quantize`，值为当前量化权重的量化方式，例如`"quantize": "w8a8"`、`"quantize": "w8a16"`74- 执行量化推理时，须在量化权重所在路径的config.json文件中添加字段`quantize`，值为当前量化权重的量化方式，例如`"quantize": "w8a8"`、`"quantize": "w8a16"`
75-- Qwen-14B执行[2k,32k]（QWen-7B为[8k,32k]）长序列推理时需增加环境变量`LONG_SEQ_ENABLE=1`。长序列推理过程具有更多计算节点，因此相比于短序列，推理性能将有下降。
76- Qwen2-7B建议采用`bf16`格式，即其权重所在路径中的config.json文件字段`torch_dtype`保持为`bfloat16`75- Qwen2-7B建议采用`bf16`格式，即其权重所在路径中的config.json文件字段`torch_dtype`保持为`bfloat16`
77- 300I DUO只支持`"torch_dtype": "float16"`76- 300I DUO只支持`"torch_dtype": "float16"`
78- 稀疏量化w8a8sc仅支持在300I DUO上使用77- 稀疏量化w8a8sc仅支持在300I DUO上使用
@@ -165,7 +164,7 @@ pip install transformers_stream_generator==0.0.4
165 - 注意：若权重生成时以TP=4进行切分，则运行时也需以TP=4运行164 - 注意：若权重生成时以TP=4进行切分，则运行时也需以TP=4运行
166 - 示例165 - 示例
167 ```shell166 ```shell
168- torchrun --nproc_per_node 2 -m examples.convert.model_slim.sparse_compressor --model_path /data1/weights/model_slim/Qwen-14b_w8a8s --save_directory /data1/weights/model_slim/Qwen-14b_w8a8sc167+ torchrun --nproc_per_node 2 -m examples.convert.model_slim.sparse_compressor --model_path /data1/weights/model_slim/Qwen2-14b_w8a8s --save_directory /data1/weights/model_slim/Qwen2-14b_w8a8sc
169 ```168 ```
170 169 
171#### Qwen2-72B W8A16量化170#### Qwen2-72B W8A16量化
@@ -229,7 +228,7 @@ bash examples/models/qwen/convert_quant_weight.sh -src ${浮点权重路径} -ds
229 - multiprocess_num必须设置为4以减小机器压力228 - multiprocess_num必须设置为4以减小机器压力
230 - 示例229 - 示例
231 ```shell230 ```shell
232- torchrun --nproc_per_node 2 -m examples.convert.model_slim.sparse_compressor --model_path /data1/weights/model_slim/Qwen-14b_w8a8s --save_directory /data1/weights/model_slim/Qwen-14b_w8a8sc --multiprocess_num 4231+ torchrun --nproc_per_node 2 -m examples.convert.model_slim.sparse_compressor --model_path /data1/weights/model_slim/Qwen2-72b_w8a8s --save_directory /data1/weights/model_slim/Qwen2-72b_w8a8sc --multiprocess_num 4
233 ```232 ```
234 - 与Qwen2-72B W8A16量化不同，量化脚本已经替用户按要求修改了config.json，用户只需要将config.json中的quant_type字段修改为"w8a8s"即可。233 - 与Qwen2-72B W8A16量化不同，量化脚本已经替用户按要求修改了config.json，用户只需要将config.json中的quant_type字段修改为"w8a8s"即可。
235 234 
@@ -393,9 +392,7 @@ bash examples/models/qwen/run_pa.sh -m ${weight_path} --trust_remote_code true
393 392 
3942.--trust_remote_code为可选参数代表是否信任本地的可执行文件，默认false。传入true，则代表信任本地可执行文件，-r为其缩写3932.--trust_remote_code为可选参数代表是否信任本地的可执行文件，默认false。传入true，则代表信任本地可执行文件，-r为其缩写
395 394 
396-3.同时支持Qwen和Qwen1.5模型推理，若启动Qwen模型推理时在`${weight_path}`中传入Qwen权重路径，若启动Qwen1.5模型推理时则在`${weight_path}`中传入Qwen1.5权重路径395+3.Qwen系列chat模型需要开启chat模式才能正常输出。
397- 
398-4.Qwen系列chat模型需要开启chat模式才能正常输出。
399执行：396执行：
400 397 
401```shell398```shell
@@ -417,15 +414,15 @@ pip install tiktoken
417- 在基础模型的权重文件夹中，新增`lora_adapter.json`文件，内容为需要预加载的Lora权重，例如：414- 在基础模型的权重文件夹中，新增`lora_adapter.json`文件，内容为需要预加载的Lora权重，例如：
418 ```json415 ```json
419 {416 {
420- "qwen_lora1": "/home/data/lora/Qwen1.5-14b-chat/adapter1",417+ "qwen_lora1": "/home/data/lora/Qwen2.5-14b-chat/adapter1",
421- "qwen_lora2": "/home/data/lora/Qwen1.5-14b-chat/adapter2"418+ "qwen_lora2": "/home/data/lora/Qwen2.5-14b-chat/adapter2"
422 }419 }
423 ```420 ```
424- 进行推理时需指定每个请求所使用的adapter权重，默认仅使用基础模型权重421- 进行推理时需指定每个请求所使用的adapter权重，默认仅使用基础模型权重
425- 运行示例422- 运行示例
426 ```shell423 ```shell
427 export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7424 export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
428- torchrun --nproc_per_node 8 --master_port 20030 -m examples.run_pa --model_path /data1/models/qwen2/Qwen1.5-14b-chat --is_chat_model --max_output_length 256 --max_batch_size 2 --input_dict '[{"prompt": "What is deep learning?", "adapter": "qwen_lora1"}, {"prompt": "What is deep learning?"}]'425+ torchrun --nproc_per_node 8 --master_port 20030 -m examples.run_pa --model_path /data1/models/qwen2/Qwen2.5-14b-chat --is_chat_model --max_output_length 256 --max_batch_size 2 --input_dict '[{"prompt": "What is deep learning?", "adapter": "qwen_lora1"}, {"prompt": "What is deep learning?"}]'
429 ```426 ```
430- 约束与限制427- 约束与限制
431 - 仅支持在Atlas 800I A2上运行428 - 仅支持在Atlas 800I A2上运行
@@ -460,19 +457,7 @@ pip install tiktoken
460 457 
461```shell458```shell
462bash run.sh pa_fp16 full_BoolQ 1 qwen /data1/models/qwen2/qwen_quant_test/ 8459bash run.sh pa_fp16 full_BoolQ 1 qwen /data1/models/qwen2/qwen_quant_test/ 8
463-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen-7b权重路径} 8460+bash run.sh pa_fp16 full_BoolQ 1 qwen ${权重路径} 8
464-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen-14b权重路径} 8
465-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen-72b权重路径} 8
466-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-14b权重路径} 8
467-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen-14b-chat权重路径} 8
468-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen-72b-chat权重路径} 8
469-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-0.5b-chat权重路径} 8
470-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-4b-chat权重路径} 4
471-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-7b权重路径} 8
472-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-14b-chat权重路径} 8
473-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-32b-chat权重路径} 8
474-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-72b权重路径} 8
475-bash run.sh pa_fp16 full_BoolQ 1 qwen ${Qwen1.5-MoE-A2.7B-Chat权重路径} 8
476bash run.sh pa_fp16 full_HumanEval_X 1 qwen ${Qwen2.5-Coder-7B权重路径} 8461bash run.sh pa_fp16 full_HumanEval_X 1 qwen ${Qwen2.5-Coder-7B权重路径} 8
477```462```
478 463 
@@ -501,21 +486,12 @@ bash run.sh pa_fp16 full_HumanEval_X 1 qwen ${Qwen2.5-Coder-7B权重路径} 8
501示例：486示例：
502 487 
503 ```shell488 ```shell
504- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen-7b权重路径} 8489+ HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${权重路径} 8
505- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen-14b权重路径} 8
506- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen-72b权重路径} 8
507- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-14b权重路径} 8
508- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen-14b-chat权重路径} 8
509- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen-72b-chat权重路径} 8
510 HCCL_DETERMINISTIC=0 LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 490 HCCL_DETERMINISTIC=0 LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120
511- bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-0.5b-chat权重路径} 8491+ bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${权重路径} 8
512 HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120492 HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120
513- bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-4b-chat权重路径} 8493+ bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${权重路径} 8
514- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-7b权重路径} 8494+ HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${权重路径} 8
515- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-14b-chat权重路径} 8
516- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-32b-chat权重路径} 8
517- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-72b权重路径} 8
518- HCCL_DETERMINISTIC=false LCCL_DETERMINISTIC=0 HCCL_BUFFSIZE=120 bash run.sh pa_fp16 performance　[[2048,2048],[1024,1024],[512,512],[256,256]] 1 qwen ${Qwen1.5-MoE-A2.7B-Chat权重路径} 8
519 ```495 ```
520 496 
521- 参考[此README文件](../../../tests/modeltest/README.md)497- 参考[此README文件](../../../tests/modeltest/README.md)
@@ -537,21 +513,6 @@ bash examples/models/qwen/run_fa.sh -m ${weight_path}
537```513```
538 514 
539# Qwen长序列推理515# Qwen长序列推理
540-Qwen长序列推理需要在Qwen权重(config.json)中将use_dynamic_ntk参数与use_logn_attn同时设置成True。如Qwen-14B-Chat：
541-注意：如果不使用长序列推理，请将use_dynamic_ntk与use_logn_attn参数同时设置成False。
542- 
543-```json
544-{
545- "architectures": [
546- "QwenLMHeadModel"
547- ],
548- // ...
549- "use_dynamic_ntk": true,
550- // ...
551- "use_logn_attn": true,
552- // ...
553-}
554-```
555Qwen2长序列推理需要在Qwen2权重(config.json)中新增rope_scaling参数。如Qwen2-72B-Instruct：516Qwen2长序列推理需要在Qwen2权重(config.json)中新增rope_scaling参数。如Qwen2-72B-Instruct：
556注意：如果不使用长序列推理，请不要添加。517注意：如果不使用长序列推理，请不要添加。
557```json518```json
@@ -571,8 +532,5 @@ Qwen2长序列推理需要在Qwen2权重(config.json)中新增rope_scaling参数
571}532}
572注：533注：
573- 除启动命令外，其他操作与执行PA相同534- 除启动命令外，其他操作与执行PA相同
574-- Qwen1.5暂不支持bf16格式，请将权重路径下config.json文件的`torch_dtype`字段修改为`float16`
575-- 暂不支持chat模式。部分chat模型输出可能存在异常，如Qwen1.5-32b-chat，若出现上述情况，请优先使用PA
576- 长序列推理过程具有更多计算节点，因此相比于短序列，推理性能将有下降。535- 长序列推理过程具有更多计算节点，因此相比于短序列，推理性能将有下降。
577-- Qwen1.5部分Chat模型(4B、32B)fa暂不支持chat推理，请优先使用pa。如需使用fa请将输入改造成续写的样式，如：`What's deep learning?`改写成`Deep learning is`
578- Qwen2 Qwen2.5系列模型当前800I A2采用bf16， 300I DUO使用fp16 536- Qwen2 Qwen2.5系列模型当前800I A2采用bf16， 300I DUO使用fp16
Dexamples/atb_models/tests/fuzztest/models/test_aquila_pa_model.cpp+0-175
@@ -1,175 +0,0 @@
1-/**
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#include <gtest/gtest.h>
14-#include <pybind11/embed.h>
15-#include <pybind11/stl.h>
16- 
17-#include "atb_speed/log.h"
18-#include "nlohmann/json.hpp"
19-#include "../utils/fuzz_utils.h"
20-#include "secodeFuzz.h"
21- 
22-namespace atb_speed {
23- 
24-void Aquila7BGetParamType(int numHiddenLayers, nlohmann::json &parameter)
25-{
26- std::vector<int> layerPackQuantType = {
27- std::rand() % 12,
28- std::rand() % 12
29- };
30- std::vector<std::vector<int>> modelPackQuantType;
31- for (int layerId = 0; layerId < numHiddenLayers; layerId++) {
32- modelPackQuantType.push_back(layerPackQuantType);
33- }
34- parameter["packQuantType"] = modelPackQuantType;
35- 
36- std::vector<int> layerLinearQuantType = {
37- std::rand() % 3 - 1,
38- std::rand() % 3 - 1,
39- std::rand() % 3 - 1,
40- std::rand() % 3 - 1,
41- std::rand() % 3 - 1,
42- std::rand() % 3 - 1,
43- std::rand() % 3 - 1
44- };
45- std::vector<std::vector<int>> modelLinearQuantType;
46- for (int layerId = 0; layerId < numHiddenLayers; layerId++) {
47- modelLinearQuantType.push_back(layerLinearQuantType);
48- }
49- parameter["linearQuantType"] = modelLinearQuantType;
50- 
51- std::vector<int> layerLinearTransposeType = {
52- std::rand() % 3 - 1,
53- std::rand() % 3 - 1,
54- std::rand() % 3 - 1,
55- std::rand() % 3 - 1,
56- std::rand() % 3 - 1,
57- std::rand() % 3 - 1,
58- std::rand() % 3 - 1
59- };
60- std::vector<std::vector<int>> modelLinearTransposeType;
61- for (int layerId = 0; layerId < numHiddenLayers; layerId++) {
62- modelLinearTransposeType.push_back(layerLinearTransposeType);
63- }
64- parameter["linearTransposeType"] = modelLinearTransposeType;
65-}
66- 
67-nlohmann::json Aquila7BGetParam(int numHiddenLayers, uint32_t &fuzzIndex)
68-{
69- nlohmann::json parameter;
70- 
71- parameter["isFA"] = FuzzUtil::GetRandomBool(fuzzIndex);
72- parameter["isEmbeddingParallel"] = FuzzUtil::GetRandomBool(fuzzIndex);
73- parameter["isPrefill"] = FuzzUtil::GetRandomBool(fuzzIndex);
74- parameter["isBF16"] = FuzzUtil::GetRandomBool(fuzzIndex);
75- parameter["isLmHeadParallel"] = FuzzUtil::GetRandomBool(fuzzIndex);
76- parameter["lmHeadTransposeType"] = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 1);
77- parameter["enableSwiGLU"] = FuzzUtil::GetRandomBool(fuzzIndex);
78- parameter["enableLcoc"] = FuzzUtil::GetRandomBool(fuzzIndex);
79- parameter["enableLora"] = FuzzUtil::GetRandomBool(fuzzIndex);
80- parameter["enableSplitFuse"] = FuzzUtil::GetRandomBool(fuzzIndex);
81- parameter["enableSpeculate"] = FuzzUtil::GetRandomBool(fuzzIndex);
82- parameter["enableCompressHead"] = FuzzUtil::GetRandomBool(fuzzIndex);
83- parameter["skipWordEmbedding"] = FuzzUtil::GetRandomBool(fuzzIndex);
84- parameter["enablePrefixCache"] = FuzzUtil::GetRandomBool(fuzzIndex);
85- parameter["enableKvQuant"] = FuzzUtil::GetRandomBool(fuzzIndex);
86- parameter["enableFA3"] = FuzzUtil::GetRandomBool(fuzzIndex);
87- parameter["enableReduceQuant"] = FuzzUtil::GetRandomBool(fuzzIndex);
88- parameter["normType"] = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 1);
89- parameter["positionEmbeddingType"] = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 1);
90- parameter["isUnpadInputs"] = FuzzUtil::GetRandomBool(fuzzIndex);
91- parameter["normEps"] = float(std::rand()) / RAND_MAX;
92- parameter["normType"] = float(std::rand()) / RAND_MAX;
93- int numAttentionHeadsPerRank = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
94- parameter["numAttentionHeadsPerRank"] = numAttentionHeadsPerRank;
95- int hiddenSizeLowerBound = 128;
96- int hiddenSizeUpperBound = 1024;
97- parameter["hiddenSizePerAttentionHead"] =
98- *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++],
99- hiddenSizeLowerBound,
100- hiddenSizeLowerBound,
101- hiddenSizeUpperBound);
102- parameter["numKeyValueHeadsPerRank"] = round(numAttentionHeadsPerRank /
103- *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, numAttentionHeadsPerRank));
104- parameter["numHiddenLayers"] = numHiddenLayers;
105- int worldSize = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
106- parameter["rank"] = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 0, 0, worldSize);
107- parameter["worldSize"] = worldSize;
108- std::vector<std::string> backendEnumTable = {"lccl", "hccl"};
109- parameter["backend"] = backendEnumTable[*(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 0, 0, 1)];
110- 
111- Aquila7BGetParamType(numHiddenLayers, parameter);
112- 
113- return parameter;
114-}
115- 
116-nlohmann::json Aquila7BGetAclParam(int numHiddenLayers, uint32_t &fuzzIndex)
117-{
118- nlohmann::json aclParameter;
119- 
120- int layerTokenOffset = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
121- std::vector<int> modelTokenOffset;
122- for (int layerId = 0; layerId < numHiddenLayers; layerId++) {
123- modelTokenOffset.push_back(layerTokenOffset);
124- }
125- aclParameter["tokenOffset"] = modelTokenOffset;
126- 
127- int layerSeqLen = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
128- std::vector<int> modelSeqLen;
129- for (int layerId = 0; layerId < numHiddenLayers; layerId++) {
130- modelSeqLen.push_back(layerSeqLen);
131- }
132- aclParameter["seqLen"] = modelSeqLen;
133- 
134- return aclParameter;
135-}
136- 
137-TEST(Aquila7BModelDTFuzz, PagedAttentionRopeModel)
138-{
139- std::srand(time(NULL));
140- ATB_SPEED_LOG_DEBUG("begin====================");
141- std::string fuzzName = "Aquila7BModelDTFuzzPagedAttentionRopeModel";
142- pybind11::module sys = pybind11::module_::import("sys");
143- sys.attr("path").attr("append")("tests/fuzztest/core/python");
144- pybind11::module modelModule = pybind11::module_::import("baichuan_fuzz");
145- 
146- DT_FUZZ_START(0, 100, const_cast<char*>(fuzzName.c_str()), 0) {
147- uint32_t fuzzIndex = 0;
148- int numHiddenLayers = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 200);
149- 
150- nlohmann::json parameter, aclParameter;
151- parameter = Aquila7BGetParam(numHiddenLayers, fuzzIndex);
152- aclParameter = Aquila7BGetAclParam(numHiddenLayers, fuzzIndex);
153- 
154- std::string parameter_string = parameter.dump();
155- std::string acl_parameter_string = aclParameter.dump();
156- 
157- try {
158- pybind11::object llamaFuzz = modelModule.attr("BaiChuanFuzz");
159- pybind11::object llamaFuzzIns = llamaFuzz("aquila_7b_DecoderModel");
160- 
161- pybind11::object ret = llamaFuzzIns.attr("set_param")(parameter_string);
162- if (ret.cast<int>() == 0) {
163- llamaFuzzIns.attr("set_weight")();
164- llamaFuzzIns.attr("set_kv_cache")();
165- llamaFuzzIns.attr("execute")(acl_parameter_string);
166- }
167- } catch (const std::exception& e) {
168- std::cerr << e.what() << std::endl;
169- }
170- }
171- DT_FUZZ_END()
172- SUCCEED();
173-}
174- 
175-}
Dexamples/atb_models/tests/fuzztest/models/test_gptneox_pa_model.cpp+0-82
@@ -1,82 +0,0 @@
1-/**
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#include <gtest/gtest.h>
14-#include <atb/atb_infer.h>
15-#include <pybind11/embed.h>
16-#include <pybind11/stl.h>
17- 
18-#include "atb_speed/log.h"
19-#include "models/gptneox/20b/layer/decoder_layer.h"
20-#include "models/gptneox/20b/model/pa_model.h"
21-#include "nlohmann/json.hpp"
22-#include "../utils/fuzz_utils.h"
23-#include "secodeFuzz.h"
24- 
25-namespace atb_speed {
26-TEST(GPTNeoxModelDTFuzz, PAModel)
27-{
28- std::srand(time(NULL));
29- ATB_SPEED_LOG_DEBUG("begin====================");
30- std::string fuzzName = "GPTNeoxModelDTFuzzPAModel";
31- pybind11::module sys = pybind11::module_::import("sys");
32- sys.attr("path").attr("append")("tests/fuzztest/core/python");
33- pybind11::module modelModule = pybind11::module_::import("gptnoex_fuzz");
34- 
35- DT_FUZZ_START(0, 100, const_cast<char*>(fuzzName.c_str()), 0) {
36- uint32_t fuzzIndex = 0;
37- 
38- nlohmann::json parameter, aclParameter;
39- parameter["headNum"] = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 1);
40- int dk = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
41- parameter["dk"] = dk;
42- int layerNum = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 200);
43- parameter["layerNum"] = layerNum;
44- parameter["layerNormEps"] = 0.1;
45- parameter["rotaryPct"] = float(std::rand()) / RAND_MAX;
46- int rankSize = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
47- parameter["rank"] = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 0, 0, rankSize);
48- parameter["rankSize"] = rankSize;
49- parameter["rmsNormEps"] = 0.1;
50- parameter["isPrefill"] = FuzzUtil::GetRandomBool(fuzzIndex);
51- parameter["qScale"] = float(std::rand()) / RAND_MAX;
52- parameter["qkScale"] = float(std::rand()) / RAND_MAX;
53- std::vector<std::string> backendEnumTable = {"lccl", "hccl"};
54- parameter["backend"] = backendEnumTable[*(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 0, 0, 1)];
55- 
56- int layerSeqLen = *(int *) DT_SetGetNumberRange(&g_Element[fuzzIndex++], 1, 1, 8);
57- std::vector<int> modelSeqLen;
58- for (int layerId = 0; layerId < layerNum; layerId++) {
59- modelSeqLen.push_back(layerSeqLen);
60- }
61- aclParameter["seqLen"] = modelSeqLen;
62- 
63- std::string parameter_string = parameter.dump();
64- std::string acl_parameter_string = aclParameter.dump();
65- try {
66- pybind11::object llamaFuzz = modelModule.attr("GPTNoexFuzz");
67- pybind11::object llamaFuzzIns = llamaFuzz("gptneox_20b_PAModel");
68- 
69- pybind11::object ret = llamaFuzzIns.attr("set_param")(parameter_string);
70- if (ret.cast<int>() == 0) {
71- llamaFuzzIns.attr("set_weight")();
72- llamaFuzzIns.attr("set_kv_cache")();
73- llamaFuzzIns.attr("execute")(acl_parameter_string);
74- }
75- } catch (const std::exception& e) {
76- std::cerr << e.what() << std::endl;
77- }
78- }
79- DT_FUZZ_END()
80- SUCCEED();
81-}
82-}
Dexamples/atb_models/tools/hello/CMakeLists.txt+0-19
@@ -1,19 +0,0 @@
1-file(GLOB_RECURSE SOURCE_FILES "${CMAKE_CURRENT_LIST_DIR}/*.cpp")
2-add_executable(acltransformer_hello ${SOURCE_FILES})
3- 
4-if(USE_TORCH_RUNNER)
5- target_link_libraries(acltransformer_hello PRIVATE acltransformer dl
6- torch c10 torch_cpu torch_python torch_npu python3 pthread)
7-else()
8- target_link_libraries(acltransformer_hello PRIVATE acltransformer dl pthread)
9-endif()
10- 
11-if(USE_LCCL_RUNNER)
12- target_link_libraries(acltransformer_hello PRIVATE lccl ascendcl)
13-endif()
14- 
15-if(USE_HCCL_RUNNER)
16- target_link_libraries(acltransformer_hello PRIVATE hccl pthread)
17-endif()
18- 
19-install(TARGETS acltransformer_hello DESTINATION bin)
Dexamples/atb_models/tools/hello/main.cpp+0-125
@@ -1,125 +0,0 @@
1-/**
2- * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
3- * MindIE is licensed under Mulan PSL v2.
4- * You can use this software according to the terms and conditions of the Mulan PSL v2.
5- * You may obtain a copy of Mulan PSL v2 at:
6- * http://license.coscl.org.cn/MulanPSL2
7- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10- * See the Mulan PSL v2 for more details.
11- */
12- 
13-#include <iostream>
14-#ifdef USE_TORCH_RUNNER
15-#include <torch_npu/csrc/core/npu/NPUCachingAllocator.h>
16-#endif
17-#include <asdops/utils/rt/rt.h>
18-#include <asdops/utils/log/log.h>
19-#include "acltransformer/operation.h"
20-#include "acltransformer/ops/add_operation.h"
21-#include "acltransformer/operation_call.h"
22-#include "acltransformer/utils/tensor_util.h"
23- 
24-using namespace AclTransformer;
25- 
26-static void SetCurrentDevice()
27-{
28- const int deviceId = 0;
29- ASD_LOG(DEBUG) << "AsdRtDeviceSetCurrent " << deviceId;
30- int ret = AsdRtDeviceSetCurrent(deviceId);
31- if (ret != ASDRT_SUCCESS) {
32- ASD_LOG(ERROR) << "AsdRtDeviceSetCurrent fail, error:" << ret;
33- return;
34- }
35- ASD_LOG(DEBUG) << "AsdRtDeviceSetCurrent success";
36-}
37- 
38-static void CreateTensor(AsdOps::Tensor &tensor)
39-{
40- tensor.dataSize = TensorUtil::CalcTensorDataSize(tensor);
41- int ret = AsdRtMemMallocDevice(&tensor.data, tensor.dataSize, ASDRT_MEM_DEFAULT);
42- ASD_LOG_IF(ret != ASDRT_SUCCESS, ERROR) << "AsdRtMemMallocDevice fail, ret:" << ret;
43-}
44- 
45-static void CreateTensors(AsdOps::SVector<AsdOps::Tensor> &inTensors, AsdOps::SVector<AsdOps::Tensor> &outTensors)
46-{
47- AsdOps::TensorDesc desc = {AsdOps::TENSOR_DTYPE_FLOAT16, AsdOps::TENSOR_FORMAT_NCHW, {2, 3}};
48- 
49- const int inTensorCount = 2;
50- inTensors.resize(inTensorCount);
51- 
52- AsdOps::Tensor &inTensor0 = inTensors.at(0);
53- inTensor0.desc = desc;
54- CreateTensor(inTensor0);
55- 
56- AsdOps::Tensor &inTensor1 = inTensors.at(1);
57- inTensor1.desc = desc;
58- CreateTensor(inTensor1);
59- 
60- outTensors.resize(1);
61- AsdOps::Tensor &outTensor0 = outTensors.at(0);
62- outTensor0.desc = desc;
63- CreateTensor(outTensor0);
64-}
65- 
66-static void FreeTensor(AsdOps::Tensor &tensor)
67-{
68- if (tensor.data) {
69- int ret = AsdRtMemFreeDevice(tensor.data);
70- ASD_LOG_IF(ret != ASDRT_SUCCESS, ERROR) << "AsdRtMemFreeDevice fail";
71- tensor.data = nullptr;
72- tensor.dataSize = 0;
73- }
74-}
75- 
76-static void FreeTensors(AsdOps::SVector<AsdOps::Tensor> &inTensors, AsdOps::SVector<AsdOps::Tensor> &outTensors)
77-{
78- for (size_t i = 0; i < inTensors.size(); ++i) {
79- FreeTensor(inTensors.at(i));
80- }
81- for (size_t i = 0; i < outTensors.size(); ++i) {
82- FreeTensor(outTensors.at(i));
83- }
84-}
85- 
86-static void InitTorch()
87-{
88-#ifdef USE_TORCH_RUNNER
89- ASD_LOG(DEBUG) << "c10_npu::NPUCachingAllocator::init";
90- c10_npu::NPUCachingAllocator::init();
91-#endif
92-}
93- 
94-int main(int argc, const char *argv[])
95-{
96- SetCurrentDevice();
97- 
98- AclTransformer::OperationCall opCall("AddOperation", AclTransformer::AddParam());
99- 
100- void *stream = nullptr;
101- int ret = AsdRtStreamCreate(&stream);
102- if (ret != 0) {
103- ASD_LOG(ERROR) << "AsdRtStreamCreate fail, ret:" << ret;
104- return 1;
105- }
106- 
107- InitTorch();
108- 
109- AsdOps::SVector<AsdOps::Tensor> inTensors;
110- AsdOps::SVector<AsdOps::Tensor> outTensors;
111- CreateTensors(inTensors, outTensors);
112- 
113- ret = opCall.ExecuteSync(inTensors, outTensors, stream);
114- if (ret != 0) {
115- ASD_LOG(ERROR) << "ExecuteSync fail, ret:" << ret;
116- } else {
117- ASD_LOG(DEBUG) << "ExecuteSync success";
118- }
119- 
120- AsdRtStreamDestroy(stream);
121- 
122- FreeTensors(inTensors, outTensors);
123- 
124- return 0;
125-}
Dexamples/ms_models/README.md+0-1
@@ -1 +0,0 @@
1-# ms_models
Dexamples/pt_models/mindietorch_models/README.md+0-1
@@ -1 +0,0 @@
1-# mindietorch_models
Dexamples/pt_models/torchair_models/README.md+0-1
@@ -1 +0,0 @@
1-# torchair_models
Dmindie_llm/text_generator/plugins/la/README.md+0-130
@@ -1,130 +0,0 @@
1-# Lookahead运行方法
2- 
3-## 1. 当前支持模型
4-qwen-7B
5- 
6-## 2. 使用run_generator.py运行
7-### 2.1 执行脚本代码示例
8- export PYTHONPATH=/LLM代码路径/examples/atb_models/:$PYTHONPATH
9- export PYTHONPATH=/LLM代码路径/:$PYTHONPATH
10- 
11- # 设置执行时运行在哪些NPU上
12- export ASCEND_RT_VISIBLE_DEVICES=0,1,2
13- 
14- export INF_NAN_MODE_ENABLE=0
15- export ATB_OPERATION_EXECUTE_ASYNC=1
16- export TASK_QUEUE_ENABLE=1
17- export ATB_CONVERT_NCHW_TO_ND=1
18- 
19- export LCCL_DETERMINISTIC=0
20- export HCCL_DETERMINISTIC=false
21- export ATB_MATMUL_SHUFFLE_K_ENABLE=1
22- export ATB_LLM_LCOC_ENABLE=1
23- 
24- # 适配长序列场景的环境变量
25- export LONG_SEQ_ENABLE=1
26- 
27- # LOG开关按需设置（推荐使用）
28- export MINDIE_LOG_LEVEL=INFO
29- export MINDIE_LOG_TO_FILE=1
30- export MINDIE_LOG_TO_STDOUT=0
31- 
32- # LOG开关按需打开, 若需要打印LA优化效果数据，LOG_LEVEL可设置为DEBUG
33- export MINDIE_LOG_LEVEL=0
34- export MINDIE_LOG_TO_FILE=0
35- export MINDIE_LOG_TO_STDOUT=1
36- export ASDOPS_LOG_LEVEL=0
37- export ASDOPS_LOG_TO_FILE=0
38- 
39- # 1表示执行的NPU数目，16表示最大batchsize，可按需调整
40- torchrun \
41- --nproc_per_node 1 \
42- --master_port 25632 \
43- -m mindie_llm.examples.run_generator \
44- --model_path /模型路径/ \
45- --max_batch_size 16 \
46- --max_output_length 20 \
47- --npu_mem 10 \
48- --plugin_params "{\"plugin_type\": \"la\", \"level\": 4, \"window\": 5, \"guess_set_size\": 5}" \
49- --speculation_gamma 30
50- 
51-### 2.2 参数配置
52-参数在run_generator.py中修改，主要涉及的参数如下（推荐配置，可按需修改）：
53- 
54-model_config中：
55- 
56- plugin_type: la
57- level: 4
58- window: 5
59- guess_set_size: 5
60- 
61- # 若出现OOM可适当调小npu_men
62- npu_mem:10
63- 
64-parse_arguments的配置示例
65- 
66- '--input_texts'：
67- default=["登鹳雀楼->王之涣\n夜雨寄北->", "Hamlet->Shakespeare\nOne hundred Years of Solitude->","登鹳雀楼->王之涣\n夜雨寄北->", "Hamlet->Shakespeare\nOne hundred Years of Solitude->"]
68- 
69- '--max_output_length'：256
70- 
71-### 2.3 运行
72-1. 编译：代码根目录下执行 bash scripts/build.sh
73-2. source output/set_env.sh
74-3. 执行2.1中的脚本。
75- 
76-## 3. 连接服务化执行
77- 
78-### 3.1 连接调度 llm_engin_test
79-#### 3.1.1 config文件配置
80-此处只呈现启动LA需要额外配置的字段
81- 
82-"ModelDeployConfig"下添加：
83-
84- "speculationGamma": 30
85-"ModelDeployConfig"中的"ModelConfig"下添加：
86- 
87- "plugin_params": "{\"plugin_type\":\"la\",\"level\": 4,\"window\": 5,\"guess_set_size\": 5}"
88-
89- 
90-[说明]<br>
91-N: level<br>
92-W: window<br>
93-G: guess_set_size<br>
94-speculationGamma: (N-1)*(W+G)
95- 
96-#### 3.1.2 执行
97- ./llm_engin_test token_input.csv
98-其中，.csv文件为输入token文件
99- 
100-### 3.2 对话验证 mindieservice_daemon
101-#### 3.2.1 config配置
102-同3.1.1
103-#### 3.2.2 执行
104-1. 执行./mindieservice_daemon
105-2. 执行对话请求<br>
106-对话请求demo代码如下：
107- 
108- curl -H "Accept: application/json" -H "Content-type: application/json" -X POST -d '{
109- "inputs": "登鹳雀楼->王之涣\n夜雨寄北->"
110- "parameters": {
111- "best_of": 1,
112- "decoder_input_details": false,
113- "details": false,
114- "do_sample": false,
115- "max_new_tokens": 256,
116- "repetition_penalty": 1,
117- "return_full_text": false,
118- "seed": null,
119- "stop":[
120- "photographer"
121- ],
122- "temperature": null,
123- "top_k": null,
124- "top_n_tokens":null,
125- "top_p":null,
126- "truncate": null,
127- "typical_p":null,
128- "watermark": true
129- },
130- "stream":false}' http://127.0.0.1:1025/generate
Dmindie_llm/text_generator/plugins/memory_decoding/README.md+0-136
@@ -1,136 +0,0 @@
1-# MemoryDecoding运行方法
2- 
3-## 0. Memory Decoding简介
4- 本算法为一种并行解码类算法，中文名称为基于动态token知识库检索的并行解码算法。
5- 其算法原理为，通过trie-tree（前缀树）的方式，缓存模型历史的输入输出，在解码阶段，从trie-tree中获取草稿token，并用作并行解码，以此提升模型的推理速度。
6- 
7- 
8-## 1. 适用场景
9- 
10- * 用户问答具有一定结构性的场景，如代码生成，代码续写等
11- * 可从prompt中获取参考的问答场景，如多轮对话、RAG、长文本等
12- * 用户问答间具有较高的相似性的场景
13- 
14- 
15-## 2. 使用run_generator.py运行
16-### 2.1 执行脚本代码示例
17- export PYTHONPATH=/LLM代码路径/examples/atb_models/:$PYTHONPATH
18- export PYTHONPATH=/LLM代码路径/:$PYTHONPATH
19- 
20- # 设置执行时运行在哪些NPU上
21- export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3
22- 
23- export INF_NAN_MODE_ENABLE=0
24- export ATB_CONVERT_NCHW_TO_ND=1
25- export LCCL_DETERMINISTIC=1
26- 
27- # LOG开关按需设置（可按需调整）
28- export MINDIE_LOG_LEVEL=INFO
29- export MINDIE_LOG_TO_FILE=1
30- export MINDIE_LOG_TO_STDOUT=1
31- 
32- # 1表示执行的NPU数目，16表示最大batchsize，可按需调整
33- torchrun \
34- --nproc_per_node 1 \
35- --master_port 25632 \
36- -m mindie_llm.examples.run_generator \
37- --model_path /模型路径/ \
38- --max_batch_size 16 \
39- --max_output_length 20 \
40- --npu_mem 10 \
41- --plugin_params "{\"plugin_type\": \"memory_decoding\", \"decoding_length\": 16}" \
42- --speculation_gamma 16
43- 
44-### 2.2 参数配置
45-参数在run_generator.py中修改，主要涉及的参数如下（推荐配置，可按需修改）：
46- 
47-model_config中：
48- 
49- plugin_type: memory_decoding
50- decoding_length: 16
51- 
52- # 若出现OOM可适当调小npu_men
53- npu_mem:10
54- 
55-parse_arguments的配置示例
56- 
57- '--input_texts'：
58- default=["What's deep learning?", "How do you feel?", "What's deep learning?", "How do you feel?"]
59- 
60- '--max_output_length'：256
61- 
62-### 2.3 运行
63-在Mindie_llm代码根目录下编译后执行3.1中的脚本即可。
64- 
65-## 3. 使用服务化执行
66- 
67-### 3.1 使用llm_engin_test执行性能测试
68-#### 3.1.1 config文件配置
69-"ModelDeployConfig"下添加：
70-
71- "speculationGamma": 16
72-"ModelDeployConfig"中的"ModelConfig"下添加：
73- 
74- "plugin_params": "{\"plugin_type\":\"memory_decoding\",\"decoding_length\": 16,\"dynamic_algo\": true}"
75-
76- 
77-#### 3.1.2 执行
78- ./llm_engin_test token_input.csv
79-其中，.csv文件为输入token文件
80- 
81-### 3.2 对话验证 mindieservice_daemon
82-#### 3.2.1 config配置
83-同3.1.1
84-#### 3.2.2 执行
85-1. 执行./mindieservice_daemon
86-2. 执行对话请求<br>
87-对话请求demo代码如下：
88- 
89- curl -H "Accept: application/json" -H "Content-type: application/json" -X POST -d '{
90- "inputs": "What's deep learning?"
91- "parameters": {
92- "best_of": 1,
93- "decoder_input_details": false,
94- "details": false,
95- "do_sample": false,
96- "max_new_tokens": 256,
97- "repetition_penalty": 1,
98- "return_full_text": false,
99- "seed": null,
100- "stop":[
101- "photographer"
102- ],
103- "temperature": null,
104- "top_k": null,
105- "top_n_tokens":null,
106- "top_p":null,
107- "truncate": null,
108- "typical_p":null,
109- "watermark": true
110- },
111- "stream":false}' http://127.0.0.1:1025/generate
112- 
113-## 参考
114- [1] @misc{zhao2024lookahead,
115- title={Lookahead: An Inference Acceleration Framework for Large Language Model with Lossless Generation Accuracy},
116- author={Yao Zhao and Zhitian Xie and Chen Liang and Chenyi Zhuang and Jinjie Gu},
117- year={2024},
118- eprint={2312.12728},
119- archivePrefix={arXiv},
120- primaryClass={id='cs.IR' full_name='Information Retrieval' is_active=True alt_name=None in_archive='cs' is_general=False description='Covers indexing, dictionaries, retrieval, content and analysis. Roughly includes material in ACM Subject Classes H.3.0, H.3.1, H.3.2, H.3.3, and H.3.4.'}}
121- [2] @misc{he2024rest,
122- title={REST: Retrieval-Based Speculative Decoding},
123- author={Zhenyu He and Zexuan Zhong and Tianle Cai and Jason D. Lee and Di He},
124- year={2024},
125- eprint={2311.08252},
126- archivePrefix={arXiv},
127- primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'}}
128- [3] @misc{yang2023inference,
129- title={Inference with Reference: Lossless Acceleration of Large Language Models},
130- author={Nan Yang and Tao Ge and Liang Wang and Binxing Jiao and Daxin Jiang and Linjun Yang and Rangan Majumder and Furu Wei},
131- year={2023},
132- eprint={2304.04487},
133- archivePrefix={arXiv},
134- primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'}}
135- 
136-