已合并
internvl自动化用例服务拉起失败bug fix #1085
Snoopy99创建于 6月30日
internvl自动化用例服务拉起失败bug fix #1085
已合并
共 3 个文件变更+17-3
| @@ -10,7 +10,7 @@ | |||
| 10 | from atb_llm.models.base.graph_manager.graph_wrapper import ATBGraphWrapper | 10 | from atb_llm.models.base.graph_manager.graph_wrapper import ATBGraphWrapper |
| 11 | from atb_llm.models.base.graph_manager.compatible_matrix import FeatureType | 11 | from atb_llm.models.base.graph_manager.compatible_matrix import FeatureType |
| 12 | from atb_llm.models.base.flash_causal_lm import FlashForCausalLM | 12 | from atb_llm.models.base.flash_causal_lm import FlashForCausalLM |
| 13 | -from atb_llm.runner.model_runner import generate_mem_pool_event_key | 13 | +from atb_llm.utils.mempool_utils import generate_mem_pool_event_key |
| 14 | from mindie_llm.text_generator.plugins.plugin_manager import MemPoolType | 14 | from mindie_llm.text_generator.plugins.plugin_manager import MemPoolType |
| 15 | 15 | ||
| 16 | 16 | ||
| @@ -10,11 +10,12 @@ | |||
| 10 | from atb_llm.models.base.graph_manager.graph_wrapper import ATBGraphWrapper | 10 | from atb_llm.models.base.graph_manager.graph_wrapper import ATBGraphWrapper |
| 11 | from atb_llm.models.base.graph_manager.compatible_matrix import FeatureType | 11 | from atb_llm.models.base.graph_manager.compatible_matrix import FeatureType |
| 12 | from atb_llm.models.base.flash_causal_lm import FlashForCausalLM | 12 | from atb_llm.models.base.flash_causal_lm import FlashForCausalLM |
| 13 | -from atb_llm.runner.model_runner import generate_mem_pool_event_key | 13 | +from atb_llm.utils.mempool_utils import generate_mem_pool_event_key |
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | class SplitFuseGraphWrapper(ATBGraphWrapper): | 16 | class SplitFuseGraphWrapper(ATBGraphWrapper): |
| 17 | "ATBGraphWrapper class for prefixcache and splitfuse" | 17 | "ATBGraphWrapper class for prefixcache and splitfuse" |
| 18 | + | ||
| 18 | pipe_key: str = generate_mem_pool_event_key(only_save_kv=False) | 19 | pipe_key: str = generate_mem_pool_event_key(only_save_kv=False) |
| 19 | 20 | ||
| 20 | def __init__(self): | 21 | def __init__(self): |
| @@ -24,7 +25,7 @@ class SplitFuseGraphWrapper(ATBGraphWrapper): | |||
| 24 | 25 | ||
| 25 | def activate(self, context: FlashForCausalLM, runtime_params, **kwargs) -> bool: | 26 | def activate(self, context: FlashForCausalLM, runtime_params, **kwargs) -> bool: |
| 26 | pa_enable = False if context.inference_mode is None else context.inference_mode.enable_prefill_pa | 27 | pa_enable = False if context.inference_mode is None else context.inference_mode.enable_prefill_pa |
| 27 | - q_lens = "\"qLen\"" in runtime_params | 28 | + q_lens = '"qLen"' in runtime_params |
| 28 | is_prefill = kwargs.get("is_prefill", False) | 29 | is_prefill = kwargs.get("is_prefill", False) |
| 29 | if q_lens and is_prefill and pa_enable: | 30 | if q_lens and is_prefill and pa_enable: |
| 30 | return True | 31 | return True |
| @@ -0,0 +1,13 @@ | |||
| 1 | +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. | ||
| 2 | +# MindIE is licensed under Mulan PSL v2. | ||
| 3 | +# You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 4 | +# You may obtain a copy of Mulan PSL v2 at: | ||
| 5 | +# http://license.coscl.org.cn/MulanPSL2 | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 7 | +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 8 | +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 9 | +# See the Mulan PSL v2 for more details. | ||
| 10 | + | ||
| 11 | + | ||
| 12 | +def generate_mem_pool_event_key(only_save_kv: bool) -> str: | ||
| 13 | + return "only_save_kv" if only_save_kv else "both_save_kv" | ||