已合并
Revert "修复ruff-check检查出来的规范性问题" #126
吴铭泾创建于 3月10日
Revert "修复ruff-check检查出来的规范性问题" #126
已合并
从已删除 :dev合入到Ascend/MindIE-Motor-CPPdev
共 46 个文件变更+144-115
| @@ -119,10 +119,10 @@ def _create_log_file(log_file): | |||
| 119 | else: | 119 | else: |
| 120 | clean_path = os.path.normpath(log_file) | 120 | clean_path = os.path.normpath(log_file) |
| 121 | if os.path.islink(clean_path): | 121 | if os.path.islink(clean_path): |
| 122 | - err_msg = "Check log file path failed because it's a symbolic." | 122 | + err_msg = f"Check log file path failed because it's a symbolic." |
| 123 | raise ValueError(err_msg) | 123 | raise ValueError(err_msg) |
| 124 | if len(clean_path) > 1024: | 124 | if len(clean_path) > 1024: |
| 125 | - err_msg = "Path of log file is too long, it should not exceed 1024 character." | 125 | + err_msg = f"Path of log file is too long, it should not exceed 1024 character." |
错误信息准确性: 第125行的错误消息中使用了单数形式的'character',但实际检查的是路径长度是否超过1024个字符。当路径长度超过限制时,应该使用复数形式'characters'更符合英语语法习惯。虽然这是一个小问题,但在国际化或面向英语用户的项目中,这种细节会影响专业性。 问题类型: 错误信息准确性 文件路径: mindie_motor/python/mindie_motor/node_manager/common/logging.py行号: 125 问题代码: err_msg = f"Path of log file is too long, it should not exceed 1024 character." 修改建议: 将错误消息中的'character'改为复数形式'characters':err_msg = f"Path of log file is too long, it should not exceed 1024 characters." --- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 126 | raise ValueError(err_msg) | 126 | raise ValueError(err_msg) |
| 127 | os.chmod(log_file, mode) | 127 | os.chmod(log_file, mode) |
| 128 | 128 | ||
| @@ -1,9 +1,12 @@ | |||
| 1 | # Copyright Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. | 1 | # Copyright Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. |
| 2 | +import argparse | ||
| 2 | import os | 3 | import os |
| 4 | +from itertools import accumulate | ||
| 3 | from dataclasses import dataclass | 5 | from dataclasses import dataclass |
| 4 | import logging | 6 | import logging |
| 5 | -from typing import List, Dict, Union | 7 | +from typing import List, Dict, Tuple, Union |
| 6 | import subprocess | 8 | import subprocess |
| 9 | +import psutil | ||
| 7 | 10 | ||
| 8 | 11 | ||
| 9 | class ENV: | 12 | class ENV: |
| @@ -11,6 +11,7 @@ | |||
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | import os | 13 | import os |
| 14 | +import json | ||
| 14 | import ipaddress | 15 | import ipaddress |
| 15 | from node_manager.common.utils import _SingletonMeta, validate_port_range | 16 | from node_manager.common.utils import _SingletonMeta, validate_port_range |
| 16 | from node_manager.common.logging import Log | 17 | from node_manager.common.logging import Log |
| @@ -41,12 +42,12 @@ class GeneralConfig(metaclass=_SingletonMeta): | |||
| 41 | 42 | ||
| 42 | def _check_server_config_valid(config): | 43 | def _check_server_config_valid(config): |
| 43 | if SERVER_CONFIG not in config: | 44 | if SERVER_CONFIG not in config: |
| 44 | - raise KeyError("Invalid Server Engine Config File") | 45 | + raise KeyError(f"Invalid Server Engine Config File") |
| 45 | server_config = config[SERVER_CONFIG] | 46 | server_config = config[SERVER_CONFIG] |
| 46 | if MANAGEMENT_PORT not in server_config: | 47 | if MANAGEMENT_PORT not in server_config: |
| 47 | - raise KeyError("Management Port is Missing in Server Engine Config File") | 48 | + raise KeyError(f"Management Port is Missing in Server Engine Config File") |
| 48 | if "distDPServerEnabled" not in server_config or not isinstance(server_config["distDPServerEnabled"], bool): | 49 | if "distDPServerEnabled" not in server_config or not isinstance(server_config["distDPServerEnabled"], bool): |
| 49 | - raise KeyError("distDPServerEnabled is Missing or Invalid in Server Engine Config File") | 50 | + raise KeyError(f"distDPServerEnabled is Missing or Invalid in Server Engine Config File") |
代码可读性: 异常消息使用f-string但未包含变量,与前两行问题相同。原字符串'distDPServerEnabled is Missing or Invalid in Server Engine Config File'已经足够清晰。 问题类型: 代码可读性 文件路径: mindie_motor/python/mindie_motor/node_manager/core/config.py行号: 50 问题代码: raise KeyError(f"distDPServerEnabled is Missing or Invalid in Server Engine Config File") 修改建议: 移除f-string前缀:raise KeyError("distDPServerEnabled is Missing or Invalid in Server Engine Config File")--- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 50 | 51 | ||
| 51 | 52 | ||
| 52 | def _load_server_engine_ip(): | 53 | def _load_server_engine_ip(): |
| @@ -55,7 +56,7 @@ class GeneralConfig(metaclass=_SingletonMeta): | |||
| 55 | raise KeyError("[Config] Please Check Environment Variable Configuration of POD_IP") | 56 | raise KeyError("[Config] Please Check Environment Variable Configuration of POD_IP") |
| 56 | address = ipaddress.ip_address(pod_ip) | 57 | address = ipaddress.ip_address(pod_ip) |
| 57 | if address.version != IPV4 and address.version != IPV6: | 58 | if address.version != IPV4 and address.version != IPV6: |
| 58 | - raise Exception('[Config] POD_IP is not ipv4 or ipv6.') | 59 | + raise Exception(f'[Config] POD_IP is not ipv4 or ipv6.') |
安全编码规范: 在异常消息中使用了f-string,但消息内容为静态字符串,没有使用任何变量插值。虽然这里使用了单引号字符串,但同样存在不必要的f-string前缀问题。 问题类型: 安全编码规范 文件路径: mindie_motor/python/mindie_motor/node_manager/core/config.py行号: 59 问题代码: raise Exception(f'[Config] POD_IP is not ipv4 or ipv6.') 修改建议: 移除不必要的f-string前缀,改为普通字符串:raise Exception('[Config] POD_IP is not ipv4 or ipv6.')--- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 59 | return pod_ip | 60 | return pod_ip |
| 60 | 61 | ||
| 61 | 62 | ||
| @@ -20,6 +20,7 @@ from node_manager.common.logging import Log | |||
| 20 | from node_manager.models.enums import ( | 20 | from node_manager.models.enums import ( |
| 21 | EngineCmd, | 21 | EngineCmd, |
| 22 | NodeRunningStatus, | 22 | NodeRunningStatus, |
| 23 | + ServiceStatus, | ||
| 23 | ControllerCmd, | 24 | ControllerCmd, |
| 24 | ) | 25 | ) |
| 25 | from node_manager.common.utils import _SingletonMeta | 26 | from node_manager.common.utils import _SingletonMeta |
| @@ -151,7 +152,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 151 | """ | 152 | """ |
| 152 | self.init_heartbeat_mng() | 153 | self.init_heartbeat_mng() |
| 153 | self.logger.info( | 154 | self.logger.info( |
| 154 | - "[_cmd_failed_further_action] Set running status = ABNORMAL" | 155 | + f"[_cmd_failed_further_action] Set running status = ABNORMAL" |
| 155 | ) | 156 | ) |
| 156 | self.heartbeat_mng.set_running_status(NodeRunningStatus.ABNORMAL.value) | 157 | self.heartbeat_mng.set_running_status(NodeRunningStatus.ABNORMAL.value) |
| 157 | 158 | ||
| @@ -174,7 +175,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 174 | - status (bool): Whether the command was executed successfully. | 175 | - status (bool): Whether the command was executed successfully. |
| 175 | - reason (str): Explanation of why the command succeeded or failed. | 176 | - reason (str): Explanation of why the command succeeded or failed. |
| 176 | """ | 177 | """ |
| 177 | - self.logger.info("FaultManager: pause engine starts.") | 178 | + self.logger.info(f"FaultManager: pause engine starts.") |
| 178 | self.init_heartbeat_mng() | 179 | self.init_heartbeat_mng() |
| 179 | _, after_state = self._find_matching_index( | 180 | _, after_state = self._find_matching_index( |
| 180 | cmd=ControllerCmd.PAUSE_ENGINE.value, | 181 | cmd=ControllerCmd.PAUSE_ENGINE.value, |
| @@ -191,14 +192,14 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 191 | ret_data = self._extract_info(ret_info_all, field=STATUS_STR) | 192 | ret_data = self._extract_info(ret_info_all, field=STATUS_STR) |
| 192 | if all(ret_data): | 193 | if all(ret_data): |
| 193 | self.logger.info( | 194 | self.logger.info( |
| 194 | - "FaultManager: Successfully executed the PAUSE ENGINE cmd." | 195 | + f"FaultManager: Successfully executed the PAUSE ENGINE cmd." |
| 195 | ) | 196 | ) |
| 196 | # 发送请求成功,命令执行成功 | 197 | # 发送请求成功,命令执行成功 |
| 197 | return {STATUS_STR: True, REASON_STR: None} | 198 | return {STATUS_STR: True, REASON_STR: None} |
| 198 | else: | 199 | else: |
| 199 | # 请求发送成功,命令执行有失败 | 200 | # 请求发送成功,命令执行有失败 |
| 200 | self.logger.error( | 201 | self.logger.error( |
| 201 | - "FaultManager: PAUSE ENGINE cmd sent successfully, but the cmd execution failed." | 202 | + f"FaultManager: PAUSE ENGINE cmd sent successfully, but the cmd execution failed." |
| 202 | ) | 203 | ) |
| 203 | self._set_heartbeat_check_allowed( | 204 | self._set_heartbeat_check_allowed( |
| 204 | True | 205 | True |
| @@ -213,7 +214,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 213 | else: | 214 | else: |
| 214 | # 请求发送失败 | 215 | # 请求发送失败 |
| 215 | # status表明当前命令的执行状态是否成功 | 216 | # status表明当前命令的执行状态是否成功 |
| 216 | - self.logger.error("FaultManager: Sending PAUSE ENGINE cmd failed.") | 217 | + self.logger.error(f"FaultManager: Sending PAUSE ENGINE cmd failed.") |
| 217 | self._set_heartbeat_check_allowed( | 218 | self._set_heartbeat_check_allowed( |
| 218 | True | 219 | True |
| 219 | ) # 默认下一步指令可能不发送cmd,启动心跳检测,kill异常pod | 220 | ) # 默认下一步指令可能不发送cmd,启动心跳检测,kill异常pod |
| @@ -230,7 +231,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 230 | } | 231 | } |
| 231 | 232 | ||
| 232 | def _reinit_npu(self) -> dict: | 233 | def _reinit_npu(self) -> dict: |
| 233 | - self.logger.info("FaultManager: REINIT NPU starts.") | 234 | + self.logger.info(f"FaultManager: REINIT NPU starts.") |
| 234 | self.init_heartbeat_mng() | 235 | self.init_heartbeat_mng() |
| 235 | _, after_state = self._find_matching_index( | 236 | _, after_state = self._find_matching_index( |
| 236 | cmd=ControllerCmd.REINIT_NPU.value, | 237 | cmd=ControllerCmd.REINIT_NPU.value, |
| @@ -248,7 +249,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 248 | else: | 249 | else: |
| 249 | # 不在pause_engine->reinit_npu->start_engine指令流中 | 250 | # 不在pause_engine->reinit_npu->start_engine指令流中 |
| 250 | self.logger.error( | 251 | self.logger.error( |
| 251 | - "not supported single cmd, must in pause_engine->reinit_npu->start_engine cmd stream" | 252 | + f"not supported single cmd, must in pause_engine->reinit_npu->start_engine cmd stream" |
| 252 | ) | 253 | ) |
| 253 | return { | 254 | return { |
| 254 | STATUS_STR: False, | 255 | STATUS_STR: False, |
| @@ -264,13 +265,13 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 264 | if all(ret_data): | 265 | if all(ret_data): |
| 265 | # 发送请求成功,命令执行成功 | 266 | # 发送请求成功,命令执行成功 |
| 266 | self.logger.info( | 267 | self.logger.info( |
| 267 | - "FaultManager: Successfully executed the REINIT NPU cmd." | 268 | + f"FaultManager: Successfully executed the REINIT NPU cmd." |
| 268 | ) | 269 | ) |
| 269 | self.heartbeat_mng.set_running_status(after_state) | 270 | self.heartbeat_mng.set_running_status(after_state) |
| 270 | else: | 271 | else: |
| 271 | # 发送请求成功,命令执行有失败 | 272 | # 发送请求成功,命令执行有失败 |
| 272 | self.logger.error( | 273 | self.logger.error( |
| 273 | - "FaultManager: REINIT NPU cmd sent successfully, but the cmd execution failed." | 274 | + f"FaultManager: REINIT NPU cmd sent successfully, but the cmd execution failed." |
| 274 | ) | 275 | ) |
| 275 | self._set_heartbeat_check_allowed( | 276 | self._set_heartbeat_check_allowed( |
| 276 | True | 277 | True |
| @@ -299,7 +300,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 299 | 300 | ||
| 300 | 301 | ||
| 301 | def _start_engine(self): | 302 | def _start_engine(self): |
| 302 | - self.logger.info("FaultManager: start engine starts.") | 303 | + self.logger.info(f"FaultManager: start engine starts.") |
| 303 | self.init_heartbeat_mng() | 304 | self.init_heartbeat_mng() |
| 304 | _, after_state = self._find_matching_index( | 305 | _, after_state = self._find_matching_index( |
| 305 | cmd=ControllerCmd.START_ENGINE.value, | 306 | cmd=ControllerCmd.START_ENGINE.value, |
| @@ -314,7 +315,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 314 | if all(ret_data): | 315 | if all(ret_data): |
| 315 | # 发送请求成功,命令执行成功 | 316 | # 发送请求成功,命令执行成功 |
| 316 | self.logger.info( | 317 | self.logger.info( |
| 317 | - "FaultManager: Successfully executed the START ENGINE cmd." | 318 | + f"FaultManager: Successfully executed the START ENGINE cmd." |
| 318 | ) | 319 | ) |
| 319 | self._set_heartbeat_check_allowed(True) | 320 | self._set_heartbeat_check_allowed(True) |
| 320 | self.heartbeat_mng.set_running_status(after_state) | 321 | self.heartbeat_mng.set_running_status(after_state) |
| @@ -322,7 +323,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 322 | else: | 323 | else: |
| 323 | # 发送请求成功,命令执行有失败 | 324 | # 发送请求成功,命令执行有失败 |
| 324 | self.logger.warning( | 325 | self.logger.warning( |
| 325 | - "FaultManager: START ENGINE cmd sent successfully, but the cmd execution failed." | 326 | + f"FaultManager: START ENGINE cmd sent successfully, but the cmd execution failed." |
| 326 | ) | 327 | ) |
| 327 | self._set_heartbeat_check_allowed( | 328 | self._set_heartbeat_check_allowed( |
| 328 | True | 329 | True |
| @@ -336,7 +337,7 @@ class FaultManager(metaclass=_SingletonMeta): | |||
| 336 | return {STATUS_STR: False, REASON_STR: reason_val_str} | 337 | return {STATUS_STR: False, REASON_STR: reason_val_str} |
| 337 | else: | 338 | else: |
| 338 | # 请求发送失败 | 339 | # 请求发送失败 |
| 339 | - self.logger.warning("FaultManager: Sending START ENGINE cmd failed.") | 340 | + self.logger.warning(f"FaultManager: Sending START ENGINE cmd failed.") |
| 340 | self._set_heartbeat_check_allowed( | 341 | self._set_heartbeat_check_allowed( |
| 341 | True | 342 | True |
| 342 | ) # 默认server不下发下一步指令,启动心跳检测,kill异常pod | 343 | ) # 默认server不下发下一步指令,启动心跳检测,kill异常pod |
| @@ -112,10 +112,10 @@ class NodeStatusMonitor(metaclass=_SingletonMeta): | |||
| 112 | # 在当前不执行指令时候才解析, 不会出现server engine状态不一致的问题 | 112 | # 在当前不执行指令时候才解析, 不会出现server engine状态不一致的问题 |
| 113 | engine_state_cur = self.engine_state[-1] # 首字段engine_state_cur[0]是时间戳 | 113 | engine_state_cur = self.engine_state[-1] # 首字段engine_state_cur[0]是时间戳 |
| 114 | if ServiceStatus.SERVICE_READY.value in engine_state_cur[1:]: | 114 | if ServiceStatus.SERVICE_READY.value in engine_state_cur[1:]: |
| 115 | - self.logger.error("[parse_engine_state] Contain SERVICE_READY in engine state while no CMD executing") | 115 | + self.logger.error(f"[parse_engine_state] Contain SERVICE_READY in engine state while no CMD executing") |
| 116 | return NodeRunningStatus.ABNORMAL.value | 116 | return NodeRunningStatus.ABNORMAL.value |
| 117 | if ServiceStatus.SERVICE_PAUSE.value in engine_state_cur[1:]: | 117 | if ServiceStatus.SERVICE_PAUSE.value in engine_state_cur[1:]: |
| 118 | - self.logger.error("[parse_engine_state] Contain SERVICE_PAUSE in engine state while no CMD executing") | 118 | + self.logger.error(f"[parse_engine_state] Contain SERVICE_PAUSE in engine state while no CMD executing") |
| 119 | return NodeRunningStatus.ABNORMAL.value | 119 | return NodeRunningStatus.ABNORMAL.value |
| 120 | if ServiceStatus.SERVICE_ABNORMAL.value in engine_state_cur[1:]: | 120 | if ServiceStatus.SERVICE_ABNORMAL.value in engine_state_cur[1:]: |
| 121 | return NodeRunningStatus.ABNORMAL.value | 121 | return NodeRunningStatus.ABNORMAL.value |
| @@ -137,15 +137,15 @@ class NodeStatusMonitor(metaclass=_SingletonMeta): | |||
| 137 | 137 | ||
| 138 | def _monitor_state(self): | 138 | def _monitor_state(self): |
| 139 | while self.running: | 139 | while self.running: |
| 140 | - self.logger.info("Monitoring EP status") | 140 | + self.logger.info(f"Monitering EP status") |
| 141 | result_all = self._query_ep_status() | 141 | result_all = self._query_ep_status() |
| 142 | if not self.heartbeat_mng.heartbeat_check_allowed: | 142 | if not self.heartbeat_mng.heartbeat_check_allowed: |
| 143 | # CMD在运行或者heartbeatmng在处理异常,不做状态更新和处理 | 143 | # CMD在运行或者heartbeatmng在处理异常,不做状态更新和处理 |
| 144 | - self.logger.info("while handling cmd, not update heartbeat") | 144 | + self.logger.info(f"while handling cmd, not update heartbeat") |
| 145 | time.sleep(self.query_interval) | 145 | time.sleep(self.query_interval) |
| 146 | continue | 146 | continue |
| 147 | if self.heartbeat_mng.get_running_status() == NodeRunningStatus.PAUSE.value: # 在处理异常 | 147 | if self.heartbeat_mng.get_running_status() == NodeRunningStatus.PAUSE.value: # 在处理异常 |
| 148 | - self.logger.error("heartBeatMng is Paused while no cmd is executing") | 148 | + self.logger.error(f"heartBeatMng is Paused while no cmd is executing") |
| 149 | time.sleep(self.query_interval) | 149 | time.sleep(self.query_interval) |
| 150 | continue | 150 | continue |
| 151 | now = datetime.now(timezone.utc).strftime("%Y/%m/%d %H:%M:%S") | 151 | now = datetime.now(timezone.utc).strftime("%Y/%m/%d %H:%M:%S") |
| @@ -283,10 +283,10 @@ class NodeStatusMonitor(metaclass=_SingletonMeta): | |||
| 283 | resp = self.client.send_alarm_info_to_ctrler(error_info) | 283 | resp = self.client.send_alarm_info_to_ctrler(error_info) |
| 284 | 284 | ||
| 285 | if resp.get(DATA_STR, {}).get("status", "") == str(ControllerReply.SEND_CONTROLLER_ALARM_SUCCESS.value): | 285 | if resp.get(DATA_STR, {}).get("status", "") == str(ControllerReply.SEND_CONTROLLER_ALARM_SUCCESS.value): |
| 286 | - self.logger.info("Has successfully send alarm to controller.") | 286 | + self.logger.info(f"Has successfully send alarm to controller.") |
| 287 | elif resp.get(DATA_STR, {}).get("status", "") == \ | 287 | elif resp.get(DATA_STR, {}).get("status", "") == \ |
| 288 | str(ControllerReply.SEND_CONTROLLER_ALARM_UNREACHEABLE.value): | 288 | str(ControllerReply.SEND_CONTROLLER_ALARM_UNREACHEABLE.value): |
| 289 | - self.logger.info("Failed to send alarm to controller, service unreachable") | 289 | + self.logger.info(f"Failed to send alarm to controller, service unreachable") |
| 290 | else: | 290 | else: |
| 291 | self.logger.error(f"Controller reply not found, ctrl_rpl={resp}") | 291 | self.logger.error(f"Controller reply not found, ctrl_rpl={resp}") |
| 292 | 292 | ||
| @@ -311,14 +311,14 @@ class HeartBeatMng(metaclass=_SingletonMeta): | |||
| 311 | 311 | ||
| 312 | def run(self): | 312 | def run(self): |
| 313 | if not GeneralConfig().has_endpoint: | 313 | if not GeneralConfig().has_endpoint: |
| 314 | - self.logger.info("Heartbeat Manager: No endpoint configured, skipping monitoring.") | 314 | + self.logger.info(f"Heartbeat Manager: No endpoint configured, skipping monitoring.") |
| 315 | return | 315 | return |
| 316 | - self.logger.info("Heartbeat Manager:Start monitoring engine state.") | 316 | + self.logger.info(f"Heartbeat Manager:Start monitoring engine state.") |
| 317 | self._status_monitor.start_monitoring() | 317 | self._status_monitor.start_monitoring() |
| 318 | 318 | ||
| 319 | def stop(self): | 319 | def stop(self): |
| 320 | self._status_monitor.stop_monitoring() | 320 | self._status_monitor.stop_monitoring() |
| 321 | - self.logger.info("Heartbeat Manager: Monitoring engine state stopped.") | 321 | + self.logger.info(f"Heartbeat Manager: Monitoring engine state stopped.") |
| 322 | 322 | ||
| 323 | def get_heartbeat_check_allowed(self) -> bool: | 323 | def get_heartbeat_check_allowed(self) -> bool: |
| 324 | return self.heartbeat_check_allowed | 324 | return self.heartbeat_check_allowed |
| @@ -15,7 +15,6 @@ import subprocess | |||
| 15 | import threading | 15 | import threading |
| 16 | import time | 16 | import time |
| 17 | from datetime import datetime, timedelta, timezone | 17 | from datetime import datetime, timedelta, timezone |
| 18 | -import importlib.util | ||
| 19 | 18 | ||
| 20 | from node_manager.common.utils import _SingletonMeta | 19 | from node_manager.common.utils import _SingletonMeta |
| 21 | from node_manager.common.logging import Log | 20 | from node_manager.common.logging import Log |
| @@ -46,7 +45,11 @@ class RuntimeParamChecker(metaclass=_SingletonMeta): | |||
| 46 | 45 | ||
| 47 | 46 | ||
| 48 | def has_msprechecker(): | 47 | def has_msprechecker(): |
| 49 | - return importlib.util.find_spec('msprechecker') is not None | 48 | + try: |
| 49 | + import msprechecker | ||
| 50 | + return True | ||
| 51 | + except ImportError: | ||
| 52 | + return False | ||
| 50 | 53 | ||
| 51 | 54 | ||
| 52 | def get_check_rule_path(): | 55 | def get_check_rule_path(): |
| @@ -80,7 +80,7 @@ class BaseDaemonManager(ABC): | |||
| 80 | try: | 80 | try: |
| 81 | pid, status = os.waitpid(-1, os.WNOHANG) | 81 | pid, status = os.waitpid(-1, os.WNOHANG) |
| 82 | except OSError: | 82 | except OSError: |
| 83 | - self.logger.error("No more child processes") | 83 | + self.logger.error(f"No more child processes") |
| 84 | return | 84 | return |
| 85 | exit_flag = pid > 0 | 85 | exit_flag = pid > 0 |
| 86 | while pid > 0: | 86 | while pid > 0: |
| @@ -13,6 +13,7 @@ | |||
| 13 | import ipaddress | 13 | import ipaddress |
| 14 | import os | 14 | import os |
| 15 | import socket | 15 | import socket |
| 16 | +import ssl | ||
| 16 | 17 | ||
| 17 | import uvicorn | 18 | import uvicorn |
| 18 | from fastapi import FastAPI, Request, HTTPException | 19 | from fastapi import FastAPI, Request, HTTPException |
| @@ -22,6 +23,7 @@ from node_manager.common.utils import _SingletonMeta | |||
| 22 | from node_manager.routes.server_api import router | 23 | from node_manager.routes.server_api import router |
| 23 | from node_manager.common.logging import Log | 24 | from node_manager.common.logging import Log |
| 24 | from node_manager.framework.utils import CertUtil | 25 | from node_manager.framework.utils import CertUtil |
| 26 | +from node_manager.framework.utils.cert_utils import CA_CERTS, TLS_CERT, TLS_KEY | ||
| 25 | 27 | ||
| 26 | logger = Log(__name__).getlog() | 28 | logger = Log(__name__).getlog() |
| 27 | 29 | ||
| @@ -15,6 +15,7 @@ import sys | |||
| 15 | import stat | 15 | import stat |
| 16 | import ssl | 16 | import ssl |
| 17 | import ctypes | 17 | import ctypes |
| 18 | +from ctypes import c_char_p | ||
| 18 | from OpenSSL import crypto | 19 | from OpenSSL import crypto |
| 19 | 20 | ||
| 20 | from node_manager.common.utils import PathCheck, safe_open | 21 | from node_manager.common.utils import PathCheck, safe_open |
| @@ -38,7 +39,7 @@ def _check_invalid_ssl_filesize(ssl_options): | |||
| 38 | def check_size(path: str): | 39 | def check_size(path: str): |
| 39 | size = os.path.getsize(path) | 40 | size = os.path.getsize(path) |
| 40 | if size > max_size: | 41 | if size > max_size: |
| 41 | - raise RuntimeError("SSL file should not exceed 10MB!") | 42 | + raise RuntimeError(f"SSL file should not exceed 10MB!") |
| 42 | 43 | ||
| 43 | max_size = 10 * 1024 * 1024 # 最大文件大小为10MB | 44 | max_size = 10 * 1024 * 1024 # 最大文件大小为10MB |
| 44 | for ssl_key in SSL_MUST_KEYS: | 45 | for ssl_key in SSL_MUST_KEYS: |
| @@ -39,7 +39,7 @@ class BaseBackend(ABC): | |||
| 39 | def fetch_log_messages(self): | 39 | def fetch_log_messages(self): |
| 40 | log_data = self.log_collector.collect_handler.log_processor.get_log_data(self.identity) | 40 | log_data = self.log_collector.collect_handler.log_processor.get_log_data(self.identity) |
| 41 | if log_data is None: | 41 | if log_data is None: |
| 42 | - self.logger.debug("No log data read from backend!") | 42 | + self.logger.debug(f"No log data read from backend!") |
| 43 | return None | 43 | return None |
| 44 | log_request_message = json.dumps(LogRequestMessage(log_data_list=[log_data], server_ip=get_local_ip()).format()) | 44 | log_request_message = json.dumps(LogRequestMessage(log_data_list=[log_data], server_ip=get_local_ip()).format()) |
| 45 | self.logger.debug(f"Log data read from backend is: {log_request_message}") | 45 | self.logger.debug(f"Log data read from backend is: {log_request_message}") |
| @@ -10,7 +10,5 @@ | |||
| 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | -__all__ = ['Collector', 'LogRequestMessage'] | ||
| 14 | - | ||
| 15 | from .log_collector import Collector | 13 | from .log_collector import Collector |
| 16 | from .data_class import LogRequestMessage | 14 | from .data_class import LogRequestMessage |
| @@ -10,6 +10,7 @@ | |||
| 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | +import json | ||
| 13 | import os | 14 | import os |
| 14 | from dataclasses import dataclass | 15 | from dataclasses import dataclass |
| 15 | from datetime import datetime, timezone | 16 | from datetime import datetime, timezone |
| @@ -42,13 +42,13 @@ class CollectHandler(FileSystemEventHandler): | |||
| 42 | 42 | ||
| 43 | def on_created(self, event): | 43 | def on_created(self, event): |
| 44 | if self._check_valid_file(event): | 44 | if self._check_valid_file(event): |
| 45 | - self.logger.info("[OM Adapter] File %s is created" % event.src_path) | 45 | + self.logger.info(f"[OM Adapter] File %s is created" % event.src_path) |
| 46 | self.log_processor.watch_files[event.src_path] = LogFile(file_path=event.src_path) | 46 | self.log_processor.watch_files[event.src_path] = LogFile(file_path=event.src_path) |
| 47 | self.log_processor.modified_log_files.add(event.src_path) | 47 | self.log_processor.modified_log_files.add(event.src_path) |
| 48 | 48 | ||
| 49 | def on_modified(self, event): | 49 | def on_modified(self, event): |
| 50 | if self._check_valid_file(event): | 50 | if self._check_valid_file(event): |
| 51 | - self.logger.debug("[OM Adapter] File %s is modified", event.src_path) # 文件内容更新频率高 | 51 | + self.logger.debug(f"[OM Adapter] File %s is modified", event.src_path) # 文件内容更新频率高 |
| 52 | if event.src_path not in self.log_processor.watch_files: | 52 | if event.src_path not in self.log_processor.watch_files: |
| 53 | self.log_processor.watch_files[event.src_path] = LogFile(file_path=event.src_path) | 53 | self.log_processor.watch_files[event.src_path] = LogFile(file_path=event.src_path) |
| 54 | self.log_processor.modified_log_files.add(event.src_path) | 54 | self.log_processor.modified_log_files.add(event.src_path) |
| @@ -61,7 +61,7 @@ class CollectHandler(FileSystemEventHandler): | |||
| 61 | 61 | ||
| 62 | def on_moved(self, event): | 62 | def on_moved(self, event): |
| 63 | if self._check_valid_file(event): | 63 | if self._check_valid_file(event): |
| 64 | - self.logger.info("[OM Adapter] File %s is changed to %s" % (event.src_path, event.dest_path)) | 64 | + self.logger.info(f"[OM Adapter] File %s is changed to %s" % (event.src_path, event.dest_path)) |
| 65 | src_log_file = self.log_processor.watch_files.pop(event.src_path, LogFile(file_path=event.dest_path)) | 65 | src_log_file = self.log_processor.watch_files.pop(event.src_path, LogFile(file_path=event.dest_path)) |
| 66 | src_log_file.file_path = event.dest_path | 66 | src_log_file.file_path = event.dest_path |
| 67 | src_log_file.last_read_position = 0 # 文件轮转后,更新读取位置 | 67 | src_log_file.last_read_position = 0 # 文件轮转后,更新读取位置 |
| @@ -72,10 +72,10 @@ class Collector: | |||
| 72 | def __init__(self, collect_path=DEFAULT_COLLECT_PATH): | 72 | def __init__(self, collect_path=DEFAULT_COLLECT_PATH): |
| 73 | self.logger = Log(__name__).getlog() | 73 | self.logger = Log(__name__).getlog() |
| 74 | if not collect_path: | 74 | if not collect_path: |
| 75 | - err_msg = "[OM Adapter] Init log monitor failed, the collect_path is empty from config.json" | 75 | + err_msg = f"[OM Adapter] Init log monitor failed, the collect_path is empty from config.json" |
| 76 | self.logger.error(err_msg) | 76 | self.logger.error(err_msg) |
| 77 | raise Exception(err_msg) | 77 | raise Exception(err_msg) |
| 78 | - self.logger.info("[OM Adapter] Log monitor path is %s" % collect_path) | 78 | + self.logger.info(f"[OM Adapter] Log monitor path is %s" % collect_path) |
代码风格与一致性: 第79行是f-string与%格式化的混合使用,与第45、51、64行是同类问题。这种模式在代码中重复出现,表明需要统一的修复方案。不一致的字符串格式化方式会影响代码的可读性和维护成本。 问题类型: 代码风格与一致性 文件路径: mindie_motor/python/mindie_motor/om_adapter/backends/log_collect/log_collector.py行号: 78 问题代码: self.logger.info(f"[OM Adapter] Log monitor path is %s" % collect_path) 修改建议: 改为纯f-string格式:`self.logger.info(f"[OM Adapter] Log monitor path is {collect_path}")`。同时建议检查整个文件中所有字符串格式化的地方,确保风格统一。--- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 79 | 79 | ||
| 80 | self.collect_handler = CollectHandler(collect_path) | 80 | self.collect_handler = CollectHandler(collect_path) |
| 81 | self.collect_observer = Observer() | 81 | self.collect_observer = Observer() |
| @@ -29,13 +29,13 @@ class MindIEBackend(BaseBackend): | |||
| 29 | self.alarm_shm = CircularShareMemory("mindie_controller_alarms_sem", "mindie_controller_alarms", | 29 | self.alarm_shm = CircularShareMemory("mindie_controller_alarms_sem", "mindie_controller_alarms", |
| 30 | 10 * 1024 * 1024) | 30 | 10 * 1024 * 1024) |
| 31 | self.logger.info("Alarm share memory created successfully!") | 31 | self.logger.info("Alarm share memory created successfully!") |
| 32 | - self.alive_shm = ByteShareMemory("smu_ctrl_heartbeat_sem", "smu_ctrl_heartbeat_shm") | 32 | + self.alive_shm = ByteShareMemory(f"smu_ctrl_heartbeat_sem", f"smu_ctrl_heartbeat_shm") |
| 33 | self.logger.info("Alive share memory created successfully!") | 33 | self.logger.info("Alive share memory created successfully!") |
| 34 | else: | 34 | else: |
| 35 | self.alarm_shm = CircularShareMemory("mindie_coordinator_alarms_sem", "mindie_coordinator_alarms", | 35 | self.alarm_shm = CircularShareMemory("mindie_coordinator_alarms_sem", "mindie_coordinator_alarms", |
| 36 | 10 * 1024 * 1024) | 36 | 10 * 1024 * 1024) |
| 37 | self.logger.info("Alarm share memory created successfully!") | 37 | self.logger.info("Alarm share memory created successfully!") |
| 38 | - self.alive_shm = ByteShareMemory("smu_coord_heartbeat_sem", "smu_coord_heartbeat_shm") | 38 | + self.alive_shm = ByteShareMemory(f"smu_coord_heartbeat_sem", f"smu_coord_heartbeat_shm") |
| 39 | self.logger.info("Alive share memory created successfully!") | 39 | self.logger.info("Alive share memory created successfully!") |
| 40 | 40 | ||
| 41 | def fetch_alarm_info(self) -> list: | 41 | def fetch_alarm_info(self) -> list: |
| @@ -66,7 +66,7 @@ class MindIEBackend(BaseBackend): | |||
| 66 | alive_timestamp = json.loads(chunk)["timestamp"] | 66 | alive_timestamp = json.loads(chunk)["timestamp"] |
| 67 | if time.time() <= alive_timestamp + 5: | 67 | if time.time() <= alive_timestamp + 5: |
| 68 | return True | 68 | return True |
| 69 | - except JSONDecodeError: | 69 | + except JSONDecodeError as json_error: |
| 70 | self.logger.error(f"Failed to read timestamp json: {chunk}") | 70 | self.logger.error(f"Failed to read timestamp json: {chunk}") |
| 71 | except Exception as e: | 71 | except Exception as e: |
| 72 | self.logger.error(e) | 72 | self.logger.error(e) |
| @@ -12,6 +12,8 @@ | |||
| 12 | 12 | ||
| 13 | import os | 13 | import os |
| 14 | import stat | 14 | import stat |
| 15 | +import ctypes | ||
| 16 | +from ctypes import c_char_p | ||
| 15 | 17 | ||
| 16 | from om_adapter.common.util import PathCheck, safe_open | 18 | from om_adapter.common.util import PathCheck, safe_open |
| 17 | from om_adapter.common.logging import Log | 19 | from om_adapter.common.logging import Log |
| @@ -34,7 +36,7 @@ def _check_invalid_ssl_filesize(ssl_options): | |||
| 34 | def check_size(path: str): | 36 | def check_size(path: str): |
| 35 | size = os.path.getsize(path) | 37 | size = os.path.getsize(path) |
| 36 | if size > max_size: | 38 | if size > max_size: |
| 37 | - raise RuntimeError("SSL file should not exceed 10MB!") | 39 | + raise RuntimeError(f"SSL file should not exceed 10MB!") |
| 38 | 40 | ||
| 39 | max_size = 10 * 1024 * 1024 # 最大文件大小为10MB | 41 | max_size = 10 * 1024 * 1024 # 最大文件大小为10MB |
| 40 | for ssl_key in SSL_MUST_KEYS: | 42 | for ssl_key in SSL_MUST_KEYS: |
| @@ -118,10 +118,10 @@ def _create_log_file(log_file): | |||
| 118 | else: | 118 | else: |
| 119 | clean_path = os.path.normpath(log_file) | 119 | clean_path = os.path.normpath(log_file) |
| 120 | if os.path.islink(clean_path): | 120 | if os.path.islink(clean_path): |
| 121 | - err_msg = "Check log file path failed because it's a symbolic." | 121 | + err_msg = f"Check log file path failed because it's a symbolic." |
| 122 | raise ValueError(err_msg) | 122 | raise ValueError(err_msg) |
| 123 | if len(clean_path) > 1024: | 123 | if len(clean_path) > 1024: |
| 124 | - err_msg = "Path of log file is too long, it should not exceed 1024 character." | 124 | + err_msg = f"Path of log file is too long, it should not exceed 1024 character." |
| 125 | raise ValueError(err_msg) | 125 | raise ValueError(err_msg) |
| 126 | os.chmod(log_file, mode) | 126 | os.chmod(log_file, mode) |
| 127 | 127 | ||
| @@ -155,7 +155,7 @@ class CCAEMonitor(BaseMonitor): | |||
| 155 | "POST", url, headers=self.headers, body=json.dumps(inventory_json).encode()) | 155 | "POST", url, headers=self.headers, body=json.dumps(inventory_json).encode()) |
| 156 | response_raise_for_status(response, "inventory") | 156 | response_raise_for_status(response, "inventory") |
| 157 | self.logger.debug("Response from inventory is: %s", response.data.decode()) | 157 | self.logger.debug("Response from inventory is: %s", response.data.decode()) |
| 158 | - except JSONDecodeError: | 158 | + except JSONDecodeError as json_error: |
| 159 | self.logger.error(f"Failed to decode inventory info: {inventories}") | 159 | self.logger.error(f"Failed to decode inventory info: {inventories}") |
| 160 | except Exception as e: | 160 | except Exception as e: |
| 161 | self.logger.error(e) | 161 | self.logger.error(e) |
| @@ -62,4 +62,4 @@ class KafkaProducer: | |||
| 62 | self.logger.error("[OM Adapter] message send failed, the reason is: %s" % err) | 62 | self.logger.error("[OM Adapter] message send failed, the reason is: %s" % err) |
| 63 | else: | 63 | else: |
| 64 | self.logger.debug( | 64 | self.logger.debug( |
| 65 | - "[OM Adapter] message send successfully topic=%s, partition=%s", msg.topic(), msg.partition()) | 65 | + f"[OM Adapter] message send successfully topic=%s, partition=%s", msg.topic(), msg.partition()) |
| @@ -46,7 +46,7 @@ def use_cxx11_abi() -> str: | |||
| 46 | logging.info(f"Detect ABI from torch, set GLOBAL_ABI_VERSION to {abi}") | 46 | logging.info(f"Detect ABI from torch, set GLOBAL_ABI_VERSION to {abi}") |
| 47 | return abi | 47 | return abi |
| 48 | except Exception: | 48 | except Exception: |
| 49 | - logging.warning("No torch detected on current environment.") | 49 | + logging.warning(f"No torch detected on current environment.") |
| 50 | 50 | ||
| 51 | # 3. Fallback to "0" with warning | 51 | # 3. Fallback to "0" with warning |
| 52 | abi = "0" | 52 | abi = "0" |
| @@ -153,12 +153,12 @@ def get_server_num(): | |||
| 153 | raise | 153 | raise |
| 154 | 154 | ||
| 155 | if (device_num % tp_num) != 0: | 155 | if (device_num % tp_num) != 0: |
| 156 | - logger.error("device_num % tp_num must equal to 0.") | 156 | + logger.error(f"device_num % tp_num must equal to 0.") |
| 157 | raise ValueError("device_num % tp_num must equal to 0.") | 157 | raise ValueError("device_num % tp_num must equal to 0.") |
| 158 | server_num = int(device_num / tp_num) | 158 | server_num = int(device_num / tp_num) |
| 159 | if server_num < 1: | 159 | if server_num < 1: |
| 160 | - logger.error(f"server_num({server_num}) must be greater than 0.") | 160 | + logger.error(f"server_num must be greater than 0.") |
| 161 | - raise ValueError(f"server_num({server_num}) must be greater than 0.") | 161 | + raise ValueError(f"server_num must be greater than 0.") |
| 162 | return server_num | 162 | return server_num |
| 163 | 163 | ||
| 164 | 164 | ||
| @@ -300,7 +300,7 @@ def check_server_config(config_file_path: str, server_id: int): | |||
| 300 | try: | 300 | try: |
| 301 | update_port_value(config_file_path, f"server{server_id}_{port_name}", port_name, port) | 301 | update_port_value(config_file_path, f"server{server_id}_{port_name}", port_name, port) |
| 302 | except Exception: | 302 | except Exception: |
| 303 | - logger.error("Update port value failed.") | 303 | + logger.error(f"Update port value failed.") |
| 304 | raise | 304 | raise |
| 305 | if server_id == 1: | 305 | if server_id == 1: |
| 306 | PortHelper.cache_port(port_name, [g_ports_allocated[f"server{server_id}_{port_name}"]]) | 306 | PortHelper.cache_port(port_name, [g_ports_allocated[f"server{server_id}_{port_name}"]]) |
| @@ -315,7 +315,7 @@ def get_coordinator_ip_from_config(config_path: str): | |||
| 315 | ms_coordinator = load_json_file(ms_coordinator_path) | 315 | ms_coordinator = load_json_file(ms_coordinator_path) |
| 316 | return ms_coordinator["http_config"]["manage_ip"] | 316 | return ms_coordinator["http_config"]["manage_ip"] |
| 317 | except Exception: | 317 | except Exception: |
| 318 | - logger.error("Get coordinator IP from config failed.") | 318 | + logger.error(f"Get coordinator IP from config failed.") |
| 319 | raise | 319 | raise |
| 320 | 320 | ||
| 321 | 321 | ||
| @@ -331,7 +331,7 @@ def check_json_files(config_path: str, config_json_files: List[str], is_gen_serv | |||
| 331 | manage_port = int(ms_coordinator["http_config"]["manage_port"]) | 331 | manage_port = int(ms_coordinator["http_config"]["manage_port"]) |
| 332 | update_port_value(ms_coordinator_path, "coordinator_manage_port", "manage_port", manage_port) | 332 | update_port_value(ms_coordinator_path, "coordinator_manage_port", "manage_port", manage_port) |
| 333 | except Exception: | 333 | except Exception: |
| 334 | - logger.error("Check and update ms_coordinator.json failed.") | 334 | + logger.error(f"Check and update ms_coordinator.json failed.") |
| 335 | raise | 335 | raise |
| 336 | 336 | ||
| 337 | try: | 337 | try: |
| @@ -343,7 +343,7 @@ def check_json_files(config_path: str, config_json_files: List[str], is_gen_serv | |||
| 343 | http_server_port = int(ms_controller["http_server"]["port"]) | 343 | http_server_port = int(ms_controller["http_server"]["port"]) |
| 344 | update_port_value(ms_controller_path, "controller_http_server_port", "port", http_server_port) | 344 | update_port_value(ms_controller_path, "controller_http_server_port", "port", http_server_port) |
| 345 | except Exception: | 345 | except Exception: |
| 346 | - logger.error("Check and update ms_controller.json failed.") | 346 | + logger.error(f"Check and update ms_controller.json failed.") |
| 347 | raise | 347 | raise |
| 348 | 348 | ||
| 349 | # check and update server json files | 349 | # check and update server json files |
| @@ -393,11 +393,7 @@ def update_keys_in_json(config, config_key, config_value): | |||
| 393 | if config_key.lower() in must_string_key: | 393 | if config_key.lower() in must_string_key: |
| 394 | config[key] = config_value | 394 | config[key] = config_value |
| 395 | else: | 395 | else: |
| 396 | - config[key] = ( | 396 | + config[key] = json.loads(config_value) if infer_type(config_value) != str else config_value |
| 397 | - json.loads(config_value) | ||
| 398 | - if not isinstance(infer_type(config_value), str) | ||
| 399 | - else config_value | ||
| 400 | - ) | ||
| 401 | return (config, True) | 397 | return (config, True) |
| 402 | 398 | ||
| 403 | new_config, is_changed = update_keys_in_json(value, config_key, config_value) | 399 | new_config, is_changed = update_keys_in_json(value, config_key, config_value) |
| @@ -13,7 +13,9 @@ import subprocess | |||
| 13 | import logging | 13 | import logging |
| 14 | import argparse | 14 | import argparse |
| 15 | from dataclasses import dataclass | 15 | from dataclasses import dataclass |
| 16 | -from typing import List, Dict, Union | 16 | +from itertools import accumulate |
| 17 | +from typing import List, Dict, Tuple, Union | ||
| 18 | +import psutil | ||
| 17 | 19 | ||
| 18 | 20 | ||
| 19 | class ENV: | 21 | class ENV: |
| @@ -57,30 +57,30 @@ class _FileUtils: | |||
| 57 | :return: check_status[True or False], err_msg[if False], real_file_path[if True] | 57 | :return: check_status[True or False], err_msg[if False], real_file_path[if True] |
| 58 | """ | 58 | """ |
| 59 | if not file_path or not isinstance(file_path, str): | 59 | if not file_path or not isinstance(file_path, str): |
| 60 | - err_msg = "The file path is empty or not a string type." | 60 | + err_msg = f"The file path is empty or not a string type." |
| 61 | return False, err_msg, None | 61 | return False, err_msg, None |
| 62 | 62 | ||
| 63 | if not base_dir or not isinstance(base_dir, str): | 63 | if not base_dir or not isinstance(base_dir, str): |
| 64 | - err_msg = "The base dir path is empty or not a string type." | 64 | + err_msg = f"The base dir path is empty or not a string type." |
| 65 | return False, err_msg, None | 65 | return False, err_msg, None |
| 66 | 66 | ||
| 67 | if len(file_path) > 1024: | 67 | if len(file_path) > 1024: |
| 68 | - err_msg = "The file path exceeds the maximum length." | 68 | + err_msg = f"The file path exceeds the maximum length." |
| 69 | return False, err_msg, None | 69 | return False, err_msg, None |
| 70 | 70 | ||
| 71 | if not allow_symlink and os.path.islink(file_path): | 71 | if not allow_symlink and os.path.islink(file_path): |
| 72 | - err_msg = "The file path is a link." | 72 | + err_msg = f"The file path is a link." |
| 73 | return False, err_msg, None | 73 | return False, err_msg, None |
| 74 | 74 | ||
| 75 | try: | 75 | try: |
| 76 | real_file_path = os.path.realpath(file_path) | 76 | real_file_path = os.path.realpath(file_path) |
| 77 | - except Exception: | 77 | + except Exception as e: |
| 78 | - err_msg = "Realpath parsing failed" | 78 | + err_msg = f"Realpath parsing failed" |
| 79 | return False, err_msg, None | 79 | return False, err_msg, None |
| 80 | 80 | ||
| 81 | base_dir = base_dir if base_dir[-1] == "/" else base_dir + '/' | 81 | base_dir = base_dir if base_dir[-1] == "/" else base_dir + '/' |
| 82 | if not cls.is_base_dir_path(base_dir, real_file_path): | 82 | if not cls.is_base_dir_path(base_dir, real_file_path): |
| 83 | - err_msg = 'the file path is not in base dir' | 83 | + err_msg = f'the file path is not in base dir' |
代码逻辑和结构: 在错误消息字符串中不必要地使用了f-string,但字符串中没有包含任何变量插值。 问题类型: 代码逻辑和结构 文件路径: mindie_motor/src/example/deploy_scripts/boot_helper/update_mindie_server_config.py行号: 83 问题代码: err_msg = f'the file path is not in base dir' 修改建议: 将f-string改回普通字符串:err_msg = 'the file path is not in base dir' --- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 84 | return False, err_msg, None | 84 | return False, err_msg, None |
| 85 | 85 | ||
| 86 | return True, None, real_file_path | 86 | return True, None, real_file_path |
| @@ -101,7 +101,7 @@ class _FileUtils: | |||
| 101 | """ | 101 | """ |
| 102 | # Check if the file exists | 102 | # Check if the file exists |
| 103 | if not cls.check_file_exists(file_path): | 103 | if not cls.check_file_exists(file_path): |
| 104 | - err_msg = "Error: File not found." | 104 | + err_msg = f"Error: File not found." |
代码逻辑和结构: 在错误消息字符串中不必要地使用了f-string,但字符串中没有包含任何变量插值。 问题类型: 代码逻辑和结构 文件路径: mindie_motor/src/example/deploy_scripts/boot_helper/update_mindie_server_config.py行号: 104 问题代码: err_msg = f"Error: File not found." 修改建议: 将f-string改回普通字符串:err_msg = "Error: File not found." --- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 105 | return False, err_msg | 105 | return False, err_msg |
| 106 | 106 | ||
| 107 | # Get the real_file_path | 107 | # Get the real_file_path |
| @@ -118,7 +118,7 @@ class _FileUtils: | |||
| 118 | # Get the file size | 118 | # Get the file size |
| 119 | file_size = fp.tell() | 119 | file_size = fp.tell() |
| 120 | if file_size < DEFAULT_MIN_FILE_SIZE or file_size > DEFAULT_MAX_FILE_SIZE: | 120 | if file_size < DEFAULT_MIN_FILE_SIZE or file_size > DEFAULT_MAX_FILE_SIZE: |
| 121 | - err_msg = "Read input file failed, file size is invalid" | 121 | + err_msg = f"Read input file failed, file size is invalid" |
代码逻辑和结构: 在错误消息字符串中不必要地使用了f-string,但字符串中没有包含任何变量插值。 问题类型: 代码逻辑和结构 文件路径: mindie_motor/src/example/deploy_scripts/boot_helper/update_mindie_server_config.py行号: 121 问题代码: err_msg = f"Read input file failed, file size is invalid" 修改建议: 将f-string改回普通字符串:err_msg = "Read input file failed, file size is invalid" --- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 122 | return False, err_msg | 122 | return False, err_msg |
| 123 | return True, None | 123 | return True, None |
| 124 | except Exception as e: | 124 | except Exception as e: |
| @@ -130,7 +130,7 @@ class _FileUtils: | |||
| 130 | try: | 130 | try: |
| 131 | file_stat = os.stat(file_path) | 131 | file_stat = os.stat(file_path) |
| 132 | except FileNotFoundError: | 132 | except FileNotFoundError: |
| 133 | - err_msg = "Error: File not found." | 133 | + err_msg = f"Error: File not found." |
代码逻辑和结构: 在错误消息字符串中不必要地使用了f-string,但字符串中没有包含任何变量插值。 问题类型: 代码逻辑和结构 文件路径: mindie_motor/src/example/deploy_scripts/boot_helper/update_mindie_server_config.py行号: 133 问题代码: err_msg = f"Error: File not found." 修改建议: 将f-string改回普通字符串:err_msg = "Error: File not found." --- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 134 | return False, err_msg | 134 | return False, err_msg |
| 135 | except PermissionError: | 135 | except PermissionError: |
| 136 | err_msg = f"Error: Permission denied to access file: {file_path}" | 136 | err_msg = f"Error: Permission denied to access file: {file_path}" |
| @@ -160,7 +160,7 @@ class _FileUtils: | |||
| 160 | try: | 160 | try: |
| 161 | file_stat = os.stat(file_path) | 161 | file_stat = os.stat(file_path) |
| 162 | except FileNotFoundError: | 162 | except FileNotFoundError: |
| 163 | - err_msg = "Error: File not found." | 163 | + err_msg = f"Error: File not found." |
代码逻辑和结构: 在错误消息字符串中不必要地使用了f-string,但字符串中没有包含任何变量插值。 问题类型: 代码逻辑和结构 文件路径: mindie_motor/src/example/deploy_scripts/boot_helper/update_mindie_server_config.py行号: 163 问题代码: err_msg = f"Error: File not found." 修改建议: 将f-string改回普通字符串:err_msg = "Error: File not found." --- 此评论由代码审查工具自动生成 ![]() ![]() | |||
| 164 | return False, err_msg | 164 | return False, err_msg |
| 165 | 165 | ||
| 166 | current_permissions = file_stat.st_mode & 0o777 | 166 | current_permissions = file_stat.st_mode & 0o777 |
| @@ -30,7 +30,7 @@ sys.path.append(os.getcwd()) | |||
| 30 | from gen_ranktable_helper.gen_global_ranktable import generate_global_ranktable | 30 | from gen_ranktable_helper.gen_global_ranktable import generate_global_ranktable |
| 31 | from utils.file_utils import safe_open | 31 | from utils.file_utils import safe_open |
| 32 | from utils.validate_config import validate_user_config | 32 | from utils.validate_config import validate_user_config |
| 33 | -from utils.validate_utils import validate_identifier, validate_path_part | 33 | +from utils.validate_utils import validate_identifier, validate_path_part, validate_command_part |
| 34 | 34 | ||
| 35 | # 配置日志格式和级别 | 35 | # 配置日志格式和级别 |
| 36 | logging.basicConfig( | 36 | logging.basicConfig( |
| @@ -299,7 +299,7 @@ def check_coordinator_memory_config(coordinator_yaml_data, ms_coordinator_json_p | |||
| 299 | else: | 299 | else: |
| 300 | return f"{bytes_val} bytes" | 300 | return f"{bytes_val} bytes" |
| 301 | 301 | ||
| 302 | - logging.info("Coordinator memory config check:") | 302 | + logging.info(f"Coordinator memory config check:") |
| 303 | logging.info(f" max_requests: {max_requests}") | 303 | logging.info(f" max_requests: {max_requests}") |
| 304 | logging.info(f" body_limit: {body_limit_mb} MB ({body_limit_bytes} bytes)") | 304 | logging.info(f" body_limit: {body_limit_mb} MB ({body_limit_bytes} bytes)") |
| 305 | logging.info(f" Theoretical max memory (with 20% margin): {format_bytes(theoretical_max_memory)}") | 305 | logging.info(f" Theoretical max memory (with 20% margin): {format_bytes(theoretical_max_memory)}") |
| @@ -334,7 +334,7 @@ def check_coordinator_memory_config(coordinator_yaml_data, ms_coordinator_json_p | |||
| 334 | ) | 334 | ) |
| 335 | raise ValueError(error_msg) | 335 | raise ValueError(error_msg) |
| 336 | 336 | ||
| 337 | - logging.info("Memory config check passed.") | 337 | + logging.info(f"Memory config check passed.") |
| 338 | 338 | ||
| 339 | 339 | ||
| 340 | def check_config(config: dict): | 340 | def check_config(config: dict): |
| @@ -1190,6 +1190,7 @@ def exec_cm_create_kubectl_multi(deploy_config, out_path): | |||
| 1190 | 1190 | ||
| 1191 | 1191 | ||
| 1192 | def exec_cm_elastic_kubectl(deploy_config, out_path): | 1192 | def exec_cm_elastic_kubectl(deploy_config, out_path): |
| 1193 | + job_id = deploy_config[CONFIG_JOB_ID] | ||
| 1193 | logging.info("Starting to execute kubectl update configmap elastic") | 1194 | logging.info("Starting to execute kubectl update configmap elastic") |
| 1194 | exec_cmd("kubectl delete configmap scaling-rule" + NAME_FLAG + deploy_config[CONFIG_JOB_ID]) | 1195 | exec_cmd("kubectl delete configmap scaling-rule" + NAME_FLAG + deploy_config[CONFIG_JOB_ID]) |
| 1195 | exec_cmd("kubectl create configmap scaling-rule --from-file=" + | 1196 | exec_cmd("kubectl create configmap scaling-rule --from-file=" + |
| @@ -231,9 +231,9 @@ def restart_service(namespace: str, boot_args): | |||
| 231 | time.sleep(10) | 231 | time.sleep(10) |
| 232 | 232 | ||
| 233 | # restart service | 233 | # restart service |
| 234 | - subprocess.run(["python3", "deploy_ac_job.py"] + boot_args) | 234 | + deploy_ac_job_res = subprocess.run(["python3", "deploy_ac_job.py"] + boot_args) |
| 235 | if is_mindie_service_detected(namespace): | 235 | if is_mindie_service_detected(namespace): |
| 236 | - logging.info("Restart service successfully!") | 236 | + logging.info(f"Restart service successfully!") |
| 237 | 237 | ||
| 238 | 238 | ||
| 239 | def get_metrics_from_metrics_api(http_pool_manager, params: CheckParams) -> str: | 239 | def get_metrics_from_metrics_api(http_pool_manager, params: CheckParams) -> str: |
| @@ -279,6 +279,7 @@ def main(): | |||
| 279 | do_inference_retries = 5 | 279 | do_inference_retries = 5 |
| 280 | do_inference_interval = 180 | 280 | do_inference_interval = 180 |
| 281 | input_content = "相对论的提出者是谁?" # Probe prompt | 281 | input_content = "相对论的提出者是谁?" # Probe prompt |
| 282 | + max_unavailable_time = 1200 # Maximum service unavailable time | ||
| 282 | http_timeout = 60 # urllib3 request timeout | 283 | http_timeout = 60 # urllib3 request timeout |
| 283 | cert_context = load_cert() | 284 | cert_context = load_cert() |
| 284 | if cert_context: | 285 | if cert_context: |
| @@ -303,7 +304,7 @@ def main(): | |||
| 303 | try: | 304 | try: |
| 304 | ms_coordinator_config = fetch_config(os.path.join(boot_config["--conf_path"], "ms_coordinator.json")) | 305 | ms_coordinator_config = fetch_config(os.path.join(boot_config["--conf_path"], "ms_coordinator.json")) |
| 305 | metric_port = ms_coordinator_config["http_config"]["external_port"] | 306 | metric_port = ms_coordinator_config["http_config"]["external_port"] |
| 306 | - except Exception: | 307 | + except Exception as e: |
| 307 | metric_port = coordinator_http_config["manage_port"] | 308 | metric_port = coordinator_http_config["manage_port"] |
| 308 | 309 | ||
| 309 | params = CheckParams( | 310 | params = CheckParams( |
| @@ -326,7 +327,7 @@ def main(): | |||
| 326 | 327 | ||
| 327 | # Check if metrics is enabled | 328 | # Check if metrics is enabled |
| 328 | if not is_metrics_mode_enabled(): | 329 | if not is_metrics_mode_enabled(): |
| 329 | - raise RuntimeError("Metrics mode is disabled, please check and set MIES_SERVICE_MONITOR_MODE=1.") | 330 | + raise RuntimeError(f"Metrics mode is disabled, please check and set MIES_SERVICE_MONITOR_MODE=1.") |
| 330 | 331 | ||
| 331 | logging.info(f"Start monitoring service with namespace: {params.namespace}, model_name: {params.model_name}, " | 332 | logging.info(f"Start monitoring service with namespace: {params.namespace}, model_name: {params.model_name}, " |
| 332 | f"coordinator_port: {params.coordinator_port}, " | 333 | f"coordinator_port: {params.coordinator_port}, " |
| @@ -350,10 +351,11 @@ def main(): | |||
| 350 | resp_text = get_metrics_from_metrics_api(http_pool_manager, params) | 351 | resp_text = get_metrics_from_metrics_api(http_pool_manager, params) |
| 351 | last_success_count = find_metric_values(resp_text, "request_success_total") | 352 | last_success_count = find_metric_values(resp_text, "request_success_total") |
| 352 | last_failed_count = find_metric_values(resp_text, "request_failed_total") | 353 | last_failed_count = find_metric_values(resp_text, "request_failed_total") |
| 354 | + last_running_count = find_metric_values(resp_text, "num_requests_running") | ||
| 353 | 355 | ||
| 354 | time.sleep(probe_interval) | 356 | time.sleep(probe_interval) |
| 355 | 357 | ||
| 356 | - logging.info("Start to examine service status...") | 358 | + logging.info(f"Start to examine service status...") |
| 357 | resp_text = get_metrics_from_metrics_api(http_pool_manager, params) | 359 | resp_text = get_metrics_from_metrics_api(http_pool_manager, params) |
| 358 | cur_success_count = find_metric_values(resp_text, "request_success_total") | 360 | cur_success_count = find_metric_values(resp_text, "request_success_total") |
| 359 | cur_failed_count = find_metric_values(resp_text, "request_failed_total") | 361 | cur_failed_count = find_metric_values(resp_text, "request_failed_total") |
| @@ -365,12 +367,12 @@ def main(): | |||
| 365 | if cur_failed_count >= 0 and last_failed_count >= 0 else -1) | 367 | if cur_failed_count >= 0 and last_failed_count >= 0 else -1) |
| 366 | 368 | ||
| 367 | if delta_success < 0 or delta_failed < 0: | 369 | if delta_success < 0 or delta_failed < 0: |
| 368 | - logging.info("Metrics values decreased, continue to monitor...") | 370 | + logging.info(f"Metrics values decreased, continue to monitor...") |
| 369 | continue | 371 | continue |
| 370 | 372 | ||
| 371 | # Fault detection logic | 373 | # Fault detection logic |
| 372 | if delta_success > 0: | 374 | if delta_success > 0: |
| 373 | - logging.info("Success inference request count increased, continue to monitor...") | 375 | + logging.info(f"Success inference request count increased, continue to monitor...") |
| 374 | continue | 376 | continue |
| 375 | elif delta_success == 0: | 377 | elif delta_success == 0: |
| 376 | if delta_failed > 0: | 378 | if delta_failed > 0: |
| @@ -379,7 +381,7 @@ def main(): | |||
| 379 | f"with interval {do_inference_interval}s") | 381 | f"with interval {do_inference_interval}s") |
| 380 | if infer_with_retry(http_pool_manager, params, do_inference_retries, do_inference_interval): | 382 | if infer_with_retry(http_pool_manager, params, do_inference_retries, do_inference_interval): |
| 381 | continue | 383 | continue |
| 382 | - logging.info("Virtual inference failed in failure increase state, restart service!") | 384 | + logging.info(f"Virtual inference failed in failure increase state, restart service!") |
| 383 | break | 385 | break |
| 384 | elif delta_failed == 0: | 386 | elif delta_failed == 0: |
| 385 | if cur_running_count == 0: # No request, idle state | 387 | if cur_running_count == 0: # No request, idle state |
| @@ -388,7 +390,7 @@ def main(): | |||
| 388 | f"with interval {do_inference_interval}s") | 390 | f"with interval {do_inference_interval}s") |
| 389 | if infer_with_retry(http_pool_manager, params, do_inference_retries, do_inference_interval): | 391 | if infer_with_retry(http_pool_manager, params, do_inference_retries, do_inference_interval): |
| 390 | continue | 392 | continue |
| 391 | - logging.info("Virtual inference failed in idle state, restart service!") | 393 | + logging.info(f"Virtual inference failed in idle state, restart service!") |
| 392 | break | 394 | break |
| 393 | elif cur_running_count > 0: # Running state, e.g. long sequence inference | 395 | elif cur_running_count > 0: # Running state, e.g. long sequence inference |
| 394 | logging.info(f"Doing virtual inference in running state, " | 396 | logging.info(f"Doing virtual inference in running state, " |
| @@ -396,7 +398,7 @@ def main(): | |||
| 396 | f"with interval {do_inference_interval}s") | 398 | f"with interval {do_inference_interval}s") |
| 397 | if infer_with_retry(http_pool_manager, params, do_inference_retries, do_inference_interval): | 399 | if infer_with_retry(http_pool_manager, params, do_inference_retries, do_inference_interval): |
| 398 | continue | 400 | continue |
| 399 | - logging.info("Virtual inference failed in running state, restart service!") | 401 | + logging.info(f"Virtual inference failed in running state, restart service!") |
| 400 | break | 402 | break |
| 401 | 403 | ||
| 402 | restart_service(params.namespace, boot_args) | 404 | restart_service(params.namespace, boot_args) |
| @@ -13,6 +13,7 @@ | |||
| 13 | import os | 13 | import os |
| 14 | import re | 14 | import re |
| 15 | import logging | 15 | import logging |
| 16 | +import string | ||
| 16 | from typing import Tuple | 17 | from typing import Tuple |
| 17 | 18 | ||
| 18 | # 配置日志格式和级别 | 19 | # 配置日志格式和级别 |
| @@ -37,7 +37,7 @@ def __is_npu_health(): | |||
| 37 | return False | 37 | return False |
| 38 | check_file_flag = FileUtils.is_symlink(yaml_file) | 38 | check_file_flag = FileUtils.is_symlink(yaml_file) |
| 39 | if check_file_flag: | 39 | if check_file_flag: |
| 40 | - logger.error("The path is a symbolic file.") | 40 | + logger.error(f"The path is a symbolic file.") |
| 41 | return False | 41 | return False |
| 42 | try: | 42 | try: |
| 43 | with open(yaml_file, 'r', encoding='utf-8') as file: | 43 | with open(yaml_file, 'r', encoding='utf-8') as file: |
| @@ -13,6 +13,7 @@ | |||
| 13 | import sys | 13 | import sys |
| 14 | import json | 14 | import json |
| 15 | import os | 15 | import os |
| 16 | +import time | ||
| 16 | import logging | 17 | import logging |
| 17 | from file_util import FileUtils | 18 | from file_util import FileUtils |
| 18 | logging.basicConfig(level=logging.INFO) | 19 | logging.basicConfig(level=logging.INFO) |
| @@ -32,7 +33,7 @@ def get_distribute_role(): | |||
| 32 | try: | 33 | try: |
| 33 | check_path_flag, err_msg, real_path = FileUtils.regular_file_path(rank_table_path) | 34 | check_path_flag, err_msg, real_path = FileUtils.regular_file_path(rank_table_path) |
| 34 | if not check_path_flag: | 35 | if not check_path_flag: |
| 35 | - logging.error("check file path failed: %s", err_msg) | 36 | + logger.error(f"check file path failed: %s", err_msg) |
| 36 | return PARSE_ERROR | 37 | return PARSE_ERROR |
| 37 | with open(real_path, 'r', encoding='utf-8') as file: | 38 | with open(real_path, 'r', encoding='utf-8') as file: |
| 38 | buf = file.read() | 39 | buf = file.read() |
| @@ -44,7 +44,7 @@ def wait_global_ranktable_completed(argv): | |||
| 44 | 44 | ||
| 45 | try: | 45 | try: |
| 46 | ipaddress.ip_address(pod_ip) | 46 | ipaddress.ip_address(pod_ip) |
| 47 | - except ValueError as e: | 47 | + except ValueError: |
| 48 | raise RuntimeError(f"Invalid POD_IP: {pod_ip}") from e | 48 | raise RuntimeError(f"Invalid POD_IP: {pod_ip}") from e |
| 49 | for group in server_group_list: | 49 | for group in server_group_list: |
| 50 | group_id = "-1" | 50 | group_id = "-1" |
| @@ -29,11 +29,11 @@ logger.addHandler(console_handler) | |||
| 29 | def __check_file_path(file_path, mode, check_permission): | 29 | def __check_file_path(file_path, mode, check_permission): |
| 30 | check_path_flag, err_msg, real_path = FileUtils.regular_file_path(file_path) | 30 | check_path_flag, err_msg, real_path = FileUtils.regular_file_path(file_path) |
| 31 | if not check_path_flag: | 31 | if not check_path_flag: |
| 32 | - logger.error("check file path failed: %s", err_msg) | 32 | + logger.error(f"check file path failed: %s", err_msg) |
| 33 | return False | 33 | return False |
| 34 | check_file_flag, err_msg = FileUtils.is_file_valid(real_path, mode=mode, check_permission=check_permission) | 34 | check_file_flag, err_msg = FileUtils.is_file_valid(real_path, mode=mode, check_permission=check_permission) |
| 35 | if not check_file_flag: | 35 | if not check_file_flag: |
| 36 | - logger.error("check file path is invalid: %s", err_msg) | 36 | + logger.error(f"check file path is invalid: %s", err_msg) |
| 37 | return False | 37 | return False |
| 38 | return True | 38 | return True |
| 39 | 39 | ||
| @@ -78,7 +78,7 @@ class TestCCAEMonitor(unittest.TestCase): | |||
| 78 | } | 78 | } |
| 79 | }] | 79 | }] |
| 80 | } | 80 | } |
| 81 | - with patch("urllib3.PoolManager.request", return_value=MockResponse(json.dumps(mock_heartbeat_response))): | 81 | + with patch("urllib3.PoolManager.request", return_value=MockResponse(json.dumps(mock_heartbeat_response))) as p: |
| 82 | self.ccae_monitor.send_heart_beat() | 82 | self.ccae_monitor.send_heart_beat() |
| 83 | self.assertTrue(self.ccae_monitor.model_id_period[MODEL_ID][0]) | 83 | self.assertTrue(self.ccae_monitor.model_id_period[MODEL_ID][0]) |
| 84 | self.assertEqual(self.ccae_monitor.model_id_period[MODEL_ID][1], 2) | 84 | self.assertEqual(self.ccae_monitor.model_id_period[MODEL_ID][1], 2) |
| @@ -89,8 +89,8 @@ class TestCCAEMonitor(unittest.TestCase): | |||
| 89 | "alarmId": "this is a new alarm" | 89 | "alarmId": "this is a new alarm" |
| 90 | }] | 90 | }] |
| 91 | with patch.object(AbstractShareMemoryUtil, "read", | 91 | with patch.object(AbstractShareMemoryUtil, "read", |
| 92 | - return_value=json.dumps(single_alarm_info)): | 92 | + return_value=json.dumps(single_alarm_info)) as shm_read_p: |
| 93 | - with patch("urllib3.PoolManager.request", return_value=MockResponse("OK")): | 93 | + with patch("urllib3.PoolManager.request", return_value=MockResponse("OK")) as response_p: |
| 94 | self.ccae_monitor.upload_alarm(self.ccae_monitor.backend.fetch_alarm_info()[0]) | 94 | self.ccae_monitor.upload_alarm(self.ccae_monitor.backend.fetch_alarm_info()[0]) |
| 95 | self.assertEqual(self.ccae_monitor.alarm_cache["this is a new alarm"], single_alarm_info[0]) | 95 | self.assertEqual(self.ccae_monitor.alarm_cache["this is a new alarm"], single_alarm_info[0]) |
| 96 | self.assertEqual(self.ccae_monitor.fetch_alarm_cache(), single_alarm_info) | 96 | self.assertEqual(self.ccae_monitor.fetch_alarm_cache(), single_alarm_info) |
| @@ -101,12 +101,12 @@ class TestCCAEMonitor(unittest.TestCase): | |||
| 101 | "alarmId": "this is a cancel alarm" | 101 | "alarmId": "this is a cancel alarm" |
| 102 | }] | 102 | }] |
| 103 | with patch.object(AbstractShareMemoryUtil, "read", | 103 | with patch.object(AbstractShareMemoryUtil, "read", |
| 104 | - return_value=json.dumps(single_alarm_info)): | 104 | + return_value=json.dumps(single_alarm_info)) as shm_read_p: |
| 105 | - with patch("urllib3.PoolManager.request", return_value=MockResponse("OK")): | 105 | + with patch("urllib3.PoolManager.request", return_value=MockResponse("OK")) as response_p: |
| 106 | self.ccae_monitor.upload_alarm(self.ccae_monitor.backend.fetch_alarm_info()[0]) | 106 | self.ccae_monitor.upload_alarm(self.ccae_monitor.backend.fetch_alarm_info()[0]) |
| 107 | self.assertEqual(len(self.ccae_monitor.alarm_cache), 0) | 107 | self.assertEqual(len(self.ccae_monitor.alarm_cache), 0) |
| 108 | 108 | ||
| 109 | def test_upload_inventory(self): | 109 | def test_upload_inventory(self): |
| 110 | - with patch("urllib3.PoolManager.request", return_value=MockResponse("OK")): | 110 | + with patch("urllib3.PoolManager.request", return_value=MockResponse("OK")) as p: |
| 111 | self.assertIsNone(self.ccae_monitor.upload_inventory(str({"inventory": None}))) | 111 | self.assertIsNone(self.ccae_monitor.upload_inventory(str({"inventory": None}))) |
| 112 | 112 | ||
| @@ -13,6 +13,7 @@ | |||
| 13 | import unittest | 13 | import unittest |
| 14 | 14 | ||
| 15 | from om_adapter.monitors.kafka_client.kafka_produce import KafkaProducer | 15 | from om_adapter.monitors.kafka_client.kafka_produce import KafkaProducer |
| 16 | +from om_adapter.config import ConfigUtil | ||
| 16 | 17 | ||
| 17 | 18 | ||
| 18 | class TestKafkaProduce(unittest.TestCase): | 19 | class TestKafkaProduce(unittest.TestCase): |
| @@ -45,7 +45,7 @@ class TestLogDataProcessor(unittest.TestCase): | |||
| 45 | self.log_processor = LogDataProcessor() | 45 | self.log_processor = LogDataProcessor() |
| 46 | self.log_processor.watch_files[self.filename] = LogFile(self.filename) | 46 | self.log_processor.watch_files[self.filename] = LogFile(self.filename) |
| 47 | self.log_processor.modified_log_files.add(self.filename) | 47 | self.log_processor.modified_log_files.add(self.filename) |
| 48 | - with patch.object(PathCheck, 'check_path_full', return_value=(True, None)): | 48 | + with patch.object(PathCheck, 'check_path_full', return_value=(True, None)) as mock_fuc: |
| 49 | # 首次读文件 | 49 | # 首次读文件 |
| 50 | log_data = self.log_processor.get_log_data(component) | 50 | log_data = self.log_processor.get_log_data(component) |
| 51 | self.assertEqual(log_data.component_type, component) | 51 | self.assertEqual(log_data.component_type, component) |
| @@ -10,9 +10,10 @@ | |||
| 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | +import os | ||
| 13 | import sys | 14 | import sys |
| 14 | import types | 15 | import types |
| 15 | -from unittest.mock import patch | 16 | +from unittest.mock import patch, MagicMock |
| 16 | 17 | ||
| 17 | 18 | ||
| 18 | class FakeGeneralConfig: | 19 | class FakeGeneralConfig: |
| @@ -19,10 +19,12 @@ sys.path.append( | |||
| 19 | ) | 19 | ) |
| 20 | import unittest | 20 | import unittest |
| 21 | from unittest.mock import patch, MagicMock | 21 | from unittest.mock import patch, MagicMock |
| 22 | +from unittest.mock import patch, MagicMock | ||
| 22 | import logging | 23 | import logging |
| 23 | 24 | ||
| 24 | import requests | 25 | import requests |
| 25 | 26 | ||
| 27 | +from node_manager.models import NodeRunningStatus | ||
| 26 | from node_manager.core.fault_mng import fault_manager | 28 | from node_manager.core.fault_mng import fault_manager |
| 27 | from node_manager.models.enums import NodeRunningStatus | 29 | from node_manager.models.enums import NodeRunningStatus |
| 28 | 30 | ||
| @@ -74,6 +76,7 @@ class TestFaultManager(unittest.TestCase): | |||
| 74 | 76 | ||
| 75 | # 下发命令 reinit npu | 77 | # 下发命令 reinit npu |
| 76 | self._set_response_data(mocked_request, 200, mock_data) | 78 | self._set_response_data(mocked_request, 200, mock_data) |
| 79 | + ret_true = {status_str: True, "reason": None} | ||
| 77 | func = self.fault_manager.get_handler("REINIT_NPU") | 80 | func = self.fault_manager.get_handler("REINIT_NPU") |
| 78 | response = func() | 81 | response = func() |
| 79 | sleep(8) | 82 | sleep(8) |
| @@ -21,7 +21,10 @@ import threading | |||
| 21 | import unittest | 21 | import unittest |
| 22 | from time import sleep | 22 | from time import sleep |
| 23 | import logging | 23 | import logging |
| 24 | +import time | ||
| 24 | from unittest.mock import patch, MagicMock | 25 | from unittest.mock import patch, MagicMock |
| 26 | +import threading | ||
| 27 | +import threading | ||
| 25 | from collections import deque | 28 | from collections import deque |
| 26 | 29 | ||
| 27 | from node_manager.models.enums import NodeRunningStatus, ServiceStatus | 30 | from node_manager.models.enums import NodeRunningStatus, ServiceStatus |
| @@ -10,6 +10,7 @@ | |||
| 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | +import os | ||
| 13 | import unittest | 14 | import unittest |
| 14 | import signal | 15 | import signal |
| 15 | from unittest.mock import patch, MagicMock | 16 | from unittest.mock import patch, MagicMock |
| @@ -9,6 +9,7 @@ | |||
| 9 | # See the Mulan PSL v2 for more details. | 9 | # See the Mulan PSL v2 for more details. |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | +import os | ||
| 12 | import threading | 13 | import threading |
| 13 | import time | 14 | import time |
| 14 | import unittest | 15 | import unittest |
| @@ -9,6 +9,7 @@ | |||
| 9 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | 9 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 10 | # See the Mulan PSL v2 for more details. | 10 | # See the Mulan PSL v2 for more details. |
| 11 | 11 | ||
| 12 | +import os | ||
| 12 | import unittest | 13 | import unittest |
| 13 | from unittest.mock import patch, Mock | 14 | from unittest.mock import patch, Mock |
| 14 | 15 | ||
| @@ -65,7 +65,7 @@ def merge_results(product, result_dir): | |||
| 65 | for case in cases: | 65 | for case in cases: |
| 66 | merge_root.extend(case) | 66 | merge_root.extend(case) |
| 67 | new_tree = ET.ElementTree(merge_root) | 67 | new_tree = ET.ElementTree(merge_root) |
| 68 | - new_tree.write("test_detail.xml", encoding='utf-8', xml_declaration=True, short_empty_elements=True) | 68 | + new_tree.write(f"test_detail.xml", encoding='utf-8', xml_declaration=True, short_empty_elements=True) |
| 69 | 69 | ||
| 70 | if __name__ == '__main__': | 70 | if __name__ == '__main__': |
| 71 | main() | 71 | main() |
| @@ -11,6 +11,7 @@ | |||
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | import os | 13 | import os |
| 14 | +import re | ||
| 14 | import stat | 15 | import stat |
| 15 | from pathlib import Path | 16 | from pathlib import Path |
| 16 | 17 | ||
| @@ -140,7 +140,7 @@ class CommandHelper(object): | |||
| 140 | def clear_output_buffer(self, terminal_id: int) -> None: | 140 | def clear_output_buffer(self, terminal_id: int) -> None: |
| 141 | '''clear old output in output.txt | 141 | '''clear old output in output.txt |
| 142 | ''' | 142 | ''' |
| 143 | - with open(self.output[terminal_id], 'w'): | 143 | + with open(self.output[terminal_id], 'w') as f: |
| 144 | pass | 144 | pass |
| 145 | 145 | ||
| 146 | def _add_output_command(self, command: str, terminal_id: int) -> str: | 146 | def _add_output_command(self, command: str, terminal_id: int) -> str: |
| @@ -45,7 +45,7 @@ def wait_compile_complete(command_helper_instance: CommandHelper, output_path: s | |||
| 45 | while retry_times > 0: | 45 | while retry_times > 0: |
| 46 | time.sleep(30 * ONE_SEC) | 46 | time.sleep(30 * ONE_SEC) |
| 47 | if has_output_file(command_helper_instance, output_path, terminal_id): | 47 | if has_output_file(command_helper_instance, output_path, terminal_id): |
| 48 | - print_to_screen('Compile completed.') | 48 | + print_to_screen(f'Compile completed.') |
| 49 | break | 49 | break |
| 50 | retry_times -= 1 | 50 | retry_times -= 1 |
| 51 | if retry_times == 0: | 51 | if retry_times == 0: |
| @@ -69,7 +69,7 @@ def compile_mies(command_helper_instance: CommandHelper, compile_config: dict): | |||
| 69 | # compile service | 69 | # compile service |
| 70 | # prepare dependency | 70 | # prepare dependency |
| 71 | command_helper_instance.exec_command(compiler_id, f'cd {mies_repo_path}', wait_time=1) | 71 | command_helper_instance.exec_command(compiler_id, f'cd {mies_repo_path}', wait_time=1) |
| 72 | - command_helper_instance.exec_command(compiler_id, 'mkdir -p third_party/install/MindIE-LLM', wait_time=1) | 72 | + command_helper_instance.exec_command(compiler_id, f'mkdir -p third_party/install/MindIE-LLM', wait_time=1) |
| 73 | unpackage_mindie_llm = f'bash {compile_config["mindie_llm_run_path"]} --extract=third_party/install/MindIE-LLM' | 73 | unpackage_mindie_llm = f'bash {compile_config["mindie_llm_run_path"]} --extract=third_party/install/MindIE-LLM' |
| 74 | command_helper_instance.exec_command(compiler_id, unpackage_mindie_llm, wait_time=1) | 74 | command_helper_instance.exec_command(compiler_id, unpackage_mindie_llm, wait_time=1) |
| 75 | command_helper_instance.exec_command( | 75 | command_helper_instance.exec_command( |
| @@ -10,6 +10,7 @@ | |||
| 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | 10 | # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 11 | # See the Mulan PSL v2 for more details. | 11 | # See the Mulan PSL v2 for more details. |
| 12 | 12 | ||
| 13 | +import os | ||
| 13 | import atexit | 14 | import atexit |
| 14 | import subprocess | 15 | import subprocess |
| 15 | import argparse | 16 | import argparse |
| @@ -18,6 +19,7 @@ import random | |||
| 18 | from command_helper import CommandHelper | 19 | from command_helper import CommandHelper |
| 19 | from compile_utils import compile_mies, deployment | 20 | from compile_utils import compile_mies, deployment |
| 20 | from utils import ( | 21 | from utils import ( |
| 22 | + PerfIndex, | ||
| 21 | print_to_screen, | 23 | print_to_screen, |
| 22 | load_config, | 24 | load_config, |
| 23 | extract_result_from_perf_csv, | 25 | extract_result_from_perf_csv, |
| @@ -116,7 +118,7 @@ if __name__ == '__main__': | |||
| 116 | if args.compile or args.deployment: | 118 | if args.compile or args.deployment: |
| 117 | try: | 119 | try: |
| 118 | deployment(command_helper_instance, server_id, environment_config) | 120 | deployment(command_helper_instance, server_id, environment_config) |
| 119 | - except Exception: | 121 | + except Exception as e: |
| 120 | print_to_screen('Deployment failed, please check the compile result and try again') | 122 | print_to_screen('Deployment failed, please check the compile result and try again') |
| 121 | 123 | ||
| 122 | results_of_performance_test = [] | 124 | results_of_performance_test = [] |
| @@ -139,7 +141,7 @@ if __name__ == '__main__': | |||
| 139 | # start service | 141 | # start service |
| 140 | command_helper_instance.exec_command(server_id, f'export MIES_CONFIG_JSON_PATH={test_case["config_path"]}', | 142 | command_helper_instance.exec_command(server_id, f'export MIES_CONFIG_JSON_PATH={test_case["config_path"]}', |
| 141 | wait_time=1) | 143 | wait_time=1) |
| 142 | - command_helper_instance.exec_command(server_id, "mindie_llm_server", | 144 | + command_helper_instance.exec_command(server_id, f"mindie_llm_server", |
| 143 | True, wait_strs=["Daemon start success"], wait_time=180) | 145 | True, wait_strs=["Daemon start success"], wait_time=180) |
| 144 | 146 | ||
| 145 | # start client | 147 | # start client |
| @@ -18,6 +18,7 @@ import random | |||
| 18 | 18 | ||
| 19 | from command_helper import CommandHelper, kill_all_service | 19 | from command_helper import CommandHelper, kill_all_service |
| 20 | from utils import ( | 20 | from utils import ( |
| 21 | + PerfIndex, | ||
| 21 | print_to_screen, | 22 | print_to_screen, |
| 22 | load_config, | 23 | load_config, |
| 23 | extract_result_from_perf_csv, | 24 | extract_result_from_perf_csv, |
| @@ -62,17 +63,17 @@ def set_env(command_helper_instance, terminal_id, env_config): | |||
| 62 | command_helper_instance.exec_command( | 63 | command_helper_instance.exec_command( |
| 63 | terminal_id, f'export RANK_TABLE_FILE={env_config["RankTableFile"]}', wait_time=1) | 64 | terminal_id, f'export RANK_TABLE_FILE={env_config["RankTableFile"]}', wait_time=1) |
| 64 | command_helper_instance.exec_command( | 65 | command_helper_instance.exec_command( |
| 65 | - terminal_id, 'export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True', wait_time=1) | 66 | + terminal_id, f'export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True', wait_time=1) |
| 66 | command_helper_instance.exec_command( | 67 | command_helper_instance.exec_command( |
| 67 | - terminal_id, 'export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3', wait_time=1) | 68 | + terminal_id, f'export ATB_WORKSPACE_MEM_ALLOC_ALG_TYPE=3', wait_time=1) |
| 68 | command_helper_instance.exec_command( | 69 | command_helper_instance.exec_command( |
| 69 | - terminal_id, 'export NPU_MEMORY_FRACTION=0.96', wait_time=1) | 70 | + terminal_id, f'export NPU_MEMORY_FRACTION=0.96', wait_time=1) |
| 70 | command_helper_instance.exec_command( | 71 | command_helper_instance.exec_command( |
| 71 | terminal_id, f'export MIES_CONTAINER_IP={env_config["ip"]}', wait_time=1) | 72 | terminal_id, f'export MIES_CONTAINER_IP={env_config["ip"]}', wait_time=1) |
| 72 | command_helper_instance.exec_command( | 73 | command_helper_instance.exec_command( |
| 73 | - terminal_id, 'export HCCL_CONNECT_TIMEOUT=7200', wait_time=1) | 74 | + terminal_id, f'export HCCL_CONNECT_TIMEOUT=7200', wait_time=1) |
| 74 | command_helper_instance.exec_command( | 75 | command_helper_instance.exec_command( |
| 75 | - terminal_id, 'HCCL_EXEC_TIMEOUT=0', wait_time=1) | 76 | + terminal_id, f'HCCL_EXEC_TIMEOUT=0', wait_time=1) |
| 76 | 77 | ||
| 77 | 78 | ||
| 78 | if __name__ == '__main__': | 79 | if __name__ == '__main__': |
| @@ -128,7 +129,7 @@ if __name__ == '__main__': | |||
| 128 | command_helper_instance.exec_command(server_id[-1], f'docker start {container_name}', wait_time=5) | 129 | command_helper_instance.exec_command(server_id[-1], f'docker start {container_name}', wait_time=5) |
| 129 | command_helper_instance.exec_command(server_id[-1], f'docker exec -it {container_name} bash', wait_time=5) | 130 | command_helper_instance.exec_command(server_id[-1], f'docker exec -it {container_name} bash', wait_time=5) |
| 130 | time.sleep(5) | 131 | time.sleep(5) |
| 131 | - print_to_screen('Check if slave node is in container using command `tmux attach -t <session-name>:2`') | 132 | + print_to_screen(f'Check if slave node is in container using command `tmux attach -t <session-name>:2`') |
| 132 | set_env(command_helper_instance, server_id[-1], env_config) | 133 | set_env(command_helper_instance, server_id[-1], env_config) |
| 133 | 134 | ||
| 134 | results_of_performance_test = [] | 135 | results_of_performance_test = [] |


mindie_motor/python/mindie_motor/node_manager/common/logging.py此评论由代码审查工具自动生成