已合并
[BugFix] 新构建方案需求适配，支持构建与测试 #605
code_mingming创建于 6月23日
[BugFix] 新构建方案需求适配，支持构建与测试 #605
已合并
共 6 个文件变更+52-10
| @@ -16,6 +16,7 @@ | |||
| 16 | # See the Mulan PSL v2 for more details. | 16 | # See the Mulan PSL v2 for more details. |
| 17 | # ------------------------------------------------------------------------- | 17 | # ------------------------------------------------------------------------- |
| 18 | import argparse | 18 | import argparse |
| 19 | +import importlib.util | ||
| 19 | import logging | 20 | import logging |
| 20 | import os | 21 | import os |
| 21 | import shutil | 22 | import shutil |
| @@ -83,18 +84,51 @@ class BuildManager: | |||
| 83 | logging.info("Archiving: %s -> %s", artifact, destination) | 84 | logging.info("Archiving: %s -> %s", artifact, destination) |
| 84 | shutil.copy2(artifact, destination) | 85 | shutil.copy2(artifact, destination) |
| 85 | 86 | ||
| 87 | + def _prepare_runtime_dependencies(self): | ||
| 88 | + """安装运行时依赖（来源 requirements.txt，由 install.sh 完成）。""" | ||
| 89 | + self._execute_command(["bash", "install.sh"], cwd=self.project_root) | ||
| 90 | + | ||
| 91 | + def _prepare_test_dependencies(self): | ||
| 92 | + """安装测试专用依赖（在运行时依赖之上额外安装，来源 test/requirements.txt）。""" | ||
| 93 | + self._execute_command( | ||
| 94 | + ["pip", "install", "-r", str(self.project_root / "test" / "requirements.txt")], | ||
| 95 | + cwd=self.project_root, | ||
| 96 | + ) | ||
| 97 | + | ||
| 98 | + def _check_torch_npu_conflict(self): | ||
| 99 | + """检测环境是否安装了 torch_npu，存在则告警。 | ||
| 100 | + | ||
| 101 | + UT 基于“纯 torch + mock torch_npu”运行，若环境真实安装了 torch_npu， | ||
| 102 | + 测试用例中的 mock 兜底逻辑会被跳过，torch_npu 指向真实 NPU 后端， | ||
| 103 | + 在无 NPU 硬件的 UT 环境下可能导致用例失败或行为异常。 | ||
| 104 | + 此处仅告警，不卸载，避免隐式修改系统级环境。 | ||
| 105 | + """ | ||
| 106 | + if importlib.util.find_spec("torch_npu") is not None: | ||
| 107 | + logging.warning( | ||
| 108 | + "torch_npu is installed in the current environment. " | ||
| 109 | + "UT runs on pure torch with mocked torch.npu; " | ||
| 110 | + "a real torch_npu may cause test failures." | ||
| 111 | + ) | ||
| 112 | + | ||
| 86 | def run(self): | 113 | def run(self): |
| 87 | os.chdir(self.project_root) | 114 | os.chdir(self.project_root) |
| 115 | + is_local = 'local' in self.args.command | ||
| 116 | + is_test = 'test' in self.args.command | ||
| 88 | 117 | ||
| 89 | # 在非 local 场景下按需更新依赖；在 local 场景下仅使用本地已有代码，不更新依赖。 | 118 | # 在非 local 场景下按需更新依赖；在 local 场景下仅使用本地已有代码，不更新依赖。 |
| 90 | - if 'local' not in self.args.command: | 119 | + if not is_local: |
| 91 | - # 补充依赖下载处理 | 120 | + if is_test: |
| 92 | - pass | 121 | + # 测试：先安装测试专用依赖，再安装运行时依赖，可减少一些重复安装 |
| 122 | + self._prepare_test_dependencies() | ||
| 123 | + self._prepare_runtime_dependencies() | ||
| 124 | + else: | ||
| 125 | + # 构建：setup.py bdist_wheel 直接读取 requirements.txt 作为 install_requires，无需预先安装 | ||
| 126 | + pass | ||
| 93 | 127 | ||
| 94 | - if 'test' in self.args.command: | 128 | + if is_test: |
| 95 | # -------------------- 单元测试 -------------------- | 129 | # -------------------- 单元测试 -------------------- |
| 96 | - self._execute_command(["bash", "install.sh"], cwd=self.project_root) | 130 | + self._check_torch_npu_conflict() |
| 97 | - self._execute_command(["bash", "run_ut.sh"], cwd=self.project_root / "test") | 131 | + self._execute_command(["bash", "run_ut.sh", "--modelslim_v1"], cwd=self.project_root / "test") |
| 98 | else: | 132 | else: |
| 99 | # -------------------- 产品构建 -------------------- | 133 | # -------------------- 产品构建 -------------------- |
| 100 | logging.info("--version: %s", self.args.version) | 134 | logging.info("--version: %s", self.args.version) |
| @@ -29,6 +29,7 @@ python --version # 应显示 Python 3.10.x | |||
| 29 | 29 | ||
| 30 | ```bash | 30 | ```bash |
| 31 | pip install pytest | 31 | pip install pytest |
| 32 | +pip install pytest-mock | ||
| 32 | pip install coverage | 33 | pip install coverage |
| 33 | pip install torch==2.1.0 | 34 | pip install torch==2.1.0 |
| 34 | pip install easydict==1.13 | 35 | pip install easydict==1.13 |
| @@ -38,7 +38,7 @@ class TestBuildQuantTips: | |||
| 38 | 38 | ||
| 39 | assert "No quant_type" in result | 39 | assert "No quant_type" in result |
| 40 | assert "default_practice_id" in result | 40 | assert "default_practice_id" in result |
| 41 | - assert QuantType.W8A8.value in result | 41 | + assert QuantType.W8A8.value in result or str(QuantType.W8A8) in result |
| 42 | 42 | ||
| 43 | def test_build_quant_tips_returns_empty_string_when_q1c0b1(self): | 43 | def test_build_quant_tips_returns_empty_string_when_q1c0b1(self): |
| 44 | """主路径：Q1C0B1（找到最佳实践且未变更）应返回空串（无需额外提示）。""" | 44 | """主路径：Q1C0B1（找到最佳实践且未变更）应返回空串（无需额外提示）。""" |
| @@ -51,7 +51,7 @@ class TestBuildQuantTips: | |||
| 51 | result = _build_quant_tips(TipsType.Q1C0B0, "qwen3", QuantType.W4A8, "default_id") | 51 | result = _build_quant_tips(TipsType.Q1C0B0, "qwen3", QuantType.W4A8, "default_id") |
| 52 | 52 | ||
| 53 | assert "qwen3" in result | 53 | assert "qwen3" in result |
| 54 | - assert "w4a8" in result # quant_type 渲染为小写 | 54 | + assert QuantType.W4A8.value in result or str(QuantType.W4A8) in result |
| 55 | 55 | ||
| 56 | def test_build_quant_tips_includes_quant_type_when_changed(self): | 56 | def test_build_quant_tips_includes_quant_type_when_changed(self): |
| 57 | """主路径：Q1C1B0 时提示应包含 quant_type（被变更的）。""" | 57 | """主路径：Q1C1B0 时提示应包含 quant_type（被变更的）。""" |
| @@ -48,7 +48,7 @@ class TestAscendV1QuantFormatConfig: | |||
| 48 | 48 | ||
| 49 | dumped = config.model_dump() | 49 | dumped = config.model_dump() |
| 50 | 50 | ||
| 51 | - assert dumped["ext"] == {} | 51 | + assert not dumped.get("ext") |
| 52 | 52 | ||
| 53 | def test_model_dump_include_ext_when_ext_nonempty(self): | 53 | def test_model_dump_include_ext_when_ext_nonempty(self): |
| 54 | config = AscendV1QuantFormatConfig(ext={"key": "value"}) | 54 | config = AscendV1QuantFormatConfig(ext={"key": "value"}) |
| @@ -48,7 +48,7 @@ class TestMindIEQuantFormatConfig: | |||
| 48 | 48 | ||
| 49 | dumped = config.model_dump() | 49 | dumped = config.model_dump() |
| 50 | 50 | ||
| 51 | - assert dumped["ext"] == {} | 51 | + assert not dumped.get("ext") |
| 52 | 52 | ||
| 53 | def test_model_dump_include_ext_when_ext_nonempty(self): | 53 | def test_model_dump_include_ext_when_ext_nonempty(self): |
| 54 | config = MindIEQuantFormatConfig(ext={"key": "value"}) | 54 | config = MindIEQuantFormatConfig(ext={"key": "value"}) |
🔵 Low Priority
test/requirements.txt中部分关键依赖未固定版本号：pytest、pytest-mock、coverage、pyyaml均未使用版本约束。主要风险在于
pyyaml— 该库历史上有多个安全漏洞（CVE-2017-18342、CVE-2020-1747 等），涉及yaml.load()任意代码执行。虽然项目本身使用yaml_safe_load（见msmodelslim/utils/security.py），但如果环境中安装了包含已知漏洞的旧版本 pyyaml，仍存在一定风险。testing_guide.md中列出了pip install pyyaml（同样未固定版本），两者一致，但最佳实践是固定已知安全的版本（如pyyaml>=6.0）。建议：建议对 pyyaml 添加最低版本约束
pyyaml>=6.0。对于其他测试工具（pytest、pytest-mock、coverage），如希望严格可复现，也可考虑固定版本或使用下限约束。