Merged
fix(dsl): 清理框架 UT 并隔离 UT 的 DSL 产物构建 #15464
zyh_gitcodeAdd on 3 days ago
fix(dsl): 清理框架 UT 并隔离 UT 的 DSL 产物构建 #15464
Merged
Total: 4 files changed+4-1138
| @@ -762,6 +762,10 @@ endfunction() | |||
| 762 | 762 | ||
| 763 | # Build PIC Host archives and publish devices into the existing ASC kernel tree. | 763 | # Build PIC Host archives and publish devices into the existing ASC kernel tree. |
| 764 | function(enable_cannbotdsl_kernel op_name source_dir) | 764 | function(enable_cannbotdsl_kernel op_name source_dir) |
| 765 | + # Unit-test targets do not publish DSL Host archives or device resources. | ||
| 766 | + if(UT_TEST_ALL OR OP_HOST_UT OR OP_API_UT OR OP_KERNEL_UT OR OP_GRAPH_UT) | ||
| 767 | + return() | ||
| 768 | + endif() | ||
| 765 | if(PREPARE_BUILD OR NOT CANNBOTDSL_COMPUTE_UNIT IN_LIST ASCEND_COMPUTE_UNIT) | 769 | if(PREPARE_BUILD OR NOT CANNBOTDSL_COMPUTE_UNIT IN_LIST ASCEND_COMPUTE_UNIT) |
| 766 | return() | 770 | return() |
| 767 | endif() | 771 | endif() |
| @@ -1,643 +0,0 @@ | |||
| 1 | -# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 2 | -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 3 | -# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 4 | -# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 5 | -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 6 | -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 7 | -# See LICENSE in the root of the software repository for the full text of the License. | ||
| 8 | - | ||
| 9 | -"""ACLNN source publication and linkage alongside the existing Native path.""" | ||
| 10 | - | ||
| 11 | -import ctypes | ||
| 12 | -import hashlib | ||
| 13 | -import importlib.util | ||
| 14 | -import json | ||
| 15 | -import shutil | ||
| 16 | -import subprocess | ||
| 17 | -import sys | ||
| 18 | -import types | ||
| 19 | -from pathlib import Path | ||
| 20 | - | ||
| 21 | -import pytest | ||
| 22 | - | ||
| 23 | -from test_cannbotdsl_build import ROOT, BUILDER, builder, cmake, write | ||
| 24 | - | ||
| 25 | - | ||
| 26 | -def publish_aclnn(root, *, mode, l0, l2, **unused): | ||
| 27 | - if mode == "kernel": | ||
| 28 | - write(root / "kernel/ascend950/sample/Sample.o", b"device") | ||
| 29 | - write(root / "kernel/ascend950/sample/Sample.json", "{}") | ||
| 30 | - return | ||
| 31 | - write(root / "runtime/runtime.cpp", 'extern "C" int runtime_value() { return 17; }') | ||
| 32 | - write( | ||
| 33 | - root / "runtime/CMakeLists.txt", | ||
| 34 | - """ | ||
| 35 | -set(CANN_ROOT "ignored" CACHE PATH "fixture") | ||
| 36 | -set(OPBASE_INCLUDE_DIR "ignored" CACHE PATH "fixture") | ||
| 37 | -set(OPBASE_LIBRARY "ignored" CACHE STRING "fixture") | ||
| 38 | -add_library(opkit_runtime STATIC runtime.cpp) | ||
| 39 | -add_library(opkit::runtime ALIAS opkit_runtime) | ||
| 40 | -set_target_properties(opkit_runtime PROPERTIES POSITION_INDEPENDENT_CODE ON) | ||
| 41 | -""", | ||
| 42 | - ) | ||
| 43 | - module = root / "aclnn/sample" | ||
| 44 | - write( | ||
| 45 | - module / "op_host/sample_tiling.cpp", | ||
| 46 | - 'extern "C" int runtime_value();\n' | ||
| 47 | - 'extern "C" int PrepareSample() { return runtime_value(); }', | ||
| 48 | - ) | ||
| 49 | - write( | ||
| 50 | - module / "op_host/sample_binding.cpp", | ||
| 51 | - 'extern "C" int SampleL0Contract() { return 0; }\n' | ||
| 52 | - 'extern "C" int OpKitPrepareDirectInvokeSample() { return 0; }', | ||
| 53 | - ) | ||
| 54 | - write( | ||
| 55 | - module / "op_api/aclnn_sample.h", | ||
| 56 | - 'extern "C" int aclnnSample();\n' | ||
| 57 | - 'extern "C" int aclnnSampleGetWorkspaceSize();\n', | ||
| 58 | - ) | ||
| 59 | - if l0: | ||
| 60 | - write( | ||
| 61 | - module / "op_api/sample.cpp", | ||
| 62 | - 'extern "C" int PrepareSample();\n' | ||
| 63 | - 'extern "C" int SampleL0() { return PrepareSample(); }', | ||
| 64 | - ) | ||
| 65 | - if l2: | ||
| 66 | - write( | ||
| 67 | - module / "op_api/aclnn_sample.cpp", | ||
| 68 | - 'extern "C" int SampleL0();\n' | ||
| 69 | - 'extern "C" int aclnnSample() { return SampleL0(); }\n' | ||
| 70 | - 'extern "C" int aclnnSampleGetWorkspaceSize() { return 0; }', | ||
| 71 | - ) | ||
| 72 | - write(module / "CMakeLists.txt", "# Generated OpKit module\n") | ||
| 73 | - write( | ||
| 74 | - root / "package.json", | ||
| 75 | - json.dumps( | ||
| 76 | - dict( | ||
| 77 | - kind="opkit", | ||
| 78 | - binary_format="device", | ||
| 79 | - mode="host", | ||
| 80 | - target="dav-3510", | ||
| 81 | - runtime="runtime", | ||
| 82 | - operators=["Sample"], | ||
| 83 | - modules={"Sample": "aclnn/sample"}, | ||
| 84 | - ) | ||
| 85 | - ), | ||
| 86 | - ) | ||
| 87 | - write( | ||
| 88 | - root / "manifest.json", | ||
| 89 | - json.dumps( | ||
| 90 | - dict( | ||
| 91 | - files={ | ||
| 92 | - str(path.relative_to(root)): hashlib.sha256( | ||
| 93 | - path.read_bytes() | ||
| 94 | - ).hexdigest() | ||
| 95 | - for path in root.rglob("*") | ||
| 96 | - if path.is_file() | ||
| 97 | - } | ||
| 98 | - ) | ||
| 99 | - ), | ||
| 100 | - ) | ||
| @@ -1,344 +0,0 @@ | |||
| 1 | -# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 2 | -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 3 | -# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 4 | -# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 5 | -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 6 | -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 7 | -# See LICENSE in the root of the software repository for the full text of the License. | ||
| 8 | - | ||
| 9 | -"""DSL publication and optional dependency integration with the ASC build.""" | ||
| 10 | - | ||
| 11 | -import ctypes | ||
| 12 | -import importlib.util | ||
| 13 | -import json | ||
| 14 | -import shlex | ||
| 15 | -import subprocess | ||
| 16 | -import sys | ||
| 17 | -import types | ||
| 18 | -from pathlib import Path | ||
| 19 | - | ||
| 20 | -import pytest | ||
| 21 | - | ||
| 22 | -ROOT = Path(__file__).resolve().parents[3] | ||
| 23 | -BUILDER = ROOT / "cmake/scripts/build_cannbotdsl.py" | ||
| 24 | - | ||
| 25 | - | ||
| 26 | -def write(path, content): | ||
| 27 | - path.parent.mkdir(parents=True, exist_ok=True) | ||
| 28 | - if isinstance(content, bytes): | ||
| 29 | - path.write_bytes(content) | ||
| 30 | - else: | ||
| 31 | - path.write_text(content) | ||
| 32 | - return path | ||
| 33 | - | ||
| 34 | - | ||
| 35 | -def cmake(*arguments, succeeds=True): | ||
| 36 | - result = subprocess.run( | ||
| 37 | - ["cmake", *map(str, arguments)], capture_output=True, text=True | ||
| 38 | - ) | ||
| 39 | - assert (result.returncode == 0) == succeeds, result.stdout + result.stderr | ||
| 40 | - return result | ||
| 41 | - | ||
| 42 | - | ||
| 43 | -def build_install(project): | ||
| 44 | - cmake("--build", project / "build") | ||
| 45 | - cmake("--install", project / "build", "--prefix", project / "installed") | ||
| 46 | - (library,) = (project / "installed").rglob("*.so") | ||
| 47 | - loaded = ctypes.CDLL(str(library)) | ||
| 48 | - assert loaded.asc_entry() == 17 | ||
| 49 | - return loaded | ||
| 50 | - | ||
| 51 | - | ||
| 52 | -@pytest.fixture | ||
| 53 | -def builder(monkeypatch): | ||
| 54 | - spec = importlib.util.spec_from_file_location("dsl_builder", BUILDER) | ||
| 55 | - module = importlib.util.module_from_spec(spec) | ||
| 56 | - spec.loader.exec_module(module) | ||
| 57 | - monkeypatch.setattr(module, "framework_files", lambda: []) | ||
| 58 | - monkeypatch.setattr( | ||
| 59 | - module, "source_exports", lambda source: dict(native=True, aclnn=False) | ||
| 60 | - ) | ||
| 61 | - return module | ||
| 62 | - | ||
| 63 | - | ||
| 64 | -def test_publication_cache_tracks_inputs_and_preserves_previous_package( | ||
| 65 | - builder, tmp_path, monkeypatch | ||
| 66 | -): | ||
| 67 | - source = write(tmp_path / "collector/entry.py", "old") | ||
| 68 | - compiler = write(tmp_path / "compiler.so", b"compiler-v1") | ||
| 69 | - monkeypatch.setattr(builder, "framework_files", lambda: [compiler]) | ||
| 70 | - calls, fail = [], False | ||
| 71 | - | ||
| 72 | - def publish(**options): | ||
| 73 | - calls.append(options["target"]) | ||
| 74 | - write(options["output"] / "archive.a", source.read_text()) | ||
| 75 | - if fail: | ||
| 76 | - raise RuntimeError("compiler failed") | ||
| 77 | - | ||
| 78 | - monkeypatch.setitem(sys.modules, "opkit", types.SimpleNamespace(build=publish)) | ||
| 79 | - output = tmp_path / "host" | ||
| 80 | - builder.build(source, output, "ascend950", "host") | ||
| 81 | - before = (output / "archive.a").stat().st_mtime_ns | ||
| 82 | - builder.build(source, output, "ascend950", "host") | ||
| 83 | - assert len(calls) == 1 and (output / "archive.a").stat().st_mtime_ns == before | ||
| 84 | - compiler.write_bytes(b"compiler-v2") | ||
| 85 | - builder.build(source, output, "ascend950", "host") | ||
| 86 | - assert len(calls) == 2 | ||
| 87 | - source.write_text("new") | ||
| 88 | - fail = True | ||
| 89 | - with pytest.raises(RuntimeError, match="compiler failed"): | ||
| 90 | - builder.build(source, output, "ascend950", "host") | ||
| 91 | - assert (output / "archive.a").read_text() == "old" | ||
| 92 | - fail = False | ||
| 93 | - builder.build(source, output, "ascend950", "host") | ||
| 94 | - assert (output / "archive.a").read_text() == "new" | ||
| 95 | - (output / "archive.a").write_text("damaged") | ||
| 96 | - builder.build(source, output, "ascend950", "host") | ||
| 97 | - assert (output / "archive.a").read_text() == "new" and len(calls) == 5 | ||
| 98 | - builder.build(source, output, "ascend960dt", "host", target="configured-target") | ||
| 99 | - state = json.loads((output / ".transformer-build.json").read_text()) | ||
| 100 | - assert state["record"]["inputs"]["target"] == calls[-1] == "configured-target" | ||
| 101 | - assert state["record"]["inputs"]["soc"] == "ascend960dt" and len(calls) == 6 | ||
| 102 | - | ||
| 103 | - | ||
| 104 | -def test_atomic_replacement_restores_previous_package(builder, tmp_path, monkeypatch): | ||
| 105 | - output = tmp_path / "package" | ||
| 106 | - write(output / "valid", "previous") | ||
| 107 | - rename = Path.rename | ||
| 108 | - | ||
| 109 | - def fail_publish(path, destination): | ||
| 110 | - if path.name == "package" and path != output: | ||
| 111 | - raise OSError("replace failed") | ||
| 112 | - return rename(path, destination) | ||
| 113 | - | ||
| 114 | - monkeypatch.setattr(Path, "rename", fail_publish) | ||
| 115 | - with pytest.raises(OSError, match="replace failed"): | ||
| 116 | - with builder.publication(output) as stage: | ||
| 117 | - write(stage / "new", "new") | ||
| 118 | - assert (output / "valid").read_text() == "previous" | ||
| 119 | - | ||
| 120 | - | ||
| 121 | -def test_input_dependencies_preserve_cmake_paths(builder, tmp_path, monkeypatch): | ||
| 122 | - source = write(tmp_path / "entry.py", "# collector") | ||
| 123 | - dependencies = [ | ||
| 124 | - write(tmp_path / name, "# dependency") | ||
| 125 | - for name in ( | ||
| 126 | - "space path.py", | ||
| 127 | - "semicolon;path.py", | ||
| 128 | - "中文输入.py", | ||
| 129 | - ) | ||
| 130 | - ] | ||
| 131 | - monkeypatch.setattr(builder, "framework_files", lambda: dependencies) | ||
| 132 | - inputs = tmp_path / "inputs.json" | ||
| 133 | - monkeypatch.setattr( | ||
| 134 | - sys, | ||
| 135 | - "argv", | ||
| 136 | - [ | ||
| 137 | - str(BUILDER), | ||
| 138 | - "--mode", | ||
| 139 | - "inputs", | ||
| 140 | - "--source", | ||
| 141 | - str(source), | ||
| 142 | - "--output", | ||
| 143 | - str(inputs), | ||
| 144 | - ], | ||
| 145 | - ) | ||
| 146 | - builder.main() | ||
| 147 | - script = write( | ||
| 148 | - tmp_path / "dependencies.cmake", | ||
| 149 | - """ | ||
| 150 | -file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/inputs.list" dependencies ENCODING UTF-8) | ||
| 151 | -set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${dependencies}") | ||
| 152 | -get_property(dependencies DIRECTORY PROPERTY CMAKE_CONFIGURE_DEPENDS) | ||
| 153 | -foreach(path IN LISTS dependencies) | ||
| 154 | - file(APPEND "${CMAKE_CURRENT_LIST_DIR}/actual.list" "${path}\\n") | ||
| 155 | -endforeach() | ||
| 156 | -""", | ||
| 157 | - ) | ||
| 158 | - cmake("-P", script) | ||
| 159 | - assert set((tmp_path / "actual.list").read_text().splitlines()) == set( | ||
| 160 | - json.loads(inputs.read_text())["files"] | ||
| 161 | - ) | ||
| 162 | - | ||
| 163 | - | ||
| 164 | -def test_kernel_scan_replacement_respects_ownership(builder, tmp_path): | ||
| 165 | - package = tmp_path / "first" | ||
| 166 | - original = write(package / "kernel/ascend950/sample/Old.o", b"DSL") | ||
| 167 | - bins = tmp_path / "binary/ascend950/bin" | ||
| 168 | - neighbor = write(bins / "sample/asc.o", b"ASC") | ||
| 169 | - builder.sync_kernels(package, bins, "ascend950") | ||
| 170 | - original.unlink() | ||
| 171 | - write(package / "kernel/ascend950/sample/New.o", b"new DSL") | ||
| 172 | - builder.sync_kernels(package, bins, "ascend950") | ||
| 173 | - assert not (bins / "sample/Old.o").exists() | ||
| 174 | - assert (bins / "sample/New.o").read_bytes() == b"new DSL" | ||
| 175 | - assert neighbor.read_bytes() == b"ASC" | ||
| 176 | - other = tmp_path / "second" | ||
| 177 | - write(other / "kernel/ascend950/sample/New.o", b"other") | ||
| 178 | - with pytest.raises(ValueError, match="conflicting"): | ||
| 179 | - builder.sync_kernels(other, bins, "ascend950") | ||
| 180 | - foreign = write(tmp_path / "foreign/sample/New.o", b"ASC") | ||
| 181 | - with pytest.raises(ValueError, match="unowned"): | ||
| 182 | - builder.sync_kernels(package, foreign.parent.parent, "ascend950") | ||
| 183 | - assert foreign.read_bytes() == b"ASC" | ||
| 184 | - | ||
| 185 | - | ||
| 186 | -def _optional_dsl_project(tmp_path, scenario): | ||
| 187 | - dependencies = tmp_path / "dependencies" | ||
| 188 | - files = { | ||
| 189 | - "opkit/__init__.py": ( | ||
| 190 | - "def build(**kwargs):\n raise RuntimeError('DSL compiler failure')\n" | ||
| 191 | - ), | ||
| 192 | - "cannbotdsl/__init__.py": "", | ||
| 193 | - "cannbotdsl/aot/__init__.py": "", | ||
| 194 | - "cannbotdsl/aot/collection.py": ( | ||
| 195 | - "def _declares_registration(source, markers): return 'export' in markers\n" | ||
| 196 | - ), | ||
| 197 | - "cannbotdsl/core/__init__.py": "", | ||
| 198 | - "cannbotdsl/core/toolchain/__init__.py": "", | ||
| 199 | - "cannbotdsl/core/toolchain/ascendc.py": ( | ||
| 200 | - "from types import SimpleNamespace\n" | ||
| 201 | - f"def resolve_bisheng(): return SimpleNamespace(path={sys.executable!r})\n" | ||
| 202 | - ), | ||
| 203 | - } | ||
| 204 | - if scenario == "bad_import": | ||
| 205 | - files["cannbotdsl/__init__.py"] = "raise OSError('broken DSL import')\n" | ||
| 206 | - for relative, content in files.items(): | ||
| 207 | - write(dependencies / relative, content) | ||
| 208 | - sdk = tmp_path / "sdk" | ||
| 209 | - bootstrap = tmp_path / "probe.py" | ||
| 210 | - calls = tmp_path / "probe-calls.log" | ||
| 211 | - blocked = {"dsl": "cannbotdsl", "dsl_stale": "cannbotdsl", "opkit": "opkit"}.get( | ||
| 212 | - scenario | ||
| 213 | - ) | ||
| 214 | - bootstrap.write_text(f""" | ||
| 215 | -import importlib.abc, pathlib, runpy, sys | ||
| 216 | -sys.path.insert(0, {str(dependencies)!r}) | ||
| 217 | -sys.argv = sys.argv[1:] | ||
| 218 | -mode = 'check' if sys.argv[0] == '-c' else sys.argv[sys.argv.index('--mode') + 1] | ||
| 219 | -with pathlib.Path({str(calls)!r}).open('a') as stream: stream.write(mode + '\\n') | ||
| 220 | -class MissingDependency(importlib.abc.MetaPathFinder): | ||
| 221 | - def find_spec(self, fullname, path=None, target=None): | ||
| 222 | - if fullname == {blocked!r}: | ||
| 223 | - with pathlib.Path({str(calls.with_suffix(".lookups"))!r}).open('a') as stream: | ||
| 224 | - stream.write(mode + ':' + fullname + '\\n') | ||
| 225 | - raise ModuleNotFoundError('blocked optional dependency: ' + fullname, name=fullname) | ||
| 226 | -sys.meta_path.insert(0, MissingDependency()) | ||
| 227 | -if mode == 'check': | ||
| 228 | - exec(sys.argv[1], {{'__name__': '__main__'}}) | ||
| 229 | -else: | ||
| 230 | - runpy.run_path(sys.argv[0], run_name='__main__') | ||
| 231 | -""") | ||
| 232 | - launcher = tmp_path / "probe-python" | ||
| 233 | - launcher.write_text( | ||
| 234 | - "#!/bin/sh\nexec " | ||
| 235 | - + shlex.quote(sys.executable) | ||
| 236 | - + " -I " | ||
| 237 | - + shlex.quote(str(bootstrap)) | ||
| 238 | - + ' "$@"\n' | ||
| 239 | - ) | ||
| 240 | - launcher.chmod(0o755) | ||
| 241 | - for name in ("first", "second"): | ||
| 242 | - write( | ||
| 243 | - tmp_path / name / (name + ".py"), | ||
| 244 | - "raise AssertionError('unexpected collection')\n", | ||
| 245 | - ) | ||
| 246 | - (tmp_path / "asc.cpp").write_text('extern "C" int asc_entry() { return 17; }') | ||
| 247 | - (tmp_path / "CMakeLists.txt").write_text(f''' | ||
| 248 | -cmake_minimum_required(VERSION 3.18.4) | ||
| 249 | -project(compatibility CXX) | ||
| 250 | -include("{ROOT}/cmake/func.cmake") | ||
| 251 | -set(OPS_ADV_DIR "{ROOT}") | ||
| 252 | -set(HI_PYTHON "{launcher}") | ||
| 253 | -set(ASCEND_CANN_PACKAGE_PATH "{sdk}") | ||
| 254 | -set(ASCEND_BINARY_OUT_DIR "${{CMAKE_BINARY_DIR}}/binary") | ||
| 255 | -set(ASCEND_COMPUTE_UNIT ascend950) | ||
| 256 | -set(CANNBOTDSL_COMPUTE_UNIT ascend950) | ||
| 257 | -set(CANNBOTDSL_TARGET dav-3510) | ||
| 258 | -set(ENABLE_OPS_HOST ON) | ||
| 259 | -set(ENABLE_OPS_KERNEL ON) | ||
| 260 | -add_library(asc SHARED asc.cpp) | ||
| 261 | -set_target_properties(asc PROPERTIES SUFFIX ".so") | ||
| 262 | -install(TARGETS asc LIBRARY DESTINATION lib) | ||
| 263 | -enable_cannbotdsl_kernel(first "{tmp_path}/first") | ||
| 264 | -enable_cannbotdsl_kernel(second "{tmp_path}/second") | ||
| 265 | -cannbotdsl_clean_resources() | ||
| 266 | -''') | ||
| 267 | - return calls | ||
| 268 | - | ||
| 269 | - | ||
| 270 | -@pytest.mark.parametrize("scenario", ["dsl", "opkit"]) | ||
| 271 | -def test_real_unavailable_probe_still_builds_and_installs_asc(tmp_path, scenario): | ||
| 272 | - calls = _optional_dsl_project(tmp_path, scenario) | ||
| 273 | - build = tmp_path / "build" | ||
| 274 | - result = cmake("-S", tmp_path, "-B", build) | ||
| 275 | - for name in ("first", "second"): | ||
| 276 | - assert f"DSL compilation skipped (not compiled): {name}" in result.stderr | ||
| 277 | - assert "cannbotdsl or opkit is unavailable in" in " ".join(result.stderr.split()) | ||
| 278 | - assert "Traceback" not in result.stderr | ||
| 279 | - assert calls.read_text() == "check\n" | ||
| 280 | - build_install(tmp_path) | ||
| 281 | - assert not (build / "cannbot_dsl").exists() | ||
| 282 | - assert calls.read_text() == "check\n" | ||
| 283 | - | ||
| 284 | - | ||
| 285 | -def test_real_unavailable_probe_cleans_owned_stale_resources_and_keeps_asc(tmp_path): | ||
| 286 | - calls = _optional_dsl_project(tmp_path, "dsl_stale") | ||
| 287 | - build = tmp_path / "build" | ||
| 288 | - root = build / "cannbot_dsl" | ||
| 289 | - package = root / "first/device" | ||
| 290 | - resources = package / "kernel/ascend950/retired" | ||
| 291 | - resources.mkdir(parents=True) | ||
| 292 | - bins = build / "binary/ascend950/bin" | ||
| 293 | - retired = bins / "retired" | ||
| 294 | - retired.mkdir(parents=True) | ||
| 295 | - owned = ["retired/Old.o", "retired/Old.json"] | ||
| 296 | - for name in ("Old.o", "Old.json"): | ||
| 297 | - (resources / name).write_bytes(b"old DSL") | ||
| 298 | - (retired / name).write_bytes(b"old DSL") | ||
| 299 | - (bins.parent / ".cannbotdsl-files.json").write_text( | ||
| 300 | - json.dumps({str(package): owned}) | ||
| 301 | - ) | ||
| 302 | - (bins / "retired.json").write_text("old DSL summary") | ||
| 303 | - index_names = ("binary_info_config.json", "relocatable_kernel_info_config.json") | ||
| 304 | - for name in index_names: | ||
| 305 | - (bins / name).write_text("old DSL index") | ||
| 306 | - asc_neighbor = bins / "asc.o" | ||
| 307 | - asc_neighbor.write_bytes(b"ASC neighbor") | ||
| 308 | - stamp = root / "first/kernel.built" | ||
| 309 | - stamp.touch() | ||
| 310 | - metadata = root / "metadata/host.json" | ||
| 311 | - metadata.parent.mkdir() | ||
| 312 | - metadata.write_text("old DSL metadata") | ||
| 313 | - registry = build / "opkit-native/host/registry.cpp" | ||
| 314 | - registry.parent.mkdir(parents=True) | ||
| 315 | - registry.write_text("old DSL registry") | ||
| 316 | - | ||
| 317 | - cmake("-S", tmp_path, "-B", build) | ||
| 318 | - build_install(tmp_path) | ||
| 319 | - assert calls.read_text() == "check\nclean\n" | ||
| 320 | - assert calls.with_suffix(".lookups").read_text() == "check:cannbotdsl\n" | ||
| 321 | - assert not retired.exists() and not (bins / "retired.json").exists() | ||
| 322 | - assert not any((bins / name).exists() for name in index_names) | ||
| 323 | - assert not stamp.exists() and not metadata.exists() and not registry.exists() | ||
| 324 | - assert asc_neighbor.read_bytes() == b"ASC neighbor" | ||
| 325 | - assert json.loads((bins.parent / ".cannbotdsl-files.json").read_text()) == {} | ||
| 326 | - assert not list((tmp_path / "installed").rglob("Old.*")) | ||
| 327 | - | ||
| 328 | - | ||
| 329 | -@pytest.mark.parametrize( | ||
| 330 | - "scenario,reason,modes", | ||
| 331 | - [ | ||
| 332 | - ("bad_import", "broken DSL import", ["check", "inputs"]), | ||
| 333 | - ("build_failure", "DSL compiler failure", ["check", "inputs", "host"]), | ||
| 334 | - ], | ||
| 335 | -) | ||
| 336 | -def test_discoverable_modules_do_not_hide_real_dsl_build_errors( | ||
| 337 | - tmp_path, scenario, reason, modes | ||
| 338 | -): | ||
| 339 | - calls = _optional_dsl_project(tmp_path, scenario) | ||
| 340 | - (tmp_path / "first/first.py").write_text("# collector\n") | ||
| 341 | - result = cmake("-S", tmp_path, "-B", tmp_path / "build", succeeds=False) | ||
| 342 | - assert reason in result.stdout + result.stderr | ||
| 343 | - assert "DSL compilation skipped" not in result.stderr | ||
| 344 | - assert calls.read_text().splitlines() == modes | ||
| @@ -1,151 +0,0 @@ | |||
| 1 | -# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 2 | -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 3 | -# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 4 | -# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 5 | -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 6 | -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 7 | -# See LICENSE in the root of the software repository for the full text of the License. | ||
| 8 | - | ||
| 9 | -"""DSL source wheel packaging and custom Torch package imports.""" | ||
| 10 | - | ||
| 11 | -import builtins | ||
| 12 | -import importlib.util | ||
| 13 | -import os | ||
| 14 | -import shutil | ||
| 15 | -import subprocess | ||
| 16 | -import sys | ||
| 17 | -import types | ||
| 18 | -import zipfile | ||
| 19 | -from pathlib import Path | ||
| 20 | - | ||
| 21 | -import pytest | ||
| 22 | - | ||
| 23 | -ROOT = Path(__file__).resolve().parents[3] | ||
| 24 | -PACKAGE = ROOT / "torch_extension/cann_ops_transformer" | ||
| 25 | - | ||
| 26 | - | ||
| 27 | -@pytest.mark.parametrize("vendor", [None, "review"]) | ||
| 28 | -def test_wheel_includes_recursive_dsl_sources_with_working_relative_imports( | ||
| 29 | - tmp_path, vendor | ||
| 30 | -): | ||
| 31 | - project = tmp_path / "project" | ||
| 32 | - setup_dir = project / "torch_extension" | ||
| 33 | - base = setup_dir / "cann_ops_transformer" | ||
| 34 | - (base / "ops/attention").mkdir(parents=True) | ||
| 35 | - for directory in (base, base / "ops", base / "ops/attention"): | ||
| 36 | - (directory / "__init__.py").write_text("") | ||
| 37 | - shutil.copy2(ROOT / "torch_extension/setup.py", setup_dir / "setup.py") | ||
| 38 | - expected_sources = {} | ||
| 39 | - for name in ("sample", "other"): | ||
| 40 | - op = project / "attention" / name | ||
| 41 | - (op / "torch_extension").mkdir(parents=True) | ||
| 42 | - (op / "torch_extension/__init__.py").write_text( | ||
| 43 | - "def sample():\n from .op_dsl.sample import compile_sample\n return compile_sample()\n" | ||
| 44 | - ) | ||
| 45 | - (op / "op_dsl/helpers").mkdir(parents=True) | ||
| 46 | - for relative, content in { | ||
| 47 | - "sample.py": "from .helpers.constants import VALUE\ndef compile_sample():\n return VALUE\n", | ||
| 48 | - "helpers/__init__.py": "", | ||
| 49 | - "helpers/constants.py": "VALUE = 17\n", | ||
| 50 | - }.items(): | ||
| 51 | - (op / "op_dsl" / relative).write_text(content) | ||
| 52 | - if name == "sample": | ||
| 53 | - expected_sources[relative] = content | ||
| 54 | - (op / "op_dsl/README.md").write_text("build documentation") | ||
| 55 | - (op / "op_dsl/temporary.o").write_bytes(b"not a runtime source") | ||
| 56 | - (op / "op_dsl/__pycache__").mkdir() | ||
| 57 | - (op / "op_dsl/__pycache__/stale.py").write_text("raise RuntimeError('stale')") | ||
| 58 | - environment = os.environ.copy() | ||
| 59 | - for key in ( | ||
| 60 | - "TORCH_EXTENSION_OPS", | ||
| 61 | - "TORCH_EXTENSION_VENDOR", | ||
| 62 | - "TORCH_EXTENSION_EXPERIMENTAL", | ||
| 63 | - ): | ||
| 64 | - environment.pop(key, None) | ||
| 65 | - if vendor: | ||
| 66 | - environment.update(TORCH_EXTENSION_OPS="sample", TORCH_EXTENSION_VENDOR=vendor) | ||
| 67 | - result = subprocess.run( | ||
| 68 | - [sys.executable, "setup.py", "bdist_wheel"], | ||
| 69 | - cwd=setup_dir, | ||
| 70 | - env=environment, | ||
| 71 | - text=True, | ||
| 72 | - capture_output=True, | ||
| 73 | - ) | ||
| 74 | - assert result.returncode == 0, result.stdout + result.stderr | ||
| 75 | - (wheel,) = (setup_dir / "dist").glob("*.whl") | ||
| 76 | - package = "cann_ops_transformer" + ("_" + vendor if vendor else "") | ||
| 77 | - prefix = package + "/ops/attention/sample/op_dsl/" | ||
| 78 | - with zipfile.ZipFile(wheel) as archive: | ||
| 79 | - names = archive.namelist() | ||
| 80 | - assert { | ||
| 81 | - name[len(prefix) :] for name in names if name.startswith(prefix) | ||
| 82 | - } == set(expected_sources) | ||
| 83 | - for relative, content in expected_sources.items(): | ||
| 84 | - assert archive.read(prefix + relative).decode() == content | ||
| 85 | - assert any("/other/op_dsl/" in name for name in names) == (vendor is None) | ||
| 86 | - installed = tmp_path / "installed" | ||
| 87 | - archive.extractall(installed) | ||
| 88 | - for key in ("ASCEND_OPP_PATH", "ASCEND_HOME_PATH", "ASCEND_CUSTOM_OPP_PATH"): | ||
| 89 | - environment.pop(key, None) | ||
| 90 | - result = subprocess.run( | ||
| 91 | - [ | ||
| 92 | - sys.executable, | ||
| 93 | - "-I", | ||
| 94 | - "-c", | ||
| 95 | - "import sys; sys.path.insert(0, sys.argv[1]); " | ||
| 96 | - "from " | ||
| 97 | - + package | ||
| 98 | - + ".ops.attention.sample import sample; assert sample() == 17", | ||
| 99 | - str(installed), | ||
| 100 | - ], | ||
| 101 | - text=True, | ||
| 102 | - capture_output=True, | ||
| 103 | - env=environment, | ||
| 104 | - ) | ||
| 105 | - assert result.returncode == 0, result.stdout + result.stderr | ||
| 106 | - | ||
| 107 | - | ||
| 108 | -def test_custom_builder_loads_without_base_package(tmp_path, monkeypatch): | ||
| 109 | - namespace = "cann_ops_transformer_custom" | ||
| 110 | - pytest.importorskip("torch") | ||
| 111 | - for suffix, directory in [ | ||
| 112 | - ("", PACKAGE), | ||
| 113 | - (".op_builder", PACKAGE / "op_builder"), | ||
| 114 | - (".utils", PACKAGE / "utils"), | ||
| 115 | - ]: | ||
| 116 | - package = types.ModuleType(namespace + suffix) | ||
| 117 | - package.__path__ = [str(directory)] | ||
| 118 | - monkeypatch.setitem(sys.modules, namespace + suffix, package) | ||
| 119 | - name = namespace + ".op_builder.builder" | ||
| 120 | - spec = importlib.util.spec_from_file_location( | ||
| 121 | - name, PACKAGE / "op_builder/builder.py" | ||
| 122 | - ) | ||
| 123 | - module = importlib.util.module_from_spec(spec) | ||
| 124 | - monkeypatch.setitem(sys.modules, name, module) | ||
| 125 | - spec.loader.exec_module(module) | ||
| 126 | - | ||
| 127 | - class Builder(module.OpBuilder): | ||
| 128 | - def sources(self): | ||
| 129 | - return [] | ||
| 130 | - | ||
| 131 | - def schema(self): | ||
| 132 | - return None | ||
| 133 | - | ||
| 134 | - def register_meta(self): | ||
| 135 | - pass | ||
| 136 | - | ||
| 137 | - builder = Builder("sample") | ||
| 138 | - monkeypatch.setattr(builder, "_ensure_initialized", lambda: None) | ||
| 139 | - monkeypatch.setattr(builder, "_remove_stale_torch_lock", lambda: None) | ||
| 140 | - extension = types.SimpleNamespace(add=lambda x, y: x + y) | ||
| 141 | - monkeypatch.setattr(builder, "_try_load_cached", lambda wrap, path: wrap(extension)) | ||
| 142 | - monkeypatch.setenv("TORCH_EXTENSIONS_DIR", str(tmp_path)) | ||
| 143 | - original_import = builtins.__import__ | ||
| 144 | - | ||
| 145 | - def without_base_package(name, *args, **kwargs): | ||
| 146 | - if name == "cann_ops_transformer" or name.startswith("cann_ops_transformer."): | ||
| 147 | - raise ModuleNotFoundError("Base package is not installed", name=name) | ||
| 148 | - return original_import(name, *args, **kwargs) | ||
| 149 | - | ||
| 150 | - monkeypatch.setattr(builtins, "__import__", without_base_package) | ||
| 151 | - assert builder.load().add(2, 3) == 5 | ||