已合并
fix: build.sh 无参数默认打包，修复 compile_acc 裸调用失败 #28
newbgirl创建于 9月21日
fix: build.sh 无参数默认打包，修复 compile_acc 裸调用失败 #28
已合并
共 6 个文件变更+77-25
| @@ -47,8 +47,12 @@ output/ | |||
| 47 | 在本目录执行： | 47 | 在本目录执行： |
| 48 | 48 | ||
| 49 | ```bash | 49 | ```bash |
| 50 | -# 编译并打包 OPP .run（默认 SOC=Ascend910B1，默认全部算子） | 50 | +# 编译 OPP .run 与 Python whl，不安装（无参数默认） |
| 51 | # → <repo>/output/Ascend910B1/oil_gas_ops_prospect_opp_<os>_<arch>.run | 51 | # → <repo>/output/Ascend910B1/oil_gas_ops_prospect_opp_<os>_<arch>.run |
| 52 | +# <repo>/output/wheels/oil_gas_ops_prospect-*.whl | ||
| 53 | +bash build.sh | ||
| 54 | + | ||
| 55 | +# 只编译并打包 OPP .run（默认 SOC=Ascend910B1，默认全部算子） | ||
| 52 | bash build.sh --pkg | 56 | bash build.sh --pkg |
| 53 | 57 | ||
| 54 | # 运行 ophost 侧单元测试（googletest） | 58 | # 运行 ophost 侧单元测试（googletest） |
| @@ -25,6 +25,8 @@ | |||
| 25 | set -euo pipefail | 25 | set -euo pipefail |
| 26 | 26 | ||
| 27 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | 27 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 28 | +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" | ||
| 29 | +OUTPUT_DIR="${REPO_ROOT}/output" | ||
| 28 | BUILD_PY="${SCRIPT_DIR}/build.py" | 30 | BUILD_PY="${SCRIPT_DIR}/build.py" |
| 29 | DEFAULT_SOC="Ascend910B1" | 31 | DEFAULT_SOC="Ascend910B1" |
| 30 | 32 | ||
| @@ -36,7 +38,8 @@ FORWARD_ARGS=() | |||
| 36 | usage() { | 38 | usage() { |
| 37 | cat <<'EOF' | 39 | cat <<'EOF' |
| 38 | Usage: | 40 | Usage: |
| 39 | - bash build.sh --pkg [options] Build and package OPP .run | 41 | + bash build.sh Build OPP .run and Python whl, no install (default) |
| 42 | + bash build.sh --pkg [options] Build and package OPP .run only | ||
| 40 | bash build.sh -u --ophost [options] Build and run ophost unit tests | 43 | bash build.sh -u --ophost [options] Build and run ophost unit tests |
| 41 | bash build.sh clean Clean ascendc/_work/ and <repo>/output/ | 44 | bash build.sh clean Clean ascendc/_work/ and <repo>/output/ |
| 42 | bash build.sh list-ops List operators in manifest | 45 | bash build.sh list-ops List operators in manifest |
| @@ -97,7 +100,20 @@ run_pkg() { | |||
| 97 | args+=("${FORWARD_ARGS[@]}") | 100 | args+=("${FORWARD_ARGS[@]}") |
| 98 | fi | 101 | fi |
| 99 | echo "[build.sh] packaging via: python3 build.py ${args[*]}" | 102 | echo "[build.sh] packaging via: python3 build.py ${args[*]}" |
| 100 | - exec python3 "${BUILD_PY}" "${args[@]}" | 103 | + python3 "${BUILD_PY}" "${args[@]}" |
| 104 | +} | ||
| 105 | + | ||
| 106 | +build_wheel() { | ||
| 107 | + # 与根目录 build.sh 的 build_python_wheel 一致：whl 落 <repo>/output/wheels/ | ||
| 108 | + local repo_root wheel_dir | ||
| 109 | + repo_root="$(cd "${SCRIPT_DIR}/.." && pwd)" | ||
| 110 | + wheel_dir="${repo_root}/output/wheels" | ||
| 111 | + echo "[build.sh] building wheel via: python3 setup.py bdist_wheel --dist-dir ${wheel_dir}" | ||
| 112 | + ( | ||
| 113 | + cd "${repo_root}" | ||
| 114 | + python3 setup.py bdist_wheel --dist-dir "${wheel_dir}" | ||
| 115 | + rm -rf build ./*.egg-info | ||
| 116 | + ) | ||
| 101 | } | 117 | } |
| 102 | 118 | ||
| 103 | run_ut() { | 119 | run_ut() { |
| @@ -127,12 +143,9 @@ case "${MODE}" in | |||
| 127 | run_ut | 143 | run_ut |
| 128 | ;; | 144 | ;; |
| 129 | "") | 145 | "") |
| 130 | - # Legacy: bash build.sh -v Ascend910B1 --ops ... | 146 | + # 无参数 = 编译 .run 与 whl，不安装 |
| 131 | - if [[ "${HAS_SOC}" -eq 1 || ${#FORWARD_ARGS[@]} -gt 0 ]]; then | 147 | + run_pkg |
| 132 | - run_pkg | 148 | + build_wheel |
| 133 | - fi | ||
| 134 | - usage >&2 | ||
| 135 | - exit 2 | ||
| 136 | ;; | 149 | ;; |
| 137 | *) | 150 | *) |
| 138 | echo "未知模式: ${MODE}" >&2 | 151 | echo "未知模式: ${MODE}" >&2 |
| @@ -24,8 +24,12 @@ | |||
| 24 | - 仓库根 `setup.py`：把扩展打进 `oil_gas_ops_prospect` whl； | 24 | - 仓库根 `setup.py`：把扩展打进 `oil_gas_ops_prospect` whl； |
| 25 | - `ascendc/pybind_setup.py`：就地编译，供 NPU 测试直接 import。 | 25 | - `ascendc/pybind_setup.py`：就地编译，供 NPU 测试直接 import。 |
| 26 | 26 | ||
| 27 | -前置条件：OPP 包已安装到 CANN（`opp/vendors/customize/op_api/lib/libcust_opapi.so` | 27 | +aclnn 头与 ``libcust_opapi.so`` 的取用顺序（见 ``_vendor_opapi``）： |
| 28 | -存在），因为扩展直接链接 msopgen 生成的 aclnn 接口。 | 28 | + |
| 29 | +1. msopgen 构建工作区产物（``ascendc/_work/CustomOp_*/build_out/``）—— | ||
| 30 | + 构建流程内无需安装 OPP 即可编扩展，且与刚构建的 `.run` 严格同源； | ||
| 31 | +2. 已安装的 CANN vendor 目录（``opp/vendors/customize/op_api/``，两种 | ||
| 32 | + 历史布局）—— 适合 `pybind_setup.py` 就地编译等仅有安装件的环境。 | ||
| 29 | """ | 33 | """ |
| 30 | # pyright: reportMissingImports=false | 34 | # pyright: reportMissingImports=false |
| 31 | 35 | ||
| @@ -56,16 +60,27 @@ def _torch_npu_include_root() -> Path: | |||
| 56 | return Path(torch.__file__).parent / ".." / "torch_npu" | 60 | return Path(torch.__file__).parent / ".." / "torch_npu" |
| 57 | 61 | ||
| 58 | 62 | ||
| 59 | -def _vendor_opapi() -> Path: | 63 | +def _vendor_opapi() -> tuple[Path, Path]: |
| 60 | - """自定义算子包的 vendor 目录（不同 CANN 版本布局略有差异）。""" | 64 | + """返回 (aclnn 头目录, libcust_opapi 库目录)。 |
| 65 | + | ||
| 66 | + 优先 msopgen 构建工作区（``_work`` 会随 ``build.sh clean`` 清理， | ||
| 67 | + 每次打包重新生成，与刚构建的 `.run` 同源）；回退已安装 vendor 目录 | ||
| 68 | + （不同 CANN 版本布局略有差异）；均不存在时返回安装布局的默认路径， | ||
| 69 | + 让编译以缺头文件的明确报错终止。 | ||
| 70 | + """ | ||
| 71 | + for custom_op in sorted((ASCENDC_DIR / "_work").glob("CustomOp_*")): | ||
| 72 | + build_out = custom_op / "build_out" | ||
| 73 | + headers, libdir = build_out / "autogen", build_out / "op_host" | ||
| 74 | + if (libdir / "libcust_opapi.so").is_file() and any(headers.glob("aclnn_*.h")): | ||
| 75 | + return headers, libdir | ||
| 61 | candidates = [ | 76 | candidates = [ |
| 62 | Path(ASCEND_DIR) / "opp" / "vendors" / "customize" / "op_api", | 77 | Path(ASCEND_DIR) / "opp" / "vendors" / "customize" / "op_api", |
| 63 | Path(ASCEND_DIR) / "vendors" / "customize" / "op_api", | 78 | Path(ASCEND_DIR) / "vendors" / "customize" / "op_api", |
| 64 | ] | 79 | ] |
| 65 | - return next( | 80 | + for p in candidates: |
| 66 | - (p for p in candidates if (p / "lib" / "libcust_opapi.so").is_file()), | 81 | + if (p / "lib" / "libcust_opapi.so").is_file(): |
| 67 | - candidates[0], | 82 | + return p / "include", p / "lib" |
| 68 | - ) | 83 | + return candidates[0] / "include", candidates[0] / "lib" |
| 69 | 84 | ||
| 70 | 85 | ||
| 71 | def discover() -> list[Path]: | 86 | def discover() -> list[Path]: |
| @@ -90,7 +105,7 @@ def build_extensions(relative_to: Path | None = None) -> list[CppExtension]: | |||
| 90 | 105 | ||
| 91 | base = Path(relative_to).resolve() if relative_to is not None else PYBIND_DIR | 106 | base = Path(relative_to).resolve() if relative_to is not None else PYBIND_DIR |
| 92 | torch_npu_root = _torch_npu_include_root() | 107 | torch_npu_root = _torch_npu_include_root() |
| 93 | - vendor_opapi = _vendor_opapi() | 108 | + opapi_include, opapi_lib = _vendor_opapi() |
| 94 | 109 | ||
| 95 | include_dirs = [ | 110 | include_dirs = [ |
| 96 | str(CSRC_DIR / "_common"), | 111 | str(CSRC_DIR / "_common"), |
| @@ -98,11 +113,11 @@ def build_extensions(relative_to: Path | None = None) -> list[CppExtension]: | |||
| 98 | str(torch_npu_root / "include" / "third_party" / "acl" / "inc"), | 113 | str(torch_npu_root / "include" / "third_party" / "acl" / "inc"), |
| 99 | str(torch_npu_root / "include" / "third_party" / "op-plugin"), | 114 | str(torch_npu_root / "include" / "third_party" / "op-plugin"), |
| 100 | f"{ASCEND_DIR}/include", | 115 | f"{ASCEND_DIR}/include", |
| 101 | - str(vendor_opapi / "include"), | 116 | + str(opapi_include), |
| 102 | ] | 117 | ] |
| 103 | library_dirs = [ | 118 | library_dirs = [ |
| 104 | f"{ASCEND_DIR}/lib64", | 119 | f"{ASCEND_DIR}/lib64", |
| 105 | - str(vendor_opapi / "lib"), | 120 | + str(opapi_lib), |
| 106 | str(torch_npu_root / "lib"), | 121 | str(torch_npu_root / "lib"), |
| 107 | ] | 122 | ] |
| 108 | libraries = ["cust_opapi", "ascendcl", "nnopbase", "opapi", "torch_npu"] | 123 | libraries = ["cust_opapi", "ascendcl", "nnopbase", "opapi", "torch_npu"] |
| @@ -1,14 +1,16 @@ | |||
| 1 | [build-system] | 1 | [build-system] |
| 2 | -# setuptools>=77 for the PEP 639 SPDX ``license`` / ``license-files`` keys. | 2 | +# ``project.license`` 采用 PEP 621 表格形式（非 PEP 639 SPDX 字符串）： |
| 3 | -requires = ["setuptools>=77", "wheel"] | 3 | +# CI 编译镜像为 python 3.10 + 旧版 setuptools（<77），SPDX 字符串形式会 |
| 4 | +# 校验失败；表格形式新旧 setuptools 均可构建（新版仅弃用告警）。 | ||
| 5 | +# LICENSE 文件由 setuptools 自动识别并随 whl 分发，无需 license-files。 | ||
| 6 | +requires = ["setuptools>=61", "wheel"] | ||
| 4 | build-backend = "setuptools.build_meta" | 7 | build-backend = "setuptools.build_meta" |
| 5 | 8 | ||
| 6 | [project] | 9 | [project] |
| 7 | name = "oil-gas-ops-prospect" | 10 | name = "oil-gas-ops-prospect" |
| 8 | version = "0.1.0" | 11 | version = "0.1.0" |
| 9 | description = "AscendC and Triton-Ascend custom operators for oil & gas exploration" | 12 | description = "AscendC and Triton-Ascend custom operators for oil & gas exploration" |
| 10 | -license = "Apache-2.0" | 13 | +license = {text = "Apache-2.0"} |
| 11 | -license-files = ["LICENSE"] | ||
| 12 | requires-python = ">=3.10" | 14 | requires-python = ">=3.10" |
| 13 | dependencies = [ | 15 | dependencies = [ |
| 14 | "torch", | 16 | "torch", |
| @@ -107,7 +107,23 @@ def _composite_mapping(composite_root: Path) -> dict[str, Path]: | |||
| 107 | 107 | ||
| 108 | 108 | ||
| 109 | def activate(repo_root: Path | None = None) -> bool: | 109 | def activate(repo_root: Path | None = None) -> bool: |
| 110 | - """把源码树挂到 ``oil_gas_ops_prospect`` 导入名下。已激活则直接返回 True。""" | 110 | + """把源码树挂到 ``oil_gas_ops_prospect`` 导入名下。已激活则直接返回 True。 |
| 111 | + | ||
| 112 | + 同时处理 torch 的设备后端自动加载（``TORCH_DEVICE_BACKEND_AUTOLOAD``）： | ||
| 113 | + LLT 等无卡镜像可能装有 torch_npu 但缺 CANN 运行环境，torch 2.7+ 默认开启的 | ||
| 114 | + 自动加载会在 ``import torch`` / 访问 ``torch.npu`` 时抛 | ||
| 115 | + ``RuntimeError: Failed to load the backend extension: torch_npu``。因此先 | ||
| 116 | + 关闭自动加载（显式置 ``1`` 可覆盖，setdefault 语义），再**显式尽早**导入 | ||
| 117 | + torch_npu：NPU 环境下 autograd 引擎按首次执行时已注册的设备初始化 ready | ||
| 118 | + queues，torch_npu 晚于首次 autograd 注册会触发 ``device_ready_queues_`` | ||
| 119 | + 内部断言；导入失败（无卡 / 缺 CANN）则静默跳过，CPU 用例照常运行。 | ||
| 120 | + whl 验证路径同样受益。 | ||
| 121 | + """ | ||
| 122 | + os.environ.setdefault("TORCH_DEVICE_BACKEND_AUTOLOAD", "0") | ||
| 123 | + try: | ||
| 124 | + import torch_npu # noqa: F401 尽早注册 NPU 设备（见上） | ||
| 125 | + except Exception: | ||
| 126 | + pass | ||
| 111 | if os.environ.get("OIL_GAS_OPS_WHL_TEST") == "1": | 127 | if os.environ.get("OIL_GAS_OPS_WHL_TEST") == "1": |
| 112 | return False | 128 | return False |
| 113 | 129 | ||
| @@ -65,6 +65,8 @@ for _p in (TRITON_ROOT / "real_fft",): | |||
| 65 | DEFAULT_ARCH = "Ascend910B1" | 65 | DEFAULT_ARCH = "Ascend910B1" |
| 66 | os.environ.setdefault("TRITON_ASCEND_ARCH", DEFAULT_ARCH) | 66 | os.environ.setdefault("TRITON_ASCEND_ARCH", DEFAULT_ARCH) |
| 67 | os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton_cache_aot") | 67 | os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton_cache_aot") |
| 68 | +# 无卡镜像可能装有 torch_npu 但缺 CANN 运行环境，torch 后端自动加载会抛错 | ||
| 69 | +os.environ.setdefault("TORCH_DEVICE_BACKEND_AUTOLOAD", "0") | ||
| 68 | ARCH = os.environ["TRITON_ASCEND_ARCH"] | 70 | ARCH = os.environ["TRITON_ASCEND_ARCH"] |
| 69 | 71 | ||
| 70 | 72 | ||