已合并
Feat：add aclblasLtMatmulAlgo init/configSetAttr/configGetAttr func #248
wangzitao创建于 7月2日
Feat：add aclblasLtMatmulAlgo init/configSetAttr/configGetAttr func #248
已合并
共 59 个文件变更+3843-2354
| @@ -86,10 +86,17 @@ set_source_files_properties( | |||
| 86 | PROPERTIES LANGUAGE ASC | 86 | PROPERTIES LANGUAGE ASC |
| 87 | ) | 87 | ) |
| 88 | 88 | ||
| 89 | +# Device kernels compile with the ASC frontend; all host code (api / internal / engine / *_host.cpp) | ||
| 90 | +# compiles as plain C++. The classification is produced by blasLt/CMakeLists.txt, so no per-file | ||
| 91 | +# language whitelist is maintained here. | ||
| 89 | set_source_files_properties( | 92 | set_source_files_properties( |
| 90 | - ${ALL_BLASLT_SRC_FILES} | 93 | + ${ALL_BLASLT_KERNEL_SRC_FILES} |
| 91 | PROPERTIES LANGUAGE ASC | 94 | PROPERTIES LANGUAGE ASC |
| 92 | ) | 95 | ) |
| 96 | +set_source_files_properties( | ||
| 97 | + ${ALL_BLASLT_HOST_SRC_FILES} | ||
| 98 | + PROPERTIES LANGUAGE CXX | ||
| 99 | +) | ||
| 93 | 100 | ||
| 94 | set(_OPS_BLAS_ASC_ROOT "${ASCEND_CANN_PACKAGE_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/asc") | 101 | set(_OPS_BLAS_ASC_ROOT "${ASCEND_CANN_PACKAGE_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/asc") |
| 95 | target_include_directories(${OPS_BLASLT} PRIVATE | 102 | target_include_directories(${OPS_BLASLT} PRIVATE |
| @@ -112,22 +119,21 @@ if(EXISTS "${_OPS_BLAS_ASC_ROOT}/include") | |||
| 112 | ) | 119 | ) |
| 113 | endif() | 120 | endif() |
| 114 | 121 | ||
| 122 | +# Private include roots for the refactored blasLt layout. internal/ headers are never installed; | ||
| 123 | +# each operator's arch dir exposes its co-located host/kernel/tiling headers. | ||
| 115 | target_include_directories(${OPS_BLASLT} PRIVATE | 124 | target_include_directories(${OPS_BLASLT} PRIVATE |
| 116 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/include | 125 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/internal/include |
| 117 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/include/kernel | 126 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/common/helper |
| 118 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/include/host | 127 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/engine |
| 119 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/utils | 128 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/common |
| 129 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/fp32/arch35 | ||
| 130 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/mxfp8/arch35 | ||
| 131 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/mxfp4/arch35 | ||
| 132 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/epilogue/arch35 | ||
| 133 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matrix_transform/engine | ||
| 134 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matrix_transform/arch35 | ||
| 120 | ) | 135 | ) |
| 121 | 136 | ||
| 122 | -if(ENABLE_BLASLT_MXFP8) | ||
| 123 | - set_source_files_properties( | ||
| 124 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul_mxfp8/arch35/matmul_mxfp8_host.cpp | ||
| 125 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul_mxfp4/arch35/matmul_mxfp4_host.cpp | ||
| 126 | - PROPERTIES LANGUAGE CXX) | ||
| 127 | -endif() | ||
| 128 | -set_source_files_properties( | ||
| 129 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/epilogue/arch35/epilogue_alpha_beta_host.cpp | ||
| 130 | - PROPERTIES LANGUAGE CXX) | ||
| 131 | target_compile_definitions(${OPS_BLASLT} PRIVATE | 137 | target_compile_definitions(${OPS_BLASLT} PRIVATE |
| 132 | ASC_DEVKIT_MAJOR=${ASC_DEVKIT_MAJOR} | 138 | ASC_DEVKIT_MAJOR=${ASC_DEVKIT_MAJOR} |
| 133 | ASC_DEVKIT_MINOR=${ASC_DEVKIT_MINOR}) | 139 | ASC_DEVKIT_MINOR=${ASC_DEVKIT_MINOR}) |
| @@ -10,52 +10,54 @@ | |||
| 10 | 10 | ||
| 11 | cmake_minimum_required(VERSION 3.16) | 11 | cmake_minimum_required(VERSION 3.16) |
| 12 | 12 | ||
| 13 | -# 收集当前目录下的基础源文件 | 13 | +# Collect every blasLt source, then classify by role so the parent scope can assign compile |
| 14 | -file(GLOB BASE_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) | 14 | +# languages without maintaining a per-file whitelist: |
| 15 | +# *_kernel.cpp -> ASC (device kernels) | ||
| 16 | +# api / internal / engine / *_host.cpp -> CXX (pure host code) | ||
| 17 | +# Architecture-specific operator code lives under <op>/<variant>/<archXX>/; only the arch dirs | ||
| 18 | +# matching the current SOC (SOC_ARCH_DIRS) are compiled. | ||
| 19 | +file(GLOB_RECURSE BLASLT_ALL_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) | ||
| 15 | 20 | ||
| 16 | -# 自动收集所有算子目录的源文件（支持子目录中的架构特定目录） | 21 | +set(BLASLT_HOST_SRCS "") |
| 17 | -set(OP_SRC_FILES "") | 22 | +set(BLASLT_KERNEL_SRCS "") |
| 18 | -file(GLOB children RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*) | 23 | + |
| 19 | -foreach(child ${children}) | 24 | +foreach(src_file ${BLASLT_ALL_SRCS}) |
| 20 | - if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${child}) | 25 | + # Skip sources under an architecture directory that does not match the current SOC. |
| 21 | - # 递归收集非架构特定源码，避免与 archXX 目录重复收集 | 26 | + set(is_arch_specific FALSE) |
| 22 | - file(GLOB_RECURSE dir_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${child}/*.cpp) | 27 | + foreach(arch_dir ${ARCH_SPECIFIC_DIRS}) |
| 23 | - foreach(src_file ${dir_srcs}) | 28 | + if(src_file MATCHES "/${arch_dir}/") |
| 24 | - set(is_arch_specific FALSE) | 29 | + set(is_arch_specific TRUE) |
| 25 | - foreach(arch_dir ${ARCH_SPECIFIC_DIRS}) | 30 | + endif() |
| 26 | - if(src_file MATCHES "/${arch_dir}/") | 31 | + endforeach() |
| 27 | - set(is_arch_specific TRUE) | 32 | + set(arch_selected FALSE) |
| 28 | - break() | 33 | + foreach(arch_dir ${SOC_ARCH_DIRS}) |
| 29 | - endif() | 34 | + if(src_file MATCHES "/${arch_dir}/") |
| 30 | - endforeach() | 35 | + set(arch_selected TRUE) |
| 31 | - if(NOT is_arch_specific) | 36 | + endif() |
| 32 | - list(APPEND OP_SRC_FILES ${src_file}) | 37 | + endforeach() |
| 33 | - endif() | 38 | + |
| 34 | - endforeach() | 39 | + if(is_arch_specific AND NOT arch_selected) |
| 40 | + # architecture directory for another SOC: do not compile it here | ||
| 41 | + elseif(src_file MATCHES "_kernel\\.cpp$") | ||
| 42 | + list(APPEND BLASLT_KERNEL_SRCS ${src_file}) | ||
| 43 | + else() | ||
| 44 | + list(APPEND BLASLT_HOST_SRCS ${src_file}) | ||
| 35 | endif() | 45 | endif() |
| 36 | endforeach() | 46 | endforeach() |
| 37 | 47 | ||
| 38 | -# 收集子目录中的架构特定源文件 | 48 | +# MXFP8/MXFP4 require asc-devkit >= 9.1 (ENABLE_BLASLT_MXFP8). Exclude the whole precision subtrees |
| 39 | -set(ARCH_SRC_FILES "") | 49 | +# rather than filtering individual files. |
| 40 | -foreach(arch_dir ${SOC_ARCH_DIRS}) | ||
| 41 | - file(GLOB_RECURSE arch_dir_srcs | ||
| 42 | - ${CMAKE_CURRENT_SOURCE_DIR}/*/${arch_dir}/*.cpp | ||
| 43 | - ) | ||
| 44 | - list(APPEND ARCH_SRC_FILES ${arch_dir_srcs}) | ||
| 45 | -endforeach() | ||
| 46 | - | ||
| 47 | -# MXFP8 需要 asc-devkit >= 9.1（ASC_DEVKIT_MAJOR >= 9 && ASC_DEVKIT_MINOR > 0） | ||
| 48 | if(NOT ENABLE_BLASLT_MXFP8) | 50 | if(NOT ENABLE_BLASLT_MXFP8) |
| 49 | - list(FILTER ARCH_SRC_FILES EXCLUDE REGEX "/matmul_mxfp8/") | 51 | + list(FILTER BLASLT_HOST_SRCS EXCLUDE REGEX "/mxfp8/") |
| 50 | - list(FILTER ARCH_SRC_FILES EXCLUDE REGEX "/matmul_mxfp4/") | 52 | + list(FILTER BLASLT_HOST_SRCS EXCLUDE REGEX "/mxfp4/") |
| 53 | + list(FILTER BLASLT_KERNEL_SRCS EXCLUDE REGEX "/mxfp8/") | ||
| 54 | + list(FILTER BLASLT_KERNEL_SRCS EXCLUDE REGEX "/mxfp4/") | ||
| 51 | message( | 55 | message( |
| 52 | STATUS | 56 | STATUS |
| 53 | "Skipping blasLt MXFP8/MXFP4 (requires asc-devkit >= 9.1, current ${ASC_DEVKIT_MAJOR}.${ASC_DEVKIT_MINOR})") | 57 | "Skipping blasLt MXFP8/MXFP4 (requires asc-devkit >= 9.1, current ${ASC_DEVKIT_MAJOR}.${ASC_DEVKIT_MINOR})") |
| 54 | endif() | 58 | endif() |
| 55 | 59 | ||
| 56 | -set(ALL_BLASLT_SRC_FILES | 60 | +# Exported to the parent scope; the top-level CMakeLists assigns LANGUAGE per list. |
| 57 | - ${BASE_SRC_FILES} | 61 | +set(ALL_BLASLT_SRC_FILES ${BLASLT_HOST_SRCS} ${BLASLT_KERNEL_SRCS} PARENT_SCOPE) |
| 58 | - ${OP_SRC_FILES} | 62 | +set(ALL_BLASLT_HOST_SRC_FILES ${BLASLT_HOST_SRCS} PARENT_SCOPE) |
| 59 | - ${ARCH_SRC_FILES} | 63 | +set(ALL_BLASLT_KERNEL_SRC_FILES ${BLASLT_KERNEL_SRCS} PARENT_SCOPE) |
| 60 | - PARENT_SCOPE | ||
| 61 | -) | ||
| @@ -1,334 +1,265 @@ | |||
| 1 | -## aclblasLtMatmul 接口实现 | 1 | +# LtMatmul算子 |
| 2 | 2 | ||
| 3 | -## 概述 | 3 | +## 算子概述 |
| 4 | 4 | ||
| 5 | -BLAS Lt 矩阵乘法（`aclblasLtMatmul`）接口实现与精度测试。 | 5 | +LtMatmul 算子实现了 BLAS Lt 通用矩阵乘法，核心运算为 D = alpha * op(A) * op(B) + beta * C。其中 A、B 为输入矩阵，C 为累加矩阵，D 为输出矩阵，alpha 和 beta 为标量，op(A)/op(B) 支持不转置（N）和转置（T）。当前实现支持 FP32、MXFP8（E4M3FN）、MXFP4（E2M1）三种输入类型组合，输出支持 FP32 和 BF16。 |
| 6 | 6 | ||
| 7 | -`aclblasLtMatmul` 实现了通用矩阵乘法运算，对应的数学表达式为： | 7 | +数学表达式： |
| 8 | 8 | ||
| 9 | ``` | 9 | ``` |
| 10 | D = alpha * op(A) * op(B) + beta * C | 10 | D = alpha * op(A) * op(B) + beta * C |
| 11 | ``` | 11 | ``` |
| 12 | 12 | ||
| 13 | -其中 A、B 为输入矩阵，C 为累加矩阵，D 为输出矩阵，alpha 和 beta 为标量，op(A)/op(B) 支持不转置（N）和转置（T）。当前实现支持 FP32、MXFP8（E4M3FN）、MXFP4（E2M1）三种输入类型组合，输出支持 FP32 和 BF16。 | 13 | +包含以下接口： |
| 14 | 14 | ||
| 15 | -## 产品支持情况 | 15 | +| 接口名 | 功能简述 | |
| 16 | +|--------|---------| | ||
| 17 | +| aclblasLtMatmul | 通用矩阵乘法（支持 FP32 / MXFP8 / MXFP4 路径） | | ||
| 16 | 18 | ||
| 17 | -| 产品 | 是否支持 | | 19 | +## 算子执行接口 |
| 18 | -| :----------------------------------------------------------- |:-------:| | 20 | + |
| 19 | -| <term>Ascend 950PR/Ascend 950DT</term> | ✓ | | 21 | +### aclblasLtMatmul |
| 20 | -| <term>Atlas A3 训练系列产品/Atlas A3 推理系列产品</term> | ✗ | | 22 | + |
| 21 | -| <term>Atlas A2 训练系列产品/Atlas A2 推理系列产品</term> | ✗ | | 23 | +#### 产品支持情况 |
| 24 | + | ||
| 25 | +- Ascend 950PR / Ascend 950DT：支持 | ||
| 26 | +- Atlas A3 训练系列产品 / Atlas A3 推理系列产品：不支持 | ||
| 27 | +- Atlas A2 训练系列产品 / Atlas A2 推理系列产品：不支持 | ||
| 22 | 28 | ||
| 23 | > MXFP8/MXFP4 量化路径依赖 CANN asc-devkit >= 9.1（`ASC_DEVKIT_MAJOR >= 9 && ASC_DEVKIT_MINOR >= 1`）。 | 29 | > MXFP8/MXFP4 量化路径依赖 CANN asc-devkit >= 9.1（`ASC_DEVKIT_MAJOR >= 9 && ASC_DEVKIT_MINOR >= 1`）。 |
| 24 | 30 | ||
| 25 | -## 目录结构介绍 | 31 | +#### 函数原型 |
| 26 | 32 | ||
| 27 | -接口实现位于 `blasLt/`： | 33 | +```cpp |
| 28 | - | 34 | +aclblasStatus_t aclblasLtMatmul(aclblasLtHandle_t lightHandle, aclblasLtMatmulDesc_t computeDesc, const void* alpha, const void* A, aclblasLtMatrixLayout_t Adesc, const void* B, aclblasLtMatrixLayout_t Bdesc, const void* beta, const void* C, aclblasLtMatrixLayout_t Cdesc, void* D, aclblasLtMatrixLayout_t Ddesc, const aclblasLtMatmulAlgo_t* algo, void* workspace, size_t workspaceSizeInBytes, aclrtStream stream) |
| 29 | -``` | ||
| 30 | -blasLt/ | ||
| 31 | -├── aclblasLt.cpp // aclBLASLt 库入口，含 aclblasLtMatmul 路由 | ||
| 32 | -├── matmul_fp32/arch35/ | ||
| 33 | -│ ├── matmul_fp32_host.cpp // FP32 Host 侧 Tiling | ||
| 34 | -│ └── matmul_fp32_kernel.cpp // FP32 Kernel 侧实现 | ||
| 35 | -├── matmul_mxfp8/arch35/ | ||
| 36 | -│ ├── matmul_mxfp8_host.cpp // MXFP8 Host 侧 Tiling | ||
| 37 | -│ └── matmul_mxfp8_kernel.cpp // MXFP8 Kernel 侧实现 | ||
| 38 | -├── matmul_mxfp4/arch35/ | ||
| 39 | -│ ├── matmul_mxfp4_host.cpp // MXFP4 Host 侧 Tiling | ||
| 40 | -│ └── matmul_mxfp4_kernel.cpp // MXFP4 Kernel 侧实现 | ||
| 41 | -└── utils/ | ||
| 42 | - └── kernel_utils.h // shared kernel helpers | ||
| 43 | ``` | 35 | ``` |
| 44 | 36 | ||
| 45 | -测试代码位于 `test/blasLtMatmul/`： | 37 | +#### 参数说明 |
| 46 | 38 | ||
| 39 | +| 参数名 | 输入/输出 | 参数类型 | 说明 | | ||
| 40 | +|--------|----------|---------|------| | ||
| 41 | +| lightHandle | 输入 | aclblasLtHandle_t | aclBLASLt 库上下文句柄，由 `aclblasLtCreate` 创建，不可为 NULL，否则返回 `ACLBLAS_STATUS_NOT_INITIALIZED`，Host 内存 | | ||
| 42 | +| computeDesc | 输入 | aclblasLtMatmulDesc_t | 矩阵乘法描述符，设置 transA/transB、epilogue、scale 指针等属性，不可为 NULL，Host 内存 | | ||
| 43 | +| alpha | 输入 | const void*（FP32） | 用于乘法的 float 标量指针，不可为 NULL，Host 内存 | | ||
| 44 | +| A | 输入 | const void* | 输入矩阵 A，数据类型由 Adesc 指定，m>0 且 n>0 时不可为 NULL，Device 内存 | | ||
| 45 | +| Adesc | 输入 | aclblasLtMatrixLayout_t | 矩阵 A 的 layout 描述符（rows/cols/ld/order/dtype），Host 内存 | | ||
| 46 | +| B | 输入 | const void* | 输入矩阵 B，数据类型由 Bdesc 指定，m>0 且 n>0 时不可为 NULL，Device 内存 | | ||
| 47 | +| Bdesc | 输入 | aclblasLtMatrixLayout_t | 矩阵 B 的 layout 描述符，Host 内存 | | ||
| 48 | +| beta | 输入 | const void*（FP32） | 用于累加的 float 标量指针，不可为 NULL，beta=0 时 C 可不参与计算，Host 内存 | | ||
| 49 | +| C | 输入 | const void* | 累加矩阵 C，beta=0 时可为 NULL，Device 内存 | | ||
| 50 | +| Cdesc | 输入 | aclblasLtMatrixLayout_t | 矩阵 C 的 layout 描述符，Host 内存 | | ||
| 51 | +| D | 输出 | void* | 输出矩阵 D，维度 m x n，m>0 且 n>0 时不可为 NULL，Device 内存 | | ||
| 52 | +| Ddesc | 输入 | aclblasLtMatrixLayout_t | 矩阵 D 的 layout 描述符，指定输出数据类型（FP32 或 BF16），Host 内存 | | ||
| 53 | +| algo | 输入 | const aclblasLtMatmulAlgo_t* | 算法描述符，可为 NULL（使用默认算法），Host 内存 | | ||
| 54 | +| workspace | 输入 | void* | 工作空间内存，可为 NULL，非 NULL 时需 16B 对齐，Device 内存 | | ||
| 55 | +| workspaceSizeInBytes | 输入 | size_t | 工作空间大小（字节），Host 内存 | | ||
| 56 | +| stream | 输入 | aclrtStream | AscendCL 执行流，Host 内存 | | ||
| 57 | + | ||
| 58 | +#### 约束说明 | ||
| 59 | + | ||
| 60 | +- M、N、K >= 0；M=0 或 N=0 时为空操作，直接返回 `ACLBLAS_STATUS_SUCCESS` | ||
| 61 | +- dtypeA / dtypeB 须为同类型组合：FP32×FP32、MXFP8×MXFP8、MXFP4×MXFP4；其他组合返回 `ACLBLAS_STATUS_NOT_SUPPORTED` | ||
| 62 | +- dtypeC 当前固定为 FP32 | ||
| 63 | +- dtypeD 支持 FP32 或 BF16；MXFP8/MXFP4 路径支持 FP32 或 BF16 输出，FP32 路径输出 FP32 | ||
| 64 | +- computeType 当前仅支持 `ACLBLAS_COMPUTE_32F` | ||
| 65 | +- transA / transB 支持 N、T，对应 `ACLBLAS_OP_N`（不转置）、`ACLBLAS_OP_T`（转置） | ||
| 66 | +- MXFP8/MXFP4 路径要求 K 为 32 的整数倍，否则返回 `ACLBLAS_STATUS_INVALID_VALUE` | ||
| 67 | +- order 当前仅支持 `ACLBLASLT_ORDER_ROW`；lda / ldb / ldc / ldd 须 >= 矩阵物理列数；MXFP4 的 ld 为逻辑元素 leading dim（2 个 FP4 元素打包为 1 字节） | ||
| 68 | +- epilogue 当前仅支持 `ACLBLASLT_EPILOGUE_DEFAULT` | ||
| 69 | +- MXFP8/MXFP4 路径须通过 computeDesc 设置 scaleA / scaleB（`ACLBLASLT_MATMUL_DESC_A/B_SCALE_POINTER`），E8M0 格式，按 K 方向每 32 元素一组，不可为 NULL | ||
| 70 | +- algo 可为 NULL，使用默认算法 | ||
| 71 | +- workspace 非 NULL 时需 16B 对齐；algo 非 NULL 时 workspaceSizeInBytes 须 >= algo->max_workspace_bytes | ||
| 72 | + | ||
| 73 | +#### 调用示例 | ||
| 74 | + | ||
| 75 | +示例代码如下，仅供参考，具体编译和执行过程请参考[编译与运行样例](../../docs/zh/develop/compile_and_run_example.md)。使用 BLASLt 接口时，需在 CMakeLists.txt 中额外链接 `libops_blasLt.so`。 | ||
| 76 | + | ||
| 77 | +```cpp | ||
| 78 | +#include <cstdio> | ||
| 79 | +#include <memory> | ||
| 80 | +#include <vector> | ||
| 81 | + | ||
| 82 | +#include "acl/acl.h" | ||
| 83 | +#include "cann_ops_blasLt.h" | ||
| 84 | + | ||
| 85 | +#define CHECK_RET(cond, return_expr) \ | ||
| 86 | + do { \ | ||
| 87 | + if (!(cond)) { \ | ||
| 88 | + return_expr; \ | ||
| 89 | + } \ | ||
| 90 | + } while (0) | ||
| 91 | + | ||
| 92 | +#define LOG_PRINT(message, ...) \ | ||
| 93 | + do { \ | ||
| 94 | + printf(message, ##__VA_ARGS__); \ | ||
| 95 | + } while (0) | ||
| 96 | + | ||
| 97 | +class AclContext { | ||
| 98 | +public: | ||
| 99 | + explicit AclContext(int32_t deviceId) : deviceId_(deviceId) {} | ||
| 100 | + | ||
| 101 | + ~AclContext() | ||
| 102 | + { | ||
| 103 | + if (stream_ != nullptr) { | ||
| 104 | + aclrtDestroyStream(stream_); | ||
| 105 | + stream_ = nullptr; | ||
| 106 | + } | ||
| 107 | + if (deviceSet_) { | ||
| 108 | + aclrtResetDevice(deviceId_); | ||
| 109 | + deviceSet_ = false; | ||
| 110 | + } | ||
| 111 | + if (aclInited_) { | ||
| 112 | + aclFinalize(); | ||
| 113 | + aclInited_ = false; | ||
| 114 | + } | ||
| 115 | + } | ||
| 116 | + | ||
| 117 | + int Init() | ||
| 118 | + { | ||
| 119 | + auto ret = aclInit(nullptr); | ||
| 120 | + CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclInit failed. ERROR: %d\n", ret); return ret); | ||
| 121 | + aclInited_ = true; | ||
| 122 | + | ||
| 123 | + ret = aclrtSetDevice(deviceId_); | ||
| 124 | + CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); return ret); | ||
| 125 | + deviceSet_ = true; | ||
| 126 | + | ||
| 127 | + ret = aclrtCreateStream(&stream_); | ||
| 128 | + CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); return ret); | ||
| 129 | + return ACL_SUCCESS; | ||
| 130 | + } | ||
| 131 | + | ||
| 132 | + aclrtStream Stream() const { return stream_; } | ||
| 133 | + | ||
| 134 | +private: | ||
| 135 | + int32_t deviceId_; | ||
| 136 | + aclrtStream stream_ = nullptr; | ||
| 137 | + bool aclInited_ = false; | ||
| 138 | + bool deviceSet_ = false; | ||
| 139 | +}; | ||
| 140 | + | ||
| 141 | +struct AclMemDeleter { | ||
| 142 | + void operator()(void* p) const { aclrtFree(p); } | ||
| 143 | +}; | ||
| 144 | +struct LtHandleDeleter { | ||
| 145 | + void operator()(aclblasLtHandle_t h) const { aclblasLtDestroy(h); } | ||
| 146 | +}; | ||
| 147 | +struct LtDescDeleter { | ||
| 148 | + void operator()(aclblasLtMatmulDesc_t d) const { aclblasLtMatmulDescDestroy(d); } | ||
| 149 | +}; | ||
| 150 | +struct LtLayoutDeleter { | ||
| 151 | + void operator()(aclblasLtMatrixLayout_t d) const { aclblasLtMatrixLayoutDestroy(d); } | ||
| 152 | +}; | ||
| 153 | + | ||
| 154 | +int aclblasLtMatmulTest(AclContext& ctx) | ||
| 155 | +{ | ||
| 156 | + aclrtStream stream = ctx.Stream(); | ||
| 157 | + | ||
| 158 | + // 1. 创建 aclBLASLt 句柄与描述符 | ||
| 159 | + aclblasLtHandle_t rawHandle = nullptr; | ||
| 160 | + auto blasRet = aclblasLtCreate(&rawHandle); | ||
| 161 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("aclblasLtCreate failed. ERROR: %d\n", blasRet); | ||
| 162 | + return blasRet); | ||
| 163 | + std::unique_ptr<std::remove_pointer<aclblasLtHandle_t>::type, LtHandleDeleter> handlePtr(rawHandle); | ||
| 164 | + | ||
| 165 | + aclblasLtMatmulDesc_t rawDesc = nullptr; | ||
| 166 | + blasRet = aclblasLtMatmulDescCreate(&rawDesc, ACLBLAS_COMPUTE_32F, ACL_FLOAT); | ||
| 167 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("aclblasLtMatmulDescCreate failed. ERROR: %d\n", blasRet); | ||
| 168 | + return blasRet); | ||
| 169 | + std::unique_ptr<std::remove_pointer<aclblasLtMatmulDesc_t>::type, LtDescDeleter> descPtr(rawDesc); | ||
| 170 | + | ||
| 171 | + const uint64_t m = 16, n = 16, k = 16; | ||
| 172 | + aclblasLtMatrixLayout_t rawAdesc = nullptr; | ||
| 173 | + aclblasLtMatrixLayout_t rawBdesc = nullptr; | ||
| 174 | + aclblasLtMatrixLayout_t rawCdesc = nullptr; | ||
| 175 | + aclblasLtMatrixLayout_t rawDdesc = nullptr; | ||
| 176 | + int32_t order = ACLBLASLT_ORDER_ROW; | ||
| 177 | + | ||
| 178 | + blasRet = aclblasLtMatrixLayoutCreate(&rawAdesc, ACL_FLOAT, m, k, k); | ||
| 179 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("MatrixLayoutCreate A failed. ERROR: %d\n", blasRet); | ||
| 180 | + return blasRet); | ||
| 181 | + blasRet = aclblasLtMatrixLayoutCreate(&rawBdesc, ACL_FLOAT, k, n, n); | ||
| 182 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("MatrixLayoutCreate B failed. ERROR: %d\n", blasRet); | ||
| 183 | + return blasRet); | ||
| 184 | + blasRet = aclblasLtMatrixLayoutCreate(&rawCdesc, ACL_FLOAT, m, n, n); | ||
| 185 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("MatrixLayoutCreate C failed. ERROR: %d\n", blasRet); | ||
| 186 | + return blasRet); | ||
| 187 | + blasRet = aclblasLtMatrixLayoutCreate(&rawDdesc, ACL_FLOAT, m, n, n); | ||
| 188 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("MatrixLayoutCreate D failed. ERROR: %d\n", blasRet); | ||
| 189 | + return blasRet); | ||
| 190 | + | ||
| 191 | + std::unique_ptr<std::remove_pointer<aclblasLtMatrixLayout_t>::type, LtLayoutDeleter> aDescPtr(rawAdesc); | ||
| 192 | + std::unique_ptr<std::remove_pointer<aclblasLtMatrixLayout_t>::type, LtLayoutDeleter> bDescPtr(rawBdesc); | ||
| 193 | + std::unique_ptr<std::remove_pointer<aclblasLtMatrixLayout_t>::type, LtLayoutDeleter> cDescPtr(rawCdesc); | ||
| 194 | + std::unique_ptr<std::remove_pointer<aclblasLtMatrixLayout_t>::type, LtLayoutDeleter> dDescPtr(rawDdesc); | ||
| 195 | + | ||
| 196 | + aclblasLtMatrixLayoutSetAttribute(aDescPtr.get(), ACLBLASLT_MATRIX_LAYOUT_ORDER, &order, sizeof(order)); | ||
| 197 | + aclblasLtMatrixLayoutSetAttribute(bDescPtr.get(), ACLBLASLT_MATRIX_LAYOUT_ORDER, &order, sizeof(order)); | ||
| 198 | + aclblasLtMatrixLayoutSetAttribute(cDescPtr.get(), ACLBLASLT_MATRIX_LAYOUT_ORDER, &order, sizeof(order)); | ||
| 199 | + aclblasLtMatrixLayoutSetAttribute(dDescPtr.get(), ACLBLASLT_MATRIX_LAYOUT_ORDER, &order, sizeof(order)); | ||
| 200 | + | ||
| 201 | + // 2. 准备 Host 数据 | ||
| 202 | + float alpha = 1.0f; | ||
| 203 | + float beta = 0.0f; | ||
| 204 | + std::vector<float> hA(m * k, 1.0f); | ||
| 205 | + std::vector<float> hB(k * n, 1.0f); | ||
| 206 | + | ||
| 207 | + // 3. 申请 Device 内存并拷贝数据 | ||
| 208 | + void* rawA = nullptr; | ||
| 209 | + void* rawB = nullptr; | ||
| 210 | + void* rawD = nullptr; | ||
| 211 | + size_t aBytes = hA.size() * sizeof(float); | ||
| 212 | + size_t bBytes = hB.size() * sizeof(float); | ||
| 213 | + size_t dBytes = m * n * sizeof(float); | ||
| 214 | + | ||
| 215 | + auto aclRet = aclrtMalloc(&rawA, aBytes, ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 216 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("aclrtMalloc for A failed. ERROR: %d\n", aclRet); return aclRet); | ||
| 217 | + aclRet = aclrtMalloc(&rawB, bBytes, ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 218 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("aclrtMalloc for B failed. ERROR: %d\n", aclRet); return aclRet); | ||
| 219 | + aclRet = aclrtMalloc(&rawD, dBytes, ACL_MEM_MALLOC_HUGE_FIRST); | ||
| 220 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("aclrtMalloc for D failed. ERROR: %d\n", aclRet); return aclRet); | ||
| 221 | + | ||
| 222 | + std::unique_ptr<void, AclMemDeleter> aDevicePtr(rawA); | ||
| 223 | + std::unique_ptr<void, AclMemDeleter> bDevicePtr(rawB); | ||
| 224 | + std::unique_ptr<void, AclMemDeleter> dDevicePtr(rawD); | ||
| 225 | + | ||
| 226 | + aclRet = aclrtMemcpy(aDevicePtr.get(), aBytes, hA.data(), aBytes, ACL_MEMCPY_HOST_TO_DEVICE); | ||
| 227 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("aclrtMemcpy for A failed. ERROR: %d\n", aclRet); return aclRet); | ||
| 228 | + aclRet = aclrtMemcpy(bDevicePtr.get(), bBytes, hB.data(), bBytes, ACL_MEMCPY_HOST_TO_DEVICE); | ||
| 229 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("aclrtMemcpy for B failed. ERROR: %d\n", aclRet); return aclRet); | ||
| 230 | + | ||
| 231 | + // 4. 调用 aclblasLtMatmul：D = alpha * A * B + beta * C（beta=0，C 可为 nullptr） | ||
| 232 | + blasRet = aclblasLtMatmul( | ||
| 233 | + handlePtr.get(), descPtr.get(), &alpha, | ||
| 234 | + aDevicePtr.get(), aDescPtr.get(), bDevicePtr.get(), bDescPtr.get(), | ||
| 235 | + &beta, nullptr, cDescPtr.get(), | ||
| 236 | + dDevicePtr.get(), dDescPtr.get(), | ||
| 237 | + nullptr, nullptr, 0, stream); | ||
| 238 | + CHECK_RET(blasRet == ACLBLAS_STATUS_SUCCESS, LOG_PRINT("aclblasLtMatmul failed. ERROR: %d\n", blasRet); | ||
| 239 | + return blasRet); | ||
| 240 | + | ||
| 241 | + // 5. 同步等待任务执行结束 | ||
| 242 | + aclRet = aclrtSynchronizeStream(stream); | ||
| 243 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("aclrtSynchronizeStream failed. ERROR: %d\n", aclRet); return aclRet); | ||
| 244 | + | ||
| 245 | + // 6. 将结果从 Device 拷贝回 Host 并打印 | ||
| 246 | + std::vector<float> hD(m * n, 0.0f); | ||
| 247 | + aclRet = aclrtMemcpy(hD.data(), dBytes, dDevicePtr.get(), dBytes, ACL_MEMCPY_DEVICE_TO_HOST); | ||
| 248 | + CHECK_RET(aclRet == ACL_SUCCESS, LOG_PRINT("copy result from device to host failed. ERROR: %d\n", aclRet); | ||
| 249 | + return aclRet); | ||
| 250 | + LOG_PRINT("D[0] = %f (expected %f)\n", hD[0], static_cast<float>(k)); | ||
| 251 | + | ||
| 252 | + return ACL_SUCCESS; | ||
| 253 | +} | ||
| 254 | + | ||
| 255 | +int main() | ||
| 256 | +{ | ||
| 257 | + AclContext ctx(0); | ||
| 258 | + auto ret = ctx.Init(); | ||
| 259 | + CHECK_RET(ret == ACL_SUCCESS, return ret); | ||
| 260 | + | ||
| 261 | + ret = aclblasLtMatmulTest(ctx); | ||
| 262 | + CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclblasLtMatmulTest failed. ERROR: %d\n", ret); return ret); | ||
| 263 | + return 0; | ||
| 264 | +} | ||
| 47 | ``` | 265 | ``` |
| 48 | -test/blasLtMatmul/ | ||
| 49 | -├── CMakeLists.txt // 编译工程文件 | ||
| 50 | -├── blasLtMatmul_param.h // 参数结构体（继承 BlasTestParamBase） | ||
| 51 | -├── blasLtMatmul_golden.h // CPU golden（封装 aclblasLtMatmul CPU 参考） | ||
| 52 | -└── arch35/ | ||
| 53 | - ├── blasLtMatmul_npu_wrapper.h // NPU wrapper（封装 aclrtMalloc/H2D/kernel/D2H/free） | ||
| 54 | - ├── blasLtMatmul_test.cpp // 精度测试（GTest 入口） | ||
| 55 | - └── blasLtMatmul_test.csv // 精度测试用例表 | ||
| 56 | -``` | ||
| 57 | - | ||
| 58 | -## 接口描述 | ||
| 59 | - | ||
| 60 | -- 接口功能： | ||
| 61 | - 执行矩阵乘法 D = alpha * op(A) * op(B) + beta * C。支持 FP32 全精度路径，以及 MXFP8/MXFP4 量化输入路径（需配合 scale factor）。 | ||
| 62 | - | ||
| 63 | -- 对应接口为： | ||
| 64 | - | ||
| 65 | -``` | ||
| 66 | -aclblasStatus_t aclblasLtMatmul( | ||
| 67 | - aclblasLtHandle_t lightHandle, | ||
| 68 | - aclblasLtMatmulDesc_t computeDesc, | ||
| 69 | - const void* alpha, | ||
| 70 | - const void* A, | ||
| 71 | - aclblasLtMatrixLayout_t Adesc, | ||
| 72 | - const void* B, | ||
| 73 | - aclblasLtMatrixLayout_t Bdesc, | ||
| 74 | - const void* beta, | ||
| 75 | - const void* C, | ||
| 76 | - aclblasLtMatrixLayout_t Cdesc, | ||
| 77 | - void* D, | ||
| 78 | - aclblasLtMatrixLayout_t Ddesc, | ||
| 79 | - const aclblasLtMatmulAlgo_t* algo, | ||
| 80 | - void* workspace, | ||
| 81 | - size_t workspaceSizeInBytes, | ||
| 82 | - aclrtStream stream); | ||
| 83 | -``` | ||
| 84 | - | ||
| 85 | -<table> | ||
| 86 | - <tr> | ||
| 87 | - <td rowspan="1" align="center">参数</td> | ||
| 88 | - <td colspan="4" align="center">aclblasLtMatmul 参数说明</td> | ||
| 89 | - </tr> | ||
| 90 | - <tr> | ||
| 91 | - <td rowspan="18" align="center">参数列表</td> | ||
| 92 | - <td align="center">Param.</td> | ||
| 93 | - <td align="center">Memory</td> | ||
| 94 | - <td align="center">in/out</td> | ||
| 95 | - <td align="center">含义</td> | ||
| 96 | - </tr> | ||
| 97 | - <tr> | ||
| 98 | - <td align="center">lightHandle</td> | ||
| 99 | - <td align="center"></td> | ||
| 100 | - <td align="center">in</td> | ||
| 101 | - <td align="center">aclBLASLt 库上下文句柄，由 aclblasLtCreate 创建。不可为 NULL，否则返回 ACLBLAS_STATUS_NOT_INITIALIZED。</td> | ||
| 102 | - </tr> | ||
| 103 | - <tr> | ||
| 104 | - <td align="center">computeDesc</td> | ||
| 105 | - <td align="center"></td> | ||
| 106 | - <td align="center">in</td> | ||
| 107 | - <td align="center">矩阵乘法描述符，设置 transA/transB、epilogue、scale 指针等属性。不可为 NULL。</td> | ||
| 108 | - </tr> | ||
| 109 | - <tr> | ||
| 110 | - <td align="center">alpha</td> | ||
| 111 | - <td align="center">host</td> | ||
| 112 | - <td align="center">in</td> | ||
| 113 | - <td align="center">用于乘法的 float 标量。不可为 NULL。</td> | ||
| 114 | - </tr> | ||
| 115 | - <tr> | ||
| 116 | - <td align="center">A</td> | ||
| 117 | - <td align="center">device</td> | ||
| 118 | - <td align="center">in</td> | ||
| 119 | - <td align="center">输入矩阵 A，数据类型由 Adesc 指定。不可为 NULL（m>0 且 n>0 时）。</td> | ||
| 120 | - </tr> | ||
| 121 | - <tr> | ||
| 122 | - <td align="center">Adesc</td> | ||
| 123 | - <td align="center"></td> | ||
| 124 | - <td align="center">in</td> | ||
| 125 | - <td align="center">矩阵 A 的 layout 描述符（rows/cols/ld/order/dtype）。</td> | ||
| 126 | - </tr> | ||
| 127 | - <tr> | ||
| 128 | - <td align="center">B</td> | ||
| 129 | - <td align="center">device</td> | ||
| 130 | - <td align="center">in</td> | ||
| 131 | - <td align="center">输入矩阵 B，数据类型由 Bdesc 指定。不可为 NULL（m>0 且 n>0 时）。</td> | ||
| 132 | - </tr> | ||
| 133 | - <tr> | ||
| 134 | - <td align="center">Bdesc</td> | ||
| 135 | - <td align="center"></td> | ||
| 136 | - <td align="center">in</td> | ||
| 137 | - <td align="center">矩阵 B 的 layout 描述符。</td> | ||
| 138 | - </tr> | ||
| 139 | - <tr> | ||
| 140 | - <td align="center">beta</td> | ||
| 141 | - <td align="center">host</td> | ||
| 142 | - <td align="center">in</td> | ||
| 143 | - <td align="center">用于累加的 float 标量。不可为 NULL。beta=0 时 C 可不参与计算。</td> | ||
| 144 | - </tr> | ||
| 145 | - <tr> | ||
| 146 | - <td align="center">C</td> | ||
| 147 | - <td align="center">device</td> | ||
| 148 | - <td align="center">in</td> | ||
| 149 | - <td align="center">累加矩阵 C。beta=0 时可为 NULL。当前测试覆盖 C=NULL 场景。</td> | ||
| 150 | - </tr> | ||
| 151 | - <tr> | ||
| 152 | - <td align="center">Cdesc</td> | ||
| 153 | - <td align="center"></td> | ||
| 154 | - <td align="center">in</td> | ||
| 155 | - <td align="center">矩阵 C 的 layout 描述符。</td> | ||
| 156 | - </tr> | ||
| 157 | - <tr> | ||
| 158 | - <td align="center">D</td> | ||
| 159 | - <td align="center">device</td> | ||
| 160 | - <td align="center">out</td> | ||
| 161 | - <td align="center">输出矩阵 D，维度 m x n。不可为 NULL（m>0 且 n>0 时）。</td> | ||
| 162 | - </tr> | ||
| 163 | - <tr> | ||
| 164 | - <td align="center">Ddesc</td> | ||
| 165 | - <td align="center"></td> | ||
| 166 | - <td align="center">in</td> | ||
| 167 | - <td align="center">矩阵 D 的 layout 描述符，指定输出数据类型（FP32 或 BF16）。</td> | ||
| 168 | - </tr> | ||
| 169 | - <tr> | ||
| 170 | - <td align="center">algo</td> | ||
| 171 | - <td align="center"></td> | ||
| 172 | - <td align="center">in</td> | ||
| 173 | - <td align="center">算法描述符，可为 NULL（使用默认算法）。</td> | ||
| 174 | - </tr> | ||
| 175 | - <tr> | ||
| 176 | - <td align="center">workspace</td> | ||
| 177 | - <td align="center">device</td> | ||
| 178 | - <td align="center">in</td> | ||
| 179 | - <td align="center">工作空间内存，可为 NULL。非 NULL 时需 16B 对齐。</td> | ||
| 180 | - </tr> | ||
| 181 | - <tr> | ||
| 182 | - <td align="center">workspaceSizeInBytes</td> | ||
| 183 | - <td align="center"></td> | ||
| 184 | - <td align="center">in</td> | ||
| 185 | - <td align="center">工作空间大小（字节）。</td> | ||
| 186 | - </tr> | ||
| 187 | - <tr> | ||
| 188 | - <td align="center">stream</td> | ||
| 189 | - <td align="center"></td> | ||
| 190 | - <td align="center">in</td> | ||
| 191 | - <td align="center">AscendCL 执行流。</td> | ||
| 192 | - </tr> | ||
| 193 | -</table> | ||
| 194 | - | ||
| 195 | -### 当前支持的入参/出参范围 | ||
| 196 | - | ||
| 197 | -<table> | ||
| 198 | - <tr> | ||
| 199 | - <td align="center">参数项</td> | ||
| 200 | - <td align="center">支持范围</td> | ||
| 201 | - <td align="center">说明</td> | ||
| 202 | - </tr> | ||
| 203 | - <tr> | ||
| 204 | - <td align="center">dtypeA / dtypeB</td> | ||
| 205 | - <td align="center">FP32；MXFP8_E4M3FN；MXFP4_E2M1</td> | ||
| 206 | - <td align="center">A/B 须为同类型组合：FP32×FP32、MXFP8×MXFP8、MXFP4×MXFP4。其他组合返回 ACLBLAS_STATUS_NOT_SUPPORTED。</td> | ||
| 207 | - </tr> | ||
| 208 | - <tr> | ||
| 209 | - <td align="center">dtypeC</td> | ||
| 210 | - <td align="center">FP32</td> | ||
| 211 | - <td align="center">累加矩阵类型，当前固定为 FP32。</td> | ||
| 212 | - </tr> | ||
| 213 | - <tr> | ||
| 214 | - <td align="center">dtypeD</td> | ||
| 215 | - <td align="center">FP32；BF16</td> | ||
| 216 | - <td align="center">MXFP8/MXFP4 路径支持 FP32 或 BF16 输出；FP32 路径输出 FP32。</td> | ||
| 217 | - </tr> | ||
| 218 | - <tr> | ||
| 219 | - <td align="center">computeType</td> | ||
| 220 | - <td align="center">ACLBLAS_COMPUTE_32F</td> | ||
| 221 | - <td align="center">所有已支持路径均使用 32F 计算精度。</td> | ||
| 222 | - </tr> | ||
| 223 | - <tr> | ||
| 224 | - <td align="center">transA / transB</td> | ||
| 225 | - <td align="center">N、T</td> | ||
| 226 | - <td align="center">对应 ACLBLAS_OP_N（不转置）、ACLBLAS_OP_T（转置）。</td> | ||
| 227 | - </tr> | ||
| 228 | - <tr> | ||
| 229 | - <td align="center">M / N / K</td> | ||
| 230 | - <td align="center">M,N,K ≥ 0</td> | ||
| 231 | - <td align="center">M=0 或 N=0 时为空操作，直接返回 SUCCESS。MXFP8/MXFP4 路径要求 K 为 32 的整数倍，否则返回 ACLBLAS_STATUS_INVALID_VALUE。</td> | ||
| 232 | - </tr> | ||
| 233 | - <tr> | ||
| 234 | - <td align="center">lda / ldb / ldc / ldd</td> | ||
| 235 | - <td align="center">ld ≥ 物理列数</td> | ||
| 236 | - <td align="center">行主序（ACLBLASLT_ORDER_ROW）存储，ld 为 leading dimension，须 ≥ 矩阵物理列数。MXFP4 的 ld 为逻辑元素 leading dim（2 个 FP4 元素打包为 1 字节）。</td> | ||
| 237 | - </tr> | ||
| 238 | - <tr> | ||
| 239 | - <td align="center">alpha / beta</td> | ||
| 240 | - <td align="center">float</td> | ||
| 241 | - <td align="center">当前测试覆盖 alpha=1.0、beta=0.0。beta=0 时 C 可为 NULL。</td> | ||
| 242 | - </tr> | ||
| 243 | - <tr> | ||
| 244 | - <td align="center">epilogue</td> | ||
| 245 | - <td align="center">ACLBLASLT_EPILOGUE_DEFAULT</td> | ||
| 246 | - <td align="center">当前仅支持默认 epilogue。</td> | ||
| 247 | - </tr> | ||
| 248 | - <tr> | ||
| 249 | - <td align="center">scaleA / scaleB</td> | ||
| 250 | - <td align="center">MXFP8/MXFP4 必填</td> | ||
| 251 | - <td align="center">通过 computeDesc 的 ACLBLASLT_MATMUL_DESC_A/B_SCALE_POINTER 设置，E8M0 格式，按 K 方向每 32 元素一组。不可为 NULL。</td> | ||
| 252 | - </tr> | ||
| 253 | - <tr> | ||
| 254 | - <td align="center">algo</td> | ||
| 255 | - <td align="center">default / NULL</td> | ||
| 256 | - <td align="center">可为 NULL，使用默认算法。</td> | ||
| 257 | - </tr> | ||
| 258 | - <tr> | ||
| 259 | - <td align="center">order</td> | ||
| 260 | - <td align="center">ACLBLASLT_ORDER_ROW</td> | ||
| 261 | - <td align="center">当前实现使用行主序。</td> | ||
| 262 | - </tr> | ||
| 263 | -</table> | ||
| 264 | - | ||
| 265 | -- 算子规格： | ||
| 266 | - <table> | ||
| 267 | - <tr><td rowspan="1" align="center">算子类型(OpType)</td><td colspan="6" align="center">aclblasLtMatmul</td></tr> | ||
| 268 | - <tr><td rowspan="7" align="center">算子输入</td><td align="center">name</td><td align="center">shape</td><td align="center">data type</td><td align="center">format</td></tr> | ||
| 269 | - <tr><td align="center">A</td><td align="center">M×K（或转置后 K×M）</td><td align="center">FP32 / MXFP8 / MXFP4</td><td align="center">ND</td></tr> | ||
| 270 | - <tr><td align="center">B</td><td align="center">K×N（或转置后 N×K）</td><td align="center">FP32 / MXFP8 / MXFP4</td><td align="center">ND</td></tr> | ||
| 271 | - <tr><td align="center">C</td><td align="center">M×N</td><td align="center">FP32</td><td align="center">ND</td></tr> | ||
| 272 | - <tr><td align="center">scaleA</td><td align="center">按 K 分组</td><td align="center">E8M0 (uint8)</td><td align="center">ND</td></tr> | ||
| 273 | - <tr><td align="center">scaleB</td><td align="center">按 K 分组</td><td align="center">E8M0 (uint8)</td><td align="center">ND</td></tr> | ||
| 274 | - <tr><td align="center">alpha/beta</td><td align="center">1</td><td align="center">float</td><td align="center">ND</td></tr> | ||
| 275 | - <tr><td rowspan="1" align="center">算子输出</td><td align="center">D</td><td align="center">M×N</td><td align="center">FP32 / BF16</td><td align="center">ND</td></tr> | ||
| 276 | - <tr><td rowspan="1" align="center">核函数名</td><td colspan="6" align="center">MatmulFp32Kernel / matmul_mxfp8_kernel_do / ltmatmul_mxfp4_kernel_do</td></tr> | ||
| 277 | - </table> | ||
| 278 | - | ||
| 279 | -- 算子实现： | ||
| 280 | - Host 侧根据 A/B 数据类型路由至 FP32、MXFP8 或 MXFP4 对应的 Tiling 与 Kernel 实现。MXFP 路径在 Kernel 内完成量化矩阵乘加，输出经 epilogue 处理写入 D。 | ||
| 281 | - | ||
| 282 | -- 调用实现： | ||
| 283 | - 通过 aclBLASLt 标准 API 调用，内部使用内核调用符 `<<<>>>` 启动 NPU 核函数。 | ||
| 284 | - | ||
| 285 | -## 测试用例覆盖 | ||
| 286 | - | ||
| 287 | -| 分组 | 用例数 | 覆盖场景 | | ||
| 288 | -|------|--------|----------| | ||
| 289 | -| L0 FP32 基础 | 7 | 小/中/大规模 NN、TN/NT/TT 转置、algo=nullptr | | ||
| 290 | -| L0 MXFP8 | 6 | K=32 基础/大规模、四种转置、C=null | | ||
| 291 | -| L0 MXFP4 | 7 | K=32 基础/大规模、四种转置、C=null、FP32 输出 | | ||
| 292 | -| L0 异常入参 | 4 | handle/desc/alpha/A 为 NULL | | ||
| 293 | -| L0 边界 | 6 | M=0/N=0 空操作、K 非 32 倍数非法、algo=nullptr | | ||
| 294 | -| L1 FP32 扩展 | 11 | 矩形矩阵、非方阵转置、瘦矩阵 M128×N32×K128 | | ||
| 295 | -| L1 MXFP8 扩展 | 22 | 多种 K 规模、矩形/奇数维度、scale 全零、K 非法值 | | ||
| 296 | -| L1 MXFP4 扩展 | 17 | 多种 K 规模、矩形/转置、scale 全零、K 非法值 | | ||
| 297 | -| TEST_F 固定用例 | 4 | NullHandle、NullComputeDesc、NullAlpha、NullA | | ||
| 298 | - | ||
| 299 | -## 编译运行 | ||
| 300 | - | ||
| 301 | -在本样例根目录下执行如下步骤，编译并执行测试。 | ||
| 302 | - | ||
| 303 | -- 配置环境变量 | ||
| 304 | - 请根据当前环境上 CANN 开发套件包的安装方式，选择对应配置环境变量的命令。 | ||
| 305 | - - 默认路径，root 用户安装 CANN 软件包 | ||
| 306 | - ```bash | ||
| 307 | - source /usr/local/Ascend/cann/set_env.sh | ||
| 308 | - ``` | ||
| 309 | - | ||
| 310 | - - 默认路径，非 root 用户安装 CANN 软件包 | ||
| 311 | - ```bash | ||
| 312 | - source $HOME/Ascend/cann/set_env.sh | ||
| 313 | - ``` | ||
| 314 | - | ||
| 315 | - - 指定路径 install_path，安装 CANN 软件包 | ||
| 316 | - ```bash | ||
| 317 | - source ${install_path}/cann/set_env.sh | ||
| 318 | - ``` | ||
| 319 | - | ||
| 320 | -- 样例执行 | ||
| 321 | - ```bash | ||
| 322 | - bash build.sh --ops=blasLtMatmul --soc=ascend950 --run | ||
| 323 | - ``` | ||
| 324 | - | ||
| 325 | - 其中 `--soc` 为**可选**参数，用于指定目标硬件平台（与上文「产品支持情况」对应）。按实际硬件选用： | ||
| 326 | - | ||
| 327 | - | 产品 | `--soc` 取值 | | ||
| 328 | - |------|----------------| | ||
| 329 | - | Ascend 950PR / Ascend 950DT | `ascend950` | | ||
| 330 | - | ||
| 331 | - 执行结果如下，说明精度对比成功。 | ||
| 332 | - ```bash | ||
| 333 | - [PASS] blasLtMatmul_test | ||
| 334 | - ``` | ||
| @@ -1,1593 +0,0 @@ | |||
| 1 | -/** | ||
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | - */ | ||
| 10 | - | ||
| 11 | - | ||
| 12 | - | ||
| 13 | - | ||
| 14 | - | ||
| 15 | - | ||
| 16 | - | ||
| 17 | - | ||
| 18 | - | ||
| 19 | - | ||
| 20 | - | ||
| 21 | - | ||
| 22 | - | ||
| 23 | - | ||
| 24 | - | ||
| 25 | - | ||
| 26 | - | ||
| 27 | - | ||
| 28 | - | ||
| 29 | - | ||
| 30 | - | ||
| 31 | - | ||
| 32 | -namespace { | ||
| 33 | - | ||
| 34 | -constexpr int ACLBLASLT_VERSION_MAJOR = 1; | ||
| 35 | -constexpr int ACLBLASLT_VERSION_MINOR = 0; | ||
| 36 | -constexpr int ACLBLASLT_VERSION_PATCH = 0; | ||
| 37 | - | ||
| 38 | -constexpr uint32_t ACLBLASLT_HANDLE_MAGIC = 0xACBA1234; | ||
| 39 | -constexpr uint32_t ACLBLASLT_LAYOUT_MAGIC = 0xACBB1234; | ||
| 40 | -constexpr uint32_t ACLBLASLT_DESC_MAGIC = 0xACBC1234; | ||
| 41 | -constexpr uint32_t ACLBLASLT_ALGO_MAGIC = 0xACBD1234; | ||
| 42 | - | ||
| 43 | -constexpr size_t DEFAULT_WORKSPACE_SIZE = 32 * 1024 * 1024; | ||
| 44 | -constexpr size_t L1_SIZE = 512 * 1024; | ||
| 45 | -constexpr size_t L0_SIZE = 256; | ||
| 46 | -constexpr uint32_t DEFAULT_AI_CORES = 8; | ||
| 47 | -constexpr double DEFAULT_PEAK_TFLOPS = 140.0; | ||
| 48 | -constexpr double DEFAULT_PEAK_GBPS = 900.0; | ||
| 49 | - | ||
| 50 | -struct AtlasA2 { | ||
| 51 | - static constexpr uint32_t BIAS_SIZE = 1024; | ||
| 52 | - static constexpr uint32_t FIXBUF_SIZE = 7 * 1024; | ||
| 53 | - static constexpr uint32_t UB_SIZE = 192 * 1024; | ||
| 54 | - static constexpr uint32_t L1_SIZE = 512 * 1024; | ||
| 55 | - static constexpr uint32_t L0A_SIZE = 64 * 1024; | ||
| 56 | - static constexpr uint32_t L0B_SIZE = 64 * 1024; | ||
| 57 | - static constexpr uint32_t L0C_SIZE = 128 * 1024; | ||
| 58 | -}; | ||
| 59 | - | ||
| 60 | -struct Ascend950 { | ||
| 61 | - static constexpr uint32_t BIAS_SIZE = 4 * 1024; | ||
| 62 | - static constexpr uint32_t FIXBUF_SIZE = 16 * 1024; | ||
| 63 | - static constexpr uint32_t UB_SIZE = 248 * 1024; | ||
| 64 | - static constexpr uint32_t L1_SIZE = 512 * 1024; | ||
| 65 | - static constexpr uint32_t L0A_SIZE = 64 * 1024; | ||
| 66 | - static constexpr uint32_t L0B_SIZE = 64 * 1024; | ||
| 67 | - static constexpr uint32_t L0C_SIZE = 256 * 1024; | ||
| 68 | -}; | ||
| 69 | - | ||
| 70 | -enum DispatchPolicyType : uint8_t | ||
| 71 | -{ | ||
| 72 | - DISPATCH_POLICY_MMAD_SYNC = 0, | ||
| 73 | - DISPATCH_POLICY_MMAD_PINGPONG = 1, | ||
| 74 | - DISPATCH_POLICY_MMAD_MULTI_STAGE = 2, | ||
| 75 | -}; | ||
| 76 | - | ||
| 77 | -struct AlgoKey { | ||
| 78 | - uint64_t m = 0; | ||
| 79 | - uint64_t n = 0; | ||
| 80 | - uint64_t k = 0; | ||
| 81 | - aclDataType aType = ACL_FLOAT; | ||
| 82 | - aclDataType bType = ACL_DT_UNDEFINED; | ||
| 83 | - aclDataType cType = ACL_DT_UNDEFINED; | ||
| 84 | - aclDataType dType = ACL_DT_UNDEFINED; | ||
| 85 | - aclblasComputeType_t computeType = ACLBLAS_COMPUTE_32F; | ||
| 86 | - aclblasLtEpilogue_t epilogue = ACLBLASLT_EPILOGUE_DEFAULT; | ||
| 87 | - bool transA = false; | ||
| 88 | - bool transB = false; | ||
| 89 | - | ||
| 90 | - bool operator==(const AlgoKey& other) const | ||
| 91 | - { | ||
| 92 | - return m == other.m && n == other.n && k == other.k && aType == other.aType && bType == other.bType && | ||
| 93 | - cType == other.cType && dType == other.dType && computeType == other.computeType && | ||
| 94 | - epilogue == other.epilogue && transA == other.transA && transB == other.transB; | ||
| 95 | - } | ||
| 96 | -}; | ||
| 97 | - | ||
| 98 | -struct AlgoKeyHasher { | ||
| 99 | - size_t operator()(const AlgoKey& x) const | ||
| 100 | - { | ||
| 101 | - size_t h = 1469598103934665603ull; | ||
| 102 | - auto mix = [&](uint64_t v) { h ^= static_cast<size_t>(v + 0x9e3779b97f4a7c15ull + (h << 6) + (h >> 2)); }; | ||
| 103 | - mix(x.m); | ||
| 104 | - mix(x.n); | ||
| 105 | - mix(x.k); | ||
| 106 | - mix(static_cast<uint64_t>(x.aType)); | ||
| 107 | - mix(static_cast<uint64_t>(x.bType)); | ||
| 108 | - mix(static_cast<uint64_t>(x.cType)); | ||
| 109 | - mix(static_cast<uint64_t>(x.dType)); | ||
| 110 | - mix(static_cast<uint64_t>(x.computeType)); | ||
| 111 | - mix(static_cast<uint64_t>(x.epilogue)); | ||
| 112 | - mix(static_cast<uint64_t>(x.transA)); | ||
| 113 | - mix(static_cast<uint64_t>(x.transB)); | ||
| 114 | - return h; | ||
| 115 | - } | ||
| 116 | -}; | ||
| 117 | - | ||
| 118 | -struct CacheEntry { | ||
| 119 | - aclblasLtMatmulAlgo_t algo; | ||
| 120 | - std::list<AlgoKey>::iterator lruIter; | ||
| 121 | -}; | ||
| 122 | - | ||
| 123 | -struct aclblasLtHandle { | ||
| 124 | - uint32_t magic = ACLBLASLT_HANDLE_MAGIC; | ||
| 125 | - bool initialized = false; | ||
| 126 | - // version info | ||
| 127 | - int versionMajor = ACLBLASLT_VERSION_MAJOR; | ||
| 128 | - int versionMinor = ACLBLASLT_VERSION_MINOR; | ||
| 129 | - // AscendCL runtime | ||
| 130 | - aclrtContext context = nullptr; | ||
| 131 | - aclrtStream defaultStream = nullptr; | ||
| 132 | - int32_t deviceId = 0; | ||
| 133 | - // workspace | ||
| 134 | - void* internalWorkspace = nullptr; | ||
| 135 | - size_t workspaceSize = 0; | ||
| 136 | - // thread safety | ||
| 137 | - std::mutex* mutex = nullptr; | ||
| 138 | - // soc spec | ||
| 139 | - int npuArch = 0; | ||
| 140 | - size_t maxSharedMemory = 0; | ||
| 141 | - // algo cache | ||
| 142 | - std::unordered_map<AlgoKey, CacheEntry, AlgoKeyHasher>* algoCache = nullptr; | ||
| 143 | - size_t algoCacheMaxSize = 128; | ||
| 144 | - std::list<AlgoKey>* lruList = nullptr; | ||
| 145 | -}; | ||
| 146 | - | ||
| 147 | -struct aclblasLtMatrixLayoutImpl { | ||
| 148 | - uint32_t magic; | ||
| 149 | - aclDataType type; | ||
| 150 | - uint64_t rows; | ||
| 151 | - uint64_t cols; | ||
| 152 | - int64_t ld; | ||
| 153 | - aclblasLtOrder_t order = ACLBLASLT_ORDER_COL; | ||
| 154 | - int32_t batchCount = 1; | ||
| 155 | - int64_t stridedBatchOffset = 0; | ||
| 156 | -}; | ||
| 157 | -static_assert( | ||
| 158 | - sizeof(aclblasLtMatrixLayoutImpl) <= sizeof(aclblasLtMatrixLayoutOpaque_t), | ||
| 159 | - "Impl of aclblasLtMatrixLayout must fit in capsule!"); | ||
| 160 | - | ||
| 161 | -struct aclblasLtMatmulDescImpl { | ||
| 162 | - uint32_t magic; | ||
| 163 | - aclblasComputeType_t computeType; | ||
| 164 | - aclDataType scaleType; | ||
| 165 | - aclblasOperation_t transA = ACLBLAS_OP_N; | ||
| 166 | - aclblasOperation_t transB = ACLBLAS_OP_N; | ||
| 167 | - aclblasLtEpilogue_t epilogue = ACLBLASLT_EPILOGUE_DEFAULT; | ||
| 168 | - const void* bias = nullptr; | ||
| 169 | - aclDataType biasDataType = ACL_DT_UNDEFINED; | ||
| 170 | - const void* scaleA = nullptr; | ||
| 171 | - const void* scaleB = nullptr; | ||
| 172 | -}; | ||
| 173 | - | ||
| 174 | -constexpr size_t kBiasPtrStorageBytes = sizeof(void*); | ||
| 175 | -static_assert( | ||
| 176 | - sizeof(aclblasLtMatmulDescImpl) <= sizeof(aclblasLtMatmulDescOpaque_t), | ||
| 177 | - "Impl of aclblasLtMatmulDesc must fit in capsule!"); | ||
| 178 | - | ||
| 179 | -struct aclblasLtMatmulPreferenceImpl { | ||
| 180 | - uint32_t magic; | ||
| 181 | - uint32_t searchMode = 0; | ||
| 182 | - size_t maxWorkspaceBytes = DEFAULT_WORKSPACE_SIZE; | ||
| 183 | - int32_t maxResults = 3; | ||
| 184 | - bool allowMixedPrecision = true; | ||
| 185 | - bool allowSplitK = true; | ||
| 186 | - // tiling | ||
| 187 | - uint32_t preferredL0M = 0; | ||
| 188 | - uint32_t preferredL0N = 0; | ||
| 189 | - uint32_t preferredL0K = 0; | ||
| 190 | - // Scheduling | ||
| 191 | - bool preferPingpong = false; | ||
| 192 | - bool preferDoubleBuffer = false; | ||
| 193 | - float minEfficiency = 0.5f; | ||
| 194 | -}; | ||
| 195 | -static_assert( | ||
| 196 | - sizeof(aclblasLtMatmulPreferenceImpl) <= sizeof(aclblasLtMatmulPreferenceOpaque_t), | ||
| 197 | - "Impl of aclblasLtMatmulPreference must fit in capsule!"); | ||
| 198 | - | ||
| 199 | -struct AscendHardwareCaps { | ||
| 200 | - uint32_t numAICores = DEFAULT_AI_CORES; | ||
| 201 | - uint32_t l0CubeSize = L0_SIZE; | ||
| 202 | - size_t l1BufferSize = L1_SIZE; | ||
| 203 | - double memoryBandwidthGBps = DEFAULT_PEAK_GBPS; | ||
| 204 | - double peakTFlops = DEFAULT_PEAK_TFLOPS; | ||
| 205 | - double bandwidthBoundThreshold = 32.0; | ||
| 206 | -}; | ||
| 207 | - | ||
| 208 | -struct AlgoCandidate { | ||
| 209 | - uint32_t algoId = 0; | ||
| 210 | - uint32_t l1TileM = 128; | ||
| 211 | - uint32_t l1TileN = 128; | ||
| 212 | - uint32_t l1TileK = 128; | ||
| 213 | - uint32_t l0TileM = 64; | ||
| 214 | - uint32_t l0TileN = 64; | ||
| 215 | - uint32_t l0TileK = 64; | ||
| 216 | - DispatchPolicyType policy = DISPATCH_POLICY_MMAD_SYNC; | ||
| 217 | - uint32_t numBuffers = 1; | ||
| 218 | - uint32_t splitKFactor = 1; | ||
| 219 | - size_t workspaceSize = 0; | ||
| 220 | - double peakPerformance = DEFAULT_PEAK_TFLOPS; | ||
| 221 | -}; | ||
| 222 | - | ||
| 223 | -struct ScoredResult { | ||
| 224 | - AlgoCandidate cand; | ||
| 225 | - double estimatedTimeMs = 0.0; | ||
| 226 | - double totalScore = 0.0; | ||
| 227 | - bool isEfficient = true; | ||
| 228 | -}; | ||
| 229 | - | ||
| 230 | -struct PackedAlgo { | ||
| 231 | - uint32_t magic; | ||
| 232 | - uint32_t algoId; | ||
| 233 | - uint16_t l1mDiv16; | ||
| 234 | - uint16_t l1nDiv16; | ||
| 235 | - uint8_t policy; | ||
| 236 | - uint8_t numBuffers; | ||
| 237 | - uint8_t splitK; | ||
| 238 | - uint8_t flags; | ||
| 239 | -}; | ||
| 240 | -static_assert(sizeof(PackedAlgo) == 16, "PackedAlgo must fit algo.data"); | ||
| 241 | - | ||
| 242 | -static uint32_t GenerateAlgoId( | ||
| 243 | - DispatchPolicyType policy, uint32_t l1m, uint32_t l1n, uint32_t l1k, uint32_t splitKFactor) | ||
| 244 | -{ | ||
| 245 | - return (static_cast<uint32_t>(policy) << 28) ^ (l1m << 16) ^ (l1n << 8) ^ (l1k << 2) ^ splitKFactor; | ||
| 246 | -} | ||
| 247 | - | ||
| 248 | -static aclblasLtMatmulAlgo_t BuildAlgoFromCandidate(const AlgoCandidate& cand) | ||
| 249 | -{ | ||
| 250 | - aclblasLtMatmulAlgo_t out{}; | ||
| 251 | - PackedAlgo packed{}; | ||
| 252 | - packed.magic = ACLBLASLT_ALGO_MAGIC; | ||
| 253 | - packed.algoId = cand.algoId; | ||
| 254 | - packed.l1mDiv16 = static_cast<uint16_t>(cand.l1TileM / 16); | ||
| 255 | - packed.l1nDiv16 = static_cast<uint16_t>(cand.l1TileN / 16); | ||
| 256 | - packed.policy = static_cast<uint8_t>(cand.policy); | ||
| 257 | - packed.numBuffers = static_cast<uint8_t>(cand.numBuffers); | ||
| 258 | - packed.splitK = static_cast<uint8_t>(cand.splitKFactor); | ||
| 259 | - (void)MemcpySSucceeds(out.data, sizeof(out.data), &packed, sizeof(packed)); | ||
| 260 | - out.max_workspace_bytes = cand.workspaceSize; | ||
| 261 | - return out; | ||
| 262 | -} | ||
| 263 | - | ||
| 264 | -static bool DecodeAlgo(const aclblasLtMatmulAlgo_t& algo, PackedAlgo* packed) | ||
| 265 | -{ | ||
| 266 | - if (packed == nullptr) { | ||
| 267 | - return false; | ||
| 268 | - } | ||
| 269 | - if (!MemcpySSucceeds(packed, sizeof(PackedAlgo), algo.data, sizeof(PackedAlgo))) { | ||
| 270 | - return false; | ||
| 271 | - } | ||
| 272 | - return packed->magic == ACLBLASLT_ALGO_MAGIC; | ||
| 273 | -} | ||
| 274 | - | ||
| 275 | -static void GetAscendHardwareCaps(int32_t, AscendHardwareCaps* caps) | ||
| 276 | -{ | ||
| 277 | - if (caps == nullptr) { | ||
| 278 | - return; | ||
| 279 | - } | ||
| 280 | - // 当前仓库保持轻量默认能力值，后续可对接真实设备查询。 | ||
| 281 | - caps->numAICores = DEFAULT_AI_CORES; | ||
| 282 | - caps->l0CubeSize = L0_SIZE; | ||
| 283 | - caps->l1BufferSize = L1_SIZE; | ||
| 284 | - caps->memoryBandwidthGBps = DEFAULT_PEAK_GBPS; | ||
| 285 | - caps->peakTFlops = DEFAULT_PEAK_TFLOPS; | ||
| 286 | - caps->bandwidthBoundThreshold = 32.0; | ||
| 287 | -} | ||
| 288 | - | ||
| 289 | -static void SelectL1TileShape( | ||
| 290 | - uint64_t m, uint64_t n, uint64_t, uint32_t numAICores, uint32_t prefL0M, uint32_t prefL0N, uint32_t prefL0K, | ||
| 291 | - uint32_t* l1M, uint32_t* l1N, uint32_t* l1K) | ||
| 292 | -{ | ||
| 293 | - const uint32_t candidates[][3] = { | ||
| 294 | - {128, 256, 256}, {256, 128, 256}, {256, 256, 128}, {128, 128, 128}, {256, 256, 64}}; | ||
| 295 | - | ||
| 296 | - float bestScore = -1.0f; | ||
| 297 | - const uint32_t* best = candidates[0]; | ||
| 298 | - | ||
| 299 | - for (const auto& cand : candidates) { | ||
| 300 | - uint32_t cm = cand[0]; | ||
| 301 | - uint32_t cn = cand[1]; | ||
| 302 | - uint32_t ck = cand[2]; | ||
| 303 | - | ||
| 304 | - uint32_t tilesM = CeilDiv<uint32_t>(m, cm); | ||
| 305 | - uint32_t tilesN = CeilDiv<uint32_t>(n, cn); | ||
| 306 | - uint32_t totalTiles = tilesM * tilesN; | ||
| 307 | - | ||
| 308 | - float balanceScore = | ||
| 309 | - 1.0f - static_cast<float>(totalTiles % std::max(1u, numAICores)) / std::max(1u, numAICores); | ||
| 310 | - size_t l1Usage = static_cast<size_t>(cm) * ck * sizeof(float) + static_cast<size_t>(cn) * ck * sizeof(float); | ||
| 311 | - float l1Util = static_cast<float>(l1Usage) / static_cast<float>(L1_SIZE); | ||
| 312 | - | ||
| 313 | - float l0Match = 0.0f; | ||
| 314 | - if (prefL0M > 0 && cm % prefL0M == 0) { | ||
| 315 | - l0Match += 0.3f; | ||
| 316 | - } | ||
| 317 | - if (prefL0N > 0 && cn % prefL0N == 0) { | ||
| 318 | - l0Match += 0.3f; | ||
| 319 | - } | ||
| 320 | - if (prefL0K > 0 && ck % prefL0K == 0) { | ||
| 321 | - l0Match += 0.4f; | ||
| 322 | - } | ||
| 323 | - | ||
| 324 | - float score = balanceScore * 0.4f + std::min(l1Util, 1.0f) * 0.3f + l0Match * 0.3f; | ||
| 325 | - if (score > bestScore) { | ||
| 326 | - bestScore = score; | ||
| 327 | - best = cand; | ||
| 328 | - } | ||
| 329 | - } | ||
| 330 | - | ||
| 331 | - *l1M = best[0]; | ||
| 332 | - *l1N = best[1]; | ||
| 333 | - *l1K = best[2]; | ||
| 334 | -} | ||
| 335 | - | ||
| 336 | -static void SelectL0TileShape( | ||
| 337 | - uint32_t l1M, uint32_t l1N, uint32_t l1K, size_t, size_t, aclDataType, aclDataType, uint32_t* l0M, uint32_t* l0N, | ||
| 338 | - uint32_t* l0K) | ||
| 339 | -{ | ||
| 340 | - *l0K = std::min(64u, l1K); | ||
| 341 | - *l0M = std::min(128u, l1M); | ||
| 342 | - *l0N = std::min(256u, l1N); | ||
| 343 | - | ||
| 344 | - while (*l0M > 16 && (l1M % *l0M != 0)) { | ||
| 345 | - --(*l0M); | ||
| 346 | - } | ||
| 347 | - while (*l0N > 16 && (l1N % *l0N != 0)) { | ||
| 348 | - --(*l0N); | ||
| 349 | - } | ||
| 350 | -} | ||
| 351 | - | ||
| 352 | -static uint32_t SelectSplitKForAscend(uint32_t l1LoopsK, uint32_t numAICores) | ||
| 353 | -{ | ||
| 354 | - if (numAICores == 0) { | ||
| 355 | - return 1; | ||
| 356 | - } | ||
| 357 | - uint32_t candidate = std::min(l1LoopsK, numAICores); | ||
| 358 | - return std::max(1u, candidate); | ||
| 359 | -} | ||
| 360 | - | ||
| 361 | -static size_t CalculateWorkspaceForAscend(uint64_t m, uint64_t n, uint32_t splitKFactor, aclblasLtEpilogue_t epilogue) | ||
| 362 | -{ | ||
| 363 | - size_t workspace = 0; | ||
| 364 | - if (splitKFactor > 1) { | ||
| 365 | - workspace += | ||
| 366 | - static_cast<size_t>(splitKFactor) * static_cast<size_t>(m) * static_cast<size_t>(n) * sizeof(float); | ||
| 367 | - } | ||
| 368 | - | ||
| 369 | - switch (epilogue) { | ||
| 370 | - case ACLBLASLT_EPILOGUE_BIAS: | ||
| 371 | - case ACLBLASLT_EPILOGUE_RELU_BIAS: | ||
| 372 | - case ACLBLASLT_EPILOGUE_GELU_BIAS: | ||
| 373 | - workspace += static_cast<size_t>(m) * sizeof(float); | ||
| 374 | - break; | ||
| 375 | - case ACLBLASLT_EPILOGUE_GELU: | ||
| 376 | - case ACLBLASLT_EPILOGUE_RELU: | ||
| 377 | - workspace += 64 * 1024; | ||
| 378 | - break; | ||
| 379 | - default: | ||
| 380 | - break; | ||
| 381 | - } | ||
| 382 | - return workspace; | ||
| 383 | -} | ||
| 384 | - | ||
| 385 | -static bool CheckHandleValid(const aclblasLtHandle* h) | ||
| 386 | -{ | ||
| 387 | - return h != nullptr && h->magic == ACLBLASLT_HANDLE_MAGIC && h->initialized && h->algoCache != nullptr && | ||
| 388 | - h->lruList != nullptr; | ||
| 389 | -} | ||
| 390 | - | ||
| 391 | -static bool BuildGemmShape( | ||
| 392 | - const aclblasLtMatmulDescImpl* desc, const aclblasLtMatrixLayoutImpl* A, const aclblasLtMatrixLayoutImpl* B, | ||
| 393 | - const aclblasLtMatrixLayoutImpl* D, uint64_t* m, uint64_t* n, uint64_t* k) | ||
| 394 | -{ | ||
| 395 | - if (desc == nullptr || A == nullptr || B == nullptr || D == nullptr || m == nullptr || n == nullptr || | ||
| 396 | - k == nullptr) { | ||
| 397 | - return false; | ||
| 398 | - } | ||
| 399 | - const bool transA = (desc->transA != ACLBLAS_OP_N); | ||
| 400 | - const bool transB = (desc->transB != ACLBLAS_OP_N); | ||
| 401 | - | ||
| 402 | - const uint64_t mA = transA ? A->cols : A->rows; | ||
| 403 | - const uint64_t kA = transA ? A->rows : A->cols; | ||
| 404 | - const uint64_t kB = transB ? B->cols : B->rows; | ||
| 405 | - const uint64_t nB = transB ? B->rows : B->cols; | ||
| 406 | - | ||
| 407 | - if (mA != D->rows || kA != kB || nB != D->cols) { | ||
| 408 | - return false; | ||
| 409 | - } | ||
| 410 | - | ||
| 411 | - *m = mA; | ||
| 412 | - *n = nB; | ||
| 413 | - *k = kA; | ||
| 414 | - return true; | ||
| 415 | -} | ||
| 416 | - | ||
| 417 | -// Pack a transform descriptor impl back into its capsule and zero-pad the remainder. | ||
| 418 | -aclblasStatus_t MatPackTransformImpl(void* capsule, size_t capsuleBytes, const void* impl, size_t implBytes) | ||
| 419 | -{ | ||
| 420 | - aclblasStatus_t copyStatus = CheckedMemcpyS(capsule, capsuleBytes, impl, implBytes); | ||
| 421 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 422 | - return copyStatus; | ||
| 423 | - } | ||
| 424 | - if (capsuleBytes > implBytes) { | ||
| 425 | - copyStatus = CheckedMemsetS( | ||
| 426 | - reinterpret_cast<char*>(capsule) + implBytes, capsuleBytes - implBytes, capsuleBytes - implBytes); | ||
| 427 | - } | ||
| 428 | - return copyStatus; | ||
| 429 | -} | ||
| 430 | - | ||
| 431 | -// Pack the transform-relevant fields of a matrix layout capsule into the plain parameter struct | ||
| 432 | -// consumed by the transform operator unit. | ||
| 433 | -MatTransformLayout MatPackTransformLayout(const aclblasLtMatrixLayoutImpl* layout) | ||
| 434 | -{ | ||
| 435 | - MatTransformLayout packed; | ||
| 436 | - packed.type = layout->type; | ||
| 437 | - packed.rows = layout->rows; | ||
| 438 | - packed.cols = layout->cols; | ||
| 439 | - packed.ld = layout->ld; | ||
| 440 | - packed.order = layout->order; | ||
| 441 | - packed.batchCount = layout->batchCount; | ||
| 442 | - return packed; | ||
| 443 | -} | ||
| 444 | - | ||
| 445 | -} // namespace | ||
| 446 | - | ||
| 447 | -extern "C" { | ||
| 448 | - | ||
| 449 | -aclblasStatus_t aclblasLtGetVersion(size_t* version) | ||
| 450 | -{ | ||
| 451 | - if (version == nullptr) { | ||
| 452 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 453 | - } | ||
| 454 | - | ||
| 455 | - *version = (static_cast<size_t>(ACLBLASLT_VERSION_MAJOR) << 24) | | ||
| 456 | - (static_cast<size_t>(ACLBLASLT_VERSION_MINOR) << 16) | static_cast<size_t>(ACLBLASLT_VERSION_PATCH); | ||
| 457 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 458 | -} | ||
| 459 | - | ||
| 460 | -aclblasStatus_t aclblasLtGetProperty(aclblasLtPropertyType_t type, int* value) | ||
| 461 | -{ | ||
| 462 | - if (value == nullptr) { | ||
| 463 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 464 | - } | ||
| 465 | - | ||
| 466 | - switch (type) { | ||
| 467 | - case ACLBLASLT_PROPERTY_MAJOR_VERSION: | ||
| 468 | - *value = ACLBLASLT_VERSION_MAJOR; | ||
| 469 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 470 | - case ACLBLASLT_PROPERTY_MINOR_VERSION: | ||
| 471 | - *value = ACLBLASLT_VERSION_MINOR; | ||
| 472 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 473 | - case ACLBLASLT_PROPERTY_PATCH_LEVEL: | ||
| 474 | - *value = ACLBLASLT_VERSION_PATCH; | ||
| 475 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 476 | - default: | ||
| 477 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 478 | - } | ||
| 479 | -} | ||
| 480 | - | ||
| 481 | -aclblasStatus_t aclblasLtCreate(aclblasLtHandle_t* lightHandle) | ||
| 482 | -{ | ||
| 483 | - if (lightHandle == nullptr) { | ||
| 484 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 485 | - } | ||
| 486 | - | ||
| 487 | - aclblasLtHandle* h = nullptr; | ||
| 488 | - auto st = AllocHandle(&h); | ||
| 489 | - if (st != ACLBLAS_STATUS_SUCCESS) { | ||
| 490 | - return st; | ||
| 491 | - } | ||
| 492 | - | ||
| 493 | - int32_t deviceId = 0; | ||
| 494 | - aclError aclRet = aclrtGetDevice(&deviceId); | ||
| 495 | - if (aclRet != ACL_SUCCESS) { | ||
| 496 | - delete h; | ||
| 497 | - return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 498 | - } | ||
| 499 | - | ||
| 500 | - aclrtContext currentCtx = nullptr; | ||
| 501 | - aclRet = aclrtGetCurrentContext(¤tCtx); | ||
| 502 | - if (aclRet != ACL_SUCCESS || currentCtx == nullptr) { | ||
| 503 | - delete h; | ||
| 504 | - return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 505 | - } | ||
| 506 | - | ||
| 507 | - h->deviceId = deviceId; | ||
| 508 | - h->context = currentCtx; | ||
| 509 | - h->defaultStream = nullptr; | ||
| 510 | - h->workspaceSize = DEFAULT_WORKSPACE_SIZE; | ||
| 511 | - h->internalWorkspace = std::malloc(h->workspaceSize); | ||
| 512 | - if (h->internalWorkspace == nullptr) { | ||
| 513 | - delete h; | ||
| 514 | - return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 515 | - } | ||
| 516 | - | ||
| 517 | - h->mutex = new (std::nothrow) std::mutex(); | ||
| 518 | - h->algoCache = new (std::nothrow) std::unordered_map<AlgoKey, CacheEntry, AlgoKeyHasher>(); | ||
| 519 | - h->lruList = new (std::nothrow) std::list<AlgoKey>(); | ||
| 520 | - if (h->mutex == nullptr || h->algoCache == nullptr || h->lruList == nullptr) { | ||
| 521 | - delete h->mutex; | ||
| 522 | - delete h->algoCache; | ||
| 523 | - delete h->lruList; | ||
| 524 | - std::free(h->internalWorkspace); | ||
| 525 | - delete h; | ||
| 526 | - return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 527 | - } | ||
| 528 | - | ||
| 529 | - h->npuArch = 2; | ||
| 530 | - h->maxSharedMemory = L1_SIZE; | ||
| 531 | - h->initialized = true; | ||
| 532 | - *lightHandle = reinterpret_cast<aclblasLtHandle_t>(h); | ||
| 533 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 534 | -} | ||
| 535 | - | ||
| 536 | -aclblasStatus_t aclblasLtDestroy(const aclblasLtHandle_t lightHandle) | ||
| 537 | -{ | ||
| 538 | - if (lightHandle == nullptr) { | ||
| 539 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 540 | - } | ||
| 541 | - | ||
| 542 | - auto* h = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 543 | - if (h->magic != ACLBLASLT_HANDLE_MAGIC) { | ||
| 544 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 545 | - } | ||
| 546 | - | ||
| 547 | - h->initialized = false; | ||
| 548 | - delete h->mutex; | ||
| 549 | - delete h->algoCache; | ||
| 550 | - delete h->lruList; | ||
| 551 | - std::free(h->internalWorkspace); | ||
| 552 | - h->mutex = nullptr; | ||
| 553 | - h->algoCache = nullptr; | ||
| 554 | - h->lruList = nullptr; | ||
| 555 | - h->internalWorkspace = nullptr; | ||
| 556 | - h->workspaceSize = 0; | ||
| 557 | - return FreeHandle(h); | ||
| 558 | -} | ||
| 559 | - | ||
| 560 | -aclblasStatus_t aclblasLtMatrixLayoutCreate( | ||
| 561 | - aclblasLtMatrixLayout_t* layout, aclDataType type, uint64_t rows, uint64_t cols, int64_t ld) | ||
| 562 | -{ | ||
| 563 | - // 1. 参数校验（BLAS/cuBLAS 允许 m=0 或 n=0 的空矩阵，仅拒绝非法 ld） | ||
| 564 | - if (layout == nullptr || ld < 0) { | ||
| 565 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 566 | - } | ||
| 567 | - *layout = nullptr; | ||
| 568 | - // 2. 堆上分配胶囊 | ||
| 569 | - auto* capsule = new (std::nothrow) aclblasLtMatrixLayoutOpaque_t(); | ||
| 570 | - if (capsule == nullptr) { | ||
| 571 | - return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 572 | - } | ||
| 573 | - // 3. 栈上创建Impl，初始化后拷贝进胶囊 | ||
| 574 | - aclblasLtMatrixLayoutImpl impl; | ||
| 575 | - impl.magic = ACLBLASLT_LAYOUT_MAGIC; | ||
| 576 | - impl.type = type; | ||
| 577 | - impl.rows = rows; | ||
| 578 | - impl.cols = cols; | ||
| 579 | - impl.ld = (ld == 0) ? static_cast<int64_t>(rows) : ld; | ||
| 580 | - // 4. Impl → 胶囊 | ||
| 581 | - static_assert(sizeof(impl) <= sizeof(*capsule), "aclblasLtMatrixLayoutImpl too large, not fit in capsule!"); | ||
| 582 | - aclblasStatus_t copyStatus = CheckedMemcpyS(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 583 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 584 | - delete capsule; | ||
| 585 | - return copyStatus; | ||
| 586 | - } | ||
| 587 | - if (sizeof(*capsule) > sizeof(impl)) { | ||
| 588 | - copyStatus = CheckedMemsetS( | ||
| 589 | - reinterpret_cast<char*>(capsule) + sizeof(impl), sizeof(*capsule) - sizeof(impl), | ||
| 590 | - sizeof(*capsule) - sizeof(impl)); | ||
| 591 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 592 | - delete capsule; | ||
| 593 | - return copyStatus; | ||
| 594 | - } | ||
| 595 | - } | ||
| 596 | - // 5. 返回胶囊指针 | ||
| 597 | - *layout = capsule; | ||
| 598 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 599 | -} | ||
| 600 | - | ||
| 601 | -aclblasStatus_t aclblasLtMatrixLayoutDestroy(const aclblasLtMatrixLayout_t layout) | ||
| 602 | -{ | ||
| 603 | - if (layout == nullptr) { | ||
| 604 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 605 | - } | ||
| 606 | - | ||
| 607 | - auto* capsule = reinterpret_cast<aclblasLtMatrixLayoutOpaque_t*>(layout); | ||
| 608 | - delete capsule; | ||
| 609 | - | ||
| 610 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 611 | -} | ||
| 612 | - | ||
| 613 | -aclblasStatus_t aclblasLtMatrixLayoutSetAttribute( | ||
| 614 | - aclblasLtMatrixLayout_t layout, aclblasLtMatrixLayoutAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 615 | -{ | ||
| 616 | - if (layout == nullptr || buf == nullptr) { | ||
| 617 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 618 | - } | ||
| 619 | - | ||
| 620 | - // 解包到栈上 | ||
| 621 | - aclblasLtMatrixLayoutImpl impl; | ||
| 622 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), layout, sizeof(impl)); | ||
| 623 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 624 | - return copyStatus; | ||
| 625 | - } | ||
| 626 | - | ||
| 627 | - switch (attr) { | ||
| 628 | - case ACLBLASLT_MATRIX_LAYOUT_TYPE: | ||
| 629 | - if (sizeInBytes != sizeof(impl.type)) { | ||
| 630 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 631 | - } | ||
| 632 | - impl.type = *reinterpret_cast<const aclDataType*>(buf); | ||
| 633 | - break; | ||
| 634 | - | ||
| 635 | - case ACLBLASLT_MATRIX_LAYOUT_ROWS: | ||
| 636 | - if (sizeInBytes != sizeof(impl.rows)) { | ||
| 637 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 638 | - } | ||
| 639 | - impl.rows = *reinterpret_cast<const uint64_t*>(buf); | ||
| 640 | - break; | ||
| 641 | - | ||
| 642 | - case ACLBLASLT_MATRIX_LAYOUT_COLS: | ||
| 643 | - if (sizeInBytes != sizeof(impl.cols)) { | ||
| 644 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 645 | - } | ||
| 646 | - impl.cols = *reinterpret_cast<const uint64_t*>(buf); | ||
| 647 | - break; | ||
| 648 | - | ||
| 649 | - case ACLBLASLT_MATRIX_LAYOUT_LD: | ||
| 650 | - if (sizeInBytes != sizeof(impl.ld)) { | ||
| 651 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 652 | - } | ||
| 653 | - impl.ld = *reinterpret_cast<const int64_t*>(buf); | ||
| 654 | - break; | ||
| 655 | - | ||
| 656 | - case ACLBLASLT_MATRIX_LAYOUT_ORDER: | ||
| 657 | - if (sizeInBytes != sizeof(impl.order)) { | ||
| 658 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 659 | - } | ||
| 660 | - impl.order = *reinterpret_cast<const aclblasLtOrder_t*>(buf); | ||
| 661 | - break; | ||
| 662 | - | ||
| 663 | - case ACLBLASLT_MATRIX_LAYOUT_BATCH_COUNT: | ||
| 664 | - if (sizeInBytes != sizeof(impl.batchCount)) { | ||
| 665 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 666 | - } | ||
| 667 | - impl.batchCount = *reinterpret_cast<const int32_t*>(buf); | ||
| 668 | - break; | ||
| 669 | - | ||
| 670 | - case ACLBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET: | ||
| 671 | - if (sizeInBytes != sizeof(impl.stridedBatchOffset)) { | ||
| 672 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 673 | - } | ||
| 674 | - impl.stridedBatchOffset = *reinterpret_cast<const int64_t*>(buf); | ||
| 675 | - break; | ||
| 676 | - | ||
| 677 | - default: | ||
| 678 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 679 | - } | ||
| 680 | - | ||
| 681 | - // 压缩回堆上 | ||
| 682 | - copyStatus = CheckedMemcpyS(layout, sizeof(*layout), &impl, sizeof(impl)); | ||
| 683 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 684 | - return copyStatus; | ||
| 685 | - } | ||
| 686 | - if (sizeof(*layout) > sizeof(impl)) { | ||
| 687 | - copyStatus = CheckedMemsetS( | ||
| 688 | - reinterpret_cast<char*>(layout) + sizeof(impl), sizeof(*layout) - sizeof(impl), | ||
| 689 | - sizeof(*layout) - sizeof(impl)); | ||
| 690 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 691 | - return copyStatus; | ||
| 692 | - } | ||
| 693 | - } | ||
| 694 | - | ||
| 695 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 696 | -} | ||
| 697 | - | ||
| 698 | -aclblasStatus_t aclblasLtMatrixLayoutGetAttribute( | ||
| 699 | - const aclblasLtMatrixLayout_t layout, aclblasLtMatrixLayoutAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 700 | - size_t* sizeWritten) | ||
| 701 | -{ | ||
| 702 | - if (layout == nullptr || buf == nullptr) { | ||
| 703 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 704 | - } | ||
| 705 | - | ||
| 706 | - aclblasLtMatrixLayoutImpl impl; | ||
| 707 | - static_assert(sizeof(impl) <= sizeof(*layout), "aclblasLtMatrixLayoutImpl too large for capsule"); | ||
| 708 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), layout, sizeof(impl)); | ||
| 709 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 710 | - return copyStatus; | ||
| 711 | - } | ||
| 712 | - | ||
| 713 | - size_t actualSize = 0; | ||
| 714 | - | ||
| 715 | - switch (attr) { | ||
| 716 | - case ACLBLASLT_MATRIX_LAYOUT_TYPE: | ||
| 717 | - actualSize = sizeof(impl.type); | ||
| 718 | - if (sizeInBytes < actualSize) { | ||
| 719 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 720 | - } | ||
| 721 | - *reinterpret_cast<aclDataType*>(buf) = impl.type; | ||
| 722 | - break; | ||
| 723 | - | ||
| 724 | - case ACLBLASLT_MATRIX_LAYOUT_ROWS: | ||
| 725 | - actualSize = sizeof(impl.rows); | ||
| 726 | - if (sizeInBytes < actualSize) { | ||
| 727 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 728 | - } | ||
| 729 | - *reinterpret_cast<uint64_t*>(buf) = impl.rows; | ||
| 730 | - break; | ||
| 731 | - | ||
| 732 | - case ACLBLASLT_MATRIX_LAYOUT_COLS: | ||
| 733 | - actualSize = sizeof(impl.cols); | ||
| 734 | - if (sizeInBytes < actualSize) { | ||
| 735 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 736 | - } | ||
| 737 | - *reinterpret_cast<uint64_t*>(buf) = impl.cols; | ||
| 738 | - break; | ||
| 739 | - | ||
| 740 | - case ACLBLASLT_MATRIX_LAYOUT_LD: | ||
| 741 | - actualSize = sizeof(impl.ld); | ||
| 742 | - if (sizeInBytes < actualSize) { | ||
| 743 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 744 | - } | ||
| 745 | - *reinterpret_cast<int64_t*>(buf) = impl.ld; | ||
| 746 | - break; | ||
| 747 | - | ||
| 748 | - case ACLBLASLT_MATRIX_LAYOUT_ORDER: | ||
| 749 | - actualSize = sizeof(impl.order); | ||
| 750 | - if (sizeInBytes < actualSize) { | ||
| 751 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 752 | - } | ||
| 753 | - *reinterpret_cast<aclblasLtOrder_t*>(buf) = impl.order; | ||
| 754 | - break; | ||
| 755 | - | ||
| 756 | - case ACLBLASLT_MATRIX_LAYOUT_BATCH_COUNT: | ||
| 757 | - actualSize = sizeof(impl.batchCount); | ||
| 758 | - if (sizeInBytes < actualSize) { | ||
| 759 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 760 | - } | ||
| 761 | - *reinterpret_cast<int32_t*>(buf) = impl.batchCount; | ||
| 762 | - break; | ||
| 763 | - | ||
| 764 | - case ACLBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET: | ||
| 765 | - actualSize = sizeof(impl.stridedBatchOffset); | ||
| 766 | - if (sizeInBytes < actualSize) { | ||
| 767 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 768 | - } | ||
| 769 | - *reinterpret_cast<int64_t*>(buf) = impl.stridedBatchOffset; | ||
| 770 | - break; | ||
| 771 | - | ||
| 772 | - default: | ||
| 773 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 774 | - } | ||
| 775 | - | ||
| 776 | - if (sizeWritten != nullptr) { | ||
| 777 | - *sizeWritten = actualSize; | ||
| 778 | - } | ||
| 779 | - | ||
| 780 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 781 | -} | ||
| 782 | - | ||
| 783 | -aclblasStatus_t aclblasLtMatmulDescCreate( | ||
| 784 | - aclblasLtMatmulDesc_t* desc, aclblasComputeType_t computeType, aclDataType scaleType) | ||
| 785 | -{ | ||
| 786 | - if (desc == nullptr) { | ||
| 787 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 788 | - } | ||
| 789 | - *desc = nullptr; | ||
| 790 | - | ||
| 791 | - auto* capsule = new (std::nothrow) aclblasLtMatmulDescOpaque_t(); | ||
| 792 | - if (capsule == nullptr) { | ||
| 793 | - return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 794 | - } | ||
| 795 | - | ||
| 796 | - aclblasLtMatmulDescImpl impl; | ||
| 797 | - impl.magic = ACLBLASLT_DESC_MAGIC; | ||
| 798 | - impl.computeType = computeType; | ||
| 799 | - impl.scaleType = scaleType; | ||
| 800 | - | ||
| 801 | - static_assert(sizeof(impl) <= sizeof(*capsule), "aclblasLtMatmulDescImpl too large, not fit in capsule!"); | ||
| 802 | - aclblasStatus_t copyStatus = CheckedMemcpyS(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 803 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 804 | - delete capsule; | ||
| 805 | - return copyStatus; | ||
| 806 | - } | ||
| 807 | - if (sizeof(*capsule) > sizeof(impl)) { | ||
| 808 | - copyStatus = CheckedMemsetS( | ||
| 809 | - reinterpret_cast<char*>(capsule) + sizeof(impl), sizeof(*capsule) - sizeof(impl), | ||
| 810 | - sizeof(*capsule) - sizeof(impl)); | ||
| 811 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 812 | - delete capsule; | ||
| 813 | - return copyStatus; | ||
| 814 | - } | ||
| 815 | - } | ||
| 816 | - | ||
| 817 | - *desc = capsule; | ||
| 818 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 819 | -} | ||
| 820 | - | ||
| 821 | -aclblasStatus_t aclblasLtMatmulDescDestroy(const aclblasLtMatmulDesc_t desc) | ||
| 822 | -{ | ||
| 823 | - if (desc == nullptr) { | ||
| 824 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 825 | - } | ||
| 826 | - | ||
| 827 | - auto* capsule = reinterpret_cast<aclblasLtMatmulDescOpaque_t*>(desc); | ||
| 828 | - delete capsule; | ||
| 829 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 830 | -} | ||
| 831 | - | ||
| 832 | -aclblasStatus_t aclblasLtMatmulDescSetAttribute( | ||
| 833 | - aclblasLtMatmulDesc_t desc, aclblasLtMatmulDescAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 834 | -{ | ||
| 835 | - if (desc == nullptr || buf == nullptr) { | ||
| 836 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 837 | - } | ||
| 838 | - | ||
| 839 | - aclblasLtMatmulDescImpl impl; | ||
| 840 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), desc, sizeof(impl)); | ||
| 841 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 842 | - return copyStatus; | ||
| 843 | - } | ||
| 844 | - | ||
| 845 | - switch (attr) { | ||
| 846 | - case ACLBLASLT_MATMUL_DESC_EPILOGUE: { | ||
| 847 | - if (sizeInBytes != sizeof(aclblasLtEpilogue_t) && sizeInBytes != sizeof(uint32_t)) { | ||
| 848 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 849 | - } | ||
| 850 | - uint32_t v = 0; | ||
| 851 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(uint32_t)); | ||
| 852 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 853 | - return copyStatus; | ||
| 854 | - } | ||
| 855 | - impl.epilogue = static_cast<aclblasLtEpilogue_t>(v); | ||
| 856 | - break; | ||
| 857 | - } | ||
| 858 | - case ACLBLASLT_MATMUL_DESC_BIAS_POINTER: { | ||
| 859 | - if (sizeInBytes != kBiasPtrStorageBytes) { | ||
| 860 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 861 | - } | ||
| 862 | - void* biasPtr = nullptr; | ||
| 863 | - copyStatus = CheckedMemcpyS(&biasPtr, kBiasPtrStorageBytes, buf, kBiasPtrStorageBytes); | ||
| 864 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 865 | - return copyStatus; | ||
| 866 | - } | ||
| 867 | - impl.bias = biasPtr; | ||
| 868 | - break; | ||
| 869 | - } | ||
| 870 | - case ACLBLASLT_MATMUL_DESC_TRANSA: { | ||
| 871 | - if (sizeInBytes != sizeof(int32_t)) { | ||
| 872 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 873 | - } | ||
| 874 | - int32_t v = 0; | ||
| 875 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(int32_t)); | ||
| 876 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 877 | - return copyStatus; | ||
| 878 | - } | ||
| 879 | - impl.transA = static_cast<aclblasOperation_t>(v); | ||
| 880 | - break; | ||
| 881 | - } | ||
| 882 | - case ACLBLASLT_MATMUL_DESC_TRANSB: { | ||
| 883 | - if (sizeInBytes != sizeof(int32_t)) { | ||
| 884 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 885 | - } | ||
| 886 | - int32_t v = 0; | ||
| 887 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(int32_t)); | ||
| 888 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 889 | - return copyStatus; | ||
| 890 | - } | ||
| 891 | - impl.transB = static_cast<aclblasOperation_t>(v); | ||
| 892 | - break; | ||
| 893 | - } | ||
| 894 | - case ACLBLASLT_MATMUL_DESC_BIAS_DATA_TYPE: { | ||
| 895 | - if (sizeInBytes != sizeof(int32_t)) { | ||
| 896 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 897 | - } | ||
| 898 | - int32_t v = 0; | ||
| 899 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(int32_t)); | ||
| 900 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 901 | - return copyStatus; | ||
| 902 | - } | ||
| 903 | - impl.biasDataType = static_cast<aclDataType>(v); | ||
| 904 | - break; | ||
| 905 | - } | ||
| 906 | - case ACLBLASLT_MATMUL_DESC_A_SCALE_POINTER: { | ||
| 907 | - if (sizeInBytes != sizeof(void*)) { | ||
| 908 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 909 | - } | ||
| 910 | - void* scalePtr = nullptr; | ||
| 911 | - copyStatus = CheckedMemcpyS(&scalePtr, sizeof(void*), buf, sizeof(void*)); | ||
| 912 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 913 | - return copyStatus; | ||
| 914 | - } | ||
| 915 | - impl.scaleA = scalePtr; | ||
| 916 | - break; | ||
| 917 | - } | ||
| 918 | - case ACLBLASLT_MATMUL_DESC_B_SCALE_POINTER: { | ||
| 919 | - if (sizeInBytes != sizeof(void*)) { | ||
| 920 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 921 | - } | ||
| 922 | - void* scalePtr = nullptr; | ||
| 923 | - copyStatus = CheckedMemcpyS(&scalePtr, sizeof(void*), buf, sizeof(void*)); | ||
| 924 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 925 | - return copyStatus; | ||
| 926 | - } | ||
| 927 | - impl.scaleB = scalePtr; | ||
| 928 | - break; | ||
| 929 | - } | ||
| 930 | - case ACLBLASLT_MATMUL_DESC_A_SCALE_MODE: | ||
| 931 | - case ACLBLASLT_MATMUL_DESC_B_SCALE_MODE: | ||
| 932 | - break; | ||
| 933 | - default: | ||
| 934 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 935 | - } | ||
| 936 | - | ||
| 937 | - copyStatus = CheckedMemcpyS(desc, sizeof(*desc), &impl, sizeof(impl)); | ||
| 938 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 939 | - return copyStatus; | ||
| 940 | - } | ||
| 941 | - if (sizeof(*desc) > sizeof(impl)) { | ||
| 942 | - copyStatus = CheckedMemsetS( | ||
| 943 | - reinterpret_cast<char*>(desc) + sizeof(impl), sizeof(*desc) - sizeof(impl), | ||
| 944 | - sizeof(*desc) - sizeof(impl)); | ||
| 945 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 946 | - return copyStatus; | ||
| 947 | - } | ||
| 948 | - } | ||
| 949 | - | ||
| 950 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 951 | -} | ||
| 952 | - | ||
| 953 | -aclblasStatus_t aclblasLtMatmulDescGetAttribute( | ||
| 954 | - aclblasLtMatmulDesc_t desc, aclblasLtMatmulDescAttribute_t attr, void* buf, size_t sizeInBytes, size_t* sizeWritten) | ||
| 955 | -{ | ||
| 956 | - if (desc == nullptr || buf == nullptr) { | ||
| 957 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 958 | - } | ||
| 959 | - | ||
| 960 | - aclblasLtMatmulDescImpl impl; | ||
| 961 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), desc, sizeof(impl)); | ||
| 962 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 963 | - return copyStatus; | ||
| 964 | - } | ||
| 965 | - | ||
| 966 | - size_t requiredSize = 0; | ||
| 967 | - const void* srcPtr = nullptr; | ||
| 968 | - | ||
| 969 | - switch (attr) { | ||
| 970 | - case ACLBLASLT_MATMUL_DESC_EPILOGUE: | ||
| 971 | - requiredSize = sizeof(impl.epilogue); | ||
| 972 | - srcPtr = &impl.epilogue; | ||
| 973 | - break; | ||
| 974 | - | ||
| 975 | - case ACLBLASLT_MATMUL_DESC_BIAS_POINTER: | ||
| 976 | - requiredSize = kBiasPtrStorageBytes; | ||
| 977 | - srcPtr = &impl.bias; | ||
| 978 | - break; | ||
| 979 | - | ||
| 980 | - case ACLBLASLT_MATMUL_DESC_TRANSA: | ||
| 981 | - requiredSize = sizeof(impl.transA); | ||
| 982 | - srcPtr = &impl.transA; | ||
| 983 | - break; | ||
| 984 | - | ||
| 985 | - case ACLBLASLT_MATMUL_DESC_TRANSB: | ||
| 986 | - requiredSize = sizeof(impl.transB); | ||
| 987 | - srcPtr = &impl.transB; | ||
| 988 | - break; | ||
| 989 | - | ||
| 990 | - case ACLBLASLT_MATMUL_DESC_BIAS_DATA_TYPE: | ||
| 991 | - requiredSize = sizeof(impl.biasDataType); | ||
| 992 | - srcPtr = &impl.biasDataType; | ||
| 993 | - break; | ||
| 994 | - | ||
| 995 | - case ACLBLASLT_MATMUL_DESC_A_SCALE_POINTER: | ||
| 996 | - requiredSize = sizeof(impl.scaleA); | ||
| 997 | - srcPtr = &impl.scaleA; | ||
| 998 | - break; | ||
| 999 | - | ||
| 1000 | - case ACLBLASLT_MATMUL_DESC_B_SCALE_POINTER: | ||
| 1001 | - requiredSize = sizeof(impl.scaleB); | ||
| 1002 | - srcPtr = &impl.scaleB; | ||
| 1003 | - break; | ||
| 1004 | - | ||
| 1005 | - default: | ||
| 1006 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 1007 | - } | ||
| 1008 | - | ||
| 1009 | - // 检查用户缓冲区大小 | ||
| 1010 | - if (sizeInBytes < requiredSize) { | ||
| 1011 | - if (sizeWritten != nullptr) { | ||
| 1012 | - *sizeWritten = requiredSize; | ||
| 1013 | - } | ||
| 1014 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1015 | - } | ||
| 1016 | - | ||
| 1017 | - copyStatus = CheckedMemcpyS(buf, sizeInBytes, srcPtr, requiredSize); | ||
| 1018 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1019 | - return copyStatus; | ||
| 1020 | - } | ||
| 1021 | - | ||
| 1022 | - if (sizeWritten != nullptr) { | ||
| 1023 | - *sizeWritten = requiredSize; | ||
| 1024 | - } | ||
| 1025 | - | ||
| 1026 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1027 | -} | ||
| 1028 | - | ||
| 1029 | -aclblasStatus_t aclblasLtMatmulPreferenceCreate(aclblasLtMatmulPreference_t* pref) | ||
| 1030 | -{ | ||
| 1031 | - if (pref == nullptr) { | ||
| 1032 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1033 | - } | ||
| 1034 | - | ||
| 1035 | - *pref = nullptr; | ||
| 1036 | - auto* capsule = new (std::nothrow) aclblasLtMatmulPreferenceOpaque_t(); | ||
| 1037 | - if (capsule == nullptr) { | ||
| 1038 | - return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 1039 | - } | ||
| 1040 | - aclblasStatus_t copyStatus = CheckedMemsetS(capsule, sizeof(*capsule), sizeof(*capsule)); | ||
| 1041 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1042 | - delete capsule; | ||
| 1043 | - return copyStatus; | ||
| 1044 | - } | ||
| 1045 | - | ||
| 1046 | - aclblasLtMatmulPreferenceImpl impl; | ||
| 1047 | - copyStatus = CheckedMemcpyS(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 1048 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1049 | - delete capsule; | ||
| 1050 | - return copyStatus; | ||
| 1051 | - } | ||
| 1052 | - if (sizeof(*capsule) > sizeof(impl)) { | ||
| 1053 | - copyStatus = CheckedMemsetS( | ||
| 1054 | - reinterpret_cast<char*>(capsule) + sizeof(impl), sizeof(*capsule) - sizeof(impl), | ||
| 1055 | - sizeof(*capsule) - sizeof(impl)); | ||
| 1056 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1057 | - delete capsule; | ||
| 1058 | - return copyStatus; | ||
| 1059 | - } | ||
| 1060 | - } | ||
| 1061 | - | ||
| 1062 | - *pref = capsule; | ||
| 1063 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1064 | -} | ||
| 1065 | - | ||
| 1066 | -aclblasStatus_t aclblasLtMatmulPreferenceDestroy(const aclblasLtMatmulPreference_t pref) | ||
| 1067 | -{ | ||
| 1068 | - if (pref == nullptr) { | ||
| 1069 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1070 | - } | ||
| 1071 | - | ||
| 1072 | - auto* capsule = reinterpret_cast<aclblasLtMatmulPreferenceOpaque_t*>(pref); | ||
| 1073 | - delete capsule; | ||
| 1074 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1075 | -} | ||
| 1076 | - | ||
| 1077 | -aclblasStatus_t aclblasLtMatmulPreferenceSetAttribute( | ||
| 1078 | - aclblasLtMatmulPreference_t pref, aclblasLtMatmulPreferenceAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 1079 | -{ | ||
| 1080 | - if (pref == nullptr || buf == nullptr) { | ||
| 1081 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1082 | - } | ||
| 1083 | - | ||
| 1084 | - aclblasLtMatmulPreferenceImpl impl; | ||
| 1085 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), pref, sizeof(impl)); | ||
| 1086 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1087 | - return copyStatus; | ||
| 1088 | - } | ||
| 1089 | - | ||
| 1090 | - switch (attr) { | ||
| 1091 | - case ACLBLASLT_MATMUL_PREF_SEARCH_MODE: { | ||
| 1092 | - if (sizeInBytes != sizeof(uint32_t)) { | ||
| 1093 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1094 | - } | ||
| 1095 | - uint32_t v = 0; | ||
| 1096 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(v)); | ||
| 1097 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1098 | - return copyStatus; | ||
| 1099 | - } | ||
| 1100 | - // 0=heuristic, 1=exhaustive, 2=fast | ||
| 1101 | - if (v > 2) { | ||
| 1102 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1103 | - } | ||
| 1104 | - impl.searchMode = v; | ||
| 1105 | - break; | ||
| 1106 | - } | ||
| 1107 | - | ||
| 1108 | - case ACLBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES: { | ||
| 1109 | - if (sizeInBytes != sizeof(size_t) && sizeInBytes != sizeof(uint64_t)) { | ||
| 1110 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1111 | - } | ||
| 1112 | - size_t v = 0; | ||
| 1113 | - const size_t copyBytes = std::min(sizeInBytes, sizeof(v)); | ||
| 1114 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, copyBytes); | ||
| 1115 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1116 | - return copyStatus; | ||
| 1117 | - } | ||
| 1118 | - if (v > INT64_MAX) { | ||
| 1119 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1120 | - } | ||
| 1121 | - impl.maxWorkspaceBytes = v; | ||
| 1122 | - break; | ||
| 1123 | - } | ||
| 1124 | - | ||
| 1125 | - default: | ||
| 1126 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 1127 | - } | ||
| 1128 | - | ||
| 1129 | - copyStatus = CheckedMemcpyS(pref, sizeof(*pref), &impl, sizeof(impl)); | ||
| 1130 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1131 | - return copyStatus; | ||
| 1132 | - } | ||
| 1133 | - | ||
| 1134 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1135 | -} | ||
| 1136 | - | ||
| 1137 | -aclblasStatus_t aclblasLtMatmulPreferenceGetAttribute( | ||
| 1138 | - aclblasLtMatmulPreference_t pref, aclblasLtMatmulPreferenceAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 1139 | - size_t* sizeWritten) | ||
| 1140 | -{ | ||
| 1141 | - if (pref == nullptr || buf == nullptr) { | ||
| 1142 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1143 | - } | ||
| 1144 | - | ||
| 1145 | - aclblasLtMatmulPreferenceImpl impl; | ||
| 1146 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), pref, sizeof(impl)); | ||
| 1147 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1148 | - return copyStatus; | ||
| 1149 | - } | ||
| 1150 | - | ||
| 1151 | - size_t requiredSize = 0; | ||
| 1152 | - const void* srcPtr = nullptr; | ||
| 1153 | - | ||
| 1154 | - switch (attr) { | ||
| 1155 | - case ACLBLASLT_MATMUL_PREF_SEARCH_MODE: | ||
| 1156 | - requiredSize = sizeof(impl.searchMode); | ||
| 1157 | - srcPtr = &impl.searchMode; | ||
| 1158 | - break; | ||
| 1159 | - | ||
| 1160 | - case ACLBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES: | ||
| 1161 | - requiredSize = sizeof(impl.maxWorkspaceBytes); | ||
| 1162 | - srcPtr = &impl.maxWorkspaceBytes; | ||
| 1163 | - break; | ||
| 1164 | - | ||
| 1165 | - default: | ||
| 1166 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 1167 | - } | ||
| 1168 | - | ||
| 1169 | - if (sizeInBytes < requiredSize) { | ||
| 1170 | - if (sizeWritten != nullptr) { | ||
| 1171 | - *sizeWritten = requiredSize; | ||
| 1172 | - } | ||
| 1173 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1174 | - } | ||
| 1175 | - | ||
| 1176 | - copyStatus = CheckedMemcpyS(buf, sizeInBytes, srcPtr, requiredSize); | ||
| 1177 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1178 | - return copyStatus; | ||
| 1179 | - } | ||
| 1180 | - | ||
| 1181 | - if (sizeWritten != nullptr) { | ||
| 1182 | - *sizeWritten = requiredSize; | ||
| 1183 | - } | ||
| 1184 | - | ||
| 1185 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1186 | -} | ||
| 1187 | - | ||
| 1188 | -aclblasStatus_t aclblasLtMatmulAlgoGetHeuristic( | ||
| 1189 | - aclblasLtHandle_t lightHandle, aclblasLtMatmulDesc_t computeDesc, aclblasLtMatrixLayout_t Adesc, | ||
| 1190 | - aclblasLtMatrixLayout_t Bdesc, aclblasLtMatrixLayout_t Cdesc, aclblasLtMatrixLayout_t Ddesc, | ||
| 1191 | - aclblasLtMatmulPreference_t preference, int requestedAlgoCount, | ||
| 1192 | - aclblasLtMatmulHeuristicResult_t heuristicResultsArray[], int* returnAlgoCount) | ||
| 1193 | -{ | ||
| 1194 | - // Validate input parameters | ||
| 1195 | - if (returnAlgoCount == nullptr) { | ||
| 1196 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1197 | - } | ||
| 1198 | - *returnAlgoCount = 0; | ||
| 1199 | - | ||
| 1200 | - if (requestedAlgoCount <= 0 || heuristicResultsArray == nullptr) { | ||
| 1201 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1202 | - } | ||
| 1203 | - | ||
| 1204 | - if (lightHandle == nullptr || computeDesc == nullptr) { | ||
| 1205 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1206 | - } | ||
| 1207 | - | ||
| 1208 | - if (Adesc == nullptr || Bdesc == nullptr || Cdesc == nullptr || Ddesc == nullptr) { | ||
| 1209 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1210 | - } | ||
| 1211 | - | ||
| 1212 | - // Get workspace size from preference | ||
| 1213 | - size_t maxWorkspace = 0; | ||
| 1214 | - if (preference != nullptr) { | ||
| 1215 | - auto* p = reinterpret_cast<aclblasLtMatmulPreferenceImpl*>(preference); | ||
| 1216 | - maxWorkspace = p->maxWorkspaceBytes; | ||
| 1217 | - } | ||
| 1218 | - | ||
| 1219 | - // Get matrix dimensions | ||
| 1220 | - auto* A = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Adesc); | ||
| 1221 | - auto* B = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Bdesc); | ||
| 1222 | - auto* D = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Ddesc); | ||
| 1223 | - auto* desc = reinterpret_cast<aclblasLtMatmulDescImpl*>(computeDesc); | ||
| 1224 | - | ||
| 1225 | - // Validate dimensions for GEMM: D = A * B + C | ||
| 1226 | - // A: m x k, B: k x n, C/D: m x n | ||
| 1227 | - uint64_t m = D->rows; | ||
| 1228 | - uint64_t n = D->cols; | ||
| 1229 | - uint64_t k = (desc->transA == ACLBLAS_OP_N) ? A->cols : A->rows; | ||
| 1230 | - | ||
| 1231 | - // Basic validation | ||
| 1232 | - if (!CheckComputeTypeCompatibility(desc->computeType, A->type, B->type)) { | ||
| 1233 | - heuristicResultsArray[0].state = ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1234 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1235 | - } | ||
| 1236 | - | ||
| 1237 | - // Fill heuristic result | ||
| 1238 | - heuristicResultsArray[0].algo.max_workspace_bytes = maxWorkspace; | ||
| 1239 | - heuristicResultsArray[0].workspaceSize = maxWorkspace; | ||
| 1240 | - heuristicResultsArray[0].state = ACLBLAS_STATUS_SUCCESS; | ||
| 1241 | - heuristicResultsArray[0].wavesCount = 1.0f; | ||
| 1242 | - if (CheckedMemsetS( | ||
| 1243 | - heuristicResultsArray[0].reserved, sizeof(heuristicResultsArray[0].reserved), | ||
| 1244 | - sizeof(heuristicResultsArray[0].reserved)) != ACLBLAS_STATUS_SUCCESS) { | ||
| 1245 | - return ACLBLAS_STATUS_INTERNAL_ERROR; | ||
| 1246 | - } | ||
| 1247 | - | ||
| 1248 | - *returnAlgoCount = 1; | ||
| 1249 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1250 | -} | ||
| 1251 | - | ||
| 1252 | -aclblasStatus_t aclblasLtMatmul( | ||
| 1253 | - aclblasLtHandle_t lightHandle, aclblasLtMatmulDesc_t computeDesc, const void* alpha, const void* A, | ||
| 1254 | - aclblasLtMatrixLayout_t Adesc, const void* B, aclblasLtMatrixLayout_t Bdesc, const void* beta, const void* C, | ||
| 1255 | - aclblasLtMatrixLayout_t Cdesc, void* D, aclblasLtMatrixLayout_t Ddesc, const aclblasLtMatmulAlgo_t* algo, | ||
| 1256 | - void* workspace, size_t workspaceSizeInBytes, aclrtStream stream) | ||
| 1257 | -{ | ||
| 1258 | - // Validate lightHandle | ||
| 1259 | - if (lightHandle == nullptr) { | ||
| 1260 | - return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 1261 | - } | ||
| 1262 | - | ||
| 1263 | - // Validate descriptors | ||
| 1264 | - if (computeDesc == nullptr || Adesc == nullptr || Bdesc == nullptr || Cdesc == nullptr || Ddesc == nullptr) { | ||
| 1265 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1266 | - } | ||
| 1267 | - | ||
| 1268 | - // Validate pointers | ||
| 1269 | - if (alpha == nullptr || beta == nullptr) { | ||
| 1270 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1271 | - } | ||
| 1272 | - | ||
| 1273 | - // Get layout info | ||
| 1274 | - auto* ALayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Adesc); | ||
| 1275 | - auto* BLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Bdesc); | ||
| 1276 | - auto* CLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Cdesc); | ||
| 1277 | - auto* DLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Ddesc); | ||
| 1278 | - auto* desc = reinterpret_cast<aclblasLtMatmulDescImpl*>(computeDesc); | ||
| 1279 | - | ||
| 1280 | - // Get dimensions | ||
| 1281 | - uint64_t m = DLayout->rows; | ||
| 1282 | - uint64_t n = DLayout->cols; | ||
| 1283 | - uint64_t k = 0; | ||
| 1284 | - | ||
| 1285 | - // Determine k based on transpose operations | ||
| 1286 | - if (desc->transA == ACLBLAS_OP_N) { | ||
| 1287 | - k = ALayout->cols; | ||
| 1288 | - } else { | ||
| 1289 | - k = ALayout->rows; | ||
| 1290 | - } | ||
| 1291 | - | ||
| 1292 | - // BLAS/cuBLAS convention: m=0 or n=0 is a no-op, succeed without touching matrices. | ||
| 1293 | - if (m == 0U || n == 0U) { | ||
| 1294 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1295 | - } | ||
| 1296 | - | ||
| 1297 | - if (A == nullptr || B == nullptr || D == nullptr) { | ||
| 1298 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1299 | - } | ||
| 1300 | - | ||
| 1301 | - // Validate workspace alignment (must be 16B aligned) | ||
| 1302 | - if (workspace != nullptr && (reinterpret_cast<uintptr_t>(workspace) & 0xF) != 0) { | ||
| 1303 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1304 | - } | ||
| 1305 | - | ||
| 1306 | - // Validate workspace size | ||
| 1307 | - if (algo != nullptr && workspaceSizeInBytes < algo->max_workspace_bytes) { | ||
| 1308 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1309 | - } | ||
| 1310 | - | ||
| 1311 | - // Actual GEMM implementation using ltmatmul routing | ||
| 1312 | - auto* handleImpl = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 1313 | - int32_t deviceId = handleImpl->deviceId; | ||
| 1314 | - int64_t cubeCoreNum = 0; | ||
| 1315 | - aclError aclRet = aclrtGetDeviceInfo(deviceId, ACL_DEV_ATTR_CUBE_CORE_NUM, &cubeCoreNum); | ||
| 1316 | - if (aclRet != ACL_SUCCESS || cubeCoreNum <= 0) { | ||
| 1317 | - cubeCoreNum = 8; // Fallback minimum | ||
| 1318 | - } | ||
| 1319 | - uint32_t numBlocks = static_cast<uint32_t>(cubeCoreNum); | ||
| 1320 | - | ||
| 1321 | - float alphaValue = *(reinterpret_cast<const float*>(alpha)); | ||
| 1322 | - float betaValue = *(reinterpret_cast<const float*>(beta)); | ||
| 1323 | - bool needEpilogue = (alphaValue != 1.0f || betaValue != 0.0f); | ||
| 1324 | - bool cOverlap = (C == D) && needEpilogue; | ||
| 1325 | - void* dRawAddr = needEpilogue && cOverlap ? workspace : D; | ||
| 1326 | - | ||
| 1327 | - aclDataType dtypeA = ALayout->type; | ||
| 1328 | - aclDataType dtypeB = BLayout->type; | ||
| 1329 | - aclDataType dtypeD = DLayout->type; | ||
| 1330 | - bool transA = (desc->transA != ACLBLAS_OP_N); | ||
| 1331 | - bool transB = (desc->transB != ACLBLAS_OP_N); | ||
| 1332 | - | ||
| 1333 | - if ((IsMxfp8Type(dtypeA) || IsMxfp4Type(dtypeA)) && (k % 32 != 0)) { | ||
| 1334 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1335 | - } | ||
| 1336 | - | ||
| 1337 | - // ===== Step 1: MMAD Kernel ===== | ||
| 1338 | - if (dtypeA == ACL_FLOAT && dtypeB == ACL_FLOAT) { | ||
| 1339 | - MatmulFp32TilingData fp32Tiling; | ||
| 1340 | - matmul_fp32_get_tiling( | ||
| 1341 | - m, n, k, transA, transB, static_cast<uint32_t>(ALayout->ld), static_cast<uint32_t>(BLayout->ld), numBlocks, | ||
| 1342 | - fp32Tiling); | ||
| 1343 | - matmul_fp32_kernel_do( | ||
| 1344 | - static_cast<uint8_t*>(const_cast<void*>(A)), | ||
| 1345 | - static_cast<uint8_t*>(const_cast<void*>(B)), | ||
| 1346 | - static_cast<uint8_t*>(dRawAddr), | ||
| 1347 | - fp32Tiling, numBlocks, stream); | ||
| 1348 | - } else if (IsMxfp8Type(dtypeA) && IsMxfp8Type(dtypeB)) { | ||
| 1349 | - void* scaleA = const_cast<void*>(desc->scaleA); | ||
| 1350 | - void* scaleB = const_cast<void*>(desc->scaleB); | ||
| 1351 | - if (scaleA == nullptr || scaleB == nullptr) { | ||
| 1352 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1353 | - } | ||
| 1354 | - QuantMatmulTilingData mxfp8Tiling; | ||
| 1355 | - matmul_mxfp8_get_tiling(m, n, k, transA, transB, numBlocks, mxfp8Tiling); | ||
| 1356 | - matmul_mxfp8_kernel_do( | ||
| 1357 | - static_cast<uint8_t*>(const_cast<void*>(A)), | ||
| 1358 | - static_cast<uint8_t*>(const_cast<void*>(B)), | ||
| 1359 | - static_cast<uint8_t*>(scaleA), | ||
| 1360 | - static_cast<uint8_t*>(scaleB), | ||
| 1361 | - static_cast<uint8_t*>(dRawAddr), | ||
| 1362 | - mxfp8Tiling, transA, transB, stream); | ||
| 1363 | - } else if (IsMxfp4Type(dtypeA) && IsMxfp4Type(dtypeB)) { | ||
| 1364 | - void* scaleA = const_cast<void*>(desc->scaleA); | ||
| 1365 | - void* scaleB = const_cast<void*>(desc->scaleB); | ||
| 1366 | - if (scaleA == nullptr || scaleB == nullptr) { | ||
| 1367 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1368 | - } | ||
| 1369 | - QuantMatmulTilingData mxfp4Tiling; | ||
| 1370 | - matmul_mxfp4_get_tiling(m, n, k, transA, transB, numBlocks, mxfp4Tiling); | ||
| 1371 | - ltmatmul_mxfp4_kernel_do( | ||
| 1372 | - static_cast<uint8_t*>(const_cast<void*>(A)), | ||
| 1373 | - static_cast<uint8_t*>(const_cast<void*>(B)), | ||
| 1374 | - static_cast<uint8_t*>(scaleA), | ||
| 1375 | - static_cast<uint8_t*>(scaleB), | ||
| 1376 | - static_cast<uint8_t*>(dRawAddr), | ||
| 1377 | - mxfp4Tiling, dtypeA, dtypeB, dtypeD, transA, transB, stream); | ||
| 1378 | - } else { | ||
| 1379 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 1380 | - } | ||
| 1381 | - | ||
| 1382 | - // ===== Step 2: Epilogue alpha*D_raw + beta*C ===== | ||
| 1383 | - if (needEpilogue) { | ||
| 1384 | - if (betaValue != 0.0f && C == nullptr) { | ||
| 1385 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1386 | - } | ||
| 1387 | - | ||
| 1388 | - aclDataType dtypeC = CLayout->type; | ||
| 1389 | - aclDataType dtypeDRaw = | ||
| 1390 | - (dtypeA == ACL_FLOAT && dtypeB == ACL_FLOAT) ? ACL_FLOAT : dtypeD; | ||
| 1391 | - const uint32_t ldc = static_cast<uint32_t>(CLayout->ld > 0 ? CLayout->ld : n); | ||
| 1392 | - const uint32_t ldd = static_cast<uint32_t>(DLayout->ld > 0 ? DLayout->ld : n); | ||
| 1393 | - const uint32_t lddRaw = (dRawAddr == D) ? ldd : static_cast<uint32_t>(n); | ||
| 1394 | - | ||
| 1395 | - if (cOverlap) { | ||
| 1396 | - const size_t dRawElemSize = (dtypeDRaw == ACL_BF16) ? sizeof(uint16_t) : sizeof(float); | ||
| 1397 | - const size_t requiredWorkspace = static_cast<size_t>(m) * static_cast<size_t>(n) * dRawElemSize; | ||
| 1398 | - if (workspace == nullptr || workspaceSizeInBytes < requiredWorkspace) { | ||
| 1399 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1400 | - } | ||
| 1401 | - } | ||
| 1402 | - | ||
| 1403 | - epilogue_alpha_beta_do( | ||
| 1404 | - static_cast<uint8_t*>(dRawAddr), | ||
| 1405 | - betaValue != 0.0f ? static_cast<uint8_t*>(const_cast<void*>(C)) : nullptr, | ||
| 1406 | - static_cast<uint8_t*>(D), | ||
| 1407 | - static_cast<uint32_t>(m), static_cast<uint32_t>(n), | ||
| 1408 | - ldc, ldd, lddRaw, | ||
| 1409 | - alphaValue, betaValue, | ||
| 1410 | - dtypeC, dtypeDRaw, dtypeD, | ||
| 1411 | - stream); | ||
| 1412 | - } | ||
| 1413 | - | ||
| 1414 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1415 | -} | ||
| 1416 | - | ||
| 1417 | -aclblasStatus_t aclblasLtMatrixTransformDescCreate( | ||
| 1418 | - aclblasLtMatrixTransformDesc_t* transformDesc, aclDataType scaleType) | ||
| 1419 | -{ | ||
| 1420 | - if (transformDesc == nullptr) { | ||
| 1421 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1422 | - } | ||
| 1423 | - *transformDesc = nullptr; | ||
| 1424 | - | ||
| 1425 | - auto* capsule = new (std::nothrow) aclblasLtMatrixTransformDescOpaque_t(); | ||
| 1426 | - if (capsule == nullptr) { | ||
| 1427 | - return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 1428 | - } | ||
| 1429 | - | ||
| 1430 | - aclblasLtMatrixTransformDescImpl impl; | ||
| 1431 | - impl.magic = ACLBLASLT_TRANSFORM_DESC_MAGIC; | ||
| 1432 | - impl.scaleType = scaleType; | ||
| 1433 | - | ||
| 1434 | - aclblasStatus_t copyStatus = MatPackTransformImpl(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 1435 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1436 | - delete capsule; | ||
| 1437 | - return copyStatus; | ||
| 1438 | - } | ||
| 1439 | - | ||
| 1440 | - *transformDesc = capsule; | ||
| 1441 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1442 | -} | ||
| 1443 | - | ||
| 1444 | -aclblasStatus_t aclblasLtMatrixTransformDescDestroy(const aclblasLtMatrixTransformDesc_t transformDesc) | ||
| 1445 | -{ | ||
| 1446 | - if (transformDesc == nullptr) { | ||
| 1447 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1448 | - } | ||
| 1449 | - auto* capsule = reinterpret_cast<aclblasLtMatrixTransformDescOpaque_t*>(transformDesc); | ||
| 1450 | - delete capsule; | ||
| 1451 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1452 | -} | ||
| 1453 | - | ||
| 1454 | -aclblasStatus_t aclblasLtMatrixTransformDescSetAttribute( | ||
| 1455 | - aclblasLtMatrixTransformDesc_t transformDesc, aclblasLtMatrixTransformDescAttribute_t attr, const void* buf, | ||
| 1456 | - size_t sizeInBytes) | ||
| 1457 | -{ | ||
| 1458 | - if (transformDesc == nullptr || buf == nullptr) { | ||
| 1459 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1460 | - } | ||
| 1461 | - | ||
| 1462 | - aclblasLtMatrixTransformDescImpl impl; | ||
| 1463 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), transformDesc, sizeof(impl)); | ||
| 1464 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1465 | - return copyStatus; | ||
| 1466 | - } | ||
| 1467 | - if (impl.magic != ACLBLASLT_TRANSFORM_DESC_MAGIC) { | ||
| 1468 | - return ACLBLAS_STATUS_INVALID_VALUE; // corrupted / foreign descriptor | ||
| 1469 | - } | ||
| 1470 | - if (sizeInBytes != sizeof(int32_t)) { | ||
| 1471 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1472 | - } | ||
| 1473 | - int32_t v = 0; | ||
| 1474 | - copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(int32_t)); | ||
| 1475 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1476 | - return copyStatus; | ||
| 1477 | - } | ||
| 1478 | - | ||
| 1479 | - switch (attr) { | ||
| 1480 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_SCALE_TYPE: | ||
| 1481 | - impl.scaleType = static_cast<aclDataType>(v); | ||
| 1482 | - break; | ||
| 1483 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_POINTER_MODE: | ||
| 1484 | - impl.pointerMode = v; | ||
| 1485 | - break; | ||
| 1486 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSA: | ||
| 1487 | - impl.transA = static_cast<aclblasOperation_t>(v); | ||
| 1488 | - break; | ||
| 1489 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSB: | ||
| 1490 | - impl.transB = static_cast<aclblasOperation_t>(v); | ||
| 1491 | - break; | ||
| 1492 | - default: | ||
| 1493 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 1494 | - } | ||
| 1495 | - | ||
| 1496 | - return MatPackTransformImpl(transformDesc, sizeof(*transformDesc), &impl, sizeof(impl)); | ||
| 1497 | -} | ||
| 1498 | - | ||
| 1499 | -aclblasStatus_t aclblasLtMatrixTransformDescGetAttribute( | ||
| 1500 | - aclblasLtMatrixTransformDesc_t transformDesc, aclblasLtMatrixTransformDescAttribute_t attr, void* buf, | ||
| 1501 | - size_t sizeInBytes, size_t* sizeWritten) | ||
| 1502 | -{ | ||
| 1503 | - if (transformDesc == nullptr || buf == nullptr) { | ||
| 1504 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1505 | - } | ||
| 1506 | - | ||
| 1507 | - aclblasLtMatrixTransformDescImpl impl; | ||
| 1508 | - aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), transformDesc, sizeof(impl)); | ||
| 1509 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1510 | - return copyStatus; | ||
| 1511 | - } | ||
| 1512 | - if (impl.magic != ACLBLASLT_TRANSFORM_DESC_MAGIC) { | ||
| 1513 | - return ACLBLAS_STATUS_INVALID_VALUE; // corrupted / foreign descriptor | ||
| 1514 | - } | ||
| 1515 | - | ||
| 1516 | - const void* srcPtr = nullptr; | ||
| 1517 | - switch (attr) { | ||
| 1518 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_SCALE_TYPE: | ||
| 1519 | - srcPtr = &impl.scaleType; | ||
| 1520 | - break; | ||
| 1521 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_POINTER_MODE: | ||
| 1522 | - srcPtr = &impl.pointerMode; | ||
| 1523 | - break; | ||
| 1524 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSA: | ||
| 1525 | - srcPtr = &impl.transA; | ||
| 1526 | - break; | ||
| 1527 | - case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSB: | ||
| 1528 | - srcPtr = &impl.transB; | ||
| 1529 | - break; | ||
| 1530 | - default: | ||
| 1531 | - return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 1532 | - } | ||
| 1533 | - | ||
| 1534 | - const size_t requiredSize = sizeof(int32_t); | ||
| 1535 | - if (sizeInBytes < requiredSize) { | ||
| 1536 | - if (sizeWritten != nullptr) { | ||
| 1537 | - *sizeWritten = requiredSize; | ||
| 1538 | - } | ||
| 1539 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1540 | - } | ||
| 1541 | - copyStatus = CheckedMemcpyS(buf, sizeInBytes, srcPtr, requiredSize); | ||
| 1542 | - if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 1543 | - return copyStatus; | ||
| 1544 | - } | ||
| 1545 | - if (sizeWritten != nullptr) { | ||
| 1546 | - *sizeWritten = requiredSize; | ||
| 1547 | - } | ||
| 1548 | - return ACLBLAS_STATUS_SUCCESS; | ||
| 1549 | -} | ||
| 1550 | - | ||
| 1551 | -aclblasStatus_t aclblasLtMatrixTransform( | ||
| 1552 | - aclblasLtHandle_t lightHandle, aclblasLtMatrixTransformDesc_t transformDesc, const void* alpha, const void* A, | ||
| 1553 | - aclblasLtMatrixLayout_t Adesc, const void* beta, const void* B, aclblasLtMatrixLayout_t Bdesc, void* C, | ||
| 1554 | - aclblasLtMatrixLayout_t Cdesc, aclrtStream stream) | ||
| 1555 | -{ | ||
| 1556 | - if (lightHandle == nullptr) { | ||
| 1557 | - return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 1558 | - } | ||
| 1559 | - if (transformDesc == nullptr || Adesc == nullptr || Cdesc == nullptr) { | ||
| 1560 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1561 | - } | ||
| 1562 | - if (alpha == nullptr) { | ||
| 1563 | - return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 1564 | - } | ||
| 1565 | - | ||
| 1566 | - auto* desc = reinterpret_cast<aclblasLtMatrixTransformDescImpl*>(transformDesc); | ||
| 1567 | - auto* ALayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Adesc); | ||
| 1568 | - auto* CLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Cdesc); | ||
| 1569 | - if (desc->magic != ACLBLASLT_TRANSFORM_DESC_MAGIC || ALayout->magic != ACLBLASLT_LAYOUT_MAGIC || | ||
| 1570 | - CLayout->magic != ACLBLASLT_LAYOUT_MAGIC) { | ||
| 1571 | - return ACLBLAS_STATUS_INVALID_VALUE; // corrupted / foreign descriptor or layout | ||
| 1572 | - } | ||
| 1573 | - | ||
| 1574 | - auto* handleImpl = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 1575 | - | ||
| 1576 | - const uint64_t rows = CLayout->rows; | ||
| 1577 | - const uint64_t cols = CLayout->cols; | ||
| 1578 | - if (rows == 0U || cols == 0U) { | ||
| 1579 | - return ACLBLAS_STATUS_SUCCESS; // empty matrix, no-op (checked before B presence) | ||
| 1580 | - } | ||
| 1581 | - | ||
| 1582 | - const MatTransformLayout aPacked = MatPackTransformLayout(ALayout); | ||
| 1583 | - const MatTransformLayout cPacked = MatPackTransformLayout(CLayout); | ||
| 1584 | - auto* BLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Bdesc); | ||
| 1585 | - const MatTransformLayout bPacked = (BLayout != nullptr) ? MatPackTransformLayout(BLayout) : MatTransformLayout{}; | ||
| 1586 | - const bool bLayoutValid = (BLayout != nullptr) && BLayout->magic == ACLBLASLT_LAYOUT_MAGIC; | ||
| 1587 | - | ||
| 1588 | - return MatTransformLaunch( | ||
| 1589 | - handleImpl->deviceId, desc, alpha, A, &aPacked, beta, B, (BLayout != nullptr) ? &bPacked : nullptr, | ||
| 1590 | - bLayoutValid, C, &cPacked, rows, cols, stream); | ||
| 1591 | -} | ||
| 1592 | - | ||
| 1593 | -} // extern "C" | ||
| @@ -0,0 +1,110 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_handle.cpp | ||
| 13 | + * \brief Public C API: handle lifecycle (create / destroy). | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | + | ||
| 26 | + | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +extern "C" { | ||
| 30 | + | ||
| 31 | +aclblasStatus_t aclblasLtCreate(aclblasLtHandle_t* lightHandle) | ||
| 32 | +{ | ||
| 33 | + if (lightHandle == nullptr) { | ||
| 34 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 35 | + } | ||
| 36 | + | ||
| 37 | + aclblasLtHandle* h = nullptr; | ||
| 38 | + auto st = AllocHandle(&h); | ||
| 39 | + if (st != ACLBLAS_STATUS_SUCCESS) { | ||
| 40 | + return st; | ||
| 41 | + } | ||
| 42 | + | ||
| 43 | + int32_t deviceId = 0; | ||
| 44 | + aclError aclRet = aclrtGetDevice(&deviceId); | ||
| 45 | + if (aclRet != ACL_SUCCESS) { | ||
| 46 | + delete h; | ||
| 47 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 48 | + } | ||
| 49 | + | ||
| 50 | + aclrtContext currentCtx = nullptr; | ||
| 51 | + aclRet = aclrtGetCurrentContext(¤tCtx); | ||
| 52 | + if (aclRet != ACL_SUCCESS || currentCtx == nullptr) { | ||
| 53 | + delete h; | ||
| 54 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 55 | + } | ||
| 56 | + | ||
| 57 | + h->deviceId = deviceId; | ||
| 58 | + h->context = currentCtx; | ||
| 59 | + h->defaultStream = nullptr; | ||
| 60 | + h->workspaceSize = DEFAULT_WORKSPACE_SIZE; | ||
| 61 | + h->internalWorkspace = std::malloc(h->workspaceSize); | ||
| 62 | + if (h->internalWorkspace == nullptr) { | ||
| 63 | + delete h; | ||
| 64 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 65 | + } | ||
| 66 | + | ||
| 67 | + h->mutex = new (std::nothrow) std::mutex(); | ||
| 68 | + h->algoCache = new (std::nothrow) std::unordered_map<AlgoKey, CacheEntry, AlgoKeyHasher>(); | ||
| 69 | + h->lruList = new (std::nothrow) std::list<AlgoKey>(); | ||
| 70 | + if (h->mutex == nullptr || h->algoCache == nullptr || h->lruList == nullptr) { | ||
| 71 | + delete h->mutex; | ||
| 72 | + delete h->algoCache; | ||
| 73 | + delete h->lruList; | ||
| 74 | + std::free(h->internalWorkspace); | ||
| 75 | + delete h; | ||
| 76 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 77 | + } | ||
| 78 | + | ||
| 79 | + h->npuArch = 2; | ||
| 80 | + h->maxSharedMemory = L1_SIZE; | ||
| 81 | + h->initialized = true; | ||
| 82 | + *lightHandle = reinterpret_cast<aclblasLtHandle_t>(h); | ||
| 83 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 84 | +} | ||
| 85 | + | ||
| 86 | +aclblasStatus_t aclblasLtDestroy(const aclblasLtHandle_t lightHandle) | ||
| 87 | +{ | ||
| 88 | + if (lightHandle == nullptr) { | ||
| 89 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 90 | + } | ||
| 91 | + | ||
| 92 | + auto* h = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 93 | + if (h->magic != ACLBLASLT_HANDLE_MAGIC) { | ||
| 94 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 95 | + } | ||
| 96 | + | ||
| 97 | + h->initialized = false; | ||
| 98 | + delete h->mutex; | ||
| 99 | + delete h->algoCache; | ||
| 100 | + delete h->lruList; | ||
| 101 | + std::free(h->internalWorkspace); | ||
| 102 | + h->mutex = nullptr; | ||
| 103 | + h->algoCache = nullptr; | ||
| 104 | + h->lruList = nullptr; | ||
| 105 | + h->internalWorkspace = nullptr; | ||
| 106 | + h->workspaceSize = 0; | ||
| 107 | + return FreeHandle(h); | ||
| 108 | +} | ||
| 109 | + | ||
| 110 | +} // extern "C" | ||
| @@ -0,0 +1,195 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_layout.cpp | ||
| 13 | + * \brief Public C API: matrix layout descriptor CRUD. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +namespace { | ||
| 25 | + | ||
| 26 | +// Validate that the caller buffer is exactly sizeof(T), then load it into field. Collapsing the | ||
| 27 | +// per-attribute size check into one helper keeps the Set/Get switches free of nested branches. | ||
| 28 | +template <typename T> | ||
| 29 | +inline aclblasStatus_t LoadLayoutField(const void* buf, size_t sizeInBytes, T& field) | ||
| 30 | +{ | ||
| 31 | + if (sizeInBytes != sizeof(T)) { | ||
| 32 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 33 | + } | ||
| 34 | + field = *reinterpret_cast<const T*>(buf); | ||
| 35 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 36 | +} | ||
| 37 | + | ||
| 38 | +template <typename T> | ||
| 39 | +inline aclblasStatus_t StoreLayoutField(void* buf, size_t sizeInBytes, const T& field, size_t& actualSize) | ||
| 40 | +{ | ||
| 41 | + actualSize = sizeof(T); | ||
| 42 | + if (sizeInBytes < actualSize) { | ||
| 43 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 44 | + } | ||
| 45 | + *reinterpret_cast<T*>(buf) = field; | ||
| 46 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 47 | +} | ||
| 48 | + | ||
| 49 | +aclblasStatus_t ApplyLayoutSetAttr( | ||
| 50 | + aclblasLtMatrixLayoutImpl& impl, aclblasLtMatrixLayoutAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 51 | +{ | ||
| 52 | + switch (attr) { | ||
| 53 | + case ACLBLASLT_MATRIX_LAYOUT_TYPE: | ||
| 54 | + return LoadLayoutField(buf, sizeInBytes, impl.type); | ||
| 55 | + case ACLBLASLT_MATRIX_LAYOUT_ROWS: | ||
| 56 | + return LoadLayoutField(buf, sizeInBytes, impl.rows); | ||
| 57 | + case ACLBLASLT_MATRIX_LAYOUT_COLS: | ||
| 58 | + return LoadLayoutField(buf, sizeInBytes, impl.cols); | ||
| 59 | + case ACLBLASLT_MATRIX_LAYOUT_LD: | ||
| 60 | + return LoadLayoutField(buf, sizeInBytes, impl.ld); | ||
| 61 | + case ACLBLASLT_MATRIX_LAYOUT_ORDER: | ||
| 62 | + return LoadLayoutField(buf, sizeInBytes, impl.order); | ||
| 63 | + case ACLBLASLT_MATRIX_LAYOUT_BATCH_COUNT: | ||
| 64 | + return LoadLayoutField(buf, sizeInBytes, impl.batchCount); | ||
| 65 | + case ACLBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET: | ||
| 66 | + return LoadLayoutField(buf, sizeInBytes, impl.stridedBatchOffset); | ||
| 67 | + default: | ||
| 68 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 69 | + } | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +aclblasStatus_t ReadLayoutGetAttr( | ||
| 73 | + const aclblasLtMatrixLayoutImpl& impl, aclblasLtMatrixLayoutAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 74 | + size_t& actualSize) | ||
| 75 | +{ | ||
| 76 | + switch (attr) { | ||
| 77 | + case ACLBLASLT_MATRIX_LAYOUT_TYPE: | ||
| 78 | + return StoreLayoutField(buf, sizeInBytes, impl.type, actualSize); | ||
| 79 | + case ACLBLASLT_MATRIX_LAYOUT_ROWS: | ||
| 80 | + return StoreLayoutField(buf, sizeInBytes, impl.rows, actualSize); | ||
| 81 | + case ACLBLASLT_MATRIX_LAYOUT_COLS: | ||
| 82 | + return StoreLayoutField(buf, sizeInBytes, impl.cols, actualSize); | ||
| 83 | + case ACLBLASLT_MATRIX_LAYOUT_LD: | ||
| 84 | + return StoreLayoutField(buf, sizeInBytes, impl.ld, actualSize); | ||
| 85 | + case ACLBLASLT_MATRIX_LAYOUT_ORDER: | ||
| 86 | + return StoreLayoutField(buf, sizeInBytes, impl.order, actualSize); | ||
| 87 | + case ACLBLASLT_MATRIX_LAYOUT_BATCH_COUNT: | ||
| 88 | + return StoreLayoutField(buf, sizeInBytes, impl.batchCount, actualSize); | ||
| 89 | + case ACLBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET: | ||
| 90 | + return StoreLayoutField(buf, sizeInBytes, impl.stridedBatchOffset, actualSize); | ||
| 91 | + default: | ||
| 92 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 93 | + } | ||
| 94 | +} | ||
| 95 | + | ||
| 96 | +} // namespace | ||
| 97 | + | ||
| 98 | +extern "C" { | ||
| 99 | + | ||
| 100 | +aclblasStatus_t aclblasLtMatrixLayoutCreate( | ||
| 101 | + aclblasLtMatrixLayout_t* layout, aclDataType type, uint64_t rows, uint64_t cols, int64_t ld) | ||
| 102 | +{ | ||
| 103 | + // Step 1: validate inputs. BLAS/cuBLAS permit empty matrices (m=0 or n=0); reject invalid ld only. | ||
| 104 | + if (layout == nullptr || ld < 0) { | ||
| 105 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 106 | + } | ||
| 107 | + *layout = nullptr; | ||
| 108 | + // Step 2: allocate the opaque capsule on the heap. | ||
| 109 | + auto* capsule = new (std::nothrow) aclblasLtMatrixLayoutOpaque_t(); | ||
| 110 | + if (capsule == nullptr) { | ||
| 111 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 112 | + } | ||
| 113 | + // Step 3: build the impl on the stack and copy it into the capsule. | ||
| 114 | + aclblasLtMatrixLayoutImpl impl; | ||
| 115 | + impl.magic = ACLBLASLT_LAYOUT_MAGIC; | ||
| 116 | + impl.type = type; | ||
| 117 | + impl.rows = rows; | ||
| 118 | + impl.cols = cols; | ||
| 119 | + impl.ld = (ld == 0) ? static_cast<int64_t>(rows) : ld; | ||
| 120 | + // Step 4: copy impl into capsule and zero the remaining bytes. | ||
| 121 | + static_assert(sizeof(impl) <= sizeof(*capsule), "aclblasLtMatrixLayoutImpl too large, not fit in capsule!"); | ||
| 122 | + aclblasStatus_t copyStatus = PackImplIntoCapsule(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 123 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 124 | + delete capsule; | ||
| 125 | + return copyStatus; | ||
| 126 | + } | ||
| 127 | + // Step 5: return the opaque layout handle. | ||
| 128 | + *layout = capsule; | ||
| 129 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 130 | +} | ||
| 131 | + | ||
| 132 | +aclblasStatus_t aclblasLtMatrixLayoutDestroy(const aclblasLtMatrixLayout_t layout) | ||
| 133 | +{ | ||
| 134 | + if (layout == nullptr) { | ||
| 135 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 136 | + } | ||
| 137 | + | ||
| 138 | + auto* capsule = reinterpret_cast<aclblasLtMatrixLayoutOpaque_t*>(layout); | ||
| 139 | + delete capsule; | ||
| 140 | + | ||
| 141 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 142 | +} | ||
| 143 | + | ||
| 144 | +aclblasStatus_t aclblasLtMatrixLayoutSetAttribute( | ||
| 145 | + aclblasLtMatrixLayout_t layout, aclblasLtMatrixLayoutAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 146 | +{ | ||
| 147 | + if (layout == nullptr || buf == nullptr) { | ||
| 148 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + // Unpack the opaque capsule into a stack-side impl for mutation. | ||
| 152 | + aclblasLtMatrixLayoutImpl impl; | ||
| 153 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), layout, sizeof(impl)); | ||
| 154 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 155 | + return copyStatus; | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + const aclblasStatus_t setStatus = ApplyLayoutSetAttr(impl, attr, buf, sizeInBytes); | ||
| 159 | + if (setStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 160 | + return setStatus; | ||
| 161 | + } | ||
| 162 | + | ||
| 163 | + // Pack the updated impl back into the opaque capsule. | ||
| 164 | + return PackImplIntoCapsule(layout, sizeof(*layout), &impl, sizeof(impl)); | ||
| 165 | +} | ||
| 166 | + | ||
| 167 | +aclblasStatus_t aclblasLtMatrixLayoutGetAttribute( | ||
| 168 | + const aclblasLtMatrixLayout_t layout, aclblasLtMatrixLayoutAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 169 | + size_t* sizeWritten) | ||
| 170 | +{ | ||
| 171 | + if (layout == nullptr || buf == nullptr) { | ||
| 172 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 173 | + } | ||
| 174 | + | ||
| 175 | + aclblasLtMatrixLayoutImpl impl; | ||
| 176 | + static_assert(sizeof(impl) <= sizeof(*layout), "aclblasLtMatrixLayoutImpl too large for capsule"); | ||
| 177 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), layout, sizeof(impl)); | ||
| 178 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 179 | + return copyStatus; | ||
| 180 | + } | ||
| 181 | + | ||
| 182 | + size_t actualSize = 0; | ||
| 183 | + const aclblasStatus_t getStatus = ReadLayoutGetAttr(impl, attr, buf, sizeInBytes, actualSize); | ||
| 184 | + if (getStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 185 | + return getStatus; | ||
| 186 | + } | ||
| 187 | + | ||
| 188 | + if (sizeWritten != nullptr) { | ||
| 189 | + *sizeWritten = actualSize; | ||
| 190 | + } | ||
| 191 | + | ||
| 192 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 193 | +} | ||
| 194 | + | ||
| 195 | +} // extern "C" | ||
| @@ -0,0 +1,92 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_matmul.cpp | ||
| 13 | + * \brief Public C API: aclblasLtMatmul — validates inputs, builds the MatmulProblem, dispatches to | ||
| 14 | + * the matmul engine. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +extern "C" { | ||
| 28 | + | ||
| 29 | +aclblasStatus_t aclblasLtMatmul( | ||
| 30 | + aclblasLtHandle_t lightHandle, aclblasLtMatmulDesc_t computeDesc, const void* alpha, const void* A, | ||
| 31 | + aclblasLtMatrixLayout_t Adesc, const void* B, aclblasLtMatrixLayout_t Bdesc, const void* beta, const void* C, | ||
| 32 | + aclblasLtMatrixLayout_t Cdesc, void* D, aclblasLtMatrixLayout_t Ddesc, const aclblasLtMatmulAlgo_t* algo, | ||
| 33 | + void* workspace, size_t workspaceSizeInBytes, aclrtStream stream) | ||
| 34 | +{ | ||
| 35 | + // Validate lightHandle | ||
| 36 | + if (lightHandle == nullptr) { | ||
| 37 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 38 | + } | ||
| 39 | + | ||
| 40 | + // Validate descriptors | ||
| 41 | + if (computeDesc == nullptr || Adesc == nullptr || Bdesc == nullptr || Cdesc == nullptr || Ddesc == nullptr) { | ||
| 42 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + // Validate pointers | ||
| 46 | + if (alpha == nullptr || beta == nullptr) { | ||
| 47 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 48 | + } | ||
| 49 | + | ||
| 50 | + // Get layout info | ||
| 51 | + auto* ALayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Adesc); | ||
| 52 | + auto* BLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Bdesc); | ||
| 53 | + auto* CLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Cdesc); | ||
| 54 | + auto* DLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Ddesc); | ||
| 55 | + auto* desc = reinterpret_cast<aclblasLtMatmulDescImpl*>(computeDesc); | ||
| 56 | + | ||
| 57 | + // Get dimensions | ||
| 58 | + uint64_t m = DLayout->rows; | ||
| 59 | + uint64_t n = DLayout->cols; | ||
| 60 | + uint64_t k = (desc->transA == ACLBLAS_OP_N) ? ALayout->cols : ALayout->rows; | ||
| 61 | + | ||
| 62 | + // BLAS/cuBLAS convention: m=0 or n=0 is a no-op, succeed without touching matrices. | ||
| 63 | + if (m == 0U || n == 0U) { | ||
| 64 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 65 | + } | ||
| 66 | + | ||
| 67 | + if (A == nullptr || B == nullptr || D == nullptr) { | ||
| 68 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 69 | + } | ||
| 70 | + | ||
| 71 | + // Validate workspace alignment (must be 16B aligned) | ||
| 72 | + if (workspace != nullptr && (reinterpret_cast<uintptr_t>(workspace) & 0xF) != 0) { | ||
| 73 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 74 | + } | ||
| 75 | + | ||
| 76 | + // Validate workspace size | ||
| 77 | + if (algo != nullptr && workspaceSizeInBytes < algo->max_workspace_bytes) { | ||
| 78 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 79 | + } | ||
| 80 | + | ||
| 81 | + auto* handleImpl = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 82 | + const uint32_t numBlocks = QueryCubeCoreNum(handleImpl->deviceId); | ||
| 83 | + | ||
| 84 | + MatmulProblem problem; | ||
| 85 | + BuildMatmulProblem( | ||
| 86 | + desc, ALayout, BLayout, CLayout, DLayout, m, n, k, alpha, beta, A, B, C, D, algo, workspace, | ||
| 87 | + workspaceSizeInBytes, numBlocks, stream, problem); | ||
| 88 | + | ||
| 89 | + return MatmulLaunch(problem); | ||
| 90 | +} | ||
| 91 | + | ||
| 92 | +} // extern "C" | ||
RblasLt/include/host/quant_matmul_mx_tiling_swat_host.h→blasLt/matmul/common/quant_matmul_mx_tiling_swat_host.h+1-1
RblasLt/epilogue/arch35/epilogue_alpha_beta_host.cpp→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_host.cpp+2-1
RblasLt/include/kernel/matmul_kernel.h→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_host.h+2-25
RblasLt/epilogue/arch35/epilogue_alpha_beta_kernel.cpp→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_kernel.cpp+1-0
RblasLt/include/host/epilogue_alpha_beta_tiling_data.h→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_tiling_data.h+0-0
RblasLt/matmul_fp32/arch35/matmul_fp32_kernel.cpp→blasLt/matmul/fp32/arch35/matmul_fp32_kernel.cpp+4-4
RblasLt/matmul_mxfp4/arch35/matmul_mxfp4_host.cpp→blasLt/matmul/mxfp4/arch35/matmul_mxfp4_host.cpp+3-4
RblasLt/matmul_mxfp4/arch35/matmul_mxfp4_kernel.cpp→blasLt/matmul/mxfp4/arch35/matmul_mxfp4_kernel.cpp+2-1
RblasLt/matmul_mxfp8/arch35/matmul_mxfp8_host.cpp→blasLt/matmul/mxfp8/arch35/matmul_mxfp8_host.cpp+2-2
RblasLt/matmul_mxfp8/arch35/matmul_mxfp8_kernel.cpp→blasLt/matmul/mxfp8/arch35/matmul_mxfp8_kernel.cpp+3-2
RblasLt/include/host/matrix_transform_get_tiling.h→blasLt/matrix_transform/arch35/matrix_transform_get_tiling.h+0-0
RblasLt/matrixtransform/arch35/matrix_transform_host.cpp→blasLt/matrix_transform/arch35/matrix_transform_host.cpp+1-1
RblasLt/matrixtransform/arch35/matrix_transform_kernel.cpp→blasLt/matrix_transform/arch35/matrix_transform_kernel.cpp+0-0
RblasLt/include/kernel/matrix_transform_kernel.h→blasLt/matrix_transform/arch35/matrix_transform_kernel.h+0-0
RblasLt/include/host/matrix_transform_perm_table.h→blasLt/matrix_transform/arch35/matrix_transform_perm_table.h+0-0
RblasLt/include/host/matrix_transform_tiling_data.h→blasLt/matrix_transform/arch35/matrix_transform_tiling_data.h+0-0
RblasLt/include/host/matrix_transform_acl_impl.h→blasLt/matrix_transform/engine/matrix_transform_engine.h+1-1