已合并
Feat:add aclblasLtMatmulAlgo init/configSetAttr/configGetAttr func #248
wangzitao创建于 7月2日
Feat:add aclblasLtMatmulAlgo init/configSetAttr/configGetAttr func #248
已合并
共 59 个文件变更+3843-2354
| @@ -86,10 +86,17 @@ set_source_files_properties( | |||
| 86 | PROPERTIES LANGUAGE ASC | 86 | PROPERTIES LANGUAGE ASC |
| 87 | ) | 87 | ) |
| 88 | 88 | ||
| 89 | +# Device kernels compile with the ASC frontend; all host code (api / internal / engine / *_host.cpp) | ||
| 90 | +# compiles as plain C++. The classification is produced by blasLt/CMakeLists.txt, so no per-file | ||
| 91 | +# language whitelist is maintained here. | ||
| 89 | set_source_files_properties( | 92 | set_source_files_properties( |
| 90 | - ${ALL_BLASLT_SRC_FILES} | 93 | + ${ALL_BLASLT_KERNEL_SRC_FILES} |
| 91 | PROPERTIES LANGUAGE ASC | 94 | PROPERTIES LANGUAGE ASC |
| 92 | ) | 95 | ) |
| 96 | +set_source_files_properties( | ||
| 97 | + ${ALL_BLASLT_HOST_SRC_FILES} | ||
| 98 | + PROPERTIES LANGUAGE CXX | ||
| 99 | +) | ||
| 93 | 100 | ||
| 94 | set(_OPS_BLAS_ASC_ROOT "${ASCEND_CANN_PACKAGE_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/asc") | 101 | set(_OPS_BLAS_ASC_ROOT "${ASCEND_CANN_PACKAGE_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/asc") |
| 95 | target_include_directories(${OPS_BLASLT} PRIVATE | 102 | target_include_directories(${OPS_BLASLT} PRIVATE |
| @@ -112,22 +119,21 @@ if(EXISTS "${_OPS_BLAS_ASC_ROOT}/include") | |||
| 112 | ) | 119 | ) |
| 113 | endif() | 120 | endif() |
| 114 | 121 | ||
| 122 | +# Private include roots for the refactored blasLt layout. internal/ headers are never installed; | ||
| 123 | +# each operator's arch dir exposes its co-located host/kernel/tiling headers. | ||
| 115 | target_include_directories(${OPS_BLASLT} PRIVATE | 124 | target_include_directories(${OPS_BLASLT} PRIVATE |
| 116 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/include | 125 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/internal/include |
| 117 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/include/kernel | 126 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/common/helper |
| 118 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/include/host | 127 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/engine |
| 119 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/utils | 128 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/common |
| 129 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/fp32/arch35 | ||
| 130 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/mxfp8/arch35 | ||
| 131 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/mxfp4/arch35 | ||
| 132 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul/epilogue/arch35 | ||
| 133 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matrix_transform/engine | ||
| 134 | + ${CMAKE_CURRENT_LIST_DIR}/blasLt/matrix_transform/arch35 | ||
| 120 | ) | 135 | ) |
| 121 | 136 | ||
| 122 | -if(ENABLE_BLASLT_MXFP8) | ||
| 123 | - set_source_files_properties( | ||
| 124 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul_mxfp8/arch35/matmul_mxfp8_host.cpp | ||
| 125 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/matmul_mxfp4/arch35/matmul_mxfp4_host.cpp | ||
| 126 | - PROPERTIES LANGUAGE CXX) | ||
| 127 | -endif() | ||
| 128 | -set_source_files_properties( | ||
| 129 | - ${CMAKE_CURRENT_LIST_DIR}/blasLt/epilogue/arch35/epilogue_alpha_beta_host.cpp | ||
| 130 | - PROPERTIES LANGUAGE CXX) | ||
| 131 | target_compile_definitions(${OPS_BLASLT} PRIVATE | 137 | target_compile_definitions(${OPS_BLASLT} PRIVATE |
| 132 | ASC_DEVKIT_MAJOR=${ASC_DEVKIT_MAJOR} | 138 | ASC_DEVKIT_MAJOR=${ASC_DEVKIT_MAJOR} |
| 133 | ASC_DEVKIT_MINOR=${ASC_DEVKIT_MINOR}) | 139 | ASC_DEVKIT_MINOR=${ASC_DEVKIT_MINOR}) |
| @@ -10,52 +10,54 @@ | |||
| 10 | 10 | ||
| 11 | cmake_minimum_required(VERSION 3.16) | 11 | cmake_minimum_required(VERSION 3.16) |
| 12 | 12 | ||
| 13 | -# 收集当前目录下的基础源文件 | 13 | +# Collect every blasLt source, then classify by role so the parent scope can assign compile |
| 14 | -file(GLOB BASE_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) | 14 | +# languages without maintaining a per-file whitelist: |
| 15 | +# *_kernel.cpp -> ASC (device kernels) | ||
| 16 | +# api / internal / engine / *_host.cpp -> CXX (pure host code) | ||
| 17 | +# Architecture-specific operator code lives under <op>/<variant>/<archXX>/; only the arch dirs | ||
| 18 | +# matching the current SOC (SOC_ARCH_DIRS) are compiled. | ||
| 19 | +file(GLOB_RECURSE BLASLT_ALL_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) | ||
| 15 | 20 | ||
| 16 | -# 自动收集所有算子目录的源文件(支持子目录中的架构特定目录) | 21 | +set(BLASLT_HOST_SRCS "") |
| 17 | -set(OP_SRC_FILES "") | 22 | +set(BLASLT_KERNEL_SRCS "") |
| 18 | -file(GLOB children RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*) | 23 | + |
| 19 | -foreach(child ${children}) | 24 | +foreach(src_file ${BLASLT_ALL_SRCS}) |
| 20 | - if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${child}) | 25 | + # Skip sources under an architecture directory that does not match the current SOC. |
| 21 | - # 递归收集非架构特定源码,避免与 archXX 目录重复收集 | 26 | + set(is_arch_specific FALSE) |
| 22 | - file(GLOB_RECURSE dir_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${child}/*.cpp) | 27 | + foreach(arch_dir ${ARCH_SPECIFIC_DIRS}) |
| 23 | - foreach(src_file ${dir_srcs}) | 28 | + if(src_file MATCHES "/${arch_dir}/") |
| 24 | - set(is_arch_specific FALSE) | 29 | + set(is_arch_specific TRUE) |
| 25 | - foreach(arch_dir ${ARCH_SPECIFIC_DIRS}) | 30 | + endif() |
| 26 | - if(src_file MATCHES "/${arch_dir}/") | 31 | + endforeach() |
| 27 | - set(is_arch_specific TRUE) | 32 | + set(arch_selected FALSE) |
| 28 | - break() | 33 | + foreach(arch_dir ${SOC_ARCH_DIRS}) |
| 29 | - endif() | 34 | + if(src_file MATCHES "/${arch_dir}/") |
| 30 | - endforeach() | 35 | + set(arch_selected TRUE) |
| 31 | - if(NOT is_arch_specific) | 36 | + endif() |
| 32 | - list(APPEND OP_SRC_FILES ${src_file}) | 37 | + endforeach() |
| 33 | - endif() | 38 | + |
| 34 | - endforeach() | 39 | + if(is_arch_specific AND NOT arch_selected) |
| 40 | + # architecture directory for another SOC: do not compile it here | ||
| 41 | + elseif(src_file MATCHES "_kernel\\.cpp$") | ||
| 42 | + list(APPEND BLASLT_KERNEL_SRCS ${src_file}) | ||
| 43 | + else() | ||
| 44 | + list(APPEND BLASLT_HOST_SRCS ${src_file}) | ||
| 35 | endif() | 45 | endif() |
| 36 | endforeach() | 46 | endforeach() |
| 37 | 47 | ||
| 38 | -# 收集子目录中的架构特定源文件 | 48 | +# MXFP8/MXFP4 require asc-devkit >= 9.1 (ENABLE_BLASLT_MXFP8). Exclude the whole precision subtrees |
| 39 | -set(ARCH_SRC_FILES "") | 49 | +# rather than filtering individual files. |
| 40 | -foreach(arch_dir ${SOC_ARCH_DIRS}) | ||
| 41 | - file(GLOB_RECURSE arch_dir_srcs | ||
| 42 | - ${CMAKE_CURRENT_SOURCE_DIR}/*/${arch_dir}/*.cpp | ||
| 43 | - ) | ||
| 44 | - list(APPEND ARCH_SRC_FILES ${arch_dir_srcs}) | ||
| 45 | -endforeach() | ||
| 46 | - | ||
| 47 | -# MXFP8 需要 asc-devkit >= 9.1(ASC_DEVKIT_MAJOR >= 9 && ASC_DEVKIT_MINOR > 0) | ||
| 48 | if(NOT ENABLE_BLASLT_MXFP8) | 50 | if(NOT ENABLE_BLASLT_MXFP8) |
| 49 | - list(FILTER ARCH_SRC_FILES EXCLUDE REGEX "/matmul_mxfp8/") | 51 | + list(FILTER BLASLT_HOST_SRCS EXCLUDE REGEX "/mxfp8/") |
| 50 | - list(FILTER ARCH_SRC_FILES EXCLUDE REGEX "/matmul_mxfp4/") | 52 | + list(FILTER BLASLT_HOST_SRCS EXCLUDE REGEX "/mxfp4/") |
| 53 | + list(FILTER BLASLT_KERNEL_SRCS EXCLUDE REGEX "/mxfp8/") | ||
| 54 | + list(FILTER BLASLT_KERNEL_SRCS EXCLUDE REGEX "/mxfp4/") | ||
| 51 | message( | 55 | message( |
| 52 | STATUS | 56 | STATUS |
| 53 | "Skipping blasLt MXFP8/MXFP4 (requires asc-devkit >= 9.1, current ${ASC_DEVKIT_MAJOR}.${ASC_DEVKIT_MINOR})") | 57 | "Skipping blasLt MXFP8/MXFP4 (requires asc-devkit >= 9.1, current ${ASC_DEVKIT_MAJOR}.${ASC_DEVKIT_MINOR})") |
| 54 | endif() | 58 | endif() |
| 55 | 59 | ||
| 56 | -set(ALL_BLASLT_SRC_FILES | 60 | +# Exported to the parent scope; the top-level CMakeLists assigns LANGUAGE per list. |
| 57 | - ${BASE_SRC_FILES} | 61 | +set(ALL_BLASLT_SRC_FILES ${BLASLT_HOST_SRCS} ${BLASLT_KERNEL_SRCS} PARENT_SCOPE) |
| 58 | - ${OP_SRC_FILES} | 62 | +set(ALL_BLASLT_HOST_SRC_FILES ${BLASLT_HOST_SRCS} PARENT_SCOPE) |
| 59 | - ${ARCH_SRC_FILES} | 63 | +set(ALL_BLASLT_KERNEL_SRC_FILES ${BLASLT_KERNEL_SRCS} PARENT_SCOPE) |
| 60 | - PARENT_SCOPE | ||
| 61 | -) | ||
| @@ -0,0 +1,110 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_handle.cpp | ||
| 13 | + * \brief Public C API: handle lifecycle (create / destroy). | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_handle_impl.h" | ||
| 19 | +#include "aclblaslt_layout_impl.h" | ||
| 20 | +#include "host_utils.h" | ||
| 21 | + | ||
| 22 | +#include <acl/acl.h> | ||
| 23 | +#include <cstdlib> | ||
| 24 | +#include <list> | ||
| 25 | +#include <mutex> | ||
| 26 | +#include <new> | ||
| 27 | +#include <unordered_map> | ||
| 28 | + | ||
| 29 | +extern "C" { | ||
| 30 | + | ||
| 31 | +aclblasStatus_t aclblasLtCreate(aclblasLtHandle_t* lightHandle) | ||
| 32 | +{ | ||
| 33 | + if (lightHandle == nullptr) { | ||
| 34 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 35 | + } | ||
| 36 | + | ||
| 37 | + aclblasLtHandle* h = nullptr; | ||
| 38 | + auto st = AllocHandle(&h); | ||
| 39 | + if (st != ACLBLAS_STATUS_SUCCESS) { | ||
| 40 | + return st; | ||
| 41 | + } | ||
| 42 | + | ||
| 43 | + int32_t deviceId = 0; | ||
| 44 | + aclError aclRet = aclrtGetDevice(&deviceId); | ||
| 45 | + if (aclRet != ACL_SUCCESS) { | ||
| 46 | + delete h; | ||
| 47 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 48 | + } | ||
| 49 | + | ||
| 50 | + aclrtContext currentCtx = nullptr; | ||
| 51 | + aclRet = aclrtGetCurrentContext(¤tCtx); | ||
| 52 | + if (aclRet != ACL_SUCCESS || currentCtx == nullptr) { | ||
| 53 | + delete h; | ||
| 54 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 55 | + } | ||
| 56 | + | ||
| 57 | + h->deviceId = deviceId; | ||
| 58 | + h->context = currentCtx; | ||
| 59 | + h->defaultStream = nullptr; | ||
| 60 | + h->workspaceSize = DEFAULT_WORKSPACE_SIZE; | ||
| 61 | + h->internalWorkspace = std::malloc(h->workspaceSize); | ||
| 62 | + if (h->internalWorkspace == nullptr) { | ||
| 63 | + delete h; | ||
| 64 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 65 | + } | ||
| 66 | + | ||
| 67 | + h->mutex = new (std::nothrow) std::mutex(); | ||
| 68 | + h->algoCache = new (std::nothrow) std::unordered_map<AlgoKey, CacheEntry, AlgoKeyHasher>(); | ||
| 69 | + h->lruList = new (std::nothrow) std::list<AlgoKey>(); | ||
| 70 | + if (h->mutex == nullptr || h->algoCache == nullptr || h->lruList == nullptr) { | ||
| 71 | + delete h->mutex; | ||
| 72 | + delete h->algoCache; | ||
| 73 | + delete h->lruList; | ||
| 74 | + std::free(h->internalWorkspace); | ||
| 75 | + delete h; | ||
| 76 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 77 | + } | ||
| 78 | + | ||
| 79 | + h->npuArch = 2; | ||
| 80 | + h->maxSharedMemory = L1_SIZE; | ||
| 81 | + h->initialized = true; | ||
| 82 | + *lightHandle = reinterpret_cast<aclblasLtHandle_t>(h); | ||
| 83 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 84 | +} | ||
| 85 | + | ||
| 86 | +aclblasStatus_t aclblasLtDestroy(const aclblasLtHandle_t lightHandle) | ||
| 87 | +{ | ||
| 88 | + if (lightHandle == nullptr) { | ||
| 89 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 90 | + } | ||
| 91 | + | ||
| 92 | + auto* h = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 93 | + if (h->magic != ACLBLASLT_HANDLE_MAGIC) { | ||
| 94 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 95 | + } | ||
| 96 | + | ||
| 97 | + h->initialized = false; | ||
| 98 | + delete h->mutex; | ||
| 99 | + delete h->algoCache; | ||
| 100 | + delete h->lruList; | ||
| 101 | + std::free(h->internalWorkspace); | ||
| 102 | + h->mutex = nullptr; | ||
| 103 | + h->algoCache = nullptr; | ||
| 104 | + h->lruList = nullptr; | ||
| 105 | + h->internalWorkspace = nullptr; | ||
| 106 | + h->workspaceSize = 0; | ||
| 107 | + return FreeHandle(h); | ||
| 108 | +} | ||
| 109 | + | ||
| 110 | +} // extern "C" | ||
| @@ -0,0 +1,195 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_layout.cpp | ||
| 13 | + * \brief Public C API: matrix layout descriptor CRUD. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_layout_impl.h" | ||
| 19 | +#include "host_utils.h" | ||
| 20 | + | ||
| 21 | +#include <cstdint> | ||
| 22 | +#include <new> | ||
| 23 | + | ||
| 24 | +namespace { | ||
| 25 | + | ||
| 26 | +// Validate that the caller buffer is exactly sizeof(T), then load it into field. Collapsing the | ||
| 27 | +// per-attribute size check into one helper keeps the Set/Get switches free of nested branches. | ||
| 28 | +template <typename T> | ||
| 29 | +inline aclblasStatus_t LoadLayoutField(const void* buf, size_t sizeInBytes, T& field) | ||
| 30 | +{ | ||
| 31 | + if (sizeInBytes != sizeof(T)) { | ||
| 32 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 33 | + } | ||
| 34 | + field = *reinterpret_cast<const T*>(buf); | ||
| 35 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 36 | +} | ||
| 37 | + | ||
| 38 | +template <typename T> | ||
| 39 | +inline aclblasStatus_t StoreLayoutField(void* buf, size_t sizeInBytes, const T& field, size_t& actualSize) | ||
| 40 | +{ | ||
| 41 | + actualSize = sizeof(T); | ||
| 42 | + if (sizeInBytes < actualSize) { | ||
| 43 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 44 | + } | ||
| 45 | + *reinterpret_cast<T*>(buf) = field; | ||
| 46 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 47 | +} | ||
| 48 | + | ||
| 49 | +aclblasStatus_t ApplyLayoutSetAttr( | ||
| 50 | + aclblasLtMatrixLayoutImpl& impl, aclblasLtMatrixLayoutAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 51 | +{ | ||
| 52 | + switch (attr) { | ||
| 53 | + case ACLBLASLT_MATRIX_LAYOUT_TYPE: | ||
| 54 | + return LoadLayoutField(buf, sizeInBytes, impl.type); | ||
| 55 | + case ACLBLASLT_MATRIX_LAYOUT_ROWS: | ||
| 56 | + return LoadLayoutField(buf, sizeInBytes, impl.rows); | ||
| 57 | + case ACLBLASLT_MATRIX_LAYOUT_COLS: | ||
| 58 | + return LoadLayoutField(buf, sizeInBytes, impl.cols); | ||
| 59 | + case ACLBLASLT_MATRIX_LAYOUT_LD: | ||
| 60 | + return LoadLayoutField(buf, sizeInBytes, impl.ld); | ||
| 61 | + case ACLBLASLT_MATRIX_LAYOUT_ORDER: | ||
| 62 | + return LoadLayoutField(buf, sizeInBytes, impl.order); | ||
| 63 | + case ACLBLASLT_MATRIX_LAYOUT_BATCH_COUNT: | ||
| 64 | + return LoadLayoutField(buf, sizeInBytes, impl.batchCount); | ||
| 65 | + case ACLBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET: | ||
| 66 | + return LoadLayoutField(buf, sizeInBytes, impl.stridedBatchOffset); | ||
| 67 | + default: | ||
| 68 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 69 | + } | ||
| 70 | +} | ||
| 71 | + | ||
| 72 | +aclblasStatus_t ReadLayoutGetAttr( | ||
| 73 | + const aclblasLtMatrixLayoutImpl& impl, aclblasLtMatrixLayoutAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 74 | + size_t& actualSize) | ||
| 75 | +{ | ||
| 76 | + switch (attr) { | ||
| 77 | + case ACLBLASLT_MATRIX_LAYOUT_TYPE: | ||
| 78 | + return StoreLayoutField(buf, sizeInBytes, impl.type, actualSize); | ||
| 79 | + case ACLBLASLT_MATRIX_LAYOUT_ROWS: | ||
| 80 | + return StoreLayoutField(buf, sizeInBytes, impl.rows, actualSize); | ||
| 81 | + case ACLBLASLT_MATRIX_LAYOUT_COLS: | ||
| 82 | + return StoreLayoutField(buf, sizeInBytes, impl.cols, actualSize); | ||
| 83 | + case ACLBLASLT_MATRIX_LAYOUT_LD: | ||
| 84 | + return StoreLayoutField(buf, sizeInBytes, impl.ld, actualSize); | ||
| 85 | + case ACLBLASLT_MATRIX_LAYOUT_ORDER: | ||
| 86 | + return StoreLayoutField(buf, sizeInBytes, impl.order, actualSize); | ||
| 87 | + case ACLBLASLT_MATRIX_LAYOUT_BATCH_COUNT: | ||
| 88 | + return StoreLayoutField(buf, sizeInBytes, impl.batchCount, actualSize); | ||
| 89 | + case ACLBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET: | ||
| 90 | + return StoreLayoutField(buf, sizeInBytes, impl.stridedBatchOffset, actualSize); | ||
| 91 | + default: | ||
| 92 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 93 | + } | ||
| 94 | +} | ||
| 95 | + | ||
| 96 | +} // namespace | ||
| 97 | + | ||
| 98 | +extern "C" { | ||
| 99 | + | ||
| 100 | +aclblasStatus_t aclblasLtMatrixLayoutCreate( | ||
| 101 | + aclblasLtMatrixLayout_t* layout, aclDataType type, uint64_t rows, uint64_t cols, int64_t ld) | ||
| 102 | +{ | ||
| 103 | + // Step 1: validate inputs. BLAS/cuBLAS permit empty matrices (m=0 or n=0); reject invalid ld only. | ||
| 104 | + if (layout == nullptr || ld < 0) { | ||
| 105 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 106 | + } | ||
| 107 | + *layout = nullptr; | ||
| 108 | + // Step 2: allocate the opaque capsule on the heap. | ||
| 109 | + auto* capsule = new (std::nothrow) aclblasLtMatrixLayoutOpaque_t(); | ||
| 110 | + if (capsule == nullptr) { | ||
| 111 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 112 | + } | ||
| 113 | + // Step 3: build the impl on the stack and copy it into the capsule. | ||
| 114 | + aclblasLtMatrixLayoutImpl impl; | ||
| 115 | + impl.magic = ACLBLASLT_LAYOUT_MAGIC; | ||
| 116 | + impl.type = type; | ||
| 117 | + impl.rows = rows; | ||
| 118 | + impl.cols = cols; | ||
| 119 | + impl.ld = (ld == 0) ? static_cast<int64_t>(rows) : ld; | ||
| 120 | + // Step 4: copy impl into capsule and zero the remaining bytes. | ||
| 121 | + static_assert(sizeof(impl) <= sizeof(*capsule), "aclblasLtMatrixLayoutImpl too large, not fit in capsule!"); | ||
| 122 | + aclblasStatus_t copyStatus = PackImplIntoCapsule(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 123 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 124 | + delete capsule; | ||
| 125 | + return copyStatus; | ||
| 126 | + } | ||
| 127 | + // Step 5: return the opaque layout handle. | ||
| 128 | + *layout = capsule; | ||
| 129 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 130 | +} | ||
| 131 | + | ||
| 132 | +aclblasStatus_t aclblasLtMatrixLayoutDestroy(const aclblasLtMatrixLayout_t layout) | ||
| 133 | +{ | ||
| 134 | + if (layout == nullptr) { | ||
| 135 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 136 | + } | ||
| 137 | + | ||
| 138 | + auto* capsule = reinterpret_cast<aclblasLtMatrixLayoutOpaque_t*>(layout); | ||
| 139 | + delete capsule; | ||
| 140 | + | ||
| 141 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 142 | +} | ||
| 143 | + | ||
| 144 | +aclblasStatus_t aclblasLtMatrixLayoutSetAttribute( | ||
| 145 | + aclblasLtMatrixLayout_t layout, aclblasLtMatrixLayoutAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 146 | +{ | ||
| 147 | + if (layout == nullptr || buf == nullptr) { | ||
| 148 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + // Unpack the opaque capsule into a stack-side impl for mutation. | ||
| 152 | + aclblasLtMatrixLayoutImpl impl; | ||
| 153 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), layout, sizeof(impl)); | ||
| 154 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 155 | + return copyStatus; | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + const aclblasStatus_t setStatus = ApplyLayoutSetAttr(impl, attr, buf, sizeInBytes); | ||
| 159 | + if (setStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 160 | + return setStatus; | ||
| 161 | + } | ||
| 162 | + | ||
| 163 | + // Pack the updated impl back into the opaque capsule. | ||
| 164 | + return PackImplIntoCapsule(layout, sizeof(*layout), &impl, sizeof(impl)); | ||
| 165 | +} | ||
| 166 | + | ||
| 167 | +aclblasStatus_t aclblasLtMatrixLayoutGetAttribute( | ||
| 168 | + const aclblasLtMatrixLayout_t layout, aclblasLtMatrixLayoutAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 169 | + size_t* sizeWritten) | ||
| 170 | +{ | ||
| 171 | + if (layout == nullptr || buf == nullptr) { | ||
| 172 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 173 | + } | ||
| 174 | + | ||
| 175 | + aclblasLtMatrixLayoutImpl impl; | ||
| 176 | + static_assert(sizeof(impl) <= sizeof(*layout), "aclblasLtMatrixLayoutImpl too large for capsule"); | ||
| 177 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), layout, sizeof(impl)); | ||
| 178 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 179 | + return copyStatus; | ||
| 180 | + } | ||
| 181 | + | ||
| 182 | + size_t actualSize = 0; | ||
| 183 | + const aclblasStatus_t getStatus = ReadLayoutGetAttr(impl, attr, buf, sizeInBytes, actualSize); | ||
| 184 | + if (getStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 185 | + return getStatus; | ||
| 186 | + } | ||
| 187 | + | ||
| 188 | + if (sizeWritten != nullptr) { | ||
| 189 | + *sizeWritten = actualSize; | ||
| 190 | + } | ||
| 191 | + | ||
| 192 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 193 | +} | ||
| 194 | + | ||
| 195 | +} // extern "C" | ||
| @@ -0,0 +1,92 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_matmul.cpp | ||
| 13 | + * \brief Public C API: aclblasLtMatmul — validates inputs, builds the MatmulProblem, dispatches to | ||
| 14 | + * the matmul engine. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#include "cann_ops_blasLt.h" | ||
| 18 | + | ||
| 19 | +#include "aclblaslt_handle_impl.h" | ||
| 20 | +#include "aclblaslt_layout_impl.h" | ||
| 21 | +#include "aclblaslt_matmul_problem.h" | ||
| 22 | +#include "matmul_engine.h" | ||
| 23 | + | ||
| 24 | +#include <acl/acl.h> | ||
| 25 | +#include <cstdint> | ||
| 26 | + | ||
| 27 | +extern "C" { | ||
| 28 | + | ||
| 29 | +aclblasStatus_t aclblasLtMatmul( | ||
| 30 | + aclblasLtHandle_t lightHandle, aclblasLtMatmulDesc_t computeDesc, const void* alpha, const void* A, | ||
| 31 | + aclblasLtMatrixLayout_t Adesc, const void* B, aclblasLtMatrixLayout_t Bdesc, const void* beta, const void* C, | ||
| 32 | + aclblasLtMatrixLayout_t Cdesc, void* D, aclblasLtMatrixLayout_t Ddesc, const aclblasLtMatmulAlgo_t* algo, | ||
| 33 | + void* workspace, size_t workspaceSizeInBytes, aclrtStream stream) | ||
| 34 | +{ | ||
| 35 | + // Validate lightHandle | ||
| 36 | + if (lightHandle == nullptr) { | ||
| 37 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 38 | + } | ||
| 39 | + | ||
| 40 | + // Validate descriptors | ||
| 41 | + if (computeDesc == nullptr || Adesc == nullptr || Bdesc == nullptr || Cdesc == nullptr || Ddesc == nullptr) { | ||
| 42 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + // Validate pointers | ||
| 46 | + if (alpha == nullptr || beta == nullptr) { | ||
| 47 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 48 | + } | ||
| 49 | + | ||
| 50 | + // Get layout info | ||
| 51 | + auto* ALayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Adesc); | ||
| 52 | + auto* BLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Bdesc); | ||
| 53 | + auto* CLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Cdesc); | ||
| 54 | + auto* DLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Ddesc); | ||
| 55 | + auto* desc = reinterpret_cast<aclblasLtMatmulDescImpl*>(computeDesc); | ||
| 56 | + | ||
| 57 | + // Get dimensions | ||
| 58 | + uint64_t m = DLayout->rows; | ||
| 59 | + uint64_t n = DLayout->cols; | ||
| 60 | + uint64_t k = (desc->transA == ACLBLAS_OP_N) ? ALayout->cols : ALayout->rows; | ||
| 61 | + | ||
| 62 | + // BLAS/cuBLAS convention: m=0 or n=0 is a no-op, succeed without touching matrices. | ||
| 63 | + if (m == 0U || n == 0U) { | ||
| 64 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 65 | + } | ||
| 66 | + | ||
| 67 | + if (A == nullptr || B == nullptr || D == nullptr) { | ||
| 68 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 69 | + } | ||
| 70 | + | ||
| 71 | + // Validate workspace alignment (must be 16B aligned) | ||
| 72 | + if (workspace != nullptr && (reinterpret_cast<uintptr_t>(workspace) & 0xF) != 0) { | ||
| 73 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 74 | + } | ||
| 75 | + | ||
| 76 | + // Validate workspace size | ||
| 77 | + if (algo != nullptr && workspaceSizeInBytes < algo->max_workspace_bytes) { | ||
| 78 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 79 | + } | ||
| 80 | + | ||
| 81 | + auto* handleImpl = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 82 | + const uint32_t numBlocks = QueryCubeCoreNum(handleImpl->deviceId); | ||
| 83 | + | ||
| 84 | + MatmulProblem problem; | ||
| 85 | + BuildMatmulProblem( | ||
| 86 | + desc, ALayout, BLayout, CLayout, DLayout, m, n, k, alpha, beta, A, B, C, D, algo, workspace, | ||
| 87 | + workspaceSizeInBytes, numBlocks, stream, problem); | ||
| 88 | + | ||
| 89 | + return MatmulLaunch(problem); | ||
| 90 | +} | ||
| 91 | + | ||
| 92 | +} // extern "C" | ||
| @@ -0,0 +1,559 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_matmul_algo.cpp | ||
| 13 | + * \brief Public C API: matmul algorithm init / config / id enumeration / heuristic. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_algo_heuristic.h" | ||
| 19 | +#include "aclblaslt_handle_impl.h" | ||
| 20 | +#include "aclblaslt_layout_impl.h" | ||
| 21 | +#include "host_utils.h" | ||
| 22 | + | ||
| 23 | +#include <algorithm> | ||
| 24 | +#include <cstdint> | ||
| 25 | +#include <vector> | ||
| 26 | + | ||
| 27 | +namespace { | ||
| 28 | + | ||
| 29 | +// ---- AlgoInit helpers ---- | ||
| 30 | + | ||
| 31 | +aclblasStatus_t ValidateAlgoInitArgs( | ||
| 32 | + aclblasLtHandle_t lightHandle, aclblasComputeType_t computeType, aclDataType scaleType, aclDataType Atype, | ||
| 33 | + aclDataType Btype, aclDataType Ctype, aclDataType Dtype, int algoId) | ||
| 34 | +{ | ||
| 35 | + if (lightHandle == nullptr) { | ||
| 36 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 37 | + } | ||
| 38 | + if (algoId < 0) { | ||
| 39 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 40 | + } | ||
| 41 | + if (!CheckComputeTypeCompatibility(computeType, Atype, Btype)) { | ||
| 42 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 43 | + } | ||
| 44 | + if (Ctype != Dtype) { | ||
| 45 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 46 | + } | ||
| 47 | + if (Dtype == ACL_FLOAT && scaleType != ACL_FLOAT && scaleType != ACL_DT_UNDEFINED) { | ||
| 48 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 49 | + } | ||
| 50 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 51 | +} | ||
| 52 | + | ||
| 53 | +// Populate packed with the default tiling, then override from a non-zero algoId. Returns false when | ||
| 54 | +// the algoId cannot be decoded into a valid tiling configuration. | ||
| 55 | +bool InitPackedAlgoForId(int algoId, PackedAlgo& packed) | ||
| 56 | +{ | ||
| 57 | + packed.magic = ACLBLASLT_ALGO_MAGIC; | ||
| 58 | + packed.l1mDiv16 = static_cast<uint16_t>(DEFAULT_L1_M / 16); | ||
| 59 | + packed.l1nDiv16 = static_cast<uint16_t>(DEFAULT_L1_N / 16); | ||
| 60 | + packed.policy = static_cast<uint8_t>(DISPATCH_POLICY_MMAD_SYNC); | ||
| 61 | + packed.numBuffers = 1; | ||
| 62 | + packed.splitK = 1; | ||
| 63 | + packed.flags = 0; | ||
| 64 | + | ||
| 65 | + if (algoId == 0) { | ||
| 66 | + packed.algoId = DEFAULT_ALGO_ID; | ||
| 67 | + packed.flags = EncodeL1KToFlags(packed.flags, DEFAULT_L1_K); | ||
| 68 | + return true; | ||
| 69 | + } | ||
| 70 | + | ||
| 71 | + packed.algoId = static_cast<uint32_t>(algoId); | ||
| 72 | + uint8_t decPolicy = 0; | ||
| 73 | + uint16_t decL1mDiv16 = 0; | ||
| 74 | + uint16_t decL1nDiv16 = 0; | ||
| 75 | + uint32_t decL1k = DEFAULT_L1_K; | ||
| 76 | + uint8_t decSplitK = 0; | ||
| 77 | + if (!DecodeAlgoIdFields(packed.algoId, &decPolicy, &decL1mDiv16, &decL1nDiv16, &decL1k, &decSplitK)) { | ||
| 78 | + return false; | ||
| 79 | + } | ||
| 80 | + packed.policy = decPolicy; | ||
| 81 | + packed.l1mDiv16 = decL1mDiv16; | ||
| 82 | + packed.l1nDiv16 = decL1nDiv16; | ||
| 83 | + packed.splitK = decSplitK; | ||
| 84 | + packed.flags = EncodeL1KToFlags(packed.flags, decL1k); | ||
| 85 | + return true; | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +// ---- ConfigSetAttribute helpers ---- | ||
| 89 | + | ||
| 90 | +aclblasStatus_t ReadConfigU32(const void* buf, size_t sizeInBytes, uint32_t& out) | ||
| 91 | +{ | ||
| 92 | + if (sizeInBytes != sizeof(uint32_t)) { | ||
| 93 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 94 | + } | ||
| 95 | + if (!MemcpySSucceeds(&out, sizeof(out), buf, sizeof(uint32_t))) { | ||
| 96 | + return ACLBLAS_STATUS_INTERNAL_ERROR; | ||
| 97 | + } | ||
| 98 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 99 | +} | ||
| 100 | + | ||
| @@ -0,0 +1,253 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_matmul_desc.cpp | ||
| 13 | + * \brief Public C API: matmul compute descriptor CRUD. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_layout_impl.h" | ||
| 19 | +#include "host_utils.h" | ||
| 20 | + | ||
| 21 | +#include <cstdint> | ||
| 22 | +#include <new> | ||
| 23 | + | ||
| 24 | +namespace { | ||
| 25 | + | ||
| 26 | +aclblasStatus_t InitMatmulDescCapsule( | ||
| 27 | + aclblasLtMatmulDescOpaque_t* capsule, aclblasComputeType_t computeType, aclDataType scaleType) | ||
| 28 | +{ | ||
| 29 | + aclblasLtMatmulDescImpl impl; | ||
| 30 | + impl.magic = ACLBLASLT_DESC_MAGIC; | ||
| 31 | + impl.computeType = computeType; | ||
| 32 | + impl.scaleType = scaleType; | ||
| 33 | + | ||
| 34 | + static_assert(sizeof(impl) <= sizeof(*capsule), "aclblasLtMatmulDescImpl too large, not fit in capsule!"); | ||
| 35 | + return PackImplIntoCapsule(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 36 | +} | ||
| 37 | + | ||
| 38 | +// ---- SetAttribute per-attribute handlers ---- | ||
| 39 | +// Each handler owns its own size validation and copy so the dispatcher switch stays branch-light, | ||
| 40 | +// keeping aclblasLtMatmulDescSetAttribute() under the cyclomatic-complexity / function-length limits. | ||
| 41 | + | ||
| 42 | +aclblasStatus_t SetDescEpilogue(aclblasLtMatmulDescImpl& impl, const void* buf, size_t sizeInBytes) | ||
| 43 | +{ | ||
| 44 | + if (sizeInBytes != sizeof(aclblasLtEpilogue_t) && sizeInBytes != sizeof(uint32_t)) { | ||
| 45 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 46 | + } | ||
| 47 | + uint32_t v = 0; | ||
| 48 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(uint32_t)); | ||
| 49 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 50 | + return copyStatus; | ||
| 51 | + } | ||
| 52 | + impl.epilogue = static_cast<aclblasLtEpilogue_t>(v); | ||
| 53 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 54 | +} | ||
| 55 | + | ||
| 56 | +aclblasStatus_t SetDescBiasPointer(aclblasLtMatmulDescImpl& impl, const void* buf, size_t sizeInBytes) | ||
| 57 | +{ | ||
| 58 | + if (sizeInBytes != BIAS_PTR_STORAGE_BYTES) { | ||
| 59 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 60 | + } | ||
| 61 | + void* biasPtr = nullptr; | ||
| 62 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&biasPtr, BIAS_PTR_STORAGE_BYTES, buf, BIAS_PTR_STORAGE_BYTES); | ||
| 63 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 64 | + return copyStatus; | ||
| 65 | + } | ||
| 66 | + impl.bias = biasPtr; | ||
| 67 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 68 | +} | ||
| 69 | + | ||
| 70 | +// Read a strict int32 payload and store it into an enum-typed field (transA/transB/biasDataType). | ||
| 71 | +template <typename E> | ||
| 72 | +aclblasStatus_t SetDescEnumFromI32(E& field, const void* buf, size_t sizeInBytes) | ||
| 73 | +{ | ||
| 74 | + if (sizeInBytes != sizeof(int32_t)) { | ||
| 75 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 76 | + } | ||
| 77 | + int32_t v = 0; | ||
| 78 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(int32_t)); | ||
| 79 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 80 | + return copyStatus; | ||
| 81 | + } | ||
| 82 | + field = static_cast<E>(v); | ||
| 83 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 84 | +} | ||
| 85 | + | ||
| 86 | +aclblasStatus_t SetDescScalePointer(const void*& field, const void* buf, size_t sizeInBytes) | ||
| 87 | +{ | ||
| 88 | + if (sizeInBytes != sizeof(void*)) { | ||
| 89 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 90 | + } | ||
| 91 | + const void* scalePtr = nullptr; | ||
| 92 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&scalePtr, sizeof(void*), buf, sizeof(void*)); | ||
| 93 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 94 | + return copyStatus; | ||
| 95 | + } | ||
| 96 | + field = scalePtr; | ||
| 97 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | +aclblasStatus_t ApplyMatmulDescSetAttr( | ||
| 101 | + aclblasLtMatmulDescImpl& impl, aclblasLtMatmulDescAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 102 | +{ | ||
| 103 | + switch (attr) { | ||
| 104 | + case ACLBLASLT_MATMUL_DESC_EPILOGUE: | ||
| 105 | + return SetDescEpilogue(impl, buf, sizeInBytes); | ||
| 106 | + case ACLBLASLT_MATMUL_DESC_BIAS_POINTER: | ||
| 107 | + return SetDescBiasPointer(impl, buf, sizeInBytes); | ||
| 108 | + case ACLBLASLT_MATMUL_DESC_TRANSA: | ||
| 109 | + return SetDescEnumFromI32(impl.transA, buf, sizeInBytes); | ||
| 110 | + case ACLBLASLT_MATMUL_DESC_TRANSB: | ||
| 111 | + return SetDescEnumFromI32(impl.transB, buf, sizeInBytes); | ||
| 112 | + case ACLBLASLT_MATMUL_DESC_BIAS_DATA_TYPE: | ||
| 113 | + return SetDescEnumFromI32(impl.biasDataType, buf, sizeInBytes); | ||
| 114 | + case ACLBLASLT_MATMUL_DESC_A_SCALE_POINTER: | ||
| 115 | + return SetDescScalePointer(impl.scaleA, buf, sizeInBytes); | ||
| 116 | + case ACLBLASLT_MATMUL_DESC_B_SCALE_POINTER: | ||
| 117 | + return SetDescScalePointer(impl.scaleB, buf, sizeInBytes); | ||
| 118 | + case ACLBLASLT_MATMUL_DESC_A_SCALE_MODE: | ||
| 119 | + case ACLBLASLT_MATMUL_DESC_B_SCALE_MODE: | ||
| 120 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 121 | + default: | ||
| 122 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 123 | + } | ||
| 124 | +} | ||
| 125 | + | ||
| 126 | +} // namespace | ||
| 127 | + | ||
| 128 | +extern "C" { | ||
| 129 | + | ||
| 130 | +aclblasStatus_t aclblasLtMatmulDescInit( | ||
| 131 | + aclblasLtMatmulDesc_t matmulDesc, aclblasComputeType_t computeType, aclDataType scaleType) | ||
| 132 | +{ | ||
| 133 | + if (matmulDesc == nullptr) { | ||
| 134 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 135 | + } | ||
| 136 | + return InitMatmulDescCapsule(matmulDesc, computeType, scaleType); | ||
| 137 | +} | ||
| 138 | + | ||
| 139 | +aclblasStatus_t aclblasLtMatmulDescCreate( | ||
| 140 | + aclblasLtMatmulDesc_t* desc, aclblasComputeType_t computeType, aclDataType scaleType) | ||
| 141 | +{ | ||
| 142 | + if (desc == nullptr) { | ||
| 143 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 144 | + } | ||
| 145 | + *desc = nullptr; | ||
| 146 | + | ||
| 147 | + auto* capsule = new (std::nothrow) aclblasLtMatmulDescOpaque_t(); | ||
| 148 | + if (capsule == nullptr) { | ||
| 149 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 150 | + } | ||
| 151 | + | ||
| 152 | + aclblasStatus_t st = InitMatmulDescCapsule(capsule, computeType, scaleType); | ||
| 153 | + if (st != ACLBLAS_STATUS_SUCCESS) { | ||
| 154 | + delete capsule; | ||
| 155 | + return st; | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + *desc = capsule; | ||
| 159 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 160 | +} | ||
| 161 | + | ||
| 162 | +aclblasStatus_t aclblasLtMatmulDescDestroy(const aclblasLtMatmulDesc_t desc) | ||
| 163 | +{ | ||
| 164 | + if (desc == nullptr) { | ||
| 165 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 166 | + } | ||
| 167 | + | ||
| 168 | + auto* capsule = reinterpret_cast<aclblasLtMatmulDescOpaque_t*>(desc); | ||
| 169 | + delete capsule; | ||
| 170 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 171 | +} | ||
| 172 | + | ||
| 173 | +aclblasStatus_t aclblasLtMatmulDescSetAttribute( | ||
| 174 | + aclblasLtMatmulDesc_t desc, aclblasLtMatmulDescAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 175 | +{ | ||
| 176 | + if (desc == nullptr || buf == nullptr) { | ||
| 177 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 178 | + } | ||
| 179 | + | ||
| 180 | + aclblasLtMatmulDescImpl impl; | ||
| 181 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), desc, sizeof(impl)); | ||
| 182 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 183 | + return copyStatus; | ||
| 184 | + } | ||
| 185 | + | ||
| 186 | + const aclblasStatus_t setStatus = ApplyMatmulDescSetAttr(impl, attr, buf, sizeInBytes); | ||
| 187 | + if (setStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 188 | + return setStatus; | ||
| 189 | + } | ||
| 190 | + | ||
| 191 | + return PackImplIntoCapsule(desc, sizeof(*desc), &impl, sizeof(impl)); | ||
| 192 | +} | ||
| 193 | + | ||
| 194 | +aclblasStatus_t aclblasLtMatmulDescGetAttribute( | ||
| 195 | + aclblasLtMatmulDesc_t desc, aclblasLtMatmulDescAttribute_t attr, void* buf, size_t sizeInBytes, size_t* sizeWritten) | ||
| 196 | +{ | ||
| 197 | + if (desc == nullptr || buf == nullptr) { | ||
| 198 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 199 | + } | ||
| 200 | + | ||
| 201 | + aclblasLtMatmulDescImpl impl; | ||
| 202 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), desc, sizeof(impl)); | ||
| 203 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 204 | + return copyStatus; | ||
| 205 | + } | ||
| 206 | + | ||
| 207 | + size_t requiredSize = 0; | ||
| 208 | + const void* srcPtr = nullptr; | ||
| 209 | + | ||
| 210 | + switch (attr) { | ||
| 211 | + case ACLBLASLT_MATMUL_DESC_EPILOGUE: | ||
| 212 | + requiredSize = sizeof(impl.epilogue); | ||
| 213 | + srcPtr = &impl.epilogue; | ||
| 214 | + break; | ||
| 215 | + | ||
| 216 | + case ACLBLASLT_MATMUL_DESC_BIAS_POINTER: | ||
| 217 | + requiredSize = BIAS_PTR_STORAGE_BYTES; | ||
| 218 | + srcPtr = &impl.bias; | ||
| 219 | + break; | ||
| 220 | + | ||
| 221 | + case ACLBLASLT_MATMUL_DESC_TRANSA: | ||
| 222 | + requiredSize = sizeof(impl.transA); | ||
| 223 | + srcPtr = &impl.transA; | ||
| 224 | + break; | ||
| 225 | + | ||
| 226 | + case ACLBLASLT_MATMUL_DESC_TRANSB: | ||
| 227 | + requiredSize = sizeof(impl.transB); | ||
| 228 | + srcPtr = &impl.transB; | ||
| 229 | + break; | ||
| 230 | + | ||
| 231 | + case ACLBLASLT_MATMUL_DESC_BIAS_DATA_TYPE: | ||
| 232 | + requiredSize = sizeof(impl.biasDataType); | ||
| 233 | + srcPtr = &impl.biasDataType; | ||
| 234 | + break; | ||
| 235 | + | ||
| 236 | + case ACLBLASLT_MATMUL_DESC_A_SCALE_POINTER: | ||
| 237 | + requiredSize = sizeof(impl.scaleA); | ||
| 238 | + srcPtr = &impl.scaleA; | ||
| 239 | + break; | ||
| 240 | + | ||
| 241 | + case ACLBLASLT_MATMUL_DESC_B_SCALE_POINTER: | ||
| 242 | + requiredSize = sizeof(impl.scaleB); | ||
| 243 | + srcPtr = &impl.scaleB; | ||
| 244 | + break; | ||
| 245 | + | ||
| 246 | + default: | ||
| 247 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 248 | + } | ||
| 249 | + | ||
| 250 | + return CopyAttributeOut(buf, sizeInBytes, srcPtr, requiredSize, sizeWritten); | ||
| 251 | +} | ||
| 252 | + | ||
| 253 | +} // extern "C" | ||
| @@ -0,0 +1,161 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_matmul_preference.cpp | ||
| 13 | + * \brief Public C API: matmul heuristic-search preference CRUD. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_layout_impl.h" | ||
| 19 | +#include "host_utils.h" | ||
| 20 | + | ||
| 21 | +#include <algorithm> | ||
| 22 | +#include <cstdint> | ||
| 23 | +#include <new> | ||
| 24 | + | ||
| 25 | +extern "C" { | ||
| 26 | + | ||
| 27 | +aclblasStatus_t aclblasLtMatmulPreferenceCreate(aclblasLtMatmulPreference_t* pref) | ||
| 28 | +{ | ||
| 29 | + if (pref == nullptr) { | ||
| 30 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 31 | + } | ||
| 32 | + | ||
| 33 | + *pref = nullptr; | ||
| 34 | + auto* capsule = new (std::nothrow) aclblasLtMatmulPreferenceOpaque_t(); | ||
| 35 | + if (capsule == nullptr) { | ||
| 36 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 37 | + } | ||
| 38 | + aclblasStatus_t copyStatus = CheckedMemsetS(capsule, sizeof(*capsule), sizeof(*capsule)); | ||
| 39 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 40 | + delete capsule; | ||
| 41 | + return copyStatus; | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | + aclblasLtMatmulPreferenceImpl impl; | ||
| 45 | + copyStatus = PackImplIntoCapsule(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 46 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 47 | + delete capsule; | ||
| 48 | + return copyStatus; | ||
| 49 | + } | ||
| 50 | + | ||
| 51 | + *pref = capsule; | ||
| 52 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 53 | +} | ||
| 54 | + | ||
| 55 | +aclblasStatus_t aclblasLtMatmulPreferenceDestroy(const aclblasLtMatmulPreference_t pref) | ||
| 56 | +{ | ||
| 57 | + if (pref == nullptr) { | ||
| 58 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 59 | + } | ||
| 60 | + | ||
| 61 | + auto* capsule = reinterpret_cast<aclblasLtMatmulPreferenceOpaque_t*>(pref); | ||
| 62 | + delete capsule; | ||
| 63 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 64 | +} | ||
| 65 | + | ||
| 66 | +aclblasStatus_t aclblasLtMatmulPreferenceSetAttribute( | ||
| 67 | + aclblasLtMatmulPreference_t pref, aclblasLtMatmulPreferenceAttribute_t attr, const void* buf, size_t sizeInBytes) | ||
| 68 | +{ | ||
| 69 | + if (pref == nullptr || buf == nullptr) { | ||
| 70 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 71 | + } | ||
| 72 | + | ||
| 73 | + aclblasLtMatmulPreferenceImpl impl; | ||
| 74 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), pref, sizeof(impl)); | ||
| 75 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 76 | + return copyStatus; | ||
| 77 | + } | ||
| 78 | + | ||
| 79 | + switch (attr) { | ||
| 80 | + case ACLBLASLT_MATMUL_PREF_SEARCH_MODE: { | ||
| 81 | + if (sizeInBytes != sizeof(uint32_t)) { | ||
| 82 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 83 | + } | ||
| 84 | + uint32_t v = 0; | ||
| 85 | + copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(v)); | ||
| 86 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 87 | + return copyStatus; | ||
| 88 | + } | ||
| 89 | + // 0=heuristic, 1=exhaustive, 2=fast | ||
| 90 | + if (v > 2) { | ||
| 91 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 92 | + } | ||
| 93 | + impl.searchMode = v; | ||
| 94 | + break; | ||
| 95 | + } | ||
| 96 | + | ||
| 97 | + case ACLBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES: { | ||
| 98 | + if (sizeInBytes != sizeof(size_t) && sizeInBytes != sizeof(uint64_t)) { | ||
| 99 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 100 | + } | ||
| 101 | + size_t v = 0; | ||
| 102 | + const size_t copyBytes = std::min(sizeInBytes, sizeof(v)); | ||
| 103 | + copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, copyBytes); | ||
| 104 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 105 | + return copyStatus; | ||
| 106 | + } | ||
| 107 | + if (v > INT64_MAX) { | ||
| 108 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 109 | + } | ||
| 110 | + impl.maxWorkspaceBytes = v; | ||
| 111 | + break; | ||
| 112 | + } | ||
| 113 | + | ||
| 114 | + default: | ||
| 115 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 116 | + } | ||
| 117 | + | ||
| 118 | + copyStatus = CheckedMemcpyS(pref, sizeof(*pref), &impl, sizeof(impl)); | ||
| 119 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 120 | + return copyStatus; | ||
| 121 | + } | ||
| 122 | + | ||
| 123 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 124 | +} | ||
| 125 | + | ||
| 126 | +aclblasStatus_t aclblasLtMatmulPreferenceGetAttribute( | ||
| 127 | + aclblasLtMatmulPreference_t pref, aclblasLtMatmulPreferenceAttribute_t attr, void* buf, size_t sizeInBytes, | ||
| 128 | + size_t* sizeWritten) | ||
| 129 | +{ | ||
| 130 | + if (pref == nullptr || buf == nullptr) { | ||
| 131 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 132 | + } | ||
| 133 | + | ||
| 134 | + aclblasLtMatmulPreferenceImpl impl; | ||
| 135 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), pref, sizeof(impl)); | ||
| 136 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 137 | + return copyStatus; | ||
| 138 | + } | ||
| 139 | + | ||
| 140 | + size_t requiredSize = 0; | ||
| 141 | + const void* srcPtr = nullptr; | ||
| 142 | + | ||
| 143 | + switch (attr) { | ||
| 144 | + case ACLBLASLT_MATMUL_PREF_SEARCH_MODE: | ||
| 145 | + requiredSize = sizeof(impl.searchMode); | ||
| 146 | + srcPtr = &impl.searchMode; | ||
| 147 | + break; | ||
| 148 | + | ||
| 149 | + case ACLBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES: | ||
| 150 | + requiredSize = sizeof(impl.maxWorkspaceBytes); | ||
| 151 | + srcPtr = &impl.maxWorkspaceBytes; | ||
| 152 | + break; | ||
| 153 | + | ||
| 154 | + default: | ||
| 155 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + return CopyAttributeOut(buf, sizeInBytes, srcPtr, requiredSize, sizeWritten); | ||
| 159 | +} | ||
| 160 | + | ||
| 161 | +} // extern "C" | ||
| @@ -0,0 +1,88 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_transform.cpp | ||
| 13 | + * \brief Public C API: aclblasLtMatrixTransform — validates inputs, packs layouts, dispatches to | ||
| 14 | + * the transform engine. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#include "cann_ops_blasLt.h" | ||
| 18 | + | ||
| 19 | +#include "aclblaslt_handle_impl.h" | ||
| 20 | +#include "aclblaslt_layout_impl.h" | ||
| 21 | +#include "matrix_transform_engine.h" | ||
| 22 | + | ||
| 23 | +#include <acl/acl.h> | ||
| 24 | +#include <cstdint> | ||
| 25 | + | ||
| 26 | +namespace { | ||
| 27 | + | ||
| 28 | +// Pack the transform-relevant fields of a matrix layout capsule into the plain parameter struct | ||
| 29 | +// consumed by the transform operator unit. | ||
| 30 | +MatTransformLayout MatPackTransformLayout(const aclblasLtMatrixLayoutImpl* layout) | ||
| 31 | +{ | ||
| 32 | + MatTransformLayout packed; | ||
| 33 | + packed.type = layout->type; | ||
| 34 | + packed.rows = layout->rows; | ||
| 35 | + packed.cols = layout->cols; | ||
| 36 | + packed.ld = layout->ld; | ||
| 37 | + packed.order = layout->order; | ||
| 38 | + packed.batchCount = layout->batchCount; | ||
| 39 | + return packed; | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +} // namespace | ||
| 43 | + | ||
| 44 | +extern "C" { | ||
| 45 | + | ||
| 46 | +aclblasStatus_t aclblasLtMatrixTransform( | ||
| 47 | + aclblasLtHandle_t lightHandle, aclblasLtMatrixTransformDesc_t transformDesc, const void* alpha, const void* A, | ||
| 48 | + aclblasLtMatrixLayout_t Adesc, const void* beta, const void* B, aclblasLtMatrixLayout_t Bdesc, void* C, | ||
| 49 | + aclblasLtMatrixLayout_t Cdesc, aclrtStream stream) | ||
| 50 | +{ | ||
| 51 | + if (lightHandle == nullptr) { | ||
| 52 | + return ACLBLAS_STATUS_NOT_INITIALIZED; | ||
| 53 | + } | ||
| 54 | + if (transformDesc == nullptr || Adesc == nullptr || Cdesc == nullptr) { | ||
| 55 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 56 | + } | ||
| 57 | + if (alpha == nullptr) { | ||
| 58 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 59 | + } | ||
| 60 | + | ||
| 61 | + auto* desc = reinterpret_cast<aclblasLtMatrixTransformDescImpl*>(transformDesc); | ||
| 62 | + auto* ALayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Adesc); | ||
| 63 | + auto* CLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Cdesc); | ||
| 64 | + if (desc->magic != ACLBLASLT_TRANSFORM_DESC_MAGIC || ALayout->magic != ACLBLASLT_LAYOUT_MAGIC || | ||
| 65 | + CLayout->magic != ACLBLASLT_LAYOUT_MAGIC) { | ||
| 66 | + return ACLBLAS_STATUS_INVALID_VALUE; // corrupted / foreign descriptor or layout | ||
| 67 | + } | ||
| 68 | + | ||
| 69 | + auto* handleImpl = reinterpret_cast<aclblasLtHandle*>(lightHandle); | ||
| 70 | + | ||
| 71 | + const uint64_t rows = CLayout->rows; | ||
| 72 | + const uint64_t cols = CLayout->cols; | ||
| 73 | + if (rows == 0U || cols == 0U) { | ||
| 74 | + return ACLBLAS_STATUS_SUCCESS; // empty matrix, no-op (checked before B presence) | ||
| 75 | + } | ||
| 76 | + | ||
| 77 | + const MatTransformLayout aPacked = MatPackTransformLayout(ALayout); | ||
| 78 | + const MatTransformLayout cPacked = MatPackTransformLayout(CLayout); | ||
| 79 | + auto* BLayout = reinterpret_cast<aclblasLtMatrixLayoutImpl*>(Bdesc); | ||
| 80 | + const MatTransformLayout bPacked = (BLayout != nullptr) ? MatPackTransformLayout(BLayout) : MatTransformLayout{}; | ||
| 81 | + const bool bLayoutValid = (BLayout != nullptr) && BLayout->magic == ACLBLASLT_LAYOUT_MAGIC; | ||
| 82 | + | ||
| 83 | + return MatTransformLaunch( | ||
| 84 | + handleImpl->deviceId, desc, alpha, A, &aPacked, beta, B, (BLayout != nullptr) ? &bPacked : nullptr, | ||
| 85 | + bLayoutValid, C, &cPacked, rows, cols, stream); | ||
| 86 | +} | ||
| 87 | + | ||
| 88 | +} // extern "C" | ||
| @@ -0,0 +1,155 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_transform_desc.cpp | ||
| 13 | + * \brief Public C API: matrix transform descriptor CRUD. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_capsule.h" | ||
| 19 | +#include "host_utils.h" | ||
| 20 | +#include "matrix_transform_engine.h" | ||
| 21 | + | ||
| 22 | +#include <cstdint> | ||
| 23 | +#include <new> | ||
| 24 | + | ||
| 25 | +namespace { | ||
| 26 | + | ||
| 27 | +// Shared Set/Get prologue: reject null args, copy the capsule into a stack impl, and reject a | ||
| 28 | +// corrupted / foreign descriptor via the magic check. | ||
| 29 | +aclblasStatus_t UnpackTransformDesc( | ||
| 30 | + const aclblasLtMatrixTransformDesc_t transformDesc, const void* buf, aclblasLtMatrixTransformDescImpl& impl) | ||
| 31 | +{ | ||
| 32 | + if (transformDesc == nullptr || buf == nullptr) { | ||
| 33 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 34 | + } | ||
| 35 | + aclblasStatus_t copyStatus = CheckedMemcpyS(&impl, sizeof(impl), transformDesc, sizeof(impl)); | ||
| 36 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 37 | + return copyStatus; | ||
| 38 | + } | ||
| 39 | + if (impl.magic != ACLBLASLT_TRANSFORM_DESC_MAGIC) { | ||
| 40 | + return ACLBLAS_STATUS_INVALID_VALUE; // corrupted / foreign descriptor | ||
| 41 | + } | ||
| 42 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +} // namespace | ||
| 46 | + | ||
| 47 | +extern "C" { | ||
| 48 | + | ||
| 49 | +aclblasStatus_t aclblasLtMatrixTransformDescCreate(aclblasLtMatrixTransformDesc_t* transformDesc, aclDataType scaleType) | ||
| 50 | +{ | ||
| 51 | + if (transformDesc == nullptr) { | ||
| 52 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 53 | + } | ||
| 54 | + *transformDesc = nullptr; | ||
| 55 | + | ||
| 56 | + auto* capsule = new (std::nothrow) aclblasLtMatrixTransformDescOpaque_t(); | ||
| 57 | + if (capsule == nullptr) { | ||
| 58 | + return ACLBLAS_STATUS_ALLOC_FAILED; | ||
| 59 | + } | ||
| 60 | + | ||
| 61 | + aclblasLtMatrixTransformDescImpl impl; | ||
| 62 | + impl.magic = ACLBLASLT_TRANSFORM_DESC_MAGIC; | ||
| 63 | + impl.scaleType = scaleType; | ||
| 64 | + | ||
| 65 | + aclblasStatus_t copyStatus = MatPackTransformImpl(capsule, sizeof(*capsule), &impl, sizeof(impl)); | ||
| 66 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 67 | + delete capsule; | ||
| 68 | + return copyStatus; | ||
| 69 | + } | ||
| 70 | + | ||
| 71 | + *transformDesc = capsule; | ||
| 72 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 73 | +} | ||
| 74 | + | ||
| 75 | +aclblasStatus_t aclblasLtMatrixTransformDescDestroy(const aclblasLtMatrixTransformDesc_t transformDesc) | ||
| 76 | +{ | ||
| 77 | + if (transformDesc == nullptr) { | ||
| 78 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 79 | + } | ||
| 80 | + auto* capsule = reinterpret_cast<aclblasLtMatrixTransformDescOpaque_t*>(transformDesc); | ||
| 81 | + delete capsule; | ||
| 82 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 83 | +} | ||
| 84 | + | ||
| 85 | +aclblasStatus_t aclblasLtMatrixTransformDescSetAttribute( | ||
| 86 | + aclblasLtMatrixTransformDesc_t transformDesc, aclblasLtMatrixTransformDescAttribute_t attr, const void* buf, | ||
| 87 | + size_t sizeInBytes) | ||
| 88 | +{ | ||
| 89 | + aclblasLtMatrixTransformDescImpl impl; | ||
| 90 | + aclblasStatus_t copyStatus = UnpackTransformDesc(transformDesc, buf, impl); | ||
| 91 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 92 | + return copyStatus; | ||
| 93 | + } | ||
| 94 | + if (sizeInBytes != sizeof(int32_t)) { | ||
| 95 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 96 | + } | ||
| 97 | + int32_t v = 0; | ||
| 98 | + copyStatus = CheckedMemcpyS(&v, sizeof(v), buf, sizeof(int32_t)); | ||
| 99 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 100 | + return copyStatus; | ||
| 101 | + } | ||
| 102 | + | ||
| 103 | + switch (attr) { | ||
| 104 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_SCALE_TYPE: | ||
| 105 | + impl.scaleType = static_cast<aclDataType>(v); | ||
| 106 | + break; | ||
| 107 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_POINTER_MODE: | ||
| 108 | + impl.pointerMode = v; | ||
| 109 | + break; | ||
| 110 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSA: | ||
| 111 | + impl.transA = static_cast<aclblasOperation_t>(v); | ||
| 112 | + break; | ||
| 113 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSB: | ||
| 114 | + impl.transB = static_cast<aclblasOperation_t>(v); | ||
| 115 | + break; | ||
| 116 | + default: | ||
| 117 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 118 | + } | ||
| 119 | + | ||
| 120 | + return MatPackTransformImpl(transformDesc, sizeof(*transformDesc), &impl, sizeof(impl)); | ||
| 121 | +} | ||
| 122 | + | ||
| 123 | +aclblasStatus_t aclblasLtMatrixTransformDescGetAttribute( | ||
| 124 | + aclblasLtMatrixTransformDesc_t transformDesc, aclblasLtMatrixTransformDescAttribute_t attr, void* buf, | ||
| 125 | + size_t sizeInBytes, size_t* sizeWritten) | ||
| 126 | +{ | ||
| 127 | + aclblasLtMatrixTransformDescImpl impl; | ||
| 128 | + aclblasStatus_t copyStatus = UnpackTransformDesc(transformDesc, buf, impl); | ||
| 129 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 130 | + return copyStatus; | ||
| 131 | + } | ||
| 132 | + | ||
| 133 | + const void* srcPtr = nullptr; | ||
| 134 | + switch (attr) { | ||
| 135 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_SCALE_TYPE: | ||
| 136 | + srcPtr = &impl.scaleType; | ||
| 137 | + break; | ||
| 138 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_POINTER_MODE: | ||
| 139 | + srcPtr = &impl.pointerMode; | ||
| 140 | + break; | ||
| 141 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSA: | ||
| 142 | + srcPtr = &impl.transA; | ||
| 143 | + break; | ||
| 144 | + case ACLBLASLT_MATRIX_TRANSFORM_DESC_TRANSB: | ||
| 145 | + srcPtr = &impl.transB; | ||
| 146 | + break; | ||
| 147 | + default: | ||
| 148 | + return ACLBLAS_STATUS_NOT_SUPPORTED; | ||
| 149 | + } | ||
| 150 | + | ||
| 151 | + const size_t requiredSize = sizeof(int32_t); | ||
| 152 | + return CopyAttributeOut(buf, sizeInBytes, srcPtr, requiredSize, sizeWritten); | ||
| 153 | +} | ||
| 154 | + | ||
| 155 | +} // extern "C" | ||
| @@ -0,0 +1,56 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_version.cpp | ||
| 13 | + * \brief Public C API: library version and property queries. | ||
| 14 | + */ | ||
| 15 | + | ||
| 16 | +#include "cann_ops_blasLt.h" | ||
| 17 | + | ||
| 18 | +#include "aclblaslt_handle_impl.h" | ||
| 19 | + | ||
| 20 | +#include <cstddef> | ||
| 21 | + | ||
| 22 | +extern "C" { | ||
| 23 | + | ||
| 24 | +aclblasStatus_t aclblasLtGetVersion(size_t* version) | ||
| 25 | +{ | ||
| 26 | + if (version == nullptr) { | ||
| 27 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 28 | + } | ||
| 29 | + | ||
| 30 | + *version = (static_cast<size_t>(ACLBLASLT_VERSION_MAJOR) << 24) | | ||
| 31 | + (static_cast<size_t>(ACLBLASLT_VERSION_MINOR) << 16) | static_cast<size_t>(ACLBLASLT_VERSION_PATCH); | ||
| 32 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 33 | +} | ||
| 34 | + | ||
| 35 | +aclblasStatus_t aclblasLtGetProperty(aclblasLtPropertyType_t type, int* value) | ||
| 36 | +{ | ||
| 37 | + if (value == nullptr) { | ||
| 38 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 39 | + } | ||
| 40 | + | ||
| 41 | + switch (type) { | ||
| 42 | + case ACLBLASLT_PROPERTY_MAJOR_VERSION: | ||
| 43 | + *value = ACLBLASLT_VERSION_MAJOR; | ||
| 44 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 45 | + case ACLBLASLT_PROPERTY_MINOR_VERSION: | ||
| 46 | + *value = ACLBLASLT_VERSION_MINOR; | ||
| 47 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 48 | + case ACLBLASLT_PROPERTY_PATCH_LEVEL: | ||
| 49 | + *value = ACLBLASLT_VERSION_PATCH; | ||
| 50 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 51 | + default: | ||
| 52 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 53 | + } | ||
| 54 | +} | ||
| 55 | + | ||
| 56 | +} // extern "C" | ||
| @@ -44,6 +44,42 @@ inline bool MemcpySSucceeds(void* dest, size_t destMax, const void* src, size_t | |||
| 44 | return memcpy_s(dest, destMax, src, count) == EOK; | 44 | return memcpy_s(dest, destMax, src, count) == EOK; |
| 45 | } | 45 | } |
| 46 | 46 | ||
| 47 | +// Copy implBytes from impl into capsule (capacity capsuleBytes) and zero the trailing reserved bytes. | ||
| 48 | +// Shared by every descriptor Create/Init/SetAttribute path to avoid re-implementing the pack step. | ||
| 49 | +inline aclblasStatus_t PackImplIntoCapsule(void* capsule, size_t capsuleBytes, const void* impl, size_t implBytes) | ||
| 50 | +{ | ||
| 51 | + aclblasStatus_t copyStatus = CheckedMemcpyS(capsule, capsuleBytes, impl, implBytes); | ||
| 52 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 53 | + return copyStatus; | ||
| 54 | + } | ||
| 55 | + if (capsuleBytes > implBytes) { | ||
| 56 | + copyStatus = CheckedMemsetS( | ||
| 57 | + reinterpret_cast<char*>(capsule) + implBytes, capsuleBytes - implBytes, capsuleBytes - implBytes); | ||
| 58 | + } | ||
| 59 | + return copyStatus; | ||
| 60 | +} | ||
| 61 | + | ||
| 62 | +// Shared GetAttribute out-parameter tail: validate the caller buffer size, copy requiredSize bytes | ||
| 63 | +// from src, and report the written size. On undersized buffers, sizeWritten still reports requiredSize. | ||
| 64 | +inline aclblasStatus_t CopyAttributeOut( | ||
| 65 | + void* buf, size_t sizeInBytes, const void* src, size_t requiredSize, size_t* sizeWritten) | ||
| 66 | +{ | ||
| 67 | + if (sizeInBytes < requiredSize) { | ||
| 68 | + if (sizeWritten != nullptr) { | ||
| 69 | + *sizeWritten = requiredSize; | ||
| 70 | + } | ||
| 71 | + return ACLBLAS_STATUS_INVALID_VALUE; | ||
| 72 | + } | ||
| 73 | + aclblasStatus_t copyStatus = CheckedMemcpyS(buf, sizeInBytes, src, requiredSize); | ||
| 74 | + if (copyStatus != ACLBLAS_STATUS_SUCCESS) { | ||
| 75 | + return copyStatus; | ||
| 76 | + } | ||
| 77 | + if (sizeWritten != nullptr) { | ||
| 78 | + *sizeWritten = requiredSize; | ||
| 79 | + } | ||
| 80 | + return ACLBLAS_STATUS_SUCCESS; | ||
| 81 | +} | ||
| 82 | + | ||
| 47 | template <typename T> | 83 | template <typename T> |
| 48 | inline aclblasStatus_t AllocHandle(T** out) | 84 | inline aclblasStatus_t AllocHandle(T** out) |
| 49 | { | 85 | { |
| @@ -0,0 +1,117 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_algo_heuristic.h | ||
| 13 | + * \brief Internal matmul algo encode/decode, the serialized PackedAlgo layout, tiling override, and | ||
| 14 | + * the heuristic candidate scoring helpers shared by the algo API and the matmul engine. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#pragma once | ||
| 18 | + | ||
| 19 | +#include "aclblaslt_handle_impl.h" | ||
| 20 | +#include "matmul_fp32_tiling_data.h" | ||
| 21 | + | ||
| 22 | +#include "cann_ops_blasLt.h" | ||
| 23 | + | ||
| 24 | +#include <acl/acl.h> | ||
| 25 | +#include <cstdint> | ||
| 26 | +#include <vector> | ||
| 27 | + | ||
| 28 | +constexpr uint32_t ACLBLASLT_ALGO_MAGIC = 0xACBD1234; | ||
| 29 | + | ||
| 30 | +constexpr uint32_t DEFAULT_L1_M = 128; | ||
| 31 | +constexpr uint32_t DEFAULT_L1_N = 128; | ||
| 32 | +constexpr uint32_t DEFAULT_L1_K = 128; | ||
| 33 | +// Default algoId for policy=MMAD_SYNC, L1 tile 128x128x128, splitK=1. The bit shifts mirror | ||
| 34 | +// GenerateAlgoId(): [28:31] policy | [16:23] l1m | [8:15] l1n | [2:9] l1k | [0:1] splitK. | ||
| 35 | +constexpr uint32_t DEFAULT_ALGO_ID = (0u << 28) ^ (128u << 16) ^ (128u << 8) ^ (128u << 2) ^ 1u; | ||
| 36 | + | ||
| 37 | +// PackedAlgo.flags bit layout: | ||
| 38 | +// bits [0:1] reduction scheme (aclblasLtReductionScheme_t) | ||
| 39 | +// bits [4:5] L1 K-tile index (0 = unset/default 128, 1 = 64, 2 = 128, 3 = 256) | ||
| 40 | +// Encoding l1k in the spare flag bits lets heuristic/matmul distinguish tiles that share the same | ||
| 41 | +// M x N but differ in K without growing the 16-byte PackedAlgo layout, so the aclblasLtMatmulAlgo_t | ||
| 42 | +// ABI stays unchanged. | ||
| 43 | +constexpr uint8_t FLAG_REDUCTION_MASK = 0x03; | ||
| 44 | +constexpr uint8_t FLAG_L1K_SHIFT = 4; | ||
| 45 | +constexpr uint8_t FLAG_L1K_MASK = 0x30; | ||
| 46 | + | ||
| 47 | +enum DispatchPolicyType : uint8_t { | ||
| 48 | + DISPATCH_POLICY_MMAD_SYNC = 0, | ||
| 49 | + DISPATCH_POLICY_MMAD_PINGPONG = 1, | ||
| 50 | + DISPATCH_POLICY_MMAD_MULTI_STAGE = 2, | ||
| 51 | +}; | ||
| 52 | + | ||
| 53 | +struct AscendHardwareCaps { | ||
| 54 | + uint32_t numAICores = DEFAULT_AI_CORES; | ||
| 55 | + uint32_t l0CubeSize = L0_SIZE; | ||
| 56 | + size_t l1BufferSize = L1_SIZE; | ||
| 57 | + double memoryBandwidthGBps = DEFAULT_PEAK_GBPS; | ||
| 58 | + double peakTFlops = DEFAULT_PEAK_TFLOPS; | ||
| 59 | + double bandwidthBoundThreshold = 32.0; | ||
| 60 | +}; | ||
| 61 | + | ||
| 62 | +struct AlgoCandidate { | ||
| 63 | + uint32_t algoId = 0; | ||
| 64 | + uint32_t l1TileM = 128; | ||
| 65 | + uint32_t l1TileN = 128; | ||
| 66 | + uint32_t l1TileK = 128; | ||
| 67 | + uint32_t l0TileM = 64; | ||
| 68 | + uint32_t l0TileN = 64; | ||
| 69 | + uint32_t l0TileK = 64; | ||
| 70 | + DispatchPolicyType policy = DISPATCH_POLICY_MMAD_SYNC; | ||
| 71 | + uint32_t numBuffers = 1; | ||
| 72 | + uint32_t splitKFactor = 1; | ||
| 73 | + size_t workspaceSize = 0; | ||
| 74 | + double peakPerformance = DEFAULT_PEAK_TFLOPS; | ||
| 75 | +}; | ||
| 76 | + | ||
| 77 | +struct PackedAlgo { | ||
| 78 | + uint32_t magic; | ||
| 79 | + uint32_t algoId; | ||
| 80 | + uint16_t l1mDiv16; | ||
| 81 | + uint16_t l1nDiv16; | ||
| 82 | + uint8_t policy; | ||
| 83 | + uint8_t numBuffers; | ||
| 84 | + uint8_t splitK; | ||
| 85 | + uint8_t flags; | ||
| 86 | +}; | ||
| 87 | +static_assert(sizeof(PackedAlgo) == 16, "PackedAlgo must fit algo.data"); | ||
| 88 | + | ||
| 89 | +// ---- algoId / flags codec ---- | ||
| 90 | +uint8_t EncodeL1KToFlags(uint8_t flags, uint32_t l1k); | ||
| 91 | +uint32_t DecodeL1KFromFlags(uint8_t flags); | ||
| 92 | +uint32_t GenerateAlgoId(DispatchPolicyType policy, uint32_t l1m, uint32_t l1n, uint32_t l1k, uint32_t splitKFactor); | ||
| 93 | + | ||
| 94 | +// ---- PackedAlgo (de)serialization ---- | ||
| 95 | +aclblasLtMatmulAlgo_t BuildAlgoFromCandidate(const AlgoCandidate& cand); | ||
| 96 | +bool DecodeAlgo(const aclblasLtMatmulAlgo_t& algo, PackedAlgo* packed); | ||
| 97 | + | ||
| 98 | +// Apply the decoded algorithm's L1 tile onto the FP32 kernel tiling (N and K tiles only take effect; | ||
| 99 | +// see the definition for the rationale). | ||
| 100 | +void ApplyAlgoTilingOverrideFp32( | ||
| 101 | + const PackedAlgo& packed, uint64_t m, uint64_t n, uint64_t k, MatmulFp32TilingData& tiling); | ||
| 102 | + | ||
| 103 | +// ---- ACLBLASLT_MATMUL_TILE_* <-> L1 M/N mapping ---- | ||
| 104 | +bool TileIdToL1MN(uint32_t tileId, uint16_t* l1mDiv16, uint16_t* l1nDiv16); | ||
| 105 | +uint32_t L1MNToTileId(uint32_t l1m, uint32_t l1n); | ||
| 106 | + | ||
| 107 | +// ---- algoId enumeration / validation ---- | ||
| 108 | +bool DecodeAlgoIdFields( | ||
| 109 | + uint32_t algoId, uint8_t* policy, uint16_t* l1mDiv16, uint16_t* l1nDiv16, uint32_t* l1k, uint8_t* splitK); | ||
| 110 | +std::vector<int> EnumerateValidAlgoIds(); | ||
| 111 | + | ||
| 112 | +// ---- heuristic scoring helpers ---- | ||
| 113 | +void GetAscendHardwareCaps(int32_t deviceId, AscendHardwareCaps* caps); | ||
| 114 | +void SelectL0TileShape( | ||
| 115 | + uint32_t l1M, uint32_t l1N, uint32_t l1K, size_t, size_t, aclDataType, aclDataType, uint32_t* l0M, uint32_t* l0N, | ||
| 116 | + uint32_t* l0K); | ||
| 117 | +size_t CalculateWorkspaceForAscend(uint64_t m, uint64_t n, uint32_t splitKFactor, aclblasLtEpilogue_t epilogue); | ||
| @@ -0,0 +1,24 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_capsule.h | ||
| 13 | + * \brief Helpers to pack an internal impl struct into its opaque public capsule, zero-padding the | ||
| 14 | + * trailing reserved bytes. Not installed. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#pragma once | ||
| 18 | + | ||
| 19 | +#include "cann_ops_blasLt.h" | ||
| 20 | + | ||
| 21 | +#include <cstddef> | ||
| 22 | + | ||
| 23 | +// Copy implBytes from impl into capsule (capacity capsuleBytes) and zero the remaining bytes. | ||
| 24 | +aclblasStatus_t MatPackTransformImpl(void* capsule, size_t capsuleBytes, const void* impl, size_t implBytes); | ||
| @@ -0,0 +1,112 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_handle_impl.h | ||
| 13 | + * \brief Internal handle struct, its algo LRU cache types, library version / hardware constants, | ||
| 14 | + * and device-capability queries. Not installed. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#pragma once | ||
| 18 | + | ||
| 19 | +#include "cann_ops_blasLt.h" | ||
| 20 | + | ||
| 21 | +#include <acl/acl.h> | ||
| 22 | +#include <cstddef> | ||
| 23 | +#include <cstdint> | ||
| 24 | +#include <list> | ||
| 25 | +#include <mutex> | ||
| 26 | +#include <unordered_map> | ||
| 27 | + | ||
| 28 | +constexpr int ACLBLASLT_VERSION_MAJOR = 1; | ||
| 29 | +constexpr int ACLBLASLT_VERSION_MINOR = 0; | ||
| 30 | +constexpr int ACLBLASLT_VERSION_PATCH = 0; | ||
| 31 | + | ||
| 32 | +constexpr uint32_t ACLBLASLT_HANDLE_MAGIC = 0xACBA1234; | ||
| 33 | + | ||
| 34 | +// Ascend on-chip / peak defaults, used by handle setup and the algo heuristic. | ||
| 35 | +constexpr size_t L1_SIZE = 512 * 1024; | ||
| 36 | +constexpr size_t L0_SIZE = 256; | ||
| 37 | +constexpr uint32_t DEFAULT_AI_CORES = 8; | ||
| 38 | +constexpr double DEFAULT_PEAK_TFLOPS = 140.0; | ||
| 39 | +constexpr double DEFAULT_PEAK_GBPS = 900.0; | ||
| 40 | + | ||
| 41 | +struct AlgoKey { | ||
| 42 | + uint64_t m = 0; | ||
| 43 | + uint64_t n = 0; | ||
| 44 | + uint64_t k = 0; | ||
| 45 | + aclDataType aType = ACL_FLOAT; | ||
| 46 | + aclDataType bType = ACL_DT_UNDEFINED; | ||
| 47 | + aclDataType cType = ACL_DT_UNDEFINED; | ||
| 48 | + aclDataType dType = ACL_DT_UNDEFINED; | ||
| 49 | + aclblasComputeType_t computeType = ACLBLAS_COMPUTE_32F; | ||
| 50 | + aclblasLtEpilogue_t epilogue = ACLBLASLT_EPILOGUE_DEFAULT; | ||
| 51 | + bool transA = false; | ||
| 52 | + bool transB = false; | ||
| 53 | + | ||
| 54 | + bool operator==(const AlgoKey& other) const | ||
| 55 | + { | ||
| 56 | + return m == other.m && n == other.n && k == other.k && aType == other.aType && bType == other.bType && | ||
| 57 | + cType == other.cType && dType == other.dType && computeType == other.computeType && | ||
| 58 | + epilogue == other.epilogue && transA == other.transA && transB == other.transB; | ||
| 59 | + } | ||
| 60 | +}; | ||
| 61 | + | ||
| 62 | +struct AlgoKeyHasher { | ||
| 63 | + size_t operator()(const AlgoKey& x) const | ||
| 64 | + { | ||
| 65 | + size_t h = 1469598103934665603ull; | ||
| 66 | + auto mix = [&](uint64_t v) { h ^= static_cast<size_t>(v + 0x9e3779b97f4a7c15ull + (h << 6) + (h >> 2)); }; | ||
| 67 | + mix(x.m); | ||
| 68 | + mix(x.n); | ||
| 69 | + mix(x.k); | ||
| 70 | + mix(static_cast<uint64_t>(x.aType)); | ||
| 71 | + mix(static_cast<uint64_t>(x.bType)); | ||
| 72 | + mix(static_cast<uint64_t>(x.cType)); | ||
| 73 | + mix(static_cast<uint64_t>(x.dType)); | ||
| 74 | + mix(static_cast<uint64_t>(x.computeType)); | ||
| 75 | + mix(static_cast<uint64_t>(x.epilogue)); | ||
| 76 | + mix(static_cast<uint64_t>(x.transA)); | ||
| 77 | + mix(static_cast<uint64_t>(x.transB)); | ||
| 78 | + return h; | ||
| 79 | + } | ||
| 80 | +}; | ||
| 81 | + | ||
| 82 | +struct CacheEntry { | ||
| 83 | + aclblasLtMatmulAlgo_t algo; | ||
| 84 | + std::list<AlgoKey>::iterator lruIter; | ||
| 85 | +}; | ||
| 86 | + | ||
| 87 | +struct aclblasLtHandle { | ||
| 88 | + uint32_t magic = ACLBLASLT_HANDLE_MAGIC; | ||
| 89 | + bool initialized = false; | ||
| 90 | + // version info | ||
| 91 | + int versionMajor = ACLBLASLT_VERSION_MAJOR; | ||
| 92 | + int versionMinor = ACLBLASLT_VERSION_MINOR; | ||
| 93 | + // AscendCL runtime | ||
| 94 | + aclrtContext context = nullptr; | ||
| 95 | + aclrtStream defaultStream = nullptr; | ||
| 96 | + int32_t deviceId = 0; | ||
| 97 | + // workspace | ||
| 98 | + void* internalWorkspace = nullptr; | ||
| 99 | + size_t workspaceSize = 0; | ||
| 100 | + // thread safety | ||
| 101 | + std::mutex* mutex = nullptr; | ||
| 102 | + // soc spec | ||
| 103 | + int npuArch = 0; | ||
| 104 | + size_t maxSharedMemory = 0; | ||
| 105 | + // algo cache | ||
| 106 | + std::unordered_map<AlgoKey, CacheEntry, AlgoKeyHasher>* algoCache = nullptr; | ||
| 107 | + size_t algoCacheMaxSize = 128; | ||
| 108 | + std::list<AlgoKey>* lruList = nullptr; | ||
| 109 | +}; | ||
| 110 | + | ||
| 111 | +// Query the number of cube cores for a device, returning a safe fallback when the runtime query fails. | ||
| 112 | +uint32_t QueryCubeCoreNum(int32_t deviceId); | ||
| @@ -0,0 +1,79 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_layout_impl.h | ||
| 13 | + * \brief Internal capsule-backed impl structs for MatrixLayout / MatmulDesc / MatmulPreference, | ||
| 14 | + * plus their capsule magics and shared defaults. Not installed. | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#pragma once | ||
| 18 | + | ||
| 19 | +#include "cann_ops_blasLt.h" | ||
| 20 | + | ||
| 21 | +#include <acl/acl.h> | ||
| 22 | +#include <cstddef> | ||
| 23 | +#include <cstdint> | ||
| 24 | + | ||
| 25 | +constexpr uint32_t ACLBLASLT_LAYOUT_MAGIC = 0xACBB1234; | ||
| 26 | +constexpr uint32_t ACLBLASLT_DESC_MAGIC = 0xACBC1234; | ||
| 27 | + | ||
| 28 | +constexpr size_t DEFAULT_WORKSPACE_SIZE = 32 * 1024 * 1024; | ||
| 29 | +constexpr size_t BIAS_PTR_STORAGE_BYTES = sizeof(void*); | ||
| 30 | + | ||
| 31 | +struct aclblasLtMatrixLayoutImpl { | ||
| 32 | + uint32_t magic; | ||
| 33 | + aclDataType type; | ||
| 34 | + uint64_t rows; | ||
| 35 | + uint64_t cols; | ||
| 36 | + int64_t ld; | ||
| 37 | + aclblasLtOrder_t order = ACLBLASLT_ORDER_COL; | ||
| 38 | + int32_t batchCount = 1; | ||
| 39 | + int64_t stridedBatchOffset = 0; | ||
| 40 | +}; | ||
| 41 | +static_assert( | ||
| 42 | + sizeof(aclblasLtMatrixLayoutImpl) <= sizeof(aclblasLtMatrixLayoutOpaque_t), | ||
| 43 | + "Impl of aclblasLtMatrixLayout must fit in capsule!"); | ||
| 44 | + | ||
| 45 | +struct aclblasLtMatmulDescImpl { | ||
| 46 | + uint32_t magic; | ||
| 47 | + aclblasComputeType_t computeType; | ||
| 48 | + aclDataType scaleType; | ||
| 49 | + aclblasOperation_t transA = ACLBLAS_OP_N; | ||
| 50 | + aclblasOperation_t transB = ACLBLAS_OP_N; | ||
| 51 | + aclblasLtEpilogue_t epilogue = ACLBLASLT_EPILOGUE_DEFAULT; | ||
| 52 | + const void* bias = nullptr; | ||
| 53 | + aclDataType biasDataType = ACL_DT_UNDEFINED; | ||
| 54 | + const void* scaleA = nullptr; | ||
| 55 | + const void* scaleB = nullptr; | ||
| 56 | +}; | ||
| 57 | +static_assert( | ||
| 58 | + sizeof(aclblasLtMatmulDescImpl) <= sizeof(aclblasLtMatmulDescOpaque_t), | ||
| 59 | + "Impl of aclblasLtMatmulDesc must fit in capsule!"); | ||
| 60 | + | ||
| 61 | +struct aclblasLtMatmulPreferenceImpl { | ||
| 62 | + uint32_t magic; | ||
| 63 | + uint32_t searchMode = 0; | ||
| 64 | + size_t maxWorkspaceBytes = DEFAULT_WORKSPACE_SIZE; | ||
| 65 | + int32_t maxResults = 3; | ||
| 66 | + bool allowMixedPrecision = true; | ||
| 67 | + bool allowSplitK = true; | ||
| 68 | + // tiling | ||
| 69 | + uint32_t preferredL0M = 0; | ||
| 70 | + uint32_t preferredL0N = 0; | ||
| 71 | + uint32_t preferredL0K = 0; | ||
| 72 | + // Scheduling | ||
| 73 | + bool preferPingpong = false; | ||
| 74 | + bool preferDoubleBuffer = false; | ||
| 75 | + float minEfficiency = 0.5f; | ||
| 76 | +}; | ||
| 77 | +static_assert( | ||
| 78 | + sizeof(aclblasLtMatmulPreferenceImpl) <= sizeof(aclblasLtMatmulPreferenceOpaque_t), | ||
| 79 | + "Impl of aclblasLtMatmulPreference must fit in capsule!"); | ||
| @@ -0,0 +1,76 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file aclblaslt_matmul_problem.h | ||
| 13 | + * \brief Plain problem description consumed by the matmul engine. The API layer fills it from the | ||
| 14 | + * validated capsules so the engine works on a POD struct (easy to unit test / mock). | ||
| 15 | + */ | ||
| 16 | + | ||
| 17 | +#pragma once | ||
| 18 | + | ||
| 19 | +#include "aclblaslt_layout_impl.h" | ||
| 20 | + | ||
| 21 | +#include "cann_ops_blasLt.h" | ||
| 22 | + | ||
| 23 | +#include <acl/acl.h> | ||
| 24 | +#include <cstddef> | ||
| 25 | +#include <cstdint> | ||
| 26 | + | ||
| 27 | +struct MatmulProblem { | ||
| 28 | + // dimensions | ||
| 29 | + uint64_t m = 0; | ||
| 30 | + uint64_t n = 0; | ||
| 31 | + uint64_t k = 0; | ||
| 32 | + bool transA = false; | ||
| 33 | + bool transB = false; | ||
| 34 | + | ||
| 35 | + // types | ||
| 36 | + aclDataType dtypeA = ACL_DT_UNDEFINED; | ||
| 37 | + aclDataType dtypeB = ACL_DT_UNDEFINED; | ||
| 38 | + aclDataType dtypeC = ACL_DT_UNDEFINED; | ||
| 39 | + aclDataType dtypeD = ACL_DT_UNDEFINED; | ||
| 40 | + | ||
| 41 | + // scalars | ||
| 42 | + float alpha = 1.0f; | ||
| 43 | + float beta = 0.0f; | ||
| 44 | + | ||
| 45 | + // device pointers | ||
| 46 | + const void* A = nullptr; | ||
| 47 | + const void* B = nullptr; | ||
| 48 | + const void* C = nullptr; | ||
| 49 | + void* D = nullptr; | ||
| 50 | + void* scaleA = nullptr; | ||
| 51 | + void* scaleB = nullptr; | ||
| 52 | + void* workspace = nullptr; | ||
| 53 | + size_t workspaceSize = 0; | ||
| 54 | + | ||
| 55 | + // layout | ||
| 56 | + uint32_t lda = 0; | ||
| 57 | + uint32_t ldb = 0; | ||
| 58 | + uint32_t ldc = 0; | ||
| 59 | + uint32_t ldd = 0; | ||
| 60 | + | ||
| 61 | + // execution context | ||
| 62 | + uint32_t numBlocks = 0; | ||
| 63 | + aclrtStream stream = nullptr; | ||
| 64 | + | ||
| 65 | + // optional caller-supplied algo (raw capsule, decoded by the engine) | ||
| 66 | + const aclblasLtMatmulAlgo_t* algo = nullptr; | ||
| 67 | +}; | ||
| 68 | + | ||
| 69 | +// Assemble a MatmulProblem from already-validated descriptor / layout impls and call parameters. | ||
| 70 | +// Dimensions (m, n, k) and the cube-core count (numBlocks) are computed by the caller and passed in. | ||
| 71 | +void BuildMatmulProblem( | ||
| 72 | + const aclblasLtMatmulDescImpl* desc, const aclblasLtMatrixLayoutImpl* aLayout, | ||
| 73 | + const aclblasLtMatrixLayoutImpl* bLayout, const aclblasLtMatrixLayoutImpl* cLayout, | ||
| 74 | + const aclblasLtMatrixLayoutImpl* dLayout, uint64_t m, uint64_t n, uint64_t k, const void* alpha, const void* beta, | ||
| 75 | + const void* A, const void* B, const void* C, void* D, const aclblasLtMatmulAlgo_t* algo, void* workspace, | ||
| 76 | + size_t workspaceSize, uint32_t numBlocks, aclrtStream stream, MatmulProblem& out); | ||
RblasLt/include/host/quant_matmul_mx_tiling_swat_host.h→blasLt/matmul/common/quant_matmul_mx_tiling_swat_host.h+1-1
RblasLt/epilogue/arch35/epilogue_alpha_beta_host.cpp→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_host.cpp+2-1
RblasLt/include/kernel/matmul_kernel.h→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_host.h+2-25
RblasLt/epilogue/arch35/epilogue_alpha_beta_kernel.cpp→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_kernel.cpp+1-0
RblasLt/include/host/epilogue_alpha_beta_tiling_data.h→blasLt/matmul/epilogue/arch35/epilogue_alpha_beta_tiling_data.h+0-0
RblasLt/matmul_fp32/arch35/matmul_fp32_kernel.cpp→blasLt/matmul/fp32/arch35/matmul_fp32_kernel.cpp+4-4
RblasLt/matmul_mxfp4/arch35/matmul_mxfp4_host.cpp→blasLt/matmul/mxfp4/arch35/matmul_mxfp4_host.cpp+3-4
RblasLt/matmul_mxfp4/arch35/matmul_mxfp4_kernel.cpp→blasLt/matmul/mxfp4/arch35/matmul_mxfp4_kernel.cpp+2-1
RblasLt/matmul_mxfp8/arch35/matmul_mxfp8_host.cpp→blasLt/matmul/mxfp8/arch35/matmul_mxfp8_host.cpp+2-2
RblasLt/matmul_mxfp8/arch35/matmul_mxfp8_kernel.cpp→blasLt/matmul/mxfp8/arch35/matmul_mxfp8_kernel.cpp+3-2
RblasLt/include/host/matrix_transform_get_tiling.h→blasLt/matrix_transform/arch35/matrix_transform_get_tiling.h+0-0
RblasLt/matrixtransform/arch35/matrix_transform_host.cpp→blasLt/matrix_transform/arch35/matrix_transform_host.cpp+1-1
RblasLt/matrixtransform/arch35/matrix_transform_kernel.cpp→blasLt/matrix_transform/arch35/matrix_transform_kernel.cpp+0-0
RblasLt/include/kernel/matrix_transform_kernel.h→blasLt/matrix_transform/arch35/matrix_transform_kernel.h+0-0
RblasLt/include/host/matrix_transform_perm_table.h→blasLt/matrix_transform/arch35/matrix_transform_perm_table.h+0-0
RblasLt/include/host/matrix_transform_tiling_data.h→blasLt/matrix_transform/arch35/matrix_transform_tiling_data.h+0-0
RblasLt/include/host/matrix_transform_acl_impl.h→blasLt/matrix_transform/engine/matrix_transform_engine.h+1-1