/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/* !
* \file matmul.asc
* \brief
*/
#include <cstdint>
#include <iostream>
#include <vector>
#include <algorithm>
#include <iterator>
#include "acl/acl.h"
#include "kernel_operator.h"
#include "lib/matmul_intf.h"
#include "tiling/platform/platform_ascendc.h"
#include "tiling/tiling_api.h"
#include "kernel_tiling/kernel_tiling.h"
#include "data_utils.h"
using namespace matmul;
__aicore__ inline uint32_t Ceiling(uint32_t a, uint32_t b)
{
return (a + b - 1) / b;
}
/**
* @brief Copy tiling data to TCubeTiling ptr from tiling gm addr.
* @param tiling: TCubeTiling ptr which needs to copy tiling data.
* @param localMemSize: Temporary local memory size required by matmul calc.
* @param tilingGM: tiling gm addr.
* @retval None
*/
__aicore__ inline void CopyTiling(TCubeTiling *tiling, uint64_t &localMemSize, GM_ADDR tilingGM)
{
uint32_t *ptr = reinterpret_cast<uint32_t *>(tiling);
auto tiling32 = reinterpret_cast<__gm__ uint32_t *>(tilingGM);
for (uint32_t i = 0; i < sizeof(TCubeTiling) / sizeof(uint32_t); i++, ptr++) {
*ptr = *(tiling32 + i);
}
localMemSize = *reinterpret_cast<__gm__ uint64_t *>(tilingGM + sizeof(TCubeTiling));
return;
}
/**
* @brief Calculate the gm offset and tail size based on the blockidx.
* @param blockIdx: Current Core blockidx.
* @param tiling: Matmul tiling data.
* @param offsetA: Gm offset of A matrix.
* @param offsetB: Gm offset of B matrix.
* @param offsetC: Gm offset of C matrix.
* @param tailM: SingleCoreM size of tail core.
* @param tailN: SingleCoreN size of tail core.
* @param isTransA: A matrix transpose.
* @param isTransB: B matrix transpose.
* @retval None
*/
__aicore__ inline void CalcGMOffset(int blockIdx, const TCubeTiling &tiling, int &offsetA, int &offsetB, int &offsetC,
int &tailM, int &tailN, bool isTransA, bool isTransB)
{
uint32_t mSingleBlocks = Ceiling(tiling.M, tiling.singleCoreM);
uint32_t mCoreIndx = blockIdx % mSingleBlocks;
uint32_t nCoreIndx = blockIdx / mSingleBlocks;
offsetA = mCoreIndx * tiling.Ka * tiling.singleCoreM;
if (isTransA) {
offsetA = mCoreIndx * tiling.singleCoreM;
}
offsetB = nCoreIndx * tiling.singleCoreN;
if (isTransB) {
offsetB = nCoreIndx * tiling.Kb * tiling.singleCoreN;
}
offsetC = mCoreIndx * tiling.N * tiling.singleCoreM + nCoreIndx * tiling.singleCoreN;
tailM = tiling.M - mCoreIndx * tiling.singleCoreM;
tailM = tailM < tiling.singleCoreM ? tailM : tiling.singleCoreM;
tailN = tiling.N - nCoreIndx * tiling.singleCoreN;
tailN = tailN < tiling.singleCoreN ? tailN : tiling.singleCoreN;
}
/**
* @brief matmul kernel function entry
* @param a: A matrix gm addr.
* @param b: B matrix gm addr.
* @param c: C matrix gm addr.
* @param workspace: Temporary gm space addr required by matmul calc.
* @param tilingGm: Tiling data addr.
* @retval None
*/
__global__ __aicore__ void matmul_custom(GM_ADDR a, GM_ADDR b, GM_ADDR c, __kfc_workspace__ GM_ADDR workspace,
GM_ADDR tilingGm)
{
using A_T = half;
using B_T = half;
using C_T = float;
AscendC::TPipe pipe;
TCubeTiling tiling;
uint64_t localMemSize = 0;
CopyTiling(&tiling, localMemSize, tilingGm);
AscendC::GlobalTensor<A_T> aGlobal;
AscendC::GlobalTensor<B_T> bGlobal;
AscendC::GlobalTensor<C_T> cGlobal;
aGlobal.SetGlobalBuffer(reinterpret_cast<__gm__ A_T *>(a), tiling.M * tiling.Ka);
bGlobal.SetGlobalBuffer(reinterpret_cast<__gm__ B_T *>(b), tiling.Ka * tiling.N);
cGlobal.SetGlobalBuffer(reinterpret_cast<__gm__ C_T *>(c), tiling.M * tiling.N);
int offsetA = 0;
int offsetB = 0;
int offsetC = 0;
bool isTransA = false;
bool isTransB = false;
int tailM = 0;
int tailN = 0;
// Calculate the gm offset and tail size based on the blockidx.
CalcGMOffset(GetBlockIdx(), tiling, offsetA, offsetB, offsetC, tailM, tailN, isTransA, isTransB);
auto gmA = aGlobal[offsetA];
auto gmB = bGlobal[offsetB];
auto gmC = cGlobal[offsetC];
Matmul<MatmulType<AscendC::TPosition::GM, CubeFormat::ND, A_T>,
MatmulType<AscendC::TPosition::GM, CubeFormat::ND, B_T>,
MatmulType<AscendC::TPosition::GM, CubeFormat::ND, C_T>> mm;
REGIST_MATMUL_OBJ(&pipe, GetSysWorkSpacePtr(), mm, &tiling); // Initialize the matmul object.
if (GetBlockIdx() >= tiling.usedCoreNum) {
return;
}
mm.SetOrgShape(tiling.M, tiling.N, tiling.Ka, tiling.Kb);
mm.SetTensorA(gmA, isTransA);
mm.SetTensorB(gmB, isTransB);
mm.SetTail(tailM, tailN);
mm.IterateAll(gmC);
mm.End();
}
void GenerateTiling(const char *socVersion, uint8_t *tilingBuf)
{
using TPosition = matmul_tiling::TPosition;
using CubeFormat = matmul_tiling::CubeFormat;
using DataType = matmul_tiling::DataType;
constexpr int32_t M = 512;
constexpr int32_t N = 1024;
constexpr int32_t K = 512;
TPosition leftPosition = TPosition::GM;
CubeFormat leftFormat = CubeFormat::ND;
DataType leftDtype = DataType::DT_FLOAT16;
bool isTransA = false;
TPosition rightPosition = TPosition::GM;
CubeFormat rightFormat = CubeFormat::ND;
DataType rightDtype = DataType::DT_FLOAT16;
bool isTransB = false;
TPosition resultPosition = TPosition::GM;
CubeFormat resultFormat = CubeFormat::ND;
DataType resultDtype = DataType::DT_FLOAT;
bool isBias = false;
constexpr int32_t SINGLECORE_M = 512;
constexpr int32_t SINGLECORE_N = 512;
optiling::TCubeTiling tilingData;
auto ascendcPlatform = platform_ascendc::PlatformAscendCManager::GetInstance(socVersion);
matmul_tiling::MultiCoreMatmulTiling tilingApi(*ascendcPlatform);
tilingApi.SetDim(ascendcPlatform->GetCoreNumAiv()); // Set the number of cores that participate in multi-core computaion is 48.
tilingApi.SetAType(leftPosition, leftFormat, leftDtype, isTransA);
tilingApi.SetBType(rightPosition, rightFormat, rightDtype, isTransB);
tilingApi.SetCType(resultPosition, resultFormat, resultDtype);
tilingApi.SetOrgShape(M, N, K);
tilingApi.SetShape(M, N, K);
tilingApi.SetBias(isBias);
tilingApi.SetBufferSpace(-1, -1, -1);
int64_t res = tilingApi.GetTiling(tilingData); // Get matmul tiling data.
if (res == -1) {
std::cout << "gen tiling failed" << std::endl;
}
uint32_t tcubeTilingSize = tilingData.GetDataSize();
tilingData.SaveToBuffer(tilingBuf, tcubeTilingSize);
uint64_t localMemSize;
ascendcPlatform->GetCoreMemSize(platform_ascendc::CoreMemType::UB, localMemSize);
*reinterpret_cast<uint64_t *>(tilingBuf + tcubeTilingSize) = localMemSize;
return;
}
int32_t main(int32_t argc, char *argv[])
{
const char *socVersion = "Ascend910B2";
auto ascendcPlatform = platform_ascendc::PlatformAscendCManager::GetInstance(socVersion);
size_t aFileSize = 512 * 512 * sizeof(uint16_t); // uint16_t represent half
size_t bFileSize = 512 * 1024 * sizeof(uint16_t); // uint16_t represent half
size_t cFileSize = 512 * 1024 * sizeof(float);
// matmul TCubeTiling + localMemorySize
size_t tilingFileSize = sizeof(TCubeTiling) + sizeof(uint64_t);
size_t userWorkspaceSize = 0;
size_t systemWorkspaceSize = static_cast<size_t>(ascendcPlatform->GetLibApiWorkSpaceSize());
size_t workspaceSize = userWorkspaceSize + systemWorkspaceSize;
uint8_t *tilingBuf = (uint8_t *)malloc(tilingFileSize);
GenerateTiling(socVersion, tilingBuf);
uint32_t numBlocks = (reinterpret_cast<TCubeTiling *>(tilingBuf)->usedCoreNum + 1) / 2;
aclInit(nullptr);
int32_t deviceId = 0;
aclrtSetDevice(deviceId);
aclrtStream stream = nullptr;
aclrtCreateStream(&stream);
uint8_t *aHost;
uint8_t *aDevice;
aclrtMallocHost((void **)(&aHost), aFileSize);
aclrtMalloc((void **)&aDevice, aFileSize, ACL_MEM_MALLOC_HUGE_FIRST);
ReadFile("./input/x1_gm.bin", aFileSize, aHost, aFileSize);
aclrtMemcpy(aDevice, aFileSize, aHost, aFileSize, ACL_MEMCPY_HOST_TO_DEVICE);
uint8_t *bHost;
uint8_t *bDevice;
aclrtMallocHost((void **)(&bHost), bFileSize);
aclrtMalloc((void **)&bDevice, bFileSize, ACL_MEM_MALLOC_HUGE_FIRST);
ReadFile("./input/x2_gm.bin", bFileSize, bHost, bFileSize);
aclrtMemcpy(bDevice, bFileSize, bHost, bFileSize, ACL_MEMCPY_HOST_TO_DEVICE);
uint8_t *workspaceDevice;
aclrtMalloc((void **)&workspaceDevice, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST);
uint8_t *tilingHost;
uint8_t *tilingDevice;
aclrtMallocHost((void **)(&tilingHost), tilingFileSize);
aclrtMalloc((void **)&tilingDevice, tilingFileSize, ACL_MEM_MALLOC_HUGE_FIRST);
aclrtMemcpy(tilingHost, tilingFileSize, tilingBuf, tilingFileSize, ACL_MEMCPY_HOST_TO_HOST);
aclrtMemcpy(tilingDevice, tilingFileSize, tilingHost, tilingFileSize, ACL_MEMCPY_HOST_TO_DEVICE);
uint8_t *cHost;
uint8_t *cDevice;
aclrtMallocHost((void **)(&cHost), cFileSize);
aclrtMalloc((void **)&cDevice, cFileSize, ACL_MEM_MALLOC_HUGE_FIRST);
matmul_custom<<<numBlocks, nullptr, stream>>>
(aDevice, bDevice, cDevice, workspaceDevice, tilingDevice);
aclrtSynchronizeStream(stream);
aclrtMemcpy(cHost, cFileSize, cDevice, cFileSize, ACL_MEMCPY_DEVICE_TO_HOST);
WriteFile("./output/output.bin", cHost, cFileSize);
aclrtFree(aDevice);
aclrtFreeHost(aHost);
aclrtFree(bDevice);
aclrtFreeHost(bHost);
aclrtFree(workspaceDevice);
aclrtFree(tilingDevice);
aclrtFreeHost(tilingHost);
aclrtFree(cDevice);
aclrtFreeHost(cHost);
aclrtDestroyStream(stream);
aclrtResetDevice(deviceId);
aclFinalize();
free(tilingBuf);
return 0;
}
