* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
#include <iostream>
#include <vector>
#include "acl/acl.h"
#include "aclnnop/aclnn_rope_with_sin_cos_cache.h"
#include <iostream>
#define CHECK_RET(cond, return_expr) \
do { \
if (!(cond)) { \
return_expr; \
} \
} while (0)
#define LOG_PRINT(message, ...) \
do { \
printf(message, ##__VA_ARGS__); \
} while (0)
int64_t GetShapeSize(const std::vector<int64_t> &shape) {
int64_t shapeSize = 1;
for (auto i : shape) {
shapeSize *= i;
}
return shapeSize;
}
void PrintOutResult(std::vector<int64_t> &shape, void **deviceAddr) {
auto size = GetShapeSize(shape);
std::vector<float> resultData(size, 0);
auto ret = aclrtMemcpy(
resultData.data(), resultData.size() * sizeof(resultData[0]), *deviceAddr,
size * sizeof(resultData[0]), ACL_MEMCPY_DEVICE_TO_HOST);
CHECK_RET(
ret == ACL_SUCCESS,
LOG_PRINT("copy result from device to host failed. ERROR: %d\n", ret);
return );
for (int64_t i = 0; i < size; i++) {
LOG_PRINT("mean result[%ld] is: %f\n", i, resultData[i]);
}
}
int Init(int32_t deviceId, aclrtStream *stream) {
auto ret = aclInit(nullptr);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclInit failed. ERROR: %d\n", ret);
return ret);
ret = aclrtSetDevice(deviceId);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret);
return ret);
ret = aclrtCreateStream(stream);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret);
return ret);
return 0;
}
template <typename T>
int CreateAclTensor(const std::vector<T> &hostData,
const std::vector<int64_t> &shape, void **deviceAddr,
aclDataType dataType, aclTensor **tensor) {
auto size = GetShapeSize(shape) * sizeof(T);
auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret);
return ret);
ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size,
ACL_MEMCPY_HOST_TO_DEVICE);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret);
return ret);
std::vector<int64_t> strides(shape.size(), 1);
for (int64_t i = shape.size() - 2; i >= 0; i--) {
strides[i] = shape[i + 1] * strides[i + 1];
}
*tensor = aclCreateTensor(shape.data(), shape.size(), dataType,
strides.data(), 0, aclFormat::ACL_FORMAT_ND,
shape.data(), shape.size(), *deviceAddr);
return 0;
}
int main() {
int32_t deviceId = 0;
aclrtStream stream;
auto ret = Init(deviceId, &stream);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("Init acl failed. ERROR: %d\n", ret);
return ret);
std::vector<int64_t> positionsShape = {2};
std::vector<int64_t> queryInShape = {2, 64};
std::vector<int64_t> keyInShape = {2, 64};
std::vector<int64_t> cosSinCacheShape = {2, 32};
std::vector<int64_t> queryOutShape = {2, 64};
std::vector<int64_t> keyOutShape = {2, 64};
void* positionsDeviceAddr = nullptr;
void* queryInDeviceAddr = nullptr;
void* keyInDeviceAddr = nullptr;
void* cosSinCacheDeviceAddr = nullptr;
void* queryOutDeviceAddr = nullptr;
void* keyOutDeviceAddr = nullptr;
aclTensor* positions = nullptr;
aclTensor* queryIn = nullptr;
aclTensor* keyIn = nullptr;
aclTensor* cosSinCache = nullptr;
int64_t headSize = 32;
bool isNeoxStyle = true;
aclTensor *queryOut = nullptr;
aclTensor *keyOut = nullptr;
std::vector<int64_t> positionsHostData = {0, 1};
std::vector<float> queryInHostData = {74, 54, 84, 125, 23, 78, 37, 72, 27, 98, 34, 107, 29, 23, 54, 60, 70, 49,
119, 54, 29, 54, 41, 99, 27, 62, 5, 46, 108, 39, 24, 123, 33, 82, 6, 40, 88,
24, 6, 116, 38, 119, 110, 5, 30, 79, 87, 18, 29, 100, 90, 24, 21, 93, 63, 68,
34, 112, 119, 48, 74, 43, 85, 64, 14, 49, 128, 59, 18, 37, 123, 76, 14, 63, 10,
39, 107, 124, 79, 16, 17, 76, 80, 47, 90, 41, 58, 82, 75, 80, 69, 37, 74, 36, 54,
26, 32, 54, 13, 100, 105, 15, 13, 69, 122, 26, 94, 59, 29, 14, 60, 8, 24, 17, 45,
33, 107, 122, 63, 111, 75, 128, 68, 31, 105, 6, 82, 99};
std::vector<float> keyInHostData = {112, 32, 66, 114, 69, 31, 117, 122, 77, 57, 78, 119, 115, 25, 54, 27, 122, 65, 15, 85,
33, 16, 36, 6, 95, 15, 43, 6, 66, 91, 14, 101, 78, 51, 110, 74, 56, 30, 127, 61, 53, 29,
32, 65, 114, 77, 26, 116, 89, 38, 75, 14, 96, 91, 87, 34, 25, 42, 57, 26, 51, 43, 23, 42,
40, 17, 98, 117, 53, 75, 68, 75, 38, 41, 115, 76, 67, 22, 76, 10, 24, 46, 85, 54, 61, 114,
10, 59, 6, 123, 58, 10, 115, 9, 13, 58, 66, 120, 23, 30, 83, 13, 11, 76, 18, 82, 57, 4,
117, 105, 8, 73, 127, 5, 91, 56, 12, 125, 20, 3, 104, 40, 46, 18, 89, 63, 99, 104};
std::vector<float> cosSinCacheHostData = {112, 32, 66, 114, 69, 31, 117, 122, 77, 57, 78, 119, 115, 25, 54, 27, 122, 65, 15, 85,
33, 16, 36, 6, 95, 15, 43, 6, 66, 91, 14, 101, 78, 51, 110, 74, 56, 30, 127, 61, 53, 29,
32, 65, 114, 77, 26, 116, 89, 38, 75, 14, 96, 91, 87, 34, 25, 42, 57, 26, 51, 43, 23, 42};
std::vector<float> queryOutHostData = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
std::vector<float> keyOutHostData = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
ret = CreateAclTensor(positionsHostData, positionsShape,
&positionsDeviceAddr, aclDataType::ACL_INT64,
&positions);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(queryInHostData, queryInShape, &queryInDeviceAddr,
aclDataType::ACL_BF16, &queryIn);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(keyInHostData, keyInShape, &keyInDeviceAddr,
aclDataType::ACL_BF16, &keyIn);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(cosSinCacheHostData, cosSinCacheShape, &cosSinCacheDeviceAddr,
aclDataType::ACL_BF16, &cosSinCache);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(queryOutHostData, queryOutShape, &queryOutDeviceAddr, aclDataType::ACL_BF16,
&queryOut);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(keyOutHostData, keyOutShape, &keyOutDeviceAddr, aclDataType::ACL_BF16,
&keyOut);
CHECK_RET(ret == ACL_SUCCESS, return ret);
uint64_t workspaceSize = 0;
aclOpExecutor *executor;
ret = aclnnRopeWithSinCosCacheGetWorkspaceSize(positions, queryIn, keyIn, cosSinCache, nullptr, headSize, isNeoxStyle,
queryOut, keyOut, &workspaceSize, &executor);
CHECK_RET(
ret == ACL_SUCCESS,
LOG_PRINT("aclnnRopeWithSinCosCacheGetWorkspaceSize failed. ERROR: %d\n", ret);
return ret);
void *workspaceAddr = nullptr;
if (workspaceSize > 0) {
ret = aclrtMalloc(&workspaceAddr, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("allocate workspace failed. ERROR: %d\n", ret);
return ret);
}
ret = aclnnRopeWithSinCosCache(workspaceAddr, workspaceSize, executor, stream);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("aclnnRopeWithSinCosCache failed. ERROR: %d\n", ret);
return ret);
ret = aclrtSynchronizeStream(stream);
CHECK_RET(ret == ACL_SUCCESS,
LOG_PRINT("aclrtSynchronizeStream failed. ERROR: %d\n", ret);
return ret);
PrintOutResult(queryOutShape, &queryOutDeviceAddr);
PrintOutResult(keyOutShape, &keyOutDeviceAddr);
aclDestroyTensor(positions);
aclDestroyTensor(queryIn);
aclDestroyTensor(keyIn);
aclDestroyTensor(cosSinCache);
aclDestroyTensor(queryOut);
aclDestroyTensor(keyOut);
aclrtFree(positionsDeviceAddr);
aclrtFree(queryInDeviceAddr);
aclrtFree(keyInDeviceAddr);
aclrtFree(cosSinCacheDeviceAddr);
aclrtFree(queryOutDeviceAddr);
aclrtFree(keyOutDeviceAddr);
if (workspaceSize > 0) {
aclrtFree(workspaceAddr);
}
aclrtDestroyStream(stream);
aclrtResetDevice(deviceId);
aclFinalize();
return 0;
}