aclnnQuantCompressor
产品支持情况
- Ascend 950PR&950DT系列产品:支持
- Atlas A3系列产品:不支持
- Atlas A2系列产品:不支持
- Atlas 200I/500 A2推理产品:不支持
- Atlas推理系列产品:不支持
- Atlas训练系列产品:不支持
功能说明
-
接口功能:QuantCompressor是推理场景下SMLA和QLI的前处理算子,是Compressor的量化版本。用于将每4或128个token的KV cache压缩成一个,然后每个token与这些压缩的KV cache进行DSA计算。在长序列的情况下,QuantCompressor可以有效地减少计算开销。与Compressor的区别在于:输入xx、WKVW^{KV}、WGateW^{Gate}为HIFLOAT8量化数据,直接以HIFLOAT8参与Matmul运算(硬件原生支持),再对Matmul输出的FLOAT32结果乘以合并后的缩放因子进行反量化,从而降低显存占用与搬运开销。主要计算过程为:
- Matmul与反量化:将HIFLOAT8量化的输入XX与WKVW^{KV}做Matmul运算得到FLOAT32结果,再乘以合并缩放因子x_descale⋅wkv_descalex\_descale \cdot wkv\_descale完成反量化得到kv_statekv\_state;将XX与WGateW^{Gate}做Matmul运算得到FLOAT32结果,再乘以合并缩放因子x_descale⋅wgate_descalex\_descale \cdot wgate\_descale完成反量化得到score_statescore\_state。其中x_descale为per-tensor缩放(单个标量),wkv_descale与wgate_descale为per-channel缩放(通道数为coff*D),合并后仍为per-channel缩放。kv_statekv\_state与score_statescore\_state根据输入的start_pos及cu_seqlens完成更新。
- 在coff为2的情况下对kv_statekv\_state和score_statescore\_state进行数据重排。
- 对score_statescore\_state按压缩比分组并与ApeApe相加,然后进行softmax运算,将softmax结果与kv_statekv\_state做Mul计算,后进行ReduceSum运算。
-
计算公式:
-
计算矩阵乘法与反量化(quant_mode=1,A8W8_A_HIFP8_PER_TENSOR_W_HIFP8_PER_CHANNEL):
使用HIFLOAT8输入进行Matmul,输出FLOAT32结果后乘以合并缩放因子(x_descale与wkv/wgate_descale的乘积)完成反量化:
C4A:[kv_statea,score_statea]=(Xhif8@[Whif8aKV,Whif8aGate])⋅(x_descale⋅[wkv_descalea,wgate_descalea]),[kv_stateb,score_stateb]=(Xhif8@[Whif8bKV,Whif8bGate])⋅(x_descale⋅[wkv_descaleb,wgate_descaleb]);C4A:\left[kv\_state^a, score\_state^a\right] = (X_{hif8} @ \left[W^{aKV}_{hif8}, W^{aGate}_{hif8}\right]) \cdot (x\_descale \cdot [wkv\_descale^a, wgate\_descale^a]), \left[kv\_state^b, score\_state^b\right] = (X_{hif8} @ \left[W^{bKV}_{hif8}, W^{bGate}_{hif8}\right]) \cdot (x\_descale \cdot [wkv\_descale^b, wgate\_descale^b]);
C128A:[kv_state,score_state]=(Xhif8@[Whif8KV,Whif8Gate])⋅(x_descale⋅[wkv_descale,wgate_descale])C128A:\left[kv\_state, score\_state\right] = (X_{hif8} @ \left[W^{KV}_{hif8}, W^{Gate}_{hif8}\right]) \cdot (x\_descale \cdot [wkv\_descale, wgate\_descale])
其中x_descale为per-tensor标量(shape为[1,]),wkv_descale、wgate_descale为per-channel向量(shape为[coff*D,],沿权重输出通道维度缩放),两者相乘后合并为per-channel缩放因子。
-
计算分组加法:
C4A:score_statei′=[score_state[4(i−1)+1:4i,:]a;score_state[4i+1:4(i+1),:]b]+Ape, i=1,2,⋯ ,s4;C4A:score\_state_i^\prime = \left[score\_state_{\left[4(i-1)+1:4i,:\right]}^a; score\_state_{\left[4i+1:4(i+1),:\right]}^b\right] + Ape,~i=1,2,\cdots, \frac{s}{4};
C128A:score_statei′=score_state[128(i−1)+1:128i,:]+Ape, i=1,2,⋯ ,s128;C128A:score\_state_i^\prime = score\_state_{\left[128(i-1)+1:128i,:\right]} + Ape,~i=1,2,\cdots, \frac{s}{128};
-
计算分组Softmax:
C4A:Si′=softmax(score_statei′), i=1,2,⋯ ,s4;C4A:S_i^\prime = softmax(score\_state_i^\prime),~i=1,2,\cdots, \frac{s}{4};
C128A:Si′=softmax(score_statei′), i=1,2,⋯ ,s128;C128A:S_i^\prime = softmax(score\_state_i^\prime),~i=1,2,\cdots, \frac{s}{128};
-
计算Hadamard乘积:
C4A:(SH)i=Si′⊙[kv_state[4(i−1)+1:4i,:]a;kv_state[4i+1:4(i+1),:]b], i=1,2,⋯ ,s4;C4A:(S_H)_i = S_i^\prime \odot \left[kv\_state^a_{\left[4(i-1)+1:4i,:\right]} ;kv\_state^b_{\left[4i+1:4(i+1),:\right]}\right],~i=1,2,\cdots, \frac{s}{4};
C128A:SH=Si′⊙kv_state;C128A:S_H = S_i^\prime \odot kv\_state;
-
沿着压缩轴分组求和:
C4A:CiComp=[1]1×8@(SH)i, i=1,2,⋯ ,s4;C4A:C_{i}^{\text{Comp}} = \left[1\right]_{1\times8} @ (S_H)_i, ~i=1,2,\cdots, \frac{s}{4};
C128A:CiComp=[1]1×128@(SH)i, i=1,2,⋯ ,s128;C128A:C_{i}^{\text{Comp}} = \left[1\right]_{1\times128} @ (S_H)_i, ~i=1,2,\cdots, \frac{s}{128};
-
函数原型
每个算子分为两段式接口,必须先调用“aclnnQuantCompressorGetWorkspaceSize”接口获取入参并根据流程计算所需workspace大小,再调用“aclnnQuantCompressor”接口执行计算。
aclnnStatus aclnnQuantCompressorGetWorkspaceSize(
const aclTensor *x,
const aclTensor *wkv,
const aclTensor *wgate,
aclTensor *stateCacheRef,
const aclTensor *ape,
const aclTensor *xDescaleOptional,
const aclTensor *wkvDescaleOptional,
const aclTensor *wgateDescaleOptional,
const aclTensor *stateBlockTableOptional,
const aclTensor *cuSeqlensOptional,
const aclTensor *sequsedOptional,
const aclTensor *startPosOptional,
int64_t quantMode,
int64_t cmpRatio,
int64_t coff,
int64_t cacheMode,
int64_t stateCacheStrideDim0,
const aclTensor *cmpKvOut,
uint64_t *workspaceSize,
aclOpExecutor **executor)
aclnnStatus aclnnQuantCompressor(
void *workspace,
uint64_t workspaceSize,
aclOpExecutor *executor,
aclrtStream stream)
aclnnQuantCompressorGetWorkspaceSize
-
参数说明
参数名 输入/输出 描述 使用说明 数据类型 数据格式 维度(shape) 非连续Tensor x(const aclTensor*) 输入 公式中的X,表示原始不经压缩的数据,HIFLOAT8量化输入。 支持B=0,S=0,T=0的空Tensor。 HIFLOAT8 ND BS合轴:[T,H]、BS非合轴:[B,S,H] × wkv(const aclTensor*) 输入 公式中的WKV,表示kv压缩权重,HIFLOAT8量化输入。 不支持空Tensor。 HIFLOAT8 ND [coff* D,H] × wgate(const aclTensor*) 输入 公式中的WGate,表示gate压缩权重,HIFLOAT8量化输入。 不支持空Tensor。 HIFLOAT8 ND [coff* D,H] × stateCacheRef(aclTensor*) 输入/输出 公式中的[kv_state, score_state],表示kv_state和score_state的历史数据,原地更新。 不支持空Tensor。 FLOAT32 ND [block_num,block_size,2* coff* D] 支持0轴非连续 ape(const aclTensor*) 输入 公式中的Ape,表示positional biases。 不支持空Tensor。 FLOAT32 ND [cmp_ratio,coff* D] × xDescaleOptional(const aclTensor*) 可选输入 x的反量化缩放因子,per-tensor缩放。quant_mode=1时必选。 不支持空Tensor。 FLOAT32 ND [1,] × wkvDescaleOptional(const aclTensor*) 可选输入 wkv的反量化缩放因子,per-channel缩放,通道数为coff*D。quant_mode=1时必选。 不支持空Tensor。 FLOAT32 ND [coff* D,] × wgateDescaleOptional(const aclTensor*) 可选输入 wgate的反量化缩放因子,per-channel缩放,通道数为coff*D。quant_mode=1时必选。 不支持空Tensor。 FLOAT32 ND [coff* D,] × stateBlockTableOptional(const aclTensor*) 可选输入 表示state_cache存储使用的block映射表。 - cacheMode=1时,shape为[B, ceil(Smax/block_size)],Smax为每个Batch中最大的Sequence Length。当x的shape为[B,S,H]时,Smax=max(startPosOptional)+S;当x的shape为[T,H]时,Smax=max(startPosOptional)+max(cuSeqlensOptional[n+1] - cuSeqlensOptional[n])。
- cacheMode=2时,shape为[B]。
- 在当前算子支持的cachemode=1/2场景下,该参数为必选。
INT32 ND cacheMode=1时:[B, ceil(Smax/block_size)]
cacheMode=2时:[B]× cuSeqlensOptional(const aclTensor*) 可选输入 表示不同Batch中的有效token数。 支持B=0,S=0,T=0的空Tensor;当x的shape为[B,S,H]时,参数必须为空。 INT32 ND 当x的shape为[T,H]时,输入shape为[B+1,] × sequsedOptional(const aclTensor*) 可选输入 表示不同Batch中实际参与压缩的token数。 如果指定为None时,表示和每个Batch上的Sequence Length相同;支持B=0的空Tensor。该入参中每个Batch的有效token数要求小于等于对应Sequence Length。当x的shape为[B,S,H]时,要求sequsedOptional[n] <= S,且不小于0;当x的shape为[T,H]时,要求sequsedOptional[n] <= cuSeqlensOptional[n+1] - cuSeqlensOptional[n],且不小于0。 INT32 ND [B,] × startPosOptional(const aclTensor*) 可选输入 表示计算起始位置。 支持B=0,T=0的空Tensor;当输入为None时,表示从0开始进行计算。 INT32 ND [B,] × quantMode(int64_t) 输入 量化模式。 取值范围为[1],1表示A8W8_A_HIFP8_PER_TENSOR_W_HIFP8_PER_CHANNEL(HIFLOAT8输入,x按per-tensor缩放、wkv/wgate按per-channel缩放反量化)。 INT64 - - - cmpRatio(int64_t) 输入 用于稀疏计算,表示数据压缩率。 取值范围为[2, 128]内的整数。 INT64 - - - coff(int64_t) 输入 表示是否进行overlap数据重排。 取值范围为{1, 2}。当coff=1时,无需进行overlap数据重排。当coff=2时,需要进行overlap数据重排。 INT64 - - - cacheMode(int64_t) 输入 表示state_cache的存储模式。 取值范围为[1, 2];1表示连续buffer,2表示循环buffer。 INT64 - - - stateCacheStrideDim0(int64_t) 输入 表示state_cache的0轴stride。 - INT64 - - - cmpKvOut(aclTensor*) 输出 表示压缩后的数据。 - 支持B=0,S=0,T=0的空Tensor。
- 尾部不足一个压缩组的pad行内容未定义(算子不写入,保留调用前内存内容),调用方应只消费有效的压缩行。
BFLOAT16 ND BS合轴:[min(T,T//cmp_ratio+B),D]、BS非合轴:[B,ceil(S/cmp_ratio),D] × workspaceSize(uint64_t*) 输出 表示需要在Device侧申请的workspace大小。 - - - - - executor(aclOpExecutor**) 输出 op执行器,包含了算子计算流程。 - - - - - -
返回值
aclnnStatus:返回状态码,具体参见aclnn返回码。
第一段接口执行参数校验与tiling时,出现以下场景时报错:
返回值 错误码 描述 ACLNN_ERR_PARAM_NULLPTR 161001 必须传入的参数(如接口核心依赖的输入/输出参数)中存在空指针。 ACLNN_ERR_PARAM_INVALID 161002 输入参数的数据类型(dtype)不在接口支持的范围内。 ACLNN_ERR_RUNTIME_ERROR 361001 API调用NPU Runtime接口时发生异常(如Runtime服务未启动、内存申请失败等)。 ACLNN_ERR_INNER_TILING_ERROR 561002 tiling发生异常:入参的shape(维度/尺寸)、属性值或可选输入的传入/缺失与当前配置不匹配(如cuSeqlensOptional在BSH布局下被传入、quantMode=1时descale缺失)。
aclnnQuantCompressor
-
参数说明
参数名 输入/输出 描述 workspace 输入 在Device侧申请的workspace内存地址。 workspaceSize 输入 在Device侧申请的workspace大小,由第一段接口aclnnQuantCompressorGetWorkspaceSize获取。 executor 输入 op执行器,包含了算子计算流程。 stream 输入 指定执行任务的Stream。 -
返回值
aclnnStatus:返回状态码,具体参见aclnn返回码。
约束说明
-
确定性计算:
- aclnnQuantCompressor默认确定性实现。
- batch一致性:通过aclrtSetSysParamOpt()配置ACL_OPT_DETERMINISTIC为3来开启batch一致性,开启后可以满足计算结果和所在批次大小、位置无关。
-
x参数维度含义:B(Batch Size)表示输入样本批量大小、S(Sequence Length)表示输入样本序列长度、H(Head Size)表示hidden层的大小、D(Head Dim)表示hidden层的最小单元大小、T表示所有Batch输入样本序列长度的累加和。
-
输入shape限制:
- stateCache支持输入shape[block_num,block_size,2*coff*D],要求blockNum>0,cacheMode=2时,需要满足blockSize >= coff * cmp_ratio + S - 1。
- 若x的维度采用BS合轴,即x的输入shape为[T,H]
- cuSeqlensOptional输入shape必须为[B+1,]。该参数中每个元素的值表示当前batch与之前所有batch的token数总和,即前缀和,因此后一个元素的值必须大于等于前一个元素的值,且第一位必须为0,最后一位必须为T。
- sequsedOptional,支持输入shape[B,],要求每个Batch的有效token数小于等于对应Sequence Length长度,即sequsedOptional[n] <= cuSeqlensOptional[n+1] - cuSeqlensOptional[n],且不小于0。
- cacheMode=1时,stateBlockTableOptional支持输入shape[B,ceil(Smax/block_size)]。Smax为每个Batch中最大的Sequence Length,即Smax=max(startPosOptional)+max(cuSeqlensOptional[n+1] - cuSeqlensOptional[n])。cacheMode=2时,stateBlockTableOptional支持输入shape[B]。
- cmpKv,输出shape为[min(T,T//cmp_ratio+B),D]:compressed_tokens + compressed_tokens + ... + compressed_tokens + pad。
- 若x的维度不采用BS合轴,即x的输入shape为[B,S,H]
- cuSeqlensOptional,参数必须为空。
- sequsedOptional,支持输入shape[B,],要求每个Batch的有效token数小于等于对应Sequence Length长度,即要求sequsedOptional[n] <= S,且不小于0。
- cacheMode=1时,stateBlockTable支持输入shape[B,ceil(Smax/block_size)]。Smax为每个Batch中最大的Sequence Length,即Smax=max(startPosOptional)+S。cacheMode=2时,stateBlockTable支持输入shape[B]。
- cmpKv,输出shape为[B,ceil(S/cmp_ratio),D]:(compressed_tokens+pad0) + (compressed_tokens+pad1) + ... + (compressed_tokens+padN)。
-
输入值域限制:
- 该接口支持B、S泛化,且存在如下场景限制:
- 部分长序列场景下,如果计算量过大可能会导致出现超过NPU内存的报错,注:这里计算量会受x输入shape的影响,值越大计算量越大。典型的长序列(即B、S的乘积或T较大)场景包括但不限于:
B S H 100 65525 4096 25 261120 4096 100 131072 4096 100 261120 4096
- 该接口支持B、S泛化,且存在如下场景限制:
-
该接口支持B、S、T取0,即shape与B、S、T值相关的入参允许传入空tensor,其余入参不支持传入空tensor。该场景下stateCache不做更新,输出cmpKv为空tensor。
-
stateBlockTableOptional元素取值范围为[0, block_num),block_num为stateCacheRef第0维大小。元素值直接用作state_cache的block索引,越界会导致内存非法访问。元素值为0时:cacheMode=1(连续buffer)下写state_cache操作跳过该位置;cacheMode=2(循环buffer)下读写操作均不跳过。算子不做重复校验,需由调用方保证元素值唯一性:cacheMode=1下元素值0为"未分配"哨兵值可重复出现,非0值须全局唯一;cacheMode=2下0为有效物理块号,所有元素值须全局唯一。重复会导致多个逻辑块/batch写同一物理块区域,造成state_cache数据踩踏覆盖。
-
输入属性限制:
- quantMode取值为1(A8W8_A_HIFP8_PER_TENSOR_W_HIFP8_PER_CHANNEL),此时xDescale、wkvDescale、wgateDescale为必选输入。
- 支持D为128/512。
- 支持H为1K~10K,512对齐。
- 支持blockSize为1~1024。
- 支持如下三种典型组合场景:
- C4A: D=512, coff=2, cmp_ratio=4;
- C4Li: D=128, coff=2, cmp_ratio=4;
- C128A: D=512, coff=1, cmp_ratio=128。
调用示例
示例代码如下,仅供参考,具体编译和执行过程请参考编译与运行样例。
/**
* Copyright (c) 2026 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file test_aclnn_quant_compressor.cpp
* \brief QuantCompressor 算子 aclnn 调用示例(A5 / ascend950)
* 场景:C4A(D=512, coff=2, cmp_ratio=4, cache_mode=1, BSH layout, HIFLOAT8)
* QuantCompressor 仅支持 ascend950。
*/
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <iostream>
#include <numeric>
#include <vector>
#include "acl/acl.h"
#include "aclnnop/aclnn_quant_compressor.h"
#include "opdev/bfloat16.h"
#include "opdev/hifloat8.h"
#define CHECK_RET(cond, return_expr) \
do { \
if (!(cond)) { \
return_expr; \
} \
} while (0)
#define LOG_PRINT(message, ...) \
do { \
printf(message, ##__VA_ARGS__); \
} while (0)
namespace {
using op::bfloat16;
using op::HiFloat8;
int64_t GetShapeSize(const std::vector<int64_t> &shape)
{
int64_t shapeSize = 1;
for (auto i : shape) {
shapeSize *= i;
}
return shapeSize;
}
int Init(int32_t deviceId, aclrtContext *context, aclrtStream *stream)
{
auto ret = aclInit(nullptr);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclInit failed. ERROR: %d\n", ret); return ret);
ret = aclrtSetDevice(deviceId);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtSetDevice failed. ERROR: %d\n", ret); return ret);
ret = aclrtCreateContext(context, deviceId);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtCreateContext failed. ERROR: %d\n", ret); return ret);
ret = aclrtSetCurrentContext(*context);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtSetCurrentContext failed. ERROR: %d\n", ret); return ret);
ret = aclrtCreateStream(stream);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtCreateStream failed. ERROR: %d\n", ret); return ret);
return 0;
}
template <typename T>
int CreateAclTensor(const std::vector<T> &hostData, const std::vector<int64_t> &shape, void **deviceAddr,
aclDataType dataType, aclTensor **tensor)
{
auto size = GetShapeSize(shape) * sizeof(T);
if (size > 0) {
auto ret = aclrtMalloc(deviceAddr, size, ACL_MEM_MALLOC_HUGE_FIRST);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtMalloc failed. ERROR: %d\n", ret); return ret);
ret = aclrtMemcpy(*deviceAddr, size, hostData.data(), size, ACL_MEMCPY_HOST_TO_DEVICE);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtMemcpy failed. ERROR: %d\n", ret); return ret);
} else {
*deviceAddr = nullptr;
}
std::vector<int64_t> strides(shape.size(), 1);
for (int64_t i = static_cast<int64_t>(shape.size()) - 2; i >= 0; i--) {
strides[i] = shape[i + 1] * strides[i + 1];
}
*tensor = aclCreateTensor(shape.data(), shape.size(), dataType, strides.data(), 0, aclFormat::ACL_FORMAT_ND,
shape.data(), shape.size(), *deviceAddr);
return 0;
}
void PrintBf16Result(const std::vector<int64_t> &shape, void **deviceAddr)
{
auto size = GetShapeSize(shape);
std::vector<bfloat16> resultData(size, bfloat16(0.0f));
auto ret = aclrtMemcpy(resultData.data(), resultData.size() * sizeof(resultData[0]), *deviceAddr,
size * sizeof(resultData[0]), ACL_MEMCPY_DEVICE_TO_HOST);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("copy result from device to host failed. ERROR: %d\n", ret); return);
for (int64_t i = 0; i < size && i < 10; i++) { // 10: max print
LOG_PRINT("cmp_kv[%ld] is: %f\n", i, static_cast<float>(resultData[i]));
}
}
} // namespace
int main()
{
// 1. device/stream 初始化
int32_t deviceId = 0;
aclrtContext context = nullptr;
aclrtStream stream = nullptr;
auto ret = Init(deviceId, &context, &stream);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("Init acl failed. ERROR: %d\n", ret); return ret);
// 场景参数: C4A (D=512, coff=2, cmp_ratio=4, cache_mode=1, BSH layout)
int64_t B = 1;
int64_t S = 4;
int64_t hiddenSize = 4096;
int64_t headDim = 512;
int64_t coff = 2;
int64_t cmpRatio = 4;
int64_t cacheMode = 1; // 1: 连续buffer (LINEAR_BUFFER)
int64_t quantMode = 1; // 1: A8W8_A_HIFP8_PER_TENSOR_W_HIFP8_PER_CHANNEL
int64_t blockSize = 128;
int64_t Smax = S;
int64_t maxBlockNumPerBatch = (Smax + blockSize - 1) / blockSize;
int64_t blockNum = B * maxBlockNumPerBatch;
int64_t coffD = coff * headDim;
int64_t stateCacheStrideDim0 = blockSize * 2 * coffD; // state_cache 0轴 stride = dim1 * dim2
// 2. 构造输入与输出 shape
std::vector<int64_t> xShape = {B, S, hiddenSize};
std::vector<int64_t> wkvShape = {coffD, hiddenSize};
std::vector<int64_t> wgateShape = {coffD, hiddenSize};
std::vector<int64_t> stateCacheShape = {blockNum, blockSize, 2 * coffD};
std::vector<int64_t> apeShape = {cmpRatio, coffD};
std::vector<int64_t> xDescaleShape = {1};
std::vector<int64_t> wkvDescaleShape = {coffD};
std::vector<int64_t> wgateDescaleShape = {coffD};
std::vector<int64_t> stateBlockTableShape = {B, maxBlockNumPerBatch};
std::vector<int64_t> startPosShape = {B};
int64_t Sr = (S + cmpRatio - 1) / cmpRatio;
std::vector<int64_t> cmpKvShape = {B, Sr, headDim};
// 3. 构造 host 数据
int64_t xSize = GetShapeSize(xShape);
int64_t wkvSize = GetShapeSize(wkvShape);
int64_t wgateSize = GetShapeSize(wgateShape);
int64_t stateCacheSize = GetShapeSize(stateCacheShape);
int64_t apeSize = GetShapeSize(apeShape);
int64_t cmpKvSize = GetShapeSize(cmpKvShape);
std::vector<HiFloat8> xHostData(xSize, HiFloat8(0.1f));
std::vector<HiFloat8> wkvHostData(wkvSize, HiFloat8(0.1f));
std::vector<HiFloat8> wgateHostData(wgateSize, HiFloat8(0.1f));
std::vector<float_t> stateCacheHostData(stateCacheSize, 0.1f);
std::vector<float_t> apeHostData(apeSize, 0.1f);
std::vector<float_t> xDescaleHostData = {0.5f};
std::vector<float_t> wkvDescaleHostData(coffD, 0.1f);
std::vector<float_t> wgateDescaleHostData(coffD, 0.1f);
std::vector<int32_t> stateBlockTableHostData;
for (int64_t i = 0; i < B * maxBlockNumPerBatch; i++) {
stateBlockTableHostData.push_back(static_cast<int32_t>(i + 1));
}
std::vector<int32_t> startPosHostData(B, 0);
std::vector<bfloat16> cmpKvHostData(cmpKvSize, bfloat16(0.0f));
// 4. 创建 aclTensor
void *xDeviceAddr = nullptr;
void *wkvDeviceAddr = nullptr;
void *wgateDeviceAddr = nullptr;
void *stateCacheDeviceAddr = nullptr;
void *apeDeviceAddr = nullptr;
void *xDescaleDeviceAddr = nullptr;
void *wkvDescaleDeviceAddr = nullptr;
void *wgateDescaleDeviceAddr = nullptr;
void *stateBlockTableDeviceAddr = nullptr;
void *startPosDeviceAddr = nullptr;
void *cmpKvDeviceAddr = nullptr;
aclTensor *x = nullptr;
aclTensor *wkv = nullptr;
aclTensor *wgate = nullptr;
aclTensor *stateCacheRef = nullptr;
aclTensor *ape = nullptr;
aclTensor *xDescale = nullptr;
aclTensor *wkvDescale = nullptr;
aclTensor *wgateDescale = nullptr;
aclTensor *stateBlockTable = nullptr;
aclTensor *startPos = nullptr;
aclTensor *cmpKvOut = nullptr;
ret = CreateAclTensor(xHostData, xShape, &xDeviceAddr, aclDataType::ACL_HIFLOAT8, &x);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(wkvHostData, wkvShape, &wkvDeviceAddr, aclDataType::ACL_HIFLOAT8, &wkv);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(wgateHostData, wgateShape, &wgateDeviceAddr, aclDataType::ACL_HIFLOAT8, &wgate);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(stateCacheHostData, stateCacheShape, &stateCacheDeviceAddr, aclDataType::ACL_FLOAT,
&stateCacheRef);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(apeHostData, apeShape, &apeDeviceAddr, aclDataType::ACL_FLOAT, &ape);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(xDescaleHostData, xDescaleShape, &xDescaleDeviceAddr, aclDataType::ACL_FLOAT, &xDescale);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(wkvDescaleHostData, wkvDescaleShape, &wkvDescaleDeviceAddr, aclDataType::ACL_FLOAT,
&wkvDescale);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(wgateDescaleHostData, wgateDescaleShape, &wgateDescaleDeviceAddr, aclDataType::ACL_FLOAT,
&wgateDescale);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(stateBlockTableHostData, stateBlockTableShape, &stateBlockTableDeviceAddr,
aclDataType::ACL_INT32, &stateBlockTable);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(startPosHostData, startPosShape, &startPosDeviceAddr, aclDataType::ACL_INT32, &startPos);
CHECK_RET(ret == ACL_SUCCESS, return ret);
ret = CreateAclTensor(cmpKvHostData, cmpKvShape, &cmpKvDeviceAddr, aclDataType::ACL_BF16, &cmpKvOut);
CHECK_RET(ret == ACL_SUCCESS, return ret);
// 5. 调用 aclnnQuantCompressorGetWorkspaceSize
uint64_t workspaceSize = 0;
aclOpExecutor *executor = nullptr;
ret = aclnnQuantCompressorGetWorkspaceSize(x, wkv, wgate, stateCacheRef, ape, xDescale, wkvDescale, wgateDescale,
stateBlockTable, nullptr, nullptr, startPos, quantMode, cmpRatio, coff,
cacheMode, stateCacheStrideDim0, cmpKvOut, &workspaceSize, &executor);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclnnQuantCompressorGetWorkspaceSize failed. ERROR: %d\n", ret);
return ret);
// 6. 申请 workspace
void *workspaceAddr = nullptr;
if (workspaceSize > 0) {
ret = aclrtMalloc(&workspaceAddr, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("allocate workspace failed. ERROR: %d\n", ret); return ret);
}
// 7. 调用 aclnnQuantCompressor
ret = aclnnQuantCompressor(workspaceAddr, workspaceSize, executor, stream);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclnnQuantCompressor failed. ERROR: %d\n", ret); return ret);
// 8. 同步等待
ret = aclrtSynchronizeStream(stream);
CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("aclrtSynchronizeStream failed. ERROR: %d\n", ret); return ret);
// 9. 获取输出
LOG_PRINT("QuantCompressor execution succeeded.\n");
PrintBf16Result(cmpKvShape, &cmpKvDeviceAddr);
// 10. 释放资源
aclDestroyTensor(x);
aclDestroyTensor(wkv);
aclDestroyTensor(wgate);
aclDestroyTensor(stateCacheRef);
aclDestroyTensor(ape);
aclDestroyTensor(xDescale);
aclDestroyTensor(wkvDescale);
aclDestroyTensor(wgateDescale);
aclDestroyTensor(stateBlockTable);
aclDestroyTensor(startPos);
aclDestroyTensor(cmpKvOut);
aclrtFree(xDeviceAddr);
aclrtFree(wkvDeviceAddr);
aclrtFree(wgateDeviceAddr);
aclrtFree(stateCacheDeviceAddr);
aclrtFree(apeDeviceAddr);
aclrtFree(xDescaleDeviceAddr);
aclrtFree(wkvDescaleDeviceAddr);
aclrtFree(wgateDescaleDeviceAddr);
aclrtFree(stateBlockTableDeviceAddr);
aclrtFree(startPosDeviceAddr);
aclrtFree(cmpKvDeviceAddr);
if (workspaceSize > 0) {
aclrtFree(workspaceAddr);
}
aclrtDestroyStream(stream);
aclrtDestroyContext(context);
aclrtResetDevice(deviceId);
aclFinalize();
return 0;
}