AAlice_zhouupdate
9d759d93Add on Sep 10History
#include <iostream>
#include <vector>
#include <getopt.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <ctime>
#include <sys/wait.h>
#include <dlfcn.h>
#include <cstdio>
#include <string>
#include <vector>
#include <iomanip>
#include <string.h>
#include <list>
#include <algorithm>
#include <stdarg.h>
#include "acl/acl.h"
#include "aclnn/aclnn_base.h"
#include "acl/acl_base.h"
#include "acl/acl_rt.h"
#include "sec_common.h"
#include "utils.h"

#include "../fuzzlib/secodeFuzz.h"
#include "aclnnop/aclnn_attention_update.h"

using namespace std;



/* ========== [CONSISTENT] 一致性压测辅助函数(gen_consistent_from_normal 生成) ========== */
static const int CONSISTENT_MAX_RUN_TIMES = 64; /* hash数组上限,防越界 */
static int CONSISTENT_RUN_TIMES = 2; /* 每用例执行次数:默认2次,对比2次输出是否一致;经环境变量 CONSISTENT_RUN_TIMES 覆盖 */

static const char* ConsistentDtypeStr(aclDataType t) {
    switch (t) {
        case ACL_FLOAT: return "FLOAT"; case ACL_FLOAT16: return "FLOAT16";
        case ACL_BF16: return "BFLOAT16"; case ACL_DOUBLE: return "DOUBLE";
        case ACL_INT8: return "INT8"; case ACL_INT16: return "INT16";
        case ACL_INT32: return "INT32"; case ACL_INT64: return "INT64";
        case ACL_UINT8: return "UINT8"; case ACL_UINT16: return "UINT16";
        case ACL_UINT32: return "UINT32"; case ACL_UINT64: return "UINT64";
        case ACL_BOOL: return "BOOL"; case ACL_COMPLEX64: return "COMPLEX64";
        case ACL_COMPLEX128: return "COMPLEX128";
        default: return "UNKNOWN";
    }
}

static const char* ConsistentFormatStr(aclFormat f) {
    switch (f) {
        case ACL_FORMAT_NCHW: return "NCHW"; case ACL_FORMAT_NHWC: return "NHWC";
        case ACL_FORMAT_ND: return "ND"; case ACL_FORMAT_NC1HWC0: return "NC1HWC0";
        case ACL_FORMAT_FRACTAL_Z: return "FRACTAL_Z"; case ACL_FORMAT_HWCN: return "HWCN";
        case ACL_FORMAT_NDHWC: return "NDHWC"; case ACL_FORMAT_FRACTAL_NZ: return "FRACTAL_NZ";
        case ACL_FORMAT_NCDHW: return "NCDHW"; case ACL_FORMAT_NC: return "NC";
        case ACL_FORMAT_NCL: return "NCL";
        default: return "OTHER";
    }
}

static size_t ConsistentSizeOfDtype(aclDataType dataType) {
    switch (dataType) {
        case ACL_FLOAT: return 4; case ACL_FLOAT16: return 2;
        case ACL_INT8: return 1; case ACL_INT32: return 4;
        case ACL_UINT8: return 1; case ACL_INT16: return 2;
        case ACL_UINT16: return 2; case ACL_UINT32: return 4;
        case ACL_INT64: return 8; case ACL_UINT64: return 8;
        case ACL_DOUBLE: return 8; case ACL_BOOL: return 1;
        case ACL_BF16: return 2; case ACL_COMPLEX64: return 8;
        case ACL_COMPLEX128: return 16;
        /* 以下 dtype 的 sizeof(T)=1(vector<int8_t>),必须显式列出,禁止落入 default:4 */
        case ACL_INT4: return 1;
        case ACL_HIFLOAT8: return 1;
        case ACL_FLOAT8_E5M2: return 1;
        case ACL_FLOAT8_E4M3FN: return 1;
        case ACL_FLOAT8_E8M0: return 1;
        case ACL_FLOAT4_E2M1: return 1;
        case ACL_FLOAT4_E1M2: return 1;
        default: return 0; /* 未知 dtype 返回 0,触发 [FUZZ_CONSISTENT_BYTES] 告警 */
    }
}

static void ConsistentPrintShape(const std::vector<int64_t>& shape) {
    printf("[");
    for (size_t k = 0; k < shape.size(); k++) {
        printf("%lld%s", (long long)shape[k], (k + 1 < shape.size()) ? "," : "");
    }
    printf("]");
}

static uint64_t ConsistentHashBuffer(const void* data, size_t len) {
    const uint8_t* p = (const uint8_t*)data;
    uint64_t hash = 1469598103934665603ULL; /* FNV-1a offset basis */
    for (size_t k = 0; k < len; k++) {
        hash ^= (uint64_t)p[k];
        hash *= 1099511628211ULL; /* FNV-1a prime */
    }
    return hash;
}

static uint64_t ConsistentHashDeviceTensor(void* deviceAddr, size_t bytes) {
    if (deviceAddr == nullptr || bytes == 0) return 0;
    void* hostBuf = malloc(bytes);
    if (hostBuf == nullptr) return 0;
    if (aclrtMemcpy(hostBuf, bytes, deviceAddr, bytes, ACL_MEMCPY_DEVICE_TO_HOST) != ACL_SUCCESS) {
        free(hostBuf);
        return 0;
    }
    uint64_t h = ConsistentHashBuffer(hostBuf, bytes);
    free(hostBuf);
    return h;
}
/* ========== [CONSISTENT] 辅助函数结束 ========== */

static int kLocalOutDtypeEnum[] = {0, 1, 27};
static int kLocalOutDtypeNum = sizeof(kLocalOutDtypeEnum) / sizeof(kLocalOutDtypeEnum[0]);

static int kUpdateTypeEnum[] = {0, 1};
static int kUpdateTypeNum = sizeof(kUpdateTypeEnum) / sizeof(kUpdateTypeEnum[0]);

static int kHeadDimValidValues[] = {8, 16, 32, 64, 128, 256, 384, 512};
static int kHeadDimNum = sizeof(kHeadDimValidValues) / sizeof(kHeadDimValidValues[0]);

int aclnnAttentionUpdate_consistent(int fuzz_num)
{
    const long long kSingleInputSizeMax = 1073741824LL;

    auto ret = Init(deviceId, &context, &stream);
    CHECK_RET(ret == ACL_SUCCESS, LOG_PRINT("Init acl failed. ERROR: %d\n", ret); return ret);
    // [CONSISTENT] md文档标注默认确定性实现,无需打开确定性计算开关
    printf("[FUZZ_DETERMINISTIC] api=aclnnAttentionUpdate, enable_deterministic=0, reason=md文档标注:aclnnAttentionUpdate默认确定性实现。\n");
    WriteDTLog("aclnnAttentionUpdate", "aclnnAttentionUpdate consistent op fuzz start \n");

    aclrtSetOpExecuteTimeOut(1091);
    DT_Set_TimeOut_Second(3000);
    DT_FUZZ_START(0, fuzz_num, (char*)__func__, 0)
    {
        int i = 0;

        void* lseDeviceAddr[16] = {nullptr};
        aclTensor* lseTensors[16] = {nullptr};
        void* localOutDeviceAddr[16] = {nullptr};
        aclTensor* localOutTensors[16] = {nullptr};
        void* outDeviceAddr = nullptr;
        aclTensor* outTensor = nullptr;
        void* lseOutDeviceAddr = nullptr;
        aclTensor* lseOutTensor = nullptr;

        uint64_t workspaceSize = 0;
        aclOpExecutor* executor;
        void* workspaceAddr = nullptr;

        // [CONSISTENT] 一致性压测:固定输入重复执行 + hash 对比用变量
        char curTilingKey[64];
        snprintf(curTilingKey, sizeof(curTilingKey), "N/A");
        int cstRunCount = 0;
        bool cstAllOk = true;
        bool cstResultPrinted = false;  // 判定路径是否已打印 FUZZ_RESULT(去重 ERROR 标签处打印)
        uint64_t cstOutTensorShapeSz = 0;  // 创建语句处捕获的 shape size(作用域安全)
        int cstOutTensorDtype = 0;  // 创建语句处捕获的 dtype(作用域安全)
        size_t cstOutTensorBytes = 0;
        void* cstOutTensorSaveBuf = nullptr;
        uint64_t cstOutTensorHashArr[CONSISTENT_MAX_RUN_TIMES];
        memset(cstOutTensorHashArr, 0, sizeof(cstOutTensorHashArr));
        uint64_t cstLseOutTensorShapeSz = 0;  // 创建语句处捕获的 shape size(作用域安全)
        int cstLseOutTensorDtype = 0;  // 创建语句处捕获的 dtype(作用域安全)
        size_t cstLseOutTensorBytes = 0;
        void* cstLseOutTensorSaveBuf = nullptr;
        uint64_t cstLseOutTensorHashArr[CONSISTENT_MAX_RUN_TIMES];
        memset(cstLseOutTensorHashArr, 0, sizeof(cstLseOutTensorHashArr));
        std::vector<size_t> cstLseDeviceAddrEltBytesArr;  // 创建点逐元素捕获的字节数
        std::vector<void*> cstLseDeviceAddrSaveBufs;
        std::vector<uint64_t> cstLseDeviceAddrHashArr[CONSISTENT_MAX_RUN_TIMES];
        std::vector<size_t> cstLocalOutDeviceAddrEltBytesArr;  // 创建点逐元素捕获的字节数
        std::vector<void*> cstLocalOutDeviceAddrSaveBufs;
        std::vector<uint64_t> cstLocalOutDeviceAddrHashArr[CONSISTENT_MAX_RUN_TIMES];

        printf("[FUZZ_LOG] plog_file: aclnnAttentionUpdate_consistent_plog_case_%d\n", fuzzi);
        LOG_PRINT("aclnnAttentionUpdate_consistent %d\n", fuzzi);
        try
        {

            // === Tensor Size distribution control ===

            int sizeBucket = *(int *)DTSetGetNumberRange(&g_Element[i++], 1, 0, 1, "i");  // [CONSISTENT] 收紧size档位(仅小/中规模), 避免大shape快照导致host OOM

            long long totalSizeMin = 0, totalSizeMax = 0;

            switch (sizeBucket) {

                case 0: totalSizeMin = 1;          totalSizeMax = 1000;          break;

                case 1: totalSizeMin = 1001;       totalSizeMax = 67108864LL;  break;  // [CONSISTENT] 中档上限收紧至64M元素(快照双份内存峰值约3GB)

                case 2: totalSizeMin = 1073741825LL; totalSizeMax = 2147483648LL; break;

                case 3: totalSizeMin = 2147483649LL; totalSizeMax = 4294967296LL; break;

            }

            int shape1 = *(int*)DTSetGetNumberRange(&g_Element[i++], 4, 1, 256, "i");
            int shape2 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");
            int shape3 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");
            int shape4 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");
            int shape5 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");
            int shape6 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");
            int shape7 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");
            int shape8 = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 1, 256, "i");

            auto localOutDtype = (aclDataType)*(int*)DT_SetGetNumberEnumV3(i++, aclDataType::ACL_FLOAT, kLocalOutDtypeEnum, kLocalOutDtypeNum);

            int dim_num = *(int*)DTSetGetNumberRange(&g_Element[i++], 2, 0, 8, "i");

            int64_t updateType = (int64_t)*(int*)DT_SetGetNumberEnum(&g_Element[i++], 0, kUpdateTypeEnum, kUpdateTypeNum);

            int sp = *(int*)DTSetGetNumberRange(&g_Element[i++], 1, 1, 16, "i");

            int headDimIndex = *(int*)DTSetGetNumberRange(&g_Element[i++], 0, 0, kHeadDimNum - 1, "i");
            int64_t headDim = kHeadDimValidValues[headDimIndex];

            std::vector<int64_t> localOutShape = {};
            if (dim_num == 0) { localOutShape = {}; } else 
            if (dim_num == 1) {
                localOutShape = {shape1, headDim};
            } else if (dim_num == 2) {
                localOutShape = {shape1, shape2, headDim};
            } else if (dim_num == 3) {
                localOutShape = {shape1, shape2, shape3, headDim};
            } else if (dim_num == 4) {
                localOutShape = {shape1, shape2, shape3, shape4, headDim};
            } else if (dim_num == 5) {
                localOutShape = {shape1, shape2, shape3, shape4, shape5, headDim};
            } else if (dim_num == 6) {
                localOutShape = {shape1, shape2, shape3, shape4, shape5, shape6, headDim};
            } else if (dim_num == 7) {
                localOutShape = {shape1, shape2, shape3, shape4, shape5, shape6, shape7, headDim};
            } else if (dim_num == 8) {
                localOutShape = {shape1, shape2, shape3, shape4, shape5, shape6, shape7, shape8, headDim};
            }

            std::vector<int64_t> lseShape = localOutShape;
            lseShape.pop_back();

            std::vector<int64_t> outShape = localOutShape;

            long long localOutSize = GetShapeSize(localOutShape);
            long long lseSize = GetShapeSize(lseShape);
            if (localOutSize > kSingleInputSizeMax || lseSize > kSingleInputSizeMax) {
                printf("shape_size too big\n"); goto ERROR;
            }

            printf("localOutDtype=%d, dim_num=%d, updateType=%lld, sp=%d, headDim=%lld, localOutSize=%lld, lseSize=%lld\n",
                   (int)localOutDtype, dim_num, (long long)updateType, sp, (long long)headDim, (long long)localOutSize, (long long)lseSize);

            aclFormat fmt = aclFormat::ACL_FORMAT_ND;

            std::vector<aclTensor*> lseList;
            for (int s = 0; s < sp; s++) {
                std::vector<float> lseHostData(lseSize, 1);
                void* lseDevAddr = nullptr;
                aclTensor* lseT = nullptr;

                // === Tensor Size distribution check ===

                long long totalSize = localOutSize + lseSize;

                if (totalSize < totalSizeMin || totalSize > totalSizeMax) { goto ERROR; }

                ret = CreateAclTensor_New(lseHostData, lseShape, &lseDevAddr, aclDataType::ACL_FLOAT, fmt, &lseT);
                CHECK_RET(ret == ACL_SUCCESS, goto ERROR);
                lseDeviceAddr[s] = lseDevAddr;                if ((size_t)s >= cstLseDeviceAddrEltBytesArr.size()) cstLseDeviceAddrEltBytesArr.resize((size_t)s + 1, 0);
                cstLseDeviceAddrEltBytesArr[s] = (size_t)GetShapeSize(lseShape) * (size_t)ConsistentSizeOfDtype(aclDataType::ACL_FLOAT);

                lseTensors[s] = lseT;
                lseList.push_back(lseT);
            }
            aclTensorList* lseTensorList = aclCreateTensorList(lseList.data(), lseList.size());

            std::vector<aclTensor*> localOutList;
            for (int s = 0; s < sp; s++) {
                if (localOutDtype == aclDataType::ACL_FLOAT) {
                    std::vector<float> localOutHostData(localOutSize, 1);
                    void* localOutDevAddr = nullptr;
                    aclTensor* localOutT = nullptr;
                    ret = CreateAclTensor_New(localOutHostData, localOutShape, &localOutDevAddr, localOutDtype, fmt, &localOutT);
                    CHECK_RET(ret == ACL_SUCCESS, goto ERROR);
                    localOutDeviceAddr[s] = localOutDevAddr;                    if ((size_t)s >= cstLocalOutDeviceAddrEltBytesArr.size()) cstLocalOutDeviceAddrEltBytesArr.resize((size_t)s + 1, 0);
                    cstLocalOutDeviceAddrEltBytesArr[s] = (size_t)GetShapeSize(localOutShape) * (size_t)ConsistentSizeOfDtype(localOutDtype);

                    localOutTensors[s] = localOutT;
                    localOutList.push_back(localOutT);
                } else if (localOutDtype == aclDataType::ACL_FLOAT16) {
                    std::vector<int16_t> localOutHostData(localOutSize, 1);
                    void* localOutDevAddr = nullptr;
                    aclTensor* localOutT = nullptr;
                    ret = CreateAclTensor_New(localOutHostData, localOutShape, &localOutDevAddr, localOutDtype, fmt, &localOutT);
                    CHECK_RET(ret == ACL_SUCCESS, goto ERROR);
                    localOutDeviceAddr[s] = localOutDevAddr;                    if ((size_t)s >= cstLocalOutDeviceAddrEltBytesArr.size()) cstLocalOutDeviceAddrEltBytesArr.resize((size_t)s + 1, 0);
                    cstLocalOutDeviceAddrEltBytesArr[s] = (size_t)GetShapeSize(localOutShape) * (size_t)ConsistentSizeOfDtype(localOutDtype);

                    localOutTensors[s] = localOutT;
                    localOutList.push_back(localOutT);
                } else if (localOutDtype == aclDataType::ACL_BF16) {
                    std::vector<int16_t> localOutHostData(localOutSize, 1);
                    void* localOutDevAddr = nullptr;
                    aclTensor* localOutT = nullptr;
                    ret = CreateAclTensor_New(localOutHostData, localOutShape, &localOutDevAddr, localOutDtype, fmt, &localOutT);
                    CHECK_RET(ret == ACL_SUCCESS, goto ERROR);
                    localOutDeviceAddr[s] = localOutDevAddr;                    if ((size_t)s >= cstLocalOutDeviceAddrEltBytesArr.size()) cstLocalOutDeviceAddrEltBytesArr.resize((size_t)s + 1, 0);
                    cstLocalOutDeviceAddrEltBytesArr[s] = (size_t)GetShapeSize(localOutShape) * (size_t)ConsistentSizeOfDtype(localOutDtype);

                    localOutTensors[s] = localOutT;
                    localOutList.push_back(localOutT);
                }
            }
            aclTensorList* localOutTensorList = aclCreateTensorList(localOutList.data(), localOutList.size());

            if (localOutDtype == aclDataType::ACL_FLOAT) {
                std::vector<float> outHostData(localOutSize, 0);
                ret = CreateAclTensor_New(outHostData, outShape, &outDeviceAddr, localOutDtype, fmt, &outTensor);
                cstOutTensorShapeSz = (uint64_t)GetShapeSize(outShape);
                cstOutTensorDtype = (int)(localOutDtype);
            } else if (localOutDtype == aclDataType::ACL_FLOAT16) {
                std::vector<int16_t> outHostData(localOutSize, 0);
                ret = CreateAclTensor_New(outHostData, outShape, &outDeviceAddr, localOutDtype, fmt, &outTensor);
                cstOutTensorShapeSz = (uint64_t)GetShapeSize(outShape);
                cstOutTensorDtype = (int)(localOutDtype);
            } else if (localOutDtype == aclDataType::ACL_BF16) {
                std::vector<int16_t> outHostData(localOutSize, 0);
                ret = CreateAclTensor_New(outHostData, outShape, &outDeviceAddr, localOutDtype, fmt, &outTensor);
                cstOutTensorShapeSz = (uint64_t)GetShapeSize(outShape);
                cstOutTensorDtype = (int)(localOutDtype);
            }
            CHECK_RET(ret == ACL_SUCCESS, goto ERROR);

            if (updateType == 1) {
                std::vector<float> lseOutHostData(lseSize, 0);
                ret = CreateAclTensor_New(lseOutHostData, lseShape, &lseOutDeviceAddr, aclDataType::ACL_FLOAT, fmt, &lseOutTensor);
                cstLseOutTensorShapeSz = (uint64_t)GetShapeSize(lseShape);
                cstLseOutTensorDtype = (int)(aclDataType::ACL_FLOAT);
                CHECK_RET(ret == ACL_SUCCESS, goto ERROR);
            } else {
                lseOutTensor = nullptr;
                lseOutDeviceAddr = nullptr;
            }

                        // ===== [CONSISTENT] 保存全部 tensor 设备侧初始内容(固定输入,含输入tensor的值) =====
            cstOutTensorBytes = (outDeviceAddr != nullptr) ? (size_t)cstOutTensorShapeSz * (size_t)ConsistentSizeOfDtype((aclDataType)cstOutTensorDtype) : 0;
            printf("[FUZZ_CONSISTENT_BYTES] case_%d: tensor=outTensor, dtype=%d(%s), shapeSize=%llu, cstBytes=%zu, ConsistentSizeOfDtype=%zu\n", fuzzi, (int)cstOutTensorDtype, ConsistentDtypeStr((aclDataType)cstOutTensorDtype), (unsigned long long)cstOutTensorShapeSz, cstOutTensorBytes, ConsistentSizeOfDtype((aclDataType)cstOutTensorDtype));
            if (cstOutTensorBytes == 0 && outDeviceAddr != nullptr) {
                printf("[FUZZ_CONSISTENT_BYTES] case_%d: WARN tensor=outTensor addr!=nullptr but cstBytes=0, dtype may be uncovered by ConsistentSizeOfDtype\n", fuzzi);
            }
            cstOutTensorSaveBuf = (cstOutTensorBytes > 0) ? malloc(cstOutTensorBytes) : nullptr;
            if (cstOutTensorSaveBuf != nullptr && outDeviceAddr != nullptr) {
                aclrtMemcpy(cstOutTensorSaveBuf, cstOutTensorBytes, outDeviceAddr, cstOutTensorBytes, ACL_MEMCPY_DEVICE_TO_HOST);
            }
            cstLseOutTensorBytes = (lseOutDeviceAddr != nullptr) ? (size_t)cstLseOutTensorShapeSz * (size_t)ConsistentSizeOfDtype((aclDataType)cstLseOutTensorDtype) : 0;
            printf("[FUZZ_CONSISTENT_BYTES] case_%d: tensor=lseOutTensor, dtype=%d(%s), shapeSize=%llu, cstBytes=%zu, ConsistentSizeOfDtype=%zu\n", fuzzi, (int)cstLseOutTensorDtype, ConsistentDtypeStr((aclDataType)cstLseOutTensorDtype), (unsigned long long)cstLseOutTensorShapeSz, cstLseOutTensorBytes, ConsistentSizeOfDtype((aclDataType)cstLseOutTensorDtype));
            if (cstLseOutTensorBytes == 0 && lseOutDeviceAddr != nullptr) {
                printf("[FUZZ_CONSISTENT_BYTES] case_%d: WARN tensor=lseOutTensor addr!=nullptr but cstBytes=0, dtype may be uncovered by ConsistentSizeOfDtype\n", fuzzi);
            }
            cstLseOutTensorSaveBuf = (cstLseOutTensorBytes > 0) ? malloc(cstLseOutTensorBytes) : nullptr;
            if (cstLseOutTensorSaveBuf != nullptr && lseOutDeviceAddr != nullptr) {
                aclrtMemcpy(cstLseOutTensorSaveBuf, cstLseOutTensorBytes, lseOutDeviceAddr, cstLseOutTensorBytes, ACL_MEMCPY_DEVICE_TO_HOST);
            }
            // [CONSISTENT] TensorList 组 lseDeviceAddr:逐元素快照(字节数已在创建点捕获)
            { size_t cstTot = 0; for (size_t cstT = 0; cstT < cstLseDeviceAddrEltBytesArr.size(); cstT++) cstTot += cstLseDeviceAddrEltBytesArr[cstT];
              printf("[FUZZ_CONSISTENT_BYTES] case_%d: tensor=lseDeviceAddr[list], num=%zu, totalBytes=%zu\n", fuzzi, cstLseDeviceAddrEltBytesArr.size(), cstTot); }
            cstLseDeviceAddrSaveBufs.assign(cstLseDeviceAddrEltBytesArr.size(), nullptr);
            for (size_t cstT = 0; cstT < cstLseDeviceAddrEltBytesArr.size(); cstT++) {
                if (cstLseDeviceAddrEltBytesArr[cstT] == 0 && lseDeviceAddr[cstT] != nullptr) {
                    printf("[FUZZ_CONSISTENT_BYTES] case_%d: WARN tensor=lseDeviceAddr[list] elt%zu addr!=nullptr but cstBytes=0\n", fuzzi, cstT);
                }
            }
            for (size_t cstT = 0; cstT < cstLseDeviceAddrEltBytesArr.size(); cstT++) {
                if (cstLseDeviceAddrEltBytesArr[cstT] > 0 && lseDeviceAddr[cstT] != nullptr) {
                    cstLseDeviceAddrSaveBufs[cstT] = malloc(cstLseDeviceAddrEltBytesArr[cstT]);
                    if (cstLseDeviceAddrSaveBufs[cstT] != nullptr) { aclrtMemcpy(cstLseDeviceAddrSaveBufs[cstT], cstLseDeviceAddrEltBytesArr[cstT], lseDeviceAddr[cstT], cstLseDeviceAddrEltBytesArr[cstT], ACL_MEMCPY_DEVICE_TO_HOST); }
                }
            }
            // [CONSISTENT] TensorList 组 localOutDeviceAddr:逐元素快照(字节数已在创建点捕获)
            { size_t cstTot = 0; for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrEltBytesArr.size(); cstT++) cstTot += cstLocalOutDeviceAddrEltBytesArr[cstT];
              printf("[FUZZ_CONSISTENT_BYTES] case_%d: tensor=localOutDeviceAddr[list], num=%zu, totalBytes=%zu\n", fuzzi, cstLocalOutDeviceAddrEltBytesArr.size(), cstTot); }
            cstLocalOutDeviceAddrSaveBufs.assign(cstLocalOutDeviceAddrEltBytesArr.size(), nullptr);
            for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrEltBytesArr.size(); cstT++) {
                if (cstLocalOutDeviceAddrEltBytesArr[cstT] == 0 && localOutDeviceAddr[cstT] != nullptr) {
                    printf("[FUZZ_CONSISTENT_BYTES] case_%d: WARN tensor=localOutDeviceAddr[list] elt%zu addr!=nullptr but cstBytes=0\n", fuzzi, cstT);
                }
            }
            for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrEltBytesArr.size(); cstT++) {
                if (cstLocalOutDeviceAddrEltBytesArr[cstT] > 0 && localOutDeviceAddr[cstT] != nullptr) {
                    cstLocalOutDeviceAddrSaveBufs[cstT] = malloc(cstLocalOutDeviceAddrEltBytesArr[cstT]);
                    if (cstLocalOutDeviceAddrSaveBufs[cstT] != nullptr) { aclrtMemcpy(cstLocalOutDeviceAddrSaveBufs[cstT], cstLocalOutDeviceAddrEltBytesArr[cstT], localOutDeviceAddr[cstT], cstLocalOutDeviceAddrEltBytesArr[cstT], ACL_MEMCPY_DEVICE_TO_HOST); }
                }
            }

            // ===== [CONSISTENT] 固定输入重复执行 CONSISTENT_RUN_TIMES 次,逐轮对全部 tensor 计算 hash =====
            cstRunCount = 0;
            cstAllOk = true;
            for (int cstRun = 0; cstRun < CONSISTENT_RUN_TIMES; cstRun++) {
                // [CONSISTENT] 恢复固定输入(输出 tensor 同时恢复初始值,覆盖 inplace/累加类算子)
                if (cstOutTensorSaveBuf != nullptr && outDeviceAddr != nullptr) {
                    aclrtMemcpy(outDeviceAddr, cstOutTensorBytes, cstOutTensorSaveBuf, cstOutTensorBytes, ACL_MEMCPY_HOST_TO_DEVICE);
                }
                if (cstLseOutTensorSaveBuf != nullptr && lseOutDeviceAddr != nullptr) {
                    aclrtMemcpy(lseOutDeviceAddr, cstLseOutTensorBytes, cstLseOutTensorSaveBuf, cstLseOutTensorBytes, ACL_MEMCPY_HOST_TO_DEVICE);
                }
                for (size_t cstT = 0; cstT < cstLseDeviceAddrSaveBufs.size(); cstT++) {
                    if (cstLseDeviceAddrSaveBufs[cstT] != nullptr && lseDeviceAddr[cstT] != nullptr) {
                        aclrtMemcpy(lseDeviceAddr[cstT], cstLseDeviceAddrEltBytesArr[cstT], cstLseDeviceAddrSaveBufs[cstT], cstLseDeviceAddrEltBytesArr[cstT], ACL_MEMCPY_HOST_TO_DEVICE);
                    }
                }
                for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrSaveBufs.size(); cstT++) {
                    if (cstLocalOutDeviceAddrSaveBufs[cstT] != nullptr && localOutDeviceAddr[cstT] != nullptr) {
                        aclrtMemcpy(localOutDeviceAddr[cstT], cstLocalOutDeviceAddrEltBytesArr[cstT], cstLocalOutDeviceAddrSaveBufs[cstT], cstLocalOutDeviceAddrEltBytesArr[cstT], ACL_MEMCPY_HOST_TO_DEVICE);
                    }
                }
                ret = aclnnAttentionUpdateGetWorkspaceSize(
                lseTensorList, localOutTensorList, updateType,
                outTensor, lseOutTensor,
                &workspaceSize, &executor);
                if (ret != ACL_SUCCESS) {
                    printf("aclnnAttentionUpdate execute failed :%d\n", (int)ret);
                    printf("[FUZZ_CONSISTENT] case_%d: faild, type=execute_error, stage=GetWorkspaceSize, run=%d/%d, ret=%d\n", fuzzi, cstRun + 1, CONSISTENT_RUN_TIMES, (int)ret);
                    cstAllOk = false;
                    break;
                }
                if (workspaceSize > 0) {
                    if (workspaceAddr != nullptr) { aclrtFree(workspaceAddr); workspaceAddr = nullptr; }
                    ret = aclrtMalloc(&workspaceAddr, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST);
                    if (ret != ACL_SUCCESS) {
                        printf("[FUZZ_CONSISTENT] case_%d: faild, type=execute_error, stage=workspace_malloc, run=%d/%d, ret=%d\n", fuzzi, cstRun + 1, CONSISTENT_RUN_TIMES, (int)ret);
                        cstAllOk = false;
                        break;
                    }
                }
                ret = aclnnAttentionUpdate(workspaceAddr, workspaceSize, executor, stream);
                if (ret != ACL_SUCCESS) {
                    printf("aclnnAttentionUpdate execute failed :%d\n", (int)ret);
                    printf("[FUZZ_CONSISTENT] case_%d: faild, type=execute_error, stage=execute, run=%d/%d, ret=%d\n", fuzzi, cstRun + 1, CONSISTENT_RUN_TIMES, (int)ret);
                    cstAllOk = false;
                    break;
                }
                ret = aclrtSynchronizeStream(stream);
                if (ret != ACL_SUCCESS) {
                    printf("[FUZZ_CONSISTENT] case_%d: faild, type=execute_error, stage=sync, run=%d/%d, ret=%d\n", fuzzi, cstRun + 1, CONSISTENT_RUN_TIMES, (int)ret);
                    cstAllOk = false;
                    break;
                }
                // [CONSISTENT] 本轮执行结果 hash
                if (outDeviceAddr != nullptr && cstOutTensorBytes > 0) {
                    cstOutTensorHashArr[cstRun] = ConsistentHashDeviceTensor(outDeviceAddr, cstOutTensorBytes);
                }
                if (lseOutDeviceAddr != nullptr && cstLseOutTensorBytes > 0) {
                    cstLseOutTensorHashArr[cstRun] = ConsistentHashDeviceTensor(lseOutDeviceAddr, cstLseOutTensorBytes);
                }
                cstLseDeviceAddrHashArr[cstRun].assign(cstLseDeviceAddrEltBytesArr.size(), 0);
                for (size_t cstT = 0; cstT < cstLseDeviceAddrEltBytesArr.size(); cstT++) {
                    if (lseDeviceAddr[cstT] != nullptr && cstLseDeviceAddrEltBytesArr[cstT] > 0) {
                        cstLseDeviceAddrHashArr[cstRun][cstT] = ConsistentHashDeviceTensor(lseDeviceAddr[cstT], cstLseDeviceAddrEltBytesArr[cstT]);
                    }
                }
                cstLocalOutDeviceAddrHashArr[cstRun].assign(cstLocalOutDeviceAddrEltBytesArr.size(), 0);
                for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrEltBytesArr.size(); cstT++) {
                    if (localOutDeviceAddr[cstT] != nullptr && cstLocalOutDeviceAddrEltBytesArr[cstT] > 0) {
                        cstLocalOutDeviceAddrHashArr[cstRun][cstT] = ConsistentHashDeviceTensor(localOutDeviceAddr[cstT], cstLocalOutDeviceAddrEltBytesArr[cstT]);
                    }
                }
                cstRunCount++;
                printf("aclnnAttentionUpdate execute success :%d\n", (int)ret);
            }

            // ===== [CONSISTENT] 对比 2 轮 hash,判定一致性(2次执行输出一致即为确定性) =====
            if (cstAllOk && cstRunCount == CONSISTENT_RUN_TIMES) {
                bool cstOutTensorSame = true;
                for (int cstR = 1; cstR < CONSISTENT_RUN_TIMES; cstR++) {
                    if (cstOutTensorHashArr[cstR] != cstOutTensorHashArr[0]) { cstOutTensorSame = false; break; }
                }
                bool cstLseOutTensorSame = true;
                for (int cstR = 1; cstR < CONSISTENT_RUN_TIMES; cstR++) {
                    if (cstLseOutTensorHashArr[cstR] != cstLseOutTensorHashArr[0]) { cstLseOutTensorSame = false; break; }
                }
                bool cstLseDeviceAddrSame = true;
                for (int cstR = 1; cstR < CONSISTENT_RUN_TIMES; cstR++) {
                    if (cstLseDeviceAddrHashArr[cstR] != cstLseDeviceAddrHashArr[0]) { cstLseDeviceAddrSame = false; break; }
                }
                bool cstLocalOutDeviceAddrSame = true;
                for (int cstR = 1; cstR < CONSISTENT_RUN_TIMES; cstR++) {
                    if (cstLocalOutDeviceAddrHashArr[cstR] != cstLocalOutDeviceAddrHashArr[0]) { cstLocalOutDeviceAddrSame = false; break; }
                }
                bool cstAllSame = cstOutTensorSame && cstLseOutTensorSame && cstLseDeviceAddrSame && cstLocalOutDeviceAddrSame;
                if (cstAllSame) {
                    printf("[FUZZ_CONSISTENT] case_%d: success, type=consistent, runs=%d, hash_consistent=all\n", fuzzi, CONSISTENT_RUN_TIMES);
                    printf("[FUZZ_RESULT] case_%d: success, ret=%d\n", fuzzi, (int)ret);
                    cstResultPrinted = true;
                } else {
                    printf("[FUZZ_CONSISTENT] case_%d: faild, type=hash_mismatch, runs=%d, inconsistent=", fuzzi, CONSISTENT_RUN_TIMES);
                    if (!cstOutTensorSame) printf("outTensor ");
                    if (!cstLseOutTensorSame) printf("lseOutTensor ");
                    printf("\n");
                    printf("[FUZZ_CONSISTENT_DETAIL] case_%d: outTensor hash:", fuzzi);
                    for (int cstR = 0; cstR < CONSISTENT_RUN_TIMES; cstR++) printf(" run%d=%llu", cstR, (unsigned long long)cstOutTensorHashArr[cstR]);
                    printf("\n");
                    printf("[FUZZ_CONSISTENT_DETAIL] case_%d: lseOutTensor hash:", fuzzi);
                    for (int cstR = 0; cstR < CONSISTENT_RUN_TIMES; cstR++) printf(" run%d=%llu", cstR, (unsigned long long)cstLseOutTensorHashArr[cstR]);
                    printf("\n");
                    printf("[FUZZ_CONSISTENT_DETAIL] case_%d: lseDeviceAddr[list] hash per element (run0):", fuzzi);
                    for (size_t cstT = 0; cstT < cstLseDeviceAddrHashArr[0].size(); cstT++) printf(" elt%zu=%llu", cstT, (unsigned long long)cstLseDeviceAddrHashArr[0][cstT]);
                    printf("\n");
                    printf("[FUZZ_CONSISTENT_DETAIL] case_%d: localOutDeviceAddr[list] hash per element (run0):", fuzzi);
                    for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrHashArr[0].size(); cstT++) printf(" elt%zu=%llu", cstT, (unsigned long long)cstLocalOutDeviceAddrHashArr[0][cstT]);
                    printf("\n");
                    // [CONSISTENT] 失败时打印全部入参信息(dtype/format/shape/attr/tiling_key)
                    printf("[FUZZ_PARAM] ");
                    printf("outTensor: dtype=%s, format=%s, ", ConsistentDtypeStr((aclDataType)cstOutTensorDtype), ConsistentFormatStr(fmt)); printf("shapeSize=%llu", (unsigned long long)cstOutTensorShapeSz); printf("; ");
                    printf("lseOutTensor: dtype=%s, format=%s, ", ConsistentDtypeStr((aclDataType)cstLseOutTensorDtype), ConsistentFormatStr(fmt)); printf("shapeSize=%llu", (unsigned long long)cstLseOutTensorShapeSz); printf("; ");
                    printf("updateType=%lld; ", (long long)updateType);
                    printf("tiling_key=%s\n", curTilingKey);
                    // ret=-2 为 hash_mismatch 专用标记(非 aclnn 错误码:执行成功但输出不确定)
                    printf("[FUZZ_RESULT] case_%d: faild, ret=%d\n", fuzzi, -2);
                    cstResultPrinted = true;
                }
            } else if (!cstAllOk) {
                // [CONSISTENT] 执行失败也打印全部入参信息
                printf("[FUZZ_PARAM] ");
                printf("outTensor: dtype=%s, format=%s, ", ConsistentDtypeStr((aclDataType)cstOutTensorDtype), ConsistentFormatStr(fmt)); printf("shapeSize=%llu", (unsigned long long)cstOutTensorShapeSz); printf("; ");
                printf("lseOutTensor: dtype=%s, format=%s, ", ConsistentDtypeStr((aclDataType)cstLseOutTensorDtype), ConsistentFormatStr(fmt)); printf("shapeSize=%llu", (unsigned long long)cstLseOutTensorShapeSz); printf("; ");
                printf("updateType=%lld; ", (long long)updateType);
                printf("tiling_key=%s\n", curTilingKey);
                printf("[FUZZ_RESULT] case_%d: faild, ret=%d\n", fuzzi, (int)ret);
                cstResultPrinted = true;
            }
        }
        catch (std::exception& e)
        {
            printf("catch exception %s\n", e.what());
            printf("[FUZZ_CONSISTENT] case_%d: faild, type=exception, ret=%d\n", fuzzi, (int)ret);
            printf("[FUZZ_RESULT] case_%d: faild, ret=%d\n", fuzzi, (int)ret);
            cstResultPrinted = true;
        }

    ERROR:
        printf("end \n");
        CheckAndLog(ret);
        pid_t pid = getpid();
        std::string cmd = "grep -rin 'aicore error' /root/ascend/log/debug/plog/";
        printf("system return value %d\n", CheckResult(pid));

        for (int s = 0; s < 16; s++) {
            if (lseTensors[s]) aclDestroyTensor(lseTensors[s]);
            if (lseDeviceAddr[s]) aclrtFree(lseDeviceAddr[s]);
            if (localOutTensors[s]) aclDestroyTensor(localOutTensors[s]);
            if (localOutDeviceAddr[s]) aclrtFree(localOutDeviceAddr[s]);
        }
        // [CONSISTENT] 释放固定输入快照
        if (cstOutTensorSaveBuf != nullptr) { free(cstOutTensorSaveBuf); cstOutTensorSaveBuf = nullptr; }
        if (cstLseOutTensorSaveBuf != nullptr) { free(cstLseOutTensorSaveBuf); cstLseOutTensorSaveBuf = nullptr; }
        for (size_t cstT = 0; cstT < cstLseDeviceAddrSaveBufs.size(); cstT++) { if (cstLseDeviceAddrSaveBufs[cstT] != nullptr) free(cstLseDeviceAddrSaveBufs[cstT]); }
        cstLseDeviceAddrSaveBufs.clear();
        for (size_t cstT = 0; cstT < cstLocalOutDeviceAddrSaveBufs.size(); cstT++) { if (cstLocalOutDeviceAddrSaveBufs[cstT] != nullptr) free(cstLocalOutDeviceAddrSaveBufs[cstT]); }
        cstLocalOutDeviceAddrSaveBufs.clear();
        aclDestroyTensor(outTensor);
        aclDestroyTensor(lseOutTensor);
        aclrtFree(outDeviceAddr);
        if (lseOutDeviceAddr) aclrtFree(lseOutDeviceAddr);
        if (workspaceSize > 0 && workspaceAddr) { aclrtFree(workspaceAddr); }
    }
    DT_FUZZ_END()

    WriteDTLog("aclnnAttentionUpdate", "aclnnAttentionUpdate consistent op fuzz total number:%u, para error number %u, execute error number %u \n", fuzz_num, aclnn_INVALID_Para, aclnn_Inner_Error + Runtime_Error);
    WriteDTLog("aclnnAttentionUpdate", "aclnnAttentionUpdate consistent op fuzz end \n");
    aclrtDestroyStream(stream);
    aclrtDestroyContext(context);
    aclrtResetDevice(deviceId);
    aclFinalize();
    return 0;
}

extern "C" int aclnnAttentionUpdate_consistent_fuzz(int argc, char *argv[]);
int aclnnAttentionUpdate_consistent_fuzz(int argc, char *argv[])
{
    // [CONSISTENT] 运行时覆盖每用例执行次数(环境变量 CONSISTENT_RUN_TIMES:默认2次,可按需覆盖)
    const char* cstRunEnv = getenv("CONSISTENT_RUN_TIMES");
    if (cstRunEnv != nullptr) {
        int cstRunV = atoi(cstRunEnv);
        if (cstRunV > 0 && cstRunV <= CONSISTENT_MAX_RUN_TIMES) { CONSISTENT_RUN_TIMES = cstRunV; }
    }
    printf("[FUZZ_CONSISTENT_RUNTIMES] run_times=%d\n", CONSISTENT_RUN_TIMES);
    int fuzz_num = MAXFUZZNUMBER;
    if (argc < 1) {
        cout << "[ERROR] input arg num must be bigger than 1 \n" << endl;
        return -1;
    }
    else if (argc > 1) {
        fuzz_num = atoi(argv[2]);
    }
    if (argc == 4) { deviceId = atoi(argv[3]); }

    printf("fuzz_num %d \n", fuzz_num);
    printf("aclnnAttentionUpdate_consistent :%d \n", aclnnAttentionUpdate_consistent(fuzz_num));
    return 0;
}