A2 PCIE5.0 MEMCPY_ASYNC硬件耗时波动问题-runtime

已开启

A2 PCIE5.0 MEMCPY_ASYNC硬件耗时波动问题 #12

guo-yanjun创建于 7月25日

7月25日创建

//g++ -o p2pDemo p2p_async_acl.cpp -I/usr/local/Ascend/ascend-toolkit/latest/include/ -L/usr/local/Ascend/ascend-toolkit/latest/lib64/ -lascendcl -std=c++11
#include "chrono"
#include "acl/acl.h"
#include
#include
#include
using namespace std;

#define API_CALL(func)
do {
auto ret = func;
if (ret != 0) {
std::cout << "call " << #func << " failed, ret = " << ret << std::endl;
exit(-1);
}
} while (0) \

constexpr uint32_t SIZE_MB = 1024 * 1024;
constexpr uint32_t CP_SIZE = 256 * SIZE_MB;
constexpr uint32_t LOOP_NUM = 40 * 500;
constexpr uint32_t WIDTH_10 = 10;
constexpr uint32_t WIDTH_20 = 20;
constexpr uint32_t WIDTH_30 = 30;

void ComputeUnidirectionalBandwidth(int32_t srcId, int32_t dstId)
{
// 源device上的准备操作
API_CALL(aclrtSetDevice(srcId));
API_CALL(aclrtDeviceEnablePeerAccess(dstId, 0));
void* srcDev = nullptr;
API_CALL(aclrtMalloc(&srcDev, CP_SIZE, ACL_MEM_TYPE_HIGH_BAND_WIDTH));
API_CALL(aclrtMemset(srcDev, CP_SIZE, 'a', CP_SIZE));

// 目的device上的准备操作
API_CALL(aclrtSetDevice(dstId));
API_CALL(aclrtDeviceEnablePeerAccess(srcId, 0));
void* dstDev = nullptr;
API_CALL(aclrtMalloc(&dstDev, CP_SIZE, ACL_MEM_TYPE_HIGH_BAND_WIDTH));
API_CALL(aclrtMemset(dstDev, CP_SIZE, 'b', CP_SIZE));

// 写带宽使用src卡上的context
API_CALL(aclrtSetDevice(srcId));
aclrtStream stream = nullptr;
API_CALL(aclrtCreateStream(&stream));
aclrtEvent startEvent = nullptr;
aclrtEvent endEvent = nullptr;
API_CALL(aclrtCreateEventWithFlag(&startEvent, ACL_EVENT_TIME_LINE));
API_CALL(aclrtCreateEventWithFlag(&endEvent, ACL_EVENT_TIME_LINE));
cout << "***************Unidirectional Bandwidth******************" << endl;
cout << left << setw(WIDTH_20) << "Duration(ms)" << setw(WIDTH_20) << "Data size(GB)";
cout << setw(WIDTH_20) << "Bandwidth(GB/s)" << endl;

for (int i = 0; i < LOOP_NUM; i++) { // 多次拷贝
    API_CALL(aclrtRecordEvent(startEvent, stream));
    API_CALL(aclrtMemcpyAsync(dstDev, CP_SIZE, srcDev, CP_SIZE, ACL_MEMCPY_DEVICE_TO_DEVICE, stream));
    API_CALL(aclrtRecordEvent(endEvent, stream));
    API_CALL(aclrtSynchronizeStream(stream));
    
    float time_ms;
    API_CALL(aclrtEventElapsedTime(&time_ms, startEvent, endEvent));
    auto bandwidth = (CP_SIZE / 1e9) / (time_ms / 1e3);
    cout << setw(WIDTH_20) << time_ms << setw(WIDTH_20) << (CP_SIZE / 1e9);
    cout << setw(WIDTH_20) << bandwidth << endl;
}

// 释放内存
API_CALL(aclrtFree(srcDev));
API_CALL(aclrtFree(dstDev));
API_CALL(aclrtResetDevice(dstId));
API_CALL(aclrtSetDevice(srcId));
API_CALL(aclrtResetDevice(srcId));

}

void ComputeBidirectionalBandwidth(int32_t srcId, int32_t dstId) {
// src上的准备操作
API_CALL(aclrtSetDevice(srcId));
API_CALL(aclrtDeviceEnablePeerAccess(dstId, 0));
void* srcSend;
API_CALL(aclrtMalloc(&srcSend, CP_SIZE, ACL_MEM_TYPE_HIGH_BAND_WIDTH));
API_CALL(aclrtMemset(srcSend, CP_SIZE, 'a', CP_SIZE));
void* srcRecv;
API_CALL(aclrtMalloc(&srcRecv, CP_SIZE, ACL_MEM_TYPE_HIGH_BAND_WIDTH));
API_CALL(aclrtMemset(srcRecv, CP_SIZE, 'b', CP_SIZE));

aclrtStream stream = nullptr;
API_CALL(aclrtCreateStream(&stream));
aclrtEvent startEvent = nullptr;
aclrtEvent endEvent = nullptr;
API_CALL(aclrtCreateEventWithFlag(&startEvent, ACL_EVENT_TIME_LINE));
API_CALL(aclrtCreateEventWithFlag(&endEvent, ACL_EVENT_TIME_LINE));

// dst上的准备操作
API_CALL(aclrtSetDevice(dstId));
API_CALL(aclrtDeviceEnablePeerAccess(srcId, 0));
void* dstSend;
API_CALL(aclrtMalloc(&dstSend, CP_SIZE, ACL_MEM_TYPE_HIGH_BAND_WIDTH));
API_CALL(aclrtMemset(dstSend, CP_SIZE, 'a', CP_SIZE));
void* dstRecv;
API_CALL(aclrtMalloc(&dstRecv, CP_SIZE, ACL_MEM_TYPE_HIGH_BAND_WIDTH));
API_CALL(aclrtMemset(dstRecv, CP_SIZE, 'b', CP_SIZE));

aclrtStream streamReverse = nullptr;
API_CALL(aclrtCreateStream(&streamReverse));
aclrtEvent startEventReverse = nullptr;
aclrtEvent endEventReverse = nullptr;
API_CALL(aclrtCreateEventWithFlag(&startEventReverse, ACL_EVENT_TIME_LINE));
API_CALL(aclrtCreateEventWithFlag(&endEventReverse, ACL_EVENT_TIME_LINE));

cout << "***************Bidirectional Bandwidth******************" << endl;
cout << setw(WIDTH_30) << "Src dur,  Dst dur(ms)" << setw(WIDTH_20) << "Data size(GB)";
cout << setw(WIDTH_20) << "Bandwidth(GB/s)" << endl;
for (int i = 0; i < LOOP_NUM; i++) { //多次拷贝
    API_CALL(aclrtSetDevice(srcId));
    API_CALL(aclrtRecordEvent(startEvent, stream));
    API_CALL(aclrtMemcpyAsync(dstRecv, CP_SIZE, srcSend, CP_SIZE, ACL_MEMCPY_DEVICE_TO_DEVICE, stream));
    API_CALL(aclrtRecordEvent(endEvent, stream));
    
    API_CALL(aclrtSetDevice(dstId));
    API_CALL(aclrtRecordEvent(startEventReverse, streamReverse));
    API_CALL(aclrtMemcpyAsync(srcRecv, CP_SIZE, dstSend, CP_SIZE, ACL_MEMCPY_DEVICE_TO_DEVICE, streamReverse));
    API_CALL(aclrtRecordEvent(endEventReverse, streamReverse));
    
    API_CALL(aclrtSetDevice(srcId));
    API_CALL(aclrtSynchronizeStream(stream));
    float time_ms;
    API_CALL(aclrtEventElapsedTime(&time_ms, startEvent, endEvent));

    API_CALL(aclrtSetDevice(dstId));
    API_CALL(aclrtSynchronizeStream(streamReverse));
    float timeReverse_ms;
    API_CALL(aclrtEventElapsedTime(&timeReverse_ms, startEventReverse, endEventReverse));
    
    auto bandwidth = ((CP_SIZE * 2) / 1e9) / (((time_ms + timeReverse_ms) / 2) / 1e3);
    cout << setw(WIDTH_10) << time_ms << setw(WIDTH_30 - WIDTH_10) << timeReverse_ms;
    cout << setw(WIDTH_20) << (CP_SIZE * 2 / 1e9) << setw(WIDTH_20) << bandwidth << endl;
}

// 释放内存
API_CALL(aclrtSetDevice(srcId));
API_CALL(aclrtFree(srcSend));
API_CALL(aclrtFree(srcRecv));
API_CALL(aclrtResetDevice(srcId));

API_CALL(aclrtSetDevice(dstId));
API_CALL(aclrtFree(dstSend));
API_CALL(aclrtFree(dstRecv));
API_CALL(aclrtResetDevice(dstId));

}

int main(const int argc, const char** argv) {
int32_t srcId = 0;
int32_t dstId = 1;
if (argc == 3) {
srcId = stoi(argv[1]);
dstId = stoi(argv[2]);
}
cout << "Device "<< srcId << " to " << dstId << endl;

API_CALL(aclInit(nullptr));
int32_t canAccessPeer = 0;
API_CALL(aclrtDeviceCanAccessPeer(&canAccessPeer, srcId, dstId));
if (canAccessPeer == 1) {
    ComputeUnidirectionalBandwidth(srcId, dstId);
    cout << "\n\n\n" << endl;
    ComputeBidirectionalBandwidth(srcId, dstId);
}
API_CALL(aclFinalize());

return 0;

}