已关闭
ACL 查询显示此芯片有两个池,但没发现如何把两个池合并使用 #535
alduy88创建于 5月20日关闭于 6月12日
5月20日 将 ykl999 设为负责人
ykl999
5月20日 评论:
5月20日 评论:
你好,针对第二个疑问
910ProB是 Ascend910A的芯片,
910A芯片字母之前还可能有Pro/Premium等修饰表示增强版子型号(如910ProA、910ProB、910PremiumA等);
他们之间的主频和aicore核数存在差异;
910A 32个aicore 主频1000MHZ
910ProB 30个aicore 主频1100MHZ
针对第一个问题可以把调用的acl脚本发下,对应每次acl脚本运行的日志


whumatrix
5月21日 评论:
5月21日 评论:
#include <acl/acl_rt.h>
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
namespace {
constexpr size_t kMiB = 1024ULL * 1024ULL;
struct Policy {
std::string name;
std::vector<aclrtMemMallocPolicy> values;
};
struct ProbeResult {
size_t bytes = 0;
aclError nextFail = ACL_SUCCESS;
};
struct SplitProbeResult {
size_t firstBytes = 0;
size_t secondBytes = 0;
aclError firstFail = ACL_SUCCESS;
aclError secondFail = ACL_SUCCESS;
};
struct PhysicalPolicy {
std::string name;
aclrtMemAttr attr;
};
int ParseIntArg(int argc, char **argv, const std::string &name, int defaultValue) {
for (int i = 1; i + 1 < argc; i++) {
if (argv[i] == name) {
return std::stoi(argv[i + 1]);
}
}
return defaultValue;
}
std::string ParseStringArg(int argc, char **argv, const std::string &name, const std::string &defaultValue) {
for (int i = 1; i + 1 < argc; i++) {
if (argv[i] == name) {
return argv[i + 1];
}
}
return defaultValue;
}
bool HasArg(int argc, char **argv, const std::string &name) {
for (int i = 1; i < argc; i++) {
if (argv[i] == name) {
return true;
}
}
return false;
}
std::vector<Policy> BuildPolicies(const std::string &name) {
std::vector<Policy> all = {
{"huge-first", {ACL_MEM_MALLOC_HUGE_FIRST}},
{"huge-only", {ACL_MEM_MALLOC_HUGE_ONLY}},
{"normal-only", {ACL_MEM_MALLOC_NORMAL_ONLY}},
{"huge-first-p2p", {ACL_MEM_MALLOC_HUGE_FIRST_P2P}},
{"huge-only-p2p", {ACL_MEM_MALLOC_HUGE_ONLY_P2P}},
{"normal-only-p2p", {ACL_MEM_MALLOC_NORMAL_ONLY_P2P}},
{"huge1g-only", {ACL_MEM_MALLOC_HUGE1G_ONLY}},
{"huge1g-only-p2p", {ACL_MEM_MALLOC_HUGE1G_ONLY_P2P}},
{"huge-then-normal", {ACL_MEM_MALLOC_HUGE_FIRST, ACL_MEM_MALLOC_NORMAL_ONLY}},
{"huge-only-then-normal", {ACL_MEM_MALLOC_HUGE_ONLY, ACL_MEM_MALLOC_NORMAL_ONLY}},
{"normal-then-huge-only", {ACL_MEM_MALLOC_NORMAL_ONLY, ACL_MEM_MALLOC_HUGE_ONLY}},
};
if (name == "all") {
return all;
}
for (auto &policy : all) {
if (policy.name == name) {
return {policy};
}
}
throw std::runtime_error("unknown --policy: " + name);
}
Policy BuildPolicy(const std::string &name) {
auto policies = BuildPolicies(name);
if (policies.size() != 1) {
throw std::runtime_error("--second-policy requires one explicit policy, not: " + name);
}
return policies[0];
}
std::vector<PhysicalPolicy> BuildPhysicalPolicies(const std::string &name) {
std::vector<PhysicalPolicy> all = {
{"hbm", ACL_HBM_MEM},
{"hbm-huge", ACL_HBM_MEM_HUGE},
{"hbm-normal", ACL_HBM_MEM_NORMAL},
{"hbm-p2p-huge", ACL_HBM_MEM_P2P_HUGE},
{"hbm-p2p-normal", ACL_HBM_MEM_P2P_NORMAL},
{"hbm-p2p-huge1g", ACL_HBM_MEM_P2P_HUGE1G},
{"mem-huge", ACL_MEM_HUGE},
{"mem-normal", ACL_MEM_NORMAL},
{"mem-p2p-huge", ACL_MEM_P2P_HUGE},
{"mem-p2p-normal", ACL_MEM_P2P_NORMAL},
{"mem-p2p-huge1g", ACL_MEM_P2P_HUGE1G},
{"mem-huge1g", ACL_MEM_HUGE1G},
};
if (name == "all") {
return all;
}
for (auto &policy : all) {
if (policy.name == name) {
return {policy};
}
}
throw std::runtime_error("unknown --physical-policy: " + name);
}
std::string FormatBytes(size_t bytes) {
double mib = static_cast<double>(bytes) / static_cast<double>(kMiB);
double gib = mib / 1024.0;
std::ostringstream os;
os << std::fixed << std::setprecision(2)
<< mib << " MiB (" << gib << " GiB)";
return os.str();
}
bool TryMalloc(size_t bytes, const Policy &policy, void **ptr, aclError *err) {
*ptr = nullptr;
*err = ACL_SUCCESS;
for (aclrtMemMallocPolicy value : policy.values) {
*err = aclrtMalloc(ptr, bytes, value);
if (*err == ACL_SUCCESS && *ptr != nullptr) {
return true;
}
}
return false;
}
ProbeResult ProbeSingle(size_t upperBytes, size_t granularityBytes, const Policy &policy) {
ProbeResult result;
if (upperBytes < granularityBytes) {
return result;
}
size_t lo = 1;
size_t hi = upperBytes / granularityBytes;
while (lo <= hi) {
size_t mid = lo + (hi - lo) / 2;
size_t bytes = mid * granularityBytes;
void *ptr = nullptr;
aclError err = ACL_SUCCESS;
if (TryMalloc(bytes, policy, &ptr, &err)) {
aclrtFree(ptr);
result.bytes = bytes;
lo = mid + 1;
} else {
result.nextFail = err;
if (mid == 0) {
break;
}
hi = mid - 1;
}
}
return result;
}
ProbeResult ProbeFill(size_t chunkBytes, size_t limitBytes, size_t granularityBytes,
const Policy &policy) {
ProbeResult result;
std::vector<void *> ptrs;
size_t total = 0;
aclError lastErr = ACL_SUCCESS;
while (chunkBytes > 0 && total + chunkBytes <= limitBytes) {
void *ptr = nullptr;
aclError err = ACL_SUCCESS;
if (!TryMalloc(chunkBytes, policy, &ptr, &err)) {
lastErr = err;
break;
}
ptrs.push_back(ptr);
total += chunkBytes;
}
size_t tailLimit = 0;
if (limitBytes > total && chunkBytes >= granularityBytes) {
tailLimit = std::min(chunkBytes - granularityBytes, limitBytes - total);
}
ProbeResult tail = ProbeSingle(tailLimit, granularityBytes, policy);
result.bytes = total + tail.bytes;
result.nextFail = tail.nextFail != ACL_SUCCESS ? tail.nextFail : lastErr;
for (void *ptr : ptrs) {
aclrtFree(ptr);
}
return result;
}
aclrtPhysicalMemProp BuildPhysicalMemProp(int device, aclrtMemAttr attr) {
aclrtPhysicalMemProp prop{};
prop.handleType = ACL_MEM_HANDLE_TYPE_NONE;
prop.allocationType = ACL_MEM_ALLOCATION_TYPE_PINNED;
prop.memAttr = attr;
prop.location.id = static_cast<uint32_t>(device);
prop.location.type = ACL_MEM_LOCATION_TYPE_DEVICE;
prop.reserve = 0;
return prop;
}
ProbeResult ProbePhysicalFill(int device, size_t chunkBytes, size_t limitBytes,
size_t granularityBytes, const PhysicalPolicy &policy) {
ProbeResult result;
std::vector<aclrtDrvMemHandle> handles;
size_t total = 0;
aclError lastErr = ACL_SUCCESS;
aclrtPhysicalMemProp prop = BuildPhysicalMemProp(device, policy.attr);
while (chunkBytes > 0 && total + chunkBytes <= limitBytes) {
aclrtDrvMemHandle handle = nullptr;
aclError err = aclrtMallocPhysical(&handle, chunkBytes, &prop, 0);
if (err != ACL_SUCCESS || handle == nullptr) {
lastErr = err;
break;
}
handles.push_back(handle);
total += chunkBytes;
}
size_t tailLimit = 0;
if (limitBytes > total && chunkBytes >= granularityBytes) {
tailLimit = std::min(chunkBytes - granularityBytes, limitBytes - total);
}
size_t lo = 1;
size_t hi = tailLimit / granularityBytes;
while (lo <= hi) {
size_t mid = lo + (hi - lo) / 2;
size_t bytes = mid * granularityBytes;
aclrtDrvMemHandle handle = nullptr;
aclError err = aclrtMallocPhysical(&handle, bytes, &prop, 0);
if (err == ACL_SUCCESS && handle != nullptr) {
aclrtFreePhysical(handle);
result.bytes = bytes;
lo = mid + 1;
} else {
result.nextFail = err;
hi = mid - 1;
}
}
result.bytes += total;
if (result.nextFail == ACL_SUCCESS) {
result.nextFail = lastErr;
}
for (aclrtDrvMemHandle handle : handles) {
aclrtFreePhysical(handle);
}
return result;
}
bool AllocateChunks(size_t bytes, size_t chunkBytes, const Policy &policy,
std::vector<void *> *ptrs, size_t *allocatedBytes, aclError *err) {
*allocatedBytes = 0;
*err = ACL_SUCCESS;
while (*allocatedBytes < bytes) {
size_t request = std::min(chunkBytes, bytes - *allocatedBytes);
void *ptr = nullptr;
if (!TryMalloc(request, policy, &ptr, err)) {
return false;
}
ptrs->push_back(ptr);
*allocatedBytes += request;
}
return true;
}
SplitProbeResult ProbeSplit(size_t firstBytes, size_t limitBytes, size_t chunkBytes,
size_t granularityBytes, const Policy &firstPolicy,
const Policy &secondPolicy) {
SplitProbeResult result;
std::vector<void *> ptrs;
if (!AllocateChunks(firstBytes, chunkBytes, firstPolicy, &ptrs, &result.firstBytes,
&result.firstFail)) {
for (void *ptr : ptrs) {
aclrtFree(ptr);
}
return result;
}
ProbeResult second = ProbeFill(chunkBytes, limitBytes - result.firstBytes,
granularityBytes, secondPolicy);
result.secondBytes = second.bytes;
result.secondFail = second.nextFail;
for (void *ptr : ptrs) {
aclrtFree(ptr);
}
return result;
}
void PrintMemInfoAttr(const char *name, aclrtMemAttr attr) {
size_t freeBytes = 0;
size_t totalBytes = 0;
aclError ret = aclrtGetMemInfo(attr, &freeBytes, &totalBytes);
if (ret == ACL_SUCCESS) {
std::cout << "meminfo " << name
<< " free=" << FormatBytes(freeBytes)
<< " total=" << FormatBytes(totalBytes) << "\n";
} else {
std::cout << "meminfo " << name << " ret=" << ret << "\n";
}
}
void PrintMemInfo() {
PrintMemInfoAttr("ACL_DDR_MEM", ACL_DDR_MEM);
PrintMemInfoAttr("ACL_HBM_MEM", ACL_HBM_MEM);
PrintMemInfoAttr("ACL_DDR_MEM_HUGE", ACL_DDR_MEM_HUGE);
PrintMemInfoAttr("ACL_DDR_MEM_NORMAL", ACL_DDR_MEM_NORMAL);
PrintMemInfoAttr("ACL_HBM_MEM_HUGE", ACL_HBM_MEM_HUGE);
PrintMemInfoAttr("ACL_HBM_MEM_NORMAL", ACL_HBM_MEM_NORMAL);
PrintMemInfoAttr("ACL_HBM_MEM_HUGE1G", ACL_HBM_MEM_HUGE1G);
PrintMemInfoAttr("ACL_DDR_MEM_P2P_HUGE", ACL_DDR_MEM_P2P_HUGE);
PrintMemInfoAttr("ACL_DDR_MEM_P2P_NORMAL", ACL_DDR_MEM_P2P_NORMAL);
PrintMemInfoAttr("ACL_HBM_MEM_P2P_HUGE", ACL_HBM_MEM_P2P_HUGE);
PrintMemInfoAttr("ACL_HBM_MEM_P2P_NORMAL", ACL_HBM_MEM_P2P_NORMAL);
PrintMemInfoAttr("ACL_HBM_MEM_P2P_HUGE1G", ACL_HBM_MEM_P2P_HUGE1G);
PrintMemInfoAttr("ACL_MEM_HUGE", ACL_MEM_HUGE);
PrintMemInfoAttr("ACL_MEM_NORMAL", ACL_MEM_NORMAL);
PrintMemInfoAttr("ACL_MEM_HUGE1G", ACL_MEM_HUGE1G);
PrintMemInfoAttr("ACL_MEM_P2P_HUGE", ACL_MEM_P2P_HUGE);
PrintMemInfoAttr("ACL_MEM_P2P_NORMAL", ACL_MEM_P2P_NORMAL);
PrintMemInfoAttr("ACL_MEM_P2P_HUGE1G", ACL_MEM_P2P_HUGE1G);
}
void PrintUsage() {
std::cerr
<< "usage: ascend_memory_probe [--device 0]\n"
<< " [--policy huge-first|huge-only|normal-only|huge1g-only|huge-then-normal|\n"
<< " huge-only-then-normal|normal-then-huge-only|all]\n"
<< " [--mode info|try|single|fill|both|split|hold] [--try-mb 0]\n"
<< " [--first-mb 8192] [--second-policy normal-only]\n"
<< " [--hold-mb 8192] [--sleep-sec 30]\n"
<< " [--mode physical-info|physical-fill]\n"
<< " [--physical-policy hbm|hbm-huge|hbm-normal|hbm-p2p-huge|hbm-p2p-normal|...\n"
<< " mem-huge|mem-normal|mem-p2p-huge|mem-p2p-normal|all]\n"
<< " [--max-mb 65536] [--chunk-mb 512] [--granularity-mb 1]\n";
}
} // namespace
int main(int argc, char **argv) {
std::cout << std::unitbuf;
std::cerr << std::unitbuf;
int device = ParseIntArg(argc, argv, "--device", 0);
int tryMb = ParseIntArg(argc, argv, "--try-mb", 0);
int firstMb = ParseIntArg(argc, argv, "--first-mb", 0);
int holdMb = ParseIntArg(argc, argv, "--hold-mb", 0);
int sleepSec = ParseIntArg(argc, argv, "--sleep-sec", 30);
int maxMb = ParseIntArg(argc, argv, "--max-mb", 65536);
int chunkMb = ParseIntArg(argc, argv, "--chunk-mb", 512);
int granularityMb = ParseIntArg(argc, argv, "--granularity-mb", 1);
std::string policyArg = ParseStringArg(argc, argv, "--policy", "huge-first");
std::string secondPolicyArg = ParseStringArg(argc, argv, "--second-policy", "normal-only");
std::string physicalPolicyArg = ParseStringArg(argc, argv, "--physical-policy", "hbm");
std::string mode = ParseStringArg(argc, argv, "--mode", "both");
if (HasArg(argc, argv, "--help") || device < 0 || maxMb <= 0 || chunkMb <= 0 ||
granularityMb <= 0 || chunkMb < granularityMb ||
!(mode == "info" || mode == "try" || mode == "single" || mode == "fill" ||
mode == "both" || mode == "split" || mode == "hold" ||
mode == "physical-info" || mode == "physical-fill")) {
PrintUsage();
return HasArg(argc, argv, "--help") ? 0 : 2;
}
try {
int deviceCount = ascinfer::AscInferAscendGetDeviceCount();
if (deviceCount <= 0) {
throw std::runtime_error("no Ascend device found");
}
if (device >= deviceCount) {
throw std::runtime_error("invalid device id");
}
ascinfer::AscInferAscendSetDevice(device);
std::cout << "device=" << device << " device_count=" << deviceCount << "\n";
PrintMemInfo();
if (mode == "info") {
return 0;
}
size_t maxBytes = static_cast<size_t>(maxMb) * kMiB;
size_t tryBytes = static_cast<size_t>(tryMb) * kMiB;
size_t firstBytes = static_cast<size_t>(firstMb) * kMiB;
size_t holdBytes = static_cast<size_t>(holdMb) * kMiB;
size_t chunkBytes = static_cast<size_t>(chunkMb) * kMiB;
size_t granularityBytes = static_cast<size_t>(granularityMb) * kMiB;
if (mode == "physical-info" || mode == "physical-fill") {
auto physicalPolicies = BuildPhysicalPolicies(physicalPolicyArg);
for (auto &policy : physicalPolicies) {
aclrtPhysicalMemProp prop = BuildPhysicalMemProp(device, policy.attr);
size_t minGranularity = 0;
size_t recGranularity = 0;
aclError minRet = aclrtMemGetAllocationGranularity(
&prop, ACL_RT_MEM_ALLOC_GRANULARITY_MINIMUM, &minGranularity);
aclError recRet = aclrtMemGetAllocationGranularity(
&prop, ACL_RT_MEM_ALLOC_GRANULARITY_RECOMMENDED, &recGranularity);
std::cout << "physical-info policy=" << policy.name
<< " min_ret=" << minRet
<< " min_granularity=" << FormatBytes(minGranularity)
<< " recommended_ret=" << recRet
<< " recommended_granularity=" << FormatBytes(recGranularity) << "\n";
if (mode == "physical-fill") {
ProbeResult result = ProbePhysicalFill(device, chunkBytes, maxBytes,
granularityBytes, policy);
std::cout << "physical-fill policy=" << policy.name
<< " chunk=" << FormatBytes(chunkBytes)
<< " max=" << FormatBytes(result.bytes)
<< " next_fail_ret=" << result.nextFail << "\n";
}
}
return 0;
}
if (mode == "hold") {
if (holdMb <= 0 || sleepSec <= 0) {
throw std::runtime_error("--mode hold requires --hold-mb and --sleep-sec > 0");
}
Policy policy = BuildPolicy(policyArg);
std::vector<void *> ptrs;
size_t allocatedBytes = 0;
aclError err = ACL_SUCCESS;
bool ok = AllocateChunks(holdBytes, chunkBytes, policy, &ptrs, &allocatedBytes, &err);
std::cout << "hold policy=" << policy.name
<< " requested=" << FormatBytes(holdBytes)
<< " allocated=" << FormatBytes(allocatedBytes)
<< " ret=" << (ok ? ACL_SUCCESS : err)
<< " sleep_sec=" << sleepSec << "\n";
if (ok) {
std::this_thread::sleep_for(std::chrono::seconds(sleepSec));
}
for (void *ptr : ptrs) {
aclrtFree(ptr);
}
return ok ? 0 : 1;
}
if (mode == "split") {
if (firstMb < 0 || firstBytes > maxBytes) {
throw std::runtime_error("--mode split requires 0 <= --first-mb <= --max-mb");
}
Policy firstPolicy = BuildPolicy(policyArg);
Policy secondPolicy = BuildPolicy(secondPolicyArg);
SplitProbeResult split = ProbeSplit(firstBytes, maxBytes, chunkBytes,
granularityBytes, firstPolicy, secondPolicy);
std::cout << "split first_policy=" << firstPolicy.name
<< " first=" << FormatBytes(split.firstBytes)
<< " first_fail_ret=" << split.firstFail
<< " second_policy=" << secondPolicy.name
<< " second=" << FormatBytes(split.secondBytes)
<< " total=" << FormatBytes(split.firstBytes + split.secondBytes)
<< " second_fail_ret=" << split.secondFail << "\n";
return 0;
}
auto policies = BuildPolicies(policyArg);
for (auto &policy : policies) {
if (mode == "try") {
if (tryBytes == 0) {
throw std::runtime_error("--mode try requires --try-mb");
}
void *ptr = nullptr;
aclError err = ACL_SUCCESS;
if (TryMalloc(tryBytes, policy, &ptr, &err)) {
std::cout << "try policy=" << policy.name
<< " bytes=" << FormatBytes(tryBytes)
<< " ret=0\n";
aclrtFree(ptr);
} else {
std::cout << "try policy=" << policy.name
<< " bytes=" << FormatBytes(tryBytes)
<< " ret=" << err << "\n";
}
continue;
}
if (mode == "single" || mode == "both") {
ProbeResult single = ProbeSingle(maxBytes, granularityBytes, policy);
std::cout << "single policy=" << policy.name
<< " max=" << FormatBytes(single.bytes)
<< " next_fail_ret=" << single.nextFail << "\n";
}
if (mode == "fill" || mode == "both") {
ProbeResult fill = ProbeFill(chunkBytes, maxBytes, granularityBytes, policy);
std::cout << "fill policy=" << policy.name
<< " chunk=" << FormatBytes(chunkBytes)
<< " max=" << FormatBytes(fill.bytes)
<< " next_fail_ret=" << fill.nextFail << "\n";
}
}
aclrtSynchronizeDevice();
} catch (const std::exception &e) {
std::cerr << "error: " << e.what() << "\n";
return 1;
}
return 0;
}
这是测试分配代码,我的环境是Ascend 910PremiumA * 8,也是出现最多分配15G多一点的显存,更多就会失败报错


rui
6月2日 评论:
6月2日 评论:
您好,
针对您反馈的"ACL_HBM_MEM_HUGE 与 ACL_HBM_MEM_NORMAL 无法合并使用,单进程最多只能用到 ~15GB"问题,我们已完成根因分析与实验验证。结论如下:
核心结论
实测结果可以看到两个内存池硬件物理隔离,CANN 不提供合并机制。 单进程最大可分配内存等于所选池的实际可用容量,而非两池总和。
关键实验数据
在设备上(Ascend 910B x86),使用不同分配策略的实测上限:
| 策略 | 最大分配量 | 说明 |
|---|---|---|
ACL_MEM_MALLOC_HUGE_ONLY |
60 GB | 大页池实际可用接近芯片总容量 |
ACL_MEM_MALLOC_NORMAL_ONLY |
8 GB | 普通池容量极小,且部分被系统预留 |
在您之前使用的 910A 32G 设备上,大页池约 16GB,普通池约 15GB,但单进程只能选其一,故上限约为 16GB。
官方文档依据-内存池属性定义:aclrtMemAttr 明确区分 ACL_HBM_MEM_HUGE 与 ACL_HBM_MEM_NORMAL — 文档链接
如有其他问题,欢迎随时沟通。


6月3日 添加了label:wait-feedback
6月12日 issue状态由 待办的 改变为 已确认
6月12日 issue状态由 已确认 改变为 已完成
6月12日 关闭了 issue
6月15日 添加了label:resolved
1.问题
ACL 查询显示此芯片有两个池,但没发现如何把两个池合并使用
ACL 查询显示:
ACL_HBM_MEM total≈31.50 GiB
ACL_HBM_MEM_HUGE total≈16.02 GiB
ACL_HBM_MEM_NORMAL total≈15.48 GiB
但实际申请测试里,huge-first、huge-only、normal-only、huge -> normal、normal -> huge、physical memory API、多进程 hold+fill,全部都卡在约 15.0-15.4GiB,没有任何一种方式能把 huge 和 normal 加起来用到 31GiB。
查了 /usr/local/Ascend、driver 配置、runtime 字符串、 /proc/svm/dev0/feature/*。 能看到 bar_mem=1、bar_mem_huge=1,但 host_mem_pool=0、mem_host_uva=0、remote_mmap=0,ACL_HBM_MEM_HUGE1G 也不支持。没有找到类似“合并 huge/normal HBM 池”或“提高 user/app HBM cap”的配置开关。
Ascend910A芯片,npu-smi info显示是910ProB。
驱动:25.5.0
cann:8.5.0