import os
import socket
import logging
from math import ceil
from typing import List, Tuple, Dict, Any
import torch
import torch_npu
import torch.distributed as dist
import torch.multiprocessing as mp
from torch.multiprocessing import Process, Manager
import pandas as pd
import numpy as np
import npu_ops_transformer
logging.basicConfig(
level=logging.INFO,
format="[%(asctime)s] [%(levelname)s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger(__name__)
is_precision_test = False
is_bandwidth_test = True
is_single_test = False
graph_type = 3
is_full_core = 1
core_num = 16 if not is_full_core else 32
aic_core_num = core_num
aiv_core_num = core_num * 2
aiv_num = aiv_core_num
loop = 100
repeat = 1
warmup = 5
profiling_path = "./Profiling_Result_bandwidth_test"
server_num = 1
is_cloud = 0
if not is_cloud:
server_index = 0
master_ip = (
"71.20.45.123" if server_num > 1 else "127.0.0.1"
)
else:
task_index = os.environ.get("VC_TASK_INDEX")
if task_index is None:
raise EnvironmentError("环境变量 VC_TASK_INDEX 未设置,请检查云环境配置")
server_index = int(task_index)
host_urls = os.environ.get("VC_WORKER_HOSTS")
if host_urls is None or host_urls == "":
raise EnvironmentError("环境变量 VC_WORKER_HOSTS 未设置,请检查云环境配置")
host_url_list = host_urls.split(",")
master_ip = socket.gethostbyname(host_url_list[0])
rank_per_dev = 8
world_size = server_num * rank_per_dev
server_ranks = slice(
server_index * rank_per_dev, server_index * rank_per_dev + rank_per_dev
)
ranks_list = list(range(world_size))
def cal_hccl_buffsize(test_bs, h, ep_world_size):
MAX_OUT_DTYPE_SIZE = 2
WIN_ADDR_ALIGN = 512
UB_ALIGN = 32
SCALE_EXPAND_IDX_BUFFER = 44
DOUBLE_DATA_BUFFER = 2
token_actual_len = (
(h * MAX_OUT_DTYPE_SIZE + UB_ALIGN - 1) // UB_ALIGN
) * UB_ALIGN + SCALE_EXPAND_IDX_BUFFER
token_need_size_dispatch = (
(token_actual_len + WIN_ADDR_ALIGN - 1) // WIN_ADDR_ALIGN
) * WIN_ADDR_ALIGN
actual_size = (
test_bs * token_need_size_dispatch * ep_world_size
) * DOUBLE_DATA_BUFFER
hccl_buffsize = ceil(actual_size / 1024 / 1024 / 2) + 5
return str(hccl_buffsize)
if is_single_test:
bs = 2
data_size = 8192
max_bs = 2
test_configs = [
{"bs": bs, "data_size": data_size, "max_bs": max_bs},
]
os.environ["HCCL_BUFFSIZE"] = cal_hccl_buffsize(bs, data_size, world_size)
else:
test_configs = [
{"bs": 1, "data_size": 1024, "max_bs": 1},
{"bs": 2, "data_size": 1024, "max_bs": 2},
{"bs": 2, "data_size": 2048, "max_bs": 2},
{"bs": 4, "data_size": 2048, "max_bs": 4},
{"bs": 4, "data_size": 4096, "max_bs": 4},
{"bs": 8, "data_size": 4096, "max_bs": 8},
{"bs": 16, "data_size": 4096, "max_bs": 16},
{"bs": 32, "data_size": 4096, "max_bs": 32},
{"bs": 32, "data_size": 8192, "max_bs": 32},
{"bs": 64, "data_size": 8192, "max_bs": 64},
{"bs": 128, "data_size": 8192, "max_bs": 128},
{"bs": 256, "data_size": 8192, "max_bs": 256},
{"bs": 256, "data_size": 16384, "max_bs": 256},
{"bs": 512, "data_size": 16384, "max_bs": 512},
{"bs": 1024, "data_size": 16384, "max_bs": 1024},
{"bs": 2048, "data_size": 16384, "max_bs": 2048},
{"bs": 4096, "data_size": 16384, "max_bs": 4096},
{"bs": 8192, "data_size": 8192, "max_bs": 8192},
]
os.environ["HCCL_BUFFSIZE"] = cal_hccl_buffsize(8192, 8192, world_size)
x_dtype = "fp16"
random_seed = 90
input_dtype = torch.bfloat16 if x_dtype == "bf16" else torch.float16
dtype_bytes = torch.empty(0, dtype=x_dtype).element_size()
if is_precision_test:
mode = 0
else:
mode = 1
comm_alg = "mte"
is_save_tensor_to_bin = False
is_check_setting = True
def save_tensor_to_bin(tensor, bin_name):
if not os.path.exists("data"):
os.makedirs("data")
tensor.cpu().numpy().tofile(f"data/{bin_name}.bin")
def print_setting():
logger.info(f"{server_num=}")
logger.info(f"{rank_per_dev=}")
logger.info(f"{world_size=}")
logger.info(f"{mode=}")
logger.info(f"{comm_alg=}")
logger.info(f"{graph_type=}")
logger.info(f"{aiv_num=}")
logger.info(f"{is_precision_test=}")
logger.info(f"{is_bandwidth_test=}")
logger.info(f"{test_configs=}")
def check_setting():
def error_log_and_exit(log_info, exit_code=1):
logger.error(log_info)
raise RuntimeError(f"ERROR: {log_info}")
for i, cfg in enumerate(test_configs):
if cfg["bs"] < 1:
error_log_and_exit(f"test_configs[{i}]: bs 应大于等于 1")
if cfg["data_size"] < 1:
error_log_and_exit(f"test_configs[{i}]: data_size 应大于等于 1")
if cfg["max_bs"] < cfg["bs"]:
error_log_and_exit(f"test_configs[{i}]: max_bs 应大于等于 bs")
if (server_index < 0) or (server_index >= server_num):
error_log_and_exit("server_index 取值范围为 [0, server_num)")
if server_num * rank_per_dev != world_size:
error_log_and_exit("server_num * rank_per_dev 必须等于 world_size")
if (graph_type == 1) or (graph_type == 2):
error_log_and_exit("当前不支持动态图模式, graph_type 合法值仅有 0 和 3")
def get_profiling(num_loop, num_repeat, num_warmup, profiling_output_path):
active = num_loop // num_repeat - num_warmup
experimental_config = torch_npu.profiler._ExperimentalConfig(
export_type=torch_npu.profiler.ExportType.Text,
aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization,
profiler_level=torch_npu.profiler.ProfilerLevel.Level1,
l2_cache=False,
data_simplification=False,
)
prof = torch_npu.profiler.profile(
activities=[
torch_npu.profiler.ProfilerActivity.CPU,
torch_npu.profiler.ProfilerActivity.NPU,
],
schedule=torch_npu.profiler.schedule(
wait=0, warmup=num_warmup, active=active, repeat=num_repeat, skip_first=0
),
on_trace_ready=torch_npu.profiler.tensorboard_trace_handler(
profiling_output_path
),
profile_memory=True,
with_stack=False,
with_flops=False,
with_modules=False,
experimental_config=experimental_config,
)
return prof
def clean_previous_profiling_result():
profiling_result = "./Profiling_Result_"
os.system(f"rm -rf {profiling_result}*")
def get_bandwidth_performance(
prof_result_path: str, data_size_bytes: int, num_world_size: int, test_bs: int
):
"""从 profiling 结果中提取性能数据并计算带宽"""
execution_times = []
for dirpath, _, filenames in os.walk(prof_result_path):
for filename in filenames:
if filename.startswith("op_summary"):
op_summary = pd.read_csv(os.path.join(dirpath, filename))
bandwidth_test_info = op_summary[
op_summary["OP Type"] == "BandwidthTest"
]
if not bandwidth_test_info.empty:
op_execute_time_series = bandwidth_test_info[
"Task Duration(us)"
].tail(50)
execution_times.extend(op_execute_time_series.tolist())
if not execution_times:
logger.warning("No performance data found")
return None
execution_times_us = np.array(execution_times)
mean_time_us = np.mean(execution_times_us)
min_time_us = np.min(execution_times_us)
max_time_us = np.max(execution_times_us)
mean_time_s = mean_time_us / 1e6
min_time_s = min_time_us / 1e6
single_rank_data_bytes = (
test_bs * data_size_bytes * ((num_world_size - 1) / num_world_size)
)
total_data_bytes = single_rank_data_bytes * num_world_size
mean_bandwidth_gbps = (
(total_data_bytes / mean_time_s) / 1024 / 1024 / 1024 / num_world_size
)
max_bandwidth_gbps = (
(total_data_bytes / min_time_s) / 1024 / 1024 / 1024 / num_world_size
)
single_rank_data_kb = single_rank_data_bytes / 1024
logger.info("=" * 60)
logger.info("Bandwidth Test Performance Results")
logger.info("=" * 60)
logger.info(f"Total data per iteration: {total_data_bytes / 1e6:.2f} MB")
logger.info(f"World size: {num_world_size}")
logger.info(f"BS per rank: {test_bs}")
logger.info(f"Data size (H): {data_size_bytes // dtype_bytes}")
logger.info("-" * 60)
logger.info(f"Mean execution time: {mean_time_us:.2f} us")
logger.info(f"Min execution time: {min_time_us:.2f} us")
logger.info("-" * 60)
logger.info(f"Mean bandwidth: {mean_bandwidth_gbps:.2f} GB/s")
logger.info(f"Max bandwidth: {max_bandwidth_gbps:.2f} GB/s")
logger.info("=" * 60)
return {
"mean_time_us": mean_time_us,
"min_time_us": min_time_us,
"max_time_us": max_time_us,
"mean_bandwidth_gbps": mean_bandwidth_gbps,
"max_bandwidth_gbps": max_bandwidth_gbps,
"total_data_bytes": total_data_bytes,
"single_rank_data_kb": single_rank_data_kb,
"data_size": data_size_bytes // dtype_bytes,
"bs": test_bs,
}
def compile_model(model):
if graph_type == 0:
raise RuntimeError("单算子模式下不应调用compile_model函数")
logger.info(f"------------------graph_type: {graph_type}------------------")
import torchair
from torchair.configs.compiler_config import CompilerConfig
if is_full_core:
compiler_config = None
else:
compiler_config = CompilerConfig()
compiler_config.ge_config.aicore_num = f"{aic_core_num}|{aiv_core_num}"
logger.info(f"当前图模式仅使用: {aic_core_num} aic核, {aiv_core_num} aiv核")
npu_backend = torchair.get_npu_backend(compiler_config=compiler_config)
if graph_type == 3:
logger.info("graph_type=3, 测试场景: aclgraph入图模式")
os.makedirs("./static_kernel", exist_ok=True)
compiler_config = CompilerConfig()
logger.info("启用aclnn静态内核(aclgraph),当前只支持全核")
compiler_config.mode = "reduce-overhead"
compiler_config.debug.aclgraph.clone_input = False
compiler_config.experimental_config.aclgraph._aclnn_static_shape_kernel = True
compiler_config.experimental_config.aclgraph._aclnn_static_shape_kernel_build_dir = "./static_kernel"
npu_backend = torchair.get_npu_backend(compiler_config=compiler_config)
compiled_model = torch.compile(
model, backend=npu_backend, dynamic=False, fullgraph=True
)
return compiled_model
class BANDWIDTH_TEST_GRAPH_Model(torch.nn.Module):
def __init__(self):
super().__init__()
def forward(
self,
x,
dstrank_id,
group,
test_world_size,
test_max_bs,
test_mode,
test_comm_alg,
test_aiv_num,
):
local_loop = loop if is_bandwidth_test else 1
input_x = x
output = None
for _ in range(local_loop):
output = npu_ops_transformer.ops.npu_bandwidth_test(
x=input_x,
dstrank_id=dstrank_id,
group=group,
world_size=test_world_size,
max_bs=test_max_bs,
mode=test_mode,
comm_alg=test_comm_alg,
aiv_num=test_aiv_num,
)
[y, receive_cnt] = output
input_x = y[:test_max_bs]
return output
def set_device(rank):
torch_npu.npu.set_device(rank % rank_per_dev)
logger.info(f"current device set: {torch_npu.npu.current_device()}")
def init_hccl_comm(rank, world_size_local):
logger.info(f"[INFO] device_{rank} 创建HCCL通信链路")
dist.init_process_group(
backend="hccl",
rank=rank,
world_size=world_size_local,
init_method=f"tcp://{master_ip}:50001",
)
logger.info(f"device_{rank} init_process_group success")
logger.info(f"device {rank} 初始化通信域")
comm_group = dist.new_group(backend="hccl", ranks=ranks_list)
logger.debug("[DEBUG] device_%s 等待所有进程初始化完成...", rank)
dist.barrier()
logger.debug("[DEBUG] device_%s barrier完成", rank)
torch.npu.synchronize()
logger.debug("[DEBUG] device_%s NPU同步完成", rank)
hcomm_info = comm_group._get_backend(torch.device("npu")).get_hccl_comm_name(rank)
logger.debug(
"[DEBUG] device_%s HCCL通信域名称: %s, 长度: %s",
rank,
hcomm_info,
len(hcomm_info) if hcomm_info else 0,
)
if not hcomm_info or len(hcomm_info) == 0:
raise RuntimeError(f"device_{rank} HCCL通信域获取失败")
dist.barrier()
torch.npu.synchronize()
logger.debug("[DEBUG] device_%s 通信域完全就绪", rank)
return hcomm_info, comm_group
def generate_inputs(
test_bs: int, test_data_size: int
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""生成测试输入数据,每个发送方内部的目标rank均衡"""
torch.manual_seed(random_seed)
x_per_rank = [
torch.empty([test_bs, test_data_size], dtype=input_dtype).uniform_(-1, 1)
for _ in range(world_size)
]
dstrank_id_per_rank = []
for _ in range(world_size):
base = test_bs // world_size
rem = test_bs % world_size
targets = []
for r in range(world_size):
count = base + (1 if r < rem else 0)
targets.extend([r] * count)
indices = torch.randperm(test_bs).tolist()
shuffled_targets = [targets[i] for i in indices]
dstrank_id_per_rank.append(torch.tensor(shuffled_targets, dtype=torch.int32))
return x_per_rank, dstrank_id_per_rank
def gen_golden_data(
rank: int,
x_per_rank: List[torch.Tensor],
dstrank_id_per_rank: List[torch.Tensor],
test_max_bs: int,
test_data_size: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""生成 golden 参考数据"""
y = x_per_rank[0].new_zeros([test_max_bs * world_size, test_data_size])
receive_cnt = torch.zeros([world_size], dtype=torch.int32)
offset = 0
for src_rank in range(world_size):
x = x_per_rank[src_rank]
dstrank_id = dstrank_id_per_rank[src_rank]
dst_rank = dstrank_id % world_size
mask = dst_rank == rank
tokens_for_me = x[mask]
count = mask.sum().item()
if count > 0:
y[offset : offset + count] = tokens_for_me
offset += count
receive_cnt[src_rank] = count
return y, receive_cnt
def compare_result(
npu_output,
golden_output,
name: str,
rtol: float = 1e-3,
atol: float = 1e-3,
verbose: bool = True,
) -> bool:
"""对比 NPU 输出与 Golden 输出"""
if isinstance(npu_output, (list, tuple)):
all_pass = True
for i, (npu_out, golden_out) in enumerate(zip(npu_output, golden_output)):
if not compare_result(
npu_out, golden_out, f"{name}_{i}", rtol, atol, verbose
):
all_pass = False
return all_pass
logger.info(f"{'=' * 60}")
logger.info(f"对比 {name}")
logger.info(f"NPU shape: {npu_output.shape}, Golden shape: {golden_output.shape}")
logger.info(f"{'=' * 60}")
if verbose:
logger.info("NPU 输出:")
logger.info(npu_output)
logger.info("Golden 输出:")
logger.info(golden_output)
if torch.equal(npu_output, golden_output):
logger.info(f"[PASS] {name} 精度对比通过 (完全一致)")
return True
diff = torch.abs(npu_output - golden_output)
max_diff = diff.max().item()
if torch.is_floating_point(npu_output):
mean_diff = diff.mean().item()
rel_diff = diff / (torch.abs(golden_output) + 1e-8)
max_rel_diff = rel_diff.max().item()
if verbose:
logger.info("差异:")
logger.info(diff)
logger.info("相对差异:")
logger.info(rel_diff)
if max_rel_diff > rtol or max_diff > atol:
logger.error(f"[ERROR] {name} 精度不满足要求:")
logger.error(f" 最大相对误差: {max_rel_diff}")
logger.error(f" 容忍范围: rtol={rtol}, atol={atol}")
return False
else:
logger.info(f"[PASS] {name} 精度对比通过 (误差在容忍范围内)")
logger.info(f" 最大绝对误差: {max_diff}, 最大相对误差: {max_rel_diff:.2e}")
else:
mean_diff = diff.float().mean().item()
if verbose:
logger.info("差异:")
logger.info(diff)
if max_diff > atol:
logger.error(f"[ERROR] {name} 精度不满足要求:")
logger.error(f" 容忍范围: atol={atol}")
return False
else:
logger.info(f"[PASS] {name} 精度对比通过 (误差在容忍范围内)")
logger.info(f" 最大绝对误差: {max_diff}")
return True
def run_bandwidth_test_npu(
queue,
rank,
x,
dstrank_id,
hcomm_info,
test_max_bs=None,
test_mode=None,
test_world_size=None,
is_perf_mode=False,
destroy_comm=True,
):
"""执行 bandwidth_test 算子"""
set_device(rank)
if hcomm_info is None:
hcomm_info, _ = init_hccl_comm(rank, test_world_size)
if is_save_tensor_to_bin:
save_tensor_to_bin(x.to(torch.float32), "x_in_" + str(rank))
save_tensor_to_bin(dstrank_id.to(torch.int32), "dstrank_id_" + str(rank))
logger.info(f"[INFO] device_{rank} 构造bandwidth_test算子输入数据")
x_npu = x.to(input_dtype).npu()
dstrank_id_npu = dstrank_id.to(torch.int32).npu()
torch.npu.synchronize()
logger.debug("[DEBUG] device_%s 开始调用算子", rank)
if graph_type == 0:
bandwidth_test_kwargs = {
"x": x_npu,
"dstrank_id": dstrank_id_npu,
"group": hcomm_info,
"world_size": test_world_size,
"max_bs": test_max_bs,
"mode": test_mode,
"comm_alg": comm_alg,
"aiv_num": aiv_num,
}
if not is_perf_mode:
y, receive_cnt = npu_ops_transformer.ops.npu_bandwidth_test(
**bandwidth_test_kwargs
)
else:
tensor = torch.ones(10000, 10000).to(torch.float16).npu()
with get_profiling(loop, repeat, warmup, profiling_path) as prof:
for i in range(loop):
torch.npu.synchronize()
y, receive_cnt = npu_ops_transformer.ops.npu_bandwidth_test(
**bandwidth_test_kwargs
)
if i == 1:
for _ in range(100):
torch.exp(tensor)
prof.step()
else:
bandwidth_test_kwargs = {
"x": x_npu,
"dstrank_id": dstrank_id_npu,
"group": hcomm_info,
"test_world_size": test_world_size,
"test_max_bs": test_max_bs,
"test_mode": test_mode,
"test_comm_alg": comm_alg,
"test_aiv_num": aiv_num,
}
if not is_perf_mode:
model = compile_model(BANDWIDTH_TEST_GRAPH_Model())
y, receive_cnt = model(**bandwidth_test_kwargs)
else:
torch._dynamo.reset()
tensor = torch.ones(10000, 10000).to(torch.float16).npu()
model = compile_model(BANDWIDTH_TEST_GRAPH_Model())
loop_time = 2
repeat_time = 1
warm_time = 1
with get_profiling(
loop_time, repeat_time, warm_time, profiling_path
) as prof:
for i in range(loop_time):
y, receive_cnt = model(**bandwidth_test_kwargs)
if i == 1:
for _ in range(100):
torch.exp(tensor)
prof.step()
torch.npu.synchronize()
torch.npu.empty_cache()
if destroy_comm:
dist.destroy_process_group()
logger.info(f"rank {rank} npu finished! \n")
if is_save_tensor_to_bin:
save_tensor_to_bin(y.to(torch.float32), "y_out_" + str(rank))
save_tensor_to_bin(receive_cnt, "receive_cnt_" + str(rank))
queue.put([rank, [y.cpu(), receive_cnt.cpu()]])
def run_bandwidth_test_npu_bandwidth(
queue, rank, x, dstrank_id, test_max_bs, test_mode, test_world_size
):
"""专门用于带宽测试的执行函数"""
run_bandwidth_test_npu(
queue,
rank,
x,
dstrank_id,
None,
test_max_bs,
test_mode,
test_world_size,
is_perf_mode=True,
destroy_comm=True,
)
def run_bandwidth_test_npu_multi_config(queue, rank, configs):
"""多配置带宽测试,共享通信域
Args:
queue: 多进程通信队列
rank: 当前进程rank
configs: 配置列表,每个元素为 (x, dstrank_id, test_max_bs, test_data_size, config_index) 元组
"""
set_device(rank)
if not configs:
queue.put([rank, []])
return
_, _, _, _, test_world_size, _ = configs[0]
logger.info(f"[INFO] device_{rank} 创建通信域(多配置共享)")
hcomm_info, _ = init_hccl_comm(rank, test_world_size)
results = []
for _, (
x,
dstrank_id,
test_max_bs,
test_data_size,
test_world_size,
config_index,
) in enumerate(configs):
test_bs = x.size(0)
logger.info(f"{'=' * 60}")
logger.info(
f"[Rank {rank}] Config [{config_index}] bs={test_bs}, data_size={test_data_size},"
f"max_bs={test_max_bs}"
)
logger.info(f"{'=' * 60}")
logger.info(f"[INFO] device_{rank} 构造bandwidth_test算子输入数据")
x_npu = x.to(input_dtype).npu()
dstrank_id_npu = dstrank_id.to(torch.int32).npu()
torch.npu.synchronize()
logger.debug("[DEBUG] device_%s 开始调用算子", rank)
dist.barrier()
config_profiling_path = f"{profiling_path}_config_{config_index}"
if graph_type == 0:
bandwidth_test_kwargs = {
"x": x_npu,
"dstrank_id": dstrank_id_npu,
"group": hcomm_info,
"world_size": test_world_size,
"max_bs": test_max_bs,
"mode": mode,
"comm_alg": comm_alg,
"aiv_num": aiv_num,
}
tensor = torch.ones(10000, 10000).to(torch.float16).npu()
with get_profiling(loop, repeat, warmup, config_profiling_path) as prof:
for j in range(loop):
torch.npu.synchronize()
y, receive_cnt = npu_ops_transformer.ops.npu_bandwidth_test(
**bandwidth_test_kwargs
)
if j == 1:
for _ in range(100):
torch.exp(tensor)
prof.step()
else:
bandwidth_test_kwargs = {
"x": x_npu,
"dstrank_id": dstrank_id_npu,
"group": hcomm_info,
"test_world_size": test_world_size,
"test_max_bs": test_max_bs,
"test_mode": mode,
"test_comm_alg": comm_alg,
"test_aiv_num": aiv_num,
}
torch._dynamo.reset()
tensor = torch.ones(10000, 10000).to(torch.float16).npu()
model = compile_model(BANDWIDTH_TEST_GRAPH_Model())
loop_time = 2
repeat_time = 1
warm_time = 1
with get_profiling(
loop_time, repeat_time, warm_time, config_profiling_path
) as prof:
for i in range(loop_time):
y, receive_cnt = model(**bandwidth_test_kwargs)
if i == 1:
for _ in range(100):
torch.exp(tensor)
prof.step()
dist.barrier()
torch.npu.synchronize()
torch.npu.empty_cache()
if is_save_tensor_to_bin:
save_tensor_to_bin(
y.to(torch.float32), f"y_out_{rank}_config{config_index}"
)
save_tensor_to_bin(receive_cnt, f"receive_cnt_{rank}_config{config_index}")
results.append(
{
"config_index": config_index,
"y": y.cpu(),
"receive_cnt": receive_cnt.cpu(),
"test_bs": test_bs,
"test_data_size": test_data_size,
}
)
logger.info(f"[INFO] device_{rank} Config [{config_index}] 完成")
dist.destroy_process_group()
queue.put([rank, results])
def run_precision_test(
x_per_rank: List[torch.Tensor],
dstrank_id_per_rank: List[torch.Tensor],
test_max_bs: int,
test_data_size: int,
) -> bool:
"""运行精度测试"""
logger.info("=" * 60)
logger.info("Precision Test")
logger.info("=" * 60)
manager = Manager()
result_queue = manager.Queue()
processes = []
for rank in range(world_size):
p = Process(
target=run_bandwidth_test_npu,
args=(
result_queue,
rank,
x_per_rank[rank],
dstrank_id_per_rank[rank],
None,
test_max_bs,
mode,
world_size,
False,
True,
),
)
p.start()
processes.append(p)
for p in processes:
p.join()
npu_outputs = {}
if result_queue.qsize() == world_size:
while not result_queue.empty():
rank, output = result_queue.get()
if output is not None:
npu_outputs[rank] = output
else:
logger.error(f"[ERROR] Rank {rank} 执行失败")
return False
else:
logger.error("Get result_queue Failed")
return False
logger.info("-" * 60)
logger.info("Precision Verification:")
logger.info("-" * 60)
all_pass = True
for rank in range(world_size):
logger.info(f"\n[Rank {rank}] 开始精度对比...")
golden_y, golden_receive_cnt = gen_golden_data(
rank, x_per_rank, dstrank_id_per_rank, test_max_bs, test_data_size
)
npu_y, npu_receive_cnt = npu_outputs[rank]
y_pass = compare_result(npu_y, golden_y, f"rank_{rank}_y")
receive_cnt_pass = compare_result(
npu_receive_cnt, golden_receive_cnt, f"rank_{rank}_receive_cnt"
)
if not (y_pass and receive_cnt_pass):
all_pass = False
return all_pass
def run_bandwidth_test(
x_per_rank: List[torch.Tensor],
dstrank_id_per_rank: List[torch.Tensor],
test_max_bs: int,
test_data_size: int,
) -> Dict[str, Any]:
"""运行带宽测试"""
logger.info("=" * 60)
logger.info("Bandwidth Test")
logger.info("=" * 60)
clean_previous_profiling_result()
manager = Manager()
result_queue = manager.Queue()
processes = []
for rank in range(world_size):
p = Process(
target=run_bandwidth_test_npu_bandwidth,
args=(
result_queue,
rank,
x_per_rank[rank],
dstrank_id_per_rank[rank],
test_max_bs,
mode,
world_size,
),
)
p.start()
processes.append(p)
for p in processes:
p.join()
npu_outputs = {}
while not result_queue.empty():
rank, output = result_queue.get()
if output is not None:
npu_outputs[rank] = output
else:
logger.error(f"[ERROR] Rank {rank} 带宽测试执行失败")
return None
data_size_bytes = test_data_size * dtype_bytes
perf_result = get_bandwidth_performance(
profiling_path, data_size_bytes, world_size, len(x_per_rank[0])
)
return perf_result
def run_multi_config_bandwidth_test() -> List[Dict[str, Any]]:
"""运行多配置带宽测试(共享通信域)"""
all_results = []
clean_previous_profiling_result()
all_inputs = []
for i, cfg in enumerate(test_configs):
test_bs = cfg["bs"]
test_data_size = cfg["data_size"]
test_max_bs = cfg["max_bs"]
x_per_rank, dstrank_id_per_rank = generate_inputs(test_bs, test_data_size)
all_inputs.append(
(x_per_rank, dstrank_id_per_rank, test_max_bs, test_data_size, i + 1)
)
manager = Manager()
result_queue = manager.Queue()
processes = []
for rank in range(world_size):
rank_configs = [
(
all_inputs[i][0][rank],
all_inputs[i][1][rank],
all_inputs[i][2],
all_inputs[i][3],
world_size,
all_inputs[i][4],
)
for i in range(len(all_inputs))
]
p = Process(
target=run_bandwidth_test_npu_multi_config,
args=(result_queue, rank, rank_configs),
)
p.start()
processes.append(p)
for p in processes:
p.join()
npu_all_results = {}
while not result_queue.empty():
rank, results = result_queue.get()
if results is not None:
npu_all_results[rank] = results
else:
logger.error(f"[ERROR] Rank {rank} 多配置测试执行失败")
return []
for i, cfg in enumerate(test_configs):
test_data_size = cfg["data_size"]
test_bs = cfg["bs"]
data_size_bytes = test_data_size * dtype_bytes
config_index = i + 1
config_profiling_path = f"{profiling_path}_config_{config_index}"
perf_result = get_bandwidth_performance(
config_profiling_path, data_size_bytes, world_size, test_bs
)
if perf_result:
perf_result["config_index"] = config_index
perf_result["config"] = cfg
all_results.append(perf_result)
print_summary_table(all_results)
return all_results
def print_summary_table(results: List[Dict[str, Any]]):
"""打印汇总结果表格"""
logger.info("=" * 90)
logger.info("Bandwidth Test Summary - All Configurations")
logger.info("=" * 90)
if not results:
logger.warning("[WARN] No valid test results")
return
header = (
f"{'No.':<5} {'BS':<5} {'H':<10} {'Total Data(KB)':<16} "
f"{'Mean BW(GB/s)':<16} {'Max BW(GB/s)':<16} {'Mean Time(us)':<14}"
)
logger.info(header)
logger.info("-" * 90)
max_bandwidth = 0
max_bw_config = None
for r in results:
cfg = r["config"]
row = (
f"{r['config_index']:<5} {cfg['bs']:<5} {cfg['data_size']:<10} "
f"{r['single_rank_data_kb']:<16.4f} {r['mean_bandwidth_gbps']:<16.2f} "
f"{r['max_bandwidth_gbps']:<16.2f} {r['mean_time_us']:<14.2f}"
)
logger.info(row)
if r["max_bandwidth_gbps"] > max_bandwidth:
max_bandwidth = r["max_bandwidth_gbps"]
max_bw_config = r
logger.info("-" * 90)
if max_bw_config:
logger.info(f"\n{'*' * 90}")
logger.info(
f"Maximum Bandwidth: {max_bw_config['max_bandwidth_gbps']:.2f} GB/s"
)
logger.info(f" H: {max_bw_config['config']['data_size']}")
logger.info(f" Total Data: {max_bw_config['single_rank_data_kb']:.4f} KB")
logger.info(f" Mean Time: {max_bw_config['mean_time_us']:.2f} us")
logger.info(f"{'*' * 90}")
def main():
print_setting()
if is_check_setting:
check_setting()
if is_bandwidth_test and len(test_configs) > 1:
all_results = run_multi_config_bandwidth_test()
logger.info("=" * 60)
logger.info("Test Summary")
logger.info("=" * 60)
logger.info(
f"Total configurations tested: {len(all_results)}/{len(test_configs)}"
)
logger.info("=" * 60)
else:
x_per_rank, dstrank_id_per_rank = generate_inputs(bs, data_size)
precision_pass = True
bandwidth_result = None
if is_precision_test:
precision_pass = run_precision_test(
x_per_rank, dstrank_id_per_rank, max_bs, data_size
)
if is_bandwidth_test:
bandwidth_result = run_bandwidth_test(
x_per_rank, dstrank_id_per_rank, max_bs, data_size
)
logger.info("=" * 60)
logger.info("Test Summary")
logger.info("=" * 60)
if is_precision_test:
logger.info(f"Precision Test: {'PASS' if precision_pass else 'FAIL'}")
if is_bandwidth_test:
if bandwidth_result:
logger.info("Bandwidth Test: Completed")
logger.info(
f" Mean Bandwidth: {bandwidth_result['mean_bandwidth_gbps']:.2f} GB/s"
)
logger.info(
f" Max Bandwidth: {bandwidth_result['max_bandwidth_gbps']:.2f} GB/s"
)
else:
logger.error("Bandwidth Test: FAILED")
logger.info("=" * 60)
if is_precision_test and not precision_pass:
exit(1)
if __name__ == "__main__":
mp.set_start_method("spawn")
main()