已关闭

[Bug]: 主线程绑核时机有bug，会导致漏绑或超前绑核 #2224

zhaoyu65创建于 6月2日关闭于 6月9日

zhaoyu65成员

6月2日创建

在提交新问题之前，请确保您已经在社区中搜索过相关问题，并使用了社区中提供的资源/工具后，仍未找到满意的解决方式。

⚠️ 安全信息提醒：请仔细检查提供的文本内容，确保其不包含敏感数据信息，包括但不限于：

API 令牌或密钥
密码或身份验证凭证
私有网址或接口地址
个人或机密数据
...

在分享配置信息或代码示例时，请将敏感信息脱敏处理，或使用 <TOKEN> 等占位符替代原有内容。

当前主线代码主线程绑核时序图：
plantuml-diagram (2).png
修改后的绑核时序图：
plantuml-diagram (1).png

环境信息

测试脚本：

import argparse
import time
import os
os.environ["ASCEND_PROCESS_LOG_PATH"] = "/home/zhaoyu"
os.environ["ASCEND_GLOBAL_LOG_LEVEL"] = "0"
os.environ["ASCEND_HOST_LOG_FILE_NUM"] = "100"
import threading
import torch
import torch.nn as nn
import torch.optim as optim
import torch_npu
from torch.utils.data import Dataset, DataLoader


def seed_everything(seed: int = 42):
    """固定随机种子，保证可复现性"""
    torch.manual_seed(seed)
    if hasattr(torch, 'npu'):
        torch.npu.manual_seed(seed)
        torch.npu.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    torch.use_deterministic_algorithms(True)

def format_affinity(affinity):
    sorted_cores = sorted(affinity)
    ranges = []
    start = sorted_cores[0]
    end = sorted_cores[0]
    for core in sorted_cores[1:]:
        if core == end + 1:
            end = core
        else:
            ranges.append(f"{start}-{end}" if start != end else f"{start}")
            start = end = core
    ranges.append(f"{start}-{end}" if start != end else f"{start}")
    return "{" + ", ".join(ranges) + "}"

# ---- 简化的 ResNet50 定义 ----

class BasicConv(nn.Module):
    def __init__(self, in_c, out_c, k=3, s=1, p=1):
        super().__init__()
        self.conv = nn.Conv2d(in_c, out_c, k, s, p, bias=False)
        self.bn = nn.BatchNorm2d(out_c)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, x):
        return self.relu(self.bn(self.conv(x)))


class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, in_c, out_c, stride=1, downsample=None):
        super().__init__()
        self.conv1 = nn.Conv2d(in_c, out_c, 1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_c)
        self.conv2 = nn.Conv2d(out_c, out_c, 3, stride, 1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_c)
        self.conv3 = nn.Conv2d(out_c, out_c * self.expansion, 1, bias=False)
        self.bn3 = nn.BatchNorm2d(out_c * self.expansion)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x if self.downsample is None else self.downsample(x)

        out = self.relu(self.bn1(self.conv1(x)))
        out = self.relu(self.bn2(self.conv2(out)))
        out = self.bn3(self.conv3(out))
        out += identity
        out = self.relu(out)
        return out


class ResNet50(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 64, 7, 2, 3, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(3, 2, 1)

        self.layer1 = self._make_layer(64, 64, 3, stride=1)
        self.layer2 = self._make_layer(256, 128, 4, stride=2)
        self.layer3 = self._make_layer(512, 256, 6, stride=2)
        self.layer4 = self._make_layer(1024, 512, 3, stride=2)

        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(512 * Bottleneck.expansion, num_classes)

    def _make_layer(self, in_c, out_c, blocks, stride):
        downsample = nn.Sequential(
            nn.Conv2d(in_c, out_c * Bottleneck.expansion, 1, stride, bias=False),
            nn.BatchNorm2d(out_c * Bottleneck.expansion),
        )
        layers = [Bottleneck(in_c, out_c, stride, downsample)]
        for _ in range(1, blocks):
            layers.append(Bottleneck(out_c * Bottleneck.expansion, out_c))
        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.maxpool(self.relu(self.bn1(self.conv1(x))))
        x = self.layer4(self.layer3(self.layer2(self.layer1(x))))
        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.fc(x)
        return x


class RandomDataset(Dataset):
    def __init__(self, num_samples, num_classes=10):
        self.images = torch.randn(num_samples, 3, 224, 224)
        self.labels = torch.randint(0, num_classes, (num_samples,))

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        return self.images[idx], self.labels[idx]


def train(rank_id, num_works=2):
    # 固定随机种子
    seed_everything(42)
    torch.npu.set_device(rank_id)

    device = torch.device(f"npu:{rank_id}")

    # 模型
    model = ResNet50(num_classes=10).to(device)

    # 参数
    batch_size = 32
    num_samples = 64
    num_epochs = 2

    dataset = RandomDataset(num_samples)
    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_works)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)

    for epoch in range(num_epochs):
        model.train()
        running_loss = 0.0
        epoch_start = time.time()

        for images, labels in dataloader:
            images = images.to(device)
            labels = labels.to(device)

            optimizer.zero_grad()
            outputs = model(images)
            loss = criterion(outputs, labels)
            print(f"main thread: before backward CPU affinity:{format_affinity(os.sched_getaffinity(0))}")
            loss.backward()
            print(f"main thread: after backward CPU affinity={format_affinity(os.sched_getaffinity(0))}")
            optimizer.step()

            running_loss += loss.item()

        epoch_time = time.time() - epoch_start
        avg_loss = running_loss / len(dataloader)
        step_time = epoch_time / len(dataloader)

        print(
            f"Epoch [{epoch + 1}/{num_epochs}] | "
            f"Loss: {avg_loss:.4f} | "
            f"Avg Step Time: {step_time:.4f}s | "
            f"Epoch Time: {epoch_time:.2f}s"
        )

    print("Training completed!")

def work1(device_id):
    torch_npu.utils.set_thread_affinity([0, 10])
    a = torch.tensor([1, 2, 3], device="npu:" + str(device_id))
    b = a.sum()
    torch_npu.npu.synchronize()
    print(f"work1: {torch_npu.npu.current_device()=}, result: {b.item()}, CPU affinity:{format_affinity(os.sched_getaffinity(0))}")

def work2(device_id):
    torch_npu.npu.set_device(device_id)
    a = torch.tensor([1, 2, 3], device="npu:" + str(device_id))
    b = a.sum()
    torch_npu.npu.synchronize()
    print(f"work2: {torch_npu.npu.current_device()=}, result: {b.item()}, CPU affinity:{format_affinity(os.sched_getaffinity(0))}")

if __name__ == "__main__":
    os.environ["CPU_AFFINITY_CONF"] = "2,force:1"
    rank = int(os.environ.get("LOCAL_RANK", 3))
    print(f"获取到rank={rank}")
    parser = argparse.ArgumentParser(description="Run work1 or work2")
    parser.add_argument("--work", type=int, choices=[0, 1, 2], required=True, help="运行模式(0:仅运行train, 1:仅运行work1, 2:仅运行work2)")
    parser.add_argument("--num_works", type=int, default=0, help="DataLoader的num_workers参数")
    args = parser.parse_args()
    work = args.work
    if work == 1:
        run_work1, run_work2 = True, False
    elif work == 2:
        run_work1, run_work2 = False, True
    elif work == 0:
        run_work1, run_work2 = False, False
    else:
        raise ValueError(f"无效的输入: {work}, 请输入0, 1或2")
    if run_work1:
        t1 = threading.Thread(target=work1, args=(rank,))
        t1.start()
        t1.join()
    if run_work2:
        t2 = threading.Thread(target=work2, args=(rank,))
        t2.start()
        t2.join()
    train(rank, num_works=args.num_works)

🐛 问题描述

当前绑核相关代码，存在如下2个问题：

1、如果用户在前向计算前，基于主线程创建了子线程，并在子线程中首先执行了NPU初始化，会造成主线程绑核失败，根因是NPU首次初始化时会执行SetMainThread，把用户创建的线程当成了主线程，导致真正的主线程绑核失败；
2、如果主线程首先完成了NPU初始化，用户的子线程首先使用了NPU，会造成主线程超前绑核。即：当前希望主线程的亲和性由autograd线程来绑定，为了避免影响前面基于主线程创建出来的子线程的亲和性。而当前的代码在StartMainThreadBind函数中会执行绑核逻辑。

主线程绑核验证结果总结

代码版本	数据处理多进程	场景	主线程绑核时机	结果是否符合预期
现有主线代码	不开启	resnet50正常做正反向	第一次反向后被绑核	✅ 符合预期
现有主线代码	不开启	正向前拉起work1线程，先于主线程执行NPU初始化	主线程绑核失败	❌ 不符合预期
现有主线代码	不开启	正向前拉起work2线程，先于主线程执行set_device	主线程超前绑核（执行反向前就被绑核）	❌ 不符合预期
现有主线代码	开启	resnet50正常做正反向	拉起数据处理进程时被超前绑核	❌ 不符合预期
现有主线代码	开启	正向前拉起work1线程，先于主线程执行NPU初始化	拉起数据处理进程时被超前绑核	❌ 不符合预期
现有主线代码	开启	正向前拉起work2线程，先于主线程执行set_device	拉起数据处理进程时被超前绑核	❌ 不符合预期

一、现有主线代码运行效果

不开启数据处理多进程

1、resnet50正常做正反向：主线程在第一次反向后被绑核。结果符合预期。

2、resnet50正向前，拉起work1线程，先于主线程执行NPU初始化。结果为主线程绑核失败，结果不符合预期。

3、resnet50正向前，拉起work2线程，先于主线程执行set_device。结果为主线程超前绑核，即在执行反向前就被绑核了，结果不符合预期。

开启数据处理多进程

1、resnet50正常做正反向：主线程在拉起数据处理进程时被超前绑核。结果不符合预期。

2、resnet50正向前，拉起work1线程，先于主线程执行NPU初始化。结果为主线程在拉起数据处理进程时被超前绑核。结果不符合预期。

3、resnet50正向前，拉起work2线程，先于主线程执行set_device。结果为主线程在拉起数据处理进程时被超前绑核。结果不符合预期。

欢迎加入社区，感谢您对社区的贡献 🎉!

zhaoyu65成员

6月2日添加了label：bug

zhaoyu65成员

6月2日评论：

一、主线程绑核验证结果总结

代码版本	数据处理多进程	场景	主线程绑核时机	结果是否符合预期
现有主线代码	不开启	resnet50正常做正反向	第一次反向后被绑核	✅ 符合预期
现有主线代码	不开启	正向前拉起work1线程，先于主线程执行NPU初始化	主线程绑核失败	❌ 不符合预期
现有主线代码	不开启	正向前拉起work2线程，先于主线程执行set_device	主线程超前绑核（执行反向前就被绑核）	❌ 不符合预期
现有主线代码	开启	resnet50正常做正反向	拉起数据处理进程时被超前绑核	❌ 不符合预期
现有主线代码	开启	正向前拉起work1线程，先于主线程执行NPU初始化	拉起数据处理进程时被超前绑核	❌ 不符合预期
现有主线代码	开启	正向前拉起work2线程，先于主线程执行set_device	拉起数据处理进程时被超前绑核	❌ 不符合预期
修改后代码	不开启	resnet50正常做正反向	第一次反向后被绑核	✅ 符合预期
修改后代码	不开启	正向前拉起work1线程，先于主线程执行NPU初始化	第一次反向时被pt_autograd_3线程绑核	✅ 符合预期
修改后代码	不开启	正向前拉起work2线程，先于主线程执行set_device	第一次反向时被pt_autograd_3线程绑核	✅ 符合预期
修改后代码	开启	resnet50正常做正反向	第一次反向后被绑核	✅ 符合预期
修改后代码	开启	正向前拉起work1线程，先于主线程执行NPU初始化	第一次反向时被pt_autograd_3线程绑核	✅ 符合预期
修改后代码	开启	正向前拉起work2线程，先于主线程执行set_device	第一次反向时被pt_autograd_3线程绑核	✅ 符合预期

测试脚本：

#!/bin/bash

DEBUG_DIR=/home/zhaoyu/debug

for num_works in 0 2; do
    if [ "$num_works" -eq 0 ]; then
        echo "num_works=0，DataLoader 不使用多线程"
    else
        echo "num_works=$num_works，DataLoader 使用 $num_works 个线程"
    fi
    for work in 0 1 2; do
        rm -rf "$DEBUG_DIR"
        echo "========== resnet50.py --work $work --num_works $num_works =========="
        python resnet50.py --work "$work" --num_works "$num_works"
        cat "$DEBUG_DIR"/plog/* 2>/dev/null | grep Affinity
        echo
    done
done

二、修改后代码运行效果

不开启数据处理多进程

1、resnet50正常做正反向：主线程在第一次反向后被绑核。结果符合预期。
2、resnet50正向前，拉起work1线程，先于主线程执行NPU初始化。结果为主线程在第一次反向时被pt_autograd_3线程绑核。结果符合预期。
3、resnet50正向前，拉起work2线程，先于主线程执行set_device。结果为主线程在第一次反向时被pt_autograd_3线程绑核。结果符合预期。

开启数据处理多进程

运行日志

num_works=0，DataLoader 不使用多线程
========== resnet50.py --work 0 --num_works 0 ==========
获取到rank=3
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[W608 14:09:48.468961030 NPUCachingAllocator.cpp:202] Warning: The current CANN and HDK(driver) versions require processing for 32 padding size, with memory allocation. (function operator())
main thread: before backward CPU affinity:{0-255}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [1/2] | Loss: 2.5761 | Avg Step Time: 1.9470s | Epoch Time: 3.89s
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [2/2] | Loss: 5.5542 | Avg Step Time: 0.5958s | Epoch Time: 1.19s
Training completed!
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.101.887 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:parseCPUAffinityConf:240: "[PTA]:Get device nums: 8 by aclrtGetDeviceCount."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.101.981 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:parseCPUAffinityConf:250: "[PTA]:Get env var CPU_AFFINITY_CONF: 2,force:1"
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.206 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:printCoreRanges:279: "[PTA]:Read CPU affinity config: Mode: 2. Core range for each device ID: Device 0: [0-31]; Device 1: [32-63]; Device 2: [64-95]; Device 3: [96-127]; Device 4: [128-159]; Device 5: [160-191]; Device 6: [192-223]; Device 7: [224-255]."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.212 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 0 get cores 0-31."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.216 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 1 get cores 32-63."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.219 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 2 get cores 64-95."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.223 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 3 get cores 96-127."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.226 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 4 get cores 128-159."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.229 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 5 get cores 160-191."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.232 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 6 get cores 192-223."
[DEBUG] APP(4136050,python):2026-06-08-14:09:47.102.235 [log_inner.cpp:87]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 7 get cores 224-255."
[INFO] APP(4136050,python):2026-06-08-14:09:47.102.238 [log_inner.cpp:84]4136050 NPUAffinityController.cpp:getThreadAffinityInfo:336: "[PTA]:CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding."
[DEBUG] APP(4136050,python):2026-06-08-14:09:48.033.639 [log_inner.cpp:87]4138342 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to acl_thread success."
[DEBUG] APP(4136050,python):2026-06-08-14:09:48.033.680 [log_inner.cpp:87]4138342 NPUAffinityController.cpp:getCpuAffinityMap:384: "[PTA]:Device 3 thread affinity map: main_thread: [96]; acl_thread: [97]; release_thread: [98]; hccl_watchdog_t: [99]; other_thread: [100-127]."
[DEBUG] APP(4136050,python):2026-06-08-14:09:48.033.728 [log_inner.cpp:87]4138343 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to release_thread success."
[DEBUG] APP(4136050,python):2026-06-08-14:09:48.033.751 [log_inner.cpp:87]4138342 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set acl_thread affinity to 97 success."
[DEBUG] APP(4136050,python):2026-06-08-14:09:48.033.789 [log_inner.cpp:87]4138343 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set release_thread affinity to 98 success."
[DEBUG] APP(4136050,python):2026-06-08-14:09:49.305.687 [log_inner.cpp:87]4138545 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4136050,python):2026-06-08-14:09:49.306.439 [log_inner.cpp:87]4138545 NPUAffinityController.cpp:StartMainThreadBind:571: "[PTA]:Device 3 set main_thread affinity to 96 success, thread name: pt_autograd_3."

========== resnet50.py --work 1 --num_works 0 ==========
获取到rank=3
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[W608 14:10:06.655494310 NPUCachingAllocator.cpp:202] Warning: The current CANN and HDK(driver) versions require processing for 32 padding size, with memory allocation. (function operator())
work1: torch_npu.npu.current_device()=0, result: 6, CPU affinity:{0-10}
main thread: before backward CPU affinity:{0-255}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [1/2] | Loss: 2.5761 | Avg Step Time: 1.6970s | Epoch Time: 3.39s
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [2/2] | Loss: 5.5542 | Avg Step Time: 0.4307s | Epoch Time: 0.86s
Training completed!
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.312 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:parseCPUAffinityConf:240: "[PTA]:Get device nums: 8 by aclrtGetDeviceCount."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.430 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:parseCPUAffinityConf:250: "[PTA]:Get env var CPU_AFFINITY_CONF: 2,force:1"
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.695 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:printCoreRanges:279: "[PTA]:Read CPU affinity config: Mode: 2. Core range for each device ID: Device 0: [0-31]; Device 1: [32-63]; Device 2: [64-95]; Device 3: [96-127]; Device 4: [128-159]; Device 5: [160-191]; Device 6: [192-223]; Device 7: [224-255]."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.701 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 0 get cores 0-31."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.705 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 1 get cores 32-63."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.708 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 2 get cores 64-95."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.711 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 3 get cores 96-127."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.715 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 4 get cores 128-159."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.718 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 5 get cores 160-191."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.721 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 6 get cores 192-223."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.724 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 7 get cores 224-255."
[INFO] APP(4140604,python):2026-06-08-14:10:05.146.727 [log_inner.cpp:84]4141955 NPUAffinityController.cpp:getThreadAffinityInfo:336: "[PTA]:CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding."
[DEBUG] APP(4140604,python):2026-06-08-14:10:05.146.862 [log_inner.cpp:87]4141955 NPUAffinityController.cpp:SetThreadAffinity:496: "[PTA]:Set thread affinity to user-defined range 0-10 success."
[DEBUG] APP(4140604,python):2026-06-08-14:10:06.220.334 [log_inner.cpp:87]4142352 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to acl_thread success."
[DEBUG] APP(4140604,python):2026-06-08-14:10:06.220.356 [log_inner.cpp:87]4142353 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to release_thread success."
[DEBUG] APP(4140604,python):2026-06-08-14:10:06.220.388 [log_inner.cpp:87]4142352 NPUAffinityController.cpp:getCpuAffinityMap:384: "[PTA]:Device 3 thread affinity map: main_thread: [96]; acl_thread: [97]; release_thread: [98]; hccl_watchdog_t: [99]; other_thread: [100-127]."
[DEBUG] APP(4140604,python):2026-06-08-14:10:06.220.452 [log_inner.cpp:87]4142352 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set acl_thread affinity to 97 success."
[DEBUG] APP(4140604,python):2026-06-08-14:10:06.220.452 [log_inner.cpp:87]4142353 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set release_thread affinity to 98 success."
[DEBUG] APP(4140604,python):2026-06-08-14:10:07.503.405 [log_inner.cpp:87]4143170 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4140604,python):2026-06-08-14:10:07.503.928 [log_inner.cpp:87]4143170 NPUAffinityController.cpp:StartMainThreadBind:571: "[PTA]:Device 3 set main_thread affinity to 96 success, thread name: pt_autograd_3."

========== resnet50.py --work 2 --num_works 0 ==========
获取到rank=3
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[W608 14:10:24.611763030 NPUCachingAllocator.cpp:202] Warning: The current CANN and HDK(driver) versions require processing for 32 padding size, with memory allocation. (function operator())
work2: torch_npu.npu.current_device()=3, result: 6, CPU affinity:{96}
main thread: before backward CPU affinity:{0-255}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [1/2] | Loss: 2.5761 | Avg Step Time: 1.9256s | Epoch Time: 3.85s
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [2/2] | Loss: 5.5542 | Avg Step Time: 0.6126s | Epoch Time: 1.23s
Training completed!
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.332 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:parseCPUAffinityConf:240: "[PTA]:Get device nums: 8 by aclrtGetDeviceCount."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.424 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:parseCPUAffinityConf:250: "[PTA]:Get env var CPU_AFFINITY_CONF: 2,force:1"
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.671 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:printCoreRanges:279: "[PTA]:Read CPU affinity config: Mode: 2. Core range for each device ID: Device 0: [0-31]; Device 1: [32-63]; Device 2: [64-95]; Device 3: [96-127]; Device 4: [128-159]; Device 5: [160-191]; Device 6: [192-223]; Device 7: [224-255]."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.676 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 0 get cores 0-31."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.680 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 1 get cores 32-63."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.684 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 2 get cores 64-95."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.687 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 3 get cores 96-127."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.691 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 4 get cores 128-159."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.694 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 5 get cores 160-191."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.697 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 6 get cores 192-223."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.926.700 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 7 get cores 224-255."
[INFO] APP(4145790,python):2026-06-08-14:10:23.926.703 [log_inner.cpp:84]4147223 NPUAffinityController.cpp:getThreadAffinityInfo:336: "[PTA]:CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.930.814 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:getCpuAffinityMap:384: "[PTA]:Device 3 thread affinity map: main_thread: [96]; acl_thread: [97]; release_thread: [98]; hccl_watchdog_t: [99]; other_thread: [100-127]."
[DEBUG] APP(4145790,python):2026-06-08-14:10:23.930.887 [log_inner.cpp:87]4147223 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4145790,python):2026-06-08-14:10:24.176.516 [log_inner.cpp:87]4147487 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to release_thread success."
[DEBUG] APP(4145790,python):2026-06-08-14:10:24.176.574 [log_inner.cpp:87]4147487 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set release_thread affinity to 98 success."
[DEBUG] APP(4145790,python):2026-06-08-14:10:24.176.668 [log_inner.cpp:87]4147486 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to acl_thread success."
[DEBUG] APP(4145790,python):2026-06-08-14:10:24.176.696 [log_inner.cpp:87]4147486 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set acl_thread affinity to 97 success."
[DEBUG] APP(4145790,python):2026-06-08-14:10:26.047.077 [log_inner.cpp:87]4148092 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4145790,python):2026-06-08-14:10:26.047.843 [log_inner.cpp:87]4148092 NPUAffinityController.cpp:StartMainThreadBind:571: "[PTA]:Device 3 set main_thread affinity to 96 success, thread name: pt_autograd_3."

num_works=2，DataLoader 使用 2 个线程
========== resnet50.py --work 0 --num_works 2 ==========
获取到rank=3
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[W608 14:10:44.531072700 NPUCachingAllocator.cpp:202] Warning: The current CANN and HDK(driver) versions require processing for 32 padding size, with memory allocation. (function operator())
main thread: before backward CPU affinity:{0-255}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [1/2] | Loss: 2.5761 | Avg Step Time: 1.7186s | Epoch Time: 3.44s
/home/zhaoyu/miniconda3/envs/pta/lib/python3.10/site-packages/torch/utils/data/dataloader.py:432: UserWarning: This DataLoader will create 2 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  self.check_worker_number_rationality()
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [2/2] | Loss: 5.5542 | Avg Step Time: 0.4638s | Epoch Time: 0.93s
Training completed!
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.337 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:parseCPUAffinityConf:240: "[PTA]:Get device nums: 8 by aclrtGetDeviceCount."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.436 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:parseCPUAffinityConf:250: "[PTA]:Get env var CPU_AFFINITY_CONF: 2,force:1"
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.680 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:printCoreRanges:279: "[PTA]:Read CPU affinity config: Mode: 2. Core range for each device ID: Device 0: [0-31]; Device 1: [32-63]; Device 2: [64-95]; Device 3: [96-127]; Device 4: [128-159]; Device 5: [160-191]; Device 6: [192-223]; Device 7: [224-255]."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.686 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 0 get cores 0-31."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.689 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 1 get cores 32-63."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.693 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 2 get cores 64-95."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.696 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 3 get cores 96-127."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.699 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 4 get cores 128-159."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.702 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 5 get cores 160-191."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.705 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 6 get cores 192-223."
[DEBUG] APP(4150170,python):2026-06-08-14:10:43.194.708 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 7 get cores 224-255."
[INFO] APP(4150170,python):2026-06-08-14:10:43.194.711 [log_inner.cpp:84]4150170 NPUAffinityController.cpp:getThreadAffinityInfo:336: "[PTA]:CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding."
[DEBUG] APP(4150170,python):2026-06-08-14:10:44.095.887 [log_inner.cpp:87]4152130 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to acl_thread success."
[DEBUG] APP(4150170,python):2026-06-08-14:10:44.095.928 [log_inner.cpp:87]4152130 NPUAffinityController.cpp:getCpuAffinityMap:384: "[PTA]:Device 3 thread affinity map: main_thread: [96]; acl_thread: [97]; release_thread: [98]; hccl_watchdog_t: [99]; other_thread: [100-127]."
[DEBUG] APP(4150170,python):2026-06-08-14:10:44.096.000 [log_inner.cpp:87]4152131 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to release_thread success."
[DEBUG] APP(4150170,python):2026-06-08-14:10:44.096.047 [log_inner.cpp:87]4152130 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set acl_thread affinity to 97 success."
[DEBUG] APP(4150170,python):2026-06-08-14:10:44.096.048 [log_inner.cpp:87]4152131 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set release_thread affinity to 98 success."
[DEBUG] APP(4150170,python):2026-06-08-14:10:44.498.909 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set other_thread affinity to 100-127 success."
[DEBUG] APP(4150170,python):2026-06-08-14:10:45.055.865 [log_inner.cpp:87]4152367 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4150170,python):2026-06-08-14:10:45.056.389 [log_inner.cpp:87]4152367 NPUAffinityController.cpp:StartMainThreadBind:571: "[PTA]:Device 3 set main_thread affinity to 96 success, thread name: pt_autograd_3."
[DEBUG] APP(4150170,python):2026-06-08-14:10:48.027.039 [log_inner.cpp:87]4150170 NPUAffinityController.cpp:ResetThreadAffinity:524: "[PTA]:Reset main_thread affinity to default range 96 success."

========== resnet50.py --work 1 --num_works 2 ==========
获取到rank=3
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[W608 14:11:01.563145330 NPUCachingAllocator.cpp:202] Warning: The current CANN and HDK(driver) versions require processing for 32 padding size, with memory allocation. (function operator())
work1: torch_npu.npu.current_device()=0, result: 6, CPU affinity:{0-10}
main thread: before backward CPU affinity:{0-255}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [1/2] | Loss: 2.5761 | Avg Step Time: 1.7454s | Epoch Time: 3.49s
/home/zhaoyu/miniconda3/envs/pta/lib/python3.10/site-packages/torch/utils/data/dataloader.py:432: UserWarning: This DataLoader will create 2 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  self.check_worker_number_rationality()
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [2/2] | Loss: 5.5542 | Avg Step Time: 0.4957s | Epoch Time: 0.99s
Training completed!
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.062.787 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:parseCPUAffinityConf:240: "[PTA]:Get device nums: 8 by aclrtGetDeviceCount."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.062.902 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:parseCPUAffinityConf:250: "[PTA]:Get env var CPU_AFFINITY_CONF: 2,force:1"
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.144 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:printCoreRanges:279: "[PTA]:Read CPU affinity config: Mode: 2. Core range for each device ID: Device 0: [0-31]; Device 1: [32-63]; Device 2: [64-95]; Device 3: [96-127]; Device 4: [128-159]; Device 5: [160-191]; Device 6: [192-223]; Device 7: [224-255]."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.150 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 0 get cores 0-31."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.154 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 1 get cores 32-63."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.158 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 2 get cores 64-95."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.161 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 3 get cores 96-127."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.165 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 4 get cores 128-159."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.168 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 5 get cores 160-191."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.171 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 6 get cores 192-223."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.174 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 7 get cores 224-255."
[INFO] APP(4154293,python):2026-06-08-14:11:00.063.178 [log_inner.cpp:84]4155667 NPUAffinityController.cpp:getThreadAffinityInfo:336: "[PTA]:CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding."
[DEBUG] APP(4154293,python):2026-06-08-14:11:00.063.315 [log_inner.cpp:87]4155667 NPUAffinityController.cpp:SetThreadAffinity:496: "[PTA]:Set thread affinity to user-defined range 0-10 success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:01.127.949 [log_inner.cpp:87]4155953 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to acl_thread success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:01.127.952 [log_inner.cpp:87]4155954 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to release_thread success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:01.127.984 [log_inner.cpp:87]4155953 NPUAffinityController.cpp:getCpuAffinityMap:384: "[PTA]:Device 3 thread affinity map: main_thread: [96]; acl_thread: [97]; release_thread: [98]; hccl_watchdog_t: [99]; other_thread: [100-127]."
[DEBUG] APP(4154293,python):2026-06-08-14:11:01.128.062 [log_inner.cpp:87]4155954 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set release_thread affinity to 98 success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:01.128.062 [log_inner.cpp:87]4155953 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set acl_thread affinity to 97 success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:02.041.417 [log_inner.cpp:87]4154293 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set other_thread affinity to 100-127 success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:02.510.865 [log_inner.cpp:87]4156495 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4154293,python):2026-06-08-14:11:02.511.449 [log_inner.cpp:87]4156495 NPUAffinityController.cpp:StartMainThreadBind:571: "[PTA]:Device 3 set main_thread affinity to 96 success, thread name: pt_autograd_3."
[DEBUG] APP(4154293,python):2026-06-08-14:11:05.649.033 [log_inner.cpp:87]4154293 NPUAffinityController.cpp:ResetThreadAffinity:524: "[PTA]:Reset main_thread affinity to default range 96 success."

========== resnet50.py --work 2 --num_works 2 ==========
获取到rank=3
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[Warning]: tiling struct [ReduceOpTilingDataV2] is conflict with one in file lp_norm_reduce.cc, line 41
[W608 14:11:19.725190150 NPUCachingAllocator.cpp:202] Warning: The current CANN and HDK(driver) versions require processing for 32 padding size, with memory allocation. (function operator())
work2: torch_npu.npu.current_device()=3, result: 6, CPU affinity:{96}
main thread: before backward CPU affinity:{0-255}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [1/2] | Loss: 2.5761 | Avg Step Time: 1.6943s | Epoch Time: 3.39s
/home/zhaoyu/miniconda3/envs/pta/lib/python3.10/site-packages/torch/utils/data/dataloader.py:432: UserWarning: This DataLoader will create 2 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  self.check_worker_number_rationality()
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
main thread: before backward CPU affinity:{96}
main thread: after backward CPU affinity={96}
Epoch [2/2] | Loss: 5.5542 | Avg Step Time: 0.4755s | Epoch Time: 0.95s
Training completed!
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.222 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:parseCPUAffinityConf:240: "[PTA]:Get device nums: 8 by aclrtGetDeviceCount."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.315 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:parseCPUAffinityConf:250: "[PTA]:Get env var CPU_AFFINITY_CONF: 2,force:1"
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.554 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:printCoreRanges:279: "[PTA]:Read CPU affinity config: Mode: 2. Core range for each device ID: Device 0: [0-31]; Device 1: [32-63]; Device 2: [64-95]; Device 3: [96-127]; Device 4: [128-159]; Device 5: [160-191]; Device 6: [192-223]; Device 7: [224-255]."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.560 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 0 get cores 0-31."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.563 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 1 get cores 32-63."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.567 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 2 get cores 64-95."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.570 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 3 get cores 96-127."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.573 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 4 get cores 128-159."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.576 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 5 get cores 160-191."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.579 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 6 get cores 192-223."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.039.582 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:324: "[PTA]:Device 7 get cores 224-255."
[INFO] APP(4159255,python):2026-06-08-14:11:19.039.586 [log_inner.cpp:84]4160714 NPUAffinityController.cpp:getThreadAffinityInfo:336: "[PTA]:CPU affinity force mode enabled, skipping affinity conflict detection, applying CPU_AFFINITY_CONF binding."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.042.407 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:getCpuAffinityMap:384: "[PTA]:Device 3 thread affinity map: main_thread: [96]; acl_thread: [97]; release_thread: [98]; hccl_watchdog_t: [99]; other_thread: [100-127]."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.042.475 [log_inner.cpp:87]4160714 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.289.972 [log_inner.cpp:87]4160961 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to release_thread success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.290.042 [log_inner.cpp:87]4160961 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set release_thread affinity to 98 success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.290.095 [log_inner.cpp:87]4160960 NPUAffinityController.cpp:SetThreadType:438: "[PTA]:Set thread name to acl_thread success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:19.290.124 [log_inner.cpp:87]4160960 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set acl_thread affinity to 97 success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:20.155.614 [log_inner.cpp:87]4159255 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set other_thread affinity to 100-127 success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:20.580.998 [log_inner.cpp:87]4161473 NPUAffinityController.cpp:SetThreadAffinity:452: "[PTA]:Device 3 set main_thread affinity to 96 success."
[DEBUG] APP(4159255,python):2026-06-08-14:11:20.581.533 [log_inner.cpp:87]4161473 NPUAffinityController.cpp:StartMainThreadBind:571: "[PTA]:Device 3 set main_thread affinity to 96 success, thread name: pt_autograd_3."
[DEBUG] APP(4159255,python):2026-06-08-14:11:23.637.440 [log_inner.cpp:87]4159255 NPUAffinityController.cpp:ResetThreadAffinity:524: "[PTA]:Reset main_thread affinity to default range 96 success."