已关闭

[Bug-Report|缺陷反馈]: adaptive_avg_pool3d_backward fp16和bf16有确定性计算问题 #1950

胡一航创建于 4月7日关闭于 4月7日

胡一航成员

4月7日创建

Thanks for sending an issue! Please fill in the following template to help quickly solve your problem.

Describe the current behavior / 问题描述 (Mandatory / 必填)

adaptive_avg_pool3d_backward fp16和bf16有确定性计算问题

Environment / 环境信息 (Mandatory / 必填)

910B

Steps to reproduce the issue / 重现步骤 (Mandatory / 必填)

from pathlib import Path

import os
import random

import numpy as np
import torch
import torch_npu

torch.npu.set_device(0)

SCRIPT_DIR = Path(file).resolve().parent
INPUT_PATH = SCRIPT_DIR / "input.bin"
GRAD_OUTPUT_PATH = SCRIPT_DIR / "output_grad_0.pt"

os.environ["ASCEND_GLOBAL_LOG_LEVEL"] = "0"

os.environ["ASCEND_SLOG_PRINT_TO_STDOUT"] = "1"

os.environ["TORCH_NPU_LOGS"] = "op_plugin"

def seed_all(seed=42):
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.use_deterministic_algorithms(True)
torch_npu.npu.manual_seed_all(seed)
torch_npu.npu.manual_seed(seed)

def compare_tensors(a, b, name="output"):
a_cpu = a.detach().cpu()
b_cpu = b.detach().cpu()

diff = (a_cpu.float() - b_cpu.float()).abs()
diff_mask = a_cpu != b_cpu
diff_idx = diff_mask.nonzero(as_tuple=False)

print(name)
print("shape:", tuple(a_cpu.shape))
print("dtype:", a_cpu.dtype)
print("equal:", torch.equal(a_cpu, b_cpu))
print("max abs diff:", diff.max().item())
print("mean abs diff:", diff.mean().item())
print("num diff:", diff_idx.shape[0])

if diff_idx.shape[0] > 0:
    first_pos = tuple(diff_idx[0].tolist())
    print("first diff index:", first_pos)
    print("first diff values:", a_cpu[first_pos].item(), b_cpu[first_pos].item())
    print(
        "first diff abs:",
        abs(a_cpu[first_pos].float().item() - b_cpu[first_pos].float().item()),
    )

def main():
seed_all()

# cmp1 = torch.load(SCRIPT_DIR / "tensor_1.pt", map_location="cpu")
# cmp2 = torch.load(SCRIPT_DIR / "tensor_2.pt", map_location="cpu")
# compare_tensors(cmp1, cmp2, name="fix_deter")

input_data = torch.load(INPUT_PATH, map_location="cpu")
x_base = input_data[0].clone()
grad_output_base = torch.load(GRAD_OUTPUT_PATH, map_location="cpu").clone()

x0 = x_base.clone().to("npu")
x1 = x_base.clone().to("npu")
grad_output0 = grad_output_base.clone().to("npu")
grad_output1 = grad_output_base.clone().to("npu")

print("op schema: aten::_adaptive_avg_pool2d_backward(Tensor grad_output, Tensor self) -> Tensor")
print("x shape:", tuple(x0.shape), x0.dtype)
print("grad_output shape:", tuple(grad_output0.shape), grad_output0.dtype)

torch.npu.synchronize()
seed_all()
out0 = torch.ops.aten._adaptive_avg_pool2d_backward.default(grad_output0, x0)
# torch.save(out0, 'tensor_2.pt')
torch.npu.synchronize()

seed_all()
out1 = torch.ops.aten._adaptive_avg_pool2d_backward.default(grad_output1, x1)
torch.npu.synchronize()

compare_tensors(out0, out1, name="aten._adaptive_avg_pool2d_backward output")

if name == "main":
main()