import math
import unittest
import itertools
import warnings
from itertools import product
import torch
import torch.autograd.forward_ad as fwAD
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.nn.functional as F
import torch_npu
import torch_npu.testing
from torch.testing._internal.common_cuda import TEST_CUDA, TEST_CUDNN, tf32_on_and_off
from torch.testing._internal.common_device_type import (
disablecuDNN,
disableMkldnn,
dtypes,
dtypesIfCUDA,
dtypesIfPRIVATEUSE1,
dtypesIfMPS,
expectedFailureMPS,
instantiate_device_type_tests,
largeTensorTest,
onlyCPU,
onlyCUDA,
onlyPRIVATEUSE1,
onlyNativeDeviceTypes,
precisionOverride,
skipCPUIfNoMkldnn,
skipCUDAIfCudnnVersionLessThan,
skipCUDAIfMiopen,
skipCUDAIfNoCudnn,
skipCUDAIfNoMiopen,
skipCUDAIfNotMiopenSuggestNHWC,
skipCUDAIfRocm,
skipCUDAIfRocmVersionLessThan,
skipMeta,
skipMPS,
)
from torch.testing._internal.common_dtype import (
floating_and_complex_types_and,
floating_types_and,
)
from torch.testing._internal.common_nn import _test_module_empty_input, NNTestCase
from torch.testing._internal.common_utils import (
download_file,
dtype2prec_DONTUSE,
gradcheck,
GRADCHECK_NONDET_TOL,
gradgradcheck,
instantiate_parametrized_tests,
MACOS_VERSION,
parametrize as parametrize_test,
run_tests,
set_default_dtype,
skipIfNotMiopenSuggestNHWC,
skipIfRocmVersionLessThan,
subtest,
TEST_SCIPY,
TEST_WITH_ROCM,
TEST_PRIVATEUSE1
)
from url import get_url
from torch_npu.testing.common_utils import SupportedDevices
AMPERE_OR_ROCM = TEST_WITH_ROCM
DEVICE_NAME = torch_npu.npu.get_device_name(0)
device_is_910A = False
if "Ascend910A" in DEVICE_NAME or "Ascend910P" in DEVICE_NAME:
device_is_910A = True
if TEST_SCIPY:
import scipy.signal
import scipy.ndimage
class TestConvolutionNN(NNTestCase):
_do_cuda_memory_leak_check = True
_do_cuda_non_default_stream = True
def test_conv_backcompat(self):
from torch.serialization import SourceChangeWarning
path = download_file(get_url('legacy_conv2d'))
with warnings.catch_warnings():
warnings.simplefilter('ignore', SourceChangeWarning)
m = torch.load(path, encoding='utf-8', weights_only=False)
input1 = torch.randn((1, 1, 1, 1), dtype=torch.float)
self.assertEqual(m(input1).size(), (1, 1, 1, 1))
def test_invalid_conv1d(self):
for dtype in [torch.half, torch.bfloat16, torch.float, torch.double, torch.cfloat, torch.cdouble]:
module = nn.Conv1d(in_channels=3, out_channels=33, kernel_size=10, stride=1, bias=True).to(dtype)
input1 = torch.randn(1, 3, 4).to(dtype)
with self.assertRaisesRegex(RuntimeError,
r'Calculated padded input size per channel: \(4\). ' +
r'Kernel size: \(10\). Kernel size can\'t be greater than actual input size'):
module(input1)
module = nn.Conv1d(in_channels=3, out_channels=6, kernel_size=3, stride=-1, bias=True).to(dtype)
input1 = torch.randn(1, 3, 4).to(dtype)
with self.assertRaisesRegex(RuntimeError, 'non-positive stride is not supported'):
module(input1)
def test_mismatch_shape_conv2d(self):
for dtype in (torch.float, torch.cfloat):
x = torch.randn(1, 10, 1, 28, 28, dtype=dtype)
w = torch.randn(6, 1, 5, 5, dtype=dtype)
with self.assertRaisesRegex(RuntimeError,
r'Expected 3D \(unbatched\) or 4D \(batched\) input to conv2d, but got ' +
r'input of size: \[1, 10, 1, 28, 28\]'):
F.conv2d(x, w)
def test_conv2d_discontiguous_weight(self):
for dtype in (torch.float, torch.cfloat):
x = torch.ones(64, 16, 16, 16, dtype=dtype)
weight = torch.arange(0, 1.0, 1 / 2.0 ** 10).reshape(32, 16, 1, 2).to(dtype)[:, :, :, ::2]
self.assertFalse(weight.is_contiguous())
y = torch.nn.functional.conv2d(x, weight, None)
if torch.backends.mkldnn.is_available():
with torch.backends.mkldnn.flags(enabled=False):
y_ = torch.nn.functional.conv2d(x, weight, None)
self.assertEqual(y, y_)
self.assertEqual(y.sum(), 4186112.)
def test_invalid_conv2d(self):
for dtype in [torch.half, torch.bfloat16, torch.float, torch.double, torch.cfloat, torch.cdouble]:
module = torch.nn.Conv2d(1, 1, kernel_size=3, dilation=2, stride=2).to(dtype)
input1 = torch.empty(1, 1, 4, 4).to(dtype)
self.assertRaises(RuntimeError, lambda: module(input1))
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, stride=1, bias=True)
input1 = torch.randn(1, 3, 1, 1)
with self.assertRaisesRegex(RuntimeError,
r'Calculated padded input size per channel: \(1 x 1\). ' +
r'Kernel size: \(10 x 10\). Kernel size can\'t be greater than actual input size'):
module(input1)
module = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=4, stride=-1, bias=True).to(dtype)
input1 = torch.randn(1, 3, 4, 4).to(dtype)
with self.assertRaisesRegex(RuntimeError, 'non-positive stride is not supported'):
module(input1)
module = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=4, stride=0, bias=True).to(dtype)
input1 = torch.randn(1, 3, 4, 4).to(dtype)
with self.assertRaisesRegex(RuntimeError, 'non-positive stride is not supported'):
module(input1)
def test_invalid_conv3d(self):
for dtype in [torch.half, torch.bfloat16, torch.float, torch.double, torch.cfloat, torch.cdouble]:
module = torch.nn.Conv3d(1, 1, kernel_size=3, dilation=2, stride=2).to(dtype)
input1 = torch.empty(1, 1, 4, 4, 4).to(dtype)
self.assertRaises(RuntimeError, lambda: module(input1))
module = torch.nn.Conv3d(1, 1, kernel_size=3, stride=-2)
input1 = torch.empty(1, 1, 4, 4, 4)
with self.assertRaisesRegex(RuntimeError, 'non-positive stride is not supported'):
module(input1)
def test_conv_invalid_groups(self):
with self.assertRaisesRegex(ValueError, 'groups must be a positive integer'):
torch.nn.Conv1d(1, 1, kernel_size=3, dilation=2, stride=2, groups=0)
with self.assertRaisesRegex(ValueError, 'groups must be a positive integer'):
torch.nn.Conv2d(1, 1, kernel_size=3, dilation=2, stride=2, groups=-1)
with self.assertRaisesRegex(ValueError, 'groups must be a positive integer'):
torch.nn.Conv3d(1, 1, kernel_size=3, dilation=2, stride=2, groups=-2)
def test_Conv1d_module_same_padding(self):
x = torch.rand(1, 1, 20)
module = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=10,
padding='same')
expect = F.conv1d(x, module.weight, module.bias, padding='same')
self.assertEqual(expect, module(x))
module = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=10,
padding='same', dilation=2)
expect = F.conv1d(x, module.weight, module.bias, padding='same', dilation=2)
self.assertEqual(expect, module(x))
module = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=10,
padding='same', padding_mode='replicate')
x_padded = F.pad(x, [4, 5], mode='replicate')
expect = F.conv1d(x_padded, module.weight, module.bias, padding='valid')
self.assertEqual(expect, module(x))
self.assertEqual(x.size(), expect.size())
with self.assertRaisesRegex(ValueError, 'Invalid padding string'):
module = nn.Conv1d(in_channels=3, out_channels=33, kernel_size=10, padding='foo')
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv1d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=2)
def test_Conv2d_module_same_padding(self):
x = torch.rand(1, 1, 9, 20)
module = nn.Conv2d(in_channels=1, out_channels=1, kernel_size=(5, 10),
padding='same')
expect = F.conv2d(x, module.weight, module.bias, padding='same')
self.assertEqual(expect, module(x))
module = nn.Conv2d(in_channels=1, out_channels=1, kernel_size=(3, 4),
padding='same', dilation=(1, 2))
expect = F.conv2d(x, module.weight, module.bias, padding='same', dilation=(1, 2))
self.assertEqual(expect, module(x))
module = nn.Conv2d(in_channels=1, out_channels=1, kernel_size=(3, 4),
padding='same', padding_mode='reflect')
x_padded = F.pad(x, [1, 2, 1, 1], mode='reflect')
expect = F.conv2d(x_padded, module.weight, module.bias, padding='valid')
self.assertEqual(expect, module(x))
self.assertEqual(x.size(), expect.size())
with self.assertRaisesRegex(ValueError, 'Invalid padding string'):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='foo')
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=2)
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=(1, 3))
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=(4, 1))
def test_Conv3d_module_same_padding(self):
x = torch.rand(1, 1, 4, 4, 4)
module = nn.Conv3d(in_channels=1, out_channels=1, kernel_size=(2, 3, 4),
padding='same')
expect = F.conv3d(x, module.weight, module.bias, padding='same')
self.assertEqual(expect, module(x))
module = nn.Conv3d(in_channels=1, out_channels=1, kernel_size=(2, 3, 4),
padding='same', dilation=(3, 2, 1))
expect = F.conv3d(x, module.weight, module.bias, padding='same', dilation=(3, 2, 1))
self.assertEqual(expect, module(x))
module = nn.Conv3d(in_channels=1, out_channels=1, kernel_size=(2, 3, 4),
padding='same', padding_mode='circular')
x_padded = F.pad(x, [1, 2, 1, 1, 0, 1], mode='circular')
expect = F.conv3d(x_padded, module.weight, module.bias, padding='valid')
self.assertEqual(expect, module(x))
self.assertEqual(x.size(), expect.size())
with self.assertRaisesRegex(ValueError, 'Invalid padding string'):
module = nn.Conv3d(in_channels=3, out_channels=33, kernel_size=10, padding='foo')
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=2)
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=(1, 1, 3))
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=(1, 4, 1))
with self.assertRaisesRegex(ValueError, "padding='same'"):
module = nn.Conv2d(in_channels=3, out_channels=33, kernel_size=10, padding='same', stride=(5, 1, 1))
@unittest.skipIf(not TEST_PRIVATEUSE1, 'NPU not available')
def test_thnn_conv_strided_padded_dilated(self):
for convfn, dims, transposed in (
(torch.nn.functional.conv2d, 2, False),
(torch.nn.functional.conv_transpose2d, 2, True),
(torch.nn.functional.conv3d, 3, False),
(torch.nn.functional.conv_transpose3d, 3, True)):
for stride, padding, dilation in (
(2, 0, 1), (1, 1, 1), (2, 1, 1), (1, 0, 2)):
kwargs = {"stride": stride, "padding": padding, "dilation": dilation}
inp_shape = (1, 2) + dims * (4,)
weight_shape = (2, 2) + dims * (1,)
inputs = torch.randn(inp_shape, dtype=torch.double, device="npu", requires_grad=True)
weight = torch.randn(weight_shape, dtype=torch.double, device="npu", requires_grad=True)
bias = torch.randn(2, dtype=torch.double, device="npu", requires_grad=True)
with torch.backends.cudnn.flags(enabled=False):
res = convfn(inputs, weight, bias, **kwargs)
res_cpu = convfn(inputs.cpu(), weight.cpu(), bias.cpu(), **kwargs)
self.assertEqual(res, res_cpu)
with torch.backends.cudnn.flags(enabled=False):
torch.autograd.gradcheck(
lambda x, w, b: convfn(x, w, b, **kwargs),
(inputs, weight, bias)
)
torch.autograd.gradcheck(
lambda x, w, b: convfn(x, w, b, **kwargs),
(inputs.cpu(), weight.cpu(), bias.cpu())
)
def test_Conv2d_inconsistent_types(self):
inputs = torch.randn(4, 1, 7, 7, dtype=torch.float)
weights = torch.randn(1, 1, 3, 3, dtype=torch.double)
self.assertRaises(RuntimeError, lambda: nn.functional.conv2d(inputs, weights))
nn.functional.conv2d(inputs.float(), weights.float())
@unittest.skipIf(not TEST_PRIVATEUSE1, 'NPU not available')
def test_Conv2d_inconsistent_types_on_GPU_without_cudnn(self):
inputs = torch.randn(4, 1, 7, 7, dtype=torch.float, device="npu")
weights = torch.randn(1, 1, 3, 3, dtype=torch.double, device="npu")
bias = torch.randn(1, dtype=torch.double, device="npu")
with torch.backends.cudnn.flags(enabled=False):
self.assertRaises(RuntimeError, lambda: nn.functional.conv2d(inputs, weights))
self.assertRaises(RuntimeError, lambda: nn.functional.conv2d(inputs, weights.float(), bias))
nn.functional.conv2d(inputs.float(), weights.float(), bias.float())
def test_Conv2d_1x1(self):
in_channels = 2
mod = torch.nn.Conv2d(2, 2, 1, bias=False).to(dtype=torch.double)
input1 = torch.randn(1, in_channels, 5, 5, requires_grad=True, dtype=torch.double)
for enabled in (False, True):
with torch.backends.mkldnn.flags(enabled=enabled):
gradcheck(F.conv2d, (input1, mod.weight))
def test_Conv2d_OneDNN(self):
def run_once(group_val=24, dilation=1):
ifm = torch.ones([1, group_val, 6, 6], dtype=torch.float32)
weights = torch.ones([group_val, 1, 3, 3], dtype=torch.float32)
op = torch.nn.Conv2d(
in_channels=group_val,
out_channels=group_val,
kernel_size=[3, 3],
stride=[2, 2],
padding=[1, 1],
dilation=[dilation, dilation],
groups=group_val,
bias=False,
padding_mode='zeros'
)
op.weight.data = weights
res = op(ifm)
grad_in = torch.ones(res.shape, dtype=torch.float32)
res.backward(grad_in)
return op.weight.grad
for gorup_val in (24, 48, 23, 25):
for dilation in (1, 2):
with torch.backends.mkldnn.flags(enabled=False):
without_onednn = run_once(gorup_val, dilation)
with torch.backends.mkldnn.flags(enabled=True):
with_onednn = run_once(gorup_val, dilation)
self.assertEqual(without_onednn, with_onednn)
@unittest.skipIf(not TEST_PRIVATEUSE1, 'PrivateUse1 not available')
def test_cudnn_non_contiguous(self):
x = torch.randn(192, 16, 50).npu()
x = x.permute(0, 2, 1).contiguous().permute(0, 2, 1)
m = torch.nn.Conv1d(
in_channels=16,
out_channels=32,
kernel_size=2,
bias=True).npu()
m(x)
@unittest.skipIf(not TEST_PRIVATEUSE1, 'PrivateUse1 not available')
def test_Conv2d_inconsistent_types_on_GPU_with_cudnn(self):
inputs = torch.randn(4, 1, 7, 7, dtype=torch.float, device="npu")
weights = torch.randn(1, 1, 3, 3, dtype=torch.double, device="npu")
bias = torch.randn(1, dtype=torch.double, device="npu")
with torch.backends.cudnn.flags(enabled=True):
self.assertRaises(RuntimeError, lambda: nn.functional.conv2d(inputs, weights))
self.assertRaises(RuntimeError, lambda: nn.functional.conv2d(inputs, weights.float(), bias))
nn.functional.conv2d(inputs.float(), weights.float(), bias.float())
def test_Conv2d_missing_argument(self):
c = nn.Conv2d(3, 3, 3)
self.assertRaises(TypeError, lambda: c(None))
def test_Conv2d_backward_twice(self):
input1 = torch.randn(2, 3, 5, 5)
c = nn.Conv2d(3, 3, 3)
o1 = c(input1)
o1.sum().backward()
self.assertRaisesRegex(RuntimeError, 'Specify retain_graph=True',
lambda: o1.sum().backward())
def test_conv_modules_raise_error_on_incorrect_input_size(self):
for dtype in [torch.half, torch.bfloat16, torch.double, torch.float]:
modules = [nn.Conv1d(3, 8, 3).to(dtype), nn.ConvTranspose1d(3, 8, 3).to(dtype),
nn.Conv2d(3, 8, 3).to(dtype), nn.ConvTranspose2d(3, 8, 3).to(dtype),
nn.Conv3d(3, 8, 3).to(dtype), nn.ConvTranspose3d(3, 8, 3).to(dtype)]
invalid_input_dims = [(1, 4), (1, 4),
(2, 5), (2, 5),
(3, 6), (3, 6)]
for invalid_dims, module in zip(invalid_input_dims, modules):
for dims in invalid_dims:
input1 = torch.empty(torch.Size((3, ) * dims))
self.assertRaises(RuntimeError, lambda: module(input1))
def test_conv_shapecheck(self):
def test(should_raise, module, input_size, dtype):
input1 = torch.empty(3, *input_size).to(dtype)
if should_raise:
self.assertRaises(RuntimeError, lambda: module(input1))
else:
module(input1)
for dtype in [torch.half, torch.bfloat16, torch.float, torch.double, torch.cfloat, torch.cdouble]:
test(True, nn.Conv1d(1, 1, 3).to(dtype), (1, 2), dtype)
test(True, nn.Conv1d(1, 1, 3, stride=2).to(dtype), (1, 2), dtype)
test(False, nn.Conv1d(1, 1, 2).to(dtype), (1, 2), dtype)
test(False, nn.Conv1d(1, 1, 2, stride=2).to(dtype), (1, 2), dtype)
test(False, nn.Conv1d(1, 1, 3, stride=2, padding=1).to(dtype), (1, 2), dtype)
test(True, nn.Conv2d(1, 1, (3, 3)).to(dtype), (1, 2, 2), dtype)
test(False, nn.Conv2d(1, 1, (3, 3)).to(dtype), (1, 3, 3), dtype)
test(False, nn.Conv2d(1, 1, (3, 3), padding=1).to(dtype), (1, 2, 2), dtype)
test(True, nn.Conv3d(1, 1, (3, 3, 3)).to(dtype), (1, 2, 2, 2), dtype)
test(False, nn.Conv3d(1, 1, (3, 3, 3)).to(dtype), (1, 3, 3, 3), dtype)
test(False, nn.Conv3d(1, 1, (3, 3, 3), padding=1).to(dtype), (1, 2, 2, 2), dtype)
def test_ConvTranspose2d_output_size(self):
m = nn.ConvTranspose2d(3, 4, 3, 3, 0, 2)
i = torch.randn(2, 3, 6, 6)
for h in range(15, 22):
for w in range(15, 22):
if 18 <= h <= 20 and 18 <= w <= 20:
output = m(i, output_size=(h, w))
self.assertEqual(output.size()[2:], (h, w))
else:
self.assertRaises(ValueError, lambda: m(i, (h, w)))
def test_ConvTranspose2d_output_size_downsample_upsample(self):
b, c, hid_c = 2, 3, 2
for h in range(13, 24):
for w in range(13, 17):
for k in range(2, 5):
for d in range(1, 5):
for s in range(1, 4):
for p in range(3):
conv = nn.Conv2d(
in_channels=c,
out_channels=hid_c,
kernel_size=k,
stride=s,
padding=p,
dilation=d,
)
t_conv = nn.ConvTranspose2d(
in_channels=hid_c,
out_channels=c,
kernel_size=k,
stride=s,
padding=p,
dilation=d,
)
i = torch.randn(b, c, h, w)
out = t_conv(conv(i), output_size=i.shape)
self.assertEqual(out.size()[2:], i.size()[2:])
def test_ConvTranspose3d_correct_output_size(self):
m = nn.ConvTranspose3d(2, 2, 2)
i = torch.rand(1, 2, 1, 1, 1)
m(i, output_size=(1, 2, 2, 2, 2))
@unittest.skipIf(not TEST_PRIVATEUSE1, 'PrivateUse1 not available')
def test_ConvTranspose2d_half_cublas_gemm(self):
with torch.backends.cudnn.flags(enabled=False):
inputs = torch.randn(1, 1, 16, 16, device='npu', dtype=torch.half)
deconv = (
nn.ConvTranspose2d(1, 1, 3, stride=2, padding=1, output_padding=1)
.npu()
.half()
)
output = deconv(inputs)
output.mean().backward()
@torch.backends.cudnn.flags(enabled=True, deterministic=True, benchmark=False)
@tf32_on_and_off(0.001)
@unittest.skipIf(TEST_WITH_ROCM, "Skipped on ROCm, since it is failing on ROCm 5.7")
def test_Conv2d_groups_nobias(self):
dev_dtypes = [("cpu", torch.float)]
if TEST_CUDA:
dev_dtypes += [("npu", torch.float), ("npu", torch.half)]
if AMPERE_OR_ROCM:
dev_dtypes += [("npu", torch.bfloat16)]
for device, dtype in dev_dtypes:
m = nn.Conv2d(4, 4, kernel_size=3, groups=2, bias=False).to(device, dtype)
i = torch.randn(2, 4, 6, 6, device=device, dtype=dtype, requires_grad=True)
output = m(i)
grad_output = torch.randn(2, 4, 4, 4, device=device, dtype=dtype)
output.backward(grad_output)
m1 = nn.Conv2d(2, 2, kernel_size=3, bias=False).to(device, dtype)
m1.weight.data.copy_(m.weight.data[:2])
i1 = i.data[:, :2].contiguous().requires_grad_(True)
output1 = m1(i1)
output1.backward(grad_output[:, :2].contiguous())
m2 = nn.Conv2d(2, 2, kernel_size=3, bias=False).to(device, dtype)
m2.weight.data.copy_(m.weight.data[2:])
i2 = i.data[:, 2:].contiguous().requires_grad_(True)
output2 = m2(i2)
output2.backward(grad_output[:, 2:].contiguous())
self.assertEqual(output, torch.cat([output1, output2], 1))
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data, m2.weight.grad.data], 0),
atol=1e-1 if dtype == torch.half else dtype2prec_DONTUSE[dtype], rtol=0)
@torch.backends.cudnn.flags(enabled=True, deterministic=True, benchmark=False)
@tf32_on_and_off(0.001)
@unittest.skipIf(TEST_WITH_ROCM, "Skipped on ROCm, since it is failing on ROCm 5.7")
def test_Conv2d_groups_nobias_v2(self):
torch.manual_seed(123)
dev_dtypes = [("cpu", torch.float)]
if TEST_PRIVATEUSE1:
dev_dtypes += [("npu", torch.float), ("npu", torch.half)]
if AMPERE_OR_ROCM:
dev_dtypes += [("npu", torch.bfloat16)]
for device, dtype in dev_dtypes:
m = nn.Conv2d(4, 16, kernel_size=3, groups=2, bias=False).to(device, dtype)
i = torch.randn(2, 4, 6, 6, device=device, dtype=dtype, requires_grad=True)
output = m(i)
grad_output = torch.randn(2, 16, 4, 4, device=device, dtype=dtype)
output.backward(grad_output)
m1 = nn.Conv2d(2, 8, kernel_size=3, bias=False).to(device, dtype)
m1.weight.data.copy_(m.weight.data[:8])
i1 = i.data[:, :2].contiguous().requires_grad_(True)
output1 = m1(i1)
output1.backward(grad_output[:, :8].contiguous())
m2 = nn.Conv2d(2, 8, kernel_size=3, bias=False).to(device, dtype)
m2.weight.data.copy_(m.weight.data[8:])
i2 = i.data[:, 2:].contiguous().requires_grad_(True)
output2 = m2(i2)
output2.backward(grad_output[:, 8:].contiguous())
self.assertEqual(output, torch.cat([output1, output2], 1))
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data, m2.weight.grad.data], 0),
atol=1e-1 if dtype == torch.half else dtype2prec_DONTUSE[dtype], rtol=0)
def test_Conv3d_groups_nobias(self):
torch.manual_seed(123)
m = nn.Conv3d(4, 16, kernel_size=3, groups=2, bias=False).to("cpu", torch.float)
i = torch.randn(2, 4, 6, 6, 6, device="cpu", dtype=torch.float, requires_grad=True)
output = m(i)
grad_output = torch.randn(2, 16, 4, 4, 4, device="cpu", dtype=torch.float)
output.backward(grad_output)
m1 = nn.Conv3d(2, 8, kernel_size=3, bias=False).to("cpu", torch.float)
m1.weight.data.copy_(m.weight.data[:8])
i1 = i.data[:, :2].contiguous().requires_grad_(True)
output1 = m1(i1)
output1.backward(grad_output[:, :8].contiguous())
m2 = nn.Conv3d(2, 8, kernel_size=3, bias=False).to("cpu", torch.float)
m2.weight.data.copy_(m.weight.data[8:])
i2 = i.data[:, 2:].contiguous().requires_grad_(True)
output2 = m2(i2)
output2.backward(grad_output[:, 8:].contiguous())
self.assertEqual(output, torch.cat([output1, output2], 1))
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[torch.float], rtol=0)
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data, m2.weight.grad.data], 0),
atol=dtype2prec_DONTUSE[torch.float], rtol=dtype2prec_DONTUSE[torch.float])
def test_Conv3d_groups_wbias(self):
torch.manual_seed(123)
m = nn.Conv3d(4, 16, kernel_size=3, groups=2, bias=True).to("cpu", torch.float)
i = torch.randn(2, 4, 6, 6, 6, device="cpu", dtype=torch.float, requires_grad=True)
output = m(i)
grad_output = torch.randn(2, 16, 4, 4, 4, device="cpu", dtype=torch.float)
output.backward(grad_output)
m1 = nn.Conv3d(2, 8, kernel_size=3, bias=True).to("cpu", torch.float)
m1.weight.data.copy_(m.weight.data[:8])
m1.bias.data.copy_(m.bias.data[:8])
i1 = i.data[:, :2].contiguous().requires_grad_(True)
output1 = m1(i1)
output1.backward(grad_output[:, :8].contiguous())
m2 = nn.Conv3d(2, 8, kernel_size=3, bias=True).to("cpu", torch.float)
m2.weight.data.copy_(m.weight.data[8:])
m2.bias.data.copy_(m.bias.data[8:])
i2 = i.data[:, 2:].contiguous().requires_grad_(True)
output2 = m2(i2)
output2.backward(grad_output[:, 8:].contiguous())
self.assertEqual(output, torch.cat([output1, output2], 1))
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[torch.float],
rtol=dtype2prec_DONTUSE[torch.float])
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data, m2.weight.grad.data], 0),
atol=dtype2prec_DONTUSE[torch.float],
rtol=dtype2prec_DONTUSE[torch.float])
self.assertEqual(m.bias.grad.data,
torch.cat([m1.bias.grad.data, m2.bias.grad.data], 0),
atol=dtype2prec_DONTUSE[torch.float], rtol=dtype2prec_DONTUSE[torch.float])
def test_conv_tbc(self):
with set_default_dtype(torch.double):
inp = torch.randn(9, 4, 5, requires_grad=True)
weight = torch.randn(3, 5, 6, requires_grad=True)
bias = torch.randn(6, requires_grad=True)
gradcheck(lambda i, w, b, pad: F.conv_tbc(i, w, b, pad), (inp, weight, bias, 3))
@unittest.skipIf(not TEST_PRIVATEUSE1, "PrivateUse1 unavailable")
@unittest.skipIf(not TEST_CUDNN, "needs cudnn")
@skipIfRocmVersionLessThan((4, 3))
@skipIfNotMiopenSuggestNHWC
def test_grouped_conv_cudnn_nhwc_support(self):
input1 = torch.randn((16, 16, 8, 8), dtype=torch.float16, device="npu").to(memory_format=torch.channels_last)
weight = torch.randn((8, 4, 3, 3), dtype=torch.float16, device="npu").to(memory_format=torch.channels_last)
torch.convolution(input1, weight, None, (1, 1), (1, 1), (1, 1), False, (0, 0), 4)
input1 = torch.randn((16, 8, 8, 8), dtype=torch.float16, device="npu").to(memory_format=torch.channels_last)
torch.convolution(input1, weight, None, (1, 1), (1, 1), (1, 1), True, (0, 0), 4)
@unittest.expectedFailure
@unittest.skipIf(not TEST_PRIVATEUSE1, "PrivateUse1 unavailable")
@unittest.skipIf(not TEST_CUDNN, "needs cudnn")
def test_conv_cudnn_memory_layout_dominance(self):
input1 = torch.randint(1, 10, (2, 8, 4, 4), dtype=torch.float32, device="npu", requires_grad=True)
conv = nn.Conv2d(8, 4, 3).npu().float()
out = conv(input1)
self.assertTrue(out.is_contiguous())
input1 = input1.contiguous(memory_format=torch.channels_last)
out = conv(input1)
self.assertTrue(out.is_contiguous())
conv.weight.data = conv.weight.contiguous(memory_format=torch.channels_last)
out = conv(input1)
self.assertTrue(out.is_contiguous(memory_format=torch.channels_last))
input1 = input1.contiguous()
out = conv(input1)
self.assertTrue(out.is_contiguous(memory_format=torch.channels_last))
@unittest.skipIf(not TEST_PRIVATEUSE1, "PrivateUse1 unavailable")
def test_cudnn_noncontiguous_weight(self):
input1 = torch.tensor([1, 1, 1], dtype=torch.double, device="npu").view(1, 1, 3)
weights1 = torch.tensor([1], dtype=torch.double, device="npu").expand(1, 1, 2)
weights2 = torch.tensor([1], dtype=torch.double, device="npu").expand(1, 1, 2).contiguous()
self.assertEqual(F.conv1d(input1, weights1, bias=None, stride=2, dilation=2),
F.conv1d(input1, weights2, bias=None, stride=2, dilation=2))
def run_grad_conv_test(self, func_forward, func_backward, dim=1, gradient='input'):
for kern, inp_size in [(3, 6), (3, 7), (4, 9)]:
for batch, stride, padding, chan_in, chan_out, dilation in \
product([1, 2], [1, 2], [0, 1, 2], [2], [3], [1]):
for has_bias in [True, False]:
input_shape = [batch, chan_in]
weight_shape = [chan_out, chan_in]
for _ in range(dim):
input_shape.append(inp_size)
weight_shape.append(kern)
input1 = torch.randn(input_shape, requires_grad=True)
weight = torch.randn(weight_shape, requires_grad=True)
if has_bias:
bias = torch.randn([chan_out], requires_grad=True)
output = func_forward(input1, weight, stride=stride, padding=padding, dilation=dilation, bias=bias)
gradient_o = torch.randn(output.shape)
gradient_w = torch.autograd.grad(output, input1 if (gradient == 'input') else weight, gradient_o)
self.assertEqual(gradient_w[0],
func_backward(
input_shape if (gradient == 'input') else input1,
weight_shape if (gradient == 'weight') else weight,
gradient_o,
stride=stride,
padding=padding,
dilation=dilation))
def test_grad_conv1d_input(self):
self.run_grad_conv_test(F.conv1d, F.grad.conv1d_input, 1, 'input')
def test_grad_conv1d_weight(self):
self.run_grad_conv_test(F.conv1d, F.grad.conv1d_weight, 1, 'weight')
def test_grad_conv2d_input(self):
self.run_grad_conv_test(F.conv2d, F.grad.conv2d_input, 2, 'input')
def test_grad_conv2d_weight(self):
self.run_grad_conv_test(F.conv2d, F.grad.conv2d_weight, 2, 'weight')
def test_grad_conv3d_input(self):
self.run_grad_conv_test(F.conv3d, F.grad.conv3d_input, 3, 'input')
def test_grad_conv3d_weight(self):
self.run_grad_conv_test(F.conv3d, F.grad.conv3d_weight, 3, 'weight')
@unittest.skipIf(not torch._nnpack_available(), "NNPACK unavailable")
def test_nnpack_conv(self):
for kern, inp_size in [(3, 6), (3, 7), (4, 9)]:
for batch, stride, padding, chan_in, chan_out in \
product([1, 2, 3, 4], [1, 2], [0, 1, 2], [2], [3]):
for has_bias in [True, False]:
input_shape = [batch, chan_in]
weight_shape = [chan_out, chan_in]
for _ in range(2):
input_shape.append(inp_size)
weight_shape.append(kern)
input1 = torch.randn(input_shape, requires_grad=True, dtype=torch.float)
weight = torch.randn(weight_shape, requires_grad=True, dtype=torch.float)
if has_bias:
bias = torch.randn([chan_out], requires_grad=True, dtype=torch.float)
output = torch._nnpack_spatial_convolution(
input1, weight, stride=stride, padding=padding, bias=bias)
output_expected = torch.nn.functional.conv2d(
input1, weight, stride=stride, padding=padding, bias=bias)
self.assertEqual(output, output_expected, atol=3e-4, rtol=0)
gradient_o = torch.randn(output.shape, dtype=torch.float)
grads = torch.autograd.grad(output, [input1, weight], gradient_o)
grads_expected = torch.autograd.grad(output_expected, [input1, weight], gradient_o)
for gr, gr_expected in zip(grads, grads_expected):
self.assertEqual(gr, gr_expected, atol=3e-4, rtol=0)
def test_conv_padding_mode(self):
with self.assertRaisesRegex(ValueError, "padding_mode must be one of"):
nn.Conv2d(3, 3, 3, padding_mode="xyz")
with self.assertRaisesRegex(ValueError, "padding_mode must be one of"):
nn.Conv2d(3, 3, 3, padding_mode=3)
with self.assertRaisesRegex(ValueError, "Only \"zeros\" "):
nn.ConvTranspose2d(3, 3, 3, padding_mode="reflect")
def test_functional_grad_conv(self):
input1 = torch.randn(1, 1, 5, requires_grad=True)
weight = torch.randn(1, 1, 3, requires_grad=True)
output = F.conv1d(input1, weight, dilation=2)
grad_output = torch.randn(output.shape)
grad_input_autograd, grad_weight_autograd = torch.autograd.grad(output, (input1, weight), grad_output)
grad_input_functional = torch.nn.grad.conv1d_input(input1.shape, weight, grad_output, dilation=2)
self.assertEqual(grad_input_functional, grad_input_autograd)
grad_weight_functional = torch.nn.grad.conv1d_weight(input1, weight.shape, grad_output, dilation=2)
self.assertEqual(grad_weight_functional, grad_weight_autograd)
input1 = torch.randn(1, 1, 5, 5, requires_grad=True)
weight = torch.randn(1, 1, 3, 3, requires_grad=True)
output = F.conv2d(input1, weight, dilation=2)
grad_output = torch.randn(output.shape)
(grad_input_autograd, grad_weight_autograd) = torch.autograd.grad(output, (input1, weight), grad_output)
grad_input_functional = torch.nn.grad.conv2d_input(input1.shape, weight, grad_output, dilation=2)
self.assertEqual(grad_input_functional, grad_input_autograd)
grad_weight_functional = torch.nn.grad.conv2d_weight(input1, weight.shape, grad_output, dilation=2)
self.assertEqual(grad_weight_functional, grad_weight_autograd)
input1 = torch.randn(1, 1, 5, 5, 5, requires_grad=True)
weight = torch.randn(1, 1, 3, 3, 3, requires_grad=True)
output = F.conv3d(input1, weight, dilation=2)
grad_output = torch.randn(output.shape)
(grad_input_autograd, grad_weight_autograd) = torch.autograd.grad(output, (input1, weight), grad_output)
grad_input_functional = torch.nn.grad.conv3d_input(input1.shape, weight, grad_output, dilation=2)
self.assertEqual(grad_input_functional, grad_input_autograd)
grad_weight_functional = torch.nn.grad.conv3d_weight(input1, weight.shape, grad_output, dilation=2)
self.assertEqual(grad_weight_functional, grad_weight_autograd)
def test_functional_grad_conv2d(self):
BATCH_SIZE = 4
IN_CH = 8
OUT_CH = 16
SPATIAL = 32
def _test_conv2d(stride, kernel_size, groups, dilation):
padding = kernel_size // 2
input1 = (
torch.empty(BATCH_SIZE, IN_CH, SPATIAL, SPATIAL)
.uniform_(-8.0, 8.0)
.requires_grad_(True)
)
weight = (
torch.empty(OUT_CH, IN_CH // groups, kernel_size, kernel_size)
.uniform_(-4.0, 4.0)
.requires_grad_(True)
)
output = F.conv2d(input1, weight,
stride=stride, padding=padding, dilation=dilation, groups=groups)
grad_output = torch.randn(output.shape)
(grad_input_autograd, grad_weight_autograd) = torch.autograd.grad(output, (input1, weight), grad_output)
grad_input_functional = torch.nn.grad.conv2d_input(input1.shape, weight, grad_output,
stride=stride, padding=padding, dilation=dilation, groups=groups)
self.assertEqual(grad_input_functional, grad_input_autograd)
grad_weight_functional = torch.nn.grad.conv2d_weight(input1, weight.shape, grad_output,
stride=stride, padding=padding, dilation=dilation, groups=groups)
self.assertEqual(grad_weight_functional, grad_weight_autograd)
strides = [1, 2]
kernel_sizes = [1, 3, 5]
groups = [1, 2, 4]
dilates = [1, 2]
for s, k, g, d in product(strides, kernel_sizes, groups, dilates):
_test_conv2d(s, k, g, d)
def test_permute_conv2d_issue_120211(self):
def reproducer(radius: int):
image = torch.rand(1, 1024, 1024, 3)
image = image.permute(0, 3, 1, 2)
kernel_x = torch.zeros([3, 1, 1, radius * 2 + 1], device=image.device)
image = torch.nn.functional.conv2d(image, kernel_x, groups=image.shape[-3])
for i in range(0, 128):
reproducer(radius=i)
def test_conv3d_issue_120406(self):
F.conv3d(torch.ones(2, 3, 8, 9, 26), torch.ones(3, 1, 1, 1, 17), groups=3)
def test_conv1d_issue_120547(self):
weight = torch.ones([16, 1, 32])
bias = torch.ones([16])
stride, padding, dilation, groups = (1, 16, 1, 16)
input1 = torch.rand((1, 1, 16))
input1 = input1.transpose(1, 2)
F.conv1d(input1, weight, bias, stride, padding, dilation, groups)
class TestConvolutionNNDeviceType(NNTestCase):
def run_conv_double_back_test(self, kern, stride, padding, chan_in, chan_out, batch_size,
inp_size, dilation, no_weight, groups=1, use_npu=False,
use_bias=True, dtype=torch.double):
if use_npu:
device = torch.device("npu")
else:
device = torch.device("cpu")
x = torch.randn(batch_size, chan_in, inp_size, inp_size, device=device,
dtype=dtype, requires_grad=True)
weight = torch.randn(chan_out, chan_in // groups, kern, kern, device=device,
dtype=dtype, requires_grad=not no_weight)
if use_bias:
bias = torch.randn(chan_out, device=device, dtype=dtype, requires_grad=True)
else:
bias = None
def func(*inputs):
if use_bias:
lx, lweight, lbias = inputs
else:
lx, lweight = inputs
lbias = None
with cudnn.flags(enabled=False):
out = F.conv2d(lx, lweight, lbias, stride, padding, dilation, groups)
return out
if use_bias:
inputs = x, weight, bias
else:
inputs = x, weight
dummy_out = func(*inputs)
grad_y = torch.randn_like(dummy_out, device=device, dtype=dtype, requires_grad=True)
if dtype == torch.float:
g, = torch.autograd.grad(dummy_out.sum(), x, create_graph=True)
return g.requires_grad
return gradgradcheck(func, inputs, (grad_y,))
@onlyPRIVATEUSE1
@skipCUDAIfNoCudnn
@dtypes(*floating_and_complex_types_and(torch.half, *[torch.bfloat16] if AMPERE_OR_ROCM else []))
def test_Conv2d_deterministic_cudnn(self, device, dtype):
inputs = torch.randn(2, 3, 5, 5, device=device, dtype=dtype, requires_grad=True)
with cudnn.flags(enabled=True, benchmark=True, deterministic=True):
conv1 = torch.nn.Conv2d(3, 3, 3).to(device, dtype)
conv2 = torch.nn.Conv2d(3, 3, 3).to(device, dtype)
conv2.bias.data.copy_(conv1.bias.data)
conv2.weight.data.copy_(conv1.weight.data)
out1 = conv1(inputs)
out2 = conv2(inputs)
self.assertEqual(out1, out2, atol=0.0, rtol=0)
y = torch.randn(out1.size(), device=device, dtype=dtype)
out1.backward(y)
out2.backward(y)
self.assertEqual(conv1.bias.grad.data, conv2.bias.grad.data, atol=0.0, rtol=0)
self.assertEqual(conv1.weight.grad.data, conv2.weight.grad.data, atol=0.0, rtol=0)
@onlyPRIVATEUSE1
@dtypes(*floating_types_and(torch.half, *[torch.bfloat16] if AMPERE_OR_ROCM else []))
def test_Conv2d_large_workspace(self, device, dtype):
sizes = [
(1, 256, 109, 175),
(1, 256, 80, 128),
(1, 256, 120, 192),
]
def run_test(benchmark):
with torch.backends.cudnn.flags(enabled=True, benchmark=benchmark):
conv = torch.nn.Conv2d(256, 256, kernel_size=3, padding=1).to(device, dtype)
for size in sizes:
x = torch.randn(size, device=device, dtype=dtype)
out = conv(x.detach().clone().requires_grad_())
out.backward(torch.ones_like(out))
run_test(benchmark=False)
run_test(benchmark=True)
@onlyPRIVATEUSE1
@dtypes(torch.half, torch.float)
def test_ConvTranspose2d_large_output_padding(self, device, dtype):
net1 = torch.nn.ConvTranspose2d(128, 64, kernel_size=3, stride=2, padding=1, output_padding=1)\
.to(device=device, dtype=dtype)
net2 = torch.nn.ConvTranspose2d(64, 32, kernel_size=3, stride=2, padding=1, output_padding=1)\
.to(device=device, dtype=dtype)
net3 = torch.nn.ConvTranspose2d(32, 3, kernel_size=3, stride=2, padding=1, output_padding=1)\
.to(device=device, dtype=dtype)
x = torch.rand(1, 128, 6, 6, device=device, dtype=dtype, requires_grad=True)
x = net1(x)
x = net2(x)
x = net3(x)
x.backward(torch.randn_like(x))
if device == 'npu':
torch_npu.npu.synchronize()
@onlyPRIVATEUSE1
@dtypes(torch.float, torch.double, torch.half)
@torch.backends.cudnn.flags(enabled=True, benchmark=False)
@tf32_on_and_off(0.01)
def test_Conv2d_depthwise_naive_groups(self, device, dtype):
for depth_multiplier in [1, 2]:
m = nn.Conv2d(2, 2 * depth_multiplier, kernel_size=3, groups=2).to(device, dtype)
i = torch.randn(2, 2, 6, 6, device=device, dtype=dtype).div_(2).requires_grad_()
output = m(i)
grad_output = torch.randn(2, 2 * depth_multiplier, 4, 4, device=device, dtype=dtype) / 2
output.backward(grad_output)
offset = 1 * depth_multiplier
m1 = nn.Conv2d(1, 1 * depth_multiplier, kernel_size=3).to(device, dtype)
m1.weight.data = m.weight.data[:offset].clone()
m1.bias.data = m.bias.data[:offset].clone()
i1 = i.detach()[:, :1].clone().requires_grad_()
output1 = m1(i1)
output1.backward(grad_output[:, :offset].contiguous())
m2 = nn.Conv2d(1, 1 * depth_multiplier, kernel_size=3).to(device, dtype)
m2.weight.data.copy_(m.weight.data[offset:])
m2.bias.data.copy_(m.bias.data[offset:])
i2 = i.detach()[:, 1:].clone().requires_grad_()
output2 = m2(i2)
output2.backward(grad_output[:, offset:].contiguous())
self.assertEqual(output, torch.cat([output1, output2], 1),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.bias.grad.data,
torch.cat([m1.bias.grad.data,
m2.bias.grad.data], 0),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data,
m2.weight.grad.data], 0),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
@onlyPRIVATEUSE1
@dtypes(torch.float, torch.double, torch.half)
@torch.backends.cudnn.flags(enabled=True, deterministic=True, benchmark=False)
@tf32_on_and_off(0.01)
def test_Conv3d_depthwise_naive_groups(self, device, dtype):
for depth_multiplier in [1, 2]:
m = nn.Conv3d(2, 2 * depth_multiplier, kernel_size=3, groups=2).to(device, dtype)
i = torch.randn(2, 2, 6, 6, 6, device=device, dtype=dtype).div_(2).requires_grad_()
output = m(i)
grad_output = torch.randn(2, 2 * depth_multiplier, 4, 4, 4, device=device, dtype=dtype) / 2
output.backward(grad_output)
offset = 1 * depth_multiplier
m1 = nn.Conv3d(1, 1 * depth_multiplier, kernel_size=3).to(device, dtype)
m1.weight.data = m.weight.data[:offset].clone()
m1.bias.data = m.bias.data[:offset].clone()
i1 = i.detach()[:, :1].clone().requires_grad_()
output1 = m1(i1)
output1.backward(grad_output[:, :offset].contiguous())
m2 = nn.Conv3d(1, 1 * depth_multiplier, kernel_size=3).to(device, dtype)
m2.weight.data.copy_(m.weight.data[offset:])
m2.bias.data.copy_(m.bias.data[offset:])
i2 = i.detach()[:, 1:].clone().requires_grad_()
output2 = m2(i2)
output2.backward(grad_output[:, offset:].contiguous())
is_cuda_sm86 = device.startswith("npu") and torch_npu.npu.get_device_capability(0) == (8, 6)
atol, rtol = (3e-4, 3e-2) if dtype == torch.float32 and is_cuda_sm86 else (dtype2prec_DONTUSE[dtype], 0)
self.assertEqual(output, torch.cat([output1, output2], 1),
atol=atol, rtol=rtol)
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.bias.grad.data,
torch.cat([m1.bias.grad.data,
m2.bias.grad.data], 0),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data,
m2.weight.grad.data], 0),
atol=atol, rtol=rtol)
@onlyPRIVATEUSE1
@dtypes(*floating_types_and(torch.half, *[torch.bfloat16] if AMPERE_OR_ROCM else []))
def test_noncontig_conv_grad(self, device, dtype):
module = nn.Conv2d(3, 5, kernel_size=3, padding=1).to(device, dtype)
input1 = torch.randn(2, 3, 10, 10, dtype=dtype, device=device, requires_grad=True)
output = module(input1)
grad = torch.randn(2, 2, 5, 10, 10, dtype=dtype, device=device)[:, 1]
assert not grad.is_contiguous()
output.backward(grad, retain_graph=True)
self.assertIsNotNone(input1.grad)
result = input1.grad.data.clone()
input1.grad.data.zero_()
output.backward(grad.contiguous())
self.assertEqual(result, input1.grad.data, atol=dtype2prec_DONTUSE[dtype], rtol=0)
@onlyPRIVATEUSE1
@dtypes(torch.double)
def test_conv_double_backward(self, device, dtype):
with torch.backends.cudnn.flags(enabled=True, deterministic=True):
batch_size = 1
for kern, inp_size, dilations in [(3, 5, [1, 2]), (4, 9, [1])]:
for stride, padding, chan_in, chan_out, dilation in product([1], [2], [2], [3], dilations):
no_weight = stride == 2
result = self.run_conv_double_back_test(kern, stride,
padding, chan_in, chan_out,
batch_size, inp_size, dilation,
no_weight, use_npu=True, dtype=dtype)
self.assertTrue(result,
"Conv double backward test failed with parameters:" +
"\nkern: " + str(kern) +
"\nstride: " + str(stride) +
"\npadding: " + str(padding) +
"\nchan_in: " + str(chan_in) +
"\nchan_out: " + str(chan_out) +
"\nbatch_size: " + str(batch_size) +
"\ninp_size: " + str(inp_size) +
"\ndilation: " + str(dilation))
def test_conv_double_backward_no_bias(self):
kern = 3
stride = 2
chan_in, chan_out = 2, 4
batch_size = 2
inp_size = 5
padding = 1
dilation = 1
no_weight = False
use_bias = True
result = self.run_conv_double_back_test(kern, stride,
padding, chan_in, chan_out,
batch_size, inp_size, dilation,
no_weight, use_bias=use_bias)
self.assertTrue(result,
"Conv double backward test failed with parameters:" +
"\nkern: " + str(kern) +
"\nstride: " + str(stride) +
"\npadding: " + str(padding) +
"\nchan_in: " + str(chan_in) +
"\nchan_out: " + str(chan_out) +
"\nbatch_size: " + str(batch_size) +
"\ninp_size: " + str(inp_size) +
"\ndilation: " + str(dilation))
def test_conv_double_backward_groups(self):
kern = 3
stride = 1
padding = 2
chan_in, chan_out = 2, 4
batch_size = 2
inp_size = 6
dilation = 1
no_weight = False
groups = 2
result = self.run_conv_double_back_test(kern, stride,
padding, chan_in * groups, chan_out * groups,
batch_size, inp_size, dilation,
no_weight, groups=groups)
self.assertTrue(result,
"Conv double backward test failed with parameters:" +
"\nkern: " + str(kern) +
"\nstride: " + str(stride) +
"\npadding: " + str(padding) +
"\nchan_in: " + str(chan_in) +
"\nchan_out: " + str(chan_out) +
"\nbatch_size: " + str(batch_size) +
"\ninp_size: " + str(inp_size) +
"\ndilation: " + str(dilation) +
"\ngroups: " + str(groups))
def test_conv_double_backward_stride(self):
batch_size = 2
for kern, inp_size, dilations in [(3, 5, [1, 2]), (3, 7, [1])]:
for stride, padding, chan_in, chan_out, dilation in product([2], [0, 1], [1], [2], dilations):
no_weight = False
self.run_conv_double_back_test(kern, stride,
padding, chan_in, chan_out,
batch_size, inp_size, dilation,
no_weight)
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
@torch.backends.cudnn.flags(enabled=True, deterministic=True, benchmark=False)
def test_conv1d_same_padding(self, device, dtype):
test_args = [
range(50, 55),
[1, 2, 3, 8],
range(1, 4),
[1],
]
for in_size, k_size, dilation, stride in itertools.product(*test_args):
x = torch.rand(1, 1, in_size, device=device, dtype=dtype)
y = torch.rand(1, 1, k_size, device=device, dtype=dtype)
z = F.conv1d(x, y, padding='same', dilation=dilation, stride=stride)
self.assertEqual(z.size(2), int(math.ceil(in_size / stride)))
x = torch.rand(1, 1, 12, device=device, dtype=dtype)
y = torch.rand(1, 1, 3, device=device, dtype=dtype)
expect = F.conv1d(x, y, padding=1)
actual = F.conv1d(x, y, padding='same')
self.assertEqual(expect, actual)
x = torch.rand(1, 1, 12, device=device, dtype=dtype)
y = torch.rand(1, 1, 4, device=device, dtype=dtype)
expect = F.conv1d(x, y, padding=3, dilation=2)
actual = F.conv1d(x, y, padding='same', dilation=2)
self.assertEqual(expect, actual)
expect = F.conv1d(x, y, padding=5, dilation=3)[..., 1:]
actual = F.conv1d(x, y, padding='same', dilation=3)
self.assertEqual(expect, actual)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
@dtypes(torch.float, torch.cfloat)
def test_conv2d_same_padding(self, device, dtype):
if dtype is torch.cfloat:
rtol, atol = 2e-6, 2e-6
else:
rtol, atol = None, None
x = torch.rand(1, 1, 10, 11, device=device, dtype=dtype)
y = torch.rand(1, 1, 4, 5, device=device, dtype=dtype)
expect = F.conv2d(x, y, padding=(2, 2))[..., 1:, :]
actual = F.conv2d(x, y, padding='same')
self.assertEqual(expect, actual, rtol=rtol, atol=atol)
y = torch.rand(1, 1, 3, 4, device=device, dtype=dtype)
expect = F.conv2d(x, y, padding=(2, 3), dilation=2)
actual = F.conv2d(x, y, padding='same', dilation=2)
self.assertEqual(expect, actual, rtol=rtol, atol=atol)
y = torch.rand(1, 1, 4, 4, device=device, dtype=dtype)
expect = F.conv2d(x, y, padding=5, dilation=3)[..., 1:, 1:]
actual = F.conv2d(x, y, padding='same', dilation=3)
self.assertEqual(expect, actual, rtol=rtol, atol=atol)
@dtypes(torch.float, torch.cfloat)
def test_conv3d_same_padding(self, device, dtype):
if dtype is torch.cfloat:
rtol, atol = 2e-6, 2e-6
else:
rtol, atol = None, None
x = torch.rand(1, 1, 10, 11, 12, device=device, dtype=dtype)
y = torch.rand(1, 1, 1, 2, 5, device=device, dtype=dtype)
expect = F.conv3d(x, y, padding=(0, 1, 2))[..., :, 1:, :]
actual = F.conv3d(x, y, padding='same')
self.assertEqual(expect, actual, rtol=rtol, atol=atol)
expect = F.conv3d(x, y, padding=(0, 1, 4), dilation=2)
actual = F.conv3d(x, y, padding='same', dilation=2)
self.assertEqual(expect, actual, rtol=rtol, atol=atol)
y = torch.rand(1, 1, 4, 4, 4, device=device, dtype=dtype)
expect = F.conv3d(x, y, padding=5, dilation=3)[..., 1:, 1:, 1:]
actual = F.conv3d(x, y, padding='same', dilation=3)
self.assertEqual(expect, actual, rtol=rtol, atol=atol)
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
def test_conv1d_valid_padding(self, device, dtype):
x = torch.rand(1, 1, 10, device=device, dtype=dtype)
y = torch.rand(1, 1, 4, device=device, dtype=dtype)
expect = F.conv1d(x, y)
actual = F.conv1d(x, y, padding='valid')
self.assertEqual(expect, actual)
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
def test_conv2d_valid_padding(self, device, dtype):
x = torch.rand(1, 1, 1, 10, device=device, dtype=dtype)
y = torch.rand(1, 1, 1, 4, device=device, dtype=dtype)
expect = F.conv2d(x, y)
actual = F.conv2d(x, y, padding='valid')
self.assertEqual(expect, actual)
@dtypes(torch.float, torch.cfloat)
def test_conv3d_valid_padding(self, device, dtype):
x = torch.rand(1, 1, 1, 1, 10, dtype=dtype, device=device)
y = torch.rand(1, 1, 1, 1, 4, dtype=dtype, device=device)
expect = F.conv3d(x, y)
actual = F.conv3d(x, y, padding='valid')
self.assertEqual(expect, actual)
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(torch.float)
def test_conv1d_same_padding_backward(self, device, dtype):
x = torch.rand(1, 1, 12, dtype=dtype, device=device, requires_grad=True)
y = torch.rand(1, 1, 4, dtype=dtype, device=device, requires_grad=True)
z = F.conv1d(x, y, padding=3, dilation=2)
z.sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
z = F.conv1d(x, y, padding='same', dilation=2)
z.sum().abs().backward()
self.assertEqual(gx_expect, x.grad)
self.assertEqual(gy_expect, y.grad)
x.grad, y.grad = None, None
z = F.conv1d(x, y, padding=2)[..., 1:]
z.sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
z = F.conv1d(x, y, padding='same')
z.sum().abs().backward()
self.assertEqual(gx_expect, x.grad)
self.assertEqual(gy_expect, y.grad)
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
@tf32_on_and_off(0.001)
def test_conv2d_same_padding_backward(self, device, dtype):
x = torch.rand(1, 1, 10, 11, device=device, dtype=dtype, requires_grad=True)
y = torch.rand(1, 1, 4, 5, device=device, dtype=dtype, requires_grad=True)
z = F.conv2d(x, y, padding=(3, 4), dilation=2)
z.sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
z = F.conv2d(x, y, padding='same', dilation=2)
z.sum().abs().backward()
self.assertEqual(gx_expect, x.grad)
self.assertEqual(gy_expect, y.grad)
x.grad, y.grad = None, None
y = torch.rand(1, 1, 4, 4, device=device, dtype=dtype, requires_grad=True)
z = F.conv2d(x, y, padding=2)[..., 1:, 1:]
z.sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
z = F.conv2d(x, y, padding='same')
z.sum().abs().backward()
self.assertEqual(gx_expect, x.grad)
self.assertEqual(gy_expect, y.grad)
@dtypes(torch.double, torch.cdouble)
@dtypesIfMPS(
torch.float, torch.cfloat
)
@expectedFailureMPS
def test_conv3d_same_padding_backward(self, device, dtype):
check_forward_ad = torch.device(device).type != 'xla'
x = torch.rand(1, 1, 1, 11, 12, dtype=dtype, device=device, requires_grad=True)
y = torch.rand(1, 1, 1, 2, 5, dtype=dtype, device=device, requires_grad=True)
z = F.conv3d(x, y, padding=(0, 1, 4), dilation=2)
z.sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
z = F.conv3d(x, y, padding='same', dilation=2)
z.sum().abs().backward()
self.assertEqual(gx_expect, x.grad)
self.assertEqual(gy_expect, y.grad)
x.grad, y.grad = None, None
gradcheck(lambda x, y: F.conv3d(x, y, padding='same', dilation=2), (x, y),
check_forward_ad=check_forward_ad, nondet_tol=1e-5)
if torch.device(device).type != 'npu':
gradgradcheck(lambda x, y: F.conv3d(x, y, padding='same', dilation=2), (x, y),
check_fwd_over_rev=True)
y = torch.rand(1, 1, 1, 4, 4, dtype=dtype, device=device, requires_grad=True)
z = F.conv3d(x, y, padding=2)[..., 1:, 1:]
z.sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
z = F.conv3d(x, y, padding='same')
z.sum().abs().backward()
self.assertEqual(gx_expect, x.grad)
self.assertEqual(gy_expect, y.grad)
gradcheck(lambda x, y: F.conv3d(x, y, padding='same'), (x, y),
check_forward_ad=check_forward_ad, nondet_tol=1e-5)
if torch.device(device).type != 'npu':
gradgradcheck(lambda x, y: F.conv3d(x, y, padding='same'), (x, y),
check_fwd_over_rev=True)
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
def test_conv1d_valid_padding_backward(self, device, dtype):
x = torch.rand(1, 1, 10, dtype=dtype, device=device, requires_grad=True)
y = torch.rand(1, 1, 4, dtype=dtype, device=device, requires_grad=True)
F.conv1d(x, y, padding=0).sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
F.conv1d(x, y, padding='valid').sum().abs().backward()
gx_actual, gy_actual = x.grad, y.grad
self.assertEqual(gx_expect, gx_actual)
self.assertEqual(gy_expect, gy_actual)
@unittest.skipIf(not TEST_SCIPY, "Scipy required for the test.")
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
@parametrize_test("mode", ('valid', 'same'))
def test_conv1d_vs_scipy(self, device, dtype, mode):
t = make_tensor((1, 10), device=device, dtype=dtype)
feat_dim = t.shape[1]
weight_even = make_tensor((1, 1, 4), device=device, dtype=dtype)
weight_odd = make_tensor((1, 1, 5), device=device, dtype=dtype)
def _test(t, weight, mode):
t_a = t.view(-1).cpu().numpy()
w_a = weight.view(-1).cpu().numpy()
expected = scipy.signal.convolve(t_a, w_a, mode=mode)
kwargs = {'padding': mode}
if mode == 'same':
p = weight.shape[2] // 2
t = torch.nn.functional.pad(t, (p, p))
kwargs.pop("padding")
weight_flipped = torch.flip(weight, (2,))
actual = torch.nn.functional.conv1d(t, weight_flipped, **kwargs).squeeze(0)
if mode == 'same':
actual = actual[:feat_dim]
self.assertEqual(actual, expected, atol=2e-5, rtol=2e-5)
with set_default_dtype(torch.float):
_test(t, weight_even, mode)
_test(t, weight_odd, mode)
@unittest.skipIf(not TEST_SCIPY, "Scipy required for the test.")
@dtypes(torch.float, torch.cfloat)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
@parametrize_test("mode", ('valid', 'same'))
def test_conv2d_vs_scipy(self, device, dtype, mode):
t = make_tensor((1, 5, 10), device=device, dtype=dtype)
weight_even = make_tensor((1, 1, 2, 4), device=device, dtype=dtype)
weight_odd = make_tensor((1, 1, 3, 5), device=device, dtype=dtype)
def _test(t, weight, mode):
t_a = t.squeeze(0).cpu().numpy()
w_a = weight.squeeze(0).squeeze(0).cpu().numpy()
expected = scipy.signal.convolve2d(t_a, w_a, mode=mode)
kwargs = {'padding': mode}
if mode == 'same':
left_right_pad = weight.shape[3] // 2
top_bottom_pad = weight.shape[2] // 2
p = (left_right_pad, left_right_pad, top_bottom_pad, top_bottom_pad)
t = torch.nn.functional.pad(t, p)
kwargs.pop("padding")
weight_flipped = torch.flip(weight, (2, 3))
actual = torch.nn.functional.conv2d(t, weight_flipped, **kwargs).squeeze(0)
if mode == 'same':
actual = actual[:5, :10]
self.assertEqual(actual, expected, rtol=2e-5, atol=5e-6)
with set_default_dtype(torch.float):
_test(t, weight_even, mode)
_test(t, weight_odd, mode)
@unittest.skipIf(not TEST_SCIPY, "Scipy required for the test.")
@skipMPS
@dtypes(torch.float, torch.cfloat)
@parametrize_test("mode", ('valid', 'same'))
def test_conv3d_vs_scipy(self, device, dtype, mode):
t = make_tensor((1, 5, 5, 10), device=device, dtype=dtype)
weight_even = make_tensor((1, 1, 2, 2, 4), device=device, dtype=dtype)
weight_odd = make_tensor((1, 1, 2, 3, 5), device=device, dtype=dtype)
def _test(t, weight, mode):
t_a = t.squeeze(0).cpu().numpy()
w_a = weight.squeeze(0).squeeze(0).cpu().numpy()
expected = scipy.signal.convolve(t_a, w_a, mode=mode)
kwargs = {'padding': mode}
if mode == 'same':
left_right_pad = weight.shape[4] // 2
top_bottom_pad = weight.shape[3] // 2
front_back_pad = weight.shape[2] // 2
p = (left_right_pad, left_right_pad, top_bottom_pad, top_bottom_pad,
front_back_pad, front_back_pad)
t = torch.nn.functional.pad(t, p)
kwargs.pop("padding")
weight_flipped = torch.flip(weight, (2, 3, 4))
actual = torch.nn.functional.conv3d(t, weight_flipped, **kwargs).squeeze(0)
if mode == 'same':
actual = actual[:5, :5, :10]
if torch.cuda.is_tf32_supported() and (dtype == torch.float or dtype == torch.complex64):
self.assertEqual(actual, expected, atol=0.05, rtol=0.05)
else:
self.assertEqual(actual, expected, rtol=2e-5, atol=5e-6)
with set_default_dtype(torch.float):
_test(t, weight_even, mode)
_test(t, weight_odd, mode)
@dtypes(torch.float, torch.complex64)
@dtypesIfMPS(
*([torch.float] if MACOS_VERSION < 14.0 else [torch.float, torch.cfloat])
)
def test_conv2d_valid_padding_backward(self, device, dtype):
x = torch.rand(1, 1, 1, 10, device=device, dtype=dtype, requires_grad=True)
y = torch.rand(1, 1, 1, 4, device=device, dtype=dtype, requires_grad=True)
F.conv2d(x, y, padding=0).sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
F.conv2d(x, y, padding='valid').sum().abs().backward()
gx_actual, gy_actual = x.grad, y.grad
self.assertEqual(gx_expect, gx_actual)
self.assertEqual(gy_expect, gy_actual)
@dtypes(torch.double, torch.cdouble)
@dtypesIfMPS(
torch.float, torch.cfloat
)
@expectedFailureMPS
def test_conv3d_valid_padding_backward(self, device, dtype):
check_forward_ad = torch.device(device).type != 'xla'
x = torch.rand(1, 1, 1, 1, 10, dtype=dtype, device=device, requires_grad=True)
y = torch.rand(1, 1, 1, 1, 4, dtype=dtype, device=device, requires_grad=True)
F.conv3d(x, y, padding=0).sum().abs().backward()
gx_expect, gy_expect = x.grad, y.grad
x.grad, y.grad = None, None
F.conv3d(x, y, padding='valid').sum().abs().backward()
gx_actual, gy_actual = x.grad, y.grad
self.assertEqual(gx_expect, gx_actual)
self.assertEqual(gy_expect, gy_actual)
gradcheck(lambda x, y: F.conv3d(x, y, padding='valid'), (x, y), check_forward_ad=check_forward_ad)
gradgradcheck(lambda x, y: F.conv3d(x, y, padding='valid'), (x, y), check_fwd_over_rev=check_forward_ad)
@parametrize_test(
arg_str="N",
arg_values=[
subtest(arg_values=(2), name="ConvTranspose2d"),
subtest(
arg_values=(3), name="ConvTranspose3d", decorators=[expectedFailureMPS]
),
],
)
def test_conv_transpose_with_output_size_and_no_batch_dim(self, device, N):
inp = torch.randn((1, 15, 13) if N == 2 else (1, 15, 13, 13), device=device)
output_size = (1, 240, 200) if N == 2 else (1, 240, 200, 200)
ConvTransposeNd = getattr(nn, f'ConvTranspose{N}d')
m = ConvTransposeNd(1, 1, kernel_size=16, stride=16, padding=7, bias=False, device=device)
output = m(inp, output_size=output_size)
self.assertEqual(output.shape, output_size)
@onlyCPU
def test_conv_contiguous_for_oneDNN(self):
for dtype in [torch.float, torch.bfloat16, torch.half]:
conv = nn.Conv2d(
1,
128,
kernel_size=(5, 2),
stride=(2, 1),
padding=(0, 1),
dilation=(1, 1),
groups=1,
bias=True,
padding_mode='zeros').to(dtype=dtype)
x = torch.rand([1, 2, 321, 201, 1]).to(dtype=dtype)
x = torch.transpose(x, 1, 4)
x2 = x[..., 0]
if torch.backends.mkldnn.is_available():
y = conv(x2)
with torch.backends.mkldnn.flags(enabled=False):
y_ = conv(x2)
self.assertEqual(y, y_)
@onlyCPU
def test_conv_ic1_channels_last_for_oneDNN(self):
for dtype in [torch.float, torch.bfloat16, torch.half]:
conv = torch.nn.Conv2d(1, 64, kernel_size=(3, 3), padding=(1, 1), bias=False)
conv = conv.to(memory_format=torch.channels_last).to(dtype=dtype)
x = torch.rand(2, 1, 100, 100).to(dtype=dtype)
if torch.backends.mkldnn.is_available():
y = conv(x)
with torch.backends.mkldnn.flags(enabled=False):
y_ = conv(x)
self.assertEqual(y, y_)
@dtypes(torch.float, torch.cfloat)
def test_conv_empty_channel(self, device, dtype):
in_channels = 0
mod = torch.nn.Conv1d(in_channels, 8, 2, stride=2, dtype=dtype).to(device)
inp = torch.randn(2, 0, 15, device=device, dtype=dtype)
_test_module_empty_input(self, mod, inp, check_size=False)
with self.assertRaisesRegex(RuntimeError, "Given groups=1, weight"):
inp = torch.randn(2, 1, 0, device=device, dtype=dtype)
mod(inp)
mod = torch.nn.Conv2d(in_channels, 33, 3, stride=2, dtype=dtype).to(device)
inp = torch.randn(2, 0, 50, 100, device=device, dtype=dtype)
_test_module_empty_input(self, mod, inp, check_size=False)
with self.assertRaisesRegex(RuntimeError, "Given groups=1, weight"):
inp = torch.randn(2, 1, 40, 0, device=device, dtype=dtype)
mod(inp)
mod = torch.nn.Conv3d(in_channels, 33, 3, stride=2, dtype=dtype).to(device)
inp = torch.randn(2, 0, 50, 20, 40, device=device, dtype=dtype)
_test_module_empty_input(self, mod, inp, check_size=False)
with self.assertRaisesRegex(RuntimeError, "Given groups=1, weight"):
inp = torch.randn(2, 1, 50, 0, 40, device=device, dtype=dtype)
mod(inp)
def test_group_conv_empty(self, device):
mod = torch.nn.Conv2d(4, 4, stride=2, kernel_size=3, padding=1, groups=4).to(device)
inp = torch.randn(0, 4, 4, 4, device=device)
_test_module_empty_input(self, mod, inp, check_size=False)
if self.device_type == 'npu' and self.has_cudnn():
with torch.backends.cudnn.flags(enabled=False):
_test_module_empty_input(self, mod, inp, check_size=False)
def test_group_convTranspose_empty(self, device):
mod = torch.nn.ConvTranspose2d(4, 4, stride=2, kernel_size=3, padding=1, groups=4).to(device)
inp = torch.randn(0, 4, 4, 4, device=device)
_test_module_empty_input(self, mod, inp, check_size=False)
if self.device_type == 'npu' and self.has_cudnn():
with torch.backends.cudnn.flags(enabled=False):
_test_module_empty_input(self, mod, inp, check_size=False)
def test_convTranspose_empty(self, device):
mod = torch.nn.ConvTranspose2d(4, 4, stride=2, kernel_size=3, padding=1).to(device)
inp = torch.randn(0, 4, 4, 4, device=device)
_test_module_empty_input(self, mod, inp, check_size=False)
if self.device_type == 'npu' and self.has_cudnn():
with torch.backends.cudnn.flags(enabled=False):
_test_module_empty_input(self, mod, inp, check_size=False)
@onlyPRIVATEUSE1
@largeTensorTest('12GB')
def test_conv_large_nosplit(self, device):
dtype = torch.half if self.device_type == 'npu' else torch.float
conv1 = nn.Conv2d(2, 2, 8, 8).to(device).to(dtype)
input_large = torch.randn(1, 2, 1024, 1024 * 1024, dtype=dtype, device=device)
conv1(input_large)
conv2 = torch.nn.Conv2d(1, 1024, 1, 1).to(device).to(dtype)
input_large = torch.randn(1, 1, 2048, 1024, dtype=dtype, device=device)
conv2(input_large)
@expectedFailureMPS
def test_conv_noncontig_weights(self, device):
for dim in (1, 2, 3):
for grouped in (False, True):
nc = 3
groups = 3 if grouped else 1
w = torch.randn([3] * dim, device=device)
w = w.expand([nc, int(nc / groups)] + list(w.shape))
w = w.detach().requires_grad_()
x = torch.randn([1, nc] + ([5] * dim), device=device, requires_grad=True)
y = getattr(F, f'conv{dim}d')(x, w, groups=groups)
y.sum().backward()
y = getattr(F, f'conv_transpose{dim}d')(x, w, groups=groups)
y.sum().backward()
def test_conv_noncontig_weights_and_bias(self, device):
for bias in [True, False]:
conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=bias).to(device, torch.float)
input_nc = torch.randn((1, 3, 224, 224, 2), device=device, dtype=torch.float)[:, :, :, :, 1]
input_c = input_nc.contiguous()
weight_nc = torch.randn((64, 3, 7, 7, 2), device=device, dtype=torch.float)[:, :, :, :, 1]
conv1.weight = nn.Parameter(weight_nc)
weight_c = conv1.weight.contiguous()
if bias:
bias_nc = torch.randn((64, 2), device=device, dtype=torch.float)[:, 1]
conv1.bias = nn.Parameter(bias_nc)
bias_c = conv1.bias.contiguous()
out1 = conv1(input_nc)
conv1.weight = nn.Parameter(weight_c)
if bias:
conv1.bias = nn.Parameter(bias_c)
out2 = conv1(input_c)
self.assertEqual(out1, out2)
@onlyPRIVATEUSE1
@largeTensorTest('12GB')
@skipIfRocmVersionLessThan((6, 0))
def test_conv_transposed_large(self, device):
dtype = torch.half if self.device_type == 'npu' else torch.float
conv = nn.ConvTranspose2d(1, 1, 1, 1, bias=False).to(device).to(dtype)
input_large = torch.randn(4096, 1, 512, 1024, dtype=dtype, device=device)
ret = conv(input_large)
maxdiff0 = (ret.narrow(0, 0, 1024) - conv(input_large.narrow(0, 0, 1024))).abs_().max().item()
maxdiff1 = (ret.narrow(0, 1024, 1024) - conv(input_large.narrow(0, 1024, 1024))).abs_().max().item()
maxdiff2 = (ret.narrow(0, 2048, 1024) - conv(input_large.narrow(0, 2048, 1024))).abs_().max().item()
maxdiff3 = (ret.narrow(0, 3072, 1024) - conv(input_large.narrow(0, 3072, 1024))).abs_().max().item()
if self.device_type == 'npu':
self.assertEqual(maxdiff0, 0, atol=2e-3, rtol=1e-5)
self.assertEqual(maxdiff1, 0, atol=2e-3, rtol=1e-5)
self.assertEqual(maxdiff2, 0, atol=2e-3, rtol=1e-5)
self.assertEqual(maxdiff3, 0, atol=2e-3, rtol=1e-5)
else:
self.assertEqual(maxdiff0, 0)
self.assertEqual(maxdiff1, 0)
self.assertEqual(maxdiff2, 0)
self.assertEqual(maxdiff3, 0)
@onlyPRIVATEUSE1
@skipCUDAIfRocm
@largeTensorTest('12GB')
def test_conv_large(self, device):
dtype = torch.half if self.device_type == 'npu' else torch.float
conv = nn.Conv2d(2, 2, 8, 8, bias=False).to(device).to(dtype)
input_large = torch.randn(4097, 2, 512, 512, dtype=dtype, device=device)
ret = conv(input_large)
self.assertEqual(ret[:2048], conv(input_large[:2048]))
self.assertEqual(ret[2048:4096], conv(input_large[2048:4096]))
self.assertEqual(ret[4096:], conv(input_large[4096:]))
conv.zero_grad()
ret.view(4097, -1).max(dim=1).values.sum().backward()
del ret
grad1 = conv.weight.grad.detach().clone()
conv.zero_grad()
conv(input_large[:2048]).view(2048, -1).max(dim=1).values.sum().backward()
conv(input_large[2048:4096]).view(2048, -1).max(dim=1).values.sum().backward()
conv(input_large[4096:]).view(1, -1).max(dim=1).values.sum().backward()
grad2 = conv.weight.grad.detach().clone()
scale = 1 / grad2.abs().mean()
grad1 = grad1 * scale
grad2 = grad2 * scale
self.assertEqual(grad1, grad2, atol=5e-2, rtol=5e-3)
@onlyPRIVATEUSE1
@skipCUDAIfRocm
@largeTensorTest("20GB", "cpu")
@largeTensorTest("60GB", "npu")
def test_conv_large_batch_1(self, device):
in_channels = 514
dim = 2048
out_channels = 1
kernel_size = 3
stride = 1
padding = 1
input_tensor = torch.ones(1, in_channels, dim, dim).npu().half()
model = (
nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding)
.npu()
.half()
)
output = model(input_tensor)
_model_cpu = model.cpu().float()
output_cpu = model(input_tensor.float().cpu())
self.assertEqual(output.cpu().float(), output_cpu, atol=1e-3, rtol=1e-3)
@onlyPRIVATEUSE1
@skipCUDAIfRocm
@largeTensorTest("24GB", "cpu")
@largeTensorTest("20GB", "npu")
def test_conv3d_large_batch_1(self, device):
x = torch.rand(1, 32, 512, 512, 256)
m = torch.nn.Conv3d(32, 1, kernel_size=1, padding=0, stride=1, bias=False)
yref = m(x)
y = m.to(device=device)(x.to(device=device))
self.assertEqual(yref, y.cpu())
@onlyPRIVATEUSE1
@skipCUDAIfNoCudnn
def test_contig_wrong_stride_cudnn(self, device):
x = torch.randn(1, 16, 5, 5, device=device)
stride = list(x.stride())
stride[0] = 20
x.set_(x.storage(), 0, x.size(), stride)
self.assertTrue(x.is_contiguous())
F.conv_transpose2d(x, torch.randn(16, 1, 1, 1, device=device))
F.conv2d(x, torch.randn(1, 16, 1, 1, device=device))
@onlyPRIVATEUSE1
@SupportedDevices(['Ascend910B'])
def test_Conv2d_size_1_kernel(self, device):
if not device_is_910A:
torch.npu.conv.allow_hf32 = False
x_cpu = torch.randn(2, 3, 5, 5)
conv_cpu = torch.nn.Conv2d(3, 3, kernel_size=1)
y_cpu = conv_cpu(x_cpu)
y = torch.rand_like(y_cpu)
y_cpu.backward(y)
with cudnn.flags(enabled=False):
conv_npu = torch.nn.Conv2d(3, 3, kernel_size=1).to(device)
conv_npu.bias.data.copy_(conv_cpu.bias.data)
conv_npu.weight.data.copy_(conv_cpu.weight.data)
y_npu = conv_npu(x_cpu.to(device))
y_npu.backward(y.to(device))
self.assertEqual(y_cpu, y_npu, atol=1e-5, rtol=0, exact_device=False)
self.assertEqual(conv_cpu.bias.grad.data, conv_npu.bias.grad.data, atol=1e-5, rtol=0, exact_device=False)
self.assertEqual(conv_cpu.weight.grad.data, conv_npu.weight.grad.data, atol=1e-5, rtol=0, exact_device=False)
@onlyPRIVATEUSE1
@SupportedDevices(["Ascend910B"])
def test_ConvTranspose2d_size_1_kernel(self, device):
x_cpu = torch.randn(2, 3, 5, 5)
conv_cpu = torch.nn.ConvTranspose2d(3, 3, kernel_size=1)
y_cpu = conv_cpu(x_cpu)
y = torch.rand_like(y_cpu)
y_cpu.backward(y)
with cudnn.flags(enabled=False):
conv_npu = torch.nn.ConvTranspose2d(3, 3, kernel_size=1).to(device)
conv_npu.bias.data.copy_(conv_cpu.bias.data)
conv_npu.weight.data.copy_(conv_cpu.weight.data)
y_npu = conv_npu(x_cpu.to(device))
y_npu.backward(y.to(device))
self.assertEqual(y_cpu, y_npu, atol=1e-5, rtol=0, exact_device=False)
self.assertEqual(conv_cpu.bias.grad.data, conv_npu.bias.grad.data, atol=1e-5, rtol=0, exact_device=False)
self.assertEqual(conv_cpu.weight.grad.data, conv_npu.weight.grad.data, atol=1e-5, rtol=0, exact_device=False)
@onlyPRIVATEUSE1
def test_ConvTranspose3d_size_1_kernel(self, device):
with set_default_dtype(torch.double):
x_cpu = torch.randn(2, 3, 3, 5, 5)
conv_cpu = torch.nn.ConvTranspose3d(3, 3, kernel_size=1)
y_cpu = conv_cpu(x_cpu)
y = torch.rand_like(y_cpu)
y_cpu.backward(y)
with cudnn.flags(enabled=False):
conv_cuda = torch.nn.ConvTranspose3d(3, 3, kernel_size=1).to(device)
conv_cuda.bias.data.copy_(conv_cpu.bias.data)
conv_cuda.weight.data.copy_(conv_cpu.weight.data)
y_cuda = conv_cuda(x_cpu.to(device))
y_cuda.backward(y.to(device))
self.assertEqual(y_cpu, y_cuda, atol=1e-5, rtol=0, exact_device=False)
self.assertEqual(conv_cpu.bias.grad.data, conv_cuda.bias.grad.data, atol=1e-5, rtol=0, exact_device=False)
self.assertEqual(conv_cpu.weight.grad.data, conv_cuda.weight.grad.data,
atol=1e-5, rtol=0, exact_device=False)
@dtypesIfPRIVATEUSE1(*floating_types_and(torch.half, *[torch.bfloat16] if AMPERE_OR_ROCM else []))
@dtypes(torch.float)
@torch.backends.cudnn.flags(enabled=True, deterministic=True, benchmark=False)
@tf32_on_and_off(0.001)
@unittest.skipIf(TEST_WITH_ROCM, "Skipped on ROCm, since it is failing on ROCm 5.7")
def test_Conv2d_naive_groups(self, device, dtype):
m = nn.Conv2d(4, 4, kernel_size=3, groups=2).to(device, dtype)
i = torch.randn(2, 4, 6, 6, device=device, dtype=dtype, requires_grad=True)
output = m(i)
grad_output = torch.randn(2, 4, 4, 4, device=device, dtype=dtype)
output.backward(grad_output)
m1 = nn.Conv2d(2, 2, kernel_size=3).to(device, dtype)
m1.weight.data.copy_(m.weight.data[:2])
m1.bias.data.copy_(m.bias.data[:2])
i1 = i.data[:, :2].contiguous().requires_grad_(True)
output1 = m1(i1)
output1.backward(grad_output[:, :2].contiguous())
m2 = nn.Conv2d(2, 2, kernel_size=3).to(device, dtype)
m2.weight.data.copy_(m.weight.data[2:])
m2.bias.data.copy_(m.bias.data[2:])
i2 = i.data[:, 2:].contiguous().requires_grad_(True)
output2 = m2(i2)
output2.backward(grad_output[:, 2:].contiguous())
self.assertEqual(output, torch.cat([output1, output2], 1))
self.assertEqual(i.grad.data,
torch.cat([i1.grad.data, i2.grad.data], 1),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.bias.grad.data,
torch.cat([m1.bias.grad.data, m2.bias.grad.data], 0),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
self.assertEqual(m.weight.grad.data,
torch.cat([m1.weight.grad.data, m2.weight.grad.data], 0),
atol=dtype2prec_DONTUSE[dtype], rtol=0)
@dtypes(torch.double, torch.cdouble)
def test_Conv2d_backward_depthwise(self, device, dtype):
x = torch.randn(2, 2, 4, 20, device=device, dtype=dtype, requires_grad=True)
weight = torch.randn(2, 1, 3, 5, device=device, dtype=dtype, requires_grad=True)
def conv2d_depthwise(x, weight):
return torch.nn.functional.conv2d(
x, weight, bias=None, stride=(1, 10), groups=2)
for cudnn_enabled in [False, True]:
with torch.backends.cudnn.flags(enabled=cudnn_enabled):
torch.autograd.gradcheck(conv2d_depthwise, (x, weight))
@onlyCPU
@dtypes(torch.float, torch.double)
def test_conv_thnn_nhwc(self, device, dtype):
def helper(mod, n, c, h, w, out_channels, kernel_size, dilation, groups, input_format, weight_format):
input1 = torch.randint(-3, 3, (n, c, h, w), dtype=dtype, device=device)\
.to(memory_format=input_format)
input1.requires_grad_()
conv = mod(c, out_channels, kernel_size, dilation=dilation, groups=groups)\
.to(device='cpu', dtype=dtype, memory_format=weight_format)
for p in conv.parameters():
p.data = torch.randint_like(p, -3, 3)
ref_input = input1.detach().clone().contiguous().requires_grad_()
ref_conv = mod(c, out_channels, kernel_size, dilation=dilation, groups=groups)
ref_conv.load_state_dict(conv.state_dict())
ref_conv = ref_conv.to(device='cpu', dtype=dtype, memory_format=torch.contiguous_format)
out = conv(input1)
ref_out = ref_conv(ref_input)
grad = torch.randint_like(out, -3, 3)
ref_grad = grad.detach().clone().contiguous()
out.backward(grad)
ref_out.backward(ref_grad)
self.assertTrue(out.is_contiguous(memory_format=torch.channels_last))
self.assertTrue(ref_out.is_contiguous())
self.assertEqual(out, ref_out, exact_dtype=False)
self.assertEqual(conv.weight.grad, ref_conv.weight.grad, exact_dtype=False)
self.assertEqual(conv.bias.grad, ref_conv.bias.grad, exact_dtype=False)
self.assertEqual(input1.grad, ref_input.grad, exact_dtype=False)
with torch.backends.mkldnn.flags(enabled=False):
formats = [[torch.channels_last, torch.channels_last],
[torch.channels_last, torch.contiguous_format],
[torch.contiguous_format, torch.channels_last]]
for input_format, weight_format in formats:
helper(nn.Conv2d, 2, 8, 4, 4, out_channels=4, kernel_size=3, dilation=1, groups=1,
input_format=input_format, weight_format=weight_format)
helper(nn.Conv2d, 2, 8, 4, 4, out_channels=8, kernel_size=3, dilation=1, groups=8,
input_format=input_format, weight_format=weight_format)
helper(nn.Conv2d, 2, 1, 10, 10, out_channels=8, kernel_size=3, dilation=1, groups=1,
input_format=torch.contiguous_format, weight_format=torch.channels_last)
helper(nn.Conv2d, 1, 16, 56, 56, out_channels=16, kernel_size=1, dilation=1, groups=1,
input_format=input_format, weight_format=weight_format)
helper(nn.Conv2d, 1, 16, 56, 56, out_channels=16, kernel_size=1, dilation=1, groups=16,
input_format=torch.channels_last, weight_format=weight_format)
helper(nn.Conv2d, 2, 8, 11, 13, out_channels=16, kernel_size=3, dilation=2, groups=1,
input_format=input_format, weight_format=weight_format)
helper(nn.Conv2d, 2, 16, 11, 13, out_channels=32, kernel_size=3, dilation=2, groups=16,
input_format=input_format, weight_format=weight_format)
helper(nn.ConvTranspose2d, 2, 8, 4, 4, out_channels=4, kernel_size=3, dilation=1, groups=1,
input_format=input_format, weight_format=weight_format)
helper(nn.ConvTranspose2d, 2, 8, 4, 4, out_channels=8, kernel_size=3, dilation=1, groups=8,
input_format=input_format, weight_format=weight_format)
helper(nn.ConvTranspose2d, 1, 16, 56, 56, out_channels=16, kernel_size=1, dilation=1, groups=1,
input_format=input_format, weight_format=weight_format)
helper(nn.ConvTranspose2d, 1, 16, 56, 56, out_channels=32, kernel_size=1, dilation=1, groups=16,
input_format=input_format, weight_format=weight_format)
@onlyPRIVATEUSE1
@skipCUDAIfRocmVersionLessThan((4, 3))
@skipCUDAIfNotMiopenSuggestNHWC
@skipCUDAIfCudnnVersionLessThan(7603)
@dtypes(torch.half, torch.float, torch.cfloat)
def test_conv_cudnn_nhwc(self, device, dtype):
def helper(n, c, h, w, out_channels, kernel_size, groups):
input1 = torch.randint(-3, 3, (n, c, h, w), dtype=dtype, device=device)\
.to(memory_format=torch.channels_last)
input1.requires_grad_()
conv = nn.Conv2d(c, out_channels, kernel_size, groups=groups)\
.to(device='npu', dtype=dtype, memory_format=torch.channels_last)
for p in conv.parameters():
p.data = torch.randint_like(p, -3, 3)
ref_input = input1.detach().clone().contiguous().double().requires_grad_()
ref_conv = nn.Conv2d(c, out_channels, kernel_size, groups=groups)
ref_conv.load_state_dict(conv.state_dict())
ref_conv = ref_conv.to(device='npu', dtype=torch.double, memory_format=torch.contiguous_format)
out = conv(input1)
ref_out = ref_conv(ref_input)
grad = torch.randint_like(out, -3, 3)
ref_grad = grad.detach().clone().double().contiguous()
out.backward(grad)
ref_out.backward(ref_grad)
self.assertTrue(out.is_contiguous(memory_format=torch.channels_last))
self.assertTrue(input1.grad.is_contiguous(memory_format=torch.channels_last))
self.assertTrue(conv.weight.grad.is_contiguous(memory_format=torch.channels_last))
self.assertTrue(ref_out.is_contiguous())
self.assertTrue(ref_input.grad.is_contiguous())
self.assertTrue(ref_conv.weight.grad.is_contiguous())
self.assertEqual(out, ref_out, exact_dtype=False)
self.assertEqual(conv.weight.grad, ref_conv.weight.grad, exact_dtype=False)
self.assertEqual(conv.bias.grad, ref_conv.bias.grad, exact_dtype=False)
self.assertEqual(input1.grad, ref_input.grad, exact_dtype=False)
helper(2, 8, 4, 4, out_channels=4, kernel_size=3, groups=1)
helper(2, 8, 4, 4, out_channels=8, kernel_size=3, groups=8)
helper(1, 16, 56, 56, out_channels=16, kernel_size=3, groups=1)
helper(1, 16, 56, 56, out_channels=16, kernel_size=3, groups=16)
@onlyPRIVATEUSE1
@skipCUDAIfRocm
@skipCUDAIfCudnnVersionLessThan(8005)
@dtypes(torch.half, torch.float)
def test_conv_cudnn_ndhwc(self, device, dtype):
def helper(n, c, d, h, w, out_channels, kernel_size, groups):
input1 = torch.randint(-2, 2, (n, c, d, h, w), dtype=dtype, device=device)\
.to(memory_format=torch.channels_last_3d)
input1.requires_grad_()
conv = nn.Conv3d(c, out_channels, kernel_size, groups=groups)\
.to(device='npu', dtype=dtype, memory_format=torch.channels_last_3d)
for p in conv.parameters():
p.data = torch.randint_like(p, -2, 2)
ref_input = input1.detach().clone().contiguous().double().requires_grad_()
ref_conv = nn.Conv3d(c, out_channels, kernel_size, groups=groups)
ref_conv.load_state_dict(conv.state_dict())
ref_conv = ref_conv.to(device='npu', dtype=torch.double, memory_format=torch.contiguous_format)
out = conv(input1)
ref_out = ref_conv(ref_input)
grad = torch.randint_like(out, -2, 2)
ref_grad = grad.detach().clone().double().contiguous()
out.backward(grad)
ref_out.backward(ref_grad)
self.assertTrue(out.is_contiguous(memory_format=torch.channels_last_3d))
self.assertTrue(input1.grad.is_contiguous(memory_format=torch.channels_last_3d))
self.assertTrue(conv.weight.grad.is_contiguous(memory_format=torch.channels_last_3d))
self.assertTrue(ref_out.is_contiguous())
self.assertTrue(ref_input.grad.is_contiguous())
self.assertTrue(ref_conv.weight.grad.is_contiguous())
self.assertEqual(out, ref_out, exact_dtype=False)
self.assertEqual(conv.weight.grad, ref_conv.weight.grad, exact_dtype=False)
self.assertEqual(conv.bias.grad, ref_conv.bias.grad, exact_dtype=False)
self.assertEqual(input1.grad, ref_input.grad, exact_dtype=False)
helper(2, 8, 4, 4, 4, out_channels=4, kernel_size=3, groups=1)
helper(2, 8, 4, 4, 4, out_channels=8, kernel_size=3, groups=8)
helper(1, 16, 18, 18, 18, out_channels=16, kernel_size=3, groups=1)
helper(1, 16, 18, 18, 18, out_channels=16, kernel_size=3, groups=16)
def _run_conv(self, layer, device, inp, grad, ref_conv, ref_input, ref_out,
input_format, weight_format, grad_format, output_format):
conv = layer(inp.size(1), grad.size(1),
ref_conv.weight.size(2)).float().to(device)
conv.load_state_dict(ref_conv.state_dict())
weight_data = conv.weight.detach().clone().contiguous(memory_format=weight_format)
conv.weight.data = weight_data.resize_(weight_data.size(), memory_format=weight_format)
input1 = inp.clone().contiguous(memory_format=input_format)
input1.resize_(input1.size(), memory_format=input_format)
input1 = input1.requires_grad_()
grad = grad.contiguous(memory_format=grad_format)
grad.resize_(grad.size(), memory_format=grad_format)
out = conv(input1)
out.backward(grad)
self.assertTrue(out.is_contiguous(memory_format=output_format))
self.assertEqual(out, ref_out)
self.assertEqual(conv.weight.grad, ref_conv.weight.grad)
self.assertEqual(conv.bias.grad, ref_conv.bias.grad)
self.assertEqual(input1.grad, ref_input.grad)
def _test_conv_cudnn_nhwc_nchw(self, layer, n, c, h, w, k, filter_size, device):
data = torch.randint(1, 10, (n, c, h, w), dtype=torch.float32, device=device)
ref_input = data.clone().contiguous().requires_grad_(True)
ref_conv = layer(c, k, filter_size).float().to(device)
ref_out = ref_conv(ref_input)
grad = torch.randint(1, 10, ref_out.size(), dtype=torch.float32, device="npu")
ref_out.backward(grad)
for w_f in [torch.contiguous_format, torch.channels_last]:
for g_f in [torch.contiguous_format, torch.channels_last]:
for input_format in [torch.contiguous_format, torch.channels_last]:
output_format = torch.contiguous_format
if torch.backends.cudnn.version() >= 7603:
if input_format == torch.channels_last:
output_format = torch.channels_last
if w_f == torch.channels_last:
if layer == nn.Conv2d and filter_size * c != 1:
output_format = torch.channels_last
if layer == nn.ConvTranspose2d and filter_size * k != 1:
output_format = torch.channels_last
self._run_conv(layer, device, data, grad, ref_conv, ref_input,
ref_out, input_format, w_f, g_f, output_format)
@onlyPRIVATEUSE1
@skipCUDAIfRocmVersionLessThan((4, 3))
@skipCUDAIfNotMiopenSuggestNHWC
@skipCUDAIfCudnnVersionLessThan(7603)
@tf32_on_and_off(0.05)
def test_conv_cudnn_mismatch_memory_format(self, device):
configs = [
[4, 2, 8, 8, 4, 2],
[4, 1, 8, 8, 4, 2],
[1, 1, 8, 8, 4, 2],
[4, 2, 2, 8, 4, 1],
[4, 2, 1, 8, 4, 1],
[4, 2, 8, 8, 4, 1],
[4, 1, 8, 8, 4, 1],
]
for n, c, h, w, k, filter_size in configs:
self._test_conv_cudnn_nhwc_nchw(nn.Conv2d, n, c, h, w, k, filter_size, device)
self._test_conv_cudnn_nhwc_nchw(nn.ConvTranspose2d, n, c, h, w, k, filter_size, device)
@onlyPRIVATEUSE1
@skipCUDAIfNoCudnn
@dtypes(torch.float, torch.double)
def test_conv_cudnn_nhwc_support(self, device, dtype):
input1 = torch.randn((1, 16, 1, 1), dtype=dtype, device="npu", requires_grad=True)
weight = torch.randn((8, 16, 3, 3), dtype=dtype, device="npu", requires_grad=True)
weight = weight.to(memory_format=torch.channels_last)
out = torch.conv2d(input1, weight, None, (2, 1), (1, 1), (1, 1), 1)
self.assertTrue(out.is_contiguous(memory_format=torch.channels_last))
out.sum().backward()
@onlyCPU
@dtypes(torch.float)
def test_conv2d_no_grad(self, device, dtype):
for batch in [1, 2, 3]:
for groups in [1, 2, 4]:
input1 = torch.rand(batch, groups, 8, 8, dtype=dtype, device=device)
m = nn.Conv2d(groups, 8, kernel_size=(3, 3), groups=groups, dtype=dtype, device=device)
with torch.no_grad():
output_ng = m(input1)
output = m(input1)
self.assertEqual(output, output_ng, rtol=1e-2, atol=1e-5)
@onlyPRIVATEUSE1
@skipCUDAIfNoCudnn
@dtypes(torch.float, torch.float16)
@precisionOverride({torch.half: 0.002, torch.float: 1e-4})
def test_cudnn_convolution_relu(self, device, dtype):
for batch, groups, image_size, kernel_size, memory_format in \
product((1, 2, 3),
(1, 2, 4),
((1, 1), (8, 8)),
((1, 1), (3, 3)),
(torch.channels_last, torch.contiguous_format)):
if image_size[0] < kernel_size[0]:
continue
inp = torch.rand(batch, groups, *image_size, dtype=dtype, device=device)
w = torch.randn(8, groups, *kernel_size, dtype=dtype, device=device)
conv2d_out = torch.conv2d(inp, w, None, (1, 1), (0, 0), (1, 1), 1)
inp = inp.to(memory_format=memory_format)
w = w.to(memory_format=memory_format)
if torch.version.hip:
cudnn_out = torch.miopen_convolution_relu(inp, w, None, (1, 1), (0, 0), (1, 1), 1)
else:
cudnn_out = torch.cudnn_convolution_relu(inp, w, None, (1, 1), (0, 0), (1, 1), 1)
self.assertTrue(cudnn_out.is_contiguous(memory_format=memory_format))
if torch.cuda.is_tf32_supported() and dtype == torch.float:
self.assertEqual(conv2d_out.relu(), cudnn_out, atol=4e-3, rtol=0.006)
else:
self.assertEqual(conv2d_out.relu(), cudnn_out)
@onlyPRIVATEUSE1
@skipCUDAIfNoCudnn
@dtypes(torch.float, torch.float16)
@precisionOverride({torch.half: 0.002, torch.float: 1e-4})
def test_cudnn_convolution_add_relu(self, device, dtype):
for batch, groups, image_size, kernel_size, memory_format in \
product((1, 2, 3),
(1, 2, 4),
((1, 1), (8, 8)),
((1, 1), (3, 3)),
(torch.channels_last, torch.contiguous_format)):
if image_size[0] < kernel_size[0]:
continue
inp = torch.rand(batch, groups, *image_size, dtype=dtype, device=device)
w = torch.randn(8, groups, *kernel_size, dtype=dtype, device=device)
conv2d_out = torch.conv2d(inp, w, None, (1, 1), (0, 0), (1, 1), 1)
alpha = 2.0
z = torch.randn_like(conv2d_out)
inp = inp.to(memory_format=memory_format)
w = w.to(memory_format=memory_format)
z = z.to(memory_format=memory_format)
if torch.version.hip:
cudnn_out = torch.miopen_convolution_add_relu(inp, w, z, alpha, None, (1, 1), (0, 0), (1, 1), 1)
else:
cudnn_out = torch.cudnn_convolution_add_relu(inp, w, z, alpha, None, (1, 1), (0, 0), (1, 1), 1)
self.assertTrue(cudnn_out.is_contiguous(memory_format=memory_format))
if torch.cuda.is_tf32_supported() and dtype == torch.float:
self.assertEqual(F.relu(conv2d_out + alpha * z), cudnn_out, atol=2e-3, rtol=0.006)
else:
self.assertEqual(F.relu(conv2d_out + alpha * z), cudnn_out)
@onlyPRIVATEUSE1
@skipCUDAIfRocm
@skipCUDAIfCudnnVersionLessThan(7603)
def test_convert_conv2d_weight_memory_format(self, device):
input1 = torch.randint(1, 10, (2, 8, 4, 4), dtype=torch.float32, device=device)
model = nn.Sequential(
nn.Conv2d(8, 4, 3),
nn.BatchNorm2d(4)).to(device).float()
for memory_format in [torch.channels_last, torch.contiguous_format]:
model = nn.utils.convert_conv2d_weight_memory_format(model, memory_format)
out = model(input1)
self.assertTrue(out.is_contiguous(memory_format=memory_format))
model = nn.Sequential(
nn.ConvTranspose2d(8, 4, 3),
nn.BatchNorm2d(4)).to(device).float()
for memory_format in [torch.channels_last, torch.contiguous_format]:
model = nn.utils.convert_conv2d_weight_memory_format(model, memory_format)
out = model(input1)
self.assertTrue(out.is_contiguous(memory_format=memory_format))
def test_conv_double_backward_strided_with_3D_input_and_weight(self, device):
input1 = torch.randn(2, 3, 6, device=device)
weight = torch.randn(3, 3, 3, device=device)
bias = torch.randn(3, device=device)
stride = (2,)
padding = (1,)
dilation = (1,)
transposed = False
output_padding = (0,)
groups = 1
output = torch.ops.aten.convolution(input1, weight, bias, stride, padding, dilation, transposed,
output_padding, groups)
ggI = torch.randn(input1.shape, device=device)
ggW = torch.randn(weight.shape, device=device)
ggB = torch.randn(bias.shape, device=device)
gO = torch.randn(output.shape, device=device)
output_mask = [True, True, True]
grad_grad_output, grad_input, grad_weight = torch.ops.aten._convolution_double_backward(
ggI, ggW, ggB, gO, weight, input1, stride, padding, dilation, transposed,
output_padding, groups, output_mask)
self.assertEqual(grad_grad_output.shape, gO.shape)
self.assertEqual(grad_input.shape, input1.shape)
self.assertEqual(grad_weight.shape, weight.shape)
@onlyPRIVATEUSE1
@largeTensorTest('40GB')
@largeTensorTest('24GB', 'cpu')
def test_conv3d_64bit_indexing(self, device):
x = torch.rand(1, 32, 512, 512, 256)
m = torch.nn.Conv3d(32, 1, kernel_size=1, padding=0, stride=1, bias=False)
yref = m(x)
y = m.to(device=device)(x.to(device=device))
self.assertEqual(yref, y)
instantiate_device_type_tests(TestConvolutionNNDeviceType, globals())
instantiate_parametrized_tests(TestConvolutionNN)
if __name__ == '__main__':
run_tests()