import matplotlib.pyplot as plt
import numpy as np
import torch.nn.functional as F
from skimage.draw import circle_perimeter as circle
class Visualizer:
def __init__(
self,
kp_size=5,
draw_border=False,
colormap="gist_rainbow",
region_bg_color=(0, 0, 0),
):
self.kp_size = kp_size
self.draw_border = draw_border
self.colormap = plt.get_cmap(colormap)
self.region_bg_color = np.array(region_bg_color)
def draw_image_with_kp(self, image, kp_array):
image = np.copy(image)
spatial_size = np.array(image.shape[:2][::-1])[np.newaxis]
kp_array = spatial_size * (kp_array + 1) / 2
num_regions = kp_array.shape[0]
for kp_ind, kp in enumerate(kp_array):
rr, cc = circle(int(kp[1]), int(kp[0]), self.kp_size, shape=image.shape[:2])
image[rr, cc] = np.array(self.colormap(kp_ind / num_regions))[:3]
return image
def create_image_column_with_kp(self, images, kp):
image_array = np.array(
[self.draw_image_with_kp(v, k) for v, k in zip(images, kp)]
)
return self.create_image_column(image_array)
def create_image_column(self, images):
if self.draw_border:
images = np.copy(images)
images[:, :, [0, -1]] = (1, 1, 1)
images[:, :, [0, -1]] = (1, 1, 1)
return np.concatenate(list(images), axis=0)
def create_image_grid(self, *args):
out = []
for arg in args:
if type(arg) == tuple:
out.append(self.create_image_column_with_kp(arg[0], arg[1]))
else:
out.append(self.create_image_column(arg))
return np.concatenate(out, axis=1)
def visualize(self, driving, source, out):
images = []
source = source.data.cpu()
source_region_params = out["source_region_params"]["shift"].data.cpu().numpy()
source = np.transpose(source, [0, 2, 3, 1])
images.append((source, source_region_params))
if "transformed_frame" in out:
transformed = out["transformed_frame"].data.cpu().numpy()
transformed = np.transpose(transformed, [0, 2, 3, 1])
transformed_kp = (
out["transformed_region_params"]["shift"].data.cpu().numpy()
)
images.append((transformed, transformed_kp))
driving_region_params = out["driving_region_params"]["shift"].data.cpu().numpy()
driving = driving.data.cpu().numpy()
driving = np.transpose(driving, [0, 2, 3, 1])
images.append((driving, driving_region_params))
if "deformed" in out:
deformed = out["deformed"].data.cpu().numpy()
deformed = np.transpose(deformed, [0, 2, 3, 1])
images.append(deformed)
prediction = out["prediction"].data.cpu().numpy()
prediction = np.transpose(prediction, [0, 2, 3, 1])
images.append(prediction)
if "occlusion_map" in out:
occlusion_map = out["occlusion_map"].data.cpu().repeat(1, 3, 1, 1)
occlusion_map = F.interpolate(occlusion_map, size=source.shape[1:3]).numpy()
occlusion_map = np.transpose(occlusion_map, [0, 2, 3, 1])
images.append(occlusion_map)
if "heatmap" in out["driving_region_params"]:
driving_heatmap = F.interpolate(
out["driving_region_params"]["heatmap"], size=source.shape[1:3]
)
driving_heatmap = np.transpose(
driving_heatmap.data.cpu().numpy(), [0, 2, 3, 1]
)
images.append(
draw_colored_heatmap(
driving_heatmap, self.colormap, self.region_bg_color
)
)
if "heatmap" in out["source_region_params"]:
source_heatmap = F.interpolate(
out["source_region_params"]["heatmap"], size=source.shape[1:3]
)
source_heatmap = np.transpose(
source_heatmap.data.cpu().numpy(), [0, 2, 3, 1]
)
images.append(
draw_colored_heatmap(
source_heatmap, self.colormap, self.region_bg_color
)
)
image = self.create_image_grid(*images)
image = (255 * image).astype(np.uint8)
return image
def draw_colored_heatmap(heatmap, colormap, bg_color):
parts = []
weights = []
bg_color = np.array(bg_color).reshape((1, 1, 1, 3))
num_regions = heatmap.shape[-1]
for i in range(num_regions):
color = np.array(colormap(i / num_regions))[:3]
color = color.reshape((1, 1, 1, 3))
part = heatmap[:, :, :, i : (i + 1)]
part = part / np.max(part, axis=(1, 2), keepdims=True)
weights.append(part)
color_part = part * color
parts.append(color_part)
weight = sum(weights)
bg_weight = 1 - np.minimum(1, weight)
weight = np.maximum(1, weight)
result = sum(parts) / weight + bg_weight * bg_color
return result