385effc5创建于 2024年8月10日历史提交
import matplotlib.pyplot as plt
import numpy as np
import torch.nn.functional as F
from skimage.draw import circle_perimeter as circle


class Visualizer:
    def __init__(
        self,
        kp_size=5,
        draw_border=False,
        colormap="gist_rainbow",
        region_bg_color=(0, 0, 0),
    ):
        self.kp_size = kp_size
        self.draw_border = draw_border
        self.colormap = plt.get_cmap(colormap)
        self.region_bg_color = np.array(region_bg_color)

    def draw_image_with_kp(self, image, kp_array):
        image = np.copy(image)
        spatial_size = np.array(image.shape[:2][::-1])[np.newaxis]
        kp_array = spatial_size * (kp_array + 1) / 2
        num_regions = kp_array.shape[0]
        for kp_ind, kp in enumerate(kp_array):
            rr, cc = circle(int(kp[1]), int(kp[0]), self.kp_size, shape=image.shape[:2])
            image[rr, cc] = np.array(self.colormap(kp_ind / num_regions))[:3]
        return image

    def create_image_column_with_kp(self, images, kp):
        image_array = np.array(
            [self.draw_image_with_kp(v, k) for v, k in zip(images, kp)]
        )
        return self.create_image_column(image_array)

    def create_image_column(self, images):
        if self.draw_border:
            images = np.copy(images)
            images[:, :, [0, -1]] = (1, 1, 1)
            images[:, :, [0, -1]] = (1, 1, 1)
        return np.concatenate(list(images), axis=0)

    def create_image_grid(self, *args):
        out = []
        for arg in args:
            if type(arg) == tuple:
                out.append(self.create_image_column_with_kp(arg[0], arg[1]))
            else:
                out.append(self.create_image_column(arg))
        return np.concatenate(out, axis=1)

    def visualize(self, driving, source, out):
        images = []

        # Source image with region centers
        source = source.data.cpu()
        source_region_params = out["source_region_params"]["shift"].data.cpu().numpy()
        source = np.transpose(source, [0, 2, 3, 1])
        images.append((source, source_region_params))

        # Equivariance visualization
        if "transformed_frame" in out:
            transformed = out["transformed_frame"].data.cpu().numpy()
            transformed = np.transpose(transformed, [0, 2, 3, 1])
            transformed_kp = (
                out["transformed_region_params"]["shift"].data.cpu().numpy()
            )
            images.append((transformed, transformed_kp))

        # Driving image with  region centers
        driving_region_params = out["driving_region_params"]["shift"].data.cpu().numpy()
        driving = driving.data.cpu().numpy()
        driving = np.transpose(driving, [0, 2, 3, 1])
        images.append((driving, driving_region_params))

        # Deformed image
        if "deformed" in out:
            deformed = out["deformed"].data.cpu().numpy()
            deformed = np.transpose(deformed, [0, 2, 3, 1])
            images.append(deformed)

        # Result
        prediction = out["prediction"].data.cpu().numpy()
        prediction = np.transpose(prediction, [0, 2, 3, 1])
        images.append(prediction)

        # Occlusion map
        if "occlusion_map" in out:
            occlusion_map = out["occlusion_map"].data.cpu().repeat(1, 3, 1, 1)
            occlusion_map = F.interpolate(occlusion_map, size=source.shape[1:3]).numpy()
            occlusion_map = np.transpose(occlusion_map, [0, 2, 3, 1])
            images.append(occlusion_map)

        # Heatmaps visualizations
        if "heatmap" in out["driving_region_params"]:
            driving_heatmap = F.interpolate(
                out["driving_region_params"]["heatmap"], size=source.shape[1:3]
            )
            driving_heatmap = np.transpose(
                driving_heatmap.data.cpu().numpy(), [0, 2, 3, 1]
            )
            images.append(
                draw_colored_heatmap(
                    driving_heatmap, self.colormap, self.region_bg_color
                )
            )

        if "heatmap" in out["source_region_params"]:
            source_heatmap = F.interpolate(
                out["source_region_params"]["heatmap"], size=source.shape[1:3]
            )
            source_heatmap = np.transpose(
                source_heatmap.data.cpu().numpy(), [0, 2, 3, 1]
            )
            images.append(
                draw_colored_heatmap(
                    source_heatmap, self.colormap, self.region_bg_color
                )
            )

        image = self.create_image_grid(*images)
        image = (255 * image).astype(np.uint8)
        return image


def draw_colored_heatmap(heatmap, colormap, bg_color):
    parts = []
    weights = []
    bg_color = np.array(bg_color).reshape((1, 1, 1, 3))
    num_regions = heatmap.shape[-1]
    for i in range(num_regions):
        color = np.array(colormap(i / num_regions))[:3]
        color = color.reshape((1, 1, 1, 3))
        part = heatmap[:, :, :, i : (i + 1)]
        part = part / np.max(part, axis=(1, 2), keepdims=True)
        weights.append(part)

        color_part = part * color
        parts.append(color_part)

    weight = sum(weights)
    bg_weight = 1 - np.minimum(1, weight)
    weight = np.maximum(1, weight)
    result = sum(parts) / weight + bg_weight * bg_color
    return result