{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "view-in-github"
},
"source": [
"<a href=\"https://colab.research.google.com/github/yerfor/Real3DPortrait/blob/main/real3dportrait_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "QS04K9oO21AW"
},
"source": [
"Check GPU"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "1ESQRDb-yVUG"
},
"outputs": [],
"source": [
"!nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv,noheader"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "y-ctmIvu3Ei8"
},
"source": [
"Installation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "gXu76wdDgaxo"
},
"outputs": [],
"source": [
"# install pytorch3d, about 15s\n",
"import os\n",
"import sys\n",
"import torch\n",
"need_pytorch3d=False\n",
"try:\n",
" import pytorch3d\n",
"except ModuleNotFoundError:\n",
" need_pytorch3d=True\n",
"if need_pytorch3d:\n",
" if torch.__version__.startswith(\"2.1.\") and sys.platform.startswith(\"linux\"):\n",
" # We try to install PyTorch3D via a released wheel.\n",
" pyt_version_str=torch.__version__.split(\"+\")[0].replace(\".\", \"\")\n",
" version_str=\"\".join([\n",
" f\"py3{sys.version_info.minor}_cu\",\n",
" torch.version.cuda.replace(\".\",\"\"),\n",
" f\"_pyt{pyt_version_str}\"\n",
" ])\n",
" !pip install fvcore iopath\n",
" !pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html\n",
" else:\n",
" # We try to install PyTorch3D from source.\n",
" !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "DuUynxmotG_-"
},
"outputs": [],
"source": [
"# install dependencies, about 5~10 min\n",
"!pip install tensorboard==2.13.0 tensorboardX==2.6.1\n",
"!pip install pyspy==0.1.1\n",
"!pip install protobuf==3.20.3\n",
"!pip install scipy==1.9.1\n",
"!pip install kornia==0.5.0\n",
"!pip install trimesh==3.22.0\n",
"!pip install einops==0.6.1 torchshow==0.5.1\n",
"!pip install imageio==2.31.1 imageio-ffmpeg==0.4.8\n",
"!pip install scikit-learn==1.3.0 scikit-image==0.21.0\n",
"!pip install av==10.0.0 lpips==0.1.4\n",
"!pip install timm==0.9.2 librosa==0.9.2\n",
"!pip install openmim==0.3.9\n",
"!mim install mmcv==2.1.0 # use mim to speed up installation for mmcv\n",
"!pip install transformers==4.33.2\n",
"!pip install pretrainedmodels==0.7.4\n",
"!pip install ninja==1.11.1\n",
"!pip install faiss-cpu==1.7.4\n",
"!pip install praat-parselmouth==0.4.3 moviepy==1.0.3\n",
"!pip install mediapipe==0.10.7\n",
"!pip install --upgrade attr\n",
"!pip install beartype==0.16.4 gateloop_transformer==0.4.0\n",
"!pip install torchode==0.2.0 torchdiffeq==0.2.3\n",
"!pip install hydra-core==1.3.2 pandas==2.1.3\n",
"!pip install pytorch_lightning==2.1.2\n",
"!pip install httpx==0.23.3\n",
"!pip install gradio==4.16.0\n",
"!pip install gdown\n",
"!pip install pyloudnorm webrtcvad pyworld==0.2.1rc0 pypinyin==0.42.0"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "0GLEV0HVu8rj"
},
"outputs": [],
"source": [
"# RESTART kernel to make sure runtime is correct if you meet runtime errors\n",
"# import os\n",
"# os.kill(os.getpid(), 9)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "5UfKHKrH6kcq"
},
"source": [
"Clone code and download checkpoints"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-gfRsd9DwIgl"
},
"outputs": [],
"source": [
"# clone Real3DPortrait repo from github\n",
"!git clone https://github.com/yerfor/Real3DPortrait\n",
"%cd Real3DPortrait"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Yju8dQY7x5OS"
},
"outputs": [],
"source": [
"# download pretrained ckpts & third-party ckpts from google drive, about 1 min\n",
"!pip install --upgrade --no-cache-dir gdown\n",
"%cd deep_3drecon/BFM\n",
"!gdown https://drive.google.com/uc?id=1SPM3IHsyNAaVMwqZZGV6QVaV7I2Hly0v\n",
"!gdown https://drive.google.com/uc?id=1MSldX9UChKEb3AXLVTPzZQcsbGD4VmGF\n",
"!gdown https://drive.google.com/uc?id=180ciTvm16peWrcpl4DOekT9eUQ-lJfMU\n",
"!gdown https://drive.google.com/uc?id=1KX9MyGueFB3M-X0Ss152x_johyTXHTfU\n",
"!gdown https://drive.google.com/uc?id=19-NyZn_I0_mkF-F5GPyFMwQJ_-WecZIL\n",
"!gdown https://drive.google.com/uc?id=11ouQ7Wr2I-JKStp2Fd1afedmWeuifhof\n",
"!gdown https://drive.google.com/uc?id=18ICIvQoKX-7feYWP61RbpppzDuYTptCq\n",
"!gdown https://drive.google.com/uc?id=1VktuY46m0v_n_d4nvOupauJkK4LF6mHE\n",
"%cd ../..\n",
"\n",
"%cd checkpoints\n",
"!gdown https://drive.google.com/uc?id=1gz8A6xestHp__GbZT5qozb43YaybRJhZ\n",
"!gdown https://drive.google.com/uc?id=1gSUIw2AkkKnlLJnNfS2FCqtaVw9tw3QF\n",
"!unzip 240210_real3dportrait_orig.zip\n",
"!unzip pretrained_ckpts.zip\n",
"!ls\n",
"%cd ..\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "LHzLro206pnA"
},
"source": [
"Inference sample"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"!python inference/real3d_infer.py -h"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "2aCDwxNivQoS"
},
"outputs": [],
"source": [
"# sample inference, about 3 min\n",
"!python inference/real3d_infer.py \\\n",
"--src_img data/raw/examples/Macron.png \\\n",
"--drv_aud data/raw/examples/Obama_5s.wav \\\n",
"--drv_pose data/raw/examples/May_5s.mp4 \\\n",
"--bg_img data/raw/examples/bg.png \\\n",
"--out_name output.mp4 \\\n",
"--out_mode concat_debug \\\n",
"--low_memory_usage"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "XL0c54l19mBG"
},
"source": [
"Display output video"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "6olmWwZP9Icj"
},
"outputs": [],
"source": [
"# borrow code from makeittalk\n",
"from IPython.display import HTML\n",
"from base64 import b64encode\n",
"import os, sys\n",
"import glob\n",
"\n",
"mp4_name = './output.mp4'\n",
"\n",
"mp4 = open('{}'.format(mp4_name),'rb').read()\n",
"data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
"\n",
"print('Display animation: {}'.format(mp4_name), file=sys.stderr)\n",
"display(HTML(\"\"\"\n",
" <video width=256 controls>\n",
" <source src=\"%s\" type=\"video/mp4\">\n",
" </video>\n",
" \"\"\" % data_url))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"WebUI"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"!python inference/app_real3dportrait.py --share"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"authorship_tag": "ABX9TyPu++zOlOS4yKF4xn4FHGtZ",
"gpuType": "T4",
"include_colab_link": true,
"private_outputs": true,
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}