import os
import cv2
import numpy as np
import scipy.io as sio
from lib import face_align
class ImageCropper():
def __init__(self, img_size, use_dlib=False):
self.use_dlib = use_dlib
if use_dlib:
import dlib
predictor_path = os.path.join('data', 'models',
'shape_predictor_68_face_landmarks.dat')
self.detector = dlib.get_frontal_face_detector()
self.predictor = dlib.shape_predictor(predictor_path)
else:
self.predictor = face_align.FaceAlignment(face_align.LandmarksType._2D,
device='cuda')
self.load_lm3d()
self.img_size = img_size
def load_lm3d(self):
Lm3D = sio.loadmat('data/models/similarity_Lm3D_all.mat')
Lm3D = Lm3D['lm']
lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1
Lm3D = np.stack([
Lm3D[lm_idx[0], :],
np.mean(Lm3D[lm_idx[[1, 2]], :], 0),
np.mean(Lm3D[lm_idx[[3, 4]], :], 0), Lm3D[lm_idx[5], :],
Lm3D[lm_idx[6], :]
], axis=0)
self.lm3D = Lm3D[[1, 2, 0, 3, 4], :]
def compute_lm_trans(self, lm):
npts = lm.shape[1]
A = np.zeros([2 * npts, 8])
A[0:2 * npts - 1:2, 0:3] = self.lm3D
A[0:2 * npts - 1:2, 3] = 1
A[1:2 * npts:2, 4:7] = self.lm3D
A[1:2 * npts:2, 7] = 1
b = np.reshape(lm.transpose(), [2 * npts, 1])
k, _, _, _ = np.linalg.lstsq(A, b, -1)
R1 = k[0:3]
R2 = k[4:7]
sTx = k[3]
sTy = k[7]
s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2
t = np.stack([sTx, sTy], axis=0)
return t, s
def process_image(self, img, t, s, im_size):
w0, h0, _ = img.shape
img = cv2.warpAffine(
img, np.array([[1, 0, w0 / 2 - t[0, 0]], [0, 1, t[1, 0] - h0 / 2]]),
(w0, h0), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
half_size = im_size // 2
scale = (102 / 224) * im_size
w = (w0 / s * scale).astype(np.int32)
h = (h0 / s * scale).astype(np.int32)
img = cv2.resize(img, (w, h), cv2.INTER_LINEAR)
if w < im_size:
top = (im_size - h) // 2
bottom = im_size - h - top
left = (im_size - w) // 2
right = im_size - w - left
img = cv2.copyMakeBorder(img, top, bottom, left, right,
cv2.BORDER_REPLICATE)
else:
left = (w / 2 - half_size).astype(np.int32)
right = left + im_size
up = (h / 2 - half_size).astype(np.int32)
below = up + im_size
img = img[left:right, up:below].copy()
return img
def get_landmarks(self, image, need_bbox=False):
if self.use_dlib:
faces = self.detector(np.array(image[..., :3]), 1)
landmarks = self.predictor(np.array(image[..., :3]), faces[0])
else:
im_size = np.array(image.shape[:2])
im_256 = cv2.resize(image, (256, 256), cv2.INTER_AREA)
landmarks, faces = self.predictor.get_landmarks(im_256)
landmarks = landmarks[0]
faces = faces[0]
landmarks = np.round(landmarks * (im_size / 256).astype(np.float32))
if need_bbox:
return landmarks, faces
else:
return landmarks
def upright_face(self, image, lm, im_size):
lm_top = np.mean(lm[:2], axis=0)
lm_bottom = np.mean(lm[3:], axis=0)
dist = lm_top - lm_bottom
angle = np.math.atan(-dist[0] / dist[1])
angle = np.rad2deg(angle)
M = cv2.getRotationMatrix2D((im_size / 2, im_size / 2), angle, 1.0)
rotated = cv2.warpAffine(image, M, (im_size, im_size))
lm_rot = np.einsum('ij,aj->ai', M[:, :2], lm)
lm_rot += M[:, 2]
lm_rot = np.round(lm_rot)
return rotated, lm_rot
def crop_image(self, image, im_size=None):
if im_size is None:
im_size = self.img_size
landmarks = self.get_landmarks(image[..., :3])
idxs = [[36, 37, 38, 39, 40, 41], [42, 43, 44, 45, 46, 47], [30], [48],
[54]]
lm = np.zeros([5, 2])
for i in range(5):
for j in idxs[i]:
if self.use_dlib:
lm[i] += np.array([landmarks.part(j).x, landmarks.part(j).y])
else:
lm[i] += np.array([landmarks[j, 0], landmarks[j, 1]])
lm[i] = lm[i] // len(idxs[i])
lm = np.stack([lm[:, 0], im_size - 1 - lm[:, 1]], axis=1)
t, s = self.compute_lm_trans(lm.transpose())
return self.process_image(image, t, s, im_size)
def crop_images(self, images, im_size=None):
outputs = []
for image in images:
outputs.append(self.crop_image(image, im_size))
return np.array(outputs)