Initial commit of faceswap.py

Author: matthewearl

faceswap.py

@@ -0,0 +1,174 @@
+#!/usr/bin/python
+
+# Copyright (c) 2015 Matthew Earl
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+#     The above copyright notice and this permission notice shall be included
+#     in all copies or substantial portions of the Software.
+# 
+#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+#     USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import cv2
+import dlib
+import numpy
+
+import sys
+
+PREDICTOR_PATH = "/home/matt/dlib-18.16/shape_predictor_68_face_landmarks.dat"
+SCALE_FACTOR = 1 
+FACE_POINTS = list(range(17, 68))
+MOUTH_POINTS = list(range(48, 61))
+RIGHT_BROW_POINTS = list(range(17, 22))
+LEFT_BROW_POINTS = list(range(22, 27))
+RIGHT_EYE_POINTS = list(range(36, 42))
+LEFT_EYE_POINTS = list(range(42, 48))
+DILATE_AMOUNT = 1
+
+detector = dlib.get_frontal_face_detector()
+predictor = dlib.shape_predictor(PREDICTOR_PATH)
+
+class TooManyFaces(Exception):
+    pass
+
+class NoFaces(Exception):
+    pass
+
+def get_landmarks(im):
+    dets = detector(im, 1)
+    
+    if len(dets) > 1:
+        raise TooManyFaces
+    if len(dets) == 0:
+        raise NoFaces
+
+    def shape_to_mat(s):
+        return numpy.matrix([[p.x, p.y] for p in s.parts()])
+
+    return shape_to_mat(predictor(im, dets[0]))
+
+def annotate_landmarks(im, shape):
+    im = im.copy()
+    for idx, point in enumerate(shape):
+        pos = (point[0, 0], point[0, 1])
+        cv2.putText(im, str(idx), pos,
+                    fontFace=cv2.FONT_HERSHEY_PLAIN,
+                    fontScale=1,
+                    color=255)
+        cv2.circle(im, pos, 5, color=255)
+    return im
+
+def get_face_mask(im, shape):
+    im = numpy.zeros(im.shape[:2], dtype=numpy.float64)
+
+    points = shape[FACE_POINTS]
+    points = cv2.convexHull(points)
+
+    cv2.fillConvexPoly(im, points, color=1)
+
+    im = numpy.array([im, im, im]).transpose((1, 2, 0))
+
+    im = (cv2.GaussianBlur(im, (DILATE_AMOUNT, DILATE_AMOUNT), 0) > 0) * 1.0
+    im = cv2.GaussianBlur(im, (DILATE_AMOUNT, DILATE_AMOUNT), 0)
+
+    return im
+    
+def transformation_from_points(points1, points2):
+    """
+    Return an affine transformation [s * R | T] such that:
+
+        sum ||s*R*p1,i + T - p2,i||^2
+
+    is minimized.
+
+    """
+    # Solve the procrustes problem by subtracting centroids, scaling by the
+    # standard deviation, and then using the SVD to calculate the rotation. See
+    # the following for more details:
+    #   https://en.wikipedia.org/wiki/Orthogonal_Procrustes_problem
+
+    points1 = points1.astype(numpy.float64)
+    points2 = points2.astype(numpy.float64)
+
+    c1 = numpy.mean(points1, axis=0)
+    c2 = numpy.mean(points2, axis=0)
+    points1 -= c1
+    points2 -= c2
+
+    s1 = numpy.std(points1)
+    s2 = numpy.std(points2)
+    points1 /= s1
+    points2 /= s2
+
+    U, S, Vt = numpy.linalg.svd(points1.T * points2)
+
+    # The R we seek is in fact the transpose of the one given by U * Vt. This
+    # is because the above formulation assumes the matrix goes on the right
+    # (with row vectors) where as our solution requires the matrix to be on the
+    # left (with column vectors).
+    R = (U * Vt).T
+
+    return numpy.vstack([numpy.hstack(((s2 / s1) * R,
+                                       c2.T - (s2 / s1) * R * c1.T)),
+                         numpy.matrix([0., 0., 1.])])
+
+def read_im_and_landmarks(fname):
+    im = cv2.imread(fname, cv2.IMREAD_COLOR)
+    im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR,
+                         im.shape[0] * SCALE_FACTOR))
+    s = get_landmarks(im)
+
+    return im, s
+
+def warp_im(im, M, dshape):
+    output_im = numpy.zeros(dshape, dtype=im.dtype)
+    cv2.warpAffine(im,
+                   M[:2],
+                   (dshape[1], dshape[0]),
+                   dst=output_im,
+                   borderMode=cv2.BORDER_TRANSPARENT,
+                   flags=cv2.WARP_INVERSE_MAP)
+    return output_im
+
+def correct_colours(im1, im2, mask):
+    im1_blur = cv2.GaussianBlur(im1, (61, 61), 0)
+    im2_blur = cv2.GaussianBlur(im2, (61, 61), 0)
+
+    return (im2.astype(numpy.float64) * im1_blur.astype(numpy.float64) /
+                                                im2_blur.astype(numpy.float64))
+
+    #factor = (im1 * mask).mean(axis=(0, 1)) / (im2 * mask).mean(axis=(0, 1))
+    #return im2 * factor
+
+im1, shape1 = read_im_and_landmarks(sys.argv[1])
+im2, shape2 = read_im_and_landmarks(sys.argv[2])
+
+#im1 = annotate_landmarks(im1, shape1)
+#im2 = annotate_landmarks(im2, shape2)
+
+M = transformation_from_points(shape1[FACE_POINTS],
+                               shape2[FACE_POINTS])
+
+mask = get_face_mask(im2, shape2)
+warped_mask = warp_im(mask, M, im1.shape)
+combined_mask = numpy.max([get_face_mask(im1, shape1), warped_mask], axis=0)
+
+warped_im2 = warp_im(im2, M, im1.shape)
+warped_corrected_im2 = correct_colours(im1, warped_im2, combined_mask)
+
+output_im = (im1.astype(numpy.float64) * (1.0 - combined_mask) +
+             warped_corrected_im2.astype(numpy.float64) * combined_mask)
+
+cv2.imwrite('output.jpg', output_im)
+