initial
This commit is contained in:
iperov
Binary file not shown.
@@ -0,0 +1,40 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import cv2
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
class DLIBExtractor(object):
|
||||
def __init__(self, dlib):
|
||||
self.scale_to = 1850
|
||||
#3100 eats ~1.687GB VRAM on 2GB 730 desktop card, but >4Gb on 6GB card,
|
||||
#but 3100 doesnt work on 2GB 850M notebook card, I cant understand this behaviour
|
||||
#1850 works on 2GB 850M notebook card, works faster than 3100, produces good result
|
||||
self.dlib = dlib
|
||||
|
||||
def __enter__(self):
|
||||
self.dlib_cnn_face_detector = self.dlib.cnn_face_detection_model_v1( str(Path(__file__).parent / "mmod_human_face_detector.dat") )
|
||||
self.dlib_cnn_face_detector ( np.zeros ( (self.scale_to, self.scale_to, 3), dtype=np.uint8), 0 )
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
|
||||
del self.dlib_cnn_face_detector
|
||||
return False #pass exception between __enter__ and __exit__ to outter level
|
||||
|
||||
def extract_from_bgr (self, input_image):
|
||||
input_image = input_image[:,:,::-1].copy()
|
||||
(h, w, ch) = input_image.shape
|
||||
|
||||
detected_faces = []
|
||||
input_scale = self.scale_to / (w if w > h else h)
|
||||
input_image = cv2.resize (input_image, ( int(w*input_scale), int(h*input_scale) ), interpolation=cv2.INTER_LINEAR)
|
||||
detected_faces = self.dlib_cnn_face_detector(input_image, 0)
|
||||
|
||||
result = []
|
||||
for d_rect in detected_faces:
|
||||
if type(d_rect) == self.dlib.mmod_rectangle:
|
||||
d_rect = d_rect.rect
|
||||
left, top, right, bottom = d_rect.left(), d_rect.top(), d_rect.right(), d_rect.bottom()
|
||||
result.append ( (int(left/input_scale), int(top/input_scale), int(right/input_scale), int(bottom/input_scale)) )
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,34 @@
|
||||
from enum import IntEnum
|
||||
|
||||
class FaceType(IntEnum):
|
||||
HALF = 0,
|
||||
FULL = 1,
|
||||
HEAD = 2,
|
||||
AVATAR = 3, #centered nose only
|
||||
MARK_ONLY = 4, #no align at all, just embedded faceinfo
|
||||
QTY = 5
|
||||
|
||||
@staticmethod
|
||||
def fromString (s):
|
||||
r = from_string_dict.get (s.lower())
|
||||
if r is None:
|
||||
raise Exception ('FaceType.fromString value error')
|
||||
return r
|
||||
|
||||
@staticmethod
|
||||
def toString (face_type):
|
||||
return to_string_list[face_type]
|
||||
|
||||
from_string_dict = {'half_face': FaceType.HALF,
|
||||
'full_face': FaceType.FULL,
|
||||
'head' : FaceType.HEAD,
|
||||
'avatar' : FaceType.AVATAR,
|
||||
'mark_only' : FaceType.MARK_ONLY,
|
||||
}
|
||||
to_string_list = [ 'half_face',
|
||||
'full_face',
|
||||
'head',
|
||||
'avatar',
|
||||
'mark_only'
|
||||
]
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import cv2
|
||||
from pathlib import Path
|
||||
|
||||
from utils import std_utils
|
||||
|
||||
|
||||
|
||||
def transform(point, center, scale, resolution):
|
||||
pt = np.array ( [point[0], point[1], 1.0] )
|
||||
h = 200.0 * scale
|
||||
m = np.eye(3)
|
||||
m[0,0] = resolution / h
|
||||
m[1,1] = resolution / h
|
||||
m[0,2] = resolution * ( -center[0] / h + 0.5 )
|
||||
m[1,2] = resolution * ( -center[1] / h + 0.5 )
|
||||
m = np.linalg.inv(m)
|
||||
return np.matmul (m, pt)[0:2]
|
||||
|
||||
def crop(image, center, scale, resolution=256.0):
|
||||
ul = transform([1, 1], center, scale, resolution).astype( np.int )
|
||||
br = transform([resolution, resolution], center, scale, resolution).astype( np.int )
|
||||
if image.ndim > 2:
|
||||
newDim = np.array([br[1] - ul[1], br[0] - ul[0], image.shape[2]], dtype=np.int32)
|
||||
newImg = np.zeros(newDim, dtype=np.uint8)
|
||||
else:
|
||||
newDim = np.array([br[1] - ul[1], br[0] - ul[0]], dtype=np.int)
|
||||
newImg = np.zeros(newDim, dtype=np.uint8)
|
||||
ht = image.shape[0]
|
||||
wd = image.shape[1]
|
||||
newX = np.array([max(1, -ul[0] + 1), min(br[0], wd) - ul[0]], dtype=np.int32)
|
||||
newY = np.array([max(1, -ul[1] + 1), min(br[1], ht) - ul[1]], dtype=np.int32)
|
||||
oldX = np.array([max(1, ul[0] + 1), min(br[0], wd)], dtype=np.int32)
|
||||
oldY = np.array([max(1, ul[1] + 1), min(br[1], ht)], dtype=np.int32)
|
||||
newImg[newY[0] - 1:newY[1], newX[0] - 1:newX[1] ] = image[oldY[0] - 1:oldY[1], oldX[0] - 1:oldX[1], :]
|
||||
newImg = cv2.resize(newImg, dsize=(int(resolution), int(resolution)), interpolation=cv2.INTER_LINEAR)
|
||||
return newImg
|
||||
|
||||
def get_pts_from_predict(a, center, scale):
|
||||
b = a.reshape ( (a.shape[0], a.shape[1]*a.shape[2]) )
|
||||
c = b.argmax(1).reshape ( (a.shape[0], 1) ).repeat(2, axis=1).astype(np.float)
|
||||
c[:,0] %= a.shape[2]
|
||||
c[:,1] = np.apply_along_axis ( lambda x: np.floor(x / a.shape[2]), 0, c[:,1] )
|
||||
|
||||
for i in range(a.shape[0]):
|
||||
pX, pY = int(c[i,0]), int(c[i,1])
|
||||
if pX > 0 and pX < 63 and pY > 0 and pY < 63:
|
||||
diff = np.array ( [a[i,pY,pX+1]-a[i,pY,pX-1], a[i,pY+1,pX]-a[i,pY-1,pX]] )
|
||||
c[i] += np.sign(diff)*0.25
|
||||
|
||||
c += 0.5
|
||||
return [ transform (c[i], center, scale, a.shape[2]) for i in range(a.shape[0]) ]
|
||||
|
||||
|
||||
class LandmarksExtractor(object):
|
||||
def __init__ (self, keras):
|
||||
self.keras = keras
|
||||
K = self.keras.backend
|
||||
class TorchBatchNorm2D(self.keras.engine.topology.Layer):
|
||||
def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, **kwargs):
|
||||
super(TorchBatchNorm2D, self).__init__(**kwargs)
|
||||
self.supports_masking = True
|
||||
self.axis = axis
|
||||
self.momentum = momentum
|
||||
self.epsilon = epsilon
|
||||
|
||||
def build(self, input_shape):
|
||||
dim = input_shape[self.axis]
|
||||
if dim is None:
|
||||
raise ValueError('Axis ' + str(self.axis) + ' of ' 'input tensor should have a defined dimension ' 'but the layer received an input with shape ' + str(input_shape) + '.')
|
||||
shape = (dim,)
|
||||
self.gamma = self.add_weight(shape=shape, name='gamma', initializer='ones', regularizer=None, constraint=None)
|
||||
self.beta = self.add_weight(shape=shape, name='beta', initializer='zeros', regularizer=None, constraint=None)
|
||||
self.moving_mean = self.add_weight(shape=shape, name='moving_mean', initializer='zeros', trainable=False)
|
||||
self.moving_variance = self.add_weight(shape=shape, name='moving_variance', initializer='ones', trainable=False)
|
||||
self.built = True
|
||||
|
||||
def call(self, inputs, training=None):
|
||||
input_shape = K.int_shape(inputs)
|
||||
|
||||
broadcast_shape = [1] * len(input_shape)
|
||||
broadcast_shape[self.axis] = input_shape[self.axis]
|
||||
|
||||
broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape)
|
||||
broadcast_moving_variance = K.reshape(self.moving_variance, broadcast_shape)
|
||||
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
|
||||
broadcast_beta = K.reshape(self.beta, broadcast_shape)
|
||||
invstd = K.ones (shape=broadcast_shape, dtype='float32') / K.sqrt(broadcast_moving_variance + K.constant(self.epsilon, dtype='float32'))
|
||||
|
||||
return (inputs - broadcast_moving_mean) * invstd * broadcast_gamma + broadcast_beta
|
||||
|
||||
def get_config(self):
|
||||
config = { 'axis': self.axis, 'momentum': self.momentum, 'epsilon': self.epsilon }
|
||||
base_config = super(TorchBatchNorm2D, self).get_config()
|
||||
return dict(list(base_config.items()) + list(config.items()))
|
||||
self.TorchBatchNorm2D = TorchBatchNorm2D
|
||||
|
||||
def __enter__(self):
|
||||
keras_model_path = Path(__file__).parent / "2DFAN-4.h5"
|
||||
if not keras_model_path.exists():
|
||||
return None
|
||||
|
||||
self.keras_model = self.keras.models.load_model ( str(keras_model_path), custom_objects={'TorchBatchNorm2D': self.TorchBatchNorm2D} )
|
||||
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
|
||||
del self.keras_model
|
||||
return False #pass exception between __enter__ and __exit__ to outter level
|
||||
|
||||
def extract_from_bgr (self, input_image, rects):
|
||||
input_image = input_image[:,:,::-1].copy()
|
||||
(h, w, ch) = input_image.shape
|
||||
|
||||
landmarks = []
|
||||
for (left, top, right, bottom) in rects:
|
||||
|
||||
center = np.array( [ (left + right) / 2.0, (top + bottom) / 2.0] )
|
||||
center[1] -= (bottom - top) * 0.12
|
||||
scale = (right - left + bottom - top) / 195.0
|
||||
|
||||
image = crop(input_image, center, scale).transpose ( (2,0,1) ).astype(np.float32) / 255.0
|
||||
image = np.expand_dims(image, 0)
|
||||
|
||||
with std_utils.suppress_stdout_stderr():
|
||||
predicted = self.keras_model.predict (image)
|
||||
|
||||
pts_img = get_pts_from_predict ( predicted[-1][0], center, scale)
|
||||
pts_img = [ ( int(pt[0]), int(pt[1]) ) for pt in pts_img ]
|
||||
landmarks.append ( ( (left, top, right, bottom),pts_img ) )
|
||||
|
||||
return landmarks
|
||||
@@ -0,0 +1,193 @@
|
||||
import colorsys
|
||||
import cv2
|
||||
import numpy as np
|
||||
from enum import IntEnum
|
||||
from mathlib.umeyama import umeyama
|
||||
from utils import image_utils
|
||||
from facelib import FaceType
|
||||
import math
|
||||
|
||||
mean_face_x = np.array([
|
||||
0.000213256, 0.0752622, 0.18113, 0.29077, 0.393397, 0.586856, 0.689483, 0.799124,
|
||||
0.904991, 0.98004, 0.490127, 0.490127, 0.490127, 0.490127, 0.36688, 0.426036,
|
||||
0.490127, 0.554217, 0.613373, 0.121737, 0.187122, 0.265825, 0.334606, 0.260918,
|
||||
0.182743, 0.645647, 0.714428, 0.793132, 0.858516, 0.79751, 0.719335, 0.254149,
|
||||
0.340985, 0.428858, 0.490127, 0.551395, 0.639268, 0.726104, 0.642159, 0.556721,
|
||||
0.490127, 0.423532, 0.338094, 0.290379, 0.428096, 0.490127, 0.552157, 0.689874,
|
||||
0.553364, 0.490127, 0.42689 ])
|
||||
|
||||
mean_face_y = np.array([
|
||||
0.106454, 0.038915, 0.0187482, 0.0344891, 0.0773906, 0.0773906, 0.0344891,
|
||||
0.0187482, 0.038915, 0.106454, 0.203352, 0.307009, 0.409805, 0.515625, 0.587326,
|
||||
0.609345, 0.628106, 0.609345, 0.587326, 0.216423, 0.178758, 0.179852, 0.231733,
|
||||
0.245099, 0.244077, 0.231733, 0.179852, 0.178758, 0.216423, 0.244077, 0.245099,
|
||||
0.780233, 0.745405, 0.727388, 0.742578, 0.727388, 0.745405, 0.780233, 0.864805,
|
||||
0.902192, 0.909281, 0.902192, 0.864805, 0.784792, 0.778746, 0.785343, 0.778746,
|
||||
0.784792, 0.824182, 0.831803, 0.824182 ])
|
||||
|
||||
landmarks_2D = np.stack( [ mean_face_x, mean_face_y ], axis=1 )
|
||||
|
||||
def get_transform_mat (image_landmarks, output_size, face_type):
|
||||
if not isinstance(image_landmarks, np.ndarray):
|
||||
image_landmarks = np.array (image_landmarks)
|
||||
|
||||
if face_type == FaceType.AVATAR:
|
||||
centroid = np.mean (image_landmarks, axis=0)
|
||||
|
||||
mat = umeyama(image_landmarks[17:], landmarks_2D, True)[0:2]
|
||||
a, c = mat[0,0], mat[1,0]
|
||||
scale = math.sqrt((a * a) + (c * c))
|
||||
|
||||
padding = (output_size / 64) * 32
|
||||
|
||||
mat = np.eye ( 2,3 )
|
||||
mat[0,2] = -centroid[0]
|
||||
mat[1,2] = -centroid[1]
|
||||
mat = mat * scale * (output_size / 3)
|
||||
mat[:,2] += output_size / 2
|
||||
else:
|
||||
if face_type == FaceType.HALF:
|
||||
padding = 0
|
||||
elif face_type == FaceType.FULL:
|
||||
padding = (output_size / 64) * 12
|
||||
elif face_type == FaceType.HEAD:
|
||||
padding = (output_size / 64) * 24
|
||||
else:
|
||||
raise ValueError ('wrong face_type')
|
||||
|
||||
mat = umeyama(image_landmarks[17:], landmarks_2D, True)[0:2]
|
||||
mat = mat * (output_size - 2 * padding)
|
||||
mat[:,2] += padding
|
||||
|
||||
return mat
|
||||
|
||||
def transform_points(points, mat, invert=False):
|
||||
if invert:
|
||||
mat = cv2.invertAffineTransform (mat)
|
||||
points = np.expand_dims(points, axis=1)
|
||||
points = cv2.transform(points, mat, points.shape)
|
||||
points = np.squeeze(points)
|
||||
return points
|
||||
|
||||
|
||||
def get_image_hull_mask (image, image_landmarks):
|
||||
if len(image_landmarks) != 68:
|
||||
raise Exception('get_image_hull_mask work only with 68 landmarks')
|
||||
|
||||
hull_mask = np.zeros(image.shape[0:2]+(1,),dtype=np.float32)
|
||||
|
||||
cv2.fillConvexPoly( hull_mask, cv2.convexHull( np.concatenate ( (image_landmarks[0:17], image_landmarks[48:], [image_landmarks[0]], [image_landmarks[8]], [image_landmarks[16]])) ), (1,) )
|
||||
cv2.fillConvexPoly( hull_mask, cv2.convexHull( np.concatenate ( (image_landmarks[27:31], [image_landmarks[33]]) ) ), (1,) )
|
||||
cv2.fillConvexPoly( hull_mask, cv2.convexHull( np.concatenate ( (image_landmarks[17:27], [image_landmarks[0]], [image_landmarks[27]], [image_landmarks[16]], [image_landmarks[33]])) ), (1,) )
|
||||
|
||||
return hull_mask
|
||||
|
||||
def get_image_eye_mask (image, image_landmarks):
|
||||
if len(image_landmarks) != 68:
|
||||
raise Exception('get_image_eye_mask work only with 68 landmarks')
|
||||
|
||||
hull_mask = np.zeros(image.shape[0:2]+(1,),dtype=np.float32)
|
||||
|
||||
cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[36:42]), (1,) )
|
||||
cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[42:48]), (1,) )
|
||||
|
||||
return hull_mask
|
||||
|
||||
def get_image_hull_mask_3D (image, image_landmarks):
|
||||
result = get_image_hull_mask(image, image_landmarks)
|
||||
|
||||
return np.repeat ( result, (3,), -1 )
|
||||
|
||||
def blur_image_hull_mask (hull_mask):
|
||||
|
||||
maxregion = np.argwhere(hull_mask==1.0)
|
||||
miny,minx = maxregion.min(axis=0)[:2]
|
||||
maxy,maxx = maxregion.max(axis=0)[:2]
|
||||
lenx = maxx - minx;
|
||||
leny = maxy - miny;
|
||||
masky = int(minx+(lenx//2))
|
||||
maskx = int(miny+(leny//2))
|
||||
lowest_len = min (lenx, leny)
|
||||
ero = int( lowest_len * 0.085 )
|
||||
blur = int( lowest_len * 0.10 )
|
||||
|
||||
hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 )
|
||||
hull_mask = cv2.blur(hull_mask, (blur, blur) )
|
||||
hull_mask = np.expand_dims (hull_mask,-1)
|
||||
|
||||
return hull_mask
|
||||
|
||||
def get_blurred_image_hull_mask(image, image_landmarks):
|
||||
return blur_image_hull_mask ( get_image_hull_mask(image, image_landmarks) )
|
||||
|
||||
mirror_idxs = [
|
||||
[0,16],
|
||||
[1,15],
|
||||
[2,14],
|
||||
[3,13],
|
||||
[4,12],
|
||||
[5,11],
|
||||
[6,10],
|
||||
[7,9],
|
||||
|
||||
[17,26],
|
||||
[18,25],
|
||||
[19,24],
|
||||
[20,23],
|
||||
[21,22],
|
||||
|
||||
[36,45],
|
||||
[37,44],
|
||||
[38,43],
|
||||
[39,42],
|
||||
[40,47],
|
||||
[41,46],
|
||||
|
||||
[31,35],
|
||||
[32,34],
|
||||
|
||||
[50,52],
|
||||
[49,53],
|
||||
[48,54],
|
||||
[59,55],
|
||||
[58,56],
|
||||
[67,65],
|
||||
[60,64],
|
||||
[61,63] ]
|
||||
|
||||
def mirror_landmarks (landmarks, val):
|
||||
result = landmarks.copy()
|
||||
|
||||
for idx in mirror_idxs:
|
||||
result [ idx ] = result [ idx[::-1] ]
|
||||
|
||||
result[:,0] = val - result[:,0] - 1
|
||||
return result
|
||||
|
||||
def draw_landmarks (image, image_landmarks, color):
|
||||
for i, (x, y) in enumerate(image_landmarks):
|
||||
cv2.circle(image, (x, y), 2, color, -1)
|
||||
#text_color = colorsys.hsv_to_rgb ( (i%4) * (0.25), 1.0, 1.0 )
|
||||
#cv2.putText(image, str(i), (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.1,text_color,1)
|
||||
|
||||
def draw_rect_landmarks (image, rect, image_landmarks, face_size, face_type):
|
||||
image_utils.draw_rect (image, rect, (255,0,0), 2 )
|
||||
draw_landmarks(image, image_landmarks, (0,255,0) )
|
||||
|
||||
image_to_face_mat = get_transform_mat (image_landmarks, face_size, face_type)
|
||||
points = transform_points ( [ (0,0), (0,face_size-1), (face_size-1, face_size-1), (face_size-1,0) ], image_to_face_mat, True)
|
||||
image_utils.draw_polygon (image, points, (0,0,255), 2)
|
||||
|
||||
def calc_face_pitch(landmarks):
|
||||
if not isinstance(landmarks, np.ndarray):
|
||||
landmarks = np.array (landmarks)
|
||||
t = ( (landmarks[6][1]-landmarks[8][1]) + (landmarks[10][1]-landmarks[8][1]) ) / 2.0
|
||||
b = landmarks[8][1]
|
||||
return float(b-t)
|
||||
def calc_face_yaw(landmarks):
|
||||
if not isinstance(landmarks, np.ndarray):
|
||||
landmarks = np.array (landmarks)
|
||||
l = ( (landmarks[27][0]-landmarks[0][0]) + (landmarks[28][0]-landmarks[1][0]) + (landmarks[29][0]-landmarks[2][0]) ) / 3.0
|
||||
r = ( (landmarks[16][0]-landmarks[27][0]) + (landmarks[15][0]-landmarks[28][0]) + (landmarks[14][0]-landmarks[29][0]) ) / 3.0
|
||||
return float(r-l)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import cv2
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .mtcnn import *
|
||||
|
||||
class MTCExtractor(object):
|
||||
def __init__(self, keras, tf, tf_session):
|
||||
self.scale_to = 1920
|
||||
self.keras = keras
|
||||
self.tf = tf
|
||||
self.tf_session = tf_session
|
||||
|
||||
self.min_face_size = self.scale_to * 0.042
|
||||
self.thresh1 = 0.7
|
||||
self.thresh2 = 0.85
|
||||
self.thresh3 = 0.6
|
||||
self.scale_factor = 0.95
|
||||
|
||||
'''
|
||||
self.min_face_size = self.scale_to * 0.042
|
||||
self.thresh1 = 7
|
||||
self.thresh2 = 85
|
||||
self.thresh3 = 6
|
||||
self.scale_factor = 0.95
|
||||
'''
|
||||
|
||||
def __enter__(self):
|
||||
with self.tf.variable_scope('pnet2'):
|
||||
data = self.tf.placeholder(self.tf.float32, (None,None,None,3), 'input')
|
||||
pnet2 = PNet(self.tf, {'data':data})
|
||||
pnet2.load(str(Path(__file__).parent/'det1.npy'), self.tf_session)
|
||||
with self.tf.variable_scope('rnet2'):
|
||||
data = self.tf.placeholder(self.tf.float32, (None,24,24,3), 'input')
|
||||
rnet2 = RNet(self.tf, {'data':data})
|
||||
rnet2.load(str(Path(__file__).parent/'det2.npy'), self.tf_session)
|
||||
with self.tf.variable_scope('onet2'):
|
||||
data = self.tf.placeholder(self.tf.float32, (None,48,48,3), 'input')
|
||||
onet2 = ONet(self.tf, {'data':data})
|
||||
onet2.load(str(Path(__file__).parent/'det3.npy'), self.tf_session)
|
||||
|
||||
self.pnet_fun = self.keras.backend.function([pnet2.layers['data']],[pnet2.layers['conv4-2'], pnet2.layers['prob1']])
|
||||
self.rnet_fun = self.keras.backend.function([rnet2.layers['data']],[rnet2.layers['conv5-2'], rnet2.layers['prob1']])
|
||||
self.onet_fun = self.keras.backend.function([onet2.layers['data']],[onet2.layers['conv6-2'], onet2.layers['conv6-3'], onet2.layers['prob1']])
|
||||
|
||||
faces, pnts = detect_face ( np.zeros ( (self.scale_to, self.scale_to, 3)), self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
|
||||
return False #pass exception between __enter__ and __exit__ to outter level
|
||||
|
||||
def extract_from_bgr (self, input_image):
|
||||
input_image = input_image[:,:,::-1].copy()
|
||||
(h, w, ch) = input_image.shape
|
||||
|
||||
|
||||
input_scale = self.scale_to / (w if w > h else h)
|
||||
input_image = cv2.resize (input_image, ( int(w*input_scale), int(h*input_scale) ), interpolation=cv2.INTER_LINEAR)
|
||||
|
||||
detected_faces, pnts = detect_face ( input_image, self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
|
||||
detected_faces = [ ( int(face[0]/input_scale), int(face[1]/input_scale), int(face[2]/input_scale), int(face[3]/input_scale)) for face in detected_faces ]
|
||||
|
||||
return detected_faces
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
from .FaceType import FaceType
|
||||
from .DLIBExtractor import DLIBExtractor
|
||||
from .MTCExtractor import MTCExtractor
|
||||
from .LandmarksExtractor import LandmarksExtractor
|
||||
from .LandmarksProcessor import *
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,761 @@
|
||||
# Source: https://github.com/davidsandberg/facenet/blob/master/src/align/
|
||||
|
||||
""" Tensorflow implementation of the face detection / alignment algorithm found at
|
||||
https://github.com/kpzhang93/MTCNN_face_detection_alignment
|
||||
"""
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2016 David Sandberg
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from six import string_types, iteritems
|
||||
|
||||
import numpy as np
|
||||
#from math import floor
|
||||
import cv2
|
||||
import os
|
||||
|
||||
def layer(op):
|
||||
"""Decorator for composable network layers."""
|
||||
|
||||
def layer_decorated(self, *args, **kwargs):
|
||||
# Automatically set a name if not provided.
|
||||
name = kwargs.setdefault('name', self.get_unique_name(op.__name__))
|
||||
# Figure out the layer inputs.
|
||||
if len(self.terminals) == 0:
|
||||
raise RuntimeError('No input variables found for layer %s.' % name)
|
||||
elif len(self.terminals) == 1:
|
||||
layer_input = self.terminals[0]
|
||||
else:
|
||||
layer_input = list(self.terminals)
|
||||
# Perform the operation and get the output.
|
||||
layer_output = op(self, layer_input, *args, **kwargs)
|
||||
# Add to layer LUT.
|
||||
self.layers[name] = layer_output
|
||||
# This output is now the input for the next layer.
|
||||
self.feed(layer_output)
|
||||
# Return self for chained calls.
|
||||
return self
|
||||
|
||||
return layer_decorated
|
||||
|
||||
class Network(object):
|
||||
|
||||
def __init__(self, tf, inputs, trainable=True):
|
||||
# The input nodes for this network
|
||||
self.tf = tf
|
||||
self.inputs = inputs
|
||||
# The current list of terminal nodes
|
||||
self.terminals = []
|
||||
# Mapping from layer names to layers
|
||||
self.layers = dict(inputs)
|
||||
# If true, the resulting variables are set as trainable
|
||||
self.trainable = trainable
|
||||
|
||||
self.setup()
|
||||
|
||||
def setup(self):
|
||||
"""Construct the network. """
|
||||
raise NotImplementedError('Must be implemented by the subclass.')
|
||||
|
||||
def load(self, data_path, session, ignore_missing=False):
|
||||
"""Load network weights.
|
||||
data_path: The path to the numpy-serialized network weights
|
||||
session: The current TensorFlow session
|
||||
ignore_missing: If true, serialized weights for missing layers are ignored.
|
||||
"""
|
||||
data_dict = np.load(data_path, encoding='latin1').item() #pylint: disable=no-member
|
||||
|
||||
for op_name in data_dict:
|
||||
with self.tf.variable_scope(op_name, reuse=True):
|
||||
for param_name, data in iteritems(data_dict[op_name]):
|
||||
try:
|
||||
var = self.tf.get_variable(param_name)
|
||||
session.run(var.assign(data))
|
||||
except ValueError:
|
||||
if not ignore_missing:
|
||||
raise
|
||||
|
||||
def feed(self, *args):
|
||||
"""Set the input(s) for the next operation by replacing the terminal nodes.
|
||||
The arguments can be either layer names or the actual layers.
|
||||
"""
|
||||
assert len(args) != 0
|
||||
self.terminals = []
|
||||
for fed_layer in args:
|
||||
if isinstance(fed_layer, string_types):
|
||||
try:
|
||||
fed_layer = self.layers[fed_layer]
|
||||
except KeyError:
|
||||
raise KeyError('Unknown layer name fed: %s' % fed_layer)
|
||||
self.terminals.append(fed_layer)
|
||||
return self
|
||||
|
||||
def get_output(self):
|
||||
"""Returns the current network output."""
|
||||
return self.terminals[-1]
|
||||
|
||||
def get_unique_name(self, prefix):
|
||||
"""Returns an index-suffixed unique name for the given prefix.
|
||||
This is used for auto-generating layer names based on the type-prefix.
|
||||
"""
|
||||
ident = sum(t.startswith(prefix) for t, _ in self.layers.items()) + 1
|
||||
return '%s_%d' % (prefix, ident)
|
||||
|
||||
def make_var(self, name, shape):
|
||||
"""Creates a new TensorFlow variable."""
|
||||
return self.tf.get_variable(name, shape, trainable=self.trainable)
|
||||
|
||||
def validate_padding(self, padding):
|
||||
"""Verifies that the padding is one of the supported ones."""
|
||||
assert padding in ('SAME', 'VALID')
|
||||
|
||||
@layer
|
||||
def conv(self,
|
||||
inp,
|
||||
k_h,
|
||||
k_w,
|
||||
c_o,
|
||||
s_h,
|
||||
s_w,
|
||||
name,
|
||||
relu=True,
|
||||
padding='SAME',
|
||||
group=1,
|
||||
biased=True):
|
||||
# Verify that the padding is acceptable
|
||||
self.validate_padding(padding)
|
||||
# Get the number of channels in the input
|
||||
c_i = int(inp.get_shape()[-1])
|
||||
# Verify that the grouping parameter is valid
|
||||
assert c_i % group == 0
|
||||
assert c_o % group == 0
|
||||
# Convolution for a given input and kernel
|
||||
convolve = lambda i, k: self.tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
|
||||
with self.tf.variable_scope(name) as scope:
|
||||
kernel = self.make_var('weights', shape=[k_h, k_w, c_i // group, c_o])
|
||||
# This is the common-case. Convolve the input without any further complications.
|
||||
output = convolve(inp, kernel)
|
||||
# Add the biases
|
||||
if biased:
|
||||
biases = self.make_var('biases', [c_o])
|
||||
output = self.tf.nn.bias_add(output, biases)
|
||||
if relu:
|
||||
# ReLU non-linearity
|
||||
output = self.tf.nn.relu(output, name=scope.name)
|
||||
return output
|
||||
|
||||
@layer
|
||||
def prelu(self, inp, name):
|
||||
with self.tf.variable_scope(name):
|
||||
i = int(inp.get_shape()[-1])
|
||||
alpha = self.make_var('alpha', shape=(i,))
|
||||
output = self.tf.nn.relu(inp) + self.tf.multiply(alpha, -self.tf.nn.relu(-inp))
|
||||
return output
|
||||
|
||||
@layer
|
||||
def max_pool(self, inp, k_h, k_w, s_h, s_w, name, padding='SAME'):
|
||||
self.validate_padding(padding)
|
||||
return self.tf.nn.max_pool(inp,
|
||||
ksize=[1, k_h, k_w, 1],
|
||||
strides=[1, s_h, s_w, 1],
|
||||
padding=padding,
|
||||
name=name)
|
||||
|
||||
@layer
|
||||
def fc(self, inp, num_out, name, relu=True):
|
||||
with self.tf.variable_scope(name):
|
||||
input_shape = inp.get_shape()
|
||||
if input_shape.ndims == 4:
|
||||
# The input is spatial. Vectorize it first.
|
||||
dim = 1
|
||||
for d in input_shape[1:].as_list():
|
||||
dim *= int(d)
|
||||
feed_in = self.tf.reshape(inp, [-1, dim])
|
||||
else:
|
||||
feed_in, dim = (inp, input_shape[-1].value)
|
||||
weights = self.make_var('weights', shape=[dim, num_out])
|
||||
biases = self.make_var('biases', [num_out])
|
||||
op = self.tf.nn.relu_layer if relu else self.tf.nn.xw_plus_b
|
||||
fc = op(feed_in, weights, biases, name=name)
|
||||
return fc
|
||||
|
||||
|
||||
"""
|
||||
Multi dimensional softmax,
|
||||
refer to https://github.com/tensorflow/tensorflow/issues/210
|
||||
compute softmax along the dimension of target
|
||||
the native softmax only supports batch_size x dimension
|
||||
"""
|
||||
@layer
|
||||
def softmax(self, target, axis, name=None):
|
||||
max_axis = self.tf.reduce_max(target, axis, keepdims=True)
|
||||
target_exp = self.tf.exp(target-max_axis)
|
||||
normalize = self.tf.reduce_sum(target_exp, axis, keepdims=True)
|
||||
softmax = self.tf.div(target_exp, normalize, name)
|
||||
return softmax
|
||||
|
||||
class PNet(Network):
|
||||
def setup(self):
|
||||
(self.feed('data') #pylint: disable=no-value-for-parameter, no-member
|
||||
.conv(3, 3, 10, 1, 1, padding='VALID', relu=False, name='conv1')
|
||||
.prelu(name='PReLU1')
|
||||
.max_pool(2, 2, 2, 2, name='pool1')
|
||||
.conv(3, 3, 16, 1, 1, padding='VALID', relu=False, name='conv2')
|
||||
.prelu(name='PReLU2')
|
||||
.conv(3, 3, 32, 1, 1, padding='VALID', relu=False, name='conv3')
|
||||
.prelu(name='PReLU3')
|
||||
.conv(1, 1, 2, 1, 1, relu=False, name='conv4-1')
|
||||
.softmax(3,name='prob1'))
|
||||
|
||||
(self.feed('PReLU3') #pylint: disable=no-value-for-parameter
|
||||
.conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
|
||||
|
||||
class RNet(Network):
|
||||
def setup(self):
|
||||
(self.feed('data') #pylint: disable=no-value-for-parameter, no-member
|
||||
.conv(3, 3, 28, 1, 1, padding='VALID', relu=False, name='conv1')
|
||||
.prelu(name='prelu1')
|
||||
.max_pool(3, 3, 2, 2, name='pool1')
|
||||
.conv(3, 3, 48, 1, 1, padding='VALID', relu=False, name='conv2')
|
||||
.prelu(name='prelu2')
|
||||
.max_pool(3, 3, 2, 2, padding='VALID', name='pool2')
|
||||
.conv(2, 2, 64, 1, 1, padding='VALID', relu=False, name='conv3')
|
||||
.prelu(name='prelu3')
|
||||
.fc(128, relu=False, name='conv4')
|
||||
.prelu(name='prelu4')
|
||||
.fc(2, relu=False, name='conv5-1')
|
||||
.softmax(1,name='prob1'))
|
||||
|
||||
(self.feed('prelu4') #pylint: disable=no-value-for-parameter
|
||||
.fc(4, relu=False, name='conv5-2'))
|
||||
|
||||
class ONet(Network):
|
||||
def setup(self):
|
||||
(self.feed('data') #pylint: disable=no-value-for-parameter, no-member
|
||||
.conv(3, 3, 32, 1, 1, padding='VALID', relu=False, name='conv1')
|
||||
.prelu(name='prelu1')
|
||||
.max_pool(3, 3, 2, 2, name='pool1')
|
||||
.conv(3, 3, 64, 1, 1, padding='VALID', relu=False, name='conv2')
|
||||
.prelu(name='prelu2')
|
||||
.max_pool(3, 3, 2, 2, padding='VALID', name='pool2')
|
||||
.conv(3, 3, 64, 1, 1, padding='VALID', relu=False, name='conv3')
|
||||
.prelu(name='prelu3')
|
||||
.max_pool(2, 2, 2, 2, name='pool3')
|
||||
.conv(2, 2, 128, 1, 1, padding='VALID', relu=False, name='conv4')
|
||||
.prelu(name='prelu4')
|
||||
.fc(256, relu=False, name='conv5')
|
||||
.prelu(name='prelu5')
|
||||
.fc(2, relu=False, name='conv6-1')
|
||||
.softmax(1, name='prob1'))
|
||||
|
||||
(self.feed('prelu5') #pylint: disable=no-value-for-parameter
|
||||
.fc(4, relu=False, name='conv6-2'))
|
||||
|
||||
(self.feed('prelu5') #pylint: disable=no-value-for-parameter
|
||||
.fc(10, relu=False, name='conv6-3'))
|
||||
|
||||
def detect_face(img, minsize, pnet, rnet, onet, threshold, factor):
|
||||
"""Detects faces in an image, and returns bounding boxes and points for them.
|
||||
img: input image
|
||||
minsize: minimum faces' size
|
||||
pnet, rnet, onet: caffemodel
|
||||
threshold: threshold=[th1, th2, th3], th1-3 are three steps's threshold
|
||||
factor: the factor used to create a scaling pyramid of face sizes to detect in the image.
|
||||
"""
|
||||
factor_count=0
|
||||
total_boxes=np.empty((0,9))
|
||||
points=np.empty(0)
|
||||
h=img.shape[0]
|
||||
w=img.shape[1]
|
||||
minl=np.amin([h, w])
|
||||
m=12.0/minsize
|
||||
minl=minl*m
|
||||
# create scale pyramid
|
||||
scales=[]
|
||||
while minl>=12:
|
||||
scales += [m*np.power(factor, factor_count)]
|
||||
minl = minl*factor
|
||||
factor_count += 1
|
||||
# first stage
|
||||
for scale in scales:
|
||||
hs=int(np.ceil(h*scale))
|
||||
ws=int(np.ceil(w*scale))
|
||||
#print ('scale %f %d %d' % (scale, ws,hs))
|
||||
im_data = imresample(img, (hs, ws))
|
||||
im_data = (im_data-127.5)*0.0078125
|
||||
img_x = np.expand_dims(im_data, 0)
|
||||
img_y = np.transpose(img_x, (0,2,1,3))
|
||||
out = pnet([img_y])
|
||||
out0 = np.transpose(out[0], (0,2,1,3))
|
||||
out1 = np.transpose(out[1], (0,2,1,3))
|
||||
|
||||
boxes, _ = generateBoundingBox(out1[0,:,:,1].copy(), out0[0,:,:,:].copy(), scale, threshold[0])
|
||||
|
||||
# inter-scale nms
|
||||
pick = nms(boxes.copy(), 0.5, 'Union')
|
||||
if boxes.size>0 and pick.size>0:
|
||||
boxes = boxes[pick,:]
|
||||
total_boxes = np.append(total_boxes, boxes, axis=0)
|
||||
|
||||
numbox = total_boxes.shape[0]
|
||||
if numbox>0:
|
||||
pick = nms(total_boxes.copy(), 0.7, 'Union')
|
||||
total_boxes = total_boxes[pick,:]
|
||||
regw = total_boxes[:,2]-total_boxes[:,0]
|
||||
regh = total_boxes[:,3]-total_boxes[:,1]
|
||||
qq1 = total_boxes[:,0]+total_boxes[:,5]*regw
|
||||
qq2 = total_boxes[:,1]+total_boxes[:,6]*regh
|
||||
qq3 = total_boxes[:,2]+total_boxes[:,7]*regw
|
||||
qq4 = total_boxes[:,3]+total_boxes[:,8]*regh
|
||||
total_boxes = np.transpose(np.vstack([qq1, qq2, qq3, qq4, total_boxes[:,4]]))
|
||||
total_boxes = rerec(total_boxes.copy())
|
||||
total_boxes[:,0:4] = np.fix(total_boxes[:,0:4]).astype(np.int32)
|
||||
dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph = pad(total_boxes.copy(), w, h)
|
||||
|
||||
numbox = total_boxes.shape[0]
|
||||
if numbox>0:
|
||||
# second stage
|
||||
tempimg = np.zeros((24,24,3,numbox))
|
||||
for k in range(0,numbox):
|
||||
tmp = np.zeros((int(tmph[k]),int(tmpw[k]),3))
|
||||
tmp[dy[k]-1:edy[k],dx[k]-1:edx[k],:] = img[y[k]-1:ey[k],x[k]-1:ex[k],:]
|
||||
if tmp.shape[0]>0 and tmp.shape[1]>0 or tmp.shape[0]==0 and tmp.shape[1]==0:
|
||||
tempimg[:,:,:,k] = imresample(tmp, (24, 24))
|
||||
else:
|
||||
return np.empty()
|
||||
tempimg = (tempimg-127.5)*0.0078125
|
||||
tempimg1 = np.transpose(tempimg, (3,1,0,2))
|
||||
out = rnet([tempimg1])
|
||||
out0 = np.transpose(out[0])
|
||||
out1 = np.transpose(out[1])
|
||||
score = out1[1,:]
|
||||
ipass = np.where(score>threshold[1])
|
||||
total_boxes = np.hstack([total_boxes[ipass[0],0:4].copy(), np.expand_dims(score[ipass].copy(),1)])
|
||||
mv = out0[:,ipass[0]]
|
||||
if total_boxes.shape[0]>0:
|
||||
pick = nms(total_boxes, 0.7, 'Union')
|
||||
total_boxes = total_boxes[pick,:]
|
||||
total_boxes = bbreg(total_boxes.copy(), np.transpose(mv[:,pick]))
|
||||
total_boxes = rerec(total_boxes.copy())
|
||||
|
||||
numbox = total_boxes.shape[0]
|
||||
if numbox>0:
|
||||
# third stage
|
||||
total_boxes = np.fix(total_boxes).astype(np.int32)
|
||||
dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph = pad(total_boxes.copy(), w, h)
|
||||
tempimg = np.zeros((48,48,3,numbox))
|
||||
for k in range(0,numbox):
|
||||
tmp = np.zeros((int(tmph[k]),int(tmpw[k]),3))
|
||||
tmp[dy[k]-1:edy[k],dx[k]-1:edx[k],:] = img[y[k]-1:ey[k],x[k]-1:ex[k],:]
|
||||
if tmp.shape[0]>0 and tmp.shape[1]>0 or tmp.shape[0]==0 and tmp.shape[1]==0:
|
||||
tempimg[:,:,:,k] = imresample(tmp, (48, 48))
|
||||
else:
|
||||
return np.empty()
|
||||
tempimg = (tempimg-127.5)*0.0078125
|
||||
tempimg1 = np.transpose(tempimg, (3,1,0,2))
|
||||
out = onet([tempimg1])
|
||||
out0 = np.transpose(out[0])
|
||||
out1 = np.transpose(out[1])
|
||||
out2 = np.transpose(out[2])
|
||||
score = out2[1,:]
|
||||
points = out1
|
||||
ipass = np.where(score>threshold[2])
|
||||
points = points[:,ipass[0]]
|
||||
total_boxes = np.hstack([total_boxes[ipass[0],0:4].copy(), np.expand_dims(score[ipass].copy(),1)])
|
||||
mv = out0[:,ipass[0]]
|
||||
|
||||
w = total_boxes[:,2]-total_boxes[:,0]+1
|
||||
h = total_boxes[:,3]-total_boxes[:,1]+1
|
||||
points[0:5,:] = np.tile(w,(5, 1))*points[0:5,:] + np.tile(total_boxes[:,0],(5, 1))-1
|
||||
points[5:10,:] = np.tile(h,(5, 1))*points[5:10,:] + np.tile(total_boxes[:,1],(5, 1))-1
|
||||
if total_boxes.shape[0]>0:
|
||||
total_boxes = bbreg(total_boxes.copy(), np.transpose(mv))
|
||||
pick = nms(total_boxes.copy(), 0.7, 'Min')
|
||||
total_boxes = total_boxes[pick,:]
|
||||
points = points[:,pick]
|
||||
|
||||
return total_boxes, points
|
||||
|
||||
|
||||
def bulk_detect_face(images, detection_window_size_ratio, pnet, rnet, onet, threshold, factor):
|
||||
"""Detects faces in a list of images
|
||||
images: list containing input images
|
||||
detection_window_size_ratio: ratio of minimum face size to smallest image dimension
|
||||
pnet, rnet, onet: caffemodel
|
||||
threshold: threshold=[th1 th2 th3], th1-3 are three steps's threshold [0-1]
|
||||
factor: the factor used to create a scaling pyramid of face sizes to detect in the image.
|
||||
"""
|
||||
all_scales = [None] * len(images)
|
||||
images_with_boxes = [None] * len(images)
|
||||
|
||||
for i in range(len(images)):
|
||||
images_with_boxes[i] = {'total_boxes': np.empty((0, 9))}
|
||||
|
||||
# create scale pyramid
|
||||
for index, img in enumerate(images):
|
||||
all_scales[index] = []
|
||||
h = img.shape[0]
|
||||
w = img.shape[1]
|
||||
minsize = int(detection_window_size_ratio * np.minimum(w, h))
|
||||
factor_count = 0
|
||||
minl = np.amin([h, w])
|
||||
if minsize <= 12:
|
||||
minsize = 12
|
||||
|
||||
m = 12.0 / minsize
|
||||
minl = minl * m
|
||||
while minl >= 12:
|
||||
all_scales[index].append(m * np.power(factor, factor_count))
|
||||
minl = minl * factor
|
||||
factor_count += 1
|
||||
|
||||
# # # # # # # # # # # # #
|
||||
# first stage - fast proposal network (pnet) to obtain face candidates
|
||||
# # # # # # # # # # # # #
|
||||
|
||||
images_obj_per_resolution = {}
|
||||
|
||||
# TODO: use some type of rounding to number module 8 to increase probability that pyramid images will have the same resolution across input images
|
||||
|
||||
for index, scales in enumerate(all_scales):
|
||||
h = images[index].shape[0]
|
||||
w = images[index].shape[1]
|
||||
|
||||
for scale in scales:
|
||||
hs = int(np.ceil(h * scale))
|
||||
ws = int(np.ceil(w * scale))
|
||||
|
||||
if (ws, hs) not in images_obj_per_resolution:
|
||||
images_obj_per_resolution[(ws, hs)] = []
|
||||
|
||||
im_data = imresample(images[index], (hs, ws))
|
||||
im_data = (im_data - 127.5) * 0.0078125
|
||||
img_y = np.transpose(im_data, (1, 0, 2)) # caffe uses different dimensions ordering
|
||||
images_obj_per_resolution[(ws, hs)].append({'scale': scale, 'image': img_y, 'index': index})
|
||||
|
||||
for resolution in images_obj_per_resolution:
|
||||
images_per_resolution = [i['image'] for i in images_obj_per_resolution[resolution]]
|
||||
outs = pnet(images_per_resolution)
|
||||
|
||||
for index in range(len(outs[0])):
|
||||
scale = images_obj_per_resolution[resolution][index]['scale']
|
||||
image_index = images_obj_per_resolution[resolution][index]['index']
|
||||
out0 = np.transpose(outs[0][index], (1, 0, 2))
|
||||
out1 = np.transpose(outs[1][index], (1, 0, 2))
|
||||
|
||||
boxes, _ = generateBoundingBox(out1[:, :, 1].copy(), out0[:, :, :].copy(), scale, threshold[0])
|
||||
|
||||
# inter-scale nms
|
||||
pick = nms(boxes.copy(), 0.5, 'Union')
|
||||
if boxes.size > 0 and pick.size > 0:
|
||||
boxes = boxes[pick, :]
|
||||
images_with_boxes[image_index]['total_boxes'] = np.append(images_with_boxes[image_index]['total_boxes'],
|
||||
boxes,
|
||||
axis=0)
|
||||
|
||||
for index, image_obj in enumerate(images_with_boxes):
|
||||
numbox = image_obj['total_boxes'].shape[0]
|
||||
if numbox > 0:
|
||||
h = images[index].shape[0]
|
||||
w = images[index].shape[1]
|
||||
pick = nms(image_obj['total_boxes'].copy(), 0.7, 'Union')
|
||||
image_obj['total_boxes'] = image_obj['total_boxes'][pick, :]
|
||||
regw = image_obj['total_boxes'][:, 2] - image_obj['total_boxes'][:, 0]
|
||||
regh = image_obj['total_boxes'][:, 3] - image_obj['total_boxes'][:, 1]
|
||||
qq1 = image_obj['total_boxes'][:, 0] + image_obj['total_boxes'][:, 5] * regw
|
||||
qq2 = image_obj['total_boxes'][:, 1] + image_obj['total_boxes'][:, 6] * regh
|
||||
qq3 = image_obj['total_boxes'][:, 2] + image_obj['total_boxes'][:, 7] * regw
|
||||
qq4 = image_obj['total_boxes'][:, 3] + image_obj['total_boxes'][:, 8] * regh
|
||||
image_obj['total_boxes'] = np.transpose(np.vstack([qq1, qq2, qq3, qq4, image_obj['total_boxes'][:, 4]]))
|
||||
image_obj['total_boxes'] = rerec(image_obj['total_boxes'].copy())
|
||||
image_obj['total_boxes'][:, 0:4] = np.fix(image_obj['total_boxes'][:, 0:4]).astype(np.int32)
|
||||
dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph = pad(image_obj['total_boxes'].copy(), w, h)
|
||||
|
||||
numbox = image_obj['total_boxes'].shape[0]
|
||||
tempimg = np.zeros((24, 24, 3, numbox))
|
||||
|
||||
if numbox > 0:
|
||||
for k in range(0, numbox):
|
||||
tmp = np.zeros((int(tmph[k]), int(tmpw[k]), 3))
|
||||
tmp[dy[k] - 1:edy[k], dx[k] - 1:edx[k], :] = images[index][y[k] - 1:ey[k], x[k] - 1:ex[k], :]
|
||||
if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0:
|
||||
tempimg[:, :, :, k] = imresample(tmp, (24, 24))
|
||||
else:
|
||||
return np.empty()
|
||||
|
||||
tempimg = (tempimg - 127.5) * 0.0078125
|
||||
image_obj['rnet_input'] = np.transpose(tempimg, (3, 1, 0, 2))
|
||||
|
||||
# # # # # # # # # # # # #
|
||||
# second stage - refinement of face candidates with rnet
|
||||
# # # # # # # # # # # # #
|
||||
|
||||
bulk_rnet_input = np.empty((0, 24, 24, 3))
|
||||
for index, image_obj in enumerate(images_with_boxes):
|
||||
if 'rnet_input' in image_obj:
|
||||
bulk_rnet_input = np.append(bulk_rnet_input, image_obj['rnet_input'], axis=0)
|
||||
|
||||
out = rnet(bulk_rnet_input)
|
||||
out0 = np.transpose(out[0])
|
||||
out1 = np.transpose(out[1])
|
||||
score = out1[1, :]
|
||||
|
||||
i = 0
|
||||
for index, image_obj in enumerate(images_with_boxes):
|
||||
if 'rnet_input' not in image_obj:
|
||||
continue
|
||||
|
||||
rnet_input_count = image_obj['rnet_input'].shape[0]
|
||||
score_per_image = score[i:i + rnet_input_count]
|
||||
out0_per_image = out0[:, i:i + rnet_input_count]
|
||||
|
||||
ipass = np.where(score_per_image > threshold[1])
|
||||
image_obj['total_boxes'] = np.hstack([image_obj['total_boxes'][ipass[0], 0:4].copy(),
|
||||
np.expand_dims(score_per_image[ipass].copy(), 1)])
|
||||
|
||||
mv = out0_per_image[:, ipass[0]]
|
||||
|
||||
if image_obj['total_boxes'].shape[0] > 0:
|
||||
h = images[index].shape[0]
|
||||
w = images[index].shape[1]
|
||||
pick = nms(image_obj['total_boxes'], 0.7, 'Union')
|
||||
image_obj['total_boxes'] = image_obj['total_boxes'][pick, :]
|
||||
image_obj['total_boxes'] = bbreg(image_obj['total_boxes'].copy(), np.transpose(mv[:, pick]))
|
||||
image_obj['total_boxes'] = rerec(image_obj['total_boxes'].copy())
|
||||
|
||||
numbox = image_obj['total_boxes'].shape[0]
|
||||
|
||||
if numbox > 0:
|
||||
tempimg = np.zeros((48, 48, 3, numbox))
|
||||
image_obj['total_boxes'] = np.fix(image_obj['total_boxes']).astype(np.int32)
|
||||
dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph = pad(image_obj['total_boxes'].copy(), w, h)
|
||||
|
||||
for k in range(0, numbox):
|
||||
tmp = np.zeros((int(tmph[k]), int(tmpw[k]), 3))
|
||||
tmp[dy[k] - 1:edy[k], dx[k] - 1:edx[k], :] = images[index][y[k] - 1:ey[k], x[k] - 1:ex[k], :]
|
||||
if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0:
|
||||
tempimg[:, :, :, k] = imresample(tmp, (48, 48))
|
||||
else:
|
||||
return np.empty()
|
||||
tempimg = (tempimg - 127.5) * 0.0078125
|
||||
image_obj['onet_input'] = np.transpose(tempimg, (3, 1, 0, 2))
|
||||
|
||||
i += rnet_input_count
|
||||
|
||||
# # # # # # # # # # # # #
|
||||
# third stage - further refinement and facial landmarks positions with onet
|
||||
# # # # # # # # # # # # #
|
||||
|
||||
bulk_onet_input = np.empty((0, 48, 48, 3))
|
||||
for index, image_obj in enumerate(images_with_boxes):
|
||||
if 'onet_input' in image_obj:
|
||||
bulk_onet_input = np.append(bulk_onet_input, image_obj['onet_input'], axis=0)
|
||||
|
||||
out = onet(bulk_onet_input)
|
||||
|
||||
out0 = np.transpose(out[0])
|
||||
out1 = np.transpose(out[1])
|
||||
out2 = np.transpose(out[2])
|
||||
score = out2[1, :]
|
||||
points = out1
|
||||
|
||||
i = 0
|
||||
ret = []
|
||||
for index, image_obj in enumerate(images_with_boxes):
|
||||
if 'onet_input' not in image_obj:
|
||||
ret.append(None)
|
||||
continue
|
||||
|
||||
onet_input_count = image_obj['onet_input'].shape[0]
|
||||
|
||||
out0_per_image = out0[:, i:i + onet_input_count]
|
||||
score_per_image = score[i:i + onet_input_count]
|
||||
points_per_image = points[:, i:i + onet_input_count]
|
||||
|
||||
ipass = np.where(score_per_image > threshold[2])
|
||||
points_per_image = points_per_image[:, ipass[0]]
|
||||
|
||||
image_obj['total_boxes'] = np.hstack([image_obj['total_boxes'][ipass[0], 0:4].copy(),
|
||||
np.expand_dims(score_per_image[ipass].copy(), 1)])
|
||||
mv = out0_per_image[:, ipass[0]]
|
||||
|
||||
w = image_obj['total_boxes'][:, 2] - image_obj['total_boxes'][:, 0] + 1
|
||||
h = image_obj['total_boxes'][:, 3] - image_obj['total_boxes'][:, 1] + 1
|
||||
points_per_image[0:5, :] = np.tile(w, (5, 1)) * points_per_image[0:5, :] + np.tile(
|
||||
image_obj['total_boxes'][:, 0], (5, 1)) - 1
|
||||
points_per_image[5:10, :] = np.tile(h, (5, 1)) * points_per_image[5:10, :] + np.tile(
|
||||
image_obj['total_boxes'][:, 1], (5, 1)) - 1
|
||||
|
||||
if image_obj['total_boxes'].shape[0] > 0:
|
||||
image_obj['total_boxes'] = bbreg(image_obj['total_boxes'].copy(), np.transpose(mv))
|
||||
pick = nms(image_obj['total_boxes'].copy(), 0.7, 'Min')
|
||||
image_obj['total_boxes'] = image_obj['total_boxes'][pick, :]
|
||||
points_per_image = points_per_image[:, pick]
|
||||
|
||||
ret.append((image_obj['total_boxes'], points_per_image))
|
||||
else:
|
||||
ret.append(None)
|
||||
|
||||
i += onet_input_count
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
# function [boundingbox] = bbreg(boundingbox,reg)
|
||||
def bbreg(boundingbox,reg):
|
||||
"""Calibrate bounding boxes"""
|
||||
if reg.shape[1]==1:
|
||||
reg = np.reshape(reg, (reg.shape[2], reg.shape[3]))
|
||||
|
||||
w = boundingbox[:,2]-boundingbox[:,0]+1
|
||||
h = boundingbox[:,3]-boundingbox[:,1]+1
|
||||
b1 = boundingbox[:,0]+reg[:,0]*w
|
||||
b2 = boundingbox[:,1]+reg[:,1]*h
|
||||
b3 = boundingbox[:,2]+reg[:,2]*w
|
||||
b4 = boundingbox[:,3]+reg[:,3]*h
|
||||
boundingbox[:,0:4] = np.transpose(np.vstack([b1, b2, b3, b4 ]))
|
||||
return boundingbox
|
||||
|
||||
def generateBoundingBox(imap, reg, scale, t):
|
||||
"""Use heatmap to generate bounding boxes"""
|
||||
stride=2
|
||||
cellsize=12
|
||||
|
||||
imap = np.transpose(imap)
|
||||
dx1 = np.transpose(reg[:,:,0])
|
||||
dy1 = np.transpose(reg[:,:,1])
|
||||
dx2 = np.transpose(reg[:,:,2])
|
||||
dy2 = np.transpose(reg[:,:,3])
|
||||
y, x = np.where(imap >= t)
|
||||
if y.shape[0]==1:
|
||||
dx1 = np.flipud(dx1)
|
||||
dy1 = np.flipud(dy1)
|
||||
dx2 = np.flipud(dx2)
|
||||
dy2 = np.flipud(dy2)
|
||||
score = imap[(y,x)]
|
||||
reg = np.transpose(np.vstack([ dx1[(y,x)], dy1[(y,x)], dx2[(y,x)], dy2[(y,x)] ]))
|
||||
if reg.size==0:
|
||||
reg = np.empty((0,3))
|
||||
bb = np.transpose(np.vstack([y,x]))
|
||||
q1 = np.fix((stride*bb+1)/scale)
|
||||
q2 = np.fix((stride*bb+cellsize-1+1)/scale)
|
||||
boundingbox = np.hstack([q1, q2, np.expand_dims(score,1), reg])
|
||||
return boundingbox, reg
|
||||
|
||||
# function pick = nms(boxes,threshold,type)
|
||||
def nms(boxes, threshold, method):
|
||||
if boxes.size==0:
|
||||
return np.empty((0,3))
|
||||
x1 = boxes[:,0]
|
||||
y1 = boxes[:,1]
|
||||
x2 = boxes[:,2]
|
||||
y2 = boxes[:,3]
|
||||
s = boxes[:,4]
|
||||
area = (x2-x1+1) * (y2-y1+1)
|
||||
I = np.argsort(s)
|
||||
pick = np.zeros_like(s, dtype=np.int16)
|
||||
counter = 0
|
||||
while I.size>0:
|
||||
i = I[-1]
|
||||
pick[counter] = i
|
||||
counter += 1
|
||||
idx = I[0:-1]
|
||||
xx1 = np.maximum(x1[i], x1[idx])
|
||||
yy1 = np.maximum(y1[i], y1[idx])
|
||||
xx2 = np.minimum(x2[i], x2[idx])
|
||||
yy2 = np.minimum(y2[i], y2[idx])
|
||||
w = np.maximum(0.0, xx2-xx1+1)
|
||||
h = np.maximum(0.0, yy2-yy1+1)
|
||||
inter = w * h
|
||||
if method is 'Min':
|
||||
o = inter / np.minimum(area[i], area[idx])
|
||||
else:
|
||||
o = inter / (area[i] + area[idx] - inter)
|
||||
I = I[np.where(o<=threshold)]
|
||||
pick = pick[0:counter]
|
||||
return pick
|
||||
|
||||
# function [dy edy dx edx y ey x ex tmpw tmph] = pad(total_boxes,w,h)
|
||||
def pad(total_boxes, w, h):
|
||||
"""Compute the padding coordinates (pad the bounding boxes to square)"""
|
||||
tmpw = (total_boxes[:,2]-total_boxes[:,0]+1).astype(np.int32)
|
||||
tmph = (total_boxes[:,3]-total_boxes[:,1]+1).astype(np.int32)
|
||||
numbox = total_boxes.shape[0]
|
||||
|
||||
dx = np.ones((numbox), dtype=np.int32)
|
||||
dy = np.ones((numbox), dtype=np.int32)
|
||||
edx = tmpw.copy().astype(np.int32)
|
||||
edy = tmph.copy().astype(np.int32)
|
||||
|
||||
x = total_boxes[:,0].copy().astype(np.int32)
|
||||
y = total_boxes[:,1].copy().astype(np.int32)
|
||||
ex = total_boxes[:,2].copy().astype(np.int32)
|
||||
ey = total_boxes[:,3].copy().astype(np.int32)
|
||||
|
||||
tmp = np.where(ex>w)
|
||||
edx.flat[tmp] = np.expand_dims(-ex[tmp]+w+tmpw[tmp],1)
|
||||
ex[tmp] = w
|
||||
|
||||
tmp = np.where(ey>h)
|
||||
edy.flat[tmp] = np.expand_dims(-ey[tmp]+h+tmph[tmp],1)
|
||||
ey[tmp] = h
|
||||
|
||||
tmp = np.where(x<1)
|
||||
dx.flat[tmp] = np.expand_dims(2-x[tmp],1)
|
||||
x[tmp] = 1
|
||||
|
||||
tmp = np.where(y<1)
|
||||
dy.flat[tmp] = np.expand_dims(2-y[tmp],1)
|
||||
y[tmp] = 1
|
||||
|
||||
return dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph
|
||||
|
||||
# function [bboxA] = rerec(bboxA)
|
||||
def rerec(bboxA):
|
||||
"""Convert bboxA to square."""
|
||||
h = bboxA[:,3]-bboxA[:,1]
|
||||
w = bboxA[:,2]-bboxA[:,0]
|
||||
l = np.maximum(w, h)
|
||||
bboxA[:,0] = bboxA[:,0]+w*0.5-l*0.5
|
||||
bboxA[:,1] = bboxA[:,1]+h*0.5-l*0.5
|
||||
bboxA[:,2:4] = bboxA[:,0:2] + np.transpose(np.tile(l,(2,1)))
|
||||
return bboxA
|
||||
|
||||
def imresample(img, sz):
|
||||
im_data = cv2.resize(img, (sz[1], sz[0]), interpolation=cv2.INTER_LINEAR) #@UndefinedVariable
|
||||
return im_data
|
||||
|
||||
# This method is kept for debugging purpose
|
||||
# h=img.shape[0]
|
||||
# w=img.shape[1]
|
||||
# hs, ws = sz
|
||||
# dx = float(w) / ws
|
||||
# dy = float(h) / hs
|
||||
# im_data = np.zeros((hs,ws,3))
|
||||
# for a1 in range(0,hs):
|
||||
# for a2 in range(0,ws):
|
||||
# for a3 in range(0,3):
|
||||
# im_data[a1,a2,a3] = img[int(floor(a1*dy)),int(floor(a2*dx)),a3]
|
||||
# return im_data
|
||||
|
||||
Reference in New Issue
Block a user