facial code refactoring
This commit is contained in:
jordi.hasianta
@@ -4,195 +4,231 @@ project_name: DBM
|
||||
created: 2020-20-07
|
||||
"""
|
||||
|
||||
from mpl_toolkits import mplot3d
|
||||
from matplotlib import pyplot as plt
|
||||
import time
|
||||
import numpy as np
|
||||
import os
|
||||
import datetime
|
||||
import glob
|
||||
import cv2
|
||||
from scipy.spatial.transform import Rotation as R
|
||||
import subprocess
|
||||
import pandas as pd
|
||||
from os.path import join
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from os.path import join
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
|
||||
import cv2
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from matplotlib import pyplot as plt
|
||||
from mpl_toolkits import mplot3d
|
||||
from scipy.spatial.transform import Rotation as R
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
|
||||
|
||||
from .face_config.face_config_reader import ConfigFaceReader
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
face_asym_dir = 'facial/face_asymmetry'
|
||||
csv_ext = '_facasym.csv'
|
||||
face_asym_dir = "facial/face_asymmetry"
|
||||
csv_ext = "_facasym.csv"
|
||||
|
||||
cv2_color_purple = (254,19,188)
|
||||
color_blue = (0,0,1.0)
|
||||
color_green = (0,1.0,0)
|
||||
color_red = (1.0,0,0)
|
||||
color_y = (1.0,1.0,0)
|
||||
cv2_color_purple = (254, 19, 188)
|
||||
color_blue = (0, 0, 1.0)
|
||||
color_green = (0, 1.0, 0)
|
||||
color_red = (1.0, 0, 0)
|
||||
color_y = (1.0, 1.0, 0)
|
||||
|
||||
error_code_message = {
|
||||
0: 'pass',
|
||||
1: 'confidence less than 80%',
|
||||
}
|
||||
error_message_code = {y:x for x,y in error_code_message.items()}
|
||||
0: "pass",
|
||||
1: "confidence less than 80%",
|
||||
}
|
||||
error_message_code = {y: x for x, y in error_code_message.items()}
|
||||
|
||||
|
||||
def visualize_vid(fn, attr=None, write_out=False):
|
||||
|
||||
vid = cv2.VideoCapture(fn)
|
||||
tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
# tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = vid.get(cv2.CAP_PROP_FPS)
|
||||
frame_width = int(vid.get(3))
|
||||
frame_height = int(vid.get(4))
|
||||
# frame_width = int(vid.get(3))
|
||||
# frame_height = int(vid.get(4))
|
||||
|
||||
if write_out:
|
||||
fig_w = 680 #680 667 676 #frame_width in order of Ali, Vennessa, synthesis
|
||||
fig_h = 659 #659 659 659 #frame_height
|
||||
out_vid = cv2.VideoWriter('out.mp4',cv2.VideoWriter_fourcc(*'MP4V'), fps, (fig_w,fig_h))
|
||||
fig_w = 680 # 680 667 676 #frame_width in order of Ali, Vennessa, synthesis
|
||||
fig_h = 659 # 659 659 659 #frame_height
|
||||
out_vid = cv2.VideoWriter(
|
||||
"out.mp4", cv2.VideoWriter_fourcc(*"MP4V"), fps, (fig_w, fig_h)
|
||||
)
|
||||
|
||||
plt.figure(figsize=(8, 8))
|
||||
try:
|
||||
frameid = 0
|
||||
while(True):
|
||||
while True:
|
||||
ret, frame = vid.read()
|
||||
if not ret:
|
||||
# Release the Video Device if ret is false
|
||||
vid.release()
|
||||
print('Released Video Resource')
|
||||
print("Released Video Resource")
|
||||
break
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
frameid += 1
|
||||
logger.info(frameid, frame.shape)
|
||||
|
||||
if 'lmks_frms' in attr:
|
||||
lmks_frms = attr['lmks_frms']
|
||||
if "lmks_frms" in attr:
|
||||
lmks_frms = attr["lmks_frms"]
|
||||
for i in range(lmks_frms[frameid].shape[0]):
|
||||
cv2.circle(frame,(int(lmks_frms[frameid][i,0]),int(lmks_frms[frameid][i,1])), 2, cv2_color_purple, -1)
|
||||
cv2.circle(
|
||||
frame,
|
||||
(int(lmks_frms[frameid][i, 0]), int(lmks_frms[frameid][i, 1])),
|
||||
2,
|
||||
cv2_color_purple,
|
||||
-1,
|
||||
)
|
||||
|
||||
if write_out:
|
||||
cv2.putText(frame,'Frame: '+str(frameid), (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 3)
|
||||
cv2.putText(
|
||||
frame,
|
||||
"Frame: " + str(frameid),
|
||||
(10, 50),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
1,
|
||||
(255, 255, 255),
|
||||
3,
|
||||
)
|
||||
|
||||
plt.subplot(211)
|
||||
plt.imshow(frame)
|
||||
plt.axis('off'); plt.pause(0.2);
|
||||
plt.axis("off")
|
||||
plt.pause(0.2)
|
||||
|
||||
if 'score_asym' in attr:
|
||||
if "score_asym" in attr:
|
||||
ax = plt.subplot(212)
|
||||
ax.cla()
|
||||
ax.set_xlim(0,140) #ax.set_xlim(0,300)
|
||||
ax.set_ylim(0,10)
|
||||
ax.set_xlim(0, 140) # ax.set_xlim(0,300)
|
||||
ax.set_ylim(0, 10)
|
||||
|
||||
sa = attr['score_asym']
|
||||
s = sa[np.where(sa[:,0] <= frameid),:][0,:,:]
|
||||
sa = attr["score_asym"]
|
||||
s = sa[np.where(sa[:, 0] <= frameid), :][0, :, :]
|
||||
|
||||
for i in range(1,s.shape[1]):
|
||||
plt.plot(s[:,0], s[:,i])
|
||||
for i in range(1, s.shape[1]):
|
||||
plt.plot(s[:, 0], s[:, i])
|
||||
|
||||
plt.legend(['mouth', 'eyebrow', 'eye', 'mouth+eye+eyebrow'])
|
||||
plt.legend(["mouth", "eyebrow", "eye", "mouth+eye+eyebrow"])
|
||||
plt.minorticks_on()
|
||||
plt.grid(b=True, which='major', color='r', linestyle='-')
|
||||
plt.grid(b=True, which='minor', color='r', linestyle='--')
|
||||
plt.grid(b=True, which="major", color="r", linestyle="-")
|
||||
plt.grid(b=True, which="minor", color="r", linestyle="--")
|
||||
|
||||
plt.savefig('tmp.png', bbox_inches='tight')
|
||||
print(cv2.imread('tmp.png').shape)
|
||||
plt.savefig("tmp.png", bbox_inches="tight")
|
||||
print(cv2.imread("tmp.png").shape)
|
||||
|
||||
plt.clf()
|
||||
if write_out:
|
||||
out_vid.write(cv2.imread('tmp.png'))
|
||||
out_vid.write(cv2.imread("tmp.png"))
|
||||
|
||||
except KeyboardInterrupt:
|
||||
# Release the Video Device
|
||||
vid.release()
|
||||
if write_out:
|
||||
out_vid.release()
|
||||
logger.info('Exception, and Video Resource Released')
|
||||
logger.info("Exception, and Video Resource Released")
|
||||
|
||||
if write_out:
|
||||
out_vid.release()
|
||||
|
||||
|
||||
def retrieve_attr(of_df):
|
||||
'''
|
||||
Retrieve landmarks and pose_translation for each frame from openface output
|
||||
Args:
|
||||
of_df: dataframe output from openface, including detected landmark coordinates
|
||||
Returns:
|
||||
lmks_frms: dictionary, with frame id as key and 68 landmark set as value
|
||||
pose_p: dictionary, with frame id as key and pose param as value
|
||||
'''
|
||||
tot_lmks = 68 # openface specific
|
||||
if len([i for i in of_df.columns.to_list() if ' x_' in i]) != tot_lmks:
|
||||
"""
|
||||
Retrieve landmarks and pose_translation for each frame from openface output
|
||||
Args:
|
||||
of_df: dataframe output from openface, including detected landmark coordinates
|
||||
Returns:
|
||||
lmks_frms: dictionary, with frame id as key and 68 landmark set as value
|
||||
pose_p: dictionary, with frame id as key and pose param as value
|
||||
"""
|
||||
tot_lmks = 68 # openface specific
|
||||
if len([i for i in of_df.columns.to_list() if " x_" in i]) != tot_lmks:
|
||||
return {}
|
||||
|
||||
lmks_frms = {}
|
||||
pose_p = {}
|
||||
|
||||
for fi in sorted(of_df['frame'].to_list()):
|
||||
lmks = np.zeros((tot_lmks,6))
|
||||
r = of_df[of_df['frame']==fi]
|
||||
for fi in sorted(of_df["frame"].to_list()):
|
||||
lmks = np.zeros((tot_lmks, 6))
|
||||
r = of_df[of_df["frame"] == fi]
|
||||
|
||||
for i in range(tot_lmks):
|
||||
lmk_y = r[' y_'+str(i)].iloc[0]
|
||||
lmk_x = r[' x_'+str(i)].iloc[0]
|
||||
lmk_X = r[' X_'+str(i)].iloc[0]
|
||||
lmk_Y = r[' Y_'+str(i)].iloc[0]
|
||||
lmk_Z = r[' Z_'+str(i)].iloc[0]
|
||||
lmk_y = r[" y_" + str(i)].iloc[0]
|
||||
lmk_x = r[" x_" + str(i)].iloc[0]
|
||||
lmk_X = r[" X_" + str(i)].iloc[0]
|
||||
lmk_Y = r[" Y_" + str(i)].iloc[0]
|
||||
lmk_Z = r[" Z_" + str(i)].iloc[0]
|
||||
|
||||
confi = r[' confidence']
|
||||
lmks[i,:] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi]
|
||||
confi = r[" confidence"]
|
||||
lmks[i, :] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi]
|
||||
|
||||
lmks_frms[fi] = lmks
|
||||
pose_p[fi] = [r[' pose_Tx'].iloc[0], r[' pose_Ty'].iloc[0], r[' pose_Tz'].iloc[0],
|
||||
r[' pose_Rx'].iloc[0], r[' pose_Ry'].iloc[0], r[' pose_Rz'].iloc[0]]
|
||||
pose_p[fi] = [
|
||||
r[" pose_Tx"].iloc[0],
|
||||
r[" pose_Ty"].iloc[0],
|
||||
r[" pose_Tz"].iloc[0],
|
||||
r[" pose_Rx"].iloc[0],
|
||||
r[" pose_Ry"].iloc[0],
|
||||
r[" pose_Rz"].iloc[0],
|
||||
]
|
||||
|
||||
return lmks_frms, pose_p
|
||||
|
||||
|
||||
def mirror_point(a, b, c, d, x1, y1, z1):
|
||||
# mirror a point w.r.t a 3D plane
|
||||
k =(-a * x1-b * y1-c * z1-d)/float((a * a + b * b + c * c))
|
||||
k = (-a * x1 - b * y1 - c * z1 - d) / float((a * a + b * b + c * c))
|
||||
|
||||
x2 = a * k + x1
|
||||
y2 = b * k + y1
|
||||
z2 = c * k + z1
|
||||
|
||||
x3 = 2 * x2-x1
|
||||
y3 = 2 * y2-y1
|
||||
z3 = 2 * z2-z1
|
||||
x3 = 2 * x2 - x1
|
||||
y3 = 2 * y2 - y1
|
||||
z3 = 2 * z2 - z1
|
||||
return [x3, y3, z3]
|
||||
|
||||
|
||||
def dist_vec2plane(vec, nrm):
|
||||
# Calculate the projected length of a vector (vec) to a plane defined by its normal (nrm)
|
||||
return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm)**2)
|
||||
return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm) ** 2)
|
||||
|
||||
|
||||
def vis_lmks3d(lmks_frms, vis_idx):
|
||||
"""
|
||||
Visualizing facial landmarks
|
||||
"""
|
||||
fig = plt.figure()
|
||||
color_type = ['b','g','r','y','c']
|
||||
# fig = plt.figure()
|
||||
color_type = ["b", "g", "r", "y", "c"]
|
||||
assert len(color_type) > len(vis_idx)
|
||||
|
||||
for fi in sorted(list(lmks_frms.keys())):
|
||||
ax = plt.axes(projection="3d")
|
||||
for i,vi in enumerate(vis_idx):
|
||||
ax.scatter(lmks_frms[fi][vi,2], lmks_frms[fi][vi,3], lmks_frms[fi][vi,4], c=color_type[i])
|
||||
for i, vi in enumerate(vis_idx):
|
||||
ax.scatter(
|
||||
lmks_frms[fi][vi, 2],
|
||||
lmks_frms[fi][vi, 3],
|
||||
lmks_frms[fi][vi, 4],
|
||||
c=color_type[i],
|
||||
)
|
||||
|
||||
ax.axes.set_xlim3d(left=-75, right=100)
|
||||
ax.axes.set_ylim3d(bottom=-200, top=25)
|
||||
ax.axes.set_zlim3d(bottom=440, top=560)
|
||||
ax.view_init(-89, -90) #elev, ariz
|
||||
plt.title(str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
|
||||
ax.view_init(-89, -90) # elev, ariz
|
||||
plt.title(str(fi))
|
||||
ax.set_xlabel("X")
|
||||
ax.set_ylabel("Y")
|
||||
ax.set_zlabel("Z")
|
||||
plt.pause(0.2)
|
||||
plt.cla()
|
||||
plt.draw()
|
||||
|
||||
|
||||
def calc_fac_asymmetry(attr, is_vis=False):
|
||||
'''
|
||||
"""
|
||||
Quantify facial asymmetry
|
||||
Args:
|
||||
attr: attribute dictionary containing necessary features for calculation, e.g.,
|
||||
@@ -200,80 +236,134 @@ def calc_fac_asymmetry(attr, is_vis=False):
|
||||
pose_param: dictionary, with frame id as key and pose param as value
|
||||
Returns:
|
||||
score_asym: 2D array of size (num_frms, num_asymm_fea), with frame id as the 0th column, and each remaining column as one asymmetry feature
|
||||
'''
|
||||
"""
|
||||
# openface landmark indices
|
||||
lmks_ref_idx = list(range(0,17)) + list(range(27,36))
|
||||
lmks_mid_idx = [27,28,29,30,33,51,62,66,57,8]
|
||||
lmks_rgt_idx = [0,1,2,3,4,5,6,7,
|
||||
17,18,19,20,21,
|
||||
36,37,38,39,40,41,
|
||||
48,49,50,
|
||||
59,58,
|
||||
60,61,
|
||||
67]
|
||||
lmks_lft_idx = [16,15,14,13,12,11,10,9,
|
||||
26,25,24,23,22,
|
||||
45,44,43,42,47,46,
|
||||
54,53,52,
|
||||
55,56,
|
||||
64,63,
|
||||
65]
|
||||
lmks_ref_idx = list(range(0, 17)) + list(range(27, 36))
|
||||
lmks_mid_idx = [27, 28, 29, 30, 33, 51, 62, 66, 57, 8]
|
||||
lmks_rgt_idx = [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
17,
|
||||
18,
|
||||
19,
|
||||
20,
|
||||
21,
|
||||
36,
|
||||
37,
|
||||
38,
|
||||
39,
|
||||
40,
|
||||
41,
|
||||
48,
|
||||
49,
|
||||
50,
|
||||
59,
|
||||
58,
|
||||
60,
|
||||
61,
|
||||
67,
|
||||
]
|
||||
lmks_lft_idx = [
|
||||
16,
|
||||
15,
|
||||
14,
|
||||
13,
|
||||
12,
|
||||
11,
|
||||
10,
|
||||
9,
|
||||
26,
|
||||
25,
|
||||
24,
|
||||
23,
|
||||
22,
|
||||
45,
|
||||
44,
|
||||
43,
|
||||
42,
|
||||
47,
|
||||
46,
|
||||
54,
|
||||
53,
|
||||
52,
|
||||
55,
|
||||
56,
|
||||
64,
|
||||
63,
|
||||
65,
|
||||
]
|
||||
|
||||
lmks_mth_idx = list(range(48,68))
|
||||
lmks_ebr_idx = list(range(17,27))
|
||||
lmks_eye_idx = list(range(36,48))
|
||||
assert len(lmks_lft_idx)==len(lmks_rgt_idx)
|
||||
lmks_mth_idx = list(range(48, 68))
|
||||
lmks_ebr_idx = list(range(17, 27))
|
||||
lmks_eye_idx = list(range(36, 48))
|
||||
assert len(lmks_lft_idx) == len(lmks_rgt_idx)
|
||||
|
||||
fea_list = ['mouth', 'eyebrow', 'eye', 'composite']
|
||||
fea_list = ["mouth", "eyebrow", "eye", "composite"]
|
||||
score_asym = np.empty(shape=(0, 0))
|
||||
|
||||
if ('lmks_frms' in attr) and ('pose_param' in attr):
|
||||
lmks_frms = attr['lmks_frms']
|
||||
pose_p = attr['pose_param']
|
||||
if ("lmks_frms" in attr) and ("pose_param" in attr):
|
||||
lmks_frms = attr["lmks_frms"]
|
||||
pose_p = attr["pose_param"]
|
||||
|
||||
if is_vis:
|
||||
vis_lmks3d(lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx])
|
||||
vis_lmks3d(
|
||||
lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx]
|
||||
)
|
||||
|
||||
score_asym = np.zeros((len(lmks_frms),len(fea_list)+1+1)) # +1: extra column for error code
|
||||
score_asym = np.zeros(
|
||||
(len(lmks_frms), len(fea_list) + 1 + 1)
|
||||
) # +1: extra column for error code
|
||||
if is_vis:
|
||||
fig = plt.figure()
|
||||
# fig = plt.figure()
|
||||
ax = plt.axes(projection="3d")
|
||||
|
||||
for s,fi in enumerate(sorted(list(lmks_frms.keys()))):
|
||||
lmks_3d = lmks_frms[fi][:,2:5]
|
||||
for s, fi in enumerate(sorted(list(lmks_frms.keys()))):
|
||||
lmks_3d = lmks_frms[fi][:, 2:5]
|
||||
pose = pose_p[fi]
|
||||
err_code = error_message_code['pass']
|
||||
err_code = error_message_code["pass"]
|
||||
|
||||
if lmks_frms[fi][0,5] < 0.8:
|
||||
err_code = error_message_code['confidence less than 80%']
|
||||
score_asym[s,:] = [fi,np.NaN,np.NaN,np.NaN,np.NaN,err_code]
|
||||
if lmks_frms[fi][0, 5] < 0.8:
|
||||
err_code = error_message_code["confidence less than 80%"]
|
||||
score_asym[s, :] = [fi, np.NaN, np.NaN, np.NaN, np.NaN, err_code]
|
||||
continue
|
||||
|
||||
rx = R.from_euler('x', pose[3])
|
||||
ry = R.from_euler('y', pose[4])
|
||||
rz = R.from_euler('z', pose[5])
|
||||
rx = R.from_euler("x", pose[3])
|
||||
ry = R.from_euler("y", pose[4])
|
||||
rz = R.from_euler("z", pose[5])
|
||||
|
||||
vec_pose = rz.apply(ry.apply(rx.apply([0,0,1])))
|
||||
anc_idx = [30, 27, 8] # for central plane estimation
|
||||
nrm = np.cross(lmks_3d[anc_idx[2],:] - lmks_3d[anc_idx[0],:],
|
||||
lmks_3d[anc_idx[1],:] - lmks_3d[anc_idx[0],:])
|
||||
vec_pose = rz.apply(ry.apply(rx.apply([0, 0, 1])))
|
||||
anc_idx = [30, 27, 8] # for central plane estimation
|
||||
nrm = np.cross(
|
||||
lmks_3d[anc_idx[2], :] - lmks_3d[anc_idx[0], :],
|
||||
lmks_3d[anc_idx[1], :] - lmks_3d[anc_idx[0], :],
|
||||
)
|
||||
|
||||
nrm = nrm / np.linalg.norm(nrm)
|
||||
a,b,c = nrm
|
||||
d = np.dot(nrm, lmks_3d[anc_idx[0],:])
|
||||
a, b, c = nrm
|
||||
d = np.dot(nrm, lmks_3d[anc_idx[0], :])
|
||||
|
||||
dist_L2R_mth = []
|
||||
dist_L2R_ebr = []
|
||||
dist_L2R_eye = []
|
||||
dist_com = []
|
||||
|
||||
lmks_rfl = np.empty((0,3))
|
||||
lmks_rfl = np.empty((0, 3))
|
||||
src_idx = lmks_lft_idx
|
||||
|
||||
for k,idx in enumerate(src_idx):
|
||||
p_rfl = np.array(mirror_point(a, b, c, -d, lmks_3d[idx,0], lmks_3d[idx,1], lmks_3d[idx,2]))
|
||||
for k, idx in enumerate(src_idx):
|
||||
p_rfl = np.array(
|
||||
mirror_point(
|
||||
a, b, c, -d, lmks_3d[idx, 0], lmks_3d[idx, 1], lmks_3d[idx, 2]
|
||||
)
|
||||
)
|
||||
lmks_rfl = np.vstack((lmks_rfl, p_rfl))
|
||||
dist = dist_vec2plane((p_rfl-lmks_3d[lmks_rgt_idx[k],:]), vec_pose)
|
||||
dist = dist_vec2plane((p_rfl - lmks_3d[lmks_rgt_idx[k], :]), vec_pose)
|
||||
|
||||
if idx in lmks_mth_idx:
|
||||
dist_L2R_mth.append(dist)
|
||||
@@ -281,15 +371,29 @@ def calc_fac_asymmetry(attr, is_vis=False):
|
||||
dist_L2R_ebr.append(dist)
|
||||
if idx in lmks_eye_idx:
|
||||
dist_L2R_eye.append(dist)
|
||||
if (idx in lmks_mth_idx) or (idx in lmks_ebr_idx) or (idx in lmks_eye_idx):
|
||||
if (
|
||||
(idx in lmks_mth_idx)
|
||||
or (idx in lmks_ebr_idx)
|
||||
or (idx in lmks_eye_idx)
|
||||
):
|
||||
dist_com.append(dist)
|
||||
score_asym[s,:] = [fi,np.mean(dist_L2R_mth),np.mean(dist_L2R_ebr),np.mean(dist_L2R_eye),np.mean(dist_com),err_code]
|
||||
score_asym[s, :] = [
|
||||
fi,
|
||||
np.mean(dist_L2R_mth),
|
||||
np.mean(dist_L2R_ebr),
|
||||
np.mean(dist_L2R_eye),
|
||||
np.mean(dist_com),
|
||||
err_code,
|
||||
]
|
||||
|
||||
if is_vis:
|
||||
ax.scatter(lmks_3d[:,0], lmks_3d[:,1], lmks_3d[:,2])
|
||||
ax.scatter(lmks_rfl[:,0], lmks_rfl[:,1], lmks_rfl[:,2], c='y')
|
||||
ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c='c')
|
||||
plt.title('mirrored landmarks, frame: '+str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
|
||||
ax.scatter(lmks_3d[:, 0], lmks_3d[:, 1], lmks_3d[:, 2])
|
||||
ax.scatter(lmks_rfl[:, 0], lmks_rfl[:, 1], lmks_rfl[:, 2], c="y")
|
||||
ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c="c")
|
||||
plt.title("mirrored landmarks, frame: " + str(fi))
|
||||
ax.set_xlabel("X")
|
||||
ax.set_ylabel("Y")
|
||||
ax.set_zlabel("Z")
|
||||
plt.pause(0.2)
|
||||
plt.cla()
|
||||
plt.draw()
|
||||
@@ -303,27 +407,38 @@ def calc_asym_feature(open_face_csv, f_cfg):
|
||||
"""
|
||||
df_list = []
|
||||
|
||||
of_df = pd.read_csv(open_face_csv, error_bad_lines=False)
|
||||
of_df = pd.read_csv(open_face_csv)
|
||||
lmks_frms, pose_p = retrieve_attr(of_df)
|
||||
|
||||
attr = {'lmks_frms': lmks_frms, 'pose_param': pose_p}
|
||||
attr = {"lmks_frms": lmks_frms, "pose_param": pose_p}
|
||||
score_asym = calc_fac_asymmetry(attr)
|
||||
|
||||
df_score_asym = pd.DataFrame(score_asym, columns=['frame', f_cfg.fac_AsymMaskMouth, f_cfg.fac_AsymMaskEyebrow,
|
||||
f_cfg.fac_AsymMaskEye, f_cfg.fac_AsymMaskCom, f_cfg.err_reason])
|
||||
df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(lambda x: error_code_message[x])
|
||||
df_score_asym = pd.DataFrame(
|
||||
score_asym,
|
||||
columns=[
|
||||
"frame",
|
||||
f_cfg.fac_AsymMaskMouth,
|
||||
f_cfg.fac_AsymMaskEyebrow,
|
||||
f_cfg.fac_AsymMaskEye,
|
||||
f_cfg.fac_AsymMaskCom,
|
||||
f_cfg.err_reason,
|
||||
],
|
||||
)
|
||||
df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(
|
||||
lambda x: error_code_message[x]
|
||||
)
|
||||
|
||||
df_score_asym['frame'] = of_df['frame']
|
||||
df_score_asym['face_id'] = of_df[' face_id']
|
||||
df_score_asym['timestamp'] = of_df[' timestamp']
|
||||
df_score_asym['confidence'] = of_df[' confidence']
|
||||
df_score_asym['success'] = of_df[' success']
|
||||
df_score_asym["frame"] = of_df["frame"]
|
||||
df_score_asym["face_id"] = of_df[" face_id"]
|
||||
df_score_asym["timestamp"] = of_df[" timestamp"]
|
||||
df_score_asym["confidence"] = of_df[" confidence"]
|
||||
df_score_asym["success"] = of_df[" success"]
|
||||
|
||||
df_list.append(df_score_asym)
|
||||
return df_list
|
||||
|
||||
|
||||
def run_face_asymmetry(video_uri, out_dir, f_cfg):
|
||||
def run_face_asymmetry(video_uri, out_dir, f_cfg, save=True):
|
||||
"""
|
||||
Processing all patient's for calculating facial asymmetry
|
||||
---------------
|
||||
@@ -334,21 +449,26 @@ def run_face_asymmetry(video_uri, out_dir, f_cfg):
|
||||
"""
|
||||
try:
|
||||
|
||||
#Baseline logic
|
||||
cfr = ConfigFaceReader()
|
||||
# Baseline logic
|
||||
ConfigFaceReader()
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
|
||||
if len(of_csv_path)>0:
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
|
||||
if len(of_csv_path) > 0:
|
||||
|
||||
of_csv = of_csv_path[0]
|
||||
asym_df_list = calc_asym_feature(of_csv, f_cfg)
|
||||
|
||||
asym_final_df = pd.concat(asym_df_list, ignore_index=True)
|
||||
asym_final_df['dbm_master_url'] = video_uri
|
||||
asym_final_df["dbm_master_url"] = video_uri
|
||||
|
||||
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext)
|
||||
if save:
|
||||
logger.info(
|
||||
"Processing Output file {} ".format(os.path.join(out_loc, fl_name))
|
||||
)
|
||||
ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext)
|
||||
return asym_final_df
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
e
|
||||
logger.error("Failed to process video file")
|
||||
|
||||
@@ -4,22 +4,25 @@ project_name: DBM
|
||||
created: 2020-20-07
|
||||
"""
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import glob
|
||||
from os.path import join
|
||||
import logging
|
||||
import os
|
||||
from os.path import join
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
|
||||
|
||||
from .face_config.face_config_reader import ConfigFaceReader
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
face_au_dir = 'facial/face_au'
|
||||
csv_ext = '_facau.csv'
|
||||
face_au_dir = "facial/face_au"
|
||||
csv_ext = "_facau.csv"
|
||||
|
||||
|
||||
def extract_col_nm_au(cols):
|
||||
@@ -30,8 +33,8 @@ def extract_col_nm_au(cols):
|
||||
Returns:
|
||||
(list) list of au column names
|
||||
"""
|
||||
cols_lmk = []
|
||||
au_tags = ' AU'
|
||||
# cols_lmk = []
|
||||
au_tags = " AU"
|
||||
cols_au = [c for c in cols if au_tags in c]
|
||||
return cols_au
|
||||
|
||||
@@ -46,19 +49,19 @@ def au_col_nm_map(df):
|
||||
"""
|
||||
dict_au_cols = {}
|
||||
for col in list(df):
|
||||
if ' AU' in col:
|
||||
idx = col.rfind('_')
|
||||
if " AU" in col:
|
||||
idx = col.rfind("_")
|
||||
if idx > -1:
|
||||
au_id = col[idx-2:idx]
|
||||
if '_r' in col:
|
||||
dict_au_cols[col] = 'fac_AU' + au_id + 'int'
|
||||
if '_c' in col:
|
||||
dict_au_cols[col] = 'fac_AU' + au_id + 'pres'
|
||||
au_id = col[idx - 2 : idx]
|
||||
if "_r" in col:
|
||||
dict_au_cols[col] = "fac_AU" + au_id + "int"
|
||||
if "_c" in col:
|
||||
dict_au_cols[col] = "fac_AU" + au_id + "pres"
|
||||
df.rename(columns=dict_au_cols, inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
def run_face_au(video_uri, out_dir, f_cfg):
|
||||
def run_face_au(video_uri, out_dir, f_cfg, save=True):
|
||||
"""
|
||||
Processing all patient's for fetching action units
|
||||
---------------
|
||||
@@ -69,29 +72,32 @@ def run_face_au(video_uri, out_dir, f_cfg):
|
||||
"""
|
||||
try:
|
||||
|
||||
#Baseline logic
|
||||
cfr = ConfigFaceReader()
|
||||
# Baseline logic
|
||||
ConfigFaceReader()
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
|
||||
if len(of_csv_path)>0:
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
|
||||
if len(of_csv_path) > 0:
|
||||
|
||||
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
|
||||
df_of = pd.read_csv(of_csv_path[0])
|
||||
df_au = df_of[extract_col_nm_au(df_of)]
|
||||
df_au = df_au.copy()
|
||||
|
||||
df_au['frame'] = df_of['frame']
|
||||
df_au['face_id'] = df_of[' face_id']
|
||||
df_au['timestamp'] = df_of[' timestamp']
|
||||
df_au['confidence'] = df_of[' confidence']
|
||||
df_au['success'] = df_of[' success']
|
||||
df_au["frame"] = df_of["frame"]
|
||||
df_au["face_id"] = df_of[" face_id"]
|
||||
df_au["timestamp"] = df_of[" timestamp"]
|
||||
df_au["confidence"] = df_of[" confidence"]
|
||||
df_au["success"] = df_of[" success"]
|
||||
|
||||
df_au = au_col_nm_map(df_au)
|
||||
df_au['dbm_master_url'] = video_uri
|
||||
|
||||
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext)
|
||||
df_au["dbm_master_url"] = video_uri
|
||||
if save:
|
||||
logger.info(
|
||||
"Processing Output file {} ".format(os.path.join(out_loc, fl_name))
|
||||
)
|
||||
ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext)
|
||||
return df_au
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
|
||||
e
|
||||
logger.error("Failed to process video file")
|
||||
|
||||
@@ -4,16 +4,22 @@ project_name: DBM
|
||||
created: 2020-20-07
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import yaml
|
||||
import boto3
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.video import DBMLIB_FACE_CONFIG
|
||||
|
||||
DBMLIB_PATH = os.path.dirname(__file__)
|
||||
DBMLIB_FACE_CONFIG = os.path.abspath(
|
||||
os.path.join(DBMLIB_PATH, "../../../../../resources/services/face_util.yml")
|
||||
)
|
||||
|
||||
|
||||
class ConfigFaceReader(object):
|
||||
"""Summary
|
||||
Read sevice end ponit
|
||||
"""
|
||||
def __init__(self,
|
||||
service_config_yml=None):
|
||||
|
||||
def __init__(self, service_config_yml=None):
|
||||
"""Summary
|
||||
Args:
|
||||
service_config_yml (None, optional): yml file defined service configuration
|
||||
@@ -24,29 +30,31 @@ class ConfigFaceReader(object):
|
||||
else:
|
||||
service_config = service_config_yml
|
||||
|
||||
with open(service_config, 'r') as ymlfile:
|
||||
config = yaml.load(ymlfile)
|
||||
self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS']
|
||||
self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS']
|
||||
self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS']
|
||||
self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS']
|
||||
self.LOWER_ACTION_UNITS = config['cdx_face_config']['LOWER_ACTION_UNITS']
|
||||
self.UPPER_ACTION_UNITS = config['cdx_face_config']['UPPER_ACTION_UNITS']
|
||||
self.happiness = config['cdx_face_config']['happiness']
|
||||
self.sadness = config['cdx_face_config']['sadness']
|
||||
self.surprise = config['cdx_face_config']['surprise']
|
||||
self.fear = config['cdx_face_config']['fear']
|
||||
self.anger = config['cdx_face_config']['anger']
|
||||
self.disgust = config['cdx_face_config']['disgust']
|
||||
self.contempt = config['cdx_face_config']['contempt']
|
||||
self.pain = config['cdx_face_config']['pain']
|
||||
self.cai = config['cdx_face_config']['CAI']
|
||||
self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',')
|
||||
self.face_expr_dir = config['cdx_face_config']['face_expr_dir']
|
||||
self.face_asym_dir = config['cdx_face_config']['face_asym_dir']
|
||||
self.AU_fl = config['cdx_face_config']['AU_filters']
|
||||
self.au_int = config['cdx_face_config']['au_intensity']
|
||||
self.au_prs = config['cdx_face_config']['au_presence']
|
||||
with open(service_config, "r") as ymlfile:
|
||||
config = yaml.load(ymlfile, Loader=yaml.CLoader)
|
||||
self.ACTION_UNITS = config["cdx_face_config"]["ACTION_UNITS"]
|
||||
self.NEG_ACTION_UNITS = config["cdx_face_config"]["NEG_ACTION_UNITS"]
|
||||
self.POS_ACTION_UNITS = config["cdx_face_config"]["POS_ACTION_UNITS"]
|
||||
self.NET_ACTION_UNITS = config["cdx_face_config"]["NET_ACTION_UNITS"]
|
||||
self.LOWER_ACTION_UNITS = config["cdx_face_config"]["LOWER_ACTION_UNITS"]
|
||||
self.UPPER_ACTION_UNITS = config["cdx_face_config"]["UPPER_ACTION_UNITS"]
|
||||
self.happiness = config["cdx_face_config"]["happiness"]
|
||||
self.sadness = config["cdx_face_config"]["sadness"]
|
||||
self.surprise = config["cdx_face_config"]["surprise"]
|
||||
self.fear = config["cdx_face_config"]["fear"]
|
||||
self.anger = config["cdx_face_config"]["anger"]
|
||||
self.disgust = config["cdx_face_config"]["disgust"]
|
||||
self.contempt = config["cdx_face_config"]["contempt"]
|
||||
self.pain = config["cdx_face_config"]["pain"]
|
||||
self.cai = config["cdx_face_config"]["CAI"]
|
||||
self.SELECTED_FEATURES = config["cdx_face_config"][
|
||||
"SELECTED_FEATURES"
|
||||
].split(",")
|
||||
self.face_expr_dir = config["cdx_face_config"]["face_expr_dir"]
|
||||
self.face_asym_dir = config["cdx_face_config"]["face_asym_dir"]
|
||||
self.AU_fl = config["cdx_face_config"]["AU_filters"]
|
||||
self.au_int = config["cdx_face_config"]["au_intensity"]
|
||||
self.au_prs = config["cdx_face_config"]["au_presence"]
|
||||
|
||||
def get_action_unit(self):
|
||||
"""Summary
|
||||
|
||||
@@ -4,24 +4,27 @@ project_name: DBM
|
||||
created: 2020-20-07
|
||||
"""
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import glob
|
||||
from os.path import join
|
||||
import logging
|
||||
import os
|
||||
from os.path import join
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
|
||||
|
||||
from .face_config.face_config_reader import ConfigFaceReader
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
face_expr_dir = 'facial/face_expressivity'
|
||||
csv_ext = '_facemo.csv'
|
||||
face_expr_dir = "facial/face_expressivity"
|
||||
csv_ext = "_facemo.csv"
|
||||
|
||||
#Openface feature extraction
|
||||
# Openface feature extraction
|
||||
def of_feature(df_of, cfr, f_cfg):
|
||||
"""
|
||||
Creating dataframe for face expressivity
|
||||
@@ -31,29 +34,31 @@ def of_feature(df_of, cfr, f_cfg):
|
||||
(list) list of expressivity score for emotions
|
||||
"""
|
||||
df_list = []
|
||||
df_of['s_confidence'] = vu.smooth(df_of[' confidence'].values, window='flat').tolist()
|
||||
df_of["s_confidence"] = vu.smooth(
|
||||
df_of[" confidence"].values, window="flat"
|
||||
).tolist()
|
||||
|
||||
if 'AU' in cfr.SELECTED_FEATURES :
|
||||
if "AU" in cfr.SELECTED_FEATURES:
|
||||
vu.calc_of_for_video(df_of, cfr, f_cfg)
|
||||
#Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1)
|
||||
# Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1)
|
||||
|
||||
if len(df_of[f_cfg.neg_exp])>0:
|
||||
df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp]/5
|
||||
if len(df_of[f_cfg.neg_exp]) > 0:
|
||||
df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp] / 5
|
||||
|
||||
if len(df_of[f_cfg.neg_exp_full])>0:
|
||||
df_of[f_cfg.neg_exp_full] = df_of[f_cfg.neg_exp_full]/5
|
||||
if len(df_of[f_cfg.neg_exp_full]) > 0:
|
||||
df_of[f_cfg.neg_exp_full] = df_of[f_cfg.neg_exp_full] / 5
|
||||
|
||||
if len(df_of[f_cfg.com_exp])>0:
|
||||
df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp]/7
|
||||
if len(df_of[f_cfg.com_exp]) > 0:
|
||||
df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp] / 7
|
||||
|
||||
if len(df_of[f_cfg.com_exp_full])>0:
|
||||
df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full]/7
|
||||
if len(df_of[f_cfg.com_exp_full]) > 0:
|
||||
df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full] / 7
|
||||
|
||||
df_list.append(df_of)
|
||||
return df_list
|
||||
|
||||
|
||||
def run_face_expressivity(video_uri, out_dir, f_cfg):
|
||||
def run_face_expressivity(video_uri, out_dir, f_cfg, save=True):
|
||||
"""
|
||||
Processing all patient's for fetching facial landmarks
|
||||
---------------
|
||||
@@ -64,22 +69,27 @@ def run_face_expressivity(video_uri, out_dir, f_cfg):
|
||||
"""
|
||||
try:
|
||||
|
||||
#Baseline logic
|
||||
# Baseline logic
|
||||
cfr = ConfigFaceReader()
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
|
||||
if len(of_csv_path)>0:
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
|
||||
if len(of_csv_path) > 0:
|
||||
|
||||
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
|
||||
df_of = pd.read_csv(of_csv_path[0])
|
||||
df_of = df_of[cfr.AU_fl]
|
||||
expr_df_list = of_feature(df_of, cfr, f_cfg)
|
||||
|
||||
exp_final_df = pd.concat(expr_df_list, ignore_index=True)
|
||||
exp_final_df['dbm_master_url'] = video_uri
|
||||
exp_final_df["dbm_master_url"] = video_uri
|
||||
|
||||
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext)
|
||||
if save:
|
||||
logger.info(
|
||||
"Processing Output file {} ".format(os.path.join(out_loc, fl_name))
|
||||
)
|
||||
ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext)
|
||||
return exp_final_df
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
e
|
||||
logger.error("Failed to process video file")
|
||||
|
||||
@@ -4,22 +4,26 @@ project_name: DBM
|
||||
created: 2020-20-07
|
||||
"""
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import glob
|
||||
from os.path import join
|
||||
import logging
|
||||
import os
|
||||
from os.path import join
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
|
||||
|
||||
from .face_config.face_config_reader import ConfigFaceReader
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
face_lmk_dir = "facial/face_landmark"
|
||||
csv_ext = "_faclmk.csv"
|
||||
|
||||
face_lmk_dir = 'facial/face_landmark'
|
||||
csv_ext = '_faclmk.csv'
|
||||
|
||||
def extract_col_nm_lmk(cols):
|
||||
"""
|
||||
@@ -30,7 +34,7 @@ def extract_col_nm_lmk(cols):
|
||||
(list) list of landmark column names
|
||||
"""
|
||||
cols_lmk = []
|
||||
lmk_tags = [' y_', ' x_', ' X_', ' Y_', ' Z_']
|
||||
lmk_tags = [" y_", " x_", " X_", " Y_", " Z_"]
|
||||
for c in cols:
|
||||
if any(t in c for t in lmk_tags):
|
||||
cols_lmk.append(c)
|
||||
@@ -45,19 +49,19 @@ def lmk_col_nm_map(df):
|
||||
"""
|
||||
dict_lmk_cols = {}
|
||||
for col in list(df):
|
||||
idx = col.rfind('_')+1
|
||||
idx = col.rfind("_") + 1
|
||||
if idx > 0:
|
||||
lmk_id = col[idx:] if len(col[idx:])>1 else '0'+col[idx:]
|
||||
if ' y_' in col:
|
||||
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'r'
|
||||
if ' x_' in col:
|
||||
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'c'
|
||||
if ' X_' in col:
|
||||
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'X'
|
||||
if ' Y_' in col:
|
||||
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Y'
|
||||
if ' Z_' in col:
|
||||
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Z'
|
||||
lmk_id = col[idx:] if len(col[idx:]) > 1 else "0" + col[idx:]
|
||||
if " y_" in col:
|
||||
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "r"
|
||||
if " x_" in col:
|
||||
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "c"
|
||||
if " X_" in col:
|
||||
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "X"
|
||||
if " Y_" in col:
|
||||
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "Y"
|
||||
if " Z_" in col:
|
||||
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "Z"
|
||||
df.rename(columns=dict_lmk_cols, inplace=True)
|
||||
return df
|
||||
|
||||
@@ -68,21 +72,31 @@ def add_disp_3D(df):
|
||||
Args:
|
||||
df: landmark dataframe
|
||||
"""
|
||||
df = df.sort_values(by=['frame'], ascending=False)
|
||||
cols_lmk = [col for col in list(df) if 'fac_LMK' in col]
|
||||
df = df.sort_values(by=["frame"], ascending=False)
|
||||
cols_lmk = [col for col in list(df) if "fac_LMK" in col]
|
||||
df_t = df[cols_lmk]
|
||||
df_diff = df_t.diff()
|
||||
df_diff = df_diff.pow(2)
|
||||
|
||||
tot_lmk = 68 # 68 landmark model
|
||||
tot_lmk = 68 # 68 landmark model
|
||||
for i in range(tot_lmk):
|
||||
lmk_id = '{:02d}'.format(i)
|
||||
df['fac_LMK'+lmk_id+'disp'] = df_diff[['fac_LMK'+lmk_id+'X', 'fac_LMK'+lmk_id+'Y', 'fac_LMK'+lmk_id+'Z']].sum(axis=1).apply(np.sqrt)
|
||||
lmk_id = "{:02d}".format(i)
|
||||
df["fac_LMK" + lmk_id + "disp"] = (
|
||||
df_diff[
|
||||
[
|
||||
"fac_LMK" + lmk_id + "X",
|
||||
"fac_LMK" + lmk_id + "Y",
|
||||
"fac_LMK" + lmk_id + "Z",
|
||||
]
|
||||
]
|
||||
.sum(axis=1)
|
||||
.apply(np.sqrt)
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def run_face_landmark(video_uri, out_dir, f_cfg):
|
||||
def run_face_landmark(video_uri, out_dir, f_cfg, save=True):
|
||||
"""
|
||||
Processing all patient's for fetching facial landmarks
|
||||
---------------
|
||||
@@ -93,29 +107,32 @@ def run_face_landmark(video_uri, out_dir, f_cfg):
|
||||
"""
|
||||
try:
|
||||
|
||||
#Baseline logic
|
||||
cfr = ConfigFaceReader()
|
||||
# Baseline logic
|
||||
ConfigFaceReader()
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
|
||||
if len(of_csv_path)>0:
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
|
||||
if len(of_csv_path) > 0:
|
||||
|
||||
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
|
||||
df_of = pd.read_csv(of_csv_path[0])
|
||||
df_lmk = df_of[extract_col_nm_lmk(df_of)]
|
||||
df_lmk = df_lmk.copy()
|
||||
|
||||
df_lmk['frame'] = df_of['frame']
|
||||
df_lmk['face_id'] = df_of[' face_id']
|
||||
df_lmk['timestamp'] = df_of[' timestamp']
|
||||
df_lmk['confidence'] = df_of[' confidence']
|
||||
df_lmk['success'] = df_of[' success']
|
||||
df_lmk["frame"] = df_of["frame"]
|
||||
df_lmk["face_id"] = df_of[" face_id"]
|
||||
df_lmk["timestamp"] = df_of[" timestamp"]
|
||||
df_lmk["confidence"] = df_of[" confidence"]
|
||||
df_lmk["success"] = df_of[" success"]
|
||||
|
||||
df_lmk = lmk_col_nm_map(df_lmk)
|
||||
df_lmk = add_disp_3D(df_lmk)
|
||||
df_lmk['dbm_master_url'] = video_uri
|
||||
df_lmk["dbm_master_url"] = video_uri
|
||||
|
||||
logger.info('Processing Output file {} '.format(join(out_loc, fl_name)))
|
||||
ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)
|
||||
if save:
|
||||
logger.info("Processing Output file {} ".format(join(out_loc, fl_name)))
|
||||
ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)
|
||||
return df_lmk
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
e
|
||||
logger.error("Failed to process video file")
|
||||
|
||||
@@ -4,19 +4,23 @@ project_name: DBM
|
||||
created: 2020-20-07
|
||||
"""
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
logger = logging.getLogger()
|
||||
|
||||
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False):
|
||||
""" Computes open_face features for the files in filepaths
|
||||
|
||||
def batch_open_face(
|
||||
filepaths, video_url, input_dir, out_dir, of_path, video_tracking=False
|
||||
):
|
||||
"""Computes open_face features for the files in filepaths
|
||||
|
||||
Args:
|
||||
-----
|
||||
@@ -33,9 +37,9 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac
|
||||
(itreable[str]) list of .csv files
|
||||
"""
|
||||
if video_tracking:
|
||||
suffix = '_openface_lmk'
|
||||
suffix = "_openface_lmk"
|
||||
else:
|
||||
suffix = '_openface'
|
||||
suffix = "_openface"
|
||||
|
||||
csv_files = []
|
||||
|
||||
@@ -48,14 +52,19 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac
|
||||
|
||||
if video_tracking and not os.path.exists(os.path.abspath(output_directory)):
|
||||
os.makedirs(os.path.abspath(output_directory))
|
||||
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
|
||||
csv_files.append(
|
||||
ut.compute_open_face_features(fp, output_directory, of_path)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
|
||||
logger.error("Failed to run OpenFace on {}\n{}".format(fp, e))
|
||||
|
||||
return csv_files
|
||||
|
||||
def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking):
|
||||
|
||||
def process_open_face(
|
||||
video_uri, input_dir, out_dir, of_path, dbm_group, video_tracking
|
||||
):
|
||||
"""
|
||||
Processing all patient's for fetching emotion expressivity
|
||||
-------------------
|
||||
@@ -67,14 +76,20 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tr
|
||||
"""
|
||||
try:
|
||||
|
||||
if dbm_group != None:
|
||||
check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation
|
||||
if dbm_group is not None:
|
||||
check_group = [
|
||||
"facial",
|
||||
"movement",
|
||||
] # add group here: if you want to use openface output for raw variable calculation
|
||||
check_val = bool(len({*check_group} & {*dbm_group}))
|
||||
if not check_val:
|
||||
return
|
||||
|
||||
filepaths = [video_uri]
|
||||
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking)
|
||||
batch_open_face(
|
||||
filepaths, video_uri, input_dir, out_dir, of_path, video_tracking
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
e
|
||||
logger.error("Failed to process video file")
|
||||
|
||||
Reference in New Issue
Block a user