bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

facial code refactoring

Browse Source
This commit is contained in:
jordi.hasianta
2022-09-15 20:52:16 +07:00
parent f8818a4047
commit a07659b206
6 changed files with 549 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
opendbm/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py
View File

@@ -4,28 +4,31 @@ project_name: DBM
created: 2020-20-07
"""
from mpl_toolkits import mplot3d
from matplotlib import pyplot as plt
import time
import numpy as np
import os
import datetime
import glob
import cv2
from scipy.spatial.transform import Rotation as R
import subprocess
import pandas as pd
from os.path import join
import logging
import os
import subprocess
import time
from os.path import join
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
import cv2
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from mpl_toolkits import mplot3d
from scipy.spatial.transform import Rotation as R
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
from .face_config.face_config_reader import ConfigFaceReader
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
face_asym_dir = 'facial/face_asymmetry'
csv_ext = '_facasym.csv'
face_asym_dir = "facial/face_asymmetry"
csv_ext = "_facasym.csv"
cv2_color_purple = (254, 19, 188)
color_blue = (0, 0, 1.0)
@@ -34,118 +37,142 @@ color_red = (1.0,0,0)
color_y = (1.0, 1.0, 0)
error_code_message = {
0: 'pass',
1: 'confidence less than 80%',
0: "pass",
1: "confidence less than 80%",
}
error_message_code = {y: x for x, y in error_code_message.items()}
def visualize_vid(fn, attr=None, write_out=False):
vid = cv2.VideoCapture(fn)
tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
# tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
fps = vid.get(cv2.CAP_PROP_FPS)
frame_width = int(vid.get(3))
frame_height = int(vid.get(4))
# frame_width = int(vid.get(3))
# frame_height = int(vid.get(4))
if write_out:
fig_w = 680 # 680 667 676 #frame_width in order of Ali, Vennessa, synthesis
fig_h = 659 # 659 659 659 #frame_height
out_vid = cv2.VideoWriter('out.mp4',cv2.VideoWriter_fourcc(*'MP4V'), fps, (fig_w,fig_h))
out_vid = cv2.VideoWriter(
"out.mp4", cv2.VideoWriter_fourcc(*"MP4V"), fps, (fig_w, fig_h)
)
plt.figure(figsize=(8, 8))
try:
frameid = 0
while(True):
while True:
ret, frame = vid.read()
if not ret:
# Release the Video Device if ret is false
vid.release()
print('Released Video Resource')
print("Released Video Resource")
break
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frameid += 1
logger.info(frameid, frame.shape)
if 'lmks_frms' in attr:
lmks_frms = attr['lmks_frms']
if "lmks_frms" in attr:
lmks_frms = attr["lmks_frms"]
for i in range(lmks_frms[frameid].shape[0]):
cv2.circle(frame,(int(lmks_frms[frameid][i,0]),int(lmks_frms[frameid][i,1])), 2, cv2_color_purple, -1)
cv2.circle(
frame,
(int(lmks_frms[frameid][i, 0]), int(lmks_frms[frameid][i, 1])),
2,
cv2_color_purple,
-1,
)
if write_out:
cv2.putText(frame,'Frame: '+str(frameid), (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 3)
cv2.putText(
frame,
"Frame: " + str(frameid),
(10, 50),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(255, 255, 255),
3,
)
plt.subplot(211)
plt.imshow(frame)
plt.axis('off'); plt.pause(0.2);
plt.axis("off")
plt.pause(0.2)
if 'score_asym' in attr:
if "score_asym" in attr:
ax = plt.subplot(212)
ax.cla()
ax.set_xlim(0, 140) # ax.set_xlim(0,300)
ax.set_ylim(0, 10)
sa = attr['score_asym']
sa = attr["score_asym"]
s = sa[np.where(sa[:, 0] <= frameid), :][0, :, :]
for i in range(1, s.shape[1]):
plt.plot(s[:, 0], s[:, i])
plt.legend(['mouth', 'eyebrow', 'eye', 'mouth+eye+eyebrow'])
plt.legend(["mouth", "eyebrow", "eye", "mouth+eye+eyebrow"])
plt.minorticks_on()
plt.grid(b=True, which='major', color='r', linestyle='-')
plt.grid(b=True, which='minor', color='r', linestyle='--')
plt.grid(b=True, which="major", color="r", linestyle="-")
plt.grid(b=True, which="minor", color="r", linestyle="--")
plt.savefig('tmp.png', bbox_inches='tight')
print(cv2.imread('tmp.png').shape)
plt.savefig("tmp.png", bbox_inches="tight")
print(cv2.imread("tmp.png").shape)
plt.clf()
if write_out:
out_vid.write(cv2.imread('tmp.png'))
out_vid.write(cv2.imread("tmp.png"))
except KeyboardInterrupt:
# Release the Video Device
vid.release()
if write_out:
out_vid.release()
logger.info('Exception, and Video Resource Released')
logger.info("Exception, and Video Resource Released")
if write_out:
out_vid.release()
def retrieve_attr(of_df):
'''
"""
Retrieve landmarks and pose_translation for each frame from openface output
Args:
of_df: dataframe output from openface, including detected landmark coordinates
Returns:
lmks_frms: dictionary, with frame id as key and 68 landmark set as value
pose_p: dictionary, with frame id as key and pose param as value
'''
"""
tot_lmks = 68 # openface specific
if len([i for i in of_df.columns.to_list() if ' x_' in i]) != tot_lmks:
if len([i for i in of_df.columns.to_list() if " x_" in i]) != tot_lmks:
return {}
lmks_frms = {}
pose_p = {}
for fi in sorted(of_df['frame'].to_list()):
for fi in sorted(of_df["frame"].to_list()):
lmks = np.zeros((tot_lmks, 6))
r = of_df[of_df['frame']==fi]
r = of_df[of_df["frame"] == fi]
for i in range(tot_lmks):
lmk_y = r[' y_'+str(i)].iloc[0]
lmk_x = r[' x_'+str(i)].iloc[0]
lmk_X = r[' X_'+str(i)].iloc[0]
lmk_Y = r[' Y_'+str(i)].iloc[0]
lmk_Z = r[' Z_'+str(i)].iloc[0]
lmk_y = r[" y_" + str(i)].iloc[0]
lmk_x = r[" x_" + str(i)].iloc[0]
lmk_X = r[" X_" + str(i)].iloc[0]
lmk_Y = r[" Y_" + str(i)].iloc[0]
lmk_Z = r[" Z_" + str(i)].iloc[0]
confi = r[' confidence']
confi = r[" confidence"]
lmks[i, :] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi]
lmks_frms[fi] = lmks
pose_p[fi] = [r[' pose_Tx'].iloc[0], r[' pose_Ty'].iloc[0], r[' pose_Tz'].iloc[0],
r[' pose_Rx'].iloc[0], r[' pose_Ry'].iloc[0], r[' pose_Rz'].iloc[0]]
pose_p[fi] = [
r[" pose_Tx"].iloc[0],
r[" pose_Ty"].iloc[0],
r[" pose_Tz"].iloc[0],
r[" pose_Rx"].iloc[0],
r[" pose_Ry"].iloc[0],
r[" pose_Rz"].iloc[0],
]
return lmks_frms, pose_p
@@ -173,26 +200,35 @@ def vis_lmks3d(lmks_frms, vis_idx):
"""
Visualizing facial landmarks
"""
fig = plt.figure()
color_type = ['b','g','r','y','c']
# fig = plt.figure()
color_type = ["b", "g", "r", "y", "c"]
assert len(color_type) > len(vis_idx)
for fi in sorted(list(lmks_frms.keys())):
ax = plt.axes(projection="3d")
for i, vi in enumerate(vis_idx):
ax.scatter(lmks_frms[fi][vi,2], lmks_frms[fi][vi,3], lmks_frms[fi][vi,4], c=color_type[i])
ax.scatter(
lmks_frms[fi][vi, 2],
lmks_frms[fi][vi, 3],
lmks_frms[fi][vi, 4],
c=color_type[i],
)
ax.axes.set_xlim3d(left=-75, right=100)
ax.axes.set_ylim3d(bottom=-200, top=25)
ax.axes.set_zlim3d(bottom=440, top=560)
ax.view_init(-89, -90) # elev, ariz
plt.title(str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
plt.title(str(fi))
ax.set_xlabel("X")
ax.set_ylabel("Y")
ax.set_zlabel("Z")
plt.pause(0.2)
plt.cla()
plt.draw()
def calc_fac_asymmetry(attr, is_vis=False):
'''
"""
Quantify facial asymmetry
Args:
attr: attribute dictionary containing necessary features for calculation, e.g.,
@@ -200,63 +236,113 @@ def calc_fac_asymmetry(attr, is_vis=False):
pose_param: dictionary, with frame id as key and pose param as value
Returns:
score_asym: 2D array of size (num_frms, num_asymm_fea), with frame id as the 0th column, and each remaining column as one asymmetry feature
'''
"""
# openface landmark indices
lmks_ref_idx = list(range(0, 17)) + list(range(27, 36))
lmks_mid_idx = [27, 28, 29, 30, 33, 51, 62, 66, 57, 8]
lmks_rgt_idx = [0,1,2,3,4,5,6,7,
17,18,19,20,21,
36,37,38,39,40,41,
48,49,50,
59,58,
60,61,
67]
lmks_lft_idx = [16,15,14,13,12,11,10,9,
26,25,24,23,22,
45,44,43,42,47,46,
54,53,52,
55,56,
64,63,
65]
lmks_rgt_idx = [
0,
1,
2,
3,
4,
5,
6,
7,
17,
18,
19,
20,
21,
36,
37,
38,
39,
40,
41,
48,
49,
50,
59,
58,
60,
61,
67,
]
lmks_lft_idx = [
16,
15,
14,
13,
12,
11,
10,
9,
26,
25,
24,
23,
22,
45,
44,
43,
42,
47,
46,
54,
53,
52,
55,
56,
64,
63,
65,
]
lmks_mth_idx = list(range(48, 68))
lmks_ebr_idx = list(range(17, 27))
lmks_eye_idx = list(range(36, 48))
assert len(lmks_lft_idx) == len(lmks_rgt_idx)
fea_list = ['mouth', 'eyebrow', 'eye', 'composite']
fea_list = ["mouth", "eyebrow", "eye", "composite"]
score_asym = np.empty(shape=(0, 0))
if ('lmks_frms' in attr) and ('pose_param' in attr):
lmks_frms = attr['lmks_frms']
pose_p = attr['pose_param']
if ("lmks_frms" in attr) and ("pose_param" in attr):
lmks_frms = attr["lmks_frms"]
pose_p = attr["pose_param"]
if is_vis:
vis_lmks3d(lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx])
vis_lmks3d(
lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx]
)
score_asym = np.zeros((len(lmks_frms),len(fea_list)+1+1)) # +1: extra column for error code
score_asym = np.zeros(
(len(lmks_frms), len(fea_list) + 1 + 1)
) # +1: extra column for error code
if is_vis:
fig = plt.figure()
# fig = plt.figure()
ax = plt.axes(projection="3d")
for s, fi in enumerate(sorted(list(lmks_frms.keys()))):
lmks_3d = lmks_frms[fi][:, 2:5]
pose = pose_p[fi]
err_code = error_message_code['pass']
err_code = error_message_code["pass"]
if lmks_frms[fi][0, 5] < 0.8:
err_code = error_message_code['confidence less than 80%']
err_code = error_message_code["confidence less than 80%"]
score_asym[s, :] = [fi, np.NaN, np.NaN, np.NaN, np.NaN, err_code]
continue
rx = R.from_euler('x', pose[3])
ry = R.from_euler('y', pose[4])
rz = R.from_euler('z', pose[5])
rx = R.from_euler("x", pose[3])
ry = R.from_euler("y", pose[4])
rz = R.from_euler("z", pose[5])
vec_pose = rz.apply(ry.apply(rx.apply([0, 0, 1])))
anc_idx = [30, 27, 8] # for central plane estimation
nrm = np.cross(lmks_3d[anc_idx[2],:] - lmks_3d[anc_idx[0],:],
lmks_3d[anc_idx[1],:] - lmks_3d[anc_idx[0],:])
nrm = np.cross(
lmks_3d[anc_idx[2], :] - lmks_3d[anc_idx[0], :],
lmks_3d[anc_idx[1], :] - lmks_3d[anc_idx[0], :],
)
nrm = nrm / np.linalg.norm(nrm)
a, b, c = nrm
@@ -271,7 +357,11 @@ def calc_fac_asymmetry(attr, is_vis=False):
src_idx = lmks_lft_idx
for k, idx in enumerate(src_idx):
p_rfl = np.array(mirror_point(a, b, c, -d, lmks_3d[idx,0], lmks_3d[idx,1], lmks_3d[idx,2]))
p_rfl = np.array(
mirror_point(
a, b, c, -d, lmks_3d[idx, 0], lmks_3d[idx, 1], lmks_3d[idx, 2]
)
)
lmks_rfl = np.vstack((lmks_rfl, p_rfl))
dist = dist_vec2plane((p_rfl - lmks_3d[lmks_rgt_idx[k], :]), vec_pose)
@@ -281,15 +371,29 @@ def calc_fac_asymmetry(attr, is_vis=False):
dist_L2R_ebr.append(dist)
if idx in lmks_eye_idx:
dist_L2R_eye.append(dist)
if (idx in lmks_mth_idx) or (idx in lmks_ebr_idx) or (idx in lmks_eye_idx):
if (
(idx in lmks_mth_idx)
or (idx in lmks_ebr_idx)
or (idx in lmks_eye_idx)
):
dist_com.append(dist)
score_asym[s,:] = [fi,np.mean(dist_L2R_mth),np.mean(dist_L2R_ebr),np.mean(dist_L2R_eye),np.mean(dist_com),err_code]
score_asym[s, :] = [
fi,
np.mean(dist_L2R_mth),
np.mean(dist_L2R_ebr),
np.mean(dist_L2R_eye),
np.mean(dist_com),
err_code,
]
if is_vis:
ax.scatter(lmks_3d[:, 0], lmks_3d[:, 1], lmks_3d[:, 2])
ax.scatter(lmks_rfl[:,0], lmks_rfl[:,1], lmks_rfl[:,2], c='y')
ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c='c')
plt.title('mirrored landmarks, frame: '+str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
ax.scatter(lmks_rfl[:, 0], lmks_rfl[:, 1], lmks_rfl[:, 2], c="y")
ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c="c")
plt.title("mirrored landmarks, frame: " + str(fi))
ax.set_xlabel("X")
ax.set_ylabel("Y")
ax.set_zlabel("Z")
plt.pause(0.2)
plt.cla()
plt.draw()
@@ -303,27 +407,38 @@ def calc_asym_feature(open_face_csv, f_cfg):
"""
df_list = []
of_df = pd.read_csv(open_face_csv, error_bad_lines=False)
of_df = pd.read_csv(open_face_csv)
lmks_frms, pose_p = retrieve_attr(of_df)
attr = {'lmks_frms': lmks_frms, 'pose_param': pose_p}
attr = {"lmks_frms": lmks_frms, "pose_param": pose_p}
score_asym = calc_fac_asymmetry(attr)
df_score_asym = pd.DataFrame(score_asym, columns=['frame', f_cfg.fac_AsymMaskMouth, f_cfg.fac_AsymMaskEyebrow,
f_cfg.fac_AsymMaskEye, f_cfg.fac_AsymMaskCom, f_cfg.err_reason])
df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(lambda x: error_code_message[x])
df_score_asym = pd.DataFrame(
score_asym,
columns=[
"frame",
f_cfg.fac_AsymMaskMouth,
f_cfg.fac_AsymMaskEyebrow,
f_cfg.fac_AsymMaskEye,
f_cfg.fac_AsymMaskCom,
f_cfg.err_reason,
],
)
df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(
lambda x: error_code_message[x]
)
df_score_asym['frame'] = of_df['frame']
df_score_asym['face_id'] = of_df[' face_id']
df_score_asym['timestamp'] = of_df[' timestamp']
df_score_asym['confidence'] = of_df[' confidence']
df_score_asym['success'] = of_df[' success']
df_score_asym["frame"] = of_df["frame"]
df_score_asym["face_id"] = of_df[" face_id"]
df_score_asym["timestamp"] = of_df[" timestamp"]
df_score_asym["confidence"] = of_df[" confidence"]
df_score_asym["success"] = of_df[" success"]
df_list.append(df_score_asym)
return df_list
def run_face_asymmetry(video_uri, out_dir, f_cfg):
def run_face_asymmetry(video_uri, out_dir, f_cfg, save=True):
"""
Processing all patient's for calculating facial asymmetry
---------------
@@ -335,20 +450,25 @@ def run_face_asymmetry(video_uri, out_dir, f_cfg):
try:
# Baseline logic
cfr = ConfigFaceReader()
ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
if len(of_csv_path) > 0:
of_csv = of_csv_path[0]
asym_df_list = calc_asym_feature(of_csv, f_cfg)
asym_final_df = pd.concat(asym_df_list, ignore_index=True)
asym_final_df['dbm_master_url'] = video_uri
asym_final_df["dbm_master_url"] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
if save:
logger.info(
"Processing Output file {} ".format(os.path.join(out_loc, fl_name))
)
ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext)
return asym_final_df
except Exception as e:
logger.error('Failed to process video file')
e
logger.error("Failed to process video file")

66
opendbm/dbm_lib/dbm_features/raw_features/video/face_au.py
View File

@@ -4,22 +4,25 @@ project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
import os
from os.path import join
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
import numpy as np
import pandas as pd
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
from .face_config.face_config_reader import ConfigFaceReader
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
face_au_dir = 'facial/face_au'
csv_ext = '_facau.csv'
face_au_dir = "facial/face_au"
csv_ext = "_facau.csv"
def extract_col_nm_au(cols):
@@ -30,8 +33,8 @@ def extract_col_nm_au(cols):
Returns:
(list) list of au column names
"""
cols_lmk = []
au_tags = ' AU'
# cols_lmk = []
au_tags = " AU"
cols_au = [c for c in cols if au_tags in c]
return cols_au
@@ -46,19 +49,19 @@ def au_col_nm_map(df):
"""
dict_au_cols = {}
for col in list(df):
if ' AU' in col:
idx = col.rfind('_')
if " AU" in col:
idx = col.rfind("_")
if idx > -1:
au_id = col[idx - 2 : idx]
if '_r' in col:
dict_au_cols[col] = 'fac_AU' + au_id + 'int'
if '_c' in col:
dict_au_cols[col] = 'fac_AU' + au_id + 'pres'
if "_r" in col:
dict_au_cols[col] = "fac_AU" + au_id + "int"
if "_c" in col:
dict_au_cols[col] = "fac_AU" + au_id + "pres"
df.rename(columns=dict_au_cols, inplace=True)
return df
def run_face_au(video_uri, out_dir, f_cfg):
def run_face_au(video_uri, out_dir, f_cfg, save=True):
"""
Processing all patient's for fetching action units
---------------
@@ -70,28 +73,31 @@ def run_face_au(video_uri, out_dir, f_cfg):
try:
# Baseline logic
cfr = ConfigFaceReader()
ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
if len(of_csv_path) > 0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_of = pd.read_csv(of_csv_path[0])
df_au = df_of[extract_col_nm_au(df_of)]
df_au = df_au.copy()
df_au['frame'] = df_of['frame']
df_au['face_id'] = df_of[' face_id']
df_au['timestamp'] = df_of[' timestamp']
df_au['confidence'] = df_of[' confidence']
df_au['success'] = df_of[' success']
df_au["frame"] = df_of["frame"]
df_au["face_id"] = df_of[" face_id"]
df_au["timestamp"] = df_of[" timestamp"]
df_au["confidence"] = df_of[" confidence"]
df_au["success"] = df_of[" success"]
df_au = au_col_nm_map(df_au)
df_au['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
df_au["dbm_master_url"] = video_uri
if save:
logger.info(
"Processing Output file {} ".format(os.path.join(out_loc, fl_name))
)
ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext)
return df_au
except Exception as e:
logger.error('Failed to process video file')
e
logger.error("Failed to process video file")

62
opendbm/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py
View File

@@ -4,16 +4,22 @@ project_name: DBM
created: 2020-20-07
"""
import os
import yaml
import boto3
from opendbm.dbm_lib.dbm_features.raw_features.video import DBMLIB_FACE_CONFIG
DBMLIB_PATH = os.path.dirname(__file__)
DBMLIB_FACE_CONFIG = os.path.abspath(
os.path.join(DBMLIB_PATH, "../../../../../resources/services/face_util.yml")
)
class ConfigFaceReader(object):
"""Summary
Read sevice end ponit
"""
def __init__(self,
service_config_yml=None):
def __init__(self, service_config_yml=None):
"""Summary
Args:
service_config_yml (None, optional): yml file defined service configuration
@@ -24,29 +30,31 @@ class ConfigFaceReader(object):
else:
service_config = service_config_yml
with open(service_config, 'r') as ymlfile:
config = yaml.load(ymlfile)
self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS']
self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS']
self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS']
self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS']
self.LOWER_ACTION_UNITS = config['cdx_face_config']['LOWER_ACTION_UNITS']
self.UPPER_ACTION_UNITS = config['cdx_face_config']['UPPER_ACTION_UNITS']
self.happiness = config['cdx_face_config']['happiness']
self.sadness = config['cdx_face_config']['sadness']
self.surprise = config['cdx_face_config']['surprise']
self.fear = config['cdx_face_config']['fear']
self.anger = config['cdx_face_config']['anger']
self.disgust = config['cdx_face_config']['disgust']
self.contempt = config['cdx_face_config']['contempt']
self.pain = config['cdx_face_config']['pain']
self.cai = config['cdx_face_config']['CAI']
self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',')
self.face_expr_dir = config['cdx_face_config']['face_expr_dir']
self.face_asym_dir = config['cdx_face_config']['face_asym_dir']
self.AU_fl = config['cdx_face_config']['AU_filters']
self.au_int = config['cdx_face_config']['au_intensity']
self.au_prs = config['cdx_face_config']['au_presence']
with open(service_config, "r") as ymlfile:
config = yaml.load(ymlfile, Loader=yaml.CLoader)
self.ACTION_UNITS = config["cdx_face_config"]["ACTION_UNITS"]
self.NEG_ACTION_UNITS = config["cdx_face_config"]["NEG_ACTION_UNITS"]
self.POS_ACTION_UNITS = config["cdx_face_config"]["POS_ACTION_UNITS"]
self.NET_ACTION_UNITS = config["cdx_face_config"]["NET_ACTION_UNITS"]
self.LOWER_ACTION_UNITS = config["cdx_face_config"]["LOWER_ACTION_UNITS"]
self.UPPER_ACTION_UNITS = config["cdx_face_config"]["UPPER_ACTION_UNITS"]
self.happiness = config["cdx_face_config"]["happiness"]
self.sadness = config["cdx_face_config"]["sadness"]
self.surprise = config["cdx_face_config"]["surprise"]
self.fear = config["cdx_face_config"]["fear"]
self.anger = config["cdx_face_config"]["anger"]
self.disgust = config["cdx_face_config"]["disgust"]
self.contempt = config["cdx_face_config"]["contempt"]
self.pain = config["cdx_face_config"]["pain"]
self.cai = config["cdx_face_config"]["CAI"]
self.SELECTED_FEATURES = config["cdx_face_config"][
"SELECTED_FEATURES"
].split(",")
self.face_expr_dir = config["cdx_face_config"]["face_expr_dir"]
self.face_asym_dir = config["cdx_face_config"]["face_asym_dir"]
self.AU_fl = config["cdx_face_config"]["AU_filters"]
self.au_int = config["cdx_face_config"]["au_intensity"]
self.au_prs = config["cdx_face_config"]["au_presence"]
def get_action_unit(self):
"""Summary

42
opendbm/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py
View File

@@ -4,22 +4,25 @@ project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
import os
from os.path import join
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
import numpy as np
import pandas as pd
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
from .face_config.face_config_reader import ConfigFaceReader
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
face_expr_dir = 'facial/face_expressivity'
csv_ext = '_facemo.csv'
face_expr_dir = "facial/face_expressivity"
csv_ext = "_facemo.csv"
# Openface feature extraction
def of_feature(df_of, cfr, f_cfg):
@@ -31,9 +34,11 @@ def of_feature(df_of, cfr, f_cfg):
(list) list of expressivity score for emotions
"""
df_list = []
df_of['s_confidence'] = vu.smooth(df_of[' confidence'].values, window='flat').tolist()
df_of["s_confidence"] = vu.smooth(
df_of[" confidence"].values, window="flat"
).tolist()
if 'AU' in cfr.SELECTED_FEATURES :
if "AU" in cfr.SELECTED_FEATURES:
vu.calc_of_for_video(df_of, cfr, f_cfg)
# Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1)
@@ -53,7 +58,7 @@ def of_feature(df_of, cfr, f_cfg):
return df_list
def run_face_expressivity(video_uri, out_dir, f_cfg):
def run_face_expressivity(video_uri, out_dir, f_cfg, save=True):
"""
Processing all patient's for fetching facial landmarks
---------------
@@ -68,18 +73,23 @@ def run_face_expressivity(video_uri, out_dir, f_cfg):
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
if len(of_csv_path) > 0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_of = pd.read_csv(of_csv_path[0])
df_of = df_of[cfr.AU_fl]
expr_df_list = of_feature(df_of, cfr, f_cfg)
exp_final_df = pd.concat(expr_df_list, ignore_index=True)
exp_final_df['dbm_master_url'] = video_uri
exp_final_df["dbm_master_url"] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
if save:
logger.info(
"Processing Output file {} ".format(os.path.join(out_loc, fl_name))
)
ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext)
return exp_final_df
except Exception as e:
logger.error('Failed to process video file')
e
logger.error("Failed to process video file")

91
opendbm/dbm_lib/dbm_features/raw_features/video/face_landmark.py
View File

@@ -4,22 +4,26 @@ project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
import os
from os.path import join
from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
import numpy as np
import pandas as pd
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu
from .face_config.face_config_reader import ConfigFaceReader
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
face_lmk_dir = 'facial/face_landmark'
csv_ext = '_faclmk.csv'
face_lmk_dir = "facial/face_landmark"
csv_ext = "_faclmk.csv"
def extract_col_nm_lmk(cols):
"""
@@ -30,7 +34,7 @@ def extract_col_nm_lmk(cols):
(list) list of landmark column names
"""
cols_lmk = []
lmk_tags = [' y_', ' x_', ' X_', ' Y_', ' Z_']
lmk_tags = [" y_", " x_", " X_", " Y_", " Z_"]
for c in cols:
if any(t in c for t in lmk_tags):
cols_lmk.append(c)
@@ -45,19 +49,19 @@ def lmk_col_nm_map(df):
"""
dict_lmk_cols = {}
for col in list(df):
idx = col.rfind('_')+1
idx = col.rfind("_") + 1
if idx > 0:
lmk_id = col[idx:] if len(col[idx:])>1 else '0'+col[idx:]
if ' y_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'r'
if ' x_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'c'
if ' X_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'X'
if ' Y_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Y'
if ' Z_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Z'
lmk_id = col[idx:] if len(col[idx:]) > 1 else "0" + col[idx:]
if " y_" in col:
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "r"
if " x_" in col:
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "c"
if " X_" in col:
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "X"
if " Y_" in col:
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "Y"
if " Z_" in col:
dict_lmk_cols[col] = "fac_LMK" + lmk_id + "Z"
df.rename(columns=dict_lmk_cols, inplace=True)
return df
@@ -68,21 +72,31 @@ def add_disp_3D(df):
Args:
df: landmark dataframe
"""
df = df.sort_values(by=['frame'], ascending=False)
cols_lmk = [col for col in list(df) if 'fac_LMK' in col]
df = df.sort_values(by=["frame"], ascending=False)
cols_lmk = [col for col in list(df) if "fac_LMK" in col]
df_t = df[cols_lmk]
df_diff = df_t.diff()
df_diff = df_diff.pow(2)
tot_lmk = 68 # 68 landmark model
for i in range(tot_lmk):
lmk_id = '{:02d}'.format(i)
df['fac_LMK'+lmk_id+'disp'] = df_diff[['fac_LMK'+lmk_id+'X', 'fac_LMK'+lmk_id+'Y', 'fac_LMK'+lmk_id+'Z']].sum(axis=1).apply(np.sqrt)
lmk_id = "{:02d}".format(i)
df["fac_LMK" + lmk_id + "disp"] = (
df_diff[
[
"fac_LMK" + lmk_id + "X",
"fac_LMK" + lmk_id + "Y",
"fac_LMK" + lmk_id + "Z",
]
]
.sum(axis=1)
.apply(np.sqrt)
)
return df
def run_face_landmark(video_uri, out_dir, f_cfg):
def run_face_landmark(video_uri, out_dir, f_cfg, save=True):
"""
Processing all patient's for fetching facial landmarks
---------------
@@ -94,28 +108,31 @@ def run_face_landmark(video_uri, out_dir, f_cfg):
try:
# Baseline logic
cfr = ConfigFaceReader()
ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv"))
if len(of_csv_path) > 0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_of = pd.read_csv(of_csv_path[0])
df_lmk = df_of[extract_col_nm_lmk(df_of)]
df_lmk = df_lmk.copy()
df_lmk['frame'] = df_of['frame']
df_lmk['face_id'] = df_of[' face_id']
df_lmk['timestamp'] = df_of[' timestamp']
df_lmk['confidence'] = df_of[' confidence']
df_lmk['success'] = df_of[' success']
df_lmk["frame"] = df_of["frame"]
df_lmk["face_id"] = df_of[" face_id"]
df_lmk["timestamp"] = df_of[" timestamp"]
df_lmk["confidence"] = df_of[" confidence"]
df_lmk["success"] = df_of[" success"]
df_lmk = lmk_col_nm_map(df_lmk)
df_lmk = add_disp_3D(df_lmk)
df_lmk['dbm_master_url'] = video_uri
df_lmk["dbm_master_url"] = video_uri
logger.info('Processing Output file {} '.format(join(out_loc, fl_name)))
if save:
logger.info("Processing Output file {} ".format(join(out_loc, fl_name)))
ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)
return df_lmk
except Exception as e:
logger.error('Failed to process video file')
e
logger.error("Failed to process video file")

41
opendbm/dbm_lib/dbm_features/raw_features/video/open_face_process.py
View File

@@ -4,18 +4,22 @@ project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import glob
import logging
import os
import numpy as np
import pandas as pd
from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False):
def batch_open_face(
filepaths, video_url, input_dir, out_dir, of_path, video_tracking=False
):
"""Computes open_face features for the files in filepaths
Args:
@@ -33,9 +37,9 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac
(itreable[str]) list of .csv files
"""
if video_tracking:
suffix = '_openface_lmk'
suffix = "_openface_lmk"
else:
suffix = '_openface'
suffix = "_openface"
csv_files = []
@@ -48,14 +52,19 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac
if video_tracking and not os.path.exists(os.path.abspath(output_directory)):
os.makedirs(os.path.abspath(output_directory))
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
csv_files.append(
ut.compute_open_face_features(fp, output_directory, of_path)
)
except Exception as e:
logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
logger.error("Failed to run OpenFace on {}\n{}".format(fp, e))
return csv_files
def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking):
def process_open_face(
video_uri, input_dir, out_dir, of_path, dbm_group, video_tracking
):
"""
Processing all patient's for fetching emotion expressivity
-------------------
@@ -67,14 +76,20 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tr
"""
try:
if dbm_group != None:
check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation
if dbm_group is not None:
check_group = [
"facial",
"movement",
] # add group here: if you want to use openface output for raw variable calculation
check_val = bool(len({*check_group} & {*dbm_group}))
if not check_val:
return
filepaths = [video_uri]
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking)
batch_open_face(
filepaths, video_uri, input_dir, out_dir, of_path, video_tracking
)
except Exception as e:
logger.error('Failed to process video file')
e
logger.error("Failed to process video file")