bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
dbm_lib/dbm_features/raw_features/video/__init__.py Normal file
View File

@@ -0,0 +1,14 @@
"""
file_name: __init__
project_name: DBM
created: 2020-20-07
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
DBMLIB_PATH = os.path.dirname(__file__)
DBMLIB_FACE_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/services/face_util.yml'))

351
dbm_lib/dbm_features/raw_features/video/face_asymmetry.py Normal file
View File

@@ -0,0 +1,351 @@
"""
file_name: face_asymmetry.py
project_name: DBM
created: 2020-20-07
"""
from mpl_toolkits import mplot3d
from matplotlib import pyplot as plt
import time
import numpy as np
import os
import datetime
import glob
import cv2
from scipy.spatial.transform import Rotation as R
import subprocess
import pandas as pd
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_asym_dir = 'video/face_asymmetry'
csv_ext = '_face_asymmetry.csv'
cv2_color_purple = (254,19,188)
color_blue = (0,0,1.0)
color_green = (0,1.0,0)
color_red = (1.0,0,0)
color_y = (1.0,1.0,0)
error_code_message = {
0: 'pass',
1: 'confidence less than 80%',
}
error_message_code = {y:x for x,y in error_code_message.items()}
def visualize_vid(fn, attr=None, write_out=False):
vid = cv2.VideoCapture(fn)
tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
fps = vid.get(cv2.CAP_PROP_FPS)
frame_width = int(vid.get(3))
frame_height = int(vid.get(4))
if write_out:
fig_w = 680 #680 667 676 #frame_width in order of Ali, Vennessa, synthesis
fig_h = 659 #659 659 659 #frame_height
out_vid = cv2.VideoWriter('out.mp4',cv2.VideoWriter_fourcc(*'MP4V'), fps, (fig_w,fig_h))
plt.figure(figsize=(8, 8))
try:
frameid = 0
while(True):
ret, frame = vid.read()
if not ret:
# Release the Video Device if ret is false
vid.release()
print('Released Video Resource')
break
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frameid += 1
logger.info(frameid, frame.shape)
if 'lmks_frms' in attr:
lmks_frms = attr['lmks_frms']
for i in range(lmks_frms[frameid].shape[0]):
cv2.circle(frame,(int(lmks_frms[frameid][i,0]),int(lmks_frms[frameid][i,1])), 2, cv2_color_purple, -1)
if write_out:
cv2.putText(frame,'Frame: '+str(frameid), (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 3)
plt.subplot(211)
plt.imshow(frame)
plt.axis('off'); plt.pause(0.2);
if 'score_asym' in attr:
ax = plt.subplot(212)
ax.cla()
ax.set_xlim(0,140) #ax.set_xlim(0,300)
ax.set_ylim(0,10)
sa = attr['score_asym']
s = sa[np.where(sa[:,0] <= frameid),:][0,:,:]
for i in range(1,s.shape[1]):
plt.plot(s[:,0], s[:,i])
plt.legend(['mouth', 'eyebrow', 'eye', 'mouth+eye+eyebrow'])
plt.minorticks_on()
plt.grid(b=True, which='major', color='r', linestyle='-')
plt.grid(b=True, which='minor', color='r', linestyle='--')
plt.savefig('tmp.png', bbox_inches='tight')
print(cv2.imread('tmp.png').shape)
plt.clf()
if write_out:
out_vid.write(cv2.imread('tmp.png'))
except KeyboardInterrupt:
# Release the Video Device
vid.release()
if write_out:
out_vid.release()
logger.info('Exception, and Video Resource Released')
if write_out:
out_vid.release()
def retrieve_attr(of_df):
'''
Retrieve landmarks and pose_translation for each frame from openface output
Args:
of_df: dataframe output from openface, including detected landmark coordinates
Returns:
lmks_frms: dictionary, with frame id as key and 68 landmark set as value
pose_p: dictionary, with frame id as key and pose param as value
'''
tot_lmks = 68 # openface specific
if len([i for i in of_df.columns.to_list() if ' x_' in i]) != tot_lmks:
return {}
lmks_frms = {}
pose_p = {}
for fi in sorted(of_df['frame'].to_list()):
lmks = np.zeros((tot_lmks,6))
r = of_df[of_df['frame']==fi]
for i in range(tot_lmks):
lmk_y = r[' y_'+str(i)].iloc[0]
lmk_x = r[' x_'+str(i)].iloc[0]
lmk_X = r[' X_'+str(i)].iloc[0]
lmk_Y = r[' Y_'+str(i)].iloc[0]
lmk_Z = r[' Z_'+str(i)].iloc[0]
confi = r[' confidence']
lmks[i,:] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi]
lmks_frms[fi] = lmks
pose_p[fi] = [r[' pose_Tx'].iloc[0], r[' pose_Ty'].iloc[0], r[' pose_Tz'].iloc[0],
r[' pose_Rx'].iloc[0], r[' pose_Ry'].iloc[0], r[' pose_Rz'].iloc[0]]
return lmks_frms, pose_p
def mirror_point(a, b, c, d, x1, y1, z1):
# mirror a point w.r.t a 3D plane
k =(-a * x1-b * y1-c * z1-d)/float((a * a + b * b + c * c))
x2 = a * k + x1
y2 = b * k + y1
z2 = c * k + z1
x3 = 2 * x2-x1
y3 = 2 * y2-y1
z3 = 2 * z2-z1
return [x3, y3, z3]
def dist_vec2plane(vec, nrm):
# Calculate the projected length of a vector (vec) to a plane defined by its normal (nrm)
return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm)**2)
def vis_lmks3d(lmks_frms, vis_idx):
"""
Visualizing facial landmarks
"""
fig = plt.figure()
color_type = ['b','g','r','y','c']
assert len(color_type) > len(vis_idx)
for fi in sorted(list(lmks_frms.keys())):
ax = plt.axes(projection="3d")
for i,vi in enumerate(vis_idx):
ax.scatter(lmks_frms[fi][vi,2], lmks_frms[fi][vi,3], lmks_frms[fi][vi,4], c=color_type[i])
ax.axes.set_xlim3d(left=-75, right=100)
ax.axes.set_ylim3d(bottom=-200, top=25)
ax.axes.set_zlim3d(bottom=440, top=560)
ax.view_init(-89, -90) #elev, ariz
plt.title(str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
plt.pause(0.2)
plt.cla()
plt.draw()
def calc_fac_asymmetry(attr, is_vis=False):
'''
Quantify facial asymmetry
Args:
attr: attribute dictionary containing necessary features for calculation, e.g.,
lmks_frms: dictionary, with frame id as key and 68 landmark set (OpenFace) as value
pose_param: dictionary, with frame id as key and pose param as value
Returns:
score_asym: 2D array of size (num_frms, num_asymm_fea), with frame id as the 0th column, and each remaining column as one asymmetry feature
'''
# openface landmark indices
lmks_ref_idx = list(range(0,17)) + list(range(27,36))
lmks_mid_idx = [27,28,29,30,33,51,62,66,57,8]
lmks_rgt_idx = [0,1,2,3,4,5,6,7,
17,18,19,20,21,
36,37,38,39,40,41,
48,49,50,
59,58,
60,61,
67]
lmks_lft_idx = [16,15,14,13,12,11,10,9,
26,25,24,23,22,
45,44,43,42,47,46,
54,53,52,
55,56,
64,63,
65]
lmks_mth_idx = list(range(48,68))
lmks_ebr_idx = list(range(17,27))
lmks_eye_idx = list(range(36,48))
assert len(lmks_lft_idx)==len(lmks_rgt_idx)
fea_list = ['mouth', 'eyebrow', 'eye', 'composite']
score_asym = np.empty(shape=(0, 0))
if ('lmks_frms' in attr) and ('pose_param' in attr):
lmks_frms = attr['lmks_frms']
pose_p = attr['pose_param']
if is_vis:
vis_lmks3d(lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx])
score_asym = np.zeros((len(lmks_frms),len(fea_list)+1+1)) # +1: extra column for error code
if is_vis:
fig = plt.figure()
ax = plt.axes(projection="3d")
for s,fi in enumerate(sorted(list(lmks_frms.keys()))):
lmks_3d = lmks_frms[fi][:,2:5]
pose = pose_p[fi]
err_code = error_message_code['pass']
if lmks_frms[fi][0,5] < 0.8:
err_code = error_message_code['confidence less than 80%']
score_asym[s,:] = [fi,np.NaN,np.NaN,np.NaN,np.NaN,err_code]
continue
rx = R.from_euler('x', pose[3])
ry = R.from_euler('y', pose[4])
rz = R.from_euler('z', pose[5])
vec_pose = rz.apply(ry.apply(rx.apply([0,0,1])))
anc_idx = [30, 27, 8] # for central plane estimation
nrm = np.cross(lmks_3d[anc_idx[2],:] - lmks_3d[anc_idx[0],:],
lmks_3d[anc_idx[1],:] - lmks_3d[anc_idx[0],:])
nrm = nrm / np.linalg.norm(nrm)
a,b,c = nrm
d = np.dot(nrm, lmks_3d[anc_idx[0],:])
dist_L2R_mth = []
dist_L2R_ebr = []
dist_L2R_eye = []
dist_com = []
lmks_rfl = np.empty((0,3))
src_idx = lmks_lft_idx
for k,idx in enumerate(src_idx):
p_rfl = np.array(mirror_point(a, b, c, -d, lmks_3d[idx,0], lmks_3d[idx,1], lmks_3d[idx,2]))
lmks_rfl = np.vstack((lmks_rfl, p_rfl))
dist = dist_vec2plane((p_rfl-lmks_3d[lmks_rgt_idx[k],:]), vec_pose)
if idx in lmks_mth_idx:
dist_L2R_mth.append(dist)
if idx in lmks_ebr_idx:
dist_L2R_ebr.append(dist)
if idx in lmks_eye_idx:
dist_L2R_eye.append(dist)
if (idx in lmks_mth_idx) or (idx in lmks_ebr_idx) or (idx in lmks_eye_idx):
dist_com.append(dist)
score_asym[s,:] = [fi,np.mean(dist_L2R_mth),np.mean(dist_L2R_ebr),np.mean(dist_L2R_eye),np.mean(dist_com),err_code]
if is_vis:
ax.scatter(lmks_3d[:,0], lmks_3d[:,1], lmks_3d[:,2])
ax.scatter(lmks_rfl[:,0], lmks_rfl[:,1], lmks_rfl[:,2], c='y')
ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c='c')
plt.title('mirrored landmarks, frame: '+str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
plt.pause(0.2)
plt.cla()
plt.draw()
return score_asym
def calc_asym_feature(open_face_csv, f_cfg):
"""
Calculating facial asymmetry features and preparing final df
"""
df_list = []
of_df = pd.read_csv(open_face_csv, error_bad_lines=False)
lmks_frms, pose_p = retrieve_attr(of_df)
attr = {'lmks_frms': lmks_frms, 'pose_param': pose_p}
score_asym = calc_fac_asymmetry(attr)
df_score_asym = pd.DataFrame(score_asym, columns=['frame', f_cfg.fac_AsymMaskMouth, f_cfg.fac_AsymMaskEyebrow,
f_cfg.fac_AsymMaskEye, f_cfg.fac_AsymMaskCom, f_cfg.err_reason])
df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(lambda x: error_code_message[x])
df_score_asym['frame'] = of_df['frame']
df_score_asym['face_id'] = of_df[' face_id']
df_score_asym['timestamp'] = of_df[' timestamp']
df_score_asym['confidence'] = of_df[' confidence']
df_score_asym['success'] = of_df[' success']
df_list.append(df_score_asym)
return df_list
def run_face_asymmetry(video_uri, out_dir, f_cfg):
"""
Processing all patient's for calculating facial asymmetry
---------------
---------------
Args:
video_uri: video path; f_cfg: face config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
of_csv = of_csv_path[0]
asym_df_list = calc_asym_feature(of_csv, f_cfg)
asym_final_df = pd.concat(asym_df_list, ignore_index=True)
asym_final_df['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext)

94
dbm_lib/dbm_features/raw_features/video/face_au.py Normal file
View File

@@ -0,0 +1,94 @@
"""
file_name: face_au.py
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_au_dir = 'video/face_au'
csv_ext = '_face_au.csv'
def extract_col_nm_au(cols):
"""
Extract action unit (au) column names from openface output (csv)
Args:
cols: column names from open face output (csv)
Returns:
(list) list of au column names
"""
cols_lmk = []
au_tags = ' AU'
cols_au = [c for c in cols if au_tags in c]
return cols_au
def au_col_nm_map(df):
"""
Rename dataframe action unit column names to match functional specifications v1.0
Args:
df: dataframe
Returns:
dataframe with mapped variables
"""
dict_au_cols = {}
for col in list(df):
if ' AU' in col:
idx = col.rfind('_')
if idx > -1:
au_id = col[idx-2:idx]
if '_r' in col:
dict_au_cols[col] = 'fac_AU' + au_id + 'int'
if '_c' in col:
dict_au_cols[col] = 'fac_AU' + au_id + 'pres'
df.rename(columns=dict_au_cols, inplace=True)
return df
def run_face_au(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching action units
---------------
---------------
Args:
video_uri: video path; f_cfg: face config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_au = df_of[extract_col_nm_au(df_of)]
df_au = df_au.copy()
df_au['frame'] = df_of['frame']
df_au['face_id'] = df_of[' face_id']
df_au['timestamp'] = df_of[' timestamp']
df_au['confidence'] = df_of[' confidence']
df_au['success'] = df_of[' success']
df_au = au_col_nm_map(df_au)
df_au['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext)

137
dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py Normal file
View File

@@ -0,0 +1,137 @@
"""
file_name: face_config_reader
project_name: DBM
created: 2020-20-07
"""
import yaml
import boto3
from dbm_lib.dbm_features.raw_features.video import DBMLIB_FACE_CONFIG
class ConfigFaceReader(object):
"""Summary
Read sevice end ponit
"""
def __init__(self,
service_config_yml=None):
"""Summary
Args:
service_config_yml (None, optional): yml file defined service configuration
"""
if service_config_yml is None:
service_config = DBMLIB_FACE_CONFIG
else:
service_config = service_config_yml
with open(service_config, 'r') as ymlfile:
config = yaml.load(ymlfile)
self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS']
self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS']
self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS']
self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS']
self.happiness = config['cdx_face_config']['happiness']
self.sadness = config['cdx_face_config']['sadness']
self.surprise = config['cdx_face_config']['surprise']
self.fear = config['cdx_face_config']['fear']
self.anger = config['cdx_face_config']['anger']
self.disgust = config['cdx_face_config']['disgust']
self.contempt = config['cdx_face_config']['contempt']
self.cai = config['cdx_face_config']['CAI']
self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',')
self.face_expr_dir = config['cdx_face_config']['face_expr_dir']
self.face_asym_dir = config['cdx_face_config']['face_asym_dir']
self.AU_fl = config['cdx_face_config']['AU_filters']
self.au_int = config['cdx_face_config']['au_intensity']
self.au_prs = config['cdx_face_config']['au_presence']
def get_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.ACTION_UNITS
def get_neg_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.NEG_ACTION_UNITS
def get_pos_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.POS_ACTION_UNITS
def get_net_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.NET_ACTION_UNITS
def get_selected_feature(self):
"""Summary
Returns:
TYPE: end point
"""
return self.SELECTED_FEATURES
def get_happiness(self):
"""Summary
Returns:
TYPE: end point
"""
return self.happiness
def get_sadness(self):
"""Summary
Returns:
TYPE: end point
"""
return self.sadness
def get_surprise(self):
"""Summary
Returns:
TYPE: end point
"""
return self.surprise
def get_fear(self):
"""Summary
Returns:
TYPE: end point
"""
return self.fear
def get_anger(self):
"""Summary
Returns:
TYPE: end point
"""
return self.anger
def get_disgust(self):
"""Summary
Returns:
TYPE: end point
"""
return self.disgust
def get_contempt(self):
"""Summary
Returns:
TYPE: end point
"""
return self.contempt
def get_cai(self):
"""Summary
Returns:
TYPE: end point
"""
return self.cai

78
dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py Normal file
View File

@@ -0,0 +1,78 @@
"""
file_name: process_emotion_expressivity
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_expr_dir = 'video/face_expressivity'
csv_ext = '_face_expressivity.csv'
#Openface feature extraction
def of_feature(df_of, cfr, f_cfg):
"""
Creating dataframe for face expressivity
Args:
of: open face attributes
Returns:
(list) list of expressivity score for emotions
"""
df_list = []
df_of['s_confidence'] = vu.smooth(df_of[' confidence'].values, window='flat').tolist()
if 'AU' in cfr.SELECTED_FEATURES :
vu.calc_of_for_video(df_of, cfr, f_cfg)
#Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1)
if len(df_of[f_cfg.neg_exp])>0:
df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp]/5
if len(df_of[f_cfg.com_exp])>0:
df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp]/7
if len(df_of[f_cfg.com_exp_full])>0:
df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full]/7
df_list.append(df_of)
return df_list
def run_face_expressivity(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching facial landmarks
---------------
---------------
Args:
video_uri: video path; f_cfg: raw variable config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_of = df_of[cfr.AU_fl]
expr_df_list = of_feature(df_of, cfr, f_cfg)
exp_final_df = pd.concat(expr_df_list, ignore_index=True)
exp_final_df['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext)

118
dbm_lib/dbm_features/raw_features/video/face_landmark.py Normal file
View File

@@ -0,0 +1,118 @@
"""
file_name: face_landmark
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_lmk_dir = 'video/face_landmark'
csv_ext = '_face_landmark.csv'
def extract_col_nm_lmk(cols):
"""
Extract landmark column names from openface output (csv)
Args:
cols: column names from open face output (csv)
Returns:
(list) list of landmark column names
"""
cols_lmk = []
lmk_tags = [' y_', ' x_', ' X_', ' Y_', ' Z_']
for c in cols:
if any(t in c for t in lmk_tags):
cols_lmk.append(c)
return cols_lmk
def lmk_col_nm_map(df):
"""
Rename dataframe landmark column names to match functional specifications v1.0
Args:
df: dataframe
"""
dict_lmk_cols = {}
for col in list(df):
idx = col.rfind('_')+1
if idx > 0:
lmk_id = col[idx:] if len(col[idx:])>1 else '0'+col[idx:]
if ' y_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'r'
if ' x_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'c'
if ' X_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'X'
if ' Y_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Y'
if ' Z_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Z'
df.rename(columns=dict_lmk_cols, inplace=True)
return df
def add_disp_3D(df):
"""
Add 3D displacement for each landmark
Args:
df: landmark dataframe
"""
df = df.sort_values(by=['frame'], ascending=False)
cols_lmk = [col for col in list(df) if 'fac_LMK' in col]
df_t = df[cols_lmk]
df_diff = df_t.diff()
df_diff = df_diff.pow(2)
tot_lmk = 68 # 68 landmark model
for i in range(tot_lmk):
lmk_id = '{:02d}'.format(i)
df['fac_LMK'+lmk_id+'disp'] = df_diff[['fac_LMK'+lmk_id+'X', 'fac_LMK'+lmk_id+'Y', 'fac_LMK'+lmk_id+'Z']].sum(axis=1).apply(np.sqrt)
return df
def run_face_landmark(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching facial landmarks
---------------
---------------
Args:
video_uri: video path; f_cfg: raw variable config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_lmk = df_of[extract_col_nm_lmk(df_of)]
df_lmk = df_lmk.copy()
df_lmk['frame'] = df_of['frame']
df_lmk['face_id'] = df_of[' face_id']
df_lmk['timestamp'] = df_of[' timestamp']
df_lmk['confidence'] = df_of[' confidence']
df_lmk['success'] = df_of[' success']
df_lmk = lmk_col_nm_map(df_lmk)
df_lmk = add_disp_3D(df_lmk)
df_lmk['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(join(out_loc, fl_name)))
ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)

67
dbm_lib/dbm_features/raw_features/video/open_face_process.py Normal file
View File

@@ -0,0 +1,67 @@
"""
file_name: process_features
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import glob
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path):
""" Computes open_face features for the files in filepaths
Args:
-----
filepaths: (itreable[str])
video_tracking: To specify whether openface's video tracking module (FaceLandmarkVid)
is being used or the default (FeatureExtract)
video_url: Raw video location on S3 bucket
input_dir: Path to the input videos
out_dir: Path to the processed output
of_path: OpenFace source code path
Returns:
--------
(itreable[str]) list of .csv files
"""
suffix = '_OF_features'
csv_files = []
for fp in filepaths:
try:
_, out_loc, fl_name = ut.filter_path(video_url, out_dir)
full_f_name = fl_name + suffix
output_directory = os.path.join(out_loc, full_f_name)
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
except Exception as e:
logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
return csv_files
def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group):
"""
Processing all patient's for fetching emotion expressivity
-------------------
-------------------
Args:
video_uri: video path; input_dir : input directory for video's; dbm_group: feature group
out_dir: (str) Output directory for processed output; of_path: OpenFace source code path
"""
if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group:
return
filepaths = [video_uri]
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path)