From a07659b2064bc351b0b110ced1f427d5bd9c380b Mon Sep 17 00:00:00 2001 From: "jordi.hasianta" Date: Thu, 15 Sep 2022 20:52:16 +0700 Subject: [PATCH] facial code refactoring --- .../raw_features/video/face_asymmetry.py | 526 +++++++++++------- .../raw_features/video/face_au.py | 78 +-- .../video/face_config/face_config_reader.py | 70 +-- .../video/face_emotion_expressivity.py | 88 +-- .../raw_features/video/face_landmark.py | 105 ++-- .../raw_features/video/open_face_process.py | 55 +- 6 files changed, 549 insertions(+), 373 deletions(-) diff --git a/opendbm/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py b/opendbm/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py index 9e1bb9c3..fbddd96c 100644 --- a/opendbm/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py +++ b/opendbm/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py @@ -4,195 +4,231 @@ project_name: DBM created: 2020-20-07 """ -from mpl_toolkits import mplot3d -from matplotlib import pyplot as plt -import time -import numpy as np -import os import datetime import glob -import cv2 -from scipy.spatial.transform import Rotation as R -import subprocess -import pandas as pd -from os.path import join import logging +import os +import subprocess +import time +from os.path import join -from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader -from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut +import cv2 +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt +from mpl_toolkits import mplot3d +from scipy.spatial.transform import Rotation as R + +from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut +from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu + +from .face_config.face_config_reader import ConfigFaceReader logging.basicConfig(level=logging.INFO) -logger=logging.getLogger() +logger = logging.getLogger() -face_asym_dir = 'facial/face_asymmetry' -csv_ext = '_facasym.csv' +face_asym_dir = "facial/face_asymmetry" +csv_ext = "_facasym.csv" -cv2_color_purple = (254,19,188) -color_blue = (0,0,1.0) -color_green = (0,1.0,0) -color_red = (1.0,0,0) -color_y = (1.0,1.0,0) +cv2_color_purple = (254, 19, 188) +color_blue = (0, 0, 1.0) +color_green = (0, 1.0, 0) +color_red = (1.0, 0, 0) +color_y = (1.0, 1.0, 0) error_code_message = { - 0: 'pass', - 1: 'confidence less than 80%', - } -error_message_code = {y:x for x,y in error_code_message.items()} + 0: "pass", + 1: "confidence less than 80%", +} +error_message_code = {y: x for x, y in error_code_message.items()} + def visualize_vid(fn, attr=None, write_out=False): vid = cv2.VideoCapture(fn) - tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT)) + # tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT)) fps = vid.get(cv2.CAP_PROP_FPS) - frame_width = int(vid.get(3)) - frame_height = int(vid.get(4)) + # frame_width = int(vid.get(3)) + # frame_height = int(vid.get(4)) if write_out: - fig_w = 680 #680 667 676 #frame_width in order of Ali, Vennessa, synthesis - fig_h = 659 #659 659 659 #frame_height - out_vid = cv2.VideoWriter('out.mp4',cv2.VideoWriter_fourcc(*'MP4V'), fps, (fig_w,fig_h)) + fig_w = 680 # 680 667 676 #frame_width in order of Ali, Vennessa, synthesis + fig_h = 659 # 659 659 659 #frame_height + out_vid = cv2.VideoWriter( + "out.mp4", cv2.VideoWriter_fourcc(*"MP4V"), fps, (fig_w, fig_h) + ) plt.figure(figsize=(8, 8)) try: frameid = 0 - while(True): + while True: ret, frame = vid.read() if not ret: # Release the Video Device if ret is false vid.release() - print('Released Video Resource') + print("Released Video Resource") break frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frameid += 1 logger.info(frameid, frame.shape) - - if 'lmks_frms' in attr: - lmks_frms = attr['lmks_frms'] + + if "lmks_frms" in attr: + lmks_frms = attr["lmks_frms"] for i in range(lmks_frms[frameid].shape[0]): - cv2.circle(frame,(int(lmks_frms[frameid][i,0]),int(lmks_frms[frameid][i,1])), 2, cv2_color_purple, -1) + cv2.circle( + frame, + (int(lmks_frms[frameid][i, 0]), int(lmks_frms[frameid][i, 1])), + 2, + cv2_color_purple, + -1, + ) if write_out: - cv2.putText(frame,'Frame: '+str(frameid), (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 3) + cv2.putText( + frame, + "Frame: " + str(frameid), + (10, 50), + cv2.FONT_HERSHEY_SIMPLEX, + 1, + (255, 255, 255), + 3, + ) plt.subplot(211) plt.imshow(frame) - plt.axis('off'); plt.pause(0.2); - - if 'score_asym' in attr: + plt.axis("off") + plt.pause(0.2) + + if "score_asym" in attr: ax = plt.subplot(212) ax.cla() - ax.set_xlim(0,140) #ax.set_xlim(0,300) - ax.set_ylim(0,10) - - sa = attr['score_asym'] - s = sa[np.where(sa[:,0] <= frameid),:][0,:,:] - - for i in range(1,s.shape[1]): - plt.plot(s[:,0], s[:,i]) - - plt.legend(['mouth', 'eyebrow', 'eye', 'mouth+eye+eyebrow']) - plt.minorticks_on() - plt.grid(b=True, which='major', color='r', linestyle='-') - plt.grid(b=True, which='minor', color='r', linestyle='--') + ax.set_xlim(0, 140) # ax.set_xlim(0,300) + ax.set_ylim(0, 10) + + sa = attr["score_asym"] + s = sa[np.where(sa[:, 0] <= frameid), :][0, :, :] + + for i in range(1, s.shape[1]): + plt.plot(s[:, 0], s[:, i]) + + plt.legend(["mouth", "eyebrow", "eye", "mouth+eye+eyebrow"]) + plt.minorticks_on() + plt.grid(b=True, which="major", color="r", linestyle="-") + plt.grid(b=True, which="minor", color="r", linestyle="--") + + plt.savefig("tmp.png", bbox_inches="tight") + print(cv2.imread("tmp.png").shape) - plt.savefig('tmp.png', bbox_inches='tight') - print(cv2.imread('tmp.png').shape) - plt.clf() if write_out: - out_vid.write(cv2.imread('tmp.png')) + out_vid.write(cv2.imread("tmp.png")) except KeyboardInterrupt: # Release the Video Device vid.release() if write_out: out_vid.release() - logger.info('Exception, and Video Resource Released') - + logger.info("Exception, and Video Resource Released") + if write_out: out_vid.release() - + def retrieve_attr(of_df): - ''' - Retrieve landmarks and pose_translation for each frame from openface output - Args: - of_df: dataframe output from openface, including detected landmark coordinates - Returns: - lmks_frms: dictionary, with frame id as key and 68 landmark set as value - pose_p: dictionary, with frame id as key and pose param as value - ''' - tot_lmks = 68 # openface specific - if len([i for i in of_df.columns.to_list() if ' x_' in i]) != tot_lmks: + """ + Retrieve landmarks and pose_translation for each frame from openface output + Args: + of_df: dataframe output from openface, including detected landmark coordinates + Returns: + lmks_frms: dictionary, with frame id as key and 68 landmark set as value + pose_p: dictionary, with frame id as key and pose param as value + """ + tot_lmks = 68 # openface specific + if len([i for i in of_df.columns.to_list() if " x_" in i]) != tot_lmks: return {} - + lmks_frms = {} pose_p = {} - - for fi in sorted(of_df['frame'].to_list()): - lmks = np.zeros((tot_lmks,6)) - r = of_df[of_df['frame']==fi] - + + for fi in sorted(of_df["frame"].to_list()): + lmks = np.zeros((tot_lmks, 6)) + r = of_df[of_df["frame"] == fi] + for i in range(tot_lmks): - lmk_y = r[' y_'+str(i)].iloc[0] - lmk_x = r[' x_'+str(i)].iloc[0] - lmk_X = r[' X_'+str(i)].iloc[0] - lmk_Y = r[' Y_'+str(i)].iloc[0] - lmk_Z = r[' Z_'+str(i)].iloc[0] - - confi = r[' confidence'] - lmks[i,:] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi] - + lmk_y = r[" y_" + str(i)].iloc[0] + lmk_x = r[" x_" + str(i)].iloc[0] + lmk_X = r[" X_" + str(i)].iloc[0] + lmk_Y = r[" Y_" + str(i)].iloc[0] + lmk_Z = r[" Z_" + str(i)].iloc[0] + + confi = r[" confidence"] + lmks[i, :] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi] + lmks_frms[fi] = lmks - pose_p[fi] = [r[' pose_Tx'].iloc[0], r[' pose_Ty'].iloc[0], r[' pose_Tz'].iloc[0], - r[' pose_Rx'].iloc[0], r[' pose_Ry'].iloc[0], r[' pose_Rz'].iloc[0]] - + pose_p[fi] = [ + r[" pose_Tx"].iloc[0], + r[" pose_Ty"].iloc[0], + r[" pose_Tz"].iloc[0], + r[" pose_Rx"].iloc[0], + r[" pose_Ry"].iloc[0], + r[" pose_Rz"].iloc[0], + ] + return lmks_frms, pose_p -def mirror_point(a, b, c, d, x1, y1, z1): +def mirror_point(a, b, c, d, x1, y1, z1): # mirror a point w.r.t a 3D plane - k =(-a * x1-b * y1-c * z1-d)/float((a * a + b * b + c * c)) - - x2 = a * k + x1 - y2 = b * k + y1 - z2 = c * k + z1 - - x3 = 2 * x2-x1 - y3 = 2 * y2-y1 - z3 = 2 * z2-z1 + k = (-a * x1 - b * y1 - c * z1 - d) / float((a * a + b * b + c * c)) + + x2 = a * k + x1 + y2 = b * k + y1 + z2 = c * k + z1 + + x3 = 2 * x2 - x1 + y3 = 2 * y2 - y1 + z3 = 2 * z2 - z1 return [x3, y3, z3] def dist_vec2plane(vec, nrm): # Calculate the projected length of a vector (vec) to a plane defined by its normal (nrm) - return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm)**2) + return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm) ** 2) def vis_lmks3d(lmks_frms, vis_idx): """ Visualizing facial landmarks """ - fig = plt.figure() - color_type = ['b','g','r','y','c'] + # fig = plt.figure() + color_type = ["b", "g", "r", "y", "c"] assert len(color_type) > len(vis_idx) - + for fi in sorted(list(lmks_frms.keys())): ax = plt.axes(projection="3d") - for i,vi in enumerate(vis_idx): - ax.scatter(lmks_frms[fi][vi,2], lmks_frms[fi][vi,3], lmks_frms[fi][vi,4], c=color_type[i]) - - ax.axes.set_xlim3d(left=-75, right=100) - ax.axes.set_ylim3d(bottom=-200, top=25) + for i, vi in enumerate(vis_idx): + ax.scatter( + lmks_frms[fi][vi, 2], + lmks_frms[fi][vi, 3], + lmks_frms[fi][vi, 4], + c=color_type[i], + ) + + ax.axes.set_xlim3d(left=-75, right=100) + ax.axes.set_ylim3d(bottom=-200, top=25) ax.axes.set_zlim3d(bottom=440, top=560) - ax.view_init(-89, -90) #elev, ariz - plt.title(str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z') + ax.view_init(-89, -90) # elev, ariz + plt.title(str(fi)) + ax.set_xlabel("X") + ax.set_ylabel("Y") + ax.set_zlabel("Z") plt.pause(0.2) plt.cla() plt.draw() + def calc_fac_asymmetry(attr, is_vis=False): - ''' + """ Quantify facial asymmetry Args: attr: attribute dictionary containing necessary features for calculation, e.g., @@ -200,96 +236,164 @@ def calc_fac_asymmetry(attr, is_vis=False): pose_param: dictionary, with frame id as key and pose param as value Returns: score_asym: 2D array of size (num_frms, num_asymm_fea), with frame id as the 0th column, and each remaining column as one asymmetry feature - ''' + """ # openface landmark indices - lmks_ref_idx = list(range(0,17)) + list(range(27,36)) - lmks_mid_idx = [27,28,29,30,33,51,62,66,57,8] - lmks_rgt_idx = [0,1,2,3,4,5,6,7, - 17,18,19,20,21, - 36,37,38,39,40,41, - 48,49,50, - 59,58, - 60,61, - 67] - lmks_lft_idx = [16,15,14,13,12,11,10,9, - 26,25,24,23,22, - 45,44,43,42,47,46, - 54,53,52, - 55,56, - 64,63, - 65] - - lmks_mth_idx = list(range(48,68)) - lmks_ebr_idx = list(range(17,27)) - lmks_eye_idx = list(range(36,48)) - assert len(lmks_lft_idx)==len(lmks_rgt_idx) + lmks_ref_idx = list(range(0, 17)) + list(range(27, 36)) + lmks_mid_idx = [27, 28, 29, 30, 33, 51, 62, 66, 57, 8] + lmks_rgt_idx = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 17, + 18, + 19, + 20, + 21, + 36, + 37, + 38, + 39, + 40, + 41, + 48, + 49, + 50, + 59, + 58, + 60, + 61, + 67, + ] + lmks_lft_idx = [ + 16, + 15, + 14, + 13, + 12, + 11, + 10, + 9, + 26, + 25, + 24, + 23, + 22, + 45, + 44, + 43, + 42, + 47, + 46, + 54, + 53, + 52, + 55, + 56, + 64, + 63, + 65, + ] - fea_list = ['mouth', 'eyebrow', 'eye', 'composite'] + lmks_mth_idx = list(range(48, 68)) + lmks_ebr_idx = list(range(17, 27)) + lmks_eye_idx = list(range(36, 48)) + assert len(lmks_lft_idx) == len(lmks_rgt_idx) + + fea_list = ["mouth", "eyebrow", "eye", "composite"] score_asym = np.empty(shape=(0, 0)) - - if ('lmks_frms' in attr) and ('pose_param' in attr): - lmks_frms = attr['lmks_frms'] - pose_p = attr['pose_param'] + + if ("lmks_frms" in attr) and ("pose_param" in attr): + lmks_frms = attr["lmks_frms"] + pose_p = attr["pose_param"] if is_vis: - vis_lmks3d(lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx]) - - score_asym = np.zeros((len(lmks_frms),len(fea_list)+1+1)) # +1: extra column for error code + vis_lmks3d( + lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx] + ) + + score_asym = np.zeros( + (len(lmks_frms), len(fea_list) + 1 + 1) + ) # +1: extra column for error code if is_vis: - fig = plt.figure() + # fig = plt.figure() ax = plt.axes(projection="3d") - - for s,fi in enumerate(sorted(list(lmks_frms.keys()))): - lmks_3d = lmks_frms[fi][:,2:5] + + for s, fi in enumerate(sorted(list(lmks_frms.keys()))): + lmks_3d = lmks_frms[fi][:, 2:5] pose = pose_p[fi] - err_code = error_message_code['pass'] - - if lmks_frms[fi][0,5] < 0.8: - err_code = error_message_code['confidence less than 80%'] - score_asym[s,:] = [fi,np.NaN,np.NaN,np.NaN,np.NaN,err_code] + err_code = error_message_code["pass"] + + if lmks_frms[fi][0, 5] < 0.8: + err_code = error_message_code["confidence less than 80%"] + score_asym[s, :] = [fi, np.NaN, np.NaN, np.NaN, np.NaN, err_code] continue - rx = R.from_euler('x', pose[3]) - ry = R.from_euler('y', pose[4]) - rz = R.from_euler('z', pose[5]) - - vec_pose = rz.apply(ry.apply(rx.apply([0,0,1]))) - anc_idx = [30, 27, 8] # for central plane estimation - nrm = np.cross(lmks_3d[anc_idx[2],:] - lmks_3d[anc_idx[0],:], - lmks_3d[anc_idx[1],:] - lmks_3d[anc_idx[0],:]) - + rx = R.from_euler("x", pose[3]) + ry = R.from_euler("y", pose[4]) + rz = R.from_euler("z", pose[5]) + + vec_pose = rz.apply(ry.apply(rx.apply([0, 0, 1]))) + anc_idx = [30, 27, 8] # for central plane estimation + nrm = np.cross( + lmks_3d[anc_idx[2], :] - lmks_3d[anc_idx[0], :], + lmks_3d[anc_idx[1], :] - lmks_3d[anc_idx[0], :], + ) + nrm = nrm / np.linalg.norm(nrm) - a,b,c = nrm - d = np.dot(nrm, lmks_3d[anc_idx[0],:]) - + a, b, c = nrm + d = np.dot(nrm, lmks_3d[anc_idx[0], :]) + dist_L2R_mth = [] dist_L2R_ebr = [] dist_L2R_eye = [] dist_com = [] - - lmks_rfl = np.empty((0,3)) + + lmks_rfl = np.empty((0, 3)) src_idx = lmks_lft_idx - - for k,idx in enumerate(src_idx): - p_rfl = np.array(mirror_point(a, b, c, -d, lmks_3d[idx,0], lmks_3d[idx,1], lmks_3d[idx,2])) + + for k, idx in enumerate(src_idx): + p_rfl = np.array( + mirror_point( + a, b, c, -d, lmks_3d[idx, 0], lmks_3d[idx, 1], lmks_3d[idx, 2] + ) + ) lmks_rfl = np.vstack((lmks_rfl, p_rfl)) - dist = dist_vec2plane((p_rfl-lmks_3d[lmks_rgt_idx[k],:]), vec_pose) - - if idx in lmks_mth_idx: + dist = dist_vec2plane((p_rfl - lmks_3d[lmks_rgt_idx[k], :]), vec_pose) + + if idx in lmks_mth_idx: dist_L2R_mth.append(dist) if idx in lmks_ebr_idx: dist_L2R_ebr.append(dist) if idx in lmks_eye_idx: dist_L2R_eye.append(dist) - if (idx in lmks_mth_idx) or (idx in lmks_ebr_idx) or (idx in lmks_eye_idx): - dist_com.append(dist) - score_asym[s,:] = [fi,np.mean(dist_L2R_mth),np.mean(dist_L2R_ebr),np.mean(dist_L2R_eye),np.mean(dist_com),err_code] + if ( + (idx in lmks_mth_idx) + or (idx in lmks_ebr_idx) + or (idx in lmks_eye_idx) + ): + dist_com.append(dist) + score_asym[s, :] = [ + fi, + np.mean(dist_L2R_mth), + np.mean(dist_L2R_ebr), + np.mean(dist_L2R_eye), + np.mean(dist_com), + err_code, + ] if is_vis: - ax.scatter(lmks_3d[:,0], lmks_3d[:,1], lmks_3d[:,2]) - ax.scatter(lmks_rfl[:,0], lmks_rfl[:,1], lmks_rfl[:,2], c='y') - ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c='c') - plt.title('mirrored landmarks, frame: '+str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z') + ax.scatter(lmks_3d[:, 0], lmks_3d[:, 1], lmks_3d[:, 2]) + ax.scatter(lmks_rfl[:, 0], lmks_rfl[:, 1], lmks_rfl[:, 2], c="y") + ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c="c") + plt.title("mirrored landmarks, frame: " + str(fi)) + ax.set_xlabel("X") + ax.set_ylabel("Y") + ax.set_zlabel("Z") plt.pause(0.2) plt.cla() plt.draw() @@ -302,28 +406,39 @@ def calc_asym_feature(open_face_csv, f_cfg): Calculating facial asymmetry features and preparing final df """ df_list = [] - - of_df = pd.read_csv(open_face_csv, error_bad_lines=False) + + of_df = pd.read_csv(open_face_csv) lmks_frms, pose_p = retrieve_attr(of_df) - - attr = {'lmks_frms': lmks_frms, 'pose_param': pose_p} + + attr = {"lmks_frms": lmks_frms, "pose_param": pose_p} score_asym = calc_fac_asymmetry(attr) - - df_score_asym = pd.DataFrame(score_asym, columns=['frame', f_cfg.fac_AsymMaskMouth, f_cfg.fac_AsymMaskEyebrow, - f_cfg.fac_AsymMaskEye, f_cfg.fac_AsymMaskCom, f_cfg.err_reason]) - df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(lambda x: error_code_message[x]) - - df_score_asym['frame'] = of_df['frame'] - df_score_asym['face_id'] = of_df[' face_id'] - df_score_asym['timestamp'] = of_df[' timestamp'] - df_score_asym['confidence'] = of_df[' confidence'] - df_score_asym['success'] = of_df[' success'] - + + df_score_asym = pd.DataFrame( + score_asym, + columns=[ + "frame", + f_cfg.fac_AsymMaskMouth, + f_cfg.fac_AsymMaskEyebrow, + f_cfg.fac_AsymMaskEye, + f_cfg.fac_AsymMaskCom, + f_cfg.err_reason, + ], + ) + df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply( + lambda x: error_code_message[x] + ) + + df_score_asym["frame"] = of_df["frame"] + df_score_asym["face_id"] = of_df[" face_id"] + df_score_asym["timestamp"] = of_df[" timestamp"] + df_score_asym["confidence"] = of_df[" confidence"] + df_score_asym["success"] = of_df[" success"] + df_list.append(df_score_asym) - return df_list + return df_list -def run_face_asymmetry(video_uri, out_dir, f_cfg): +def run_face_asymmetry(video_uri, out_dir, f_cfg, save=True): """ Processing all patient's for calculating facial asymmetry --------------- @@ -333,22 +448,27 @@ def run_face_asymmetry(video_uri, out_dir, f_cfg): out_dir: (str) Output directory for processed output """ try: - - #Baseline logic - cfr = ConfigFaceReader() + + # Baseline logic + ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) - if len(of_csv_path)>0: + of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv")) + if len(of_csv_path) > 0: of_csv = of_csv_path[0] asym_df_list = calc_asym_feature(of_csv, f_cfg) asym_final_df = pd.concat(asym_df_list, ignore_index=True) - asym_final_df['dbm_master_url'] = video_uri + asym_final_df["dbm_master_url"] = video_uri + + if save: + logger.info( + "Processing Output file {} ".format(os.path.join(out_loc, fl_name)) + ) + ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext) + return asym_final_df - logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) - ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext) - except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + e + logger.error("Failed to process video file") diff --git a/opendbm/dbm_lib/dbm_features/raw_features/video/face_au.py b/opendbm/dbm_lib/dbm_features/raw_features/video/face_au.py index eb208714..a313e097 100644 --- a/opendbm/dbm_lib/dbm_features/raw_features/video/face_au.py +++ b/opendbm/dbm_lib/dbm_features/raw_features/video/face_au.py @@ -4,22 +4,25 @@ project_name: DBM created: 2020-20-07 """ -import os -import numpy as np -import pandas as pd import datetime import glob -from os.path import join import logging +import os +from os.path import join -from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader -from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut +import numpy as np +import pandas as pd + +from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut +from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu + +from .face_config.face_config_reader import ConfigFaceReader logging.basicConfig(level=logging.INFO) -logger=logging.getLogger() +logger = logging.getLogger() -face_au_dir = 'facial/face_au' -csv_ext = '_facau.csv' +face_au_dir = "facial/face_au" +csv_ext = "_facau.csv" def extract_col_nm_au(cols): @@ -30,8 +33,8 @@ def extract_col_nm_au(cols): Returns: (list) list of au column names """ - cols_lmk = [] - au_tags = ' AU' + # cols_lmk = [] + au_tags = " AU" cols_au = [c for c in cols if au_tags in c] return cols_au @@ -46,19 +49,19 @@ def au_col_nm_map(df): """ dict_au_cols = {} for col in list(df): - if ' AU' in col: - idx = col.rfind('_') + if " AU" in col: + idx = col.rfind("_") if idx > -1: - au_id = col[idx-2:idx] - if '_r' in col: - dict_au_cols[col] = 'fac_AU' + au_id + 'int' - if '_c' in col: - dict_au_cols[col] = 'fac_AU' + au_id + 'pres' + au_id = col[idx - 2 : idx] + if "_r" in col: + dict_au_cols[col] = "fac_AU" + au_id + "int" + if "_c" in col: + dict_au_cols[col] = "fac_AU" + au_id + "pres" df.rename(columns=dict_au_cols, inplace=True) return df -def run_face_au(video_uri, out_dir, f_cfg): +def run_face_au(video_uri, out_dir, f_cfg, save=True): """ Processing all patient's for fetching action units --------------- @@ -68,30 +71,33 @@ def run_face_au(video_uri, out_dir, f_cfg): out_dir: (str) Output directory for processed output """ try: - - #Baseline logic - cfr = ConfigFaceReader() + + # Baseline logic + ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) - if len(of_csv_path)>0: + of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv")) + if len(of_csv_path) > 0: - df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) + df_of = pd.read_csv(of_csv_path[0]) df_au = df_of[extract_col_nm_au(df_of)] df_au = df_au.copy() - df_au['frame'] = df_of['frame'] - df_au['face_id'] = df_of[' face_id'] - df_au['timestamp'] = df_of[' timestamp'] - df_au['confidence'] = df_of[' confidence'] - df_au['success'] = df_of[' success'] + df_au["frame"] = df_of["frame"] + df_au["face_id"] = df_of[" face_id"] + df_au["timestamp"] = df_of[" timestamp"] + df_au["confidence"] = df_of[" confidence"] + df_au["success"] = df_of[" success"] df_au = au_col_nm_map(df_au) - df_au['dbm_master_url'] = video_uri + df_au["dbm_master_url"] = video_uri + if save: + logger.info( + "Processing Output file {} ".format(os.path.join(out_loc, fl_name)) + ) + ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext) + return df_au - logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) - ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext) - except Exception as e: - logger.error('Failed to process video file') - \ No newline at end of file + e + logger.error("Failed to process video file") diff --git a/opendbm/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py b/opendbm/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py index e3777db7..f7e97c4f 100644 --- a/opendbm/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py +++ b/opendbm/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py @@ -4,50 +4,58 @@ project_name: DBM created: 2020-20-07 """ +import os + import yaml -import boto3 -from opendbm.dbm_lib.dbm_features.raw_features.video import DBMLIB_FACE_CONFIG + +DBMLIB_PATH = os.path.dirname(__file__) +DBMLIB_FACE_CONFIG = os.path.abspath( + os.path.join(DBMLIB_PATH, "../../../../../resources/services/face_util.yml") +) + class ConfigFaceReader(object): """Summary Read sevice end ponit """ - def __init__(self, - service_config_yml=None): + + def __init__(self, service_config_yml=None): """Summary Args: service_config_yml (None, optional): yml file defined service configuration """ - + if service_config_yml is None: service_config = DBMLIB_FACE_CONFIG else: service_config = service_config_yml - - with open(service_config, 'r') as ymlfile: - config = yaml.load(ymlfile) - self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS'] - self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS'] - self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS'] - self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS'] - self.LOWER_ACTION_UNITS = config['cdx_face_config']['LOWER_ACTION_UNITS'] - self.UPPER_ACTION_UNITS = config['cdx_face_config']['UPPER_ACTION_UNITS'] - self.happiness = config['cdx_face_config']['happiness'] - self.sadness = config['cdx_face_config']['sadness'] - self.surprise = config['cdx_face_config']['surprise'] - self.fear = config['cdx_face_config']['fear'] - self.anger = config['cdx_face_config']['anger'] - self.disgust = config['cdx_face_config']['disgust'] - self.contempt = config['cdx_face_config']['contempt'] - self.pain = config['cdx_face_config']['pain'] - self.cai = config['cdx_face_config']['CAI'] - self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',') - self.face_expr_dir = config['cdx_face_config']['face_expr_dir'] - self.face_asym_dir = config['cdx_face_config']['face_asym_dir'] - self.AU_fl = config['cdx_face_config']['AU_filters'] - self.au_int = config['cdx_face_config']['au_intensity'] - self.au_prs = config['cdx_face_config']['au_presence'] - + + with open(service_config, "r") as ymlfile: + config = yaml.load(ymlfile, Loader=yaml.CLoader) + self.ACTION_UNITS = config["cdx_face_config"]["ACTION_UNITS"] + self.NEG_ACTION_UNITS = config["cdx_face_config"]["NEG_ACTION_UNITS"] + self.POS_ACTION_UNITS = config["cdx_face_config"]["POS_ACTION_UNITS"] + self.NET_ACTION_UNITS = config["cdx_face_config"]["NET_ACTION_UNITS"] + self.LOWER_ACTION_UNITS = config["cdx_face_config"]["LOWER_ACTION_UNITS"] + self.UPPER_ACTION_UNITS = config["cdx_face_config"]["UPPER_ACTION_UNITS"] + self.happiness = config["cdx_face_config"]["happiness"] + self.sadness = config["cdx_face_config"]["sadness"] + self.surprise = config["cdx_face_config"]["surprise"] + self.fear = config["cdx_face_config"]["fear"] + self.anger = config["cdx_face_config"]["anger"] + self.disgust = config["cdx_face_config"]["disgust"] + self.contempt = config["cdx_face_config"]["contempt"] + self.pain = config["cdx_face_config"]["pain"] + self.cai = config["cdx_face_config"]["CAI"] + self.SELECTED_FEATURES = config["cdx_face_config"][ + "SELECTED_FEATURES" + ].split(",") + self.face_expr_dir = config["cdx_face_config"]["face_expr_dir"] + self.face_asym_dir = config["cdx_face_config"]["face_asym_dir"] + self.AU_fl = config["cdx_face_config"]["AU_filters"] + self.au_int = config["cdx_face_config"]["au_intensity"] + self.au_prs = config["cdx_face_config"]["au_presence"] + def get_action_unit(self): """Summary Returns: @@ -137,4 +145,4 @@ class ConfigFaceReader(object): Returns: TYPE: end point """ - return self.cai \ No newline at end of file + return self.cai diff --git a/opendbm/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py b/opendbm/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py index d053717d..3a63c7eb 100644 --- a/opendbm/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py +++ b/opendbm/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py @@ -4,24 +4,27 @@ project_name: DBM created: 2020-20-07 """ -import os -import numpy as np -import pandas as pd import datetime import glob -from os.path import join import logging +import os +from os.path import join -from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader -from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut +import numpy as np +import pandas as pd + +from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut +from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu + +from .face_config.face_config_reader import ConfigFaceReader logging.basicConfig(level=logging.INFO) -logger=logging.getLogger() +logger = logging.getLogger() -face_expr_dir = 'facial/face_expressivity' -csv_ext = '_facemo.csv' +face_expr_dir = "facial/face_expressivity" +csv_ext = "_facemo.csv" -#Openface feature extraction +# Openface feature extraction def of_feature(df_of, cfr, f_cfg): """ Creating dataframe for face expressivity @@ -31,29 +34,31 @@ def of_feature(df_of, cfr, f_cfg): (list) list of expressivity score for emotions """ df_list = [] - df_of['s_confidence'] = vu.smooth(df_of[' confidence'].values, window='flat').tolist() - - if 'AU' in cfr.SELECTED_FEATURES : + df_of["s_confidence"] = vu.smooth( + df_of[" confidence"].values, window="flat" + ).tolist() + + if "AU" in cfr.SELECTED_FEATURES: vu.calc_of_for_video(df_of, cfr, f_cfg) - #Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1) - - if len(df_of[f_cfg.neg_exp])>0: - df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp]/5 - - if len(df_of[f_cfg.neg_exp_full])>0: - df_of[f_cfg.neg_exp_full] = df_of[f_cfg.neg_exp_full]/5 - - if len(df_of[f_cfg.com_exp])>0: - df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp]/7 - - if len(df_of[f_cfg.com_exp_full])>0: - df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full]/7 - + # Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1) + + if len(df_of[f_cfg.neg_exp]) > 0: + df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp] / 5 + + if len(df_of[f_cfg.neg_exp_full]) > 0: + df_of[f_cfg.neg_exp_full] = df_of[f_cfg.neg_exp_full] / 5 + + if len(df_of[f_cfg.com_exp]) > 0: + df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp] / 7 + + if len(df_of[f_cfg.com_exp_full]) > 0: + df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full] / 7 + df_list.append(df_of) - return df_list + return df_list -def run_face_expressivity(video_uri, out_dir, f_cfg): +def run_face_expressivity(video_uri, out_dir, f_cfg, save=True): """ Processing all patient's for fetching facial landmarks --------------- @@ -63,23 +68,28 @@ def run_face_expressivity(video_uri, out_dir, f_cfg): out_dir: (str) Output directory for processed output """ try: - - #Baseline logic + + # Baseline logic cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) - if len(of_csv_path)>0: + of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv")) + if len(of_csv_path) > 0: - df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) + df_of = pd.read_csv(of_csv_path[0]) df_of = df_of[cfr.AU_fl] expr_df_list = of_feature(df_of, cfr, f_cfg) exp_final_df = pd.concat(expr_df_list, ignore_index=True) - exp_final_df['dbm_master_url'] = video_uri + exp_final_df["dbm_master_url"] = video_uri + + if save: + logger.info( + "Processing Output file {} ".format(os.path.join(out_loc, fl_name)) + ) + ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext) + return exp_final_df - logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) - ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext) - except Exception as e: - logger.error('Failed to process video file') + e + logger.error("Failed to process video file") diff --git a/opendbm/dbm_lib/dbm_features/raw_features/video/face_landmark.py b/opendbm/dbm_lib/dbm_features/raw_features/video/face_landmark.py index 33550391..6b92f9f4 100644 --- a/opendbm/dbm_lib/dbm_features/raw_features/video/face_landmark.py +++ b/opendbm/dbm_lib/dbm_features/raw_features/video/face_landmark.py @@ -4,22 +4,26 @@ project_name: DBM created: 2020-20-07 """ -import os -import numpy as np -import pandas as pd import datetime import glob -from os.path import join import logging +import os +from os.path import join -from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader -from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut +import numpy as np +import pandas as pd + +from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut +from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu + +from .face_config.face_config_reader import ConfigFaceReader logging.basicConfig(level=logging.INFO) -logger=logging.getLogger() +logger = logging.getLogger() + +face_lmk_dir = "facial/face_landmark" +csv_ext = "_faclmk.csv" -face_lmk_dir = 'facial/face_landmark' -csv_ext = '_faclmk.csv' def extract_col_nm_lmk(cols): """ @@ -30,7 +34,7 @@ def extract_col_nm_lmk(cols): (list) list of landmark column names """ cols_lmk = [] - lmk_tags = [' y_', ' x_', ' X_', ' Y_', ' Z_'] + lmk_tags = [" y_", " x_", " X_", " Y_", " Z_"] for c in cols: if any(t in c for t in lmk_tags): cols_lmk.append(c) @@ -45,19 +49,19 @@ def lmk_col_nm_map(df): """ dict_lmk_cols = {} for col in list(df): - idx = col.rfind('_')+1 + idx = col.rfind("_") + 1 if idx > 0: - lmk_id = col[idx:] if len(col[idx:])>1 else '0'+col[idx:] - if ' y_' in col: - dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'r' - if ' x_' in col: - dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'c' - if ' X_' in col: - dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'X' - if ' Y_' in col: - dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Y' - if ' Z_' in col: - dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Z' + lmk_id = col[idx:] if len(col[idx:]) > 1 else "0" + col[idx:] + if " y_" in col: + dict_lmk_cols[col] = "fac_LMK" + lmk_id + "r" + if " x_" in col: + dict_lmk_cols[col] = "fac_LMK" + lmk_id + "c" + if " X_" in col: + dict_lmk_cols[col] = "fac_LMK" + lmk_id + "X" + if " Y_" in col: + dict_lmk_cols[col] = "fac_LMK" + lmk_id + "Y" + if " Z_" in col: + dict_lmk_cols[col] = "fac_LMK" + lmk_id + "Z" df.rename(columns=dict_lmk_cols, inplace=True) return df @@ -68,21 +72,31 @@ def add_disp_3D(df): Args: df: landmark dataframe """ - df = df.sort_values(by=['frame'], ascending=False) - cols_lmk = [col for col in list(df) if 'fac_LMK' in col] + df = df.sort_values(by=["frame"], ascending=False) + cols_lmk = [col for col in list(df) if "fac_LMK" in col] df_t = df[cols_lmk] df_diff = df_t.diff() df_diff = df_diff.pow(2) - tot_lmk = 68 # 68 landmark model + tot_lmk = 68 # 68 landmark model for i in range(tot_lmk): - lmk_id = '{:02d}'.format(i) - df['fac_LMK'+lmk_id+'disp'] = df_diff[['fac_LMK'+lmk_id+'X', 'fac_LMK'+lmk_id+'Y', 'fac_LMK'+lmk_id+'Z']].sum(axis=1).apply(np.sqrt) + lmk_id = "{:02d}".format(i) + df["fac_LMK" + lmk_id + "disp"] = ( + df_diff[ + [ + "fac_LMK" + lmk_id + "X", + "fac_LMK" + lmk_id + "Y", + "fac_LMK" + lmk_id + "Z", + ] + ] + .sum(axis=1) + .apply(np.sqrt) + ) return df -def run_face_landmark(video_uri, out_dir, f_cfg): +def run_face_landmark(video_uri, out_dir, f_cfg, save=True): """ Processing all patient's for fetching facial landmarks --------------- @@ -92,30 +106,33 @@ def run_face_landmark(video_uri, out_dir, f_cfg): out_dir: (str) Output directory for processed output """ try: - - #Baseline logic - cfr = ConfigFaceReader() + + # Baseline logic + ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) - if len(of_csv_path)>0: + of_csv_path = glob.glob(join(out_loc, fl_name + "_openface/*.csv")) + if len(of_csv_path) > 0: - df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) + df_of = pd.read_csv(of_csv_path[0]) df_lmk = df_of[extract_col_nm_lmk(df_of)] df_lmk = df_lmk.copy() - df_lmk['frame'] = df_of['frame'] - df_lmk['face_id'] = df_of[' face_id'] - df_lmk['timestamp'] = df_of[' timestamp'] - df_lmk['confidence'] = df_of[' confidence'] - df_lmk['success'] = df_of[' success'] + df_lmk["frame"] = df_of["frame"] + df_lmk["face_id"] = df_of[" face_id"] + df_lmk["timestamp"] = df_of[" timestamp"] + df_lmk["confidence"] = df_of[" confidence"] + df_lmk["success"] = df_of[" success"] df_lmk = lmk_col_nm_map(df_lmk) df_lmk = add_disp_3D(df_lmk) - df_lmk['dbm_master_url'] = video_uri + df_lmk["dbm_master_url"] = video_uri + + if save: + logger.info("Processing Output file {} ".format(join(out_loc, fl_name))) + ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext) + return df_lmk - logger.info('Processing Output file {} '.format(join(out_loc, fl_name))) - ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext) - except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + e + logger.error("Failed to process video file") diff --git a/opendbm/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/opendbm/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 3a052fd0..fa55b98c 100644 --- a/opendbm/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/opendbm/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -4,19 +4,23 @@ project_name: DBM created: 2020-20-07 """ -import os -import numpy as np -import pandas as pd import glob import logging +import os + +import numpy as np +import pandas as pd from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) -logger=logging.getLogger() +logger = logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False): - """ Computes open_face features for the files in filepaths + +def batch_open_face( + filepaths, video_url, input_dir, out_dir, of_path, video_tracking=False +): + """Computes open_face features for the files in filepaths Args: ----- @@ -31,31 +35,36 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac Returns: -------- (itreable[str]) list of .csv files - """ + """ if video_tracking: - suffix = '_openface_lmk' + suffix = "_openface_lmk" else: - suffix = '_openface' + suffix = "_openface" csv_files = [] - + for fp in filepaths: try: _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) - + if video_tracking and not os.path.exists(os.path.abspath(output_directory)): os.makedirs(os.path.abspath(output_directory)) - csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) + csv_files.append( + ut.compute_open_face_features(fp, output_directory, of_path) + ) except Exception as e: - logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e)) + logger.error("Failed to run OpenFace on {}\n{}".format(fp, e)) return csv_files -def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking): + +def process_open_face( + video_uri, input_dir, out_dir, of_path, dbm_group, video_tracking +): """ Processing all patient's for fetching emotion expressivity ------------------- @@ -66,15 +75,21 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tr """ try: - - if dbm_group != None: - check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation + + if dbm_group is not None: + check_group = [ + "facial", + "movement", + ] # add group here: if you want to use openface output for raw variable calculation check_val = bool(len({*check_group} & {*dbm_group})) if not check_val: return - + filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking) + batch_open_face( + filepaths, video_uri, input_dir, out_dir, of_path, video_tracking + ) except Exception as e: - logger.error('Failed to process video file') + e + logger.error("Failed to process video file")