From 993bd860e46aba5fbf548fb6d360e7c6e506866f Mon Sep 17 00:00:00 2001 From: Vijay Yadev Date: Mon, 30 Nov 2020 20:31:33 -0500 Subject: [PATCH] eye gaze --- dbm_lib/config/config_raw_feature.py | 8 + dbm_lib/controller/process_feature.py | 5 +- .../raw_features/movement/eye_gaze.py | 148 ++++++++++++++++++ resources/features/derived_feature.yml | 14 +- resources/features/raw_feature.yml | 8 + 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/movement/eye_gaze.py diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 3dc4834b..955780b7 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -222,6 +222,14 @@ class ConfigRawReader(object): self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw'] self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll'] self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist'] + self.mov_leye_x = config['raw_feature']['mov_leye_x'] + self.mov_leye_y = config['raw_feature']['mov_leye_y'] + self.mov_leye_z = config['raw_feature']['mov_leye_z'] + self.mov_reye_x = config['raw_feature']['mov_reye_x'] + self.mov_reye_y = config['raw_feature']['mov_reye_y'] + self.mov_reye_z = config['raw_feature']['mov_reye_z'] + self.mov_eleft_disp = config['raw_feature']['mov_eleft_disp'] + self.mov_eright_disp = config['raw_feature']['mov_eright_disp'] #NLP features self.nlp_transcribe = config['raw_feature']['nlp_transcribe'] diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index be781a7f..555f4cc9 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features import subprocess @@ -123,6 +123,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) + + logger.info('processing eye gaze....') + eye_gaze.run_eye_gaze(video_uri, out_dir, r_config) def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): """ diff --git a/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py new file mode 100644 index 00000000..a574a2d9 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py @@ -0,0 +1,148 @@ +""" +file_name: eye_gaze +project_name: DBM +created: 2020-30-11 +""" + +import os +import glob +import pandas as pd +import numpy as np +from scipy.spatial import distance +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +eye_pose_dir = 'movement/gaze' +eye_pose_ext = '_eyegaze.csv' + +def eye_motion_df(l_disp, r_disp, error_list, r_config): + """ + Generating eye movement dataframe + + Args: + l_disp: displacement list(left eye); l_disp: displacement list(right eye) + r_config: raw variable config file object + + Reutrns: + Final eye displacement dataframe + """ + df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp]) + df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp]) + + df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False) + df_eye_motion[r_config.err_reason] = error_list + return df_eye_motion + +def filter_motion(df_of, df_disp, col_l, col_r, r_config): + """ + Filtering final eye movement dataframe + + Args: + df_of: Openface raw out dataframe; col_r: right eye column + col_l: left eye column; r_config: raw variable config file object + """ + + df_of = df_of[col_l + col_r + [' confidence']] + df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan + + df_filter = df_of[col_l + col_r] + df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z, + r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z] + + df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False) + return df_motion + +def eye_disp(of_results, col, r_config): + """ + Computing head velocity frame by frame + + Args: + of_results: Openface raw out dataframe + r_config: Face config file object + + Reutrns: + Final head velocity frame by frame output + """ + distance_list = [] + error_list = [] + + of_results = of_results[col+ [' confidence']] + for index, row in of_results.iterrows(): + dst = np.nan + + if index == 0 or float(row[' confidence']) < 0.8: #Threshold < 0.8 + distance_list.append(dst) + + if float(row[' confidence']) < 0.8: + error_list.append('confidence less than 80%') + + else: + error_list.append('Pass') + continue + + if index > 0: + + point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1]) + point_y = (row[col[0]],row[col[1]],row[col[2]]) + try: + dst = distance.euclidean(point_x, point_y) + except: + pass + + distance_list.append(abs(dst)) + error_list.append('Pass') + + return distance_list, error_list + +def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config): + """ + Computing eye motion variables + Args: + df_of: Openface dataframe + out_loc: Output path for saving output csv's + fl_name: file name for output csv + r_config: raw variable config file object + + """ + + col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z'] + col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z'] + + gazel_disp, err_l = eye_disp(df_of, col_l, r_config) + gazer_disp, err_r = eye_disp(df_of, col_r, r_config) + + df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config) + df_disp['dbm_master_url'] = video_uri + + df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config) + ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext) + +def run_eye_gaze(video_uri, out_dir, r_config): + """ + Processing all patient's for getting eye movement artifacts + -------------------------------- + -------------------------------- + Args: + video_uri: video path; input_dir : input directory for video's + out_dir: (str) Output directory for processed output; r_config: raw variable config object + """ + try: + + #filtering path to generate input & output path + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + + if len(of_csv_path)>0: + + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config) + except Exception as e: + logger.error('Failed to process video file') \ No newline at end of file diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index 7346eb36..dd1751fd 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,8 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH', + 'EYE_GAZE'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -23,6 +24,7 @@ derive_feature: MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks NLP_SPEECH_LOC: _nlp + EYE_GAZE_LOC: _eyegaze #Facial category feature group FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom'] @@ -65,6 +67,8 @@ derive_feature: MOV_HM: ['head_vel'] MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] + EYE_GAZE: ['mov_leye_x', 'mov_leye_y', 'mov_leye_z', 'mov_reye_x', 'mov_reye_y', 'mov_reye_z', 'mov_eleft_disp', + 'mov_eright_disp'] #NLP category feature group NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen', @@ -255,6 +259,14 @@ derive_feature: mov_blink_ear: ['mean', 'std'] vid_dur: ['count'] mov_blinkdur: ['mean', 'std'] + mov_leye_x: ['mean', 'std'] + mov_leye_y: ['mean', 'std'] + mov_leye_z: ['mean', 'std'] + mov_reye_x: ['mean', 'std'] + mov_reye_y: ['mean', 'std'] + mov_reye_z: ['mean', 'std'] + mov_eleft_disp: ['mean', 'std'] + mov_eright_disp: ['mean', 'std'] #NLP feature nlp_numSentences: ['mean'] diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index b9c673b8..3f15bd9b 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -196,6 +196,14 @@ raw_feature: mov_Hpose_Yaw: mov_hposeyaw mov_Hpose_Roll: mov_hposeroll mov_Hpose_Dist: mov_hposedist + mov_leye_x: mov_lefteyex + mov_leye_y: mov_lefteyey + mov_leye_z: mov_lefteyez + mov_reye_x: mov_righteyex + mov_reye_y: mov_righteyey + mov_reye_z: mov_righteyez + mov_eleft_disp: mov_leyedisp + mov_eright_disp: mov_reyedisp #NLP markers nlp_transcribe: nlp_transcribe