bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eye gaze

Browse Source
This commit is contained in:
Vijay Yadev
2020-11-30 20:31:33 -05:00
parent bfa42917b9
commit 993bd860e4
5 changed files with 181 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
dbm_lib/config/config_raw_feature.py
View File

@@ -222,6 +222,14 @@ class ConfigRawReader(object):
self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw']
self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll']
self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist']
self.mov_leye_x = config['raw_feature']['mov_leye_x']
self.mov_leye_y = config['raw_feature']['mov_leye_y']
self.mov_leye_z = config['raw_feature']['mov_leye_z']
self.mov_reye_x = config['raw_feature']['mov_reye_x']
self.mov_reye_y = config['raw_feature']['mov_reye_y']
self.mov_reye_z = config['raw_feature']['mov_reye_z']
self.mov_eleft_disp = config['raw_feature']['mov_eleft_disp']
self.mov_eright_disp = config['raw_feature']['mov_eright_disp']
#NLP features
self.nlp_transcribe = config['raw_feature']['nlp_transcribe']

5
dbm_lib/controller/process_feature.py
View File

@@ -7,7 +7,7 @@ created: 2020-20-07
from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq
from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc
from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze
from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features
import subprocess
@@ -124,6 +124,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
logger.info('processing eye blink....')
eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model)
logger.info('processing eye gaze....')
eye_gaze.run_eye_gaze(video_uri, out_dir, r_config)
def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path):
"""
processing nlp features

148
dbm_lib/dbm_features/raw_features/movement/eye_gaze.py Normal file
View File

@@ -0,0 +1,148 @@
"""
file_name: eye_gaze
project_name: DBM
created: 2020-30-11
"""
import os
import glob
import pandas as pd
import numpy as np
from scipy.spatial import distance
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
eye_pose_dir = 'movement/gaze'
eye_pose_ext = '_eyegaze.csv'
def eye_motion_df(l_disp, r_disp, error_list, r_config):
"""
Generating eye movement dataframe
Args:
l_disp: displacement list(left eye); l_disp: displacement list(right eye)
r_config: raw variable config file object
Reutrns:
Final eye displacement dataframe
"""
df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp])
df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp])
df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False)
df_eye_motion[r_config.err_reason] = error_list
return df_eye_motion
def filter_motion(df_of, df_disp, col_l, col_r, r_config):
"""
Filtering final eye movement dataframe
Args:
df_of: Openface raw out dataframe; col_r: right eye column
col_l: left eye column; r_config: raw variable config file object
"""
df_of = df_of[col_l + col_r + [' confidence']]
df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan
df_filter = df_of[col_l + col_r]
df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z,
r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z]
df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False)
return df_motion
def eye_disp(of_results, col, r_config):
"""
Computing head velocity frame by frame
Args:
of_results: Openface raw out dataframe
r_config: Face config file object
Reutrns:
Final head velocity frame by frame output
"""
distance_list = []
error_list = []
of_results = of_results[col+ [' confidence']]
for index, row in of_results.iterrows():
dst = np.nan
if index == 0 or float(row[' confidence']) < 0.8: #Threshold < 0.8
distance_list.append(dst)
if float(row[' confidence']) < 0.8:
error_list.append('confidence less than 80%')
else:
error_list.append('Pass')
continue
if index > 0:
point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1])
point_y = (row[col[0]],row[col[1]],row[col[2]])
try:
dst = distance.euclidean(point_x, point_y)
except:
pass
distance_list.append(abs(dst))
error_list.append('Pass')
return distance_list, error_list
def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config):
"""
Computing eye motion variables
Args:
df_of: Openface dataframe
out_loc: Output path for saving output csv's
fl_name: file name for output csv
r_config: raw variable config file object
"""
col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z']
col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z']
gazel_disp, err_l = eye_disp(df_of, col_l, r_config)
gazer_disp, err_r = eye_disp(df_of, col_r, r_config)
df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config)
df_disp['dbm_master_url'] = video_uri
df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config)
ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext)
def run_eye_gaze(video_uri, out_dir, r_config):
"""
Processing all patient's for getting eye movement artifacts
--------------------------------
--------------------------------
Args:
video_uri: video path; input_dir : input directory for video's
out_dir: (str) Output directory for processed output; r_config: raw variable config object
"""
try:
#filtering path to generate input & output path
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
of_csv = of_csv_path[0]
df_of = pd.read_csv(of_csv, error_bad_lines=False)
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config)
except Exception as e:
logger.error('Failed to process video file')

14
resources/features/derived_feature.yml
View File

@@ -2,7 +2,8 @@ derive_feature:
#DBM Feature Group
FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM',
'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH']
'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH',
'EYE_GAZE']
#Feature group output file extensions
FAC_ASYM_LOC: _facasym
@@ -23,6 +24,7 @@ derive_feature:
MOV_HP_LOC: _headpose
EYE_BLINK_LOC: _eyeblinks
NLP_SPEECH_LOC: _nlp
EYE_GAZE_LOC: _eyegaze
#Facial category feature group
FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom']
@@ -65,6 +67,8 @@ derive_feature:
MOV_HM: ['head_vel']
MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll']
EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur']
EYE_GAZE: ['mov_leye_x', 'mov_leye_y', 'mov_leye_z', 'mov_reye_x', 'mov_reye_y', 'mov_reye_z', 'mov_eleft_disp',
'mov_eright_disp']
#NLP category feature group
NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen',
@@ -255,6 +259,14 @@ derive_feature:
mov_blink_ear: ['mean', 'std']
vid_dur: ['count']
mov_blinkdur: ['mean', 'std']
mov_leye_x: ['mean', 'std']
mov_leye_y: ['mean', 'std']
mov_leye_z: ['mean', 'std']
mov_reye_x: ['mean', 'std']
mov_reye_y: ['mean', 'std']
mov_reye_z: ['mean', 'std']
mov_eleft_disp: ['mean', 'std']
mov_eright_disp: ['mean', 'std']
#NLP feature
nlp_numSentences: ['mean']

8
resources/features/raw_feature.yml
View File

@@ -196,6 +196,14 @@ raw_feature:
mov_Hpose_Yaw: mov_hposeyaw
mov_Hpose_Roll: mov_hposeroll
mov_Hpose_Dist: mov_hposedist
mov_leye_x: mov_lefteyex
mov_leye_y: mov_lefteyey
mov_leye_z: mov_lefteyez
mov_reye_x: mov_righteyex
mov_reye_y: mov_righteyey
mov_reye_z: mov_righteyez
mov_eleft_disp: mov_leyedisp
mov_eright_disp: mov_reyedisp
#NLP markers
nlp_transcribe: nlp_transcribe