Merge pull request #9 from AiCure/face_tremor
Face tremor variable added
This commit is contained in:
vjbytes102
@@ -230,6 +230,16 @@ class ConfigRawReader(object):
|
||||
self.mov_amp_trem_index = config['raw_feature']['mov_amp_trem_index']
|
||||
self.mov_amp_trem_pindex = config['raw_feature']['mov_amp_trem_pindex']
|
||||
|
||||
self.fac_tremor_median_5 = config['raw_feature']['fac_tremor_median_5']
|
||||
self.fac_tremor_median_12 = config['raw_feature']['fac_tremor_median_12']
|
||||
self.fac_tremor_median_8 = config['raw_feature']['fac_tremor_median_8']
|
||||
self.fac_tremor_median_48 = config['raw_feature']['fac_tremor_median_48']
|
||||
self.fac_tremor_median_54 = config['raw_feature']['fac_tremor_median_54']
|
||||
self.fac_tremor_median_28 = config['raw_feature']['fac_tremor_median_28']
|
||||
self.fac_tremor_median_51 = config['raw_feature']['fac_tremor_median_51']
|
||||
self.fac_tremor_median_66 = config['raw_feature']['fac_tremor_median_66']
|
||||
self.fac_tremor_median_57 = config['raw_feature']['fac_tremor_median_57']
|
||||
|
||||
self.mov_leye_x = config['raw_feature']['mov_leye_x']
|
||||
self.mov_leye_y = config['raw_feature']['mov_leye_y']
|
||||
self.mov_leye_z = config['raw_feature']['mov_leye_z']
|
||||
|
||||
@@ -7,9 +7,10 @@ created: 2020-20-07
|
||||
from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq
|
||||
from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc
|
||||
from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark
|
||||
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze, voice_tremor
|
||||
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze, voice_tremor, facial_tremor
|
||||
from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features
|
||||
|
||||
|
||||
import subprocess
|
||||
import logging
|
||||
from os.path import isfile, splitext, basename, dirname, join
|
||||
@@ -121,7 +122,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
|
||||
|
||||
logger.info('processing head movement....')
|
||||
head_motion.run_head_movement(video_uri, out_dir, r_config)
|
||||
|
||||
|
||||
logger.info('processing eye blink....')
|
||||
eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model)
|
||||
|
||||
@@ -130,6 +131,10 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
|
||||
|
||||
logger.info('processing voice tremor....')
|
||||
voice_tremor.run_vtremor(video_uri, out_dir, r_config)
|
||||
|
||||
logger.info('processing facial tremor....')
|
||||
facial_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True)
|
||||
|
||||
|
||||
def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path):
|
||||
"""
|
||||
@@ -146,6 +151,7 @@ def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path):
|
||||
transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path)
|
||||
speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog)
|
||||
|
||||
|
||||
def remove_file(file_path):
|
||||
"""
|
||||
removing wav file
|
||||
|
||||
@@ -31,7 +31,11 @@ def formant_list(formant,snd):
|
||||
Returns:
|
||||
List of first through fourth formant for each frame
|
||||
"""
|
||||
f1_list, f2_list, f3_list, f4_list = ([], ) * 4
|
||||
f1_list = []
|
||||
f2_list = []
|
||||
f3_list = []
|
||||
f4_list = []
|
||||
|
||||
dur = snd.duration-0.02
|
||||
dur_round = round(dur, 2)
|
||||
|
||||
@@ -126,4 +130,4 @@ def run_formant(video_uri, out_dir, r_config):
|
||||
|
||||
calc_formant(video_uri, audio_file, out_loc, fl_name, r_config)
|
||||
except Exception as e:
|
||||
logger.error('Failed to process audio file')
|
||||
logger.error('Failed to process audio file')
|
||||
|
||||
@@ -13,4 +13,5 @@ import os
|
||||
DBMLIB_PATH = os.path.dirname(__file__)
|
||||
DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH,
|
||||
'../../../../resources/libraries/voice_tremor.praat'))
|
||||
DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json'))
|
||||
DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json'))
|
||||
|
||||
|
||||
164
dbm_lib/dbm_features/raw_features/movement/facial_tremor.py
Normal file
164
dbm_lib/dbm_features/raw_features/movement/facial_tremor.py
Normal file
@@ -0,0 +1,164 @@
|
||||
import sys, os, glob, cv2, re
|
||||
import pickle, json
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import numpy.ma as ma
|
||||
import logging
|
||||
from os.path import join
|
||||
|
||||
from dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
from dbm_lib.dbm_features.raw_features.util.math_util import *
|
||||
|
||||
from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
|
||||
ft_dir = 'movement/facial_tremor'
|
||||
csv_ext = '_fac_tremor.csv'
|
||||
model_ext = '_fac_model.csv'
|
||||
fac_features_ext = '_fac_features.csv'
|
||||
|
||||
def compute_features(out_dir, df_of, r_config):
|
||||
""" Computes features
|
||||
|
||||
Returns: features in vector format
|
||||
"""
|
||||
config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read())
|
||||
|
||||
pattern_x = re.compile("l\d+_x")
|
||||
pattern_y = re.compile("l\d+_y")
|
||||
|
||||
# assumption: distance of face to camera remains at roughly static
|
||||
|
||||
# logic break
|
||||
landmark_columns = []
|
||||
for col in df_of.columns:
|
||||
if pattern_x.match(col) or pattern_y.match(col):
|
||||
landmark_columns.append(col)
|
||||
|
||||
df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)]
|
||||
df_of.reset_index(inplace=True)
|
||||
|
||||
num_frames = len(df_of)
|
||||
logger.info("Number of frames to be processed: {}".format(str(num_frames)))
|
||||
landmarks = config['landmarks']
|
||||
|
||||
try:
|
||||
if num_frames == 0:
|
||||
error_reason = "No frames with visible face."
|
||||
logger.error(error_reason)
|
||||
return empty_frame(landmarks, r_config, error_reason)
|
||||
|
||||
# if num_frames < 60:
|
||||
# error_reason = 'Number of frames with visible face < 60. Video too short'
|
||||
# logger.error(error_reason)
|
||||
# return empty_frame(landmarks, f_cfg, error_reason)
|
||||
|
||||
first_row = df_of.iloc[0]
|
||||
|
||||
facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']])
|
||||
faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']])
|
||||
|
||||
if facew == 0 or faceh == 0:
|
||||
error_reason = 'face width or height = 0. Check landmark values'
|
||||
logger.error(error_reason)
|
||||
return empty_frame(landmarks, r_config)
|
||||
|
||||
fac_disp = calc_displacement_vec(df_of, landmarks, num_frames)
|
||||
|
||||
# if verbose:
|
||||
# logger.info("Displacement output: {}".format(str(fac_disp)))
|
||||
|
||||
fac_disp_median = np.median(fac_disp, axis = 1)
|
||||
fac_disp_mean = np.mean(fac_disp, axis = 1)
|
||||
|
||||
if len(fac_disp.shape)!=2:
|
||||
error_reason = 'fac_disp is not 2D. smth went wrong with disp calc'
|
||||
logger.error(error_reason)
|
||||
return empty_frame(landmarks, r_config, error_reason)
|
||||
|
||||
if len(fac_disp[0])<=1:
|
||||
error_reason = 'Video too short. smth went wrong with disp calc'
|
||||
logger.error(error_reason)
|
||||
return empty_frame(landmarks, r_config, error_reason)
|
||||
|
||||
fac_corr_mat = np.corrcoef(fac_disp, rowvar = True)
|
||||
# extract relevant row from cov matrix
|
||||
ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk]
|
||||
fac_corr = fac_corr_mat[ref_lmk_index][0]
|
||||
|
||||
fac_area = config['ref_area'] / (facew * faceh)
|
||||
|
||||
# if verbose:
|
||||
# logger.info("Face area: {}".format(fac_area))
|
||||
# logger.info("Face Displacement Median: {}".format(str(fac_disp_median)))
|
||||
# logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean)))
|
||||
|
||||
fac_features1 = np.multiply(fac_area * fac_disp_median, (1. - fac_corr))
|
||||
fac_features2 = np.multiply(fac_area * fac_disp_mean, (1. - fac_corr))
|
||||
|
||||
# base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr))
|
||||
|
||||
fac_features_dict = {}
|
||||
for i, landmark in enumerate(landmarks):
|
||||
fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]]
|
||||
raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
|
||||
fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]]
|
||||
|
||||
fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]]
|
||||
fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]]
|
||||
|
||||
fac_features_dict[r_config.err_reason] = ['']
|
||||
data = pd.DataFrame.from_dict(fac_features_dict)
|
||||
logger.info('Concluded computing tremor features')
|
||||
|
||||
return data
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Error computing tremor features: {}'.format(str(e)))
|
||||
return empty_frame(landmarks, r_config, str(e))
|
||||
|
||||
def empty_frame(landmarks, r_config, error_reason):
|
||||
fac_features_dict = {}
|
||||
for i, landmark in enumerate(landmarks):
|
||||
raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
|
||||
fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan]
|
||||
|
||||
fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan]
|
||||
fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan]
|
||||
fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan]
|
||||
|
||||
fac_features_dict[r_config.err_reason] = [error_reason]
|
||||
empty_frame = pd.DataFrame.from_dict(fac_features_dict)
|
||||
return empty_frame
|
||||
|
||||
def fac_tremor_process(video_uri, out_dir, r_config, model_output=False):
|
||||
"""
|
||||
processing input videos
|
||||
|
||||
|
||||
"""
|
||||
# try:
|
||||
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv'))
|
||||
|
||||
if len(of_csv_path)>0:
|
||||
of_csv = of_csv_path[0]
|
||||
df_of = pd.read_csv(of_csv, error_bad_lines=False)
|
||||
|
||||
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
|
||||
feats = compute_features(of_csv_path , df_of, r_config)
|
||||
|
||||
# if model_output:
|
||||
# result = score(feats, r_config)
|
||||
# feats = pd.concat([feats, result], axis=1)
|
||||
|
||||
ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext)
|
||||
|
||||
|
||||
|
||||
# except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
@@ -179,11 +179,12 @@ def run_head_movement(video_uri, out_dir, r_config):
|
||||
out_dir: (str) Output directory for processed output; r_config: raw variable config object
|
||||
"""
|
||||
try:
|
||||
|
||||
|
||||
#filtering path to generate input & output path
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
|
||||
|
||||
|
||||
if len(of_csv_path)>0:
|
||||
|
||||
of_csv = of_csv_path[0]
|
||||
@@ -192,4 +193,4 @@ def run_head_movement(video_uri, out_dir, r_config):
|
||||
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config)
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
logger.error('Failed to process video file')
|
||||
|
||||
@@ -47,7 +47,7 @@ def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name):
|
||||
df_tremor = tremor_praat(audio_file, r_config)
|
||||
df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release
|
||||
|
||||
logger.info('Processing Output file {} '.format(out_loc))
|
||||
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
|
||||
|
||||
def prepare_empty_vt(out_loc, fl_name, r_config, error_txt):
|
||||
@@ -61,7 +61,7 @@ def prepare_empty_vt(out_loc, fl_name, r_config, error_txt):
|
||||
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
|
||||
df_tremor = pd.DataFrame(out_val, columns = cols)
|
||||
|
||||
logger.info('Saving Output file {} '.format(out_loc))
|
||||
logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name)))
|
||||
ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
|
||||
|
||||
def run_vtremor(video_uri, out_dir, r_config):
|
||||
@@ -74,9 +74,9 @@ def run_vtremor(video_uri, out_dir, r_config):
|
||||
out_dir: (str) Output directory for processed output
|
||||
"""
|
||||
try:
|
||||
|
||||
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
|
||||
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
|
||||
|
||||
if len(aud_filter)>0:
|
||||
|
||||
audio_file = aud_filter[0]
|
||||
|
||||
57
dbm_lib/dbm_features/raw_features/util/math_util.py
Normal file
57
dbm_lib/dbm_features/raw_features/util/math_util.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
file_name: facial_tremor
|
||||
project_name: cdx_analysis
|
||||
created: 2019-03-16
|
||||
author: Deshana Desai
|
||||
"""
|
||||
import sys, os, glob, cv2
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
def euclidean_distance(point1, point2):
|
||||
"""
|
||||
Compute euclidean distance between points
|
||||
"""
|
||||
|
||||
return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)
|
||||
|
||||
|
||||
# def detect_peaks()
|
||||
|
||||
|
||||
def expand_landmarks(landmarks):
|
||||
"""
|
||||
util method to expand landmark list:
|
||||
eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']]
|
||||
"""
|
||||
return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks]
|
||||
|
||||
|
||||
|
||||
def calc_displacement_vec(df, landmarks, num_frames):
|
||||
"""
|
||||
Calculates displacement vector frame by frame
|
||||
"""
|
||||
|
||||
landmarks = expand_landmarks(landmarks)
|
||||
|
||||
disp_vec = np.zeros((len(landmarks), num_frames))
|
||||
prev_point = np.zeros((len(landmarks), 2))
|
||||
|
||||
# initialize
|
||||
for j, pair in enumerate(landmarks):
|
||||
first_row = df.iloc[0]
|
||||
prev_point[j] = (first_row[pair[0]], first_row[pair[1]])
|
||||
|
||||
|
||||
for i in range(num_frames):
|
||||
frame_row = df.iloc[i]
|
||||
for j, pair in enumerate(landmarks):
|
||||
x, y = pair[0], pair[1]
|
||||
current = (frame_row[x], frame_row[y])
|
||||
deviation = euclidean_distance( current, prev_point[j])
|
||||
disp_vec[j][i] = deviation
|
||||
prev_point[j] = current
|
||||
|
||||
return disp_vec
|
||||
@@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
|
||||
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path):
|
||||
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False):
|
||||
""" Computes open_face features for the files in filepaths
|
||||
|
||||
|
||||
Args:
|
||||
-----
|
||||
filepaths: (itreable[str])
|
||||
@@ -27,38 +27,43 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path):
|
||||
input_dir: Path to the input videos
|
||||
out_dir: Path to the processed output
|
||||
of_path: OpenFace source code path
|
||||
|
||||
|
||||
Returns:
|
||||
--------
|
||||
(itreable[str]) list of .csv files
|
||||
"""
|
||||
|
||||
suffix = '_OF_features'
|
||||
"""
|
||||
if video_tracking:
|
||||
suffix = '_OF_video_features'
|
||||
else:
|
||||
suffix = '_OF_features'
|
||||
|
||||
csv_files = []
|
||||
|
||||
for fp in filepaths:
|
||||
try:
|
||||
|
||||
|
||||
_, out_loc, fl_name = ut.filter_path(video_url, out_dir)
|
||||
full_f_name = fl_name + suffix
|
||||
output_directory = os.path.join(out_loc, full_f_name)
|
||||
|
||||
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
|
||||
|
||||
if video_tracking and not os.path.exists(os.path.abspath(output_directory)):
|
||||
os.makedirs(os.path.abspath(output_directory))
|
||||
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
|
||||
|
||||
|
||||
return csv_files
|
||||
|
||||
def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group):
|
||||
|
||||
def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking):
|
||||
"""
|
||||
Processing all patient's for fetching emotion expressivity
|
||||
-------------------
|
||||
-------------------
|
||||
Args:
|
||||
video_uri: video path; input_dir : input directory for video's; dbm_group: feature group
|
||||
out_dir: (str) Output directory for processed output; of_path: OpenFace source code path
|
||||
|
||||
out_dir: (str) Output directory for processed output; of_path: OpenFace source code path
|
||||
|
||||
"""
|
||||
try:
|
||||
|
||||
@@ -69,7 +74,7 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group):
|
||||
return
|
||||
|
||||
filepaths = [video_uri]
|
||||
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path)
|
||||
|
||||
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking)
|
||||
|
||||
except Exception as e:
|
||||
logger.error('Failed to process video file')
|
||||
|
||||
@@ -155,7 +155,7 @@ int main(int argc, char **argv)
|
||||
std::string base_filename = path.substr(path.find_last_of("/\\") + 1);
|
||||
base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,"");
|
||||
results.open(out_dir + '/' + base_filename + "_landmark_output.csv");
|
||||
// confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv");
|
||||
confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv");
|
||||
int lx = 0;
|
||||
int ly = 0;
|
||||
for(lx = 0; lx < 2; lx++){
|
||||
@@ -163,7 +163,7 @@ int main(int argc, char **argv)
|
||||
|
||||
if (lx == 0){
|
||||
results << "l" << ly << "_x,";
|
||||
// confidence << "c" << ly <<",";
|
||||
confidence << "c" << ly <<",";
|
||||
}
|
||||
if (lx == 1){
|
||||
results << "l" << ly << "_y,";
|
||||
@@ -172,7 +172,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ;
|
||||
results << std::endl;
|
||||
// confidence << std::endl;
|
||||
confidence << std::endl;
|
||||
|
||||
int counter = 0;
|
||||
|
||||
@@ -207,7 +207,6 @@ int main(int argc, char **argv)
|
||||
// Gaze tracking, absolute gaze direction
|
||||
cv::Point3f gazeDirection0(0, 0, -1);
|
||||
cv::Point3f gazeDirection1(0, 0, -1);
|
||||
cv::Vec6d pose_estimate(0, 0, 0, 0, 0, 0);
|
||||
|
||||
// If tracking succeeded and we have an eye model, estimate gaze
|
||||
if (detection_success && face_model.eye_model)
|
||||
@@ -217,9 +216,9 @@ int main(int argc, char **argv)
|
||||
}
|
||||
|
||||
// Work out the pose of the head from the tracked model
|
||||
if (detection_success){
|
||||
pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
|
||||
}
|
||||
|
||||
cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -234,11 +233,11 @@ int main(int argc, char **argv)
|
||||
|
||||
// Displaying the tracking visualizations
|
||||
// std::cout<< "setting observation landmarks"<<std::endl;
|
||||
// visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
|
||||
// visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
|
||||
// visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
|
||||
// visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
|
||||
// visualizer.SetFps(fps_tracker.GetFPS());
|
||||
visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
|
||||
visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
|
||||
visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
|
||||
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
|
||||
visualizer.SetFps(fps_tracker.GetFPS());
|
||||
|
||||
// std::cout << "openfacerec set obs landmarks"<<std::endl;
|
||||
// std::cout<< fps_tracker.GetFPS() <<std::endl;
|
||||
@@ -264,15 +263,15 @@ int main(int argc, char **argv)
|
||||
}
|
||||
results <<std::endl;
|
||||
|
||||
// for(i=0;i<68;i++){
|
||||
// if (i==67){
|
||||
// confidence << face_model.landmark_likelihoods[0][i];
|
||||
// }
|
||||
// else{
|
||||
// confidence << face_model.landmark_likelihoods[0][i] << ",";
|
||||
// }
|
||||
// }
|
||||
// confidence <<std::endl;
|
||||
for(i=0;i<68;i++){
|
||||
if (i==67){
|
||||
confidence << face_model.landmark_likelihoods[0][i];
|
||||
}
|
||||
else{
|
||||
confidence << face_model.landmark_likelihoods[0][i] << ",";
|
||||
}
|
||||
}
|
||||
confidence <<std::endl;
|
||||
|
||||
// detect key presses (due to pecularities of OpenCV, you can get it when displaying images)
|
||||
//char character_press = visualizer.ShowObservation();
|
||||
@@ -311,4 +310,3 @@ int main(int argc, char **argv)
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,8 @@ import time
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger=logging.getLogger()
|
||||
|
||||
#for ftremor
|
||||
OPENFACE_PATH_VIDEO = 'pkg/OpenFace/build/bin/FaceLandmarkVid'
|
||||
OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction'
|
||||
DEEP_SPEECH = 'pkg/DeepSpeech'
|
||||
DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat'
|
||||
@@ -33,10 +35,14 @@ def common_video(video_file, args, r_config):
|
||||
"""
|
||||
out_path = os.path.join(args.output_path, 'raw_variables')
|
||||
pf.audio_to_wav(video_file)
|
||||
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group)
|
||||
|
||||
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group,video_tracking=False)
|
||||
pf.process_facial(video_file, out_path, args.dbm_group, r_config)
|
||||
pf.process_acoustic(video_file, out_path, args.dbm_group, r_config)
|
||||
pf.process_nlp(video_file, out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH)
|
||||
|
||||
pf.process_nlp(video_file, out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH)
|
||||
if args.dbm_group == None or len(args.dbm_group)>0 and 'movement' in args.dbm_group:
|
||||
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group, video_tracking=True)
|
||||
|
||||
pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL)
|
||||
pf.remove_file(video_file)
|
||||
|
||||
@@ -3,7 +3,7 @@ derive_feature:
|
||||
#DBM Feature Group
|
||||
FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM',
|
||||
'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH',
|
||||
'EYE_GAZE', 'MOV_VT']
|
||||
'EYE_GAZE', 'MOV_VT', 'MOV_FT']
|
||||
|
||||
#Feature group output file extensions
|
||||
FAC_ASYM_LOC: _facasym
|
||||
@@ -26,6 +26,7 @@ derive_feature:
|
||||
NLP_SPEECH_LOC: _nlp
|
||||
EYE_GAZE_LOC: _eyegaze
|
||||
MOV_VT_LOC: _vtremor
|
||||
MOV_FT_LOC: _fac_tremor
|
||||
|
||||
#Facial category feature group
|
||||
FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom']
|
||||
@@ -70,6 +71,7 @@ derive_feature:
|
||||
EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur']
|
||||
MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq',
|
||||
'mov_amp_trem_index', 'mov_amp_trem_pindex']
|
||||
MOV_FT: ['fac_tremor_median_5','fac_tremor_median_12','fac_tremor_median_8','fac_tremor_median_48','fac_tremor_median_54','fac_tremor_median_28','fac_tremor_median_51','fac_tremor_median_66','fac_tremor_median_57']
|
||||
|
||||
EYE_GAZE: ['mov_leye_x', 'mov_leye_y', 'mov_leye_z', 'mov_reye_x', 'mov_reye_y', 'mov_reye_z', 'mov_eleft_disp',
|
||||
'mov_eright_disp']
|
||||
@@ -270,6 +272,16 @@ derive_feature:
|
||||
mov_amp_trem_freq: ['mean']
|
||||
mov_amp_trem_index: ['mean']
|
||||
mov_amp_trem_pindex: ['mean']
|
||||
|
||||
fac_tremor_median_5: ['mean']
|
||||
fac_tremor_median_12: ['mean']
|
||||
fac_tremor_median_8: ['mean']
|
||||
fac_tremor_median_48: ['mean']
|
||||
fac_tremor_median_54: ['mean']
|
||||
fac_tremor_median_28: ['mean']
|
||||
fac_tremor_median_51: ['mean']
|
||||
fac_tremor_median_66: ['mean']
|
||||
fac_tremor_median_57: ['mean']
|
||||
|
||||
mov_leye_x: ['mean', 'std']
|
||||
mov_leye_y: ['mean', 'std']
|
||||
@@ -298,3 +310,4 @@ derive_feature:
|
||||
nlp_mattr: ['mean']
|
||||
nlp_wordsPerMin: ['mean']
|
||||
nlp_totalTime: ['mean']
|
||||
|
||||
|
||||
1
resources/features/facial/config.json
Normal file
1
resources/features/facial/config.json
Normal file
@@ -0,0 +1 @@
|
||||
{"ref_lmk": 28, "ref_area": 350000, "face_width_left": "l15_x", "face_width_right": "l1_x", "face_height_left": "l8_y", "face_height_right": "l27_y", "landmarks": [5, 12, 8, 48, 54, 28, 51, 66, 57], "model_path": "resources/facial/svm_bin_fac_tremor.sav", "feature_order": ["fac_features_mean_5", "fac_features_mean_12", "fac_features_mean_8", "fac_features_mean_48", "fac_features_mean_54", "fac_features_mean_28", "fac_features_mean_51", "fac_features_mean_66", "fac_features_mean_57", "fac_features_median_5", "fac_features_median_12", "fac_features_median_8", "fac_features_median_48", "fac_features_median_54", "fac_features_median_28", "fac_features_median_51", "fac_features_median_66", "fac_features_median_57"]}
|
||||
@@ -204,6 +204,16 @@ raw_feature:
|
||||
mov_amp_trem_index: mov_amptremindex
|
||||
mov_amp_trem_pindex: mov_amptrempindex
|
||||
|
||||
fac_tremor_median_5: fac_tremor_median_5
|
||||
fac_tremor_median_12: fac_tremor_median_12
|
||||
fac_tremor_median_8: fac_tremor_median_8
|
||||
fac_tremor_median_48: fac_tremor_median_48
|
||||
fac_tremor_median_54: fac_tremor_median_54
|
||||
fac_tremor_median_28: fac_tremor_median_28
|
||||
fac_tremor_median_51: fac_tremor_median_51
|
||||
fac_tremor_median_66: fac_tremor_median_66
|
||||
fac_tremor_median_57: fac_tremor_median_57
|
||||
|
||||
mov_leye_x: mov_lefteyex
|
||||
mov_leye_y: mov_lefteyey
|
||||
mov_leye_z: mov_lefteyez
|
||||
|
||||
Reference in New Issue
Block a user