bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added facial tremor variable

Browse Source
This commit is contained in:
Ubuntu
2020-12-01 19:54:35 +00:00
parent 4cacbb8b37
commit 9a751ca434
9 changed files with 78 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dbm_lib/controller/process_feature.py
View File

@@ -128,7 +128,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
voice_tremor.run_vtremor(video_uri, out_dir, r_config) voice_tremor.run_vtremor(video_uri, out_dir, r_config)
logger.info('processing facial tremor....') logger.info('processing facial tremor....')
face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) facial_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True)
def remove_file(file_path): def remove_file(file_path):
""" """

2
dbm_lib/dbm_features/raw_features/movement/__init__.py
View File

@@ -13,4 +13,4 @@ import os
DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_PATH = os.path.dirname(__file__)
DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH,
'../../../../resources/libraries/voice_tremor.praat')) '../../../../resources/libraries/voice_tremor.praat'))
DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json'))

43
dbm_lib/dbm_features/raw_features/movement/facial_tremor.py
View File

@@ -25,7 +25,7 @@ def compute_features(out_dir, df_of, r_config):
Returns: features in vector format Returns: features in vector format
""" """
config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read())
logger.info('json file read')
pattern_x = re.compile("l\d+_x") pattern_x = re.compile("l\d+_x")
pattern_y = re.compile("l\d+_y") pattern_y = re.compile("l\d+_y")
@@ -40,7 +40,7 @@ def compute_features(out_dir, df_of, r_config):
df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)]
df_of.reset_index(inplace=True) df_of.reset_index(inplace=True)
num_frames = len(df) num_frames = len(df_of)
logger.info("Number of frames to be processed: {}".format(str(num_frames))) logger.info("Number of frames to be processed: {}".format(str(num_frames)))
landmarks = config['landmarks'] landmarks = config['landmarks']
@@ -104,7 +104,7 @@ def compute_features(out_dir, df_of, r_config):
for i, landmark in enumerate(landmarks): for i, landmark in enumerate(landmarks):
fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]]
raw_variable_map = 'fac_tremor_median_{}'.format(landmark) raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]]
fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]]
fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]]
@@ -123,7 +123,7 @@ def empty_frame(landmarks, r_config, error_reason):
fac_features_dict = {} fac_features_dict = {}
for i, landmark in enumerate(landmarks): for i, landmark in enumerate(landmarks):
raw_variable_map = 'fac_tremor_median_{}'.format(landmark) raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan]
fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan]
fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan]
@@ -133,27 +133,32 @@ def empty_frame(landmarks, r_config, error_reason):
empty_frame = pd.DataFrame.from_dict(fac_features_dict) empty_frame = pd.DataFrame.from_dict(fac_features_dict)
return empty_frame return empty_frame
def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): def fac_tremor_process(video_uri, out_dir, r_config, model_output=False):
""" """
processing input videos processing input videos
""" """
try: # try:
logger.info('filtering path: ',video_uri,out_dir)
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv'))
if len(of_csv_path)>0: input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv = of_csv_path[0] of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv'))
df_of = pd.read_csv(of_csv, error_bad_lines=False)
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) if len(of_csv_path)>0:
of_csv = of_csv_path[0]
df_of = pd.read_csv(of_csv, error_bad_lines=False)
feats = compute_features(of_csv_path , df_of, r_config) logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
if model_output:
result = score(feats, r_config)
feats = pd.concat([feats, result], axis=1)
ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) feats = compute_features(of_csv_path , df_of, r_config)
except Exception as e: # if model_output:
# result = score(feats, r_config)
# feats = pd.concat([feats, result], axis=1)
ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext)
# except Exception as e:
logger.error('Failed to process video file') logger.error('Failed to process video file')

1
dbm_lib/dbm_features/raw_features/movement/head_motion.py
View File

@@ -184,6 +184,7 @@ def run_head_movement(video_uri, out_dir, r_config):
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0: if len(of_csv_path)>0:
of_csv = of_csv_path[0] of_csv = of_csv_path[0]

6
dbm_lib/dbm_features/raw_features/movement/voice_tremor.py
View File

@@ -47,7 +47,7 @@ def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name):
df_tremor = tremor_praat(audio_file, r_config) df_tremor = tremor_praat(audio_file, r_config)
df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release
logger.info('Processing Output file {} '.format(out_loc)) logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): def prepare_empty_vt(out_loc, fl_name, r_config, error_txt):
@@ -61,7 +61,7 @@ def prepare_empty_vt(out_loc, fl_name, r_config, error_txt):
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
df_tremor = pd.DataFrame(out_val, columns = cols) df_tremor = pd.DataFrame(out_val, columns = cols)
logger.info('Saving Output file {} '.format(out_loc)) logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
def run_vtremor(video_uri, out_dir, r_config): def run_vtremor(video_uri, out_dir, r_config):
@@ -74,9 +74,9 @@ def run_vtremor(video_uri, out_dir, r_config):
out_dir: (str) Output directory for processed output out_dir: (str) Output directory for processed output
""" """
try: try:
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0: if len(aud_filter)>0:
audio_file = aud_filter[0] audio_file = aud_filter[0]

9
dbm_lib/dbm_features/raw_features/video/open_face_process.py
View File

@@ -15,7 +15,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
logger=logging.getLogger() logger=logging.getLogger()
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False):
""" Computes open_face features for the files in filepaths """ Computes open_face features for the files in filepaths
Args: Args:
@@ -33,7 +33,7 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_track
(itreable[str]) list of .csv files (itreable[str]) list of .csv files
""" """
if video_tracking: if video_tracking:
suffix = '_OF_video_features/' suffix = '_OF_video_features'
else: else:
suffix = '_OF_features' suffix = '_OF_features'
@@ -45,7 +45,8 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_track
_, out_loc, fl_name = ut.filter_path(video_url, out_dir) _, out_loc, fl_name = ut.filter_path(video_url, out_dir)
full_f_name = fl_name + suffix full_f_name = fl_name + suffix
output_directory = os.path.join(out_loc, full_f_name) output_directory = os.path.join(out_loc, full_f_name)
if not os.path.isdir(output_directory):
os.mkdir(output_directory)
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
except Exception as e: except Exception as e:
@@ -69,7 +70,7 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tr
return return
filepaths = [video_uri] filepaths = [video_uri]
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking)
except Exception as e: except Exception as e:
logger.error('Failed to process video file') logger.error('Failed to process video file')

42
pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp
View File

@@ -155,7 +155,7 @@ int main(int argc, char **argv)
std::string base_filename = path.substr(path.find_last_of("/\\") + 1); std::string base_filename = path.substr(path.find_last_of("/\\") + 1);
base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,""); base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,"");
results.open(out_dir + '/' + base_filename + "_landmark_output.csv"); results.open(out_dir + '/' + base_filename + "_landmark_output.csv");
// confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv");
int lx = 0; int lx = 0;
int ly = 0; int ly = 0;
for(lx = 0; lx < 2; lx++){ for(lx = 0; lx < 2; lx++){
@@ -163,7 +163,7 @@ int main(int argc, char **argv)
if (lx == 0){ if (lx == 0){
results << "l" << ly << "_x,"; results << "l" << ly << "_x,";
// confidence << "c" << ly <<","; confidence << "c" << ly <<",";
} }
if (lx == 1){ if (lx == 1){
results << "l" << ly << "_y,"; results << "l" << ly << "_y,";
@@ -172,7 +172,7 @@ int main(int argc, char **argv)
} }
results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ; results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ;
results << std::endl; results << std::endl;
// confidence << std::endl; confidence << std::endl;
int counter = 0; int counter = 0;
@@ -207,7 +207,6 @@ int main(int argc, char **argv)
// Gaze tracking, absolute gaze direction // Gaze tracking, absolute gaze direction
cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection0(0, 0, -1);
cv::Point3f gazeDirection1(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1);
cv::Vec6d pose_estimate(0, 0, 0, 0, 0, 0);
// If tracking succeeded and we have an eye model, estimate gaze // If tracking succeeded and we have an eye model, estimate gaze
if (detection_success && face_model.eye_model) if (detection_success && face_model.eye_model)
@@ -217,9 +216,9 @@ int main(int argc, char **argv)
} }
// Work out the pose of the head from the tracked model // Work out the pose of the head from the tracked model
if (detection_success){
pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
}
@@ -234,11 +233,11 @@ int main(int argc, char **argv)
// Displaying the tracking visualizations // Displaying the tracking visualizations
// std::cout<< "setting observation landmarks"<<std::endl; // std::cout<< "setting observation landmarks"<<std::endl;
// visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
// visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
// visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
// visualizer.SetFps(fps_tracker.GetFPS()); visualizer.SetFps(fps_tracker.GetFPS());
// std::cout << "openfacerec set obs landmarks"<<std::endl; // std::cout << "openfacerec set obs landmarks"<<std::endl;
// std::cout<< fps_tracker.GetFPS() <<std::endl; // std::cout<< fps_tracker.GetFPS() <<std::endl;
@@ -264,15 +263,15 @@ int main(int argc, char **argv)
} }
results <<std::endl; results <<std::endl;
// for(i=0;i<68;i++){ for(i=0;i<68;i++){
// if (i==67){ if (i==67){
// confidence << face_model.landmark_likelihoods[0][i]; confidence << face_model.landmark_likelihoods[0][i];
// } }
// else{ else{
// confidence << face_model.landmark_likelihoods[0][i] << ","; confidence << face_model.landmark_likelihoods[0][i] << ",";
// } }
// } }
// confidence <<std::endl; confidence <<std::endl;
// detect key presses (due to pecularities of OpenCV, you can get it when displaying images) // detect key presses (due to pecularities of OpenCV, you can get it when displaying images)
//char character_press = visualizer.ShowObservation(); //char character_press = visualizer.ShowObservation();
@@ -311,4 +310,3 @@ int main(int argc, char **argv)
return 0; return 0;
} }

9
process_data.py
View File

@@ -20,7 +20,7 @@ logging.basicConfig(level=logging.INFO)
logger=logging.getLogger() logger=logging.getLogger()
#for ftremor #for ftremor
OPENFACE_PATH_VIDEO = '/pkg/OpenFace/build/bin/FaceLandmarkVid' OPENFACE_PATH_VIDEO = 'pkg/OpenFace/build/bin/FaceLandmarkVid'
OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction' OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction'
DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat' DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat'
@@ -34,10 +34,13 @@ def common_video(video_file, args, r_config):
""" """
out_path = os.path.join(args.output_path, 'raw_variables') out_path = os.path.join(args.output_path, 'raw_variables')
pf.audio_to_wav(video_file) pf.audio_to_wav(video_file)
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group)
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group,video_tracking=False)
pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_facial(video_file, out_path, args.dbm_group, r_config)
pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config)
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group,video_tracking=True)
if args.dbm_group == None or len(args.dbm_group)>0 and 'movement' in args.dbm_group:
of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group, video_tracking=True)
pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL)
pf.remove_file(video_file) pf.remove_file(video_file)

13
resources/features/derived_feature.yml
View File

@@ -2,7 +2,7 @@ derive_feature:
#DBM Feature Group #DBM Feature Group
FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM',
'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT'] 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT', 'MOV_FT']
#Feature group output file extensions #Feature group output file extensions
FAC_ASYM_LOC: _facasym FAC_ASYM_LOC: _facasym
@@ -23,6 +23,7 @@ derive_feature:
MOV_HP_LOC: _headpose MOV_HP_LOC: _headpose
EYE_BLINK_LOC: _eyeblinks EYE_BLINK_LOC: _eyeblinks
MOV_VT_LOC: _vtremor MOV_VT_LOC: _vtremor
MOV_FT_LOC: _fac_tremor
#Facial category feature group #Facial category feature group
@@ -68,6 +69,7 @@ derive_feature:
EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur']
MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq', MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq',
'mov_amp_trem_index', 'mov_amp_trem_pindex'] 'mov_amp_trem_index', 'mov_amp_trem_pindex']
MOV_FT: ['fac_tremor_median_5','fac_tremor_median_12','fac_tremor_median_8','fac_tremor_median_48','fac_tremor_median_54','fac_tremor_median_28','fac_tremor_median_51','fac_tremor_median_66','fac_tremor_median_57']
#Calculation for variables #Calculation for variables
# Facial Asymmetry # Facial Asymmetry
@@ -258,3 +260,12 @@ derive_feature:
mov_amp_trem_freq: ['mean'] mov_amp_trem_freq: ['mean']
mov_amp_trem_index: ['mean'] mov_amp_trem_index: ['mean']
mov_amp_trem_pindex: ['mean'] mov_amp_trem_pindex: ['mean']
fac_tremor_median_5: ['mean']
fac_tremor_median_12: ['mean']
fac_tremor_median_8: ['mean']
fac_tremor_median_48: ['mean']
fac_tremor_median_54: ['mean']
fac_tremor_median_28: ['mean']
fac_tremor_median_51: ['mean']
fac_tremor_median_66: ['mean']
fac_tremor_median_57: ['mean']