diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 666fb534..b0fc683f 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -194,6 +194,9 @@ class ConfigRawReader(object): self.neu_exp = config['raw_feature']['neu_exp'] self.cai_exp = config['raw_feature']['cai_exp'] self.com_exp = config['raw_feature']['com_exp'] + self.com_lower_exp = config['raw_feature']['com_lower_exp'] + self.com_upper_exp = config['raw_feature']['com_upper_exp'] + self.pai_exp = config['raw_feature']['pai_exp'] self.hap_exp_full = config['raw_feature']['hap_exp_full'] self.sad_exp_full = config['raw_feature']['sad_exp_full'] self.sur_exp_full = config['raw_feature']['sur_exp_full'] @@ -206,6 +209,9 @@ class ConfigRawReader(object): self.neu_exp_full = config['raw_feature']['neu_exp_full'] self.cai_exp_full = config['raw_feature']['cai_exp_full'] self.com_exp_full = config['raw_feature']['com_exp_full'] + self.com_lower_exp_full = config['raw_feature']['com_lower_exp_full'] + self.com_upper_exp_full = config['raw_feature']['com_upper_exp_full'] + self.pai_exp_full = config['raw_feature']['pai_exp_full'] self.fac_AsymMaskMouth = config['raw_feature']['fac_AsymMaskMouth'] self.fac_AsymMaskEye = config['raw_feature']['fac_AsymMaskEye'] self.fac_AsymMaskEyebrow = config['raw_feature']['fac_AsymMaskEyebrow'] diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 6009864e..630709aa 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -152,13 +152,13 @@ def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path): speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog) -def remove_file(file_path): +def remove_file(file_path, file_ext = '.wav'): """ removing wav file """ file_dir = dirname(file_path) file_name, _ = splitext(basename(file_path)) - wav_file = glob.glob(join(file_dir, file_name + '.wav')) + wav_file = glob.glob(join(file_dir, file_name + file_ext)) if len(wav_file)> 0: os.remove(wav_file[0]) diff --git a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py index 2d2eff92..64568ac7 100644 --- a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py +++ b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py @@ -18,7 +18,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -formant_dir = 'audio/formant_freq' +formant_dir = 'acoustic/formant_freq' csv_ext = '_formant.csv' error_txt = 'error: length less than 0.064' diff --git a/dbm_lib/dbm_features/raw_features/audio/gne.py b/dbm_lib/dbm_features/raw_features/audio/gne.py index 245597c9..00a98f81 100644 --- a/dbm_lib/dbm_features/raw_features/audio/gne.py +++ b/dbm_lib/dbm_features/raw_features/audio/gne.py @@ -19,8 +19,8 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -gne_dir = 'audio/glottal_noise' -ff_dir = 'audio/pitch' +gne_dir = 'acoustic/glottal_noise' +ff_dir = 'acoustic/pitch' csv_ext = '_gne.csv' def gne_ratio(sound): diff --git a/dbm_lib/dbm_features/raw_features/audio/hnr.py b/dbm_lib/dbm_features/raw_features/audio/hnr.py index 18fff61e..d3dfec72 100644 --- a/dbm_lib/dbm_features/raw_features/audio/hnr.py +++ b/dbm_lib/dbm_features/raw_features/audio/hnr.py @@ -18,7 +18,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -hnr_dir = 'audio/harmonic_noise' +hnr_dir = 'acoustic/harmonic_noise' csv_ext = '_hnr.csv' error_txt = 'error: length less than 0.064' diff --git a/dbm_lib/dbm_features/raw_features/audio/intensity.py b/dbm_lib/dbm_features/raw_features/audio/intensity.py index abdd52f1..383267f8 100644 --- a/dbm_lib/dbm_features/raw_features/audio/intensity.py +++ b/dbm_lib/dbm_features/raw_features/audio/intensity.py @@ -17,7 +17,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -intensity_dir = 'audio/intensity' +intensity_dir = 'acoustic/intensity' csv_ext = '_intensity.csv' error_txt = 'error: length less than 0.064' diff --git a/dbm_lib/dbm_features/raw_features/audio/jitter.py b/dbm_lib/dbm_features/raw_features/audio/jitter.py index 0b759ee8..443e15ae 100644 --- a/dbm_lib/dbm_features/raw_features/audio/jitter.py +++ b/dbm_lib/dbm_features/raw_features/audio/jitter.py @@ -20,8 +20,8 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -jitter_dir = 'audio/jitter' -ff_dir = 'audio/pitch' +jitter_dir = 'acoustic/jitter' +ff_dir = 'acoustic/pitch' csv_ext = '_jitter.csv' def audio_jitter(sound): diff --git a/dbm_lib/dbm_features/raw_features/audio/mfcc.py b/dbm_lib/dbm_features/raw_features/audio/mfcc.py index 48c43f5d..505b3205 100644 --- a/dbm_lib/dbm_features/raw_features/audio/mfcc.py +++ b/dbm_lib/dbm_features/raw_features/audio/mfcc.py @@ -19,7 +19,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -mfcc_dir = 'audio/mfcc' +mfcc_dir = 'acoustic/mfcc' csv_ext = '_mfcc.csv' error_txt = 'error: length less than 0.064' diff --git a/dbm_lib/dbm_features/raw_features/audio/pause_segment.py b/dbm_lib/dbm_features/raw_features/audio/pause_segment.py index 8849eca6..b5c43534 100644 --- a/dbm_lib/dbm_features/raw_features/audio/pause_segment.py +++ b/dbm_lib/dbm_features/raw_features/audio/pause_segment.py @@ -20,7 +20,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -pause_seg_dir = 'audio/pause_segment' +pause_seg_dir = 'acoustic/pause_segment' csv_ext = '_pausechar.csv' def get_timing_cues(seg_starts_sec, seg_ends_sec, r_config): diff --git a/dbm_lib/dbm_features/raw_features/audio/pitch_freq.py b/dbm_lib/dbm_features/raw_features/audio/pitch_freq.py index 48464233..66f35718 100644 --- a/dbm_lib/dbm_features/raw_features/audio/pitch_freq.py +++ b/dbm_lib/dbm_features/raw_features/audio/pitch_freq.py @@ -18,7 +18,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -ff_dir = 'audio/pitch' +ff_dir = 'acoustic/pitch' csv_ext = '_pitch.csv' error_txt = 'error: length less than 0.064' diff --git a/dbm_lib/dbm_features/raw_features/audio/shimmer.py b/dbm_lib/dbm_features/raw_features/audio/shimmer.py index 56d39454..c33f163b 100644 --- a/dbm_lib/dbm_features/raw_features/audio/shimmer.py +++ b/dbm_lib/dbm_features/raw_features/audio/shimmer.py @@ -21,8 +21,8 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -shimmer_dir = 'audio/shimmer' -ff_dir = 'audio/pitch' +shimmer_dir = 'acoustic/shimmer' +ff_dir = 'acoustic/pitch' csv_ext = '_shimmer.csv' def audio_shimmer(sound): diff --git a/dbm_lib/dbm_features/raw_features/audio/voice_frame_score.py b/dbm_lib/dbm_features/raw_features/audio/voice_frame_score.py index 3e2ecf1d..1d8d0bb3 100644 --- a/dbm_lib/dbm_features/raw_features/audio/voice_frame_score.py +++ b/dbm_lib/dbm_features/raw_features/audio/voice_frame_score.py @@ -17,7 +17,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -vfs_dir = 'audio/voice_frame_score' +vfs_dir = 'acoustic/voice_frame_score' csv_ext = '_voiceprev.csv' error_txt = 'error: length less than 0.064' diff --git a/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py index a574a2d9..c52893db 100644 --- a/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py +++ b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py @@ -135,7 +135,7 @@ def run_eye_gaze(video_uri, out_dir, r_config): #filtering path to generate input & output path input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) if len(of_csv_path)>0: diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py index 7753eb0b..929cc19f 100644 --- a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -142,7 +142,7 @@ def fac_tremor_process(video_uri, out_dir, r_config, model_output=False): # try: input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface_lmk/*.csv')) if len(of_csv_path)>0: of_csv = of_csv_path[0] diff --git a/dbm_lib/dbm_features/raw_features/movement/head_motion.py b/dbm_lib/dbm_features/raw_features/movement/head_motion.py index 59179c2e..709b4f6c 100644 --- a/dbm_lib/dbm_features/raw_features/movement/head_motion.py +++ b/dbm_lib/dbm_features/raw_features/movement/head_motion.py @@ -182,7 +182,7 @@ def run_head_movement(video_uri, out_dir, r_config): #filtering path to generate input & output path input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) if len(of_csv_path)>0: diff --git a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py index 3f0cfba7..4217a968 100644 --- a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py +++ b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py @@ -18,9 +18,9 @@ from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -speech_dir = 'nlp/speech_feature' +speech_dir = 'speech/speech_feature' speech_ext = '_nlp.csv' -transcribe_ext = 'nlp/transcribe/*_transcribe.csv' +transcribe_ext = 'speech/deepspeech/*_transcribe.csv' def run_speech_feature(video_uri, out_dir, r_config, tran_tog): """ diff --git a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py index f567e967..0e23d824 100644 --- a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py +++ b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py @@ -17,7 +17,7 @@ from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -formant_dir = 'nlp/transcribe' +formant_dir = 'speech/deepspeech' csv_ext = '_transcribe.csv' error_txt = 'error: length less than 0.1' diff --git a/dbm_lib/dbm_features/raw_features/util/nlp_util.py b/dbm_lib/dbm_features/raw_features/util/nlp_util.py index 6a07141e..a13edc9b 100644 --- a/dbm_lib/dbm_features/raw_features/util/nlp_util.py +++ b/dbm_lib/dbm_features/raw_features/util/nlp_util.py @@ -41,7 +41,7 @@ def deepspeech(AUDIO_FILE,deep_path): stdout,stderr = out.communicate() except: return "error", "error" - print(stderr) + #print(stderr) return stdout,stderr def deep_speech_output_clean(result): @@ -209,4 +209,4 @@ def process_speech(transcribe_df,r_config): speech_dict['dbm_master_url'] = master_url df_speech = pd.DataFrame([speech_dict]) - return df_speech \ No newline at end of file + return df_speech diff --git a/dbm_lib/dbm_features/raw_features/util/util.py b/dbm_lib/dbm_features/raw_features/util/util.py index 8ee050cc..7ed15c11 100644 --- a/dbm_lib/dbm_features/raw_features/util/util.py +++ b/dbm_lib/dbm_features/raw_features/util/util.py @@ -86,7 +86,7 @@ def compute_open_face_features(input_filepath, bn, _ = os.path.splitext(os.path.basename(input_filepath)) if not output_directory: - output_directory = os.path.join(os.path.dirname(input_filepath), bn + '_OF_features') + output_directory = os.path.join(os.path.dirname(input_filepath), bn + '_openface') output_csv = os.path.join(output_directory, bn + '.csv') if not os.path.isfile(output_csv) or clobber: diff --git a/dbm_lib/dbm_features/raw_features/util/video_util.py b/dbm_lib/dbm_features/raw_features/util/video_util.py index 013c073c..2a60480e 100644 --- a/dbm_lib/dbm_features/raw_features/util/video_util.py +++ b/dbm_lib/dbm_features/raw_features/util/video_util.py @@ -135,10 +135,11 @@ def calc_of_for_video(of,face_cfg,fe_cfg): Creating dataframe for emotion expressivity """ new_cols = [fe_cfg.hap_exp,fe_cfg.sad_exp,fe_cfg.sur_exp,fe_cfg.fea_exp,fe_cfg.ang_exp,fe_cfg.dis_exp,fe_cfg.con_exp, - fe_cfg.neg_exp,fe_cfg.pos_exp,fe_cfg.neu_exp,fe_cfg.cai_exp,fe_cfg.com_exp,fe_cfg.happ_occ,fe_cfg.sad_occ, - fe_cfg.sur_occ,fe_cfg.fea_occ,fe_cfg.ang_occ,fe_cfg.dis_occ,fe_cfg.con_occ,fe_cfg.hap_exp_full, - fe_cfg.sad_exp_full,fe_cfg.sur_exp_full,fe_cfg.fea_exp_full,fe_cfg.ang_exp_full,fe_cfg.dis_exp_full, - fe_cfg.con_exp_full,fe_cfg.neg_exp_full,fe_cfg.pos_exp_full,fe_cfg.neu_exp_full,fe_cfg.cai_exp_full, + fe_cfg.pai_exp,fe_cfg.neg_exp,fe_cfg.pos_exp,fe_cfg.neu_exp,fe_cfg.com_lower_exp,fe_cfg.com_upper_exp, + fe_cfg.cai_exp,fe_cfg.com_exp,fe_cfg.happ_occ,fe_cfg.sad_occ,fe_cfg.sur_occ,fe_cfg.fea_occ,fe_cfg.ang_occ, + fe_cfg.dis_occ,fe_cfg.con_occ,fe_cfg.hap_exp_full,fe_cfg.sad_exp_full,fe_cfg.sur_exp_full,fe_cfg.fea_exp_full, + fe_cfg.ang_exp_full,fe_cfg.dis_exp_full,fe_cfg.con_exp_full,fe_cfg.pai_exp_full,fe_cfg.neg_exp_full, + fe_cfg.pos_exp_full,fe_cfg.neu_exp_full,fe_cfg.cai_exp_full,fe_cfg.com_lower_exp_full,fe_cfg.com_upper_exp_full, fe_cfg.com_exp_full] of[new_cols] = pd.DataFrame([[0] * len(new_cols)], index=of.index) of[fe_cfg.err_reason] = 'Pass' @@ -167,6 +168,12 @@ def calc_of_for_video(of,face_cfg,fe_cfg): emotion_exp(face_cfg.cai,of,[fe_cfg.cai_exp,fe_cfg.cai_exp_full],fe_cfg.err_reason) #Composite Expressivity emotion_exp(face_cfg.ACTION_UNITS,of,[fe_cfg.com_exp,fe_cfg.com_exp_full],fe_cfg.err_reason) + #Composite lower face expressivity + emotion_exp(face_cfg.LOWER_ACTION_UNITS,of,[fe_cfg.com_lower_exp,fe_cfg.com_lower_exp_full],fe_cfg.err_reason) + #Composite upper face Expressivity + emotion_exp(face_cfg.UPPER_ACTION_UNITS,of,[fe_cfg.com_upper_exp,fe_cfg.com_upper_exp_full],fe_cfg.err_reason) + #Composite pain expressivity + emotion_exp(face_cfg.pain,of,[fe_cfg.pai_exp,fe_cfg.pai_exp_full],fe_cfg.err_reason) #AU happiness presence emotion_pres(face_cfg.happiness,of,fe_cfg.happ_occ,fe_cfg.err_reason) #AU Sad presence diff --git a/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py b/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py index 025c9497..4249abcb 100644 --- a/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py +++ b/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py @@ -25,7 +25,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -face_asym_dir = 'video/face_asymmetry' +face_asym_dir = 'facial/face_asymmetry' csv_ext = '_facasym.csv' cv2_color_purple = (254,19,188) @@ -339,7 +339,7 @@ def run_face_asymmetry(video_uri, out_dir, f_cfg): cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) if len(of_csv_path)>0: of_csv = of_csv_path[0] diff --git a/dbm_lib/dbm_features/raw_features/video/face_au.py b/dbm_lib/dbm_features/raw_features/video/face_au.py index 79015abb..3a3f3338 100644 --- a/dbm_lib/dbm_features/raw_features/video/face_au.py +++ b/dbm_lib/dbm_features/raw_features/video/face_au.py @@ -19,7 +19,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -face_au_dir = 'video/face_au' +face_au_dir = 'facial/face_au' csv_ext = '_facau.csv' @@ -74,7 +74,7 @@ def run_face_au(video_uri, out_dir, f_cfg): cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) if len(of_csv_path)>0: df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) diff --git a/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py b/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py index 4d882976..54cf9bcd 100644 --- a/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py +++ b/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py @@ -29,7 +29,9 @@ class ConfigFaceReader(object): self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS'] self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS'] self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS'] - self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS'] + self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS'] + self.LOWER_ACTION_UNITS = config['cdx_face_config']['LOWER_ACTION_UNITS'] + self.UPPER_ACTION_UNITS = config['cdx_face_config']['UPPER_ACTION_UNITS'] self.happiness = config['cdx_face_config']['happiness'] self.sadness = config['cdx_face_config']['sadness'] self.surprise = config['cdx_face_config']['surprise'] @@ -37,6 +39,7 @@ class ConfigFaceReader(object): self.anger = config['cdx_face_config']['anger'] self.disgust = config['cdx_face_config']['disgust'] self.contempt = config['cdx_face_config']['contempt'] + self.pain = config['cdx_face_config']['pain'] self.cai = config['cdx_face_config']['CAI'] self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',') self.face_expr_dir = config['cdx_face_config']['face_expr_dir'] diff --git a/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py b/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py index e1b16be8..d47966cf 100644 --- a/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py +++ b/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py @@ -19,7 +19,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -face_expr_dir = 'video/face_expressivity' +face_expr_dir = 'facial/face_expressivity' csv_ext = '_facemo.csv' #Openface feature extraction @@ -69,7 +69,7 @@ def run_face_expressivity(video_uri, out_dir, f_cfg): cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) if len(of_csv_path)>0: df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) diff --git a/dbm_lib/dbm_features/raw_features/video/face_landmark.py b/dbm_lib/dbm_features/raw_features/video/face_landmark.py index 26a01d40..cc309508 100644 --- a/dbm_lib/dbm_features/raw_features/video/face_landmark.py +++ b/dbm_lib/dbm_features/raw_features/video/face_landmark.py @@ -19,7 +19,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -face_lmk_dir = 'video/face_landmark' +face_lmk_dir = 'facial/face_landmark' csv_ext = '_faclmk.csv' def extract_col_nm_lmk(cols): @@ -98,7 +98,7 @@ def run_face_landmark(video_uri, out_dir, f_cfg): cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv')) if len(of_csv_path)>0: df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index d42819e2..6f2e3847 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -33,9 +33,9 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac (itreable[str]) list of .csv files """ if video_tracking: - suffix = '_OF_video_features' + suffix = '_openface_lmk' else: - suffix = '_OF_features' + suffix = '_openface' csv_files = [] diff --git a/process_data.py b/process_data.py index e27a6764..55617e4b 100644 --- a/process_data.py +++ b/process_data.py @@ -101,7 +101,7 @@ def process_raw_video_dir(args, s_config, r_config): r_config: raw feature config object """ if args.output_path != None: - vid_loc = glob.glob(args.input_path + '/*.mp4') + glob.glob(args.input_path + '/*.mov') + vid_loc = glob.glob(args.input_path + '/*.mp4') + glob.glob(args.input_path + '/*.mov') + glob.glob(args.input_path + '/*.MOV') if len(vid_loc) == 0: logger.info('Directory does not have any MP4 files.') @@ -111,10 +111,12 @@ def process_raw_video_dir(args, s_config, r_config): for vid_file in vid_loc: try: fname, file_ext = os.path.splitext(vid_file) - if file_ext == '.mov': + + if file_ext.lower() == '.mov': convert_file(vid_file) common_video(fname+'.mp4', args, r_config) - + + remove_convert(vid_file, '.mp4') #removing files(ffmpeg converted ) after processing except Exception as e: logger.error('Failed to process mp4 file.') pf.remove_file(vid_file) @@ -128,7 +130,7 @@ def process_raw_audio_dir(args, s_config, r_config): r_config: raw feature config object """ if args.output_path != None: - audio_loc = glob.glob(args.input_path + '/*.wav') + glob.glob(args.input_path + '/*.mp3') + audio_loc = glob.glob(args.input_path + '/*.wav') + glob.glob(args.input_path + '/*.mp3') + glob.glob(args.input_path + '/*.MP3') if len(audio_loc) == 0: logger.info('Directory does not have any WAV files.') @@ -138,31 +140,47 @@ def process_raw_audio_dir(args, s_config, r_config): for audio in audio_loc: try: fname, file_ext = os.path.splitext(audio) - if file_ext == '.mp3': + if file_ext.lower() == '.mp3': convert_file(audio) + out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(fname+'.wav', out_path, args.dbm_group, r_config) pf.process_nlp(fname +'.wav', out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) + + remove_convert(audio, '.wav') #removing files(ffmpeg converted) after processing except Exception as e: logger.error('Failed to process wav file.') def convert_file(input_filepath): + """ + Converting mp3/mov to wav/mp4 files + """ _, file_ext = os.path.splitext(os.path.basename(input_filepath)) fname, _ = splitext(input_filepath) - - if file_ext == '.mp3': + call = [] + + if file_ext.lower() == '.mp3': output_filepath = fname + '.wav' logger.info('Converting audio from {} to wav'.format(input_filepath)) call = ['ffmpeg', '-i', input_filepath, output_filepath] - if file_ext == '.mov': + if file_ext.lower() == '.mov': output_filepath = fname + '.mp4' logger.info('Converting video from {} to mp4'.format(input_filepath)) call = ['ffmpeg', '-i', input_filepath, '-vcodec', 'h264','-acodec','aac', '-strict', '-2', output_filepath] - subprocess.check_output(call) - - + if len(call)>0: + subprocess.check_output(call) + +def remove_convert(input_filepath, file_ext): + """ + removing converted files after processing + """ + expected_ext = ['.mp3', '.mov'] + input_loc, inp_ext = os.path.splitext(input_filepath) + + if inp_ext.lower() in expected_ext: + pf.remove_file(input_loc + file_ext, file_ext) def process_derive(args, r_config, d_config, input_type): """ @@ -201,13 +219,16 @@ if __name__=="__main__": if file_ext.lower() in ['.mp4','.mov']: if file_ext.lower() == '.mov': convert_file(args.input_path) + process_raw_video_file(args, s_config, r_config) + remove_convert(args.input_path, '.mp4') elif file_ext.lower() in ['.wav','.mp3']: if file_ext.lower() == '.mp3': convert_file(args.input_path) + process_raw_audio_file(args, s_config, r_config) - + remove_convert(args.input_path, '.wav') else: logger.error('No WAV/MP3 or MOV/MP4 files detected in input path') else: diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index e0f51bb8..43ee0e78 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -38,7 +38,7 @@ derive_feature: FAC_EXP: ['hap_exp', 'sad_exp', 'sur_exp', 'fea_exp', 'ang_exp', 'dis_exp', 'con_exp', 'happ_occ', 'sad_occ', 'sur_occ', 'fea_occ', 'ang_occ', 'dis_occ', 'con_occ', 'pos_exp', 'neg_exp', 'com_exp', 'hap_exp_full', 'sad_exp_full', 'sur_exp_full','fea_exp_full', 'ang_exp_full', 'dis_exp_full', 'con_exp_full', 'pos_exp_full', - 'neg_exp_full', 'com_exp_full'] + 'neg_exp_full', 'com_exp_full', 'com_lower_exp','com_upper_exp', 'pai_exp', 'pai_exp_full'] FAC_LMK: ['fac_LMK00disp', 'fac_LMK01disp', 'fac_LMK02disp', 'fac_LMK03disp', 'fac_LMK04disp', 'fac_LMK05disp', 'fac_LMK06disp', 'fac_LMK07disp', 'fac_LMK08disp', 'fac_LMK09disp', 'fac_LMK10disp', 'fac_LMK11disp', 'fac_LMK12disp', 'fac_LMK13disp', 'fac_LMK14disp', 'fac_LMK15disp', 'fac_LMK16disp', @@ -134,17 +134,20 @@ derive_feature: ang_exp: ['mean', 'std'] dis_exp: ['mean', 'std'] con_exp: ['mean', 'std'] - happ_occ: ['mean', 'std', 'pct'] - sad_occ: ['mean', 'std', 'pct'] - sur_occ: ['mean', 'std', 'pct'] - fea_occ: ['mean', 'std', 'pct'] - ang_occ: ['mean', 'std', 'pct'] - dis_occ: ['mean', 'std', 'pct'] - con_occ: ['mean', 'std', 'pct'] + happ_occ: ['pct'] + sad_occ: ['pct'] + sur_occ: ['pct'] + fea_occ: ['pct'] + ang_occ: ['pct'] + dis_occ: ['pct'] + con_occ: ['pct'] pos_exp: ['mean', 'std', 'pct'] neg_exp: ['mean', 'std', 'pct'] neu_exp: ['mean', 'std', 'pct'] com_exp: ['mean', 'std', 'pct'] + com_lower_exp: ['mean','std','pct'] + com_upper_exp: ['mean','std','pct'] + pai_exp: ['mean','std','pct'] hap_exp_full: ['mean', 'std'] sad_exp_full: ['mean', 'std'] sur_exp_full: ['mean', 'std'] @@ -156,6 +159,9 @@ derive_feature: neg_exp_full: ['mean', 'std'] neu_exp_full: ['mean', 'std'] com_exp_full: ['mean', 'std'] + com_lower_exp_full: ['mean','std'] + com_upper_exp_full: ['mean', 'std'] + pai_exp_full: ['mean','std'] #Facial Landmarks fac_LMK00disp: ['mean', 'std'] diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index 78a4ae7b..e05aff04 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -26,6 +26,9 @@ raw_feature: neu_exp: neu_exp cai_exp: cai_exp com_exp: fac_comintsoft + com_lower_exp: fac_comlowintsoft + com_upper_exp: fac_comuppintsoft + pai_exp: fac_paiintsoft hap_exp_full: fac_hapinthard sad_exp_full: fac_sadinthard sur_exp_full: fac_surinthard @@ -38,6 +41,9 @@ raw_feature: neu_exp_full: neu_exp_full cai_exp_full: cai_exp_full com_exp_full: fac_cominthard + com_lower_exp_full: fac_comlowinthard + com_upper_exp_full: fac_comuppinthard + pai_exp_full: fac_paiinthard #Facial asymmetry fac_AsymMaskMouth: fac_asymmaskmouth diff --git a/resources/services/face_util.yml b/resources/services/face_util.yml index d398cdf1..241d83c6 100644 --- a/resources/services/face_util.yml +++ b/resources/services/face_util.yml @@ -1,5 +1,7 @@ cdx_face_config: ACTION_UNITS: [[6, 12],[1, 4, 15],[1, 2, 5, 26],[1, 2, 4, 5, 7, 20, 26],[4, 5, 7, 23],[9, 15],[12, 14]] + LOWER_ACTION_UNITS: [[12], [15], [26], [20, 26], [23], [15], [12, 14]] + UPPER_ACTION_UNITS: [[6], [1, 4], [1, 2, 5], [1, 2, 4, 5, 7], [4, 5, 7], [9]] NEG_ACTION_UNITS: [[1, 4, 15], [1, 2, 4, 5, 7, 20, 26], [4, 5, 7, 23], [9, 15], [12, 14]] POS_ACTION_UNITS: [[6, 12]] NET_ACTION_UNITS: [[1, 2, 5, 26]] @@ -10,6 +12,7 @@ cdx_face_config: anger: [[4, 5, 7, 23]] disgust: [[9, 15]] contempt: [[12, 14]] + pain: [[4, 6, 7, 9, 10, 12, 20, 26]] CAI: [[6, 12],[1, 4, 15],[2, 5, 26],[7, 20, 26],[23],[9],[12, 14]] SELECTED_FEATURES: AU,POSE face_expr_dir: /video/face_expressivity