bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

face_features

Browse Source
This commit is contained in:
Vijay Yadev
2020-12-15 14:50:26 -05:00
parent 18fbd84c33
commit f6c5add7f0
8 changed files with 72 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dbm_lib/config/config_raw_feature.py
View File

@@ -194,6 +194,9 @@ class ConfigRawReader(object):
self.neu_exp = config['raw_feature']['neu_exp'] self.neu_exp = config['raw_feature']['neu_exp']
self.cai_exp = config['raw_feature']['cai_exp'] self.cai_exp = config['raw_feature']['cai_exp']
self.com_exp = config['raw_feature']['com_exp'] self.com_exp = config['raw_feature']['com_exp']
self.com_lower_exp = config['raw_feature']['com_lower_exp']
self.com_upper_exp = config['raw_feature']['com_upper_exp']
self.pai_exp = config['raw_feature']['pai_exp']
self.hap_exp_full = config['raw_feature']['hap_exp_full'] self.hap_exp_full = config['raw_feature']['hap_exp_full']
self.sad_exp_full = config['raw_feature']['sad_exp_full'] self.sad_exp_full = config['raw_feature']['sad_exp_full']
self.sur_exp_full = config['raw_feature']['sur_exp_full'] self.sur_exp_full = config['raw_feature']['sur_exp_full']
@@ -206,6 +209,9 @@ class ConfigRawReader(object):
self.neu_exp_full = config['raw_feature']['neu_exp_full'] self.neu_exp_full = config['raw_feature']['neu_exp_full']
self.cai_exp_full = config['raw_feature']['cai_exp_full'] self.cai_exp_full = config['raw_feature']['cai_exp_full']
self.com_exp_full = config['raw_feature']['com_exp_full'] self.com_exp_full = config['raw_feature']['com_exp_full']
self.com_lower_exp_full = config['raw_feature']['com_lower_exp_full']
self.com_upper_exp_full = config['raw_feature']['com_upper_exp_full']
self.pai_exp_full = config['raw_feature']['pai_exp_full']
self.fac_AsymMaskMouth = config['raw_feature']['fac_AsymMaskMouth'] self.fac_AsymMaskMouth = config['raw_feature']['fac_AsymMaskMouth']
self.fac_AsymMaskEye = config['raw_feature']['fac_AsymMaskEye'] self.fac_AsymMaskEye = config['raw_feature']['fac_AsymMaskEye']
self.fac_AsymMaskEyebrow = config['raw_feature']['fac_AsymMaskEyebrow'] self.fac_AsymMaskEyebrow = config['raw_feature']['fac_AsymMaskEyebrow']

4
dbm_lib/controller/process_feature.py
View File

@@ -152,13 +152,13 @@ def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path):
speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog) speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog)
def remove_file(file_path): def remove_file(file_path, file_ext = '.wav'):
""" """
removing wav file removing wav file
""" """
file_dir = dirname(file_path) file_dir = dirname(file_path)
file_name, _ = splitext(basename(file_path)) file_name, _ = splitext(basename(file_path))
wav_file = glob.glob(join(file_dir, file_name + '.wav')) wav_file = glob.glob(join(file_dir, file_name + file_ext))
if len(wav_file)> 0: if len(wav_file)> 0:
os.remove(wav_file[0]) os.remove(wav_file[0])

15
dbm_lib/dbm_features/raw_features/util/video_util.py
View File

@@ -135,10 +135,11 @@ def calc_of_for_video(of,face_cfg,fe_cfg):
Creating dataframe for emotion expressivity Creating dataframe for emotion expressivity
""" """
new_cols = [fe_cfg.hap_exp,fe_cfg.sad_exp,fe_cfg.sur_exp,fe_cfg.fea_exp,fe_cfg.ang_exp,fe_cfg.dis_exp,fe_cfg.con_exp, new_cols = [fe_cfg.hap_exp,fe_cfg.sad_exp,fe_cfg.sur_exp,fe_cfg.fea_exp,fe_cfg.ang_exp,fe_cfg.dis_exp,fe_cfg.con_exp,
fe_cfg.neg_exp,fe_cfg.pos_exp,fe_cfg.neu_exp,fe_cfg.cai_exp,fe_cfg.com_exp,fe_cfg.happ_occ,fe_cfg.sad_occ, fe_cfg.pai_exp,fe_cfg.neg_exp,fe_cfg.pos_exp,fe_cfg.neu_exp,fe_cfg.com_lower_exp,fe_cfg.com_upper_exp,
fe_cfg.sur_occ,fe_cfg.fea_occ,fe_cfg.ang_occ,fe_cfg.dis_occ,fe_cfg.con_occ,fe_cfg.hap_exp_full, fe_cfg.cai_exp,fe_cfg.com_exp,fe_cfg.happ_occ,fe_cfg.sad_occ,fe_cfg.sur_occ,fe_cfg.fea_occ,fe_cfg.ang_occ,
fe_cfg.sad_exp_full,fe_cfg.sur_exp_full,fe_cfg.fea_exp_full,fe_cfg.ang_exp_full,fe_cfg.dis_exp_full, fe_cfg.dis_occ,fe_cfg.con_occ,fe_cfg.hap_exp_full,fe_cfg.sad_exp_full,fe_cfg.sur_exp_full,fe_cfg.fea_exp_full,
fe_cfg.con_exp_full,fe_cfg.neg_exp_full,fe_cfg.pos_exp_full,fe_cfg.neu_exp_full,fe_cfg.cai_exp_full, fe_cfg.ang_exp_full,fe_cfg.dis_exp_full,fe_cfg.con_exp_full,fe_cfg.pai_exp_full,fe_cfg.neg_exp_full,
fe_cfg.pos_exp_full,fe_cfg.neu_exp_full,fe_cfg.cai_exp_full,fe_cfg.com_lower_exp_full,fe_cfg.com_upper_exp_full,
fe_cfg.com_exp_full] fe_cfg.com_exp_full]
of[new_cols] = pd.DataFrame([[0] * len(new_cols)], index=of.index) of[new_cols] = pd.DataFrame([[0] * len(new_cols)], index=of.index)
of[fe_cfg.err_reason] = 'Pass' of[fe_cfg.err_reason] = 'Pass'
@@ -167,6 +168,12 @@ def calc_of_for_video(of,face_cfg,fe_cfg):
emotion_exp(face_cfg.cai,of,[fe_cfg.cai_exp,fe_cfg.cai_exp_full],fe_cfg.err_reason) emotion_exp(face_cfg.cai,of,[fe_cfg.cai_exp,fe_cfg.cai_exp_full],fe_cfg.err_reason)
#Composite Expressivity #Composite Expressivity
emotion_exp(face_cfg.ACTION_UNITS,of,[fe_cfg.com_exp,fe_cfg.com_exp_full],fe_cfg.err_reason) emotion_exp(face_cfg.ACTION_UNITS,of,[fe_cfg.com_exp,fe_cfg.com_exp_full],fe_cfg.err_reason)
#Composite lower face expressivity
emotion_exp(face_cfg.LOWER_ACTION_UNITS,of,[fe_cfg.com_lower_exp,fe_cfg.com_lower_exp_full],fe_cfg.err_reason)
#Composite upper face Expressivity
emotion_exp(face_cfg.UPPER_ACTION_UNITS,of,[fe_cfg.com_upper_exp,fe_cfg.com_upper_exp_full],fe_cfg.err_reason)
#Composite pain expressivity
emotion_exp(face_cfg.pain,of,[fe_cfg.pai_exp,fe_cfg.pai_exp_full],fe_cfg.err_reason)
#AU happiness presence #AU happiness presence
emotion_pres(face_cfg.happiness,of,fe_cfg.happ_occ,fe_cfg.err_reason) emotion_pres(face_cfg.happiness,of,fe_cfg.happ_occ,fe_cfg.err_reason)
#AU Sad presence #AU Sad presence

3
dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py
View File

@@ -30,6 +30,8 @@ class ConfigFaceReader(object):
self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS'] self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS']
self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS'] self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS']
self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS'] self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS']
self.LOWER_ACTION_UNITS = config['cdx_face_config']['LOWER_ACTION_UNITS']
self.UPPER_ACTION_UNITS = config['cdx_face_config']['UPPER_ACTION_UNITS']
self.happiness = config['cdx_face_config']['happiness'] self.happiness = config['cdx_face_config']['happiness']
self.sadness = config['cdx_face_config']['sadness'] self.sadness = config['cdx_face_config']['sadness']
self.surprise = config['cdx_face_config']['surprise'] self.surprise = config['cdx_face_config']['surprise']
@@ -37,6 +39,7 @@ class ConfigFaceReader(object):
self.anger = config['cdx_face_config']['anger'] self.anger = config['cdx_face_config']['anger']
self.disgust = config['cdx_face_config']['disgust'] self.disgust = config['cdx_face_config']['disgust']
self.contempt = config['cdx_face_config']['contempt'] self.contempt = config['cdx_face_config']['contempt']
self.pain = config['cdx_face_config']['pain']
self.cai = config['cdx_face_config']['CAI'] self.cai = config['cdx_face_config']['CAI']
self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',') self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',')
self.face_expr_dir = config['cdx_face_config']['face_expr_dir'] self.face_expr_dir = config['cdx_face_config']['face_expr_dir']

35
process_data.py
View File

@@ -101,7 +101,7 @@ def process_raw_video_dir(args, s_config, r_config):
r_config: raw feature config object r_config: raw feature config object
""" """
if args.output_path != None: if args.output_path != None:
vid_loc = glob.glob(args.input_path + '/*.mp4') + glob.glob(args.input_path + '/*.mov') vid_loc = glob.glob(args.input_path + '/*.mp4') + glob.glob(args.input_path + '/*.mov') + glob.glob(args.input_path + '/*.MOV')
if len(vid_loc) == 0: if len(vid_loc) == 0:
logger.info('Directory does not have any MP4 files.') logger.info('Directory does not have any MP4 files.')
@@ -111,10 +111,12 @@ def process_raw_video_dir(args, s_config, r_config):
for vid_file in vid_loc: for vid_file in vid_loc:
try: try:
fname, file_ext = os.path.splitext(vid_file) fname, file_ext = os.path.splitext(vid_file)
if file_ext == '.mov':
if file_ext.lower() == '.mov':
convert_file(vid_file) convert_file(vid_file)
common_video(fname+'.mp4', args, r_config) common_video(fname+'.mp4', args, r_config)
remove_convert(vid_file, '.mp4') #removing files(ffmpeg converted ) after processing
except Exception as e: except Exception as e:
logger.error('Failed to process mp4 file.') logger.error('Failed to process mp4 file.')
pf.remove_file(vid_file) pf.remove_file(vid_file)
@@ -128,7 +130,7 @@ def process_raw_audio_dir(args, s_config, r_config):
r_config: raw feature config object r_config: raw feature config object
""" """
if args.output_path != None: if args.output_path != None:
audio_loc = glob.glob(args.input_path + '/*.wav') + glob.glob(args.input_path + '/*.mp3') audio_loc = glob.glob(args.input_path + '/*.wav') + glob.glob(args.input_path + '/*.mp3') + glob.glob(args.input_path + '/*.MP3')
if len(audio_loc) == 0: if len(audio_loc) == 0:
logger.info('Directory does not have any WAV files.') logger.info('Directory does not have any WAV files.')
@@ -138,31 +140,47 @@ def process_raw_audio_dir(args, s_config, r_config):
for audio in audio_loc: for audio in audio_loc:
try: try:
fname, file_ext = os.path.splitext(audio) fname, file_ext = os.path.splitext(audio)
if file_ext == '.mp3': if file_ext.lower() == '.mp3':
convert_file(audio) convert_file(audio)
out_path = os.path.join(args.output_path, 'raw_variables') out_path = os.path.join(args.output_path, 'raw_variables')
pf.process_acoustic(fname+'.wav', out_path, args.dbm_group, r_config) pf.process_acoustic(fname+'.wav', out_path, args.dbm_group, r_config)
pf.process_nlp(fname +'.wav', out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) pf.process_nlp(fname +'.wav', out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH)
remove_convert(audio, '.wav') #removing files(ffmpeg converted) after processing
except Exception as e: except Exception as e:
logger.error('Failed to process wav file.') logger.error('Failed to process wav file.')
def convert_file(input_filepath): def convert_file(input_filepath):
"""
Converting mp3/mov to wav/mp4 files
"""
_, file_ext = os.path.splitext(os.path.basename(input_filepath)) _, file_ext = os.path.splitext(os.path.basename(input_filepath))
fname, _ = splitext(input_filepath) fname, _ = splitext(input_filepath)
call = []
if file_ext == '.mp3': if file_ext.lower() == '.mp3':
output_filepath = fname + '.wav' output_filepath = fname + '.wav'
logger.info('Converting audio from {} to wav'.format(input_filepath)) logger.info('Converting audio from {} to wav'.format(input_filepath))
call = ['ffmpeg', '-i', input_filepath, output_filepath] call = ['ffmpeg', '-i', input_filepath, output_filepath]
if file_ext == '.mov': if file_ext.lower() == '.mov':
output_filepath = fname + '.mp4' output_filepath = fname + '.mp4'
logger.info('Converting video from {} to mp4'.format(input_filepath)) logger.info('Converting video from {} to mp4'.format(input_filepath))
call = ['ffmpeg', '-i', input_filepath, '-vcodec', 'h264','-acodec','aac', '-strict', '-2', output_filepath] call = ['ffmpeg', '-i', input_filepath, '-vcodec', 'h264','-acodec','aac', '-strict', '-2', output_filepath]
if len(call)>0:
subprocess.check_output(call) subprocess.check_output(call)
def remove_convert(input_filepath, file_ext):
"""
removing converted files after processing
"""
expected_ext = ['.mp3', '.mov']
input_loc, inp_ext = os.path.splitext(input_filepath)
if inp_ext.lower() in expected_ext:
pf.remove_file(input_loc + file_ext, file_ext)
def process_derive(args, r_config, d_config, input_type): def process_derive(args, r_config, d_config, input_type):
""" """
@@ -201,13 +219,16 @@ if __name__=="__main__":
if file_ext.lower() in ['.mp4','.mov']: if file_ext.lower() in ['.mp4','.mov']:
if file_ext.lower() == '.mov': if file_ext.lower() == '.mov':
convert_file(args.input_path) convert_file(args.input_path)
process_raw_video_file(args, s_config, r_config) process_raw_video_file(args, s_config, r_config)
remove_convert(args.input_path, '.mp4')
elif file_ext.lower() in ['.wav','.mp3']: elif file_ext.lower() in ['.wav','.mp3']:
if file_ext.lower() == '.mp3': if file_ext.lower() == '.mp3':
convert_file(args.input_path) convert_file(args.input_path)
process_raw_audio_file(args, s_config, r_config)
process_raw_audio_file(args, s_config, r_config)
remove_convert(args.input_path, '.wav')
else: else:
logger.error('No WAV/MP3 or MOV/MP4 files detected in input path') logger.error('No WAV/MP3 or MOV/MP4 files detected in input path')
else: else:

8
resources/features/derived_feature.yml
View File

@@ -38,7 +38,7 @@ derive_feature:
FAC_EXP: ['hap_exp', 'sad_exp', 'sur_exp', 'fea_exp', 'ang_exp', 'dis_exp', 'con_exp', 'happ_occ', 'sad_occ', FAC_EXP: ['hap_exp', 'sad_exp', 'sur_exp', 'fea_exp', 'ang_exp', 'dis_exp', 'con_exp', 'happ_occ', 'sad_occ',
'sur_occ', 'fea_occ', 'ang_occ', 'dis_occ', 'con_occ', 'pos_exp', 'neg_exp', 'com_exp', 'hap_exp_full', 'sur_occ', 'fea_occ', 'ang_occ', 'dis_occ', 'con_occ', 'pos_exp', 'neg_exp', 'com_exp', 'hap_exp_full',
'sad_exp_full', 'sur_exp_full','fea_exp_full', 'ang_exp_full', 'dis_exp_full', 'con_exp_full', 'pos_exp_full', 'sad_exp_full', 'sur_exp_full','fea_exp_full', 'ang_exp_full', 'dis_exp_full', 'con_exp_full', 'pos_exp_full',
'neg_exp_full', 'com_exp_full'] 'neg_exp_full', 'com_exp_full', 'com_lower_exp','com_upper_exp', 'pai_exp', 'pai_exp_full']
FAC_LMK: ['fac_LMK00disp', 'fac_LMK01disp', 'fac_LMK02disp', 'fac_LMK03disp', 'fac_LMK04disp', FAC_LMK: ['fac_LMK00disp', 'fac_LMK01disp', 'fac_LMK02disp', 'fac_LMK03disp', 'fac_LMK04disp',
'fac_LMK05disp', 'fac_LMK06disp', 'fac_LMK07disp', 'fac_LMK08disp', 'fac_LMK09disp', 'fac_LMK10disp', 'fac_LMK05disp', 'fac_LMK06disp', 'fac_LMK07disp', 'fac_LMK08disp', 'fac_LMK09disp', 'fac_LMK10disp',
'fac_LMK11disp', 'fac_LMK12disp', 'fac_LMK13disp', 'fac_LMK14disp', 'fac_LMK15disp', 'fac_LMK16disp', 'fac_LMK11disp', 'fac_LMK12disp', 'fac_LMK13disp', 'fac_LMK14disp', 'fac_LMK15disp', 'fac_LMK16disp',
@@ -145,6 +145,9 @@ derive_feature:
neg_exp: ['mean', 'std', 'pct'] neg_exp: ['mean', 'std', 'pct']
neu_exp: ['mean', 'std', 'pct'] neu_exp: ['mean', 'std', 'pct']
com_exp: ['mean', 'std', 'pct'] com_exp: ['mean', 'std', 'pct']
com_lower_exp: ['mean','std','pct']
com_upper_exp: ['mean','std','pct']
pai_exp: ['mean','std','pct']
hap_exp_full: ['mean', 'std'] hap_exp_full: ['mean', 'std']
sad_exp_full: ['mean', 'std'] sad_exp_full: ['mean', 'std']
sur_exp_full: ['mean', 'std'] sur_exp_full: ['mean', 'std']
@@ -156,6 +159,9 @@ derive_feature:
neg_exp_full: ['mean', 'std'] neg_exp_full: ['mean', 'std']
neu_exp_full: ['mean', 'std'] neu_exp_full: ['mean', 'std']
com_exp_full: ['mean', 'std'] com_exp_full: ['mean', 'std']
com_lower_exp_full: ['mean','std']
com_upper_exp_full: ['mean', 'std']
pai_exp_full: ['mean','std']
#Facial Landmarks #Facial Landmarks
fac_LMK00disp: ['mean', 'std'] fac_LMK00disp: ['mean', 'std']

6
resources/features/raw_feature.yml
View File

@@ -26,6 +26,9 @@ raw_feature:
neu_exp: neu_exp neu_exp: neu_exp
cai_exp: cai_exp cai_exp: cai_exp
com_exp: fac_comintsoft com_exp: fac_comintsoft
com_lower_exp: fac_comlowintsoft
com_upper_exp: fac_comuppintsoft
pai_exp: fac_paiintsoft
hap_exp_full: fac_hapinthard hap_exp_full: fac_hapinthard
sad_exp_full: fac_sadinthard sad_exp_full: fac_sadinthard
sur_exp_full: fac_surinthard sur_exp_full: fac_surinthard
@@ -38,6 +41,9 @@ raw_feature:
neu_exp_full: neu_exp_full neu_exp_full: neu_exp_full
cai_exp_full: cai_exp_full cai_exp_full: cai_exp_full
com_exp_full: fac_cominthard com_exp_full: fac_cominthard
com_lower_exp_full: fac_comlowinthard
com_upper_exp_full: fac_comuppinthard
pai_exp_full: fac_paiinthard
#Facial asymmetry #Facial asymmetry
fac_AsymMaskMouth: fac_asymmaskmouth fac_AsymMaskMouth: fac_asymmaskmouth

3
resources/services/face_util.yml
View File

@@ -1,5 +1,7 @@
cdx_face_config: cdx_face_config:
ACTION_UNITS: [[6, 12],[1, 4, 15],[1, 2, 5, 26],[1, 2, 4, 5, 7, 20, 26],[4, 5, 7, 23],[9, 15],[12, 14]] ACTION_UNITS: [[6, 12],[1, 4, 15],[1, 2, 5, 26],[1, 2, 4, 5, 7, 20, 26],[4, 5, 7, 23],[9, 15],[12, 14]]
LOWER_ACTION_UNITS: [[12], [15], [26], [20, 26], [23], [15], [12, 14]]
UPPER_ACTION_UNITS: [[6], [1, 4], [1, 2, 5], [1, 2, 4, 5, 7], [4, 5, 7], [9]]
NEG_ACTION_UNITS: [[1, 4, 15], [1, 2, 4, 5, 7, 20, 26], [4, 5, 7, 23], [9, 15], [12, 14]] NEG_ACTION_UNITS: [[1, 4, 15], [1, 2, 4, 5, 7, 20, 26], [4, 5, 7, 23], [9, 15], [12, 14]]
POS_ACTION_UNITS: [[6, 12]] POS_ACTION_UNITS: [[6, 12]]
NET_ACTION_UNITS: [[1, 2, 5, 26]] NET_ACTION_UNITS: [[1, 2, 5, 26]]
@@ -10,6 +12,7 @@ cdx_face_config:
anger: [[4, 5, 7, 23]] anger: [[4, 5, 7, 23]]
disgust: [[9, 15]] disgust: [[9, 15]]
contempt: [[12, 14]] contempt: [[12, 14]]
pain: [[4, 6, 7, 9, 10, 12, 20, 26]]
CAI: [[6, 12],[1, 4, 15],[2, 5, 26],[7, 20, 26],[23],[9],[12, 14]] CAI: [[6, 12],[1, 4, 15],[2, 5, 26],[7, 20, 26],[23],[9],[12, 14]]
SELECTED_FEATURES: AU,POSE SELECTED_FEATURES: AU,POSE
face_expr_dir: /video/face_expressivity face_expr_dir: /video/face_expressivity