move pkg, resources, dbm_lib, to under 1 opendbm directory

opendbm/dbm_lib/dbm_features/raw_features/audio/formant_freq.py

@@ -0,0 +1,133 @@
+"""
+file_name: formant_freq
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import parselmouth
+import numpy as np
+import parselmouth
+import librosa
+import glob
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+formant_dir = 'acoustic/formant_freq'
+csv_ext = '_formant.csv'
+error_txt = 'error: length less than 0.064'
+
+def formant_list(formant,snd):
+    """
+    Getting formant frequency per second
+    Args:
+        formant: Formant object for sound wave
+        snd: Parselmouth sound object
+    Returns:
+        List of first through fourth formant for each frame
+    """
+    f1_list = []
+    f2_list = []
+    f3_list = []
+    f4_list = []
+    
+    dur = snd.duration-0.02
+    dur_round = round(dur, 2)
+    
+    time_list = np.arange(0.001, dur_round, 0.001)
+    for time in time_list:
+        
+        f1 = formant.get_value_at_time(1,time)
+        f2 = formant.get_value_at_time(2,time)
+        f3 = formant.get_value_at_time(3,time)
+        f4 = formant.get_value_at_time(4,time)
+        
+        f1_list.append(f1)
+        f2_list.append(f2)
+        f3_list.append(f3)
+        f4_list.append(f4)
+    return f1_list,f2_list,f3_list,f4_list
+
+def formant_score(path):
+    """
+    Using parselmouth library fetching Formant Frequency
+    Args:
+        path: (.wav) audio file location
+    Returns:
+        (list) list of Formant freq for each voice frame
+    """
+    sound_pat = parselmouth.Sound(path)
+    formant = sound_pat.to_formant_burg(time_step=.001)
+    f_score = formant_list(formant,sound_pat)
+    return f_score
+
+def calc_formant(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing Formant freq matrix
+    Args:
+        audio_file: (.wav) parsed audio file; fl_name: input file name
+        out_loc: (str) Output directory; r_config: raw variable config
+    """
+    
+    f1_list,f2_list,f3_list,f4_list = formant_score(audio_file)
+    df_formant = pd.DataFrame(f1_list, columns=[r_config.aco_fm1])
+    
+    df_formant[r_config.aco_fm2] = f2_list
+    df_formant[r_config.aco_fm3] = f3_list
+    df_formant[r_config.aco_fm4] = f4_list
+    
+    df_formant.replace('', np.nan, regex=True,inplace=True)
+    df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    
+    df_formant['Frames'] = df_formant.index
+    df_formant['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
+    
+def empty_fm(video_uri, out_loc, fl_name, r_config):
+    
+    """
+    Preparing empty formant frequency matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_fm1, r_config.aco_fm2, r_config.aco_fm3, r_config.aco_fm4, r_config.err_reason]
+    out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
+    df_fm = pd.DataFrame(out_val, columns = cols)
+    df_fm['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
+
+def run_formant(video_uri, out_dir, r_config):
+    
+    """
+    Processing all patient's for fetching Formant freq
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                empty_fm(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_formant(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/gne.py

@@ -0,0 +1,161 @@
+"""
+file_name: gne
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import numpy as np
+import os
+import glob
+import parselmouth
+import librosa
+import more_itertools as mit
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+gne_dir = 'acoustic/glottal_noise'
+ff_dir =  'acoustic/pitch'
+csv_ext = '_gne.csv'
+
+def gne_ratio(sound):
+    """
+    Using parselmouth library fetching glottal noise excitation ratio
+    Args:
+        sound: parselmouth object
+    Returns:
+        (list) list of gne ratio for each voice frame
+    """
+    harmonicity_gne = sound.to_harmonicity_gne()
+    gne_all_bands = harmonicity_gne.values
+    gne_all_bands = np.where(gne_all_bands==-200, np.NaN, gne_all_bands)
+    
+    gne = np.nanmax(gne_all_bands) # following http://www.fon.hum.uva.nl/rob/NKI_TEVA/TEVA/HTML/NKI_TEVA.pdf
+    return gne
+
+def empty_gne(video_uri, out_loc, fl_name, r_config, error_txt):
+    """
+    Preparing empty GNE matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_gne, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    
+    df_gne = pd.DataFrame(out_val, columns = cols)
+    df_gne['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_gne, out_loc, fl_name, gne_dir, csv_ext)
+    
+def segment_pitch(dir_path, r_config):
+    """
+    segmenting pitch freq for each voice segment
+    """
+    com_speech_sort, voiced_yes, voiced_no  = ([], ) * 3
+    for file in os.listdir(dir_path):
+        try:
+            
+            if file.endswith('_pitch.csv'):
+                
+                ff_df = pd.read_csv((dir_path+'/'+file))
+                voice_label = ff_df[r_config.aco_voiceLabel]
+                
+                indices_yes = [i for i, x in enumerate(voice_label) if x == "yes"]
+                voiced_yes = [list(group) for group in mit.consecutive_groups(indices_yes)]
+                
+                indices_no = [i for i, x in enumerate(voice_label) if x == "no"]
+                voiced_no = [list(group) for group in mit.consecutive_groups(indices_no)]
+                
+                com_speech = voiced_yes + voiced_no
+                com_speech_sort = sorted(com_speech, key=lambda x: x[0])
+        except:
+            pass
+        
+    return com_speech_sort, voiced_yes, voiced_no
+
+def segment_gne(com_speech_sort, voiced_yes, voiced_no, gne_all_frames, audio_file):
+    """
+    calculating gne for each voice segment
+    """
+    snd = parselmouth.Sound(audio_file)
+    pitch = snd.to_pitch(time_step=.001)
+    
+    for idx, vs in enumerate(com_speech_sort):
+        try:
+            
+            max_gne = np.NaN
+            if vs in voiced_yes and len(vs)>1:
+                
+                start_time = pitch.get_time_from_frame_number(vs[0])
+                end_time = pitch.get_time_from_frame_number(vs[-1])
+
+                snd_start = int(snd.get_frame_number_from_time(start_time))
+                snd_end = int(snd.get_frame_number_from_time(end_time))
+
+                samples = parselmouth.Sound(snd.as_array()[0][snd_start:snd_end])
+                max_gne = gne_ratio(samples)
+        except:
+            pass
+
+        gne_all_frames[idx] = max_gne
+    return gne_all_frames
+    
+def calc_gne(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing gne matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        out_loc: (str) Output directory for csv's
+    """
+    dir_path = os.path.join(out_loc, ff_dir)
+    if os.path.isdir(dir_path):
+        voice_seg = segment_pitch(dir_path, r_config)
+        
+        gne_all_frames = [np.NaN] * len(voice_seg[0])
+        gne_segment_frames = segment_gne(voice_seg[0], voice_seg[1], voice_seg[2], gne_all_frames, audio_file)
+        
+        df_gne = pd.DataFrame(gne_segment_frames, columns=[r_config.aco_gne])
+        df_gne[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+        
+        df_gne['Frames'] = df_gne.index
+        df_gne['dbm_master_url'] = video_uri
+        
+        logger.info('Processing Output file {} '.format(out_loc))
+        ut.save_output(df_gne, out_loc, fl_name, gne_dir, csv_ext)
+        
+    else:
+        error_txt = 'error: pitch freq not available'
+        empty_gne(video_uri, out_loc, fl_name, r_config, error_txt)
+
+def run_gne(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for fetching glottal noise ratio
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                error_txt = 'error: length less than 0.064'
+                empty_gne(video_uri, out_loc, fl_name, r_config, error_txt)
+                return
+
+            calc_gne(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/hnr.py

@@ -0,0 +1,96 @@
+"""
+file_name: hnr
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import numpy as np
+import os
+import glob
+import parselmouth
+import librosa
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+hnr_dir = 'acoustic/harmonic_noise'
+csv_ext = '_hnr.csv'
+error_txt = 'error: length less than 0.064'
+
+def hnr_ratio(filepath):
+    """
+    Using parselmouth library fetching harmonic noise ratio ratio
+    Args:
+        path: (.wav) audio file location
+    Returns:
+        (list) list of hnr ratio for each voice frame, min,max and mean hnr
+    """
+    sound = parselmouth.Sound(filepath)
+    harmonicity = sound.to_harmonicity_ac(time_step=.001)
+    
+    hnr_all_frames = harmonicity.values#[harmonicity.values != -200] nan it (****)
+    hnr_all_frames = np.where(hnr_all_frames==-200, np.NaN, hnr_all_frames)
+    return hnr_all_frames.transpose()
+
+def calc_hnr(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing harmonic noise matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        out_loc: (str) Output directory for csv's
+    """
+    
+    hnr_all_frames = hnr_ratio(audio_file)
+    df_hnr = pd.DataFrame(hnr_all_frames, columns=[r_config.aco_hnr])
+    
+    df_hnr['Frames'] = df_hnr.index
+    df_hnr['dbm_master_url'] = video_uri
+    df_hnr[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_hnr, out_loc, fl_name, hnr_dir, csv_ext)
+    
+def empty_hnr(video_uri, out_loc, fl_name, r_config):
+    """
+    Preparing empty HNR matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_hnr, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    df_hnr = pd.DataFrame(out_val, columns = cols)
+    df_hnr['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_hnr, out_loc, fl_name, hnr_dir, csv_ext)
+
+def run_hnr(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for fetching harmonic noise ratio
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                empty_hnr(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_hnr(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/intensity.py

@@ -0,0 +1,92 @@
+"""
+file_name: intensity
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import numpy as np
+import glob
+import parselmouth
+import librosa
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+intensity_dir = 'acoustic/intensity'
+csv_ext = '_intensity.csv'
+error_txt = 'error: length less than 0.064'
+
+def intensity_score(path):
+    """
+    Using parselmouth library fetching Intensity
+    Args:
+        path: (.wav) audio file location
+    Returns:
+        (list) list of Intensity for each voice frame
+    """
+    sound_pat = parselmouth.Sound(path)
+    intensity = sound_pat.to_intensity(time_step=.001)
+    return intensity.values[0]
+
+def calc_intensity(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing Intensity matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        out_loc: (str) Output directory for csv's
+    """
+    
+    intensity_frames = intensity_score(audio_file)
+    df_intensity = pd.DataFrame(intensity_frames, columns=[r_config.aco_int])
+    
+    df_intensity['Frames'] = df_intensity.index
+    df_intensity['dbm_master_url'] = video_uri
+    df_intensity[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_intensity, out_loc, fl_name, intensity_dir, csv_ext)
+    
+def empty_intensity(video_uri, out_loc, fl_name, r_config):
+    """
+    Preparing empty Intensity matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_int, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    df_int = pd.DataFrame(out_val, columns = cols)
+    df_int['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_int, out_loc, fl_name, intensity_dir, csv_ext)
+
+def run_intensity(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for fetching Intensity
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                empty_intensity(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_intensity(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/jitter.py

@@ -0,0 +1,159 @@
+"""
+file_name: jitter_processing
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import numpy as np
+import os
+import glob
+import parselmouth
+import librosa
+import numpy as np
+import more_itertools as mit
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+jitter_dir = 'acoustic/jitter'
+ff_dir =  'acoustic/pitch'
+csv_ext = '_jitter.csv'
+
+def audio_jitter(sound):
+    """
+    Using parselmouth library fetching jitter
+    Args:
+        sound: parselmouth object
+    Returns:
+        (list) list of jitters for each voice frame
+    """
+    pointProcess = parselmouth.praat.call(sound, "To PointProcess (periodic, cc)...", 80, 500)
+    jitter = parselmouth.praat.call(pointProcess, "Get jitter (local)", 0, 0, 0.0001, 0.02, 1.3)
+    return jitter
+
+def empty_jitter(video_uri, out_loc, fl_name, r_config, error_txt):
+    """
+    Preparing empty jitter matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_jitter, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    df_jitter = pd.DataFrame(out_val, columns = cols)
+    df_jitter['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_jitter, out_loc, fl_name, jitter_dir, csv_ext)
+    
+def segment_pitch(dir_path, r_config):
+    """
+    segmenting pitch freq for each voice segment
+    """
+    com_speech_sort, voiced_yes, voiced_no  = ([], ) * 3
+    for file in os.listdir(dir_path):
+        try:
+            
+            if file.endswith('_pitch.csv'):
+                
+                ff_df = pd.read_csv((dir_path+'/'+file))
+                voice_label = ff_df[r_config.aco_voiceLabel]
+                
+                indices_yes = [i for i, x in enumerate(voice_label) if x == "yes"]
+                voiced_yes = [list(group) for group in mit.consecutive_groups(indices_yes)]
+                
+                indices_no = [i for i, x in enumerate(voice_label) if x == "no"]
+                voiced_no = [list(group) for group in mit.consecutive_groups(indices_no)]
+                
+                com_speech = voiced_yes + voiced_no
+                com_speech_sort = sorted(com_speech, key=lambda x: x[0])
+        except:
+            pass
+        
+    return com_speech_sort, voiced_yes, voiced_no
+
+def segment_jitter(com_speech_sort, voiced_yes, voiced_no, jitter_frames, audio_file):
+    """
+    calculating jitter for each voice segment
+    """
+    snd = parselmouth.Sound(audio_file)
+    pitch = snd.to_pitch(time_step=.001)
+    
+    for idx, vs in enumerate(com_speech_sort):
+        try:
+            
+            jitter = np.NaN
+            if vs in voiced_yes and len(vs)>1:
+                
+                start_time = pitch.get_time_from_frame_number(vs[0])
+                end_time = pitch.get_time_from_frame_number(vs[-1])
+
+                snd_start = int(snd.get_frame_number_from_time(start_time))
+                snd_end = int(snd.get_frame_number_from_time(end_time))
+
+                samples = parselmouth.Sound(snd.as_array()[0][snd_start:snd_end])
+                jitter = audio_jitter(samples)
+        except:
+            pass
+
+        jitter_frames[idx] = jitter
+    return jitter_frames
+    
+def calc_jitter(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing jitter matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        out_loc: (str) Output directory for csv
+        r_config: config.config_raw_feature.pyConfigFeatureNmReader object
+    """
+    dir_path = os.path.join(out_loc, ff_dir)
+    if os.path.isdir(dir_path):
+        voice_seg = segment_pitch(dir_path, r_config)
+        
+        jitter_frames = [np.NaN] * len(voice_seg[0])
+        jitter_segment_frames = segment_jitter(voice_seg[0], voice_seg[1], voice_seg[2], jitter_frames, audio_file)
+        
+        df_jitter = pd.DataFrame(jitter_segment_frames, columns=[r_config.aco_jitter])
+        df_jitter[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+        
+        df_jitter['Frames'] = df_jitter.index
+        df_jitter['dbm_master_url'] = video_uri
+        
+        logger.info('Processing Output file {} '.format(out_loc))
+        ut.save_output(df_jitter, out_loc, fl_name, jitter_dir, csv_ext)
+        
+    else:
+        error_txt = 'error: fundamental freq not available'
+        empty_jitter(video_uri, out_loc, fl_name, r_config, error_txt)
+    
+def run_jitter(video_uri, out_dir, r_config):
+    """
+    Processing all patient's videos for fetching jitter
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output 
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                error_txt = 'error: length less than 0.064'
+                empty_jitter(video_uri, out_loc, fl_name, r_config, error_txt)
+                return
+
+            calc_jitter(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/mfcc.py

@@ -0,0 +1,105 @@
+"""
+file_name: mfcc
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import os
+import glob
+import parselmouth
+import librosa
+import numpy as np
+import librosa
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+mfcc_dir = 'acoustic/mfcc'
+csv_ext = '_mfcc.csv'
+error_txt = 'error: length less than 0.064'
+
+def empty_mfcc(video_uri, out_loc, fl_name, r_config):
+    
+    """
+    Preparing empty empty_mfcc matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_mfcc1, r_config.aco_mfcc2, r_config.aco_mfcc3, r_config.aco_mfcc4, r_config.aco_mfcc5,
+            r_config.aco_mfcc6, r_config.aco_mfcc7, r_config.aco_mfcc8, r_config.aco_mfcc9, r_config.aco_mfcc10, 
+            r_config.aco_mfcc11, r_config.aco_mfcc12, r_config.err_reason]
+    out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 
+                error_txt]]
+    df_mfcc = pd.DataFrame(out_val, columns = cols)
+    df_mfcc['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_mfcc, out_loc, fl_name, mfcc_dir, csv_ext)
+
+def audio_mfcc(path):
+    """
+    Using parselmouth library fetching mfccs
+    Args:
+        path: (.wav) audio file location
+    Returns:
+        (list) list of mfccs for each voice frame
+    """
+    sound = parselmouth.Sound(path)
+    mfcc_object = sound.to_mfcc(time_step=.001,number_of_coefficients=12)
+    mfccs = mfcc_object.to_array()
+    mfccs = np.delete(mfccs, (0), axis=0)
+    return mfccs
+    
+def calc_mfcc(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing mfcc matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        out_loc: output location to save csv
+        fl_name: (str) name of audio file
+        r_config: config.config_raw_feature.pyConfigFeatureNmReader object
+    """
+    dict_ = {}
+    mfccs = audio_mfcc(audio_file)
+    
+    for i in range(1,13):
+        conf_str = r_config.base_raw['raw_feature']
+        dict_[conf_str['aco_mfcc' + str(i)]] = mfccs[i-1, :]
+        
+    df = pd.DataFrame(dict_)
+    df['Frames'] = df.index
+    
+    df[r_config.err_reason] = 'Pass'# may replace based on threshold in future release
+    df['dbm_master_url'] = video_uri
+    
+    ut.save_output(df, out_loc, fl_name, mfcc_dir, csv_ext)
+    
+def run_mfcc(video_uri, out_dir, r_config):
+    """
+    Processing all patients to fetch mfccs
+    
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output 
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                empty_mfcc(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_mfcc(video_uri, audio_file, out_loc, fl_name, r_config) 
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/pause_segment.py

@@ -0,0 +1,171 @@
+"""
+file_name: pause_segment
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import glob
+from pydub import AudioSegment
+import librosa
+import pandas as pd
+import numpy as np
+import webrtcvad
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import vad_utilities as vu, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+pause_seg_dir = 'acoustic/pause_segment'
+csv_ext = '_pausechar.csv'
+
+def get_timing_cues(seg_starts_sec, seg_ends_sec, r_config):
+    """
+    Get timing cues from segmented speech
+    Args:
+        seg_starts_sec: Audio segment start time in seconds
+        seg_ends_sec: Audio segment end time in seconds
+    Returns:
+        Dictionary with pause features
+    """
+    total_time = seg_ends_sec[-1] - seg_starts_sec[0]
+    speaking_time = np.sum(np.asarray(seg_ends_sec) - np.asarray(seg_starts_sec))
+    num_pauses = len(seg_starts_sec) - 1
+    pause_len = np.zeros(num_pauses)
+    
+    for p in range(num_pauses):
+        pause_len[p] = seg_starts_sec[p+1] - seg_ends_sec[p]
+        
+    if len(pause_len)>0:
+        pause_len_mean = np.mean(pause_len)
+        pause_len_std = np.std(pause_len)
+        pause_time = np.sum(pause_len)
+        
+    else:
+        pause_len_mean = 0
+        pause_len_std = 0
+        pause_time = 0
+        
+    pause_frac = pause_time / total_time
+    timing_dict = {r_config.aco_totaltime: total_time, r_config.aco_speakingtime: speaking_time,
+                   r_config.aco_numpauses: num_pauses, r_config.aco_pausetime: pause_time, r_config.aco_pausefrac: pause_frac}
+    return timing_dict
+
+def process_silence(audio_file, r_config):
+    """
+    Returns dataframe for pause between words using voice activity detection
+    Args:
+        audio_file: Audio file location
+    Returns:
+        Dataframe value
+    """
+    feat_dict_list = []
+    y, sr = vu.read_wave(audio_file)
+    
+    # 3 is most aggressive (splits most), 0 least (better for low snr)
+    aggressiveness = 3
+    frame_dur_ms = 20
+    
+    #pause segment(long & short pad) 
+    long_pad_around_voice_ms = 200
+    short_pad_around_voice_ms = 100
+    
+    if len(y)>0:
+        vad = webrtcvad.Vad(aggressiveness)
+        
+        frames = vu.frame_generator(frame_dur_ms, y, sr)
+        frames = list(frames)
+        
+        #longer pad time screens out little blips, but misses short silences
+        long_seg_starts, long_seg_ends = vu.vad_get_segment_times(sr, frame_dur_ms, long_pad_around_voice_ms, vad, frames)
+        
+        #Logic to handle blank audio file
+        if len(long_seg_starts) == 0 or len(long_seg_ends) == 0:
+            return ''
+        
+        t_start = long_seg_starts[0]
+        t_end = long_seg_ends[-1]
+        # shorter pad time captures short silences (but misfires on little blips)
+        short_seg_starts, short_seg_ends = vu.vad_get_segment_times(sr, frame_dur_ms, short_pad_around_voice_ms, vad, frames)
+        
+        seg_starts = []
+        seg_ends = []
+        for k in range(len(short_seg_starts)):  # logic to clean up some typical misfires
+            if (short_seg_starts[k] >=t_start) and (short_seg_starts[k] <= t_end):
+                
+                seg_starts.append(short_seg_starts[k])
+                seg_ends.append(short_seg_ends[k])
+        if len(seg_starts) == 0 or len(seg_ends) == 0:
+            return ''
+        
+        timing_dict = get_timing_cues(seg_starts, seg_ends, r_config)
+        feat_dict_list.append(timing_dict)
+        
+    df = pd.DataFrame(feat_dict_list)
+    df[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    return df
+
+def empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt):
+    """
+    Preparing empty Pause Segment matrix if something fails
+    """
+    cols = [r_config.aco_totaltime, r_config.aco_speakingtime, r_config.aco_numpauses, r_config.aco_pausetime,
+            r_config.aco_pausefrac, r_config.err_reason]
+    out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
+    df_pause = pd.DataFrame(out_val, columns = cols)
+    df_pause['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_pause, out_loc, fl_name, pause_seg_dir, csv_ext)
+
+def run_pause_segment(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for getting Pause Segment
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                error_txt = 'error: length less than 0.064'
+                empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt)
+                return
+
+            logger.info('Converting stereo sound to mono-lD')
+            sound_mono = AudioSegment.from_wav(audio_file)
+            sound_mono = sound_mono.set_channels(1)
+            sound_mono = sound_mono.set_frame_rate(48000)
+
+            mono_wav = os.path.join(input_loc, fl_name + '_mono.wav')
+            sound_mono.export(mono_wav, format="wav")
+
+            df_pause_seg = process_silence(mono_wav, r_config)
+            os.remove(mono_wav)#removing mono wav file
+
+            if isinstance(df_pause_seg, pd.DataFrame) and len(df_pause_seg)>0:
+                logger.info('Processing Output file {} '.format(out_loc))
+
+                df_pause_seg['dbm_master_url'] = video_uri
+                ut.save_output(df_pause_seg, out_loc, fl_name, pause_seg_dir, csv_ext)
+
+            else:
+                error_txt = 'error: webrtcvad returns no segment'
+                empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt)
+                
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/pitch_freq.py

@@ -0,0 +1,113 @@
+"""
+file_name: pitch_freq
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import os
+import glob
+import parselmouth
+import librosa
+import numpy as np
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+ff_dir = 'acoustic/pitch'
+csv_ext = '_pitch.csv'
+error_txt = 'error: length less than 0.064'
+
+def audio_pitch(path):
+    """
+    Using parselmouth library fetching pitch/fundamental frequency
+    Args:
+        path: (.wav) audio file location
+    Returns:
+        (list) list of pitch/fundamental frequency for each voice frame
+    """
+    sound_pat = parselmouth.Sound(path)
+    pitch = sound_pat.to_pitch(time_step=.001)
+    pitch_values = pitch.selected_array['frequency']
+    
+    return list(pitch_values)
+
+def label_speech(row,fd_freq):
+    """
+    identify whether frame is voiced or not
+    Args:
+        row: (item) pitch frequency value
+    Returns:
+        (str) yes or no indicator for voice
+    """
+    if row[fd_freq] > 0 :
+        return 'yes'
+    else:
+        return 'no'
+
+def calc_pitch(video_uri, audio_file, out_loc, fl_name, r_config):
+    
+    """
+    Preparing pitch frequency matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        row: (dataframe) subject details from master csv
+        new_out_base_dir: (str) Output directory for csv
+    """
+
+    ff_frames = audio_pitch(audio_file)
+    df_ffreq = pd.DataFrame(ff_frames, columns=[r_config.aco_ff])
+    
+    df_ffreq['Frames'] = df_ffreq.index
+    df_ffreq[r_config.aco_voiceLabel] = df_ffreq.apply(lambda row: label_speech(row, r_config.aco_ff),axis=1)
+    
+    df_ffreq[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    df_ffreq['dbm_master_url'] = video_uri
+    
+    logger.info('Processing Output file {} '.format(out_loc))
+    ut.save_output(df_ffreq, out_loc, fl_name, ff_dir, csv_ext)
+    
+def empty_pitch(video_uri, out_loc, fl_name, r_config):
+    """
+    Preparing empty pitch frequency matrix if something fails
+    """
+    
+    df_ffreq = pd.DataFrame([[np.nan, np.nan, 'no', error_txt]], 
+                            columns=['Frames', r_config.aco_ff, r_config.aco_voiceLabel, r_config.err_reason])
+    df_ffreq['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_ffreq, out_loc, fl_name, ff_dir, csv_ext)  
+
+def run_pitch(video_uri, out_dir, r_config):
+    
+    """
+    Processing audio for fetching pitch
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                empty_pitch(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_pitch(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/shimmer.py

@@ -0,0 +1,160 @@
+"""
+file_name: shimmer_processing
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import numpy as np
+import os
+import glob
+import parselmouth
+import librosa
+import numpy as np
+import more_itertools as mit
+from os.path import join
+
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+shimmer_dir = 'acoustic/shimmer'
+ff_dir = 'acoustic/pitch'
+csv_ext = '_shimmer.csv'
+
+def audio_shimmer(sound):
+    """
+    Using parselmouth library fetching shimmer
+    Args:
+        sound: parselmouth object
+    Returns:
+        (list) list of shimmers for each voice frame
+    """
+    pointProcess = parselmouth.praat.call(sound, "To PointProcess (periodic, cc)...", 80, 500)
+    shimmer = parselmouth.praat.call([sound, pointProcess], "Get shimmer (local)", 0, 0, 0.0001, 0.02, 1.3, 1.6)
+    return shimmer
+
+def empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt):
+    """
+    Preparing empty shimmer matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_shimmer, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    df_shimmer = pd.DataFrame(out_val, columns = cols)
+    df_shimmer['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_shimmer, out_loc, fl_name, shimmer_dir, csv_ext)
+    
+def segment_pitch(dir_path, r_config):
+    """
+    segmenting pitch freq for each voice segment
+    """
+    com_speech_sort, voiced_yes, voiced_no  = ([], ) * 3
+    for file in os.listdir(dir_path):
+        try:
+            
+            if file.endswith('_pitch.csv'):
+                
+                ff_df = pd.read_csv((dir_path+'/'+file))
+                voice_label = ff_df[r_config.aco_voiceLabel]
+                
+                indices_yes = [i for i, x in enumerate(voice_label) if x == "yes"]
+                voiced_yes = [list(group) for group in mit.consecutive_groups(indices_yes)]
+                
+                indices_no = [i for i, x in enumerate(voice_label) if x == "no"]
+                voiced_no = [list(group) for group in mit.consecutive_groups(indices_no)]
+                
+                com_speech = voiced_yes + voiced_no
+                com_speech_sort = sorted(com_speech, key=lambda x: x[0])
+        except:
+            pass
+        
+    return com_speech_sort, voiced_yes, voiced_no
+
+def segment_shimmer(com_speech_sort, voiced_yes, voiced_no, shimmer_frames, audio_file):
+    """
+    calculating shimmer for each voice segment
+    """
+    snd = parselmouth.Sound(audio_file)
+    pitch = snd.to_pitch(time_step=.001)
+    
+    for idx, vs in enumerate(com_speech_sort):
+        try:
+            
+            shimmer = np.NaN
+            if vs in voiced_yes and len(vs)>1:
+                
+                start_time = pitch.get_time_from_frame_number(vs[0])
+                end_time = pitch.get_time_from_frame_number(vs[-1])
+
+                snd_start = int(snd.get_frame_number_from_time(start_time))
+                snd_end = int(snd.get_frame_number_from_time(end_time))
+
+                samples = parselmouth.Sound(snd.as_array()[0][snd_start:snd_end])
+                shimmer = audio_shimmer(samples)
+        except:
+            pass
+
+        shimmer_frames[idx] = shimmer
+    return shimmer_frames
+    
+def calc_shimmer(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+    Preparing shimmer matrix
+    Args:
+        audio_file: (.wav) parsed audio file
+        out_loc: (str) Output directory for csv
+        r_config: config.config_raw_feature.pyConfigFeatureNmReader object
+    """
+    dir_path = os.path.join(out_loc, ff_dir)
+    if os.path.isdir(dir_path):
+        voice_seg = segment_pitch(dir_path, r_config)
+        
+        shimmer_frames = [np.NaN] * len(voice_seg[0])
+        shimmer_segment_frames = segment_shimmer(voice_seg[0], voice_seg[1], voice_seg[2], shimmer_frames, audio_file)
+        
+        df_shimmer = pd.DataFrame(shimmer_segment_frames, columns=[r_config.aco_shimmer])
+        df_shimmer[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+        
+        df_shimmer['Frames'] = df_shimmer.index
+        df_shimmer['dbm_master_url'] = video_uri
+        
+        logger.info('Processing Output file {} '.format(out_loc))
+        ut.save_output(df_shimmer, out_loc, fl_name, shimmer_dir, csv_ext)
+        
+    else:
+        error_txt = 'error: fundamental freq not available'
+        empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt)
+    
+def run_shimmer(video_uri, out_dir, r_config):
+    """
+    Processing all patients to fetch shimmer
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output 
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                error_txt = 'error: length less than 0.064'
+                empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt)
+                return
+
+            calc_shimmer(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/audio/voice_frame_score.py

@@ -0,0 +1,111 @@
+"""
+file_name: voice_frame_score
+project_name: DBM
+created: 2020-20-07
+"""
+
+import parselmouth
+import pandas as pd
+import numpy as np
+import glob
+import librosa
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+vfs_dir = 'acoustic/voice_frame_score'
+csv_ext = '_voiceprev.csv'
+error_txt = 'error: length less than 0.064'
+
+def audio_pitch_frame(pitch):
+    """
+        Computing total number of speech and participant voiced frames
+        Args:
+            pitch: speech pitch
+        Returns:
+            (float) total voice frames and participant voiced frames
+    """
+    total_frames = pitch.get_number_of_frames()
+    voiced_frames = pitch.count_voiced_frames()
+    return total_frames, voiced_frames
+
+def voice_segment(path):
+    """
+        Using parselmouth library for fundamental frequency
+        Args:
+            path: (.wav) audio file location
+        Returns:
+            (float) total voice frames, participant voiced frames and voiced frames percentage
+    """
+    sound_pat = parselmouth.Sound(path)
+    pitch = sound_pat.to_pitch()
+    total_frames,voiced_frames = audio_pitch_frame(pitch)
+    
+    voiced_percentage = (voiced_frames/total_frames)*100
+    return voiced_percentage, voiced_frames, total_frames
+
+def calc_vfs(video_uri, audio_file, out_loc, fl_name, r_config):
+    """
+        creating dataframe matrix for voice frame score
+        Args:
+            audio_file: Audio file path
+            new_out_base_dir: AWS instance output base directory path
+            f_nm_config: Config file object
+    """
+
+    voice_percentage,voiced_frames, total_frames = voice_segment(audio_file)
+    df_vfs = pd.DataFrame([voiced_frames], columns=[r_config.aco_voiceFrame])
+    
+    df_vfs[r_config.aco_totVoiceFrame] = [total_frames]
+    df_vfs[r_config.aco_voicePct] = [voice_percentage]
+    df_vfs[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    
+    df_vfs['Frames'] = df_vfs.index
+    df_vfs['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext) 
+    
+def empty_vfs(video_uri, out_loc, fl_name, r_config):
+    """
+    Preparing empty VFS matrix if something fails
+    """
+    cols = ['Frames', r_config.aco_voiceFrame, r_config.aco_totVoiceFrame, r_config.aco_voicePct, r_config.err_reason]
+    out_val = [[np.nan, np.nan, np.nan, np.nan, error_txt]]
+    df_vfs = pd.DataFrame(out_val, columns = cols)
+    df_vfs['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)  
+
+def run_vfs(video_uri, out_dir, r_config):
+    """
+    Processing all participants for fetching voice frame score
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.064:
+                logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
+
+                empty_vfs(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_vfs(video_uri, audio_file, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process audio file')

opendbm/dbm_lib/dbm_features/raw_features/movement/__init__.py

@@ -0,0 +1,17 @@
+"""
+file_name: init
+project_name: DBM
+created: 2020-20-07
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+DBMLIB_PATH = os.path.dirname(__file__)
+DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH,
+                                                  '../../../../resources/libraries/voice_tremor.praat'))
+DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json'))
+

opendbm/dbm_lib/dbm_features/raw_features/movement/eye_blink.py

@@ -0,0 +1,160 @@
+"""
+file_name: eye_blink
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import glob
+from scipy.spatial import distance as dist
+from scipy.signal import find_peaks
+from imutils.video import FileVideoStream
+from imutils.video import VideoStream
+from imutils import face_utils
+from moviepy.editor import VideoFileClip
+import numpy as np
+import pandas as pd
+import imutils
+import time
+import dlib
+import cv2
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+movement_expr_dir = 'movement/eye_blink'
+csv_ext = '_eyeblinks.csv'
+
+def eye_aspect_ratio(eye):
+    """
+        Computing eye aspect ratio for an individual frame
+        Args:
+            eye: Eye landmarks
+        Return:
+            Eye aspect ratio for a frame
+    """
+    # euclidean distance for vertical eye landmarks
+    dist_cor1 = dist.euclidean(eye[1], eye[5])
+    dist_cor2 = dist.euclidean(eye[2], eye[4])
+    
+    # euclidean distance for horizontal eye landmark
+    dist_cor3 = dist.euclidean(eye[0], eye[3])
+    
+    ear = (dist_cor1 + dist_cor2) / (2.0 * dist_cor3)
+    return ear
+
+def blink_detection(video_path,facial_landmarks,raw_config):
+    """
+        Blink detection for each frame
+        Args:
+            video_path: MP4 file location
+            facial_landmarks: Facial landmark pre-trained model path
+            raw_config: Raw configuration file object
+        Return:
+            Dataframe with blink informatiom like blink frame, duration etc.
+    """
+    TOT_FRAME = 1
+    blink_frame = []
+    ear_frame = []
+    
+    clip = VideoFileClip(video_path, has_mask=True)
+    vid_length = clip.duration
+    
+    identifier = dlib.get_frontal_face_detector() #dlib's face detector (HOG-based)
+    forecaster = dlib.shape_predictor(facial_landmarks) # the facial landmark predictor
+    
+    #left and right eye landmarks
+    (left_beg, left_end) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
+    (right_beg, right_end) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]
+    
+    f_stream = True
+    vid_stream = FileVideoStream(video_path).start()
+
+    while True:
+        try:
+            #check if stream/frame available in video
+            if f_stream and not vid_stream.more():
+                break
+                
+            #reading & converting frame into grayscale
+            vid_frame = vid_stream.read()
+            vid_frame = imutils.resize(vid_frame, width=450)
+            gray = cv2.cvtColor(vid_frame, cv2.COLOR_BGR2GRAY)
+            
+            #detecting face
+            rects = identifier(gray, 0)
+            for rect in rects:
+                
+                lmk = forecaster(gray, rect)
+                lmk = face_utils.shape_to_np(lmk)
+                
+                l_eye = lmk[left_beg:left_end] #Extracting left eye ratio
+                r_eye = lmk[right_beg:right_end] #Extracting right eye ratio
+                l_ear = eye_aspect_ratio(l_eye) # eye aspect ratio for left eye
+                r_ear = eye_aspect_ratio(r_eye) # eye aspect ratio for right eye
+                
+                ear = (l_ear + r_ear) / 2.0 # average the eye aspect ratio
+                blink_frame.append(TOT_FRAME)
+                ear_frame.append(ear)
+                
+            TOT_FRAME += 1
+        except Exception as e:
+            #logger.error("blink detection processing failed for: {}".format(video_path))
+            continue
+            
+    blink_df = pd.DataFrame(ear_frame, columns =[raw_config.mov_blink_ear])
+    blink_df[raw_config.vid_dur] = vid_length
+    blink_df[raw_config.fps] = int(TOT_FRAME/vid_length)
+    blink_df[raw_config.mov_blinkframes] = blink_frame
+    
+    peaks, _ = find_peaks(blink_df[raw_config.mov_blink_ear]*-1, prominence=0.1)#prominence = 0.1 based on tuning
+    final_blink_df = blink_df.iloc[peaks,:].reset_index(drop=True)
+    
+    u_blink_df = blink_dur(final_blink_df,raw_config)
+    u_blink_df['dbm_master_url'] = video_path
+    return u_blink_df
+
+def blink_dur(blink_df,raw_config):
+    """
+        Computing blink duration between each blink
+        Args:
+            blink_df : Dataframe with blink informatiom like blink frame
+            raw_config: Raw configuration file object
+        Returns:
+            Updated dataframe with blink duration
+    """
+    dur_list = []
+    if len(blink_df)>0:
+        blink_df[raw_config.mov_blinkdur] = blink_df[raw_config.mov_blinkframes].diff().fillna(
+            blink_df[raw_config.mov_blinkframes])
+    else:
+        blink_df[raw_config.mov_blinkdur] = np.nan
+    blink_df[raw_config.mov_blinkdur] = blink_df[raw_config.mov_blinkdur]/blink_df[raw_config.fps]
+    return blink_df
+
+def run_eye_blink(video_uri, out_dir, r_config, facial_landmarks):
+    """
+    Processing all patient's for getting eye blink artifacts
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; input_dir : input directory for video's
+        out_dir: (str) Output directory for processed output; r_config: raw variable config object;
+        facial_landmarks: landmark model path
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        vid_file_path = os.path.exists(video_uri)
+        if vid_file_path==True:
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            df_blink = blink_detection(video_uri, facial_landmarks, r_config)
+            ut.save_output(df_blink, out_loc, fl_name, movement_expr_dir, csv_ext)
+            
+    except Exception as e:
+        logger.error('Failed to process video file')
+        

opendbm/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py

@@ -0,0 +1,148 @@
+"""
+file_name: eye_gaze
+project_name: DBM
+created: 2020-30-11
+"""
+
+import os
+import glob
+import pandas as pd
+import numpy as np
+from scipy.spatial import distance
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+eye_pose_dir = 'movement/gaze'
+eye_pose_ext = '_eyegaze.csv'
+
+def eye_motion_df(l_disp, r_disp, error_list, r_config):
+    """
+    Generating eye movement dataframe
+
+    Args:
+        l_disp: displacement list(left eye); l_disp: displacement list(right eye)
+        r_config: raw variable config file object
+
+    Reutrns:
+        Final eye displacement dataframe
+    """
+    df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp])
+    df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp])
+    
+    df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False)
+    df_eye_motion[r_config.err_reason] = error_list
+    return df_eye_motion
+
+def filter_motion(df_of, df_disp, col_l, col_r, r_config):
+    """
+    Filtering final eye movement dataframe
+    
+    Args:
+        df_of: Openface raw out dataframe; col_r: right eye column
+        col_l: left eye column; r_config: raw variable config file object
+    """
+    
+    df_of = df_of[col_l + col_r + [' confidence']]
+    df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan
+    
+    df_filter =  df_of[col_l + col_r]
+    df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z,
+                      r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z]
+    
+    df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False)
+    return df_motion
+
+def eye_disp(of_results, col, r_config):
+    """
+    Computing head velocity frame by frame
+
+    Args:
+        of_results: Openface raw out dataframe
+        r_config: Face config file object
+
+    Reutrns:
+        Final head velocity frame by frame output
+    """
+    distance_list = []
+    error_list = []
+    
+    of_results = of_results[col+ [' confidence']]
+    for index, row in of_results.iterrows():
+        dst = np.nan
+        
+        if index == 0 or float(row[' confidence']) < 0.8:  #Threshold < 0.8
+            distance_list.append(dst)
+            
+            if float(row[' confidence']) < 0.8:
+                error_list.append('confidence less than 80%')
+                
+            else:
+                error_list.append('Pass')
+            continue
+            
+        if index > 0:
+            
+            point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1])
+            point_y = (row[col[0]],row[col[1]],row[col[2]])
+            try:
+                dst = distance.euclidean(point_x, point_y)
+            except:
+                pass
+            
+            distance_list.append(abs(dst))
+            error_list.append('Pass')
+            
+    return distance_list, error_list
+        
+def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config):
+    """
+    Computing eye motion variables
+    Args:
+        df_of: Openface dataframe
+        out_loc: Output path for saving output csv's
+        fl_name: file name for output csv
+        r_config: raw variable config file object
+        
+    """
+    
+    col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z'] 
+    col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z']
+    
+    gazel_disp, err_l = eye_disp(df_of, col_l, r_config)
+    gazer_disp, err_r = eye_disp(df_of, col_r, r_config)
+    
+    df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config)
+    df_disp['dbm_master_url'] = video_uri
+    
+    df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config)
+    ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext)
+    
+def run_eye_gaze(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for getting eye movement artifacts 
+    --------------------------------
+    --------------------------------
+    Args:
+        video_uri: video path; input_dir : input directory for video's
+        out_dir: (str) Output directory for processed output; r_config: raw variable config object
+    """
+    try:
+        
+        #filtering path to generate input & output path
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
+
+        if len(of_csv_path)>0:
+
+            of_csv = of_csv_path[0]
+            df_of = pd.read_csv(of_csv, error_bad_lines=False)
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py

@@ -0,0 +1,164 @@
+import sys, os, glob, cv2, re
+import pickle, json
+import pandas as pd
+import numpy as np
+import numpy.ma as ma
+import logging
+from os.path import join
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+from opendbm.dbm_lib.dbm_features.raw_features.util.math_util import *
+
+from opendbm.dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+ft_dir = 'movement/facial_tremor'
+csv_ext = '_fac_tremor.csv'
+model_ext = '_fac_model.csv'
+fac_features_ext = '_fac_features.csv'
+
+def compute_features(out_dir, df_of, r_config):
+    """ Computes features
+
+    Returns: features in vector format
+    """
+    config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read())
+
+    pattern_x = re.compile("l\d+_x")
+    pattern_y = re.compile("l\d+_y")
+
+    # assumption: distance of face to camera remains at roughly static
+
+    # logic break
+    landmark_columns = []
+    for col in df_of.columns:
+        if pattern_x.match(col) or pattern_y.match(col):
+            landmark_columns.append(col)
+
+    df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)]
+    df_of.reset_index(inplace=True)
+
+    num_frames = len(df_of)
+    logger.info("Number of frames to be processed: {}".format(str(num_frames)))
+    landmarks = config['landmarks']
+
+    try:
+        if num_frames == 0:
+            error_reason = "No frames with visible face."
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config, error_reason)
+
+#             if num_frames < 60:
+#                 error_reason = 'Number of frames with visible face < 60. Video too short'
+#                 logger.error(error_reason)
+#                 return empty_frame(landmarks, f_cfg, error_reason)
+
+        first_row = df_of.iloc[0]
+
+        facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']])
+        faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']])
+
+        if facew == 0 or faceh == 0:
+            error_reason = 'face width or height = 0. Check landmark values'
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config)
+
+        fac_disp = calc_displacement_vec(df_of, landmarks, num_frames)
+
+        # if verbose:
+            # logger.info("Displacement output: {}".format(str(fac_disp)))
+
+        fac_disp_median = np.median(fac_disp, axis = 1)
+        fac_disp_mean = np.mean(fac_disp, axis = 1)
+
+        if len(fac_disp.shape)!=2:
+            error_reason = 'fac_disp is not 2D. smth went wrong with disp calc'
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config, error_reason)
+
+        if len(fac_disp[0])<=1:
+            error_reason = 'Video too short. smth went wrong with disp calc'
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config, error_reason)
+
+        fac_corr_mat = np.corrcoef(fac_disp, rowvar = True)
+        # extract relevant row from cov matrix
+        ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk]
+        fac_corr = fac_corr_mat[ref_lmk_index][0]
+
+        fac_area = config['ref_area'] / (facew * faceh)
+
+        # if verbose:
+        #     logger.info("Face area: {}".format(fac_area))
+        #     logger.info("Face Displacement Median: {}".format(str(fac_disp_median)))
+        #     logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean)))
+
+        fac_features1 =  np.multiply(fac_area * fac_disp_median, (1. - fac_corr))
+        fac_features2 =  np.multiply(fac_area * fac_disp_mean, (1. - fac_corr))
+
+#         base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr))
+
+        fac_features_dict = {}
+        for i, landmark in enumerate(landmarks):
+            fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]]
+            raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
+            fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]]
+
+            fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]]
+            fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]]
+
+        fac_features_dict[r_config.err_reason] = ['']
+        data = pd.DataFrame.from_dict(fac_features_dict)
+        logger.info('Concluded computing tremor features')
+
+        return data
+
+    except Exception as e:
+        logger.error('Error computing tremor features: {}'.format(str(e)))
+        return empty_frame(landmarks, r_config, str(e))
+
+def empty_frame(landmarks, r_config, error_reason):
+    fac_features_dict = {}
+    for i, landmark in enumerate(landmarks):
+        raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
+        fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan]
+
+        fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan]
+        fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan]
+        fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan]
+
+    fac_features_dict[r_config.err_reason] = [error_reason]
+    empty_frame = pd.DataFrame.from_dict(fac_features_dict)
+    return empty_frame
+
+def fac_tremor_process(video_uri, out_dir, r_config, model_output=False):
+    """
+    processing input videos
+    
+    
+    """
+#     try:
+        
+    input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+    of_csv_path = glob.glob(join(out_loc, fl_name + '_openface_lmk/*.csv'))
+
+    if len(of_csv_path)>0:
+        of_csv = of_csv_path[0]
+        df_of = pd.read_csv(of_csv, error_bad_lines=False)
+
+        logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+
+        feats = compute_features(of_csv_path , df_of, r_config)
+        
+#         if model_output:
+#             result = score(feats, r_config)
+#             feats = pd.concat([feats, result], axis=1)
+            
+        ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext)
+
+
+
+#     except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/movement/head_motion.py

@@ -0,0 +1,196 @@
+"""
+file_name: head_mov
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import glob
+import pandas as pd
+import numpy as np
+from scipy.spatial import distance
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+h_mov_dir = 'movement/head_movement'
+h_pose_dir = 'movement/head_pose'
+h_mov_ext = '_headmov.csv'
+h_pose_ext = '_headpose.csv'
+
+def head_pose_dist(of_results):
+    """
+    Computing head pose distance frame by frame
+
+    Args:
+        of_results: Openface raw out dataframe
+        f_nm_config: Face config file object
+
+    Reutrns:
+        Final head pose distance frame by frame output
+    """
+    distance_list = []
+    error_list = []
+    for index, row in of_results.iterrows():
+        dst = np.nan
+        
+        if index == 0 or float(row[' confidence']) < 0.2: #Threshold < 0.2
+            distance_list.append(dst)
+            
+            if float(row[' confidence']) < 0.2:
+                error_list.append('confidence less than 20%')
+                
+            else:
+                error_list.append('Pass')
+            continue
+            
+        if index > 0:
+            
+            point_x = (of_results[' pose_Rx'][index-1], of_results[' pose_Ry'][index-1], of_results[' pose_Rz'][index-1])
+            point_y = (row[' pose_Rx'],row[' pose_Ry'],row[' pose_Rz'])
+            try:
+                dst = distance.euclidean(point_x, point_y)
+            except:
+                pass
+            distance_list.append(abs(dst))
+            error_list.append('Pass')
+    return distance_list, error_list
+
+def head_pose(of_results,r_config):
+    """
+    Generating head pose estimation dataframe
+
+    Args:
+        distance_val: distance list
+        f_nm_config: raw variable config file object
+
+    Reutrns:
+        Final head pose estimation dataframe
+    """
+    pose_dist_list, error_list = head_pose_dist(of_results)
+    of_results.loc[(of_results[' confidence'].astype(float) < 0.2), [' pose_Rx',' pose_Ry',' pose_Rz']] = np.nan
+    pose_of =  of_results[[' pose_Rx',' pose_Ry',' pose_Rz']]
+    pose_of.columns = [r_config.mov_Hpose_Pitch, r_config.mov_Hpose_Yaw, r_config.mov_Hpose_Roll]
+    pose_of[r_config.mov_Hpose_Dist] = pose_dist_list
+    pose_of[r_config.err_reason] = error_list
+    
+    return pose_of
+
+def head_motion_df(distance_val, error_list, r_config):
+    """
+    Generating head movement dataframe
+
+    Args:
+        distance_val: distance list
+        r_config: raw variable config file object
+
+    Reutrns:
+        Final head velocity dataframe
+    """
+    head_motion = r_config.head_vel
+    df_head_motion = pd.DataFrame(distance_val, columns=[head_motion])
+    df_head_motion['Frames'] = df_head_motion.index
+    
+    new_df_intensity = df_head_motion[['Frames', head_motion]]
+    new_df_intensity[r_config.err_reason] = error_list
+    
+    return new_df_intensity
+
+def head_vel(of_results, r_config):
+    """
+    Computing head velocity frame by frame
+
+    Args:
+        of_results: Openface raw out dataframe
+        r_config: Face config file object
+
+    Reutrns:
+        Final head velocity frame by frame output
+    """
+    distance_list = []
+    error_list = []
+    for index, row in of_results.iterrows():
+        dst = np.nan
+        
+        if index == 0 or float(row[' confidence']) < 0.2:  #Threshold < 0.2
+            distance_list.append(dst)
+            
+            if float(row[' confidence']) < 0.2:
+                error_list.append('confidence less than 20%')
+                
+            else:
+                error_list.append('Pass')
+            continue
+            
+        if index > 0:
+            
+            point_x = (of_results[' pose_Tx'][index-1], of_results[' pose_Ty'][index-1], of_results[' pose_Tz'][index-1])
+            point_y = (row[' pose_Tx'],row[' pose_Ty'],row[' pose_Tz'])
+            try:
+                dst = distance.euclidean(point_x, point_y)
+            except:
+                pass
+            
+            if abs(dst)>200:
+                dst = np.nan
+                error_list.append('Out of range')
+                
+            else:
+                error_list.append('Pass')
+            distance_list.append(dst)
+    df_velocity = head_motion_df(distance_list, error_list, r_config)
+    
+    return df_velocity
+        
+def calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config):
+    """
+    Computing head motion and head pose variables
+    Args:
+        df_of: Openface dataframe
+        out_loc: Output path for saving output csv's
+        fl_name: file name for output csv
+        r_config: raw variable config file object
+        
+    """
+    
+    col = [' confidence',' pose_Rx',' pose_Ry',' pose_Rz',' pose_Tx', ' pose_Ty', ' pose_Tz']
+    df_of = df_of[col]
+    
+    df_hmotion = head_vel(df_of, r_config)
+    df_hmotion['dbm_master_url'] = video_uri
+    
+    df_pose = head_pose(df_of, r_config)
+    df_pose['dbm_master_url'] = video_uri
+    
+    ut.save_output(df_hmotion, out_loc, fl_name, h_mov_dir, h_mov_ext)
+    ut.save_output(df_pose, out_loc, fl_name, h_pose_dir, h_pose_ext)
+    
+def run_head_movement(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for getting movement artifacts for cdx_analysis workflow
+    --------------------------------
+    --------------------------------
+    Args:
+        video_uri: video path; input_dir : input directory for video's
+        out_dir: (str) Output directory for processed output; r_config: raw variable config object
+    """
+    try:
+
+        #filtering path to generate input & output path
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
+
+
+        if len(of_csv_path)>0:
+
+            of_csv = of_csv_path[0]
+            df_of = pd.read_csv(of_csv, error_bad_lines=False)
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py

@@ -0,0 +1,94 @@
+import pandas as pd
+import os
+import glob
+from os.path import join
+import parselmouth
+from parselmouth.praat import call, run_file
+import numpy as np
+import librosa
+import json
+import re
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+from opendbm.dbm_lib.dbm_features.raw_features.movement import DBMLIB_VTREMOR_LIB
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+vt_dir = 'movement/voice_tremor'
+csv_ext = '_vtremor.csv'
+
+#Executing praat script using parselmouth function
+def tremor_praat(snd_file,r_cfg):
+    """
+        Generating Voice tremor endpoint dataframe
+        Args:
+            snd_file: (.wav) parsed audio file
+            r_cfg: Raw variable configuration file
+        Returns tremor endpoint dataframe
+    """
+    snd = parselmouth.Sound(snd_file)
+    tremor_var = run_file(snd,DBMLIB_VTREMOR_LIB,capture_output=True)
+    new_tremor_var = re.sub('--undefined--', '0', tremor_var[1])
+    res = json.loads(new_tremor_var)
+    tremor_df = pd.DataFrame(res,index=['0',])
+    tremor_df.columns = [r_cfg.mov_freq_trem_freq,r_cfg.mov_amp_trem_freq,r_cfg.mov_freq_trem_index,
+                         r_cfg.mov_amp_trem_index,r_cfg.mov_freq_trem_pindex,r_cfg.mov_amp_trem_pindex]
+    return tremor_df
+
+def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name):
+    """
+        Preparing voice tremor matrix
+        Args:
+            audio_file: (.wav) parsed audio file ; r_config: raw config object
+            out_loc: (str) Output directory for csv ; fl_name: file name
+    """
+    df_tremor = tremor_praat(audio_file, r_config)
+    df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+
+    logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+    ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
+
+def prepare_empty_vt(out_loc, fl_name, r_config, error_txt):
+
+    """
+    Preparing empty voice tremor matrix
+    """
+    cols = [r_config.mov_freq_trem_freq, r_config.mov_amp_trem_freq, r_config.mov_freq_trem_index,
+            r_config.mov_amp_trem_index, r_config.mov_freq_trem_pindex, r_config.mov_amp_trem_pindex, r_config.err_reason]
+
+    out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
+    df_tremor = pd.DataFrame(out_val, columns = cols)
+
+    logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name)))
+    ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
+
+def run_vtremor(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for fetching Formant freq
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.5:
+                logger.info('Output file {} size is less than 0.5sec'.format(audio_file))
+                
+                error_txt = 'error: length less than 0.5 sec'
+                prepare_empty_vt(video_uri, out_loc, fl_name, error_txt)
+                return
+            prepare_vtrem_output(audio_file, out_loc, r_config, fl_name)
+    except Exception as e:
+        logger.error('Failed to compute Voice Tremor {} for {}'.format(e,video_uri))
+        prepare_empty_vt(out_loc, fl_name, r_config, e)

opendbm/dbm_lib/dbm_features/raw_features/nlp/speech_features.py

@@ -0,0 +1,50 @@
+"""
+file_name: speech_features
+project_name: DBM
+created: 2020-13-11
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import glob
+from os.path import join
+import logging
+import shutil
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import nlp_util as n_util, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+speech_dir = 'speech/speech_feature'
+speech_ext = '_nlp.csv'
+transcribe_ext = 'speech/deepspeech/*_transcribe.csv'
+
+def run_speech_feature(video_uri, out_dir, r_config, tran_tog):
+    """
+    Processing all patient's for fetching nlp features
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        transcribe_path = glob.glob(join(out_loc, transcribe_ext))
+        if len(transcribe_path)>0:
+
+            transcribe_df = pd.read_csv(transcribe_path[0])
+            df_speech= n_util.process_speech(transcribe_df, r_config)
+
+            logger.info('Saving Output file {} '.format(out_loc))
+            ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext)
+
+            if (tran_tog == None) or (tran_tog != 'on'):
+                shutil.rmtree(os.path.dirname(transcribe_path[0]))
+            
+    except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/nlp/transcribe.py

@@ -0,0 +1,83 @@
+"""
+file_name: transcribe
+project_name: DBM
+created: 2020-10-11
+"""
+
+import pandas as pd
+import numpy as np
+import librosa
+import glob
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import nlp_util as n_util, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+formant_dir = 'speech/deepspeech'
+csv_ext = '_transcribe.csv'
+error_txt = 'error: length less than 0.1'
+
+def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur):
+    """
+    Preparing Formant freq matrix
+    Args:
+        audio_file: (.wav) parsed audio file; fl_name: input file name
+        out_loc: (str) Output directory; r_config: raw variable config
+    """
+    
+    text = n_util.process_deepspeech(audio_file, deep_path)
+    df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe])
+    
+    df_formant.replace('', np.nan, regex=True,inplace=True)
+    df_formant[r_config.nlp_totalTime] = aud_dur
+    df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    df_formant['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
+    
+def empty_transcribe(video_uri, out_loc, fl_name, r_config):
+    
+    """
+    Preparing empty formant frequency matrix if something fails
+    """
+    cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    df_fm = pd.DataFrame(out_val, columns = cols)
+    df_fm['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
+
+def run_transcribe(video_uri, out_dir, r_config, deep_path):
+    
+    """
+    Processing all patient's for fetching Formant freq
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output; deep_path: deepspeech build path
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.1:
+                logger.info('Output file {} size is less than 0.1 sec'.format(audio_file))
+
+                empty_transcribe(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur)
+    except Exception as e:
+        logger.error('Failed to process audio file')
+        

opendbm/dbm_lib/dbm_features/raw_features/util/math_util.py

@@ -0,0 +1,57 @@
+"""
+file_name: facial_tremor
+project_name: cdx_analysis
+created: 2019-03-16
+author: Deshana Desai
+"""
+import sys, os, glob, cv2
+import pandas as pd
+import numpy as np
+
+
+def euclidean_distance(point1, point2):
+    """
+    Compute euclidean distance between points
+    """
+
+    return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)
+
+
+# def detect_peaks()
+
+
+def expand_landmarks(landmarks):
+    """
+    util method to expand landmark list:
+    eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']]
+    """
+    return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks]
+
+
+
+def calc_displacement_vec(df, landmarks, num_frames):
+    """
+    Calculates displacement vector frame by frame
+    """
+
+    landmarks = expand_landmarks(landmarks)
+
+    disp_vec = np.zeros((len(landmarks), num_frames))
+    prev_point = np.zeros((len(landmarks), 2))
+
+    # initialize
+    for j, pair in enumerate(landmarks):
+        first_row = df.iloc[0]
+        prev_point[j] = (first_row[pair[0]], first_row[pair[1]])
+
+
+    for i in range(num_frames):
+        frame_row = df.iloc[i]
+        for j, pair in enumerate(landmarks):
+            x, y = pair[0], pair[1]
+            current = (frame_row[x], frame_row[y])
+            deviation = euclidean_distance( current, prev_point[j])
+            disp_vec[j][i] = deviation
+            prev_point[j] = current
+
+    return disp_vec

opendbm/dbm_lib/dbm_features/raw_features/util/nlp_util.py

@@ -0,0 +1,212 @@
+"""
+file_name: nlp_util
+project_name: DBM
+created: 2020-10-11
+"""
+
+import subprocess
+import json
+import numpy as np
+import pandas as pd
+import os
+import logging
+
+import nltk
+import re
+from lexicalrichness import LexicalRichness
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer 
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+#Speech to text using Deepspeech 0.9.1
+def deepspeech(AUDIO_FILE,deep_path):
+    """
+        Extracting text from audio using Deep Speech neural network trained model
+        Returns:
+            Text: text which is extracted from audio
+    """
+    api = 'deepspeech'
+    arg_speech0 = '--model'
+    arg_speech_path0 = os.path.join(deep_path, 'deepspeech-0.9.1-models.pbmm')
+    arg_speech1 = '--scorer'
+    arg_speech_path1 = os.path.join(deep_path, 'deepspeech-0.9.1-models.scorer')
+    arg_audio = "--audio"
+    
+    out = subprocess.Popen([api, arg_speech0, arg_speech_path0, arg_speech1, arg_speech_path1, arg_audio, AUDIO_FILE],
+                           stdout=subprocess.PIPE, 
+                           stderr=subprocess.STDOUT)
+    logger.info('Deepspeech output...... {}'.format(out))
+    try:
+        stdout,stderr = out.communicate()
+    except:
+        return "error", "error"
+    #print(stderr)
+    return stdout,stderr
+
+def deep_speech_output_clean(result):
+    """
+        Parsing deep speech output(text)
+        Return: 
+            Text from speech
+    """
+    text = ""
+    if len(result)>0:
+        res_split = str(result[0]).split('\\n')
+        
+        if len(res_split)>0:
+            for i in range(len(res_split)):
+                if 'Inference took' in res_split[i]:
+                    text = res_split[i + 1]
+                    return text
+    return text
+
+def process_deepspeech(audio_file,deep_path):
+    """
+    Transcribing audio to extract text from speech
+    """
+    deep_output = deepspeech(audio_file,deep_path)
+    deep_text= deep_speech_output_clean(deep_output)
+    
+    return deep_text
+
+def nltk_download():
+    
+    try:
+        nltk.data.find('tokenizers/punkt')
+        
+    except LookupError:
+        logger.info('punkt is not available')
+        nltk.download('punkt')
+        
+    try:
+        nltk.data.find('averaged_perceptron_tagger')
+        
+    except LookupError:
+        logger.info('averaged_perceptron_tagger is not available')
+        nltk.download('averaged_perceptron_tagger')
+
+def empty_speech(r_config, master_url, error_txt):
+    """
+    Preparing empty speech matrix with error
+    Args:
+        r_config: raw config file object
+        error_txt: Error message during transcription
+        
+    Returns:
+            Empty dataframe for speech features with error
+    """
+    
+    col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns,
+          r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns,
+          r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns,
+          r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin,
+          r_config.nlp_totalTime, r_config.err_reason]
+    
+    df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col)
+    df_speech['dbm_master_url'] = master_url
+    
+    return df_speech
+
+def divide_var(speech_var1, spech_var2):
+    """
+    divide variables
+    """
+    speech_var = np.nan
+    if spech_var2!=0:
+        speech_var = speech_var1/spech_var2
+    return speech_var
+
+def process_speech(transcribe_df,r_config):
+    """
+        Preparing speech features
+        Args:
+            transcribe_df: Transcribed dataframe
+            r_config: raw config file object
+        Returns:
+            Dataframe for speech features
+    """
+    transcribe_df = transcribe_df.replace(np.nan, '', regex=True)
+    err_transcribe = transcribe_df[r_config.err_reason].iloc[0]
+    transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0]
+    total_time = transcribe_df[r_config.nlp_totalTime].iloc[0]
+    master_url = transcribe_df['dbm_master_url'].iloc[0]
+    
+    #clean transcribe
+    transcribe = transcribe.replace(",", "")
+    transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe))
+    
+    if err_transcribe != 'Pass':
+        df_speech = empty_speech(r_config, master_url, error_txt)
+        
+        return df_speech
+    
+    speech_dict = {}
+    nltk_download()
+    
+    sentences = nltk.tokenize.sent_tokenize(transcribe)
+    words_all = nltk.tokenize.word_tokenize(transcribe)
+    num_sentences = len(sentences)
+    
+    speech_dict[r_config.nlp_numSentences] = num_sentences
+    
+    #nlp_singPron
+    i_s = transcribe.count('I')
+    me_s = transcribe.count('me')
+    my_s = transcribe.count('my')
+    sing_count = i_s + me_s + my_s
+    
+    speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences)
+    
+    tagged = nltk.pos_tag(transcribe.split())
+    tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag'])
+    
+    #Past tense per answer
+    all_POSs = tagged_df['pos_tag'].tolist()
+    speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences)
+    
+    #Pronoun per answer
+    pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$')
+    speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences)
+    
+    #Verb per answer
+    verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \
+                      + all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ')
+    speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences)
+    
+    #Adjective per answer
+    adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS')
+    speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences)
+    
+    #Noun per answer
+    nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS')
+    speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences)
+    
+    #Sentiment analysis
+    vader = SentimentIntensityAnalyzer()
+    sentence_valences = []
+    
+    for s in sentences:
+        sentiment_dict = vader.polarity_scores(s) 
+        sentence_valences.append(sentiment_dict['compound'])
+        
+    speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan
+    non_punc = list(value for value in words_all if value not in ['.','!','?'])
+    
+    non_punc_as_str = " ".join(str(non_punc))
+    lex = LexicalRichness(non_punc_as_str)
+    speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan
+    
+    #Number of words per minute
+    speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60
+    speech_dict[r_config.nlp_totalTime] = total_time
+    speech_dict['dbm_master_url'] = master_url
+    
+    df_speech = pd.DataFrame([speech_dict])
+    return df_speech

opendbm/dbm_lib/dbm_features/raw_features/util/util.py

@@ -0,0 +1,112 @@
+"""
+file_name: util
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import glob
+import numpy as np
+import subprocess
+
+def filter_path(video_url, out_dir):
+    
+    """
+    Filtering video uri path to prepare input and ouptut location
+    
+    Args:
+        video_url: S3 bucket path for video
+        out_dir: Output directory path
+        
+    """
+    
+    fl_name,_ = os.path.splitext(os.path.basename(video_url))
+    input_loc = os.path.dirname(video_url)
+    out_loc = os.path.join(out_dir, fl_name)
+    return input_loc, out_loc, fl_name
+
+def save_output(df, out_loc, fl_name, f_dir, f_ext):
+    """
+    creating output directory for Audio features
+    Args:
+        df: (dataframe) feature dataframe[ex: Formant freq, pitch]
+        out_loc: (dir) Output location where we want to save raw output
+        fl_name: file name
+        f_dir: directory name for a feature
+        f_ext: extension for a feature [ex: '_pose.csv']
+    """
+    full_f_name = fl_name + f_ext
+    dir_path = os.path.join(out_loc, f_dir)
+    
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+        
+    sav_path = os.path.join(dir_path,full_f_name)
+    df.to_csv(sav_path, index=False)
+
+def audio_process(base_dir,video_url):
+    """
+    Parsing cleaned audio files(Audio files without IMA voice)
+    Args:
+        base_dir: Base path for raw data
+        video_url: Raw video file path
+    """
+    new_video_url = base_dir+'/'.join(video_url[2:])
+    split_val = new_video_url.split('/')
+    wav_path = '/'.join(split_val[0:len(split_val)-1])
+    audio_split_check = glob.glob(wav_path + '/*_split.wav')
+    return audio_split_check
+
+def compute_open_face_features(input_filepath,
+                         output_directory,
+                         open_face_executable,
+                         au_static=False,
+                         tracked_visualization=False,
+                         clobber=False,
+                         verbose=True):
+    """
+    Runs OpenFace on an input video.
+    See https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments
+    Args:
+        input_filepath:
+        output_directory:
+        au_static:
+        tracked_visualization:
+        open_face_executable:
+        clobber: (bool) if True existing files will be overwritten
+        verbose:
+    Returns:
+        (str) path to output csv file
+    Raises:
+        IOError if OpenFace executable is missing
+    """
+
+    if not os.path.isfile(open_face_executable):
+        raise IOError("OpenFace executable {} could not be found.".format(open_face_executable))
+
+    bn, _ = os.path.splitext(os.path.basename(input_filepath))
+    if not output_directory:
+        output_directory = os.path.join(os.path.dirname(input_filepath), bn + '_openface')
+
+    output_csv = os.path.join(output_directory, bn + '.csv')
+    if not os.path.isfile(output_csv) or clobber:
+        call = [open_face_executable, ]
+        if au_static:
+            call += ['-au_static', ]
+
+        if tracked_visualization:
+            call += ['-tracked', ]
+
+        call += ['-q', '-2Dfp', '-3Dfp', '-pdmparams', '-pose', '-aus', '-gaze']
+        call += ['-f', input_filepath, '-out_dir', output_directory]
+
+        if verbose:
+            print('Computing OpenFace features {} from video file'.format(input_filepath))
+        subprocess.check_output(call)
+        if verbose:
+            print('OpenFace features saved to {}'.format(output_directory))
+    else:
+        if verbose:
+            print('Output file {} already exists'.format(output_csv))
+
+    return os.path.join(output_directory, bn + '.csv')

opendbm/dbm_lib/dbm_features/raw_features/util/vad_utilities.py

@@ -0,0 +1,221 @@
+"""
+file_name: vad_utilities
+project_name: DBM
+created: 2020-20-07
+"""
+
+# code from https://github.com/wiseman/py-webrtcvad/blob/master/example.py
+import collections
+import contextlib
+import sys
+import wave
+
+def read_wave(path):
+    """Reads a .wav file.
+    Takes the path, and returns (PCM audio data, sample rate).
+    """
+    with contextlib.closing(wave.open(path, 'rb')) as wf:
+        num_channels = wf.getnchannels()
+        assert num_channels == 1
+        sample_width = wf.getsampwidth()
+        assert sample_width == 2
+        sample_rate = wf.getframerate()
+        assert sample_rate in (8000, 16000, 32000, 48000)
+        pcm_data = wf.readframes(wf.getnframes())
+        return pcm_data, sample_rate
+
+
+class Frame(object):
+    """Represents a "frame" of audio data."""
+    def __init__(self, bytes, timestamp, duration):
+        self.bytes = bytes
+        self.timestamp = timestamp
+        self.duration = duration
+
+def frame_generator(frame_duration_ms, audio, sample_rate):
+    """Generates audio frames from PCM audio data.
+    Takes the desired frame duration in milliseconds, the PCM data, and
+    the sample rate.
+    Yields Frames of the requested duration.
+    """
+    n = int(sample_rate * (frame_duration_ms / 1000.0) * 2)
+    offset = 0
+    timestamp = 0.0
+    duration = (float(n) / sample_rate) / 2.0
+    while offset + n < len(audio):
+        yield Frame(audio[offset:offset + n], timestamp, duration)
+        timestamp += duration
+        offset += n
+
+
+def vad_collector(sample_rate, frame_duration_ms,
+                  padding_duration_ms, vad, frames):
+    """Filters out non-voiced audio frames.
+    Given a webrtcvad.Vad and a source of audio frames, yields only
+    the voiced audio.
+    Uses a padded, sliding window algorithm over the audio frames.
+    When more than 90% of the frames in the window are voiced (as
+    reported by the VAD), the collector triggers and begins yielding
+    audio frames. Then the collector waits until 90% of the frames in
+    the window are unvoiced to detrigger.
+    The window is padded at the front and back to provide a small
+    amount of silence or the beginnings/endings of speech around the
+    voiced frames.
+    Arguments:
+    sample_rate - The audio sample rate, in Hz.
+    frame_duration_ms - The frame duration in milliseconds.
+    padding_duration_ms - The amount to pad the window, in milliseconds.
+    vad - An instance of webrtcvad.Vad.
+    frames - a source of audio frames (sequence or generator).
+    Returns: A generator that yields PCM audio data.
+    """
+    num_padding_frames = int(padding_duration_ms / frame_duration_ms)
+    # We use a deque for our sliding window/ring buffer.
+    ring_buffer = collections.deque(maxlen=num_padding_frames)
+    # We have two states: TRIGGERED and NOTTRIGGERED. We start in the
+    # NOTTRIGGERED state.
+    triggered = False
+
+    voiced_frames = []
+    for frame in frames:
+        is_speech = vad.is_speech(frame.bytes, sample_rate)
+
+        sys.stdout.write('1' if is_speech else '0')
+        if not triggered:
+            ring_buffer.append((frame, is_speech))
+            num_voiced = len([f for f, speech in ring_buffer if speech])
+            # If we're NOTTRIGGERED and more than 90% of the frames in
+            # the ring buffer are voiced frames, then enter the
+            # TRIGGERED state.
+            if num_voiced > 0.9 * ring_buffer.maxlen:
+                triggered = True
+                sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
+                # We want to yield all the audio we see from now until
+                # we are NOTTRIGGERED, but we have to start with the
+                # audio that's already in the ring buffer.
+                for f, s in ring_buffer:
+                    voiced_frames.append(f)
+                ring_buffer.clear()
+        else:
+            # We're in the TRIGGERED state, so collect the audio data
+            # and add it to the ring buffer.
+            voiced_frames.append(frame)
+            ring_buffer.append((frame, is_speech))
+            num_unvoiced = len([f for f, speech in ring_buffer if not speech])
+            # If more than 90% of the frames in the ring buffer are
+            # unvoiced, then enter NOTTRIGGERED and yield whatever
+            # audio we've collected.
+            if num_unvoiced > 0.9 * ring_buffer.maxlen:
+                sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
+                triggered = False
+                yield b''.join([f.bytes for f in voiced_frames])
+                ring_buffer.clear()
+                voiced_frames = []
+    if triggered:  # BT if were in triggered state at end of signal, set output time
+        sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
+    sys.stdout.write('\n')
+    # If we have any leftover voiced audio when we run out of input,
+    # yield it.
+    if voiced_frames:
+        yield b''.join([f.bytes for f in voiced_frames])
+
+
+
+def vad_get_segment_times(sample_rate, frame_duration_ms,
+                  padding_duration_ms, vad, frames):
+    """Filters out non-voiced audio frames.
+    BT: based on vad_collector, but returns start and end times for voiced segs
+
+    Given a webrtcvad.Vad and a source of audio frames, yields only
+    the voiced audio.
+    Uses a padded, sliding window algorithm over the audio frames.
+    When more than 90% of the frames in the window are voiced (as
+    reported by the VAD), the collector triggers and begins yielding
+    audio frames. Then the collector waits until 90% of the frames in
+    the window are unvoiced to detrigger.
+    The window is padded at the front and back to provide a small
+    amount of silence or the beginnings/endings of speech around the
+    voiced frames.
+    Arguments:
+    sample_rate - The audio sample rate, in Hz.
+    frame_duration_ms - The frame duration in milliseconds.
+    padding_duration_ms - The amount to pad the window, in milliseconds.
+    vad - An instance of webrtcvad.Vad.
+    frames - a source of audio frames (sequence or generator).
+    Returns: lists of start and end segments
+    """
+
+    num_padding_frames = int(padding_duration_ms / frame_duration_ms)
+    # We use a deque for our sliding window/ring buffer.
+    ring_buffer = collections.deque(maxlen=num_padding_frames)
+    # We have two states: TRIGGERED and NOTTRIGGERED. We start in the
+    # NOTTRIGGERED state.
+    triggered = False
+
+    start_times = []
+    end_times = []
+
+    for frame in frames:
+        is_speech = vad.is_speech(frame.bytes, sample_rate)
+
+        #sys.stdout.write('1' if is_speech else '0')
+        if not triggered:
+            ring_buffer.append((frame, is_speech))
+            num_voiced = len([f for f, speech in ring_buffer if speech])
+            # If we're NOTTRIGGERED and more than 90% of the frames in
+            # the ring buffer are voiced frames, then enter the
+            # TRIGGERED state.
+            if num_voiced > 0.9 * ring_buffer.maxlen:
+                triggered = True
+                #sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
+                start_times.append(ring_buffer[0][0].timestamp)  # BT
+                ring_buffer.clear()
+        else:
+            # We're in the TRIGGERED state, so collect the audio data
+            # and add it to the ring buffer.
+            ring_buffer.append((frame, is_speech))
+            num_unvoiced = len([f for f, speech in ring_buffer if not speech])
+            # If more than 90% of the frames in the ring buffer are
+            # unvoiced, then enter NOTTRIGGERED and yield whatever
+            # audio we've collected.
+            if num_unvoiced > 0.9 * ring_buffer.maxlen:
+                #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
+                end_times.append(ring_buffer[0][0].timestamp + frame.duration)  # BT
+                triggered = False
+
+    if triggered:  # BT if were in triggered state at end of signal, set output time
+        #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
+        if len(ring_buffer)>0:
+            end_times.append(ring_buffer[0][0].timestamp )  # BT
+        else:
+            # only get here in very rare case that we triggered on 2nd-to-last frame
+            end_times.append(frame.timestamp + frame.duration)
+    #sys.stdout.write('\n')
+
+    return(start_times, end_times)
+
+
+def filter_seg_times(seg_starts, seg_ends, pad_at_start = 0.5, len_to_keep=2.5 ):
+    """
+    do some filtering on the segments found to select part for analysis
+    rule: find the first segment that is at least (pad_at_start+len_to_keep sec long.
+    Discard the firstpad_at_start sec, keep the next len_to_keep sec
+    if no such segments, then return empty list
+
+    returns sel_start, sel_end, sel_end_longer
+    """
+    sel_start = []
+    sel_end = []
+    sel_end_longer = []
+
+    not_found = True
+    for iseg in range(len(seg_starts)):
+        seg_dur = seg_ends[iseg]-seg_starts[iseg]
+        if (not_found & (seg_dur > (pad_at_start + len_to_keep))):
+            t_start = seg_starts[iseg] + pad_at_start
+            sel_start.append(t_start)
+            sel_end.append(t_start + len_to_keep)
+            sel_end_longer.append(max(t_start + len_to_keep, seg_ends[iseg]-pad_at_start))
+            not_found = False
+
+    return sel_start, sel_end, sel_end_longer

opendbm/dbm_lib/dbm_features/raw_features/util/video_util.py

@@ -0,0 +1,191 @@
+"""
+file_name: video_util
+project_name: DBM
+created: 2020-20-07
+"""
+
+import pandas as pd
+import numpy as np
+import glob
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+
+def smooth(x,window_len=11,window='hanning'):
+    """smooth the data using a window with requested size.
+    
+    This method is based on the convolution of a scaled window with the signal.
+    The signal is prepared by introducing reflected copies of the signal 
+    (with the window size) in both ends so that transient parts are minimized
+    in the begining and end part of the output signal.
+    
+    input:
+        x: the input signal 
+        window_len: the dimension of the smoothing window; should be an odd integer
+        window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
+            flat window will produce a moving average smoothing.
+
+    output:
+        the smoothed signal
+        
+    example:
+
+    t=linspace(-2,2,0.1)
+    x=sin(t)+randn(len(t))*0.1
+    y=smooth(x)
+    
+    see also: 
+    
+    numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve
+    scipy.signal.lfilter
+ 
+    TODO: the window parameter could be the window itself if an array instead of a string
+    NOTE: length(output) != length(input), to correct this: return y[(window_len/2-1):-(window_len/2)] instead of just y.
+    """
+    if x.ndim != 1:
+        raise (ValueError, "smooth only accepts 1 dimension arrays.")
+    if x.size < window_len:
+        raise (ValueError, "Input vector needs to be bigger than window size.")
+    if window_len<3:
+        return x
+    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
+        raise (ValueError, "Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")
+    s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]
+    #print(len(s))
+    if window == 'flat': #moving average
+        w=np.ones(window_len,'d')
+    else:
+        w=eval('np.'+window+'(window_len)')
+    y=np.convolve(w/w.sum(),s,mode='valid')
+    return y[int(window_len/2):-int(window_len/2)]
+
+def filter_by_confidence_and_thresh(x, fea, thresh):
+    if x['s_confidence'] > 0.2 and np.fabs(x[fea]) < thresh:
+        return x[fea]
+    else:
+        return np.NaN
+    
+def add_au_emotion(x, emotion,emotion_type,exp_type):
+    """
+    computing individula emotion expressivity matrix
+    Args:
+        emotion: Action Unit
+    """
+    error_reason = 'Pass'
+    if x['s_confidence'] > 0.8: #if using smooth, no need for 'success'
+        sum_r = 0
+        cnt = 0
+        for au in emotion:
+            au_c_label = " AU{:02d}_c".format(au)
+            au_r_label = " AU{:02d}_r".format(au)
+            if x[au_c_label]==1 and (not np.isnan(x[au_r_label])): #there are data with face in, but au_c=0
+                sum_r += x[au_r_label]
+                cnt += 6
+            if exp_type=='full' and x[au_c_label]==0: #Logic to compute emotion expressivity when all AU's are present 
+                cnt = 0 
+                break
+        if cnt > 0: 
+            sum_r /= cnt
+        else:
+            sum_r = 0
+        v_emo = x[emotion_type] + sum_r 
+    else:
+        v_emo = np.NaN
+        error_reason = 'confidence less than 80%'
+    
+    return v_emo, error_reason
+    
+def add_au_occ(x, emotion,emotion_type):
+    """
+    computing individula emotion presence
+    Args:
+        emotion: Action Unit
+    """
+    au_pres = []
+    em_pres = 0
+    error_reason = 'Pass'
+    if x['s_confidence'] > 0.8: #if using smooth, no need for 'success'
+        for au in emotion:
+            au_c_label = " AU{:02d}_c".format(au)
+            if x[au_c_label]==1: #there are data with face in, but au_c=0
+                au_pres.append(1)
+                
+        if len(au_pres) == len(emotion):
+            em_pres = 1
+    else:
+        em_pres = np.NaN
+        error_reason = 'confidence less than 80%'
+    return em_pres, error_reason
+    
+def emotion_exp(em_au,of,em_col,err_col):
+    """
+        Computing individual emotion expressivity and adding it to dataframe
+    """
+    for emotion in em_au:
+        of[[em_col[0],err_col]]=of.apply(add_au_emotion, args=(emotion,em_col[0],'partial',), axis=1, result_type='expand')
+        of[[em_col[1],err_col]]=of.apply(add_au_emotion, args=(emotion,em_col[1],'full',), axis=1, result_type='expand')
+        
+def emotion_pres(em_au,of,em_col,err_col):
+    """
+        Computing individual emotion expressivity and adding it to dataframe
+    """
+    for emotion in em_au:
+        of[[em_col,err_col]]=of.apply(add_au_occ, args=(emotion,em_col,), axis=1, result_type='expand')
+    
+def calc_of_for_video(of,face_cfg,fe_cfg):
+    """
+        Creating dataframe for emotion expressivity
+    """
+    new_cols = [fe_cfg.hap_exp,fe_cfg.sad_exp,fe_cfg.sur_exp,fe_cfg.fea_exp,fe_cfg.ang_exp,fe_cfg.dis_exp,fe_cfg.con_exp,
+               fe_cfg.pai_exp,fe_cfg.neg_exp,fe_cfg.pos_exp,fe_cfg.neu_exp,fe_cfg.com_lower_exp,fe_cfg.com_upper_exp,
+                fe_cfg.cai_exp,fe_cfg.com_exp,fe_cfg.happ_occ,fe_cfg.sad_occ,fe_cfg.sur_occ,fe_cfg.fea_occ,fe_cfg.ang_occ,
+                fe_cfg.dis_occ,fe_cfg.con_occ,fe_cfg.hap_exp_full,fe_cfg.sad_exp_full,fe_cfg.sur_exp_full,fe_cfg.fea_exp_full,
+                fe_cfg.ang_exp_full,fe_cfg.dis_exp_full,fe_cfg.con_exp_full,fe_cfg.pai_exp_full,fe_cfg.neg_exp_full,
+                fe_cfg.pos_exp_full,fe_cfg.neu_exp_full,fe_cfg.cai_exp_full,fe_cfg.com_lower_exp_full,fe_cfg.com_upper_exp_full,
+                fe_cfg.com_exp_full]
+    of[new_cols] = pd.DataFrame([[0] * len(new_cols)], index=of.index)
+    of[fe_cfg.err_reason] = 'Pass'
+    
+    #Composite happiness expressivity
+    emotion_exp(face_cfg.happiness,of,[fe_cfg.hap_exp,fe_cfg.hap_exp_full],fe_cfg.err_reason)
+    #Composite sadness expressivity
+    emotion_exp(face_cfg.sadness,of,[fe_cfg.sad_exp,fe_cfg.sad_exp_full],fe_cfg.err_reason)
+    #Composite surprise expressivity
+    emotion_exp(face_cfg.surprise,of,[fe_cfg.sur_exp,fe_cfg.sur_exp_full],fe_cfg.err_reason)
+    #Composite fear expressivity
+    emotion_exp(face_cfg.fear,of,[fe_cfg.fea_exp,fe_cfg.fea_exp_full],fe_cfg.err_reason)
+    #Composite anger expressivity
+    emotion_exp(face_cfg.anger,of,[fe_cfg.ang_exp,fe_cfg.ang_exp_full],fe_cfg.err_reason)
+    #Composite disgust expressivity
+    emotion_exp(face_cfg.disgust,of,[fe_cfg.dis_exp,fe_cfg.dis_exp_full],fe_cfg.err_reason)
+    #Composite contempt expressivity
+    emotion_exp(face_cfg.contempt,of,[fe_cfg.con_exp,fe_cfg.con_exp_full],fe_cfg.err_reason)
+    #Composite Negative Expressivity
+    emotion_exp(face_cfg.NEG_ACTION_UNITS,of,[fe_cfg.neg_exp,fe_cfg.neg_exp_full],fe_cfg.err_reason)
+    #Composite Positive Expressivity
+    emotion_exp(face_cfg.POS_ACTION_UNITS,of,[fe_cfg.pos_exp,fe_cfg.pos_exp_full],fe_cfg.err_reason)
+    #Composite Neutral Expressivity
+    emotion_exp(face_cfg.NET_ACTION_UNITS,of,[fe_cfg.neu_exp,fe_cfg.neu_exp_full],fe_cfg.err_reason)
+    #Composite Activation Expressivity
+    emotion_exp(face_cfg.cai,of,[fe_cfg.cai_exp,fe_cfg.cai_exp_full],fe_cfg.err_reason)
+    #Composite Expressivity
+    emotion_exp(face_cfg.ACTION_UNITS,of,[fe_cfg.com_exp,fe_cfg.com_exp_full],fe_cfg.err_reason)
+    #Composite lower face expressivity
+    emotion_exp(face_cfg.LOWER_ACTION_UNITS,of,[fe_cfg.com_lower_exp,fe_cfg.com_lower_exp_full],fe_cfg.err_reason)
+    #Composite upper face Expressivity
+    emotion_exp(face_cfg.UPPER_ACTION_UNITS,of,[fe_cfg.com_upper_exp,fe_cfg.com_upper_exp_full],fe_cfg.err_reason)
+    #Composite pain expressivity
+    emotion_exp(face_cfg.pain,of,[fe_cfg.pai_exp,fe_cfg.pai_exp_full],fe_cfg.err_reason) 
+    #AU happiness presence
+    emotion_pres(face_cfg.happiness,of,fe_cfg.happ_occ,fe_cfg.err_reason)
+    #AU Sad presence
+    emotion_pres(face_cfg.sadness,of,fe_cfg.sad_occ,fe_cfg.err_reason)
+    #AU Surprise presence
+    emotion_pres(face_cfg.surprise,of,fe_cfg.sur_occ,fe_cfg.err_reason)
+    #AU fear presence
+    emotion_pres(face_cfg.fear,of,fe_cfg.fea_occ,fe_cfg.err_reason)
+    #AU anger presence
+    emotion_pres(face_cfg.anger,of,fe_cfg.ang_occ,fe_cfg.err_reason)
+    #AU disgust presence
+    emotion_pres(face_cfg.disgust,of,fe_cfg.dis_occ,fe_cfg.err_reason)
+    #AU contempt presence
+    emotion_pres(face_cfg.contempt,of,fe_cfg.con_occ,fe_cfg.err_reason)

opendbm/dbm_lib/dbm_features/raw_features/video/__init__.py

@@ -0,0 +1,14 @@
+"""
+file_name: __init__
+project_name: DBM
+created: 2020-20-07
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+DBMLIB_PATH = os.path.dirname(__file__)
+DBMLIB_FACE_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/services/face_util.yml'))

opendbm/dbm_lib/dbm_features/raw_features/video/face_asymmetry.py

@@ -0,0 +1,354 @@
+"""
+file_name: face_asymmetry.py
+project_name: DBM
+created: 2020-20-07
+"""
+
+from mpl_toolkits import mplot3d
+from matplotlib import pyplot as plt
+import time
+import numpy as np
+import os
+import datetime
+import glob
+import cv2
+from scipy.spatial.transform import Rotation as R
+import subprocess
+import pandas as pd
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
+from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+face_asym_dir = 'facial/face_asymmetry'
+csv_ext = '_facasym.csv'
+
+cv2_color_purple = (254,19,188)
+color_blue = (0,0,1.0)
+color_green = (0,1.0,0)
+color_red = (1.0,0,0)
+color_y = (1.0,1.0,0)
+
+error_code_message = {
+    0: 'pass', 
+    1: 'confidence less than 80%',
+    }
+error_message_code = {y:x for x,y in error_code_message.items()}
+
+def visualize_vid(fn, attr=None, write_out=False):
+
+    vid = cv2.VideoCapture(fn)
+    tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
+    fps = vid.get(cv2.CAP_PROP_FPS)
+    frame_width = int(vid.get(3))
+    frame_height = int(vid.get(4))
+
+    if write_out:
+        fig_w = 680 #680 667 676 #frame_width in order of Ali, Vennessa, synthesis
+        fig_h = 659 #659 659 659 #frame_height
+        out_vid = cv2.VideoWriter('out.mp4',cv2.VideoWriter_fourcc(*'MP4V'), fps, (fig_w,fig_h))
+
+    plt.figure(figsize=(8, 8))
+    try:
+        frameid = 0
+        while(True):
+            ret, frame = vid.read()
+            if not ret:
+                # Release the Video Device if ret is false
+                vid.release()
+                print('Released Video Resource')
+                break
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            frameid += 1
+            logger.info(frameid, frame.shape)
+            
+            if 'lmks_frms' in attr:
+                lmks_frms = attr['lmks_frms']
+                for i in range(lmks_frms[frameid].shape[0]):
+                    cv2.circle(frame,(int(lmks_frms[frameid][i,0]),int(lmks_frms[frameid][i,1])), 2, cv2_color_purple, -1)
+
+            if write_out:
+                cv2.putText(frame,'Frame: '+str(frameid), (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 3)
+
+            plt.subplot(211)
+            plt.imshow(frame)
+            plt.axis('off'); plt.pause(0.2); 
+            
+            if 'score_asym' in attr:
+                ax = plt.subplot(212)
+                ax.cla()
+                ax.set_xlim(0,140) #ax.set_xlim(0,300)
+                ax.set_ylim(0,10)
+                
+                sa = attr['score_asym']
+                s = sa[np.where(sa[:,0] <= frameid),:][0,:,:]
+                
+                for i in range(1,s.shape[1]):
+                    plt.plot(s[:,0], s[:,i])
+                    
+                plt.legend(['mouth', 'eyebrow', 'eye', 'mouth+eye+eyebrow'])
+                plt.minorticks_on()
+                plt.grid(b=True, which='major', color='r', linestyle='-')
+                plt.grid(b=True, which='minor', color='r', linestyle='--')
+
+                plt.savefig('tmp.png', bbox_inches='tight')
+                print(cv2.imread('tmp.png').shape)
+                
+            plt.clf()
+            if write_out:
+                out_vid.write(cv2.imread('tmp.png'))
+
+    except KeyboardInterrupt:
+        # Release the Video Device
+        vid.release()
+        if write_out:
+            out_vid.release()
+        logger.info('Exception, and Video Resource Released')
+        
+    if write_out:
+        out_vid.release()
+
+        
+def retrieve_attr(of_df):
+    '''
+        Retrieve landmarks and pose_translation for each frame from openface output
+        Args:
+            of_df: dataframe output from openface, including detected landmark coordinates
+        Returns:
+            lmks_frms: dictionary, with frame id as key and 68 landmark set as value
+            pose_p: dictionary, with frame id as key and pose param as value
+    '''
+    tot_lmks = 68 # openface specific
+    if len([i for i in of_df.columns.to_list() if ' x_' in i]) != tot_lmks:
+        return {}
+    
+    lmks_frms = {}
+    pose_p = {}
+    
+    for fi in sorted(of_df['frame'].to_list()):
+        lmks = np.zeros((tot_lmks,6))
+        r = of_df[of_df['frame']==fi]
+        
+        for i in range(tot_lmks):
+            lmk_y = r[' y_'+str(i)].iloc[0]
+            lmk_x = r[' x_'+str(i)].iloc[0]
+            lmk_X = r[' X_'+str(i)].iloc[0]
+            lmk_Y = r[' Y_'+str(i)].iloc[0]
+            lmk_Z = r[' Z_'+str(i)].iloc[0]
+            
+            confi = r[' confidence']
+            lmks[i,:] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi]
+            
+        lmks_frms[fi] = lmks
+        pose_p[fi] = [r[' pose_Tx'].iloc[0], r[' pose_Ty'].iloc[0], r[' pose_Tz'].iloc[0],
+                      r[' pose_Rx'].iloc[0], r[' pose_Ry'].iloc[0], r[' pose_Rz'].iloc[0]]
+    
+    return lmks_frms, pose_p
+
+
+def mirror_point(a, b, c, d, x1, y1, z1): 
+    # mirror a point w.r.t a 3D plane
+    k =(-a * x1-b * y1-c * z1-d)/float((a * a + b * b + c * c)) 
+    
+    x2 = a * k + x1 
+    y2 = b * k + y1 
+    z2 = c * k + z1 
+    
+    x3 = 2 * x2-x1 
+    y3 = 2 * y2-y1 
+    z3 = 2 * z2-z1 
+    return [x3, y3, z3]
+
+
+def dist_vec2plane(vec, nrm):
+    # Calculate the projected length of a vector (vec) to a plane defined by its normal (nrm)
+    return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm)**2)
+
+
+def vis_lmks3d(lmks_frms, vis_idx):
+    """
+    Visualizing facial landmarks
+    """
+    fig = plt.figure()
+    color_type = ['b','g','r','y','c']
+    assert len(color_type) > len(vis_idx)
+    
+    for fi in sorted(list(lmks_frms.keys())):
+        ax = plt.axes(projection="3d")
+        for i,vi in enumerate(vis_idx):
+            ax.scatter(lmks_frms[fi][vi,2], lmks_frms[fi][vi,3], lmks_frms[fi][vi,4], c=color_type[i])
+            
+        ax.axes.set_xlim3d(left=-75, right=100) 
+        ax.axes.set_ylim3d(bottom=-200, top=25) 
+        ax.axes.set_zlim3d(bottom=440, top=560)
+        ax.view_init(-89, -90) #elev, ariz
+        plt.title(str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
+        plt.pause(0.2)
+        plt.cla()
+        plt.draw()
+
+def calc_fac_asymmetry(attr, is_vis=False):
+    '''
+    Quantify facial asymmetry
+    Args:
+        attr: attribute dictionary containing necessary features for calculation, e.g.,
+            lmks_frms: dictionary, with frame id as key and 68 landmark set (OpenFace) as value
+            pose_param: dictionary, with frame id as key and pose param as value
+    Returns:
+        score_asym: 2D array of size (num_frms, num_asymm_fea), with frame id as the 0th column, and each remaining column as one asymmetry feature
+    '''
+    # openface landmark indices
+    lmks_ref_idx = list(range(0,17)) + list(range(27,36))
+    lmks_mid_idx = [27,28,29,30,33,51,62,66,57,8]
+    lmks_rgt_idx = [0,1,2,3,4,5,6,7,
+                    17,18,19,20,21,
+                    36,37,38,39,40,41,
+                    48,49,50,
+                    59,58,
+                    60,61,
+                    67]
+    lmks_lft_idx = [16,15,14,13,12,11,10,9,
+                    26,25,24,23,22,
+                    45,44,43,42,47,46,
+                    54,53,52,
+                    55,56,
+                    64,63,
+                    65]
+    
+    lmks_mth_idx = list(range(48,68))
+    lmks_ebr_idx = list(range(17,27))
+    lmks_eye_idx = list(range(36,48))
+    assert len(lmks_lft_idx)==len(lmks_rgt_idx)
+
+    fea_list = ['mouth', 'eyebrow', 'eye', 'composite']
+    score_asym = np.empty(shape=(0, 0))
+    
+    if ('lmks_frms' in attr) and ('pose_param' in attr):
+        lmks_frms = attr['lmks_frms']
+        pose_p = attr['pose_param']
+
+        if is_vis:
+            vis_lmks3d(lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx])
+        
+        score_asym = np.zeros((len(lmks_frms),len(fea_list)+1+1)) # +1: extra column for error code
+        if is_vis:
+            fig = plt.figure()
+            ax = plt.axes(projection="3d")
+            
+        for s,fi in enumerate(sorted(list(lmks_frms.keys()))):
+            lmks_3d = lmks_frms[fi][:,2:5]
+            pose = pose_p[fi]
+            err_code = error_message_code['pass']
+            
+            if lmks_frms[fi][0,5] < 0.8:
+                err_code = error_message_code['confidence less than 80%']
+                score_asym[s,:] = [fi,np.NaN,np.NaN,np.NaN,np.NaN,err_code]
+                continue
+
+            rx = R.from_euler('x', pose[3])
+            ry = R.from_euler('y', pose[4])
+            rz = R.from_euler('z', pose[5])
+            
+            vec_pose = rz.apply(ry.apply(rx.apply([0,0,1])))
+            anc_idx = [30, 27, 8] # for central plane estimation
+            nrm = np.cross(lmks_3d[anc_idx[2],:] - lmks_3d[anc_idx[0],:],
+                            lmks_3d[anc_idx[1],:] - lmks_3d[anc_idx[0],:])
+            
+            nrm = nrm / np.linalg.norm(nrm)
+            a,b,c = nrm
+            d = np.dot(nrm, lmks_3d[anc_idx[0],:])
+            
+            dist_L2R_mth = []
+            dist_L2R_ebr = []
+            dist_L2R_eye = []
+            dist_com = []
+            
+            lmks_rfl = np.empty((0,3))
+            src_idx = lmks_lft_idx
+            
+            for k,idx in enumerate(src_idx):
+                p_rfl = np.array(mirror_point(a, b, c, -d, lmks_3d[idx,0], lmks_3d[idx,1], lmks_3d[idx,2]))
+                lmks_rfl = np.vstack((lmks_rfl, p_rfl))
+                dist = dist_vec2plane((p_rfl-lmks_3d[lmks_rgt_idx[k],:]), vec_pose)
+                
+                if idx in lmks_mth_idx: 
+                    dist_L2R_mth.append(dist)
+                if idx in lmks_ebr_idx:
+                    dist_L2R_ebr.append(dist)
+                if idx in lmks_eye_idx:
+                    dist_L2R_eye.append(dist)
+                if (idx in lmks_mth_idx) or (idx in lmks_ebr_idx) or (idx in lmks_eye_idx):
+                    dist_com.append(dist) 
+            score_asym[s,:] = [fi,np.mean(dist_L2R_mth),np.mean(dist_L2R_ebr),np.mean(dist_L2R_eye),np.mean(dist_com),err_code]
+
+            if is_vis:
+                ax.scatter(lmks_3d[:,0], lmks_3d[:,1], lmks_3d[:,2])
+                ax.scatter(lmks_rfl[:,0], lmks_rfl[:,1], lmks_rfl[:,2], c='y')
+                ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c='c')
+                plt.title('mirrored landmarks, frame: '+str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
+                plt.pause(0.2)
+                plt.cla()
+                plt.draw()
+
+    return score_asym
+
+
+def calc_asym_feature(open_face_csv, f_cfg):
+    """
+    Calculating facial asymmetry features and preparing final df
+    """
+    df_list = []
+    
+    of_df = pd.read_csv(open_face_csv, error_bad_lines=False)
+    lmks_frms, pose_p = retrieve_attr(of_df)
+    
+    attr = {'lmks_frms': lmks_frms, 'pose_param': pose_p}
+    score_asym = calc_fac_asymmetry(attr)
+    
+    df_score_asym = pd.DataFrame(score_asym, columns=['frame', f_cfg.fac_AsymMaskMouth, f_cfg.fac_AsymMaskEyebrow, 
+                                                      f_cfg.fac_AsymMaskEye, f_cfg.fac_AsymMaskCom, f_cfg.err_reason])
+    df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(lambda x: error_code_message[x])
+    
+    df_score_asym['frame'] = of_df['frame']
+    df_score_asym['face_id'] = of_df[' face_id']
+    df_score_asym['timestamp'] = of_df[' timestamp']
+    df_score_asym['confidence'] = of_df[' confidence']
+    df_score_asym['success'] = of_df[' success']
+    
+    df_list.append(df_score_asym)
+    return df_list 
+
+
+def run_face_asymmetry(video_uri, out_dir, f_cfg):
+    """
+    Processing all patient's for calculating facial asymmetry
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; f_cfg: face config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        #Baseline logic
+        cfr = ConfigFaceReader()
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
+        if len(of_csv_path)>0:
+
+            of_csv = of_csv_path[0]
+            asym_df_list = calc_asym_feature(of_csv, f_cfg)
+
+            asym_final_df = pd.concat(asym_df_list, ignore_index=True)
+            asym_final_df['dbm_master_url'] = video_uri
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext)
+        
+    except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/video/face_au.py

@@ -0,0 +1,97 @@
+"""
+file_name: face_au.py
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import datetime
+import glob
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
+from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+face_au_dir = 'facial/face_au'
+csv_ext = '_facau.csv'
+
+
+def extract_col_nm_au(cols):
+    """
+    Extract action unit (au) column names from openface output (csv)
+    Args:
+        cols: column names from open face output (csv)
+    Returns:
+        (list) list of au column names
+    """
+    cols_lmk = []
+    au_tags = ' AU'
+    cols_au = [c for c in cols if au_tags in c]
+    return cols_au
+
+
+def au_col_nm_map(df):
+    """
+    Rename dataframe action unit column names to match functional specifications v1.0
+    Args:
+        df: dataframe
+    Returns:
+        dataframe with mapped variables
+    """
+    dict_au_cols = {}
+    for col in list(df):
+        if ' AU' in col:
+            idx = col.rfind('_')
+            if idx > -1:
+                au_id = col[idx-2:idx]
+                if '_r' in col:
+                    dict_au_cols[col] = 'fac_AU' + au_id + 'int'
+                if '_c' in col:
+                    dict_au_cols[col] = 'fac_AU' + au_id + 'pres'
+    df.rename(columns=dict_au_cols, inplace=True)
+    return df
+
+
+def run_face_au(video_uri, out_dir, f_cfg):
+    """
+    Processing all patient's for fetching action units
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; f_cfg: face config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        #Baseline logic
+        cfr = ConfigFaceReader()
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
+        if len(of_csv_path)>0:
+
+            df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
+            df_au = df_of[extract_col_nm_au(df_of)]
+            df_au = df_au.copy()
+
+            df_au['frame'] = df_of['frame']
+            df_au['face_id'] = df_of[' face_id']
+            df_au['timestamp'] = df_of[' timestamp']
+            df_au['confidence'] = df_of[' confidence']
+            df_au['success'] = df_of[' success']
+
+            df_au = au_col_nm_map(df_au)
+            df_au['dbm_master_url'] = video_uri
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext)
+            
+    except Exception as e:
+        logger.error('Failed to process video file')
+    

opendbm/dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py

@@ -0,0 +1,140 @@
+"""
+file_name: face_config_reader
+project_name: DBM
+created: 2020-20-07
+"""
+
+import yaml
+import boto3
+from opendbm.dbm_lib.dbm_features.raw_features.video import DBMLIB_FACE_CONFIG
+
+class ConfigFaceReader(object):
+    """Summary
+    Read sevice end ponit
+    """
+    def __init__(self,
+                 service_config_yml=None):
+        """Summary
+        Args:
+            service_config_yml (None, optional): yml file defined service configuration
+        """
+        
+        if service_config_yml is None:
+            service_config = DBMLIB_FACE_CONFIG
+        else:
+            service_config = service_config_yml
+        
+        with open(service_config, 'r') as ymlfile:
+            config = yaml.load(ymlfile)
+            self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS']
+            self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS']
+            self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS']
+            self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS']  
+            self.LOWER_ACTION_UNITS = config['cdx_face_config']['LOWER_ACTION_UNITS']
+            self.UPPER_ACTION_UNITS = config['cdx_face_config']['UPPER_ACTION_UNITS']
+            self.happiness = config['cdx_face_config']['happiness']
+            self.sadness = config['cdx_face_config']['sadness']
+            self.surprise = config['cdx_face_config']['surprise']
+            self.fear = config['cdx_face_config']['fear']
+            self.anger = config['cdx_face_config']['anger']
+            self.disgust = config['cdx_face_config']['disgust']
+            self.contempt = config['cdx_face_config']['contempt']
+            self.pain = config['cdx_face_config']['pain']
+            self.cai = config['cdx_face_config']['CAI']
+            self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',')
+            self.face_expr_dir = config['cdx_face_config']['face_expr_dir']
+            self.face_asym_dir = config['cdx_face_config']['face_asym_dir']
+            self.AU_fl = config['cdx_face_config']['AU_filters']
+            self.au_int = config['cdx_face_config']['au_intensity']
+            self.au_prs = config['cdx_face_config']['au_presence']
+        
+    def get_action_unit(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.ACTION_UNITS
+
+    def get_neg_action_unit(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.NEG_ACTION_UNITS
+
+    def get_pos_action_unit(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.POS_ACTION_UNITS
+
+    def get_net_action_unit(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.NET_ACTION_UNITS
+
+    def get_selected_feature(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.SELECTED_FEATURES
+
+    def get_happiness(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.happiness
+
+    def get_sadness(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.sadness
+
+    def get_surprise(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.surprise
+
+    def get_fear(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.fear
+
+    def get_anger(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.anger
+
+    def get_disgust(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.disgust
+
+    def get_contempt(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.contempt
+
+    def get_cai(self):
+        """Summary
+        Returns:
+            TYPE: end point
+        """
+        return self.cai

opendbm/dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py

@@ -0,0 +1,85 @@
+"""
+file_name: process_emotion_expressivity
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import datetime
+import glob
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
+from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+face_expr_dir = 'facial/face_expressivity'
+csv_ext = '_facemo.csv'
+
+#Openface feature extraction
+def of_feature(df_of, cfr, f_cfg):
+    """
+    Creating dataframe for face expressivity
+    Args:
+        of: open face attributes
+    Returns:
+        (list) list of expressivity score for emotions
+    """
+    df_list = []
+    df_of['s_confidence'] = vu.smooth(df_of[' confidence'].values, window='flat').tolist()
+    
+    if 'AU' in cfr.SELECTED_FEATURES :
+        vu.calc_of_for_video(df_of, cfr, f_cfg)
+    #Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1)
+    
+    if len(df_of[f_cfg.neg_exp])>0:
+        df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp]/5
+        
+    if len(df_of[f_cfg.neg_exp_full])>0:
+        df_of[f_cfg.neg_exp_full] = df_of[f_cfg.neg_exp_full]/5
+        
+    if len(df_of[f_cfg.com_exp])>0:
+        df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp]/7
+        
+    if len(df_of[f_cfg.com_exp_full])>0:
+        df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full]/7
+        
+    df_list.append(df_of)
+    return df_list 
+
+
+def run_face_expressivity(video_uri, out_dir, f_cfg):
+    """
+    Processing all patient's for fetching facial landmarks
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; f_cfg: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        #Baseline logic
+        cfr = ConfigFaceReader()
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
+        if len(of_csv_path)>0:
+
+            df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
+            df_of = df_of[cfr.AU_fl]
+            expr_df_list = of_feature(df_of, cfr, f_cfg)
+
+            exp_final_df = pd.concat(expr_df_list, ignore_index=True)
+            exp_final_df['dbm_master_url'] = video_uri
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext)
+            
+    except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/video/face_landmark.py

@@ -0,0 +1,121 @@
+"""
+file_name: face_landmark
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import datetime
+import glob
+from os.path import join
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
+from opendbm.dbm_lib.dbm_features.raw_features.util import video_util as vu, util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+face_lmk_dir = 'facial/face_landmark'
+csv_ext = '_faclmk.csv'
+
+def extract_col_nm_lmk(cols):
+    """
+    Extract landmark column names from openface output (csv)
+    Args:
+        cols: column names from open face output (csv)
+    Returns:
+        (list) list of landmark column names
+    """
+    cols_lmk = []
+    lmk_tags = [' y_', ' x_', ' X_', ' Y_', ' Z_']
+    for c in cols:
+        if any(t in c for t in lmk_tags):
+            cols_lmk.append(c)
+    return cols_lmk
+
+
+def lmk_col_nm_map(df):
+    """
+    Rename dataframe landmark column names to match functional specifications v1.0
+    Args:
+        df: dataframe
+    """
+    dict_lmk_cols = {}
+    for col in list(df):
+        idx = col.rfind('_')+1
+        if idx > 0:
+            lmk_id = col[idx:] if len(col[idx:])>1 else '0'+col[idx:]
+            if ' y_' in col:
+                dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'r'
+            if ' x_' in col:
+                dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'c'
+            if ' X_' in col:
+                dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'X'
+            if ' Y_' in col:
+                dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Y'
+            if ' Z_' in col:
+                dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Z'
+    df.rename(columns=dict_lmk_cols, inplace=True)
+    return df
+
+
+def add_disp_3D(df):
+    """
+    Add 3D displacement for each landmark
+    Args:
+        df: landmark dataframe
+    """
+    df = df.sort_values(by=['frame'], ascending=False)
+    cols_lmk = [col for col in list(df) if 'fac_LMK' in col]
+    df_t = df[cols_lmk]
+    df_diff = df_t.diff()
+    df_diff = df_diff.pow(2)
+
+    tot_lmk = 68 # 68 landmark model
+    for i in range(tot_lmk):
+        lmk_id = '{:02d}'.format(i)
+        df['fac_LMK'+lmk_id+'disp'] = df_diff[['fac_LMK'+lmk_id+'X', 'fac_LMK'+lmk_id+'Y', 'fac_LMK'+lmk_id+'Z']].sum(axis=1).apply(np.sqrt)
+
+    return df
+
+
+def run_face_landmark(video_uri, out_dir, f_cfg):
+    """
+    Processing all patient's for fetching facial landmarks
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path;  f_cfg: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        #Baseline logic
+        cfr = ConfigFaceReader()
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_openface/*.csv'))
+        if len(of_csv_path)>0:
+
+            df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
+            df_lmk = df_of[extract_col_nm_lmk(df_of)]
+            df_lmk = df_lmk.copy()
+
+            df_lmk['frame'] = df_of['frame']
+            df_lmk['face_id'] = df_of[' face_id']
+            df_lmk['timestamp'] = df_of[' timestamp']
+            df_lmk['confidence'] = df_of[' confidence']
+            df_lmk['success'] = df_of[' success']
+
+            df_lmk = lmk_col_nm_map(df_lmk)
+            df_lmk = add_disp_3D(df_lmk)
+            df_lmk['dbm_master_url'] = video_uri
+
+            logger.info('Processing Output file {} '.format(join(out_loc, fl_name)))
+            ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)
+    
+    except Exception as e:
+        logger.error('Failed to process video file')

opendbm/dbm_lib/dbm_features/raw_features/video/open_face_process.py

@@ -0,0 +1,80 @@
+"""
+file_name: process_features
+project_name: DBM
+created: 2020-20-07
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import glob
+import logging
+
+from opendbm.dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False):
+    """ Computes open_face features for the files in filepaths
+
+    Args:
+    -----
+        filepaths: (itreable[str])
+        video_tracking: To specify whether openface's video tracking module (FaceLandmarkVid)
+                    is being used or the default (FeatureExtract)
+        video_url: Raw video location on S3 bucket
+        input_dir: Path to the input videos
+        out_dir: Path to the processed output
+        of_path: OpenFace source code path
+
+    Returns:
+    --------
+        (itreable[str]) list of .csv files
+    """    
+    if video_tracking:
+        suffix = '_openface_lmk'
+    else:
+        suffix = '_openface'
+
+    csv_files = []
+    
+    for fp in filepaths:
+        try:
+
+            _, out_loc, fl_name = ut.filter_path(video_url, out_dir)
+            full_f_name = fl_name + suffix
+            output_directory = os.path.join(out_loc, full_f_name)
+            
+            if video_tracking and not os.path.exists(os.path.abspath(output_directory)):
+                os.makedirs(os.path.abspath(output_directory))
+            csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
+
+        except Exception as e:
+            logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
+
+    return csv_files
+
+def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking):
+    """
+    Processing all patient's for fetching emotion expressivity
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; input_dir : input directory for video's; dbm_group: feature group
+        out_dir: (str) Output directory for processed output; of_path: OpenFace source code path
+
+    """
+    try:
+        
+        if dbm_group != None:
+            check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation
+            check_val = bool(len({*check_group} & {*dbm_group}))
+            if not check_val:
+                return
+        
+        filepaths = [video_uri]
+        csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking)
+
+    except Exception as e:
+        logger.error('Failed to process video file')