added pain AUs

dbm_lib/config/config_raw_feature.py

@@ -229,3 +229,51 @@ class ConfigRawReader(object):
             self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw']
             self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll']
             self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist']
+
+            self.mov_freq_trem_freq = config['raw_feature']['mov_freq_trem_freq']
+            self.mov_freq_trem_index = config['raw_feature']['mov_freq_trem_index']
+            self.mov_freq_trem_pindex = config['raw_feature']['mov_freq_trem_pindex']
+            self.mov_amp_trem_freq = config['raw_feature']['mov_amp_trem_freq']
+            self.mov_amp_trem_index = config['raw_feature']['mov_amp_trem_index']
+            self.mov_amp_trem_pindex = config['raw_feature']['mov_amp_trem_pindex']
+
+            self.fac_tremor_median_5 = config['raw_feature']['fac_tremor_median_5']
+            self.fac_tremor_median_12 = config['raw_feature']['fac_tremor_median_12']
+            self.fac_tremor_median_8 = config['raw_feature']['fac_tremor_median_8']
+            self.fac_tremor_median_48 = config['raw_feature']['fac_tremor_median_48']
+            self.fac_tremor_median_54 = config['raw_feature']['fac_tremor_median_54']
+            self.fac_tremor_median_28 = config['raw_feature']['fac_tremor_median_28']
+            self.fac_tremor_median_51 = config['raw_feature']['fac_tremor_median_51']
+            self.fac_tremor_median_66 = config['raw_feature']['fac_tremor_median_66']
+            self.fac_tremor_median_57 = config['raw_feature']['fac_tremor_median_57']
+
+            self.mov_leye_x = config['raw_feature']['mov_leye_x']
+            self.mov_leye_y = config['raw_feature']['mov_leye_y']
+            self.mov_leye_z = config['raw_feature']['mov_leye_z']
+            self.mov_reye_x = config['raw_feature']['mov_reye_x']
+            self.mov_reye_y = config['raw_feature']['mov_reye_y']
+            self.mov_reye_z = config['raw_feature']['mov_reye_z']
+            self.mov_eleft_disp = config['raw_feature']['mov_eleft_disp']
+            self.mov_eright_disp = config['raw_feature']['mov_eright_disp']
+
+            #NLP features
+            self.nlp_transcribe = config['raw_feature']['nlp_transcribe']
+            self.nlp_numSentences = config['raw_feature']['nlp_numSentences']
+            self.nlp_singPronPerAns = config['raw_feature']['nlp_singPronPerAns']
+            self.nlp_singPronPerSen = config['raw_feature']['nlp_singPronPerSen']
+            self.nlp_pastTensePerAns = config['raw_feature']['nlp_pastTensePerAns']
+            self.nlp_pastTensePerSen = config['raw_feature']['nlp_pastTensePerSen']
+            self.nlp_pronounsPerAns = config['raw_feature']['nlp_pronounsPerAns']
+            self.nlp_pronounsPerSen = config['raw_feature']['nlp_pronounsPerSen']
+            self.nlp_verbsPerAns = config['raw_feature']['nlp_verbsPerAns']
+            self.nlp_verbsPerSen = config['raw_feature']['nlp_verbsPerSen']
+            self.nlp_adjectivesPerAns = config['raw_feature']['nlp_adjectivesPerAns']
+            self.nlp_adjectivesPerSen = config['raw_feature']['nlp_adjectivesPerSen']
+            self.nlp_nounsPerAns = config['raw_feature']['nlp_nounsPerAns']
+            self.nlp_nounsPerSen = config['raw_feature']['nlp_nounsPerSen']
+            self.nlp_sentiment_mean = config['raw_feature']['nlp_sentiment_mean']
+            self.nlp_mattr = config['raw_feature']['nlp_mattr']
+            self.nlp_wordsPerMin = config['raw_feature']['nlp_wordsPerMin']
+            self.nlp_totalTime = config['raw_feature']['nlp_totalTime']
+         
+

dbm_lib/controller/process_feature.py

@@ -7,7 +7,9 @@ created: 2020-20-07
 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq
 from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc
 from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark
-from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink
+from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze, voice_tremor, facial_tremor
+from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features
+
 
 import subprocess
 import logging
@@ -19,13 +21,13 @@ logging.basicConfig(level=logging.INFO)
 logger=logging.getLogger()
 
 def audio_to_wav(input_filepath):
-    """ Extracts a video's audio file and saves it to wav 
+    """ Extracts a video's audio file and saves it to wav
     Args:
-        input_filepath: (str)  
+        input_filepath: (str)
     Returns:
     """
     try:
-        
+
         fname, _ = splitext(input_filepath)
         output_filepath = fname + '.wav'
 
@@ -37,7 +39,7 @@ def audio_to_wav(input_filepath):
             logger.info('wav output saved in {}'.format(output_filepath))
         else:
             logger.info('Output file {} already exists'.format(output_filepath))
-        
+
     except Exception as e:
         logger.error('Failed to extract audio from Video')
 
@@ -50,38 +52,38 @@ def process_acoustic(video_uri, out_dir, dbm_group, r_config):
     """
     if dbm_group != None and len(dbm_group)>0 and 'acoustic' not in dbm_group:
         return
-    
+
     logger.info('Processing acoustic variables from data in {}'.format(video_uri))
     logger.info('processing audio intensity....')
     intensity.run_intensity(video_uri, out_dir, r_config)
-    
+
     logger.info('processing audio pitch freq....')
     pitch_freq.run_pitch(video_uri, out_dir, r_config)
-    
+
     logger.info('processing HNR....')
     hnr.run_hnr(video_uri, out_dir, r_config)
-    
+
     logger.info('processing GNE....')
     gne.run_gne(video_uri, out_dir, r_config)
-    
+
     logger.info('processing voice frame score....')
     voice_frame_score.run_vfs(video_uri, out_dir, r_config)
-    
+
     logger.info('processing formant frequency....')
     formant_freq.run_formant(video_uri, out_dir, r_config)
-    
+
     logger.info('processing pause segment....')
     pause_segment.run_pause_segment(video_uri, out_dir, r_config)
-    
+
     logger.info('processing jitter....')
     jitter.run_jitter(video_uri, out_dir, r_config)
-    
+
     logger.info('processing shimmer....')
     shimmer.run_shimmer(video_uri, out_dir, r_config)
-    
+
     logger.info('processing mfcc....')
     mfcc.run_mfcc(video_uri, out_dir, r_config)
-    
+
 def process_facial(video_uri, out_dir, dbm_group, r_config):
     """
     processing facial features
@@ -91,20 +93,20 @@ def process_facial(video_uri, out_dir, dbm_group, r_config):
     """
     if dbm_group != None and len(dbm_group)>0 and 'facial' not in dbm_group:
         return
-    
+
     logger.info('Processing facial variables from data in {}'.format(video_uri))
     logger.info('processing facial asymmetry....')
     face_asymmetry.run_face_asymmetry(video_uri, out_dir, r_config)
-    
+
     logger.info('processing facial Action Unit....')
     face_au.run_face_au(video_uri, out_dir, r_config)
-    
+
     logger.info('processing facial expressivity....')
     face_emotion_expressivity.run_face_expressivity(video_uri, out_dir, r_config)
-    
+
     logger.info('processing facial landmark....')
     face_landmark.run_face_landmark(video_uri, out_dir, r_config)
-    
+
 def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
     """
     processing facial features
@@ -115,23 +117,48 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
     """
     if dbm_group != None and len(dbm_group)>0 and 'movement' not in dbm_group:
         return
-    
+
     logger.info('Processing movement variables from data in {}'.format(video_uri))
+
     logger.info('processing head movement....')
     head_motion.run_head_movement(video_uri, out_dir, r_config)
-    
+   
     logger.info('processing eye blink....')
     eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model)
+
+    logger.info('processing eye gaze....')
+    eye_gaze.run_eye_gaze(video_uri, out_dir, r_config)
     
+    logger.info('processing voice tremor....')
+    voice_tremor.run_vtremor(video_uri, out_dir, r_config)
+
+    logger.info('processing facial tremor....')
+    facial_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True)
+
+    
+def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path):
+    """
+    processing nlp features
+    Args:
+        video_uri: video path; out_dir: raw variable output dir
+        dbm_group: list of features to process; r_config: raw feature config object
+        deep_path: deep speech build path
+    """
+    if dbm_group != None and len(dbm_group)>0 and 'speech' not in dbm_group:
+        return
+    
+    logger.info('Processing nlp variables from data in {}'.format(video_uri))
+    transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path)
+    speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog)
+    
+
 def remove_file(file_path):
     """
-    removing wav file 
+    removing wav file
     """
     file_dir = dirname(file_path)
     file_name, _ = splitext(basename(file_path))
     wav_file = glob.glob(join(file_dir, file_name + '.wav'))
-    
+
     if len(wav_file)> 0:
         os.remove(wav_file[0])
-    
-    

dbm_lib/dbm_features/raw_features/audio/formant_freq.py

@@ -31,7 +31,11 @@ def formant_list(formant,snd):
     Returns:
         List of first through fourth formant for each frame
     """
-    f1_list, f2_list, f3_list, f4_list = ([], ) * 4
+    f1_list = []
+    f2_list = []
+    f3_list = []
+    f4_list = []
+    
     dur = snd.duration-0.02
     dur_round = round(dur, 2)
     
@@ -126,4 +130,4 @@ def run_formant(video_uri, out_dir, r_config):
 
             calc_formant(video_uri, audio_file, out_loc, fl_name, r_config)
     except Exception as e:
-        logger.error('Failed to process audio file')
+        logger.error('Failed to process audio file')

dbm_lib/dbm_features/raw_features/movement/__init__.py

@@ -11,3 +11,7 @@ from __future__ import print_function
 import os
 
 DBMLIB_PATH = os.path.dirname(__file__)
+DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH,
+                                                  '../../../../resources/libraries/voice_tremor.praat'))
+DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json'))
+

dbm_lib/dbm_features/raw_features/movement/eye_gaze.py

@@ -0,0 +1,148 @@
+"""
+file_name: eye_gaze
+project_name: DBM
+created: 2020-30-11
+"""
+
+import os
+import glob
+import pandas as pd
+import numpy as np
+from scipy.spatial import distance
+from os.path import join
+import logging
+
+from dbm_lib.dbm_features.raw_features.util import util as ut
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+eye_pose_dir = 'movement/gaze'
+eye_pose_ext = '_eyegaze.csv'
+
+def eye_motion_df(l_disp, r_disp, error_list, r_config):
+    """
+    Generating eye movement dataframe
+
+    Args:
+        l_disp: displacement list(left eye); l_disp: displacement list(right eye)
+        r_config: raw variable config file object
+
+    Reutrns:
+        Final eye displacement dataframe
+    """
+    df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp])
+    df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp])
+    
+    df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False)
+    df_eye_motion[r_config.err_reason] = error_list
+    return df_eye_motion
+
+def filter_motion(df_of, df_disp, col_l, col_r, r_config):
+    """
+    Filtering final eye movement dataframe
+    
+    Args:
+        df_of: Openface raw out dataframe; col_r: right eye column
+        col_l: left eye column; r_config: raw variable config file object
+    """
+    
+    df_of = df_of[col_l + col_r + [' confidence']]
+    df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan
+    
+    df_filter =  df_of[col_l + col_r]
+    df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z,
+                      r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z]
+    
+    df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False)
+    return df_motion
+
+def eye_disp(of_results, col, r_config):
+    """
+    Computing head velocity frame by frame
+
+    Args:
+        of_results: Openface raw out dataframe
+        r_config: Face config file object
+
+    Reutrns:
+        Final head velocity frame by frame output
+    """
+    distance_list = []
+    error_list = []
+    
+    of_results = of_results[col+ [' confidence']]
+    for index, row in of_results.iterrows():
+        dst = np.nan
+        
+        if index == 0 or float(row[' confidence']) < 0.8:  #Threshold < 0.8
+            distance_list.append(dst)
+            
+            if float(row[' confidence']) < 0.8:
+                error_list.append('confidence less than 80%')
+                
+            else:
+                error_list.append('Pass')
+            continue
+            
+        if index > 0:
+            
+            point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1])
+            point_y = (row[col[0]],row[col[1]],row[col[2]])
+            try:
+                dst = distance.euclidean(point_x, point_y)
+            except:
+                pass
+            
+            distance_list.append(abs(dst))
+            error_list.append('Pass')
+            
+    return distance_list, error_list
+        
+def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config):
+    """
+    Computing eye motion variables
+    Args:
+        df_of: Openface dataframe
+        out_loc: Output path for saving output csv's
+        fl_name: file name for output csv
+        r_config: raw variable config file object
+        
+    """
+    
+    col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z'] 
+    col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z']
+    
+    gazel_disp, err_l = eye_disp(df_of, col_l, r_config)
+    gazer_disp, err_r = eye_disp(df_of, col_r, r_config)
+    
+    df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config)
+    df_disp['dbm_master_url'] = video_uri
+    
+    df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config)
+    ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext)
+    
+def run_eye_gaze(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for getting eye movement artifacts 
+    --------------------------------
+    --------------------------------
+    Args:
+        video_uri: video path; input_dir : input directory for video's
+        out_dir: (str) Output directory for processed output; r_config: raw variable config object
+    """
+    try:
+        
+        #filtering path to generate input & output path
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
+
+        if len(of_csv_path)>0:
+
+            of_csv = of_csv_path[0]
+            df_of = pd.read_csv(of_csv, error_bad_lines=False)
+
+            logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+            calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config)
+    except Exception as e:
+        logger.error('Failed to process video file')

dbm_lib/dbm_features/raw_features/movement/facial_tremor.py

@@ -0,0 +1,164 @@
+import sys, os, glob, cv2, re
+import pickle, json
+import pandas as pd
+import numpy as np
+import numpy.ma as ma
+import logging
+from os.path import join
+
+from dbm_lib.dbm_features.raw_features.util import util as ut
+from dbm_lib.dbm_features.raw_features.util.math_util import *
+
+from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+ft_dir = 'movement/facial_tremor'
+csv_ext = '_fac_tremor.csv'
+model_ext = '_fac_model.csv'
+fac_features_ext = '_fac_features.csv'
+
+def compute_features(out_dir, df_of, r_config):
+    """ Computes features
+
+    Returns: features in vector format
+    """
+    config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read())
+
+    pattern_x = re.compile("l\d+_x")
+    pattern_y = re.compile("l\d+_y")
+
+    # assumption: distance of face to camera remains at roughly static
+
+    # logic break
+    landmark_columns = []
+    for col in df_of.columns:
+        if pattern_x.match(col) or pattern_y.match(col):
+            landmark_columns.append(col)
+
+    df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)]
+    df_of.reset_index(inplace=True)
+
+    num_frames = len(df_of)
+    logger.info("Number of frames to be processed: {}".format(str(num_frames)))
+    landmarks = config['landmarks']
+
+    try:
+        if num_frames == 0:
+            error_reason = "No frames with visible face."
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config, error_reason)
+
+#             if num_frames < 60:
+#                 error_reason = 'Number of frames with visible face < 60. Video too short'
+#                 logger.error(error_reason)
+#                 return empty_frame(landmarks, f_cfg, error_reason)
+
+        first_row = df_of.iloc[0]
+
+        facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']])
+        faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']])
+
+        if facew == 0 or faceh == 0:
+            error_reason = 'face width or height = 0. Check landmark values'
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config)
+
+        fac_disp = calc_displacement_vec(df_of, landmarks, num_frames)
+
+        # if verbose:
+            # logger.info("Displacement output: {}".format(str(fac_disp)))
+
+        fac_disp_median = np.median(fac_disp, axis = 1)
+        fac_disp_mean = np.mean(fac_disp, axis = 1)
+
+        if len(fac_disp.shape)!=2:
+            error_reason = 'fac_disp is not 2D. smth went wrong with disp calc'
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config, error_reason)
+
+        if len(fac_disp[0])<=1:
+            error_reason = 'Video too short. smth went wrong with disp calc'
+            logger.error(error_reason)
+            return empty_frame(landmarks, r_config, error_reason)
+
+        fac_corr_mat = np.corrcoef(fac_disp, rowvar = True)
+        # extract relevant row from cov matrix
+        ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk]
+        fac_corr = fac_corr_mat[ref_lmk_index][0]
+
+        fac_area = config['ref_area'] / (facew * faceh)
+
+        # if verbose:
+        #     logger.info("Face area: {}".format(fac_area))
+        #     logger.info("Face Displacement Median: {}".format(str(fac_disp_median)))
+        #     logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean)))
+
+        fac_features1 =  np.multiply(fac_area * fac_disp_median, (1. - fac_corr))
+        fac_features2 =  np.multiply(fac_area * fac_disp_mean, (1. - fac_corr))
+
+#         base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr))
+
+        fac_features_dict = {}
+        for i, landmark in enumerate(landmarks):
+            fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]]
+            raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
+            fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]]
+
+            fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]]
+            fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]]
+
+        fac_features_dict[r_config.err_reason] = ['']
+        data = pd.DataFrame.from_dict(fac_features_dict)
+        logger.info('Concluded computing tremor features')
+
+        return data
+
+    except Exception as e:
+        logger.error('Error computing tremor features: {}'.format(str(e)))
+        return empty_frame(landmarks, r_config, str(e))
+
+def empty_frame(landmarks, r_config, error_reason):
+    fac_features_dict = {}
+    for i, landmark in enumerate(landmarks):
+        raw_variable_map = 'fac_tremor_median_{}'.format(landmark)
+        fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan]
+
+        fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan]
+        fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan]
+        fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan]
+
+    fac_features_dict[r_config.err_reason] = [error_reason]
+    empty_frame = pd.DataFrame.from_dict(fac_features_dict)
+    return empty_frame
+
+def fac_tremor_process(video_uri, out_dir, r_config, model_output=False):
+    """
+    processing input videos
+    
+    
+    """
+#     try:
+        
+    input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+    of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv'))
+
+    if len(of_csv_path)>0:
+        of_csv = of_csv_path[0]
+        df_of = pd.read_csv(of_csv, error_bad_lines=False)
+
+        logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+
+        feats = compute_features(of_csv_path , df_of, r_config)
+        
+#         if model_output:
+#             result = score(feats, r_config)
+#             feats = pd.concat([feats, result], axis=1)
+            
+        ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext)
+
+
+
+#     except Exception as e:
+        logger.error('Failed to process video file')

dbm_lib/dbm_features/raw_features/movement/head_motion.py

@@ -179,11 +179,12 @@ def run_head_movement(video_uri, out_dir, r_config):
         out_dir: (str) Output directory for processed output; r_config: raw variable config object
     """
     try:
-        
+
         #filtering path to generate input & output path
         input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
         of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
 
+
         if len(of_csv_path)>0:
 
             of_csv = of_csv_path[0]
@@ -192,4 +193,4 @@ def run_head_movement(video_uri, out_dir, r_config):
             logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
             calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config)
     except Exception as e:
-        logger.error('Failed to process video file')
+        logger.error('Failed to process video file')

dbm_lib/dbm_features/raw_features/movement/voice_tremor.py

@@ -0,0 +1,94 @@
+import pandas as pd
+import os
+import glob
+from os.path import join
+import parselmouth
+from parselmouth.praat import call, run_file
+import numpy as np
+import librosa
+import json
+import re
+import logging
+
+from dbm_lib.dbm_features.raw_features.util import util as ut
+from dbm_lib.dbm_features.raw_features.movement import DBMLIB_VTREMOR_LIB
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+vt_dir = 'movement/voice_tremor'
+csv_ext = '_vtremor.csv'
+
+#Executing praat script using parselmouth function
+def tremor_praat(snd_file,r_cfg):
+    """
+        Generating Voice tremor endpoint dataframe
+        Args:
+            snd_file: (.wav) parsed audio file
+            r_cfg: Raw variable configuration file
+        Returns tremor endpoint dataframe
+    """
+    snd = parselmouth.Sound(snd_file)
+    tremor_var = run_file(snd,DBMLIB_VTREMOR_LIB,capture_output=True)
+    new_tremor_var = re.sub('--undefined--', '0', tremor_var[1])
+    res = json.loads(new_tremor_var)
+    tremor_df = pd.DataFrame(res,index=['0',])
+    tremor_df.columns = [r_cfg.mov_freq_trem_freq,r_cfg.mov_amp_trem_freq,r_cfg.mov_freq_trem_index,
+                         r_cfg.mov_amp_trem_index,r_cfg.mov_freq_trem_pindex,r_cfg.mov_amp_trem_pindex]
+    return tremor_df
+
+def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name):
+    """
+        Preparing voice tremor matrix
+        Args:
+            audio_file: (.wav) parsed audio file ; r_config: raw config object
+            out_loc: (str) Output directory for csv ; fl_name: file name
+    """
+    df_tremor = tremor_praat(audio_file, r_config)
+    df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+
+    logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
+    ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
+
+def prepare_empty_vt(out_loc, fl_name, r_config, error_txt):
+
+    """
+    Preparing empty voice tremor matrix
+    """
+    cols = [r_config.mov_freq_trem_freq, r_config.mov_amp_trem_freq, r_config.mov_freq_trem_index,
+            r_config.mov_amp_trem_index, r_config.mov_freq_trem_pindex, r_config.mov_amp_trem_pindex, r_config.err_reason]
+
+    out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
+    df_tremor = pd.DataFrame(out_val, columns = cols)
+
+    logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name)))
+    ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext)
+
+def run_vtremor(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for fetching Formant freq
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.5:
+                logger.info('Output file {} size is less than 0.5sec'.format(audio_file))
+                
+                error_txt = 'error: length less than 0.5 sec'
+                prepare_empty_vt(video_uri, out_loc, fl_name, error_txt)
+                return
+            prepare_vtrem_output(audio_file, out_loc, r_config, fl_name)
+    except Exception as e:
+        logger.error('Failed to compute Voice Tremor {} for {}'.format(e,video_uri))
+        prepare_empty_vt(out_loc, fl_name, r_config, e)

dbm_lib/dbm_features/raw_features/nlp/speech_features.py

@@ -0,0 +1,51 @@
+"""
+file_name: speech_features
+project_name: DBM
+created: 2020-13-11
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import glob
+from os.path import join
+import logging
+import shutil
+
+from dbm_lib.dbm_features.raw_features.util import util as ut
+from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+speech_dir = 'nlp/speech_feature'
+speech_ext = '_nlp.csv'
+transcribe_ext = 'nlp/transcribe/*_transcribe.csv'
+
+def run_speech_feature(video_uri, out_dir, r_config, tran_tog):
+    """
+    Processing all patient's for fetching nlp features
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        transcribe_path = glob.glob(join(out_loc, transcribe_ext))
+        if len(transcribe_path)>0:
+
+            transcribe_df = pd.read_csv(transcribe_path[0])
+            df_speech= n_util.process_speech(transcribe_df, r_config)
+
+            logger.info('Saving Output file {} '.format(out_loc))
+            ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext)
+
+            if (tran_tog == None) or (tran_tog != 'on'):
+                shutil.rmtree(os.path.dirname(transcribe_path[0]))
+            
+    except Exception as e:
+        logger.error('Failed to process video file')

dbm_lib/dbm_features/raw_features/nlp/transcribe.py

@@ -0,0 +1,84 @@
+"""
+file_name: transcribe
+project_name: DBM
+created: 2020-10-11
+"""
+
+import pandas as pd
+import numpy as np
+import librosa
+import glob
+from os.path import join
+import logging
+
+from dbm_lib.dbm_features.raw_features.util import util as ut
+from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+formant_dir = 'nlp/transcribe'
+csv_ext = '_transcribe.csv'
+error_txt = 'error: length less than 0.1'
+
+def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur):
+    """
+    Preparing Formant freq matrix
+    Args:
+        audio_file: (.wav) parsed audio file; fl_name: input file name
+        out_loc: (str) Output directory; r_config: raw variable config
+    """
+    
+    text = n_util.process_deepspeech(audio_file, deep_path)
+    df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe])
+    
+    df_formant.replace('', np.nan, regex=True,inplace=True)
+    df_formant[r_config.nlp_totalTime] = aud_dur
+    df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
+    df_formant['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
+    
+def empty_transcribe(video_uri, out_loc, fl_name, r_config):
+    
+    """
+    Preparing empty formant frequency matrix if something fails
+    """
+    cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
+    df_fm = pd.DataFrame(out_val, columns = cols)
+    df_fm['dbm_master_url'] = video_uri
+    
+    logger.info('Saving Output file {} '.format(out_loc))
+    ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
+
+def run_transcribe(video_uri, out_dir, r_config, deep_path):
+    
+    """
+    Processing all patient's for fetching Formant freq
+    ---------------
+    ---------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output; deep_path: deepspeech build path
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+        aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
+        if len(aud_filter)>0:
+
+            audio_file = aud_filter[0]
+            aud_dur = librosa.get_duration(filename=audio_file)
+
+            if float(aud_dur) < 0.1:
+                logger.info('Output file {} size is less than 0.1 sec'.format(audio_file))
+
+                empty_transcribe(video_uri, out_loc, fl_name, r_config)
+                return
+
+            calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur)
+    except Exception as e:
+        logger.error('Failed to process audio file')
+        

dbm_lib/dbm_features/raw_features/util/math_util.py

@@ -0,0 +1,57 @@
+"""
+file_name: facial_tremor
+project_name: cdx_analysis
+created: 2019-03-16
+author: Deshana Desai
+"""
+import sys, os, glob, cv2
+import pandas as pd
+import numpy as np
+
+
+def euclidean_distance(point1, point2):
+    """
+    Compute euclidean distance between points
+    """
+
+    return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)
+
+
+# def detect_peaks()
+
+
+def expand_landmarks(landmarks):
+    """
+    util method to expand landmark list:
+    eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']]
+    """
+    return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks]
+
+
+
+def calc_displacement_vec(df, landmarks, num_frames):
+    """
+    Calculates displacement vector frame by frame
+    """
+
+    landmarks = expand_landmarks(landmarks)
+
+    disp_vec = np.zeros((len(landmarks), num_frames))
+    prev_point = np.zeros((len(landmarks), 2))
+
+    # initialize
+    for j, pair in enumerate(landmarks):
+        first_row = df.iloc[0]
+        prev_point[j] = (first_row[pair[0]], first_row[pair[1]])
+
+
+    for i in range(num_frames):
+        frame_row = df.iloc[i]
+        for j, pair in enumerate(landmarks):
+            x, y = pair[0], pair[1]
+            current = (frame_row[x], frame_row[y])
+            deviation = euclidean_distance( current, prev_point[j])
+            disp_vec[j][i] = deviation
+            prev_point[j] = current
+
+    return disp_vec

dbm_lib/dbm_features/raw_features/util/nlp_util.py

@@ -0,0 +1,212 @@
+"""
+file_name: nlp_util
+project_name: DBM
+created: 2020-10-11
+"""
+
+import subprocess
+import json
+import numpy as np
+import pandas as pd
+import os
+import logging
+
+import nltk
+import re
+from lexicalrichness import LexicalRichness
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer 
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+#Speech to text using Deepspeech 0.9.1
+def deepspeech(AUDIO_FILE,deep_path):
+    """
+        Extracting text from audio using Deep Speech neural network trained model
+        Returns:
+            Text: text which is extracted from audio
+    """
+    api = 'deepspeech'
+    arg_speech0 = '--model'
+    arg_speech_path0 = os.path.join(deep_path, 'deepspeech-0.9.1-models.pbmm')
+    arg_speech1 = '--scorer'
+    arg_speech_path1 = os.path.join(deep_path, 'deepspeech-0.9.1-models.scorer')
+    arg_audio = "--audio"
+    
+    out = subprocess.Popen([api, arg_speech0, arg_speech_path0, arg_speech1, arg_speech_path1, arg_audio, AUDIO_FILE],
+                           stdout=subprocess.PIPE, 
+                           stderr=subprocess.STDOUT)
+    logger.info('Deepspeech output...... {}'.format(out))
+    try:
+        stdout,stderr = out.communicate()
+    except:
+        return "error", "error"
+    print(stderr)
+    return stdout,stderr
+
+def deep_speech_output_clean(result):
+    """
+        Parsing deep speech output(text)
+        Return: 
+            Text from speech
+    """
+    text = ""
+    if len(result)>0:
+        res_split = str(result[0]).split('\\n')
+        
+        if len(res_split)>0:
+            for i in range(len(res_split)):
+                if 'Inference took' in res_split[i]:
+                    text = res_split[i + 1]
+                    return text
+    return text
+
+def process_deepspeech(audio_file,deep_path):
+    """
+    Transcribing audio to extract text from speech
+    """
+    deep_output = deepspeech(audio_file,deep_path)
+    deep_text= deep_speech_output_clean(deep_output)
+    
+    return deep_text
+
+def nltk_download():
+    
+    try:
+        nltk.data.find('tokenizers/punkt')
+        
+    except LookupError:
+        logger.info('punkt is not available')
+        nltk.download('punkt')
+        
+    try:
+        nltk.data.find('averaged_perceptron_tagger')
+        
+    except LookupError:
+        logger.info('averaged_perceptron_tagger is not available')
+        nltk.download('averaged_perceptron_tagger')
+
+def empty_speech(r_config, master_url, error_txt):
+    """
+    Preparing empty speech matrix with error
+    Args:
+        r_config: raw config file object
+        error_txt: Error message during transcription
+        
+    Returns:
+            Empty dataframe for speech features with error
+    """
+    
+    col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns,
+          r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns,
+          r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns,
+          r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin,
+          r_config.nlp_totalTime, r_config.err_reason]
+    
+    df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col)
+    df_speech['dbm_master_url'] = master_url
+    
+    return df_speech
+
+def divide_var(speech_var1, spech_var2):
+    """
+    divide variables
+    """
+    speech_var = np.nan
+    if spech_var2!=0:
+        speech_var = speech_var1/spech_var2
+    return speech_var
+
+def process_speech(transcribe_df,r_config):
+    """
+        Preparing speech features
+        Args:
+            transcribe_df: Transcribed dataframe
+            r_config: raw config file object
+        Returns:
+            Dataframe for speech features
+    """
+    transcribe_df = transcribe_df.replace(np.nan, '', regex=True)
+    err_transcribe = transcribe_df[r_config.err_reason].iloc[0]
+    transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0]
+    total_time = transcribe_df[r_config.nlp_totalTime].iloc[0]
+    master_url = transcribe_df['dbm_master_url'].iloc[0]
+    
+    #clean transcribe
+    transcribe = transcribe.replace(",", "")
+    transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe))
+    
+    if err_transcribe != 'Pass':
+        df_speech = empty_speech(r_config, master_url, error_txt)
+        
+        return df_speech
+    
+    speech_dict = {}
+    nltk_download()
+    
+    sentences = nltk.tokenize.sent_tokenize(transcribe)
+    words_all = nltk.tokenize.word_tokenize(transcribe)
+    num_sentences = len(sentences)
+    
+    speech_dict[r_config.nlp_numSentences] = num_sentences
+    
+    #nlp_singPron
+    i_s = transcribe.count('I')
+    me_s = transcribe.count('me')
+    my_s = transcribe.count('my')
+    sing_count = i_s + me_s + my_s
+    
+    speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences)
+    
+    tagged = nltk.pos_tag(transcribe.split())
+    tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag'])
+    
+    #Past tense per answer
+    all_POSs = tagged_df['pos_tag'].tolist()
+    speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences)
+    
+    #Pronoun per answer
+    pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$')
+    speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences)
+    
+    #Verb per answer
+    verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \
+                      + all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ')
+    speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences)
+    
+    #Adjective per answer
+    adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS')
+    speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences)
+    
+    #Noun per answer
+    nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS')
+    speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences)
+    
+    #Sentiment analysis
+    vader = SentimentIntensityAnalyzer()
+    sentence_valences = []
+    
+    for s in sentences:
+        sentiment_dict = vader.polarity_scores(s) 
+        sentence_valences.append(sentiment_dict['compound'])
+        
+    speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan
+    non_punc = list(value for value in words_all if value not in ['.','!','?'])
+    
+    non_punc_as_str = " ".join(str(non_punc))
+    lex = LexicalRichness(non_punc_as_str)
+    speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan
+    
+    #Number of words per minute
+    speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60
+    speech_dict[r_config.nlp_totalTime] = total_time
+    speech_dict['dbm_master_url'] = master_url
+    
+    df_speech = pd.DataFrame([speech_dict])
+    return df_speech

dbm_lib/dbm_features/raw_features/util/vad_utilities.py

@@ -158,7 +158,7 @@ def vad_get_segment_times(sample_rate, frame_duration_ms,
     for frame in frames:
         is_speech = vad.is_speech(frame.bytes, sample_rate)
 
-        sys.stdout.write('1' if is_speech else '0')
+        #sys.stdout.write('1' if is_speech else '0')
         if not triggered:
             ring_buffer.append((frame, is_speech))
             num_voiced = len([f for f, speech in ring_buffer if speech])
@@ -167,7 +167,7 @@ def vad_get_segment_times(sample_rate, frame_duration_ms,
             # TRIGGERED state.
             if num_voiced > 0.9 * ring_buffer.maxlen:
                 triggered = True
-                sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
+                #sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
                 start_times.append(ring_buffer[0][0].timestamp)  # BT
                 ring_buffer.clear()
         else:
@@ -179,18 +179,18 @@ def vad_get_segment_times(sample_rate, frame_duration_ms,
             # unvoiced, then enter NOTTRIGGERED and yield whatever
             # audio we've collected.
             if num_unvoiced > 0.9 * ring_buffer.maxlen:
-                sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
+                #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
                 end_times.append(ring_buffer[0][0].timestamp + frame.duration)  # BT
                 triggered = False
 
     if triggered:  # BT if were in triggered state at end of signal, set output time
-        sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
+        #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
         if len(ring_buffer)>0:
             end_times.append(ring_buffer[0][0].timestamp )  # BT
         else:
             # only get here in very rare case that we triggered on 2nd-to-last frame
             end_times.append(frame.timestamp + frame.duration)
-    sys.stdout.write('\n')
+    #sys.stdout.write('\n')
 
     return(start_times, end_times)
 

dbm_lib/dbm_features/raw_features/video/open_face_process.py

@@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut
 logging.basicConfig(level=logging.INFO)
 logger=logging.getLogger()
 
-def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path):
+def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False):
     """ Computes open_face features for the files in filepaths
-    
+
     Args:
     -----
         filepaths: (itreable[str])
@@ -27,46 +27,54 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path):
         input_dir: Path to the input videos
         out_dir: Path to the processed output
         of_path: OpenFace source code path
-                    
+
     Returns:
     --------
         (itreable[str]) list of .csv files
-    """
-    
-    suffix = '_OF_features'
+    """    
+    if video_tracking:
+        suffix = '_OF_video_features'
+    else:
+        suffix = '_OF_features'
+
     csv_files = []
     
     for fp in filepaths:
         try:
-            
+
             _, out_loc, fl_name = ut.filter_path(video_url, out_dir)
             full_f_name = fl_name + suffix
             output_directory = os.path.join(out_loc, full_f_name)
-
-            csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
             
+            if video_tracking and not os.path.exists(os.path.abspath(output_directory)):
+                os.makedirs(os.path.abspath(output_directory))
+            csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
+
         except Exception as e:
             logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
-            
+
     return csv_files
-        
-def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group):
+
+def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking):
     """
     Processing all patient's for fetching emotion expressivity
     -------------------
     -------------------
     Args:
         video_uri: video path; input_dir : input directory for video's; dbm_group: feature group
-        out_dir: (str) Output directory for processed output; of_path: OpenFace source code path 
-        
+        out_dir: (str) Output directory for processed output; of_path: OpenFace source code path
+
     """
     try:
         
-        if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group:
-            return
-
-        filepaths = [video_uri]
-        csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path)
+        if dbm_group != None:
+            check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation
+            check_val = bool(len({*check_group} & {*dbm_group}))
+            if not check_val:
+                return
         
+        filepaths = [video_uri]
+        csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking)
+
     except Exception as e:
-        logger.error('Failed to process video file')
+        logger.error('Failed to process video file')