bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into tremor_vars

Browse Source
This commit is contained in:
vkoesmahargyo
2020-12-02 10:55:49 -05:00
committed by GitHub
parent d3920a26a2 28abbb459a
commit 7f2fe4ac2b
15 changed files with 730 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
dbm_lib/config/config_raw_feature.py
View File

@@ -222,18 +222,40 @@ class ConfigRawReader(object):
self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw']
self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll']
self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist']
self.mov_freq_trem_freq = config['raw_feature']['mov_freq_trem_freq']
self.mov_freq_trem_index = config['raw_feature']['mov_freq_trem_index']
self.mov_freq_trem_pindex = config['raw_feature']['mov_freq_trem_pindex']
self.mov_amp_trem_freq = config['raw_feature']['mov_amp_trem_freq']
self.mov_amp_trem_index = config['raw_feature']['mov_amp_trem_index']
self.mov_amp_trem_pindex = config['raw_feature']['mov_amp_trem_pindex']
self.fac_tremor_median_5 = config['raw_feature']['fac_tremor_median_5']
self.fac_tremor_median_12 = config['raw_feature']['fac_tremor_median_12']
self.fac_tremor_median_8 = config['raw_feature']['fac_tremor_median_8']
self.fac_tremor_median_48 = config['raw_feature']['fac_tremor_median_48']
self.fac_tremor_median_54 = config['raw_feature']['fac_tremor_median_54']
self.fac_tremor_median_28 = config['raw_feature']['fac_tremor_median_28']
self.fac_tremor_median_51 = config['raw_feature']['fac_tremor_median_51']
self.fac_tremor_median_66 = config['raw_feature']['fac_tremor_median_66']
self.fac_tremor_median_57 = config['raw_feature']['fac_tremor_median_57']
self.mov_leye_x = config['raw_feature']['mov_leye_x']
self.mov_leye_y = config['raw_feature']['mov_leye_y']
self.mov_leye_z = config['raw_feature']['mov_leye_z']
self.mov_reye_x = config['raw_feature']['mov_reye_x']
self.mov_reye_y = config['raw_feature']['mov_reye_y']
self.mov_reye_z = config['raw_feature']['mov_reye_z']
self.mov_eleft_disp = config['raw_feature']['mov_eleft_disp']
self.mov_eright_disp = config['raw_feature']['mov_eright_disp']
#NLP features
self.nlp_transcribe = config['raw_feature']['nlp_transcribe']
self.nlp_numSentences = config['raw_feature']['nlp_numSentences']
self.nlp_singPronPerAns = config['raw_feature']['nlp_singPronPerAns']
self.nlp_singPronPerSen = config['raw_feature']['nlp_singPronPerSen']
self.nlp_pastTensePerAns = config['raw_feature']['nlp_pastTensePerAns']
self.nlp_pastTensePerSen = config['raw_feature']['nlp_pastTensePerSen']
self.nlp_pronounsPerAns = config['raw_feature']['nlp_pronounsPerAns']
self.nlp_pronounsPerSen = config['raw_feature']['nlp_pronounsPerSen']
self.nlp_verbsPerAns = config['raw_feature']['nlp_verbsPerAns']
self.nlp_verbsPerSen = config['raw_feature']['nlp_verbsPerSen']
self.nlp_adjectivesPerAns = config['raw_feature']['nlp_adjectivesPerAns']
self.nlp_adjectivesPerSen = config['raw_feature']['nlp_adjectivesPerSen']
self.nlp_nounsPerAns = config['raw_feature']['nlp_nounsPerAns']
self.nlp_nounsPerSen = config['raw_feature']['nlp_nounsPerSen']
self.nlp_sentiment_mean = config['raw_feature']['nlp_sentiment_mean']
self.nlp_mattr = config['raw_feature']['nlp_mattr']
self.nlp_wordsPerMin = config['raw_feature']['nlp_wordsPerMin']
self.nlp_totalTime = config['raw_feature']['nlp_totalTime']

26
dbm_lib/controller/process_feature.py
View File

@@ -7,7 +7,8 @@ created: 2020-20-07
from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq
from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc
from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor, facial_tremor
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze, voice_tremor
from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features
import subprocess
import logging
@@ -127,12 +128,27 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
logger.info('processing eye blink....')
eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model)
logger.info('processing eye gaze....')
eye_gaze.run_eye_gaze(video_uri, out_dir, r_config)
logger.info('processing voice tremor....')
voice_tremor.run_vtremor(video_uri, out_dir, r_config)
logger.info('processing facial tremor....')
face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True)
def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path):
"""
processing nlp features
Args:
video_uri: video path; out_dir: raw variable output dir
dbm_group: list of features to process; r_config: raw feature config object
deep_path: deep speech build path
"""
if dbm_group != None and len(dbm_group)>0 and 'nlp' not in dbm_group:
return
logger.info('Processing nlp variables from data in {}'.format(video_uri))
transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path)
speech_features.run_speech_feature(video_uri, out_dir, r_config)
def remove_file(file_path):
"""
removing wav file

148
dbm_lib/dbm_features/raw_features/movement/eye_gaze.py Normal file
View File

@@ -0,0 +1,148 @@
"""
file_name: eye_gaze
project_name: DBM
created: 2020-30-11
"""
import os
import glob
import pandas as pd
import numpy as np
from scipy.spatial import distance
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
eye_pose_dir = 'movement/gaze'
eye_pose_ext = '_eyegaze.csv'
def eye_motion_df(l_disp, r_disp, error_list, r_config):
"""
Generating eye movement dataframe
Args:
l_disp: displacement list(left eye); l_disp: displacement list(right eye)
r_config: raw variable config file object
Reutrns:
Final eye displacement dataframe
"""
df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp])
df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp])
df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False)
df_eye_motion[r_config.err_reason] = error_list
return df_eye_motion
def filter_motion(df_of, df_disp, col_l, col_r, r_config):
"""
Filtering final eye movement dataframe
Args:
df_of: Openface raw out dataframe; col_r: right eye column
col_l: left eye column; r_config: raw variable config file object
"""
df_of = df_of[col_l + col_r + [' confidence']]
df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan
df_filter = df_of[col_l + col_r]
df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z,
r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z]
df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False)
return df_motion
def eye_disp(of_results, col, r_config):
"""
Computing head velocity frame by frame
Args:
of_results: Openface raw out dataframe
r_config: Face config file object
Reutrns:
Final head velocity frame by frame output
"""
distance_list = []
error_list = []
of_results = of_results[col+ [' confidence']]
for index, row in of_results.iterrows():
dst = np.nan
if index == 0 or float(row[' confidence']) < 0.8: #Threshold < 0.8
distance_list.append(dst)
if float(row[' confidence']) < 0.8:
error_list.append('confidence less than 80%')
else:
error_list.append('Pass')
continue
if index > 0:
point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1])
point_y = (row[col[0]],row[col[1]],row[col[2]])
try:
dst = distance.euclidean(point_x, point_y)
except:
pass
distance_list.append(abs(dst))
error_list.append('Pass')
return distance_list, error_list
def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config):
"""
Computing eye motion variables
Args:
df_of: Openface dataframe
out_loc: Output path for saving output csv's
fl_name: file name for output csv
r_config: raw variable config file object
"""
col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z']
col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z']
gazel_disp, err_l = eye_disp(df_of, col_l, r_config)
gazer_disp, err_r = eye_disp(df_of, col_r, r_config)
df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config)
df_disp['dbm_master_url'] = video_uri
df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config)
ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext)
def run_eye_gaze(video_uri, out_dir, r_config):
"""
Processing all patient's for getting eye movement artifacts
--------------------------------
--------------------------------
Args:
video_uri: video path; input_dir : input directory for video's
out_dir: (str) Output directory for processed output; r_config: raw variable config object
"""
try:
#filtering path to generate input & output path
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
of_csv = of_csv_path[0]
df_of = pd.read_csv(of_csv, error_bad_lines=False)
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config)
except Exception as e:
logger.error('Failed to process video file')

47
dbm_lib/dbm_features/raw_features/nlp/speech_features.py Normal file
View File

@@ -0,0 +1,47 @@
"""
file_name: speech_features
project_name: DBM
created: 2020-13-11
"""
import os
import numpy as np
import pandas as pd
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
speech_dir = 'nlp/speech_feature'
speech_ext = '_nlp.csv'
transcribe_ext = 'nlp/transcribe/*_transcribe.csv'
def run_speech_feature(video_uri, out_dir, r_config):
"""
Processing all patient's for fetching nlp features
-------------------
-------------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
try:
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
transcribe_path = glob.glob(join(out_loc, transcribe_ext))
if len(transcribe_path)>0:
transcribe_df = pd.read_csv(transcribe_path[0])
df_speech= n_util.process_speech(transcribe_df, r_config)
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext)
except Exception as e:
logger.error('Failed to process video file')

84
dbm_lib/dbm_features/raw_features/nlp/transcribe.py Normal file
View File

@@ -0,0 +1,84 @@
"""
file_name: transcribe
project_name: DBM
created: 2020-10-11
"""
import pandas as pd
import numpy as np
import librosa
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
formant_dir = 'nlp/transcribe'
csv_ext = '_transcribe.csv'
error_txt = 'error: length less than 0.1'
def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur):
"""
Preparing Formant freq matrix
Args:
audio_file: (.wav) parsed audio file; fl_name: input file name
out_loc: (str) Output directory; r_config: raw variable config
"""
text = n_util.process_deepspeech(audio_file, deep_path)
df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe])
df_formant.replace('', np.nan, regex=True,inplace=True)
df_formant[r_config.nlp_totalTime] = aud_dur
df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_formant['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
def empty_transcribe(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty formant frequency matrix if something fails
"""
cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_fm = pd.DataFrame(out_val, columns = cols)
df_fm['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
def run_transcribe(video_uri, out_dir, r_config, deep_path):
"""
Processing all patient's for fetching Formant freq
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output; deep_path: deepspeech build path
"""
try:
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.1:
logger.info('Output file {} size is less than 0.1 sec'.format(audio_file))
empty_transcribe(video_uri, out_loc, fl_name, r_config)
return
calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur)
except Exception as e:
logger.error('Failed to process audio file')

212
dbm_lib/dbm_features/raw_features/util/nlp_util.py Normal file
View File

@@ -0,0 +1,212 @@
"""
file_name: nlp_util
project_name: DBM
created: 2020-10-11
"""
import subprocess
import json
import numpy as np
import pandas as pd
import os
import logging
import nltk
import re
from lexicalrichness import LexicalRichness
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
#Speech to text using Deepspeech 0.9.1
def deepspeech(AUDIO_FILE,deep_path):
"""
Extracting text from audio using Deep Speech neural network trained model
Returns:
Text: text which is extracted from audio
"""
api = 'deepspeech'
arg_speech0 = '--model'
arg_speech_path0 = os.path.join(deep_path, 'deepspeech-0.9.1-models.pbmm')
arg_speech1 = '--scorer'
arg_speech_path1 = os.path.join(deep_path, 'deepspeech-0.9.1-models.scorer')
arg_audio = "--audio"
out = subprocess.Popen([api, arg_speech0, arg_speech_path0, arg_speech1, arg_speech_path1, arg_audio, AUDIO_FILE],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
logger.info('Deepspeech output...... {}'.format(out))
try:
stdout,stderr = out.communicate()
except:
return "error", "error"
print(stderr)
return stdout,stderr
def deep_speech_output_clean(result):
"""
Parsing deep speech output(text)
Return:
Text from speech
"""
text = ""
if len(result)>0:
res_split = str(result[0]).split('\\n')
if len(res_split)>0:
for i in range(len(res_split)):
if 'Inference took' in res_split[i]:
text = res_split[i + 1]
return text
return text
def process_deepspeech(audio_file,deep_path):
"""
Transcribing audio to extract text from speech
"""
deep_output = deepspeech(audio_file,deep_path)
deep_text= deep_speech_output_clean(deep_output)
return deep_text
def nltk_download():
try:
nltk.data.find('tokenizers/punkt')
except LookupError:
logger.info('punkt is not available')
nltk.download('punkt')
try:
nltk.data.find('averaged_perceptron_tagger')
except LookupError:
logger.info('averaged_perceptron_tagger is not available')
nltk.download('averaged_perceptron_tagger')
def empty_speech(r_config, master_url, error_txt):
"""
Preparing empty speech matrix with error
Args:
r_config: raw config file object
error_txt: Error message during transcription
Returns:
Empty dataframe for speech features with error
"""
col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns,
r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns,
r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns,
r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin,
r_config.nlp_totalTime, r_config.err_reason]
df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col)
df_speech['dbm_master_url'] = master_url
return df_speech
def divide_var(speech_var1, spech_var2):
"""
divide variables
"""
speech_var = np.nan
if spech_var2!=0:
speech_var = speech_var1/spech_var2
return speech_var
def process_speech(transcribe_df,r_config):
"""
Preparing speech features
Args:
transcribe_df: Transcribed dataframe
r_config: raw config file object
Returns:
Dataframe for speech features
"""
err_transcribe = transcribe_df[r_config.err_reason].iloc[0]
transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0]
total_time = transcribe_df[r_config.nlp_totalTime].iloc[0]
master_url = transcribe_df['dbm_master_url'].iloc[0]
#clean transcribe
transcribe = transcribe.replace(",", "")
transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe))
if err_transcribe != 'Pass':
df_speech = empty_speech(r_config, master_url, error_txt)
return df_speech
speech_dict = {}
nltk_download()
sentences = nltk.tokenize.sent_tokenize(transcribe)
words_all = nltk.tokenize.word_tokenize(transcribe)
num_sentences = len(sentences)
speech_dict[r_config.nlp_numSentences] = num_sentences
#nlp_singPron
i_s = transcribe.count('I')
me_s = transcribe.count('me')
my_s = transcribe.count('my')
sing_count = i_s + me_s + my_s
speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan
speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences)
tagged = nltk.pos_tag(transcribe.split())
tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag'])
#Past tense per answer
all_POSs = tagged_df['pos_tag'].tolist()
speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan
speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences)
#Pronoun per answer
pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$')
speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan
speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences)
#Verb per answer
verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \
+ all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ')
speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan
speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences)
#Adjective per answer
adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS')
speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan
speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences)
#Noun per answer
nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS')
speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan
speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences)
#Sentiment analysis
vader = SentimentIntensityAnalyzer()
sentence_valences = []
for s in sentences:
sentiment_dict = vader.polarity_scores(s)
sentence_valences.append(sentiment_dict['compound'])
speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan
non_punc = list(value for value in words_all if value not in ['.','!','?'])
non_punc_as_str = " ".join(str(non_punc))
lex = LexicalRichness(non_punc_as_str)
speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan
#Number of words per minute
speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60
speech_dict[r_config.nlp_totalTime] = total_time
speech_dict['dbm_master_url'] = master_url
df_speech = pd.DataFrame([speech_dict])
return df_speech

11
dbm_lib/dbm_features/raw_features/video/open_face_process.py
View File

@@ -62,11 +62,14 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group):
"""
try:
if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group:
return
if dbm_group != None:
check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation
check_val = bool(len({*check_group} & {*dbm_group}))
if not check_val:
return
filepaths = [video_uri]
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path)
except Exception as e:
logger.error('Failed to process video file')
logger.error('Failed to process video file')