diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 679c0845..3dc4834b 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -225,4 +225,21 @@ class ConfigRawReader(object): #NLP features self.nlp_transcribe = config['raw_feature']['nlp_transcribe'] + self.nlp_numSentences = config['raw_feature']['nlp_numSentences'] + self.nlp_singPronPerAns = config['raw_feature']['nlp_singPronPerAns'] + self.nlp_singPronPerSen = config['raw_feature']['nlp_singPronPerSen'] + self.nlp_pastTensePerAns = config['raw_feature']['nlp_pastTensePerAns'] + self.nlp_pastTensePerSen = config['raw_feature']['nlp_pastTensePerSen'] + self.nlp_pronounsPerAns = config['raw_feature']['nlp_pronounsPerAns'] + self.nlp_pronounsPerSen = config['raw_feature']['nlp_pronounsPerSen'] + self.nlp_verbsPerAns = config['raw_feature']['nlp_verbsPerAns'] + self.nlp_verbsPerSen = config['raw_feature']['nlp_verbsPerSen'] + self.nlp_adjectivesPerAns = config['raw_feature']['nlp_adjectivesPerAns'] + self.nlp_adjectivesPerSen = config['raw_feature']['nlp_adjectivesPerSen'] + self.nlp_nounsPerAns = config['raw_feature']['nlp_nounsPerAns'] + self.nlp_nounsPerSen = config['raw_feature']['nlp_nounsPerSen'] + self.nlp_sentiment_mean = config['raw_feature']['nlp_sentiment_mean'] + self.nlp_mattr = config['raw_feature']['nlp_mattr'] + self.nlp_wordsPerMin = config['raw_feature']['nlp_wordsPerMin'] + self.nlp_totalTime = config['raw_feature']['nlp_totalTime'] \ No newline at end of file diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 902ef2c6..be781a7f 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -8,7 +8,7 @@ from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink -from dbm_lib.dbm_features.raw_features.nlp import transcribe +from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features import subprocess import logging @@ -137,6 +137,7 @@ def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): logger.info('Processing nlp variables from data in {}'.format(video_uri)) transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path) + speech_features.run_speech_feature(video_uri, out_dir, r_config) def remove_file(file_path): """ diff --git a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py new file mode 100644 index 00000000..6aebd547 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py @@ -0,0 +1,47 @@ +""" +file_name: speech_features +project_name: DBM +created: 2020-13-11 +""" + +import os +import numpy as np +import pandas as pd +import glob +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +speech_dir = 'nlp/speech_feature' +speech_ext = '_nlp.csv' +transcribe_ext = 'nlp/transcribe/*_transcribe.csv' + +def run_speech_feature(video_uri, out_dir, r_config): + """ + Processing all patient's for fetching nlp features + ------------------- + ------------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output + """ + try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + + transcribe_path = glob.glob(join(out_loc, transcribe_ext)) + if len(transcribe_path)>0: + + transcribe_df = pd.read_csv(transcribe_path[0]) + df_speech= n_util.process_speech(transcribe_df, r_config) + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext) + + except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py index 3914f78a..f567e967 100644 --- a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py +++ b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py @@ -21,7 +21,7 @@ formant_dir = 'nlp/transcribe' csv_ext = '_transcribe.csv' error_txt = 'error: length less than 0.1' -def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path): +def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur): """ Preparing Formant freq matrix Args: @@ -33,6 +33,7 @@ def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe]) df_formant.replace('', np.nan, regex=True,inplace=True) + df_formant[r_config.nlp_totalTime] = aud_dur df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release df_formant['dbm_master_url'] = video_uri @@ -44,8 +45,8 @@ def empty_transcribe(video_uri, out_loc, fl_name, r_config): """ Preparing empty formant frequency matrix if something fails """ - cols = [r_config.nlp_transcribe, r_config.err_reason] - out_val = [[np.nan, error_txt]] + cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason] + out_val = [[np.nan, np.nan, error_txt]] df_fm = pd.DataFrame(out_val, columns = cols) df_fm['dbm_master_url'] = video_uri @@ -77,6 +78,7 @@ def run_transcribe(video_uri, out_dir, r_config, deep_path): empty_transcribe(video_uri, out_loc, fl_name, r_config) return - calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path) + calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur) except Exception as e: - logger.error('Failed to process audio file') \ No newline at end of file + logger.error('Failed to process audio file') + \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/util/nlp_util.py b/dbm_lib/dbm_features/raw_features/util/nlp_util.py index 3288240b..fc1ac3d1 100644 --- a/dbm_lib/dbm_features/raw_features/util/nlp_util.py +++ b/dbm_lib/dbm_features/raw_features/util/nlp_util.py @@ -11,6 +11,11 @@ import pandas as pd import os import logging +import nltk +import re +from lexicalrichness import LexicalRichness +from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + logging.basicConfig(level=logging.INFO) logger=logging.getLogger() @@ -64,3 +69,144 @@ def process_deepspeech(audio_file,deep_path): deep_text= deep_speech_output_clean(deep_output) return deep_text + +def nltk_download(): + + try: + nltk.data.find('tokenizers/punkt') + + except LookupError: + logger.info('punkt is not available') + nltk.download('punkt') + + try: + nltk.data.find('averaged_perceptron_tagger') + + except LookupError: + logger.info('averaged_perceptron_tagger is not available') + nltk.download('averaged_perceptron_tagger') + +def empty_speech(r_config, master_url, error_txt): + """ + Preparing empty speech matrix with error + Args: + r_config: raw config file object + error_txt: Error message during transcription + + Returns: + Empty dataframe for speech features with error + """ + + col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns, + r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns, + r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns, + r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin, + r_config.nlp_totalTime, r_config.err_reason] + + df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col) + df_speech['dbm_master_url'] = master_url + + return df_speech + +def divide_var(speech_var1, spech_var2): + """ + divide variables + """ + speech_var = np.nan + if spech_var2!=0: + speech_var = speech_var1/spech_var2 + return speech_var + +def process_speech(transcribe_df,r_config): + """ + Preparing speech features + Args: + transcribe_df: Transcribed dataframe + r_config: raw config file object + Returns: + Dataframe for speech features + """ + + err_transcribe = transcribe_df[r_config.err_reason].iloc[0] + transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0] + total_time = transcribe_df[r_config.nlp_totalTime].iloc[0] + master_url = transcribe_df['dbm_master_url'].iloc[0] + + #clean transcribe + transcribe = transcribe.replace(",", "") + transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe)) + + if err_transcribe != 'Pass': + df_speech = empty_speech(r_config, master_url, error_txt) + + return df_speech + + speech_dict = {} + nltk_download() + + sentences = nltk.tokenize.sent_tokenize(transcribe) + words_all = nltk.tokenize.word_tokenize(transcribe) + num_sentences = len(sentences) + + speech_dict[r_config.nlp_numSentences] = num_sentences + + #nlp_singPron + i_s = transcribe.count('I') + me_s = transcribe.count('me') + my_s = transcribe.count('my') + sing_count = i_s + me_s + my_s + + speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences) + + tagged = nltk.pos_tag(transcribe.split()) + tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag']) + + #Past tense per answer + all_POSs = tagged_df['pos_tag'].tolist() + speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences) + + #Pronoun per answer + pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$') + speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences) + + #Verb per answer + verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \ + + all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ') + speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences) + + #Adjective per answer + adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS') + speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences) + + #Noun per answer + nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS') + speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences) + + #Sentiment analysis + vader = SentimentIntensityAnalyzer() + sentence_valences = [] + + for s in sentences: + sentiment_dict = vader.polarity_scores(s) + sentence_valences.append(sentiment_dict['compound']) + + speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan + non_punc = list(value for value in words_all if value not in ['.','!','?']) + + non_punc_as_str = " ".join(str(non_punc)) + lex = LexicalRichness(non_punc_as_str) + speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan + + #Number of words per minute + speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60 + speech_dict[r_config.nlp_totalTime] = total_time + speech_dict['dbm_master_url'] = master_url + + df_speech = pd.DataFrame([speech_dict]) + return df_speech \ No newline at end of file diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index a0759d79..7346eb36 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,7 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -22,6 +22,7 @@ derive_feature: MOV_HM_LOC: _headmov MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks + NLP_SPEECH_LOC: _nlp #Facial category feature group FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom'] @@ -65,6 +66,12 @@ derive_feature: MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] + #NLP category feature group + NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen', + 'nlp_pronounsPerAns', 'nlp_pronounsPerSen', 'nlp_verbsPerAns', 'nlp_verbsPerSen', 'nlp_adjectivesPerAns', + 'nlp_adjectivesPerSen', 'nlp_nounsPerAns', 'nlp_nounsPerSen', 'nlp_sentiment_mean', 'nlp_mattr', 'nlp_wordsPerMin', + 'nlp_totalTime'] + #Calculation for variables # Facial Asymmetry fac_AsymMaskMouth: ['mean', 'std'] @@ -248,3 +255,22 @@ derive_feature: mov_blink_ear: ['mean', 'std'] vid_dur: ['count'] mov_blinkdur: ['mean', 'std'] + + #NLP feature + nlp_numSentences: ['mean'] + nlp_singPronPerAns: ['mean'] + nlp_singPronPerSen: ['mean'] + nlp_pastTensePerAns: ['mean'] + nlp_pastTensePerSen: ['mean'] + nlp_pronounsPerAns: ['mean'] + nlp_pronounsPerSen: ['mean'] + nlp_verbsPerAns: ['mean'] + nlp_verbsPerSen: ['mean'] + nlp_adjectivesPerAns: ['mean'] + nlp_adjectivesPerSen: ['mean'] + nlp_nounsPerAns: ['mean'] + nlp_nounsPerSen: ['mean'] + nlp_sentiment_mean: ['mean'] + nlp_mattr: ['mean'] + nlp_wordsPerMin: ['mean'] + nlp_totalTime: ['mean'] diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index f8b00883..b9c673b8 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -199,3 +199,21 @@ raw_feature: #NLP markers nlp_transcribe: nlp_transcribe + nlp_numSentences: nlp_numSentences + nlp_singPronPerAns: nlp_singPronPerAns + nlp_singPronPerSen: nlp_singPronPerSen + nlp_pastTensePerAns: nlp_pastTensePerAns + nlp_pastTensePerSen: nlp_pastTensePerSen + nlp_pronounsPerAns: nlp_pronounsPerAns + nlp_pronounsPerSen: nlp_pronounsPerSen + nlp_verbsPerAns: nlp_verbsPerAns + nlp_verbsPerSen: nlp_verbsPerSen + nlp_adjectivesPerAns: nlp_adjectivesPerAns + nlp_adjectivesPerSen: nlp_adjectivesPerSen + nlp_nounsPerAns: nlp_nounsPerAns + nlp_nounsPerSen: nlp_nounsPerSen + nlp_sentiment_mean: nlp_sentiment_mean + nlp_mattr: nlp_mattr + nlp_wordsPerMin: nlp_wordsPerMin + nlp_totalTime: nlp_totalTime + \ No newline at end of file