nlp feature

dbm_lib/config/config_raw_feature.py

@@ -225,4 +225,21 @@ class ConfigRawReader(object):
 
             #NLP features
             self.nlp_transcribe = config['raw_feature']['nlp_transcribe']
+            self.nlp_numSentences = config['raw_feature']['nlp_numSentences']
+            self.nlp_singPronPerAns = config['raw_feature']['nlp_singPronPerAns']
+            self.nlp_singPronPerSen = config['raw_feature']['nlp_singPronPerSen']
+            self.nlp_pastTensePerAns = config['raw_feature']['nlp_pastTensePerAns']
+            self.nlp_pastTensePerSen = config['raw_feature']['nlp_pastTensePerSen']
+            self.nlp_pronounsPerAns = config['raw_feature']['nlp_pronounsPerAns']
+            self.nlp_pronounsPerSen = config['raw_feature']['nlp_pronounsPerSen']
+            self.nlp_verbsPerAns = config['raw_feature']['nlp_verbsPerAns']
+            self.nlp_verbsPerSen = config['raw_feature']['nlp_verbsPerSen']
+            self.nlp_adjectivesPerAns = config['raw_feature']['nlp_adjectivesPerAns']
+            self.nlp_adjectivesPerSen = config['raw_feature']['nlp_adjectivesPerSen']
+            self.nlp_nounsPerAns = config['raw_feature']['nlp_nounsPerAns']
+            self.nlp_nounsPerSen = config['raw_feature']['nlp_nounsPerSen']
+            self.nlp_sentiment_mean = config['raw_feature']['nlp_sentiment_mean']
+            self.nlp_mattr = config['raw_feature']['nlp_mattr']
+            self.nlp_wordsPerMin = config['raw_feature']['nlp_wordsPerMin']
+            self.nlp_totalTime = config['raw_feature']['nlp_totalTime']
             

dbm_lib/controller/process_feature.py

@@ -8,7 +8,7 @@ from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr,
 from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc
 from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark
 from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink
-from dbm_lib.dbm_features.raw_features.nlp import transcribe
+from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features
 
 import subprocess
 import logging
@@ -137,6 +137,7 @@ def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path):
     
     logger.info('Processing nlp variables from data in {}'.format(video_uri))
     transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path)
+    speech_features.run_speech_feature(video_uri, out_dir, r_config)
     
 def remove_file(file_path):
     """

dbm_lib/dbm_features/raw_features/nlp/speech_features.py

@@ -0,0 +1,47 @@
+"""
+file_name: speech_features
+project_name: DBM
+created: 2020-13-11
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import glob
+from os.path import join
+import logging
+
+from dbm_lib.dbm_features.raw_features.util import util as ut
+from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util
+
+logging.basicConfig(level=logging.INFO)
+logger=logging.getLogger()
+
+speech_dir = 'nlp/speech_feature'
+speech_ext = '_nlp.csv'
+transcribe_ext = 'nlp/transcribe/*_transcribe.csv'
+
+def run_speech_feature(video_uri, out_dir, r_config):
+    """
+    Processing all patient's for fetching nlp features
+    -------------------
+    -------------------
+    Args:
+        video_uri: video path; r_config: raw variable config object
+        out_dir: (str) Output directory for processed output
+    """
+    try:
+        
+        input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
+
+        transcribe_path = glob.glob(join(out_loc, transcribe_ext))
+        if len(transcribe_path)>0:
+
+            transcribe_df = pd.read_csv(transcribe_path[0])
+            df_speech= n_util.process_speech(transcribe_df, r_config)
+
+            logger.info('Saving Output file {} '.format(out_loc))
+            ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext)
+            
+    except Exception as e:
+        logger.error('Failed to process video file')

dbm_lib/dbm_features/raw_features/nlp/transcribe.py

@@ -21,7 +21,7 @@ formant_dir = 'nlp/transcribe'
 csv_ext = '_transcribe.csv'
 error_txt = 'error: length less than 0.1'
 
-def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path):
+def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur):
     """
     Preparing Formant freq matrix
     Args:
@@ -33,6 +33,7 @@ def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path
     df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe])
     
     df_formant.replace('', np.nan, regex=True,inplace=True)
+    df_formant[r_config.nlp_totalTime] = aud_dur
     df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
     df_formant['dbm_master_url'] = video_uri
     
@@ -44,8 +45,8 @@ def empty_transcribe(video_uri, out_loc, fl_name, r_config):
     """
     Preparing empty formant frequency matrix if something fails
     """
-    cols = [r_config.nlp_transcribe, r_config.err_reason]
-    out_val = [[np.nan, error_txt]]
+    cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason]
+    out_val = [[np.nan, np.nan, error_txt]]
     df_fm = pd.DataFrame(out_val, columns = cols)
     df_fm['dbm_master_url'] = video_uri
     
@@ -77,6 +78,7 @@ def run_transcribe(video_uri, out_dir, r_config, deep_path):
                 empty_transcribe(video_uri, out_loc, fl_name, r_config)
                 return
 
-            calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path)
+            calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur)
     except Exception as e:
         logger.error('Failed to process audio file')
+        

dbm_lib/dbm_features/raw_features/util/nlp_util.py

@@ -11,6 +11,11 @@ import pandas as pd
 import os
 import logging
 
+import nltk
+import re
+from lexicalrichness import LexicalRichness
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer 
+
 logging.basicConfig(level=logging.INFO)
 logger=logging.getLogger()
 
@@ -64,3 +69,144 @@ def process_deepspeech(audio_file,deep_path):
     deep_text= deep_speech_output_clean(deep_output)
     
     return deep_text
+
+def nltk_download():
+    
+    try:
+        nltk.data.find('tokenizers/punkt')
+        
+    except LookupError:
+        logger.info('punkt is not available')
+        nltk.download('punkt')
+        
+    try:
+        nltk.data.find('averaged_perceptron_tagger')
+        
+    except LookupError:
+        logger.info('averaged_perceptron_tagger is not available')
+        nltk.download('averaged_perceptron_tagger')
+
+def empty_speech(r_config, master_url, error_txt):
+    """
+    Preparing empty speech matrix with error
+    Args:
+        r_config: raw config file object
+        error_txt: Error message during transcription
+        
+    Returns:
+            Empty dataframe for speech features with error
+    """
+    
+    col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns,
+          r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns,
+          r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns,
+          r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin,
+          r_config.nlp_totalTime, r_config.err_reason]
+    
+    df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col)
+    df_speech['dbm_master_url'] = master_url
+    
+    return df_speech
+
+def divide_var(speech_var1, spech_var2):
+    """
+    divide variables
+    """
+    speech_var = np.nan
+    if spech_var2!=0:
+        speech_var = speech_var1/spech_var2
+    return speech_var
+
+def process_speech(transcribe_df,r_config):
+    """
+        Preparing speech features
+        Args:
+            transcribe_df: Transcribed dataframe
+            r_config: raw config file object
+        Returns:
+            Dataframe for speech features
+    """
+    
+    err_transcribe = transcribe_df[r_config.err_reason].iloc[0]
+    transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0]
+    total_time = transcribe_df[r_config.nlp_totalTime].iloc[0]
+    master_url = transcribe_df['dbm_master_url'].iloc[0]
+    
+    #clean transcribe
+    transcribe = transcribe.replace(",", "")
+    transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe))
+    
+    if err_transcribe != 'Pass':
+        df_speech = empty_speech(r_config, master_url, error_txt)
+        
+        return df_speech
+    
+    speech_dict = {}
+    nltk_download()
+    
+    sentences = nltk.tokenize.sent_tokenize(transcribe)
+    words_all = nltk.tokenize.word_tokenize(transcribe)
+    num_sentences = len(sentences)
+    
+    speech_dict[r_config.nlp_numSentences] = num_sentences
+    
+    #nlp_singPron
+    i_s = transcribe.count('I')
+    me_s = transcribe.count('me')
+    my_s = transcribe.count('my')
+    sing_count = i_s + me_s + my_s
+    
+    speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences)
+    
+    tagged = nltk.pos_tag(transcribe.split())
+    tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag'])
+    
+    #Past tense per answer
+    all_POSs = tagged_df['pos_tag'].tolist()
+    speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences)
+    
+    #Pronoun per answer
+    pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$')
+    speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan
+    speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences)
+    
+    #Verb per answer
+    verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \
+                      + all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ')
+    speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences)
+    
+    #Adjective per answer
+    adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS')
+    speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences)
+    
+    #Noun per answer
+    nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS')
+    speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan
+    speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences)
+    
+    #Sentiment analysis
+    vader = SentimentIntensityAnalyzer()
+    sentence_valences = []
+    
+    for s in sentences:
+        sentiment_dict = vader.polarity_scores(s) 
+        sentence_valences.append(sentiment_dict['compound'])
+        
+    speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan
+    non_punc = list(value for value in words_all if value not in ['.','!','?'])
+    
+    non_punc_as_str = " ".join(str(non_punc))
+    lex = LexicalRichness(non_punc_as_str)
+    speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan
+    
+    #Number of words per minute
+    speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60
+    speech_dict[r_config.nlp_totalTime] = total_time
+    speech_dict['dbm_master_url'] = master_url
+    
+    df_speech = pd.DataFrame([speech_dict])
+    return df_speech

resources/features/derived_feature.yml

@@ -2,7 +2,7 @@ derive_feature:
     
     #DBM Feature Group
     FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', 
-    'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK']
+    'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH']
     
     #Feature group output file extensions
     FAC_ASYM_LOC: _facasym
@@ -22,6 +22,7 @@ derive_feature:
     MOV_HM_LOC: _headmov
     MOV_HP_LOC: _headpose
     EYE_BLINK_LOC: _eyeblinks
+    NLP_SPEECH_LOC: _nlp
     
     #Facial category feature group
     FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom']
@@ -65,6 +66,12 @@ derive_feature:
     MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll']
     EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur']
     
+    #NLP category feature group
+    NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen',
+    'nlp_pronounsPerAns', 'nlp_pronounsPerSen', 'nlp_verbsPerAns', 'nlp_verbsPerSen', 'nlp_adjectivesPerAns', 
+    'nlp_adjectivesPerSen', 'nlp_nounsPerAns', 'nlp_nounsPerSen', 'nlp_sentiment_mean', 'nlp_mattr', 'nlp_wordsPerMin',
+    'nlp_totalTime']
+    
     #Calculation for variables
     # Facial Asymmetry
     fac_AsymMaskMouth: ['mean', 'std']
@@ -248,3 +255,22 @@ derive_feature:
     mov_blink_ear: ['mean', 'std']
     vid_dur: ['count']
     mov_blinkdur: ['mean', 'std']
+    
+    #NLP feature
+    nlp_numSentences: ['mean']
+    nlp_singPronPerAns: ['mean']
+    nlp_singPronPerSen: ['mean']
+    nlp_pastTensePerAns: ['mean']
+    nlp_pastTensePerSen: ['mean']
+    nlp_pronounsPerAns: ['mean']
+    nlp_pronounsPerSen: ['mean']
+    nlp_verbsPerAns: ['mean']
+    nlp_verbsPerSen: ['mean']
+    nlp_adjectivesPerAns: ['mean']
+    nlp_adjectivesPerSen: ['mean']
+    nlp_nounsPerAns: ['mean']
+    nlp_nounsPerSen: ['mean']
+    nlp_sentiment_mean: ['mean']
+    nlp_mattr: ['mean']
+    nlp_wordsPerMin: ['mean']
+    nlp_totalTime: ['mean']

resources/features/raw_feature.yml

@@ -199,3 +199,21 @@ raw_feature:
 
     #NLP markers
     nlp_transcribe: nlp_transcribe
+    nlp_numSentences: nlp_numSentences
+    nlp_singPronPerAns: nlp_singPronPerAns
+    nlp_singPronPerSen: nlp_singPronPerSen
+    nlp_pastTensePerAns: nlp_pastTensePerAns
+    nlp_pastTensePerSen: nlp_pastTensePerSen
+    nlp_pronounsPerAns: nlp_pronounsPerAns
+    nlp_pronounsPerSen: nlp_pronounsPerSen
+    nlp_verbsPerAns: nlp_verbsPerAns
+    nlp_verbsPerSen: nlp_verbsPerSen
+    nlp_adjectivesPerAns: nlp_adjectivesPerAns
+    nlp_adjectivesPerSen: nlp_adjectivesPerSen
+    nlp_nounsPerAns: nlp_nounsPerAns
+    nlp_nounsPerSen: nlp_nounsPerSen
+    nlp_sentiment_mean: nlp_sentiment_mean
+    nlp_mattr: nlp_mattr
+    nlp_wordsPerMin: nlp_wordsPerMin
+    nlp_totalTime: nlp_totalTime
+