From 920a7633cdf261846cc55bc31bc3c75142edc90c Mon Sep 17 00:00:00 2001 From: Vijay Yadev Date: Wed, 11 Nov 2020 21:57:04 -0500 Subject: [PATCH 01/26] nlp_transcribe --- Dockerfile | 13 ++- dbm_lib/config/config_raw_feature.py | 3 + dbm_lib/controller/process_feature.py | 15 ++++ .../raw_features/nlp/transcribe.py | 82 +++++++++++++++++++ .../raw_features/util/nlp_util.py | 66 +++++++++++++++ process_data.py | 6 ++ process_dbm.sh | 3 + requirements.txt | 1 + resources/features/raw_feature.yml | 3 + 9 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/nlp/transcribe.py create mode 100644 dbm_lib/dbm_features/raw_features/util/nlp_util.py diff --git a/Dockerfile b/Dockerfile index c3dfd352..5d425d61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y python3-pip \ && apt-get install -y libavcodec-dev \ && apt-get install -y libavformat-dev \ && apt-get install -y libavdevice-dev \ - && apt-get install -y libboost-all-dev + && apt-get install -y libboost-all-dev \ + && apt-get install -y git \ + && apt-get install -y sox RUN ln -sfn /usr/bin/pip3 /usr/bin/pip COPY . /app @@ -24,8 +26,15 @@ RUN dpkg --configure -a RUN su -c ./install.sh RUN echo "Done OpenFace!" -WORKDIR /app +RUN echo "Cloning DeepSpeech..." +WORKDIR /app/pkg +RUN git clone https://github.com/mozilla/DeepSpeech.git +WORKDIR /app/pkg/DeepSpeech +RUN wget https://github.com/mozilla/DeepSpeech/releases/download/v0.9.1/deepspeech-0.9.1-models.pbmm +RUN wget https://github.com/mozilla/DeepSpeech/releases/download/v0.9.1/deepspeech-0.9.1-models.scorer + +WORKDIR /app RUN pip install --upgrade pip RUN pip install -r requirements.txt RUN echo "Requirement txt done!" diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 5494e21d..679c0845 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -222,4 +222,7 @@ class ConfigRawReader(object): self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw'] self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll'] self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist'] + + #NLP features + self.nlp_transcribe = config['raw_feature']['nlp_transcribe'] \ No newline at end of file diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 3282edc2..902ef2c6 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -8,6 +8,7 @@ from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink +from dbm_lib.dbm_features.raw_features.nlp import transcribe import subprocess import logging @@ -123,6 +124,20 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) +def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): + """ + processing nlp features + Args: + video_uri: video path; out_dir: raw variable output dir + dbm_group: list of features to process; r_config: raw feature config object + deep_path: deep speech build path + """ + if dbm_group != None and len(dbm_group)>0 and 'nlp' not in dbm_group: + return + + logger.info('Processing nlp variables from data in {}'.format(video_uri)) + transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path) + def remove_file(file_path): """ removing wav file diff --git a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py new file mode 100644 index 00000000..3914f78a --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py @@ -0,0 +1,82 @@ +""" +file_name: transcribe +project_name: DBM +created: 2020-10-11 +""" + +import pandas as pd +import numpy as np +import librosa +import glob +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +formant_dir = 'nlp/transcribe' +csv_ext = '_transcribe.csv' +error_txt = 'error: length less than 0.1' + +def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path): + """ + Preparing Formant freq matrix + Args: + audio_file: (.wav) parsed audio file; fl_name: input file name + out_loc: (str) Output directory; r_config: raw variable config + """ + + text = n_util.process_deepspeech(audio_file, deep_path) + df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe]) + + df_formant.replace('', np.nan, regex=True,inplace=True) + df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release + df_formant['dbm_master_url'] = video_uri + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext) + +def empty_transcribe(video_uri, out_loc, fl_name, r_config): + + """ + Preparing empty formant frequency matrix if something fails + """ + cols = [r_config.nlp_transcribe, r_config.err_reason] + out_val = [[np.nan, error_txt]] + df_fm = pd.DataFrame(out_val, columns = cols) + df_fm['dbm_master_url'] = video_uri + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext) + +def run_transcribe(video_uri, out_dir, r_config, deep_path): + + """ + Processing all patient's for fetching Formant freq + --------------- + --------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output; deep_path: deepspeech build path + """ + try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) + if len(aud_filter)>0: + + audio_file = aud_filter[0] + aud_dur = librosa.get_duration(filename=audio_file) + + if float(aud_dur) < 0.1: + logger.info('Output file {} size is less than 0.1 sec'.format(audio_file)) + + empty_transcribe(video_uri, out_loc, fl_name, r_config) + return + + calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path) + except Exception as e: + logger.error('Failed to process audio file') \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/util/nlp_util.py b/dbm_lib/dbm_features/raw_features/util/nlp_util.py new file mode 100644 index 00000000..3288240b --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/util/nlp_util.py @@ -0,0 +1,66 @@ +""" +file_name: nlp_util +project_name: DBM +created: 2020-10-11 +""" + +import subprocess +import json +import numpy as np +import pandas as pd +import os +import logging + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +#Speech to text using Deepspeech 0.9.1 +def deepspeech(AUDIO_FILE,deep_path): + """ + Extracting text from audio using Deep Speech neural network trained model + Returns: + Text: text which is extracted from audio + """ + api = 'deepspeech' + arg_speech0 = '--model' + arg_speech_path0 = os.path.join(deep_path, 'deepspeech-0.9.1-models.pbmm') + arg_speech1 = '--scorer' + arg_speech_path1 = os.path.join(deep_path, 'deepspeech-0.9.1-models.scorer') + arg_audio = "--audio" + + out = subprocess.Popen([api, arg_speech0, arg_speech_path0, arg_speech1, arg_speech_path1, arg_audio, AUDIO_FILE], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + logger.info('Deepspeech output...... {}'.format(out)) + try: + stdout,stderr = out.communicate() + except: + return "error", "error" + print(stderr) + return stdout,stderr + +def deep_speech_output_clean(result): + """ + Parsing deep speech output(text) + Return: + Text from speech + """ + text = "" + if len(result)>0: + res_split = str(result[0]).split('\\n') + + if len(res_split)>0: + for i in range(len(res_split)): + if 'Inference took' in res_split[i]: + text = res_split[i + 1] + return text + return text + +def process_deepspeech(audio_file,deep_path): + """ + Transcribing audio to extract text from speech + """ + deep_output = deepspeech(audio_file,deep_path) + deep_text= deep_speech_output_clean(deep_output) + + return deep_text diff --git a/process_data.py b/process_data.py index 0283f9f2..90b37d43 100644 --- a/process_data.py +++ b/process_data.py @@ -20,6 +20,7 @@ logging.basicConfig(level=logging.INFO) logger=logging.getLogger() OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction' +DEEP_SPEECH = 'pkg/DeepSpeech' DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat' def common_video(video_file, args, r_config): @@ -36,6 +37,8 @@ def common_video(video_file, args, r_config): pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) + pf.process_nlp(video_file, out_path, args.dbm_group, r_config, DEEP_SPEECH) + pf.remove_file(video_file) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) @@ -79,6 +82,7 @@ def process_raw_audio_file(args, s_config, r_config): out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio_file[0], out_path, args.dbm_group, r_config) + pf.process_nlp(audio_file[0], out_path, args.dbm_group, r_config, DEEP_SPEECH) else: logger.info('Enter correct audio(*.wav) file path.') @@ -130,6 +134,8 @@ def process_raw_audio_dir(args, s_config, r_config): out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio, out_path, args.dbm_group, r_config) + pf.process_nlp(audio, out_path, args.dbm_group, r_config, DEEP_SPEECH) + except Exception as e: logger.error('Failed to process wav file.') diff --git a/process_dbm.sh b/process_dbm.sh index cad71b7f..361d2aed 100644 --- a/process_dbm.sh +++ b/process_dbm.sh @@ -55,6 +55,9 @@ fi if [[ $dbm_group == *"movement"* ]]; then dbm_new="$dbm_new movement" fi +if [[ $dbm_group == *"nlp"* ]]; then + dbm_new="$dbm_new nlp" +fi #docker commands to run container docker create -ti --name dbm_container dbm bash diff --git a/requirements.txt b/requirements.txt index 01f03057..1ecea7d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ more_itertools scipy==1.2.0 pyyaml pydub +deepspeech \ No newline at end of file diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index 982bf631..f8b00883 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -196,3 +196,6 @@ raw_feature: mov_Hpose_Yaw: mov_hposeyaw mov_Hpose_Roll: mov_hposeroll mov_Hpose_Dist: mov_hposedist + + #NLP markers + nlp_transcribe: nlp_transcribe From dae5eb3cd4799d25c1bdad19d9a7e5a598a25ce3 Mon Sep 17 00:00:00 2001 From: Vijay Yadev Date: Fri, 13 Nov 2020 01:03:23 -0500 Subject: [PATCH 02/26] nlp feature --- dbm_lib/config/config_raw_feature.py | 17 ++ dbm_lib/controller/process_feature.py | 3 +- .../raw_features/nlp/speech_features.py | 47 ++++++ .../raw_features/nlp/transcribe.py | 12 +- .../raw_features/util/nlp_util.py | 146 ++++++++++++++++++ resources/features/derived_feature.yml | 28 +++- resources/features/raw_feature.yml | 18 +++ 7 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/nlp/speech_features.py diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 679c0845..3dc4834b 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -225,4 +225,21 @@ class ConfigRawReader(object): #NLP features self.nlp_transcribe = config['raw_feature']['nlp_transcribe'] + self.nlp_numSentences = config['raw_feature']['nlp_numSentences'] + self.nlp_singPronPerAns = config['raw_feature']['nlp_singPronPerAns'] + self.nlp_singPronPerSen = config['raw_feature']['nlp_singPronPerSen'] + self.nlp_pastTensePerAns = config['raw_feature']['nlp_pastTensePerAns'] + self.nlp_pastTensePerSen = config['raw_feature']['nlp_pastTensePerSen'] + self.nlp_pronounsPerAns = config['raw_feature']['nlp_pronounsPerAns'] + self.nlp_pronounsPerSen = config['raw_feature']['nlp_pronounsPerSen'] + self.nlp_verbsPerAns = config['raw_feature']['nlp_verbsPerAns'] + self.nlp_verbsPerSen = config['raw_feature']['nlp_verbsPerSen'] + self.nlp_adjectivesPerAns = config['raw_feature']['nlp_adjectivesPerAns'] + self.nlp_adjectivesPerSen = config['raw_feature']['nlp_adjectivesPerSen'] + self.nlp_nounsPerAns = config['raw_feature']['nlp_nounsPerAns'] + self.nlp_nounsPerSen = config['raw_feature']['nlp_nounsPerSen'] + self.nlp_sentiment_mean = config['raw_feature']['nlp_sentiment_mean'] + self.nlp_mattr = config['raw_feature']['nlp_mattr'] + self.nlp_wordsPerMin = config['raw_feature']['nlp_wordsPerMin'] + self.nlp_totalTime = config['raw_feature']['nlp_totalTime'] \ No newline at end of file diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 902ef2c6..be781a7f 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -8,7 +8,7 @@ from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink -from dbm_lib.dbm_features.raw_features.nlp import transcribe +from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features import subprocess import logging @@ -137,6 +137,7 @@ def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): logger.info('Processing nlp variables from data in {}'.format(video_uri)) transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path) + speech_features.run_speech_feature(video_uri, out_dir, r_config) def remove_file(file_path): """ diff --git a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py new file mode 100644 index 00000000..6aebd547 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py @@ -0,0 +1,47 @@ +""" +file_name: speech_features +project_name: DBM +created: 2020-13-11 +""" + +import os +import numpy as np +import pandas as pd +import glob +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +speech_dir = 'nlp/speech_feature' +speech_ext = '_nlp.csv' +transcribe_ext = 'nlp/transcribe/*_transcribe.csv' + +def run_speech_feature(video_uri, out_dir, r_config): + """ + Processing all patient's for fetching nlp features + ------------------- + ------------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output + """ + try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + + transcribe_path = glob.glob(join(out_loc, transcribe_ext)) + if len(transcribe_path)>0: + + transcribe_df = pd.read_csv(transcribe_path[0]) + df_speech= n_util.process_speech(transcribe_df, r_config) + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext) + + except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py index 3914f78a..f567e967 100644 --- a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py +++ b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py @@ -21,7 +21,7 @@ formant_dir = 'nlp/transcribe' csv_ext = '_transcribe.csv' error_txt = 'error: length less than 0.1' -def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path): +def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur): """ Preparing Formant freq matrix Args: @@ -33,6 +33,7 @@ def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe]) df_formant.replace('', np.nan, regex=True,inplace=True) + df_formant[r_config.nlp_totalTime] = aud_dur df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release df_formant['dbm_master_url'] = video_uri @@ -44,8 +45,8 @@ def empty_transcribe(video_uri, out_loc, fl_name, r_config): """ Preparing empty formant frequency matrix if something fails """ - cols = [r_config.nlp_transcribe, r_config.err_reason] - out_val = [[np.nan, error_txt]] + cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason] + out_val = [[np.nan, np.nan, error_txt]] df_fm = pd.DataFrame(out_val, columns = cols) df_fm['dbm_master_url'] = video_uri @@ -77,6 +78,7 @@ def run_transcribe(video_uri, out_dir, r_config, deep_path): empty_transcribe(video_uri, out_loc, fl_name, r_config) return - calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path) + calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur) except Exception as e: - logger.error('Failed to process audio file') \ No newline at end of file + logger.error('Failed to process audio file') + \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/util/nlp_util.py b/dbm_lib/dbm_features/raw_features/util/nlp_util.py index 3288240b..fc1ac3d1 100644 --- a/dbm_lib/dbm_features/raw_features/util/nlp_util.py +++ b/dbm_lib/dbm_features/raw_features/util/nlp_util.py @@ -11,6 +11,11 @@ import pandas as pd import os import logging +import nltk +import re +from lexicalrichness import LexicalRichness +from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + logging.basicConfig(level=logging.INFO) logger=logging.getLogger() @@ -64,3 +69,144 @@ def process_deepspeech(audio_file,deep_path): deep_text= deep_speech_output_clean(deep_output) return deep_text + +def nltk_download(): + + try: + nltk.data.find('tokenizers/punkt') + + except LookupError: + logger.info('punkt is not available') + nltk.download('punkt') + + try: + nltk.data.find('averaged_perceptron_tagger') + + except LookupError: + logger.info('averaged_perceptron_tagger is not available') + nltk.download('averaged_perceptron_tagger') + +def empty_speech(r_config, master_url, error_txt): + """ + Preparing empty speech matrix with error + Args: + r_config: raw config file object + error_txt: Error message during transcription + + Returns: + Empty dataframe for speech features with error + """ + + col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns, + r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns, + r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns, + r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin, + r_config.nlp_totalTime, r_config.err_reason] + + df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col) + df_speech['dbm_master_url'] = master_url + + return df_speech + +def divide_var(speech_var1, spech_var2): + """ + divide variables + """ + speech_var = np.nan + if spech_var2!=0: + speech_var = speech_var1/spech_var2 + return speech_var + +def process_speech(transcribe_df,r_config): + """ + Preparing speech features + Args: + transcribe_df: Transcribed dataframe + r_config: raw config file object + Returns: + Dataframe for speech features + """ + + err_transcribe = transcribe_df[r_config.err_reason].iloc[0] + transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0] + total_time = transcribe_df[r_config.nlp_totalTime].iloc[0] + master_url = transcribe_df['dbm_master_url'].iloc[0] + + #clean transcribe + transcribe = transcribe.replace(",", "") + transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe)) + + if err_transcribe != 'Pass': + df_speech = empty_speech(r_config, master_url, error_txt) + + return df_speech + + speech_dict = {} + nltk_download() + + sentences = nltk.tokenize.sent_tokenize(transcribe) + words_all = nltk.tokenize.word_tokenize(transcribe) + num_sentences = len(sentences) + + speech_dict[r_config.nlp_numSentences] = num_sentences + + #nlp_singPron + i_s = transcribe.count('I') + me_s = transcribe.count('me') + my_s = transcribe.count('my') + sing_count = i_s + me_s + my_s + + speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences) + + tagged = nltk.pos_tag(transcribe.split()) + tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag']) + + #Past tense per answer + all_POSs = tagged_df['pos_tag'].tolist() + speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences) + + #Pronoun per answer + pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$') + speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences) + + #Verb per answer + verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \ + + all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ') + speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences) + + #Adjective per answer + adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS') + speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences) + + #Noun per answer + nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS') + speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences) + + #Sentiment analysis + vader = SentimentIntensityAnalyzer() + sentence_valences = [] + + for s in sentences: + sentiment_dict = vader.polarity_scores(s) + sentence_valences.append(sentiment_dict['compound']) + + speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan + non_punc = list(value for value in words_all if value not in ['.','!','?']) + + non_punc_as_str = " ".join(str(non_punc)) + lex = LexicalRichness(non_punc_as_str) + speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan + + #Number of words per minute + speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60 + speech_dict[r_config.nlp_totalTime] = total_time + speech_dict['dbm_master_url'] = master_url + + df_speech = pd.DataFrame([speech_dict]) + return df_speech \ No newline at end of file diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index a0759d79..7346eb36 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,7 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -22,6 +22,7 @@ derive_feature: MOV_HM_LOC: _headmov MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks + NLP_SPEECH_LOC: _nlp #Facial category feature group FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom'] @@ -65,6 +66,12 @@ derive_feature: MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] + #NLP category feature group + NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen', + 'nlp_pronounsPerAns', 'nlp_pronounsPerSen', 'nlp_verbsPerAns', 'nlp_verbsPerSen', 'nlp_adjectivesPerAns', + 'nlp_adjectivesPerSen', 'nlp_nounsPerAns', 'nlp_nounsPerSen', 'nlp_sentiment_mean', 'nlp_mattr', 'nlp_wordsPerMin', + 'nlp_totalTime'] + #Calculation for variables # Facial Asymmetry fac_AsymMaskMouth: ['mean', 'std'] @@ -248,3 +255,22 @@ derive_feature: mov_blink_ear: ['mean', 'std'] vid_dur: ['count'] mov_blinkdur: ['mean', 'std'] + + #NLP feature + nlp_numSentences: ['mean'] + nlp_singPronPerAns: ['mean'] + nlp_singPronPerSen: ['mean'] + nlp_pastTensePerAns: ['mean'] + nlp_pastTensePerSen: ['mean'] + nlp_pronounsPerAns: ['mean'] + nlp_pronounsPerSen: ['mean'] + nlp_verbsPerAns: ['mean'] + nlp_verbsPerSen: ['mean'] + nlp_adjectivesPerAns: ['mean'] + nlp_adjectivesPerSen: ['mean'] + nlp_nounsPerAns: ['mean'] + nlp_nounsPerSen: ['mean'] + nlp_sentiment_mean: ['mean'] + nlp_mattr: ['mean'] + nlp_wordsPerMin: ['mean'] + nlp_totalTime: ['mean'] diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index f8b00883..b9c673b8 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -199,3 +199,21 @@ raw_feature: #NLP markers nlp_transcribe: nlp_transcribe + nlp_numSentences: nlp_numSentences + nlp_singPronPerAns: nlp_singPronPerAns + nlp_singPronPerSen: nlp_singPronPerSen + nlp_pastTensePerAns: nlp_pastTensePerAns + nlp_pastTensePerSen: nlp_pastTensePerSen + nlp_pronounsPerAns: nlp_pronounsPerAns + nlp_pronounsPerSen: nlp_pronounsPerSen + nlp_verbsPerAns: nlp_verbsPerAns + nlp_verbsPerSen: nlp_verbsPerSen + nlp_adjectivesPerAns: nlp_adjectivesPerAns + nlp_adjectivesPerSen: nlp_adjectivesPerSen + nlp_nounsPerAns: nlp_nounsPerAns + nlp_nounsPerSen: nlp_nounsPerSen + nlp_sentiment_mean: nlp_sentiment_mean + nlp_mattr: nlp_mattr + nlp_wordsPerMin: nlp_wordsPerMin + nlp_totalTime: nlp_totalTime + \ No newline at end of file From a5f50210f9f38d1b8628851c6a6ef6134bff270b Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Fri, 13 Nov 2020 02:01:49 -0500 Subject: [PATCH 03/26] nlp requirement --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1ecea7d7..278428b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,7 @@ more_itertools scipy==1.2.0 pyyaml pydub -deepspeech \ No newline at end of file +deepspeech +nltk +lexicalrichness +vaderSentiment From bfa42917b9328c1e37fbfd87746d1a6fd9f2aae2 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Fri, 13 Nov 2020 20:14:59 -0500 Subject: [PATCH 04/26] requirement updates --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 278428b1..6c90cf09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ deepspeech nltk lexicalrichness vaderSentiment +textblob From f9f5b4ec5f5cf54c34e28b15d153657cbf4d0130 Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Wed, 18 Nov 2020 23:33:30 -0500 Subject: [PATCH 05/26] added vocal tremor --- dbm_lib/config/config_raw_feature.py | 26 +- dbm_lib/controller/process_feature.py | 60 +-- .../raw_features/movement/__init__.py | 2 + .../raw_features/movement/voice_tremor.py | 93 ++++ process_data.py | 46 +- process_dbm.sh | 8 +- resources/features/derived_feature.yml | 82 ++-- resources/features/raw_feature.yml | 29 +- resources/libraries/voice_tremor.praat | 422 ++++++++++++++++++ 9 files changed, 663 insertions(+), 105 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/movement/voice_tremor.py create mode 100644 resources/libraries/voice_tremor.praat diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 5494e21d..e0878dcf 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -17,7 +17,7 @@ class ConfigRawReader(object): Args: feature_config_yml (None, optional): yml file defined service configuration """ - + if feature_config_yml is None: feature_config = DBMLIB_FEATURE_CONFIG else: @@ -25,15 +25,15 @@ class ConfigRawReader(object): with open(feature_config, 'r') as ymlfile: config = yaml.load(ymlfile) - + #Verbal features self.base_raw = config self.err_reason = config['raw_feature']['error_reason'] - + #Output range self.mov_headvel_start = config['raw_feature']['mov_headvel_start'] self.mov_headvel_end = config['raw_feature']['mov_headvel_end'] - + #Acoustic variable self.aco_int = config['raw_feature']['aco_int'] self.aco_ff = config['raw_feature']['aco_ff'] @@ -210,7 +210,7 @@ class ConfigRawReader(object): self.fac_AsymMaskEye = config['raw_feature']['fac_AsymMaskEye'] self.fac_AsymMaskEyebrow = config['raw_feature']['fac_AsymMaskEyebrow'] self.fac_AsymMaskCom = config['raw_feature']['fac_AsymMaskCom'] - + #Movement features self.head_vel = config['raw_feature']['head_vel'] self.mov_blink_ear = config['raw_feature']['mov_blink_ear'] @@ -222,4 +222,18 @@ class ConfigRawReader(object): self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw'] self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll'] self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist'] - \ No newline at end of file + self.mov_freq_trem_freq = config['raw_feature']['mov_freq_trem_freq'] + self.mov_freq_trem_index = config['raw_feature']['mov_freq_trem_index'] + self.mov_freq_trem_pindex = config['raw_feature']['mov_freq_trem_pindex'] + self.mov_amp_trem_freq = config['raw_feature']['mov_amp_trem_freq'] + self.mov_amp_trem_index = config['raw_feature']['mov_amp_trem_index'] + self.mov_amp_trem_pindex = config['raw_feature']['mov_amp_trem_pindex'] + self.fac_tremor_median_5 = config['raw_feature']['fac_tremor_median_5'] + self.fac_tremor_median_12 = config['raw_feature']['fac_tremor_median_12'] + self.fac_tremor_median_8 = config['raw_feature']['fac_tremor_median_8'] + self.fac_tremor_median_48 = config['raw_feature']['fac_tremor_median_48'] + self.fac_tremor_median_54 = config['raw_feature']['fac_tremor_median_54'] + self.fac_tremor_median_28 = config['raw_feature']['fac_tremor_median_28'] + self.fac_tremor_median_51 = config['raw_feature']['fac_tremor_median_51'] + self.fac_tremor_median_66 = config['raw_feature']['fac_tremor_median_66'] + self.fac_tremor_median_57 = config['raw_feature']['fac_tremor_median_57'] diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 3282edc2..f5112da2 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor import subprocess import logging @@ -19,13 +19,13 @@ logging.basicConfig(level=logging.INFO) logger=logging.getLogger() def audio_to_wav(input_filepath): - """ Extracts a video's audio file and saves it to wav + """ Extracts a video's audio file and saves it to wav Args: - input_filepath: (str) + input_filepath: (str) Returns: """ try: - + fname, _ = splitext(input_filepath) output_filepath = fname + '.wav' @@ -37,7 +37,7 @@ def audio_to_wav(input_filepath): logger.info('wav output saved in {}'.format(output_filepath)) else: logger.info('Output file {} already exists'.format(output_filepath)) - + except Exception as e: logger.error('Failed to extract audio from Video') @@ -50,38 +50,41 @@ def process_acoustic(video_uri, out_dir, dbm_group, r_config): """ if dbm_group != None and len(dbm_group)>0 and 'acoustic' not in dbm_group: return - + logger.info('Processing acoustic variables from data in {}'.format(video_uri)) logger.info('processing audio intensity....') intensity.run_intensity(video_uri, out_dir, r_config) - + logger.info('processing audio pitch freq....') pitch_freq.run_pitch(video_uri, out_dir, r_config) - + logger.info('processing HNR....') hnr.run_hnr(video_uri, out_dir, r_config) - + logger.info('processing GNE....') gne.run_gne(video_uri, out_dir, r_config) - + logger.info('processing voice frame score....') voice_frame_score.run_vfs(video_uri, out_dir, r_config) - + logger.info('processing formant frequency....') formant_freq.run_formant(video_uri, out_dir, r_config) - + logger.info('processing pause segment....') pause_segment.run_pause_segment(video_uri, out_dir, r_config) - + logger.info('processing jitter....') jitter.run_jitter(video_uri, out_dir, r_config) - + logger.info('processing shimmer....') shimmer.run_shimmer(video_uri, out_dir, r_config) - + logger.info('processing mfcc....') mfcc.run_mfcc(video_uri, out_dir, r_config) - + + logger.info('processing voice tremor....') + voice_tremor.run_vtremor(video_uri, out_dir, r_config) + def process_facial(video_uri, out_dir, dbm_group, r_config): """ processing facial features @@ -91,20 +94,20 @@ def process_facial(video_uri, out_dir, dbm_group, r_config): """ if dbm_group != None and len(dbm_group)>0 and 'facial' not in dbm_group: return - + logger.info('Processing facial variables from data in {}'.format(video_uri)) logger.info('processing facial asymmetry....') face_asymmetry.run_face_asymmetry(video_uri, out_dir, r_config) - + logger.info('processing facial Action Unit....') face_au.run_face_au(video_uri, out_dir, r_config) - + logger.info('processing facial expressivity....') face_emotion_expressivity.run_face_expressivity(video_uri, out_dir, r_config) - + logger.info('processing facial landmark....') face_landmark.run_face_landmark(video_uri, out_dir, r_config) - + def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): """ processing facial features @@ -115,23 +118,24 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): """ if dbm_group != None and len(dbm_group)>0 and 'movement' not in dbm_group: return - + logger.info('Processing movement variables from data in {}'.format(video_uri)) logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) - + logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) - + + logger.info('processing voice tremor....') + voice_tremor.run_vtremor(video_uri, out_dir, r_config) + def remove_file(file_path): """ - removing wav file + removing wav file """ file_dir = dirname(file_path) file_name, _ = splitext(basename(file_path)) wav_file = glob.glob(join(file_dir, file_name + '.wav')) - + if len(wav_file)> 0: os.remove(wav_file[0]) - - \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index f2a2d4b9..dca2f624 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -11,3 +11,5 @@ from __future__ import print_function import os DBMLIB_PATH = os.path.dirname(__file__) +DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, + '../../../../resources/libraries/voice_tremor.praat')) diff --git a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py new file mode 100644 index 00000000..52f9c398 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py @@ -0,0 +1,93 @@ +import pandas as pd +import os +import glob +from os.path import join +import parselmouth +from parselmouth.praat import call, run_file +import numpy as np +import librosa +import json +import re +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.movement import DBMLIB_VTREMOR_LIB + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +vt_dir = 'movement/voice_tremor' +csv_ext = '_vtremor.csv' + +#Executing praat script using parselmouth function +def tremor_praat(snd_file,r_cfg): + """ + Generating Voice tremor endpoint dataframe + Args: + snd_file: (.wav) parsed audio file + r_cfg: Raw variable configuration file + Returns tremor endpoint dataframe + """ + snd = parselmouth.Sound(snd_file) + tremor_var = run_file(snd,DBMLIB_VTREMOR_LIB,capture_output=True) + new_tremor_var = re.sub('--undefined--', '0', tremor_var[1]) + res = json.loads(new_tremor_var) + tremor_df = pd.DataFrame(res,index=['0',]) + tremor_df.columns = [r_cfg.mov_freq_trem_freq,r_cfg.mov_amp_trem_freq,r_cfg.mov_freq_trem_index, + r_cfg.mov_amp_trem_index,r_cfg.mov_freq_trem_pindex,r_cfg.mov_amp_trem_pindex] + return tremor_df + +def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name): + """ + Preparing voice tremor matrix + Args: + audio_file: (.wav) parsed audio file ; r_config: raw config object + out_loc: (str) Output directory for csv ; fl_name: file name + """ + df_tremor = tremor_praat(audio_file, r_config) + df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release + + logger.info('Processing Output file {} '.format(out_loc)) + ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) + +def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): + + """ + Preparing empty voice tremor matrix + """ + cols = [r_config.mov_freq_trem_freq, r_config.mov_amp_trem_freq, r_config.mov_freq_trem_index, + r_config.mov_amp_trem_index, r_config.mov_freq_trem_pindex, r_config.mov_amp_trem_pindex, r_config.err_reason] + + out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] + df_tremor = pd.DataFrame(out_val, columns = cols) + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) + +def run_vtremor(video_uri, out_dir, r_config): + """ + Processing all patient's for fetching Formant freq + --------------- + --------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output + """ + try: + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) + + if len(aud_filter)>0: + + audio_file = aud_filter[0] + aud_dur = librosa.get_duration(filename=audio_file) + + if float(aud_dur) < 0.064: + logger.info('Output file {} size is less than 0.064sec'.format(audio_file)) + + prepare_empty_vt(video_uri, out_loc, fl_name, r_config) + return + prepare_vtrem_output(audio_file, out_loc, r_config, fl_name) + except Exception as e: + logger.error('Failed to compute Voice Tremor {} for {}'.format(e,video_uri)) + prepare_empty_vt(out_loc, fl_name, r_config, e) diff --git a/process_data.py b/process_data.py index 0283f9f2..31f85fdb 100644 --- a/process_data.py +++ b/process_data.py @@ -33,11 +33,10 @@ def common_video(video_file, args, r_config): out_path = os.path.join(args.output_path, 'raw_variables') pf.audio_to_wav(video_file) of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group) - pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) - pf.remove_file(video_file) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) + pf.remove_file(video_file) def process_raw_video_file(args, s_config, r_config): """ @@ -57,7 +56,7 @@ def process_raw_video_file(args, s_config, r_config): else: logger.info('Enter correct video(*.mp4) file path.') - + except Exception as e: logger.error('Failed to process mp4 file.') pf.remove_file(video_file[0]) @@ -76,15 +75,15 @@ def process_raw_audio_file(args, s_config, r_config): if len(audio_file)>0: logger.info('Calculating raw variables...') - + out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio_file[0], out_path, args.dbm_group, r_config) - + else: logger.info('Enter correct audio(*.wav) file path.') except Exception as e: logger.error('Failed to process wav file.') - + def process_raw_video_dir(args, s_config, r_config): """ Processing video file @@ -95,16 +94,15 @@ def process_raw_video_dir(args, s_config, r_config): """ if args.output_path != None: vid_loc = glob.glob(args.input_path + '/*.mp4') - + if len(vid_loc) == 0: logger.info('Directory does not have any MP4 files.') return - + logger.info('Calculating raw variables...') for vid_file in vid_loc: try: - - common_video(vid_file, args, r_config) + common_video(vid_file, args, r_config) except Exception as e: logger.error('Failed to process mp4 file.') pf.remove_file(vid_file) @@ -119,20 +117,20 @@ def process_raw_audio_dir(args, s_config, r_config): """ if args.output_path != None: audio_loc = glob.glob(args.input_path + '/*.wav') - + if len(audio_loc) == 0: logger.info('Directory does not have any WAV files.') return - + logger.info('Calculating raw variables...') for audio in audio_loc: try: - + out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio, out_path, args.dbm_group, r_config) except Exception as e: logger.error('Failed to process wav file.') - + def process_derive(args, r_config, d_config, input_type): """ Processing dbm derived variables @@ -141,28 +139,28 @@ def process_derive(args, r_config, d_config, input_type): input_file = glob.glob(args.input_path) else: input_file = glob.glob(args.input_path + '/*') - + out_raw_path = os.path.join(args.output_path, 'raw_variables') out_derive_path = os.path.join(args.output_path, 'derived_variables') - + logger.info('Calculating derived variables...') feature_df = der.run_derive(input_file, out_raw_path, out_derive_path, r_config, d_config) - + if __name__=="__main__": start_time = time.time() parser = argparse.ArgumentParser(description="Process video/audio......") - + parser.add_argument("--input_path", help="path to the input files", required=True) parser.add_argument("--output_path", help="path to the raw and derived variable output", required=True) parser.add_argument("--dbm_group", help="list of feature groups", nargs='+') - + args = parser.parse_args() s_config = config_reader.ConfigReader() r_config = config_raw_feature.ConfigRawReader() d_config = config_derive_feature.ConfigDeriveReader() - + _, file_ext = os.path.splitext(os.path.basename(args.input_path)) - + if file_ext: input_type = 'file' if file_ext.lower() == '.mp4': @@ -170,14 +168,14 @@ if __name__=="__main__": elif file_ext.lower() == '.wav': process_raw_audio_file(args, s_config, r_config) - + else: logger.error('No WAV or MP4 files detected in input path') else: input_type = 'dir' process_raw_video_dir(args, s_config, r_config) process_raw_audio_dir(args, s_config, r_config) - + process_derive(args, r_config, d_config, input_type) exec_time = time.time() - start_time - logger.info('Done! Processing time: {} seconds'.format(exec_time)) \ No newline at end of file + logger.info('Done! Processing time: {} seconds'.format(exec_time)) diff --git a/process_dbm.sh b/process_dbm.sh index cad71b7f..424154bf 100644 --- a/process_dbm.sh +++ b/process_dbm.sh @@ -4,9 +4,9 @@ helpFunction() { echo "" echo "Usage: $0 --input_path parameterA --output_path parameterB --dbm_group parameterC" - echo -e "\t--input_path Description of what is parameterA" - echo -e "\t--output_path Description of what is parameterB" - echo -e "\t-dbm_group Description of what is parameterC" + echo -e "\t--input_path: path to the input files" + echo -e "\t--output_path: path to the raw and derived variable output" + echo -e "\t--dbm_group: list of feature groups" exit 1 # Exit script after printing help } @@ -72,4 +72,4 @@ docker cp dbm_container:/app/output $output_path docker stop dbm_container docker rm dbm_container -exit \ No newline at end of file +exit diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index a0759d79..43db29dc 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -1,9 +1,9 @@ derive_feature: - + #DBM Feature Group - FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK'] - + FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT'] + #Feature group output file extensions FAC_ASYM_LOC: _facasym FAC_AU_LOC: _facau @@ -22,31 +22,33 @@ derive_feature: MOV_HM_LOC: _headmov MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks - + MOV_VT_LOC: _vtremor + + #Facial category feature group FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom'] - FAC_AU: ['fac_AU01int', 'fac_AU02int', 'fac_AU04int', 'fac_AU05int', 'fac_AU06int', 'fac_AU07int', 'fac_AU09int', - 'fac_AU10int', 'fac_AU12int', 'fac_AU14int', 'fac_AU15int', 'fac_AU17int', 'fac_AU20int', 'fac_AU23int', - 'fac_AU25int', 'fac_AU26int', 'fac_AU45int', 'fac_AU01pres', 'fac_AU02pres', 'fac_AU04pres', 'fac_AU05pres', - 'fac_AU06pres', 'fac_AU07pres', 'fac_AU09pres', 'fac_AU10pres', 'fac_AU12pres', 'fac_AU14pres', 'fac_AU15pres', + FAC_AU: ['fac_AU01int', 'fac_AU02int', 'fac_AU04int', 'fac_AU05int', 'fac_AU06int', 'fac_AU07int', 'fac_AU09int', + 'fac_AU10int', 'fac_AU12int', 'fac_AU14int', 'fac_AU15int', 'fac_AU17int', 'fac_AU20int', 'fac_AU23int', + 'fac_AU25int', 'fac_AU26int', 'fac_AU45int', 'fac_AU01pres', 'fac_AU02pres', 'fac_AU04pres', 'fac_AU05pres', + 'fac_AU06pres', 'fac_AU07pres', 'fac_AU09pres', 'fac_AU10pres', 'fac_AU12pres', 'fac_AU14pres', 'fac_AU15pres', 'fac_AU17pres', 'fac_AU20pres', 'fac_AU23pres', 'fac_AU25pres', 'fac_AU26pres', 'fac_AU28pres', 'fac_AU45pres'] FAC_EXP: ['hap_exp', 'sad_exp', 'sur_exp', 'fea_exp', 'ang_exp', 'dis_exp', 'con_exp', 'happ_occ', 'sad_occ', 'sur_occ', 'fea_occ', 'ang_occ', 'dis_occ', 'con_occ', 'pos_exp', 'neg_exp', 'com_exp', 'hap_exp_full', 'sad_exp_full', 'sur_exp_full','fea_exp_full', 'ang_exp_full', 'dis_exp_full', 'con_exp_full', 'pos_exp_full', 'neg_exp_full', 'com_exp_full'] - FAC_LMK: ['fac_LMK00disp', 'fac_LMK01disp', 'fac_LMK02disp', 'fac_LMK03disp', 'fac_LMK04disp', - 'fac_LMK05disp', 'fac_LMK06disp', 'fac_LMK07disp', 'fac_LMK08disp', 'fac_LMK09disp', 'fac_LMK10disp', - 'fac_LMK11disp', 'fac_LMK12disp', 'fac_LMK13disp', 'fac_LMK14disp', 'fac_LMK15disp', 'fac_LMK16disp', - 'fac_LMK17disp', 'fac_LMK18disp', 'fac_LMK19disp', 'fac_LMK20disp', 'fac_LMK21disp', 'fac_LMK22disp', - 'fac_LMK23disp', 'fac_LMK24disp', 'fac_LMK25disp', 'fac_LMK26disp', 'fac_LMK27disp', 'fac_LMK28disp', - 'fac_LMK29disp', 'fac_LMK30disp', 'fac_LMK31disp', 'fac_LMK32disp', 'fac_LMK33disp', 'fac_LMK34disp', - 'fac_LMK35disp', 'fac_LMK36disp', 'fac_LMK37disp', 'fac_LMK38disp', 'fac_LMK39disp', 'fac_LMK40disp', - 'fac_LMK41disp', 'fac_LMK42disp', 'fac_LMK43disp', 'fac_LMK44disp', 'fac_LMK45disp', 'fac_LMK46disp', - 'fac_LMK47disp', 'fac_LMK48disp', 'fac_LMK49disp', 'fac_LMK50disp', 'fac_LMK51disp', 'fac_LMK52disp', - 'fac_LMK53disp', 'fac_LMK54disp', 'fac_LMK55disp', 'fac_LMK56disp', 'fac_LMK57disp', 'fac_LMK58disp', - 'fac_LMK59disp', 'fac_LMK60disp', 'fac_LMK61disp', 'fac_LMK62disp', 'fac_LMK63disp', 'fac_LMK64disp', + FAC_LMK: ['fac_LMK00disp', 'fac_LMK01disp', 'fac_LMK02disp', 'fac_LMK03disp', 'fac_LMK04disp', + 'fac_LMK05disp', 'fac_LMK06disp', 'fac_LMK07disp', 'fac_LMK08disp', 'fac_LMK09disp', 'fac_LMK10disp', + 'fac_LMK11disp', 'fac_LMK12disp', 'fac_LMK13disp', 'fac_LMK14disp', 'fac_LMK15disp', 'fac_LMK16disp', + 'fac_LMK17disp', 'fac_LMK18disp', 'fac_LMK19disp', 'fac_LMK20disp', 'fac_LMK21disp', 'fac_LMK22disp', + 'fac_LMK23disp', 'fac_LMK24disp', 'fac_LMK25disp', 'fac_LMK26disp', 'fac_LMK27disp', 'fac_LMK28disp', + 'fac_LMK29disp', 'fac_LMK30disp', 'fac_LMK31disp', 'fac_LMK32disp', 'fac_LMK33disp', 'fac_LMK34disp', + 'fac_LMK35disp', 'fac_LMK36disp', 'fac_LMK37disp', 'fac_LMK38disp', 'fac_LMK39disp', 'fac_LMK40disp', + 'fac_LMK41disp', 'fac_LMK42disp', 'fac_LMK43disp', 'fac_LMK44disp', 'fac_LMK45disp', 'fac_LMK46disp', + 'fac_LMK47disp', 'fac_LMK48disp', 'fac_LMK49disp', 'fac_LMK50disp', 'fac_LMK51disp', 'fac_LMK52disp', + 'fac_LMK53disp', 'fac_LMK54disp', 'fac_LMK55disp', 'fac_LMK56disp', 'fac_LMK57disp', 'fac_LMK58disp', + 'fac_LMK59disp', 'fac_LMK60disp', 'fac_LMK61disp', 'fac_LMK62disp', 'fac_LMK63disp', 'fac_LMK64disp', 'fac_LMK65disp', 'fac_LMK66disp', 'fac_LMK67disp'] - + #Acoustic category feature group ACO_INT: ['aco_int'] ACO_FF: ['aco_ff'] @@ -59,19 +61,21 @@ derive_feature: ACO_VFS: ['aco_voicePct'] ACO_MFCC: ['aco_mfcc1','aco_mfcc2','aco_mfcc3','aco_mfcc4','aco_mfcc5','aco_mfcc6','aco_mfcc7','aco_mfcc8','aco_mfcc9', 'aco_mfcc10','aco_mfcc11','aco_mfcc12'] - + #Movement category feature group MOV_HM: ['head_vel'] MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] - + MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq', + 'mov_amp_trem_index', 'mov_amp_trem_pindex'] + #Calculation for variables # Facial Asymmetry fac_AsymMaskMouth: ['mean', 'std'] fac_AsymMaskEyebrow: ['mean', 'std'] fac_AsymMaskEye: ['mean', 'std'] fac_AsymMaskCom: ['mean', 'std'] - + #Facial Action Unit fac_AU01int: ['mean', 'std'] fac_AU02int: ['mean', 'std'] @@ -103,12 +107,12 @@ derive_feature: fac_AU15pres: ['pct'] fac_AU17pres: ['pct'] fac_AU20pres: ['pct'] - fac_AU23pres: ['pct'] + fac_AU23pres: ['pct'] fac_AU25pres: ['pct'] fac_AU26pres: ['pct'] fac_AU28pres: ['pct'] fac_AU45pres: ['pct'] - + #Facial Expressivity hap_exp: ['mean', 'std'] sad_exp: ['mean', 'std'] @@ -129,7 +133,7 @@ derive_feature: neu_exp: ['mean', 'std', 'pct'] com_exp: ['mean', 'std', 'pct'] hap_exp_full: ['mean', 'std'] - sad_exp_full: ['mean', 'std'] + sad_exp_full: ['mean', 'std'] sur_exp_full: ['mean', 'std'] fea_exp_full: ['mean', 'std'] ang_exp_full: ['mean', 'std'] @@ -139,7 +143,7 @@ derive_feature: neg_exp_full: ['mean', 'std'] neu_exp_full: ['mean', 'std'] com_exp_full: ['mean', 'std'] - + #Facial Landmarks fac_LMK00disp: ['mean', 'std'] fac_LMK01disp: ['mean', 'std'] @@ -151,7 +155,7 @@ derive_feature: fac_LMK07disp: ['mean', 'std'] fac_LMK08disp: ['mean', 'std'] fac_LMK09disp: ['mean', 'std'] - fac_LMK10disp: ['mean', 'std'] + fac_LMK10disp: ['mean', 'std'] fac_LMK11disp: ['mean', 'std'] fac_LMK12disp: ['mean', 'std'] fac_LMK13disp: ['mean', 'std'] @@ -163,22 +167,22 @@ derive_feature: fac_LMK19disp: ['mean', 'std'] fac_LMK20disp: ['mean', 'std'] fac_LMK21disp: ['mean', 'std'] - fac_LMK22disp: ['mean', 'std'] + fac_LMK22disp: ['mean', 'std'] fac_LMK23disp: ['mean', 'std'] fac_LMK24disp: ['mean', 'std'] fac_LMK25disp: ['mean', 'std'] fac_LMK26disp: ['mean', 'std'] fac_LMK27disp: ['mean', 'std'] fac_LMK28disp: ['mean', 'std'] - fac_LMK29disp: ['mean', 'std'] + fac_LMK29disp: ['mean', 'std'] fac_LMK30disp: ['mean', 'std'] fac_LMK31disp: ['mean', 'std'] fac_LMK32disp: ['mean', 'std'] fac_LMK33disp: ['mean', 'std'] - fac_LMK34disp: ['mean', 'std'] + fac_LMK34disp: ['mean', 'std'] fac_LMK35disp: ['mean', 'std'] fac_LMK36disp: ['mean', 'std'] - fac_LMK37disp: ['mean', 'std'] + fac_LMK37disp: ['mean', 'std'] fac_LMK38disp: ['mean', 'std'] fac_LMK39disp: ['mean', 'std'] fac_LMK40disp: ['mean', 'std'] @@ -193,7 +197,7 @@ derive_feature: fac_LMK49disp: ['mean', 'std'] fac_LMK50disp: ['mean', 'std'] fac_LMK51disp: ['mean', 'std'] - fac_LMK52disp: ['mean', 'std'] + fac_LMK52disp: ['mean', 'std'] fac_LMK53disp: ['mean', 'std'] fac_LMK54disp: ['mean', 'std'] fac_LMK55disp: ['mean', 'std'] @@ -209,7 +213,7 @@ derive_feature: fac_LMK65disp: ['mean', 'std'] fac_LMK66disp: ['mean', 'std'] fac_LMK67disp: ['mean', 'std'] - + #Acoustic feature aco_int: ['mean', 'std', 'range'] aco_ff: ['mean', 'std', 'range'] @@ -238,7 +242,7 @@ derive_feature: aco_mfcc10: ['mean'] aco_mfcc11: ['mean'] aco_mfcc12: ['mean'] - + #Movement feature head_vel: ['mean', 'std'] mov_Hpose_Dist: ['mean', 'std'] @@ -248,3 +252,9 @@ derive_feature: mov_blink_ear: ['mean', 'std'] vid_dur: ['count'] mov_blinkdur: ['mean', 'std'] + mov_freq_trem_freq: ['mean'] + mov_freq_trem_index: ['mean'] + mov_freq_trem_pindex: ['mean'] + mov_amp_trem_freq: ['mean'] + mov_amp_trem_index: ['mean'] + mov_amp_trem_pindex: ['mean'] diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index 982bf631..fc61ff99 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -1,11 +1,11 @@ raw_feature: #error reason error_reason: error_reason - + #Output range mov_headvel_start: 0 mov_headvel_end: 200 - + #Facial markers hap_exp: fac_hapintsoft sad_exp: fac_sadintsoft @@ -38,13 +38,13 @@ raw_feature: neu_exp_full: neu_exp_full cai_exp_full: cai_exp_full com_exp_full: fac_cominthard - + #Facial asymmetry fac_AsymMaskMouth: fac_asymmaskmouth fac_AsymMaskEye: fac_asymmaskeye fac_AsymMaskEyebrow: fac_asymmaskeyebrow fac_AsymMaskCom: fac_asymmaskcom - + #Facial landmark fac_LMK00disp: fac_LMK00disp fac_LMK01disp: fac_LMK01disp @@ -114,7 +114,7 @@ raw_feature: fac_LMK65disp: fac_LMK65disp fac_LMK66disp: fac_LMK66disp fac_LMK67disp: fac_LMK67disp - + #Facial action unit fac_AU01int: fac_AU01int fac_AU02int: fac_AU02int @@ -151,7 +151,7 @@ raw_feature: fac_AU26pres: fac_AU26pres fac_AU28pres: fac_AU28pres fac_AU45pres: fac_AU45pres - + #Verbal markers aco_int: aco_int aco_ff: aco_ff @@ -184,7 +184,7 @@ raw_feature: aco_speakingtime: aco_speakingtime aco_numpauses: aco_numpauses aco_pausefrac: aco_pausefrac - + #Movement markers head_vel: mov_headvel mov_blink_ear: mov_blink_ear @@ -196,3 +196,18 @@ raw_feature: mov_Hpose_Yaw: mov_hposeyaw mov_Hpose_Roll: mov_hposeroll mov_Hpose_Dist: mov_hposedist + mov_freq_trem_freq: mov_freqtremfreq + mov_freq_trem_index: mov_freqtremindex + mov_freq_trem_pindex: mov_freqtrempindex + mov_amp_trem_freq: mov_amptremfreq + mov_amp_trem_index: mov_amptremindex + mov_amp_trem_pindex: mov_amptrempindex + fac_tremor_median_5: fac_tremor_median_5 + fac_tremor_median_12: fac_tremor_median_12 + fac_tremor_median_8: fac_tremor_median_8 + fac_tremor_median_48: fac_tremor_median_48 + fac_tremor_median_54: fac_tremor_median_54 + fac_tremor_median_28: fac_tremor_median_28 + fac_tremor_median_51: fac_tremor_median_51 + fac_tremor_median_66: fac_tremor_median_66 + fac_tremor_median_57: fac_tremor_median_57 diff --git a/resources/libraries/voice_tremor.praat b/resources/libraries/voice_tremor.praat new file mode 100644 index 00000000..efeca086 --- /dev/null +++ b/resources/libraries/voice_tremor.praat @@ -0,0 +1,422 @@ + +###################################### +# Global Settings +###################################### +sourcedirec$ = "./"; directory of sounds to be analyzed +minPi = 60; minimal Pitch [Hz] +maxPi = 350; maximal Pitch [Hz] +ts = 0.015; analysis time step [s] +tremthresh = 0.15; minimal autocorr.-coefficient to assume "tremor" +minTr = 1.5; minimal tremor frequency [Hz] +maxTr = 15; maximal tremor frequency [Hz] + + + +###################################### +# Sound (.wav) in, results (.txt) out +###################################### + +# record/load and select the sound to be analyzed!!! + +info$ = Info +name$ = extractWord$(info$, "Object name: ") + +slength = Get total duration +call ftrem +call atrem + +echo +...{"FTrF": 'ftrf:2#', "ATrF":'atrf:2',"FTrI":'ftri:3',"ATrI":'atri:3',"FTrP":'ftrp:3',"ATrP":'atrp:3'} + + + +###################################### +# Frequency Tremor Analysis +###################################### +procedure ftrem + To Pitch (cc)... ts minPi 15 yes 0.03 0.3 0.01 0.35 0.14 maxPi + +#Edit +#pause + +# because PRAAT only runs "Subtract linear fit" if the last frame is "voiceless" (!?): +# numberOfFrames+1 (1) + numberOfFrames = Get number of frames + x1 = Get time from frame number... 1 + am_F0 = Get mean... 0 0 Hertz + + Create Matrix... ftrem_0 0 slength numberOfFrames+1 ts x1 1 1 1 1 1 0 + for i from 1 to numberOfFrames + select Pitch 'name$' + f0 = Get value in frame... i Hertz + select Matrix ftrem_0 +# write zeros to matrix where frames are voiceless + if f0 = undefined + Set value... 1 i 0 + else + Set value... 1 i f0 + endif + endfor + +# remove the linear F0 trend (F0 declination) + To Pitch + Subtract linear fit... Hertz + Rename... ftrem_0_lin + +# undo (1) + Create Matrix... ftrem 0 slength numberOfFrames ts x1 1 1 1 1 1 0 + for i from 1 to numberOfFrames + select Pitch ftrem_0_lin + f0 = Get value in frame... i Hertz + select Matrix ftrem +# write zeros to matrix where frames are voiceless + if f0 = undefined + Set value... 1 i 0 + else + Set value... 1 i f0 + endif + endfor + + To Pitch + +# normalize F0-contour by mean F0 + select Matrix ftrem + Formula... (self-am_F0)/am_F0 + +# since zeros in the Matrix (unvoiced frames) become normalized to -1 but +# unvoiced frames should be zero (if anything) +# write zeros to matrix where frames are voiceless + for i from 1 to numberOfFrames + select Pitch ftrem + f0 = Get value in frame... i Hertz + if f0 = undefined + select Matrix ftrem + Set value... 1 i 0 + endif + endfor + +# to calculate autocorrelation (cc-method): + select Matrix ftrem + To Sound (slice)... 1 +# calculate Frequency of Frequency Tremor [Hz] + To Pitch (cc)... slength minTr 15 yes 0.01 tremthresh 0.01 0.35 0.14 maxTr + Rename... ftrem_norm + + ftrf = Get mean... 0 0 Hertz + +# calculate Intensity Index of Frequency Tremor [%] + select Sound ftrem + plus Pitch ftrem_norm + To PointProcess (peaks)... yes no + Rename... Maxima + numberofMaxPoints = Get number of points + ftri_max = 0 + noFMax = 0 + for iPoint from 1 to numberofMaxPoints + select PointProcess Maxima + ti = Get time from index... iPoint + select Sound ftrem + ftri_Point = Get value at time... Average ti Sinc70 + if ftri_Point = undefined + ftri_Point = 0 + noFMax += 1 + endif + ftri_max += abs(ftri_Point) + endfor + +select Sound ftrem +plus PointProcess Maxima +#Edit +#pause + +# ftri_max:= (mean) procentual deviation of F0-maxima from mean F0 at ftrf + numberofMaxima = numberofMaxPoints - noFMax + ftri_max = 100 * ftri_max/numberofMaxima + + select Sound ftrem + plus Pitch ftrem_norm + To PointProcess (peaks)... no yes + Rename... Minima + numberofMinPoints = Get number of points + ftri_min = 0 + noFMin = 0 + for iPoint from 1 to numberofMinPoints + select PointProcess Minima + ti = Get time from index... iPoint + select Sound ftrem + ftri_Point = Get value at time... Average ti Sinc70 + if ftri_Point = undefined + ftri_Point = 0 + noFMin += 1 + endif + ftri_min += abs(ftri_Point) + endfor + +select Sound ftrem +plus PointProcess Minima +#Edit +#pause + + +# ftri_min:= (mean) procentual deviation of F0-minima from mean F0 at ftrf + numberofMinima = numberofMinPoints - noFMin + ftri_min = 100 * ftri_min/numberofMinima + + ftri = (ftri_max + ftri_min) / 2 + + ftrp = ftri * ftrf/(ftrf+1) + +# uncomment to inspect frequnecy tremor objects: +# pause + + select Pitch ftrem +# uncomment if only frequency tremor is to be analyzed: +# plus Pitch 'name$' + plus Matrix ftrem_0 + plus Pitch ftrem_0 + plus Pitch ftrem_0_lin + plus Matrix ftrem + plus Sound ftrem + plus Pitch ftrem_norm + plus PointProcess Maxima + plus PointProcess Minima + Remove + +endproc + + +###################################### +# Amplitude Tremor Analysis +###################################### +procedure atrem + select Sound 'name$' +# uncomment if only amplitude tremor is to be analyzed: +# To Pitch (cc)... ts minPi 15 yes 0.03 0.3 0.01 0.35 0.14 maxPi +# select Sound 'name$' + plus Pitch 'name$' + To PointProcess (cc) + select Sound 'name$' + plus PointProcess 'name$'_'name$' + +# amplitudes are integrals of intensity over periods -- not intensity maxima + To AmplitudeTier (period)... 0 0 0.0001 0.02 1.7 + +#Edit +#pause + +# from here on out: prepare to autocorrelate AmplitudeTier-data +# sample AmplitudeTier at (constant) rate ts + numbOfAmpPoints = Get number of points + first_ampP = Get time from index... 1 + last_ampP = Get time from index... numbOfAmpPoints + +# to be able to -- automatically -- read Amp. values... + Down to TableOfReal + + select Pitch 'name$' + frameNo1 = Get frame number from time... first_ampP + hiframe1 = ceiling(frameNo1) + t_hiframe1 = Get time from frame number... hiframe1 + + frameNoN = Get frame number from time... last_ampP + loframeN = floor(frameNoN) + +# number of Amp. points if (re-)sampled at ts + numbOfPoints_neu = loframeN - hiframe1 + 1 + +# to enable autocorrelation of the Amp.-contour: ->Matrix->Sound + + Create Matrix... atrem_nlc 0 slength numbOfPoints_neu+1 ts t_hiframe1 1 1 1 1 1 2 +# get the mean of the amplitude contour in time windows of constant duration + for point_neu from 1 to numbOfPoints_neu + t = (point_neu-1) * ts + t_hiframe1 + tl = t - ts/2 + tu = t + ts/2 + + select AmplitudeTier 'name$'_'name$'_'name$' + loil = Get low index from time... tl + hiil = Get high index from time... tl + loiu = Get low index from time... tu + hiiu = Get high index from time... tu + + select TableOfReal 'name$'_'name$'_'name$' + if loil = 0 + lotl = 0; time before the first amp. point + druck_lol = Get value... hiil 2; amplitude value before the first amp. point + else + lotl = Get value... loil 1; time value of Amp.Point before tl in the PointProcess [s] + druck_lol = Get value... loil 2; amplitude value before tl in the PointProcess [Pa, ranged from 0 to 1] + endif + + hitl = Get value... hiil 1 + druck_hil = Get value... hiil 2; amplitude value after tl in the PointProcess + + lotu = Get value... loiu 1 + druck_lou = Get value... loiu 2; amplitude value before tu in the PointProcess + + if hiiu = numbOfAmpPoints + 1 + hitu = slength; time after the last amp. point + druck_hiu = Get value... hiil 2; amplitude value after the last amp. point + else + hitu = Get value... hiiu 1; time value after tu in the PointProcess + druck_hiu = Get value... hiiu 2; amplitude value after tu in the PointProcess + endif + + nPinter = loiu - loil; = hiiu - hiil; number of amp.-points between tl and tu + if nPinter > 0 + itinter = 0 + tinter = 0 + druck_tin = 0 + deltat = 0 + for iinter from 1 to nPinter + hilft = itinter + itinter = Get value... loil+iinter 1 + idruck_tin = Get value... loil+iinter 2 + + ideltat = itinter - hilft + druck_tin += idruck_tin * ideltat + tinter += itinter + deltat += ideltat + endfor + + tin = tinter/nPinter + druck_tin = druck_tin/deltat + endif + + druck_tl = ((hitl-tl)*druck_lol + (tl-lotl)*druck_hil) / (hitl-lotl) + druck_tu = ((hitu-tu)*druck_lou + (tu-lotu)*druck_hiu) / (hitu-lotu) + + if nPinter = 0; loil = loiu; hiil = hiiu + druck_mean = (druck_tl + druck_tu) / 2 + else + druck_mean = ((tin-tl)*(druck_tl + druck_tin)/2 + (tu-tin)*(druck_tin + druck_tu)/2) / (tu-tl) + endif + + select Matrix atrem_nlc + Set value... 1 point_neu druck_mean + endfor + + To Pitch + am_Int = Get mean... 0 0 Hertz + +# because PRAAT classifies frequencies in Pitch objects <=0 as "voiceless" and +# therefore parts with extreme INTENSITIES would be considered as "voiceless" +# (irrelevant) after "Subtract linear fit" (1) +# "1" is added to the original Pa-values (ranged from 0 to 1) + select Matrix atrem_nlc + Formula... self+1 + +# because PRAAT only runs "Subtract linear fit" if the last frame is "voiceless"...?(2) + Set value... 1 numbOfPoints_neu+1 0 + +# remove the linear amp.-trend (amplitude declination) +#Formula... self*1000; better for viewing + To Pitch + Rename... hilf_lincorr + Subtract linear fit... Hertz + Rename... atrem + +# undo (1)... + To Matrix + Formula... self-1 + +# normalize Amp. contour by mean Amp. + Formula... (self-am_Int)/am_Int + +# remove last frame, undo (2) + Create Matrix... atrem_besser 0 slength numbOfPoints_neu ts t_hiframe1 1 1 1 1 1 0 + for point_neu from 1 to numbOfPoints_neu + select Matrix atrem + spring = Get value in cell... 1 point_neu + select Matrix atrem_besser + Set value... 1 point_neu spring + endfor + +# to calculate autocorrelation (cc-method) + To Sound (slice)... 1 +# calculate Frequency of Ampitude Tremor [Hz] + To Pitch (cc)... slength minTr 15 yes 0.01 tremthresh 0.01 0.35 0.14 maxTr + Rename... atrem_norm + + atrf = Get mean... 0 0 Hertz + +# calculate Intensity Index of Amplitude Tremor [%] + select Sound atrem_besser + plus Pitch atrem_norm + To PointProcess (peaks)... yes no + Rename... Maxima + numberofMaxPoints = Get number of points + atri_max = 0 + noAMax = 0 + for iPoint from 1 to numberofMaxPoints + select PointProcess Maxima + ti = Get time from index... iPoint + select Sound atrem_besser + atri_Point = Get value at time... 0 ti Sinc70 + if atri_Point = undefined + atri_Point = 0 + noAMax += 1 + endif + atri_max += abs(atri_Point) + endfor + +select Sound atrem_besser +plus PointProcess Maxima +#Edit +#pause + +# atri_max:= (mean) procentual deviation of Amp. maxima from mean Amp.[Pa] at atrf + numberofMaxima = numberofMaxPoints - noAMax + atri_max = 100 * atri_max / numberofMaxima + + select Sound atrem_besser + plus Pitch atrem_norm + To PointProcess (peaks)... no yes + Rename... Minima + numberofMinPoints = Get number of points + atri_min = 0 + noAMin = 0 + for iPoint from 1 to numberofMinPoints + select PointProcess Minima + ti = Get time from index... iPoint + select Sound atrem_besser + atri_Point = Get value at time... 0 ti Sinc70 + if atri_Point = undefined + atri_Point = 0 + noAMin += 1 + endif + atri_min += abs(atri_Point) + endfor + +select Sound atrem_besser +plus PointProcess Minima +#Edit +#pause + +# atri_min:= (mean) procentual deviation of Amp. minima from mean Amp.[Pa] at atrf + numberofMinima = numberofMinPoints - noAMin + atri_min = 100 * atri_min / numberofMinima + + atri = (atri_max + atri_min) / 2 + + atrp = atri * atrf/(atrf+1) + +# uncomment to inspect amplitude tremor objects: +# pause + + select Pitch 'name$' + plus PointProcess 'name$'_'name$' + plus AmplitudeTier 'name$'_'name$'_'name$' + plus TableOfReal 'name$'_'name$'_'name$' + plus Matrix atrem_nlc + plus Pitch atrem_nlc + plus Pitch hilf_lincorr + plus Pitch atrem + plus Matrix atrem + plus Matrix atrem_besser + plus Sound atrem_besser + plus Pitch atrem_norm + plus PointProcess Maxima + plus PointProcess Minima + Remove +endproc \ No newline at end of file From d3920a26a25bbfb93ac81bf6e3020f552e123606 Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Wed, 25 Nov 2020 09:59:42 -0500 Subject: [PATCH 06/26] facial tremor testing --- dbm_lib/controller/process_feature.py | 6 +- .../raw_features/movement/__init__.py | 1 + .../raw_features/movement/facial_tremor.py | 159 ++++++++++++++++++ .../raw_features/util/math_util.py | 57 +++++++ resources/features/facial/config.json | 1 + 5 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 dbm_lib/dbm_features/raw_features/movement/facial_tremor.py create mode 100644 dbm_lib/dbm_features/raw_features/util/math_util.py create mode 100644 resources/features/facial/config.json diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index f5112da2..488b835c 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor, facial_tremor import subprocess import logging @@ -120,6 +120,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): return logger.info('Processing movement variables from data in {}'.format(video_uri)) + logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) @@ -129,6 +130,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing voice tremor....') voice_tremor.run_vtremor(video_uri, out_dir, r_config) + logger.info('processing facial tremor....') + face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + def remove_file(file_path): """ removing wav file diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index dca2f624..03fd8909 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,3 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py new file mode 100644 index 00000000..e6bfb1a5 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -0,0 +1,159 @@ +import sys, os, glob, cv2, re +import pickle, json +import pandas as pd +import numpy as np +import numpy.ma as ma +import logging +from os.path import join + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util.math_util import * + +from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +ft_dir = 'movement/facial_tremor' +csv_ext = '_fac_tremor.csv' +model_ext = '_fac_model.csv' +fac_features_ext = '_fac_features.csv' + +def compute_features(out_dir, df_of, r_config): + """ Computes features + + Returns: features in vector format + """ + config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) + logger.info('json file read') + pattern_x = re.compile("l\d+_x") + pattern_y = re.compile("l\d+_y") + + # assumption: distance of face to camera remains at roughly static + + # logic break + landmark_columns = [] + for col in df_of.columns: + if pattern_x.match(col) or pattern_y.match(col): + landmark_columns.append(col) + + df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] + df_of.reset_index(inplace=True) + + num_frames = len(df) + logger.info("Number of frames to be processed: {}".format(str(num_frames))) + landmarks = config['landmarks'] + + try: + if num_frames == 0: + error_reason = "No frames with visible face." + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + +# if num_frames < 60: +# error_reason = 'Number of frames with visible face < 60. Video too short' +# logger.error(error_reason) +# return empty_frame(landmarks, f_cfg, error_reason) + + first_row = df_of.iloc[0] + + facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']]) + faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']]) + + if facew == 0 or faceh == 0: + error_reason = 'face width or height = 0. Check landmark values' + logger.error(error_reason) + return empty_frame(landmarks, r_config) + + fac_disp = calc_displacement_vec(df_of, landmarks, num_frames) + + # if verbose: + # logger.info("Displacement output: {}".format(str(fac_disp))) + + fac_disp_median = np.median(fac_disp, axis = 1) + fac_disp_mean = np.mean(fac_disp, axis = 1) + + if len(fac_disp.shape)!=2: + error_reason = 'fac_disp is not 2D. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + if len(fac_disp[0])<=1: + error_reason = 'Video too short. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + fac_corr_mat = np.corrcoef(fac_disp, rowvar = True) + # extract relevant row from cov matrix + ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk] + fac_corr = fac_corr_mat[ref_lmk_index][0] + + fac_area = config['ref_area'] / (facew * faceh) + + # if verbose: + # logger.info("Face area: {}".format(fac_area)) + # logger.info("Face Displacement Median: {}".format(str(fac_disp_median))) + # logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean))) + + fac_features1 = np.multiply(fac_area * fac_disp_median, (1. - fac_corr)) + fac_features2 = np.multiply(fac_area * fac_disp_mean, (1. - fac_corr)) + +# base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr)) + + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] + + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] + fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] + + fac_features_dict[r_config.err_reason] = [''] + data = pd.DataFrame.from_dict(fac_features_dict) + logger.info('Concluded computing tremor features') + + return data + + except Exception as e: + logger.error('Error computing tremor features: {}'.format(str(e))) + return empty_frame(landmarks, r_config, str(e)) + +def empty_frame(landmarks, r_config, error_reason): + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] + + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan] + + fac_features_dict[r_config.err_reason] = [error_reason] + empty_frame = pd.DataFrame.from_dict(fac_features_dict) + return empty_frame + +def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): + """ + processing input videos + """ + try: + logger.info('filtering path: ',video_uri,out_dir) + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + + if len(of_csv_path)>0: + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + + feats = compute_features(of_csv_path , df_of, r_config) + if model_output: + result = score(feats, r_config) + feats = pd.concat([feats, result], axis=1) + + ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) + + except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/util/math_util.py b/dbm_lib/dbm_features/raw_features/util/math_util.py new file mode 100644 index 00000000..283acadc --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/util/math_util.py @@ -0,0 +1,57 @@ +""" +file_name: facial_tremor +project_name: cdx_analysis +created: 2019-03-16 +author: Deshana Desai +""" +import sys, os, glob, cv2 +import pandas as pd +import numpy as np + + +def euclidean_distance(point1, point2): + """ + Compute euclidean distance between points + """ + + return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2) + + +# def detect_peaks() + + +def expand_landmarks(landmarks): + """ + util method to expand landmark list: + eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']] + """ + return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks] + + + +def calc_displacement_vec(df, landmarks, num_frames): + """ + Calculates displacement vector frame by frame + """ + + landmarks = expand_landmarks(landmarks) + + disp_vec = np.zeros((len(landmarks), num_frames)) + prev_point = np.zeros((len(landmarks), 2)) + + # initialize + for j, pair in enumerate(landmarks): + first_row = df.iloc[0] + prev_point[j] = (first_row[pair[0]], first_row[pair[1]]) + + + for i in range(num_frames): + frame_row = df.iloc[i] + for j, pair in enumerate(landmarks): + x, y = pair[0], pair[1] + current = (frame_row[x], frame_row[y]) + deviation = euclidean_distance( current, prev_point[j]) + disp_vec[j][i] = deviation + prev_point[j] = current + + return disp_vec diff --git a/resources/features/facial/config.json b/resources/features/facial/config.json new file mode 100644 index 00000000..ebad1b06 --- /dev/null +++ b/resources/features/facial/config.json @@ -0,0 +1 @@ +{"ref_lmk": 28, "ref_area": 350000, "face_width_left": "l15_x", "face_width_right": "l1_x", "face_height_left": "l8_y", "face_height_right": "l27_y", "landmarks": [5, 12, 8, 48, 54, 28, 51, 66, 57], "model_path": "resources/facial/svm_bin_fac_tremor.sav", "feature_order": ["fac_features_mean_5", "fac_features_mean_12", "fac_features_mean_8", "fac_features_mean_48", "fac_features_mean_54", "fac_features_mean_28", "fac_features_mean_51", "fac_features_mean_66", "fac_features_mean_57", "fac_features_median_5", "fac_features_median_12", "fac_features_median_8", "fac_features_median_48", "fac_features_median_54", "fac_features_median_28", "fac_features_median_51", "fac_features_median_66", "fac_features_median_57"]} From 517e54bf816ca9b445cc29e7aba4ba1e2f347022 Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Mon, 30 Nov 2020 09:43:09 -0500 Subject: [PATCH 07/26] facial tremor testing --- dbm_lib/controller/process_feature.py | 6 ++- .../raw_features/movement/__init__.py | 1 + .../raw_features/video/open_face_process.py | 37 ++++++++++--------- process_data.py | 3 ++ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index f5112da2..488b835c 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor, facial_tremor import subprocess import logging @@ -120,6 +120,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): return logger.info('Processing movement variables from data in {}'.format(video_uri)) + logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) @@ -129,6 +130,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing voice tremor....') voice_tremor.run_vtremor(video_uri, out_dir, r_config) + logger.info('processing facial tremor....') + face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + def remove_file(file_path): """ removing wav file diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index dca2f624..03fd8909 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,3 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 292e2a98..61828b08 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): """ Computes open_face features for the files in filepaths - + Args: ----- filepaths: (itreable[str]) @@ -27,46 +27,49 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): input_dir: Path to the input videos out_dir: Path to the processed output of_path: OpenFace source code path - + Returns: -------- (itreable[str]) list of .csv files """ - - suffix = '_OF_features' + if video_tracking: + suffix = '_OF_video_features/' + else: + suffix = '_OF_features' + csv_files = [] - + for fp in filepaths: try: - + _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) - + except Exception as e: logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e)) - + return csv_files - -def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): + +def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking): """ Processing all patient's for fetching emotion expressivity ------------------- ------------------- Args: video_uri: video path; input_dir : input directory for video's; dbm_group: feature group - out_dir: (str) Output directory for processed output; of_path: OpenFace source code path - + out_dir: (str) Output directory for processed output; of_path: OpenFace source code path + """ try: - + if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group: return filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) - + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) + except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/process_data.py b/process_data.py index 31f85fdb..07d8f4af 100644 --- a/process_data.py +++ b/process_data.py @@ -19,6 +19,8 @@ import time logging.basicConfig(level=logging.INFO) logger=logging.getLogger() +#for ftremor +OPENFACE_PATH_VIDEO = '/pkg/OpenFace/build/bin/FaceLandmarkVid' OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction' DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat' @@ -35,6 +37,7 @@ def common_video(video_file, args, r_config): of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group) pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group,video_tracking=True) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) From 0285dd51ffb29891ba8b629d104509f63eec54ed Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Mon, 30 Nov 2020 09:46:48 -0500 Subject: [PATCH 08/26] facial tremor testing --- dbm_lib/controller/process_feature.py | 9 +- .../raw_features/movement/__init__.py | 1 + .../raw_features/movement/facial_tremor.py | 159 ++++++++++++++++++ .../raw_features/video/open_face_process.py | 37 ++-- process_data.py | 3 + resources/features/facial/config.json | 1 + 6 files changed, 189 insertions(+), 21 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/movement/facial_tremor.py create mode 100644 resources/features/facial/config.json diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index f5112da2..00463600 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor, facial_tremor import subprocess import logging @@ -82,9 +82,6 @@ def process_acoustic(video_uri, out_dir, dbm_group, r_config): logger.info('processing mfcc....') mfcc.run_mfcc(video_uri, out_dir, r_config) - logger.info('processing voice tremor....') - voice_tremor.run_vtremor(video_uri, out_dir, r_config) - def process_facial(video_uri, out_dir, dbm_group, r_config): """ processing facial features @@ -120,6 +117,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): return logger.info('Processing movement variables from data in {}'.format(video_uri)) + logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) @@ -129,6 +127,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing voice tremor....') voice_tremor.run_vtremor(video_uri, out_dir, r_config) + logger.info('processing facial tremor....') + face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + def remove_file(file_path): """ removing wav file diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index dca2f624..03fd8909 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,3 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py new file mode 100644 index 00000000..7df1b861 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -0,0 +1,159 @@ +import sys, os, glob, cv2, re +import pickle, json +import pandas as pd +import numpy as np +import numpy.ma as ma +import logging +from os.path import join + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util.math_util import * + +from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +ft_dir = 'movement/facial_tremor' +csv_ext = '_fac_tremor.csv' +model_ext = '_fac_model.csv' +fac_features_ext = '_fac_features.csv' + +def compute_features(out_dir, df_of, r_config): + """ Computes features + + Returns: features in vector format + """ + config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) + logger.info('json file read') + pattern_x = re.compile("l\d+_x") + pattern_y = re.compile("l\d+_y") + + # assumption: distance of face to camera remains at roughly static + + # logic break + landmark_columns = [] + for col in df_of.columns: + if pattern_x.match(col) or pattern_y.match(col): + landmark_columns.append(col) + + df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] + df_of.reset_index(inplace=True) + + num_frames = len(df) + logger.info("Number of frames to be processed: {}".format(str(num_frames))) + landmarks = config['landmarks'] + + try: + if num_frames == 0: + error_reason = "No frames with visible face." + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + +# if num_frames < 60: +# error_reason = 'Number of frames with visible face < 60. Video too short' +# logger.error(error_reason) +# return empty_frame(landmarks, f_cfg, error_reason) + + first_row = df_of.iloc[0] + + facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']]) + faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']]) + + if facew == 0 or faceh == 0: + error_reason = 'face width or height = 0. Check landmark values' + logger.error(error_reason) + return empty_frame(landmarks, r_config) + + fac_disp = calc_displacement_vec(df_of, landmarks, num_frames) + + # if verbose: + # logger.info("Displacement output: {}".format(str(fac_disp))) + + fac_disp_median = np.median(fac_disp, axis = 1) + fac_disp_mean = np.mean(fac_disp, axis = 1) + + if len(fac_disp.shape)!=2: + error_reason = 'fac_disp is not 2D. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + if len(fac_disp[0])<=1: + error_reason = 'Video too short. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + fac_corr_mat = np.corrcoef(fac_disp, rowvar = True) + # extract relevant row from cov matrix + ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk] + fac_corr = fac_corr_mat[ref_lmk_index][0] + + fac_area = config['ref_area'] / (facew * faceh) + + # if verbose: + # logger.info("Face area: {}".format(fac_area)) + # logger.info("Face Displacement Median: {}".format(str(fac_disp_median))) + # logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean))) + + fac_features1 = np.multiply(fac_area * fac_disp_median, (1. - fac_corr)) + fac_features2 = np.multiply(fac_area * fac_disp_mean, (1. - fac_corr)) + +# base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr)) + + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] + + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] + fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] + + fac_features_dict[r_config.err_reason] = [''] + data = pd.DataFrame.from_dict(fac_features_dict) + logger.info('Concluded computing tremor features') + + return data + + except Exception as e: + logger.error('Error computing tremor features: {}'.format(str(e))) + return empty_frame(landmarks, r_config, str(e)) + +def empty_frame(landmarks, r_config, error_reason): + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] + + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan] + + fac_features_dict[r_config.err_reason] = [error_reason] + empty_frame = pd.DataFrame.from_dict(fac_features_dict) + return empty_frame + +def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): + """ + processing input videos + """ + try: + logger.info('filtering path: ',video_uri,out_dir) + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) + + if len(of_csv_path)>0: + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + + feats = compute_features(of_csv_path , df_of, r_config) + if model_output: + result = score(feats, r_config) + feats = pd.concat([feats, result], axis=1) + + ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) + + except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 292e2a98..61828b08 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): """ Computes open_face features for the files in filepaths - + Args: ----- filepaths: (itreable[str]) @@ -27,46 +27,49 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): input_dir: Path to the input videos out_dir: Path to the processed output of_path: OpenFace source code path - + Returns: -------- (itreable[str]) list of .csv files """ - - suffix = '_OF_features' + if video_tracking: + suffix = '_OF_video_features/' + else: + suffix = '_OF_features' + csv_files = [] - + for fp in filepaths: try: - + _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) - + except Exception as e: logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e)) - + return csv_files - -def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): + +def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking): """ Processing all patient's for fetching emotion expressivity ------------------- ------------------- Args: video_uri: video path; input_dir : input directory for video's; dbm_group: feature group - out_dir: (str) Output directory for processed output; of_path: OpenFace source code path - + out_dir: (str) Output directory for processed output; of_path: OpenFace source code path + """ try: - + if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group: return filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) - + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) + except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/process_data.py b/process_data.py index 31f85fdb..07d8f4af 100644 --- a/process_data.py +++ b/process_data.py @@ -19,6 +19,8 @@ import time logging.basicConfig(level=logging.INFO) logger=logging.getLogger() +#for ftremor +OPENFACE_PATH_VIDEO = '/pkg/OpenFace/build/bin/FaceLandmarkVid' OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction' DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat' @@ -35,6 +37,7 @@ def common_video(video_file, args, r_config): of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group) pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group,video_tracking=True) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) diff --git a/resources/features/facial/config.json b/resources/features/facial/config.json new file mode 100644 index 00000000..ebad1b06 --- /dev/null +++ b/resources/features/facial/config.json @@ -0,0 +1 @@ +{"ref_lmk": 28, "ref_area": 350000, "face_width_left": "l15_x", "face_width_right": "l1_x", "face_height_left": "l8_y", "face_height_right": "l27_y", "landmarks": [5, 12, 8, 48, 54, 28, 51, 66, 57], "model_path": "resources/facial/svm_bin_fac_tremor.sav", "feature_order": ["fac_features_mean_5", "fac_features_mean_12", "fac_features_mean_8", "fac_features_mean_48", "fac_features_mean_54", "fac_features_mean_28", "fac_features_mean_51", "fac_features_mean_66", "fac_features_mean_57", "fac_features_median_5", "fac_features_median_12", "fac_features_median_8", "fac_features_median_48", "fac_features_median_54", "fac_features_median_28", "fac_features_median_51", "fac_features_median_66", "fac_features_median_57"]} From 4cacbb8b37b1ad594952803daebca53fb0137803 Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Mon, 30 Nov 2020 09:49:25 -0500 Subject: [PATCH 09/26] added math_util --- .../raw_features/util/math_util.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 dbm_lib/dbm_features/raw_features/util/math_util.py diff --git a/dbm_lib/dbm_features/raw_features/util/math_util.py b/dbm_lib/dbm_features/raw_features/util/math_util.py new file mode 100644 index 00000000..283acadc --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/util/math_util.py @@ -0,0 +1,57 @@ +""" +file_name: facial_tremor +project_name: cdx_analysis +created: 2019-03-16 +author: Deshana Desai +""" +import sys, os, glob, cv2 +import pandas as pd +import numpy as np + + +def euclidean_distance(point1, point2): + """ + Compute euclidean distance between points + """ + + return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2) + + +# def detect_peaks() + + +def expand_landmarks(landmarks): + """ + util method to expand landmark list: + eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']] + """ + return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks] + + + +def calc_displacement_vec(df, landmarks, num_frames): + """ + Calculates displacement vector frame by frame + """ + + landmarks = expand_landmarks(landmarks) + + disp_vec = np.zeros((len(landmarks), num_frames)) + prev_point = np.zeros((len(landmarks), 2)) + + # initialize + for j, pair in enumerate(landmarks): + first_row = df.iloc[0] + prev_point[j] = (first_row[pair[0]], first_row[pair[1]]) + + + for i in range(num_frames): + frame_row = df.iloc[i] + for j, pair in enumerate(landmarks): + x, y = pair[0], pair[1] + current = (frame_row[x], frame_row[y]) + deviation = euclidean_distance( current, prev_point[j]) + disp_vec[j][i] = deviation + prev_point[j] = current + + return disp_vec From 993bd860e46aba5fbf548fb6d360e7c6e506866f Mon Sep 17 00:00:00 2001 From: Vijay Yadev Date: Mon, 30 Nov 2020 20:31:33 -0500 Subject: [PATCH 10/26] eye gaze --- dbm_lib/config/config_raw_feature.py | 8 + dbm_lib/controller/process_feature.py | 5 +- .../raw_features/movement/eye_gaze.py | 148 ++++++++++++++++++ resources/features/derived_feature.yml | 14 +- resources/features/raw_feature.yml | 8 + 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/movement/eye_gaze.py diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 3dc4834b..955780b7 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -222,6 +222,14 @@ class ConfigRawReader(object): self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw'] self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll'] self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist'] + self.mov_leye_x = config['raw_feature']['mov_leye_x'] + self.mov_leye_y = config['raw_feature']['mov_leye_y'] + self.mov_leye_z = config['raw_feature']['mov_leye_z'] + self.mov_reye_x = config['raw_feature']['mov_reye_x'] + self.mov_reye_y = config['raw_feature']['mov_reye_y'] + self.mov_reye_z = config['raw_feature']['mov_reye_z'] + self.mov_eleft_disp = config['raw_feature']['mov_eleft_disp'] + self.mov_eright_disp = config['raw_feature']['mov_eright_disp'] #NLP features self.nlp_transcribe = config['raw_feature']['nlp_transcribe'] diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index be781a7f..555f4cc9 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features import subprocess @@ -123,6 +123,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) + + logger.info('processing eye gaze....') + eye_gaze.run_eye_gaze(video_uri, out_dir, r_config) def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): """ diff --git a/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py new file mode 100644 index 00000000..a574a2d9 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py @@ -0,0 +1,148 @@ +""" +file_name: eye_gaze +project_name: DBM +created: 2020-30-11 +""" + +import os +import glob +import pandas as pd +import numpy as np +from scipy.spatial import distance +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +eye_pose_dir = 'movement/gaze' +eye_pose_ext = '_eyegaze.csv' + +def eye_motion_df(l_disp, r_disp, error_list, r_config): + """ + Generating eye movement dataframe + + Args: + l_disp: displacement list(left eye); l_disp: displacement list(right eye) + r_config: raw variable config file object + + Reutrns: + Final eye displacement dataframe + """ + df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp]) + df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp]) + + df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False) + df_eye_motion[r_config.err_reason] = error_list + return df_eye_motion + +def filter_motion(df_of, df_disp, col_l, col_r, r_config): + """ + Filtering final eye movement dataframe + + Args: + df_of: Openface raw out dataframe; col_r: right eye column + col_l: left eye column; r_config: raw variable config file object + """ + + df_of = df_of[col_l + col_r + [' confidence']] + df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan + + df_filter = df_of[col_l + col_r] + df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z, + r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z] + + df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False) + return df_motion + +def eye_disp(of_results, col, r_config): + """ + Computing head velocity frame by frame + + Args: + of_results: Openface raw out dataframe + r_config: Face config file object + + Reutrns: + Final head velocity frame by frame output + """ + distance_list = [] + error_list = [] + + of_results = of_results[col+ [' confidence']] + for index, row in of_results.iterrows(): + dst = np.nan + + if index == 0 or float(row[' confidence']) < 0.8: #Threshold < 0.8 + distance_list.append(dst) + + if float(row[' confidence']) < 0.8: + error_list.append('confidence less than 80%') + + else: + error_list.append('Pass') + continue + + if index > 0: + + point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1]) + point_y = (row[col[0]],row[col[1]],row[col[2]]) + try: + dst = distance.euclidean(point_x, point_y) + except: + pass + + distance_list.append(abs(dst)) + error_list.append('Pass') + + return distance_list, error_list + +def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config): + """ + Computing eye motion variables + Args: + df_of: Openface dataframe + out_loc: Output path for saving output csv's + fl_name: file name for output csv + r_config: raw variable config file object + + """ + + col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z'] + col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z'] + + gazel_disp, err_l = eye_disp(df_of, col_l, r_config) + gazer_disp, err_r = eye_disp(df_of, col_r, r_config) + + df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config) + df_disp['dbm_master_url'] = video_uri + + df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config) + ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext) + +def run_eye_gaze(video_uri, out_dir, r_config): + """ + Processing all patient's for getting eye movement artifacts + -------------------------------- + -------------------------------- + Args: + video_uri: video path; input_dir : input directory for video's + out_dir: (str) Output directory for processed output; r_config: raw variable config object + """ + try: + + #filtering path to generate input & output path + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + + if len(of_csv_path)>0: + + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config) + except Exception as e: + logger.error('Failed to process video file') \ No newline at end of file diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index 7346eb36..dd1751fd 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,8 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH', + 'EYE_GAZE'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -23,6 +24,7 @@ derive_feature: MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks NLP_SPEECH_LOC: _nlp + EYE_GAZE_LOC: _eyegaze #Facial category feature group FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom'] @@ -65,6 +67,8 @@ derive_feature: MOV_HM: ['head_vel'] MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] + EYE_GAZE: ['mov_leye_x', 'mov_leye_y', 'mov_leye_z', 'mov_reye_x', 'mov_reye_y', 'mov_reye_z', 'mov_eleft_disp', + 'mov_eright_disp'] #NLP category feature group NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen', @@ -255,6 +259,14 @@ derive_feature: mov_blink_ear: ['mean', 'std'] vid_dur: ['count'] mov_blinkdur: ['mean', 'std'] + mov_leye_x: ['mean', 'std'] + mov_leye_y: ['mean', 'std'] + mov_leye_z: ['mean', 'std'] + mov_reye_x: ['mean', 'std'] + mov_reye_y: ['mean', 'std'] + mov_reye_z: ['mean', 'std'] + mov_eleft_disp: ['mean', 'std'] + mov_eright_disp: ['mean', 'std'] #NLP feature nlp_numSentences: ['mean'] diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index b9c673b8..3f15bd9b 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -196,6 +196,14 @@ raw_feature: mov_Hpose_Yaw: mov_hposeyaw mov_Hpose_Roll: mov_hposeroll mov_Hpose_Dist: mov_hposedist + mov_leye_x: mov_lefteyex + mov_leye_y: mov_lefteyey + mov_leye_z: mov_lefteyez + mov_reye_x: mov_righteyex + mov_reye_y: mov_righteyey + mov_reye_z: mov_righteyez + mov_eleft_disp: mov_leyedisp + mov_eright_disp: mov_reyedisp #NLP markers nlp_transcribe: nlp_transcribe From ccde369840cfb984a41d9e733025d6af67aa03b4 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Tue, 1 Dec 2020 12:49:56 -0500 Subject: [PATCH 11/26] handling openface execution --- .../raw_features/video/open_face_process.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 292e2a98..34c31257 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -62,11 +62,16 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): """ try: - if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group: + if dbm_group != None: return - + + check_group = ['facial','movement'] + check_val = bool(len({*check_group} & {*dbm_group})) + if not check_val: + return + filepaths = [video_uri] csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') From 9a751ca4341d19811f164f3eedbcdb72ee893980 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 1 Dec 2020 19:54:35 +0000 Subject: [PATCH 12/26] added facial tremor variable --- dbm_lib/controller/process_feature.py | 4 +- .../raw_features/movement/__init__.py | 2 +- .../raw_features/movement/facial_tremor.py | 43 +++++++++++-------- .../raw_features/movement/head_motion.py | 5 ++- .../raw_features/movement/voice_tremor.py | 6 +-- .../raw_features/video/open_face_process.py | 13 +++--- .../exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 42 +++++++++--------- process_data.py | 9 ++-- resources/features/derived_feature.yml | 13 +++++- 9 files changed, 78 insertions(+), 59 deletions(-) diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 00463600..0a9d3dca 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -120,7 +120,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) - + logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) @@ -128,7 +128,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): voice_tremor.run_vtremor(video_uri, out_dir, r_config) logger.info('processing facial tremor....') - face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + facial_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) def remove_file(file_path): """ diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index 03fd8909..cafb6e11 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,4 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) -DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py index 7df1b861..7753eb0b 100644 --- a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -25,7 +25,7 @@ def compute_features(out_dir, df_of, r_config): Returns: features in vector format """ config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) - logger.info('json file read') + pattern_x = re.compile("l\d+_x") pattern_y = re.compile("l\d+_y") @@ -40,7 +40,7 @@ def compute_features(out_dir, df_of, r_config): df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] df_of.reset_index(inplace=True) - num_frames = len(df) + num_frames = len(df_of) logger.info("Number of frames to be processed: {}".format(str(num_frames))) landmarks = config['landmarks'] @@ -104,7 +104,7 @@ def compute_features(out_dir, df_of, r_config): for i, landmark in enumerate(landmarks): fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] raw_variable_map = 'fac_tremor_median_{}'.format(landmark) - fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] + fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]] fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] @@ -123,7 +123,7 @@ def empty_frame(landmarks, r_config, error_reason): fac_features_dict = {} for i, landmark in enumerate(landmarks): raw_variable_map = 'fac_tremor_median_{}'.format(landmark) - fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] + fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan] fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] @@ -133,27 +133,32 @@ def empty_frame(landmarks, r_config, error_reason): empty_frame = pd.DataFrame.from_dict(fac_features_dict) return empty_frame -def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): +def fac_tremor_process(video_uri, out_dir, r_config, model_output=False): """ processing input videos + + """ - try: - logger.info('filtering path: ',video_uri,out_dir) - input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) +# try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) - if len(of_csv_path)>0: - of_csv = of_csv_path[0] - df_of = pd.read_csv(of_csv, error_bad_lines=False) + if len(of_csv_path)>0: + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) - logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) - feats = compute_features(of_csv_path , df_of, r_config) - if model_output: - result = score(feats, r_config) - feats = pd.concat([feats, result], axis=1) + feats = compute_features(of_csv_path , df_of, r_config) + +# if model_output: +# result = score(feats, r_config) +# feats = pd.concat([feats, result], axis=1) + + ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext) - ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) - except Exception as e: + +# except Exception as e: logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/movement/head_motion.py b/dbm_lib/dbm_features/raw_features/movement/head_motion.py index c45bc123..59179c2e 100644 --- a/dbm_lib/dbm_features/raw_features/movement/head_motion.py +++ b/dbm_lib/dbm_features/raw_features/movement/head_motion.py @@ -179,11 +179,12 @@ def run_head_movement(video_uri, out_dir, r_config): out_dir: (str) Output directory for processed output; r_config: raw variable config object """ try: - + #filtering path to generate input & output path input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + if len(of_csv_path)>0: of_csv = of_csv_path[0] @@ -192,4 +193,4 @@ def run_head_movement(video_uri, out_dir, r_config): logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config) except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py index 52f9c398..18f664b6 100644 --- a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py +++ b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py @@ -47,7 +47,7 @@ def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name): df_tremor = tremor_praat(audio_file, r_config) df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release - logger.info('Processing Output file {} '.format(out_loc)) + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): @@ -61,7 +61,7 @@ def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] df_tremor = pd.DataFrame(out_val, columns = cols) - logger.info('Saving Output file {} '.format(out_loc)) + logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name))) ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) def run_vtremor(video_uri, out_dir, r_config): @@ -74,9 +74,9 @@ def run_vtremor(video_uri, out_dir, r_config): out_dir: (str) Output directory for processed output """ try: + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) - if len(aud_filter)>0: audio_file = aud_filter[0] diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 61828b08..51364521 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,7 +15,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False): """ Computes open_face features for the files in filepaths Args: @@ -31,21 +31,22 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_track Returns: -------- (itreable[str]) list of .csv files - """ + """ if video_tracking: - suffix = '_OF_video_features/' + suffix = '_OF_video_features' else: suffix = '_OF_features' csv_files = [] - + for fp in filepaths: try: _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) - + if not os.path.isdir(output_directory): + os.mkdir(output_directory) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) except Exception as e: @@ -69,7 +70,7 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tr return filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking) except Exception as e: logger.error('Failed to process video file') diff --git a/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index fafc36a9..058f4dc1 100644 --- a/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -155,7 +155,7 @@ int main(int argc, char **argv) std::string base_filename = path.substr(path.find_last_of("/\\") + 1); base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,""); results.open(out_dir + '/' + base_filename + "_landmark_output.csv"); -// confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); + confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); int lx = 0; int ly = 0; for(lx = 0; lx < 2; lx++){ @@ -163,7 +163,7 @@ int main(int argc, char **argv) if (lx == 0){ results << "l" << ly << "_x,"; -// confidence << "c" << ly <<","; + confidence << "c" << ly <<","; } if (lx == 1){ results << "l" << ly << "_y,"; @@ -172,7 +172,7 @@ int main(int argc, char **argv) } results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ; results << std::endl; -// confidence << std::endl; + confidence << std::endl; int counter = 0; @@ -207,7 +207,6 @@ int main(int argc, char **argv) // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); - cv::Vec6d pose_estimate(0, 0, 0, 0, 0, 0); // If tracking succeeded and we have an eye model, estimate gaze if (detection_success && face_model.eye_model) @@ -217,9 +216,9 @@ int main(int argc, char **argv) } // Work out the pose of the head from the tracked model - if (detection_success){ - pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - } + + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + @@ -234,11 +233,11 @@ int main(int argc, char **argv) // Displaying the tracking visualizations // std::cout<< "setting observation landmarks"<0 and 'movement' in args.dbm_group: + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group, video_tracking=True) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index 43db29dc..fc2f3926 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,7 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT', 'MOV_FT'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -23,6 +23,7 @@ derive_feature: MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks MOV_VT_LOC: _vtremor + MOV_FT_LOC: _fac_tremor #Facial category feature group @@ -68,6 +69,7 @@ derive_feature: EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq', 'mov_amp_trem_index', 'mov_amp_trem_pindex'] + MOV_FT: ['fac_tremor_median_5','fac_tremor_median_12','fac_tremor_median_8','fac_tremor_median_48','fac_tremor_median_54','fac_tremor_median_28','fac_tremor_median_51','fac_tremor_median_66','fac_tremor_median_57'] #Calculation for variables # Facial Asymmetry @@ -258,3 +260,12 @@ derive_feature: mov_amp_trem_freq: ['mean'] mov_amp_trem_index: ['mean'] mov_amp_trem_pindex: ['mean'] + fac_tremor_median_5: ['mean'] + fac_tremor_median_12: ['mean'] + fac_tremor_median_8: ['mean'] + fac_tremor_median_48: ['mean'] + fac_tremor_median_54: ['mean'] + fac_tremor_median_28: ['mean'] + fac_tremor_median_51: ['mean'] + fac_tremor_median_66: ['mean'] + fac_tremor_median_57: ['mean'] From 5ab0d2f1e153f1a34695c8b4aabd9bcda988d941 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Tue, 1 Dec 2020 14:58:26 -0500 Subject: [PATCH 13/26] update in dbm group check --- .../raw_features/video/open_face_process.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 34c31257..695b35fb 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -63,12 +63,10 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): try: if dbm_group != None: - return - - check_group = ['facial','movement'] - check_val = bool(len({*check_group} & {*dbm_group})) - if not check_val: - return + check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation + check_val = bool(len({*check_group} & {*dbm_group})) + if not check_val: + return filepaths = [video_uri] csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) From 476123baa8d5247dadbcdc840f3733c321026aec Mon Sep 17 00:00:00 2001 From: vkoesmahargyo Date: Wed, 2 Dec 2020 11:01:07 -0500 Subject: [PATCH 14/26] Update derived_feature.yml --- resources/features/derived_feature.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index b29c8251..c9914b87 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -271,7 +271,7 @@ derive_feature: mov_amp_trem_index: ['mean'] mov_amp_trem_pindex: ['mean'] - mov_leye_x: ['mean', 'std'] + mov_leye_x: ['mean', 'std'] mov_leye_y: ['mean', 'std'] mov_leye_z: ['mean', 'std'] mov_reye_x: ['mean', 'std'] From 248546860e814e1baedb92abae3b868beef52a87 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 2 Dec 2020 17:21:01 +0000 Subject: [PATCH 15/26] resolved conflicts --- dbm_lib/controller/process_feature.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index f0caa338..d91d479d 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -83,9 +83,6 @@ def process_acoustic(video_uri, out_dir, dbm_group, r_config): logger.info('processing mfcc....') mfcc.run_mfcc(video_uri, out_dir, r_config) - logger.info('processing voice tremor....') - voice_tremor.run_vtremor(video_uri, out_dir, r_config) - def process_facial(video_uri, out_dir, dbm_group, r_config): """ processing facial features From b4e729200886b68b7bfdea7a9bcfc58fad2d91c8 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 2 Dec 2020 12:28:31 -0500 Subject: [PATCH 16/26] removing face tremor --- .../raw_features/movement/facial_tremor.py | 159 ------------------ 1 file changed, 159 deletions(-) delete mode 100644 dbm_lib/dbm_features/raw_features/movement/facial_tremor.py diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py deleted file mode 100644 index e6bfb1a5..00000000 --- a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py +++ /dev/null @@ -1,159 +0,0 @@ -import sys, os, glob, cv2, re -import pickle, json -import pandas as pd -import numpy as np -import numpy.ma as ma -import logging -from os.path import join - -from dbm_lib.dbm_features.raw_features.util import util as ut -from dbm_lib.dbm_features.raw_features.util.math_util import * - -from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG - -logging.basicConfig(level=logging.INFO) -logger=logging.getLogger() - -ft_dir = 'movement/facial_tremor' -csv_ext = '_fac_tremor.csv' -model_ext = '_fac_model.csv' -fac_features_ext = '_fac_features.csv' - -def compute_features(out_dir, df_of, r_config): - """ Computes features - - Returns: features in vector format - """ - config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) - logger.info('json file read') - pattern_x = re.compile("l\d+_x") - pattern_y = re.compile("l\d+_y") - - # assumption: distance of face to camera remains at roughly static - - # logic break - landmark_columns = [] - for col in df_of.columns: - if pattern_x.match(col) or pattern_y.match(col): - landmark_columns.append(col) - - df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] - df_of.reset_index(inplace=True) - - num_frames = len(df) - logger.info("Number of frames to be processed: {}".format(str(num_frames))) - landmarks = config['landmarks'] - - try: - if num_frames == 0: - error_reason = "No frames with visible face." - logger.error(error_reason) - return empty_frame(landmarks, r_config, error_reason) - -# if num_frames < 60: -# error_reason = 'Number of frames with visible face < 60. Video too short' -# logger.error(error_reason) -# return empty_frame(landmarks, f_cfg, error_reason) - - first_row = df_of.iloc[0] - - facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']]) - faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']]) - - if facew == 0 or faceh == 0: - error_reason = 'face width or height = 0. Check landmark values' - logger.error(error_reason) - return empty_frame(landmarks, r_config) - - fac_disp = calc_displacement_vec(df_of, landmarks, num_frames) - - # if verbose: - # logger.info("Displacement output: {}".format(str(fac_disp))) - - fac_disp_median = np.median(fac_disp, axis = 1) - fac_disp_mean = np.mean(fac_disp, axis = 1) - - if len(fac_disp.shape)!=2: - error_reason = 'fac_disp is not 2D. smth went wrong with disp calc' - logger.error(error_reason) - return empty_frame(landmarks, r_config, error_reason) - - if len(fac_disp[0])<=1: - error_reason = 'Video too short. smth went wrong with disp calc' - logger.error(error_reason) - return empty_frame(landmarks, r_config, error_reason) - - fac_corr_mat = np.corrcoef(fac_disp, rowvar = True) - # extract relevant row from cov matrix - ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk] - fac_corr = fac_corr_mat[ref_lmk_index][0] - - fac_area = config['ref_area'] / (facew * faceh) - - # if verbose: - # logger.info("Face area: {}".format(fac_area)) - # logger.info("Face Displacement Median: {}".format(str(fac_disp_median))) - # logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean))) - - fac_features1 = np.multiply(fac_area * fac_disp_median, (1. - fac_corr)) - fac_features2 = np.multiply(fac_area * fac_disp_mean, (1. - fac_corr)) - -# base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr)) - - fac_features_dict = {} - for i, landmark in enumerate(landmarks): - fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] - raw_variable_map = 'fac_tremor_median_{}'.format(landmark) - fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] - - fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] - fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] - - fac_features_dict[r_config.err_reason] = [''] - data = pd.DataFrame.from_dict(fac_features_dict) - logger.info('Concluded computing tremor features') - - return data - - except Exception as e: - logger.error('Error computing tremor features: {}'.format(str(e))) - return empty_frame(landmarks, r_config, str(e)) - -def empty_frame(landmarks, r_config, error_reason): - fac_features_dict = {} - for i, landmark in enumerate(landmarks): - raw_variable_map = 'fac_tremor_median_{}'.format(landmark) - fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] - - fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] - fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] - fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan] - - fac_features_dict[r_config.err_reason] = [error_reason] - empty_frame = pd.DataFrame.from_dict(fac_features_dict) - return empty_frame - -def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): - """ - processing input videos - """ - try: - logger.info('filtering path: ',video_uri,out_dir) - input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) - - if len(of_csv_path)>0: - of_csv = of_csv_path[0] - df_of = pd.read_csv(of_csv, error_bad_lines=False) - - logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) - - feats = compute_features(of_csv_path , df_of, r_config) - if model_output: - result = score(feats, r_config) - feats = pd.concat([feats, result], axis=1) - - ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) - - except Exception as e: - logger.error('Failed to process video file') From 377de48690941b117bb20350829ac464cfd21d40 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 2 Dec 2020 12:28:56 -0500 Subject: [PATCH 17/26] removing util --- .../raw_features/util/math_util.py | 57 ------------------- 1 file changed, 57 deletions(-) delete mode 100644 dbm_lib/dbm_features/raw_features/util/math_util.py diff --git a/dbm_lib/dbm_features/raw_features/util/math_util.py b/dbm_lib/dbm_features/raw_features/util/math_util.py deleted file mode 100644 index 283acadc..00000000 --- a/dbm_lib/dbm_features/raw_features/util/math_util.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -file_name: facial_tremor -project_name: cdx_analysis -created: 2019-03-16 -author: Deshana Desai -""" -import sys, os, glob, cv2 -import pandas as pd -import numpy as np - - -def euclidean_distance(point1, point2): - """ - Compute euclidean distance between points - """ - - return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2) - - -# def detect_peaks() - - -def expand_landmarks(landmarks): - """ - util method to expand landmark list: - eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']] - """ - return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks] - - - -def calc_displacement_vec(df, landmarks, num_frames): - """ - Calculates displacement vector frame by frame - """ - - landmarks = expand_landmarks(landmarks) - - disp_vec = np.zeros((len(landmarks), num_frames)) - prev_point = np.zeros((len(landmarks), 2)) - - # initialize - for j, pair in enumerate(landmarks): - first_row = df.iloc[0] - prev_point[j] = (first_row[pair[0]], first_row[pair[1]]) - - - for i in range(num_frames): - frame_row = df.iloc[i] - for j, pair in enumerate(landmarks): - x, y = pair[0], pair[1] - current = (frame_row[x], frame_row[y]) - deviation = euclidean_distance( current, prev_point[j]) - disp_vec[j][i] = deviation - prev_point[j] = current - - return disp_vec From c72d56827e9095d014d741865d6dfa4c97a9b266 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 2 Dec 2020 12:33:00 -0500 Subject: [PATCH 18/26] update in voice tremor --- .../dbm_features/raw_features/movement/voice_tremor.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py index 52f9c398..ef853e65 100644 --- a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py +++ b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py @@ -82,10 +82,11 @@ def run_vtremor(video_uri, out_dir, r_config): audio_file = aud_filter[0] aud_dur = librosa.get_duration(filename=audio_file) - if float(aud_dur) < 0.064: - logger.info('Output file {} size is less than 0.064sec'.format(audio_file)) - - prepare_empty_vt(video_uri, out_loc, fl_name, r_config) + if float(aud_dur) < 0.5: + logger.info('Output file {} size is less than 0.5sec'.format(audio_file)) + + error_txt = 'error: length less than 0.5 sec' + prepare_empty_vt(video_uri, out_loc, fl_name, error_txt) return prepare_vtrem_output(audio_file, out_loc, r_config, fl_name) except Exception as e: From f9eeb3d27b5953902a7dd78d0c3554cdd8d3d0fe Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 2 Dec 2020 12:42:13 -0500 Subject: [PATCH 19/26] update in controller --- process_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_data.py b/process_data.py index cd4cc707..746ac18d 100644 --- a/process_data.py +++ b/process_data.py @@ -37,7 +37,7 @@ def common_video(video_file, args, r_config): pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) pf.process_nlp(video_file, out_path, args.dbm_group, r_config, DEEP_SPEECH) - pf.remove_file(video_file) + pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) From 38c7f66d370f19f38a3b9a026b727e5c4d271ff4 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 2 Dec 2020 12:43:53 -0500 Subject: [PATCH 20/26] removing face tremor json --- resources/features/facial/config.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 resources/features/facial/config.json diff --git a/resources/features/facial/config.json b/resources/features/facial/config.json deleted file mode 100644 index ebad1b06..00000000 --- a/resources/features/facial/config.json +++ /dev/null @@ -1 +0,0 @@ -{"ref_lmk": 28, "ref_area": 350000, "face_width_left": "l15_x", "face_width_right": "l1_x", "face_height_left": "l8_y", "face_height_right": "l27_y", "landmarks": [5, 12, 8, 48, 54, 28, 51, 66, 57], "model_path": "resources/facial/svm_bin_fac_tremor.sav", "feature_order": ["fac_features_mean_5", "fac_features_mean_12", "fac_features_mean_8", "fac_features_mean_48", "fac_features_mean_54", "fac_features_mean_28", "fac_features_mean_51", "fac_features_mean_66", "fac_features_mean_57", "fac_features_median_5", "fac_features_median_12", "fac_features_median_8", "fac_features_median_48", "fac_features_median_54", "fac_features_median_28", "fac_features_median_51", "fac_features_median_66", "fac_features_median_57"]} From a83a42393c2368e75e49cb9fcd0145711e7ba866 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Fri, 4 Dec 2020 15:05:12 -0500 Subject: [PATCH 21/26] removing unused lib --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6c90cf09..bee0e653 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ sk-video watchtower opencv-python webrtcvad -pysptk imutils dlib==19.13.0 coloredlogs From 2c7f57160137716feb67b13e75cddd4d30464016 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Mon, 7 Dec 2020 15:26:35 -0500 Subject: [PATCH 22/26] condition for FT --- dbm_lib/dbm_features/raw_features/video/open_face_process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 51364521..b427656e 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -45,8 +45,9 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) - if not os.path.isdir(output_directory): - os.mkdir(output_directory) + + if video_tracking and not os.path.exists(os.path.abspath(output_directory)): + os.makedirs(os.path.abspath(output_directory)) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) except Exception as e: From 117ffb3d57e649aabad9b4eb0aef13c957a07bcc Mon Sep 17 00:00:00 2001 From: Vijay Yadev Date: Mon, 7 Dec 2020 20:32:08 -0500 Subject: [PATCH 23/26] transcription logs --- dbm_lib/controller/process_feature.py | 6 +++--- .../dbm_features/raw_features/nlp/speech_features.py | 6 +++++- dbm_lib/dbm_features/raw_features/util/nlp_util.py | 2 +- .../dbm_features/raw_features/util/vad_utilities.py | 10 +++++----- process_data.py | 7 ++++--- process_dbm.sh | 9 +++++++-- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index d91d479d..271b0d49 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -131,7 +131,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing voice tremor....') voice_tremor.run_vtremor(video_uri, out_dir, r_config) -def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): +def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path): """ processing nlp features Args: @@ -139,12 +139,12 @@ def process_nlp(video_uri, out_dir, dbm_group, r_config, deep_path): dbm_group: list of features to process; r_config: raw feature config object deep_path: deep speech build path """ - if dbm_group != None and len(dbm_group)>0 and 'nlp' not in dbm_group: + if dbm_group != None and len(dbm_group)>0 and 'speech' not in dbm_group: return logger.info('Processing nlp variables from data in {}'.format(video_uri)) transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path) - speech_features.run_speech_feature(video_uri, out_dir, r_config) + speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog) def remove_file(file_path): """ diff --git a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py index 6aebd547..3f0cfba7 100644 --- a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py +++ b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py @@ -10,6 +10,7 @@ import pandas as pd import glob from os.path import join import logging +import shutil from dbm_lib.dbm_features.raw_features.util import util as ut from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util @@ -21,7 +22,7 @@ speech_dir = 'nlp/speech_feature' speech_ext = '_nlp.csv' transcribe_ext = 'nlp/transcribe/*_transcribe.csv' -def run_speech_feature(video_uri, out_dir, r_config): +def run_speech_feature(video_uri, out_dir, r_config, tran_tog): """ Processing all patient's for fetching nlp features ------------------- @@ -42,6 +43,9 @@ def run_speech_feature(video_uri, out_dir, r_config): logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext) + + if (tran_tog == None) or (tran_tog != 'on'): + shutil.rmtree(os.path.dirname(transcribe_path[0])) except Exception as e: logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/util/nlp_util.py b/dbm_lib/dbm_features/raw_features/util/nlp_util.py index fc1ac3d1..6a07141e 100644 --- a/dbm_lib/dbm_features/raw_features/util/nlp_util.py +++ b/dbm_lib/dbm_features/raw_features/util/nlp_util.py @@ -126,7 +126,7 @@ def process_speech(transcribe_df,r_config): Returns: Dataframe for speech features """ - + transcribe_df = transcribe_df.replace(np.nan, '', regex=True) err_transcribe = transcribe_df[r_config.err_reason].iloc[0] transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0] total_time = transcribe_df[r_config.nlp_totalTime].iloc[0] diff --git a/dbm_lib/dbm_features/raw_features/util/vad_utilities.py b/dbm_lib/dbm_features/raw_features/util/vad_utilities.py index b769ba19..8f11a2e8 100644 --- a/dbm_lib/dbm_features/raw_features/util/vad_utilities.py +++ b/dbm_lib/dbm_features/raw_features/util/vad_utilities.py @@ -158,7 +158,7 @@ def vad_get_segment_times(sample_rate, frame_duration_ms, for frame in frames: is_speech = vad.is_speech(frame.bytes, sample_rate) - sys.stdout.write('1' if is_speech else '0') + #sys.stdout.write('1' if is_speech else '0') if not triggered: ring_buffer.append((frame, is_speech)) num_voiced = len([f for f, speech in ring_buffer if speech]) @@ -167,7 +167,7 @@ def vad_get_segment_times(sample_rate, frame_duration_ms, # TRIGGERED state. if num_voiced > 0.9 * ring_buffer.maxlen: triggered = True - sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,)) + #sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,)) start_times.append(ring_buffer[0][0].timestamp) # BT ring_buffer.clear() else: @@ -179,18 +179,18 @@ def vad_get_segment_times(sample_rate, frame_duration_ms, # unvoiced, then enter NOTTRIGGERED and yield whatever # audio we've collected. if num_unvoiced > 0.9 * ring_buffer.maxlen: - sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) + #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) end_times.append(ring_buffer[0][0].timestamp + frame.duration) # BT triggered = False if triggered: # BT if were in triggered state at end of signal, set output time - sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) + #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) if len(ring_buffer)>0: end_times.append(ring_buffer[0][0].timestamp ) # BT else: # only get here in very rare case that we triggered on 2nd-to-last frame end_times.append(frame.timestamp + frame.duration) - sys.stdout.write('\n') + #sys.stdout.write('\n') return(start_times, end_times) diff --git a/process_data.py b/process_data.py index 746ac18d..57b5a52f 100644 --- a/process_data.py +++ b/process_data.py @@ -36,7 +36,7 @@ def common_video(video_file, args, r_config): of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group) pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) - pf.process_nlp(video_file, out_path, args.dbm_group, r_config, DEEP_SPEECH) + pf.process_nlp(video_file, out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) @@ -81,7 +81,7 @@ def process_raw_audio_file(args, s_config, r_config): out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio_file[0], out_path, args.dbm_group, r_config) - pf.process_nlp(audio_file[0], out_path, args.dbm_group, r_config, DEEP_SPEECH) + pf.process_nlp(audio_file[0], out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) else: logger.info('Enter correct audio(*.wav) file path.') @@ -132,7 +132,7 @@ def process_raw_audio_dir(args, s_config, r_config): out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio, out_path, args.dbm_group, r_config) - pf.process_nlp(audio, out_path, args.dbm_group, r_config, DEEP_SPEECH) + pf.process_nlp(audio, out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) except Exception as e: logger.error('Failed to process wav file.') @@ -159,6 +159,7 @@ if __name__=="__main__": parser.add_argument("--input_path", help="path to the input files", required=True) parser.add_argument("--output_path", help="path to the raw and derived variable output", required=True) parser.add_argument("--dbm_group", help="list of feature groups", nargs='+') + parser.add_argument("--tr", help="speech transcription toogle") args = parser.parse_args() s_config = config_reader.ConfigReader() diff --git a/process_dbm.sh b/process_dbm.sh index c573eccc..a74f50ea 100644 --- a/process_dbm.sh +++ b/process_dbm.sh @@ -7,6 +7,7 @@ helpFunction() echo -e "\t--input_path: path to the input files" echo -e "\t--output_path: path to the raw and derived variable output" echo -e "\t--dbm_group: list of feature groups" + echo -e "\t--tr: Toggle for speech transcription(optional)" exit 1 # Exit script after printing help } @@ -15,6 +16,7 @@ while [ $# -gt 0 ]; do --input_path=*) input_path="${1#*=}" ;; --output_path=*) output_path="${1#*=}" ;; --dbm_group=*) dbm_group="${1#*=}" ;; + --tr=*) tr="${1#*=}" ;; *) helpFunction ;; esac shift @@ -55,8 +57,11 @@ fi if [[ $dbm_group == *"movement"* ]]; then dbm_new="$dbm_new movement" fi -if [[ $dbm_group == *"nlp"* ]]; then - dbm_new="$dbm_new nlp" +if [[ $dbm_group == *"speech"* ]]; then + dbm_new="$dbm_new speech" +fi +if [[ $dbm_group == *"speech"* ]] && [[ ${tr,,} == "on" ]]; then + dbm_new="$dbm_new --tr ${tr,,}" fi #docker commands to run container From 2407db6d11658623edca454e81a745219538befe Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Tue, 8 Dec 2020 14:16:27 -0500 Subject: [PATCH 24/26] ignore warning --- process_dbm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process_dbm.sh b/process_dbm.sh index a74f50ea..7cb38519 100644 --- a/process_dbm.sh +++ b/process_dbm.sh @@ -71,9 +71,9 @@ docker cp $input_path dbm_container:/app/raw_data docker start dbm_container if [ -z "$dbm_new" ] then - docker exec -it dbm_container /bin/bash -c "python3 process_data.py --input_path /app/raw_data --output_path /app/output" + docker exec -it dbm_container /bin/bash -c "python3 -W ignore process_data.py --input_path /app/raw_data --output_path /app/output" else - docker exec -it dbm_container /bin/bash -c "python3 process_data.py --input_path /app/raw_data --output_path /app/output --dbm_group$dbm_new" + docker exec -it dbm_container /bin/bash -c "python3 -W ignore process_data.py --input_path /app/raw_data --output_path /app/output --dbm_group$dbm_new" fi docker cp dbm_container:/app/output $output_path From bebaa201bac51597042b5ece47e0004d601f2f26 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Tue, 8 Dec 2020 17:13:05 -0500 Subject: [PATCH 25/26] bash fix for mac --- process_dbm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process_dbm.sh b/process_dbm.sh index 7cb38519..10991adf 100644 --- a/process_dbm.sh +++ b/process_dbm.sh @@ -60,8 +60,8 @@ fi if [[ $dbm_group == *"speech"* ]]; then dbm_new="$dbm_new speech" fi -if [[ $dbm_group == *"speech"* ]] && [[ ${tr,,} == "on" ]]; then - dbm_new="$dbm_new --tr ${tr,,}" +if [[ $dbm_group == *"speech"* ]] && [[ ${tr} == "on" ]]; then + dbm_new="$dbm_new --tr ${tr}" fi #docker commands to run container From dad92abfbce9cb2826801044b1e7937a4da7e262 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 9 Dec 2020 18:17:09 -0500 Subject: [PATCH 26/26] list initialization --- dbm_lib/dbm_features/raw_features/audio/formant_freq.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py index 592ebdc3..2d2eff92 100644 --- a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py +++ b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py @@ -31,7 +31,11 @@ def formant_list(formant,snd): Returns: List of first through fourth formant for each frame """ - f1_list, f2_list, f3_list, f4_list = ([], ) * 4 + f1_list = [] + f2_list = [] + f3_list = [] + f4_list = [] + dur = snd.duration-0.02 dur_round = round(dur, 2) @@ -126,4 +130,4 @@ def run_formant(video_uri, out_dir, r_config): calc_formant(video_uri, audio_file, out_loc, fl_name, r_config) except Exception as e: - logger.error('Failed to process audio file') \ No newline at end of file + logger.error('Failed to process audio file')