diff --git a/Dockerfile b/Dockerfile index c3dfd352..5d425d61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,9 @@ RUN apt-get update && apt-get install -y python3-pip \ && apt-get install -y libavcodec-dev \ && apt-get install -y libavformat-dev \ && apt-get install -y libavdevice-dev \ - && apt-get install -y libboost-all-dev + && apt-get install -y libboost-all-dev \ + && apt-get install -y git \ + && apt-get install -y sox RUN ln -sfn /usr/bin/pip3 /usr/bin/pip COPY . /app @@ -24,8 +26,15 @@ RUN dpkg --configure -a RUN su -c ./install.sh RUN echo "Done OpenFace!" -WORKDIR /app +RUN echo "Cloning DeepSpeech..." +WORKDIR /app/pkg +RUN git clone https://github.com/mozilla/DeepSpeech.git +WORKDIR /app/pkg/DeepSpeech +RUN wget https://github.com/mozilla/DeepSpeech/releases/download/v0.9.1/deepspeech-0.9.1-models.pbmm +RUN wget https://github.com/mozilla/DeepSpeech/releases/download/v0.9.1/deepspeech-0.9.1-models.scorer + +WORKDIR /app RUN pip install --upgrade pip RUN pip install -r requirements.txt RUN echo "Requirement txt done!" diff --git a/dbm_lib/config/config_raw_feature.py b/dbm_lib/config/config_raw_feature.py index 554359cd..bee0643a 100644 --- a/dbm_lib/config/config_raw_feature.py +++ b/dbm_lib/config/config_raw_feature.py @@ -229,3 +229,51 @@ class ConfigRawReader(object): self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw'] self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll'] self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist'] + + self.mov_freq_trem_freq = config['raw_feature']['mov_freq_trem_freq'] + self.mov_freq_trem_index = config['raw_feature']['mov_freq_trem_index'] + self.mov_freq_trem_pindex = config['raw_feature']['mov_freq_trem_pindex'] + self.mov_amp_trem_freq = config['raw_feature']['mov_amp_trem_freq'] + self.mov_amp_trem_index = config['raw_feature']['mov_amp_trem_index'] + self.mov_amp_trem_pindex = config['raw_feature']['mov_amp_trem_pindex'] + + self.fac_tremor_median_5 = config['raw_feature']['fac_tremor_median_5'] + self.fac_tremor_median_12 = config['raw_feature']['fac_tremor_median_12'] + self.fac_tremor_median_8 = config['raw_feature']['fac_tremor_median_8'] + self.fac_tremor_median_48 = config['raw_feature']['fac_tremor_median_48'] + self.fac_tremor_median_54 = config['raw_feature']['fac_tremor_median_54'] + self.fac_tremor_median_28 = config['raw_feature']['fac_tremor_median_28'] + self.fac_tremor_median_51 = config['raw_feature']['fac_tremor_median_51'] + self.fac_tremor_median_66 = config['raw_feature']['fac_tremor_median_66'] + self.fac_tremor_median_57 = config['raw_feature']['fac_tremor_median_57'] + + self.mov_leye_x = config['raw_feature']['mov_leye_x'] + self.mov_leye_y = config['raw_feature']['mov_leye_y'] + self.mov_leye_z = config['raw_feature']['mov_leye_z'] + self.mov_reye_x = config['raw_feature']['mov_reye_x'] + self.mov_reye_y = config['raw_feature']['mov_reye_y'] + self.mov_reye_z = config['raw_feature']['mov_reye_z'] + self.mov_eleft_disp = config['raw_feature']['mov_eleft_disp'] + self.mov_eright_disp = config['raw_feature']['mov_eright_disp'] + + #NLP features + self.nlp_transcribe = config['raw_feature']['nlp_transcribe'] + self.nlp_numSentences = config['raw_feature']['nlp_numSentences'] + self.nlp_singPronPerAns = config['raw_feature']['nlp_singPronPerAns'] + self.nlp_singPronPerSen = config['raw_feature']['nlp_singPronPerSen'] + self.nlp_pastTensePerAns = config['raw_feature']['nlp_pastTensePerAns'] + self.nlp_pastTensePerSen = config['raw_feature']['nlp_pastTensePerSen'] + self.nlp_pronounsPerAns = config['raw_feature']['nlp_pronounsPerAns'] + self.nlp_pronounsPerSen = config['raw_feature']['nlp_pronounsPerSen'] + self.nlp_verbsPerAns = config['raw_feature']['nlp_verbsPerAns'] + self.nlp_verbsPerSen = config['raw_feature']['nlp_verbsPerSen'] + self.nlp_adjectivesPerAns = config['raw_feature']['nlp_adjectivesPerAns'] + self.nlp_adjectivesPerSen = config['raw_feature']['nlp_adjectivesPerSen'] + self.nlp_nounsPerAns = config['raw_feature']['nlp_nounsPerAns'] + self.nlp_nounsPerSen = config['raw_feature']['nlp_nounsPerSen'] + self.nlp_sentiment_mean = config['raw_feature']['nlp_sentiment_mean'] + self.nlp_mattr = config['raw_feature']['nlp_mattr'] + self.nlp_wordsPerMin = config['raw_feature']['nlp_wordsPerMin'] + self.nlp_totalTime = config['raw_feature']['nlp_totalTime'] + + diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 3282edc2..6009864e 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,9 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, eye_gaze, voice_tremor, facial_tremor +from dbm_lib.dbm_features.raw_features.nlp import transcribe, speech_features + import subprocess import logging @@ -19,13 +21,13 @@ logging.basicConfig(level=logging.INFO) logger=logging.getLogger() def audio_to_wav(input_filepath): - """ Extracts a video's audio file and saves it to wav + """ Extracts a video's audio file and saves it to wav Args: - input_filepath: (str) + input_filepath: (str) Returns: """ try: - + fname, _ = splitext(input_filepath) output_filepath = fname + '.wav' @@ -37,7 +39,7 @@ def audio_to_wav(input_filepath): logger.info('wav output saved in {}'.format(output_filepath)) else: logger.info('Output file {} already exists'.format(output_filepath)) - + except Exception as e: logger.error('Failed to extract audio from Video') @@ -50,38 +52,38 @@ def process_acoustic(video_uri, out_dir, dbm_group, r_config): """ if dbm_group != None and len(dbm_group)>0 and 'acoustic' not in dbm_group: return - + logger.info('Processing acoustic variables from data in {}'.format(video_uri)) logger.info('processing audio intensity....') intensity.run_intensity(video_uri, out_dir, r_config) - + logger.info('processing audio pitch freq....') pitch_freq.run_pitch(video_uri, out_dir, r_config) - + logger.info('processing HNR....') hnr.run_hnr(video_uri, out_dir, r_config) - + logger.info('processing GNE....') gne.run_gne(video_uri, out_dir, r_config) - + logger.info('processing voice frame score....') voice_frame_score.run_vfs(video_uri, out_dir, r_config) - + logger.info('processing formant frequency....') formant_freq.run_formant(video_uri, out_dir, r_config) - + logger.info('processing pause segment....') pause_segment.run_pause_segment(video_uri, out_dir, r_config) - + logger.info('processing jitter....') jitter.run_jitter(video_uri, out_dir, r_config) - + logger.info('processing shimmer....') shimmer.run_shimmer(video_uri, out_dir, r_config) - + logger.info('processing mfcc....') mfcc.run_mfcc(video_uri, out_dir, r_config) - + def process_facial(video_uri, out_dir, dbm_group, r_config): """ processing facial features @@ -91,20 +93,20 @@ def process_facial(video_uri, out_dir, dbm_group, r_config): """ if dbm_group != None and len(dbm_group)>0 and 'facial' not in dbm_group: return - + logger.info('Processing facial variables from data in {}'.format(video_uri)) logger.info('processing facial asymmetry....') face_asymmetry.run_face_asymmetry(video_uri, out_dir, r_config) - + logger.info('processing facial Action Unit....') face_au.run_face_au(video_uri, out_dir, r_config) - + logger.info('processing facial expressivity....') face_emotion_expressivity.run_face_expressivity(video_uri, out_dir, r_config) - + logger.info('processing facial landmark....') face_landmark.run_face_landmark(video_uri, out_dir, r_config) - + def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): """ processing facial features @@ -115,23 +117,48 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): """ if dbm_group != None and len(dbm_group)>0 and 'movement' not in dbm_group: return - + logger.info('Processing movement variables from data in {}'.format(video_uri)) + logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) - + logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) + + logger.info('processing eye gaze....') + eye_gaze.run_eye_gaze(video_uri, out_dir, r_config) + logger.info('processing voice tremor....') + voice_tremor.run_vtremor(video_uri, out_dir, r_config) + + logger.info('processing facial tremor....') + facial_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + + +def process_nlp(video_uri, out_dir, dbm_group, tran_tog, r_config, deep_path): + """ + processing nlp features + Args: + video_uri: video path; out_dir: raw variable output dir + dbm_group: list of features to process; r_config: raw feature config object + deep_path: deep speech build path + """ + if dbm_group != None and len(dbm_group)>0 and 'speech' not in dbm_group: + return + + logger.info('Processing nlp variables from data in {}'.format(video_uri)) + transcribe.run_transcribe(video_uri, out_dir, r_config, deep_path) + speech_features.run_speech_feature(video_uri, out_dir, r_config, tran_tog) + + def remove_file(file_path): """ - removing wav file + removing wav file """ file_dir = dirname(file_path) file_name, _ = splitext(basename(file_path)) wav_file = glob.glob(join(file_dir, file_name + '.wav')) - + if len(wav_file)> 0: os.remove(wav_file[0]) - - \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py index 592ebdc3..2d2eff92 100644 --- a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py +++ b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py @@ -31,7 +31,11 @@ def formant_list(formant,snd): Returns: List of first through fourth formant for each frame """ - f1_list, f2_list, f3_list, f4_list = ([], ) * 4 + f1_list = [] + f2_list = [] + f3_list = [] + f4_list = [] + dur = snd.duration-0.02 dur_round = round(dur, 2) @@ -126,4 +130,4 @@ def run_formant(video_uri, out_dir, r_config): calc_formant(video_uri, audio_file, out_loc, fl_name, r_config) except Exception as e: - logger.error('Failed to process audio file') \ No newline at end of file + logger.error('Failed to process audio file') diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index f2a2d4b9..2ed5b79a 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -11,3 +11,7 @@ from __future__ import print_function import os DBMLIB_PATH = os.path.dirname(__file__) +DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, + '../../../../resources/libraries/voice_tremor.praat')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json')) + diff --git a/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py new file mode 100644 index 00000000..a574a2d9 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/eye_gaze.py @@ -0,0 +1,148 @@ +""" +file_name: eye_gaze +project_name: DBM +created: 2020-30-11 +""" + +import os +import glob +import pandas as pd +import numpy as np +from scipy.spatial import distance +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +eye_pose_dir = 'movement/gaze' +eye_pose_ext = '_eyegaze.csv' + +def eye_motion_df(l_disp, r_disp, error_list, r_config): + """ + Generating eye movement dataframe + + Args: + l_disp: displacement list(left eye); l_disp: displacement list(right eye) + r_config: raw variable config file object + + Reutrns: + Final eye displacement dataframe + """ + df_eye_left = pd.DataFrame(l_disp, columns=[r_config.mov_eleft_disp]) + df_eye_right = pd.DataFrame(r_disp, columns=[r_config.mov_eright_disp]) + + df_eye_motion = pd.concat([df_eye_left, df_eye_right], axis=1, sort=False) + df_eye_motion[r_config.err_reason] = error_list + return df_eye_motion + +def filter_motion(df_of, df_disp, col_l, col_r, r_config): + """ + Filtering final eye movement dataframe + + Args: + df_of: Openface raw out dataframe; col_r: right eye column + col_l: left eye column; r_config: raw variable config file object + """ + + df_of = df_of[col_l + col_r + [' confidence']] + df_of.loc[(df_of[' confidence'].astype(float) < 0.8), col_l + col_r] = np.nan + + df_filter = df_of[col_l + col_r] + df_filter.columns = [r_config.mov_leye_x, r_config.mov_leye_y, r_config.mov_leye_z, + r_config.mov_reye_x, r_config.mov_reye_y, r_config.mov_reye_z] + + df_motion = pd.concat([df_filter, df_disp], axis=1, sort=False) + return df_motion + +def eye_disp(of_results, col, r_config): + """ + Computing head velocity frame by frame + + Args: + of_results: Openface raw out dataframe + r_config: Face config file object + + Reutrns: + Final head velocity frame by frame output + """ + distance_list = [] + error_list = [] + + of_results = of_results[col+ [' confidence']] + for index, row in of_results.iterrows(): + dst = np.nan + + if index == 0 or float(row[' confidence']) < 0.8: #Threshold < 0.8 + distance_list.append(dst) + + if float(row[' confidence']) < 0.8: + error_list.append('confidence less than 80%') + + else: + error_list.append('Pass') + continue + + if index > 0: + + point_x = (of_results[col[0]][index-1], of_results[col[1]][index-1], of_results[col[2]][index-1]) + point_y = (row[col[0]],row[col[1]],row[col[2]]) + try: + dst = distance.euclidean(point_x, point_y) + except: + pass + + distance_list.append(abs(dst)) + error_list.append('Pass') + + return distance_list, error_list + +def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config): + """ + Computing eye motion variables + Args: + df_of: Openface dataframe + out_loc: Output path for saving output csv's + fl_name: file name for output csv + r_config: raw variable config file object + + """ + + col_l = [ ' gaze_0_x', ' gaze_0_y', ' gaze_0_z'] + col_r = [ ' gaze_1_x', ' gaze_1_y', ' gaze_1_z'] + + gazel_disp, err_l = eye_disp(df_of, col_l, r_config) + gazer_disp, err_r = eye_disp(df_of, col_r, r_config) + + df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config) + df_disp['dbm_master_url'] = video_uri + + df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config) + ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext) + +def run_eye_gaze(video_uri, out_dir, r_config): + """ + Processing all patient's for getting eye movement artifacts + -------------------------------- + -------------------------------- + Args: + video_uri: video path; input_dir : input directory for video's + out_dir: (str) Output directory for processed output; r_config: raw variable config object + """ + try: + + #filtering path to generate input & output path + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + + if len(of_csv_path)>0: + + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config) + except Exception as e: + logger.error('Failed to process video file') \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py new file mode 100644 index 00000000..7753eb0b --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -0,0 +1,164 @@ +import sys, os, glob, cv2, re +import pickle, json +import pandas as pd +import numpy as np +import numpy.ma as ma +import logging +from os.path import join + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util.math_util import * + +from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +ft_dir = 'movement/facial_tremor' +csv_ext = '_fac_tremor.csv' +model_ext = '_fac_model.csv' +fac_features_ext = '_fac_features.csv' + +def compute_features(out_dir, df_of, r_config): + """ Computes features + + Returns: features in vector format + """ + config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) + + pattern_x = re.compile("l\d+_x") + pattern_y = re.compile("l\d+_y") + + # assumption: distance of face to camera remains at roughly static + + # logic break + landmark_columns = [] + for col in df_of.columns: + if pattern_x.match(col) or pattern_y.match(col): + landmark_columns.append(col) + + df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] + df_of.reset_index(inplace=True) + + num_frames = len(df_of) + logger.info("Number of frames to be processed: {}".format(str(num_frames))) + landmarks = config['landmarks'] + + try: + if num_frames == 0: + error_reason = "No frames with visible face." + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + +# if num_frames < 60: +# error_reason = 'Number of frames with visible face < 60. Video too short' +# logger.error(error_reason) +# return empty_frame(landmarks, f_cfg, error_reason) + + first_row = df_of.iloc[0] + + facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']]) + faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']]) + + if facew == 0 or faceh == 0: + error_reason = 'face width or height = 0. Check landmark values' + logger.error(error_reason) + return empty_frame(landmarks, r_config) + + fac_disp = calc_displacement_vec(df_of, landmarks, num_frames) + + # if verbose: + # logger.info("Displacement output: {}".format(str(fac_disp))) + + fac_disp_median = np.median(fac_disp, axis = 1) + fac_disp_mean = np.mean(fac_disp, axis = 1) + + if len(fac_disp.shape)!=2: + error_reason = 'fac_disp is not 2D. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + if len(fac_disp[0])<=1: + error_reason = 'Video too short. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + fac_corr_mat = np.corrcoef(fac_disp, rowvar = True) + # extract relevant row from cov matrix + ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk] + fac_corr = fac_corr_mat[ref_lmk_index][0] + + fac_area = config['ref_area'] / (facew * faceh) + + # if verbose: + # logger.info("Face area: {}".format(fac_area)) + # logger.info("Face Displacement Median: {}".format(str(fac_disp_median))) + # logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean))) + + fac_features1 = np.multiply(fac_area * fac_disp_median, (1. - fac_corr)) + fac_features2 = np.multiply(fac_area * fac_disp_mean, (1. - fac_corr)) + +# base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr)) + + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]] + + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] + fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] + + fac_features_dict[r_config.err_reason] = [''] + data = pd.DataFrame.from_dict(fac_features_dict) + logger.info('Concluded computing tremor features') + + return data + + except Exception as e: + logger.error('Error computing tremor features: {}'.format(str(e))) + return empty_frame(landmarks, r_config, str(e)) + +def empty_frame(landmarks, r_config, error_reason): + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan] + + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan] + + fac_features_dict[r_config.err_reason] = [error_reason] + empty_frame = pd.DataFrame.from_dict(fac_features_dict) + return empty_frame + +def fac_tremor_process(video_uri, out_dir, r_config, model_output=False): + """ + processing input videos + + + """ +# try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) + + if len(of_csv_path)>0: + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + + feats = compute_features(of_csv_path , df_of, r_config) + +# if model_output: +# result = score(feats, r_config) +# feats = pd.concat([feats, result], axis=1) + + ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext) + + + +# except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/movement/head_motion.py b/dbm_lib/dbm_features/raw_features/movement/head_motion.py index c45bc123..59179c2e 100644 --- a/dbm_lib/dbm_features/raw_features/movement/head_motion.py +++ b/dbm_lib/dbm_features/raw_features/movement/head_motion.py @@ -179,11 +179,12 @@ def run_head_movement(video_uri, out_dir, r_config): out_dir: (str) Output directory for processed output; r_config: raw variable config object """ try: - + #filtering path to generate input & output path input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + if len(of_csv_path)>0: of_csv = of_csv_path[0] @@ -192,4 +193,4 @@ def run_head_movement(video_uri, out_dir, r_config): logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config) except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py new file mode 100644 index 00000000..868fa152 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py @@ -0,0 +1,94 @@ +import pandas as pd +import os +import glob +from os.path import join +import parselmouth +from parselmouth.praat import call, run_file +import numpy as np +import librosa +import json +import re +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.movement import DBMLIB_VTREMOR_LIB + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +vt_dir = 'movement/voice_tremor' +csv_ext = '_vtremor.csv' + +#Executing praat script using parselmouth function +def tremor_praat(snd_file,r_cfg): + """ + Generating Voice tremor endpoint dataframe + Args: + snd_file: (.wav) parsed audio file + r_cfg: Raw variable configuration file + Returns tremor endpoint dataframe + """ + snd = parselmouth.Sound(snd_file) + tremor_var = run_file(snd,DBMLIB_VTREMOR_LIB,capture_output=True) + new_tremor_var = re.sub('--undefined--', '0', tremor_var[1]) + res = json.loads(new_tremor_var) + tremor_df = pd.DataFrame(res,index=['0',]) + tremor_df.columns = [r_cfg.mov_freq_trem_freq,r_cfg.mov_amp_trem_freq,r_cfg.mov_freq_trem_index, + r_cfg.mov_amp_trem_index,r_cfg.mov_freq_trem_pindex,r_cfg.mov_amp_trem_pindex] + return tremor_df + +def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name): + """ + Preparing voice tremor matrix + Args: + audio_file: (.wav) parsed audio file ; r_config: raw config object + out_loc: (str) Output directory for csv ; fl_name: file name + """ + df_tremor = tremor_praat(audio_file, r_config) + df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) + +def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): + + """ + Preparing empty voice tremor matrix + """ + cols = [r_config.mov_freq_trem_freq, r_config.mov_amp_trem_freq, r_config.mov_freq_trem_index, + r_config.mov_amp_trem_index, r_config.mov_freq_trem_pindex, r_config.mov_amp_trem_pindex, r_config.err_reason] + + out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] + df_tremor = pd.DataFrame(out_val, columns = cols) + + logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name))) + ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) + +def run_vtremor(video_uri, out_dir, r_config): + """ + Processing all patient's for fetching Formant freq + --------------- + --------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output + """ + try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) + if len(aud_filter)>0: + + audio_file = aud_filter[0] + aud_dur = librosa.get_duration(filename=audio_file) + + if float(aud_dur) < 0.5: + logger.info('Output file {} size is less than 0.5sec'.format(audio_file)) + + error_txt = 'error: length less than 0.5 sec' + prepare_empty_vt(video_uri, out_loc, fl_name, error_txt) + return + prepare_vtrem_output(audio_file, out_loc, r_config, fl_name) + except Exception as e: + logger.error('Failed to compute Voice Tremor {} for {}'.format(e,video_uri)) + prepare_empty_vt(out_loc, fl_name, r_config, e) diff --git a/dbm_lib/dbm_features/raw_features/nlp/speech_features.py b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py new file mode 100644 index 00000000..3f0cfba7 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/nlp/speech_features.py @@ -0,0 +1,51 @@ +""" +file_name: speech_features +project_name: DBM +created: 2020-13-11 +""" + +import os +import numpy as np +import pandas as pd +import glob +from os.path import join +import logging +import shutil + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +speech_dir = 'nlp/speech_feature' +speech_ext = '_nlp.csv' +transcribe_ext = 'nlp/transcribe/*_transcribe.csv' + +def run_speech_feature(video_uri, out_dir, r_config, tran_tog): + """ + Processing all patient's for fetching nlp features + ------------------- + ------------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output + """ + try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + + transcribe_path = glob.glob(join(out_loc, transcribe_ext)) + if len(transcribe_path)>0: + + transcribe_df = pd.read_csv(transcribe_path[0]) + df_speech= n_util.process_speech(transcribe_df, r_config) + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext) + + if (tran_tog == None) or (tran_tog != 'on'): + shutil.rmtree(os.path.dirname(transcribe_path[0])) + + except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/nlp/transcribe.py b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py new file mode 100644 index 00000000..f567e967 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/nlp/transcribe.py @@ -0,0 +1,84 @@ +""" +file_name: transcribe +project_name: DBM +created: 2020-10-11 +""" + +import pandas as pd +import numpy as np +import librosa +import glob +from os.path import join +import logging + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util import nlp_util as n_util + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +formant_dir = 'nlp/transcribe' +csv_ext = '_transcribe.csv' +error_txt = 'error: length less than 0.1' + +def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur): + """ + Preparing Formant freq matrix + Args: + audio_file: (.wav) parsed audio file; fl_name: input file name + out_loc: (str) Output directory; r_config: raw variable config + """ + + text = n_util.process_deepspeech(audio_file, deep_path) + df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe]) + + df_formant.replace('', np.nan, regex=True,inplace=True) + df_formant[r_config.nlp_totalTime] = aud_dur + df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release + df_formant['dbm_master_url'] = video_uri + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext) + +def empty_transcribe(video_uri, out_loc, fl_name, r_config): + + """ + Preparing empty formant frequency matrix if something fails + """ + cols = [r_config.nlp_transcribe, r_config.nlp_totalTime, r_config.err_reason] + out_val = [[np.nan, np.nan, error_txt]] + df_fm = pd.DataFrame(out_val, columns = cols) + df_fm['dbm_master_url'] = video_uri + + logger.info('Saving Output file {} '.format(out_loc)) + ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext) + +def run_transcribe(video_uri, out_dir, r_config, deep_path): + + """ + Processing all patient's for fetching Formant freq + --------------- + --------------- + Args: + video_uri: video path; r_config: raw variable config object + out_dir: (str) Output directory for processed output; deep_path: deepspeech build path + """ + try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) + if len(aud_filter)>0: + + audio_file = aud_filter[0] + aud_dur = librosa.get_duration(filename=audio_file) + + if float(aud_dur) < 0.1: + logger.info('Output file {} size is less than 0.1 sec'.format(audio_file)) + + empty_transcribe(video_uri, out_loc, fl_name, r_config) + return + + calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur) + except Exception as e: + logger.error('Failed to process audio file') + \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/util/math_util.py b/dbm_lib/dbm_features/raw_features/util/math_util.py new file mode 100644 index 00000000..283acadc --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/util/math_util.py @@ -0,0 +1,57 @@ +""" +file_name: facial_tremor +project_name: cdx_analysis +created: 2019-03-16 +author: Deshana Desai +""" +import sys, os, glob, cv2 +import pandas as pd +import numpy as np + + +def euclidean_distance(point1, point2): + """ + Compute euclidean distance between points + """ + + return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2) + + +# def detect_peaks() + + +def expand_landmarks(landmarks): + """ + util method to expand landmark list: + eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']] + """ + return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks] + + + +def calc_displacement_vec(df, landmarks, num_frames): + """ + Calculates displacement vector frame by frame + """ + + landmarks = expand_landmarks(landmarks) + + disp_vec = np.zeros((len(landmarks), num_frames)) + prev_point = np.zeros((len(landmarks), 2)) + + # initialize + for j, pair in enumerate(landmarks): + first_row = df.iloc[0] + prev_point[j] = (first_row[pair[0]], first_row[pair[1]]) + + + for i in range(num_frames): + frame_row = df.iloc[i] + for j, pair in enumerate(landmarks): + x, y = pair[0], pair[1] + current = (frame_row[x], frame_row[y]) + deviation = euclidean_distance( current, prev_point[j]) + disp_vec[j][i] = deviation + prev_point[j] = current + + return disp_vec diff --git a/dbm_lib/dbm_features/raw_features/util/nlp_util.py b/dbm_lib/dbm_features/raw_features/util/nlp_util.py new file mode 100644 index 00000000..6a07141e --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/util/nlp_util.py @@ -0,0 +1,212 @@ +""" +file_name: nlp_util +project_name: DBM +created: 2020-10-11 +""" + +import subprocess +import json +import numpy as np +import pandas as pd +import os +import logging + +import nltk +import re +from lexicalrichness import LexicalRichness +from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +#Speech to text using Deepspeech 0.9.1 +def deepspeech(AUDIO_FILE,deep_path): + """ + Extracting text from audio using Deep Speech neural network trained model + Returns: + Text: text which is extracted from audio + """ + api = 'deepspeech' + arg_speech0 = '--model' + arg_speech_path0 = os.path.join(deep_path, 'deepspeech-0.9.1-models.pbmm') + arg_speech1 = '--scorer' + arg_speech_path1 = os.path.join(deep_path, 'deepspeech-0.9.1-models.scorer') + arg_audio = "--audio" + + out = subprocess.Popen([api, arg_speech0, arg_speech_path0, arg_speech1, arg_speech_path1, arg_audio, AUDIO_FILE], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + logger.info('Deepspeech output...... {}'.format(out)) + try: + stdout,stderr = out.communicate() + except: + return "error", "error" + print(stderr) + return stdout,stderr + +def deep_speech_output_clean(result): + """ + Parsing deep speech output(text) + Return: + Text from speech + """ + text = "" + if len(result)>0: + res_split = str(result[0]).split('\\n') + + if len(res_split)>0: + for i in range(len(res_split)): + if 'Inference took' in res_split[i]: + text = res_split[i + 1] + return text + return text + +def process_deepspeech(audio_file,deep_path): + """ + Transcribing audio to extract text from speech + """ + deep_output = deepspeech(audio_file,deep_path) + deep_text= deep_speech_output_clean(deep_output) + + return deep_text + +def nltk_download(): + + try: + nltk.data.find('tokenizers/punkt') + + except LookupError: + logger.info('punkt is not available') + nltk.download('punkt') + + try: + nltk.data.find('averaged_perceptron_tagger') + + except LookupError: + logger.info('averaged_perceptron_tagger is not available') + nltk.download('averaged_perceptron_tagger') + +def empty_speech(r_config, master_url, error_txt): + """ + Preparing empty speech matrix with error + Args: + r_config: raw config file object + error_txt: Error message during transcription + + Returns: + Empty dataframe for speech features with error + """ + + col = [r_config.nlp_numSentences, r_config.nlp_singPronPerAns, r_config.nlp_singPronPerSen, r_config.nlp_pastTensePerAns, + r_config.nlp_pastTensePerSen, r_config.nlp_pronounsPerAns, r_config.nlp_pronounsPerSen, r_config.nlp_verbsPerAns, + r_config.nlp_verbsPerSen, r_config.nlp_adjectivesPerAns, r_config.nlp_adjectivesPerSen, r_config.nlp_nounsPerAns, + r_config.nlp_nounsPerSen, r_config.nlp_sentiment_mean, r_config.nlp_mattr, r_config.nlp_wordsPerMin, + r_config.nlp_totalTime, r_config.err_reason] + + df_speech = pd.DataFrame([[np.nan] * len(col) + [error_txt]], columns = col) + df_speech['dbm_master_url'] = master_url + + return df_speech + +def divide_var(speech_var1, spech_var2): + """ + divide variables + """ + speech_var = np.nan + if spech_var2!=0: + speech_var = speech_var1/spech_var2 + return speech_var + +def process_speech(transcribe_df,r_config): + """ + Preparing speech features + Args: + transcribe_df: Transcribed dataframe + r_config: raw config file object + Returns: + Dataframe for speech features + """ + transcribe_df = transcribe_df.replace(np.nan, '', regex=True) + err_transcribe = transcribe_df[r_config.err_reason].iloc[0] + transcribe = transcribe_df[r_config.nlp_transcribe].iloc[0] + total_time = transcribe_df[r_config.nlp_totalTime].iloc[0] + master_url = transcribe_df['dbm_master_url'].iloc[0] + + #clean transcribe + transcribe = transcribe.replace(",", "") + transcribe = " ".join(re.findall(r"[\w']+|[.!?]", transcribe)) + + if err_transcribe != 'Pass': + df_speech = empty_speech(r_config, master_url, error_txt) + + return df_speech + + speech_dict = {} + nltk_download() + + sentences = nltk.tokenize.sent_tokenize(transcribe) + words_all = nltk.tokenize.word_tokenize(transcribe) + num_sentences = len(sentences) + + speech_dict[r_config.nlp_numSentences] = num_sentences + + #nlp_singPron + i_s = transcribe.count('I') + me_s = transcribe.count('me') + my_s = transcribe.count('my') + sing_count = i_s + me_s + my_s + + speech_dict[r_config.nlp_singPronPerAns] = sing_count if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_singPronPerSen] = divide_var(speech_dict[r_config.nlp_singPronPerAns], num_sentences) + + tagged = nltk.pos_tag(transcribe.split()) + tagged_df = pd.DataFrame(tagged, columns=['word', 'pos_tag']) + + #Past tense per answer + all_POSs = tagged_df['pos_tag'].tolist() + speech_dict[r_config.nlp_pastTensePerAns] = all_POSs.count('VBD') if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_pastTensePerSen] = divide_var(speech_dict[r_config.nlp_pastTensePerAns], num_sentences) + + #Pronoun per answer + pronounsPerAns = all_POSs.count('PRP') + all_POSs.count('PRP$') + speech_dict[r_config.nlp_pronounsPerAns] = pronounsPerAns if len(words_all)>0 else np.nan + speech_dict[r_config.nlp_pronounsPerSen] = divide_var(speech_dict[r_config.nlp_pronounsPerAns], num_sentences) + + #Verb per answer + verbPerAns = all_POSs.count('VB') + all_POSs.count('VBD') + all_POSs.count('VBG') \ + + all_POSs.count('VBN') + all_POSs.count('VBP') + all_POSs.count('VBZ') + speech_dict[r_config.nlp_verbsPerAns] = verbPerAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_verbsPerSen] = divide_var(speech_dict[r_config.nlp_verbsPerAns], num_sentences) + + #Adjective per answer + adjectivesAns = all_POSs.count('JJ') + all_POSs.count('JJR') + all_POSs.count('JJS') + speech_dict[r_config.nlp_adjectivesPerAns] = adjectivesAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_adjectivesPerSen] = divide_var(speech_dict[r_config.nlp_adjectivesPerAns], num_sentences) + + #Noun per answer + nounsAns = all_POSs.count('NN') + all_POSs.count('NNP') + all_POSs.count('NNS') + speech_dict[r_config.nlp_nounsPerAns] = nounsAns if len(words_all) > 0 else np.nan + speech_dict[r_config.nlp_nounsPerSen] = divide_var(speech_dict[r_config.nlp_nounsPerAns], num_sentences) + + #Sentiment analysis + vader = SentimentIntensityAnalyzer() + sentence_valences = [] + + for s in sentences: + sentiment_dict = vader.polarity_scores(s) + sentence_valences.append(sentiment_dict['compound']) + + speech_dict[r_config.nlp_sentiment_mean] = np.mean(sentence_valences) if len(sentence_valences) > 0 else np.nan + non_punc = list(value for value in words_all if value not in ['.','!','?']) + + non_punc_as_str = " ".join(str(non_punc)) + lex = LexicalRichness(non_punc_as_str) + speech_dict[r_config.nlp_mattr] = lex.mattr(window_size=lex.words) if lex.words > 0 else np.nan + + #Number of words per minute + speech_dict[r_config.nlp_wordsPerMin] = divide_var(len(non_punc), total_time)*60 + speech_dict[r_config.nlp_totalTime] = total_time + speech_dict['dbm_master_url'] = master_url + + df_speech = pd.DataFrame([speech_dict]) + return df_speech \ No newline at end of file diff --git a/dbm_lib/dbm_features/raw_features/util/vad_utilities.py b/dbm_lib/dbm_features/raw_features/util/vad_utilities.py index b769ba19..8f11a2e8 100644 --- a/dbm_lib/dbm_features/raw_features/util/vad_utilities.py +++ b/dbm_lib/dbm_features/raw_features/util/vad_utilities.py @@ -158,7 +158,7 @@ def vad_get_segment_times(sample_rate, frame_duration_ms, for frame in frames: is_speech = vad.is_speech(frame.bytes, sample_rate) - sys.stdout.write('1' if is_speech else '0') + #sys.stdout.write('1' if is_speech else '0') if not triggered: ring_buffer.append((frame, is_speech)) num_voiced = len([f for f, speech in ring_buffer if speech]) @@ -167,7 +167,7 @@ def vad_get_segment_times(sample_rate, frame_duration_ms, # TRIGGERED state. if num_voiced > 0.9 * ring_buffer.maxlen: triggered = True - sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,)) + #sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,)) start_times.append(ring_buffer[0][0].timestamp) # BT ring_buffer.clear() else: @@ -179,18 +179,18 @@ def vad_get_segment_times(sample_rate, frame_duration_ms, # unvoiced, then enter NOTTRIGGERED and yield whatever # audio we've collected. if num_unvoiced > 0.9 * ring_buffer.maxlen: - sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) + #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) end_times.append(ring_buffer[0][0].timestamp + frame.duration) # BT triggered = False if triggered: # BT if were in triggered state at end of signal, set output time - sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) + #sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration)) if len(ring_buffer)>0: end_times.append(ring_buffer[0][0].timestamp ) # BT else: # only get here in very rare case that we triggered on 2nd-to-last frame end_times.append(frame.timestamp + frame.duration) - sys.stdout.write('\n') + #sys.stdout.write('\n') return(start_times, end_times) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 292e2a98..d42819e2 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False): """ Computes open_face features for the files in filepaths - + Args: ----- filepaths: (itreable[str]) @@ -27,46 +27,54 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): input_dir: Path to the input videos out_dir: Path to the processed output of_path: OpenFace source code path - + Returns: -------- (itreable[str]) list of .csv files - """ - - suffix = '_OF_features' + """ + if video_tracking: + suffix = '_OF_video_features' + else: + suffix = '_OF_features' + csv_files = [] for fp in filepaths: try: - + _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) - - csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) + if video_tracking and not os.path.exists(os.path.abspath(output_directory)): + os.makedirs(os.path.abspath(output_directory)) + csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) + except Exception as e: logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e)) - + return csv_files - -def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): + +def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking): """ Processing all patient's for fetching emotion expressivity ------------------- ------------------- Args: video_uri: video path; input_dir : input directory for video's; dbm_group: feature group - out_dir: (str) Output directory for processed output; of_path: OpenFace source code path - + out_dir: (str) Output directory for processed output; of_path: OpenFace source code path + """ try: - if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group: - return - - filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) + if dbm_group != None: + check_group = ['facial','movement'] #add group here: if you want to use openface output for raw variable calculation + check_val = bool(len({*check_group} & {*dbm_group})) + if not check_val: + return + filepaths = [video_uri] + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking) + except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index fafc36a9..058f4dc1 100644 --- a/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -155,7 +155,7 @@ int main(int argc, char **argv) std::string base_filename = path.substr(path.find_last_of("/\\") + 1); base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,""); results.open(out_dir + '/' + base_filename + "_landmark_output.csv"); -// confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); + confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); int lx = 0; int ly = 0; for(lx = 0; lx < 2; lx++){ @@ -163,7 +163,7 @@ int main(int argc, char **argv) if (lx == 0){ results << "l" << ly << "_x,"; -// confidence << "c" << ly <<","; + confidence << "c" << ly <<","; } if (lx == 1){ results << "l" << ly << "_y,"; @@ -172,7 +172,7 @@ int main(int argc, char **argv) } results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ; results << std::endl; -// confidence << std::endl; + confidence << std::endl; int counter = 0; @@ -207,7 +207,6 @@ int main(int argc, char **argv) // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); - cv::Vec6d pose_estimate(0, 0, 0, 0, 0, 0); // If tracking succeeded and we have an eye model, estimate gaze if (detection_success && face_model.eye_model) @@ -217,9 +216,9 @@ int main(int argc, char **argv) } // Work out the pose of the head from the tracked model - if (detection_success){ - pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - } + + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + @@ -234,11 +233,11 @@ int main(int argc, char **argv) // Displaying the tracking visualizations // std::cout<< "setting observation landmarks"<0 and 'movement' in args.dbm_group: + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group, video_tracking=True) + pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) + pf.remove_file(video_file) def process_raw_video_file(args, s_config, r_config): """ @@ -57,7 +65,7 @@ def process_raw_video_file(args, s_config, r_config): else: logger.info('Enter correct video(*.mp4) file path.') - + except Exception as e: logger.error('Failed to process mp4 file.') pf.remove_file(video_file[0]) @@ -76,15 +84,16 @@ def process_raw_audio_file(args, s_config, r_config): if len(audio_file)>0: logger.info('Calculating raw variables...') - + out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio_file[0], out_path, args.dbm_group, r_config) + pf.process_nlp(audio_file[0], out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) else: logger.info('Enter correct audio(*.wav) file path.') except Exception as e: logger.error('Failed to process wav file.') - + def process_raw_video_dir(args, s_config, r_config): """ Processing video file @@ -95,16 +104,15 @@ def process_raw_video_dir(args, s_config, r_config): """ if args.output_path != None: vid_loc = glob.glob(args.input_path + '/*.mp4') - + if len(vid_loc) == 0: logger.info('Directory does not have any MP4 files.') return - + logger.info('Calculating raw variables...') for vid_file in vid_loc: try: - - common_video(vid_file, args, r_config) + common_video(vid_file, args, r_config) except Exception as e: logger.error('Failed to process mp4 file.') pf.remove_file(vid_file) @@ -119,20 +127,22 @@ def process_raw_audio_dir(args, s_config, r_config): """ if args.output_path != None: audio_loc = glob.glob(args.input_path + '/*.wav') - + if len(audio_loc) == 0: logger.info('Directory does not have any WAV files.') return - + logger.info('Calculating raw variables...') for audio in audio_loc: try: - + out_path = os.path.join(args.output_path, 'raw_variables') pf.process_acoustic(audio, out_path, args.dbm_group, r_config) + pf.process_nlp(audio, out_path, args.dbm_group, args.tr, r_config, DEEP_SPEECH) + except Exception as e: logger.error('Failed to process wav file.') - + def process_derive(args, r_config, d_config, input_type): """ Processing dbm derived variables @@ -141,28 +151,29 @@ def process_derive(args, r_config, d_config, input_type): input_file = glob.glob(args.input_path) else: input_file = glob.glob(args.input_path + '/*') - + out_raw_path = os.path.join(args.output_path, 'raw_variables') out_derive_path = os.path.join(args.output_path, 'derived_variables') - + logger.info('Calculating derived variables...') feature_df = der.run_derive(input_file, out_raw_path, out_derive_path, r_config, d_config) - + if __name__=="__main__": start_time = time.time() parser = argparse.ArgumentParser(description="Process video/audio......") - + parser.add_argument("--input_path", help="path to the input files", required=True) parser.add_argument("--output_path", help="path to the raw and derived variable output", required=True) parser.add_argument("--dbm_group", help="list of feature groups", nargs='+') - + parser.add_argument("--tr", help="speech transcription toogle") + args = parser.parse_args() s_config = config_reader.ConfigReader() r_config = config_raw_feature.ConfigRawReader() d_config = config_derive_feature.ConfigDeriveReader() - + _, file_ext = os.path.splitext(os.path.basename(args.input_path)) - + if file_ext: input_type = 'file' if file_ext.lower() == '.mp4': @@ -170,14 +181,14 @@ if __name__=="__main__": elif file_ext.lower() == '.wav': process_raw_audio_file(args, s_config, r_config) - + else: logger.error('No WAV or MP4 files detected in input path') else: input_type = 'dir' process_raw_video_dir(args, s_config, r_config) process_raw_audio_dir(args, s_config, r_config) - + process_derive(args, r_config, d_config, input_type) exec_time = time.time() - start_time - logger.info('Done! Processing time: {} seconds'.format(exec_time)) \ No newline at end of file + logger.info('Done! Processing time: {} seconds'.format(exec_time)) diff --git a/process_dbm.sh b/process_dbm.sh index cad71b7f..10991adf 100644 --- a/process_dbm.sh +++ b/process_dbm.sh @@ -4,9 +4,10 @@ helpFunction() { echo "" echo "Usage: $0 --input_path parameterA --output_path parameterB --dbm_group parameterC" - echo -e "\t--input_path Description of what is parameterA" - echo -e "\t--output_path Description of what is parameterB" - echo -e "\t-dbm_group Description of what is parameterC" + echo -e "\t--input_path: path to the input files" + echo -e "\t--output_path: path to the raw and derived variable output" + echo -e "\t--dbm_group: list of feature groups" + echo -e "\t--tr: Toggle for speech transcription(optional)" exit 1 # Exit script after printing help } @@ -15,6 +16,7 @@ while [ $# -gt 0 ]; do --input_path=*) input_path="${1#*=}" ;; --output_path=*) output_path="${1#*=}" ;; --dbm_group=*) dbm_group="${1#*=}" ;; + --tr=*) tr="${1#*=}" ;; *) helpFunction ;; esac shift @@ -55,6 +57,12 @@ fi if [[ $dbm_group == *"movement"* ]]; then dbm_new="$dbm_new movement" fi +if [[ $dbm_group == *"speech"* ]]; then + dbm_new="$dbm_new speech" +fi +if [[ $dbm_group == *"speech"* ]] && [[ ${tr} == "on" ]]; then + dbm_new="$dbm_new --tr ${tr}" +fi #docker commands to run container docker create -ti --name dbm_container dbm bash @@ -63,13 +71,13 @@ docker cp $input_path dbm_container:/app/raw_data docker start dbm_container if [ -z "$dbm_new" ] then - docker exec -it dbm_container /bin/bash -c "python3 process_data.py --input_path /app/raw_data --output_path /app/output" + docker exec -it dbm_container /bin/bash -c "python3 -W ignore process_data.py --input_path /app/raw_data --output_path /app/output" else - docker exec -it dbm_container /bin/bash -c "python3 process_data.py --input_path /app/raw_data --output_path /app/output --dbm_group$dbm_new" + docker exec -it dbm_container /bin/bash -c "python3 -W ignore process_data.py --input_path /app/raw_data --output_path /app/output --dbm_group$dbm_new" fi docker cp dbm_container:/app/output $output_path docker stop dbm_container docker rm dbm_container -exit \ No newline at end of file +exit diff --git a/requirements.txt b/requirements.txt index 01f03057..bee0e653 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ sk-video watchtower opencv-python webrtcvad -pysptk imutils dlib==19.13.0 coloredlogs @@ -20,3 +19,8 @@ more_itertools scipy==1.2.0 pyyaml pydub +deepspeech +nltk +lexicalrichness +vaderSentiment +textblob diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index ecff066a..090b652a 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,8 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'NLP_SPEECH', + 'EYE_GAZE', 'MOV_VT', 'MOV_FT'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -22,6 +23,10 @@ derive_feature: MOV_HM_LOC: _headmov MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks + NLP_SPEECH_LOC: _nlp + EYE_GAZE_LOC: _eyegaze + MOV_VT_LOC: _vtremor + MOV_FT_LOC: _fac_tremor #Facial category feature group FAC_ASYM: ['fac_AsymMaskMouth', 'fac_AsymMaskEyebrow', 'fac_AsymMaskEye', 'fac_AsymMaskCom'] @@ -33,7 +38,9 @@ derive_feature: FAC_EXP: ['hap_exp', 'sad_exp', 'sur_exp', 'fea_exp', 'ang_exp', 'dis_exp', 'con_exp', 'happ_occ', 'sad_occ', 'sur_occ', 'fea_occ', 'ang_occ', 'dis_occ', 'con_occ', 'pos_exp', 'neg_exp', 'com_exp','com_lower_exp','com_upper_exp', 'hap_exp_full', 'sad_exp_full', 'sur_exp_full','fea_exp_full', 'ang_exp_full', 'dis_exp_full', 'con_exp_full', 'pos_exp_full', - 'neg_exp_full', 'com_exp_full','com_lower_exp_full','com_upper_exp_full'] + 'neg_exp_full', 'com_exp_full','com_lower_exp_full','com_upper_exp_full', 'pai_exp', 'pai_exp_full'] + + FAC_LMK: ['fac_LMK00disp', 'fac_LMK01disp', 'fac_LMK02disp', 'fac_LMK03disp', 'fac_LMK04disp', 'fac_LMK05disp', 'fac_LMK06disp', 'fac_LMK07disp', 'fac_LMK08disp', 'fac_LMK09disp', 'fac_LMK10disp', 'fac_LMK11disp', 'fac_LMK12disp', 'fac_LMK13disp', 'fac_LMK14disp', 'fac_LMK15disp', 'fac_LMK16disp', @@ -64,6 +71,18 @@ derive_feature: MOV_HM: ['head_vel'] MOV_HP: ['mov_Hpose_Dist','mov_Hpose_Pitch','mov_Hpose_Yaw','mov_Hpose_Roll'] EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] + MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq', + 'mov_amp_trem_index', 'mov_amp_trem_pindex'] + MOV_FT: ['fac_tremor_median_5','fac_tremor_median_12','fac_tremor_median_8','fac_tremor_median_48','fac_tremor_median_54','fac_tremor_median_28','fac_tremor_median_51','fac_tremor_median_66','fac_tremor_median_57'] + + EYE_GAZE: ['mov_leye_x', 'mov_leye_y', 'mov_leye_z', 'mov_reye_x', 'mov_reye_y', 'mov_reye_z', 'mov_eleft_disp', + 'mov_eright_disp'] + + #NLP category feature group + NLP_SPEECH: ['nlp_numSentences', 'nlp_singPronPerAns', 'nlp_singPronPerSen', 'nlp_pastTensePerAns', 'nlp_pastTensePerSen', + 'nlp_pronounsPerAns', 'nlp_pronounsPerSen', 'nlp_verbsPerAns', 'nlp_verbsPerSen', 'nlp_adjectivesPerAns', + 'nlp_adjectivesPerSen', 'nlp_nounsPerAns', 'nlp_nounsPerSen', 'nlp_sentiment_mean', 'nlp_mattr', 'nlp_wordsPerMin', + 'nlp_totalTime'] #Calculation for variables # Facial Asymmetry @@ -254,3 +273,49 @@ derive_feature: mov_blink_ear: ['mean', 'std'] vid_dur: ['count'] mov_blinkdur: ['mean', 'std'] + + mov_freq_trem_freq: ['mean'] + mov_freq_trem_index: ['mean'] + mov_freq_trem_pindex: ['mean'] + mov_amp_trem_freq: ['mean'] + mov_amp_trem_index: ['mean'] + mov_amp_trem_pindex: ['mean'] + + fac_tremor_median_5: ['mean'] + fac_tremor_median_12: ['mean'] + fac_tremor_median_8: ['mean'] + fac_tremor_median_48: ['mean'] + fac_tremor_median_54: ['mean'] + fac_tremor_median_28: ['mean'] + fac_tremor_median_51: ['mean'] + fac_tremor_median_66: ['mean'] + fac_tremor_median_57: ['mean'] + + mov_leye_x: ['mean', 'std'] + mov_leye_y: ['mean', 'std'] + mov_leye_z: ['mean', 'std'] + mov_reye_x: ['mean', 'std'] + mov_reye_y: ['mean', 'std'] + mov_reye_z: ['mean', 'std'] + mov_eleft_disp: ['mean', 'std'] + mov_eright_disp: ['mean', 'std'] + + #NLP feature + nlp_numSentences: ['mean'] + nlp_singPronPerAns: ['mean'] + nlp_singPronPerSen: ['mean'] + nlp_pastTensePerAns: ['mean'] + nlp_pastTensePerSen: ['mean'] + nlp_pronounsPerAns: ['mean'] + nlp_pronounsPerSen: ['mean'] + nlp_verbsPerAns: ['mean'] + nlp_verbsPerSen: ['mean'] + nlp_adjectivesPerAns: ['mean'] + nlp_adjectivesPerSen: ['mean'] + nlp_nounsPerAns: ['mean'] + nlp_nounsPerSen: ['mean'] + nlp_sentiment_mean: ['mean'] + nlp_mattr: ['mean'] + nlp_wordsPerMin: ['mean'] + nlp_totalTime: ['mean'] + diff --git a/resources/features/facial/config.json b/resources/features/facial/config.json new file mode 100644 index 00000000..ebad1b06 --- /dev/null +++ b/resources/features/facial/config.json @@ -0,0 +1 @@ +{"ref_lmk": 28, "ref_area": 350000, "face_width_left": "l15_x", "face_width_right": "l1_x", "face_height_left": "l8_y", "face_height_right": "l27_y", "landmarks": [5, 12, 8, 48, 54, 28, 51, 66, 57], "model_path": "resources/facial/svm_bin_fac_tremor.sav", "feature_order": ["fac_features_mean_5", "fac_features_mean_12", "fac_features_mean_8", "fac_features_mean_48", "fac_features_mean_54", "fac_features_mean_28", "fac_features_mean_51", "fac_features_mean_66", "fac_features_mean_57", "fac_features_median_5", "fac_features_median_12", "fac_features_median_8", "fac_features_median_48", "fac_features_median_54", "fac_features_median_28", "fac_features_median_51", "fac_features_median_66", "fac_features_median_57"]} diff --git a/resources/features/raw_feature.yml b/resources/features/raw_feature.yml index 736393bb..72a0d069 100644 --- a/resources/features/raw_feature.yml +++ b/resources/features/raw_feature.yml @@ -203,3 +203,52 @@ raw_feature: mov_Hpose_Yaw: mov_hposeyaw mov_Hpose_Roll: mov_hposeroll mov_Hpose_Dist: mov_hposedist + + mov_freq_trem_freq: mov_freqtremfreq + mov_freq_trem_index: mov_freqtremindex + mov_freq_trem_pindex: mov_freqtrempindex + mov_amp_trem_freq: mov_amptremfreq + mov_amp_trem_index: mov_amptremindex + mov_amp_trem_pindex: mov_amptrempindex + + fac_tremor_median_5: fac_tremor_median_5 + fac_tremor_median_12: fac_tremor_median_12 + fac_tremor_median_8: fac_tremor_median_8 + fac_tremor_median_48: fac_tremor_median_48 + fac_tremor_median_54: fac_tremor_median_54 + fac_tremor_median_28: fac_tremor_median_28 + fac_tremor_median_51: fac_tremor_median_51 + fac_tremor_median_66: fac_tremor_median_66 + fac_tremor_median_57: fac_tremor_median_57 + + mov_leye_x: mov_lefteyex + mov_leye_y: mov_lefteyey + mov_leye_z: mov_lefteyez + mov_reye_x: mov_righteyex + mov_reye_y: mov_righteyey + mov_reye_z: mov_righteyez + mov_eleft_disp: mov_leyedisp + mov_eright_disp: mov_reyedisp + + #NLP markers + nlp_transcribe: nlp_transcribe + nlp_numSentences: nlp_numSentences + nlp_singPronPerAns: nlp_singPronPerAns + nlp_singPronPerSen: nlp_singPronPerSen + nlp_pastTensePerAns: nlp_pastTensePerAns + nlp_pastTensePerSen: nlp_pastTensePerSen + nlp_pronounsPerAns: nlp_pronounsPerAns + nlp_pronounsPerSen: nlp_pronounsPerSen + nlp_verbsPerAns: nlp_verbsPerAns + nlp_verbsPerSen: nlp_verbsPerSen + nlp_adjectivesPerAns: nlp_adjectivesPerAns + nlp_adjectivesPerSen: nlp_adjectivesPerSen + nlp_nounsPerAns: nlp_nounsPerAns + nlp_nounsPerSen: nlp_nounsPerSen + nlp_sentiment_mean: nlp_sentiment_mean + nlp_mattr: nlp_mattr + nlp_wordsPerMin: nlp_wordsPerMin + nlp_totalTime: nlp_totalTime + + + diff --git a/resources/libraries/voice_tremor.praat b/resources/libraries/voice_tremor.praat new file mode 100644 index 00000000..efeca086 --- /dev/null +++ b/resources/libraries/voice_tremor.praat @@ -0,0 +1,422 @@ + +###################################### +# Global Settings +###################################### +sourcedirec$ = "./"; directory of sounds to be analyzed +minPi = 60; minimal Pitch [Hz] +maxPi = 350; maximal Pitch [Hz] +ts = 0.015; analysis time step [s] +tremthresh = 0.15; minimal autocorr.-coefficient to assume "tremor" +minTr = 1.5; minimal tremor frequency [Hz] +maxTr = 15; maximal tremor frequency [Hz] + + + +###################################### +# Sound (.wav) in, results (.txt) out +###################################### + +# record/load and select the sound to be analyzed!!! + +info$ = Info +name$ = extractWord$(info$, "Object name: ") + +slength = Get total duration +call ftrem +call atrem + +echo +...{"FTrF": 'ftrf:2#', "ATrF":'atrf:2',"FTrI":'ftri:3',"ATrI":'atri:3',"FTrP":'ftrp:3',"ATrP":'atrp:3'} + + + +###################################### +# Frequency Tremor Analysis +###################################### +procedure ftrem + To Pitch (cc)... ts minPi 15 yes 0.03 0.3 0.01 0.35 0.14 maxPi + +#Edit +#pause + +# because PRAAT only runs "Subtract linear fit" if the last frame is "voiceless" (!?): +# numberOfFrames+1 (1) + numberOfFrames = Get number of frames + x1 = Get time from frame number... 1 + am_F0 = Get mean... 0 0 Hertz + + Create Matrix... ftrem_0 0 slength numberOfFrames+1 ts x1 1 1 1 1 1 0 + for i from 1 to numberOfFrames + select Pitch 'name$' + f0 = Get value in frame... i Hertz + select Matrix ftrem_0 +# write zeros to matrix where frames are voiceless + if f0 = undefined + Set value... 1 i 0 + else + Set value... 1 i f0 + endif + endfor + +# remove the linear F0 trend (F0 declination) + To Pitch + Subtract linear fit... Hertz + Rename... ftrem_0_lin + +# undo (1) + Create Matrix... ftrem 0 slength numberOfFrames ts x1 1 1 1 1 1 0 + for i from 1 to numberOfFrames + select Pitch ftrem_0_lin + f0 = Get value in frame... i Hertz + select Matrix ftrem +# write zeros to matrix where frames are voiceless + if f0 = undefined + Set value... 1 i 0 + else + Set value... 1 i f0 + endif + endfor + + To Pitch + +# normalize F0-contour by mean F0 + select Matrix ftrem + Formula... (self-am_F0)/am_F0 + +# since zeros in the Matrix (unvoiced frames) become normalized to -1 but +# unvoiced frames should be zero (if anything) +# write zeros to matrix where frames are voiceless + for i from 1 to numberOfFrames + select Pitch ftrem + f0 = Get value in frame... i Hertz + if f0 = undefined + select Matrix ftrem + Set value... 1 i 0 + endif + endfor + +# to calculate autocorrelation (cc-method): + select Matrix ftrem + To Sound (slice)... 1 +# calculate Frequency of Frequency Tremor [Hz] + To Pitch (cc)... slength minTr 15 yes 0.01 tremthresh 0.01 0.35 0.14 maxTr + Rename... ftrem_norm + + ftrf = Get mean... 0 0 Hertz + +# calculate Intensity Index of Frequency Tremor [%] + select Sound ftrem + plus Pitch ftrem_norm + To PointProcess (peaks)... yes no + Rename... Maxima + numberofMaxPoints = Get number of points + ftri_max = 0 + noFMax = 0 + for iPoint from 1 to numberofMaxPoints + select PointProcess Maxima + ti = Get time from index... iPoint + select Sound ftrem + ftri_Point = Get value at time... Average ti Sinc70 + if ftri_Point = undefined + ftri_Point = 0 + noFMax += 1 + endif + ftri_max += abs(ftri_Point) + endfor + +select Sound ftrem +plus PointProcess Maxima +#Edit +#pause + +# ftri_max:= (mean) procentual deviation of F0-maxima from mean F0 at ftrf + numberofMaxima = numberofMaxPoints - noFMax + ftri_max = 100 * ftri_max/numberofMaxima + + select Sound ftrem + plus Pitch ftrem_norm + To PointProcess (peaks)... no yes + Rename... Minima + numberofMinPoints = Get number of points + ftri_min = 0 + noFMin = 0 + for iPoint from 1 to numberofMinPoints + select PointProcess Minima + ti = Get time from index... iPoint + select Sound ftrem + ftri_Point = Get value at time... Average ti Sinc70 + if ftri_Point = undefined + ftri_Point = 0 + noFMin += 1 + endif + ftri_min += abs(ftri_Point) + endfor + +select Sound ftrem +plus PointProcess Minima +#Edit +#pause + + +# ftri_min:= (mean) procentual deviation of F0-minima from mean F0 at ftrf + numberofMinima = numberofMinPoints - noFMin + ftri_min = 100 * ftri_min/numberofMinima + + ftri = (ftri_max + ftri_min) / 2 + + ftrp = ftri * ftrf/(ftrf+1) + +# uncomment to inspect frequnecy tremor objects: +# pause + + select Pitch ftrem +# uncomment if only frequency tremor is to be analyzed: +# plus Pitch 'name$' + plus Matrix ftrem_0 + plus Pitch ftrem_0 + plus Pitch ftrem_0_lin + plus Matrix ftrem + plus Sound ftrem + plus Pitch ftrem_norm + plus PointProcess Maxima + plus PointProcess Minima + Remove + +endproc + + +###################################### +# Amplitude Tremor Analysis +###################################### +procedure atrem + select Sound 'name$' +# uncomment if only amplitude tremor is to be analyzed: +# To Pitch (cc)... ts minPi 15 yes 0.03 0.3 0.01 0.35 0.14 maxPi +# select Sound 'name$' + plus Pitch 'name$' + To PointProcess (cc) + select Sound 'name$' + plus PointProcess 'name$'_'name$' + +# amplitudes are integrals of intensity over periods -- not intensity maxima + To AmplitudeTier (period)... 0 0 0.0001 0.02 1.7 + +#Edit +#pause + +# from here on out: prepare to autocorrelate AmplitudeTier-data +# sample AmplitudeTier at (constant) rate ts + numbOfAmpPoints = Get number of points + first_ampP = Get time from index... 1 + last_ampP = Get time from index... numbOfAmpPoints + +# to be able to -- automatically -- read Amp. values... + Down to TableOfReal + + select Pitch 'name$' + frameNo1 = Get frame number from time... first_ampP + hiframe1 = ceiling(frameNo1) + t_hiframe1 = Get time from frame number... hiframe1 + + frameNoN = Get frame number from time... last_ampP + loframeN = floor(frameNoN) + +# number of Amp. points if (re-)sampled at ts + numbOfPoints_neu = loframeN - hiframe1 + 1 + +# to enable autocorrelation of the Amp.-contour: ->Matrix->Sound + + Create Matrix... atrem_nlc 0 slength numbOfPoints_neu+1 ts t_hiframe1 1 1 1 1 1 2 +# get the mean of the amplitude contour in time windows of constant duration + for point_neu from 1 to numbOfPoints_neu + t = (point_neu-1) * ts + t_hiframe1 + tl = t - ts/2 + tu = t + ts/2 + + select AmplitudeTier 'name$'_'name$'_'name$' + loil = Get low index from time... tl + hiil = Get high index from time... tl + loiu = Get low index from time... tu + hiiu = Get high index from time... tu + + select TableOfReal 'name$'_'name$'_'name$' + if loil = 0 + lotl = 0; time before the first amp. point + druck_lol = Get value... hiil 2; amplitude value before the first amp. point + else + lotl = Get value... loil 1; time value of Amp.Point before tl in the PointProcess [s] + druck_lol = Get value... loil 2; amplitude value before tl in the PointProcess [Pa, ranged from 0 to 1] + endif + + hitl = Get value... hiil 1 + druck_hil = Get value... hiil 2; amplitude value after tl in the PointProcess + + lotu = Get value... loiu 1 + druck_lou = Get value... loiu 2; amplitude value before tu in the PointProcess + + if hiiu = numbOfAmpPoints + 1 + hitu = slength; time after the last amp. point + druck_hiu = Get value... hiil 2; amplitude value after the last amp. point + else + hitu = Get value... hiiu 1; time value after tu in the PointProcess + druck_hiu = Get value... hiiu 2; amplitude value after tu in the PointProcess + endif + + nPinter = loiu - loil; = hiiu - hiil; number of amp.-points between tl and tu + if nPinter > 0 + itinter = 0 + tinter = 0 + druck_tin = 0 + deltat = 0 + for iinter from 1 to nPinter + hilft = itinter + itinter = Get value... loil+iinter 1 + idruck_tin = Get value... loil+iinter 2 + + ideltat = itinter - hilft + druck_tin += idruck_tin * ideltat + tinter += itinter + deltat += ideltat + endfor + + tin = tinter/nPinter + druck_tin = druck_tin/deltat + endif + + druck_tl = ((hitl-tl)*druck_lol + (tl-lotl)*druck_hil) / (hitl-lotl) + druck_tu = ((hitu-tu)*druck_lou + (tu-lotu)*druck_hiu) / (hitu-lotu) + + if nPinter = 0; loil = loiu; hiil = hiiu + druck_mean = (druck_tl + druck_tu) / 2 + else + druck_mean = ((tin-tl)*(druck_tl + druck_tin)/2 + (tu-tin)*(druck_tin + druck_tu)/2) / (tu-tl) + endif + + select Matrix atrem_nlc + Set value... 1 point_neu druck_mean + endfor + + To Pitch + am_Int = Get mean... 0 0 Hertz + +# because PRAAT classifies frequencies in Pitch objects <=0 as "voiceless" and +# therefore parts with extreme INTENSITIES would be considered as "voiceless" +# (irrelevant) after "Subtract linear fit" (1) +# "1" is added to the original Pa-values (ranged from 0 to 1) + select Matrix atrem_nlc + Formula... self+1 + +# because PRAAT only runs "Subtract linear fit" if the last frame is "voiceless"...?(2) + Set value... 1 numbOfPoints_neu+1 0 + +# remove the linear amp.-trend (amplitude declination) +#Formula... self*1000; better for viewing + To Pitch + Rename... hilf_lincorr + Subtract linear fit... Hertz + Rename... atrem + +# undo (1)... + To Matrix + Formula... self-1 + +# normalize Amp. contour by mean Amp. + Formula... (self-am_Int)/am_Int + +# remove last frame, undo (2) + Create Matrix... atrem_besser 0 slength numbOfPoints_neu ts t_hiframe1 1 1 1 1 1 0 + for point_neu from 1 to numbOfPoints_neu + select Matrix atrem + spring = Get value in cell... 1 point_neu + select Matrix atrem_besser + Set value... 1 point_neu spring + endfor + +# to calculate autocorrelation (cc-method) + To Sound (slice)... 1 +# calculate Frequency of Ampitude Tremor [Hz] + To Pitch (cc)... slength minTr 15 yes 0.01 tremthresh 0.01 0.35 0.14 maxTr + Rename... atrem_norm + + atrf = Get mean... 0 0 Hertz + +# calculate Intensity Index of Amplitude Tremor [%] + select Sound atrem_besser + plus Pitch atrem_norm + To PointProcess (peaks)... yes no + Rename... Maxima + numberofMaxPoints = Get number of points + atri_max = 0 + noAMax = 0 + for iPoint from 1 to numberofMaxPoints + select PointProcess Maxima + ti = Get time from index... iPoint + select Sound atrem_besser + atri_Point = Get value at time... 0 ti Sinc70 + if atri_Point = undefined + atri_Point = 0 + noAMax += 1 + endif + atri_max += abs(atri_Point) + endfor + +select Sound atrem_besser +plus PointProcess Maxima +#Edit +#pause + +# atri_max:= (mean) procentual deviation of Amp. maxima from mean Amp.[Pa] at atrf + numberofMaxima = numberofMaxPoints - noAMax + atri_max = 100 * atri_max / numberofMaxima + + select Sound atrem_besser + plus Pitch atrem_norm + To PointProcess (peaks)... no yes + Rename... Minima + numberofMinPoints = Get number of points + atri_min = 0 + noAMin = 0 + for iPoint from 1 to numberofMinPoints + select PointProcess Minima + ti = Get time from index... iPoint + select Sound atrem_besser + atri_Point = Get value at time... 0 ti Sinc70 + if atri_Point = undefined + atri_Point = 0 + noAMin += 1 + endif + atri_min += abs(atri_Point) + endfor + +select Sound atrem_besser +plus PointProcess Minima +#Edit +#pause + +# atri_min:= (mean) procentual deviation of Amp. minima from mean Amp.[Pa] at atrf + numberofMinima = numberofMinPoints - noAMin + atri_min = 100 * atri_min / numberofMinima + + atri = (atri_max + atri_min) / 2 + + atrp = atri * atrf/(atrf+1) + +# uncomment to inspect amplitude tremor objects: +# pause + + select Pitch 'name$' + plus PointProcess 'name$'_'name$' + plus AmplitudeTier 'name$'_'name$'_'name$' + plus TableOfReal 'name$'_'name$'_'name$' + plus Matrix atrem_nlc + plus Pitch atrem_nlc + plus Pitch hilf_lincorr + plus Pitch atrem + plus Matrix atrem + plus Matrix atrem_besser + plus Sound atrem_besser + plus Pitch atrem_norm + plus PointProcess Maxima + plus PointProcess Minima + Remove +endproc \ No newline at end of file