From 517e54bf816ca9b445cc29e7aba4ba1e2f347022 Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Mon, 30 Nov 2020 09:43:09 -0500 Subject: [PATCH 1/6] facial tremor testing --- dbm_lib/controller/process_feature.py | 6 ++- .../raw_features/movement/__init__.py | 1 + .../raw_features/video/open_face_process.py | 37 ++++++++++--------- process_data.py | 3 ++ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index f5112da2..488b835c 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor, facial_tremor import subprocess import logging @@ -120,6 +120,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): return logger.info('Processing movement variables from data in {}'.format(video_uri)) + logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) @@ -129,6 +130,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing voice tremor....') voice_tremor.run_vtremor(video_uri, out_dir, r_config) + logger.info('processing facial tremor....') + face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + def remove_file(file_path): """ removing wav file diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index dca2f624..03fd8909 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,3 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 292e2a98..61828b08 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): """ Computes open_face features for the files in filepaths - + Args: ----- filepaths: (itreable[str]) @@ -27,46 +27,49 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): input_dir: Path to the input videos out_dir: Path to the processed output of_path: OpenFace source code path - + Returns: -------- (itreable[str]) list of .csv files """ - - suffix = '_OF_features' + if video_tracking: + suffix = '_OF_video_features/' + else: + suffix = '_OF_features' + csv_files = [] - + for fp in filepaths: try: - + _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) - + except Exception as e: logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e)) - + return csv_files - -def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): + +def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking): """ Processing all patient's for fetching emotion expressivity ------------------- ------------------- Args: video_uri: video path; input_dir : input directory for video's; dbm_group: feature group - out_dir: (str) Output directory for processed output; of_path: OpenFace source code path - + out_dir: (str) Output directory for processed output; of_path: OpenFace source code path + """ try: - + if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group: return filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) - + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) + except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/process_data.py b/process_data.py index 31f85fdb..07d8f4af 100644 --- a/process_data.py +++ b/process_data.py @@ -19,6 +19,8 @@ import time logging.basicConfig(level=logging.INFO) logger=logging.getLogger() +#for ftremor +OPENFACE_PATH_VIDEO = '/pkg/OpenFace/build/bin/FaceLandmarkVid' OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction' DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat' @@ -35,6 +37,7 @@ def common_video(video_file, args, r_config): of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group) pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group,video_tracking=True) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) From 0285dd51ffb29891ba8b629d104509f63eec54ed Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Mon, 30 Nov 2020 09:46:48 -0500 Subject: [PATCH 2/6] facial tremor testing --- dbm_lib/controller/process_feature.py | 9 +- .../raw_features/movement/__init__.py | 1 + .../raw_features/movement/facial_tremor.py | 159 ++++++++++++++++++ .../raw_features/video/open_face_process.py | 37 ++-- process_data.py | 3 + resources/features/facial/config.json | 1 + 6 files changed, 189 insertions(+), 21 deletions(-) create mode 100644 dbm_lib/dbm_features/raw_features/movement/facial_tremor.py create mode 100644 resources/features/facial/config.json diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index f5112da2..00463600 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -7,7 +7,7 @@ created: 2020-20-07 from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark -from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor +from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink, voice_tremor, facial_tremor import subprocess import logging @@ -82,9 +82,6 @@ def process_acoustic(video_uri, out_dir, dbm_group, r_config): logger.info('processing mfcc....') mfcc.run_mfcc(video_uri, out_dir, r_config) - logger.info('processing voice tremor....') - voice_tremor.run_vtremor(video_uri, out_dir, r_config) - def process_facial(video_uri, out_dir, dbm_group, r_config): """ processing facial features @@ -120,6 +117,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): return logger.info('Processing movement variables from data in {}'.format(video_uri)) + logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) @@ -129,6 +127,9 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing voice tremor....') voice_tremor.run_vtremor(video_uri, out_dir, r_config) + logger.info('processing facial tremor....') + face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + def remove_file(file_path): """ removing wav file diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index dca2f624..03fd8909 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,3 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py new file mode 100644 index 00000000..7df1b861 --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -0,0 +1,159 @@ +import sys, os, glob, cv2, re +import pickle, json +import pandas as pd +import numpy as np +import numpy.ma as ma +import logging +from os.path import join + +from dbm_lib.dbm_features.raw_features.util import util as ut +from dbm_lib.dbm_features.raw_features.util.math_util import * + +from dbm_lib.dbm_features.raw_features.movement import DBMLIB_FTREMOR_CONFIG + +logging.basicConfig(level=logging.INFO) +logger=logging.getLogger() + +ft_dir = 'movement/facial_tremor' +csv_ext = '_fac_tremor.csv' +model_ext = '_fac_model.csv' +fac_features_ext = '_fac_features.csv' + +def compute_features(out_dir, df_of, r_config): + """ Computes features + + Returns: features in vector format + """ + config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) + logger.info('json file read') + pattern_x = re.compile("l\d+_x") + pattern_y = re.compile("l\d+_y") + + # assumption: distance of face to camera remains at roughly static + + # logic break + landmark_columns = [] + for col in df_of.columns: + if pattern_x.match(col) or pattern_y.match(col): + landmark_columns.append(col) + + df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] + df_of.reset_index(inplace=True) + + num_frames = len(df) + logger.info("Number of frames to be processed: {}".format(str(num_frames))) + landmarks = config['landmarks'] + + try: + if num_frames == 0: + error_reason = "No frames with visible face." + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + +# if num_frames < 60: +# error_reason = 'Number of frames with visible face < 60. Video too short' +# logger.error(error_reason) +# return empty_frame(landmarks, f_cfg, error_reason) + + first_row = df_of.iloc[0] + + facew = abs(first_row[config['face_width_left']] - first_row[config['face_width_right']]) + faceh = abs(first_row[config['face_height_left']] - first_row[config['face_height_right']]) + + if facew == 0 or faceh == 0: + error_reason = 'face width or height = 0. Check landmark values' + logger.error(error_reason) + return empty_frame(landmarks, r_config) + + fac_disp = calc_displacement_vec(df_of, landmarks, num_frames) + + # if verbose: + # logger.info("Displacement output: {}".format(str(fac_disp))) + + fac_disp_median = np.median(fac_disp, axis = 1) + fac_disp_mean = np.mean(fac_disp, axis = 1) + + if len(fac_disp.shape)!=2: + error_reason = 'fac_disp is not 2D. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + if len(fac_disp[0])<=1: + error_reason = 'Video too short. smth went wrong with disp calc' + logger.error(error_reason) + return empty_frame(landmarks, r_config, error_reason) + + fac_corr_mat = np.corrcoef(fac_disp, rowvar = True) + # extract relevant row from cov matrix + ref_lmk_index = [i for i, lmk in enumerate(landmarks) if config['ref_lmk']==lmk] + fac_corr = fac_corr_mat[ref_lmk_index][0] + + fac_area = config['ref_area'] / (facew * faceh) + + # if verbose: + # logger.info("Face area: {}".format(fac_area)) + # logger.info("Face Displacement Median: {}".format(str(fac_disp_median))) + # logger.info("Face Displacement Mean: {}".format(str(fac_disp_mean))) + + fac_features1 = np.multiply(fac_area * fac_disp_median, (1. - fac_corr)) + fac_features2 = np.multiply(fac_area * fac_disp_mean, (1. - fac_corr)) + +# base_fac_features = np.dot(fac_area * fac_disp_median, (1. - fac_corr)) + + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] + + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] + fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] + + fac_features_dict[r_config.err_reason] = [''] + data = pd.DataFrame.from_dict(fac_features_dict) + logger.info('Concluded computing tremor features') + + return data + + except Exception as e: + logger.error('Error computing tremor features: {}'.format(str(e))) + return empty_frame(landmarks, r_config, str(e)) + +def empty_frame(landmarks, r_config, error_reason): + fac_features_dict = {} + for i, landmark in enumerate(landmarks): + raw_variable_map = 'fac_tremor_median_{}'.format(landmark) + fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] + + fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] + fac_features_dict['fac_corr_{}'.format(landmark)] = [np.nan] + + fac_features_dict[r_config.err_reason] = [error_reason] + empty_frame = pd.DataFrame.from_dict(fac_features_dict) + return empty_frame + +def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): + """ + processing input videos + """ + try: + logger.info('filtering path: ',video_uri,out_dir) + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) + + if len(of_csv_path)>0: + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) + + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + + feats = compute_features(of_csv_path , df_of, r_config) + if model_output: + result = score(feats, r_config) + feats = pd.concat([feats, result], axis=1) + + ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) + + except Exception as e: + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 292e2a98..61828b08 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,9 +15,9 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): """ Computes open_face features for the files in filepaths - + Args: ----- filepaths: (itreable[str]) @@ -27,46 +27,49 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path): input_dir: Path to the input videos out_dir: Path to the processed output of_path: OpenFace source code path - + Returns: -------- (itreable[str]) list of .csv files """ - - suffix = '_OF_features' + if video_tracking: + suffix = '_OF_video_features/' + else: + suffix = '_OF_features' + csv_files = [] - + for fp in filepaths: try: - + _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) - + except Exception as e: logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e)) - + return csv_files - -def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group): + +def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tracking): """ Processing all patient's for fetching emotion expressivity ------------------- ------------------- Args: video_uri: video path; input_dir : input directory for video's; dbm_group: feature group - out_dir: (str) Output directory for processed output; of_path: OpenFace source code path - + out_dir: (str) Output directory for processed output; of_path: OpenFace source code path + """ try: - + if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group: return filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path) - + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) + except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/process_data.py b/process_data.py index 31f85fdb..07d8f4af 100644 --- a/process_data.py +++ b/process_data.py @@ -19,6 +19,8 @@ import time logging.basicConfig(level=logging.INFO) logger=logging.getLogger() +#for ftremor +OPENFACE_PATH_VIDEO = '/pkg/OpenFace/build/bin/FaceLandmarkVid' OPENFACE_PATH = 'pkg/OpenFace/build/bin/FeatureExtraction' DLIB_SHAPE_MODEL = 'pkg/shape_detector/shape_predictor_68_face_landmarks.dat' @@ -35,6 +37,7 @@ def common_video(video_file, args, r_config): of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH, args.dbm_group) pf.process_facial(video_file, out_path, args.dbm_group, r_config) pf.process_acoustic(video_file, out_path, args.dbm_group, r_config) + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group,video_tracking=True) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) diff --git a/resources/features/facial/config.json b/resources/features/facial/config.json new file mode 100644 index 00000000..ebad1b06 --- /dev/null +++ b/resources/features/facial/config.json @@ -0,0 +1 @@ +{"ref_lmk": 28, "ref_area": 350000, "face_width_left": "l15_x", "face_width_right": "l1_x", "face_height_left": "l8_y", "face_height_right": "l27_y", "landmarks": [5, 12, 8, 48, 54, 28, 51, 66, 57], "model_path": "resources/facial/svm_bin_fac_tremor.sav", "feature_order": ["fac_features_mean_5", "fac_features_mean_12", "fac_features_mean_8", "fac_features_mean_48", "fac_features_mean_54", "fac_features_mean_28", "fac_features_mean_51", "fac_features_mean_66", "fac_features_mean_57", "fac_features_median_5", "fac_features_median_12", "fac_features_median_8", "fac_features_median_48", "fac_features_median_54", "fac_features_median_28", "fac_features_median_51", "fac_features_median_66", "fac_features_median_57"]} From 4cacbb8b37b1ad594952803daebca53fb0137803 Mon Sep 17 00:00:00 2001 From: Vidya Koesmahargyo Date: Mon, 30 Nov 2020 09:49:25 -0500 Subject: [PATCH 3/6] added math_util --- .../raw_features/util/math_util.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 dbm_lib/dbm_features/raw_features/util/math_util.py diff --git a/dbm_lib/dbm_features/raw_features/util/math_util.py b/dbm_lib/dbm_features/raw_features/util/math_util.py new file mode 100644 index 00000000..283acadc --- /dev/null +++ b/dbm_lib/dbm_features/raw_features/util/math_util.py @@ -0,0 +1,57 @@ +""" +file_name: facial_tremor +project_name: cdx_analysis +created: 2019-03-16 +author: Deshana Desai +""" +import sys, os, glob, cv2 +import pandas as pd +import numpy as np + + +def euclidean_distance(point1, point2): + """ + Compute euclidean distance between points + """ + + return np.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2) + + +# def detect_peaks() + + +def expand_landmarks(landmarks): + """ + util method to expand landmark list: + eg: [1,2] -> [['l1_x', 'l1_y'], ['l2_x', 'l2_y']] + """ + return [['l{}_x'.format(l), 'l{}_y'.format(l)] for l in landmarks] + + + +def calc_displacement_vec(df, landmarks, num_frames): + """ + Calculates displacement vector frame by frame + """ + + landmarks = expand_landmarks(landmarks) + + disp_vec = np.zeros((len(landmarks), num_frames)) + prev_point = np.zeros((len(landmarks), 2)) + + # initialize + for j, pair in enumerate(landmarks): + first_row = df.iloc[0] + prev_point[j] = (first_row[pair[0]], first_row[pair[1]]) + + + for i in range(num_frames): + frame_row = df.iloc[i] + for j, pair in enumerate(landmarks): + x, y = pair[0], pair[1] + current = (frame_row[x], frame_row[y]) + deviation = euclidean_distance( current, prev_point[j]) + disp_vec[j][i] = deviation + prev_point[j] = current + + return disp_vec From 9a751ca4341d19811f164f3eedbcdb72ee893980 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 1 Dec 2020 19:54:35 +0000 Subject: [PATCH 4/6] added facial tremor variable --- dbm_lib/controller/process_feature.py | 4 +- .../raw_features/movement/__init__.py | 2 +- .../raw_features/movement/facial_tremor.py | 43 +++++++++++-------- .../raw_features/movement/head_motion.py | 5 ++- .../raw_features/movement/voice_tremor.py | 6 +-- .../raw_features/video/open_face_process.py | 13 +++--- .../exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 42 +++++++++--------- process_data.py | 9 ++-- resources/features/derived_feature.yml | 13 +++++- 9 files changed, 78 insertions(+), 59 deletions(-) diff --git a/dbm_lib/controller/process_feature.py b/dbm_lib/controller/process_feature.py index 00463600..0a9d3dca 100644 --- a/dbm_lib/controller/process_feature.py +++ b/dbm_lib/controller/process_feature.py @@ -120,7 +120,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): logger.info('processing head movement....') head_motion.run_head_movement(video_uri, out_dir, r_config) - + logger.info('processing eye blink....') eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model) @@ -128,7 +128,7 @@ def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model): voice_tremor.run_vtremor(video_uri, out_dir, r_config) logger.info('processing facial tremor....') - face_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) + facial_tremor.fac_tremor_process(video_uri, out_dir, r_config, model_output=True) def remove_file(file_path): """ diff --git a/dbm_lib/dbm_features/raw_features/movement/__init__.py b/dbm_lib/dbm_features/raw_features/movement/__init__.py index 03fd8909..cafb6e11 100644 --- a/dbm_lib/dbm_features/raw_features/movement/__init__.py +++ b/dbm_lib/dbm_features/raw_features/movement/__init__.py @@ -13,4 +13,4 @@ import os DBMLIB_PATH = os.path.dirname(__file__) DBMLIB_VTREMOR_LIB = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/libraries/voice_tremor.praat')) -DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/facial/config.json')) +DBMLIB_FTREMOR_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/features/facial/config.json')) diff --git a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py index 7df1b861..7753eb0b 100644 --- a/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py +++ b/dbm_lib/dbm_features/raw_features/movement/facial_tremor.py @@ -25,7 +25,7 @@ def compute_features(out_dir, df_of, r_config): Returns: features in vector format """ config = json.loads(open(DBMLIB_FTREMOR_CONFIG,'r').read()) - logger.info('json file read') + pattern_x = re.compile("l\d+_x") pattern_y = re.compile("l\d+_y") @@ -40,7 +40,7 @@ def compute_features(out_dir, df_of, r_config): df_of= df_of[(df_of[landmark_columns]!= 0).any(axis=1)] df_of.reset_index(inplace=True) - num_frames = len(df) + num_frames = len(df_of) logger.info("Number of frames to be processed: {}".format(str(num_frames))) landmarks = config['landmarks'] @@ -104,7 +104,7 @@ def compute_features(out_dir, df_of, r_config): for i, landmark in enumerate(landmarks): fac_features_dict['fac_features_mean_{}'.format(landmark)] = [fac_features2[i]] raw_variable_map = 'fac_tremor_median_{}'.format(landmark) - fac_features_dict[r_config.raw_feature[raw_variable_map]] = [fac_features1[i]] + fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [fac_features1[i]] fac_features_dict['fac_disp_median_{}'.format(landmark)] = [fac_disp_median[i]] fac_features_dict['fac_corr_{}'.format(landmark)] = [fac_corr[i]] @@ -123,7 +123,7 @@ def empty_frame(landmarks, r_config, error_reason): fac_features_dict = {} for i, landmark in enumerate(landmarks): raw_variable_map = 'fac_tremor_median_{}'.format(landmark) - fac_features_dict[r_config.raw_feature[raw_variable_map]] = [np.nan] + fac_features_dict[r_config.base_raw['raw_feature'][raw_variable_map]] = [np.nan] fac_features_dict['fac_features_mean_{}'.format(landmark)] = [np.nan] fac_features_dict['fac_disp_median_{}'.format(landmark)] = [np.nan] @@ -133,27 +133,32 @@ def empty_frame(landmarks, r_config, error_reason): empty_frame = pd.DataFrame.from_dict(fac_features_dict) return empty_frame -def fac_tremor_process(video_uri,out_dir,r_config, model_output=False): +def fac_tremor_process(video_uri, out_dir, r_config, model_output=False): """ processing input videos + + """ - try: - logger.info('filtering path: ',video_uri,out_dir) - input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) - of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) +# try: + + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) + of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_video_features/*.csv')) - if len(of_csv_path)>0: - of_csv = of_csv_path[0] - df_of = pd.read_csv(of_csv, error_bad_lines=False) + if len(of_csv_path)>0: + of_csv = of_csv_path[0] + df_of = pd.read_csv(of_csv, error_bad_lines=False) - logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) - feats = compute_features(of_csv_path , df_of, r_config) - if model_output: - result = score(feats, r_config) - feats = pd.concat([feats, result], axis=1) + feats = compute_features(of_csv_path , df_of, r_config) + +# if model_output: +# result = score(feats, r_config) +# feats = pd.concat([feats, result], axis=1) + + ut.save_output(feats, out_loc, fl_name, ft_dir, csv_ext) - ut.output_audio_feature(feats, new_out_base_dir, '/'+fac_dir, fac_ext) - except Exception as e: + +# except Exception as e: logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/movement/head_motion.py b/dbm_lib/dbm_features/raw_features/movement/head_motion.py index c45bc123..59179c2e 100644 --- a/dbm_lib/dbm_features/raw_features/movement/head_motion.py +++ b/dbm_lib/dbm_features/raw_features/movement/head_motion.py @@ -179,11 +179,12 @@ def run_head_movement(video_uri, out_dir, r_config): out_dir: (str) Output directory for processed output; r_config: raw variable config object """ try: - + #filtering path to generate input & output path input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) + if len(of_csv_path)>0: of_csv = of_csv_path[0] @@ -192,4 +193,4 @@ def run_head_movement(video_uri, out_dir, r_config): logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config) except Exception as e: - logger.error('Failed to process video file') \ No newline at end of file + logger.error('Failed to process video file') diff --git a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py index 52f9c398..18f664b6 100644 --- a/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py +++ b/dbm_lib/dbm_features/raw_features/movement/voice_tremor.py @@ -47,7 +47,7 @@ def prepare_vtrem_output(audio_file, out_loc, r_config, fl_name): df_tremor = tremor_praat(audio_file, r_config) df_tremor[r_config.err_reason] = 'Pass'# will replace with threshold in future release - logger.info('Processing Output file {} '.format(out_loc)) + logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name))) ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): @@ -61,7 +61,7 @@ def prepare_empty_vt(out_loc, fl_name, r_config, error_txt): out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] df_tremor = pd.DataFrame(out_val, columns = cols) - logger.info('Saving Output file {} '.format(out_loc)) + logger.info('Saving Output file {} '.format(os.path.join(out_loc, fl_name))) ut.save_output(df_tremor, out_loc, fl_name, vt_dir, csv_ext) def run_vtremor(video_uri, out_dir, r_config): @@ -74,9 +74,9 @@ def run_vtremor(video_uri, out_dir, r_config): out_dir: (str) Output directory for processed output """ try: + input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) - if len(aud_filter)>0: audio_file = aud_filter[0] diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 61828b08..51364521 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -15,7 +15,7 @@ from dbm_lib.dbm_features.raw_features.util import util as ut logging.basicConfig(level=logging.INFO) logger=logging.getLogger() -def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_tracking=False): +def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_tracking=False): """ Computes open_face features for the files in filepaths Args: @@ -31,21 +31,22 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path,video_track Returns: -------- (itreable[str]) list of .csv files - """ + """ if video_tracking: - suffix = '_OF_video_features/' + suffix = '_OF_video_features' else: suffix = '_OF_features' csv_files = [] - + for fp in filepaths: try: _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) - + if not os.path.isdir(output_directory): + os.mkdir(output_directory) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) except Exception as e: @@ -69,7 +70,7 @@ def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group,video_tr return filepaths = [video_uri] - csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path,video_tracking) + csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path, video_tracking) except Exception as e: logger.error('Failed to process video file') diff --git a/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index fafc36a9..058f4dc1 100644 --- a/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -155,7 +155,7 @@ int main(int argc, char **argv) std::string base_filename = path.substr(path.find_last_of("/\\") + 1); base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,""); results.open(out_dir + '/' + base_filename + "_landmark_output.csv"); -// confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); + confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv"); int lx = 0; int ly = 0; for(lx = 0; lx < 2; lx++){ @@ -163,7 +163,7 @@ int main(int argc, char **argv) if (lx == 0){ results << "l" << ly << "_x,"; -// confidence << "c" << ly <<","; + confidence << "c" << ly <<","; } if (lx == 1){ results << "l" << ly << "_y,"; @@ -172,7 +172,7 @@ int main(int argc, char **argv) } results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ; results << std::endl; -// confidence << std::endl; + confidence << std::endl; int counter = 0; @@ -207,7 +207,6 @@ int main(int argc, char **argv) // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); - cv::Vec6d pose_estimate(0, 0, 0, 0, 0, 0); // If tracking succeeded and we have an eye model, estimate gaze if (detection_success && face_model.eye_model) @@ -217,9 +216,9 @@ int main(int argc, char **argv) } // Work out the pose of the head from the tracked model - if (detection_success){ - pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); - } + + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + @@ -234,11 +233,11 @@ int main(int argc, char **argv) // Displaying the tracking visualizations // std::cout<< "setting observation landmarks"<0 and 'movement' in args.dbm_group: + of.process_open_face(video_file, os.path.dirname(video_file), out_path, OPENFACE_PATH_VIDEO, args.dbm_group, video_tracking=True) pf.process_movement(video_file, out_path, args.dbm_group, r_config, DLIB_SHAPE_MODEL) pf.remove_file(video_file) diff --git a/resources/features/derived_feature.yml b/resources/features/derived_feature.yml index 43db29dc..fc2f3926 100644 --- a/resources/features/derived_feature.yml +++ b/resources/features/derived_feature.yml @@ -2,7 +2,7 @@ derive_feature: #DBM Feature Group FEATURE_GROUP: ['FAC_ASYM', 'FAC_AU', 'FAC_EXP', 'FAC_LMK', 'ACO_INT', 'ACO_FF', 'ACO_HNR', 'ACO_GNE', 'ACO_FM', - 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT'] + 'ACO_JITTER','ACO_SHIMMER', 'ACO_PAUSE', 'ACO_VFS', 'ACO_MFCC', 'MOV_HM', 'MOV_HP', 'EYE_BLINK', 'MOV_VT', 'MOV_FT'] #Feature group output file extensions FAC_ASYM_LOC: _facasym @@ -23,6 +23,7 @@ derive_feature: MOV_HP_LOC: _headpose EYE_BLINK_LOC: _eyeblinks MOV_VT_LOC: _vtremor + MOV_FT_LOC: _fac_tremor #Facial category feature group @@ -68,6 +69,7 @@ derive_feature: EYE_BLINK: ['mov_blink_ear', 'vid_dur', 'mov_blinkdur'] MOV_VT: ['mov_freq_trem_freq', 'mov_freq_trem_index', 'mov_freq_trem_pindex', 'mov_amp_trem_freq', 'mov_amp_trem_index', 'mov_amp_trem_pindex'] + MOV_FT: ['fac_tremor_median_5','fac_tremor_median_12','fac_tremor_median_8','fac_tremor_median_48','fac_tremor_median_54','fac_tremor_median_28','fac_tremor_median_51','fac_tremor_median_66','fac_tremor_median_57'] #Calculation for variables # Facial Asymmetry @@ -258,3 +260,12 @@ derive_feature: mov_amp_trem_freq: ['mean'] mov_amp_trem_index: ['mean'] mov_amp_trem_pindex: ['mean'] + fac_tremor_median_5: ['mean'] + fac_tremor_median_12: ['mean'] + fac_tremor_median_8: ['mean'] + fac_tremor_median_48: ['mean'] + fac_tremor_median_54: ['mean'] + fac_tremor_median_28: ['mean'] + fac_tremor_median_51: ['mean'] + fac_tremor_median_66: ['mean'] + fac_tremor_median_57: ['mean'] From 2c7f57160137716feb67b13e75cddd4d30464016 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Mon, 7 Dec 2020 15:26:35 -0500 Subject: [PATCH 5/6] condition for FT --- dbm_lib/dbm_features/raw_features/video/open_face_process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/video/open_face_process.py b/dbm_lib/dbm_features/raw_features/video/open_face_process.py index 51364521..b427656e 100644 --- a/dbm_lib/dbm_features/raw_features/video/open_face_process.py +++ b/dbm_lib/dbm_features/raw_features/video/open_face_process.py @@ -45,8 +45,9 @@ def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path, video_trac _, out_loc, fl_name = ut.filter_path(video_url, out_dir) full_f_name = fl_name + suffix output_directory = os.path.join(out_loc, full_f_name) - if not os.path.isdir(output_directory): - os.mkdir(output_directory) + + if video_tracking and not os.path.exists(os.path.abspath(output_directory)): + os.makedirs(os.path.abspath(output_directory)) csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path)) except Exception as e: From dad92abfbce9cb2826801044b1e7937a4da7e262 Mon Sep 17 00:00:00 2001 From: vjbytes102 Date: Wed, 9 Dec 2020 18:17:09 -0500 Subject: [PATCH 6/6] list initialization --- dbm_lib/dbm_features/raw_features/audio/formant_freq.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py index 592ebdc3..2d2eff92 100644 --- a/dbm_lib/dbm_features/raw_features/audio/formant_freq.py +++ b/dbm_lib/dbm_features/raw_features/audio/formant_freq.py @@ -31,7 +31,11 @@ def formant_list(formant,snd): Returns: List of first through fourth formant for each frame """ - f1_list, f2_list, f3_list, f4_list = ([], ) * 4 + f1_list = [] + f2_list = [] + f3_list = [] + f4_list = [] + dur = snd.duration-0.02 dur_round = round(dur, 2) @@ -126,4 +130,4 @@ def run_formant(video_uri, out_dir, r_config): calc_formant(video_uri, audio_file, out_loc, fl_name, r_config) except Exception as e: - logger.error('Failed to process audio file') \ No newline at end of file + logger.error('Failed to process audio file')