bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
README.md
View File

@@ -1 +1,62 @@
# open_dbm
# DBM Open Source Lib
---------------------------------------------
Installation - Dev Mode
---------------------------------------------
- Clone repo from github
git clone https://github.com/AiCure/open_dbm.git
- Prepare a python 3 virtualenv
pip install virtualenv (optional, only needed if it's not installed)
virtualenv <your_env_name>
source <your_env_name>/bin/activate
- Install package
pip install -r requirements.txt
sudo apt-get -y install ffmpeg
- Install Openface 2.0.6 (follow below installation steps)
cd pkg/OpenFace
bash download_models.sh
bash install.sh
---------------------------------------------
Documentation
---------------------------------------------
Please add documentation link here
---------------------------------------------
Example usage
---------------------------------------------
- Process video/audio file
python process_file.py --input_path abc.mp4 --output_raw_path data/output --output_derived_path output/derived --dbm_group acoustic facial movement
- Process multiple files in a directory
python process_dir.py --input_path abc/ --output_raw_path data/output --output_derived_path output/derived --dbm_group acoustic facial movement
```
where,
input_path: Required – Path of the file being processed. This can be either an .mp4 file or a .wav file.
output_raw_path: Required – Path where the raw variable data will be stored.
output_derived_path: Optional - Path to store derived variable. If not specified, derived variables won't be calculated.
dbm_group: Optional –
Type of variables to calculate. If not specified, all variables will be calculated. The following inputs are acceptable:
- facial, in which case only facial variables are calculated
- acoustic, in which case only acoustic variables are calculated
- movement, in which case only movement variables are calculated
If the input_path was a .wav file, then only acoustic variables can be calculated. If otherwise, a warning message will be printed.
```

16
dbm_lib/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
"""
file_name: init
project_name: DBM
created: 2020-20-07
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
DBMLIB_PATH = os.path.dirname(__file__)
DBMLIB_SERVICE_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/services/services.yml'))
DBMLIB_FEATURE_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/raw_feature.yml'))
DBMLIB_DERIVE_FEATURE_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../resources/features/derived_feature.yml'))

29
dbm_lib/config/config_derive_feature.py Normal file
View File

@@ -0,0 +1,29 @@
"""
file_name: config_derive_feature
project_name: DBM
created: 2020-20-07
"""
import yaml
from dbm_lib import DBMLIB_DERIVE_FEATURE_CONFIG
class ConfigDeriveReader(object):
"""Summary
Read sevice end ponit
"""
def __init__(self,
feature_config_yml=None):
"""Summary
Args:
feature_config_yml (None, optional): yml file defined service configuration
"""
if feature_config_yml is None:
feature_config = DBMLIB_DERIVE_FEATURE_CONFIG
else:
feature_config = feature_config_yml
with open(feature_config, 'r') as ymlfile:
config = yaml.load(ymlfile)
self.base_derive = config

225
dbm_lib/config/config_raw_feature.py Normal file
View File

@@ -0,0 +1,225 @@
"""
file_name: config_raw_feature
project_name: DBM
created: 2020-20-07
"""
import yaml
from dbm_lib import DBMLIB_FEATURE_CONFIG
class ConfigRawReader(object):
"""Summary
Read sevice end ponit
"""
def __init__(self,
feature_config_yml=None):
"""Summary
Args:
feature_config_yml (None, optional): yml file defined service configuration
"""
if feature_config_yml is None:
feature_config = DBMLIB_FEATURE_CONFIG
else:
feature_config = feature_config_yml
with open(feature_config, 'r') as ymlfile:
config = yaml.load(ymlfile)
#Verbal features
self.base_raw = config
self.err_reason = config['raw_feature']['error_reason']
#Output range
self.mov_headvel_start = config['raw_feature']['mov_headvel_start']
self.mov_headvel_end = config['raw_feature']['mov_headvel_end']
#Acoustic variable
self.aco_int = config['raw_feature']['aco_int']
self.aco_ff = config['raw_feature']['aco_ff']
self.aco_voiceLabel = config['raw_feature']['aco_voiceLabel']
self.aco_hnr = config['raw_feature']['aco_hnr']
self.aco_gne = config['raw_feature']['aco_gne']
self.aco_fm1 = config['raw_feature']['aco_fm1']
self.aco_fm2 = config['raw_feature']['aco_fm2']
self.aco_fm3 = config['raw_feature']['aco_fm3']
self.aco_fm4 = config['raw_feature']['aco_fm4']
self.aco_jitter = config['raw_feature']['aco_jitter']
self.aco_shimmer = config['raw_feature']['aco_shimmer']
self.aco_mfcc1 = config['raw_feature']['aco_mfcc1']
self.aco_mfcc2 = config['raw_feature']['aco_mfcc2']
self.aco_mfcc3 = config['raw_feature']['aco_mfcc3']
self.aco_mfcc4 = config['raw_feature']['aco_mfcc4']
self.aco_mfcc5 = config['raw_feature']['aco_mfcc5']
self.aco_mfcc6 = config['raw_feature']['aco_mfcc6']
self.aco_mfcc7 = config['raw_feature']['aco_mfcc7']
self.aco_mfcc8 = config['raw_feature']['aco_mfcc8']
self.aco_mfcc9 = config['raw_feature']['aco_mfcc9']
self.aco_mfcc10 = config['raw_feature']['aco_mfcc10']
self.aco_mfcc11 = config['raw_feature']['aco_mfcc11']
self.aco_mfcc12 = config['raw_feature']['aco_mfcc12']
self.aco_voiceFrame = config['raw_feature']['aco_voiceFrame']
self.aco_totVoiceFrame = config['raw_feature']['aco_totVoiceFrame']
self.aco_voicePct = config['raw_feature']['aco_voicePct']
self.aco_pausetime = config['raw_feature']['aco_pausetime']
self.aco_totaltime = config['raw_feature']['aco_totaltime']
self.aco_speakingtime = config['raw_feature']['aco_speakingtime']
self.aco_numpauses = config['raw_feature']['aco_numpauses']
self.aco_pausefrac = config['raw_feature']['aco_pausefrac']
#Facial Action Unit (for consistency)
self.fac_AU01int = config['raw_feature']['fac_AU01int']
self.fac_AU02int = config['raw_feature']['fac_AU02int']
self.fac_AU04int = config['raw_feature']['fac_AU04int']
self.fac_AU05int = config['raw_feature']['fac_AU05int']
self.fac_AU06int = config['raw_feature']['fac_AU06int']
self.fac_AU07int = config['raw_feature']['fac_AU07int']
self.fac_AU09int = config['raw_feature']['fac_AU09int']
self.fac_AU10int = config['raw_feature']['fac_AU10int']
self.fac_AU12int = config['raw_feature']['fac_AU12int']
self.fac_AU14int = config['raw_feature']['fac_AU14int']
self.fac_AU15int = config['raw_feature']['fac_AU15int']
self.fac_AU17int = config['raw_feature']['fac_AU17int']
self.fac_AU20int = config['raw_feature']['fac_AU20int']
self.fac_AU23int = config['raw_feature']['fac_AU23int']
self.fac_AU25int = config['raw_feature']['fac_AU25int']
self.fac_AU26int = config['raw_feature']['fac_AU26int']
self.fac_AU45int = config['raw_feature']['fac_AU45int']
self.fac_AU01pres = config['raw_feature']['fac_AU01pres']
self.fac_AU02pres = config['raw_feature']['fac_AU02pres']
self.fac_AU04pres = config['raw_feature']['fac_AU04pres']
self.fac_AU05pres = config['raw_feature']['fac_AU05pres']
self.fac_AU06pres = config['raw_feature']['fac_AU06pres']
self.fac_AU07pres = config['raw_feature']['fac_AU07pres']
self.fac_AU09pres = config['raw_feature']['fac_AU09pres']
self.fac_AU10pres = config['raw_feature']['fac_AU10pres']
self.fac_AU12pres = config['raw_feature']['fac_AU12pres']
self.fac_AU14pres = config['raw_feature']['fac_AU14pres']
self.fac_AU15pres = config['raw_feature']['fac_AU15pres']
self.fac_AU17pres = config['raw_feature']['fac_AU17pres']
self.fac_AU20pres = config['raw_feature']['fac_AU20pres']
self.fac_AU23pres = config['raw_feature']['fac_AU23pres']
self.fac_AU25pres = config['raw_feature']['fac_AU25pres']
self.fac_AU26pres = config['raw_feature']['fac_AU26pres']
self.fac_AU28pres = config['raw_feature']['fac_AU28pres']
self.fac_AU45pres = config['raw_feature']['fac_AU45pres']
#Facial Landmarks (for consistency)
self.fac_LMK00disp = config['raw_feature']['fac_LMK00disp']
self.fac_LMK01disp = config['raw_feature']['fac_LMK01disp']
self.fac_LMK02disp = config['raw_feature']['fac_LMK02disp']
self.fac_LMK03disp = config['raw_feature']['fac_LMK03disp']
self.fac_LMK04disp = config['raw_feature']['fac_LMK04disp']
self.fac_LMK05disp = config['raw_feature']['fac_LMK05disp']
self.fac_LMK06disp = config['raw_feature']['fac_LMK06disp']
self.fac_LMK07disp = config['raw_feature']['fac_LMK07disp']
self.fac_LMK08disp = config['raw_feature']['fac_LMK08disp']
self.fac_LMK09disp = config['raw_feature']['fac_LMK09disp']
self.fac_LMK10disp = config['raw_feature']['fac_LMK10disp']
self.fac_LMK11disp = config['raw_feature']['fac_LMK11disp']
self.fac_LMK12disp = config['raw_feature']['fac_LMK12disp']
self.fac_LMK13disp = config['raw_feature']['fac_LMK13disp']
self.fac_LMK14disp = config['raw_feature']['fac_LMK14disp']
self.fac_LMK15disp = config['raw_feature']['fac_LMK15disp']
self.fac_LMK16disp = config['raw_feature']['fac_LMK16disp']
self.fac_LMK17disp = config['raw_feature']['fac_LMK17disp']
self.fac_LMK18disp = config['raw_feature']['fac_LMK18disp']
self.fac_LMK19disp = config['raw_feature']['fac_LMK19disp']
self.fac_LMK20disp = config['raw_feature']['fac_LMK20disp']
self.fac_LMK21disp = config['raw_feature']['fac_LMK21disp']
self.fac_LMK22disp = config['raw_feature']['fac_LMK22disp']
self.fac_LMK23disp = config['raw_feature']['fac_LMK23disp']
self.fac_LMK24disp = config['raw_feature']['fac_LMK24disp']
self.fac_LMK25disp = config['raw_feature']['fac_LMK25disp']
self.fac_LMK26disp = config['raw_feature']['fac_LMK26disp']
self.fac_LMK27disp = config['raw_feature']['fac_LMK27disp']
self.fac_LMK28disp = config['raw_feature']['fac_LMK28disp']
self.fac_LMK29disp = config['raw_feature']['fac_LMK29disp']
self.fac_LMK30disp = config['raw_feature']['fac_LMK30disp']
self.fac_LMK31disp = config['raw_feature']['fac_LMK31disp']
self.fac_LMK32disp = config['raw_feature']['fac_LMK32disp']
self.fac_LMK33disp = config['raw_feature']['fac_LMK33disp']
self.fac_LMK34disp = config['raw_feature']['fac_LMK34disp']
self.fac_LMK35disp = config['raw_feature']['fac_LMK35disp']
self.fac_LMK36disp = config['raw_feature']['fac_LMK36disp']
self.fac_LMK37disp = config['raw_feature']['fac_LMK37disp']
self.fac_LMK38disp = config['raw_feature']['fac_LMK38disp']
self.fac_LMK39disp = config['raw_feature']['fac_LMK39disp']
self.fac_LMK40disp = config['raw_feature']['fac_LMK40disp']
self.fac_LMK41disp = config['raw_feature']['fac_LMK41disp']
self.fac_LMK42disp = config['raw_feature']['fac_LMK42disp']
self.fac_LMK43disp = config['raw_feature']['fac_LMK43disp']
self.fac_LMK44disp = config['raw_feature']['fac_LMK44disp']
self.fac_LMK45disp = config['raw_feature']['fac_LMK45disp']
self.fac_LMK46disp = config['raw_feature']['fac_LMK46disp']
self.fac_LMK47disp = config['raw_feature']['fac_LMK47disp']
self.fac_LMK48disp = config['raw_feature']['fac_LMK48disp']
self.fac_LMK49disp = config['raw_feature']['fac_LMK49disp']
self.fac_LMK50disp = config['raw_feature']['fac_LMK50disp']
self.fac_LMK51disp = config['raw_feature']['fac_LMK51disp']
self.fac_LMK52disp = config['raw_feature']['fac_LMK52disp']
self.fac_LMK53disp = config['raw_feature']['fac_LMK53disp']
self.fac_LMK54disp = config['raw_feature']['fac_LMK54disp']
self.fac_LMK55disp = config['raw_feature']['fac_LMK55disp']
self.fac_LMK56disp = config['raw_feature']['fac_LMK56disp']
self.fac_LMK57disp = config['raw_feature']['fac_LMK57disp']
self.fac_LMK58disp = config['raw_feature']['fac_LMK58disp']
self.fac_LMK59disp = config['raw_feature']['fac_LMK59disp']
self.fac_LMK60disp = config['raw_feature']['fac_LMK60disp']
self.fac_LMK61disp = config['raw_feature']['fac_LMK61disp']
self.fac_LMK62disp = config['raw_feature']['fac_LMK62disp']
self.fac_LMK63disp = config['raw_feature']['fac_LMK63disp']
self.fac_LMK64disp = config['raw_feature']['fac_LMK64disp']
self.fac_LMK65disp = config['raw_feature']['fac_LMK65disp']
self.fac_LMK66disp = config['raw_feature']['fac_LMK66disp']
self.fac_LMK67disp = config['raw_feature']['fac_LMK67disp']
#Facial features
self.hap_exp = config['raw_feature']['hap_exp']
self.sad_exp = config['raw_feature']['sad_exp']
self.sur_exp = config['raw_feature']['sur_exp']
self.fea_exp = config['raw_feature']['fea_exp']
self.ang_exp = config['raw_feature']['ang_exp']
self.dis_exp = config['raw_feature']['dis_exp']
self.con_exp = config['raw_feature']['con_exp']
self.happ_occ = config['raw_feature']['happ_occ']
self.sad_occ = config['raw_feature']['sad_occ']
self.sur_occ = config['raw_feature']['sur_occ']
self.fea_occ = config['raw_feature']['fea_occ']
self.ang_occ = config['raw_feature']['ang_occ']
self.dis_occ = config['raw_feature']['dis_occ']
self.con_occ = config['raw_feature']['con_occ']
self.pos_exp = config['raw_feature']['pos_exp']
self.neg_exp = config['raw_feature']['neg_exp']
self.neu_exp = config['raw_feature']['neu_exp']
self.cai_exp = config['raw_feature']['cai_exp']
self.com_exp = config['raw_feature']['com_exp']
self.hap_exp_full = config['raw_feature']['hap_exp_full']
self.sad_exp_full = config['raw_feature']['sad_exp_full']
self.sur_exp_full = config['raw_feature']['sur_exp_full']
self.fea_exp_full = config['raw_feature']['fea_exp_full']
self.ang_exp_full = config['raw_feature']['ang_exp_full']
self.dis_exp_full = config['raw_feature']['dis_exp_full']
self.con_exp_full = config['raw_feature']['con_exp_full']
self.pos_exp_full = config['raw_feature']['pos_exp_full']
self.neg_exp_full = config['raw_feature']['neg_exp_full']
self.neu_exp_full = config['raw_feature']['neu_exp_full']
self.cai_exp_full = config['raw_feature']['cai_exp_full']
self.com_exp_full = config['raw_feature']['com_exp_full']
self.fac_AsymMaskMouth = config['raw_feature']['fac_AsymMaskMouth']
self.fac_AsymMaskEye = config['raw_feature']['fac_AsymMaskEye']
self.fac_AsymMaskEyebrow = config['raw_feature']['fac_AsymMaskEyebrow']
self.fac_AsymMaskCom = config['raw_feature']['fac_AsymMaskCom']
#Movement features
self.head_vel = config['raw_feature']['head_vel']
self.mov_blink_ear = config['raw_feature']['mov_blink_ear']
self.vid_dur = config['raw_feature']['vid_dur']
self.fps = config['raw_feature']['fps']
self.mov_blinkframes = config['raw_feature']['mov_blinkframes']
self.mov_blinkdur = config['raw_feature']['mov_blinkdur']
self.mov_Hpose_Pitch = config['raw_feature']['mov_Hpose_Pitch']
self.mov_Hpose_Yaw = config['raw_feature']['mov_Hpose_Yaw']
self.mov_Hpose_Roll = config['raw_feature']['mov_Hpose_Roll']
self.mov_Hpose_Dist = config['raw_feature']['mov_Hpose_Dist']

67
dbm_lib/config/config_reader.py Normal file
View File

@@ -0,0 +1,67 @@
"""
file_name: config_reader
project_name: DBM
created: 2020-20-07
"""
import yaml
from dbm_lib import DBMLIB_SERVICE_CONFIG
class ConfigReader(object):
"""Summary
Read sevice end ponit
"""
def __init__(self,
service_config_yml=None):
"""Summary
Args:
service_config_yml (None, optional): yml file defined service configuration
"""
if service_config_yml is None:
service_config = DBMLIB_SERVICE_CONFIG
else:
service_config = service_config_yml
with open(service_config, 'r') as ymlfile:
config = yaml.load(ymlfile)
self.input_dir = config['cdx_configuration']['input_dir']
self.output_dir = config['cdx_configuration']['output_dir']
self.out_derived_dir = config['cdx_configuration']['out_derived_dir']
self.of_path = config['cdx_configuration']['open_face_path']
self.facial_landmarks = config['cdx_configuration']['facial_landmarks']
self.feature_group = config['cdx_configuration']['feature_group']
def get_open_face_path(self):
"""Summary
Returns:
TYPE: end point
"""
return self.of_path
def get_input_dir(self):
"""Summary
Returns:
TYPE: end point
"""
return self.input_dir
def get_output_dir(self):
"""Summary
Returns:
TYPE: end point
"""
return self.output_dir
def get_out_derived_dir(self):
"""Summary
Returns:
TYPE: end point
"""
return self.out_derived_dir
def get_fac_landmark_path(self):
"""Summary
Returns:
TYPE: end point
"""
return self.facial_landmarks

129
dbm_lib/controller/process_feature.py Normal file
View File

@@ -0,0 +1,129 @@
"""
file_name: process_features
project_name: DBM
created: 2020-20-07
"""
from dbm_lib.dbm_features.raw_features.audio import intensity, pitch_freq, hnr, gne, voice_frame_score, formant_freq
from dbm_lib.dbm_features.raw_features.audio import pause_segment, jitter, shimmer, mfcc
from dbm_lib.dbm_features.raw_features.video import face_asymmetry, face_au, face_emotion_expressivity, face_landmark
from dbm_lib.dbm_features.raw_features.movement import head_motion, eye_blink
import subprocess
import logging
from os.path import isfile, splitext, basename, dirname, join
import glob
import os
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
def audio_to_wav(input_filepath):
""" Extracts a video's audio file and saves it to wav
Args:
input_filepath: (str)
Returns:
"""
fname, _ = splitext(input_filepath)
output_filepath = fname + '.wav'
if not isfile(output_filepath):
call = ['ffmpeg', '-i', input_filepath, '-vn', '-acodec', 'pcm_s16le', '-ar', '44100', output_filepath]
logger.info('Converting audio from {} to wav'.format(input_filepath))
subprocess.check_output(call)
logger.info('wav output saved in {}'.format(output_filepath))
else:
logger.info('Output file {} already exists'.format(output_filepath))
def process_acoustic(video_uri, out_dir, dbm_group, r_config):
"""
processing acoustic features
Args:
video_uri: video path; out_dir: raw variable output dir
dbm_group: list of features group to process; r_config: raw feature config object
"""
if dbm_group != None and len(dbm_group)>0 and 'acoustic' not in dbm_group:
return
logger.info('processing audio intensity....')
intensity.run_intensity(video_uri, out_dir, r_config)
logger.info('processing audio pitch freq....')
pitch_freq.run_pitch(video_uri, out_dir, r_config)
logger.info('processing HNR....')
hnr.run_hnr(video_uri, out_dir, r_config)
logger.info('processing GNE....')
gne.run_gne(video_uri, out_dir, r_config)
logger.info('processing voice frame score....')
voice_frame_score.run_vfs(video_uri, out_dir, r_config)
logger.info('processing formant frequency....')
formant_freq.run_formant(video_uri, out_dir, r_config)
logger.info('processing pause segment....')
pause_segment.run_pause_segment(video_uri, out_dir, r_config)
logger.info('processing jitter....')
jitter.run_jitter(video_uri, out_dir, r_config)
logger.info('processing shimmer....')
shimmer.run_shimmer(video_uri, out_dir, r_config)
logger.info('processing mfcc....')
mfcc.run_mfcc(video_uri, out_dir, r_config)
def process_facial(video_uri, out_dir, dbm_group, r_config):
"""
processing facial features
Args:
video_uri: video path; out_dir: raw variable output dir
dbm_group: list of features to process; r_config: raw feature config object
"""
if dbm_group != None and len(dbm_group)>0 and 'facial' not in dbm_group:
return
logger.info('processing facial asymmetry....')
face_asymmetry.run_face_asymmetry(video_uri, out_dir, r_config)
logger.info('processing facial Action Unit....')
face_au.run_face_au(video_uri, out_dir, r_config)
logger.info('processing facial expressivity....')
face_emotion_expressivity.run_face_expressivity(video_uri, out_dir, r_config)
logger.info('processing facial landmark....')
face_landmark.run_face_landmark(video_uri, out_dir, r_config)
def process_movement(video_uri, out_dir, dbm_group, r_config, dlib_model):
"""
processing facial features
Args:
video_uri: video path; out_dir: raw variable output dir
dbm_group: list of features to process; r_config: raw feature config object
dlib_model: shape predictor model path
"""
if dbm_group != None and len(dbm_group)>0 and 'movement' not in dbm_group:
return
logger.info('processing head movement....')
head_motion.run_head_movement(video_uri, out_dir, r_config)
logger.info('processing eye blink....')
eye_blink.run_eye_blink(video_uri, out_dir, r_config, dlib_model)
def remove_file(file_path):
"""
removing wav file
"""
file_dir = dirname(file_path)
file_name, _ = splitext(basename(file_path))
wav_file = glob.glob(join(file_dir, file_name + '.wav'))
if len(wav_file)> 0:
os.remove(wav_file[0])

10
dbm_lib/dbm_features/derived_features/__init__.py Normal file
View File

@@ -0,0 +1,10 @@
"""
file_name: init
project_name: DBM
created: 2020-20-07
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

151
dbm_lib/dbm_features/derived_features/derive.py Normal file
View File

@@ -0,0 +1,151 @@
"""
file_name: derive
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import glob
import os
import logging
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
def dict_to_df(feature_dict, file):
"""
Converting ditionary to dataframe
"""
final_dict = {k: v for d in feature_dict for k, v in d.items()}
feature_df = pd.DataFrame([final_dict])
feature_df['dbm_master_url'] = file
return feature_df
def save_derive_output(df_list, feature, out_loc):
"""
Saving derive variable output
"""
if len(df_list)>0:
logger.info("Saving derived variable output for {}".format(feature))
df = pd.concat(df_list, ignore_index=True)
feature_dir = 'derive_' + feature
out_dir = os.path.join(out_loc, feature)
file_name = os.path.join(out_dir, feature_dir + '.csv')
if not os.path.exists(out_dir):
os.makedirs(out_dir)
df.to_csv(file_name, index=False)
def feature_output(df_fea, exp_var, cal_type):
"""
Computing mean value of dataframe columns
"""
exp_val = np.nan
try:
df_ = df_fea[exp_var].astype(float).copy()
df_ = df_.dropna().reset_index(drop=True)
if len(df_)>0:
if cal_type == 'mean':
exp_val = df_.mean(axis = 0, skipna = True)
elif cal_type == 'std':
exp_val = df_.std(axis = 0, skipna = True)
elif cal_type == 'count':#use case for eye blink
exp_var = 'blink_count'
exp_val = (len(df_)/df_[0])*60
elif cal_type == 'pct':
if len(df_)>0:
exp_val = len(df_[df_ > 0])/len(df_)
elif cal_type == 'range':
exp_val = max(df_) - min(df_)
except Exception as e:
logger.error('Failed to compute calculation: {}'.format(e))
pass
var_name = exp_var + '_' + cal_type
exp_val = float("{0:.4f}".format(exp_val))
var_val = (var_name, exp_val)
return var_val
def cal_type_dict(var_df, raw_df, d_cfg_Obj, r_cfg_Obj):
var_name = str(var_df['var_id'])
#fetching key based on variable name from raw config
var_key = list(r_cfg_Obj.keys())[list(r_cfg_Obj.values()).index(var_name)]
cal_type = d_cfg_Obj[var_key] # calculation type from config
var_val = [feature_output(raw_df, var_name, cal) for cal in cal_type]
var_val_dict = dict(var_val)
return var_val_dict
def compute_feature(raw_df, var_cols, d_cfg_Obj, r_cfg_Obj):
"""
Computing features
"""
#Variable data frame for each feature group
var_df = pd.DataFrame(var_cols,columns=['var_id'])
feature_dict = {}
if len(raw_df)>0:
feature_dict = var_df.apply(cal_type_dict, args=(raw_df, d_cfg_Obj, r_cfg_Obj, ), axis=1)
return feature_dict
def calc_derive(input_file, input_dir, output_dir, r_cfg_Obj, d_cfg_Obj, feature):
"""
Calculating derived variable
"""
df_list = []
for file in input_file:
file_name, _ = os.path.splitext(os.path.basename(file))
input_loc = os.path.join(input_dir, file_name)
var_cols = [r_cfg_Obj[x] for x in d_cfg_Obj[feature]]
fea_loc = d_cfg_Obj[feature + '_LOC']
fea_res = glob.glob(os.path.join(input_loc, '*/*/*' + fea_loc + '.csv'))
if len(fea_res)>0:
raw_df = pd.read_csv(fea_res[0])
feature_dict = compute_feature(raw_df, var_cols, d_cfg_Obj, r_cfg_Obj)
if len(feature_dict)>0:
feature_df = dict_to_df(feature_dict, file)
df_list.append(feature_df)
save_derive_output(df_list, feature, output_dir)
def run_derive(input_file, input_dir, output_dir, r_config, d_config):
"""
Processing derived variable
"""
d_cfg_Obj = d_config.base_derive['derive_feature']
r_cfg_Obj = r_config.base_raw['raw_feature']
feature_group = d_cfg_Obj['FEATURE_GROUP']
#Iterating over feature group
for feature in feature_group:
try:
calc_derive(input_file, input_dir, output_dir, r_cfg_Obj, d_cfg_Obj, feature)
except Exception as e:
logger.error('Failed to process derived variables.')

125
dbm_lib/dbm_features/raw_features/audio/formant_freq.py Normal file
View File

@@ -0,0 +1,125 @@
"""
file_name: formant_freq
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import parselmouth
import numpy as np
import parselmouth
import librosa
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
formant_dir = 'audio/formant_freq'
csv_ext = '_formant.csv'
error_txt = 'error: length less than 0.064'
def formant_list(formant,snd):
"""
Getting formant frequency per second
Args:
formant: Formant object for sound wave
snd: Parselmouth sound object
Returns:
List of first through fourth formant for each frame
"""
f1_list, f2_list, f3_list, f4_list = ([], ) * 4
dur = snd.duration-0.02
dur_round = round(dur, 2)
time_list = np.arange(0.001, dur_round, 0.001)
for time in time_list:
f1 = formant.get_value_at_time(1,time)
f2 = formant.get_value_at_time(2,time)
f3 = formant.get_value_at_time(3,time)
f4 = formant.get_value_at_time(4,time)
f1_list.append(f1)
f2_list.append(f2)
f3_list.append(f3)
f4_list.append(f4)
return f1_list,f2_list,f3_list,f4_list
def formant_score(path):
"""
Using parselmouth library fetching Formant Frequency
Args:
path: (.wav) audio file location
Returns:
(list) list of Formant freq for each voice frame
"""
sound_pat = parselmouth.Sound(path)
formant = sound_pat.to_formant_burg(time_step=.001)
f_score = formant_list(formant,sound_pat)
return f_score
def calc_formant(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing Formant freq matrix
Args:
audio_file: (.wav) parsed audio file; fl_name: input file name
out_loc: (str) Output directory; r_config: raw variable config
"""
f1_list,f2_list,f3_list,f4_list = formant_score(audio_file)
df_formant = pd.DataFrame(f1_list, columns=[r_config.aco_fm1])
df_formant[r_config.aco_fm2] = f2_list
df_formant[r_config.aco_fm3] = f3_list
df_formant[r_config.aco_fm4] = f4_list
df_formant.replace('', np.nan, regex=True,inplace=True)
df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_formant['Frames'] = df_formant.index
df_formant['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
def empty_fm(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty formant frequency matrix if something fails
"""
cols = ['Frames', r_config.aco_fm1, r_config.aco_fm2, r_config.aco_fm3, r_config.aco_fm4, r_config.err_reason]
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
df_fm = pd.DataFrame(out_val, columns = cols)
df_fm['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
def run_formant(video_uri, out_dir, r_config):
"""
Processing all patient's for fetching Formant freq
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
empty_fm(video_uri, out_loc, fl_name, r_config)
return
calc_formant(video_uri, audio_file, out_loc, fl_name, r_config)

157
dbm_lib/dbm_features/raw_features/audio/gne.py Normal file
View File

@@ -0,0 +1,157 @@
"""
file_name: gne
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import os
import glob
import parselmouth
import librosa
import more_itertools as mit
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
gne_dir = 'audio/glottal_noise'
ff_dir = 'audio/pitch'
csv_ext = '_gne_frame.csv'
def gne_ratio(sound):
"""
Using parselmouth library fetching glottal noise excitation ratio
Args:
sound: parselmouth object
Returns:
(list) list of gne ratio for each voice frame
"""
harmonicity_gne = sound.to_harmonicity_gne()
gne_all_bands = harmonicity_gne.values
gne_all_bands = np.where(gne_all_bands==-200, np.NaN, gne_all_bands)
gne = np.nanmax(gne_all_bands) # following http://www.fon.hum.uva.nl/rob/NKI_TEVA/TEVA/HTML/NKI_TEVA.pdf
return gne
def empty_gne(video_uri, out_loc, fl_name, r_config, error_txt):
"""
Preparing empty GNE matrix if something fails
"""
cols = ['Frames', r_config.aco_gne, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_gne = pd.DataFrame(out_val, columns = cols)
df_gne['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_gne, out_loc, fl_name, gne_dir, csv_ext)
def segment_pitch(dir_path, r_config):
"""
segmenting pitch freq for each voice segment
"""
com_speech_sort, voiced_yes, voiced_no = ([], ) * 3
for file in os.listdir(dir_path):
try:
if file.endswith('_pitch.csv'):
ff_df = pd.read_csv((dir_path+'/'+file))
voice_label = ff_df[r_config.aco_voiceLabel]
indices_yes = [i for i, x in enumerate(voice_label) if x == "yes"]
voiced_yes = [list(group) for group in mit.consecutive_groups(indices_yes)]
indices_no = [i for i, x in enumerate(voice_label) if x == "no"]
voiced_no = [list(group) for group in mit.consecutive_groups(indices_no)]
com_speech = voiced_yes + voiced_no
com_speech_sort = sorted(com_speech, key=lambda x: x[0])
except:
pass
return com_speech_sort, voiced_yes, voiced_no
def segment_gne(com_speech_sort, voiced_yes, voiced_no, gne_all_frames, audio_file):
"""
calculating gne for each voice segment
"""
snd = parselmouth.Sound(audio_file)
pitch = snd.to_pitch(time_step=.001)
for idx, vs in enumerate(com_speech_sort):
try:
max_gne = np.NaN
if vs in voiced_yes and len(vs)>1:
start_time = pitch.get_time_from_frame_number(vs[0])
end_time = pitch.get_time_from_frame_number(vs[-1])
snd_start = int(snd.get_frame_number_from_time(start_time))
snd_end = int(snd.get_frame_number_from_time(end_time))
samples = parselmouth.Sound(snd.as_array()[0][snd_start:snd_end])
max_gne = gne_ratio(samples)
except:
pass
gne_all_frames[idx] = max_gne
return gne_all_frames
def calc_gne(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing gne matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv's
"""
dir_path = os.path.join(out_loc, ff_dir)
if os.path.isdir(dir_path):
voice_seg = segment_pitch(dir_path, r_config)
gne_all_frames = [np.NaN] * len(voice_seg[0])
gne_segment_frames = segment_gne(voice_seg[0], voice_seg[1], voice_seg[2], gne_all_frames, audio_file)
df_gne = pd.DataFrame(gne_segment_frames, columns=[r_config.aco_gne])
df_gne[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_gne['Frames'] = df_gne.index
df_gne['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(out_loc))
ut.save_output(df_gne, out_loc, fl_name, gne_dir, csv_ext)
else:
error_txt = 'error: pitch freq not available'
empty_gne(video_uri, out_loc, fl_name, r_config, error_txt)
def run_gne(video_uri, out_dir, r_config):
"""
Processing all patient's for fetching glottal noise ratio
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
error_txt = 'error: length less than 0.064'
empty_gne(video_uri, out_loc, fl_name, r_config, error_txt)
return
calc_gne(video_uri, audio_file, out_loc, fl_name, r_config)

92
dbm_lib/dbm_features/raw_features/audio/hnr.py Normal file
View File

@@ -0,0 +1,92 @@
"""
file_name: hnr
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import os
import glob
import parselmouth
import librosa
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
hnr_dir = 'audio/harmonic_noise'
csv_ext = '_hnr_frame.csv'
error_txt = 'error: length less than 0.064'
def hnr_ratio(filepath):
"""
Using parselmouth library fetching harmonic noise ratio ratio
Args:
path: (.wav) audio file location
Returns:
(list) list of hnr ratio for each voice frame, min,max and mean hnr
"""
sound = parselmouth.Sound(filepath)
harmonicity = sound.to_harmonicity_ac(time_step=.001)
hnr_all_frames = harmonicity.values#[harmonicity.values != -200] nan it (****)
hnr_all_frames = np.where(hnr_all_frames==-200, np.NaN, hnr_all_frames)
return hnr_all_frames.transpose()
def calc_hnr(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing harmonic noise matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv's
"""
hnr_all_frames = hnr_ratio(audio_file)
df_hnr = pd.DataFrame(hnr_all_frames, columns=[r_config.aco_hnr])
df_hnr['Frames'] = df_hnr.index
df_hnr['dbm_master_url'] = video_uri
df_hnr[r_config.err_reason] = 'Pass'# will replace with threshold in future release
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_hnr, out_loc, fl_name, hnr_dir, csv_ext)
def empty_hnr(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty HNR matrix if something fails
"""
cols = ['Frames', r_config.aco_hnr, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_hnr = pd.DataFrame(out_val, columns = cols)
df_hnr['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_hnr, out_loc, fl_name, hnr_dir, csv_ext)
def run_hnr(video_uri, out_dir, r_config):
"""
Processing all patient's for fetching harmonic noise ratio
-------------------
-------------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
empty_hnr(video_uri, out_loc, fl_name, r_config)
return
calc_hnr(video_uri, audio_file, out_loc, fl_name, r_config)

88
dbm_lib/dbm_features/raw_features/audio/intensity.py Normal file
View File

@@ -0,0 +1,88 @@
"""
file_name: intensity
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import glob
import parselmouth
import librosa
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
intensity_dir = 'audio/intensity'
csv_ext = '_intensity.csv'
error_txt = 'error: length less than 0.064'
def intensity_score(path):
"""
Using parselmouth library fetching Intensity
Args:
path: (.wav) audio file location
Returns:
(list) list of Intensity for each voice frame
"""
sound_pat = parselmouth.Sound(path)
intensity = sound_pat.to_intensity(time_step=.001)
return intensity.values[0]
def calc_intensity(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing Intensity matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv's
"""
intensity_frames = intensity_score(audio_file)
df_intensity = pd.DataFrame(intensity_frames, columns=[r_config.aco_int])
df_intensity['Frames'] = df_intensity.index
df_intensity['dbm_master_url'] = video_uri
df_intensity[r_config.err_reason] = 'Pass'# will replace with threshold in future release
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_intensity, out_loc, fl_name, intensity_dir, csv_ext)
def empty_intensity(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty Intensity matrix if something fails
"""
cols = ['Frames', r_config.aco_int, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_int = pd.DataFrame(out_val, columns = cols)
df_int['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_int, out_loc, fl_name, intensity_dir, csv_ext)
def run_intensity(video_uri, out_dir, r_config):
"""
Processing all patient's for fetching Intensity
-------------------
-------------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
empty_intensity(video_uri, out_loc, fl_name, r_config)
return
calc_intensity(video_uri, audio_file, out_loc, fl_name, r_config)

155
dbm_lib/dbm_features/raw_features/audio/jitter.py Normal file
View File

@@ -0,0 +1,155 @@
"""
file_name: jitter_processing
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import os
import glob
import parselmouth
import librosa
import numpy as np
import more_itertools as mit
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
jitter_dir = 'audio/jitter'
ff_dir = 'audio/pitch'
csv_ext = '_jitter.csv'
def audio_jitter(sound):
"""
Using parselmouth library fetching jitter
Args:
sound: parselmouth object
Returns:
(list) list of jitters for each voice frame
"""
pointProcess = parselmouth.praat.call(sound, "To PointProcess (periodic, cc)...", 80, 500)
jitter = parselmouth.praat.call(pointProcess, "Get jitter (local)", 0, 0, 0.0001, 0.02, 1.3)
return jitter
def empty_jitter(video_uri, out_loc, fl_name, r_config, error_txt):
"""
Preparing empty jitter matrix if something fails
"""
cols = ['Frames', r_config.aco_jitter, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_jitter = pd.DataFrame(out_val, columns = cols)
df_jitter['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_jitter, out_loc, fl_name, jitter_dir, csv_ext)
def segment_pitch(dir_path, r_config):
"""
segmenting pitch freq for each voice segment
"""
com_speech_sort, voiced_yes, voiced_no = ([], ) * 3
for file in os.listdir(dir_path):
try:
if file.endswith('_pitch.csv'):
ff_df = pd.read_csv((dir_path+'/'+file))
voice_label = ff_df[r_config.aco_voiceLabel]
indices_yes = [i for i, x in enumerate(voice_label) if x == "yes"]
voiced_yes = [list(group) for group in mit.consecutive_groups(indices_yes)]
indices_no = [i for i, x in enumerate(voice_label) if x == "no"]
voiced_no = [list(group) for group in mit.consecutive_groups(indices_no)]
com_speech = voiced_yes + voiced_no
com_speech_sort = sorted(com_speech, key=lambda x: x[0])
except:
pass
return com_speech_sort, voiced_yes, voiced_no
def segment_jitter(com_speech_sort, voiced_yes, voiced_no, jitter_frames, audio_file):
"""
calculating jitter for each voice segment
"""
snd = parselmouth.Sound(audio_file)
pitch = snd.to_pitch(time_step=.001)
for idx, vs in enumerate(com_speech_sort):
try:
jitter = np.NaN
if vs in voiced_yes and len(vs)>1:
start_time = pitch.get_time_from_frame_number(vs[0])
end_time = pitch.get_time_from_frame_number(vs[-1])
snd_start = int(snd.get_frame_number_from_time(start_time))
snd_end = int(snd.get_frame_number_from_time(end_time))
samples = parselmouth.Sound(snd.as_array()[0][snd_start:snd_end])
jitter = audio_jitter(samples)
except:
pass
jitter_frames[idx] = jitter
return jitter_frames
def calc_jitter(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing jitter matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv
r_config: config.config_raw_feature.pyConfigFeatureNmReader object
"""
dir_path = os.path.join(out_loc, ff_dir)
if os.path.isdir(dir_path):
voice_seg = segment_pitch(dir_path, r_config)
jitter_frames = [np.NaN] * len(voice_seg[0])
jitter_segment_frames = segment_jitter(voice_seg[0], voice_seg[1], voice_seg[2], jitter_frames, audio_file)
df_jitter = pd.DataFrame(jitter_segment_frames, columns=[r_config.aco_jitter])
df_jitter[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_jitter['Frames'] = df_jitter.index
df_jitter['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(out_loc))
ut.save_output(df_jitter, out_loc, fl_name, jitter_dir, csv_ext)
else:
error_txt = 'error: fundamental freq not available'
empty_jitter(video_uri, out_loc, fl_name, r_config, error_txt)
def run_jitter(video_uri, out_dir, r_config):
"""
Processing all patient's videos for fetching jitter
-------------------
-------------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
error_txt = 'error: length less than 0.064'
empty_jitter(video_uri, out_loc, fl_name, r_config, error_txt)
return
calc_jitter(video_uri, audio_file, out_loc, fl_name, r_config)

102
dbm_lib/dbm_features/raw_features/audio/mfcc.py Normal file
View File

@@ -0,0 +1,102 @@
"""
file_name: mfcc
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import os
import glob
import parselmouth
import librosa
import numpy as np
import librosa
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
mfcc_dir = 'audio/mfcc'
csv_ext = '_mfcc.csv'
error_txt = 'error: length less than 0.064'
def empty_mfcc(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty empty_mfcc matrix if something fails
"""
cols = ['Frames', r_config.aco_mfcc1, r_config.aco_mfcc2, r_config.aco_mfcc3, r_config.aco_mfcc4, r_config.aco_mfcc5,
r_config.aco_mfcc6, r_config.aco_mfcc7, r_config.aco_mfcc8, r_config.aco_mfcc9, r_config.aco_mfcc10,
r_config.aco_mfcc11, r_config.aco_mfcc12, r_config.err_reason]
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan,
error_txt]]
df_mfcc = pd.DataFrame(out_val, columns = cols)
df_mfcc['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_mfcc, out_loc, fl_name, mfcc_dir, csv_ext)
def audio_mfcc(path):
"""
Using parselmouth library fetching mfccs
Args:
path: (.wav) audio file location
Returns:
(list) list of mfccs for each voice frame
"""
sound = parselmouth.Sound(path)
mfcc_object = sound.to_mfcc(time_step=.001,number_of_coefficients=12)
mfccs = mfcc_object.to_array()
mfccs = np.delete(mfccs, (0), axis=0)
return mfccs
def calc_mfcc(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing mfcc matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: output location to save csv
fl_name: (str) name of audio file
r_config: config.config_raw_feature.pyConfigFeatureNmReader object
"""
dict_ = {}
mfccs = audio_mfcc(audio_file)
for i in range(1,13):
conf_str = r_config.base_raw['raw_feature']
dict_[conf_str['aco_mfcc' + str(i)]] = mfccs[i-1, :]
df = pd.DataFrame(dict_)
df['Frames'] = df.index
df[r_config.err_reason] = 'Pass'# may replace based on threshold in future release
df['dbm_master_url'] = video_uri
ut.save_output(df, out_loc, fl_name, mfcc_dir, csv_ext)
def run_mfcc(video_uri, out_dir, r_config):
"""
Processing all patients to fetch mfccs
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
empty_mfcc(video_uri, out_loc, fl_name, r_config)
return
calc_mfcc(video_uri, audio_file, out_loc, fl_name, r_config)

167
dbm_lib/dbm_features/raw_features/audio/pause_segment.py Normal file
View File

@@ -0,0 +1,167 @@
"""
file_name: pause_segment
project_name: DBM
created: 2020-20-07
"""
import os
import glob
from pydub import AudioSegment
import librosa
import pandas as pd
import numpy as np
import webrtcvad
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import vad_utilities as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
pause_seg_dir = 'audio/pause_segment'
csv_ext = '_pause_segment.csv'
def get_timing_cues(seg_starts_sec, seg_ends_sec, r_config):
"""
Get timing cues from segmented speech
Args:
seg_starts_sec: Audio segment start time in seconds
seg_ends_sec: Audio segment end time in seconds
Returns:
Dictionary with pause features
"""
total_time = seg_ends_sec[-1] - seg_starts_sec[0]
speaking_time = np.sum(np.asarray(seg_ends_sec) - np.asarray(seg_starts_sec))
num_pauses = len(seg_starts_sec) - 1
pause_len = np.zeros(num_pauses)
for p in range(num_pauses):
pause_len[p] = seg_starts_sec[p+1] - seg_ends_sec[p]
if len(pause_len)>0:
pause_len_mean = np.mean(pause_len)
pause_len_std = np.std(pause_len)
pause_time = np.sum(pause_len)
else:
pause_len_mean = 0
pause_len_std = 0
pause_time = 0
pause_frac = pause_time / total_time
timing_dict = {r_config.aco_totaltime: total_time, r_config.aco_speakingtime: speaking_time,
r_config.aco_numpauses: num_pauses, r_config.aco_pausetime: pause_time, r_config.aco_pausefrac: pause_frac}
return timing_dict
def process_silence(audio_file, r_config):
"""
Returns dataframe for pause between words using voice activity detection
Args:
audio_file: Audio file location
Returns:
Dataframe value
"""
feat_dict_list = []
y, sr = vu.read_wave(audio_file)
# 3 is most aggressive (splits most), 0 least (better for low snr)
aggressiveness = 3
frame_dur_ms = 20
#pause segment(long & short pad)
long_pad_around_voice_ms = 200
short_pad_around_voice_ms = 100
if len(y)>0:
vad = webrtcvad.Vad(aggressiveness)
frames = vu.frame_generator(frame_dur_ms, y, sr)
frames = list(frames)
#longer pad time screens out little blips, but misses short silences
long_seg_starts, long_seg_ends = vu.vad_get_segment_times(sr, frame_dur_ms, long_pad_around_voice_ms, vad, frames)
#Logic to handle blank audio file
if len(long_seg_starts) == 0 or len(long_seg_ends) == 0:
return ''
t_start = long_seg_starts[0]
t_end = long_seg_ends[-1]
# shorter pad time captures short silences (but misfires on little blips)
short_seg_starts, short_seg_ends = vu.vad_get_segment_times(sr, frame_dur_ms, short_pad_around_voice_ms, vad, frames)
seg_starts = []
seg_ends = []
for k in range(len(short_seg_starts)): # logic to clean up some typical misfires
if (short_seg_starts[k] >=t_start) and (short_seg_starts[k] <= t_end):
seg_starts.append(short_seg_starts[k])
seg_ends.append(short_seg_ends[k])
if len(seg_starts) == 0 or len(seg_ends) == 0:
return ''
timing_dict = get_timing_cues(seg_starts, seg_ends, r_config)
feat_dict_list.append(timing_dict)
df = pd.DataFrame(feat_dict_list)
df[r_config.err_reason] = 'Pass'# will replace with threshold in future release
return df
def empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt):
"""
Preparing empty Pause Segment matrix if something fails
"""
cols = [r_config.aco_totaltime, r_config.aco_speakingtime, r_config.aco_numpauses, r_config.aco_pausetime,
r_config.aco_pausefrac, r_config.err_reason]
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
df_pause = pd.DataFrame(out_val, columns = cols)
df_pause['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_pause, out_loc, fl_name, pause_seg_dir, csv_ext)
def run_pause_segment(video_uri, out_dir, r_config):
"""
Processing all patient's for getting Pause Segment
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
error_txt = 'error: length less than 0.064'
empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt)
return
logger.info('Converting stereo sound to mono-lD')
sound_mono = AudioSegment.from_wav(audio_file)
sound_mono = sound_mono.set_channels(1)
sound_mono = sound_mono.set_frame_rate(48000)
mono_wav = os.path.join(input_loc, fl_name + '_mono.wav')
sound_mono.export(mono_wav, format="wav")
df_pause_seg = process_silence(mono_wav, r_config)
os.remove(mono_wav)#removing mono wav file
if isinstance(df_pause_seg, pd.DataFrame) and len(df_pause_seg)>0:
logger.info('Processing Output file {} '.format(out_loc))
df_pause_seg['dbm_master_url'] = video_uri
ut.save_output(df_pause_seg, out_loc, fl_name, pause_seg_dir, csv_ext)
else:
error_txt = 'error: webrtcvad returns no segment'
empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt)

109
dbm_lib/dbm_features/raw_features/audio/pitch_freq.py Normal file
View File

@@ -0,0 +1,109 @@
"""
file_name: pitch_freq
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import os
import glob
import parselmouth
import librosa
import numpy as np
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
ff_dir = 'audio/pitch'
csv_ext = '_pitch.csv'
error_txt = 'error: length less than 0.064'
def audio_pitch(path):
"""
Using parselmouth library fetching pitch/fundamental frequency
Args:
path: (.wav) audio file location
Returns:
(list) list of pitch/fundamental frequency for each voice frame
"""
sound_pat = parselmouth.Sound(path)
pitch = sound_pat.to_pitch(time_step=.001)
pitch_values = pitch.selected_array['frequency']
return list(pitch_values)
def label_speech(row,fd_freq):
"""
identify whether frame is voiced or not
Args:
row: (item) pitch frequency value
Returns:
(str) yes or no indicator for voice
"""
if row[fd_freq] > 0 :
return 'yes'
else:
return 'no'
def calc_pitch(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing pitch frequency matrix
Args:
audio_file: (.wav) parsed audio file
row: (dataframe) subject details from master csv
new_out_base_dir: (str) Output directory for csv
"""
ff_frames = audio_pitch(audio_file)
df_ffreq = pd.DataFrame(ff_frames, columns=[r_config.aco_ff])
df_ffreq['Frames'] = df_ffreq.index
df_ffreq[r_config.aco_voiceLabel] = df_ffreq.apply(lambda row: label_speech(row, r_config.aco_ff),axis=1)
df_ffreq[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_ffreq['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(out_loc))
ut.save_output(df_ffreq, out_loc, fl_name, ff_dir, csv_ext)
def empty_pitch(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty pitch frequency matrix if something fails
"""
df_ffreq = pd.DataFrame([[np.nan, np.nan, 'no', error_txt]],
columns=['Frames', r_config.aco_ff, r_config.aco_voiceLabel, r_config.err_reason])
df_ffreq['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_ffreq, out_loc, fl_name, ff_dir, csv_ext)
def run_pitch(video_uri, out_dir, r_config):
"""
Processing audio for fetching pitch
-------------------
-------------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
empty_pitch(video_uri, out_loc, fl_name, r_config)
return
calc_pitch(video_uri, audio_file, out_loc, fl_name, r_config)

157
dbm_lib/dbm_features/raw_features/audio/shimmer.py Normal file
View File

@@ -0,0 +1,157 @@
"""
file_name: shimmer_processing
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import os
import glob
import parselmouth
import librosa
import numpy as np
import more_itertools as mit
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
shimmer_dir = 'audio/shimmer'
ff_dir = 'audio/pitch'
csv_ext = '_shimmer.csv'
def audio_shimmer(sound):
"""
Using parselmouth library fetching shimmer
Args:
sound: parselmouth object
Returns:
(list) list of shimmers for each voice frame
"""
pointProcess = parselmouth.praat.call(sound, "To PointProcess (periodic, cc)...", 80, 500)
shimmer = parselmouth.praat.call([sound, pointProcess], "Get shimmer (local)", 0, 0, 0.0001, 0.02, 1.3, 1.6)
return shimmer
def empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt):
"""
Preparing empty shimmer matrix if something fails
"""
cols = ['Frames', r_config.aco_shimmer, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_shimmer = pd.DataFrame(out_val, columns = cols)
df_shimmer['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_shimmer, out_loc, fl_name, shimmer_dir, csv_ext)
def segment_pitch(dir_path, r_config):
"""
segmenting pitch freq for each voice segment
"""
com_speech_sort, voiced_yes, voiced_no = ([], ) * 3
for file in os.listdir(dir_path):
try:
if file.endswith('_pitch.csv'):
ff_df = pd.read_csv((dir_path+'/'+file))
voice_label = ff_df[r_config.aco_voiceLabel]
indices_yes = [i for i, x in enumerate(voice_label) if x == "yes"]
voiced_yes = [list(group) for group in mit.consecutive_groups(indices_yes)]
indices_no = [i for i, x in enumerate(voice_label) if x == "no"]
voiced_no = [list(group) for group in mit.consecutive_groups(indices_no)]
com_speech = voiced_yes + voiced_no
com_speech_sort = sorted(com_speech, key=lambda x: x[0])
except:
pass
return com_speech_sort, voiced_yes, voiced_no
def segment_shimmer(com_speech_sort, voiced_yes, voiced_no, shimmer_frames, audio_file):
"""
calculating shimmer for each voice segment
"""
snd = parselmouth.Sound(audio_file)
pitch = snd.to_pitch(time_step=.001)
for idx, vs in enumerate(com_speech_sort):
try:
shimmer = np.NaN
if vs in voiced_yes and len(vs)>1:
start_time = pitch.get_time_from_frame_number(vs[0])
end_time = pitch.get_time_from_frame_number(vs[-1])
snd_start = int(snd.get_frame_number_from_time(start_time))
snd_end = int(snd.get_frame_number_from_time(end_time))
samples = parselmouth.Sound(snd.as_array()[0][snd_start:snd_end])
shimmer = audio_shimmer(samples)
except:
pass
shimmer_frames[idx] = shimmer
return shimmer_frames
def calc_shimmer(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing shimmer matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv
r_config: config.config_raw_feature.pyConfigFeatureNmReader object
"""
dir_path = os.path.join(out_loc, ff_dir)
if os.path.isdir(dir_path):
voice_seg = segment_pitch(dir_path, r_config)
shimmer_frames = [np.NaN] * len(voice_seg[0])
shimmer_segment_frames = segment_shimmer(voice_seg[0], voice_seg[1], voice_seg[2], shimmer_frames, audio_file)
df_shimmer = pd.DataFrame(shimmer_segment_frames, columns=[r_config.aco_shimmer])
df_shimmer[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_shimmer['Frames'] = df_shimmer.index
df_shimmer['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(out_loc))
ut.save_output(df_shimmer, out_loc, fl_name, shimmer_dir, csv_ext)
else:
error_txt = 'error: fundamental freq not available'
empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt)
def run_shimmer(video_uri, out_dir, r_config):
"""
Processing all patients to fetch shimmer
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
error_txt = 'error: length less than 0.064'
empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt)
return
calc_shimmer(video_uri, audio_file, out_loc, fl_name, r_config)

107
dbm_lib/dbm_features/raw_features/audio/voice_frame_score.py Normal file
View File

@@ -0,0 +1,107 @@
"""
file_name: voice_frame_score
project_name: DBM
created: 2020-20-07
"""
import parselmouth
import pandas as pd
import numpy as np
import glob
import librosa
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
vfs_dir = 'audio/voice_frame_score'
csv_ext = '_vfs.csv'
error_txt = 'error: length less than 0.064'
def audio_pitch_frame(pitch):
"""
Computing total number of speech and participant voiced frames
Args:
pitch: speech pitch
Returns:
(float) total voice frames and participant voiced frames
"""
total_frames = pitch.get_number_of_frames()
voiced_frames = pitch.count_voiced_frames()
return total_frames, voiced_frames
def voice_segment(path):
"""
Using parselmouth library for fundamental frequency
Args:
path: (.wav) audio file location
Returns:
(float) total voice frames, participant voiced frames and voiced frames percentage
"""
sound_pat = parselmouth.Sound(path)
pitch = sound_pat.to_pitch()
total_frames,voiced_frames = audio_pitch_frame(pitch)
voiced_percentage = (voiced_frames/total_frames)*100
return voiced_percentage, voiced_frames, total_frames
def calc_vfs(video_uri, audio_file, out_loc, fl_name, r_config):
"""
creating dataframe matrix for voice frame score
Args:
audio_file: Audio file path
new_out_base_dir: AWS instance output base directory path
f_nm_config: Config file object
"""
voice_percentage,voiced_frames, total_frames = voice_segment(audio_file)
df_vfs = pd.DataFrame([voiced_frames], columns=[r_config.aco_voiceFrame])
df_vfs[r_config.aco_totVoiceFrame] = [total_frames]
df_vfs[r_config.aco_voicePct] = [voice_percentage]
df_vfs[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_vfs['Frames'] = df_vfs.index
df_vfs['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)
def empty_vfs(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty VFS matrix if something fails
"""
cols = ['Frames', r_config.aco_voiceFrame, r_config.aco_totVoiceFrame, r_config.aco_voicePct, r_config.err_reason]
out_val = [[np.nan, np.nan, np.nan, np.nan, error_txt]]
df_vfs = pd.DataFrame(out_val, columns = cols)
df_vfs['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)
def run_vfs(video_uri, out_dir, r_config):
"""
Processing all participants for fetching voice frame score
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter)>0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(audio_file))
empty_vfs(video_uri, out_loc, fl_name, r_config)
return
calc_vfs(video_uri, audio_file, out_loc, fl_name, r_config)

13
dbm_lib/dbm_features/raw_features/movement/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
"""
file_name: init
project_name: DBM
created: 2020-20-07
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
DBMLIB_PATH = os.path.dirname(__file__)

156
dbm_lib/dbm_features/raw_features/movement/eye_blink.py Normal file
View File

@@ -0,0 +1,156 @@
"""
file_name: eye_blink
project_name: DBM
created: 2020-20-07
"""
import os
import glob
from scipy.spatial import distance as dist
from scipy.signal import find_peaks
from imutils.video import FileVideoStream
from imutils.video import VideoStream
from imutils import face_utils
from moviepy.editor import VideoFileClip
import numpy as np
import pandas as pd
import imutils
import time
import dlib
import cv2
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
movement_expr_dir = 'movement/eye_blink'
csv_ext = '_eye_blink.csv'
def eye_aspect_ratio(eye):
"""
Computing eye aspect ratio for an individual frame
Args:
eye: Eye landmarks
Return:
Eye aspect ratio for a frame
"""
# euclidean distance for vertical eye landmarks
dist_cor1 = dist.euclidean(eye[1], eye[5])
dist_cor2 = dist.euclidean(eye[2], eye[4])
# euclidean distance for horizontal eye landmark
dist_cor3 = dist.euclidean(eye[0], eye[3])
ear = (dist_cor1 + dist_cor2) / (2.0 * dist_cor3)
return ear
def blink_detection(video_path,facial_landmarks,raw_config):
"""
Blink detection for each frame
Args:
video_path: MP4 file location
facial_landmarks: Facial landmark pre-trained model path
raw_config: Raw configuration file object
Return:
Dataframe with blink informatiom like blink frame, duration etc.
"""
TOT_FRAME = 1
blink_frame = []
ear_frame = []
clip = VideoFileClip(video_path, has_mask=True)
vid_length = clip.duration
identifier = dlib.get_frontal_face_detector() #dlib's face detector (HOG-based)
forecaster = dlib.shape_predictor(facial_landmarks) # the facial landmark predictor
#left and right eye landmarks
(left_beg, left_end) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
(right_beg, right_end) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]
f_stream = True
vid_stream = FileVideoStream(video_path).start()
while True:
try:
#check if stream/frame available in video
if f_stream and not vid_stream.more():
break
#reading & converting frame into grayscale
vid_frame = vid_stream.read()
vid_frame = imutils.resize(vid_frame, width=450)
gray = cv2.cvtColor(vid_frame, cv2.COLOR_BGR2GRAY)
#detecting face
rects = identifier(gray, 0)
for rect in rects:
lmk = forecaster(gray, rect)
lmk = face_utils.shape_to_np(lmk)
l_eye = lmk[left_beg:left_end] #Extracting left eye ratio
r_eye = lmk[right_beg:right_end] #Extracting right eye ratio
l_ear = eye_aspect_ratio(l_eye) # eye aspect ratio for left eye
r_ear = eye_aspect_ratio(r_eye) # eye aspect ratio for right eye
ear = (l_ear + r_ear) / 2.0 # average the eye aspect ratio
blink_frame.append(TOT_FRAME)
ear_frame.append(ear)
TOT_FRAME += 1
except Exception as e:
#logger.error("blink detection processing failed for: {}".format(video_path))
continue
blink_df = pd.DataFrame(ear_frame, columns =[raw_config.mov_blink_ear])
blink_df[raw_config.vid_dur] = vid_length
blink_df[raw_config.fps] = int(TOT_FRAME/vid_length)
blink_df[raw_config.mov_blinkframes] = blink_frame
peaks, _ = find_peaks(blink_df[raw_config.mov_blink_ear]*-1, prominence=0.1)#prominence = 0.1 based on tuning
final_blink_df = blink_df.iloc[peaks,:].reset_index(drop=True)
u_blink_df = blink_dur(final_blink_df,raw_config)
u_blink_df['dbm_master_url'] = video_path
return u_blink_df
def blink_dur(blink_df,raw_config):
"""
Computing blink duration between each blink
Args:
blink_df : Dataframe with blink informatiom like blink frame
raw_config: Raw configuration file object
Returns:
Updated dataframe with blink duration
"""
dur_list = []
if len(blink_df)>0:
blink_df[raw_config.mov_blinkdur] = blink_df[raw_config.mov_blinkframes].diff().fillna(
blink_df[raw_config.mov_blinkframes])
else:
blink_df[raw_config.mov_blinkdur] = np.nan
blink_df[raw_config.mov_blinkdur] = blink_df[raw_config.mov_blinkdur]/blink_df[raw_config.fps]
return blink_df
def run_eye_blink(video_uri, out_dir, r_config, facial_landmarks):
"""
Processing all patient's for getting eye blink artifacts
---------------
---------------
Args:
video_uri: video path; input_dir : input directory for video's
out_dir: (str) Output directory for processed output; r_config: raw variable config object;
facial_landmarks: landmark model path
"""
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
vid_file_path = os.path.exists(video_uri)
if vid_file_path==True:
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
df_blink = blink_detection(video_uri, facial_landmarks, r_config)
ut.save_output(df_blink, out_loc, fl_name, movement_expr_dir, csv_ext)

193
dbm_lib/dbm_features/raw_features/movement/head_motion.py Normal file
View File

@@ -0,0 +1,193 @@
"""
file_name: head_mov
project_name: DBM
created: 2020-20-07
"""
import os
import glob
import pandas as pd
import numpy as np
from scipy.spatial import distance
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
h_mov_dir = 'movement/head_movement'
h_pose_dir = 'movement/head_pose'
h_mov_ext = '_head_movement.csv'
h_pose_ext = '_head_pose.csv'
def head_pose_dist(of_results):
"""
Computing head pose distance frame by frame
Args:
of_results: Openface raw out dataframe
f_nm_config: Face config file object
Reutrns:
Final head pose distance frame by frame output
"""
distance_list = []
error_list = []
for index, row in of_results.iterrows():
dst = np.nan
if index == 0 or float(row[' confidence']) < 0.2: #Threshold < 0.2
distance_list.append(dst)
if float(row[' confidence']) < 0.2:
error_list.append('confidence less than 20%')
else:
error_list.append('Pass')
continue
if index > 0:
point_x = (of_results[' pose_Rx'][index-1], of_results[' pose_Ry'][index-1], of_results[' pose_Rz'][index-1])
point_y = (row[' pose_Rx'],row[' pose_Ry'],row[' pose_Rz'])
try:
dst = distance.euclidean(point_x, point_y)
except:
pass
distance_list.append(abs(dst))
error_list.append('Pass')
return distance_list, error_list
def head_pose(of_results,r_config):
"""
Generating head pose estimation dataframe
Args:
distance_val: distance list
f_nm_config: raw variable config file object
Reutrns:
Final head pose estimation dataframe
"""
pose_dist_list, error_list = head_pose_dist(of_results)
of_results.loc[(of_results[' confidence'].astype(float) < 0.2), [' pose_Rx',' pose_Ry',' pose_Rz']] = np.nan
pose_of = of_results[[' pose_Rx',' pose_Ry',' pose_Rz']]
pose_of.columns = [r_config.mov_Hpose_Pitch, r_config.mov_Hpose_Yaw, r_config.mov_Hpose_Roll]
pose_of[r_config.mov_Hpose_Dist] = pose_dist_list
pose_of[r_config.err_reason] = error_list
return pose_of
def head_motion_df(distance_val, error_list, r_config):
"""
Generating head movement dataframe
Args:
distance_val: distance list
r_config: raw variable config file object
Reutrns:
Final head velocity dataframe
"""
head_motion = r_config.head_vel
df_head_motion = pd.DataFrame(distance_val, columns=[head_motion])
df_head_motion['Frames'] = df_head_motion.index
new_df_intensity = df_head_motion[['Frames', head_motion]]
new_df_intensity[r_config.err_reason] = error_list
return new_df_intensity
def head_vel(of_results, r_config):
"""
Computing head velocity frame by frame
Args:
of_results: Openface raw out dataframe
r_config: Face config file object
Reutrns:
Final head velocity frame by frame output
"""
distance_list = []
error_list = []
for index, row in of_results.iterrows():
dst = np.nan
if index == 0 or float(row[' confidence']) < 0.2: #Threshold < 0.2
distance_list.append(dst)
if float(row[' confidence']) < 0.2:
error_list.append('confidence less than 20%')
else:
error_list.append('Pass')
continue
if index > 0:
point_x = (of_results[' pose_Tx'][index-1], of_results[' pose_Ty'][index-1], of_results[' pose_Tz'][index-1])
point_y = (row[' pose_Tx'],row[' pose_Ty'],row[' pose_Tz'])
try:
dst = distance.euclidean(point_x, point_y)
except:
pass
if abs(dst)>200:
dst = np.nan
error_list.append('Out of range')
else:
error_list.append('Pass')
distance_list.append(dst)
df_velocity = head_motion_df(distance_list, error_list, r_config)
return df_velocity
def calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config):
"""
Computing head motion and head pose variables
Args:
df_of: Openface dataframe
out_loc: Output path for saving output csv's
fl_name: file name for output csv
r_config: raw variable config file object
"""
col = [' confidence',' pose_Rx',' pose_Ry',' pose_Rz',' pose_Tx', ' pose_Ty', ' pose_Tz']
df_of = df_of[col]
df_hmotion = head_vel(df_of, r_config)
df_hmotion['dbm_master_url'] = video_uri
df_pose = head_pose(df_of, r_config)
df_pose['dbm_master_url'] = video_uri
ut.save_output(df_hmotion, out_loc, fl_name, h_mov_dir, h_mov_ext)
ut.save_output(df_pose, out_loc, fl_name, h_pose_dir, h_pose_ext)
def run_head_movement(video_uri, out_dir, r_config):
"""
Processing all patient's for getting movement artifacts for cdx_analysis workflow
--------------------------------
--------------------------------
Args:
video_uri: video path; input_dir : input directory for video's
out_dir: (str) Output directory for processed output; r_config: raw variable config object
"""
#filtering path to generate input & output path
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
of_csv = of_csv_path[0]
df_of = pd.read_csv(of_csv, error_bad_lines=False)
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config)

112
dbm_lib/dbm_features/raw_features/util/util.py Normal file
View File

@@ -0,0 +1,112 @@
"""
file_name: util
project_name: DBM
created: 2020-20-07
"""
import os
import glob
import numpy as np
import subprocess
def filter_path(video_url, out_dir):
"""
Filtering video uri path to prepare input and ouptut location
Args:
video_url: S3 bucket path for video
out_dir: Output directory path
"""
fl_name,_ = os.path.splitext(os.path.basename(video_url))
input_loc = os.path.dirname(video_url)
out_loc = os.path.join(out_dir, fl_name)
return input_loc, out_loc, fl_name
def save_output(df, out_loc, fl_name, f_dir, f_ext):
"""
creating output directory for Audio features
Args:
df: (dataframe) feature dataframe[ex: Formant freq, pitch]
out_loc: (dir) Output location where we want to save raw output
fl_name: file name
f_dir: directory name for a feature
f_ext: extension for a feature [ex: '_pose.csv']
"""
full_f_name = fl_name + f_ext
dir_path = os.path.join(out_loc, f_dir)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
sav_path = os.path.join(dir_path,full_f_name)
df.to_csv(sav_path, index=False)
def audio_process(base_dir,video_url):
"""
Parsing cleaned audio files(Audio files without IMA voice)
Args:
base_dir: Base path for raw data
video_url: Raw video file path
"""
new_video_url = base_dir+'/'.join(video_url[2:])
split_val = new_video_url.split('/')
wav_path = '/'.join(split_val[0:len(split_val)-1])
audio_split_check = glob.glob(wav_path + '/*_split.wav')
return audio_split_check
def compute_open_face_features(input_filepath,
output_directory,
open_face_executable,
au_static=False,
tracked_visualization=False,
clobber=False,
verbose=True):
"""
Runs OpenFace on an input video.
See https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments
Args:
input_filepath:
output_directory:
au_static:
tracked_visualization:
open_face_executable:
clobber: (bool) if True existing files will be overwritten
verbose:
Returns:
(str) path to output csv file
Raises:
IOError if OpenFace executable is missing
"""
if not os.path.isfile(open_face_executable):
raise IOError("OpenFace executable {} could not be found.".format(open_face_executable))
bn, _ = os.path.splitext(os.path.basename(input_filepath))
if not output_directory:
output_directory = os.path.join(os.path.dirname(input_filepath), bn + '_OF_features')
output_csv = os.path.join(output_directory, bn + '.csv')
if not os.path.isfile(output_csv) or clobber:
call = [open_face_executable, ]
if au_static:
call += ['-au_static', ]
if tracked_visualization:
call += ['-tracked', ]
call += ['-q', '-2Dfp', '-3Dfp', '-pdmparams', '-pose', '-aus', '-gaze']
call += ['-f', input_filepath, '-out_dir', output_directory]
if verbose:
print('Computing OpenFace features {} from video file'.format(input_filepath))
subprocess.check_output(call)
if verbose:
print('OpenFace features saved to {}'.format(output_directory))
else:
if verbose:
print('Output file {} already exists'.format(output_csv))
return os.path.join(output_directory, bn + '.csv')

221
dbm_lib/dbm_features/raw_features/util/vad_utilities.py Normal file
View File

@@ -0,0 +1,221 @@
"""
file_name: vad_utilities
project_name: DBM
created: 2020-20-07
"""
# code from https://github.com/wiseman/py-webrtcvad/blob/master/example.py
import collections
import contextlib
import sys
import wave
def read_wave(path):
"""Reads a .wav file.
Takes the path, and returns (PCM audio data, sample rate).
"""
with contextlib.closing(wave.open(path, 'rb')) as wf:
num_channels = wf.getnchannels()
assert num_channels == 1
sample_width = wf.getsampwidth()
assert sample_width == 2
sample_rate = wf.getframerate()
assert sample_rate in (8000, 16000, 32000, 48000)
pcm_data = wf.readframes(wf.getnframes())
return pcm_data, sample_rate
class Frame(object):
"""Represents a "frame" of audio data."""
def __init__(self, bytes, timestamp, duration):
self.bytes = bytes
self.timestamp = timestamp
self.duration = duration
def frame_generator(frame_duration_ms, audio, sample_rate):
"""Generates audio frames from PCM audio data.
Takes the desired frame duration in milliseconds, the PCM data, and
the sample rate.
Yields Frames of the requested duration.
"""
n = int(sample_rate * (frame_duration_ms / 1000.0) * 2)
offset = 0
timestamp = 0.0
duration = (float(n) / sample_rate) / 2.0
while offset + n < len(audio):
yield Frame(audio[offset:offset + n], timestamp, duration)
timestamp += duration
offset += n
def vad_collector(sample_rate, frame_duration_ms,
padding_duration_ms, vad, frames):
"""Filters out non-voiced audio frames.
Given a webrtcvad.Vad and a source of audio frames, yields only
the voiced audio.
Uses a padded, sliding window algorithm over the audio frames.
When more than 90% of the frames in the window are voiced (as
reported by the VAD), the collector triggers and begins yielding
audio frames. Then the collector waits until 90% of the frames in
the window are unvoiced to detrigger.
The window is padded at the front and back to provide a small
amount of silence or the beginnings/endings of speech around the
voiced frames.
Arguments:
sample_rate - The audio sample rate, in Hz.
frame_duration_ms - The frame duration in milliseconds.
padding_duration_ms - The amount to pad the window, in milliseconds.
vad - An instance of webrtcvad.Vad.
frames - a source of audio frames (sequence or generator).
Returns: A generator that yields PCM audio data.
"""
num_padding_frames = int(padding_duration_ms / frame_duration_ms)
# We use a deque for our sliding window/ring buffer.
ring_buffer = collections.deque(maxlen=num_padding_frames)
# We have two states: TRIGGERED and NOTTRIGGERED. We start in the
# NOTTRIGGERED state.
triggered = False
voiced_frames = []
for frame in frames:
is_speech = vad.is_speech(frame.bytes, sample_rate)
sys.stdout.write('1' if is_speech else '0')
if not triggered:
ring_buffer.append((frame, is_speech))
num_voiced = len([f for f, speech in ring_buffer if speech])
# If we're NOTTRIGGERED and more than 90% of the frames in
# the ring buffer are voiced frames, then enter the
# TRIGGERED state.
if num_voiced > 0.9 * ring_buffer.maxlen:
triggered = True
sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
# We want to yield all the audio we see from now until
# we are NOTTRIGGERED, but we have to start with the
# audio that's already in the ring buffer.
for f, s in ring_buffer:
voiced_frames.append(f)
ring_buffer.clear()
else:
# We're in the TRIGGERED state, so collect the audio data
# and add it to the ring buffer.
voiced_frames.append(frame)
ring_buffer.append((frame, is_speech))
num_unvoiced = len([f for f, speech in ring_buffer if not speech])
# If more than 90% of the frames in the ring buffer are
# unvoiced, then enter NOTTRIGGERED and yield whatever
# audio we've collected.
if num_unvoiced > 0.9 * ring_buffer.maxlen:
sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
triggered = False
yield b''.join([f.bytes for f in voiced_frames])
ring_buffer.clear()
voiced_frames = []
if triggered: # BT if were in triggered state at end of signal, set output time
sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
sys.stdout.write('\n')
# If we have any leftover voiced audio when we run out of input,
# yield it.
if voiced_frames:
yield b''.join([f.bytes for f in voiced_frames])
def vad_get_segment_times(sample_rate, frame_duration_ms,
padding_duration_ms, vad, frames):
"""Filters out non-voiced audio frames.
BT: based on vad_collector, but returns start and end times for voiced segs
Given a webrtcvad.Vad and a source of audio frames, yields only
the voiced audio.
Uses a padded, sliding window algorithm over the audio frames.
When more than 90% of the frames in the window are voiced (as
reported by the VAD), the collector triggers and begins yielding
audio frames. Then the collector waits until 90% of the frames in
the window are unvoiced to detrigger.
The window is padded at the front and back to provide a small
amount of silence or the beginnings/endings of speech around the
voiced frames.
Arguments:
sample_rate - The audio sample rate, in Hz.
frame_duration_ms - The frame duration in milliseconds.
padding_duration_ms - The amount to pad the window, in milliseconds.
vad - An instance of webrtcvad.Vad.
frames - a source of audio frames (sequence or generator).
Returns: lists of start and end segments
"""
num_padding_frames = int(padding_duration_ms / frame_duration_ms)
# We use a deque for our sliding window/ring buffer.
ring_buffer = collections.deque(maxlen=num_padding_frames)
# We have two states: TRIGGERED and NOTTRIGGERED. We start in the
# NOTTRIGGERED state.
triggered = False
start_times = []
end_times = []
for frame in frames:
is_speech = vad.is_speech(frame.bytes, sample_rate)
sys.stdout.write('1' if is_speech else '0')
if not triggered:
ring_buffer.append((frame, is_speech))
num_voiced = len([f for f, speech in ring_buffer if speech])
# If we're NOTTRIGGERED and more than 90% of the frames in
# the ring buffer are voiced frames, then enter the
# TRIGGERED state.
if num_voiced > 0.9 * ring_buffer.maxlen:
triggered = True
sys.stdout.write('+(%s)' % (ring_buffer[0][0].timestamp,))
start_times.append(ring_buffer[0][0].timestamp) # BT
ring_buffer.clear()
else:
# We're in the TRIGGERED state, so collect the audio data
# and add it to the ring buffer.
ring_buffer.append((frame, is_speech))
num_unvoiced = len([f for f, speech in ring_buffer if not speech])
# If more than 90% of the frames in the ring buffer are
# unvoiced, then enter NOTTRIGGERED and yield whatever
# audio we've collected.
if num_unvoiced > 0.9 * ring_buffer.maxlen:
sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
end_times.append(ring_buffer[0][0].timestamp + frame.duration) # BT
triggered = False
if triggered: # BT if were in triggered state at end of signal, set output time
sys.stdout.write('-(%s)' % (frame.timestamp + frame.duration))
if len(ring_buffer)>0:
end_times.append(ring_buffer[0][0].timestamp ) # BT
else:
# only get here in very rare case that we triggered on 2nd-to-last frame
end_times.append(frame.timestamp + frame.duration)
sys.stdout.write('\n')
return(start_times, end_times)
def filter_seg_times(seg_starts, seg_ends, pad_at_start = 0.5, len_to_keep=2.5 ):
"""
do some filtering on the segments found to select part for analysis
rule: find the first segment that is at least (pad_at_start+len_to_keep sec long.
Discard the firstpad_at_start sec, keep the next len_to_keep sec
if no such segments, then return empty list
returns sel_start, sel_end, sel_end_longer
"""
sel_start = []
sel_end = []
sel_end_longer = []
not_found = True
for iseg in range(len(seg_starts)):
seg_dur = seg_ends[iseg]-seg_starts[iseg]
if (not_found & (seg_dur > (pad_at_start + len_to_keep))):
t_start = seg_starts[iseg] + pad_at_start
sel_start.append(t_start)
sel_end.append(t_start + len_to_keep)
sel_end_longer.append(max(t_start + len_to_keep, seg_ends[iseg]-pad_at_start))
not_found = False
return sel_start, sel_end, sel_end_longer

183
dbm_lib/dbm_features/raw_features/util/video_util.py Normal file
View File

@@ -0,0 +1,183 @@
"""
file_name: video_util
project_name: DBM
created: 2020-20-07
"""
import pandas as pd
import numpy as np
import glob
from dbm_lib.dbm_features.raw_features.util import util as ut
def smooth(x,window_len=11,window='hanning'):
"""smooth the data using a window with requested size.
This method is based on the convolution of a scaled window with the signal.
The signal is prepared by introducing reflected copies of the signal
(with the window size) in both ends so that transient parts are minimized
in the begining and end part of the output signal.
input:
x: the input signal
window_len: the dimension of the smoothing window; should be an odd integer
window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
flat window will produce a moving average smoothing.
output:
the smoothed signal
example:
t=linspace(-2,2,0.1)
x=sin(t)+randn(len(t))*0.1
y=smooth(x)
see also:
numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve
scipy.signal.lfilter
TODO: the window parameter could be the window itself if an array instead of a string
NOTE: length(output) != length(input), to correct this: return y[(window_len/2-1):-(window_len/2)] instead of just y.
"""
if x.ndim != 1:
raise (ValueError, "smooth only accepts 1 dimension arrays.")
if x.size < window_len:
raise (ValueError, "Input vector needs to be bigger than window size.")
if window_len<3:
return x
if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
raise (ValueError, "Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")
s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]
#print(len(s))
if window == 'flat': #moving average
w=np.ones(window_len,'d')
else:
w=eval('np.'+window+'(window_len)')
y=np.convolve(w/w.sum(),s,mode='valid')
return y[int(window_len/2):-int(window_len/2)]
def filter_by_confidence_and_thresh(x, fea, thresh):
if x['s_confidence'] > 0.2 and np.fabs(x[fea]) < thresh:
return x[fea]
else:
return np.NaN
def add_au_emotion(x, emotion,emotion_type,exp_type):
"""
computing individula emotion expressivity matrix
Args:
emotion: Action Unit
"""
error_reason = 'Pass'
if x['s_confidence'] > 0.8: #if using smooth, no need for 'success'
sum_r = 0
cnt = 0
for au in emotion:
au_c_label = " AU{:02d}_c".format(au)
au_r_label = " AU{:02d}_r".format(au)
if x[au_c_label]==1 and (not np.isnan(x[au_r_label])): #there are data with face in, but au_c=0
sum_r += x[au_r_label]
cnt += 6
if exp_type=='full' and x[au_c_label]==0: #Logic to compute emotion expressivity when all AU's are present
cnt = 0
break
if cnt > 0:
sum_r /= cnt
else:
sum_r = 0
v_emo = x[emotion_type] + sum_r
else:
v_emo = np.NaN
error_reason = 'confidence less than 80%'
return v_emo, error_reason
def add_au_occ(x, emotion,emotion_type):
"""
computing individula emotion presence
Args:
emotion: Action Unit
"""
au_pres = []
em_pres = 0
error_reason = 'Pass'
if x['s_confidence'] > 0.8: #if using smooth, no need for 'success'
for au in emotion:
au_c_label = " AU{:02d}_c".format(au)
if x[au_c_label]==1: #there are data with face in, but au_c=0
au_pres.append(1)
if len(au_pres) == len(emotion):
em_pres = 1
else:
em_pres = np.NaN
error_reason = 'confidence less than 80%'
return em_pres, error_reason
def emotion_exp(em_au,of,em_col,err_col):
"""
Computing individual emotion expressivity and adding it to dataframe
"""
for emotion in em_au:
of[[em_col[0],err_col]]=of.apply(add_au_emotion, args=(emotion,em_col[0],'partial',), axis=1, result_type='expand')
of[[em_col[1],err_col]]=of.apply(add_au_emotion, args=(emotion,em_col[1],'full',), axis=1, result_type='expand')
def emotion_pres(em_au,of,em_col,err_col):
"""
Computing individual emotion expressivity and adding it to dataframe
"""
for emotion in em_au:
of[[em_col,err_col]]=of.apply(add_au_occ, args=(emotion,em_col,), axis=1, result_type='expand')
def calc_of_for_video(of,face_cfg,fe_cfg):
"""
Creating dataframe for emotion expressivity
"""
new_cols = [fe_cfg.hap_exp,fe_cfg.sad_exp,fe_cfg.sur_exp,fe_cfg.fea_exp,fe_cfg.ang_exp,fe_cfg.dis_exp,fe_cfg.con_exp,
fe_cfg.neg_exp,fe_cfg.pos_exp,fe_cfg.neu_exp,fe_cfg.cai_exp,fe_cfg.com_exp,fe_cfg.happ_occ,fe_cfg.sad_occ,
fe_cfg.sur_occ,fe_cfg.fea_occ,fe_cfg.ang_occ,fe_cfg.dis_occ,fe_cfg.con_occ,fe_cfg.hap_exp_full,
fe_cfg.sad_exp_full,fe_cfg.sur_exp_full,fe_cfg.fea_exp_full,fe_cfg.ang_exp_full,fe_cfg.dis_exp_full,
fe_cfg.con_exp_full,fe_cfg.neg_exp_full,fe_cfg.pos_exp_full,fe_cfg.neu_exp_full,fe_cfg.cai_exp_full,
fe_cfg.com_exp_full]
of[new_cols] = pd.DataFrame([[0] * len(new_cols)], index=of.index)
of[fe_cfg.err_reason] = 'Pass'
#Composite happiness expressivity
emotion_exp(face_cfg.happiness,of,[fe_cfg.hap_exp,fe_cfg.hap_exp_full],fe_cfg.err_reason)
#Composite sadness expressivity
emotion_exp(face_cfg.sadness,of,[fe_cfg.sad_exp,fe_cfg.sad_exp_full],fe_cfg.err_reason)
#Composite surprise expressivity
emotion_exp(face_cfg.surprise,of,[fe_cfg.sur_exp,fe_cfg.sur_exp_full],fe_cfg.err_reason)
#Composite fear expressivity
emotion_exp(face_cfg.fear,of,[fe_cfg.fea_exp,fe_cfg.fea_exp_full],fe_cfg.err_reason)
#Composite anger expressivity
emotion_exp(face_cfg.anger,of,[fe_cfg.ang_exp,fe_cfg.ang_exp_full],fe_cfg.err_reason)
#Composite disgust expressivity
emotion_exp(face_cfg.disgust,of,[fe_cfg.dis_exp,fe_cfg.dis_exp_full],fe_cfg.err_reason)
#Composite contempt expressivity
emotion_exp(face_cfg.contempt,of,[fe_cfg.con_exp,fe_cfg.con_exp_full],fe_cfg.err_reason)
#Composite Negative Expressivity
emotion_exp(face_cfg.NEG_ACTION_UNITS,of,[fe_cfg.neg_exp,fe_cfg.neg_exp_full],fe_cfg.err_reason)
#Composite Positive Expressivity
emotion_exp(face_cfg.POS_ACTION_UNITS,of,[fe_cfg.pos_exp,fe_cfg.pos_exp_full],fe_cfg.err_reason)
#Composite Neutral Expressivity
emotion_exp(face_cfg.NET_ACTION_UNITS,of,[fe_cfg.neu_exp,fe_cfg.neu_exp_full],fe_cfg.err_reason)
#Composite Activation Expressivity
emotion_exp(face_cfg.cai,of,[fe_cfg.cai_exp,fe_cfg.cai_exp_full],fe_cfg.err_reason)
#Composite Expressivity
emotion_exp(face_cfg.ACTION_UNITS,of,[fe_cfg.com_exp,fe_cfg.com_exp_full],fe_cfg.err_reason)
#AU happiness presence
emotion_pres(face_cfg.happiness,of,fe_cfg.happ_occ,fe_cfg.err_reason)
#AU Sad presence
emotion_pres(face_cfg.sadness,of,fe_cfg.sad_occ,fe_cfg.err_reason)
#AU Surprise presence
emotion_pres(face_cfg.surprise,of,fe_cfg.sur_occ,fe_cfg.err_reason)
#AU fear presence
emotion_pres(face_cfg.fear,of,fe_cfg.fea_occ,fe_cfg.err_reason)
#AU anger presence
emotion_pres(face_cfg.anger,of,fe_cfg.ang_occ,fe_cfg.err_reason)
#AU disgust presence
emotion_pres(face_cfg.disgust,of,fe_cfg.dis_occ,fe_cfg.err_reason)
#AU contempt presence
emotion_pres(face_cfg.contempt,of,fe_cfg.con_occ,fe_cfg.err_reason)

14
dbm_lib/dbm_features/raw_features/video/__init__.py Normal file
View File

@@ -0,0 +1,14 @@
"""
file_name: __init__
project_name: DBM
created: 2020-20-07
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
DBMLIB_PATH = os.path.dirname(__file__)
DBMLIB_FACE_CONFIG = os.path.abspath(os.path.join(DBMLIB_PATH, '../../../../resources/services/face_util.yml'))

351
dbm_lib/dbm_features/raw_features/video/face_asymmetry.py Normal file
View File

@@ -0,0 +1,351 @@
"""
file_name: face_asymmetry.py
project_name: DBM
created: 2020-20-07
"""
from mpl_toolkits import mplot3d
from matplotlib import pyplot as plt
import time
import numpy as np
import os
import datetime
import glob
import cv2
from scipy.spatial.transform import Rotation as R
import subprocess
import pandas as pd
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_asym_dir = 'video/face_asymmetry'
csv_ext = '_face_asymmetry.csv'
cv2_color_purple = (254,19,188)
color_blue = (0,0,1.0)
color_green = (0,1.0,0)
color_red = (1.0,0,0)
color_y = (1.0,1.0,0)
error_code_message = {
0: 'pass',
1: 'confidence less than 80%',
}
error_message_code = {y:x for x,y in error_code_message.items()}
def visualize_vid(fn, attr=None, write_out=False):
vid = cv2.VideoCapture(fn)
tot = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
fps = vid.get(cv2.CAP_PROP_FPS)
frame_width = int(vid.get(3))
frame_height = int(vid.get(4))
if write_out:
fig_w = 680 #680 667 676 #frame_width in order of Ali, Vennessa, synthesis
fig_h = 659 #659 659 659 #frame_height
out_vid = cv2.VideoWriter('out.mp4',cv2.VideoWriter_fourcc(*'MP4V'), fps, (fig_w,fig_h))
plt.figure(figsize=(8, 8))
try:
frameid = 0
while(True):
ret, frame = vid.read()
if not ret:
# Release the Video Device if ret is false
vid.release()
print('Released Video Resource')
break
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frameid += 1
logger.info(frameid, frame.shape)
if 'lmks_frms' in attr:
lmks_frms = attr['lmks_frms']
for i in range(lmks_frms[frameid].shape[0]):
cv2.circle(frame,(int(lmks_frms[frameid][i,0]),int(lmks_frms[frameid][i,1])), 2, cv2_color_purple, -1)
if write_out:
cv2.putText(frame,'Frame: '+str(frameid), (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 3)
plt.subplot(211)
plt.imshow(frame)
plt.axis('off'); plt.pause(0.2);
if 'score_asym' in attr:
ax = plt.subplot(212)
ax.cla()
ax.set_xlim(0,140) #ax.set_xlim(0,300)
ax.set_ylim(0,10)
sa = attr['score_asym']
s = sa[np.where(sa[:,0] <= frameid),:][0,:,:]
for i in range(1,s.shape[1]):
plt.plot(s[:,0], s[:,i])
plt.legend(['mouth', 'eyebrow', 'eye', 'mouth+eye+eyebrow'])
plt.minorticks_on()
plt.grid(b=True, which='major', color='r', linestyle='-')
plt.grid(b=True, which='minor', color='r', linestyle='--')
plt.savefig('tmp.png', bbox_inches='tight')
print(cv2.imread('tmp.png').shape)
plt.clf()
if write_out:
out_vid.write(cv2.imread('tmp.png'))
except KeyboardInterrupt:
# Release the Video Device
vid.release()
if write_out:
out_vid.release()
logger.info('Exception, and Video Resource Released')
if write_out:
out_vid.release()
def retrieve_attr(of_df):
'''
Retrieve landmarks and pose_translation for each frame from openface output
Args:
of_df: dataframe output from openface, including detected landmark coordinates
Returns:
lmks_frms: dictionary, with frame id as key and 68 landmark set as value
pose_p: dictionary, with frame id as key and pose param as value
'''
tot_lmks = 68 # openface specific
if len([i for i in of_df.columns.to_list() if ' x_' in i]) != tot_lmks:
return {}
lmks_frms = {}
pose_p = {}
for fi in sorted(of_df['frame'].to_list()):
lmks = np.zeros((tot_lmks,6))
r = of_df[of_df['frame']==fi]
for i in range(tot_lmks):
lmk_y = r[' y_'+str(i)].iloc[0]
lmk_x = r[' x_'+str(i)].iloc[0]
lmk_X = r[' X_'+str(i)].iloc[0]
lmk_Y = r[' Y_'+str(i)].iloc[0]
lmk_Z = r[' Z_'+str(i)].iloc[0]
confi = r[' confidence']
lmks[i,:] = [lmk_x, lmk_y, lmk_X, lmk_Y, lmk_Z, confi]
lmks_frms[fi] = lmks
pose_p[fi] = [r[' pose_Tx'].iloc[0], r[' pose_Ty'].iloc[0], r[' pose_Tz'].iloc[0],
r[' pose_Rx'].iloc[0], r[' pose_Ry'].iloc[0], r[' pose_Rz'].iloc[0]]
return lmks_frms, pose_p
def mirror_point(a, b, c, d, x1, y1, z1):
# mirror a point w.r.t a 3D plane
k =(-a * x1-b * y1-c * z1-d)/float((a * a + b * b + c * c))
x2 = a * k + x1
y2 = b * k + y1
z2 = c * k + z1
x3 = 2 * x2-x1
y3 = 2 * y2-y1
z3 = 2 * z2-z1
return [x3, y3, z3]
def dist_vec2plane(vec, nrm):
# Calculate the projected length of a vector (vec) to a plane defined by its normal (nrm)
return np.sqrt(np.dot(vec, vec) - np.dot(vec, nrm)**2)
def vis_lmks3d(lmks_frms, vis_idx):
"""
Visualizing facial landmarks
"""
fig = plt.figure()
color_type = ['b','g','r','y','c']
assert len(color_type) > len(vis_idx)
for fi in sorted(list(lmks_frms.keys())):
ax = plt.axes(projection="3d")
for i,vi in enumerate(vis_idx):
ax.scatter(lmks_frms[fi][vi,2], lmks_frms[fi][vi,3], lmks_frms[fi][vi,4], c=color_type[i])
ax.axes.set_xlim3d(left=-75, right=100)
ax.axes.set_ylim3d(bottom=-200, top=25)
ax.axes.set_zlim3d(bottom=440, top=560)
ax.view_init(-89, -90) #elev, ariz
plt.title(str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
plt.pause(0.2)
plt.cla()
plt.draw()
def calc_fac_asymmetry(attr, is_vis=False):
'''
Quantify facial asymmetry
Args:
attr: attribute dictionary containing necessary features for calculation, e.g.,
lmks_frms: dictionary, with frame id as key and 68 landmark set (OpenFace) as value
pose_param: dictionary, with frame id as key and pose param as value
Returns:
score_asym: 2D array of size (num_frms, num_asymm_fea), with frame id as the 0th column, and each remaining column as one asymmetry feature
'''
# openface landmark indices
lmks_ref_idx = list(range(0,17)) + list(range(27,36))
lmks_mid_idx = [27,28,29,30,33,51,62,66,57,8]
lmks_rgt_idx = [0,1,2,3,4,5,6,7,
17,18,19,20,21,
36,37,38,39,40,41,
48,49,50,
59,58,
60,61,
67]
lmks_lft_idx = [16,15,14,13,12,11,10,9,
26,25,24,23,22,
45,44,43,42,47,46,
54,53,52,
55,56,
64,63,
65]
lmks_mth_idx = list(range(48,68))
lmks_ebr_idx = list(range(17,27))
lmks_eye_idx = list(range(36,48))
assert len(lmks_lft_idx)==len(lmks_rgt_idx)
fea_list = ['mouth', 'eyebrow', 'eye', 'composite']
score_asym = np.empty(shape=(0, 0))
if ('lmks_frms' in attr) and ('pose_param' in attr):
lmks_frms = attr['lmks_frms']
pose_p = attr['pose_param']
if is_vis:
vis_lmks3d(lmks_frms, [lmks_lft_idx, lmks_rgt_idx, lmks_mid_idx, lmks_ref_idx])
score_asym = np.zeros((len(lmks_frms),len(fea_list)+1+1)) # +1: extra column for error code
if is_vis:
fig = plt.figure()
ax = plt.axes(projection="3d")
for s,fi in enumerate(sorted(list(lmks_frms.keys()))):
lmks_3d = lmks_frms[fi][:,2:5]
pose = pose_p[fi]
err_code = error_message_code['pass']
if lmks_frms[fi][0,5] < 0.8:
err_code = error_message_code['confidence less than 80%']
score_asym[s,:] = [fi,np.NaN,np.NaN,np.NaN,np.NaN,err_code]
continue
rx = R.from_euler('x', pose[3])
ry = R.from_euler('y', pose[4])
rz = R.from_euler('z', pose[5])
vec_pose = rz.apply(ry.apply(rx.apply([0,0,1])))
anc_idx = [30, 27, 8] # for central plane estimation
nrm = np.cross(lmks_3d[anc_idx[2],:] - lmks_3d[anc_idx[0],:],
lmks_3d[anc_idx[1],:] - lmks_3d[anc_idx[0],:])
nrm = nrm / np.linalg.norm(nrm)
a,b,c = nrm
d = np.dot(nrm, lmks_3d[anc_idx[0],:])
dist_L2R_mth = []
dist_L2R_ebr = []
dist_L2R_eye = []
dist_com = []
lmks_rfl = np.empty((0,3))
src_idx = lmks_lft_idx
for k,idx in enumerate(src_idx):
p_rfl = np.array(mirror_point(a, b, c, -d, lmks_3d[idx,0], lmks_3d[idx,1], lmks_3d[idx,2]))
lmks_rfl = np.vstack((lmks_rfl, p_rfl))
dist = dist_vec2plane((p_rfl-lmks_3d[lmks_rgt_idx[k],:]), vec_pose)
if idx in lmks_mth_idx:
dist_L2R_mth.append(dist)
if idx in lmks_ebr_idx:
dist_L2R_ebr.append(dist)
if idx in lmks_eye_idx:
dist_L2R_eye.append(dist)
if (idx in lmks_mth_idx) or (idx in lmks_ebr_idx) or (idx in lmks_eye_idx):
dist_com.append(dist)
score_asym[s,:] = [fi,np.mean(dist_L2R_mth),np.mean(dist_L2R_ebr),np.mean(dist_L2R_eye),np.mean(dist_com),err_code]
if is_vis:
ax.scatter(lmks_3d[:,0], lmks_3d[:,1], lmks_3d[:,2])
ax.scatter(lmks_rfl[:,0], lmks_rfl[:,1], lmks_rfl[:,2], c='y')
ax.scatter(pose_p[fi][0], pose_p[fi][1], pose_p[fi][2], c='c')
plt.title('mirrored landmarks, frame: '+str(fi)); ax.set_xlabel('X'); ax.set_ylabel('Y'); ax.set_zlabel('Z')
plt.pause(0.2)
plt.cla()
plt.draw()
return score_asym
def calc_asym_feature(open_face_csv, f_cfg):
"""
Calculating facial asymmetry features and preparing final df
"""
df_list = []
of_df = pd.read_csv(open_face_csv, error_bad_lines=False)
lmks_frms, pose_p = retrieve_attr(of_df)
attr = {'lmks_frms': lmks_frms, 'pose_param': pose_p}
score_asym = calc_fac_asymmetry(attr)
df_score_asym = pd.DataFrame(score_asym, columns=['frame', f_cfg.fac_AsymMaskMouth, f_cfg.fac_AsymMaskEyebrow,
f_cfg.fac_AsymMaskEye, f_cfg.fac_AsymMaskCom, f_cfg.err_reason])
df_score_asym[f_cfg.err_reason] = df_score_asym[f_cfg.err_reason].apply(lambda x: error_code_message[x])
df_score_asym['frame'] = of_df['frame']
df_score_asym['face_id'] = of_df[' face_id']
df_score_asym['timestamp'] = of_df[' timestamp']
df_score_asym['confidence'] = of_df[' confidence']
df_score_asym['success'] = of_df[' success']
df_list.append(df_score_asym)
return df_list
def run_face_asymmetry(video_uri, out_dir, f_cfg):
"""
Processing all patient's for calculating facial asymmetry
---------------
---------------
Args:
video_uri: video path; f_cfg: face config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
of_csv = of_csv_path[0]
asym_df_list = calc_asym_feature(of_csv, f_cfg)
asym_final_df = pd.concat(asym_df_list, ignore_index=True)
asym_final_df['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext)

94
dbm_lib/dbm_features/raw_features/video/face_au.py Normal file
View File

@@ -0,0 +1,94 @@
"""
file_name: face_au.py
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_au_dir = 'video/face_au'
csv_ext = '_face_au.csv'
def extract_col_nm_au(cols):
"""
Extract action unit (au) column names from openface output (csv)
Args:
cols: column names from open face output (csv)
Returns:
(list) list of au column names
"""
cols_lmk = []
au_tags = ' AU'
cols_au = [c for c in cols if au_tags in c]
return cols_au
def au_col_nm_map(df):
"""
Rename dataframe action unit column names to match functional specifications v1.0
Args:
df: dataframe
Returns:
dataframe with mapped variables
"""
dict_au_cols = {}
for col in list(df):
if ' AU' in col:
idx = col.rfind('_')
if idx > -1:
au_id = col[idx-2:idx]
if '_r' in col:
dict_au_cols[col] = 'fac_AU' + au_id + 'int'
if '_c' in col:
dict_au_cols[col] = 'fac_AU' + au_id + 'pres'
df.rename(columns=dict_au_cols, inplace=True)
return df
def run_face_au(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching action units
---------------
---------------
Args:
video_uri: video path; f_cfg: face config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_au = df_of[extract_col_nm_au(df_of)]
df_au = df_au.copy()
df_au['frame'] = df_of['frame']
df_au['face_id'] = df_of[' face_id']
df_au['timestamp'] = df_of[' timestamp']
df_au['confidence'] = df_of[' confidence']
df_au['success'] = df_of[' success']
df_au = au_col_nm_map(df_au)
df_au['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(df_au, out_loc, fl_name, face_au_dir, csv_ext)

137
dbm_lib/dbm_features/raw_features/video/face_config/face_config_reader.py Normal file
View File

@@ -0,0 +1,137 @@
"""
file_name: face_config_reader
project_name: DBM
created: 2020-20-07
"""
import yaml
import boto3
from dbm_lib.dbm_features.raw_features.video import DBMLIB_FACE_CONFIG
class ConfigFaceReader(object):
"""Summary
Read sevice end ponit
"""
def __init__(self,
service_config_yml=None):
"""Summary
Args:
service_config_yml (None, optional): yml file defined service configuration
"""
if service_config_yml is None:
service_config = DBMLIB_FACE_CONFIG
else:
service_config = service_config_yml
with open(service_config, 'r') as ymlfile:
config = yaml.load(ymlfile)
self.ACTION_UNITS = config['cdx_face_config']['ACTION_UNITS']
self.NEG_ACTION_UNITS = config['cdx_face_config']['NEG_ACTION_UNITS']
self.POS_ACTION_UNITS = config['cdx_face_config']['POS_ACTION_UNITS']
self.NET_ACTION_UNITS = config['cdx_face_config']['NET_ACTION_UNITS']
self.happiness = config['cdx_face_config']['happiness']
self.sadness = config['cdx_face_config']['sadness']
self.surprise = config['cdx_face_config']['surprise']
self.fear = config['cdx_face_config']['fear']
self.anger = config['cdx_face_config']['anger']
self.disgust = config['cdx_face_config']['disgust']
self.contempt = config['cdx_face_config']['contempt']
self.cai = config['cdx_face_config']['CAI']
self.SELECTED_FEATURES = config['cdx_face_config']['SELECTED_FEATURES'].split(',')
self.face_expr_dir = config['cdx_face_config']['face_expr_dir']
self.face_asym_dir = config['cdx_face_config']['face_asym_dir']
self.AU_fl = config['cdx_face_config']['AU_filters']
self.au_int = config['cdx_face_config']['au_intensity']
self.au_prs = config['cdx_face_config']['au_presence']
def get_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.ACTION_UNITS
def get_neg_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.NEG_ACTION_UNITS
def get_pos_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.POS_ACTION_UNITS
def get_net_action_unit(self):
"""Summary
Returns:
TYPE: end point
"""
return self.NET_ACTION_UNITS
def get_selected_feature(self):
"""Summary
Returns:
TYPE: end point
"""
return self.SELECTED_FEATURES
def get_happiness(self):
"""Summary
Returns:
TYPE: end point
"""
return self.happiness
def get_sadness(self):
"""Summary
Returns:
TYPE: end point
"""
return self.sadness
def get_surprise(self):
"""Summary
Returns:
TYPE: end point
"""
return self.surprise
def get_fear(self):
"""Summary
Returns:
TYPE: end point
"""
return self.fear
def get_anger(self):
"""Summary
Returns:
TYPE: end point
"""
return self.anger
def get_disgust(self):
"""Summary
Returns:
TYPE: end point
"""
return self.disgust
def get_contempt(self):
"""Summary
Returns:
TYPE: end point
"""
return self.contempt
def get_cai(self):
"""Summary
Returns:
TYPE: end point
"""
return self.cai

78
dbm_lib/dbm_features/raw_features/video/face_emotion_expressivity.py Normal file
View File

@@ -0,0 +1,78 @@
"""
file_name: process_emotion_expressivity
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_expr_dir = 'video/face_expressivity'
csv_ext = '_face_expressivity.csv'
#Openface feature extraction
def of_feature(df_of, cfr, f_cfg):
"""
Creating dataframe for face expressivity
Args:
of: open face attributes
Returns:
(list) list of expressivity score for emotions
"""
df_list = []
df_of['s_confidence'] = vu.smooth(df_of[' confidence'].values, window='flat').tolist()
if 'AU' in cfr.SELECTED_FEATURES :
vu.calc_of_for_video(df_of, cfr, f_cfg)
#Normalizing facial expressivity for Composite and Negative expr(Range 0 to 1)
if len(df_of[f_cfg.neg_exp])>0:
df_of[f_cfg.neg_exp] = df_of[f_cfg.neg_exp]/5
if len(df_of[f_cfg.com_exp])>0:
df_of[f_cfg.com_exp] = df_of[f_cfg.com_exp]/7
if len(df_of[f_cfg.com_exp_full])>0:
df_of[f_cfg.com_exp_full] = df_of[f_cfg.com_exp_full]/7
df_list.append(df_of)
return df_list
def run_face_expressivity(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching facial landmarks
---------------
---------------
Args:
video_uri: video path; f_cfg: raw variable config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_of = df_of[cfr.AU_fl]
expr_df_list = of_feature(df_of, cfr, f_cfg)
exp_final_df = pd.concat(expr_df_list, ignore_index=True)
exp_final_df['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(os.path.join(out_loc, fl_name)))
ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext)

118
dbm_lib/dbm_features/raw_features/video/face_landmark.py Normal file
View File

@@ -0,0 +1,118 @@
"""
file_name: face_landmark
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import datetime
import glob
from os.path import join
import logging
from dbm_lib.dbm_features.raw_features.video.face_config.face_config_reader import ConfigFaceReader
from dbm_lib.dbm_features.raw_features.util import video_util as vu
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
face_lmk_dir = 'video/face_landmark'
csv_ext = '_face_landmark.csv'
def extract_col_nm_lmk(cols):
"""
Extract landmark column names from openface output (csv)
Args:
cols: column names from open face output (csv)
Returns:
(list) list of landmark column names
"""
cols_lmk = []
lmk_tags = [' y_', ' x_', ' X_', ' Y_', ' Z_']
for c in cols:
if any(t in c for t in lmk_tags):
cols_lmk.append(c)
return cols_lmk
def lmk_col_nm_map(df):
"""
Rename dataframe landmark column names to match functional specifications v1.0
Args:
df: dataframe
"""
dict_lmk_cols = {}
for col in list(df):
idx = col.rfind('_')+1
if idx > 0:
lmk_id = col[idx:] if len(col[idx:])>1 else '0'+col[idx:]
if ' y_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'r'
if ' x_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'c'
if ' X_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'X'
if ' Y_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Y'
if ' Z_' in col:
dict_lmk_cols[col] = 'fac_LMK' + lmk_id + 'Z'
df.rename(columns=dict_lmk_cols, inplace=True)
return df
def add_disp_3D(df):
"""
Add 3D displacement for each landmark
Args:
df: landmark dataframe
"""
df = df.sort_values(by=['frame'], ascending=False)
cols_lmk = [col for col in list(df) if 'fac_LMK' in col]
df_t = df[cols_lmk]
df_diff = df_t.diff()
df_diff = df_diff.pow(2)
tot_lmk = 68 # 68 landmark model
for i in range(tot_lmk):
lmk_id = '{:02d}'.format(i)
df['fac_LMK'+lmk_id+'disp'] = df_diff[['fac_LMK'+lmk_id+'X', 'fac_LMK'+lmk_id+'Y', 'fac_LMK'+lmk_id+'Z']].sum(axis=1).apply(np.sqrt)
return df
def run_face_landmark(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching facial landmarks
---------------
---------------
Args:
video_uri: video path; f_cfg: raw variable config object
out_dir: (str) Output directory for processed output
"""
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path)>0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_lmk = df_of[extract_col_nm_lmk(df_of)]
df_lmk = df_lmk.copy()
df_lmk['frame'] = df_of['frame']
df_lmk['face_id'] = df_of[' face_id']
df_lmk['timestamp'] = df_of[' timestamp']
df_lmk['confidence'] = df_of[' confidence']
df_lmk['success'] = df_of[' success']
df_lmk = lmk_col_nm_map(df_lmk)
df_lmk = add_disp_3D(df_lmk)
df_lmk['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(join(out_loc, fl_name)))
ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)

67
dbm_lib/dbm_features/raw_features/video/open_face_process.py Normal file
View File

@@ -0,0 +1,67 @@
"""
file_name: process_features
project_name: DBM
created: 2020-20-07
"""
import os
import numpy as np
import pandas as pd
import glob
import logging
from dbm_lib.dbm_features.raw_features.util import util as ut
logging.basicConfig(level=logging.INFO)
logger=logging.getLogger()
def batch_open_face(filepaths,video_url, input_dir, out_dir, of_path):
""" Computes open_face features for the files in filepaths
Args:
-----
filepaths: (itreable[str])
video_tracking: To specify whether openface's video tracking module (FaceLandmarkVid)
is being used or the default (FeatureExtract)
video_url: Raw video location on S3 bucket
input_dir: Path to the input videos
out_dir: Path to the processed output
of_path: OpenFace source code path
Returns:
--------
(itreable[str]) list of .csv files
"""
suffix = '_OF_features'
csv_files = []
for fp in filepaths:
try:
_, out_loc, fl_name = ut.filter_path(video_url, out_dir)
full_f_name = fl_name + suffix
output_directory = os.path.join(out_loc, full_f_name)
csv_files.append(ut.compute_open_face_features(fp,output_directory,of_path))
except Exception as e:
logger.error('Failed to run OpenFace on {}\n{}'.format(fp, e))
return csv_files
def process_open_face(video_uri, input_dir, out_dir, of_path, dbm_group):
"""
Processing all patient's for fetching emotion expressivity
-------------------
-------------------
Args:
video_uri: video path; input_dir : input directory for video's; dbm_group: feature group
out_dir: (str) Output directory for processed output; of_path: OpenFace source code path
"""
if dbm_group != None and len(dbm_group) == 1 and 'acoustic' in dbm_group:
return
filepaths = [video_uri]
csv_filepaths = batch_open_face(filepaths, video_uri, input_dir, out_dir, of_path)

10
pkg/OpenFace/.dockerignore Normal file
View File

@@ -0,0 +1,10 @@
.git
docker-compose.yml
build
matlab_runners
matlab_version
python_scripts
test-dump
samples
model_training
gui

6
pkg/OpenFace/.env Normal file
View File

@@ -0,0 +1,6 @@
## This is read by docker-compose. You can overwrite these at runtime, e.g.:
## DOCKERUSER=bobfoo docker-compose build
DOCKERUSER=openface
DOCKERTAG=latest
DATA_MOUNT=/tmp/openface

93
pkg/OpenFace/.travis.yml Normal file
View File

@@ -0,0 +1,93 @@
language: cpp
dist: xenial
sudo: required
branches:
only:
- master
- develop
- feature/opencv4
compiler:
- gcc
os:
- linux
- osx
before_install:
# G++ 8 compiler, OpenCV dependencies, dlib, openblas, and tbb
- if [ ${TRAVIS_OS_NAME} = linux ]; then
cd ../;
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test;
sudo apt-get update -qq;
sudo apt-get install -qq g++-8;
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90;
sudo apt-get install git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev;
sudo apt-get install python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev;
sudo apt-get install cmake;
sudo apt-get install libopenblas-dev;
wget http://dlib.net/files/dlib-19.13.tar.bz2;
tar xf dlib-19.13.tar.bz2;
cd dlib-19.13;
mkdir build;
cd build;
cmake ..;
cmake --build . --config Release;
sudo make install;
sudo ldconfig;
cd ../..;
wget https://github.com/opencv/opencv/archive/4.0.0.zip;
unzip -qq 4.0.0.zip;
cd opencv-4.0.0;
mkdir build;
cd build;
fi
# g++ TODO these should not be separated?
- if [ "$CXX" = "g++" ]; then
if [ ${TRAVIS_OS_NAME} = linux ]; then
$CXX --version;
sudo cmake -D CMAKE_BUILD_TYPE=RELEASE -D WITH_V4L=ON -D WITH_OPENCL=OFF -D INSTALL_C_EXAMPLES=OFF -D WITH_TBB=ON -D BUILD_EXAMPLES=OFF -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF ..;
sudo make -j4;
sudo make install;
cd ../..;
fi
fi
# clang
- if [ "$CXX" = "clang++" ]; then
if [ ${TRAVIS_OS_NAME} = linux ]; then
$CXX --version;
sudo cmake -D CMAKE_BUILD_TYPE=RELEASE -D WITH_V4L=ON -D WITH_OPENCL=OFF -D INSTALL_C_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF -D WITH_TBB=ON ..;
sudo make -j4;
sudo make install;
cd ../..;
fi
fi
- if [ ${TRAVIS_OS_NAME} = osx ]; then
brew update;
brew install tbb;
brew install openblas;
brew install dlib;
brew install opencv;
cd ../;
fi
script:
- cd OpenFace
- $CXX --version
- chmod 777 ./download_models.sh
- ./download_models.sh
- mkdir build
- cd build
- cmake -D CMAKE_BUILD_TYPE=RELEASE ..
- make
- ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -q -mloc model/main_clm_general.txt
- ../build/bin/FaceLandmarkImg -fdir ../samples -out_dir data -multi_view 1 -wild -q
- ../build/bin/FaceLandmarkImg -f ../samples/sample1.jpg -out_dir data -multi_view 1 -wild -q
- ../build/bin/FeatureExtraction -fdir "../samples/image_sequence" -out_dir output -q
- ../build/bin/FaceLandmarkVidMulti -fdir ../samples/image_sequence -q -mloc model/main_clm_general.txt
- ../build/bin/FaceLandmarkVidMulti -f ../samples/multi_face.avi -q -mloc model/main_clnf_general.txt

247
pkg/OpenFace/CMakeLists.txt Normal file
View File

@@ -0,0 +1,247 @@
cmake_minimum_required (VERSION 3.8)
set(CMAKE_CXX_STANDARD 17)
project(OpenFace VERSION 2.0.2)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/")
set(CMAKE_CONFIG_DIR etc/OpenFace)
set(CONFIG_DIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_CONFIG_DIR}")
add_definitions(-DCONFIG_DIR="${CONFIG_DIR}")
# make sure we'll use OpenBLAS only: there's a header file naming difference between different
# implementations; so OpenFace wants OpenBLAS;
find_package(OpenBLAS REQUIRED)
if ( ${OpenBLAS_FOUND} )
MESSAGE("OpenBLAS information:")
MESSAGE(" OpenBLAS_LIBRARIES: ${OpenBLAS_LIB}")
else()
MESSAGE(FATAL_ERROR "OpenBLAS not found in the system.")
endif()
if ( ${OpenBLAS_INCLUDE_FOUND} )
MESSAGE(" OpenBLAS_INCLUDE: ${OpenBLAS_INCLUDE_DIR}")
else()
MESSAGE(WARNING "OpenBLAS include not found in the system. Using the one vended with OpenFace.")
set(OpenBLAS_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/lib/3rdParty/OpenBLAS/include")
MESSAGE(" OpenBLAS_INCLUDE: ${OpenBLAS_INCLUDE_DIR}")
endif()
find_package( OpenCV 4.0 REQUIRED COMPONENTS core imgproc calib3d highgui objdetect)
if(${OpenCV_FOUND})
MESSAGE("OpenCV information:")
MESSAGE(" OpenCV_INCLUDE_DIRS: ${OpenCV_INCLUDE_DIRS}")
MESSAGE(" OpenCV_LIBRARIES: ${OpenCV_LIBRARIES}")
MESSAGE(" OpenCV_LIBRARY_DIRS: ${OpenCV_LINK_DIRECTORIES}")
else()
MESSAGE(FATAL_ERROR "OpenCV not found in the system.")
endif()
find_package( Boost 1.5.9 COMPONENTS filesystem system)
if(${Boost_FOUND})
MESSAGE("Boost information:")
MESSAGE(" Boost_VERSION: ${Boost_VERSION}")
MESSAGE(" Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
MESSAGE(" Boost_LIBRARIES: ${Boost_LIBRARIES}")
MESSAGE(" Boost_LIBRARY_DIRS: ${Boost_LIBRARY_DIRS}")
else()
MESSAGE("Boost not found in the system.")
endif()
# Move LandmarkDetector model
file(GLOB files "lib/local/LandmarkDetector/model/*.txt")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model)
endforeach()
# Move the hierarchical LandmarkDetector models
file(GLOB files "lib/local/LandmarkDetector/model/model*")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model)
install(DIRECTORY ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model)
endforeach()
# Move detection validation models
file(GLOB files "lib/local/LandmarkDetector/model/detection_validation/*.txt")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model/detection_validation)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model/detection_validation)
endforeach()
# Move patch experts
file(GLOB files "lib/local/LandmarkDetector/model/patch_experts/*.txt")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model/patch_experts)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model/patch_experts)
endforeach()
# Move CEN patch experts
file(GLOB files "lib/local/LandmarkDetector/model/patch_experts/*.dat")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model/patch_experts)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model/patch_experts)
endforeach()
# Move MTCNN face detector
file(GLOB files "lib/local/LandmarkDetector/model/mtcnn_detector/*.txt")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model/mtcnn_detector)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model/mtcnn_detector)
endforeach()
# Move MTCNN face detector
file(GLOB files "lib/local/LandmarkDetector/model/mtcnn_detector/*.dat")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model/mtcnn_detector)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model/mtcnn_detector)
endforeach()
# Move Point Distribution models
file(GLOB files "lib/local/LandmarkDetector/model/pdms/*.txt")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model/pdms)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model/pdms)
endforeach()
# Move OpenCV classifiers
file(GLOB files "lib/3rdParty/OpenCV3.4/classifiers/*.xml")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/classifiers)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/classifiers)
endforeach()
# Move AU prediction modules
file(GLOB files "lib/local/FaceAnalyser/AU_predictors/*.txt")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/AU_predictors)
install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/AU_predictors)
endforeach()
# Move AU prediction modules
file(GLOB files "lib/local/FaceAnalyser/AU_predictors/svr*")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/AU_predictors)
install(DIRECTORY ${file} DESTINATION ${CMAKE_CONFIG_DIR}/AU_predictors)
endforeach()
# Move AU prediction modules
file(GLOB files "lib/local/FaceAnalyser/AU_predictors/svm*")
foreach(file ${files})
file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/AU_predictors)
install(DIRECTORY ${file} DESTINATION ${CMAKE_CONFIG_DIR}/AU_predictors)
endforeach()
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
if (GCC_VERSION VERSION_LESS 8.0)
MESSAGE(FATAL_ERROR "Need a 8.0 or newer GCC compiler. Current GCC: ${GCC_VERSION}")
else ()
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse -msse2 -msse3")
endif ()
endif ()
# dlib
find_package(dlib 19.13)
if(${dlib_FOUND})
message("dlib information:")
message(" dlib version: ${dlib_VERSION}")
if (NOT TARGET dlib)
add_library(dlib INTERFACE IMPORTED GLOBAL)
endif()
else()
message(FATAL_ERROR "dlib not found in the system, please install dlib")
endif()
# suppress auto_ptr deprecation warnings
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
add_compile_options("-Wno-deprecated-declarations")
endif()
# LandmarkDetector library
add_subdirectory(lib/local/LandmarkDetector)
# Facial Expression analysis library
add_subdirectory(lib/local/FaceAnalyser)
# Gaze estimation library
add_subdirectory(lib/local/GazeAnalyser)
# Utilities library
add_subdirectory(lib/local/Utilities)
# test if this file is a top list file
# thus we're building an OpenFace as a standalone
# project; otherwise OpenFace is being built as a
# part or larger tree;
if(CMAKE_CURRENT_SOURCE_DIR STREQUAL "${CMAKE_SOURCE_DIR}")
# for a standalone builds - allow installing package configs;
message(STATUS "Standalone mode detected; Enabling configuration/targets export.")
# export libraries for reuse
include(CMakePackageConfigHelpers)
set(LIB_INSTALL_DIR lib)
set(CONFIG_DEST_DIR ${LIB_INSTALL_DIR}/cmake/OpenFace/)
set(OpenFace_LIBRARIES OpenFace::GazeAnalyser OpenFace::FaceAnalyser OpenFace::LandmarkDetector OpenFace::Utilities)
# export targets [build tree]
export(EXPORT OpenFaceTargets
NAMESPACE OpenFace::
FILE "${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_DEST_DIR}/OpenFaceTargets.cmake")
# write package version file
write_basic_package_version_file(
"${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_DEST_DIR}/OpenFaceConfigVersion.cmake"
COMPATIBILITY AnyNewerVersion)
# define [build tree] bindir relative include dir
foreach(lib ${OpenFace_LIBRARIES})
if(TARGET ${lib})
get_target_property(libname ${lib} "NAME")
file(RELATIVE_PATH rel_incdir ${CMAKE_CURRENT_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/lib/local/${libname}/include")
list(APPEND OPENFACE_INCLUDE_DIRS ${rel_incdir})
endif()
endforeach()
list(REMOVE_DUPLICATES OPENFACE_INCLUDE_DIRS)
# write package config file from template [build tree]
# all PATH_VARS should be relative to a ${CMAKE_CURRENT_BINARY_DIR}
# as it's the "prefix" of our non installed package in the build tree
configure_package_config_file(cmake/OpenFaceConfig.cmake.in
"${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_DEST_DIR}/OpenFaceConfig.cmake"
INSTALL_DESTINATION ${CONFIG_DEST_DIR}
PATH_VARS OPENFACE_INCLUDE_DIRS)
# store current build dir in the CMake package registry
# export(PACKAGE OpenFace)
# install exported targets [install tree]
install(EXPORT OpenFaceTargets
FILE OpenFaceTargets.cmake
NAMESPACE OpenFace::
DESTINATION ${CONFIG_DEST_DIR})
# redefine [install tree] prefix relative include dir
set(OPENFACE_INCLUDE_DIRS "include/OpenFace")
# write package config file from template [install tree]
configure_package_config_file(cmake/OpenFaceConfig.cmake.in
"${CMAKE_CURRENT_BINARY_DIR}/OpenFace/OpenFaceConfig.cmake"
INSTALL_DESTINATION ${CONFIG_DEST_DIR}
PATH_VARS OPENFACE_INCLUDE_DIRS)
# install package configs
install(FILES
"${CMAKE_CURRENT_BINARY_DIR}/OpenFace/OpenFaceConfig.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_DEST_DIR}/OpenFaceConfigVersion.cmake"
DESTINATION ${CONFIG_DEST_DIR})
endif()
# executables
add_subdirectory(exe/FaceLandmarkImg)
add_subdirectory(exe/FaceLandmarkVid)
add_subdirectory(exe/FaceLandmarkVidMulti)
add_subdirectory(exe/FeatureExtraction)

32
pkg/OpenFace/Copyright.txt Normal file
View File

@@ -0,0 +1,32 @@
Copyright (C) 2017, University of Southern California, University of Cambridge, and Carnegie Mellon University, all rights reserved
ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Notwithstanding the license granted herein, Licensee acknowledges that certain components of the Software may be covered by so-called “open source” software licenses (“Open Source Components”), which means any software licenses approved as open source licenses by the Open Source Initiative or any substantially similar licenses, including without limitation any license that, as a condition of distribution of the software licensed under such license, requires that the distributor make the software available in source code format. Carnegie Mellon shall provide a list of Open Source Components for a particular version of the Software upon Licensee’s request. Licensee will comply with the applicable terms of such licenses and to the extent required by the licenses covering Open Source Components, the terms of such licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the licenses applicable to Open Source Components prohibit any of the restrictions in this License Agreement with respect to such Open Source Component, such restrictions will not apply to such Open Source Component. To the extent the terms of the licenses applicable to Open Source Components require Carnegie Mellon to make an offer to provide source code or related information in connection with the Software, such offer is hereby made. Any request for source code or related information should be directed to Tadas Baltrusaitis. Licensee acknowledges receipt of notices for the Open Source Components for the initial delivery of the Software.
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////

50
pkg/OpenFace/OpenFace-license.txt Normal file
View File

@@ -0,0 +1,50 @@
OPENFACE
SOFTWARE LICENSE AGREEMENT
ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
This is a license agreement ("Agreement") between your academic institution or non-profit organization or self (called "Licensee" or "You" in this Agreement) and Carnegie Mellon University (called "Licensor" in this Agreement). All rights not specifically granted to you in this Agreement are reserved for Licensor.
RESERVATION OF OWNERSHIP AND GRANT OF LICENSE:
Licensor retains exclusive ownership of any copy of the Software (as defined below) licensed under this Agreement and hereby grants to Licensee a personal, non-exclusive, non-transferable license to use the Software for noncommercial research purposes, without the right to sublicense, pursuant to the terms and conditions of this Agreement. As used in this Agreement, the term "Software" means (i) the actual copy of all or any portion of code for program routines made accessible to Licensee by Licensor pursuant to this Agreement, inclusive of backups, updates, and/or merged copies permitted hereunder or subsequently supplied by Licensor, including all or any file structures, programming instructions, user interfaces and screen formats and sequences as well as any and all documentation and instructions related to it, and (ii) all or any derivatives and/or modifications created or made by You to any of the items specified in (i).
CONFIDENTIALITY: Licensee acknowledges that the Software is proprietary to Licensor, and as such, Licensee agrees to receive all such materials in confidence and use the Software only in accordance with the terms of this Agreement. Licensee agrees to use reasonable effort to protect the Software from unauthorized use, reproduction, distribution, or publication.
COPYRIGHT: The Software is owned by Licensor and is protected by United
States copyright laws and applicable international treaties and/or conventions.
PERMITTED USES: The Software may be used for your own noncommercial internal research purposes. You understand and agree that Licensor is not obligated to implement any suggestions and/or feedback you might provide regarding the Software, but to the extent Licensor does so, you are not entitled to any compensation related thereto.
DERIVATIVES: You may create derivatives of or make modifications to the Software, however, You agree that all and any such derivatives and modifications will be owned by Licensor and become a part of the Software licensed to You under this Agreement. You may only use such derivatives and modifications for your own noncommercial internal research purposes, and you may not otherwise use, distribute or copy such derivatives and modifications in violation of this Agreement. You must provide to Licensor one copy of all such derivatives and modifications in a recognized electronic format by way of electronic mail sent to Tadas Baltrusaitis at tadyla@gmail.com within thirty (30) days of the publication date of any publication that relates to any such derivatives or modifications. You understand that Licensor is not obligated to distribute or otherwise make available any derivatives or modifications provided by You.
BACKUPS: If Licensee is an organization, it may make that number of copies of the Software necessary for internal noncommercial use at a single site within its organization provided that all information appearing in or on the original labels, including the copyright and trademark notices are copied onto the labels of the copies.
USES NOT PERMITTED: You may not distribute, copy or use the Software except as explicitly permitted herein. Licensee has not been granted any trademark license as part of this Agreement and may not use the name or mark "OPENFACE", "Carnegie Mellon" or any renditions thereof without the prior written permission of Licensor.
You may not sell, rent, lease, sublicense, lend, time-share or transfer, in whole or in part, or provide third parties access to prior or present versions (or any parts thereof) of the Software.
ASSIGNMENT: You may not assign this Agreement or your rights hereunder without the prior written consent of Licensor. Any attempted assignment without such consent shall be null and void.
TERM: The term of the license granted by this Agreement is from Licensee's acceptance of this Agreement by clicking "I Agree" below or by using the Software until terminated as provided below.
The Agreement automatically terminates without notice if you fail to comply with any provision of this Agreement. Licensee may terminate this Agreement by ceasing using the Software. Upon any termination of this Agreement, Licensee will delete any and all copies of the Software. You agree that all provisions which operate to protect the proprietary rights of Licensor shall remain in force should breach occur and that the obligation of confidentiality described in this Agreement is binding in perpetuity and, as such, survives the term of the Agreement.
FEE: Provided Licensee abides completely by the terms and conditions of this Agreement, there is no fee due to Licensor for Licensee's use of the Software in accordance with this Agreement.
DISCLAIMER OF WARRANTIES: THE SOFTWARE IS PROVIDED "AS-IS" WITHOUT WARRANTY OF ANY KIND INCLUDING ANY WARRANTIES OF PERFORMANCE OR MERCHANTABILITY OR FITNESS FOR A PARTICULAR USE OR PURPOSE OR OF NON-INFRINGEMENT. LICENSEE BEARS ALL RISK RELATING TO QUALITY AND PERFORMANCE OF THE SOFTWARE AND RELATED MATERIALS.
SUPPORT AND MAINTENANCE: No Software support or training by the Licensor is provided as part of this Agreement.
EXCLUSIVE REMEDY AND LIMITATION OF LIABILITY: To the maximum extent permitted under applicable law, Licensor shall not be liable for direct, indirect, special, incidental, or consequential damages or lost profits related to Licensee's use of and/or inability to use the Software, even if Licensor is advised of the possibility of such damage.
EXPORT REGULATION: Licensee agrees to comply with any and all applicable
U.S. export control laws, regulations, and/or other laws related to embargoes and sanction programs administered by the Office of Foreign Assets Control.
SEVERABILITY: If any provision(s) of this Agreement shall be held to be invalid, illegal, or unenforceable by a court or other tribunal of competent jurisdiction, the validity, legality and enforceability of the remaining provisions shall not in any way be affected or impaired thereby.
NO IMPLIED WAIVERS: No failure or delay by Licensor in enforcing any right or remedy under this Agreement shall be construed as a waiver of any future or other exercise of such right or remedy by Licensor.
GOVERNING LAW: This Agreement shall be construed and enforced in accordance with the laws of the Commonwealth of Pennsylvania without reference to conflict of laws principles. You consent to the personal jurisdiction of the courts of this County and waive their rights to venue outside of Allegheny County, Pennsylvania.
ENTIRE AGREEMENT AND AMENDMENTS: This Agreement constitutes the sole and entire agreement between Licensee and Licensor as to the matter set forth herein and supersedes any previous agreements, understandings, and arrangements between the parties relating hereto.

185
pkg/OpenFace/OpenFace.sln Normal file
View File

@@ -0,0 +1,185 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 17
VisualStudioVersion = 15.8.2
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LandmarkDetector", "lib\local\LandmarkDetector\LandmarkDetector.vcxproj", "{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceAnalyser", "lib\local\FaceAnalyser\FaceAnalyser.vcxproj", "{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FeatureExtraction", "exe\FeatureExtraction\FeatureExtraction.vcxproj", "{8A23C00D-767D-422D-89A3-CF225E3DAB4B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Libraries", "Libraries", "{99FEBA13-BDDF-4076-B57E-D8EF4076E20D}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Executables", "Executables", "{9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkVidMulti", "exe\FaceLandmarkVidMulti\FaceLandmarkVidMulti.vcxproj", "{C3FAF36F-44BC-4454-87C2-C5106575FE50}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Recording", "exe\Recording\Recording.vcxproj", "{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkVid", "exe\FaceLandmarkVid\FaceLandmarkVid.vcxproj", "{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkImg", "exe\FaceLandmarkImg\FaceLandmarkImg.vcxproj", "{DDC3535E-526C-44EC-9DF4-739E2D3A323B}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GazeAnalyser", "lib\local\GazeAnalyser\GazeAnalyser.vcxproj", "{5F915541-F531-434F-9C81-79F5DB58012B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "UtilLibs", "UtilLibs", "{652CCE53-4997-4B43-9A99-28D075199C99}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Utilities", "lib\local\Utilities\Utilities.vcxproj", "{8E741EA2-9386-4CF2-815E-6F9B08991EAC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gui", "gui", "{E59CF005-539F-484F-9AA6-9F08AC2DB31E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeadPoseLive", "gui\HeadPose-live\HeadPoseLive.csproj", "{F396362D-821E-4EA6-9BBF-1F6050844118}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenFaceDemo", "gui\OpenFaceDemo\OpenFaceDemo.csproj", "{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenFaceOffline", "gui\OpenFaceOffline\OpenFaceOffline.csproj", "{A4760F41-2B1F-4144-B7B2-62785AFFE79B}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CppInerop", "lib\local\CppInerop\CppInerop.vcxproj", "{78196985-EE54-411F-822B-5A23EDF80642}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CameraEnumerator", "lib\3rdParty\CameraEnumerator\CameraEnumerator.vcxproj", "{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Debug|Win32.ActiveCfg = Debug|Win32
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Debug|Win32.Build.0 = Debug|Win32
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Debug|x64.ActiveCfg = Debug|x64
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Debug|x64.Build.0 = Debug|x64
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Release|Win32.ActiveCfg = Release|Win32
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Release|Win32.Build.0 = Release|Win32
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Release|x64.ActiveCfg = Release|x64
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8}.Release|x64.Build.0 = Release|x64
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Debug|Win32.ActiveCfg = Debug|Win32
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Debug|Win32.Build.0 = Debug|Win32
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Debug|x64.ActiveCfg = Debug|x64
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Debug|x64.Build.0 = Debug|x64
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Release|Win32.ActiveCfg = Release|Win32
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Release|Win32.Build.0 = Release|Win32
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Release|x64.ActiveCfg = Release|x64
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7}.Release|x64.Build.0 = Release|x64
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Debug|Win32.ActiveCfg = Debug|Win32
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Debug|Win32.Build.0 = Debug|Win32
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Debug|x64.ActiveCfg = Debug|x64
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Debug|x64.Build.0 = Debug|x64
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Release|Win32.ActiveCfg = Release|Win32
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Release|Win32.Build.0 = Release|Win32
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Release|x64.ActiveCfg = Release|x64
{8A23C00D-767D-422D-89A3-CF225E3DAB4B}.Release|x64.Build.0 = Release|x64
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Debug|Win32.ActiveCfg = Debug|Win32
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Debug|Win32.Build.0 = Debug|Win32
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Debug|x64.ActiveCfg = Debug|x64
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Debug|x64.Build.0 = Debug|x64
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Release|Win32.ActiveCfg = Release|Win32
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Release|Win32.Build.0 = Release|Win32
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Release|x64.ActiveCfg = Release|x64
{C3FAF36F-44BC-4454-87C2-C5106575FE50}.Release|x64.Build.0 = Release|x64
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Debug|Win32.ActiveCfg = Debug|Win32
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Debug|Win32.Build.0 = Debug|Win32
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Debug|x64.ActiveCfg = Debug|x64
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Debug|x64.Build.0 = Debug|x64
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Release|Win32.ActiveCfg = Release|Win32
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Release|Win32.Build.0 = Release|Win32
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Release|x64.ActiveCfg = Release|x64
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}.Release|x64.Build.0 = Release|x64
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Debug|Win32.ActiveCfg = Debug|Win32
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Debug|Win32.Build.0 = Debug|Win32
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Debug|x64.ActiveCfg = Debug|x64
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Debug|x64.Build.0 = Debug|x64
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Release|Win32.ActiveCfg = Release|Win32
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Release|Win32.Build.0 = Release|Win32
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Release|x64.ActiveCfg = Release|x64
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}.Release|x64.Build.0 = Release|x64
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Debug|Win32.ActiveCfg = Debug|Win32
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Debug|Win32.Build.0 = Debug|Win32
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Debug|x64.ActiveCfg = Debug|x64
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Debug|x64.Build.0 = Debug|x64
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|Win32.ActiveCfg = Release|Win32
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|Win32.Build.0 = Release|Win32
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|x64.ActiveCfg = Release|x64
{DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|x64.Build.0 = Release|x64
{5F915541-F531-434F-9C81-79F5DB58012B}.Debug|Win32.ActiveCfg = Debug|Win32
{5F915541-F531-434F-9C81-79F5DB58012B}.Debug|Win32.Build.0 = Debug|Win32
{5F915541-F531-434F-9C81-79F5DB58012B}.Debug|x64.ActiveCfg = Debug|x64
{5F915541-F531-434F-9C81-79F5DB58012B}.Debug|x64.Build.0 = Debug|x64
{5F915541-F531-434F-9C81-79F5DB58012B}.Release|Win32.ActiveCfg = Release|Win32
{5F915541-F531-434F-9C81-79F5DB58012B}.Release|Win32.Build.0 = Release|Win32
{5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.ActiveCfg = Release|x64
{5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.Build.0 = Release|x64
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|Win32.ActiveCfg = Debug|Win32
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|Win32.Build.0 = Debug|Win32
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|x64.ActiveCfg = Debug|x64
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|x64.Build.0 = Debug|x64
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|Win32.ActiveCfg = Release|Win32
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|Win32.Build.0 = Release|Win32
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|x64.ActiveCfg = Release|x64
{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|x64.Build.0 = Release|x64
{F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|Win32.ActiveCfg = Debug|x86
{F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|Win32.Build.0 = Debug|x86
{F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|x64.ActiveCfg = Debug|Any CPU
{F396362D-821E-4EA6-9BBF-1F6050844118}.Debug|x64.Build.0 = Debug|Any CPU
{F396362D-821E-4EA6-9BBF-1F6050844118}.Release|Win32.ActiveCfg = Release|x86
{F396362D-821E-4EA6-9BBF-1F6050844118}.Release|Win32.Build.0 = Release|x86
{F396362D-821E-4EA6-9BBF-1F6050844118}.Release|x64.ActiveCfg = Release|Any CPU
{F396362D-821E-4EA6-9BBF-1F6050844118}.Release|x64.Build.0 = Release|Any CPU
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|Win32.ActiveCfg = Debug|x86
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|Win32.Build.0 = Debug|x86
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|x64.ActiveCfg = Debug|Any CPU
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Debug|x64.Build.0 = Debug|Any CPU
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|Win32.ActiveCfg = Release|x86
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|Win32.Build.0 = Release|x86
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|x64.ActiveCfg = Release|Any CPU
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}.Release|x64.Build.0 = Release|Any CPU
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Debug|Win32.ActiveCfg = Debug|x86
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Debug|Win32.Build.0 = Debug|x86
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Debug|x64.ActiveCfg = Debug|Any CPU
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Debug|x64.Build.0 = Debug|Any CPU
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|Win32.ActiveCfg = Release|x86
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|Win32.Build.0 = Release|x86
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|x64.ActiveCfg = Release|Any CPU
{A4760F41-2B1F-4144-B7B2-62785AFFE79B}.Release|x64.Build.0 = Release|Any CPU
{78196985-EE54-411F-822B-5A23EDF80642}.Debug|Win32.ActiveCfg = Debug|Win32
{78196985-EE54-411F-822B-5A23EDF80642}.Debug|Win32.Build.0 = Debug|Win32
{78196985-EE54-411F-822B-5A23EDF80642}.Debug|x64.ActiveCfg = Debug|x64
{78196985-EE54-411F-822B-5A23EDF80642}.Debug|x64.Build.0 = Debug|x64
{78196985-EE54-411F-822B-5A23EDF80642}.Release|Win32.ActiveCfg = Release|Win32
{78196985-EE54-411F-822B-5A23EDF80642}.Release|Win32.Build.0 = Release|Win32
{78196985-EE54-411F-822B-5A23EDF80642}.Release|x64.ActiveCfg = Release|x64
{78196985-EE54-411F-822B-5A23EDF80642}.Release|x64.Build.0 = Release|x64
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Debug|Win32.ActiveCfg = Debug|Win32
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Debug|Win32.Build.0 = Debug|Win32
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Debug|x64.ActiveCfg = Debug|x64
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Debug|x64.Build.0 = Debug|x64
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Release|Win32.ActiveCfg = Release|Win32
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Release|Win32.Build.0 = Release|Win32
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Release|x64.ActiveCfg = Release|x64
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{BDC1D107-DE17-4705-8E7B-CDDE8BFB2BF8} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D}
{0E7FC556-0E80-45EA-A876-DDE4C2FEDCD7} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D}
{8A23C00D-767D-422D-89A3-CF225E3DAB4B} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
{C3FAF36F-44BC-4454-87C2-C5106575FE50} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
{34032CF2-1B99-4A25-9050-E9C13DD4CD0A} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
{DDC3535E-526C-44EC-9DF4-739E2D3A323B} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
{5F915541-F531-434F-9C81-79F5DB58012B} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D}
{8E741EA2-9386-4CF2-815E-6F9B08991EAC} = {652CCE53-4997-4B43-9A99-28D075199C99}
{F396362D-821E-4EA6-9BBF-1F6050844118} = {E59CF005-539F-484F-9AA6-9F08AC2DB31E}
{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90} = {E59CF005-539F-484F-9AA6-9F08AC2DB31E}
{A4760F41-2B1F-4144-B7B2-62785AFFE79B} = {E59CF005-539F-484F-9AA6-9F08AC2DB31E}
{78196985-EE54-411F-822B-5A23EDF80642} = {652CCE53-4997-4B43-9A99-28D075199C99}
{50B7D4BF-E33B-41D0-AA89-76BBA57BF5CC} = {652CCE53-4997-4B43-9A99-28D075199C99}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {228609CD-6688-47E7-8D5B-5EE684F8A7A1}
EndGlobalSection
EndGlobal

96
pkg/OpenFace/README.md Normal file
View File

@@ -0,0 +1,96 @@
# OpenFace 2.2.0: a facial behavior analysis toolkit
[![Build Status](https://travis-ci.org/TadasBaltrusaitis/OpenFace.svg?branch=master)](https://travis-ci.org/TadasBaltrusaitis/OpenFace)
[![Build status](https://ci.appveyor.com/api/projects/status/8msiklxfbhlnsmxp/branch/master?svg=true)](https://ci.appveyor.com/project/TadasBaltrusaitis/openface/branch/master)
Over the past few years, there has been an increased interest in automatic facial behavior analysis
and understanding. We present OpenFace – a tool intended for computer vision and machine learning
researchers, affective computing community and people interested in building interactive
applications based on facial behavior analysis. OpenFace is the first toolkit capable of facial
landmark detection, head pose estimation, facial action unit recognition, and eye-gaze estimation
with available source code for both running and training the models. The computer vision algorithms
which represent the core of OpenFace demonstrate state-of-the-art results in all of the above
mentioned tasks. Furthermore, our tool is capable of real-time performance and is able to run from a
simple webcam without any specialist hardware.
![Multicomp logo](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/muticomp_logo_black.png)
OpenFace was originally developed by Tadas Baltrušaitis in collaboration with CMU MultiComp Lab led by Prof. Louis-Philippe Morency. Some of the original algorithms were created while at Rainbow Group, Cambridge University. The OpenFace library is still actively developed at the CMU MultiComp Lab in collaboration with Tadas Baltršaitis. Special thanks to researcher who helped developing, implementing and testing the algorithms present in OpenFace: Amir Zadeh and Yao Chong Lim on work on the CE-CLM model and Erroll Wood for the gaze estimation work.
## WIKI
**For instructions of how to install/compile/use the project please see [WIKI](https://github.com/TadasBaltrusaitis/OpenFace/wiki)**
## Functionality
The system is capable of performing a number of facial analysis tasks:
* Facial Landmark Detection
![Sample facial landmark detection image](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/multi_face_img.png)
* Facial Landmark and head pose tracking (links to YouTube videos)
<a href="https://www.youtube.com/watch?v=V7rV0uy7heQ" target="_blank"><img src="http://img.youtube.com/vi/V7rV0uy7heQ/0.jpg" alt="Multiple Face Tracking" width="240" height="180" border="10" /></a>
<a href="https://www.youtube.com/watch?v=vYOa8Pif5lY" target="_blank"><img src="http://img.youtube.com/vi/vYOa8Pif5lY/0.jpg" alt="Multiple Face Tracking" width="240" height="180" border="10" /></a>
* Facial Action Unit Recognition
<img src="https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/au_sample.png" height="280" width="600" >
* Gaze tracking (image of it in action)
<img src="https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/gaze_ex.png" height="182" width="600" >
* Facial Feature Extraction (aligned faces and HOG features)
![Sample aligned face and HOG image](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/appearance.png)
## Citation
If you use any of the resources provided on this page in any of your publications we ask you to cite the following work and the work for a relevant submodule you used.
#### Overall system
**OpenFace 2.0: Facial Behavior Analysis Toolkit**
Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency,
_IEEE International Conference on Automatic Face and Gesture Recognition_, 2018
#### Facial landmark detection and tracking
**Convolutional experts constrained local model for facial landmark detection**
A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency.
_Computer Vision and Pattern Recognition Workshops_, 2017
**Constrained Local Neural Fields for robust facial landmark detection in the wild**
Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency.
in IEEE Int. _Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge_, 2013.
#### Eye gaze tracking
**Rendering of Eyes for Eye-Shape Registration and Gaze Estimation**
Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
in _IEEE International Conference on Computer Vision (ICCV)_, 2015
#### Facial Action Unit detection
**Cross-dataset learning and person-specific normalisation for automatic Action Unit detection**
Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
in _Facial Expression Recognition and Analysis Challenge_,
_IEEE International Conference on Automatic Face and Gesture Recognition_, 2015
# Commercial license
For inquiries about the commercial licensing of the OpenFace toolkit please visit https://www.flintbox.com/public/project/50632/
# Final remarks
I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and a research project. If you encounter any problems/bugs/issues please contact me on github or by emailing me at tadyla@gmail.com for any bug reports/questions/suggestions. I prefer questions and bug reports on github as that provides visibility to others who might be encountering same issues or who have the same questions.
# Copyright
Copyright can be found in the Copyright.txt
You have to respect dlib, OpenBLAS, and OpenCV licenses.
Furthermore you have to respect the licenses of the datasets used for model training - https://github.com/TadasBaltrusaitis/OpenFace/wiki/Datasets

34
pkg/OpenFace/appveyor.yml Normal file
View File

@@ -0,0 +1,34 @@
version: 1.0.{build}
image: Visual Studio 2017
branches:
only:
- develop
- feature/opencv4
- master
- feature/boost_removal
max_jobs: 4
configuration:
- Release
- Debug
platform:
- x64
- Win32
# scripts that run after cloning repository
install:
- ps: '& ".\download_libraries.ps1"'
- ps: '& ".\download_models.ps1"'
build:
project: OpenFace.sln
verbosity: minimal
test_script:
# C++
- cmd: dir
- cmd: if exist x64 (cd x64)
- cmd: if exist Debug (cd Debug)
- cmd: if exist Release (cd Release)
- cmd: dir
- cmd: if exist "../samples" (FaceLandmarkImg.exe -inroot ../samples -f sample1.jpg -out_dir out_data -q) else (FaceLandmarkImg.exe -inroot ../../samples -f sample1.jpg -out_dir out_data -q)
- cmd: if exist "../samples" (FaceLandmarkVidMulti.exe -fdir ../samples/image_sequence -q -mloc model/main_clnf_general.txt) else (FaceLandmarkVidMulti.exe -fdir ../../samples/image_sequence -q -mloc model/main_clnf_general.txt)
- cmd: if exist "../samples" (FeatureExtraction.exe -fdir "../samples/image_sequence" -q) else (FeatureExtraction.exe -fdir "../../samples/image_sequence" -q)
- cmd: if exist "../samples" (FaceLandmarkVid.exe -f "../samples/default.wmv" -q) else (FaceLandmarkVid.exe -f "../../samples/default.wmv" -q)

18
pkg/OpenFace/cmake/OpenFaceConfig.cmake.in Normal file
View File

@@ -0,0 +1,18 @@
@PACKAGE_INIT@
include(CMakeFindDependencyMacro)
find_dependency(dlib 18.18)
find_dependency(OpenCV 3.3)
find_dependency(Boost)
find_dependency(BLAS)
find_dependency(TBB)
include("${CMAKE_CURRENT_LIST_DIR}/OpenFaceTargets.cmake")
set_and_check(OpenFace_INCLUDE_DIRS "@PACKAGE_OPENFACE_INCLUDE_DIRS@;
${dlib_INCLUDE_DIRS};
${Boost_INCLUDE_DIRS};
${Boost_INCLUDE_DIRS}/boost;")
set(OpenFace_LIBRARIES @OpenFace_LIBRARIES@)

86
pkg/OpenFace/cmake/modules/CMakePushCheckState.cmake Normal file
View File

@@ -0,0 +1,86 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
#.rst:
# CMakePushCheckState
# -------------------
#
#
#
# This module defines three macros: CMAKE_PUSH_CHECK_STATE()
# CMAKE_POP_CHECK_STATE() and CMAKE_RESET_CHECK_STATE() These macros can
# be used to save, restore and reset (i.e., clear contents) the state of
# the variables CMAKE_REQUIRED_FLAGS, CMAKE_REQUIRED_DEFINITIONS,
# CMAKE_REQUIRED_LIBRARIES, CMAKE_REQUIRED_INCLUDES and CMAKE_EXTRA_INCLUDE_FILES
# used by the various Check-files coming with CMake, like e.g.
# check_function_exists() etc. The variable contents are pushed on a
# stack, pushing multiple times is supported. This is useful e.g. when
# executing such tests in a Find-module, where they have to be set, but
# after the Find-module has been executed they should have the same
# value as they had before.
#
# CMAKE_PUSH_CHECK_STATE() macro receives optional argument RESET.
# Whether it's specified, CMAKE_PUSH_CHECK_STATE() will set all
# CMAKE_REQUIRED_* variables to empty values, same as
# CMAKE_RESET_CHECK_STATE() call will do.
#
# Usage:
#
# ::
#
# cmake_push_check_state(RESET)
# set(CMAKE_REQUIRED_DEFINITIONS -DSOME_MORE_DEF)
# check_function_exists(...)
# cmake_reset_check_state()
# set(CMAKE_REQUIRED_DEFINITIONS -DANOTHER_DEF)
# check_function_exists(...)
# cmake_pop_check_state()
macro(CMAKE_RESET_CHECK_STATE)
set(CMAKE_EXTRA_INCLUDE_FILES)
set(CMAKE_REQUIRED_INCLUDES)
set(CMAKE_REQUIRED_DEFINITIONS)
set(CMAKE_REQUIRED_LIBRARIES)
set(CMAKE_REQUIRED_FLAGS)
set(CMAKE_REQUIRED_QUIET)
endmacro()
macro(CMAKE_PUSH_CHECK_STATE)
if(NOT DEFINED _CMAKE_PUSH_CHECK_STATE_COUNTER)
set(_CMAKE_PUSH_CHECK_STATE_COUNTER 0)
endif()
math(EXPR _CMAKE_PUSH_CHECK_STATE_COUNTER "${_CMAKE_PUSH_CHECK_STATE_COUNTER}+1")
set(_CMAKE_EXTRA_INCLUDE_FILES_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER} ${CMAKE_EXTRA_INCLUDE_FILES})
set(_CMAKE_REQUIRED_INCLUDES_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER} ${CMAKE_REQUIRED_INCLUDES})
set(_CMAKE_REQUIRED_DEFINITIONS_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER} ${CMAKE_REQUIRED_DEFINITIONS})
set(_CMAKE_REQUIRED_LIBRARIES_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER} ${CMAKE_REQUIRED_LIBRARIES})
set(_CMAKE_REQUIRED_FLAGS_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER} ${CMAKE_REQUIRED_FLAGS})
set(_CMAKE_REQUIRED_QUIET_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER} ${CMAKE_REQUIRED_QUIET})
if (ARGC GREATER 0 AND ARGV0 STREQUAL "RESET")
cmake_reset_check_state()
endif()
endmacro()
macro(CMAKE_POP_CHECK_STATE)
# don't pop more than we pushed
if("${_CMAKE_PUSH_CHECK_STATE_COUNTER}" GREATER "0")
set(CMAKE_EXTRA_INCLUDE_FILES ${_CMAKE_EXTRA_INCLUDE_FILES_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER}})
set(CMAKE_REQUIRED_INCLUDES ${_CMAKE_REQUIRED_INCLUDES_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER}})
set(CMAKE_REQUIRED_DEFINITIONS ${_CMAKE_REQUIRED_DEFINITIONS_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER}})
set(CMAKE_REQUIRED_LIBRARIES ${_CMAKE_REQUIRED_LIBRARIES_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER}})
set(CMAKE_REQUIRED_FLAGS ${_CMAKE_REQUIRED_FLAGS_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER}})
set(CMAKE_REQUIRED_QUIET ${_CMAKE_REQUIRED_QUIET_SAVE_${_CMAKE_PUSH_CHECK_STATE_COUNTER}})
math(EXPR _CMAKE_PUSH_CHECK_STATE_COUNTER "${_CMAKE_PUSH_CHECK_STATE_COUNTER}-1")
endif()
endmacro()

66
pkg/OpenFace/cmake/modules/CheckFortranFunctionExists.cmake Normal file
View File

@@ -0,0 +1,66 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
#.rst:
# CheckFortranFunctionExists
# --------------------------
#
# macro which checks if the Fortran function exists
#
# CHECK_FORTRAN_FUNCTION_EXISTS(FUNCTION VARIABLE)
#
# ::
#
# FUNCTION - the name of the Fortran function
# VARIABLE - variable to store the result
# Will be created as an internal cache variable.
#
#
#
# The following variables may be set before calling this macro to modify
# the way the check is run:
#
# ::
#
# CMAKE_REQUIRED_LIBRARIES = list of libraries to link
macro(CHECK_FORTRAN_FUNCTION_EXISTS FUNCTION VARIABLE)
if(NOT DEFINED ${VARIABLE})
message(STATUS "Looking for Fortran ${FUNCTION}")
if(CMAKE_REQUIRED_LIBRARIES)
set(CHECK_FUNCTION_EXISTS_ADD_LIBRARIES
LINK_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
else()
set(CHECK_FUNCTION_EXISTS_ADD_LIBRARIES)
endif()
file(WRITE
${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCompiler.f
"
program TESTFortran
external ${FUNCTION}
call ${FUNCTION}()
end program TESTFortran
"
)
try_compile(${VARIABLE}
${CMAKE_BINARY_DIR}
${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCompiler.f
${CHECK_FUNCTION_EXISTS_ADD_LIBRARIES}
OUTPUT_VARIABLE OUTPUT
)
# message(STATUS "${OUTPUT}")
if(${VARIABLE})
set(${VARIABLE} 1 CACHE INTERNAL "Have Fortran function ${FUNCTION}")
message(STATUS "Looking for Fortran ${FUNCTION} - found")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
"Determining if the Fortran ${FUNCTION} exists passed with the following output:\n"
"${OUTPUT}\n\n")
else()
message(STATUS "Looking for Fortran ${FUNCTION} - not found")
set(${VARIABLE} "" CACHE INTERNAL "Have Fortran function ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
"Determining if the Fortran ${FUNCTION} exists failed with the following output:\n"
"${OUTPUT}\n\n")
endif()
endif()
endmacro()

98
pkg/OpenFace/cmake/modules/CheckFunctionExists.cmake Normal file
View File

@@ -0,0 +1,98 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
#.rst:
# CheckFunctionExists
# -------------------
#
# Check if a C function can be linked::
#
# check_function_exists(<function> <variable>)
#
# Check that the ``<function>`` is provided by libraries on the system and store
# the result in a ``<variable>``. ``<variable>`` will be created as an internal
# cache variable.
#
# The following variables may be set before calling this macro to modify the
# way the check is run:
#
# ::
#
# CMAKE_REQUIRED_FLAGS = string of compile command line flags
# CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar)
# CMAKE_REQUIRED_INCLUDES = list of include directories
# CMAKE_REQUIRED_LIBRARIES = list of libraries to link
# CMAKE_REQUIRED_QUIET = execute quietly without messages
#
# .. note::
#
# Prefer using :Module:`CheckSymbolExists` instead of this module,
# for the following reasons:
#
# * ``check_function_exists()`` can't detect functions that are inlined
# in headers or specified as a macro.
#
# * ``check_function_exists()`` can't detect anything in the 32-bit
# versions of the Win32 API, because of a mismatch in calling conventions.
#
# * ``check_function_exists()`` only verifies linking, it does not verify
# that the function is declared in system headers.
macro(CHECK_FUNCTION_EXISTS FUNCTION VARIABLE)
if(NOT DEFINED "${VARIABLE}" OR "x${${VARIABLE}}" STREQUAL "x${VARIABLE}")
set(MACRO_CHECK_FUNCTION_DEFINITIONS
"-DCHECK_FUNCTION_EXISTS=${FUNCTION} ${CMAKE_REQUIRED_FLAGS}")
if(NOT CMAKE_REQUIRED_QUIET)
message(STATUS "Looking for ${FUNCTION}")
endif()
if(CMAKE_REQUIRED_LIBRARIES)
set(CHECK_FUNCTION_EXISTS_ADD_LIBRARIES
LINK_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
else()
set(CHECK_FUNCTION_EXISTS_ADD_LIBRARIES)
endif()
if(CMAKE_REQUIRED_INCLUDES)
set(CHECK_FUNCTION_EXISTS_ADD_INCLUDES
"-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}")
else()
set(CHECK_FUNCTION_EXISTS_ADD_INCLUDES)
endif()
if(CMAKE_C_COMPILER_LOADED)
set(_cfe_source ${CMAKE_ROOT}/Modules/CheckFunctionExists.c)
elseif(CMAKE_CXX_COMPILER_LOADED)
set(_cfe_source ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CheckFunctionExists/CheckFunctionExists.cxx)
configure_file(${CMAKE_ROOT}/Modules/CheckFunctionExists.c "${_cfe_source}" COPYONLY)
else()
message(FATAL_ERROR "CHECK_FUNCTION_EXISTS needs either C or CXX language enabled")
endif()
try_compile(${VARIABLE}
${CMAKE_BINARY_DIR}
${_cfe_source}
COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
${CHECK_FUNCTION_EXISTS_ADD_LIBRARIES}
CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
"${CHECK_FUNCTION_EXISTS_ADD_INCLUDES}"
OUTPUT_VARIABLE OUTPUT)
unset(_cfe_source)
if(${VARIABLE})
set(${VARIABLE} 1 CACHE INTERNAL "Have function ${FUNCTION}")
if(NOT CMAKE_REQUIRED_QUIET)
message(STATUS "Looking for ${FUNCTION} - found")
endif()
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
"Determining if the function ${FUNCTION} exists passed with the following output:\n"
"${OUTPUT}\n\n")
else()
if(NOT CMAKE_REQUIRED_QUIET)
message(STATUS "Looking for ${FUNCTION} - not found")
endif()
set(${VARIABLE} "" CACHE INTERNAL "Have function ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
"Determining if the function ${FUNCTION} exists failed with the following output:\n"
"${OUTPUT}\n\n")
endif()
endif()
endmacro()

708
pkg/OpenFace/cmake/modules/FindBLAS.cmake Normal file
View File

@@ -0,0 +1,708 @@
#.rst:
# FindBLAS
# --------
#
# Find BLAS library
#
# This module finds an installed fortran library that implements the
# BLAS linear-algebra interface (see http://www.netlib.org/blas/). The
# list of libraries searched for is taken from the autoconf macro file,
# acx_blas.m4 (distributed at
# http://ac-archive.sourceforge.net/ac-archive/acx_blas.html).
#
# This module sets the following variables:
#
# ::
#
# BLAS_FOUND - set to true if a library implementing the BLAS interface
# is found
# BLAS_LINKER_FLAGS - uncached list of required linker flags (excluding -l
# and -L).
# BLAS_LIBRARIES - uncached list of libraries (using full path name) to
# link against to use BLAS
# BLAS95_LIBRARIES - uncached list of libraries (using full path name)
# to link against to use BLAS95 interface
# BLAS95_FOUND - set to true if a library implementing the BLAS f95 interface
# is found
# BLA_STATIC if set on this determines what kind of linkage we do (static)
# BLA_VENDOR if set checks only the specified vendor, if not set checks
# all the possibilities
# BLA_F95 if set on tries to find the f95 interfaces for BLAS/LAPACK
#
# ######### ## List of vendors (BLA_VENDOR) valid in this module #
# Goto,OpenBLAS,ATLAS PhiPACK,CXML,DXML,SunPerf,SCSL,SGIMATH,IBMESSL,
# Intel10_32 (intel mkl v10 32 bit),Intel10_64lp (intel mkl v10 64 bit,
# lp thread model, lp64 model), # Intel10_64lp_seq (intel mkl v10 64
# bit,sequential code, lp64 model), # Intel( older versions of mkl 32
# and 64 bit), ACML,ACML_MP,ACML_GPU,Apple, NAS, Generic C/CXX should be
# enabled to use Intel mkl
#=============================================================================
# Copyright 2007-2009 Kitware, Inc.
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
include(${CMAKE_CURRENT_LIST_DIR}/CheckFunctionExists.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/CheckFortranFunctionExists.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/CMakePushCheckState.cmake)
cmake_push_check_state()
set(CMAKE_REQUIRED_QUIET ${BLAS_FIND_QUIETLY})
set(_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
# Check the language being used
get_property( _LANGUAGES_ GLOBAL PROPERTY ENABLED_LANGUAGES )
if( _LANGUAGES_ MATCHES Fortran )
set( _CHECK_FORTRAN TRUE )
elseif( (_LANGUAGES_ MATCHES C) OR (_LANGUAGES_ MATCHES CXX) )
set( _CHECK_FORTRAN FALSE )
else()
if(BLAS_FIND_REQUIRED)
message(FATAL_ERROR "FindBLAS requires Fortran, C, or C++ to be enabled.")
else()
message(STATUS "Looking for BLAS... - NOT found (Unsupported languages)")
return()
endif()
endif()
macro(Check_Fortran_Libraries LIBRARIES _prefix _name _flags _list _thread)
# This macro checks for the existence of the combination of fortran libraries
# given by _list. If the combination is found, this macro checks (using the
# Check_Fortran_Function_Exists macro) whether can link against that library
# combination using the name of a routine given by _name using the linker
# flags given by _flags. If the combination of libraries is found and passes
# the link test, LIBRARIES is set to the list of complete library paths that
# have been found. Otherwise, LIBRARIES is set to FALSE.
# N.B. _prefix is the prefix applied to the names of all cached variables that
# are generated internally and marked advanced by this macro.
set(_libdir ${ARGN})
set(_libraries_work TRUE)
set(${LIBRARIES})
set(_combined_name)
if (NOT _libdir)
if (WIN32)
set(_libdir ENV LIB)
elseif (APPLE)
set(_libdir ENV DYLD_LIBRARY_PATH)
else ()
set(_libdir ENV LD_LIBRARY_PATH)
endif ()
endif ()
foreach(_library ${_list})
set(_combined_name ${_combined_name}_${_library})
if(_libraries_work)
if (BLA_STATIC)
if (WIN32)
set(CMAKE_FIND_LIBRARY_SUFFIXES .lib ${CMAKE_FIND_LIBRARY_SUFFIXES})
endif ()
if (APPLE)
set(CMAKE_FIND_LIBRARY_SUFFIXES .lib ${CMAKE_FIND_LIBRARY_SUFFIXES})
else ()
set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
endif ()
else ()
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
# for ubuntu's libblas3gf and liblapack3gf packages
set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES} .so.3gf)
endif ()
endif ()
find_library(${_prefix}_${_library}_LIBRARY
NAMES ${_library}
PATHS ${_libdir}
)
mark_as_advanced(${_prefix}_${_library}_LIBRARY)
set(${LIBRARIES} ${${LIBRARIES}} ${${_prefix}_${_library}_LIBRARY})
set(_libraries_work ${${_prefix}_${_library}_LIBRARY})
endif()
endforeach()
if(_libraries_work)
# Test this combination of libraries.
set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}} ${_thread})
# message("DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}")
if (_CHECK_FORTRAN)
check_fortran_function_exists("${_name}" ${_prefix}${_combined_name}_WORKS)
else()
check_function_exists("${_name}_" ${_prefix}${_combined_name}_WORKS)
endif()
set(CMAKE_REQUIRED_LIBRARIES)
mark_as_advanced(${_prefix}${_combined_name}_WORKS)
set(_libraries_work ${${_prefix}${_combined_name}_WORKS})
endif()
if(NOT _libraries_work)
set(${LIBRARIES} FALSE)
endif()
#message("DEBUG: ${LIBRARIES} = ${${LIBRARIES}}")
endmacro()
set(BLAS_LINKER_FLAGS)
set(BLAS_LIBRARIES)
set(BLAS95_LIBRARIES)
if (NOT $ENV{BLA_VENDOR} STREQUAL "")
set(BLA_VENDOR $ENV{BLA_VENDOR})
else ()
if(NOT BLA_VENDOR)
set(BLA_VENDOR "All")
endif()
endif ()
if (BLA_VENDOR STREQUAL "Goto" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
# gotoblas (http://www.tacc.utexas.edu/tacc-projects/gotoblas2)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"goto2"
""
)
endif()
endif ()
if (BLA_VENDOR STREQUAL "OpenBLAS" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
# OpenBLAS (http://www.openblas.net)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"openblas"
""
)
endif()
endif ()
if (BLA_VENDOR STREQUAL "ATLAS" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
# BLAS in ATLAS library? (http://math-atlas.sourceforge.net/)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
dgemm
""
"f77blas;atlas"
""
)
endif()
endif ()
# BLAS in PhiPACK libraries? (requires generic BLAS lib, too)
if (BLA_VENDOR STREQUAL "PhiPACK" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"sgemm;dgemm;blas"
""
)
endif()
endif ()
# BLAS in Alpha CXML library?
if (BLA_VENDOR STREQUAL "CXML" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"cxml"
""
)
endif()
endif ()
# BLAS in Alpha DXML library? (now called CXML, see above)
if (BLA_VENDOR STREQUAL "DXML" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"dxml"
""
)
endif()
endif ()
# BLAS in Sun Performance library?
if (BLA_VENDOR STREQUAL "SunPerf" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
"-xlic_lib=sunperf"
"sunperf;sunmath"
""
)
if(BLAS_LIBRARIES)
set(BLAS_LINKER_FLAGS "-xlic_lib=sunperf")
endif()
endif()
endif ()
# BLAS in SCSL library? (SGI/Cray Scientific Library)
if (BLA_VENDOR STREQUAL "SCSL" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"scsl"
""
)
endif()
endif ()
# BLAS in SGIMATH library?
if (BLA_VENDOR STREQUAL "SGIMATH" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"complib.sgimath"
""
)
endif()
endif ()
# BLAS in IBM ESSL library? (requires generic BLAS lib, too)
if (BLA_VENDOR STREQUAL "IBMESSL" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"essl;blas"
""
)
endif()
endif ()
#BLAS in acml library?
if (BLA_VENDOR MATCHES "ACML" OR BLA_VENDOR STREQUAL "All")
if( ((BLA_VENDOR STREQUAL "ACML") AND (NOT BLAS_ACML_LIB_DIRS)) OR
((BLA_VENDOR STREQUAL "ACML_MP") AND (NOT BLAS_ACML_MP_LIB_DIRS)) OR
((BLA_VENDOR STREQUAL "ACML_GPU") AND (NOT BLAS_ACML_GPU_LIB_DIRS))
)
# try to find acml in "standard" paths
if( WIN32 )
file( GLOB _ACML_ROOT "C:/AMD/acml*/ACML-EULA.txt" )
else()
file( GLOB _ACML_ROOT "/opt/acml*/ACML-EULA.txt" )
endif()
if( WIN32 )
file( GLOB _ACML_GPU_ROOT "C:/AMD/acml*/GPGPUexamples" )
else()
file( GLOB _ACML_GPU_ROOT "/opt/acml*/GPGPUexamples" )
endif()
list(GET _ACML_ROOT 0 _ACML_ROOT)
list(GET _ACML_GPU_ROOT 0 _ACML_GPU_ROOT)
if( _ACML_ROOT )
get_filename_component( _ACML_ROOT ${_ACML_ROOT} PATH )
if( SIZEOF_INTEGER EQUAL 8 )
set( _ACML_PATH_SUFFIX "_int64" )
else()
set( _ACML_PATH_SUFFIX "" )
endif()
if( CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" )
set( _ACML_COMPILER32 "ifort32" )
set( _ACML_COMPILER64 "ifort64" )
elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "SunPro" )
set( _ACML_COMPILER32 "sun32" )
set( _ACML_COMPILER64 "sun64" )
elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "PGI" )
set( _ACML_COMPILER32 "pgi32" )
if( WIN32 )
set( _ACML_COMPILER64 "win64" )
else()
set( _ACML_COMPILER64 "pgi64" )
endif()
elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "Open64" )
# 32 bit builds not supported on Open64 but for code simplicity
# We'll just use the same directory twice
set( _ACML_COMPILER32 "open64_64" )
set( _ACML_COMPILER64 "open64_64" )
elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "NAG" )
set( _ACML_COMPILER32 "nag32" )
set( _ACML_COMPILER64 "nag64" )
else()
set( _ACML_COMPILER32 "gfortran32" )
set( _ACML_COMPILER64 "gfortran64" )
endif()
if( BLA_VENDOR STREQUAL "ACML_MP" )
set(_ACML_MP_LIB_DIRS
"${_ACML_ROOT}/${_ACML_COMPILER32}_mp${_ACML_PATH_SUFFIX}/lib"
"${_ACML_ROOT}/${_ACML_COMPILER64}_mp${_ACML_PATH_SUFFIX}/lib" )
else()
set(_ACML_LIB_DIRS
"${_ACML_ROOT}/${_ACML_COMPILER32}${_ACML_PATH_SUFFIX}/lib"
"${_ACML_ROOT}/${_ACML_COMPILER64}${_ACML_PATH_SUFFIX}/lib" )
endif()
endif()
elseif(BLAS_${BLA_VENDOR}_LIB_DIRS)
set(_${BLA_VENDOR}_LIB_DIRS ${BLAS_${BLA_VENDOR}_LIB_DIRS})
endif()
if( BLA_VENDOR STREQUAL "ACML_MP" )
foreach( BLAS_ACML_MP_LIB_DIRS ${_ACML_MP_LIB_DIRS})
check_fortran_libraries (
BLAS_LIBRARIES
BLAS
sgemm
"" "acml_mp;acml_mv" "" ${BLAS_ACML_MP_LIB_DIRS}
)
if( BLAS_LIBRARIES )
break()
endif()
endforeach()
elseif( BLA_VENDOR STREQUAL "ACML_GPU" )
foreach( BLAS_ACML_GPU_LIB_DIRS ${_ACML_GPU_LIB_DIRS})
check_fortran_libraries (
BLAS_LIBRARIES
BLAS
sgemm
"" "acml;acml_mv;CALBLAS" "" ${BLAS_ACML_GPU_LIB_DIRS}
)
if( BLAS_LIBRARIES )
break()
endif()
endforeach()
else()
foreach( BLAS_ACML_LIB_DIRS ${_ACML_LIB_DIRS} )
check_fortran_libraries (
BLAS_LIBRARIES
BLAS
sgemm
"" "acml;acml_mv" "" ${BLAS_ACML_LIB_DIRS}
)
if( BLAS_LIBRARIES )
break()
endif()
endforeach()
endif()
# Either acml or acml_mp should be in LD_LIBRARY_PATH but not both
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"acml;acml_mv"
""
)
endif()
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"acml_mp;acml_mv"
""
)
endif()
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"acml;acml_mv;CALBLAS"
""
)
endif()
endif () # ACML
# Apple BLAS library?
if (BLA_VENDOR STREQUAL "Apple" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
dgemm
""
"Accelerate"
""
)
endif()
endif ()
if (BLA_VENDOR STREQUAL "NAS" OR BLA_VENDOR STREQUAL "All")
if ( NOT BLAS_LIBRARIES )
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
dgemm
""
"vecLib"
""
)
endif ()
endif ()
# Generic BLAS library?
if (BLA_VENDOR STREQUAL "Generic" OR BLA_VENDOR STREQUAL "All")
if(NOT BLAS_LIBRARIES)
check_fortran_libraries(
BLAS_LIBRARIES
BLAS
sgemm
""
"blas"
""
)
endif()
endif ()
#BLAS in intel mkl 10 library? (em64t 64bit)
if (BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
if (NOT WIN32)
set(LM "-lm")
endif ()
if (_LANGUAGES_ MATCHES C OR _LANGUAGES_ MATCHES CXX)
if(BLAS_FIND_QUIETLY OR NOT BLAS_FIND_REQUIRED)
find_package(Threads)
else()
find_package(Threads REQUIRED)
endif()
set(BLAS_SEARCH_LIBS "")
if(BLA_F95)
set(BLAS_mkl_SEARCH_SYMBOL SGEMM)
set(_LIBRARIES BLAS95_LIBRARIES)
if (WIN32)
if (BLA_STATIC)
set(BLAS_mkl_DLL_SUFFIX "")
else()
set(BLAS_mkl_DLL_SUFFIX "_dll")
endif()
# Find the main file (32-bit or 64-bit)
set(BLAS_SEARCH_LIBS_WIN_MAIN "")
if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN
"mkl_blas95${BLAS_mkl_DLL_SUFFIX} mkl_intel_c${BLAS_mkl_DLL_SUFFIX}")
endif()
if (BLA_VENDOR MATCHES "^Intel10_64lp" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN
"mkl_blas95_lp64${BLAS_mkl_DLL_SUFFIX} mkl_intel_lp64${BLAS_mkl_DLL_SUFFIX}")
endif ()
# Add threading/sequential libs
set(BLAS_SEARCH_LIBS_WIN_THREAD "")
if (BLA_VENDOR MATCHES "_seq$" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD
"mkl_sequential${BLAS_mkl_DLL_SUFFIX}")
endif()
if (NOT BLA_VENDOR MATCHES "_seq$" OR BLA_VENDOR STREQUAL "All")
# old version
list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD
"libguide40 mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}")
# mkl >= 10.3
list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD
"libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}")
endif()
# Cartesian product of the above
foreach (MAIN ${BLAS_SEARCH_LIBS_WIN_MAIN})
foreach (THREAD ${BLAS_SEARCH_LIBS_WIN_THREAD})
list(APPEND BLAS_SEARCH_LIBS
"${MAIN} ${THREAD} mkl_core${BLAS_mkl_DLL_SUFFIX}")
endforeach()
endforeach()
else ()
if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS
"mkl_blas95 mkl_intel mkl_intel_thread mkl_core guide")
endif ()
if (BLA_VENDOR STREQUAL "Intel10_64lp" OR BLA_VENDOR STREQUAL "All")
# old version
list(APPEND BLAS_SEARCH_LIBS
"mkl_blas95 mkl_intel_lp64 mkl_intel_thread mkl_core guide")
# mkl >= 10.3
if (CMAKE_C_COMPILER MATCHES ".+gcc")
list(APPEND BLAS_SEARCH_LIBS
"mkl_blas95_lp64 mkl_intel_lp64 mkl_gnu_thread mkl_core gomp")
else ()
list(APPEND BLAS_SEARCH_LIBS
"mkl_blas95_lp64 mkl_intel_lp64 mkl_intel_thread mkl_core iomp5")
endif ()
endif ()
if (BLA_VENDOR STREQUAL "Intel10_64lp_seq" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS
"mkl_intel_lp64 mkl_sequential mkl_core")
endif ()
endif ()
else ()
set(BLAS_mkl_SEARCH_SYMBOL sgemm)
set(_LIBRARIES BLAS_LIBRARIES)
if (WIN32)
if (BLA_STATIC)
set(BLAS_mkl_DLL_SUFFIX "")
else()
set(BLAS_mkl_DLL_SUFFIX "_dll")
endif()
# Find the main file (32-bit or 64-bit)
set(BLAS_SEARCH_LIBS_WIN_MAIN "")
if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN
"mkl_intel_c${BLAS_mkl_DLL_SUFFIX}")
endif()
if (BLA_VENDOR MATCHES "^Intel10_64lp" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS_WIN_MAIN
"mkl_intel_lp64${BLAS_mkl_DLL_SUFFIX}")
endif ()
# Add threading/sequential libs
set(BLAS_SEARCH_LIBS_WIN_THREAD "")
if (NOT BLA_VENDOR MATCHES "_seq$" OR BLA_VENDOR STREQUAL "All")
# old version
list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD
"libguide40 mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}")
# mkl >= 10.3
list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD
"libiomp5md mkl_intel_thread${BLAS_mkl_DLL_SUFFIX}")
endif()
if (BLA_VENDOR MATCHES "_seq$" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS_WIN_THREAD
"mkl_sequential${BLAS_mkl_DLL_SUFFIX}")
endif()
# Cartesian product of the above
foreach (MAIN ${BLAS_SEARCH_LIBS_WIN_MAIN})
foreach (THREAD ${BLAS_SEARCH_LIBS_WIN_THREAD})
list(APPEND BLAS_SEARCH_LIBS
"${MAIN} ${THREAD} mkl_core${BLAS_mkl_DLL_SUFFIX}")
endforeach()
endforeach()
else ()
if (BLA_VENDOR STREQUAL "Intel10_32" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS
"mkl_intel mkl_intel_thread mkl_core guide")
endif ()
if (BLA_VENDOR STREQUAL "Intel10_64lp" OR BLA_VENDOR STREQUAL "All")
# old version
list(APPEND BLAS_SEARCH_LIBS
"mkl_intel_lp64 mkl_intel_thread mkl_core guide")
# mkl >= 10.3
if (CMAKE_C_COMPILER MATCHES ".+gcc")
list(APPEND BLAS_SEARCH_LIBS
"mkl_intel_lp64 mkl_gnu_thread mkl_core gomp")
else ()
list(APPEND BLAS_SEARCH_LIBS
"mkl_intel_lp64 mkl_intel_thread mkl_core iomp5")
endif ()
endif ()
if (BLA_VENDOR STREQUAL "Intel10_64lp_seq" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS
"mkl_intel_lp64 mkl_sequential mkl_core")
endif ()
#older vesions of intel mkl libs
if (BLA_VENDOR STREQUAL "Intel" OR BLA_VENDOR STREQUAL "All")
list(APPEND BLAS_SEARCH_LIBS
"mkl")
list(APPEND BLAS_SEARCH_LIBS
"mkl_ia32")
list(APPEND BLAS_SEARCH_LIBS
"mkl_em64t")
endif ()
endif ()
endif ()
foreach (IT ${BLAS_SEARCH_LIBS})
string(REPLACE " " ";" SEARCH_LIBS ${IT})
if (${_LIBRARIES})
else ()
check_fortran_libraries(
${_LIBRARIES}
BLAS
${BLAS_mkl_SEARCH_SYMBOL}
""
"${SEARCH_LIBS}"
"${CMAKE_THREAD_LIBS_INIT};${LM}"
)
endif ()
endforeach ()
endif ()
endif ()
if(BLA_F95)
if(BLAS95_LIBRARIES)
set(BLAS95_FOUND TRUE)
else()
set(BLAS95_FOUND FALSE)
endif()
if(NOT BLAS_FIND_QUIETLY)
if(BLAS95_FOUND)
message(STATUS "A library with BLAS95 API found.")
else()
if(BLAS_FIND_REQUIRED)
message(FATAL_ERROR
"A required library with BLAS95 API not found. Please specify library location.")
else()
message(STATUS
"A library with BLAS95 API not found. Please specify library location.")
endif()
endif()
endif()
set(BLAS_FOUND TRUE)
set(BLAS_LIBRARIES "${BLAS95_LIBRARIES}")
else()
if(BLAS_LIBRARIES)
set(BLAS_FOUND TRUE)
else()
set(BLAS_FOUND FALSE)
endif()
if(NOT BLAS_FIND_QUIETLY)
if(BLAS_FOUND)
message(STATUS "A library with BLAS API found.")
else()
if(BLAS_FIND_REQUIRED)
message(FATAL_ERROR
"A required library with BLAS API not found. Please specify library location."
)
else()
message(STATUS
"A library with BLAS API not found. Please specify library location."
)
endif()
endif()
endif()
endif()
cmake_pop_check_state()
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_blas_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})

111
pkg/OpenFace/cmake/modules/FindOpenBLAS.cmake Normal file
View File

@@ -0,0 +1,111 @@
#COPYRIGHT
#
#All contributions by the University of California:
#Copyright (c) 2014, 2015, The Regents of the University of California (Regents)
#All rights reserved.
#
#All other contributions:
#Copyright (c) 2014, 2015, the respective contributors
#All rights reserved.
#
#Caffe uses a shared copyright model: each contributor holds copyright over
#their contributions to Caffe. The project versioning records all such
#contribution and copyright details. If a contributor wants to further mark
#their specific copyright on a particular contribution, they should indicate
#their copyright solely in the commit message of the change when it is
#committed.
#
#LICENSE
#
#Redistribution and use in source and binary forms, with or without
#modification, are permitted provided that the following conditions are met:
#
#1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
#ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
#ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#CONTRIBUTION AGREEMENT
#
#By contributing to the BVLC/caffe repository through pull-request, comment,
#or otherwise, the contributor releases their content to the
#license and copyright terms herein.
SET(Open_BLAS_INCLUDE_SEARCH_PATHS
$ENV{OpenBLAS_HOME}
$ENV{OpenBLAS_HOME}/include
/opt/OpenBLAS/include
/usr/local/include/openblas
/usr/include/openblas
/usr/local/include/openblas-base
/usr/include/openblas-base
/usr/include/x86_64-linux-gnu
/usr/local/include
/usr/include
/usr/local/opt/openblas/include
)
SET(Open_BLAS_LIB_SEARCH_PATHS
$ENV{OpenBLAS}cd
$ENV{OpenBLAS}/lib
$ENV{OpenBLAS_HOME}
$ENV{OpenBLAS_HOME}/lib
/opt/OpenBLAS/lib
/usr/local/lib64
/usr/local/lib
/lib/openblas-base
/lib64/
/lib/
/usr/lib/openblas-base
/usr/lib/x86_64-linux-gnu
/usr/lib64
/usr/lib
/usr/local/opt/openblas/lib
)
FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES f77blas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS} NO_DEFAULT_PATH)
FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS} NO_DEFAULT_PATH)
SET(OpenBLAS_FOUND ON)
SET(OpenBLAS_INCLUDE_FOUND ON)
# Check include files
IF(NOT OpenBLAS_INCLUDE_DIR)
SET(OpenBLAS_INCLUDE_FOUND OFF)
MESSAGE(STATUS "Could not find OpenBLAS include, defaulting to using OpenFace vended ones")
ENDIF()
# Check libraries
IF(NOT OpenBLAS_LIB)
SET(OpenBLAS_FOUND OFF)
MESSAGE(STATUS "Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off")
ENDIF()
IF (OpenBLAS_FOUND)
IF (NOT OpenBLAS_FIND_QUIETLY)
MESSAGE(STATUS "Found OpenBLAS libraries: ${OpenBLAS_LIB}")
MESSAGE(STATUS "Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}")
ENDIF (NOT OpenBLAS_FIND_QUIETLY)
ELSE (OpenBLAS_FOUND)
IF (OpenBLAS_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find OpenBLAS")
ENDIF (OpenBLAS_FIND_REQUIRED)
ENDIF (OpenBLAS_FOUND)
MARK_AS_ADVANCED(
OpenBLAS_INCLUDE_DIR
OpenBLAS_LIB
OpenBLAS
)

307
pkg/OpenFace/cmake/modules/FindTBB.cmake Normal file
View File

@@ -0,0 +1,307 @@
# The MIT License (MIT)
#
# Copyright (c) 2015 Justus Calvin
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# FindTBB
# -------
#
# Find TBB include directories and libraries.
#
# Usage:
#
# find_package(TBB [major[.minor]] [EXACT]
# [QUIET] [REQUIRED]
# [[COMPONENTS] [components...]]
# [OPTIONAL_COMPONENTS components...])
#
# where the allowed components are tbbmalloc and tbb_preview. Users may modify
# the behavior of this module with the following variables:
#
# * TBB_ROOT_DIR - The base directory the of TBB installation.
# * TBB_INCLUDE_DIR - The directory that contains the TBB headers files.
# * TBB_LIBRARY - The directory that contains the TBB library files.
# * TBB_<library>_LIBRARY - The path of the TBB the corresponding TBB library.
# These libraries, if specified, override the
# corresponding library search results, where <library>
# may be tbb, tbb_debug, tbbmalloc, tbbmalloc_debug,
# tbb_preview, or tbb_preview_debug.
# * TBB_USE_DEBUG_BUILD - The debug version of tbb libraries, if present, will
# be used instead of the release version.
#
# Users may modify the behavior of this module with the following environment
# variables:
#
# * TBB_INSTALL_DIR
# * TBBROOT
# * LIBRARY_PATH
#
# This module will set the following variables:
#
# * TBB_FOUND - Set to false, or undefined, if we haven’t found, or
# don’t want to use TBB.
# * TBB_<component>_FOUND - If False, optional <component> part of TBB sytem is
# not available.
# * TBB_VERSION - The full version string
# * TBB_VERSION_MAJOR - The major version
# * TBB_VERSION_MINOR - The minor version
# * TBB_INTERFACE_VERSION - The interface version number defined in
# tbb/tbb_stddef.h.
# * TBB_<library>_LIBRARY_RELEASE - The path of the TBB release version of
# <library>, where <library> may be tbb, tbb_debug,
# tbbmalloc, tbbmalloc_debug, tbb_preview, or
# tbb_preview_debug.
# * TBB_<library>_LIBRARY_DEGUG - The path of the TBB release version of
# <library>, where <library> may be tbb, tbb_debug,
# tbbmalloc, tbbmalloc_debug, tbb_preview, or
# tbb_preview_debug.
#
# The following varibles should be used to build and link with TBB:
#
# * TBB_INCLUDE_DIRS - The include directory for TBB.
# * TBB_LIBRARIES - The libraries to link against to use TBB.
# * TBB_LIBRARIES_RELEASE - The release libraries to link against to use TBB.
# * TBB_LIBRARIES_DEBUG - The debug libraries to link against to use TBB.
# * TBB_DEFINITIONS - Definitions to use when compiling code that uses
# TBB.
# * TBB_DEFINITIONS_RELEASE - Definitions to use when compiling release code that
# uses TBB.
# * TBB_DEFINITIONS_DEBUG - Definitions to use when compiling debug code that
# uses TBB.
#
# This module will also create the "tbb" target that may be used when building
# executables and libraries.
include(FindPackageHandleStandardArgs)
if("${TBB_LIBRARIES}" STREQUAL "")
##################################
# Check the build type
##################################
if(NOT DEFINED TBB_USE_DEBUG_BUILD)
if(CMAKE_BUILD_TYPE MATCHES "(Debug|DEBUG|debug|RelWithDebInfo|RELWITHDEBINFO|relwithdebinfo)")
set(TBB_BUILD_TYPE DEBUG)
else()
set(TBB_BUILD_TYPE RELEASE)
endif()
elseif(TBB_USE_DEBUG_BUILD)
set(TBB_BUILD_TYPE DEBUG)
else()
set(TBB_BUILD_TYPE RELEASE)
endif()
##################################
# Set the TBB search directories
##################################
# Define search paths based on user input and environment variables
set(TBB_SEARCH_DIR ${TBB_ROOT_DIR} $ENV{TBB_INSTALL_DIR} $ENV{TBBROOT})
# Define the search directories based on the current platform
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
set(TBB_DEFAULT_SEARCH_DIR "C:/Program Files/Intel/TBB"
"C:/Program Files (x86)/Intel/TBB")
# Set the target architecture
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
set(TBB_ARCHITECTURE "intel64")
else()
set(TBB_ARCHITECTURE "ia32")
endif()
# Set the TBB search library path search suffix based on the version of VC
if(WINDOWS_STORE)
set(TBB_LIB_PATH_SUFFIX "lib/${TBB_ARCHITECTURE}/vc11_ui")
elseif(MSVC14)
set(TBB_LIB_PATH_SUFFIX "lib/${TBB_ARCHITECTURE}/vc14")
elseif(MSVC12)
set(TBB_LIB_PATH_SUFFIX "lib/${TBB_ARCHITECTURE}/vc12")
elseif(MSVC11)
set(TBB_LIB_PATH_SUFFIX "lib/${TBB_ARCHITECTURE}/vc11")
elseif(MSVC10)
set(TBB_LIB_PATH_SUFFIX "lib/${TBB_ARCHITECTURE}/vc10")
endif()
# Add the library path search suffix for the VC independent version of TBB
list(APPEND TBB_LIB_PATH_SUFFIX "lib/${TBB_ARCHITECTURE}/vc_mt")
elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
# OS X
set(TBB_DEFAULT_SEARCH_DIR
"/opt/intel/tbb"
"/usr/local/Cellar/tbb")
# TODO: Check to see which C++ library is being used by the compiler.
if(NOT ${CMAKE_SYSTEM_VERSION} VERSION_LESS 13.0)
# The default C++ library on OS X 10.9 and later is libc++
set(TBB_LIB_PATH_SUFFIX "lib/libc++" "lib")
else()
set(TBB_LIB_PATH_SUFFIX "lib")
endif()
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
# Linux
set(TBB_DEFAULT_SEARCH_DIR "/opt/intel/tbb")
# TODO: Check compiler version to see the suffix should be <arch>/gcc4.1 or
# <arch>/gcc4.1. For now, assume that the compiler is more recent than
# gcc 4.4.x or later.
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
set(TBB_LIB_PATH_SUFFIX "lib/intel64/gcc4.4")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
set(TBB_LIB_PATH_SUFFIX "lib/ia32/gcc4.4")
endif()
endif()
MESSAGE("Searching for TBB in: ${TBB_INCLUDE_DIRS}")
##################################
# Find the TBB include dir
##################################
find_path(TBB_INCLUDE_DIRS tbb/tbb.h
HINTS ${TBB_INCLUDE_DIR} ${TBB_SEARCH_DIR}
PATHS ${TBB_DEFAULT_SEARCH_DIR}
PATH_SUFFIXES include)
##################################
# Set version strings
##################################
if(TBB_INCLUDE_DIRS)
file(READ "${TBB_INCLUDE_DIRS}/tbb/tbb_stddef.h" _tbb_version_file)
string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1"
TBB_VERSION_MAJOR "${_tbb_version_file}")
string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1"
TBB_VERSION_MINOR "${_tbb_version_file}")
string(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1"
TBB_INTERFACE_VERSION "${_tbb_version_file}")
set(TBB_VERSION "${TBB_VERSION_MAJOR}.${TBB_VERSION_MINOR}")
endif()
##################################
# Find TBB components
##################################
if(TBB_VERSION VERSION_LESS 4.3)
set(TBB_SEARCH_COMPOMPONENTS tbb_preview tbbmalloc tbb)
else()
set(TBB_SEARCH_COMPOMPONENTS tbb_preview tbbmalloc_proxy tbbmalloc tbb)
endif()
# Find each component
foreach(_comp ${TBB_SEARCH_COMPOMPONENTS})
if(";${TBB_FIND_COMPONENTS};tbb;" MATCHES ";${_comp};")
# Search for the libraries
find_library(TBB_${_comp}_LIBRARY_RELEASE ${_comp}
HINTS ${TBB_LIBRARY} ${TBB_SEARCH_DIR}
PATHS ${TBB_DEFAULT_SEARCH_DIR} ENV LIBRARY_PATH
PATH_SUFFIXES ${TBB_LIB_PATH_SUFFIX})
find_library(TBB_${_comp}_LIBRARY_DEBUG ${_comp}_debug
HINTS ${TBB_LIBRARY} ${TBB_SEARCH_DIR}
PATHS ${TBB_DEFAULT_SEARCH_DIR} ENV LIBRARY_PATH
PATH_SUFFIXES ${TBB_LIB_PATH_SUFFIX})
if(TBB_${_comp}_LIBRARY_DEBUG)
list(APPEND TBB_LIBRARIES_DEBUG "${TBB_${_comp}_LIBRARY_DEBUG}")
endif()
if(TBB_${_comp}_LIBRARY_RELEASE)
list(APPEND TBB_LIBRARIES_RELEASE "${TBB_${_comp}_LIBRARY_RELEASE}")
endif()
if(TBB_${_comp}_LIBRARY_${TBB_BUILD_TYPE} AND NOT TBB_${_comp}_LIBRARY)
set(TBB_${_comp}_LIBRARY "${TBB_${_comp}_LIBRARY_${TBB_BUILD_TYPE}}")
endif()
if(TBB_${_comp}_LIBRARY AND EXISTS "${TBB_${_comp}_LIBRARY}")
set(TBB_${_comp}_FOUND TRUE)
else()
set(TBB_${_comp}_FOUND FALSE)
endif()
# Mark internal variables as advanced
mark_as_advanced(TBB_${_comp}_LIBRARY_RELEASE)
mark_as_advanced(TBB_${_comp}_LIBRARY_DEBUG)
mark_as_advanced(TBB_${_comp}_LIBRARY)
endif()
endforeach()
##################################
# Set compile flags and libraries
##################################
set(TBB_DEFINITIONS_RELEASE "")
set(TBB_DEFINITIONS_DEBUG "-DTBB_USE_DEBUG=1")
if(TBB_LIBRARIES_${TBB_BUILD_TYPE})
set(TBB_DEFINITIONS "${TBB_DEFINITIONS_${TBB_BUILD_TYPE}}")
set(TBB_LIBRARIES "${TBB_LIBRARIES_${TBB_BUILD_TYPE}}")
elseif(TBB_LIBRARIES_RELEASE)
set(TBB_DEFINITIONS "${TBB_DEFINITIONS_RELEASE}")
set(TBB_LIBRARIES "${TBB_LIBRARIES_RELEASE}")
elseif(TBB_LIBRARIES_DEBUG)
set(TBB_DEFINITIONS "${TBB_DEFINITIONS_DEBUG}")
set(TBB_LIBRARIES "${TBB_LIBRARIES_DEBUG}")
endif()
find_package_handle_standard_args(TBB
REQUIRED_VARS TBB_INCLUDE_DIRS TBB_LIBRARIES
HANDLE_COMPONENTS
VERSION_VAR TBB_VERSION)
##################################
# Create targets
##################################
if(NOT CMAKE_VERSION VERSION_LESS 3.0 AND TBB_FOUND)
add_library(tbb SHARED IMPORTED)
set_target_properties(tbb PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIRS}
IMPORTED_LOCATION ${TBB_LIBRARIES})
if(TBB_LIBRARIES_RELEASE AND TBB_LIBRARIES_DEBUG)
set_target_properties(tbb PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "$<$<OR:$<CONFIG:Debug>,$<CONFIG:RelWithDebInfo>>:TBB_USE_DEBUG=1>"
IMPORTED_LOCATION_DEBUG ${TBB_LIBRARIES_DEBUG}
IMPORTED_LOCATION_RELWITHDEBINFO ${TBB_LIBRARIES_DEBUG}
IMPORTED_LOCATION_RELEASE ${TBB_LIBRARIES_RELEASE}
IMPORTED_LOCATION_MINSIZEREL ${TBB_LIBRARIES_RELEASE}
)
elseif(TBB_LIBRARIES_RELEASE)
set_target_properties(tbb PROPERTIES IMPORTED_LOCATION ${TBB_LIBRARIES_RELEASE})
else()
set_target_properties(tbb PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "${TBB_DEFINITIONS_DEBUG}"
IMPORTED_LOCATION ${TBB_LIBRARIES_DEBUG}
)
endif()
endif()
mark_as_advanced(TBB_INCLUDE_DIRS TBB_LIBRARIES)
unset(TBB_ARCHITECTURE)
unset(TBB_BUILD_TYPE)
unset(TBB_LIB_PATH_SUFFIX)
unset(TBB_DEFAULT_SEARCH_DIR)
endif()

24
pkg/OpenFace/docker-compose.yml Normal file
View File

@@ -0,0 +1,24 @@
---
# Variables can be set from the shell: `DOCKERTAG=foo docker-compose build`
# or from a .env file. See docker-compose documentation for details
# Variable DOCKERUSER should be set to your dockerhub user
# Alternatively, use a docker registry url as the image name
version: '3.7'
services:
## Image runtime service
## This can be used to add volume mounts or pass environment variables
## Todo: make a service which can use the container interactively
openface:
container_name: openface
build:
context: .
dockerfile: docker/Dockerfile
image: "${DOCKERUSER}/openface:${DOCKERTAG}"
tty: true
volumes:
- "${DATA_MOUNT}:${DATA_MOUNT}"
environment:
DATA_MOUNT: "${DATA_MOUNT}"
command: ["bash"]
...

110
pkg/OpenFace/docker/Dockerfile Normal file
View File

@@ -0,0 +1,110 @@
# ==================== Building Model Layer ===========================
# This is a little trick to improve caching and minimize rebuild time
# and bandwidth. Note that RUN commands only cache-miss if the prior layers
# miss, or the dockerfile changes prior to this step.
# To update these patch files, be sure to run build with --no-cache
FROM alpine as model_data
RUN apk --no-cache --update-cache add wget
WORKDIR /data/patch_experts
RUN wget -q https://www.dropbox.com/s/7na5qsjzz8yfoer/cen_patches_0.25_of.dat &&\
wget -q https://www.dropbox.com/s/k7bj804cyiu474t/cen_patches_0.35_of.dat &&\
wget -q https://www.dropbox.com/s/ixt4vkbmxgab1iu/cen_patches_0.50_of.dat &&\
wget -q https://www.dropbox.com/s/2t5t1sdpshzfhpj/cen_patches_1.00_of.dat
## ==================== Install Ubuntu Base libs ===========================
## This will be our base image for OpenFace, and also the base for the compiler
## image. We only need packages which are linked
FROM ubuntu:18.04 as ubuntu_base
LABEL maintainer="Michael McDermott <mikemcdermott23@gmail.com>"
ARG DEBIAN_FRONTEND=noninteractive
# todo: minimize this even more
RUN apt-get update -qq &&\
apt-get install -qq curl &&\
apt-get install -qq --no-install-recommends \
libopenblas-dev liblapack-dev \
libavcodec-dev libavformat-dev libswscale-dev \
libtbb2 libtbb-dev libjpeg-dev \
libpng-dev libtiff-dev &&\
rm -rf /var/lib/apt/lists/*
## ==================== Build-time dependency libs ======================
## This will build and install opencv and dlib into an additional dummy
## directory, /root/diff, so we can later copy in these artifacts,
## minimizing docker layer size
## Protip: ninja is faster than `make -j` and less likely to lock up system
FROM ubuntu_base as cv_deps
WORKDIR /root/build-dep
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update -qq && apt-get install -qq -y \
cmake ninja-build pkg-config build-essential checkinstall\
g++-8 &&\
rm -rf /var/lib/apt/lists/* &&\
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
## llvm clang-3.7 libc++-dev libc++abi-dev \
## ==================== Building dlib ===========================
RUN curl http://dlib.net/files/dlib-19.13.tar.bz2 -LO &&\
tar xf dlib-19.13.tar.bz2 && \
rm dlib-19.13.tar.bz2 &&\
mv dlib-19.13 dlib &&\
mkdir -p dlib/build &&\
cd dlib/build &&\
cmake -DCMAKE_BUILD_TYPE=Release -G Ninja .. &&\
ninja && \
ninja install && \
DESTDIR=/root/diff ninja install &&\
ldconfig
## ==================== Building OpenCV ======================
ENV OPENCV_VERSION=4.1.0
RUN curl https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz -LO &&\
tar xf ${OPENCV_VERSION}.tar.gz && \
rm ${OPENCV_VERSION}.tar.gz &&\
mv opencv-${OPENCV_VERSION} opencv && \
mkdir -p opencv/build && \
cd opencv/build && \
cmake -D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=/usr/local \
-D WITH_TBB=ON -D WITH_CUDA=OFF \
-DWITH_QT=OFF -DWITH_GTK=OFF\
-G Ninja .. && \
ninja && \
ninja install &&\
DESTDIR=/root/diff ninja install
## ==================== Building OpenFace ===========================
FROM cv_deps as openface
WORKDIR /root/openface
COPY ./ ./
COPY --from=model_data /data/patch_experts/* \
/root/openface/lib/local/LandmarkDetector/model/patch_experts/
RUN mkdir -p build && cd build && \
cmake -D CMAKE_BUILD_TYPE=RELEASE -G Ninja .. && \
ninja &&\
DESTDIR=/root/diff ninja install
## ==================== Streamline container ===========================
## Clean up - start fresh and only copy in necessary stuff
## This shrinks the image from ~8 GB to ~1.6 GB
FROM ubuntu_base as final
WORKDIR /root
# Copy in only necessary libraries
COPY --from=openface /root/diff /
# Since we "imported" the build artifacts, we need to reconfigure ld
RUN ldconfig

57
pkg/OpenFace/docker/README.md Normal file
View File

@@ -0,0 +1,57 @@
# Docker building instructions
This image can be build with just `docker`, but it is highly recommend to use
`docker-compose` as this greatly simplifies and improves the process.
## Quick start
To start with the container hosted by the repo maintainer, run
`docker run -it --rm --name openface algebr/openface:latest`
This will drop you into a shell with binaries such as FaceLandmarkImg. For example,
try this code (in the container):
```bash
curl -o tesla.jpg https://upload.wikimedia.org/wikipedia/commons/thumb/b/b7/Nicola_Tesla_LCCN2014684845.jpg/559px-Nicola_Tesla_LCCN2014684845.jpg
FaceLandmarkImg -f tesla.jpg
```
Then, copy the output to the host system (from host terminal):
```bash
docker cp openface:/root/processed /tmp/
cd /tmp/processed
```
Tip: On Ubuntu and other *nixes with X running, you can open a file directly
like this:
```bash
xdg-open /tmp/processed/tesla.jpg
```
## Building
In repo root, run `docker-compose build` to automatically build and tag.
There are two variables which can be used to modify the tag, `$DOCKERUSER` and
`$DOCKERTAG`. DC will automatically tag image as
`${DOCKERUSER}/openface:${DOCKERTAG}`
## OpenFace service (in progress)
To run OpenFace like a service, you can start the container with bind mounts
in order to pass data into and out of the container easily.
`$DATA_MOUNT` by default is set to `/tmp/openface`. This can be overridden by
modifying `.env` file, setting it in your shell environment, or passing in
before `docker-compose` at runtime. Note: output will be `root` owner.
```bash
export DATA_MOUNT=/tmp/openface
mkdir -p $DATA_MOUNT/tesla # this is just to ensure this is writable by user
docker-compose up -d openface && sync # sync is to wait till service starts
curl -o $DATA_MOUNT/tesla.jpg \
https://upload.wikimedia.org/wikipedia/commons/thumb/b/b7/Nicola_Tesla_LCCN2014684845.jpg/559px-Nicola_Tesla_LCCN2014684845.jpg
docker exec -it openface FaceLandmarkImg -f $DATA_MOUNT/tesla.jpg -out_dir $DATA_MOUNT/tesla
docker exec -it openface chown -R $UID:$UID $DATA_MOUNT # chown to current user
docker-compose down # stop service if you wish
```

127
pkg/OpenFace/download_libraries.ps1 Normal file
View File

@@ -0,0 +1,127 @@
# Download the OpenCV libraries from the cloud (stored in Dropbox and OneDrive)
# ffmpeg x64 dll
$destination = "lib/3rdParty/OpenCV/bin/opencv_ffmpeg410_64.dll"
$out_dir = Join-Path (Get-Location) "lib/3rdParty/OpenCV/bin"
if(!(Test-Path $out_dir))
{
New-Item -ItemType directory -Path $out_dir
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/gvkd4549wsjvn3u/opencv_ffmpeg410_64.dll?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153244&authkey=AEuLAF197Sy7S3M"
Invoke-WebRequest $source -OutFile $destination
}
# Release x64 dll
$destination = "lib/3rdParty/OpenCV/x64/v141/bin/Release/opencv_world410.dll"
$out_dir = Join-Path (Get-Location) "lib/3rdParty/OpenCV/x64/v141/bin/Release"
if(!(Test-Path $out_dir))
{
New-Item -ItemType directory -Path $out_dir
}
#if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
#{
# $source = "https://www.dropbox.com/s/c81shi8br57xytv/opencv_world410.dll?dl=1"
# Invoke-WebRequest $source -OutFile $destination
#}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153246&authkey=AJkwseAGKqL3PpU"
Invoke-WebRequest $source -OutFile $destination
}
# Debug x64 dll
$destination = "lib/3rdParty/OpenCV/x64/v141/bin/Debug/opencv_world410d.dll"
$out_dir = Join-Path (Get-Location) "lib/3rdParty/OpenCV/x64/v141/bin/Debug"
if(!(Test-Path $out_dir))
{
New-Item -ItemType directory -Path $out_dir
}
#if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
#{
# $source = "https://www.dropbox.com/s/8a4kmvpj5a09jdz/opencv_world410d.dll?dl=1"
# Invoke-WebRequest $source -OutFile $destination
#}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153247&authkey=AJE4pLhzjtkPDWs"
Invoke-WebRequest $source -OutFile $destination
}
# ffmpeg x32 dll
$destination = "lib/3rdParty/OpenCV/bin/opencv_ffmpeg410.dll"
$out_dir = Join-Path (Get-Location) "lib/3rdParty/OpenCV/bin"
if(!(Test-Path $out_dir))
{
New-Item -ItemType directory -Path $out_dir
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/7pada6etpui97f7/opencv_ffmpeg410.dll?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153252&authkey=AEwNPWYZ7sOOhXo"
Invoke-WebRequest $source -OutFile $destination
}
# Release x32 dll
$destination = "lib/3rdParty/OpenCV/x86/v141/bin/Release/opencv_world410.dll"
$out_dir = Join-Path (Get-Location) "lib/3rdParty/OpenCV/x86/v141/bin/Release"
if(!(Test-Path $out_dir))
{
New-Item -ItemType directory -Path $out_dir
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/qf4rqphvrj2k4d1/opencv_world410.dll?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153255&authkey=AJ0S3GY4fCYKFXo"
Invoke-WebRequest $source -OutFile $destination
}
# Debug x32 dll
$destination = "lib/3rdParty/OpenCV/x86/v141/bin/Debug/opencv_world410d.dll"
$out_dir = Join-Path (Get-Location) "lib/3rdParty/OpenCV/x86/v141/bin/Debug"
if(!(Test-Path $out_dir))
{
New-Item -ItemType directory -Path $out_dir
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/kafps88dbdlg5y2/opencv_world410d.dll?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153256&authkey=ALbDTeRByHmWO-M"
Invoke-WebRequest $source -OutFile $destination
}

75
pkg/OpenFace/download_models.ps1 Normal file
View File

@@ -0,0 +1,75 @@
# Download the models from the cloud (stored in Dropbox and OneDrive)
# Determine correct path to the model files
if([System.IO.Directory]::Exists( (Join-Path (Get-Location) 'lib') ))
{
# If the lib folder exists, code is compiled from source
$modelPath = "lib/local/LandmarkDetector/"
}
else
{
# Otherwise, binaries are used
$modelPath = ""
}
# Start with 0.25 scale models
$destination = $modelPath + "model/patch_experts/cen_patches_0.25_of.dat"
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/7na5qsjzz8yfoer/cen_patches_0.25_of.dat?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153072&authkey=AKqoZtcN0PSIZH4"
Invoke-WebRequest $source -OutFile $destination
}
# 0.35 scale models
$destination = $modelPath + "model/patch_experts/cen_patches_0.35_of.dat"
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/k7bj804cyiu474t/cen_patches_0.35_of.dat?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153079&authkey=ANpDR1n3ckL_0gs"
Invoke-WebRequest $source -OutFile $destination
}
# 0.5 scale models
$destination = $modelPath + "model/patch_experts/cen_patches_0.50_of.dat"
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/ixt4vkbmxgab1iu/cen_patches_0.50_of.dat?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153074&authkey=AGi-e30AfRc_zvs"
Invoke-WebRequest $source -OutFile $destination
}
# 1.0 scale models
$destination = $modelPath + "model/patch_experts/cen_patches_1.00_of.dat"
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://www.dropbox.com/s/2t5t1sdpshzfhpj/cen_patches_1.00_of.dat?dl=1"
Invoke-WebRequest $source -OutFile $destination
}
if(!([System.IO.File]::Exists( (Join-Path (Get-Location) $destination) )))
{
$source = "https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153070&authkey=AD6KjtYipphwBPc"
Invoke-WebRequest $source -OutFile $destination
}

27
pkg/OpenFace/download_models.sh Normal file
View File

@@ -0,0 +1,27 @@
cd lib/local/LandmarkDetector/model/patch_experts
wget https://www.dropbox.com/s/7na5qsjzz8yfoer/cen_patches_0.25_of.dat
if [ $? -ne 0 ]
then
wget https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153072&authkey=AKqoZtcN0PSIZH4
fi
wget https://www.dropbox.com/s/k7bj804cyiu474t/cen_patches_0.35_of.dat
if [ $? -ne 0 ]
then
wget https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153079&authkey=ANpDR1n3ckL_0gs
fi
wget https://www.dropbox.com/s/ixt4vkbmxgab1iu/cen_patches_0.50_of.dat
if [ $? -ne 0 ]
then
wget https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153074&authkey=AGi-e30AfRc_zvs
fi
wget https://www.dropbox.com/s/2t5t1sdpshzfhpj/cen_patches_1.00_of.dat
if [ $? -ne 0 ]
then
wget https://onedrive.live.com/download?cid=2E2ADA578BFF6E6E&resid=2E2ADA578BFF6E6E%2153070&authkey=AD6KjtYipphwBPc
fi
cd ../../../../../

10
pkg/OpenFace/exe/FaceLandmarkImg/CMakeLists.txt Normal file
View File

@@ -0,0 +1,10 @@
# Local libraries
include_directories(${LandmarkDetector_SOURCE_DIR}/include)
add_executable(FaceLandmarkImg FaceLandmarkImg.cpp)
target_link_libraries(FaceLandmarkImg LandmarkDetector)
target_link_libraries(FaceLandmarkImg FaceAnalyser)
target_link_libraries(FaceLandmarkImg GazeAnalyser)
target_link_libraries(FaceLandmarkImg Utilities)
install (TARGETS FaceLandmarkImg DESTINATION bin)

255
pkg/OpenFace/exe/FaceLandmarkImg/FaceLandmarkImg.cpp Normal file
View File

@@ -0,0 +1,255 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// FaceLandmarkImg.cpp : Defines the entry point for the console application for detecting landmarks in images.
// dlib
#include <dlib/image_processing/frontal_face_detector.h>
#include "LandmarkCoreIncludes.h"
#include <FaceAnalyser.h>
#include <GazeEstimation.h>
#include <ImageCapture.h>
#include <Visualizer.h>
#include <VisualizationUtils.h>
#include <RecorderOpenFace.h>
#include <RecorderOpenFaceParameters.h>
#ifndef CONFIG_DIR
#define CONFIG_DIR "~"
#endif
std::vector<std::string> get_arguments(int argc, char **argv)
{
std::vector<std::string> arguments;
for (int i = 0; i < argc; ++i)
{
arguments.push_back(std::string(argv[i]));
}
return arguments;
}
int main(int argc, char **argv)
{
//Convert arguments to more convenient vector form
std::vector<std::string> arguments = get_arguments(argc, argv);
// no arguments: output usage
if (arguments.size() == 1)
{
std::cout << "For command line arguments see:" << std::endl;
std::cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
// Prepare for image reading
Utilities::ImageCapture image_reader;
// The sequence reader chooses what to open based on command line arguments provided
if (!image_reader.Open(arguments))
{
std::cout << "Could not open any images" << std::endl;
return 1;
}
// Load the models if images found
LandmarkDetector::FaceModelParameters det_parameters(arguments);
// The modules that are being used for tracking
std::cout << "Loading the model" << std::endl;
LandmarkDetector::CLNF face_model(det_parameters.model_location);
if (!face_model.loaded_successfully)
{
std::cout << "ERROR: Could not load the landmark detector" << std::endl;
return 1;
}
std::cout << "Model loaded" << std::endl;
// Load facial feature extractor and AU analyser (make sure it is static)
FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
face_analysis_params.OptimizeForImages();
FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
// If bounding boxes not provided, use a face detector
cv::CascadeClassifier classifier(det_parameters.haar_face_detector_location);
dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector();
LandmarkDetector::FaceDetectorMTCNN face_detector_mtcnn(det_parameters.mtcnn_face_detector_location);
// If can't find MTCNN face detector, default to HOG one
if (det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR && face_detector_mtcnn.empty())
{
std::cout << "INFO: defaulting to HOG-SVM face detector" << std::endl;
det_parameters.curr_face_detector = LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR;
}
// A utility for visualizing the results
Utilities::Visualizer visualizer(arguments);
cv::Mat rgb_image;
rgb_image = image_reader.GetNextImage();
if (!face_model.eye_model)
{
std::cout << "WARNING: no eye model found" << std::endl;
}
if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0)
{
std::cout << "WARNING: no Action Unit models found" << std::endl;
}
std::cout << "Starting tracking" << std::endl;
while (!rgb_image.empty())
{
Utilities::RecorderOpenFaceParameters recording_params(arguments, false, false,
image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy);
if (!face_model.eye_model)
{
recording_params.setOutputGaze(false);
}
Utilities::RecorderOpenFace open_face_rec(image_reader.name, recording_params, arguments);
visualizer.SetImage(rgb_image, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy);
// Making sure the image is in uchar grayscale (some face detectors use RGB, landmark detector uses grayscale)
cv::Mat_<uchar> grayscale_image = image_reader.GetGrayFrame();
// Detect faces in an image
std::vector<cv::Rect_<float> > face_detections;
if (image_reader.has_bounding_boxes)
{
face_detections = image_reader.GetBoundingBoxes();
}
else
{
if (det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR)
{
std::vector<float> confidences;
LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, face_detector_hog, confidences);
}
else if (det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::HAAR_DETECTOR)
{
LandmarkDetector::DetectFaces(face_detections, grayscale_image, classifier);
}
else
{
std::vector<float> confidences;
LandmarkDetector::DetectFacesMTCNN(face_detections, rgb_image, face_detector_mtcnn, confidences);
}
}
// Detect landmarks around detected faces
int face_det = 0;
// perform landmark detection for every face detected
for (size_t face = 0; face < face_detections.size(); ++face)
{
// if there are multiple detections go through them
bool success = LandmarkDetector::DetectLandmarksInImage(rgb_image, face_detections[face], face_model, det_parameters, grayscale_image);
// Estimate head pose and eye gaze
cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy);
// Gaze tracking, absolute gaze direction
cv::Point3f gaze_direction0(0, 0, -1);
cv::Point3f gaze_direction1(0, 0, -1);
cv::Vec2f gaze_angle(0, 0);
if (face_model.eye_model)
{
GazeAnalysis::EstimateGaze(face_model, gaze_direction0, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy, true);
GazeAnalysis::EstimateGaze(face_model, gaze_direction1, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy, false);
gaze_angle = GazeAnalysis::GetGazeAngle(gaze_direction0, gaze_direction1);
}
cv::Mat sim_warped_img;
cv::Mat_<double> hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0;
// Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization
if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog)
{
face_analyser.PredictStaticAUsAndComputeFeatures(rgb_image, face_model.detected_landmarks);
face_analyser.GetLatestAlignedFace(sim_warped_img);
face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols);
}
// Displaying the tracking visualizations
visualizer.SetObservationFaceAlign(sim_warped_img);
visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
visualizer.SetObservationLandmarks(face_model.detected_landmarks, 1.0, face_model.GetVisibilities()); // Set confidence to high to make sure we always visualize
visualizer.SetObservationPose(pose_estimate, 1.0);
visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy), face_model.detection_certainty);
visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
// Setting up the recorder output
open_face_rec.SetObservationHOG(face_model.detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy),
face_model.params_global, face_model.params_local, face_model.detection_certainty, face_model.detection_success);
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy));
open_face_rec.SetObservationFaceAlign(sim_warped_img);
open_face_rec.SetObservationFaceID(face);
open_face_rec.WriteObservation();
}
if (face_detections.size() > 0)
{
visualizer.ShowObservation();
}
open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
open_face_rec.WriteObservationTracked();
open_face_rec.Close();
// Grabbing the next frame in the sequence
rgb_image = image_reader.GetNextImage();
}
return 0;
}

201
pkg/OpenFace/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj Normal file
View File

@@ -0,0 +1,201 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{DDC3535E-526C-44EC-9DF4-739E2D3A323B}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>FaceLandmarkImg</RootNamespace>
<ProjectName>FaceLandmarkImg</ProjectName>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>FaceLandmarkImg</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>FaceLandmarkImg</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<TargetName>FaceLandmarkImg</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>FaceLandmarkImg</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="FaceLandmarkImg.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\lib\local\FaceAnalyser\FaceAnalyser.vcxproj">
<Project>{0e7fc556-0e80-45ea-a876-dde4c2fedcd7}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\GazeAnalyser\GazeAnalyser.vcxproj">
<Project>{5f915541-f531-434f-9c81-79f5db58012b}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

23
pkg/OpenFace/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj.user Normal file
View File

@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LocalDebuggerCommandArguments>
</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LocalDebuggerCommandArguments>
</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LocalDebuggerCommandArguments>
</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LocalDebuggerCommandArguments>
</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
</Project>

10
pkg/OpenFace/exe/FaceLandmarkVid/CMakeLists.txt Normal file
View File

@@ -0,0 +1,10 @@
# Local libraries
include_directories(${LandmarkDetector_SOURCE_DIR}/include)
add_executable(FaceLandmarkVid FaceLandmarkVid.cpp)
target_link_libraries(FaceLandmarkVid LandmarkDetector)
target_link_libraries(FaceLandmarkVid FaceAnalyser)
target_link_libraries(FaceLandmarkVid GazeAnalyser)
target_link_libraries(FaceLandmarkVid Utilities)
install (TARGETS FaceLandmarkVid DESTINATION bin)

314
pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.cpp Normal file
View File

@@ -0,0 +1,314 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// FaceTrackingVid.cpp : Defines the entry point for the console application for tracking faces in videos.
// Libraries for landmark detection (includes CLNF and CLM modules)
#include "LandmarkCoreIncludes.h"
#include "GazeEstimation.h"
#include <FaceAnalyser.h>
#include <SequenceCapture.h>
#include <Visualizer.h>
#include <VisualizationUtils.h>
#include <RecorderOpenFace.h>
#include <RecorderOpenFaceParameters.h>
#include <iostream>
#include <fstream>
#define INFO_STREAM( stream ) \
std::cout << stream << std::endl
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
#define ERROR_STREAM( stream ) \
std::cout << "Error: " << stream << std::endl
static void printErrorAndAbort(const std::string & error)
{
std::cout << error << std::endl;
abort();
}
#define FATAL_STREAM( stream ) \
printErrorAndAbort( std::string( "Fatal error: " ) + stream )
std::vector<std::string> get_arguments(int argc, std::string *out_dir, char **argv)
{
std::vector<std::string> arguments;
for (int i = 0; i < argc; ++i)
{
arguments.push_back(std::string(argv[i]));
if (std::string(argv[i]).compare("-out_dir") == 0)
{
*out_dir = std::string(argv[i+1]);
}
}
return arguments;
}
int main(int argc, char **argv)
{
std::string out_dir = ".";
std::vector<std::string> arguments = get_arguments(argc, &out_dir, argv);
std::cout<< "out_dir:" << out_dir <<std::endl;
// no arguments: output usage
if (arguments.size() == 1)
{
std::cout << "For command line arguments see:" << std::endl;
std::cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
LandmarkDetector::FaceModelParameters det_parameters(arguments);
// The modules that are being used for tracking
LandmarkDetector::CLNF face_model(det_parameters.model_location);
if (!face_model.loaded_successfully)
{
std::cout << "ERROR: Could not load the landmark detector" << std::endl;
return 1;
}
if (!face_model.eye_model)
{
std::cout << "WARNING: no eye model found" << std::endl;
}
// Open a sequence
Utilities::SequenceCapture sequence_reader;
// A utility for visualizing the results (show just the tracks)
Utilities::Visualizer visualizer(true, false, false, false);
// Tracking FPS for visualization
Utilities::FpsTracker fps_tracker;
fps_tracker.AddFrame();
int sequence_number = 0;
std::string ext = ".mp4";
while (true) // this is not a for loop as we might also be reading from a webcam
{
// The sequence reader chooses what to open based on command line arguments provided
if (!sequence_reader.Open(arguments))
break;
INFO_STREAM("Device or file opened");
cv::Mat rgb_image = sequence_reader.GetNextFrame();
INFO_STREAM("Starting tracking");
std::ofstream results;
std::ofstream confidence;
std::string path = sequence_reader.name;
std::string base_filename = path.substr(path.find_last_of("/\\") + 1);
base_filename = base_filename.replace(base_filename.find(ext),sizeof(ext)-1,"");
results.open(out_dir + '/' + base_filename + "_landmark_output.csv");
// confidence.open(out_dir + '/' + base_filename + "_landmark_likelihoods.csv");
int lx = 0;
int ly = 0;
for(lx = 0; lx < 2; lx++){
for(ly = 0; ly < 68; ly++){
if (lx == 0){
results << "l" << ly << "_x,";
// confidence << "c" << ly <<",";
}
if (lx == 1){
results << "l" << ly << "_y,";
}
}
}
results << "pose_Tx,pose_Ty,pose_Tz,pose_Rx,pose_Ry,pose_Rz" ;
results << std::endl;
// confidence << std::endl;
int counter = 0;
// FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
// face_analysis_params.OptimizeForImages();
// FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
while (!rgb_image.empty()) // this is not a for loop as we might also be reading from a webcam
{
// Added lines
// Utilities::RecorderOpenFaceParameters recording_params(arguments, false, false,
// sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// std::string stem = sequence_reader.name;
// stem = stem.replace(stem.find(ext),sizeof(ext)-1,"_");
// Utilities::RecorderOpenFace open_face_rec(stem+std::to_string(counter)+ext, recording_params, arguments);
// Reading the images
cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
// The actual facial landmark detection / tracking
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_model, det_parameters, grayscale_image);
// Gaze tracking, absolute gaze direction
cv::Point3f gazeDirection0(0, 0, -1);
cv::Point3f gazeDirection1(0, 0, -1);
cv::Vec6d pose_estimate(0, 0, 0, 0, 0, 0);
// If tracking succeeded and we have an eye model, estimate gaze
if (detection_success && face_model.eye_model)
{
GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
}
// Work out the pose of the head from the tracked model
if (detection_success){
pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
}
// cv::Mat sim_warped_img;
// face_analyser.PredictStaticAUsAndComputeFeatures(rgb_image, face_model.detected_landmarks);
// face_analyser.GetLatestAlignedFace(sim_warped_img);
// Keeping track of FPS
fps_tracker.AddFrame();
// Displaying the tracking visualizations
// std::cout<< "setting observation landmarks"<<std::endl;
// visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
// visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
// visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
// visualizer.SetFps(fps_tracker.GetFPS());
// std::cout << "openfacerec set obs landmarks"<<std::endl;
// std::cout<< fps_tracker.GetFPS() <<std::endl;
// open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
// face_model.params_global, face_model.params_local, face_model.detection_certainty, face_model.detection_success);
// open_face_rec.SetObservationPose(pose_estimate);
// open_face_rec.SetObservationFaceAlign(sim_warped_img);
int i;
for (i=0;i< 136;i++){
results << face_model.detected_landmarks[0][i] << ",";
}
for (i=0;i< 6;i++){
if (i==5){
results << pose_estimate[i];
}
else{
results << pose_estimate[i] << ",";
}
}
results <<std::endl;
// for(i=0;i<68;i++){
// if (i==67){
// confidence << face_model.landmark_likelihoods[0][i];
// }
// else{
// confidence << face_model.landmark_likelihoods[0][i] << ",";
// }
// }
// confidence <<std::endl;
// detect key presses (due to pecularities of OpenCV, you can get it when displaying images)
//char character_press = visualizer.ShowObservation();
char character_press = 't';
// restart the tracker
if (character_press == 'r')
{
face_model.Reset();
}
// quit the application
else if (character_press == 'q')
{
return(0);
}
// added lines
// open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
// open_face_rec.WriteObservationTracked();
// open_face_rec.Close();
// Grabbing the next frame in the sequence
rgb_image = sequence_reader.GetNextFrame();
counter++;
}
// Reset the model, for the next video
face_model.Reset();
sequence_reader.Close();
sequence_number++;
results.close();
}
return 0;
}

203
pkg/OpenFace/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj Normal file
View File

@@ -0,0 +1,203 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{34032CF2-1B99-4A25-9050-E9C13DD4CD0A}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>FaceLandmarkVid</RootNamespace>
<ProjectName>FaceLandmarkVid</ProjectName>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>FaceLandmarkVid</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>FaceLandmarkVid</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<TargetName>FaceLandmarkVid</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>FaceLandmarkVid</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="FaceLandmarkVid.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\lib\local\FaceAnalyser\FaceAnalyser.vcxproj">
<Project>{0e7fc556-0e80-45ea-a876-dde4c2fedcd7}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\GazeAnalyser\GazeAnalyser.vcxproj">
<Project>{5f915541-f531-434f-9c81-79f5db58012b}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

10
pkg/OpenFace/exe/FaceLandmarkVidMulti/CMakeLists.txt Normal file
View File

@@ -0,0 +1,10 @@
# Local libraries
include_directories(${LandmarkDetector_SOURCE_DIR}/include)
add_executable(FaceLandmarkVidMulti FaceLandmarkVidMulti.cpp)
target_link_libraries(FaceLandmarkVidMulti LandmarkDetector)
target_link_libraries(FaceLandmarkVidMulti FaceAnalyser)
target_link_libraries(FaceLandmarkVidMulti GazeAnalyser)
target_link_libraries(FaceLandmarkVidMulti Utilities)
install (TARGETS FaceLandmarkVidMulti DESTINATION bin)

477
pkg/OpenFace/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp Normal file
View File

@@ -0,0 +1,477 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// FaceTrackingVidMulti.cpp : Defines the entry point for the multiple face tracking console application.
#include "LandmarkCoreIncludes.h"
#include "VisualizationUtils.h"
#include "Visualizer.h"
#include "SequenceCapture.h"
#include <RecorderOpenFace.h>
#include <RecorderOpenFaceParameters.h>
#include <GazeEstimation.h>
#include <FaceAnalyser.h>
#define INFO_STREAM( stream ) \
std::cout << stream << std::endl
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
#define ERROR_STREAM( stream ) \
std::cout << "Error: " << stream << std::endl
static void printErrorAndAbort(const std::string & error)
{
std::cout << error << std::endl;
abort();
}
#define FATAL_STREAM( stream ) \
printErrorAndAbort( std::string( "Fatal error: " ) + stream )
std::vector<std::string> get_arguments(int argc, char **argv)
{
std::vector<std::string> arguments;
for (int i = 0; i < argc; ++i)
{
arguments.push_back(std::string(argv[i]));
}
return arguments;
}
double IOU(cv::Rect_<float> rect1, cv::Rect_<float> rect2)
{
double intersection_area = (rect1 & rect2).area();
double union_area = rect1.area() + rect2.area() - intersection_area;
return intersection_area / union_area;
}
void RemoveOverlapingModels(std::vector<LandmarkDetector::CLNF>& face_models, std::vector<bool>& active_models)
{
// Go over the model and eliminate detections that are not informative (there already is a tracker there)
for (size_t model1 = 0; model1 < active_models.size(); ++model1)
{
if (active_models[model1])
{
// See if the detections intersect
cv::Rect_<float> model1_rect = face_models[model1].GetBoundingBox();
for (int model2 = model1 + 1; model2 < active_models.size(); ++model2)
{
if(active_models[model2])
{
cv::Rect_<float> model2_rect = face_models[model2].GetBoundingBox();
// If the model is already tracking what we're detecting ignore the detection, this is determined by amount of overlap
if (IOU(model1_rect, model2_rect) > 0.5)
{
active_models[model1] = false;
face_models[model1].Reset();
}
}
}
}
}
}
void NonOverlapingDetections(const std::vector<LandmarkDetector::CLNF>& clnf_models, std::vector<cv::Rect_<float> >& face_detections)
{
// Go over the model and eliminate detections that are not informative (there already is a tracker there)
for (size_t model = 0; model < clnf_models.size(); ++model)
{
// See if the detections intersect
cv::Rect_<float> model_rect = clnf_models[model].GetBoundingBox();
for (int detection = face_detections.size() - 1; detection >= 0; --detection)
{
// If the model is already tracking what we're detecting ignore the detection, this is determined by amount of overlap
if (IOU(model_rect, face_detections[detection]) > 0.5)
{
face_detections.erase(face_detections.begin() + detection);
}
}
}
}
int main(int argc, char **argv)
{
std::vector<std::string> arguments = get_arguments(argc, argv);
// no arguments: output usage
if (arguments.size() == 1)
{
std::cout << "For command line arguments see:" << std::endl;
std::cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
LandmarkDetector::FaceModelParameters det_params(arguments);
// This is so that the model would not try re-initialising itself
det_params.reinit_video_every = -1;
det_params.curr_face_detector = LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR;
std::vector<LandmarkDetector::FaceModelParameters> det_parameters;
det_parameters.push_back(det_params);
// The modules that are being used for tracking
std::vector<LandmarkDetector::CLNF> face_models;
std::vector<bool> active_models;
int num_faces_max = 4;
LandmarkDetector::CLNF face_model(det_parameters[0].model_location);
if (!face_model.loaded_successfully)
{
std::cout << "ERROR: Could not load the landmark detector" << std::endl;
return 1;
}
// Loading the face detectors
face_model.face_detector_HAAR.load(det_parameters[0].haar_face_detector_location);
face_model.haar_face_detector_location = det_parameters[0].haar_face_detector_location;
face_model.face_detector_MTCNN.Read(det_parameters[0].mtcnn_face_detector_location);
face_model.mtcnn_face_detector_location = det_parameters[0].mtcnn_face_detector_location;
// If can't find MTCNN face detector, default to HOG one
if (det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR && face_model.face_detector_MTCNN.empty())
{
std::cout << "INFO: defaulting to HOG-SVM face detector" << std::endl;
det_parameters[0].curr_face_detector = LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR;
}
face_models.reserve(num_faces_max);
face_models.push_back(face_model);
active_models.push_back(false);
for (int i = 1; i < num_faces_max; ++i)
{
face_models.push_back(face_model);
active_models.push_back(false);
det_parameters.push_back(det_params);
}
// Load facial feature extractor and AU analyser (make sure it is static, as we don't reidentify faces)
FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
face_analysis_params.OptimizeForImages();
FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
if (!face_model.eye_model)
{
std::cout << "WARNING: no eye model found" << std::endl;
}
if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0)
{
std::cout << "WARNING: no Action Unit models found" << std::endl;
}
// Open a sequence
Utilities::SequenceCapture sequence_reader;
// A utility for visualizing the results (show just the tracks)
Utilities::Visualizer visualizer(arguments);
// Tracking FPS for visualization
Utilities::FpsTracker fps_tracker;
fps_tracker.AddFrame();
int sequence_number = 0;
while (true) // this is not a for loop as we might also be reading from a webcam
{
// The sequence reader chooses what to open based on command line arguments provided
if (!sequence_reader.Open(arguments))
break;
INFO_STREAM("Device or file opened");
cv::Mat rgb_image = sequence_reader.GetNextFrame();
int frame_count = 0;
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(),
sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps);
if (!face_model.eye_model)
{
recording_params.setOutputGaze(false);
}
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
if (sequence_reader.IsWebcam())
{
INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)");
visualizer.vis_track = true;
}
if (recording_params.outputAUs())
{
INFO_STREAM("WARNING: using a AU detection in multiple face mode, it might not be as accurate and is experimental");
}
// For reporting progress
double reported_completion = 0;
INFO_STREAM("Starting tracking");
while (!rgb_image.empty())
{
// Reading the images
cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
std::vector<cv::Rect_<float> > face_detections;
bool all_models_active = true;
for (unsigned int model = 0; model < face_models.size(); ++model)
{
if (!active_models[model])
{
all_models_active = false;
}
}
// Get the detections (every 8th frame and when there are free models available for tracking)
if (frame_count % 8 == 0 && !all_models_active)
{
if (det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR)
{
std::vector<float> confidences;
LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, face_models[0].face_detector_HOG, confidences);
}
else if (det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HAAR_DETECTOR)
{
LandmarkDetector::DetectFaces(face_detections, grayscale_image, face_models[0].face_detector_HAAR);
}
else
{
std::vector<float> confidences;
LandmarkDetector::DetectFacesMTCNN(face_detections, rgb_image, face_models[0].face_detector_MTCNN, confidences);
}
}
// Keep only non overlapping detections (so as not to start tracking where the face is already tracked)
NonOverlapingDetections(face_models, face_detections);
std::vector<bool> face_detections_used(face_detections.size(), false);
// Go through every model and update the tracking
for (unsigned int model = 0; model < face_models.size(); ++model)
{
bool detection_success = false;
// If the current model has failed more than 4 times in a row, remove it
if (face_models[model].failures_in_a_row > 4)
{
active_models[model] = false;
face_models[model].Reset();
}
// If the model is inactive reactivate it with new detections
if (!active_models[model])
{
for (size_t detection_ind = 0; detection_ind < face_detections.size(); ++detection_ind)
{
// if it was not taken by another tracker take it
if (!face_detections_used[detection_ind])
{
face_detections_used[detection_ind] = true;
// Reinitialise the model
face_models[model].Reset();
// This ensures that a wider window is used for the initial landmark localisation
face_models[model].detection_success = false;
detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_detections[detection_ind], face_models[model], det_parameters[model], grayscale_image);
// This activates the model
active_models[model] = true;
// break out of the loop as the tracker has been reinitialised
break;
}
}
}
else
{
// The actual facial landmark detection / tracking
detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_models[model], det_parameters[model], grayscale_image);
}
}
// Remove models that end up tracking overlapping faces
// even if initial bounding boxes were not overlapping, they could have ended up converging to the same face
RemoveOverlapingModels(face_models, active_models);
// Keeping track of FPS
fps_tracker.AddFrame();
visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// Go through every model and detect eye gaze, record results and visualise the results
for (size_t model = 0; model < face_models.size(); ++model)
{
// Visualising and recording the results
if (active_models[model])
{
// Estimate head pose and eye gaze
cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
cv::Point3f gaze_direction0(0, 0, 0); cv::Point3f gaze_direction1(0, 0, 0); cv::Vec2d gaze_angle(0, 0);
// Detect eye gazes
if (face_models[model].detection_success && face_model.eye_model)
{
GazeAnalysis::EstimateGaze(face_models[model], gaze_direction0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(face_models[model], gaze_direction1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
gaze_angle = GazeAnalysis::GetGazeAngle(gaze_direction0, gaze_direction1);
}
// Face analysis step
cv::Mat sim_warped_img;
cv::Mat_<double> hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0;
// Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization
if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog)
{
face_analyser.PredictStaticAUsAndComputeFeatures(rgb_image, face_models[model].detected_landmarks);
face_analyser.GetLatestAlignedFace(sim_warped_img);
face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols);
}
// Visualize the features
visualizer.SetObservationFaceAlign(sim_warped_img);
visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
visualizer.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].detection_certainty);
visualizer.SetObservationPose(LandmarkDetector::GetPose(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty);
visualizer.SetObservationGaze(gaze_direction0, gaze_direction1, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_models[model].detection_certainty);
visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
// Output features
open_face_rec.SetObservationHOG(face_models[model].detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
open_face_rec.SetObservationLandmarks(face_models[model].detected_landmarks, face_models[model].GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
face_models[model].params_global, face_models[model].params_local, face_models[model].detection_certainty, face_models[model].detection_success);
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationGaze(gaze_direction0, gaze_direction1, gaze_angle, LandmarkDetector::CalculateAllEyeLandmarks(face_models[model]), LandmarkDetector::Calculate3DEyeLandmarks(face_models[model], sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
open_face_rec.SetObservationFaceAlign(sim_warped_img);
open_face_rec.SetObservationFaceID(model);
open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber());
open_face_rec.WriteObservation();
}
}
visualizer.SetFps(fps_tracker.GetFPS());
// Record frame
open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
open_face_rec.WriteObservationTracked();
// show visualization and detect key presses
char character_press = visualizer.ShowObservation();
// restart the trackers
if (character_press == 'r')
{
for (size_t i = 0; i < face_models.size(); ++i)
{
face_models[i].Reset();
active_models[i] = false;
}
}
// quit the application
else if (character_press == 'q')
{
return 0;
}
// Reporting progress
if (sequence_reader.GetProgress() >= reported_completion / 10.0)
{
std::cout << reported_completion * 10 << "% ";
if (reported_completion == 10)
{
std::cout << std::endl;
}
reported_completion = reported_completion + 1;
}
// Update the frame count
frame_count++;
// Grabbing the next frame in the sequence
rgb_image = sequence_reader.GetNextFrame();
}
frame_count = 0;
// Reset the model, for the next video
for (size_t model = 0; model < face_models.size(); ++model)
{
face_models[model].Reset();
active_models[model] = false;
}
INFO_STREAM("Closing output recorder");
open_face_rec.Close();
INFO_STREAM("Closing input reader");
sequence_reader.Close();
INFO_STREAM("Closed successfully");
sequence_number++;
}
return 0;
}

185
pkg/OpenFace/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.vcxproj Normal file
View File

@@ -0,0 +1,185 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{C3FAF36F-44BC-4454-87C2-C5106575FE50}</ProjectGuid>
<RootNamespace>FaceLandmarkVidMulti</RootNamespace>
<ProjectName>FaceLandmarkVidMulti</ProjectName>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<TargetName>FaceLandmarkVidMulti</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>FaceLandmarkVidMulti</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<TargetName>FaceLandmarkVidMulti</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>FaceLandmarkVidMulti</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\Utilities\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="FaceLandmarkVidMulti.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\lib\local\FaceAnalyser\FaceAnalyser.vcxproj">
<Project>{0e7fc556-0e80-45ea-a876-dde4c2fedcd7}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\GazeAnalyser\GazeAnalyser.vcxproj">
<Project>{5f915541-f531-434f-9c81-79f5db58012b}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

10
pkg/OpenFace/exe/FeatureExtraction/CMakeLists.txt Normal file
View File

@@ -0,0 +1,10 @@
# Local libraries
include_directories(${LandmarkDetector_SOURCE_DIR}/include)
add_executable(FeatureExtraction FeatureExtraction.cpp)
target_link_libraries(FeatureExtraction LandmarkDetector)
target_link_libraries(FeatureExtraction FaceAnalyser)
target_link_libraries(FeatureExtraction GazeAnalyser)
target_link_libraries(FeatureExtraction Utilities)
install (TARGETS FeatureExtraction DESTINATION bin)

273
pkg/OpenFace/exe/FeatureExtraction/FeatureExtraction.cpp Normal file
View File

@@ -0,0 +1,273 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// FeatureExtraction.cpp : Defines the entry point for the feature extraction console application.
// Local includes
#include "LandmarkCoreIncludes.h"
#include <Face_utils.h>
#include <FaceAnalyser.h>
#include <GazeEstimation.h>
#include <RecorderOpenFace.h>
#include <RecorderOpenFaceParameters.h>
#include <SequenceCapture.h>
#include <Visualizer.h>
#include <VisualizationUtils.h>
#ifndef CONFIG_DIR
#define CONFIG_DIR "~"
#endif
#define INFO_STREAM( stream ) \
std::cout << stream << std::endl
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
#define ERROR_STREAM( stream ) \
std::cout << "Error: " << stream << std::endl
static void printErrorAndAbort(const std::string & error)
{
std::cout << error << std::endl;
}
#define FATAL_STREAM( stream ) \
printErrorAndAbort( std::string( "Fatal error: " ) + stream )
std::vector<std::string> get_arguments(int argc, char **argv)
{
std::vector<std::string> arguments;
// First argument is reserved for the name of the executable
for (int i = 0; i < argc; ++i)
{
arguments.push_back(std::string(argv[i]));
}
return arguments;
}
int main(int argc, char **argv)
{
std::vector<std::string> arguments = get_arguments(argc, argv);
// no arguments: output usage
if (arguments.size() == 1)
{
std::cout << "For command line arguments see:" << std::endl;
std::cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
return 0;
}
// Load the modules that are being used for tracking and face analysis
// Load face landmark detector
LandmarkDetector::FaceModelParameters det_parameters(arguments);
// Always track gaze in feature extraction
LandmarkDetector::CLNF face_model(det_parameters.model_location);
if (!face_model.loaded_successfully)
{
std::cout << "ERROR: Could not load the landmark detector" << std::endl;
return 1;
}
// Load facial feature extractor and AU analyser
FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments);
FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params);
if (!face_model.eye_model)
{
std::cout << "WARNING: no eye model found" << std::endl;
}
if (face_analyser.GetAUClassNames().size() == 0 && face_analyser.GetAUClassNames().size() == 0)
{
std::cout << "WARNING: no Action Unit models found" << std::endl;
}
Utilities::SequenceCapture sequence_reader;
// A utility for visualizing the results
Utilities::Visualizer visualizer(arguments);
// Tracking FPS for visualization
Utilities::FpsTracker fps_tracker;
fps_tracker.AddFrame();
while (true) // this is not a for loop as we might also be reading from a webcam
{
// The sequence reader chooses what to open based on command line arguments provided
if (!sequence_reader.Open(arguments))
break;
INFO_STREAM("Device or file opened");
if (sequence_reader.IsWebcam())
{
INFO_STREAM("WARNING: using a webcam in feature extraction, Action Unit predictions will not be as accurate in real-time webcam mode");
INFO_STREAM("WARNING: using a webcam in feature extraction, forcing visualization of tracking to allow quitting the application (press q)");
visualizer.vis_track = true;
}
cv::Mat captured_image;
Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(),
sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps);
if (!face_model.eye_model)
{
recording_params.setOutputGaze(false);
}
Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments);
if (recording_params.outputGaze() && !face_model.eye_model)
std::cout << "WARNING: no eye model defined, but outputting gaze" << std::endl;
captured_image = sequence_reader.GetNextFrame();
// For reporting progress
double reported_completion = 0;
INFO_STREAM("Starting tracking");
while (!captured_image.empty())
{
// Converting to grayscale
cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
// The actual facial landmark detection / tracking
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters, grayscale_image);
// Gaze tracking, absolute gaze direction
cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0);
if (detection_success && face_model.eye_model)
{
GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1);
}
// Do face alignment
cv::Mat sim_warped_img;
cv::Mat_<double> hog_descriptor; int num_hog_rows = 0, num_hog_cols = 0;
// Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization
if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog || visualizer.vis_aus)
{
face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam());
face_analyser.GetLatestAlignedFace(sim_warped_img);
face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols);
}
// Work out the pose of the head from the tracked model
cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
// Keeping track of FPS
fps_tracker.AddFrame();
// Displaying the tracking visualizations
visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
visualizer.SetObservationFaceAlign(sim_warped_img);
visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols);
visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
visualizer.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
visualizer.SetFps(fps_tracker.GetFPS());
// detect key presses
char character_press = visualizer.ShowObservation();
// quit processing the current sequence (useful when in Webcam mode)
if (character_press == 'q')
{
break;
}
// Setting up the recorder output
open_face_rec.SetObservationHOG(detection_success, hog_descriptor, num_hog_rows, num_hog_cols, 31); // The number of channels in HOG is fixed at the moment, as using FHOG
open_face_rec.SetObservationVisualization(visualizer.GetVisImage());
open_face_rec.SetObservationActionUnits(face_analyser.GetCurrentAUsReg(), face_analyser.GetCurrentAUsClass());
open_face_rec.SetObservationLandmarks(face_model.detected_landmarks, face_model.GetShape(sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy),
face_model.params_global, face_model.params_local, face_model.detection_certainty, detection_success);
open_face_rec.SetObservationPose(pose_estimate);
open_face_rec.SetObservationGaze(gazeDirection0, gazeDirection1, gazeAngle, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy));
open_face_rec.SetObservationTimestamp(sequence_reader.time_stamp);
open_face_rec.SetObservationFaceID(0);
open_face_rec.SetObservationFrameNumber(sequence_reader.GetFrameNumber());
open_face_rec.SetObservationFaceAlign(sim_warped_img);
open_face_rec.WriteObservation();
open_face_rec.WriteObservationTracked();
// Reporting progress
if (sequence_reader.GetProgress() >= reported_completion / 10.0)
{
std::cout << reported_completion * 10 << "% ";
if (reported_completion == 10)
{
std::cout << std::endl;
}
reported_completion = reported_completion + 1;
}
// Grabbing the next frame in the sequence
captured_image = sequence_reader.GetNextFrame();
}
INFO_STREAM("Closing output recorder");
open_face_rec.Close();
INFO_STREAM("Closing input reader");
sequence_reader.Close();
INFO_STREAM("Closed successfully");
if (recording_params.outputAUs())
{
INFO_STREAM("Postprocessing the Action Unit predictions");
face_analyser.PostprocessOutputFile(open_face_rec.GetCSVFile());
}
// Reset the models for the next video
face_analyser.Reset();
face_model.Reset();
}
return 0;
}

204
pkg/OpenFace/exe/FeatureExtraction/FeatureExtraction.vcxproj Normal file
View File

@@ -0,0 +1,204 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{8A23C00D-767D-422D-89A3-CF225E3DAB4B}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>FeatureExtraction</RootNamespace>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\dlib\dlib.props" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\lib\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>FeatureExtraction</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>FeatureExtraction</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<TargetName>FeatureExtraction</TargetName>
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>FeatureExtraction</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;$(SolutionDir)\lib\local\Utilities\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<OpenMPSupport>false</OpenMPSupport>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="FeatureExtraction.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\lib\local\FaceAnalyser\FaceAnalyser.vcxproj">
<Project>{0e7fc556-0e80-45ea-a876-dde4c2fedcd7}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\GazeAnalyser\GazeAnalyser.vcxproj">
<Project>{5f915541-f531-434f-9c81-79f5db58012b}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

201
pkg/OpenFace/exe/Recording/Record.cpp Normal file
View File

@@ -0,0 +1,201 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// Record.cpp : A useful function for quick recording from a webcam for test purposes
#include <fstream>
#include <sstream>
#include <iostream>
#include <windows.h>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <stdio.h>
#include <time.h>
#include <filesystem>
#define INFO_STREAM( stream ) \
std::cout << stream << std::endl
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
#define ERROR_STREAM( stream ) \
std::cout << "Error: " << stream << std::endl
static void printErrorAndAbort( const std::string & error )
{
std::cout << error << std::endl;
abort();
}
#define FATAL_STREAM( stream ) \
printErrorAndAbort( std::string( "Fatal error: " ) + stream )
// Get current date/time, format is YYYY-MM-DD.HH:mm:ss
const std::string currentDateTime() {
time_t now = time(0);
struct tm tstruct;
char buf[80];
localtime_s(&tstruct, &now);
// Visit http://www.cplusplus.com/reference/clibrary/ctime/strftime/
// for more information about date/time format
strftime(buf, sizeof(buf), "%Y-%m-%d-%H-%M", &tstruct);
return buf;
}
std::vector<std::string> get_arguments(int argc, char **argv)
{
std::vector<std::string> arguments;
for(int i = 1; i < argc; ++i)
{
arguments.push_back(std::string(argv[i]));
}
return arguments;
}
int main (int argc, char **argv)
{
std::vector<std::string> arguments = get_arguments(argc, argv);
// Some initial parameters that can be overriden from command line
std::string outroot, outfile;
TCHAR NPath[200];
GetCurrentDirectory(200, NPath);
// By default write to same directory
outroot = NPath;
outroot = outroot + "/recording/";
outfile = currentDateTime() + ".avi";
// By default try webcam
int device = 0;
for (size_t i = 0; i < arguments.size(); i++)
{
if( strcmp( arguments[i].c_str(), "-dev") == 0 )
{
std::stringstream ss;
ss << arguments[i+1].c_str();
ss >> device;
}
else if (strcmp(arguments[i].c_str(), "-r") == 0)
{
outroot = arguments[i+1];
}
else if (strcmp(arguments[i].c_str(), "-of") == 0)
{
outroot = arguments[i+1];
}
else
{
WARN_STREAM( "invalid argument" );
}
i++;
}
// Do some grabbing
cv::VideoCapture vCap;
INFO_STREAM( "Attempting to capture from device: " << device );
vCap = cv::VideoCapture( device );
if (!vCap.isOpened()) {
FATAL_STREAM("Failed to open video source");
return 1;
}
cv::Mat img;
vCap >> img;
std::filesystem::path dir(outroot);
std::filesystem::create_directory(dir);
std::string out_file = outroot + outfile;
// saving the videos
cv::VideoWriter video_writer(out_file, cv::VideoWriter::fourcc('D','I','V','X'), 30, img.size(), true);
std::ofstream outlog;
outlog.open((outroot + outfile + ".log").c_str(), std::ios_base::out);
outlog << "frame, time(ms)" << std::endl;
double freq = cv::getTickFrequency();
double init_time = (double)cv::getTickCount();
int frameProc = 0;
while(!img.empty())
{
cv::namedWindow("rec",1);
vCap >> img;
double curr_time = (cv::getTickCount() - init_time) / freq;
curr_time *= 1000;
video_writer << img;
outlog << frameProc + 1 << " " << curr_time;
outlog << std::endl;
cv::imshow("rec", img);
frameProc++;
// detect key presses
char c = cv::waitKey(1);
// quit the application
if(c=='q')
{
outlog.close();
return(0);
}
}
return 0;
}

146
pkg/OpenFace/exe/Recording/Recording.vcxproj Normal file
View File

@@ -0,0 +1,146 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC}</ProjectGuid>
<RootNamespace>Recording</RootNamespace>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v141</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\lib\3rdParty\OpenCV\openCV.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<IntDir>$(ProjectDir)$(Configuration)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<OpenMPSupport>false</OpenMPSupport>
<CallingConvention>Cdecl</CallingConvention>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<Profile>true</Profile>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<OpenMPSupport>false</OpenMPSupport>
<CallingConvention>Cdecl</CallingConvention>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<Profile>true</Profile>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="Record.cpp" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

17
pkg/OpenFace/exe/Recording/Recording.vcxproj.user Normal file
View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LocalDebuggerCommandArguments>
</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
</Project>

158
pkg/OpenFace/exe/releases/package_windows_executables.m Normal file
View File

@@ -0,0 +1,158 @@
clear;
version = '2.2.0';
out_x86 = sprintf('OpenFace_%s_win_x86', version);
out_x64 = sprintf('OpenFace_%s_win_x64', version);
mkdir(out_x86);
mkdir(out_x64);
in_x86 = '../../Release/';
in_x64 = '../../x64/Release/';
% Copy models
copyfile([in_x86, 'AU_predictors'], [out_x86, '/AU_predictors'])
copyfile([in_x86, 'classifiers'], [out_x86, '/classifiers'])
copyfile([in_x86, 'model'], [out_x86, '/model'])
copyfile([in_x64, 'AU_predictors'], [out_x64, '/AU_predictors'])
copyfile([in_x64, 'classifiers'], [out_x64, '/classifiers'])
copyfile([in_x64, 'model'], [out_x64, '/model'])
copyfile('readme.txt', out_x86);
copyfile('../../download_models.ps1', out_x86);
copyfile('readme.txt', out_x64);
copyfile('../../download_models.ps1', out_x64);
%% Copy libraries
libs_x86 = dir([in_x86, '*.lib'])';
for lib = libs_x86
copyfile([in_x86, '/', lib.name], [out_x86, '/', lib.name])
end
libs_x64 = dir([in_x64, '*.lib'])';
for lib = libs_x64
copyfile([in_x64, '/', lib.name], [out_x64, '/', lib.name])
end
%% Copy dlls
dlls_x86 = dir([in_x86, '*.dll'])';
for dll = dlls_x86
copyfile([in_x86, '/', dll.name], [out_x86, '/', dll.name])
end
dlls_x64 = dir([in_x64, '*.dll'])';
for dll = dlls_x64
copyfile([in_x64, '/', dll.name], [out_x64, '/', dll.name])
end
% Copy zmq dll's
mkdir([out_x64, '/amd64']);
copyfile([in_x64, '/amd64'], [out_x64, '/amd64']);
mkdir([out_x64, '/i386']);
copyfile([in_x64, '/i386'], [out_x64, '/i386']);
mkdir([out_x86, '/amd64']);
copyfile([in_x86, '/amd64'], [out_x86, '/amd64']);
mkdir([out_x86, '/i386']);
copyfile([in_x86, '/i386'], [out_x86, '/i386']);
%% Copy exe's
exes_x86 = dir([in_x86, '*.exe'])';
for exe = exes_x86
copyfile([in_x86, '/', exe.name], [out_x86, '/', exe.name])
end
exes_x64 = dir([in_x64, '*.exe'])';
for exe = exes_x64
copyfile([in_x64, '/', exe.name], [out_x64, '/', exe.name])
end
%% Copy license and copyright
copyfile('../../Copyright.txt', [out_x86, '/Copyright.txt']);
copyfile('../../OpenFace-license.txt', [out_x86, '/OpenFace-license.txt']);
copyfile('../../Copyright.txt', [out_x64, '/Copyright.txt']);
copyfile('../../OpenFace-license.txt', [out_x64, '/OpenFace-license.txt']);
%% Copy icons etc. needed for GUI
img_x86 = dir([in_x86, '*.ico'])';
for img = img_x86
copyfile([in_x86, '/', img.name], [out_x86, '/', img.name])
end
img_x64 = dir([in_x64, '*.ico'])';
for img = img_x64
copyfile([in_x64, '/', img.name], [out_x64, '/', img.name])
end
img_x86 = dir([in_x86, '*.png'])';
for img = img_x86
copyfile([in_x86, '/', img.name], [out_x86, '/', img.name])
end
img_x64 = dir([in_x86, '*.png'])';
for img = img_x64
copyfile([in_x64, '/', img.name], [out_x64, '/', img.name])
end
%% Copy sample images for testing
copyfile('../../samples', [out_x86, '/samples']);
copyfile('../../samples', [out_x64 '/samples']);
%% Test if everything worked by running examples
cd(out_x64);
vid_test = sprintf('FaceLandmarkVid.exe -f samples/default.wmv');
dos(vid_test);
feat_test = sprintf('FeatureExtraction.exe -f samples/default.wmv -verbose');
dos(feat_test);
img_test = sprintf('FaceLandmarkImg.exe -fdir samples -verbose');
dos(img_test);
vid_test = sprintf('FaceLandmarkVidMulti.exe -f samples/multi_face.avi -verbose');
dos(vid_test);
rmdir('processed', 's');
%%
cd('..');
cd(out_x86);
vid_test = sprintf('FaceLandmarkVid.exe -f samples/default.wmv');
dos(vid_test);
feat_test = sprintf('FeatureExtraction.exe -f samples/default.wmv -verbose');
dos(feat_test);
img_test = sprintf('FaceLandmarkImg.exe -fdir samples -verbose');
dos(img_test);
vid_test = sprintf('FaceLandmarkVidMulti.exe -f samples/multi_face.avi -verbose');
dos(vid_test);
rmdir('processed', 's');
cd('..');

156
pkg/OpenFace/exe/releases/package_windows_executables_no_au.m Normal file
View File

@@ -0,0 +1,156 @@
clear;
version = '0.4.1';
out_x86 = sprintf('OpenFace_%s_win_x86_landmarks', version);
out_x64 = sprintf('OpenFace_%s_win_x64_landmarks', version);
mkdir(out_x86);
mkdir(out_x64);
in_x86 = '../../Release/';
in_x64 = '../../x64/Release/';
% Copy models
copyfile([in_x86, 'AU_predictors'], [out_x86, '/AU_predictors'])
rmdir([ out_x86, '/AU_predictors/svm_combined'], 's');
rmdir([ out_x86, '/AU_predictors/svr_combined'], 's');
copyfile([in_x86, 'classifiers'], [out_x86, '/classifiers'])
copyfile([in_x86, 'model'], [out_x86, '/model'])
copyfile([in_x64, 'AU_predictors'], [out_x64, '/AU_predictors'])
rmdir([ out_x64, '/AU_predictors/svm_combined'], 's');
rmdir([ out_x64, '/AU_predictors/svr_combined'], 's');
copyfile([in_x64, 'classifiers'], [out_x64, '/classifiers'])
copyfile([in_x64, 'model'], [out_x64, '/model'])
%% Copy libraries
libs_x86 = dir([in_x86, '*.lib'])';
for lib = libs_x86
copyfile([in_x86, '/', lib.name], [out_x86, '/', lib.name])
end
libs_x64 = dir([in_x64, '*.lib'])';
for lib = libs_x64
copyfile([in_x64, '/', lib.name], [out_x64, '/', lib.name])
end
%% Copy dlls
dlls_x86 = dir([in_x86, '*.dll'])';
for dll = dlls_x86
copyfile([in_x86, '/', dll.name], [out_x86, '/', dll.name])
end
dlls_x64 = dir([in_x64, '*.dll'])';
for dll = dlls_x64
copyfile([in_x64, '/', dll.name], [out_x64, '/', dll.name])
end
% Copy zmq dll's
mkdir([out_x64, '/amd64']);
copyfile([in_x64, '/amd64'], [out_x64, '/amd64']);
mkdir([out_x64, '/i386']);
copyfile([in_x64, '/i386'], [out_x64, '/i386']);
mkdir([out_x86, '/amd64']);
copyfile([in_x86, '/amd64'], [out_x86, '/amd64']);
mkdir([out_x86, '/i386']);
copyfile([in_x86, '/i386'], [out_x86, '/i386']);
%% Copy exe's
exes_x86 = dir([in_x86, '*.exe'])';
for exe = exes_x86
copyfile([in_x86, '/', exe.name], [out_x86, '/', exe.name])
end
exes_x64 = dir([in_x64, '*.exe'])';
for exe = exes_x64
copyfile([in_x64, '/', exe.name], [out_x64, '/', exe.name])
end
%% Copy license and copyright
copyfile('../../Copyright.txt', [out_x86, '/Copyright.txt']);
copyfile('../../OpenFace-license.txt', [out_x86, '/OpenFace-license.txt']);
copyfile('../../Copyright.txt', [out_x64, '/Copyright.txt']);
copyfile('../../OpenFace-license.txt', [out_x64, '/OpenFace-license.txt']);
%% Copy icons etc. needed for GUI
img_x86 = dir([in_x86, '*.ico'])';
for img = img_x86
copyfile([in_x86, '/', img.name], [out_x86, '/', img.name])
end
img_x64 = dir([in_x64, '*.ico'])';
for img = img_x64
copyfile([in_x64, '/', img.name], [out_x64, '/', img.name])
end
img_x86 = dir([in_x86, '*.png'])';
for img = img_x86
copyfile([in_x86, '/', img.name], [out_x86, '/', img.name])
end
img_x64 = dir([in_x86, '*.png'])';
for img = img_x64
copyfile([in_x64, '/', img.name], [out_x64, '/', img.name])
end
%% Copy sample images for testing
copyfile('../../samples', [out_x86, '/samples']);
copyfile('../../samples', [out_x64 '/samples']);
%% Test if everything worked by running examples
cd(out_x64);
vid_test = sprintf('FaceLandmarkVid.exe -f samples/default.wmv');
dos(vid_test);
feat_test = sprintf('FeatureExtraction.exe -f samples/default.wmv -verbose');
dos(feat_test);
img_test = sprintf('FaceLandmarkImg.exe -fdir samples -verbose');
dos(img_test);
vid_test = sprintf('FaceLandmarkVidMulti.exe -f samples/multi_face.avi -verbose');
dos(vid_test);
rmdir('processed', 's');
%%
cd('..');
cd(out_x86);
vid_test = sprintf('FaceLandmarkVid.exe -f samples/default.wmv');
dos(vid_test);
feat_test = sprintf('FeatureExtraction.exe -f samples/default.wmv -verbose');
dos(feat_test);
img_test = sprintf('FaceLandmarkImg.exe -fdir samples -verbose');
dos(img_test);
vid_test = sprintf('FaceLandmarkVidMulti.exe -f samples/multi_face.avi -verbose');
dos(vid_test);
rmdir('processed', 's');
cd('..');

1
pkg/OpenFace/exe/releases/readme.txt Normal file
View File

@@ -0,0 +1 @@
Please run the download_models.ps1 script or visit https://github.com/TadasBaltrusaitis/OpenFace/wiki/Model-download for information of how to get the models required to run the executables.

6
pkg/OpenFace/gui/HeadPose-live/App.config Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
</startup>
</configuration>

9
pkg/OpenFace/gui/HeadPose-live/App.xaml Normal file
View File

@@ -0,0 +1,9 @@
<Application x:Class="HeadPose_live.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="clr-namespace:HeadPose_live"
StartupUri="MainWindow.xaml">
<Application.Resources>
</Application.Resources>
</Application>

17
pkg/OpenFace/gui/HeadPose-live/App.xaml.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data;
using System.Linq;
using System.Threading.Tasks;
using System.Windows;
namespace HeadPose_live
{
/// <summary>
/// Interaction logic for App.xaml
/// </summary>
public partial class App : Application
{
}
}

179
pkg/OpenFace/gui/HeadPose-live/HeadPoseLive.csproj Normal file
View File

@@ -0,0 +1,179 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{F396362D-821E-4EA6-9BBF-1F6050844118}</ProjectGuid>
<OutputType>WinExe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>HeadPoseLive</RootNamespace>
<AssemblyName>HeadPoseLive</AssemblyName>
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<ProjectTypeGuids>{60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<WarningLevel>4</WarningLevel>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>..\..\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>..\..\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>..\..\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>..\..\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup>
<ApplicationIcon>logo1.ico</ApplicationIcon>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Data" />
<Reference Include="System.Drawing" />
<Reference Include="System.Xml" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xaml">
<RequiredTargetFramework>4.0</RequiredTargetFramework>
</Reference>
<Reference Include="WindowsBase" />
<Reference Include="PresentationCore" />
<Reference Include="PresentationFramework" />
<Reference Include="ZeroMQ, Version=4.1.0.22, Culture=neutral, PublicKeyToken=4a9630883fd6c563, processorArchitecture=MSIL">
<HintPath>..\..\packages\ZeroMQ.4.1.0.22\lib\net40\ZeroMQ.dll</HintPath>
<Private>True</Private>
</Reference>
</ItemGroup>
<ItemGroup>
<ApplicationDefinition Include="App.xaml">
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</ApplicationDefinition>
<Compile Include="Liability.xaml.cs">
<DependentUpon>Liability.xaml</DependentUpon>
</Compile>
<Compile Include="TextEntryWindow.xaml.cs">
<DependentUpon>TextEntryWindow.xaml</DependentUpon>
</Compile>
<Page Include="Liability.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
<Page Include="MainWindow.xaml">
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</Page>
<Compile Include="App.xaml.cs">
<DependentUpon>App.xaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
<Compile Include="MainWindow.xaml.cs">
<DependentUpon>MainWindow.xaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
<Page Include="TextEntryWindow.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Properties\Resources.Designer.cs">
<AutoGen>True</AutoGen>
<DesignTime>True</DesignTime>
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
<Compile Include="Properties\Settings.Designer.cs">
<AutoGen>True</AutoGen>
<DependentUpon>Settings.settings</DependentUpon>
<DesignTimeSharedInput>True</DesignTimeSharedInput>
</Compile>
<EmbeddedResource Include="Properties\Resources.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
</EmbeddedResource>
<None Include="packages.config" />
<None Include="Properties\Settings.settings">
<Generator>SettingsSingleFileGenerator</Generator>
<LastGenOutput>Settings.Designer.cs</LastGenOutput>
</None>
<AppDesigner Include="Properties\" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\lib\local\CppInerop\CppInerop.vcxproj">
<Project>{78196985-ee54-411f-822b-5a23edf80642}</Project>
<Name>CppInerop</Name>
</ProjectReference>
<ProjectReference Include="..\OpenFaceDemo\OpenFaceDemo.csproj">
<Project>{e143a2aa-312e-4dfe-b61d-9a87ccbc8e90}</Project>
<Name>OpenFaceDemo</Name>
</ProjectReference>
<ProjectReference Include="..\OpenFaceOffline\OpenFaceOffline.csproj">
<Project>{a4760f41-2b1f-4144-b7b2-62785affe79b}</Project>
<Name>OpenFaceOffline</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<Resource Include="logo1.ico" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup>
<PostBuildEvent>xcopy /I /E /Y /D "$(ProjectDir)logo1.ico" "$(ProjectDir)$(OutDir)"
xcopy /I /E /Y /D "$(ProjectDir)logo1.png" "$(ProjectDir)$(OutDir)"</PostBuildEvent>
</PropertyGroup>
<Import Project="..\..\packages\ZeroMQ.4.1.0.22\build\net40\ZeroMQ.targets" Condition="Exists('..\..\packages\ZeroMQ.4.1.0.22\build\net40\ZeroMQ.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\packages\ZeroMQ.4.1.0.22\build\net40\ZeroMQ.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\ZeroMQ.4.1.0.22\build\net40\ZeroMQ.targets'))" />
</Target>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

26
pkg/OpenFace/gui/HeadPose-live/Liability.xaml Normal file
View File

@@ -0,0 +1,26 @@
<Window x:Class="HeadPoseLive.Liability"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="Liability" Height="300" Width="400" ResizeMode="NoResize" WindowStartupLocation="CenterScreen">
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="auto"/>
<RowDefinition Height="30"/>
</Grid.RowDefinitions>
<TextBlock TextWrapping="WrapWithOverflow" Margin="10,10,10,10" TextAlignment="Justify" Grid.Row="0">This software is provided by the copyright holders and contributors "as is" and
any express or implied warranties, including, but not limited to, the implied
warranties of merchantability and fitness for a particular purpose are disclaimed.
In no event shall copyright holders or contributors be liable for any direct,
indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or services;
loss of use, data, or profits; or business interruption) however caused
and on any theory of liability, whether in contract, strict liability,
or tort (including negligence or otherwise) arising in any way out of
the use of this software, even if advised of the possibility of such damage.</TextBlock>
<Button Grid.Row="1" HorizontalAlignment="Center" Click="Button_Click" Name="ContinueButton">
Continue
</Button>
</Grid>
</Window>

83
pkg/OpenFace/gui/HeadPose-live/Liability.xaml.cs Normal file
View File

@@ -0,0 +1,83 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Shapes;
namespace HeadPoseLive
{
/// <summary>
/// Interaction logic for Liability.xaml
/// </summary>
public partial class Liability : Window
{
public bool continue_pressed = false;
public Liability()
{
InitializeComponent();
this.KeyDown += new KeyEventHandler(TextEntry_KeyDown);
FocusManager.SetFocusedElement(this, ContinueButton);
}
private void Button_Click(object sender, RoutedEventArgs e)
{
continue_pressed = true;
this.Close();
}
private void TextEntry_KeyDown(object sender, KeyEventArgs e)
{
if (e.Key == Key.Enter)
{
continue_pressed = true;
DialogResult = true;
}
}
}
}

94
pkg/OpenFace/gui/HeadPose-live/MainWindow.xaml Normal file
View File

@@ -0,0 +1,94 @@
<Window x:Class="HeadPoseLive.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:Ophthalm_experiments="clr-namespace:HeadPoseLive"
Title="Cambridge Face Tracker - Head Pose Experiments" MinHeight="400" MinWidth="640" WindowStartupLocation="CenterScreen" UseLayoutRounding="True" Closing="Window_Closing">
<Window.Resources>
<Style TargetType="{x:Type Image}">
<Setter Property="RenderOptions.BitmapScalingMode"
Value="Fant" />
</Style>
</Window.Resources>
<Grid Name="MainGrid" HorizontalAlignment="Stretch" MinWidth="620">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="100"/>
<ColumnDefinition MinWidth="300"/>
<ColumnDefinition Width="auto"/>
<ColumnDefinition MinWidth="10" Width="auto"/>
</Grid.ColumnDefinitions>
<Grid.RowDefinitions>
<RowDefinition Height="10"/>
<RowDefinition MinHeight="200"/>
<RowDefinition Height="50"/>
</Grid.RowDefinitions>
<Label HorizontalAlignment="Left" VerticalAlignment="Bottom" Grid.Column="1" Grid.Row="1" Background="DarkSalmon" Canvas.ZIndex="1" MouseDown="ResetButton_Click" Name="ResetButton" Content="Reset"/>
<StackPanel Grid.Row="2" Grid.Column="1" Orientation="Horizontal" HorizontalAlignment="Center">
<Button Width="110" Height="26" FontSize="16" Click="startRecordingButton_Click" Name="RecordingButton">Record trial 0</Button>
</StackPanel>
<StackPanel Grid.Row="1" Grid.Column="2" VerticalAlignment="Top">
<Button Name="PauseButton" Click="PauseButton_Click">
Pause
</Button>
<Button Name="ScreenshotButton" Click="ScreenshotButton_Click" Margin="0,10,0,0">
Take a screenshot
</Button>
<Button Name="MirrorImageButton" Click="MirrorButton_Click" Margin="0,10,0,0">
Mirror image
</Button>
</StackPanel>
<StackPanel Grid.Row="1" Grid.Column="2" VerticalAlignment="Center" Width="185">
<Label Name="headOrientationLabel" Margin="0,0,0,0" FontSize="18" HorizontalContentAlignment="Left">Head Orientation</Label>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" Width="80" HorizontalContentAlignment="Left">Turn:</Label>
<Label Name="YawLabel" FontSize="16" MinWidth="30" HorizontalContentAlignment="Right">0°</Label>
<Label Name="YawLabelDir" FontSize="16" Width="70" HorizontalContentAlignment="Left">straight</Label>
</StackPanel>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" Width="80" HorizontalContentAlignment="Left">Up/down:</Label>
<Label Name="PitchLabel" FontSize="16" Width="30" HorizontalContentAlignment="Right">0°</Label>
<Label Name="PitchLabelDir" FontSize="16" Width="70" HorizontalContentAlignment="Left">straight</Label>
</StackPanel>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" Width="80" HorizontalContentAlignment="Left">Tilt:</Label>
<Label Name="RollLabel" FontSize="16" Width="30" HorizontalContentAlignment="Right">0°</Label>
<Label Name="RollLabelDir" FontSize="16" Width="70" HorizontalContentAlignment="Left">straight</Label>
</StackPanel>
<Label Name="gazeLabel" Margin="0,0,0,0" FontSize="18" HorizontalContentAlignment="Left">Gaze Orientation</Label>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" Width="80" HorizontalContentAlignment="Left">Left-right:</Label>
<Label Name="YawLabelGaze" FontSize="16" MinWidth="30" HorizontalContentAlignment="Right">0°</Label>
<Label Name="YawLabelGazeDir" FontSize="16" Width="70" HorizontalContentAlignment="Left">straight</Label>
</StackPanel>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" Width="80" HorizontalContentAlignment="Left">Up/down:</Label>
<Label Name="PitchLabelGaze" FontSize="16" Width="30" HorizontalContentAlignment="Right">0°</Label>
<Label Name="PitchLabelGazeDir" FontSize="16" Width="70" HorizontalContentAlignment="Left">straight</Label>
</StackPanel>
<Label Name="headPoseLabel" Margin="0,0,0,0" FontSize="18" HorizontalContentAlignment="Left">Pose</Label>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" HorizontalContentAlignment="Left" Width="20">X:</Label>
<Label Name="XPoseLabel" FontSize="16" HorizontalContentAlignment="Right" Width="70">0 mm</Label>
</StackPanel>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" HorizontalContentAlignment="Left" Width="20">Y:</Label>
<Label Name="YPoseLabel" FontSize="16" HorizontalContentAlignment="Right" Width="70">0 mm</Label>
</StackPanel>
<StackPanel Orientation="Horizontal">
<Label Margin="5,0,0,0" FontSize="16" HorizontalContentAlignment="Left" Width="20">Z:</Label>
<Label Name="ZPoseLabel" FontSize="16" HorizontalContentAlignment="Right" Width="70">0 mm</Label>
</StackPanel>
</StackPanel>
<StackPanel Grid.Row="2" Grid.Column="2" VerticalAlignment="Bottom">
<Button FontSize="16" HorizontalAlignment="Center" Click="CompleteButton_Click" Name="CompleteButton" >Start new subject</Button>
</StackPanel>
<StackPanel HorizontalAlignment="Left" VerticalAlignment="Top" Width="80" Grid.Column="0" Grid.Row="0" Grid.RowSpan="2">
<Image RenderOptions.BitmapScalingMode="Fant" RenderOptions.EdgeMode="Aliased" x:Name="logoLabel" Source="/logo1.png" Stretch="Uniform" />
</StackPanel>
</Grid>
</Window>

756
pkg/OpenFace/gui/HeadPose-live/MainWindow.xaml.cs Normal file
View File

@@ -0,0 +1,756 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing.Imaging;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Media.Imaging;
using OpenCVWrappers;
using CppInterop;
using CppInterop.LandmarkDetector;
using System.Windows.Threading;
using GazeAnalyser_Interop;
using FaceDetectorInterop;
using ZeroMQ;
using System.Drawing;
using System.Collections.Concurrent;
namespace HeadPoseLive
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
// Timing for measuring FPS
#region High-Resolution Timing
static DateTime startTime;
static Stopwatch sw = new Stopwatch();
static MainWindow()
{
startTime = DateTime.Now;
sw.Start();
}
public static DateTime CurrentTime
{
get { return startTime + sw.Elapsed; }
}
#endregion
OpenFaceOffline.FpsTracker processing_fps = new OpenFaceOffline.FpsTracker();
Thread processing_thread;
Thread rec_thread;
string subject_id;
bool record_video;
bool record_head_pose;
// Controls if the view should be mirrored or not
volatile bool mirror_image = false;
// Capturing and displaying the images
OpenFaceOffline.OverlayImage webcam_img;
// Some members for displaying the results
private WriteableBitmap latest_img;
// For tracking
bool reset = false;
// For recording
string record_root = "./head_pose_live_recordings/subject";
string output_root;
private Object recording_lock = new Object();
int trial_id = 0;
bool recording = false;
int img_width;
int img_height;
double seconds_to_record = 10;
System.IO.StreamWriter recording_success_file = null;
ConcurrentQueue<Tuple<RawImage, bool, List<float>>> recording_objects;
// For broadcasting the results
ZeroMQ.ZContext zero_mq_context;
ZeroMQ.ZSocket zero_mq_socket;
volatile bool running = true;
volatile bool pause = false;
public void StartExperiment()
{
// Inquire more from the user
// Get the entry dialogue now for the subject ID
trial_id = 0;
TextEntryWindow subject_id_window = new TextEntryWindow();
subject_id_window.Icon = this.Icon;
subject_id_window.WindowStartupLocation = WindowStartupLocation.CenterScreen;
if (subject_id_window.ShowDialog() == true)
{
subject_id = subject_id_window.ResponseText;
// Remove trailing spaces and full stops at the end of the folder name
int old_length;
do
{
old_length = subject_id.Length;
subject_id = subject_id.Trim();
if (subject_id.Length > 0)
{
while (subject_id[subject_id.Length - 1].Equals('.'))
{
subject_id = subject_id.Substring(0, subject_id.Length - 1);
}
}
} while (subject_id.Length != old_length);
output_root = record_root + subject_id + "/";
if (System.IO.Directory.Exists(output_root))
{
string messageBoxText = "The recording for subject already exists, are you sure you want to continue?";
string caption = "Directory exists!";
MessageBoxButton button = MessageBoxButton.YesNo;
MessageBoxImage icon = MessageBoxImage.Warning;
MessageBoxResult result = MessageBox.Show(messageBoxText, caption, button, icon);
if (result == MessageBoxResult.No)
{
this.Close();
}
// Else find the latest trial from which to continu
int trial_id_not_found = 0;
while (System.IO.File.Exists(output_root + '/' + "trial_" + trial_id_not_found + ".avi"))
{
trial_id_not_found++;
}
trial_id = trial_id_not_found;
}
System.IO.Directory.CreateDirectory(output_root);
record_video = subject_id_window.RecordVideo;
record_head_pose = subject_id_window.RecordHeadPose;
RecordingButton.Content = "Record trial: " + trial_id;
}
else
{
this.Close();
}
}
public MainWindow()
{
InitializeComponent();
DateTime now = DateTime.Now;
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
// Warn about the liability
Liability liab = new Liability();
liab.Icon = BitmapFrame.Create(iconUri);
liab.ShowDialog();
if (!liab.continue_pressed)
{
this.Close();
return;
}
BitmapImage src = new BitmapImage();
src.BeginInit();
src.UriSource = new Uri("logo1.png", UriKind.RelativeOrAbsolute);
src.CacheOption = BitmapCacheOption.OnLoad;
src.EndInit();
logoLabel.Source = src;
// First make the user chooose a webcam
OpenFaceOffline.CameraSelection cam_select = new OpenFaceOffline.CameraSelection();
cam_select.Icon = BitmapFrame.Create(iconUri);
if (!cam_select.no_cameras_found)
{
cam_select.ShowDialog();
}
if (cam_select.camera_selected)
{
// Create the capture device
int cam_id = cam_select.selected_camera.Item1;
img_width = cam_select.selected_camera.Item2;
img_height = cam_select.selected_camera.Item3;
UtilitiesOF.SequenceReader reader = new UtilitiesOF.SequenceReader(cam_id, img_width, img_height);
if (reader.IsOpened())
{
// Create the ZeroMQ context for broadcasting the results
zero_mq_context = ZeroMQ.ZContext.Create();
zero_mq_socket = new ZSocket(zero_mq_context, ZeroMQ.ZSocketType.PUB);
// Bind on localhost port 5000
zero_mq_socket.Bind("tcp://127.0.0.1:5000");
processing_thread = new Thread(() => VideoLoop(reader));
processing_thread.Name = "Webcam processing";
processing_thread.Start();
}
else
{
string messageBoxText = "Failed to open a webcam";
string caption = "Webcam failure";
MessageBoxButton button = MessageBoxButton.OK;
MessageBoxImage icon = MessageBoxImage.Warning;
// Display message box
MessageBox.Show(messageBoxText, caption, button, icon);
this.Close();
}
// Create an overlay image for display purposes
webcam_img = new OpenFaceOffline.OverlayImage();
webcam_img.SetValue(Grid.RowProperty, 1);
webcam_img.SetValue(Grid.ColumnProperty, 1);
MainGrid.Children.Add(webcam_img);
StartExperiment();
}
else
{
cam_select.Close();
this.Close();
}
}
private bool ProcessFrame(CLNF landmark_detector, GazeAnalyserManaged gaze_analyser, FaceModelParameters model_params, RawImage frame, RawImage grayscale_frame, float fx, float fy, float cx, float cy)
{
bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(frame, model_params, grayscale_frame);
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, fx, fy, cx, cy);
return detection_succeeding;
}
// Capturing and processing the video frame by frame
private void RecordingLoop()
{
// Set up the recording objects first
Thread.CurrentThread.IsBackground = true;
System.IO.StreamWriter output_head_pose_file = null;
if (record_head_pose)
{
String filename_poses = output_root + "/trial_" + trial_id + ".poses.txt";
output_head_pose_file = new System.IO.StreamWriter(filename_poses);
output_head_pose_file.WriteLine("time(ms), success, pose_X(mm), pose_Y(mm), pose_Z(mm), pitch(deg), yaw(deg), roll(deg)");
}
VideoWriter video_writer = null;
if (record_video)
{
double fps = processing_fps.GetFPS();
String filename_video = output_root + "/trial_" + trial_id + ".avi";
video_writer = new VideoWriter(filename_video, img_width, img_height, fps, true);
}
// The timiing should be when the item is captured, but oh well
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
while (recording)
{
Tuple<RawImage, bool, List<float>> recording_object;
if (recording_objects.TryDequeue(out recording_object))
{
if (record_video)
{
video_writer.Write(recording_object.Item1);
}
if (record_head_pose)
{
String output_pose_line = stopWatch.ElapsedMilliseconds.ToString();
if (recording_object.Item2)
output_pose_line += ", 1";
else
output_pose_line += ", 0";
for (int i = 0; i < recording_object.Item3.Count; ++i)
{
double num = recording_object.Item3[i];
if (i > 2)
{
output_pose_line += ", " + num * 180 / Math.PI;
}
else
{
output_pose_line += ", " + num;
}
}
output_head_pose_file.WriteLine(output_pose_line);
}
}
Thread.Sleep(10);
}
// Clean up the recording
if (record_head_pose)
{
output_head_pose_file.Close();
}
}
// Capturing and processing the video frame by frame
private void VideoLoop(UtilitiesOF.SequenceReader reader)
{
Thread.CurrentThread.IsBackground = true;
String root = AppDomain.CurrentDomain.BaseDirectory;
FaceModelParameters model_params = new FaceModelParameters(root, true, false, false);
// Initialize the face detector
FaceDetector face_detector = new FaceDetector(model_params.GetHaarLocation(), model_params.GetMTCNNLocation());
// If MTCNN model not available, use HOG
if (!face_detector.IsMTCNNLoaded())
{
model_params.SetFaceDetector(false, true, false);
}
CLNF face_model = new CLNF(model_params);
GazeAnalyserManaged gaze_analyser = new GazeAnalyserManaged();
DateTime? startTime = CurrentTime;
var lastFrameTime = CurrentTime;
while (running)
{
//////////////////////////////////////////////
// CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING
//////////////////////////////////////////////
RawImage frame = reader.GetNextImage();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
var grayFrame = reader.GetCurrentFrameGray();
if (mirror_image)
{
frame.Mirror();
grayFrame.Mirror();
}
bool detectionSucceeding = ProcessFrame(face_model, gaze_analyser, model_params, frame, grayFrame, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
lock (recording_lock)
{
if (recording)
{
// Add objects to recording queues
List<float> pose = new List<float>();
face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
recording_objects.Enqueue(new Tuple<RawImage, bool, List<float>>(frame, detectionSucceeding, pose));
}
}
List<Tuple<System.Windows.Point, System.Windows.Point>> lines = null;
List<Tuple<float, float>> eye_landmarks = null;
List<System.Windows.Point> landmarks = new List<System.Windows.Point>();
List<Tuple<System.Windows.Point, System.Windows.Point>> gaze_lines = null;
Tuple<float, float> gaze_angle = new Tuple<float, float>(0, 0);
var visibilities = face_model.GetVisibilities();
double scale = face_model.GetRigidParams()[0];
if (detectionSucceeding)
{
List<Tuple<float, float>> landmarks_doubles = face_model.CalculateAllLandmarks();
foreach (var p in landmarks_doubles)
landmarks.Add(new System.Windows.Point(p.Item1, p.Item2));
eye_landmarks = face_model.CalculateVisibleEyeLandmarks();
gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
gaze_angle = gaze_analyser.GetGazeAngle();
lines = face_model.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
}
if (reset)
{
face_model.Reset();
reset = false;
}
// Visualisation updating
try
{
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
if (latest_img == null)
latest_img = frame.CreateWriteableBitmap();
List<float> pose = new List<float>();
face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
int yaw_abs = Math.Abs(yaw);
int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
int roll_abs = Math.Abs(roll);
int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
int pitch_abs = Math.Abs(pitch);
YawLabel.Content = yaw_abs + "°";
RollLabel.Content = roll_abs + "°";
PitchLabel.Content = pitch_abs + "°";
if (yaw > 0)
YawLabelDir.Content = "Right";
else if (yaw < 0)
YawLabelDir.Content = "Left";
else
YawLabelDir.Content = "Straight";
if (pitch > 0)
PitchLabelDir.Content = "Down";
else if (pitch < 0)
PitchLabelDir.Content = "Up";
else
PitchLabelDir.Content = "Straight";
if (roll > 0)
RollLabelDir.Content = "Left";
else if (roll < 0)
RollLabelDir.Content = "Right";
else
RollLabelDir.Content = "Straight";
XPoseLabel.Content = (int)pose[0] + " mm";
YPoseLabel.Content = (int)pose[1] + " mm";
ZPoseLabel.Content = (int)pose[2] + " mm";
String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
YawLabelGaze.Content = x_angle;
PitchLabelGaze.Content = y_angle;
if (gaze_angle.Item1 > 0)
YawLabelGazeDir.Content = "Right";
else if (gaze_angle.Item1 < 0)
YawLabelGazeDir.Content = "Left";
else
YawLabelGazeDir.Content = "Straight";
if (gaze_angle.Item2 > 0)
PitchLabelGazeDir.Content = "Down";
else if (gaze_angle.Item2 < 0)
PitchLabelGazeDir.Content = "Up";
else
PitchLabelGazeDir.Content = "Straight";
double confidence = face_model.GetConfidence();
if (confidence < 0)
confidence = 0;
else if (confidence > 1)
confidence = 1;
frame.UpdateWriteableBitmap(latest_img);
webcam_img.Clear();
webcam_img.Source = latest_img;
webcam_img.Confidence.Add(confidence);
webcam_img.FPS = processing_fps.GetFPS();
if(detectionSucceeding)
{
webcam_img.OverlayLines.Add(lines);
webcam_img.OverlayPoints.Add(landmarks);
webcam_img.OverlayPointsVisibility.Add(visibilities);
webcam_img.FaceScale.Add(scale);
List<System.Windows.Point> eye_landmark_points = new List<System.Windows.Point>();
foreach (var p in eye_landmarks)
{
eye_landmark_points.Add(new System.Windows.Point(p.Item1, p.Item2));
}
webcam_img.OverlayEyePoints.Add(eye_landmark_points);
webcam_img.GazeLines.Add(gaze_lines);
// Publish the information for other applications
String str_head_pose = String.Format("{0}:{1:F2}, {2:F2}, {3:F2}, {4:F2}, {5:F2}, {6:F2}", "HeadPose", pose[0], pose[1], pose[2],
pose[3] * 180 / Math.PI, pose[4] * 180 / Math.PI, pose[5] * 180 / Math.PI);
zero_mq_socket.Send(new ZFrame(str_head_pose, Encoding.UTF8));
String str_gaze = String.Format("{0}:{1:F2}, {2:F2}", "GazeAngle", gaze_angle.Item1 * (180.0 / Math.PI), gaze_angle.Item2 * (180.0 / Math.PI));
zero_mq_socket.Send(new ZFrame(str_gaze, Encoding.UTF8));
}
}));
while (running & pause)
{
Thread.Sleep(10);
}
}
catch (TaskCanceledException)
{
// Quitting
break;
}
}
reader.Close();
System.Console.Out.WriteLine("Thread finished");
}
private void startRecordingButton_Click(object sender, RoutedEventArgs e)
{
lock (recording_lock)
{
RecordingButton.IsEnabled = false;
CompleteButton.IsEnabled = false;
PauseButton.IsEnabled = false;
recording_objects = new ConcurrentQueue<Tuple<RawImage, bool, List<float>>>();
recording = true;
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
// Start the recording thread
rec_thread = new Thread(RecordingLoop);
rec_thread.Start();
double d = seconds_to_record * 1000;
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
while (d > 1000)
{
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
RecordingButton.Content = ((int)(d / 1000)).ToString() + " seconds remaining";
}));
System.Threading.Thread.Sleep(1000);
d = seconds_to_record * 1000 - stopWatch.ElapsedMilliseconds;
}
if (d > 0)
{
System.Threading.Thread.Sleep((int)(d));
}
recording = false;
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
RecordingButton.Content = "0 seconds remaining";
}));
Dispatcher.Invoke(() =>
{
lock (recording_lock)
{
// Wait for the recording thread to finish before enabling
rec_thread.Join();
string messageBoxText = "Was the tracking successful?";
string caption = "Success of tracking";
MessageBoxButton button = MessageBoxButton.YesNo;
MessageBoxImage icon = MessageBoxImage.Question;
MessageBoxResult result = MessageBox.Show(messageBoxText, caption, button, icon);
if (recording_success_file == null)
{
recording_success_file = new System.IO.StreamWriter(output_root + "/recording_success.txt", true);
}
if (result == MessageBoxResult.Yes)
{
recording_success_file.WriteLine('1');
}
else
{
recording_success_file.WriteLine('0');
}
recording_success_file.Flush();
trial_id++;
RecordingButton.Content = "Record trial: " + trial_id;
RecordingButton.IsEnabled = true;
CompleteButton.IsEnabled = true;
PauseButton.IsEnabled = true;
}
});
}).Start();
}
}
private void ResetButton_Click(object sender, RoutedEventArgs e)
{
reset = true;
}
private void CompleteButton_Click(object sender, RoutedEventArgs e)
{
StartExperiment();
}
private void MirrorButton_Click(object sender, RoutedEventArgs e)
{
mirror_image = !mirror_image;
}
private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
{
// Let it finish recording
recording = false;
if (rec_thread != null)
{
rec_thread.Join();
}
// Stop capture and tracking
running = false;
if (processing_thread != null)
{
processing_thread.Join();
}
}
private void MorrorButton_Click(object sender, RoutedEventArgs e)
{
}
private void PauseButton_Click(object sender, RoutedEventArgs e)
{
pause = !pause;
if (pause)
{
PauseButton.Content = "Resume";
}
else
{
PauseButton.Content = "Pause";
}
}
private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
{
PresentationSource source = PresentationSource.FromVisual(Application.Current.MainWindow);
var topLeft = source.CompositionTarget.TransformToDevice.Transform(new System.Windows.Point(this.Left, this.Top));
var bottomRight = source.CompositionTarget.TransformToDevice.Transform(new System.Windows.Point(this.Left + this.Width, this.Top + this.Height));
int Width = (int)(bottomRight.X - topLeft.X);
int Height = (int)(bottomRight.Y - topLeft.Y);
using (Bitmap bmpScreenCapture = new Bitmap(Width,
Height))
{
using (System.Drawing.Graphics g = Graphics.FromImage(bmpScreenCapture))
{
g.CopyFromScreen((int)(topLeft.X),
(int)(topLeft.Y),
0, 0,
bmpScreenCapture.Size,
CopyPixelOperation.SourceCopy);
// Write out the bitmap here encoded by a time-stamp?
String fname = output_root + DateTime.Now.ToString("yyyy-MMM-dd--HH-mm-ss") + ".png";
bmpScreenCapture.Save(fname, ImageFormat.Png);
}
}
}
}
}

55
pkg/OpenFace/gui/HeadPose-live/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,55 @@
using System.Reflection;
using System.Resources;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Windows;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("HeadPose-live")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("HeadPose-live")]
[assembly: AssemblyCopyright("Copyright © Microsoft 2017")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
//In order to begin building localizable applications, set
//<UICulture>CultureYouAreCodingWith</UICulture> in your .csproj file
//inside a <PropertyGroup>. For example, if you are using US english
//in your source files, set the <UICulture> to en-US. Then uncomment
//the NeutralResourceLanguage attribute below. Update the "en-US" in
//the line below to match the UICulture setting in the project file.
//[assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.Satellite)]
[assembly: ThemeInfo(
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
//(used if a resource is not found in the page,
// or application resource dictionaries)
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
//(used if a resource is not found in the page,
// app, or any theme specific resource dictionaries)
)]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

63
pkg/OpenFace/gui/HeadPose-live/Properties/Resources.Designer.cs generated Normal file
View File

@@ -0,0 +1,63 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace HeadPoseLive.Properties {
using System;
/// <summary>
/// A strongly-typed resource class, for looking up localized strings, etc.
/// </summary>
// This class was auto-generated by the StronglyTypedResourceBuilder
// class via a tool like ResGen or Visual Studio.
// To add or remove a member, edit your .ResX file then rerun ResGen
// with the /str option, or rebuild your VS project.
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
internal class Resources {
private static global::System.Resources.ResourceManager resourceMan;
private static global::System.Globalization.CultureInfo resourceCulture;
[global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
internal Resources() {
}
/// <summary>
/// Returns the cached ResourceManager instance used by this class.
/// </summary>
[global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
internal static global::System.Resources.ResourceManager ResourceManager {
get {
if (object.ReferenceEquals(resourceMan, null)) {
global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("HeadPoseLive.Properties.Resources", typeof(Resources).Assembly);
resourceMan = temp;
}
return resourceMan;
}
}
/// <summary>
/// Overrides the current thread's CurrentUICulture property for all
/// resource lookups using this strongly typed resource class.
/// </summary>
[global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
internal static global::System.Globalization.CultureInfo Culture {
get {
return resourceCulture;
}
set {
resourceCulture = value;
}
}
}
}

117
pkg/OpenFace/gui/HeadPose-live/Properties/Resources.resx Normal file
View File

@@ -0,0 +1,117 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
<resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
<resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
<data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
<data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
<data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
<value>[base64 mime encoded serialized .NET Framework object]</value>
</data>
<data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
: System.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="metadata">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" />
<xsd:attribute name="type" type="xsd:string" />
<xsd:attribute name="mimetype" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="assembly">
<xsd:complexType>
<xsd:attribute name="alias" type="xsd:string" />
<xsd:attribute name="name" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" msdata:Ordinal="1" />
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
</xsd:complexType>
</xsd:element>
<xsd:element name="resheader">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" />
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>2.0</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
</root>

26
pkg/OpenFace/gui/HeadPose-live/Properties/Settings.Designer.cs generated Normal file
View File

@@ -0,0 +1,26 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace HeadPoseLive.Properties {
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "14.0.0.0")]
internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase {
private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings())));
public static Settings Default {
get {
return defaultInstance;
}
}
}
}

7
pkg/OpenFace/gui/HeadPose-live/Properties/Settings.settings Normal file
View File

@@ -0,0 +1,7 @@
<?xml version='1.0' encoding='utf-8'?>
<SettingsFile xmlns="uri:settings" CurrentProfile="(Default)">
<Profiles>
<Profile Name="(Default)" />
</Profiles>
<Settings />
</SettingsFile>

17
pkg/OpenFace/gui/HeadPose-live/TextEntryWindow.xaml Normal file
View File

@@ -0,0 +1,17 @@
<Window x:Class="HeadPoseLive.TextEntryWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="Configuration and subject information" Height="210" Width="300">
<Grid>
<StackPanel FocusManager.FocusedElement="{Binding ElementName=ResponseTextBox}">
<TextBlock HorizontalAlignment="Center" Text="Enter subject ID" FontSize="20"/>
<TextBox Margin="0,4,0,0" x:Name="ResponseTextBox" FontSize="20" Width="120" TextChanged="ResponseTextBox_TextChanged" />
<Label Name="warningLabel" Visibility="Collapsed" FontStyle="Italic" Foreground="Red" HorizontalAlignment="Center">Can't use the following characters: " \ / | &lt; > : * ?</Label>
<CheckBox HorizontalAlignment="Center" x:Name="RecordVideoCheckBox" Margin="-25,10,0,0" Content="Record video"/>
<CheckBox Margin="0,6,0,0" IsChecked="True" HorizontalAlignment="Center" x:Name="RecordHeadPoseCheckBox" Content="Record head pose"/>
<Button Margin="0,8,0,0" Content="OK" Click="OKButton_Click" Width="100" VerticalAlignment="Bottom"/>
</StackPanel>
</Grid>
</Window>

129
pkg/OpenFace/gui/HeadPose-live/TextEntryWindow.xaml.cs Normal file
View File

@@ -0,0 +1,129 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Shapes;
namespace HeadPoseLive
{
/// <summary>
/// Interaction logic for TextEntryWindow.xaml
/// </summary>
public partial class TextEntryWindow : Window
{
public TextEntryWindow()
{
InitializeComponent();
this.KeyDown += new KeyEventHandler(TextEntry_KeyDown);
}
public string ResponseText
{
get { return ResponseTextBox.Text; }
set { ResponseTextBox.Text = value; }
}
public bool RecordVideo
{
get { return (bool)RecordVideoCheckBox.IsChecked; }
set { RecordVideoCheckBox.IsChecked = value; }
}
public bool RecordHeadPose
{
get { return (bool)RecordHeadPoseCheckBox.IsChecked; }
set { RecordHeadPoseCheckBox.IsChecked = value; }
}
private void OKButton_Click(object sender, System.Windows.RoutedEventArgs e)
{
DialogResult = true;
}
private void TextEntry_KeyDown(object sender, KeyEventArgs e)
{
if (e.Key == Key.Enter)
{
DialogResult = true;
}
}
// Do not allow illegal characters like
private void ResponseTextBox_TextChanged(object sender, TextChangedEventArgs e)
{
Regex regex = new Regex("[/:*?<>|\"]");
Regex regex2 = new Regex(@"[\\]");
MatchCollection matches = regex.Matches(ResponseTextBox.Text);
MatchCollection matches2 = regex2.Matches(ResponseTextBox.Text);
if (matches.Count > 0 || matches2.Count > 0)
{
for (int i = matches.Count - 1; i >= 0; --i)
{
// Remove the illegal characters
ResponseTextBox.Text = ResponseTextBox.Text.Substring(0, matches[i].Index) + ResponseTextBox.Text.Substring(matches[i].Index + 1);
}
//tell the user
for (int i = matches2.Count - 1; i >= 0; --i)
{
// Remove the illegal characters
ResponseTextBox.Text = ResponseTextBox.Text.Substring(0, matches2[i].Index) + ResponseTextBox.Text.Substring(matches2[i].Index + 1);
}
warningLabel.Visibility = System.Windows.Visibility.Visible;
ResponseTextBox.SelectionStart = ResponseTextBox.Text.Length;
}
else
{
warningLabel.Visibility = System.Windows.Visibility.Collapsed;
}
}
}
}

BIN
pkg/OpenFace/gui/HeadPose-live/logo1.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.6 MiB

BIN
pkg/OpenFace/gui/HeadPose-live/logo1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 199 KiB

4
pkg/OpenFace/gui/HeadPose-live/packages.config Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="ZeroMQ" version="4.1.0.22" targetFramework="net452" />
</packages>

6
pkg/OpenFace/gui/OpenFaceDemo/App.config Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
</startup>
</configuration>

9
pkg/OpenFace/gui/OpenFaceDemo/App.xaml Normal file
View File

@@ -0,0 +1,9 @@
<Application x:Class="OpenFaceDemo.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="clr-namespace:OpenFaceDemo"
StartupUri="MainWindow.xaml">
<Application.Resources>
</Application.Resources>
</Application>

17
pkg/OpenFace/gui/OpenFaceDemo/App.xaml.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data;
using System.Linq;
using System.Threading.Tasks;
using System.Windows;
namespace OpenFaceDemo
{
/// <summary>
/// Interaction logic for App.xaml
/// </summary>
public partial class App : Application
{
}
}

51
pkg/OpenFace/gui/OpenFaceDemo/MainWindow.xaml Normal file
View File

@@ -0,0 +1,51 @@
<Window x:Class="OpenFaceDemo.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:local="clr-namespace:OpenFaceDemo"
xmlns:of="clr-namespace:OpenFaceOffline;assembly=OpenFaceOffline"
mc:Ignorable="d"
Title="OpenFace Analyser" Height="800" Width="1300" MinWidth="700" MinHeight="450" Closing="Window_Closing" WindowStartupLocation="CenterScreen" KeyDown="Window_KeyDown">
<Grid Name="MainGrid" Margin="-1,1,1.333,-1.333">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="1.8*"/>
<ColumnDefinition Width="1.8*"/>
<ColumnDefinition Width="1.8*"/>
</Grid.ColumnDefinitions>
<Grid.RowDefinitions>
<RowDefinition Height="20"/>
<RowDefinition Height="1.5*" MinHeight="100"/>
<RowDefinition Height="1.5*" MinHeight="100"/>
<RowDefinition Height="1.5*" MinHeight="100"/>
<RowDefinition Height="1.6*"/>
<RowDefinition Height="auto"/>
</Grid.RowDefinitions>
<Menu IsMainMenu="True" Grid.Row="0" Grid.Column="0" Grid.ColumnSpan="4">
<MenuItem Header="File">
<MenuItem Header="Open webcam" Click="openWebcamClick">
</MenuItem>
</MenuItem>
</Menu>
<Border Name="VideoBorder" Grid.Row="1" Grid.Column="0" Grid.ColumnSpan="2" Grid.RowSpan="3" BorderBrush="Black" BorderThickness="1" Background="LightGray" Margin="5,5,0,0" >
<of:OverlayImage x:Name="video" MouseDown="video_MouseDown" />
</Border>
<local:AxesTimeSeriesPlot NumVertGrid="5" x:Name="headPosePlot" ShowLegend="True" MinVal="-1" MaxVal="1" MinHeight="180" Grid.Row="4" Grid.Column="0" Padding="60 20 30 40" RangeLabel="Head pose" Orientation="Horizontal">
</local:AxesTimeSeriesPlot>
<local:AxesTimeSeriesPlot MinHeight="180" Grid.Row="4" Grid.Column="1" Padding="60 20 30 40" x:Name="gazePlot" ShowLegend="True" RangeLabel="Eye gaze" Orientation="Horizontal" MinVal="-20" MaxVal="20" NumVertGrid="5">
</local:AxesTimeSeriesPlot>
<local:AxesTimeSeriesPlot Grid.Column="2" Grid.Row="1" MinHeight="130" ShowXLabel="False" Padding="60 20 30 10" x:Name="smilePlot" ShowLegend="True" XTicks="False" RangeLabel="Lips" Orientation="Horizontal" MinVal="0" MaxVal="1" NumVertGrid="5">
</local:AxesTimeSeriesPlot>
<local:AxesTimeSeriesPlot Grid.Column="2" Grid.Row="2" MinHeight="130" ShowXLabel="False" Padding="60 20 30 10" x:Name="browPlot" ShowLegend="True" XTicks="False" RangeLabel="Brows" Orientation="Horizontal" MinVal="0" MaxVal="1" NumVertGrid="5">
</local:AxesTimeSeriesPlot>
<local:AxesTimeSeriesPlot Grid.Column="2" Grid.Row="3" MinHeight="130" ShowXLabel="False" x:Name="eyePlot" ShowLegend="True" Padding="60 20 30 10" XTicks="False" RangeLabel="Other" Orientation="Horizontal" MinVal="0" MaxVal="1" NumVertGrid="5">
</local:AxesTimeSeriesPlot>
</Grid>
</Window>

494
pkg/OpenFace/gui/OpenFaceDemo/MainWindow.xaml.cs Normal file
View File

@@ -0,0 +1,494 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
using System;
using System.Collections.Generic;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Threading;
using System.Windows.Threading;
using System.Diagnostics;
// Internal libraries
using OpenFaceOffline;
using OpenCVWrappers;
using CppInterop.LandmarkDetector;
using FaceAnalyser_Interop;
using FaceDetectorInterop;
using GazeAnalyser_Interop;
using UtilitiesOF;
namespace OpenFaceDemo
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
// -----------------------------------------------------------------
// Members
// -----------------------------------------------------------------
// Timing for measuring FPS
#region High-Resolution Timing
static DateTime startTime;
static Stopwatch sw = new Stopwatch();
static MainWindow()
{
startTime = DateTime.Now;
sw.Start();
}
public static DateTime CurrentTime
{
get { return startTime + sw.Elapsed; }
}
#endregion
Thread processing_thread;
// Some members for displaying the results
private WriteableBitmap latest_img;
private volatile bool thread_running;
FpsTracker processing_fps = new FpsTracker();
// Controlling the model reset
volatile bool reset = false;
Point? resetPoint = null;
// For selecting webcams
CameraSelection cam_sec;
// For tracking
FaceModelParameters face_model_params;
FaceAnalyserManaged face_analyser;
CLNF landmark_detector;
GazeAnalyserManaged gaze_analyser;
public MainWindow()
{
InitializeComponent();
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
String root = AppDomain.CurrentDomain.BaseDirectory;
// TODO, create a demo version of parameters
face_model_params = new FaceModelParameters(root, true, false, false);
face_model_params.optimiseForVideo();
// Initialize the face detector
FaceDetector face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());
// If MTCNN model not available, use HOG
if (!face_detector.IsMTCNNLoaded())
{
face_model_params.SetFaceDetector(false, true, false);
}
landmark_detector = new CLNF(face_model_params);
face_analyser = new FaceAnalyserManaged(root, true, 112, true);
gaze_analyser = new GazeAnalyserManaged();
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
headPosePlot.AssocColor(0, Colors.Blue);
headPosePlot.AssocColor(1, Colors.Red);
headPosePlot.AssocColor(2, Colors.Green);
headPosePlot.AssocName(1, "Turn");
headPosePlot.AssocName(2, "Tilt");
headPosePlot.AssocName(0, "Up/Down");
headPosePlot.AssocThickness(0, 2);
headPosePlot.AssocThickness(1, 2);
headPosePlot.AssocThickness(2, 2);
gazePlot.AssocColor(0, Colors.Red);
gazePlot.AssocColor(1, Colors.Blue);
gazePlot.AssocName(0, "Left-right");
gazePlot.AssocName(1, "Up-down");
gazePlot.AssocThickness(0, 2);
gazePlot.AssocThickness(1, 2);
smilePlot.AssocColor(0, Colors.Green);
smilePlot.AssocColor(1, Colors.Red);
smilePlot.AssocName(0, "Smile");
smilePlot.AssocName(1, "Frown");
smilePlot.AssocThickness(0, 2);
smilePlot.AssocThickness(1, 2);
browPlot.AssocColor(0, Colors.Green);
browPlot.AssocColor(1, Colors.Red);
browPlot.AssocName(0, "Raise");
browPlot.AssocName(1, "Furrow");
browPlot.AssocThickness(0, 2);
browPlot.AssocThickness(1, 2);
eyePlot.AssocColor(0, Colors.Green);
eyePlot.AssocColor(1, Colors.Red);
eyePlot.AssocName(0, "Eye widen");
eyePlot.AssocName(1, "Nose wrinkle");
eyePlot.AssocThickness(0, 2);
eyePlot.AssocThickness(1, 2);
}));
}
private void StopTracking()
{
// First complete the running of the thread
if (processing_thread != null)
{
// Tell the other thread to finish
thread_running = false;
processing_thread.Join();
}
}
// The main function call for processing the webcam feed
private void ProcessingLoop(SequenceReader reader)
{
thread_running = true;
Thread.CurrentThread.IsBackground = true;
DateTime? startTime = CurrentTime;
var lastFrameTime = CurrentTime;
landmark_detector.Reset();
face_analyser.Reset();
int frame_id = 0;
double old_gaze_x = 0;
double old_gaze_y = 0;
double smile_cumm = 0;
double frown_cumm = 0;
double brow_up_cumm = 0;
double brow_down_cumm = 0;
double widen_cumm = 0;
double wrinkle_cumm = 0;
while (thread_running)
{
// Loading an image file
RawImage frame = reader.GetNextImage();
RawImage gray_frame = reader.GetCurrentFrameGray();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
// The face analysis step
bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(frame, face_model_params, gray_frame);
face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, true);
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
double confidence = landmark_detector.GetConfidence();
if (confidence < 0)
confidence = 0;
else if (confidence > 1)
confidence = 1;
List<float> pose = new List<float>();
landmark_detector.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
List<float> non_rigid_params = landmark_detector.GetNonRigidParams();
float scale = landmark_detector.GetRigidParams()[0];
double time_stamp = (DateTime.Now - (DateTime)startTime).TotalMilliseconds;
List<Tuple<Point, Point>> lines = null;
List<Tuple<float, float>> landmarks = null;
List<Tuple<float, float>> eye_landmarks = null;
List<Tuple<Point, Point>> gaze_lines = null;
List<bool> visibilities = null;
Tuple<float, float> gaze_angle = gaze_analyser.GetGazeAngle();
if (detection_succeeding)
{
landmarks = landmark_detector.CalculateVisibleLandmarks();
eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks();
lines = landmark_detector.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
visibilities = landmark_detector.GetVisibilities();
}
// Visualisation
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
var au_regs = face_analyser.GetCurrentAUsReg();
if (au_regs.Count > 0)
{
double smile = (au_regs["AU12"] + au_regs["AU06"] + au_regs["AU25"]) / 13.0;
double frown = (au_regs["AU15"] + au_regs["AU17"]) / 12.0;
double brow_up = (au_regs["AU01"] + au_regs["AU02"]) / 10.0;
double brow_down = au_regs["AU04"] / 5.0;
double eye_widen = au_regs["AU05"] / 3.0;
double nose_wrinkle = au_regs["AU09"] / 4.0;
Dictionary<int, double> smileDict = new Dictionary<int, double>();
smileDict[0] = 0.7 * smile_cumm + 0.3 * smile;
smileDict[1] = 0.7 * frown_cumm + 0.3 * frown;
smilePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = smileDict, Confidence = confidence });
Dictionary<int, double> browDict = new Dictionary<int, double>();
browDict[0] = 0.7 * brow_up_cumm + 0.3 * brow_up;
browDict[1] = 0.7 * brow_down_cumm + 0.3 * brow_down;
browPlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = browDict, Confidence = confidence });
Dictionary<int, double> eyeDict = new Dictionary<int, double>();
eyeDict[0] = 0.7 * widen_cumm + 0.3 * eye_widen;
eyeDict[1] = 0.7 * wrinkle_cumm + 0.3 * nose_wrinkle;
eyePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = eyeDict, Confidence = confidence });
smile_cumm = smileDict[0];
frown_cumm = smileDict[1];
brow_up_cumm = browDict[0];
brow_down_cumm = browDict[1];
widen_cumm = eyeDict[0];
wrinkle_cumm = eyeDict[1];
}
else
{
// If no AUs present disable the AU visualization
MainGrid.ColumnDefinitions[2].Width = new GridLength(0);
eyePlot.Visibility = Visibility.Collapsed;
browPlot.Visibility = Visibility.Collapsed;
smilePlot.Visibility = Visibility.Collapsed;
}
Dictionary<int, double> poseDict = new Dictionary<int, double>();
poseDict[0] = -pose[3];
poseDict[1] = pose[4];
poseDict[2] = pose[5];
headPosePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = poseDict, Confidence = confidence });
Dictionary<int, double> gazeDict = new Dictionary<int, double>();
gazeDict[0] = gaze_angle.Item1 * (180.0 / Math.PI);
gazeDict[0] = 0.5 * old_gaze_x + 0.5 * gazeDict[0];
gazeDict[1] = -gaze_angle.Item2 * (180.0 / Math.PI);
gazeDict[1] = 0.5 * old_gaze_y + 0.5 * gazeDict[1];
gazePlot.AddDataPoint(new DataPointGraph() { Time = CurrentTime, values = gazeDict, Confidence = confidence });
old_gaze_x = gazeDict[0];
old_gaze_y = gazeDict[1];
if (latest_img == null)
{
latest_img = frame.CreateWriteableBitmap();
}
frame.UpdateWriteableBitmap(latest_img);
video.Source = latest_img;
video.FPS = processing_fps.GetFPS();
// First clear the old results
video.Clear();
video.Confidence.Add(confidence);
if(detection_succeeding)
{
video.FaceScale.Add(scale);
video.OverlayLines.Add(lines);
List<Point> landmark_points = new List<Point>();
foreach (var p in landmarks)
{
landmark_points.Add(new Point(p.Item1, p.Item2));
}
List<Point> eye_landmark_points = new List<Point>();
foreach (var p in eye_landmarks)
{
eye_landmark_points.Add(new Point(p.Item1, p.Item2));
}
video.OverlayPoints.Add(landmark_points);
video.OverlayEyePoints.Add(eye_landmark_points);
video.OverlayPointsVisibility.Add(visibilities);
video.OverlayEyePoints.Add(eye_landmark_points);
video.GazeLines.Add(gaze_lines);
}
}));
if (reset)
{
// TODO add
if (resetPoint.HasValue)
{
//landmark_detector.Reset(resetPoint.Value.X, resetPoint.Value.Y);
resetPoint = null;
}
else
{
// TODO add
//landmark_detector.Reset();
}
// TODO add
//face_analyser.Reset();
reset = false;
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
headPosePlot.ClearDataPoints();
headPosePlot.ClearDataPoints();
gazePlot.ClearDataPoints();
smilePlot.ClearDataPoints();
browPlot.ClearDataPoints();
eyePlot.ClearDataPoints();
}));
}
frame_id++;
}
reader.Close();
latest_img = null;
}
// --------------------------------------------------------
// Button handling
// --------------------------------------------------------
private void openWebcamClick(object sender, RoutedEventArgs e)
{
StopTracking();
if (cam_sec == null)
{
cam_sec = new CameraSelection();
}
else
{
cam_sec = new CameraSelection(cam_sec.cams);
cam_sec.Visibility = System.Windows.Visibility.Visible;
}
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
cam_sec.Icon = BitmapFrame.Create(iconUri);
if (!cam_sec.no_cameras_found)
cam_sec.ShowDialog();
if (cam_sec.camera_selected)
{
int cam_id = cam_sec.selected_camera.Item1;
int width = cam_sec.selected_camera.Item2;
int height = cam_sec.selected_camera.Item3;
SequenceReader reader = new SequenceReader(cam_id, width, height);
processing_thread = new Thread(() => ProcessingLoop(reader));
processing_thread.Name = "Webcam processing";
processing_thread.Start();
}
}
// Cleanup stuff when closing the window
private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
{
if (processing_thread != null)
{
// Stop capture and tracking
thread_running = false;
processing_thread.Join();
}
if (face_analyser != null)
face_analyser.Dispose();
if(landmark_detector != null)
landmark_detector.Dispose();
}
private void Window_KeyDown(object sender, KeyEventArgs e)
{
if (e.Key == Key.R)
{
reset = true;
}
}
private void video_MouseDown(object sender, MouseButtonEventArgs e)
{
var clickPos = e.GetPosition(video);
resetPoint = new Point(clickPos.X / video.ActualWidth, clickPos.Y / video.ActualHeight);
reset = true;
}
}
}

159
pkg/OpenFace/gui/OpenFaceDemo/OpenFaceDemo.csproj Normal file
View File

@@ -0,0 +1,159 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{E143A2AA-312E-4DFE-B61D-9A87CCBC8E90}</ProjectGuid>
<OutputType>WinExe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>OpenFaceDemo</RootNamespace>
<AssemblyName>OpenFaceDemo</AssemblyName>
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<ProjectTypeGuids>{60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<WarningLevel>4</WarningLevel>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>..\..\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<GenerateSerializationAssemblies>Auto</GenerateSerializationAssemblies>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>..\..\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>..\..\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>..\..\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup>
<ApplicationIcon>logo1.ico</ApplicationIcon>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xaml">
<RequiredTargetFramework>4.0</RequiredTargetFramework>
</Reference>
<Reference Include="WindowsBase" />
<Reference Include="PresentationCore" />
<Reference Include="PresentationFramework" />
</ItemGroup>
<ItemGroup>
<ApplicationDefinition Include="App.xaml">
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</ApplicationDefinition>
<Compile Include="UI_items\AxesTimeSeriesPlot.xaml.cs">
<DependentUpon>AxesTimeSeriesPlot.xaml</DependentUpon>
</Compile>
<Page Include="MainWindow.xaml">
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</Page>
<Compile Include="App.xaml.cs">
<DependentUpon>App.xaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
<Compile Include="MainWindow.xaml.cs">
<DependentUpon>MainWindow.xaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
<Page Include="UI_items\AxesTimeSeriesPlot.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Properties\Resources.Designer.cs">
<AutoGen>True</AutoGen>
<DesignTime>True</DesignTime>
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
<Compile Include="Properties\Settings.Designer.cs">
<AutoGen>True</AutoGen>
<DependentUpon>Settings.settings</DependentUpon>
<DesignTimeSharedInput>True</DesignTimeSharedInput>
</Compile>
<EmbeddedResource Include="Properties\Resources.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
</EmbeddedResource>
<None Include="Properties\Settings.settings">
<Generator>SettingsSingleFileGenerator</Generator>
<LastGenOutput>Settings.Designer.cs</LastGenOutput>
</None>
<AppDesigner Include="Properties\" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OpenFaceOffline\OpenFaceOffline.csproj">
<Project>{a4760f41-2b1f-4144-b7b2-62785affe79b}</Project>
<Name>OpenFaceOffline</Name>
</ProjectReference>
<ProjectReference Include="..\..\lib\local\CppInerop\CppInerop.vcxproj">
<Project>{78196985-ee54-411f-822b-5a23edf80642}</Project>
<Name>CppInerop</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<Resource Include="logo1.ico" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup>
<PreBuildEvent>
</PreBuildEvent>
</PropertyGroup>
<PropertyGroup>
<PostBuildEvent>xcopy /I /E /Y /D "$(ProjectDir)logo1.ico" "$(ProjectDir)$(OutDir)"</PostBuildEvent>
</PropertyGroup>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

55
pkg/OpenFace/gui/OpenFaceDemo/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,55 @@
using System.Reflection;
using System.Resources;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Windows;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("OpenFaceDemo")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("OpenFaceDemo")]
[assembly: AssemblyCopyright("Copyright © Microsoft 2016")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
//In order to begin building localizable applications, set
//<UICulture>CultureYouAreCodingWith</UICulture> in your .csproj file
//inside a <PropertyGroup>. For example, if you are using US english
//in your source files, set the <UICulture> to en-US. Then uncomment
//the NeutralResourceLanguage attribute below. Update the "en-US" in
//the line below to match the UICulture setting in the project file.
//[assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.Satellite)]
[assembly: ThemeInfo(
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
//(used if a resource is not found in the page,
// or application resource dictionaries)
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
//(used if a resource is not found in the page,
// app, or any theme specific resource dictionaries)
)]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

71
pkg/OpenFace/gui/OpenFaceDemo/Properties/Resources.Designer.cs generated Normal file
View File

@@ -0,0 +1,71 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace OpenFaceDemo.Properties
{
/// <summary>
/// A strongly-typed resource class, for looking up localized strings, etc.
/// </summary>
// This class was auto-generated by the StronglyTypedResourceBuilder
// class via a tool like ResGen or Visual Studio.
// To add or remove a member, edit your .ResX file then rerun ResGen
// with the /str option, or rebuild your VS project.
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
internal class Resources
{
private static global::System.Resources.ResourceManager resourceMan;
private static global::System.Globalization.CultureInfo resourceCulture;
[global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
internal Resources()
{
}
/// <summary>
/// Returns the cached ResourceManager instance used by this class.
/// </summary>
[global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
internal static global::System.Resources.ResourceManager ResourceManager
{
get
{
if ((resourceMan == null))
{
global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("OpenFaceDemo.Properties.Resources", typeof(Resources).Assembly);
resourceMan = temp;
}
return resourceMan;
}
}
/// <summary>
/// Overrides the current thread's CurrentUICulture property for all
/// resource lookups using this strongly typed resource class.
/// </summary>
[global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
internal static global::System.Globalization.CultureInfo Culture
{
get
{
return resourceCulture;
}
set
{
resourceCulture = value;
}
}
}
}

117
pkg/OpenFace/gui/OpenFaceDemo/Properties/Resources.resx Normal file
View File

@@ -0,0 +1,117 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
<resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
<resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
<data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
<data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
<data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
<value>[base64 mime encoded serialized .NET Framework object]</value>
</data>
<data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
: System.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="metadata">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" />
<xsd:attribute name="type" type="xsd:string" />
<xsd:attribute name="mimetype" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="assembly">
<xsd:complexType>
<xsd:attribute name="alias" type="xsd:string" />
<xsd:attribute name="name" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" msdata:Ordinal="1" />
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
</xsd:complexType>
</xsd:element>
<xsd:element name="resheader">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" />
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>2.0</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
</root>

30
pkg/OpenFace/gui/OpenFaceDemo/Properties/Settings.Designer.cs generated Normal file
View File

@@ -0,0 +1,30 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace OpenFaceDemo.Properties
{
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "11.0.0.0")]
internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase
{
private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings())));
public static Settings Default
{
get
{
return defaultInstance;
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More