bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/OpenFace/matlab_runners/Head Pose Experiments/Euler2Rot.m Normal file
View File

@@ -0,0 +1,11 @@
function [Rot] = Euler2Rot(euler)
rx = euler(1);
ry = euler(2);
rz = euler(3);
Rx = [1 0 0; 0 cos(rx) -sin(rx); 0 sin(rx) cos(rx)];
Ry = [cos(ry) 0 sin(ry); 0 1 0; -sin(ry) 0 cos(ry)];
Rz = [cos(rz) -sin(rz) 0; sin(rz) cos(rz) 0; 0 0 1];
Rot = Rx * Ry * Rz;

12
pkg/OpenFace/matlab_runners/Head Pose Experiments/Rot2Euler.m Normal file
View File

@@ -0,0 +1,12 @@
function [euler] = Rot2Euler(R)
q0 = sqrt( 1 + R(1,1) + R(2,2) + R(3,3) ) / 2;
q1 = (R(3,2) - R(2,3)) / (4*q0) ;
q2 = (R(1,3) - R(3,1)) / (4*q0) ;
q3 = (R(2,1) - R(1,2)) / (4*q0) ;
yaw = asin(2*(q0*q2 + q1*q3));
pitch= atan2(2*(q0*q1-q2*q3), q0*q0-q1*q1-q2*q2+q3*q3);
roll = atan2(2*(q0*q3-q1*q2), q0*q0+q1*q1-q2*q2-q3*q3);
euler = [pitch, yaw, roll];

82
pkg/OpenFace/matlab_runners/Head Pose Experiments/calcBUerror.m Normal file
View File

@@ -0,0 +1,82 @@
function [meanError, all_rot_preds, all_rot_gts, all_errors, rels_all, seq_ids] = calcBUerror(resDir, gtDir)
seqNames = {'jam1','jam2','jam3','jam4','jam5','jam6','jam7','jam8','jam9', ...
'jim1','jim2','jim3','jim4','jim5','jim6','jim7','jim8','jim9', ...
'llm1','llm2','llm3','llm4','llm5','llm6','llm7','llm8','llm9', ...
'ssm1','ssm2','ssm3','ssm4','ssm5','ssm6','ssm7','ssm8','ssm9', ...
'vam1','vam2','vam3','vam4','vam5','vam6','vam7','vam8','vam9'};
rotMeanErr = zeros(numel(seqNames),3);
rotRMS = zeros(numel(seqNames),3);
rot = cell(1,numel(seqNames));
rotg = cell(1,numel(seqNames));
rels_all = [];
seq_ids = {};
for i = 1:numel(seqNames)
fname = [resDir seqNames{i} '.csv'];
if(i == 1)
% First read in the column names
tab = readtable(fname);
column_names = tab.Properties.VariableNames;
confidence_id = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'confidence'));
rot_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'pose_R'));
end
all_params = dlmread(fname, ',', 1, 0);
rot{i} = all_params(:, rot_ids);
rels = all_params(:, confidence_id);
posesGround = load ([gtDir seqNames{i} '.dat']);
% the reliabilities of head pose
rels_all = cat(1, rels_all, rels);
% Flip because of different conventions
rot{i}(:,2) = -rot{i}(:,2);
rot{i}(:,3) = -rot{i}(:,3);
% Convert to radians
rotg{i} = posesGround(2:end,[7 6 5]) * (pi/180);
% Correct the first frame so it corresponds to (0,0,0), as slightly
% different pose might be assumed frontal and this corrects for
% that
% Work out the correction matrix for ground truth
rot_corr_gt = Euler2Rot(rotg{i}(1,:));
for r_e = 1:size(rotg{i},1)
rot_curr_gt = Euler2Rot(rotg{i}(r_e,:));
rot_new_gt = rot_corr_gt' * rot_curr_gt;
rotg{i}(r_e,:) = Rot2Euler(rot_new_gt);
end
% Work out the correction matrix for estimates
rot_corr_est = Euler2Rot(rot{i}(1,:));
for r_e = 1:size(rot{i},1)
rot_curr_est = Euler2Rot(rot{i}(r_e,:));
rot_new_est = rot_corr_est' * rot_curr_est;
rot{i}(r_e,:) = Rot2Euler(rot_new_est);
end
% Convert to degrees
rotg{i} = rotg{i} * 180 / pi;
rot{i} = rot{i} * 180 / pi;
rotMeanErr(i,:) = mean(abs((rot{i}(:,:)-rotg{i}(:,:))));
rotRMS(i,:) = sqrt(mean(((rot{i}(:,:)-rotg{i}(:,:))).^2));
seq_ids = cat(1, seq_ids, repmat(seqNames(i), size(rot{i},1), 1));
end
allRot = cell2mat(rot');
allRotg = cell2mat(rotg');
meanError = mean(abs((allRot(:,:)-allRotg(:,:))));
all_errors = abs(allRot-allRotg);
rmsError = sqrt(mean(((allRot(:,:)-allRotg(:,:))).^2));
errorVariance = var(abs((allRot(:,:)-allRotg(:,:))));
all_rot_preds = allRot;
all_rot_gts = allRotg;

101
pkg/OpenFace/matlab_runners/Head Pose Experiments/calcBiwiError.m Normal file
View File

@@ -0,0 +1,101 @@
function [meanError, all_rot_preds, all_rot_gts, meanErrors, all_errors, rels_all, seq_ids] = calcBiwiError(resDir, gtDir)
seqNames = {'01','02','03','04','05','06','07','08','09', ...
'10', '11','12','13','14','15','16','17','18','19', ...
'20', '21','22','23','24'};
rotMeanErr = zeros(numel(seqNames),3);
rotRMS = zeros(numel(seqNames),3);
rot = cell(1,numel(seqNames));
rotg = cell(1,numel(seqNames));
rels_all = [];
seq_ids = {};
for i=1:numel(seqNames)
posesGround = load ([gtDir '/' seqNames{i} '/groundTruthPose.txt']);
fname = [resDir seqNames{i} '.csv'];
if(i == 1)
% First read in the column names
tab = readtable(fname);
column_names = tab.Properties.VariableNames;
confidence_id = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'confidence'));
rot_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'pose_R'));
t_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'pose_T'));
end
all_params = dlmread(fname, ',', 1, 0);
T = all_params(:,t_ids);
tx = T(:,1);
ty = T(:,2);
tz = T(:,3);
rot{i} = all_params(:, rot_ids);
rels = all_params(:, confidence_id);
% the reliabilities of head pose
rels_all = cat(1, rels_all, rels);
rotg{i} = posesGround(:,[5 6 7]);
% Correct the first frame so it corresponds to (0,0,0), as slightly
% different pose might be assumed frontal and this corrects for
% that
% Work out the correction matrix for ground truth
rot_corr_gt = Euler2Rot(rotg{i}(1,:));
for r_e = 1:size(rotg{i},1)
rot_curr_gt = Euler2Rot(rotg{i}(r_e,:));
rot_new_gt = rot_corr_gt' * rot_curr_gt;
rotg{i}(r_e,:) = Rot2Euler(rot_new_gt);
end
% First move the orientation to camera space
zx = sqrt(tx.^2 + tz.^2);
eul_x = atan2(ty, zx);
zy = sqrt(ty.^2 + tz.^2);
eul_y = -atan2(tx, zy);
for r_e = 1:size(rot{i},1)
cam_rot = Euler2Rot([eul_x(r_e), eul_y(r_e), 0]);
h_rot = Euler2Rot(rot{i}(r_e,:));
c_rot = cam_rot * h_rot;
rot{i}(r_e,:) = Rot2Euler(c_rot);
end
% Work out the correction matrix for estimates
rot_corr_est = Euler2Rot(rot{i}(1,:));
for r_e = 1:size(rot{i},1)
rot_curr_est = Euler2Rot(rot{i}(r_e,:));
rot_new_est = rot_corr_est' * rot_curr_est;
rot{i}(r_e,:) = Rot2Euler(rot_new_est);
end
rotg{i} = rotg{i} * 180 / pi;
rot{i} = rot{i} * 180 / pi;
rotMeanErr(i,:) = mean(abs((rot{i}(:,:)-rotg{i}(:,:))));
rotRMS(i,:) = sqrt(mean(((rot{i}(:,:)-rotg{i}(:,:))).^2));
seq_ids = cat(1, seq_ids, repmat(seqNames(i), size(rot{i},1), 1));
end
%%
meanErrors = rotMeanErr;
allRot = cell2mat(rot');
allRotg = cell2mat(rotg');
meanError = mean(abs((allRot(:,:)-allRotg(:,:))));
all_errors = abs(allRot-allRotg);
rmsError = sqrt(mean(((allRot(:,:)-allRotg(:,:))).^2));
errorVariance = std(abs((allRot(:,:)-allRotg(:,:))));
all_rot_preds = allRot;
all_rot_gts = allRotg;

85
pkg/OpenFace/matlab_runners/Head Pose Experiments/calcIctError.m Normal file
View File

@@ -0,0 +1,85 @@
function [meanError, all_rot_preds, all_rot_gts, meanErrors, all_errors, rels_all, seq_ids] = calcIctError(resDir, gtDir)
%CALCICTERROR Summary of this function goes here
% Detailed explanation goes here
polhemus = 'polhemusNorm.csv';
sequences = dir([resDir '*.csv']);
rotMeanErr = zeros(numel(sequences),3);
rotRMS = zeros(numel(sequences),3);
rot = cell(1,numel(sequences));
rotg = cell(1,numel(sequences));
rels_all = [];
seq_ids = {};
for i = 1:numel(sequences)
[~, name,~] = fileparts(sequences(i).name);
fname = [resDir '/' sequences(i).name];
if(i == 1)
% First read in the column names
tab = readtable(fname);
column_names = tab.Properties.VariableNames;
confidence_id = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'confidence'));
rot_ids = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'pose_R'));
end
all_params = dlmread(fname, ',', 1, 0);
rot{i} = all_params(:, rot_ids);
rels = all_params(:, confidence_id);
% the reliabilities of head pose
rels_all = cat(1, rels_all, rels);
[txg tyg tzg rxg ryg rzg] = textread([gtDir name '/' polhemus], '%f,%f,%f,%f,%f,%f');
rotg{i} = [rxg ryg rzg];
% Correct the first frame so it corresponds to (0,0,0), as slightly
% different pose might be assumed frontal and this corrects for
% that
% Work out the correction matrix for ground truth
rot_corr_gt = Euler2Rot(rotg{i}(1,:));
for r_e = 1:size(rotg{i},1)
rot_curr_gt = Euler2Rot(rotg{i}(r_e,:));
rot_new_gt = rot_corr_gt' * rot_curr_gt;
rotg{i}(r_e,:) = Rot2Euler(rot_new_gt);
end
% Work out the correction matrix for estimates
rot_corr_est = Euler2Rot(rot{i}(1,:));
for r_e = 1:size(rot{i},1)
rot_curr_est = Euler2Rot(rot{i}(r_e,:));
rot_new_est = rot_corr_est' * rot_curr_est;
rot{i}(r_e,:) = Rot2Euler(rot_new_est);
end
% Convert the ground truth and estimates to degrees
rot{i} = rot{i} * (180/ pi);
rotg{i} = rotg{i} * (180/ pi);
% Now compute the errors
rotMeanErr(i,:) = mean(abs((rot{i}(:,:)-rotg{i}(:,:))));
rotRMS(i,:) = sqrt(mean(((rot{i}(:,:)-rotg{i}(:,:))).^2));
seq_ids = cat(1, seq_ids, repmat({[name 'ict']}, size(rot{i},1), 1));
end
allRot = cell2mat(rot');
allRotg = cell2mat(rotg');
meanErrors = rotMeanErr;
meanError = mean(abs((allRot(:,:)-allRotg(:,:))));
all_errors = abs(allRot-allRotg);
rmsError = sqrt(mean(((allRot(:,:)-allRotg(:,:))).^2));
errorVariance = var(abs((allRot(:,:)-allRotg(:,:))));
all_rot_preds = allRot;
all_rot_gts = allRotg;
end

58
pkg/OpenFace/matlab_runners/Head Pose Experiments/chehra_err_biwi.m Normal file
View File

@@ -0,0 +1,58 @@
gtDir = [database_root biwi_dir];
seqNames = {'01','02','03','04','05','06','07','08','09', ...
'10', '11','12','13','14','15','16','17','18','19', ...
'20', '21','22','23','24'};
rotMeanErr = zeros(numel(seqNames),3);
rotRMS = zeros(numel(seqNames),3);
rot = cell(1,numel(seqNames));
rotg = cell(1,numel(seqNames));
chehra_res = pose_results;
tic;
for i=1:numel(seqNames)
posesGround = load ([gtDir '/' seqNames{i} '/groundTruthPose.txt']);
rotg{i} = posesGround(:,[5 6 7]);
rot{i} = chehra_res(i).pose * pi / 180;
rot{i}(:,2) = -rot{i}(:,2);
% Correct the first frame so it corresponds to (0,0,0), as slightly
% different pose might be assumed frontal and this corrects for
% that
% Work out the correction matrix for ground truth
rot_corr_gt = Euler2Rot(rotg{i}(1,:));
for r_e = 1:size(rotg{i},1)
rot_curr_gt = Euler2Rot(rotg{i}(r_e,:));
rot_new_gt = rot_corr_gt' * rot_curr_gt;
rotg{i}(r_e,:) = Rot2Euler(rot_new_gt);
end
% Work out the correction matrix for estimates
rot_corr_est = Euler2Rot(rot{i}(1,:));
for r_e = 1:size(rot{i},1)
rot_curr_est = Euler2Rot(rot{i}(r_e,:));
rot_new_est = rot_corr_est' * rot_curr_est;
rot{i}(r_e,:) = Rot2Euler(rot_new_est);
end
rotg{i} = rotg{i} * 180 / pi;
rot{i} = rot{i} * 180 / pi;
rotMeanErr(i,:) = mean(abs((rot{i}(:,:)-rotg{i}(:,:))));
rotRMS(i,:) = sqrt(mean(((rot{i}(:,:)-rotg{i}(:,:))).^2));
end
%%
meanErrors = rotMeanErr;
allRot = cell2mat(rot');
allRotg = cell2mat(rotg');
meanError = mean(abs((allRot(:,:)-allRotg(:,:))));
all_errors = abs(allRot-allRotg);
rmsError = sqrt(mean(((allRot(:,:)-allRotg(:,:))).^2));
errorVariance = std(abs((allRot(:,:)-allRotg(:,:))));
all_rot_preds = allRot;
all_rot_gts = allRotg;

51
pkg/OpenFace/matlab_runners/Head Pose Experiments/find_paired_matches.m Normal file
View File

@@ -0,0 +1,51 @@
% Working out corrections for head pose and model correlations
clear
%%
% first need to run run_clm_head_pose_tests_clnf
if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
database_root = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];
else
database_root = 'F:/Dropbox/Dropbox/AAM/test data/';
end
buDir = [database_root, '/bu/uniform-light/'];
resFolderBUclnf_general = [database_root, '/bu/uniform-light/CLMr3/'];
[~, pred_hp_bu, gt_hp_bu, ~, rels_bu] = calcBUerror(resFolderBUclnf_general, buDir);
biwi_dir = '/biwi pose/';
biwi_results_root = '/biwi pose results/';
res_folder_clnf_general = '/biwi pose results//CLMr4/';
[~, pred_hp_biwi, gt_hp_biwi, ~, ~, rels_biwi] = calcBiwiError([database_root res_folder_clnf_general], [database_root biwi_dir]);
ict_dir = ['ict/'];
ict_results_root = ['ict results/'];
res_folder_ict_clnf_general = 'ict results//CLMr4/';
[~, pred_hp_ict, gt_hp_ict, ~, ~, rel_ict] = calcIctError([database_root res_folder_ict_clnf_general], [database_root ict_dir]);
% Finding matching pairs to make sure they are independently distributed?
%
%%
all_hps = cat(1, pred_hp_bu, pred_hp_biwi, pred_hp_ict);
all_gts = cat(1, gt_hp_bu, gt_hp_biwi, gt_hp_ict);
all_rels = cat(1, rels_bu, rels_biwi, rel_ict);
rel_frames = all_rels > 0.8;
all_err = mean(abs(all_gts - all_hps), 2);
all_hps = all_hps(rel_frames, :);
all_gts = all_gts(rel_frames, :);
% Variation along pitch when others are close to 0
pitch_bins = [-40:5:40];
for p = pitch_bins
rel_frames = find(abs(all_gts(:,2))<3 & abs(all_gts(:,3))<3 & abs(all_gts(:,1) - p)<3);
if ~isempty(rel_frames)
corr_coeff = corr(all_hps(rel_frames,1), all_gts(rel_frames,1));
fprintf('%d, %.3f\n', numel(rel_frames), corr_coeff);
end
end
plot(find(abs(all_gts(:,1))<1 & abs(all_gts(:,3))<1));
plot(find(abs(all_gts(:,1))<1 & abs(all_gts(:,2))<1));

120286
pkg/OpenFace/matlab_runners/Head Pose Experiments/head_pose_scatter/pitch.eps Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/head_pose_scatter/pitch.tif Normal file
View File

Binary file not shown.

94740
pkg/OpenFace/matlab_runners/Head Pose Experiments/head_pose_scatter/roll.eps Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/head_pose_scatter/roll.tif Normal file
View File

Binary file not shown.

132419
pkg/OpenFace/matlab_runners/Head Pose Experiments/head_pose_scatter/yaw.eps Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/head_pose_scatter/yaw.tif Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_OF.mat Normal file
View File

Binary file not shown.

4
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_OF.txt Normal file
View File

@@ -0,0 +1,4 @@
Dataset and model, pitch, yaw, roll, mean, median
biwi error: 7.091, 5.170, 4.657, 5.639, 2.609
bu error: 2.739, 3.349, 2.459, 2.849, 1.974
ict error: 3.498, 3.986, 3.298, 3.594, 1.966

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.mat Normal file
View File

Binary file not shown.

4
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.txt Normal file
View File

@@ -0,0 +1,4 @@
Dataset and model, pitch, yaw, roll, mean, median
biwi error: 6.119, 5.033, 3.673, 4.942, 2.411
bu error: 2.396, 3.247, 2.409, 2.684, 1.835
ict error: 3.134, 3.512, 3.136, 3.261, 1.867

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_OF_CLNF.mat Normal file
View File

Binary file not shown.

4
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_OF_CLNF.txt Normal file
View File

@@ -0,0 +1,4 @@
Dataset and model, pitch, yaw, roll, mean, median
biwi error: 7.444, 5.100, 4.608, 5.717, 2.556
bu error: 2.776, 3.347, 2.447, 2.857, 1.987
ict error: 3.837, 3.698, 3.552, 3.696, 1.935

BIN
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_clm.mat Normal file
View File

Binary file not shown.

6
pkg/OpenFace/matlab_runners/Head Pose Experiments/results/Pose_clm.txt Normal file
View File

@@ -0,0 +1,6 @@
Dataset and model, pitch, yaw, roll, mean, median
biwi error clm: 8.181, 8.214, 6.539, 7.645, 3.336
biwi error clm-z: 7.693, 6.455, 6.267, 6.805, 3.138
bu error clm general: 2.992, 4.298, 2.579, 3.290, 2.151
ict error clm: 4.830, 4.192, 4.457, 4.493, 2.511
ict error clm-z: 5.089, 3.935, 4.670, 4.565, 2.493

36
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_biwi_experiment.m Normal file
View File

@@ -0,0 +1,36 @@
function [output_dir] = run_biwi_experiment(rootDir, biwiDir, verbose, varargin)
% Biwi dataset experiment
if(isunix)
executable = '"../../build/bin/FeatureExtraction"';
else
executable = '"../../x64/Release/FeatureExtraction.exe"';
end
output_dir = 'experiments/biwi_out';
dbSeqDir = dir([rootDir biwiDir]);
dbSeqDir = dbSeqDir(3:end);
output_dir = cat(2, output_dir, '/');
command = sprintf('%s -inroot "%s" -out_dir "%s" -fx 505 -fy 505 -cx 320 -cy 240 -pose -vis-track ', executable, rootDir, output_dir);
if(verbose)
command = cat(2, command, [' -tracked ' outputVideo]);
end
if(any(strcmp('model', varargin)))
command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']);
end
for i=1:numel(dbSeqDir)
inputFile = [biwiDir dbSeqDir(i).name '/colour.avi'];
command = sprintf('%s -f "%s" -of "%s" ', command, inputFile, dbSeqDir(i).name);
end
if(isunix)
unix(command, '-echo')
else
dos(command);
end

36
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_bu_experiment.m Normal file
View File

@@ -0,0 +1,36 @@
function [output_dir] = run_bu_experiment(bu_dir, verbose, varargin)
if(isunix)
executable = '"../../build/bin/FeatureExtraction"';
else
executable = '"../../x64/Release/FeatureExtraction.exe"';
end
output_dir = 'experiments/bu_out/';
buFiles = dir([bu_dir '*.avi']);
% Only outputing the pose (-pose)
command = sprintf('%s -inroot "%s" -out_dir "%s" -fx 500 -fy 500 -cx 160 -cy 120 -pose -vis-track ', executable, bu_dir, output_dir);
for i=1:numel(buFiles)
inputFile = [buFiles(i).name];
command = cat(2, command, sprintf(' -f "%s" ', inputFile));
end
if(verbose)
command = cat(2, command, [' -tracked ' outputVideo]);
end
if(any(strcmp('model', varargin)))
command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']);
end
if(isunix)
unix(command, '-echo')
else
dos(command);
end
end

75
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_f2f_videos.m Normal file
View File

@@ -0,0 +1,75 @@
executable = '"../../x64/Release/FeatureExtraction.exe"';
outputDir = 'D:\Datasets\face2face\2007_processed/';
% First collect the filenames of the data to be used
input_label_dir = 'D:\Datasets\face2face\f2f-2007-all-transcriptions/';
folds = dir([input_label_dir, '*Rapport*']);
listener_file_labels = {};
speaker_file_labels = {};
listener_vid_files = {};
speaker_vid_files = {};
speaker_vid_dir = 'D:\Datasets\face2face\f2f-2007-all-movie-speaker/';
listener_vid_dir = 'D:\Datasets\face2face\f2f-2007-all-movie-listener/';
verbose = true;
for i=1:numel(folds)
listener_file = dir([input_label_dir, folds(i).name, '/*.L.nod.eaf']);
speaker_file = dir([input_label_dir, folds(i).name, '/*.S.nod.eaf']);
if(~isempty(listener_file))
% Need to find the appropriate video file if it exists
num = listener_file.name(end-15:end-10);
vid_file_dir = dir([listener_vid_dir, '/*', num, '*']);
vid_file = dir([listener_vid_dir, '/', vid_file_dir.name, '/*.mp4']);
if(~isempty(vid_file))
listener_vid_files = cat(1, listener_vid_files, [listener_vid_dir, '/', vid_file_dir.name, '/', vid_file.name]);
listener_file_labels = cat(1, listener_file_labels, [input_label_dir, '/' folds(i).name, '/' listener_file.name]);
end
end
if(~isempty(speaker_file))
num = speaker_file.name(end-15:end-10);
vid_file_dir = dir([speaker_vid_dir, '/*', num, '*']);
vid_file = dir([speaker_vid_dir, '/', vid_file_dir.name, '/*.mp4']);
if(~isempty(vid_file))
speaker_vid_files = cat(1, speaker_vid_files, [speaker_vid_dir, '/', vid_file_dir.name, '/', vid_file.name]);
speaker_file_labels = cat(1, speaker_file_labels, [input_label_dir, '/' folds(i).name, '/' speaker_file.name]);
end
end
end
file_labels = cat(1, listener_file_labels, speaker_file_labels);
video_files = cat(1, listener_vid_files, speaker_vid_files);
parfor i=1:numel(file_labels)
[~,short_name,vid_ext] = fileparts(video_files{i});
command = executable;
inputFile = video_files{i};
outputFile = [outputDir short_name '.txt'];
outputEaf = [outputDir short_name '.eaf'];
command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']);
if(verbose)
outputVideo = [outputDir short_name '.track.avi'];
command = cat(2, command, [' -ov "' outputVideo '"']);
end
command = cat(2, command, [' -no2Dfp -no3Dfp -noMparams -noAUs -noGaze']);
dos(command);
copyfile(file_labels{i}, outputEaf);
copyfile(video_files{i}, [outputDir '/' short_name, '.mp4']);
end

78
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_f2f_videos_2006.m Normal file
View File

@@ -0,0 +1,78 @@
executable = '"../../x64/Release/FeatureExtraction.exe"';
outputDir = 'D:\Datasets\face2face\2006_processed/';
% First collect the filenames of the data to be used
input_label_dir = 'D:\Datasets\face2face\rapport-oct-2006-all-transcriptions/';
folds = dir([input_label_dir, '*SES*']);
listener_file_labels = {};
speaker_file_labels = {};
listener_vid_files = {};
speaker_vid_files = {};
speaker_vid_dir = 'D:\Datasets\face2face\rapport-oct-2006-all-movie-speaker/';
listener_vid_dir = 'D:\Datasets\face2face\rapport-oct-2006-all-movie-listener/';
verbose = true;
for i=1:numel(folds)
listener_file = dir([input_label_dir, folds(i).name, '/*.L.nod.eaf']);
speaker_file = dir([input_label_dir, folds(i).name, '/*.S.nod.eaf']);
if(~isempty(listener_file))
% Need to find the appropriate video file if it exists
num = listener_file.name(end-13:end-10);
vid_file_dir = dir([listener_vid_dir, '/*', num, '*']);
vid_file = dir([listener_vid_dir, '/', vid_file_dir.name, '/*.mp4']);
if(~isempty(vid_file))
listener_vid_files = cat(1, listener_vid_files, [listener_vid_dir, '/', vid_file_dir.name, '/', vid_file.name]);
listener_file_labels = cat(1, listener_file_labels, [input_label_dir, '/' folds(i).name, '/' listener_file.name]);
end
end
if(~isempty(speaker_file))
num = speaker_file.name(end-13:end-10);
vid_file_dir = dir([speaker_vid_dir, '/*', num, '*']);
vid_file = dir([speaker_vid_dir, '/', vid_file_dir.name, '/*.mp4']);
if(~isempty(vid_file))
speaker_vid_files = cat(1, speaker_vid_files, [speaker_vid_dir, '/', vid_file_dir.name, '/', vid_file.name]);
speaker_file_labels = cat(1, speaker_file_labels, [input_label_dir, '/' folds(i).name, '/' speaker_file.name]);
end
end
end
% file_labels = cat(1, listener_file_labels, speaker_file_labels);
% video_files = cat(1, listener_vid_files, speaker_vid_files);
file_labels = listener_file_labels;
video_files = listener_vid_files;
parfor i=1:numel(file_labels)
[~,short_name,vid_ext] = fileparts(video_files{i});
command = executable;
inputFile = video_files{i};
outputFile = [outputDir short_name '.txt'];
outputEaf = [outputDir short_name '.eaf'];
command = cat(2, command, [' -f "' inputFile '" -of "' outputFile '"']);
if(verbose)
outputVideo = [outputDir short_name '.track.avi'];
command = cat(2, command, [' -ov "' outputVideo '"']);
end
command = cat(2, command, [' -no2Dfp -no3Dfp -noMparams -noAUs -noGaze']);
dos(command);
copyfile(file_labels{i}, outputEaf);
copyfile(video_files{i}, [outputDir '/' short_name, '.mp4']);
end

53
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CECLM.m Normal file
View File

@@ -0,0 +1,53 @@
clear;
%%
% Find the dataset location
if exist('D:/Datasets/HeadPose', 'file')
database_root = 'D:/Datasets/HeadPose/';
elseif(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
database_root = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];
elseif(exist([getenv('USERPROFILE') 'F:/Dropbox/Dropbox/AAM/test data/'], 'file'))
database_root = 'F:/Dropbox/Dropbox/AAM/test data/';
elseif(exist('F:/Dropbox/AAM/test data/', 'file'))
database_root = 'F:/Dropbox/AAM/test data/';
elseif(exist('/multicomp/datasets/head_pose_dbs', 'file'))
database_root = '/multicomp/datasets/head_pose_dbs/';
elseif(exist('/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data', 'file'))
database_root = '/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data';
end
buDir = [database_root, '/bu/uniform-light/'];
%%
[resFolderBU_OF] = run_bu_experiment(buDir, false, 'model', 'model/main_ceclm_general.txt');
[bu_error_OF, pred_hp_bu, gt_hp_bu, all_errors_bu_OF, rels_bu] = calcBUerror(resFolderBU_OF, buDir);
%%
% Run the Biwi test
biwi_dir = '/biwi pose/';
[res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, false, 'model', 'model/main_ceclm_general.txt');
% Calculate the resulting errors
[biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError(res_folder_biwi_OF, [database_root biwi_dir]);
%% Run the ICT test
ict_dir = ['/ict/'];
% Intensity
[res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, false, 'model', 'model/main_ceclm_general.txt');
% Calculate the resulting errors
[ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]);
%% Save the results
filename = 'results/Pose_OF_CECLM';
save(filename);
% Also save them in a reasonable .txt format for easy comparison
f = fopen('results/Pose_OF_CECLM.txt', 'w');
fprintf(f, 'Dataset and model, pitch, yaw, roll, mean, median\n');
fprintf(f, 'biwi error: %.3f, %.3f, %.3f, %.3f, %.3f\n', biwi_error_OF, mean(all_errors_biwi_OF(:)), median(all_errors_biwi_OF(:)));
fprintf(f, 'bu error: %.3f, %.3f, %.3f, %.3f, %.3f\n', bu_error_OF, mean(all_errors_bu_OF(:)), median(all_errors_bu_OF(:)));
fprintf(f, 'ict error: %.3f, %.3f, %.3f, %.3f, %.3f\n', ict_error_OF, mean(all_errors_ict_OF(:)), median(all_errors_ict_OF(:)));
fclose(f);
clear 'f'

52
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CLNF.m Normal file
View File

@@ -0,0 +1,52 @@
clear;
%%
% Find the dataset location
if exist('D:/Datasets/HeadPose', 'file')
database_root = 'D:/Datasets/HeadPose/';
elseif(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
database_root = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];
elseif(exist(['F:/Dropbox/Dropbox/AAM/test data/'], 'file'))
database_root = 'F:/Dropbox/Dropbox/AAM/test data/';
elseif(exist(['F:/Dropbox/AAM/test data/'], 'file'))
database_root = 'F:/Dropbox/AAM/test data/';
elseif(exist('/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data', 'file'))
database_root = '/media/tadas/5E08AE0D08ADE3ED/Dropbox/AAM/test data';
end
buDir = [database_root, '/bu/uniform-light/'];
% The fast and accurate clnf
%%
[resFolderBU_OF] = run_bu_experiment(buDir, false, 'model', 'model/main_clnf_general.txt');
[bu_error_OF, pred_hp_bu, gt_hp_bu, all_errors_bu_OF, rels_bu] = calcBUerror(resFolderBU_OF, buDir);
%%
% Run the Biwi test
biwi_dir = '/biwi pose/';
[res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, false, 'model', 'model/main_clnf_general.txt');
% Calculate the resulting errors
[biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError(res_folder_biwi_OF, [database_root biwi_dir]);
%% Run the ICT test
ict_dir = ['/ict/'];
% Intensity
[res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, false, 'model', 'model/main_clnf_general.txt');
% Calculate the resulting errors
[ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]);
%% Save the results
filename = 'results/Pose_OF_CLNF';
save(filename);
% Also save them in a reasonable .txt format for easy comparison
f = fopen('results/Pose_OF_CLNF.txt', 'w');
fprintf(f, 'Dataset and model, pitch, yaw, roll, mean, median\n');
fprintf(f, 'biwi error: %.3f, %.3f, %.3f, %.3f, %.3f\n', biwi_error_OF, mean(all_errors_biwi_OF(:)), median(all_errors_biwi_OF(:)));
fprintf(f, 'bu error: %.3f, %.3f, %.3f, %.3f, %.3f\n', bu_error_OF, mean(all_errors_bu_OF(:)), median(all_errors_bu_OF(:)));
fprintf(f, 'ict error: %.3f, %.3f, %.3f, %.3f, %.3f\n', ict_error_OF, mean(all_errors_ict_OF(:)), median(all_errors_ict_OF(:)));
fclose(f);
clear 'f'

41
pkg/OpenFace/matlab_runners/Head Pose Experiments/run_ict_experiment.m Normal file
View File

@@ -0,0 +1,41 @@
function [output_dir] = run_ict_experiment(rootDir, ictDir, verbose, varargin)
%EVALUATEICTDATABASE Summary of this function goes here
% Detailed explanation goes here
if(isunix)
executable = '"../../build/bin/FeatureExtraction"';
else
executable = '"../../x64/Release/FeatureExtraction.exe"';
end
output_dir = 'experiments/ict_out';
dbSeqDir = dir([rootDir ictDir]);
dbSeqDir = dbSeqDir(3:end);
output_dir = cat(2, output_dir, '/');
command = sprintf('%s -inroot "%s" -out_dir "%s" -fx 535 -fy 536 -cx 327 -cy 241 -pose -vis-track ', executable, rootDir, output_dir);
if(verbose)
command = cat(2, command, [' -tracked ' outputVideo]);
end
if(any(strcmp('model', varargin)))
command = cat(2, command, [' -mloc "', varargin{find(strcmp('model', varargin))+1}, '"']);
end
for i=1:numel(dbSeqDir)
inputFile = [ictDir dbSeqDir(i).name '/colour undist.avi'];
command = cat(2, command, [' -f "' inputFile '" -of "' dbSeqDir(i).name '" ']);
end
if(isunix)
unix(command, '-echo')
else
dos(command);
end
end

98
pkg/OpenFace/matlab_runners/Head Pose Experiments/work_out_correlations.m Normal file
View File

@@ -0,0 +1,98 @@
% Working out corrections for head pose and model correlations
clear
%%
% first need to run run_clm_head_pose_tests_clnf
if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
database_root = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];
else
database_root = 'F:/Dropbox/Dropbox/AAM/test data/';
end
buDir = [database_root, '/bu/uniform-light/'];
resFolderBUclnf_general = [database_root, '/bu/uniform-light/CLMr3/'];
[~, pred_hp_bu, gt_hp_bu, ~, rels_bu] = calcBUerror(resFolderBUclnf_general, buDir);
biwi_dir = '/biwi pose/';
biwi_results_root = '/biwi pose results/';
res_folder_clnf_general = '/biwi pose results//CLMr4/';
[~, pred_hp_biwi, gt_hp_biwi, ~, ~, rels_biwi] = calcBiwiError([database_root res_folder_clnf_general], [database_root biwi_dir]);
ict_dir = ['ict/'];
ict_results_root = ['ict results/'];
res_folder_ict_clnf_general = 'ict results//CLMr4/';
[~, pred_hp_ict, gt_hp_ict, ~, ~, rel_ict] = calcIctError([database_root res_folder_ict_clnf_general], [database_root ict_dir]);
resFolderBUCLM_general = [database_root, '/bu/uniform-light/CLMr1/'];
[~, pred_hp_bu_clm, pred_gt_bu_clm, all_errors_bu_svr_general, rels_bu_clm] = calcBUerror(resFolderBUCLM_general, buDir);
biwi_dir = '/biwi pose/';
res_folder_clnf_general = '/biwi pose results//CLMr1/';
[~, pred_hp_biwi_clm, gt_hp_biwi_clm, ~, ~, rels_biwi_clm] = calcBiwiError([database_root res_folder_clnf_general], [database_root biwi_dir]);
biwi_dir = '/biwi pose/';
res_folder_clnf_general = '/biwi pose results//CLMr2_depth/';
[~, pred_hp_biwi_clmz, gt_hp_biwi_clmz, ~, ~, rels_biwi_clmz] = calcBiwiError([database_root res_folder_clnf_general], [database_root biwi_dir]);
res_folder_ict_clnf_general = 'ict results//CLMr1/';
[~, pred_hp_ict_clm, gt_hp_ict_clm, ~, ~, rel_ict_clm] = calcIctError([database_root res_folder_ict_clnf_general], [database_root ict_dir]);
ict_results_root = ['ict results/'];
res_folder_ict_clnf_general = 'ict results//CLMr2_depth/';
[~, pred_hp_ict_clmz, gt_hp_ict_clmz, ~, ~, rel_ict_clmz] = calcIctError([database_root res_folder_ict_clnf_general], [database_root ict_dir]);
%%
%all_hps = cat(1, pred_hp_bu, pred_hp_biwi, pred_hp_ict, pred_hp_bu_clm, pred_hp_biwi_clm, pred_hp_biwi_clmz, pred_hp_ict_clm, pred_hp_ict_clmz);
%all_gts = cat(1, gt_hp_bu, gt_hp_biwi, gt_hp_ict, pred_gt_bu_clm, gt_hp_biwi_clm, gt_hp_biwi_clmz, gt_hp_ict_clm, gt_hp_ict_clmz);
% all_rels = cat(1, rels_bu, rels_biwi, rel_ict, rels_bu_clm, rels_biwi_clm, rels_biwi_clmz, rel_ict_clm, rel_ict_clmz);
all_hps = cat(1, pred_hp_bu, pred_hp_biwi, pred_hp_ict);
all_gts = cat(1, gt_hp_bu, gt_hp_biwi, gt_hp_ict);
all_rels = cat(1, rels_bu, rels_biwi, rel_ict);
rel_frames = all_rels > 0.8;
all_err = mean(abs(all_gts - all_hps), 2);
corr(all_hps, all_gts)
corr(all_hps(rel_frames, :), all_gts(rel_frames, :))
centres_all = [0 0 0;
0 -20 0;
0 -45 0;
0 -70 0;
0 20 0;
0 45 0;
0 70 0];
ids = zeros(size(all_err,1),1);
mins = [];
for i=1:size(centres_all, 1)
mins = cat(2, mins, mean(abs(bsxfun(@plus, all_hps, -centres_all(i,:))), 2));
end
% find the center id for each of the frames
[~, ids] = min(mins');
ids = ids';
%%
% pitch_err = mean(abs(all_hps(rel_frames,1) - all_gts(rel_frames,1)))
% yaw_err = mean(abs(all_hps(rel_frames,2) - all_gts(rel_frames,2)))
% roll_err = mean(abs(all_hps(rel_frames,3) - all_gts(rel_frames,3)))
%% draw errors properly
yaw_bins = [-50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50];
err_yaw_bin = zeros(size(yaw_bins));
std_yaw_bin = zeros(size(yaw_bins));
yaw_bin = bsxfun(@plus, all_gts(:,2), yaw_bins);
[~, ids] = min(abs(yaw_bin'));
ids = ids';
for i=1:numel(yaw_bins)
rel_bins = ids == i & rel_frames;
err_bin(i) = mean(abs(all_hps(rel_bins,2) - all_gts(rel_bins,2)));
std_bin(i) = std(abs(all_hps(rel_bins,2) - all_gts(rel_bins,2)));
end