bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
pkg/OpenFace/model_training/learn_error_mapping/Collect_menpo_train_imgs.m Normal file
View File

@@ -0,0 +1,45 @@
function [images, detections, labels] = Copy_menpo_train_imgs(root_dir)
load('face_detections/menpo_train_dets.mat');
% Have three bounding box locations (frontal tuned, profile tuned)
detections = zeros(numel(bboxes), 4);
labels = cell(numel(bboxes),1);
for i=1:numel(bboxes)
images(i).img = [root_dir, bboxes(i).name];
% If face detected
if(~isempty(bboxes(i).bbox))
bbox = bboxes(i).bbox(1:4);
% Correct the MTCNN bounding box
width = bbox(3) - bbox(1);
height = bbox(4) - bbox(2);
tx = bbox(1);
ty = bbox(2);
% Frontal faces
new_width = width * 1.0323;
new_height = height * 0.7751;
new_tx = width * -0.0075 + tx;
new_ty = height * 0.2459 + ty;
detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
else % If face not detected, use the mean location of the face in training data
img_size = size(imread([root_dir, bboxes(i).name]));
img_width = img_size(2);
img_height = img_size(1);
width = img_width * 0.4421;
height = img_height * 0.445;
tx = img_width * 0.5048 - 0.5 * width;
ty = img_height * 0.5166 - 0.5 * height;
detections(i,:) = [tx, ty, tx + width, ty + height];
end
labels{i} = bboxes(i).gt_landmarks;
end
end

71
pkg/OpenFace/model_training/learn_error_mapping/Script_CECLM_general_cross_data_multi_hyp.m Normal file
View File

@@ -0,0 +1,71 @@
function Script_CECLM_general_cross_data_multi_hyp()
addpath(genpath('../'));
[images, detections, labels] = Collect_menpo_train_imgs('D:\Datasets\menpo/');
%% loading the patch experts
[patches, pdm, clmParams] = Load_CECLM_general();
%% Fitting the model to the provided image
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
num_views = size(views,1);
clmParams.numPatchIters = 1;
% for recording purposes
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), num_views);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),num_views);
all_lmark_lhoods = zeros(num_points, numel(images),num_views);
all_views_used = zeros(numel(images),num_views);
errors_view = zeros(numel(images),num_views);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
verbose = true;
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% Find the best orientation
for v = 1:size(views,1)
[shape,~,~,lhood,lmark_lhoods,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
shapes_all{i,v} = shape;
all_views_used(i,v) = view_used;
all_lmark_lhoods(:, i,v) = lmark_lhoods;
lhoods(i,v) = lhood;
errors_view(i,v) = compute_error_menpo_1(labels(i), {shape});
end
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
end
toc
experiment.lhoods = lhoods;
experiment.errors_view = errors_view;
experiment.all_views_used = all_views_used;
%%
output_results = ['results/results_ceclm_general.mat'];
save(output_results, 'experiment');
end

71
pkg/OpenFace/model_training/learn_error_mapping/Script_CECLM_menpo_multi_hyp.m Normal file
View File

@@ -0,0 +1,71 @@
function Script_CECLM_menpo_multi_hyp()
addpath(genpath('../'));
[images, detections, labels] = Collect_menpo_train_imgs('D:\Datasets\menpo/');
%% loading the patch experts
[patches, pdm, clmParams] = Load_CECLM_menpo();
%% Fitting the model to the provided image
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
num_views = size(views,1);
clmParams.numPatchIters = 1;
% for recording purposes
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), num_views);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),num_views);
all_lmark_lhoods = zeros(num_points, numel(images),num_views);
all_views_used = zeros(numel(images),num_views);
errors_view = zeros(numel(images),num_views);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
verbose = true;
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% Find the best orientation
for v = 1:size(views,1)
[shape,~,~,lhood,lmark_lhoods,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
shapes_all{i,v} = shape;
all_views_used(i,v) = view_used;
all_lmark_lhoods(:, i,v) = lmark_lhoods;
lhoods(i,v) = lhood;
errors_view(i,v) = compute_error_menpo_1(labels(i), {shape});
end
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
end
toc
experiment.lhoods = lhoods;
experiment.errors_view = errors_view;
experiment.all_views_used = all_views_used;
%%
output_results = ['results/results_ceclm_menpo.mat'];
save(output_results, 'experiment');
end

225
pkg/OpenFace/model_training/learn_error_mapping/Script_CECLM_menpo_sparse_sel_frontal.m Normal file
View File

@@ -0,0 +1,225 @@
clear
addpath(genpath('../'));
[images, detections, labels] = Collect_menpo_train_imgs('G:\datasets\menpo/');
%% Setup recording
[~, pdm, ~, ~] = Load_CECLM_OF();
%% Identify orientation of each of the training images
orientations = zeros(numel(images), 3);
for i=1:numel(images)
landmarks = standardise_landmarks(labels{i});
[ a, R, T, T3D, params, error, shapeOrtho ] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, landmarks);
orientations(i,:) = Rot2Euler(R);
end
%% Keep only the relevant orientations for that view
angle_dist = sum(abs(orientations)')';
to_keep = angle_dist < 25 * pi/180;
images = images(to_keep);
detections = detections(to_keep,:);
labels = labels(to_keep);
%%
% Change if you want to visualize the outputs
verbose = false;
if(verbose)
f = figure;
end
view = 1;
%% Mirror indices
mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
remove_all = cell(4, 1);
errors_all_med = cell(4,1);
errors_all_mean = cell(4,1);
% To start where we finished across scales
gparams = cell(numel(images), 1);
lparams = cell(numel(images), 1);
for scale = 1:4
remove_all_c = [];
errors_all_med_c = [];
errors_all_mean_c = [];
[patches, pdm, clmParams, early_term_params] = Load_CECLM_OF();
clmParams.numPatchIters = 1;
clmParams.startScale = scale;
patches = patches(1:scale);
for s=1:scale
patches(s).correlations = patches(s).correlations(view,:);
patches(s).centers = patches(s).centers(view,:);
patches(s).rms_errors = patches(s).rms_errors(view,:);
patches(s).visibilities = patches(s).visibilities(view,:);
patches(s).patch_experts = patches(s).patch_experts(view,:);
end
visi_old = patches(scale).visibilities;
shapes_base = cell(numel(labels), 1);
% First establish the basic error without removing landmarks at this
% scale
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% Precomputing patch experts
if(scale == 1)
[shape] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, []);
else
[shape] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'gparam', gparams{i}, 'lparam', lparams{i});
end
shapes_base{i} = shape;
% v_points = logical(patches(scale).visibilities(1,:))';
% DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
error_base_mean = mean(compute_error_menpo_1(labels, shapes_base));
error_base_med = median(compute_error_menpo_1(labels, shapes_base));
fprintf('Base error scale %d - %.3f\n', scale, error_base_med);
for s=1:30
% First remove the indicies already decided to remove
patches(scale).visibilities = visi_old;
patches(scale).visibilities(view, remove_all_c) = 0;
% Indices to test removing
% Identify which indices to remove
to_rem_potential = find(patches(scale).visibilities(view,:));
% Now remove the mirror inds from them
to_remove = intersect(mirror_inds(:,2), to_rem_potential);
for i=numel(to_remove):-1:1
to_rem_potential(to_rem_potential==to_remove(i)) = [];
end
to_test_num = numel(to_rem_potential);
gparams_all = cell(numel(images), to_test_num);
lparams_all = cell(numel(images), to_test_num);
shapes_all = cell(numel(images), to_test_num);
labels_all = cell(numel(images), to_test_num);
to_rems = to_rem_potential;
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% Precomputing patch experts
if(scale == 1)
[~,~,~,~,~,~, precomp] = Fitting_from_bb_precomp(image, [], bbox, pdm, patches, clmParams, []);
else
[~,~,~,~,~,~, precomp] = Fitting_from_bb_precomp(image, [], bbox, pdm, patches, clmParams, [], 'gparam', gparams{i}, 'lparam', lparams{i});
end
for v=1:numel(to_rem_potential)
patches(scale).visibilities(view, to_rem_potential(v)) = 0;
mirr_id = union(mirror_inds(find(mirror_inds(:,1)==to_rem_potential(v),1),2),... ;
mirror_inds(find(mirror_inds(:,2)==to_rem_potential(v),1),1));
if(~isempty(mirr_id))
patches(scale).visibilities(view, mirr_id) = 0;
end
%% Fitting the model to the provided images
if(scale == 1)
[shape,gparams_all{i,v},lparams_all{i,v},lhood,lmark_lhood,view_used] = Fitting_from_bb_precomp(image, [], bbox, pdm, patches, clmParams, precomp);
else
[shape,gparams_all{i,v},lparams_all{i,v},lhood,lmark_lhood,view_used] = Fitting_from_bb_precomp(image, [], bbox, pdm, patches, clmParams, precomp, 'gparam', gparams{i}, 'lparam', lparams{i});
end
shapes_all{i,v} = shape;
labels_all{i,v} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
if(verbose)
v_points = logical(patches(scale).visibilities(view_used,:))';
DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
% Clean up
patches(scale).visibilities = visi_old;
patches(scale).visibilities(view, remove_all_c) = 0;
end
end
errors_mean = zeros(numel(to_rem_potential),1);
errors_median = zeros(numel(to_rem_potential),1);
for v=1:numel(to_rem_potential)
errors_mean(v) = mean(compute_error_menpo_1(labels_all(:,v), shapes_all(:,v)));
errors_median(v) = median(compute_error_menpo_1(labels_all(:,v), shapes_all(:,v)));
end
[~, id] = min(errors_mean);
val_med = errors_median(id);
val_mean = errors_mean(id);
remove_all_c = cat(2, remove_all_c, to_rems(id));
errors_all_med_c = cat(2, errors_all_med_c, val_med);
errors_all_mean_c = cat(2, errors_all_mean_c, val_mean);
mirr_id = mirror_inds(find(mirror_inds(:,1)==to_rems(id),1),2);
if(~isempty(mirr_id))
remove_all_c = cat(2, remove_all_c, mirr_id);
errors_all_med_c = cat(2, errors_all_med_c, val_med);
errors_all_mean_c = cat(2, errors_all_mean_c, val_mean);
end
remove_all{scale} = remove_all_c;
errors_all_med{scale} = errors_all_med_c;
errors_all_mean{scale} = errors_all_mean_c;
%%
output_results = 'sparse_selection/menpo_frontal_sparse.mat';
save(output_results, 'remove_all', 'errors_all_med', 'errors_all_mean');
fprintf('Curr error rem %d - %.3f\n', s, errors_all_med_c(end));
% Going to the next scale
if(0.99 * errors_all_med_c(end) > error_base_med)
break;
end
end
gparams = gparams_all(:,id);
lparams = lparams_all(:,id);
end

187
pkg/OpenFace/model_training/learn_error_mapping/Script_CECLM_menpo_sparse_sel_profile.m Normal file
View File

@@ -0,0 +1,187 @@
function Script_CECLM_menpo_sparse_sel_profile()
addpath(genpath('../'));
[images, detections, labels] = Collect_menpo_train_imgs('G:\datasets\menpo/');
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
patches = patches(1);
%% Setup recording
clmParams.numPatchIters = 1;
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
%% Identify orientation of each of the training images
orientations = zeros(numel(images), 3);
centres_all = patches.centers;
num_centers = size(centres_all,1);
views_all = zeros(numel(images), 1);
for i=1:numel(images)
landmarks = standardise_landmarks(labels{i});
[ a, R, T, T3D, params, error, shapeOrtho ] = fit_PDM_ortho_proj_to_2D(pdm.M, pdm.E, pdm.V, landmarks);
orientations(i,:) = Rot2Euler(R);
[~, view] = min(sum(abs(centres_all * pi/180 - repmat(orientations(i,:) , num_centers, 1)),2));
% views_all(i) = view;
end
%% Keep only the relevant orientations for that view
or_id = 2;
view = or_id;
mirr_view = 9 - view;
orientation_1 = patches.centers(or_id,:) * pi/180;
orientation_2 = orientation_1;
orientation_2(2) = -orientation_2(2);
% Keep the ones closest to orientation of interest
angle_dist_1 = sum(abs(bsxfun(@plus, orientations, -orientation_1))')';
to_keep_1 = angle_dist_1 < 15 * pi/180;
angle_dist_2 = sum(abs(bsxfun(@plus, orientations, -orientation_2))')';
to_keep_2 = angle_dist_2 < 15 * pi/180;
to_keep = to_keep_1 | to_keep_2;
% Keep track of which view is used
to_keep_1 = to_keep_1(to_keep);
to_keep_2 = to_keep_2(to_keep);
images = images(to_keep);
detections = detections(to_keep,:);
labels = labels(to_keep);
shapes_all = cell(numel(images), 1);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
%%
% Change if you want to visualize the outputs
verbose = false;
if(verbose)
f = figure;
end
remove_all = [];
remove_all_mirr = [];
errors_all_med = [];
errors_all_mean = [];
visi_old = patches(1).visibilities;
rng(0);
%% Mirror indices
mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
for s=1:30
% First remove the indicies already decided to remove
patches(1).visibilities = visi_old;
patches(1).visibilities(view, remove_all) = 0;
patches(1).visibilities(mirr_view, remove_all_mirr) = 0;
% Indices to test removing
% Identify which indices are up for removal
to_rem_potential = find(patches(1).visibilities(view,:));
to_test_num = numel(to_rem_potential);
errors_mean = zeros(to_test_num, 1);
errors_median = zeros(to_test_num, 1);
to_rems = zeros(to_test_num, 1);
to_rems_mirr = zeros(to_test_num, 1);
for v=1:numel(to_rem_potential)
to_rem_c = to_rem_potential(v);
% See if there's a mirror index to this and remove both
patches(1).visibilities(view, to_rem_c) = 0;
to_rems(v) = to_rem_c;
mirr_id = union(mirror_inds(find(mirror_inds(:,1)==to_rem_c,1),2),... ;
mirror_inds(find(mirror_inds(:,2)==to_rem_c,1),1));
if(~isempty(mirr_id))
patches(1).visibilities(mirr_view, mirr_id) = 0;
to_rems_mirr(v) = mirr_id;
else
patches(1).visibilities(mirr_view, to_rem_c) = 0;
to_rems_mirr(v) = to_rem_c;
end
%% Fitting the model to the provided images
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% pick the view
if(to_keep_1(i))
or = orientation_1;
else
or = orientation_2;
end
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', or);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
end
% Reset the visibilities
patches(1).visibilities = visi_old;
patches(1).visibilities(view, remove_all) = 0;
patches(1).visibilities(mirr_view, remove_all_mirr) = 0;
errors_mean(v) = mean(compute_error_menpo_1(labels_all(1:i), shapes_all(1:i)));
errors_median(v) = median(compute_error_menpo_1(labels_all(1:i), shapes_all(1:i)));
end
[~, id] = min(errors_mean);
val_med = errors_median(id);
val_mean = errors_mean(id);
remove_all = cat(2, remove_all, to_rems(id));
remove_all_mirr = cat(2, remove_all_mirr, to_rems_mirr(id));
errors_all_med = cat(2, errors_all_med, val_med);
errors_all_mean = cat(2, errors_all_mean, val_mean);
%%
output_results = 'sparse_selection/menpo_profile_1_sparse.mat';
save(output_results, 'remove_all', 'remove_all_mirr', 'errors_all_med', 'errors_all_mean');
end
toc
end

71
pkg/OpenFace/model_training/learn_error_mapping/Script_CECLM_of_multi_hyp.m Normal file
View File

@@ -0,0 +1,71 @@
function Script_CECLM_of_multi_hyp()
addpath(genpath('../'));
[images, detections, labels] = Collect_menpo_train_imgs('G:\datasets\menpo/');
%% loading the patch experts
[patches, pdm, clmParams] = Load_CECLM_OF();
%% Fitting the model to the provided image
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
num_views = size(views,1);
clmParams.numPatchIters = 1;
% for recording purposes
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), num_views);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),num_views);
all_lmark_lhoods = zeros(num_points, numel(images),num_views);
all_views_used = zeros(numel(images),num_views);
errors_view = zeros(numel(images),num_views);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
verbose = true;
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% Find the best orientation
for v = 1:size(views,1)
[shape,~,~,lhood,lmark_lhoods,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
shapes_all{i,v} = shape;
all_views_used(i,v) = view_used;
all_lmark_lhoods(:, i,v) = lmark_lhoods;
lhoods(i,v) = lhood;
errors_view(i,v) = compute_error_menpo_1(labels(i), {shape});
end
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
end
toc
experiment.lhoods = lhoods;
experiment.errors_view = errors_view;
experiment.all_views_used = all_views_used;
%%
output_results = ['results/results_ceclm_of.mat'];
save(output_results, 'experiment');
end

BIN
pkg/OpenFace/model_training/learn_error_mapping/cen_general_mapping.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/cen_menpo_mapping.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/cen_menpo_mapping_of.mat Normal file
View File

Binary file not shown.

85
pkg/OpenFace/model_training/learn_error_mapping/compute_error_menpo_1.m Normal file
View File

@@ -0,0 +1,85 @@
function [ error_per_image, frontal_ids ] = compute_error_menpo_1( ground_truth_all, detected_points_all )
%compute_error
% compute the average point-to-point Euclidean error normalized by the
% inter-ocular distance (measured as the Euclidean distance between the
% outer corners of the eyes)
%
% Inputs:
% grounth_truth_all, size: num_of_points x 2 x num_of_images
% detected_points_all, size: num_of_points x 2 x num_of_images
% Output:
% error_per_image, size: num_of_images x 1
num_of_images = numel(ground_truth_all);
error_per_image = zeros(num_of_images,1);
frontal_ids = true(num_of_images,1);
for i =1:num_of_images
detected_points = detected_points_all{i} + 0.5;
ground_truth_points = ground_truth_all{i};
num_of_points = size(ground_truth_points,1);
normalization = (max(ground_truth_points(:,1))-min(ground_truth_points(:,1)))+...
(max(ground_truth_points(:,2))-min(ground_truth_points(:,2)));
normalization = normalization / 2;
if(num_of_points==39)
frontal_ids(i) = false;
landmark_labels = zeros(68,2);
% Need to map to the profile points, and normalize based on size
% instead
left_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,32; 23,39; 24,38; 25,37; 26,42; 27,41;
28,52; 29,51; 30,50; 31,49; 32,60; 33,59; 34,58;
35,63; 36,62; 37,61; 38,68; 39,67];
right_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,36; 23,44; 24,45; 25,46; 26,47; 27,48;
28,52; 29,53; 30,54; 31,55; 32,56; 33,57; 34,58;
35,63; 36,64; 37,65; 38,66; 39,67];
% Determine if the points are clock-wise or counter clock-wise
% Clock-wise points are facing left, counter-clock-wise right
sum = 0;
for k=1:11
step = (ground_truth_points(k+1,1) - ground_truth_points(k,1)) * (ground_truth_points(k+1,2) + ground_truth_points(k,2));
sum = sum + step;
end
if(sum > 0)
% First need to resample the face outline as there are 9
% points in the near-frontal and 10 points in profile for
% the outline of the face
outline = iterate_piece_wise(ground_truth_points(1:10,:), 9);
brow = iterate_piece_wise(ground_truth_points(13:16,:), 5);
landmark_labels(1:9,:) = outline;
landmark_labels(18:22,:) = brow;
landmark_labels(left_to_frontal_map(:,2),:) = ground_truth_points(left_to_frontal_map(:,1),:);
else
outline = iterate_piece_wise(ground_truth_points(10:-1:1,:), 9);
brow = iterate_piece_wise(ground_truth_points(16:-1:13,:), 5);
landmark_labels(9:17,:) = outline;
landmark_labels(23:27,:) = brow;
landmark_labels(right_to_frontal_map(:,2),:) = ground_truth_points(right_to_frontal_map(:,1),:);
end
detected_points = detected_points(landmark_labels(:,1)~=0,:);
ground_truth_points = landmark_labels(landmark_labels(:,1)~=0,:);
end
num_of_points = size(ground_truth_points,1);
sum=0;
for j=1:num_of_points
sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:));
end
error_per_image(i) = sum/(num_of_points*normalization);
end
end

1
pkg/OpenFace/model_training/learn_error_mapping/early_term_cen.txt Normal file
View File

@@ -0,0 +1 @@
1.000 0.954 0.736 0.903 0.903 0.736 0.954 0.000 0.072 -0.279 -0.380 -0.380 -0.279 0.072 -0.270 -0.230 0.070 0.210 0.210 0.070 -0.230

1
pkg/OpenFace/model_training/learn_error_mapping/early_term_cen_of.txt Normal file
View File

@@ -0,0 +1 @@
1.000 1.005 1.054 0.976 0.976 1.054 1.005 0.000 0.049 0.121 -0.370 -0.370 0.121 0.049 -0.230 -0.200 -0.140 0.260 0.260 -0.140 -0.200

BIN
pkg/OpenFace/model_training/learn_error_mapping/face_detections/menpo_test_frontal.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/face_detections/menpo_test_profile.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/face_detections/menpo_train_dets.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/face_detections/menpo_valid_dets.mat Normal file
View File

Binary file not shown.

32
pkg/OpenFace/model_training/learn_error_mapping/iterate_piece_wise.m Normal file
View File

@@ -0,0 +1,32 @@
function [ pts_new ] = iterate_piece_wise( pts_orig, new_num_points )
%ITERATE_PIECE_WISE Summary of this function goes here
% Detailed explanation goes here
% Reinterpolate the new points, but make sure they are still on the
% same lines
num_orig = size(pts_orig,1);
% Divide the number of original segments by number of new segments
step_size = (num_orig - 1) / (new_num_points-1);
pts_new = zeros(new_num_points,2);
% Clamp the beginning and end, as they will be the same
pts_new(1,:) = pts_orig(1,:);
pts_new(end,:) = pts_orig(end,:);
for i=1:new_num_points-2
low_point = floor(1 + i * step_size);
high_point = ceil(1 + i * step_size);
coeff_1 = floor(1 + i * step_size) - i * step_size;
coeff_2 = 1 - coeff_1;
new_pt = coeff_1 * pts_orig(low_point,:) + coeff_2 * pts_orig(high_point,:);
pts_new(i+1,:) = new_pt;
end
end

78
pkg/OpenFace/model_training/learn_error_mapping/learn_error_pred_general.m Normal file
View File

@@ -0,0 +1,78 @@
clear;
load('./results/results_ceclm_general.mat');
% Generate a lot of possibilities and find best
weights = [1,1,1,0,0,0];
score_base = ranking_score(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
scores = zeros(1,1000);
score = score_base;
weights_all = zeros(1000,6);
for i=1:1000
scores(i) = score;
weights_all(i,:) = weights;
% weights = weights + (0.999^i)*0.3 * ranking_score_grad(weights, l_hoods_view, experiment.all_views_used, errors_view);
weights = weights + (0.999^i)*0.3 * ranking_score_grad(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
score = ranking_score(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
end
[~,id] = max(scores);
weights_lhoods = weights_all(id,:);
%% Learn the cuttof
view_used = experiment.all_views_used;
weights_scale = ones(size( experiment.lhoods));
% As views 2-7, 3-6, 4-5 are mirrors of each other, trust their
% assessment the same amount
weights_scale(view_used == 2) = weights_lhoods(1);
weights_scale(view_used == 3) = weights_lhoods(2);
weights_scale(view_used == 4) = weights_lhoods(3);
weights_scale(view_used == 5) = weights_lhoods(3);
weights_scale(view_used == 6) = weights_lhoods(2);
weights_scale(view_used == 7) = weights_lhoods(1);
weights_add = zeros(size(experiment.lhoods));
weights_add(view_used == 2) = weights_lhoods(4);
weights_add(view_used == 3) = weights_lhoods(5);
weights_add(view_used == 4) = weights_lhoods(6);
weights_add(view_used == 5) = weights_lhoods(6);
weights_add(view_used == 6) = weights_lhoods(5);
weights_add(view_used == 7) = weights_lhoods(4);
lhoods = experiment.lhoods .* weights_scale + weights_add;
% As a very reliable detection, make sure that 98% of cases have high
% accuracy (less than 0.1)
% That is find the smallest number which leads to good accuracy
cutoffs = [];
for i=1:4
ids = view_used==i;
if(i > 1)
if(i==2)
mirr_id = 7;
elseif(i==3)
mirr_id = 6;
elseif(i==4)
mirr_id = 5;
end
ids = ids | view_used==mirr_id;
end
lhood_view = lhoods(ids);
error = experiment.errors_view(ids);
for c=-2:0.01:2
if(mean(error(lhood_view >c)<0.1) >= 0.98)
break;
end
end
cutoffs = cat(1, cutoffs, c);
end
cutoffs = [cutoffs(1), cutoffs(2), cutoffs(3), cutoffs(4), cutoffs(4), cutoffs(3), cutoffs(2)];
weights_scale = weights_lhoods(1:end/2);
weights_scale = [1, weights_scale(1), weights_scale(2), weights_scale(3), weights_scale(3), weights_scale(2), weights_scale(1)];
weights_add = weights_lhoods(end/2+1:end);
weights_add = [0, weights_add(1), weights_add(2), weights_add(3), weights_add(3), weights_add(2), weights_add(1)];
early_term_params.weights_scale = weights_scale;
early_term_params.weights_add = weights_add;
early_term_params.cutoffs = cutoffs;
save('cen_general_mapping', 'early_term_params');

78
pkg/OpenFace/model_training/learn_error_mapping/learn_error_pred_menpo.m Normal file
View File

@@ -0,0 +1,78 @@
clear;
load('./results/results_ceclm_menpo.mat');
% Generate a lot of possibilities and find best
weights = [1,1,1,0,0,0];
score_base = ranking_score(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
scores = zeros(1,1000);
score = score_base;
weights_all = zeros(1000,6);
for i=1:1000
scores(i) = score;
weights_all(i,:) = weights;
% weights = weights + (0.999^i)*0.3 * ranking_score_grad(weights, l_hoods_view, experiment.all_views_used, errors_view);
weights = weights + (0.999^i)*0.3 * ranking_score_grad(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
score = ranking_score(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
end
[~,id] = max(scores);
weights_lhoods = weights_all(id,:);
%% Learn the cuttof
view_used = experiment.all_views_used;
weights_scale = ones(size( experiment.lhoods));
% As views 2-7, 3-6, 4-5 are mirrors of each other, trust their
% assessment the same amount
weights_scale(view_used == 2) = weights_lhoods(1);
weights_scale(view_used == 3) = weights_lhoods(2);
weights_scale(view_used == 4) = weights_lhoods(3);
weights_scale(view_used == 5) = weights_lhoods(3);
weights_scale(view_used == 6) = weights_lhoods(2);
weights_scale(view_used == 7) = weights_lhoods(1);
weights_add = zeros(size(experiment.lhoods));
weights_add(view_used == 2) = weights_lhoods(4);
weights_add(view_used == 3) = weights_lhoods(5);
weights_add(view_used == 4) = weights_lhoods(6);
weights_add(view_used == 5) = weights_lhoods(6);
weights_add(view_used == 6) = weights_lhoods(5);
weights_add(view_used == 7) = weights_lhoods(4);
lhoods = experiment.lhoods .* weights_scale + weights_add;
% As a very reliable detection, make sure that 98% of cases have high
% accuracy (less than 0.1)
% That is find the smallest number which leads to good accuracy
cutoffs = [];
for i=1:4
ids = view_used==i;
if(i > 1)
if(i==2)
mirr_id = 7;
elseif(i==3)
mirr_id = 6;
elseif(i==4)
mirr_id = 5;
end
ids = ids | view_used==mirr_id;
end
lhood_view = lhoods(ids);
error = experiment.errors_view(ids);
for c=-2:0.01:2
if(mean(error(lhood_view >c)<0.1) >= 0.99)
break;
end
end
cutoffs = cat(1, cutoffs, c);
end
cutoffs = [cutoffs(1), cutoffs(2), cutoffs(3), cutoffs(4), cutoffs(4), cutoffs(3), cutoffs(2)];
weights_scale = weights_lhoods(1:end/2);
weights_scale = [1, weights_scale(1), weights_scale(2), weights_scale(3), weights_scale(3), weights_scale(2), weights_scale(1)];
weights_add = weights_lhoods(end/2+1:end);
weights_add = [0, weights_add(1), weights_add(2), weights_add(3), weights_add(3), weights_add(2), weights_add(1)];
early_term_params.weights_scale = weights_scale;
early_term_params.weights_add = weights_add;
early_term_params.cutoffs = cutoffs;
save('cen_menpo_mapping', 'early_term_params');

78
pkg/OpenFace/model_training/learn_error_mapping/learn_error_pred_of.m Normal file
View File

@@ -0,0 +1,78 @@
clear;
load('./results/results_ceclm_menpo.mat');
% Generate a lot of possibilities and find best
weights = [1,1,1,0,0,0];
score_base = ranking_score(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
scores = zeros(1,1000);
score = score_base;
weights_all = zeros(1000,6);
for i=1:1000
scores(i) = score;
weights_all(i,:) = weights;
% weights = weights + (0.999^i)*0.3 * ranking_score_grad(weights, l_hoods_view, experiment.all_views_used, errors_view);
weights = weights + (0.999^i)*0.3 * ranking_score_grad(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
score = ranking_score(weights, experiment.lhoods, experiment.all_views_used, experiment.errors_view);
end
[~,id] = max(scores);
weights_lhoods = weights_all(id,:);
%% Learn the cuttof
view_used = experiment.all_views_used;
weights_scale = ones(size( experiment.lhoods));
% As views 2-7, 3-6, 4-5 are mirrors of each other, trust their
% assessment the same amount
weights_scale(view_used == 2) = weights_lhoods(1);
weights_scale(view_used == 3) = weights_lhoods(2);
weights_scale(view_used == 4) = weights_lhoods(3);
weights_scale(view_used == 5) = weights_lhoods(3);
weights_scale(view_used == 6) = weights_lhoods(2);
weights_scale(view_used == 7) = weights_lhoods(1);
weights_add = zeros(size(experiment.lhoods));
weights_add(view_used == 2) = weights_lhoods(4);
weights_add(view_used == 3) = weights_lhoods(5);
weights_add(view_used == 4) = weights_lhoods(6);
weights_add(view_used == 5) = weights_lhoods(6);
weights_add(view_used == 6) = weights_lhoods(5);
weights_add(view_used == 7) = weights_lhoods(4);
lhoods = experiment.lhoods .* weights_scale + weights_add;
% As a very reliable detection, make sure that 98% of cases have high
% accuracy (less than 0.1)
% That is find the smallest number which leads to good accuracy
cutoffs = [];
for i=1:4
ids = view_used==i;
if(i > 1)
if(i==2)
mirr_id = 7;
elseif(i==3)
mirr_id = 6;
elseif(i==4)
mirr_id = 5;
end
ids = ids | view_used==mirr_id;
end
lhood_view = lhoods(ids);
error = experiment.errors_view(ids);
for c=-2:0.01:2
if(mean(error(lhood_view >c)<0.1) >= 0.99)
break;
end
end
cutoffs = cat(1, cutoffs, c);
end
cutoffs = [cutoffs(1), cutoffs(2), cutoffs(3), cutoffs(4), cutoffs(4), cutoffs(3), cutoffs(2)];
weights_scale = weights_lhoods(1:end/2);
weights_scale = [1, weights_scale(1), weights_scale(2), weights_scale(3), weights_scale(3), weights_scale(2), weights_scale(1)];
weights_add = weights_lhoods(end/2+1:end);
weights_add = [0, weights_add(1), weights_add(2), weights_add(3), weights_add(3), weights_add(2), weights_add(1)];
early_term_params.weights_scale = weights_scale;
early_term_params.weights_add = weights_add;
early_term_params.cutoffs = cutoffs;
save('cen_menpo_mapping', 'early_term_params');

18
pkg/OpenFace/model_training/learn_error_mapping/output_corrections.m Normal file
View File

@@ -0,0 +1,18 @@
clear;
load('cen_of_mapping.mat');
fe = fopen('early_term_cen_of.txt', 'w');
for i=1:numel(early_term_params.weights_scale)
fprintf(fe, '%.3f ', early_term_params.weights_scale(i));
end
for i=1:numel(early_term_params.weights_add)
fprintf(fe, '%.3f ', early_term_params.weights_add(i));
end
for i=1:numel(early_term_params.cutoffs)
fprintf(fe, '%.3f ', early_term_params.cutoffs(i));
end
fclose(fe);

41
pkg/OpenFace/model_training/learn_error_mapping/ranking_score.m Normal file
View File

@@ -0,0 +1,41 @@
function [ score ] = ranking_score( weights, lhoods, view_used, errors )
%RANKING_SCORE Summary of this function goes here
% Detailed explanation goes here
weights_scale = ones(size(lhoods));
% As views 2-7, 3-6, 4-5 are mirrors of each other, trust their
% assessment the same amount
% weights_scale(view_used == 1) = weights(1);
weights_scale(view_used == 2) = weights(1);
weights_scale(view_used == 3) = weights(2);
weights_scale(view_used == 4) = weights(3);
weights_scale(view_used == 5) = weights(3);
weights_scale(view_used == 6) = weights(2);
weights_scale(view_used == 7) = weights(1);
weights_add = zeros(size(lhoods));
% weights_add(view_used == 1) = weights(5);
weights_add(view_used == 2) = weights(4);
weights_add(view_used == 3) = weights(5);
weights_add(view_used == 4) = weights(6);
weights_add(view_used == 5) = weights(6);
weights_add(view_used == 6) = weights(5);
weights_add(view_used == 7) = weights(4);
lhoods = lhoods .* weights_scale + weights_add;
[~, inds] = sort(lhoods');
inds_best = inds(end,:);
inds_second_best = inds(end-1,:);
% [~,max_lhoods] = max(lhoods');
[~, min_errors] = min(errors');
% Count both the best and second best errors
score = mean(inds_best == min_errors);
% score = mean(inds_best == min_errors | inds_second_best == min_errors);
% inds_to_err = sub2ind(size(errors),[1:length(max_lhoods)]',max_lhoods');
% err = mean(errors(inds_to_err));
% err2 = median(errors(inds_to_err));
% score = err;
end

23
pkg/OpenFace/model_training/learn_error_mapping/ranking_score_grad.m Normal file
View File

@@ -0,0 +1,23 @@
function [ partial_grad ] = ranking_score_grad( weights, lhoods, views_used, errors )
%RANKING_SCORE Summary of this function goes here
% Detailed explanation goes here
% central_score = ranking_score(weights, lhoods, errors);
delta = 0.075;
partial_grad = zeros(size(weights));
for i=1:numel(weights)
w_m = weights;
w_p = weights;
w_m(i) = w_m(i) - delta/2;
w_p(i) = w_p(i) + delta/2;
partial_grad(i) = ranking_score(w_p, lhoods, views_used, errors)-ranking_score(w_m, lhoods, views_used, errors);
end
partial_grad = partial_grad ./ delta;
end

14
pkg/OpenFace/model_training/learn_error_mapping/readme.txt Normal file
View File

@@ -0,0 +1,14 @@
This function allows to learn a better relationship between landmark
detection likelihoods and the resulting landmark error.
First run:
Script_CECLM_general_cross_data_multi_hyp.m
Script_CECLM_menpo_multi_hyp.m
Script_CECLM_of.m
Then:
learn_error_pred_general.m
learn_error_pred_menpo.m
learn_error_pred_of.m
To use in C++ also generate the early_term_cen_of.txt file using output_corrections.m, this is already generated so no need to re-run it.

BIN
pkg/OpenFace/model_training/learn_error_mapping/results/results_ceclm_general.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/results/results_ceclm_menpo.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/results/results_ceclm_of.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/learn_error_mapping/sparse_selection/menpo_frontal_sparse.mat Normal file
View File

Binary file not shown.

46
pkg/OpenFace/model_training/learn_error_mapping/standardise_landmarks.m Normal file
View File

@@ -0,0 +1,46 @@
function [landmark_labels] = standardise_landmarks(landmarks)
left_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,32; 23,39; 24,38; 25,37; 26,42; 27,41;
28,52; 29,51; 30,50; 31,49; 32,60; 33,59; 34,58;
35,63; 36,62; 37,61; 38,68; 39,67];
right_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,36; 23,44; 24,45; 25,46; 26,47; 27,48;
28,52; 29,53; 30,54; 31,55; 32,56; 33,57; 34,58;
35,63; 36,64; 37,65; 38,66; 39,67];
landmark_labels = zeros(68,2);
if(size(landmarks,1) == 39)
% Determine if the points are clock-wise or counter clock-wise
% Clock-wise points are facing left, counter-clock-wise right
sum = 0;
for k=1:11
step = (landmarks(k+1,1) - landmarks(k,1)) * (landmarks(k+1,2) + landmarks(k,2));
sum = sum + step;
end
if(sum > 0)
% First need to resample the face outline as there are 9
% points in the near-frontal and 10 points in profile for
% the outline of the face
outline = iterate_piece_wise(landmarks(1:10,:), 9);
brow = iterate_piece_wise(landmarks(13:16,:), 5);
landmark_labels(1:9,:) = outline;
landmark_labels(18:22,:) = brow;
landmark_labels(left_to_frontal_map(:,2),:) = landmarks(left_to_frontal_map(:,1),:);
else
outline = iterate_piece_wise(landmarks(10:-1:1,:), 9);
brow = iterate_piece_wise(landmarks(16:-1:13,:), 5);
landmark_labels(9:17,:) = outline;
landmark_labels(23:27,:) = brow;
landmark_labels(right_to_frontal_map(:,2),:) = landmarks(right_to_frontal_map(:,1),:);
end
else
landmark_labels = landmarks;
end
end