bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
pkg/OpenFace/matlab_version/experiments_menpo/Collect_menpo_imgs.m Normal file
View File

@@ -0,0 +1,49 @@
function [images, detections, labels] = Collect_menpo_imgs(root_dir)
load('face_detections/menpo_train_dets.mat');
bb = bboxes;
load('face_detections/menpo_valid_dets.mat');
bboxes = cat(2, bb, bboxes);
% Have three bounding box locations (frontal tuned, profile tuned)
detections = zeros(numel(bboxes), 4);
labels = cell(numel(bboxes),1);
for i=1:numel(bboxes)
images(i).img = [root_dir, bboxes(i).name];
% If face detected
if(~isempty(bboxes(i).bbox))
bbox = bboxes(i).bbox(1:4);
% Correct the MTCNN bounding box
width = bbox(3) - bbox(1);
height = bbox(4) - bbox(2);
tx = bbox(1);
ty = bbox(2);
% Frontal faces
new_width = width * 1.0323;
new_height = height * 0.7751;
new_tx = width * -0.0075 + tx;
new_ty = height * 0.2459 + ty;
detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
else % If face not detected, use the mean location of the face in training data
img_size = size(imread([root_dir, bboxes(i).name]));
img_width = img_size(2);
img_height = img_size(1);
width = img_width * 0.4421;
height = img_height * 0.445;
tx = img_width * 0.5048 - 0.5 * width;
ty = img_height * 0.5166 - 0.5 * height;
detections(i,:) = [tx, ty, tx + width, ty + height];
end
labels{i} = bboxes(i).gt_landmarks;
end
end

45
pkg/OpenFace/matlab_version/experiments_menpo/Collect_valid_imgs.m Normal file
View File

@@ -0,0 +1,45 @@
function [images, detections, labels] = Collect_valid_imgs(root_dir)
load('face_detections/menpo_valid_dets.mat');
% Have three bounding box locations (frontal tuned, profile tuned)
detections = zeros(numel(bboxes), 4);
labels = cell(numel(bboxes),1);
for i=1:numel(bboxes)
images(i).img = [root_dir, bboxes(i).name];
% If face detected
if(~isempty(bboxes(i).bbox))
bbox = bboxes(i).bbox(1:4);
% Correct the MTCNN bounding box
width = bbox(3) - bbox(1);
height = bbox(4) - bbox(2);
tx = bbox(1);
ty = bbox(2);
% Frontal faces
new_width = width * 1.0323;
new_height = height * 0.7751;
new_tx = width * -0.0075 + tx;
new_ty = height * 0.2459 + ty;
detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
else % If face not detected, use the mean location of the face in training data
img_size = size(imread([root_dir, bboxes(i).name]));
img_width = img_size(2);
img_height = img_size(1);
width = img_width * 0.4421;
height = img_height * 0.445;
tx = img_width * 0.5048 - 0.5 * width;
ty = img_height * 0.5166 - 0.5 * height;
detections(i,:) = [tx, ty, tx + width, ty + height];
end
labels{i} = bboxes(i).gt_landmarks;
end
end

35
pkg/OpenFace/matlab_version/experiments_menpo/Construct_error_table.m Normal file
View File

@@ -0,0 +1,35 @@
% Run this in order to construct the results table for the CE-CLM paper
Extract_table_results_68_cross_data;
file_out = fopen('results/Menpo_68.txt', 'w');
fprintf(file_out, 'Errors with outline (68 points)\n');
fprintf(file_out, '------------------------------\n');
fprintf(file_out, 'Method\tfrontal\tprofile\n');
fprintf(file_out, 'CLNF\t%.2f\t%.2f\n', median(clnf_error_frontal)*100, median(clnf_error_profile)*100);
fprintf(file_out, 'CFAN\t%.2f\t%.2f\n', median(cfan_error_frontal)*100, median(cfan_error_profile)*100);
fprintf(file_out, 'CFSS\t%.2f\t%.2f\n', median(cfss_error_frontal)*100, median(cfss_error_profile)*100);
fprintf(file_out, 'TCDCN\t%.2f\t%.2f\n', median(tcdcn_error_frontal)*100, median(tcdcn_error_profile)*100);
fprintf(file_out, '3DDFA\t%.2f\t%.2f\n', median(error_3ddfa_frontal)*100, median(error_3ddfa_profile)*100);
fprintf(file_out, '------------------------------\n');
fprintf(file_out, 'CE-CLM\t%.2f\t%.2f\n', median(ceclm_error_frontal)*100, median(ceclm_error_profile)*100);
fclose(file_out);
Extract_table_results_49_cross_data;
file_out = fopen('results/Menpo_49.txt', 'w');
fprintf(file_out, 'Errors without outline (49 points)\n');
fprintf(file_out, '------------------------------\n');
fprintf(file_out, 'Method\tfrontal\tprofile\n');
fprintf(file_out, 'CLNF\t%.2f\t%.2f\n', median(clnf_error_frontal)*100, median(clnf_error_profile)*100);
fprintf(file_out, 'SDM \t%.2f\t%.2f\n', median(sdm_error_frontal)*100, median(sdm_error_profile)*100);
fprintf(file_out, 'CFAN\t%.2f\t%.2f\n', median(cfan_error_frontal)*100, median(cfan_error_profile)*100);
fprintf(file_out, 'DRMF\t%.2f\t%.2f\n', median(drmf_error_frontal)*100, median(drmf_error_profile)*100);
fprintf(file_out, 'CFSS\t%.2f\t%.2f\n', median(cfss_error_frontal)*100, median(cfss_error_profile)*100);
fprintf(file_out, 'PO-CR\t%.2f\t%.2f\n', median(pocr_error_frontal)*100, median(pocr_error_profile)*100);
fprintf(file_out, 'TCDCN\t%.2f\t%.2f\n', median(tcdcn_error_frontal)*100, median(tcdcn_error_profile)*100);
fprintf(file_out, '3DDFA\t%.2f\t%.2f\n', median(error_3ddfa_frontal)*100, median(error_3ddfa_profile)*100);
fprintf(file_out, '------------------------------\n');
fprintf(file_out, 'CE-CLM\t%.2f\t%.2f\n', median(ceclm_error_frontal)*100, median(ceclm_error_profile)*100);
fclose(file_out);

119
pkg/OpenFace/matlab_version/experiments_menpo/Display_menpo_results_49_cross_data.m Normal file
View File

@@ -0,0 +1,119 @@
%%
clear;
Extract_table_results_49_cross_data;
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
[error_x, error_y] = cummErrorCurve(ceclm_error_frontal);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(clnf_error_frontal);
plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfan_error_frontal);
plot(error_x, error_y, 'DisplayName', 'CFAN', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(error_3ddfa_frontal);
plot(error_x, error_y, 'DisplayName', '3DDFA', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfss_error_frontal);
plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(tcdcn_error_frontal);
plot(error_x, error_y, 'DisplayName', 'TCDCN', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(pocr_error_frontal);
plot(error_x, error_y, 'DisplayName', 'PO-CR', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(sdm_error_frontal);
plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(drmf_error_frontal);
plot(error_x, error_y, 'DisplayName', 'DRMF', 'LineWidth',line_width);
set(gca,'xtick',[0:0.01:0.07])
xlim([0.01,0.07]);
xlabel('Size normalised MAE','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
% title('Fitting on Menpo frontal images','FontSize',60,'FontName','Helvetica');
leg = legend('show', 'Location', 'SouthEast');
set(leg,'FontSize',30)
[error_x, error_y] = cummErrorCurve(ceclm_error_frontal);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
print -dpdf results/menpo-frontal_49.pdf
print -dpng results/menpo-frontal_49.png
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
[error_x, error_y] = cummErrorCurve(ceclm_error_profile);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(clnf_error_profile);
plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfan_error_profile);
plot(error_x, error_y, 'DisplayName', 'CFAN', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(error_3ddfa_profile);
plot(error_x, error_y, 'DisplayName', '3DDFA', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfss_error_profile);
plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(tcdcn_error_profile);
plot(error_x, error_y, 'DisplayName', 'TCDCN', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(pocr_error_profile);
plot(error_x, error_y, 'DisplayName', 'PO-CR', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(sdm_error_profile);
plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(drmf_error_profile);
plot(error_x, error_y, 'DisplayName', 'DRMF', 'LineWidth',line_width);
set(gca,'xtick',[0.01:0.02:0.11])
xlim([0.02,0.09]);
xlabel('Size normalised MAE','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
% title('Fitting on Menpo frontal images','FontSize',60,'FontName','Helvetica');
leg = legend('show', 'Location', 'SouthEast');
set(leg,'FontSize',30)
[error_x, error_y] = cummErrorCurve(ceclm_error_profile);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
print -dpdf results/menpo-profile_49.pdf
print -dpng results/menpo-profile_49.png

105
pkg/OpenFace/matlab_version/experiments_menpo/Display_menpo_results_68_cross_data.m Normal file
View File

@@ -0,0 +1,105 @@
%%
clear;
Extract_table_results_68_cross_data;
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
[error_x, error_y] = cummErrorCurve(ceclm_error_frontal);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(clnf_error_frontal);
plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfan_error_frontal);
plot(error_x, error_y, 'DisplayName', 'CFAN', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(error_3ddfa_frontal);
plot(error_x, error_y, 'DisplayName', '3DDFA', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfss_error_frontal);
plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(tcdcn_error_frontal);
plot(error_x, error_y, 'DisplayName', 'TCDCN', 'LineWidth',line_width);
set(gca,'xtick',[0:0.01:0.07])
xlim([0.01,0.07]);
xlabel('Size normalised MAE','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
% title('Fitting on Menpo frontal images','FontSize',60,'FontName','Helvetica');
leg = legend('show', 'Location', 'SouthEast');
set(leg,'FontSize',50)
[error_x, error_y] = cummErrorCurve(ceclm_error_frontal);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
print -dpdf results/menpo-frontal_68.pdf
print -dpng results/menpo-frontal_68.png
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
[error_x, error_y] = cummErrorCurve(ceclm_error_profile);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(clnf_error_profile);
plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfan_error_profile);
plot(error_x, error_y, 'DisplayName', 'CFAN', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(error_3ddfa_profile);
plot(error_x, error_y, 'DisplayName', '3DDFA', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(cfss_error_profile);
plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
[error_x, error_y] = cummErrorCurve(tcdcn_error_profile);
plot(error_x, error_y, 'DisplayName', 'TCDCN', 'LineWidth',line_width);
set(gca,'xtick',[0.01:0.02:0.11])
xlim([0.03,0.11]);
xlabel('Size normalised MAE','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
% title('Fitting on Menpo frontal images','FontSize',60,'FontName','Helvetica');
leg = legend('show', 'Location', 'SouthEast');
set(leg,'FontSize',50)
[error_x, error_y] = cummErrorCurve(ceclm_error_profile);
plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
print -dpdf results/menpo-profile_68.pdf
print -dpng results/menpo-profile_68.png

19
pkg/OpenFace/matlab_version/experiments_menpo/Extract_menpo_labels_corrected.m Normal file
View File

@@ -0,0 +1,19 @@
root_dir = 'D:\Datasets\menpo/';
load('menpo_train_dets.mat');
bb = bboxes;
load('menpo_valid_dets.mat');
bboxes = cat(2, bb, bboxes);
labels = cell(numel(bboxes),1);
for i=1:numel(bboxes)
pts_name = [root_dir, bboxes(i).name(1:end-3), 'pts'];
lmarks = importdata(pts_name, ' ', 3);
labels{i} = lmarks.data;
img = imread([root_dir, bboxes(i).name]);
imshow(img); hold on;plot(labels{i}(:,1), labels{i}(:,2), '.r');hold off;
drawnow expose;
end
save('results/menpo_labels', 'labels');

70
pkg/OpenFace/matlab_version/experiments_menpo/Extract_table_results_49_cross_data.m Normal file
View File

@@ -0,0 +1,70 @@
clear
load('results/results_clnf_cross-data_general.mat');
load('results/menpo_labels.mat');
[clnf_error, frontal_ids] = compute_error_menpo_small( labels, experiment.shapes);
clnf_error_frontal = clnf_error(frontal_ids);
clnf_error_profile = clnf_error(~frontal_ids);
load('results/menpo_train_chehra.mat');
[drmf_error, frontal_ids] = compute_error_menpo_small( labels, shapes);
drmf_error_frontal = drmf_error(frontal_ids);
drmf_error_profile = drmf_error(~frontal_ids);
load('results/menpo_train_sdm.mat');
for i = 1:numel(experiment.shapes)
shapes{i} = shapes{i}+0.5;
end
[sdm_error, frontal_ids] = compute_error_menpo_small( labels, shapes);
sdm_error_frontal = sdm_error(frontal_ids);
sdm_error_profile = sdm_error(~frontal_ids);
load('results/Menpo_train_pocr.mat');
[pocr_error, frontal_ids] = compute_error_menpo_small( labels, experiments.shapes);
pocr_error_frontal = pocr_error(frontal_ids);
pocr_error_profile = pocr_error(~frontal_ids);
load('results/results_ceclm_cross-data.mat');
[ceclm_error, frontal_ids] = compute_error_menpo_small( labels, experiment.shapes);
ceclm_error_frontal = ceclm_error(frontal_ids);
ceclm_error_profile = ceclm_error(~frontal_ids);
load('results/tcdcn_menpo.mat');
for i = 1:numel(shapes)
shapes{i} = shapes{i};
end
[tcdcn_error, frontal_ids] = compute_error_menpo_small(labels, shapes);
tcdcn_error_frontal = tcdcn_error(frontal_ids);
tcdcn_error_profile = tcdcn_error(~frontal_ids);
load('results/CFAN_menpo_train.mat');
for i = 1:numel(shapes)
shapes{i} = shapes{i}-0.5;
end
[cfan_error, frontal_ids] = compute_error_menpo_small(labels, shapes);
cfan_error_frontal = cfan_error(frontal_ids);
cfan_error_profile = cfan_error(~frontal_ids);
load('results/menpo_train_3DDFA.mat');
for i = 1:numel(shapes)
shapes{i} = shapes{i}-0.5;
end
[error_3ddfa, frontal_ids] = compute_error_menpo_small(labels, shapes);
error_3ddfa_frontal = error_3ddfa(frontal_ids);
error_3ddfa_profile = error_3ddfa(~frontal_ids);
load('results/Menpo-CFSS_train.mat');
shapes = cell(size(estimatedPoseFull,1),1);
for i = 1:numel(shapes)
shape = cat(2, estimatedPoseFull(i,1:68)', estimatedPoseFull(i,69:end)');
shapes{i} = shape-0.5;
end
[cfss_error, frontal_ids] = compute_error_menpo_small(labels, shapes);
cfss_error_frontal = cfss_error(frontal_ids);
cfss_error_profile = cfss_error(~frontal_ids);

55
pkg/OpenFace/matlab_version/experiments_menpo/Extract_table_results_68_cross_data.m Normal file
View File

@@ -0,0 +1,55 @@
clear
%%
load('results/results_clnf_cross-data_general.mat');
load('results/menpo_labels.mat');
[clnf_error, frontal_ids] = compute_error_menpo_1( labels, experiment.shapes);
clnf_error_frontal = clnf_error(frontal_ids);
clnf_error_profile = clnf_error(~frontal_ids);
load('results/results_ceclm_cross-data.mat');
[ceclm_error, frontal_ids] = compute_error_menpo_1( labels, experiment.shapes);
ceclm_error_frontal = ceclm_error(frontal_ids);
ceclm_error_profile = ceclm_error(~frontal_ids);
load('results/CFAN_menpo_train.mat');
for i = 1:numel(shapes)
shapes{i} = shapes{i}-0.5;
end
[cfan_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
cfan_error_frontal = cfan_error(frontal_ids);
cfan_error_profile = cfan_error(~frontal_ids);
load('results/tcdcn_menpo.mat');
for i = 1:numel(shapes)
shapes{i} = shapes{i};
end
[tcdcn_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
tcdcn_error_frontal = tcdcn_error(frontal_ids);
tcdcn_error_profile = tcdcn_error(~frontal_ids);
load('results/menpo_train_3DDFA.mat');
for i = 1:numel(shapes)
shapes{i} = shapes{i}-0.5;
end
[error_3ddfa, frontal_ids] = compute_error_menpo_1(labels, shapes);
error_3ddfa_frontal = error_3ddfa(frontal_ids);
error_3ddfa_profile = error_3ddfa(~frontal_ids);
load('results/Menpo-CFSS_train.mat');
shapes = cell(size(estimatedPoseFull,1),1);
for i = 1:numel(shapes)
shape = cat(2, estimatedPoseFull(i,1:68)', estimatedPoseFull(i,69:end)');
shapes{i} = shape-0.5;
end
[cfss_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
cfss_error_frontal = cfss_error(frontal_ids);
cfss_error_profile = cfss_error(~frontal_ids);

97
pkg/OpenFace/matlab_version/experiments_menpo/Script_CECLM_cross_data.m Normal file
View File

@@ -0,0 +1,97 @@
function Script_CECLM_cross_data()
addpath(genpath('../'));
[images, detections, labels] = Collect_menpo_imgs('G:\Datasets\menpo/');
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
% As early termination weights were trained on part of menpo turn them off,
% to perform a clean cross-data experiment
early_term_params.weights_scale(:) = 1;
early_term_params.weights_add(:) = 0;
early_term_params.cutoffs(:) = -0.2;
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
%% Setup recording
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), 1);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Change if you want to visualize the outputs
verbose = false;
output_img = false;
if(output_img)
output_root = './ceclm_gen_out/';
if(~exist(output_root, 'dir'))
mkdir(output_root);
end
end
if(verbose)
f = figure;
end
%% Fitting the model to the provided images
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% The actual work get's done here
[shape,~,~,lhood,lmark_lhood,view_used] =...
Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(output_img)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnImg(image_orig, shape, sprintf('%s/%s%d.jpg', output_root, 'fit', i), bbox, v_points);
end
if(verbose)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
end
toc
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
%%
output_results = 'results/results_ceclm_cross-data.mat';
save(output_results, 'experiment');
end

95
pkg/OpenFace/matlab_version/experiments_menpo/Script_CECLM_menpo_test_frontal.m Normal file
View File

@@ -0,0 +1,95 @@
function Script_CECLM_menpo_test_frontal()
addpath(genpath('../'));
addpath(genpath('./menpo_challenge_helpers'));
[images, detections] = Collect_menpo_test_frontal('G:\Datasets\menpo\testset\semifrontal/');
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
%% Setup recording
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Change if you want to visualize the outputs
verbose = false;
output_img = false;
if(output_img)
output_root = './ceclm_menpo_test/';
if(~exist(output_root, 'dir'))
mkdir(output_root);
end
end
if(verbose)
f = figure;
end
out_pts = './out_semifrontal/';
if(~exist(out_pts, 'dir'))
mkdir(out_pts);
end
%%
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
[shape,~,~,lhood,lmark_lhood,view_used] =...
Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params);
shape = shape + 0.5;
[~, name_org, ~] = fileparts(images(i).img);
name = [out_pts, name_org, '.pts'];
shape = write_menpo_frontal(shape, name);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(output_img)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnImg(image_orig, shape, sprintf('%s/%s%d.jpg', output_root, 'fit', i), bbox, v_points);
end
if(verbose)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
end
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
%%
output_results = 'results/results_test_menpo_semifrontal.mat';
save(output_results, 'experiment');
end

99
pkg/OpenFace/matlab_version/experiments_menpo/Script_CECLM_menpo_test_profile.m Normal file
View File

@@ -0,0 +1,99 @@
function Script_CECLM_menpo_test_profile()
addpath(genpath('../'));
addpath(genpath('./menpo_challenge_helpers'));
[images, detections] = Collect_menpo_test_profile('G:\Datasets\menpo\testset\profile/');
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
%% Setup recording
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Change if you want to visualize the outputs
verbose = false;
output_img = false;
if(output_img)
output_root = './ceclm_menpo_test/';
if(~exist(output_root, 'dir'))
mkdir(output_root);
end
end
if(verbose)
f = figure;
end
out_pts = './out_profile/';
if(~exist(out_pts, 'dir'))
mkdir(out_pts);
end
load('menpo_challenge_helpers/conversion.mat');
%%
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
[shape,g_params,~,lhood,lmark_lhood,view_used] =...
Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params);
shape = shape + 0.5;
[~, name_org, ~] = fileparts(images(i).img);
name = [out_pts, name_org, '.pts'];
if(g_params(3) > 0)
shape = write_menpo_profile(shape, name, a_left, vis_pts_left);
else
shape = write_menpo_profile(shape, name, a_right, vis_pts_right);
end
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(output_img)
DrawFaceOnImg(image_orig, shape, sprintf('%s/%s%d.jpg', output_root, 'fit', i), bbox);
end
if(verbose)
DrawFaceOnFig(image_orig, shape, bbox);
end
end
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
%%
output_results = 'results/results_test_menpo_profile.mat';
save(output_results, 'experiment');
end

86
pkg/OpenFace/matlab_version/experiments_menpo/Script_CECLM_menpo_valid.m Normal file
View File

@@ -0,0 +1,86 @@
function Script_CECLM_menpo_valid()
addpath(genpath('../'));
[images, detections, labels] = Collect_valid_imgs('G:\Datasets\menpo/');
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
%% Setup recording
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), 1);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Change if you want to visualize the outputs
verbose = false;
output_img = false;
if(output_img)
output_root = './ceclm_menpo_out/';
if(~exist(output_root, 'dir'))
mkdir(output_root);
end
end
if(verbose)
f = figure;
end
%% Fitting the model to the provided images
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
[shape,~,~,lhood,lmark_lhood,view_used] =...
Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(output_img)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnImg(image_orig, shape, sprintf('%s/%s%d.jpg', output_root, 'fit', i), bbox, v_points);
end
if(verbose)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
end
toc
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
%%
output_results = 'results/results_ceclm_menpo_valid.mat';
save(output_results, 'experiment');
end

80
pkg/OpenFace/matlab_version/experiments_menpo/Script_CLNF_cross_data_general.m Normal file
View File

@@ -0,0 +1,80 @@
function Script_CLNF_cross_data_general()
addpath(genpath('../'));
% Replace this with the location of the Menpo dataset
[images, detections, labels] = Collect_menpo_imgs('G:\Datasets\menpo/');
%% loading the patch experts
[ patches, pdm, clmParams ] = Load_CLNF_general();
views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
views = views * pi/180;
[ patches_51, pdm_51, clmParams_51, inds_full, inds_inner ] = Load_CLNF_inner();
shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
lhoods = zeros(numel(images),1);
% for recording purposes
experiment.params = clmParams;
num_points = numel(pdm.M)/3;
shapes_all = cell(numel(images), 1);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
verbose = false; % set to true to visualise the fitting
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = detections(i,:);
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views);
% Perform inner landmark fitting now
[shape, shape_inner] = Fitting_from_bb_hierarch(image, pdm, pdm_51, patches_51, clmParams_51, shape, inds_full, inds_inner);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
labels_all{i} = labels{i};
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose)
v_points = logical(patches(1).visibilities(view_used,:))';
DrawFaceOnFig(image_orig, shape, bbox, v_points);
end
end
toc
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
%%
output_results = 'results/results_clnf_cross-data_general.mat';
save(output_results, 'experiment');
end

85
pkg/OpenFace/matlab_version/experiments_menpo/compute_error_menpo_1.m Normal file
View File

@@ -0,0 +1,85 @@
function [ error_per_image, frontal_ids ] = compute_error_menpo_1( ground_truth_all, detected_points_all )
%compute_error
% compute the average point-to-point Euclidean error normalized by the
% inter-ocular distance (measured as the Euclidean distance between the
% outer corners of the eyes)
%
% Inputs:
% grounth_truth_all, size: num_of_points x 2 x num_of_images
% detected_points_all, size: num_of_points x 2 x num_of_images
% Output:
% error_per_image, size: num_of_images x 1
num_of_images = numel(ground_truth_all);
error_per_image = zeros(num_of_images,1);
frontal_ids = true(num_of_images,1);
for i =1:num_of_images
detected_points = detected_points_all{i} + 0.5;
ground_truth_points = ground_truth_all{i};
num_of_points = size(ground_truth_points,1);
normalization = (max(ground_truth_points(:,1))-min(ground_truth_points(:,1)))+...
(max(ground_truth_points(:,2))-min(ground_truth_points(:,2)));
normalization = normalization / 2;
if(num_of_points==39)
frontal_ids(i) = false;
landmark_labels = zeros(68,2);
% Need to map to the profile points, and normalize based on size
% instead
left_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,32; 23,39; 24,38; 25,37; 26,42; 27,41;
28,52; 29,51; 30,50; 31,49; 32,60; 33,59; 34,58;
35,63; 36,62; 37,61; 38,68; 39,67];
right_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,36; 23,44; 24,45; 25,46; 26,47; 27,48;
28,52; 29,53; 30,54; 31,55; 32,56; 33,57; 34,58;
35,63; 36,64; 37,65; 38,66; 39,67];
% Determine if the points are clock-wise or counter clock-wise
% Clock-wise points are facing left, counter-clock-wise right
sum = 0;
for k=1:11
step = (ground_truth_points(k+1,1) - ground_truth_points(k,1)) * (ground_truth_points(k+1,2) + ground_truth_points(k,2));
sum = sum + step;
end
if(sum > 0)
% First need to resample the face outline as there are 9
% points in the near-frontal and 10 points in profile for
% the outline of the face
outline = iterate_piece_wise(ground_truth_points(1:10,:), 9);
brow = iterate_piece_wise(ground_truth_points(13:16,:), 5);
landmark_labels(1:9,:) = outline;
landmark_labels(18:22,:) = brow;
landmark_labels(left_to_frontal_map(:,2),:) = ground_truth_points(left_to_frontal_map(:,1),:);
else
outline = iterate_piece_wise(ground_truth_points(10:-1:1,:), 9);
brow = iterate_piece_wise(ground_truth_points(16:-1:13,:), 5);
landmark_labels(9:17,:) = outline;
landmark_labels(23:27,:) = brow;
landmark_labels(right_to_frontal_map(:,2),:) = ground_truth_points(right_to_frontal_map(:,1),:);
end
detected_points = detected_points(landmark_labels(:,1)~=0,:);
ground_truth_points = landmark_labels(landmark_labels(:,1)~=0,:);
end
num_of_points = size(ground_truth_points,1);
sum=0;
for j=1:num_of_points
sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:));
end
error_per_image(i) = sum/(num_of_points*normalization);
end
end

97
pkg/OpenFace/matlab_version/experiments_menpo/compute_error_menpo_small.m Normal file
View File

@@ -0,0 +1,97 @@
function [ error_per_image, frontal_ids ] = compute_error_menpo_small( ground_truth_all, detected_points_all )
%compute_error
% compute the average point-to-point Euclidean error normalized by the
% inter-ocular distance (measured as the Euclidean distance between the
% outer corners of the eyes)
%
% Inputs:
% grounth_truth_all, size: num_of_points x 2 x num_of_images
% detected_points_all, size: num_of_points x 2 x num_of_images
% Output:
% error_per_image, size: num_of_images x 1
% This script uses the 49 point convention instead of the 68 point one
num_of_images = numel(ground_truth_all);
error_per_image = zeros(num_of_images,1);
frontal_ids = true(num_of_images,1);
for i =1:num_of_images
detected_points = detected_points_all{i} + 0.5;
ground_truth_points = ground_truth_all{i};
num_of_points = size(ground_truth_points,1);
normalization = (max(ground_truth_points(:,1))-min(ground_truth_points(:,1)))+...
(max(ground_truth_points(:,2))-min(ground_truth_points(:,2)));
normalization = normalization / 2;
landmark_labels = ground_truth_points;
if(num_of_points==39)
frontal_ids(i) = false;
landmark_labels = zeros(68,2);
% Need to map to the profile points, and normalize based on size
% instead
left_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,32; 23,39; 24,38; 25,37; 26,42; 27,41;
28,52; 29,51; 30,50; 31,49; 32,60; 33,59; 34,58;
35,63; 36,62; 37,61; 38,68; 39,67];
right_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
21,34; 22,36; 23,44; 24,45; 25,46; 26,47; 27,48;
28,52; 29,53; 30,54; 31,55; 32,56; 33,57; 34,58;
35,63; 36,64; 37,65; 38,66; 39,67];
% Determine if the points are clock-wise or counter clock-wise
% Clock-wise points are facing left, counter-clock-wise right
sum = 0;
for k=1:11
step = (ground_truth_points(k+1,1) - ground_truth_points(k,1)) * (ground_truth_points(k+1,2) + ground_truth_points(k,2));
sum = sum + step;
end
if(sum > 0)
% First need to resample the face outline as there are 9
% points in the near-frontal and 10 points in profile for
% the outline of the face
outline = iterate_piece_wise(ground_truth_points(1:10,:), 9);
brow = iterate_piece_wise(ground_truth_points(13:16,:), 5);
landmark_labels(1:9,:) = outline;
landmark_labels(18:22,:) = brow;
landmark_labels(left_to_frontal_map(:,2),:) = ground_truth_points(left_to_frontal_map(:,1),:);
else
outline = iterate_piece_wise(ground_truth_points(10:-1:1,:), 9);
brow = iterate_piece_wise(ground_truth_points(16:-1:13,:), 5);
landmark_labels(9:17,:) = outline;
landmark_labels(23:27,:) = brow;
landmark_labels(right_to_frontal_map(:,2),:) = ground_truth_points(right_to_frontal_map(:,1),:);
end
end
% First get rid of inner mouth points and then the face outline
landmark_labels = landmark_labels([1:60,62:64,66:end],:,:);
landmark_labels = landmark_labels(18:end,:);
% Do the same for 68 markup version
if(size(detected_points,1) == 68)
detected_points = detected_points([1:60,62:64,66:end],:,:);
detected_points = detected_points(18:end,:);
end
% Remove the invisible points in profile
ground_truth_points = landmark_labels(landmark_labels(:,1)~=0,:);
detected_points = detected_points(landmark_labels(:,1)~=0,:);
num_of_points = size(ground_truth_points,1);
sum=0;
for j=1:num_of_points
sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:));
end
error_per_image(i) = sum/(num_of_points*normalization);
end
end

19
pkg/OpenFace/matlab_version/experiments_menpo/cummErrorCurve.m Normal file
View File

@@ -0,0 +1,19 @@
function [x, y] = cummErrorCurve( errorVec )
%CUMMERRORCURVE Summary of this function goes here
% Detailed explanation goes here
spacing = 0.001;
sampling = [0:spacing:max(errorVec)];
x = sampling;
y = zeros(numel(sampling,1));
for i=1:numel(sampling)
y(i) = sum(errorVec < sampling(i)) / numel(errorVec);
end
end

BIN
pkg/OpenFace/matlab_version/experiments_menpo/face_detections/menpo_test_frontal.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/face_detections/menpo_test_profile.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/face_detections/menpo_train_dets.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/face_detections/menpo_valid_dets.mat Normal file
View File

Binary file not shown.

32
pkg/OpenFace/matlab_version/experiments_menpo/iterate_piece_wise.m Normal file
View File

@@ -0,0 +1,32 @@
function [ pts_new ] = iterate_piece_wise( pts_orig, new_num_points )
%ITERATE_PIECE_WISE Summary of this function goes here
% Detailed explanation goes here
% Reinterpolate the new points, but make sure they are still on the
% same lines
num_orig = size(pts_orig,1);
% Divide the number of original segments by number of new segments
step_size = (num_orig - 1) / (new_num_points-1);
pts_new = zeros(new_num_points,2);
% Clamp the beginning and end, as they will be the same
pts_new(1,:) = pts_orig(1,:);
pts_new(end,:) = pts_orig(end,:);
for i=1:new_num_points-2
low_point = floor(1 + i * step_size);
high_point = ceil(1 + i * step_size);
coeff_1 = floor(1 + i * step_size) - i * step_size;
coeff_2 = 1 - coeff_1;
new_pt = coeff_1 * pts_orig(low_point,:) + coeff_2 * pts_orig(high_point,:);
pts_new(i+1,:) = new_pt;
end
end

18
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/Collect_menpo_names.m Normal file
View File

@@ -0,0 +1,18 @@
function [images, labels] = Collect_menpo_names(root_dir)
load('menpo_train_dets.mat');
bb = bboxes;
load('menpo_valid_dets.mat');
bboxes = cat(2, bb, bboxes);
% Have three bounding box locations (frontal tuned, profile tuned)
labels = cell(numel(bboxes),1);
for i=1:numel(bboxes)
images(i).img = [bboxes(i).name];
labels{i} = bboxes(i).gt_landmarks;
end
end

43
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/Collect_menpo_test_frontal.m Normal file
View File

@@ -0,0 +1,43 @@
function [images, detections] = Collect_menpo_test_frontal(root_dir)
load('face_detections/menpo_test_frontal.mat');
% Have three bounding box locations (frontal tuned, profile tuned)
detections = zeros(numel(bboxes), 4);
for i=1:numel(bboxes)
images(i).img = [root_dir, bboxes(i).name];
% If face detected
if(~isempty(bboxes(i).bbox))
bbox = bboxes(i).bbox(1:4);
% Correct the MTCNN bounding box
width = bbox(3) - bbox(1);
height = bbox(4) - bbox(2);
tx = bbox(1);
ty = bbox(2);
% Frontal faces
new_width = width * 1.0323;
new_height = height * 0.7751;
new_tx = width * -0.0075 + tx;
new_ty = height * 0.2459 + ty;
detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
else % If face not detected, use the mean location of the face in training data
img_size = size(imread([root_dir, bboxes(i).name]));
img_width = img_size(2);
img_height = img_size(1);
width = img_width * 0.4421;
height = img_height * 0.445;
tx = img_width * 0.5048 - 0.5 * width;
ty = img_height * 0.5166 - 0.5 * height;
detections(i,:) = [tx, ty, tx + width, ty + height];
end
end
end

43
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/Collect_menpo_test_profile.m Normal file
View File

@@ -0,0 +1,43 @@
function [images, detections] = Collect_menpo_test_profile(root_dir)
load('face_detections/menpo_test_profile.mat');
% Have three bounding box locations (frontal tuned, profile tuned)
detections = zeros(numel(bboxes), 4);
for i=1:numel(bboxes)
images(i).img = [root_dir, bboxes(i).name];
% If face detected
if(~isempty(bboxes(i).bbox))
bbox = bboxes(i).bbox(1:4);
% Correct the MTCNN bounding box
width = bbox(3) - bbox(1);
height = bbox(4) - bbox(2);
tx = bbox(1);
ty = bbox(2);
% Frontal faces
new_width = width * 1.0323;
new_height = height * 0.7751;
new_tx = width * -0.0075 + tx;
new_ty = height * 0.2459 + ty;
detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
else % If face not detected, use the mean location of the face in training data
img_size = size(imread([root_dir, bboxes(i).name]));
img_width = img_size(2);
img_height = img_size(1);
width = img_width * 0.4421;
height = img_height * 0.445;
tx = img_width * 0.5048 - 0.5 * width;
ty = img_height * 0.5166 - 0.5 * height;
detections(i,:) = [tx, ty, tx + width, ty + height];
end
end
end

163
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/Script_CECLM_menpo_test_frontal_old.m Normal file
View File

@@ -0,0 +1,163 @@
function Script_CECLM_menpo_test_frontal()
addpath('menpo_challenge_helpers');
addpath('../PDM_helpers/');
addpath('../fitting/normxcorr2_mex_ALL');
addpath('../fitting/');
addpath('../CCNF/');
addpath('../models/');
[images, detections] = Collect_menpo_test_frontal('D:\Datasets\menpo\testset\semifrontal/');
%% loading the patch experts
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
%% Fitting the model to the provided image
output_root = './menpo_fit_ceclm_test_semifrontal/';
out_pts = './out_semifrontal/';
mkdir(out_pts);
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = 0.9 * [35, 27, 20, 20];
clmParams.sigmaMeanShift = 1.5 * [1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% for recording purposes
experiment.params = clmParams;
num_points = numel(M)/3;
shapes_all = cell(numel(images), 1);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
verbose = true;
tic
if(verbose)
f = figure;
end
% load('../pdm_generation/menpo_pdm/conversion.mat');
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% have a multi-view version
if(multi_view)
views = [0,0,0; 0,-70,40; 0,70,-40; 0,-30,0; 0,-60,0; 0,-90,0; 0,30,0; 0,60,0; 0,90,0; 0,0,30; 0,0,-30;];
views = views * pi/180;
shapes = zeros(num_points, 2, size(views,1));
ls = zeros(size(views,1),1);
lmark_lhoods = zeros(num_points,size(views,1));
views_used = zeros(size(views,1),1);
g_params = zeros(size(views, 1), 6);
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),g_params(v,:),~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
end
[lhood, v_ind] = max(ls);
lmark_lhood = lmark_lhoods(:,v_ind);
shape = shapes(:,:,v_ind);
view_used = views_used(v_ind);
g_params = g_params(v_ind,:);
else
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
end
shape = shape + 0.5;
[~, name_org, ~] = fileparts(images(i).img);
name = [out_pts, name_org, '.pts'];
shape = write_menpo_frontal(shape, name);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose && mod(i,20) == 0)
% f = figure('visible','off');
try
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
plot(shape(:,1)-0.5, shape(:,2)-0.5,'.r','MarkerSize',20);
plot(shape(:,1)-0.5, shape(:,2)-0.5,'.b','MarkerSize',10);
rectangle('Position', [bbox(1), bbox(2), bbox(3) - bbox(1), bbox(4) - bbox(2)], 'EdgeColor', 'r', 'LineWidth', 2);
% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
print(f, '-djpeg', sprintf('%s/%s.jpg', output_root, name_org));
% close(f);
hold off;
drawnow expose
catch warn
end
end
end
toc
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
% fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
% numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
%%
output_results = 'results/results_test_menpo_semifrontal.mat';
save(output_results, 'experiments');
end

167
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/Script_CECLM_menpo_test_profile_old.m Normal file
View File

@@ -0,0 +1,167 @@
function Script_CECLM_menpo_test_profile()
addpath('menpo_challenge_helpers');
addpath('../PDM_helpers/');
addpath('../fitting/normxcorr2_mex_ALL');
addpath('../fitting/');
addpath('../CCNF/');
addpath('../models/');
[images, detections] = Collect_menpo_test_profile('D:\Datasets\menpo\testset\profile/');
%% loading the patch experts
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
%% Fitting the model to the provided image
output_root = './menpo_fit_ceclm_test_profile/';
mkdir(output_root);
out_pts = './out_profile/';
mkdir(out_pts);
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = 0.9 * [35, 27, 20, 20];
clmParams.sigmaMeanShift = 1.5 * [1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% for recording purposes
experiment.params = clmParams;
num_points = numel(M)/3;
shapes_all = cell(numel(images), 1);
labels_all = cell(numel(images), 1);
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
verbose = true;
tic
if(verbose)
f = figure;
end
load('../pdm_generation/menpo_pdm/menpo_chin/conversion.mat');
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = squeeze(detections(i,:));
% have a multi-view version
if(multi_view)
views = [0,0,0; 0,-70,40; 0,70,-40; 0,-30,0; 0,-60,0; 0,-90,0; 0,30,0; 0,60,0; 0,90,0; 0,0,30; 0,0,-30;];
views = views * pi/180;
shapes = zeros(num_points, 2, size(views,1));
ls = zeros(size(views,1),1);
lmark_lhoods = zeros(num_points,size(views,1));
views_used = zeros(size(views,1),1);
g_params = zeros(size(views, 1), 6);
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),g_params(v,:),~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
end
[lhood, v_ind] = max(ls);
lmark_lhood = lmark_lhoods(:,v_ind);
shape = shapes(:,:,v_ind);
view_used = views_used(v_ind);
g_params = g_params(v_ind,:);
else
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
end
shape = shape + 0.5;
[~, name_org, ~] = fileparts(images(i).img);
name = [out_pts, name_org, '.pts'];
if(g_params(3) > 0)
shape = write_menpo_profile(shape, name, a_left, vis_pts_left);
else
shape = write_menpo_profile(shape, name, a_right, vis_pts_right);
end
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all{i} = shape;
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose && mod(i,20) == 0)
% f = figure('visible','off');
try
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
plot(shape(:,1)-0.5, shape(:,2)-0.5,'.r','MarkerSize',20);
plot(shape(:,1)-0.5, shape(:,2)-0.5,'.b','MarkerSize',10);
rectangle('Position', [bbox(1), bbox(2), bbox(3) - bbox(1), bbox(4) - bbox(2)], 'EdgeColor', 'r', 'LineWidth', 2);
% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
print(f, '-djpeg', sprintf('%s/%s.jpg', output_root, name_org));
% close(f);
hold off;
drawnow expose
catch warn
end
end
end
toc
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
% fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
% numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
%%
output_results = 'results/results_test_menpo_profile.mat';
save(output_results, 'experiments');
end

45
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/Test_menpo_formats.m Normal file
View File

@@ -0,0 +1,45 @@
clear;
load('results/results_valid_dclm_menpo.mat');
% First compute the error
[errors, frontal_ids] = compute_error_menpo_1(experiments.labels, experiments.shapes);
labels_f = experiments.labels(frontal_ids);
shapes_f = experiments.shapes(frontal_ids);
errors_f = errors(frontal_ids);
for i=1:10
write_menpo_frontal(shapes_f{i} + 0.5, sprintf('tmp/%d.pts', i));
preds = importdata(sprintf('tmp/%d.pts', i), ' ', 3);
landmarks = preds.data;
[err] = compute_error_menpo_unb(labels_f(i), {landmarks});
end
%%
labels_p = experiments.labels(~frontal_ids);
shapes_p = experiments.shapes(~frontal_ids);
errors_p = errors(~frontal_ids);
v_used = experiments.all_views_used(~frontal_ids);
load('../pdm_generation/menpo_pdm/conversion.mat');
load('../pdm_generation/menpo_pdm/menpo_68_pts_valid_profile.mat');
ind_left = 1;
ind_right = 1;
for i=1:10
if(v_used(i) == 2 || v_used(i) == 3 || v_used(i) == 6)
write_menpo_profile(shapes_p{i} + 0.5, sprintf('tmp/%d.pts', i), a_left, vis_pts_left);
labs = all_pts_orig_left(:,:,ind_left);
ind_left = ind_left + 1;
else
write_menpo_profile(shapes_p{i} + 0.5, sprintf('tmp/%d.pts', i), a_right, vis_pts_right);
labs = all_pts_orig_right(:,:,ind_right);
ind_right = ind_right + 1;
end
preds = importdata(sprintf('tmp/%d.pts', i), ' ', 3);
landmarks = preds.data;
[err] = compute_error_prof_unb({labs}, {landmarks});
end

BIN
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/conversion.mat Normal file
View File

Binary file not shown.

19
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/write_menpo_frontal.m Normal file
View File

@@ -0,0 +1,19 @@
function [ out_dets ] = write_menpo_frontal( detections, name )
%WRITE_MEPO_PROFILE Summary of this function goes here
% Detailed explanation goes here
f = fopen(name, 'w');
fprintf(f, 'version: 1\n');
fprintf(f, 'n_points: 68\n');
fprintf(f, '{\n');
xs = detections(:,1);
ys = detections(:,2);
for i=1:size(xs,1)
fprintf(f, '%.3f %.3f\n', xs(i), ys(i));
end
fprintf(f, '}\n');
fclose(f);
out_dets = cat(2, xs, ys);
end

20
pkg/OpenFace/matlab_version/experiments_menpo/menpo_challenge_helpers/write_menpo_profile.m Normal file
View File

@@ -0,0 +1,20 @@
function [ out_dets ] = write_menpo_profile( detections, name, conversion, visibilities )
%WRITE_MEPO_PROFILE Summary of this function goes here
% Detailed explanation goes here
f = fopen(name, 'w');
fprintf(f, 'version: 1\n');
fprintf(f, 'n_points: 39\n');
fprintf(f, '{\n');
dets = conversion * cat(1,detections(visibilities,1), detections(visibilities,2));
xs = dets(1:end/2,:);
ys = dets(end/2+1:end,:);
for i=1:size(xs,1)
fprintf(f, '%.3f %.3f\n', xs(i), ys(i));
end
fprintf(f, '}\n');
fclose(f);
out_dets = cat(2, xs, ys);
end

BIN
pkg/OpenFace/matlab_version/experiments_menpo/menpo_gt.mat Normal file
View File

Binary file not shown.

21
pkg/OpenFace/matlab_version/experiments_menpo/readme.txt Normal file
View File

@@ -0,0 +1,21 @@
Scripts for evaluating CE-CLM, and CLNF methods on the Menpo dataset - https://ibug.doc.ic.ac.uk/resources/2nd-facial-landmark-tracking-competition-menpo-ben/
To run the models on the data using models trained on 300W + MultiPIE on all Menpo training data:
- Script_CECLM_cross_data.m
- Script_CLNF_cross_data_general.m
To run CE-CLM model trained on 300W + MultiPIE + Menpo on menpo validation data:
- Script CECLM_menpo_valid.m
To visualize the results of the models against baselines (using inner face markup - 49, and full face markup - 68):
Display_ceclm_results_49.m
Display_ceclm_results_68.m
To construct the error table:
Construct_error_table.m
For a fair comparison all the models and baselines are initialized using the bounding boxes generated by using an MTCNN face detector (https://github.com/kpzhang93/MTCNN_face_detection_alignment).
Scripts to reproduce the CE-CLM submission to the Menpo challenge:
Script_CECLM_menpo_test_frontal.m
Script_CECLM_menpo_test_profile.m

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/CFAN_menpo_train.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/Menpo-CFSS_train.mat Normal file
View File

Binary file not shown.

13
pkg/OpenFace/matlab_version/experiments_menpo/results/Menpo_49.txt Normal file
View File

@@ -0,0 +1,13 @@
Errors without outline (49 points)
------------------------------
Method frontal profile
CLNF 2.10 4.43
SDM 2.54 36.73
CFAN 2.34 28.09
DRMF 3.44 36.14
CFSS 1.90 8.42
PO-CR 2.03 36.04
TCDCN 2.81 8.69
3DDFA 3.59 5.47
------------------------------
CE-CLM 1.75 3.34

10
pkg/OpenFace/matlab_version/experiments_menpo/results/Menpo_68.txt Normal file
View File

@@ -0,0 +1,10 @@
Errors with outline (68 points)
------------------------------
Method frontal profile
CLNF 2.66 6.68
CFAN 2.87 25.33
CFSS 2.32 9.99
TCDCN 3.32 9.82
3DDFA 4.51 6.02
------------------------------
CE-CLM 2.25 5.41

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/Menpo_train_pocr.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/errors_all.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-frontal_49.pdf Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-frontal_49.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 215 KiB

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-frontal_68.pdf Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-frontal_68.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 196 KiB

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-profile_49.pdf Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-profile_49.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 166 KiB

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-profile_68.pdf Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo-profile_68.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 174 KiB

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo_labels.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo_train_3DDFA.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo_train_chehra.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/menpo_train_sdm.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/results_ceclm_cross-data.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/results_ceclm_menpo_valid.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/results_clnf_cross-data_general.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/experiments_menpo/results/tcdcn_menpo.mat Normal file
View File

Binary file not shown.