bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

360
pkg/OpenFace/matlab_version/face_validation/Create_data_test.m Normal file
View File

@@ -0,0 +1,360 @@
function Create_data_test()
load '../models/pdm/pdm_68_aligned_menpo';
load '../models/tri_68.mat';
% This script uses the same format used for patch expert training, and
% expects the data to be there (this can be found in
% https://github.com/TadasBaltrusaitis/CCNF)
% Replace with your location of training data
dataset_loc = 'C:\Users\tbaltrus\Documents\CCNF\patch_experts\data_preparation/prepared_data/';
addpath('../PDM_helpers/');
addpath('./paw_helpers/');
% Collect Menpo, Multi-PIE and 300W data for training the validator
scale = '0.5';
prefix_menpo= 'menpo_valid_';
prefix_mpie_300W = 'combined_';
% Find the available positive training data
data_files = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_menpo, scale));
data_files_c = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_mpie_300W, scale));
centres_all = [];
for i=1:numel(data_files)
% Load the orientation of the training data
load([dataset_loc, '/', data_files(i).name], 'centres');
centres_all = cat(1, centres_all, centres);
end
% Construct mirror indices (which views need to be flipped to create other
% profile training data)
mirror_inds = zeros(size(centres_all,1), 1);
for i=1:numel(data_files)
% mirrored image has inverse yaw
mirrored_centre = centres_all(i,:);
mirrored_centre(2) = -mirrored_centre(2);
% if mirrored version has same orientation, do not need mirroring
if(~isequal(mirrored_centre, centres_all(i,:)))
centres_all = cat(1, centres_all, mirrored_centre);
mirror_inds = cat(1, mirror_inds, i);
end
end
% Replace with your location of training data
outputLocation = 'D:\Datasets/detection_validation/prep_data/';
num_more_neg = 10;
% Make sure same data generated all the time
rng(0);
neg_image_loc = 'D:\Datasets\INRIAPerson\INRIAPerson\Train\neg/';
neg_images = cat(1,dir([neg_image_loc, '/*.jpg']),dir([neg_image_loc, '/*.png']));
max_img_used = 4000;
% do it separately for centers due to memory limitations
for r=1:size(centres_all,1)
a_mod = 0.4;
mirror = false;
if(mirror_inds(r) ~= 0 )
mirror = true;
label_mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
% Make sure we take the subset of visibilities from all the
% datasets
load([dataset_loc, '/', data_files_c(mirror_inds(r)).name], 'visiIndex');
visiIndex_t = visiIndex;
load([dataset_loc, '/', data_files(mirror_inds(r)).name]);
visiIndex = visiIndex_t & visiIndex;
else
load([dataset_loc, '/', data_files_c(r).name], 'visiIndex');
visiIndex_t = visiIndex;
load([dataset_loc, '/', data_files(r).name]);
visiIndex = visiIndex_t & visiIndex;
end
visiCurrent = logical(visiIndex);
if(mirror)
centres = [centres(1), -centres(2), -centres(3)];
tmp1 = visiCurrent(label_mirror_inds(:,1));
tmp2 = visiCurrent(label_mirror_inds(:,2));
visiCurrent(label_mirror_inds(:,2)) = tmp1;
visiCurrent(label_mirror_inds(:,1)) = tmp2;
end
visibleVerts = 1:numel(visiCurrent);
visibleVerts = visibleVerts(visiCurrent)-1;
% Correct the triangulation to take into account the vertex
% visibilities
triangulation = [];
shape = a_mod * Euler2Rot(centres * pi/180) * reshape(M, numel(M)/3, 3)';
shape = shape';
for i=1:size(T,1)
visib = 0;
for j=1:numel(visibleVerts)
if(T(i,1)==visibleVerts(j))
visib = visib+1;
end
if(T(i,2)==visibleVerts(j))
visib = visib+1;
end
if(T(i,3)==visibleVerts(j))
visib = visib+1;
end
end
% Only if all three of the vertices are visible
if(visib == 3)
% Also want to remove triangles facing the wrong way (self occluded)
v1 = [shape(T(i,1)+1,1), shape(T(i,1)+1,2), shape(T(i,1)+1,3)];
v2 = [shape(T(i,2)+1,1), shape(T(i,2)+1,2), shape(T(i,2)+1,3)];
v3 = [shape(T(i,3)+1,1), shape(T(i,3)+1,2), shape(T(i,3)+1,3)];
normal = cross((v2-v1), v3 - v2);
normal = normal / norm(normal);
direction = normal * [0,0,1]';
% And only if the triangle is facing the camera
if(direction > 0)
triangulation = cat(1, triangulation, T(i,:));
end
end
end
% Initialise the warp
[ alphas, betas, triX, mask, minX, minY, nPix ] = InitialisePieceWiseAffine(triangulation, shape);
mask = logical(mask);
imgs_to_use = randperm(size(landmark_locations, 1));
if(size(landmark_locations, 1) > max_img_used)
imgs_to_use = imgs_to_use(1:max_img_used);
end
% Extracting relevant filenames
examples = zeros(numel(imgs_to_use) * (num_more_neg+1), nPix);
errors = zeros(numel(imgs_to_use) * (num_more_neg+1), 1);
unused_pos = 0;
curr_filled = 0;
for j=imgs_to_use
labels = squeeze(landmark_locations(j,:,:));
img = squeeze(all_images(j,:,:));
if(mirror)
img = fliplr(img);
imgSize = size(img);
flippedLbls = labels;
flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1) + 1;
tmp1 = flippedLbls(label_mirror_inds(:,1),:);
tmp2 = flippedLbls(label_mirror_inds(:,2),:);
flippedLbls(label_mirror_inds(:,2),:) = tmp1;
flippedLbls(label_mirror_inds(:,1),:) = tmp2;
labels = flippedLbls;
end
% If for some reason some of the labels are not visible in the
% current sample skip this label
non_existent_labels = labels(:,1)==0 | labels(:,2)==0;
non_existent_inds = find(non_existent_labels)-1;
if(numel(intersect(triangulation(:), non_existent_inds)) > 0)
unused_pos = unused_pos + 1;
continue;
end
% Centering the pixel so that 0,0 is center of the top left pixel
labels = labels - 1;
curr_filled = curr_filled + 1;
[features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
examples(curr_filled,:) = features;
errors(curr_filled,:) = 0;
% Extract the correct PDM parameters for the model (we will perturb
% them for some negative examples)
[ a_orig, R_orig, trans_orig, ~, params_orig] = fit_PDM_ortho_proj_to_2D(M, E, V, labels);
eul_orig = Rot2Euler(R_orig);
% a slightly perturbed example, too tight
% from 0.3 to 0.9
a_mod = a_orig * (0.6 + (randi(7) - 4)*0.1);
p_global = [a_mod; eul_orig'; trans_orig];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
% Compute the badness of fit
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% a slightly perturbed example, too broad
% from 1.2 to 0.6
a_mod = a_orig * (1.4 + (randi(5) - 3)*0.1);
p_global = [a_mod; eul_orig'; trans_orig];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% A somewhat offset example
trans_mod = trans_orig + randn(2,1) * 20;
p_global = [a_orig; eul_orig'; trans_mod];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% A rotated sample
eul_mod = eul_orig + randn(1,3)*0.3;
p_global = [a_orig; eul_mod'; trans_orig];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% A sample with modified shape parameters
p_global = [a_orig; eul_orig'; trans_orig];
params_mod = params_orig + randn(size(params_orig)).*sqrt(E);
labels_mod = GetShapeOrtho(M, V, params_mod, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% pick a random image from negative inriaperson dataset, use original location if
% first, otherwhise resize it to fit
for n=6:num_more_neg
n_img = randi(numel(neg_images));
neg_image = imread([neg_image_loc, neg_images(n_img).name]);
if(size(neg_image,3) == 3)
neg_image = rgb2gray(neg_image);
end
[h_neg, w_neg] = size(neg_image);
% if the current labels fit just use them, if not, then resize
% to fit
max_x = max(labels(:,1));
max_y = max(labels(:,2));
if(max_x > w_neg || max_y > h_neg)
neg_image = imresize(neg_image, [max_y, max_x]);
end
[features] = ExtractFaceFeatures(neg_image, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
% Set high error to 3
errors(curr_filled,:) = 3;
end
if(mod(curr_filled, 10) == 0)
fprintf('%d/%d done\n', curr_filled/(num_more_neg+1), numel(imgs_to_use));
end
% add the pos example to the background
end
examples = examples(1:curr_filled,:);
errors = errors(1:curr_filled);
filename = sprintf('%s/face_validator_test_%d.mat', outputLocation, r);
save(filename, 'examples', 'errors', 'alphas', 'betas', 'triangulation', 'minX', 'minY', 'nPix', 'shape', 'triX', 'mask', 'centres');
end
end
function [features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY)
% Make sure labels are within range
[hRes, wRes] = size(img);
labels(labels(:,1) < 0,1) = 0;
labels(labels(:,2) < 0,2) = 0;
labels(labels(:,1) > wRes-1,1) = wRes-1;
labels(labels(:,2) > hRes-1,2) = hRes-1;
crop_img = Crop(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
crop_img(isnan(crop_img)) = 0;
% vectorised version
features = reshape(crop_img(logical(mask)), 1, nPix);
% normalisations
features = (features - mean(features));
norms = std(features);
if(norms==0)
norms = 1;
end
features = features / norms;
end

369
pkg/OpenFace/matlab_version/face_validation/Create_data_train.m Normal file
View File

@@ -0,0 +1,369 @@
function Create_data_train()
load '../models/pdm/pdm_68_aligned_menpo';
load '../models/tri_68.mat';
% This script uses the same format used for patch expert training, and
% expects the data to be there (this can be found in
% https://github.com/TadasBaltrusaitis/CCNF)
% Replace with your location of training data
dataset_loc = 'C:\Users\tbaltrus\Documents\CCNF\patch_experts\data_preparation/prepared_data/';
addpath('../PDM_helpers/');
addpath('./paw_helpers/');
% Collect Menpo, Multi-PIE and 300W data for training the validator
scale = '0.5';
prefix_menpo= 'menpo_train_';
prefix_mpie_300W = 'combined_';
% Find the available positive training data
data_files = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_menpo, scale));
data_files_c = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_mpie_300W, scale));
centres_all = [];
for i=1:numel(data_files)
% Load the orientation of the training data
load([dataset_loc, '/', data_files(i).name], 'centres');
centres_all = cat(1, centres_all, centres);
end
% Construct mirror indices (which views need to be flipped to create other
% profile training data)
mirror_inds = zeros(size(centres_all,1), 1);
for i=1:numel(data_files)
% mirrored image has inverse yaw
mirrored_centre = centres_all(i,:);
mirrored_centre(2) = -mirrored_centre(2);
% if mirrored version has same orientation, do not need mirroring
if(~isequal(mirrored_centre, centres_all(i,:)))
centres_all = cat(1, centres_all, mirrored_centre);
mirror_inds = cat(1, mirror_inds, i);
end
end
% Replace with your location of training data
outputLocation = 'D:\Datasets/detection_validation/prep_data/';
num_more_neg = 10;
% Make sure same data generated all the time
rng(0);
% Negative samples from teh INRIAPerson dataset
neg_image_loc = 'D:\Datasets\INRIAPerson\INRIAPerson\Train\neg/';
neg_images = cat(1,dir([neg_image_loc, '/*.jpg']),dir([neg_image_loc, '/*.png']));
max_img_used = 8000;
% do it separately for centers due to memory limitations
for r=1:size(centres_all,1)
a_mod = 0.4;
mirror = false;
if(mirror_inds(r) ~= 0 )
mirror = true;
label_mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
load([dataset_loc, '/', data_files_c(mirror_inds(r)).name]);
all_images_t = all_images;
landmark_locations_t = landmark_locations;
visiIndex_t = visiIndex;
load([dataset_loc, '/', data_files(mirror_inds(r)).name]);
% Combining Menpo + MPIE + 300W
all_images = cat(1, all_images, all_images_t);
landmark_locations = cat(1, landmark_locations, landmark_locations_t);
% Taking a subset of visibilities from all the datasets
visiIndex = visiIndex_t & visiIndex;
else
load([dataset_loc, '/', data_files_c(r).name]);
all_images_t = all_images;
landmark_locations_t = landmark_locations;
visiIndex_t = visiIndex;
load([dataset_loc, '/', data_files(r).name]);
all_images = cat(1, all_images, all_images_t);
landmark_locations = cat(1, landmark_locations, landmark_locations_t);
visiIndex = visiIndex_t & visiIndex;
end
visiCurrent = logical(visiIndex);
if(mirror)
centres = [centres(1), -centres(2), -centres(3)];
tmp1 = visiCurrent(label_mirror_inds(:,1));
tmp2 = visiCurrent(label_mirror_inds(:,2));
visiCurrent(label_mirror_inds(:,2)) = tmp1;
visiCurrent(label_mirror_inds(:,1)) = tmp2;
end
visibleVerts = 1:numel(visiCurrent);
visibleVerts = visibleVerts(visiCurrent)-1;
% Correct the triangulation to take into account the vertex
% visibilities
triangulation = [];
shape = a_mod * Euler2Rot(centres * pi/180) * reshape(M, numel(M)/3, 3)';
shape = shape';
for i=1:size(T,1)
visib = 0;
for j=1:numel(visibleVerts)
if(T(i,1)==visibleVerts(j))
visib = visib+1;
end
if(T(i,2)==visibleVerts(j))
visib = visib+1;
end
if(T(i,3)==visibleVerts(j))
visib = visib+1;
end
end
% Only if all three of the vertices are visible
if(visib == 3)
% Also want to remove triangles facing the wrong way (self occluded)
v1 = [shape(T(i,1)+1,1), shape(T(i,1)+1,2), shape(T(i,1)+1,3)];
v2 = [shape(T(i,2)+1,1), shape(T(i,2)+1,2), shape(T(i,2)+1,3)];
v3 = [shape(T(i,3)+1,1), shape(T(i,3)+1,2), shape(T(i,3)+1,3)];
normal = cross((v2-v1), v3 - v2);
normal = normal / norm(normal);
direction = normal * [0,0,1]';
% And only if the triangle is facing the camera
if(direction > 0)
triangulation = cat(1, triangulation, T(i,:));
end
end
end
% Initialise the warp
[ alphas, betas, triX, mask, minX, minY, nPix ] = InitialisePieceWiseAffine(triangulation, shape);
mask = logical(mask);
imgs_to_use = randperm(size(landmark_locations, 1));
if(size(landmark_locations, 1) > max_img_used)
imgs_to_use = imgs_to_use(1:max_img_used);
end
% Extracting relevant filenames
examples = zeros(numel(imgs_to_use) * (num_more_neg+1), nPix);
errors = zeros(numel(imgs_to_use) * (num_more_neg+1), 1);
unused_pos = 0;
curr_filled = 0;
for j=imgs_to_use
labels = squeeze(landmark_locations(j,:,:));
img = squeeze(all_images(j,:,:));
if(mirror)
img = fliplr(img);
imgSize = size(img);
flippedLbls = labels;
flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1) + 1;
tmp1 = flippedLbls(label_mirror_inds(:,1),:);
tmp2 = flippedLbls(label_mirror_inds(:,2),:);
flippedLbls(label_mirror_inds(:,2),:) = tmp1;
flippedLbls(label_mirror_inds(:,1),:) = tmp2;
labels = flippedLbls;
end
% If for some reason some of the labels are not visible in the
% current sample skip this label
non_existent_labels = labels(:,1)==0 | labels(:,2)==0;
non_existent_inds = find(non_existent_labels)-1;
if(numel(intersect(triangulation(:), non_existent_inds)) > 0)
unused_pos = unused_pos + 1;
continue;
end
% Centering the pixel so that 0,0 is center of the top left pixel
labels = labels - 1;
curr_filled = curr_filled + 1;
[features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
examples(curr_filled,:) = features;
errors(curr_filled,:) = 0;
% Extract the correct PDM parameters for the model (we will perturb
% them for some negative examples)
[ a_orig, R_orig, trans_orig, ~, params_orig] = fit_PDM_ortho_proj_to_2D(M, E, V, labels);
eul_orig = Rot2Euler(R_orig);
% a slightly perturbed example, too tight
% from 0.3 to 0.9
a_mod = a_orig * (0.6 + (randi(7) - 4)*0.1);
p_global = [a_mod; eul_orig'; trans_orig];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
% Compute the badness of fit
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% a slightly perturbed example, too broad
% from 1.2 to 0.6
a_mod = a_orig * (1.4 + (randi(5) - 3)*0.1);
p_global = [a_mod; eul_orig'; trans_orig];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
% sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% A somewhat offset example
trans_mod = trans_orig + randn(2,1) * 20;
p_global = [a_orig; eul_orig'; trans_mod];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% A rotated sample
eul_mod = eul_orig + randn(1,3)*0.3;
p_global = [a_orig; eul_mod'; trans_orig];
labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% A sample with modified shape parameters
p_global = [a_orig; eul_orig'; trans_orig];
params_mod = params_orig + randn(size(params_orig)).*sqrt(E);
labels_mod = GetShapeOrtho(M, V, params_mod, p_global);
labels_mod = labels_mod(:,1:2);
[features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
errors(curr_filled,:) = error;
% pick a random image from negative inriaperson dataset, use original location if
% first, otherwhise resize it to fit
for n=6:num_more_neg
n_img = randi(numel(neg_images));
neg_image = imread([neg_image_loc, neg_images(n_img).name]);
if(size(neg_image,3) == 3)
neg_image = rgb2gray(neg_image);
end
[h_neg, w_neg] = size(neg_image);
% if the current labels fit just use them, if not, then resize
% to fit
max_x = max(labels(:,1));
max_y = max(labels(:,2));
if(max_x > w_neg || max_y > h_neg)
neg_image = imresize(neg_image, [max_y, max_x]);
end
[features] = ExtractFaceFeatures(neg_image, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
curr_filled = curr_filled + 1;
examples(curr_filled,:) = features;
% Set high error to 3
errors(curr_filled,:) = 3;
end
if(mod(curr_filled, 10) == 0)
fprintf('%d/%d done\n', curr_filled/(num_more_neg+1), numel(imgs_to_use));
end
% add the pos example to the background
end
examples = examples(1:curr_filled,:);
errors = errors(1:curr_filled);
filename = sprintf('%s/face_validator_train_%d.mat', outputLocation, r);
save(filename, 'examples', 'errors', 'alphas', 'betas', 'triangulation', 'minX', 'minY', 'nPix', 'shape', 'triX', 'mask', 'centres');
end
end
function [features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY)
% Make sure labels are within range
[hRes, wRes] = size(img);
labels(labels(:,1) < 0,1) = 0;
labels(labels(:,2) < 0,2) = 0;
labels(labels(:,1) > wRes-1,1) = wRes-1;
labels(labels(:,2) > hRes-1,2) = hRes-1;
crop_img = Crop(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
crop_img(isnan(crop_img)) = 0;
% vectorised version
features = reshape(crop_img(logical(mask)), 1, nPix);
% normalisations
features = (features - mean(features));
norms = std(features);
if(norms==0)
norms = 1;
end
features = features / norms;
end

139
pkg/OpenFace/matlab_version/face_validation/WriteOutFaceCheckersCNNbinary.m Normal file
View File

@@ -0,0 +1,139 @@
function WriteOutFaceCheckersCNNbinary(locationTxt, faceCheckers)
addpath('../PDM_helpers\');
% use little-endian
faceCheckerFile = fopen(locationTxt, 'w', 'l');
views = numel(faceCheckers);
% Type 0 - linear SVR, 1 - feed forward neural net, 2 - CNN, 3 - new
% CNN
fwrite(faceCheckerFile, 3, 'uint'); % 4 bytes
% Number of face checkers
fwrite(faceCheckerFile, views, 'uint'); % 4 bytes
% Matrices representing view orientations
for i=1:views
% this indicates that we're writing a 3x1 double matrix
writeMatrixBin(faceCheckerFile, faceCheckers(i).centres', 6);
end
for i = 1:views
% The normalisation models
% Mean of images
writeMatrixBin(faceCheckerFile, faceCheckers(i).mean_ex, 6);
% Standard deviation of images
writeMatrixBin(faceCheckerFile, faceCheckers(i).std_ex, 6);
cnn = faceCheckers(i).cnn;
num_depth_layers = size(cnn.layers,2);
% Get the number of layers
fwrite(faceCheckerFile, num_depth_layers, 'uint'); % 4 bytes
% For disambiguation between FC and conv layers
res = vl_simplenn(cnn, single(faceCheckers(i).mask), [], []);
for layers=1:num_depth_layers
% write layer type: 0 - convolutional, 1 - max pooling (2x2 stride 2), 2 -
% fully connected, 3 - relu, 4 - sigmoid
if(cnn.layers{layers}.type == 'conv')
% First check if it is an FC layer (they are represented
% like that in matconvnet)
if(numel(res(layers).x) == numel(cnn.layers{layers}.weights{1}(:,:,:,1)))
% This is the fully connected layer
fwrite(faceCheckerFile, 2, 'uint'); % 4 bytes
% the bias term
writeMatrixBin(faceCheckerFile, cnn.layers{layers}.weights{2}(:), 5);
% the weights
% Convert the filters to a matrix
weights_c = cnn.layers{layers}.weights{1};
size_w = size(weights_c);
weights = zeros(size_w(1)*size_w(2)*size_w(3), size_w(4));
weights(:) = weights_c;
writeMatrixBin(faceCheckerFile, weights, 5);
else
% write the type (convolutional)
fwrite(faceCheckerFile, 0, 'uint'); % 4 bytes
num_in_map = size(cnn.layers{layers}.weights{1},3);
% write the number of input maps
fwrite(faceCheckerFile, num_in_map, 'uint'); % 4 bytes
num_out_kerns = size(cnn.layers{layers}.weights{1},4);
% write the number of kernels for each output map
fwrite(faceCheckerFile, num_out_kerns, 'uint'); % 4 bytes
% Write output map bias terms
for k2=1:num_out_kerns
fwrite(faceCheckerFile, cnn.layers{layers}.weights{2}(k2), 'float32'); % 4 bytes
end
for k=1:num_in_map
for k2=1:num_out_kerns
% Write out the kernel
W = squeeze(cnn.layers{layers}.weights{1}(:,:,k,k2));
writeMatrixBin(faceCheckerFile, W, 5);
end
end
end
elseif(cnn.layers{layers}.type == 'pool')
fwrite(faceCheckerFile, 1, 'uint'); % 4 bytes, indicate max pooling layer, no params, assume (2x2 stride 2)
elseif(cnn.layers{layers}.type == 'relu')
fwrite(faceCheckerFile, 3, 'uint'); % 4 bytes, indicate relu layer, no params
end
end
% Piecewise affine warp
nPix = faceCheckers(i).nPix;
minX = faceCheckers(i).minX;
minY = faceCheckers(i).minY;
destination = reshape(faceCheckers(i).destination, numel(faceCheckers(i).destination), 1);
triangulation = faceCheckers(i).triangulation;
triX = faceCheckers(i).triX;
mask = faceCheckers(i).mask;
alphas = faceCheckers(i).alphas;
betas = faceCheckers(i).betas;
fwrite(faceCheckerFile, nPix, 'uint'); % 4 bytes
fwrite(faceCheckerFile, minX, 'float64'); % 8 bytes
fwrite(faceCheckerFile, minY, 'float64'); % 8 bytes
% Destination shape
writeMatrixBin(faceCheckerFile, destination, 6);
% Triangulation
writeMatrixBin(faceCheckerFile, triangulation, 4);
% Triangle map
writeMatrixBin(faceCheckerFile, triX, 4);
% Mask
writeMatrixBin(faceCheckerFile, mask, 4);
% Alphas
writeMatrixBin(faceCheckerFile, alphas, 6);
% Betas
writeMatrixBin(faceCheckerFile, betas, 6);
end
fclose(faceCheckerFile);
end

632
pkg/OpenFace/matlab_version/face_validation/cnn_train_reg.m Normal file
View File

@@ -0,0 +1,632 @@
function [net, stats] = cnn_train_reg(net, imdb, getBatch, varargin)
%cnn_train_reg An example implementation of SGD for training CNNs
% CNN_TRAIN() is an example learner implementing stochastic
% gradient descent with momentum to train a CNN. It can be used
% with different datasets and tasks by providing a suitable
% getBatch function.
%
% The function automatically restarts after each training epoch by
% checkpointing.
%
% The function supports training on CPU or on one or more GPUs
% (specify the list of GPU IDs in the `gpus` option).
% Copyright (C) 2014-16 Andrea Vedaldi.
% All rights reserved.
%
% This file is part of the VLFeat library and is made available under
% the terms of the BSD license (see the COPYING file).
% This is a modified version for regression using the CNN_TRAIN from
% MatConvNet
addpath(fullfile(vl_rootnn, 'examples'));
opts.expDir = fullfile('data','exp') ;
opts.continue = true ;
opts.batchSize = 256 ;
opts.numSubBatches = 1 ;
opts.train = [] ;
opts.val = [] ;
opts.gpus = [] ;
opts.epochSize = inf;
opts.prefetch = false ;
opts.numEpochs = 300 ;
opts.learningRate = 0.001 ;
opts.weightDecay = 0.0005 ;
opts.solver = [] ; % Empty array means use the default SGD solver
[opts, varargin] = vl_argparse(opts, varargin) ;
if ~isempty(opts.solver)
assert(isa(opts.solver, 'function_handle') && nargout(opts.solver) == 2,...
'Invalid solver; expected a function handle with two outputs.') ;
% Call without input arguments, to get default options
opts.solverOpts = opts.solver() ;
end
opts.momentum = 0.9 ;
opts.saveSolverState = true ;
opts.nesterovUpdate = false ;
opts.randomSeed = 0 ;
opts.memoryMapFile = fullfile(tempdir, 'matconvnet.bin') ;
opts.profile = false ;
opts.parameterServer.method = 'mmap' ;
opts.parameterServer.prefix = 'mcn' ;
opts.conserveMemory = true ;
opts.backPropDepth = +inf ;
opts.sync = false ;
opts.cudnn = true ;
opts.errorFunction = 'regression' ;
opts.errorLabels = {} ;
opts.plotDiagnostics = false ;
opts.plotStatistics = true;
opts.postEpochFn = [] ; % postEpochFn(net,params,state) called after each epoch; can return a new learning rate, 0 to stop, [] for no change
opts = vl_argparse(opts, varargin) ;
if ~exist(opts.expDir, 'dir'), mkdir(opts.expDir) ; end
if isempty(opts.train), opts.train = find(imdb.images.set==1) ; end
if isempty(opts.val), opts.val = find(imdb.images.set==2) ; end
if isscalar(opts.train) && isnumeric(opts.train) && isnan(opts.train)
opts.train = [] ;
end
if isscalar(opts.val) && isnumeric(opts.val) && isnan(opts.val)
opts.val = [] ;
end
hasError = true ;
opts.errorLabels = {'correlation', 'rmse'};
% -------------------------------------------------------------------------
% Initialization
% -------------------------------------------------------------------------
net = vl_simplenn_tidy(net); % fill in some eventually missing values
net.layers{end-1}.precious = 1; % do not remove predictions, used for error
vl_simplenn_display(net, 'batchSize', opts.batchSize) ;
evaluateMode = isempty(opts.train) ;
if ~evaluateMode
for i=1:numel(net.layers)
J = numel(net.layers{i}.weights) ;
if ~isfield(net.layers{i}, 'learningRate')
net.layers{i}.learningRate = ones(1, J) ;
end
if ~isfield(net.layers{i}, 'weightDecay')
net.layers{i}.weightDecay = ones(1, J) ;
end
end
end
state.getBatch = getBatch ;
stats = [] ;
% -------------------------------------------------------------------------
% Train and validate
% -------------------------------------------------------------------------
modelPath = @(ep) fullfile(opts.expDir, sprintf('net-epoch-%d.mat', ep));
modelFigPath = fullfile(opts.expDir, 'net-train.pdf') ;
start = opts.continue * findLastCheckpoint(opts.expDir) ;
if start >= 1
fprintf('%s: resuming by loading epoch %d\n', mfilename, start) ;
[net, state, stats] = loadState(modelPath(start)) ;
else
state = [] ;
end
for epoch=start+1:opts.numEpochs
% Set the random seed based on the epoch and opts.randomSeed.
% This is important for reproducibility, including when training
% is restarted from a checkpoint.
rng(epoch + opts.randomSeed) ;
prepareGPUs(opts, epoch == start+1) ;
% Train for one epoch.
params = opts ;
params.epoch = epoch ;
params.learningRate = opts.learningRate(min(epoch, numel(opts.learningRate))) ;
params.train = opts.train(randperm(numel(opts.train))) ; % shuffle
params.train = params.train(1:min(opts.epochSize, numel(opts.train)));
params.imdb = imdb ;
params.getBatch = getBatch ;
if numel(params.gpus) <= 1
[net, state] = processEpoch(net, state, params, 'train') ;
[net, state] = processEpoch(net, state, params, 'val') ;
if ~evaluateMode
saveState(modelPath(epoch), net, state) ;
end
lastStats = state.stats ;
else
spmd
[net, state] = processEpoch(net, state, params, 'train') ;
[net, state] = processEpoch(net, state, params, 'val') ;
if labindex == 1 && ~evaluateMode
saveState(modelPath(epoch), net, state) ;
end
lastStats = state.stats ;
end
lastStats = accumulateStats(lastStats) ;
end
stats.train(epoch) = lastStats.train ;
stats.val(epoch) = lastStats.val ;
clear lastStats ;
if ~evaluateMode
saveStats(modelPath(epoch), stats) ;
end
if params.plotStatistics
switchFigure(1) ; clf ;
plots = setdiff(...
cat(2,...
fieldnames(stats.train)', ...
fieldnames(stats.val)'), {'num', 'time'}) ;
for p = plots
p = char(p) ;
values = zeros(0, epoch) ;
leg = {} ;
for f = {'train', 'val'}
f = char(f) ;
if isfield(stats.(f), p)
tmp = [stats.(f).(p)] ;
values(end+1,:) = tmp(1,:)' ;
leg{end+1} = f ;
end
end
subplot(1,numel(plots),find(strcmp(p,plots))) ;
plot(1:epoch, values','o-') ;
xlabel('epoch') ;
title(p) ;
legend(leg{:}) ;
grid on ;
end
drawnow ;
print(1, modelFigPath, '-dpdf') ;
end
if ~isempty(opts.postEpochFn)
if nargout(opts.postEpochFn) == 0
opts.postEpochFn(net, params, state) ;
else
lr = opts.postEpochFn(net, params, state) ;
if ~isempty(lr), opts.learningRate = lr; end
if opts.learningRate == 0, break; end
end
end
end
% Return the best performing model
[~,best_epoch] = min(cat(1,stats.val.rmse));
fprintf('%s: Best model in epoch %d\n', mfilename, best_epoch) ;
[net, state, stats] = loadState(modelPath(best_epoch)) ;
% With multiple GPUs, return one copy
if isa(net, 'Composite'), net = net{1} ; end
% -------------------------------------------------------------------------
function [net, state] = processEpoch(net, state, params, mode)
% -------------------------------------------------------------------------
% Note that net is not strictly needed as an output argument as net
% is a handle class. However, this fixes some aliasing issue in the
% spmd caller.
% initialize with momentum 0
if isempty(state) || isempty(state.solverState)
for i = 1:numel(net.layers)
state.solverState{i} = cell(1, numel(net.layers{i}.weights)) ;
state.solverState{i}(:) = {0} ;
end
end
% move CNN to GPU as needed
numGpus = numel(params.gpus) ;
if numGpus >= 1
net = vl_simplenn_move(net, 'gpu') ;
for i = 1:numel(state.solverState)
for j = 1:numel(state.solverState{i})
s = state.solverState{i}{j} ;
if isnumeric(s)
state.solverState{i}{j} = gpuArray(s) ;
elseif isstruct(s)
state.solverState{i}{j} = structfun(@gpuArray, s, 'UniformOutput', false) ;
end
end
end
end
if numGpus > 1
parserv = ParameterServer(params.parameterServer) ;
vl_simplenn_start_parserv(net, parserv) ;
else
parserv = [] ;
end
% profile
if params.profile
if numGpus <= 1
profile clear ;
profile on ;
else
mpiprofile reset ;
mpiprofile on ;
end
end
subset = params.(mode) ;
num = 0 ;
stats.num = 0 ; % return something even if subset = []
stats.time = 0 ;
adjustTime = 0 ;
res = [] ;
preds_all = [];
labels_all = [];
err = 0;
start = tic ;
for t=1:params.batchSize:numel(subset)
fprintf('%s: epoch %02d: %3d/%3d:', mode, params.epoch, ...
fix((t-1)/params.batchSize)+1, ceil(numel(subset)/params.batchSize)) ;
batchSize = min(params.batchSize, numel(subset) - t + 1) ;
for s=1:params.numSubBatches
% get this image batch and prefetch the next
batchStart = t + (labindex-1) + (s-1) * numlabs ;
batchEnd = min(t+params.batchSize-1, numel(subset)) ;
batch = subset(batchStart : params.numSubBatches * numlabs : batchEnd) ;
num = num + numel(batch) ;
if numel(batch) == 0, continue ; end
[im, labels] = params.getBatch(params.imdb, batch) ;
if params.prefetch
if s == params.numSubBatches
batchStart = t + (labindex-1) + params.batchSize ;
batchEnd = min(t+2*params.batchSize-1, numel(subset)) ;
else
batchStart = batchStart + numlabs ;
end
nextBatch = subset(batchStart : params.numSubBatches * numlabs : batchEnd) ;
params.getBatch(params.imdb, nextBatch) ;
end
if numGpus >= 1
im = gpuArray(im) ;
end
if strcmp(mode, 'train')
dzdy = 1 ;
evalMode = 'normal' ;
else
dzdy = [] ;
evalMode = 'test' ;
end
net.layers{end}.class = labels ;
res = vl_simplenn(net, im, dzdy, res, ...
'accumulate', s ~= 1, ...
'mode', evalMode, ...
'conserveMemory', params.conserveMemory, ...
'backPropDepth', params.backPropDepth, ...
'sync', params.sync, ...
'cudnn', params.cudnn, ...
'parameterServer', parserv, ...
'holdOn', s < params.numSubBatches) ;
predictions = gather(res(end-1).x) ;
[~,predictions] = sort(predictions, 3, 'descend') ;
predictions = squeeze(predictions);
num_bins = size(predictions,1);
predictions = predictions(1,:);
% Convert the class labels into the continuous values
labels = unQuantizeContinuous(squeeze(labels), 0, 3, num_bins)';
predictions = unQuantizeContinuous(squeeze(predictions), 0, 3, num_bins)';
preds_all = cat(1, preds_all, predictions);
labels_all = cat(1, labels_all, labels);
err = [err(1)+sum(double(gather(res(end).x)));...
corr(labels_all, preds_all);...
sqrt(mean((labels_all-preds_all).^2))];
end
% accumulate gradient
if strcmp(mode, 'train')
if ~isempty(parserv), parserv.sync() ; end
[net, res, state] = accumulateGradients(net, res, state, params, batchSize, parserv) ;
end
% get statistics
time = toc(start) + adjustTime ;
batchTime = time - stats.time ;
stats = extractStats(net, params, [err(1)/num; err(2); err(3)]) ;
stats.num = num ;
stats.time = time ;
currentSpeed = batchSize / batchTime ;
averageSpeed = (t + batchSize - 1) / time ;
if t == 3*params.batchSize + 1
% compensate for the first three iterations, which are outliers
adjustTime = 4*batchTime - time ;
stats.time = time + adjustTime ;
end
fprintf(' %.1f (%.1f) Hz', averageSpeed, currentSpeed) ;
for f = setdiff(fieldnames(stats)', {'num', 'time'})
f = char(f) ;
fprintf(' %s: %.3f', f, stats.(f)) ;
end
fprintf('\n') ;
% collect diagnostic statistics
if strcmp(mode, 'train') && params.plotDiagnostics
switchFigure(2) ; clf ;
diagn = [res.stats] ;
diagnvar = horzcat(diagn.variation) ;
diagnpow = horzcat(diagn.power) ;
subplot(2,2,1) ; barh(diagnvar) ;
set(gca,'TickLabelInterpreter', 'none', ...
'YTick', 1:numel(diagnvar), ...
'YTickLabel',horzcat(diagn.label), ...
'YDir', 'reverse', ...
'XScale', 'log', ...
'XLim', [1e-5 1], ...
'XTick', 10.^(-5:1)) ;
grid on ; title('Variation');
subplot(2,2,2) ; barh(sqrt(diagnpow)) ;
set(gca,'TickLabelInterpreter', 'none', ...
'YTick', 1:numel(diagnpow), ...
'YTickLabel',{diagn.powerLabel}, ...
'YDir', 'reverse', ...
'XScale', 'log', ...
'XLim', [1e-5 1e5], ...
'XTick', 10.^(-5:5)) ;
grid on ; title('Power');
subplot(2,2,3); plot(squeeze(res(end-1).x)) ;
drawnow ;
end
end
% Save back to state.
state.stats.(mode) = stats ;
if params.profile
if numGpus <= 1
state.prof.(mode) = profile('info') ;
profile off ;
else
state.prof.(mode) = mpiprofile('info');
mpiprofile off ;
end
end
if ~params.saveSolverState
state.solverState = [] ;
else
for i = 1:numel(state.solverState)
for j = 1:numel(state.solverState{i})
s = state.solverState{i}{j} ;
if isnumeric(s)
state.solverState{i}{j} = gather(s) ;
elseif isstruct(s)
state.solverState{i}{j} = structfun(@gather, s, 'UniformOutput', false) ;
end
end
end
end
net = vl_simplenn_move(net, 'cpu') ;
% -------------------------------------------------------------------------
function [net, res, state] = accumulateGradients(net, res, state, params, batchSize, parserv)
% -------------------------------------------------------------------------
numGpus = numel(params.gpus) ;
otherGpus = setdiff(1:numGpus, labindex) ;
for l=numel(net.layers):-1:1
for j=numel(res(l).dzdw):-1:1
if ~isempty(parserv)
tag = sprintf('l%d_%d',l,j) ;
parDer = parserv.pull(tag) ;
else
parDer = res(l).dzdw{j} ;
end
if j == 3 && strcmp(net.layers{l}.type, 'bnorm')
% special case for learning bnorm moments
thisLR = net.layers{l}.learningRate(j) ;
net.layers{l}.weights{j} = vl_taccum(...
1 - thisLR, ...
net.layers{l}.weights{j}, ...
thisLR / batchSize, ...
parDer) ;
else
% Standard gradient training.
thisDecay = params.weightDecay * net.layers{l}.weightDecay(j) ;
thisLR = params.learningRate * net.layers{l}.learningRate(j) ;
if thisLR>0 || thisDecay>0
% Normalize gradient and incorporate weight decay.
parDer = vl_taccum(1/batchSize, parDer, ...
thisDecay, net.layers{l}.weights{j}) ;
if isempty(params.solver)
% Default solver is the optimised SGD.
% Update momentum.
state.solverState{l}{j} = vl_taccum(...
params.momentum, state.solverState{l}{j}, ...
-1, parDer) ;
% Nesterov update (aka one step ahead).
if params.nesterovUpdate
delta = params.momentum * state.solverState{l}{j} - parDer ;
else
delta = state.solverState{l}{j} ;
end
% Update parameters.
net.layers{l}.weights{j} = vl_taccum(...
1, net.layers{l}.weights{j}, ...
thisLR, delta) ;
else
% call solver function to update weights
[net.layers{l}.weights{j}, state.solverState{l}{j}] = ...
params.solver(net.layers{l}.weights{j}, state.solverState{l}{j}, ...
parDer, params.solverOpts, thisLR) ;
end
end
end
% if requested, collect some useful stats for debugging
if params.plotDiagnostics
variation = [] ;
label = '' ;
switch net.layers{l}.type
case {'conv','convt'}
if isnumeric(state.solverState{l}{j})
variation = thisLR * mean(abs(state.solverState{l}{j}(:))) ;
end
power = mean(res(l+1).x(:).^2) ;
if j == 1 % fiters
base = mean(net.layers{l}.weights{j}(:).^2) ;
label = 'filters' ;
else % biases
base = sqrt(power) ;%mean(abs(res(l+1).x(:))) ;
label = 'biases' ;
end
variation = variation / base ;
label = sprintf('%s_%s', net.layers{l}.name, label) ;
end
res(l).stats.variation(j) = variation ;
res(l).stats.power = power ;
res(l).stats.powerLabel = net.layers{l}.name ;
res(l).stats.label{j} = label ;
end
end
end
% -------------------------------------------------------------------------
function stats = accumulateStats(stats_)
% -------------------------------------------------------------------------
for s = {'train', 'val'}
s = char(s) ;
total = 0 ;
% initialize stats stucture with same fields and same order as
% stats_{1}
stats__ = stats_{1} ;
names = fieldnames(stats__.(s))' ;
values = zeros(1, numel(names)) ;
fields = cat(1, names, num2cell(values)) ;
stats.(s) = struct(fields{:}) ;
for g = 1:numel(stats_)
stats__ = stats_{g} ;
num__ = stats__.(s).num ;
total = total + num__ ;
for f = setdiff(fieldnames(stats__.(s))', 'num')
f = char(f) ;
stats.(s).(f) = stats.(s).(f) + stats__.(s).(f) * num__ ;
if g == numel(stats_)
stats.(s).(f) = stats.(s).(f) / total ;
end
end
end
stats.(s).num = total ;
end
% -------------------------------------------------------------------------
function stats = extractStats(net, params, errors)
% -------------------------------------------------------------------------
stats.objective = errors(1) ;
for i = 1:numel(params.errorLabels)
stats.(params.errorLabels{i}) = errors(i+1) ;
end
% -------------------------------------------------------------------------
function saveState(fileName, net, state)
% -------------------------------------------------------------------------
save(fileName, 'net', 'state') ;
% -------------------------------------------------------------------------
function saveStats(fileName, stats)
% -------------------------------------------------------------------------
if exist(fileName)
save(fileName, 'stats', '-append') ;
else
save(fileName, 'stats') ;
end
% -------------------------------------------------------------------------
function [net, state, stats] = loadState(fileName)
% -------------------------------------------------------------------------
load(fileName, 'net', 'state', 'stats') ;
net = vl_simplenn_tidy(net) ;
if isempty(whos('stats'))
error('Epoch ''%s'' was only partially saved. Delete this file and try again.', ...
fileName) ;
end
% -------------------------------------------------------------------------
function epoch = findLastCheckpoint(modelDir)
% -------------------------------------------------------------------------
list = dir(fullfile(modelDir, 'net-epoch-*.mat')) ;
tokens = regexp({list.name}, 'net-epoch-([\d]+).mat', 'tokens') ;
epoch = cellfun(@(x) sscanf(x{1}{1}, '%d'), tokens) ;
epoch = max([epoch 0]) ;
% -------------------------------------------------------------------------
function switchFigure(n)
% -------------------------------------------------------------------------
if get(0,'CurrentFigure') ~= n
try
set(0,'CurrentFigure',n) ;
catch
figure(n) ;
end
end
% -------------------------------------------------------------------------
function clearMex()
% -------------------------------------------------------------------------
%clear vl_tmove vl_imreadjpeg ;
disp('Clearing mex files') ;
clear mex ;
clear vl_tmove vl_imreadjpeg ;
% -------------------------------------------------------------------------
function prepareGPUs(params, cold)
% -------------------------------------------------------------------------
numGpus = numel(params.gpus) ;
if numGpus > 1
% check parallel pool integrity as it could have timed out
pool = gcp('nocreate') ;
if ~isempty(pool) && pool.NumWorkers ~= numGpus
delete(pool) ;
end
pool = gcp('nocreate') ;
if isempty(pool)
parpool('local', numGpus) ;
cold = true ;
end
end
if numGpus >= 1 && cold
fprintf('%s: resetting GPU\n', mfilename) ;
clearMex() ;
if numGpus == 1
disp(gpuDevice(params.gpus)) ;
else
spmd
clearMex() ;
disp(gpuDevice(params.gpus(labindex))) ;
end
end
end

58
pkg/OpenFace/matlab_version/face_validation/face_check_cnn.m Normal file
View File

@@ -0,0 +1,58 @@
function [ decision ] = face_check_cnn( img, shape, global_params, cnns )
%FACE_CHECK_CNN Summary of this function goes here
% Detailed explanation goes here
%%
if(size(img,3) == 3)
img = rgb2gray(img);
end
% first need to determine the view
centres = cat(1, cnns.centres);
dists = centres*pi/180 - repmat(global_params(2:4)',size(centres,1),1);
[~,view_id] = min(sum(dists.^2,2));
mask_small = cnns(view_id).mask(1:size(cnns(view_id).triX,1), 1:size(cnns(view_id).triX,2));
if(size(cnns(view_id).destination,1) == 66 && size(shape,1) == 68)
label_inds = [1:60,62:64,66:68];
shape = shape(label_inds,:);
end
img_crop = Crop(img, shape, cnns(view_id).triangulation,...
cnns(view_id).triX, mask_small,...
cnns(view_id).alphas, cnns(view_id).betas,...
cnns(view_id).nPix, cnns(view_id).minX, ...
cnns(view_id).minY);
%%
img_crop = reshape(img_crop(logical(cnns(view_id).mask)), 1, cnns(view_id).nPix);
img_crop(isnan(img_crop)) = 0;
% normalisation (local)
img_crop = (img_crop - mean(img_crop));
norms = std(img_crop);
if(norms==0)
norms = 1;
end
img_crop = img_crop / norms;
% normalisation (global)
img_crop = img_crop - cnns(view_id).mean_ex;
img_crop = img_crop ./ cnns(view_id).std_ex;
mask = cnns(view_id).mask;
img = zeros(size(mask));
img(mask) = img_crop;
cnn = cnns(view_id).cnn;
%%
res = vl_simplenn(cnn, single(img), [], []);
res = gather(res(end).x);
num_bins = numel(res);
[~,res] = sort(res, 3, 'descend') ;
res = squeeze(res);
res = res(1,:);
decision = unQuantizeContinuous(res, 0, 3, num_bins)';

37
pkg/OpenFace/matlab_version/face_validation/initializeFaceCNN.m Normal file
View File

@@ -0,0 +1,37 @@
function net = initializeFaceCNN_simple_5(num_bins)
f=1/100 ;
net.layers = {} ;
net.layers{end+1} = struct('type', 'conv', ...
'weights', {{f*randn(9,9,1,10, 'single'), zeros(1, 10, 'single')}}, ...
'stride', 1, ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'relu') ;
net.layers{end+1} = struct('type', 'pool', ...
'method', 'max', ...
'pool', [2 2], ...
'stride', 2, ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'conv', ...
'weights', {{f*randn(7,7,10,10, 'single'), zeros(1,10,'single')}}, ...
'stride', 1, ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'relu') ;
net.layers{end+1} = struct('type', 'pool', ...
'method', 'max', ...
'pool', [2 2], ...
'stride', 2, ...
'pad', 0) ;
% This is basically an FC layer
net.layers{end+1} = struct('type', 'conv', ...
'weights', {{f*randn(10,10,10,50, 'single'), zeros(1,50,'single')}}, ...
'stride', 1, ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'relu') ;
net.layers{end+1} = struct('type', 'conv', ...
'weights', {{f*randn(1,1,50,num_bins, 'single'), zeros(1,num_bins,'single')}}, ...
'stride', 1, ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'softmaxloss') ;
net = vl_simplenn_tidy(net) ;

33
pkg/OpenFace/matlab_version/face_validation/paw_helpers/CalculateCoefficients.m Normal file
View File

@@ -0,0 +1,33 @@
function [ coeffs ] = CalculateCoefficients( alphas, betas, triangulation, destination )
%CALCULATECOEFFICIENTS Summary of this function goes here
% Detailed explanation goes here
xs = destination(:,1);
ys = destination(:,2);
triangulation = triangulation + 1;
numTri = size(triangulation, 1);
coeffs = zeros(numTri, 3);
for l=1:numTri
i = triangulation(l,1);
j = triangulation(l,2);
k = triangulation(l,3);
c1 = xs(i);
c2 = xs(j) - c1;
c3 = xs(k) - c1;
c4 = ys(i);
c5 = ys(j) - c4;
c6 = ys(k) - c4;
coeffs(l,1) = c1 + c2*alphas(l,1) + c3*betas(l,1);
coeffs(l,2) = c2*alphas(l,2) + c3*betas(l,2);
coeffs(l,3) = c2*alphas(l,3) + c3*betas(l,3);
coeffs(l,4) = c4 + c5*alphas(l,1) + c6*betas(l,1);
coeffs(l,5) = c5*alphas(l,2) + c6*betas(l,2);
coeffs(l,6) = c5*alphas(l,3) + c6*betas(l,3);
end
end

15
pkg/OpenFace/matlab_version/face_validation/paw_helpers/Crop.m Normal file
View File

@@ -0,0 +1,15 @@
function [warpedImage] = Crop(inputImage, destinationPoints, triangulation, triX, mask, alphas, betas, nPix, minX, minY)
coeffs = CalculateCoefficients(alphas, betas, triangulation, destinationPoints);
[ mapX, mapY ] = WarpRegion( minX, minY, mask, triX, coeffs );
if(size(inputImage,3) == 1)
warpedImage = Remap(inputImage, mapX, mapY);
else
warpedImage = [];
for i=1:size(inputImage,3)
warpedImage = cat(3, warpedImage, Remap(inputImage(:,:,i), mapX, mapY));
end
end
end

113
pkg/OpenFace/matlab_version/face_validation/paw_helpers/InitialisePieceWiseAffine.m Normal file
View File

@@ -0,0 +1,113 @@
function [ alphas, betas, triX, mask, xmin, ymin, npix ] = InitialisePieceWiseAffine( triangulation, sourcePoints )
%INITIALISEPIECEWICEAFFINE Summary of this function goes here
% Detailed explanation goes here
triangulation = triangulation + 1;
numPoints = size(sourcePoints, 1);
numTris = size(triangulation, 1);
alphas = zeros(size(triangulation, 1), 3);
betas = zeros(size(triangulation, 1), 3);
xs = sourcePoints(:,1);
ys = sourcePoints(:,2);
for i = 1:numTris
j = triangulation(i, 1);
k = triangulation(i, 2);
l = triangulation(i, 3);
c1 = ys(l) - ys(j);
c2 = xs(l) - xs(j);
c4 = ys(k) - ys(j);
c3 = xs(k) - xs(j);
c5 = c3*c1 - c2*c4;
alphas(i, 1) = (ys(j) * c2 - xs(j) * c1) / c5;
alphas(i, 2) = c1/c5;
alphas(i, 3) = -c2/c5;
betas(i, 1) = (xs(j) * c4 - ys(j) * c3)/c5;
betas(i, 2) = -c4/c5;
betas(i, 3) = c3/c5;
end
xmin = min(xs);
ymin = min(ys);
xmax = max(xs);
ymax = max(ys);
w = int32(xmax - xmin + 1);
h = int32(ymax - ymin + 1);
mask = zeros(h, w);
triX = zeros(h, w);
shape = [xs, ys];
for i = 1:h
for j = 1:w
currTri = findTriangle(double([double(j)-1+xmin, double(i)-1+ymin])', triangulation, shape);
if(currTri ~= -1)
triX(i, j) = currTri - 1;
mask(i, j) = 1;
else
triX(i, j) = -1;
end
end
end
npix = sum(sum(mask));
end
function [tri] = findTriangle(point, tris, controlPoints)
numTris = size(tris, 1);
tri = -1;
for i=1:numTris
if(PointInTriangle(point, controlPoints(tris(i,1),:)', controlPoints(tris(i,2),:)', controlPoints(tris(i,3),:)'))
tri = i;
break;
end
end
end
function inTriangle = PointInTriangle(point, v1, v2, v3)
inTriangle = SameSide(point, v1,v2,v3) && SameSide(point, v2,v1,v3) && SameSide(point, v3,v1,v2);
end
function sameSide = SameSide(toTest,v1,v2,v3)
x0 = toTest(1);
y0 = toTest(2);
x1 = v1(1);
x2 = v2(1);
x3 = v3(1);
y1 = v1(2);
y2 = v2(2);
y3 = v3(2);
x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2);
y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2);
if(x*y >= 0)
sameSide = 1;
else
sameSide = 0;
end
% cross1 = cross( v3 - v2, toTest - v2);
% cross2 = cross( v3 - v2, v1 - v2);
%
% sameSide = (cross1 * cross2') >= 0;
end

17
pkg/OpenFace/matlab_version/face_validation/paw_helpers/Remap.m Normal file
View File

@@ -0,0 +1,17 @@
function [ outputTexture ] = Remap( inputTexture, mapX, mapY )
%REMAP Summary of this function goes here
% Detailed explanation goes here
outputTexture = zeros(size(mapX));
[X,Y] = meshgrid(0:size(inputTexture,2)-1,0:size(inputTexture,1)-1);
inds = find(mapX ~= -1);
xSources = mapX(inds);
ySources = mapY(inds);
Z = interp2(X, Y, double(inputTexture), xSources, ySources);
outputTexture(inds) = Z;
end

49
pkg/OpenFace/matlab_version/face_validation/paw_helpers/WarpRegion.m Normal file
View File

@@ -0,0 +1,49 @@
function [ mapX, mapY ] = WarpRegion( xmin, ymin, mask, triX, coeffs )
%WARPREGION Summary of this function goes here
% Detailed explanation goes here
%%
[h, w] = size(mask);
mapX = zeros(size(mask));
mapY = zeros(size(mask));
ys = [1:h]' * ones(1, w) + ymin - 1;
xs = ([1:w]' * ones(1, h))' + xmin - 1;
for t=0:size(coeffs,1)-1
trimap = triX == t;
a = coeffs(t+1,:);
xo = a(1) + a(2) * xs + a(3) * ys;
mapX(trimap) = xo(trimap);
yo = a(4) + a(5) * xs + a(6) * ys;
mapY(trimap) = yo(trimap);
end
mapX(~mask) = -1;
mapY(~mask) = -1;
%%
% [h, w] = size(mask);
% mapX_2 = zeros(size(mask));
% mapY_2 = zeros(size(mask));
%
% ys = [1:h]' * ones(1, w) + ymin - 1;
% xs = ([1:w]' * ones(1, h))' + xmin - 1;
%
% ys = ys(:);
% xs = xs(:);
%
% xos = coeffs(1,:) + bsxfun(@times, coeffs(2,:), xs) + bsxfun(@times, coeffs(3,:), ys);
% yos = coeffs(4,:) + bsxfun(@times, coeffs(5,:), xs) + bsxfun(@times, coeffs(6,:), ys);
%
% maps = repmat(trimap(:),1, size(coeffs,1));
% maps = repmat
end

12
pkg/OpenFace/matlab_version/face_validation/quantizeContinuous.m Normal file
View File

@@ -0,0 +1,12 @@
function [quantized] = quantizeContinuous(values, min_v, max_v, num_bins)
step_size = (max_v - min_v) / num_bins;
bin_centres = min_v + step_size/2 : step_size : max_v;
bin_centres = repmat(bin_centres, numel(values),1);
values = repmat(values, 1, num_bins);
[~,quantized] = min(abs(values - bin_centres)');
quantized = quantized';
end

21
pkg/OpenFace/matlab_version/face_validation/readme.txt Normal file
View File

@@ -0,0 +1,21 @@
Code for facial landmark detection validation (knowing if detection succeeded), to be use for face tracking in videos so as not to do face detection every frame.
To create the training data run:
Create_data_train.m and Create_data_test.m
The data generation code requires you to have the patch expert training data (Menpo, Multi-PIE and 300W data, not included) for positive examples, and inriaperson dataset for negative samples (not included as well).
To train Convolutional Neural Network based face landmark validation model use:
Train_face_checker_cnn.m
This will produce trained/face_checker_cnn_*.mat and trained/face_checker_cnn_*.txt files that can be used in C++ and matlab versions of OpenFace for face validation. Old versions can also be found in trained folder (they are simpler CNN models trained on smaller datasets).
This will also produces tris*.txt files that can be used in the C++ version of the OpenFace,just place it in the lib\local\LandmarkDetector\model\detection_validation folder and edit the appropriate "main_*.txt" files.
The code uses piece-wise affine warping to a neutral shape with an CNN regressor for error estimation (see http://www.cl.cam.ac.uk/~tb346/ThesisFinal.pdf Section 4.6.2 for a very similar model but with SVR regressor)
Dependencies:
- vlfeat-0.9.20 and extract it in the current directory (http://www.vlfeat.org/download.html)
- MatConvNet from http://www.vlfeat.org/matconvnet/ (tested with version 1.0-beta24), and install following the instructions
Change the setup.m to match the locations of vlfeat and MatConvNet

25
pkg/OpenFace/matlab_version/face_validation/setup.m Normal file
View File

@@ -0,0 +1,25 @@
function setup(varargin)
%run vlfeat-0.9.20/toolbox/vl_setup ;
run C:\matconvnet\matconvnet-1.0-beta25\matlab/vl_setupnn ;
addpath C:\matconvnet\matconvnet-1.0-beta25\examples ;
opts.useGpu = false ;
opts.verbose = false ;
opts = vl_argparse(opts, varargin) ;
try
vl_nnconv(single(1),single(1),[]) ;
catch
warning('VL_NNCONV() does not seem to be compiled. Trying to compile it now.') ;
vl_compilenn('enableGpu', opts.useGpu, 'verbose', opts.verbose) ;
end
if opts.useGpu
try
vl_nnconv(gpuArray(single(1)),gpuArray(single(1)),[]) ;
catch
vl_compilenn('enableGpu', opts.useGpu, 'verbose', opts.verbose) ;
warning('GPU support does not seem to be compiled in MatConvNet. Trying to compile it now') ;
end
end

210
pkg/OpenFace/matlab_version/face_validation/train_CNN_model.m Normal file
View File

@@ -0,0 +1,210 @@
function train_CNN_model(varargin)
setup('useGPU', true);
tic
% Load the data
root_loc = 'D:\Datasets\detection_validation';
location = [root_loc, '/prep_data/'];
faceCheckersLoc_train = dir([location 'face_validator_train_*']);
faceCheckersLoc_test = dir([location 'face_validator_test_*']);
corrs_all = [];
rmses_all = [];
faceCheckers = struct;
% As we will be training a classifier that will act as a regressor,
% binarize it
num_bins = 30;
for i=1:numel(faceCheckersLoc_train)
load([location faceCheckersLoc_train(i).name]);
% set a max value to the error
errors(errors > 3) = 3;
errors_train = quantizeContinuous(errors, 0, 3, num_bins);
examples_train = single(examples);
clear examples
mean_ex = mean(examples_train);
std_ex = std(examples_train);
std_ex = std_ex / 256;
examples_train = bsxfun(@times, bsxfun(@minus, examples_train, mean_ex), 1./std_ex);
num_examples_train = size(examples_train,1);
% Add rows and columns untill we have 60 x 60 image
% keep adding rows
while(size(mask,1) < 60)
mask = cat(1, mask, false(1, size(mask,2)));
triX = cat(1, triX, -ones(1, size(mask,2)));
end
% keep adding cols
while(size(mask,2) < 60)
mask = cat(2, mask, false(size(mask,1),1));
triX = cat(2, triX, -ones(size(triX,1),1));
end
examples_r = single(zeros(size(mask, 1), size(mask, 2), num_examples_train));
img_curr = zeros(size(mask));
for e=1:num_examples_train
img_curr(mask) = examples_train(e,:);
examples_r(:, :, e) = img_curr;
end
imdb.images.data = examples_r;
clear examples_r
imdb.images.label = errors_train';
imdb.images.id = 1:numel(errors_train);
% Split data for training and validation (20%)
imdb.images.set = ones(1, numel(errors_train));
imdb.images.set(round(4*end/5):end) = 2;
% Visualize some of the data
figure(10) ; clf ; colormap gray ;
subplot(1,2,1) ;
pos_samples = squeeze(imdb.images.data(:,:,imdb.images.label==1));
vl_imarraysc(pos_samples(:,:,1:20)) ;
axis image off ;
title('Positive training data') ;
subplot(1,2,2) ;
neg_samples = squeeze(imdb.images.data(:,:,imdb.images.label>5));
vl_imarraysc(neg_samples(:,:,1:20)) ;
axis image off ;
title('Negative training data') ;
net = initializeFaceCNN(num_bins) ;
trainOpts.batchSize = 100 ;
trainOpts.numEpochs = 20;
trainOpts.continue = true ;
trainOpts.gpus = [1] ;
trainOpts.learningRate = 0.001 ;
trainOpts.expDir = ['trained/intermediate/face_validator_' num2str(i)];
trainOpts.errorFunction = 'regression';
trainOpts = vl_argparse(trainOpts, varargin);
% Call training function in MatConvNet
[net,info] = cnn_train_reg(net, imdb, @getBatch, trainOpts) ;
% Move the CNN back to the CPU if it was trained on the GPU
if numel(trainOpts.gpus) > 0
net = vl_simplenn_move(net, 'cpu') ;
end
% Save the result for later use
net.layers(end) = [] ;
net.imageMean = mean_ex ;
net.imageStd = std_ex;
% save('data/face-experiment/facecnn.mat', '-struct', 'net') ;
% Evaluate on the test data, also evaluate using correlation and RMSE
load([location faceCheckersLoc_test(i).name]);
% set a max value to the error
errors(errors > 3) = 3;
errors_test = errors;
errors_q = quantizeContinuous(errors, 0, 3, num_bins);
errors_test_q = errors_q;
examples_test = single(examples);
examples_test = bsxfun(@times, bsxfun(@minus, examples_test, net.imageMean), 1./net.imageStd);
num_examples_test = size(examples_test,1);
% keep adding rows
while(size(mask,1) < 60)
mask = cat(1, mask, false(1, size(mask,2)));
triX = cat(1, triX, -ones(1, size(mask,2)));
end
% keep adding cols
while(size(mask,2) < 60)
mask = cat(2, mask, false(size(mask,1),1));
triX = cat(2, triX, -ones(size(triX,1),1));
end
examples_r = single(zeros(size(mask, 1), size(mask, 2), num_examples_test));
img_curr = zeros(size(mask));
for e=1:num_examples_test
img_curr(mask) = examples_test(e,:);
examples_r(:, :, e) = img_curr;
end
examples_test = zeros(size(examples_r,1), size(examples_r,2), 1, size(examples_r,3));
examples_test(:) = single(examples_r(:));
ids = 1:99:num_examples_test;
errors_all_test = zeros(num_examples_test,1);
for k=1:numel(ids)-1
examples_test_sm = single(examples_test(:,:,:,ids(k):ids(k+1)-1));
res = vl_simplenn(net, examples_test_sm, [], []);
res = gather(res(end).x);
[~,res] = sort(res, 3, 'descend') ;
res = squeeze(res);
res = res(1,:);
errors_test_rec = unQuantizeContinuous(res, 0, 3, num_bins)';
errors_all_test(ids(k):ids(k+1)-1) = errors_test_rec;
end
errors_all_test = errors_all_test(1:ids(end));
corrs_test = corr(errors_all_test, errors_test(1:numel(errors_all_test)));
rmse_test = sqrt(mean((errors_all_test-errors_test(1:numel(errors_all_test))).^2));
corrs_all = cat(1, corrs_all, corrs_test);
rmses_all = cat(1, rmses_all, rmse_test);
net.corrs = corrs_test;
net.rmse = rmse_test;
save([trainOpts.expDir, '/facecnn.mat'], '-struct', 'net');
% The CNN
faceCheckers(i).cnn = net;
% The orientation
faceCheckers(i).centres = centres;
% The info for preprocessing
faceCheckers(i).mask = mask;
faceCheckers(i).nPix = nPix;
faceCheckers(i).minX = minX;
faceCheckers(i).minY = minY;
faceCheckers(i).destination = shape(:,1:2);
tri = load([location faceCheckersLoc_train(i).name], 'triangulation');
faceCheckers(i).triangulation = tri.triangulation;
faceCheckers(i).triX = triX;
faceCheckers(i).mask = mask;
faceCheckers(i).alphas = alphas;
faceCheckers(i).betas = betas;
faceCheckers(i).mean_ex = mean_ex;
faceCheckers(i).std_ex = std_ex;
WriteOutFaceCheckersCNNbinary("trained/validator_cnn.txt", faceCheckers);
end
save('trained/faceCheckers.mat', 'faceCheckers', 'corrs_all', 'rmses_all');
end
% --------------------------------------------------------------------
function [im, labels] = getBatch(imdb, batch)
% --------------------------------------------------------------------
im = imdb.images.data(:,:,batch) ;
im = reshape(im, 60, 60, 1, []) ;
labels = imdb.images.label(1,batch) ;
end

BIN
pkg/OpenFace/matlab_version/face_validation/trained/faceCheckers.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/face_validation/trained/face_check_cnn_66.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/face_validation/trained/face_check_cnn_66.txt Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/face_validation/trained/face_check_cnn_68.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/face_validation/trained/face_check_cnn_68.txt Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/face_validation/trained/tris_66.mat Normal file
View File

Binary file not shown.

518
pkg/OpenFace/matlab_version/face_validation/trained/tris_66.txt Normal file
View File

@@ -0,0 +1,518 @@
# Number of triangulations
7
# Triangulation 1
# triangulation
91
3
4
23 20 21
23 21 22
36 0 1
15 16 45
17 0 36
16 26 45
18 17 37
26 25 44
37 17 36
45 26 44
19 18 38
25 24 43
38 18 37
44 25 43
20 19 38
24 23 43
21 20 39
23 22 42
39 20 38
43 23 42
22 21 27
27 21 39
22 27 42
27 28 42
28 27 39
42 28 47
28 39 40
36 1 41
46 15 45
41 1 2
14 15 46
29 28 40
28 29 47
41 2 40
47 14 46
2 29 40
29 14 47
29 2 3
13 14 29
30 29 31
35 29 30
29 3 31
35 13 29
33 30 32
34 30 33
32 30 31
35 30 34
31 3 4
12 13 35
4 5 48
11 12 54
5 6 48
10 11 54
48 6 59
55 10 54
6 7 59
9 10 55
59 7 58
56 9 55
58 8 57
57 8 56
7 8 58
8 9 56
31 4 48
54 12 35
49 31 48
54 35 53
50 31 49
53 35 52
32 31 50
35 34 52
33 32 50
34 33 52
51 33 50
52 33 51
49 48 60
50 49 60
61 50 60
51 50 61
52 51 61
61 62 52
53 52 62
54 53 62
55 54 63
56 55 63
64 56 63
57 56 64
64 65 57
58 57 65
59 58 65
65 48 59
# Triangulation 2
# triangulation
90
3
4
23 20 21
23 21 22
36 0 1
15 16 45
17 0 36
16 26 45
18 17 37
26 25 44
37 17 36
45 26 44
19 18 38
25 24 43
38 18 37
44 25 43
20 19 38
24 23 43
21 20 39
23 22 42
39 20 38
43 23 42
22 21 27
27 21 39
22 27 42
27 28 42
28 27 39
42 28 47
28 39 40
36 1 41
46 15 45
41 1 2
14 15 46
29 28 40
28 29 47
41 2 40
47 14 46
2 29 40
29 14 47
29 2 3
13 14 29
30 29 31
29 3 31
35 13 29
33 30 32
34 30 33
32 30 31
35 30 34
31 3 4
12 13 35
4 5 48
11 12 54
5 6 48
10 11 54
48 6 59
55 10 54
6 7 59
9 10 55
59 7 58
56 9 55
58 8 57
57 8 56
7 8 58
8 9 56
31 4 48
54 12 35
49 31 48
54 35 53
50 31 49
53 35 52
32 31 50
35 34 52
33 32 50
34 33 52
51 33 50
52 33 51
49 48 60
50 49 60
61 50 60
51 50 61
52 51 61
61 62 52
53 52 62
54 53 62
55 54 63
56 55 63
64 56 63
57 56 64
64 65 57
58 57 65
59 58 65
65 48 59
# Triangulation 3
# triangulation
77
3
4
23 20 21
23 21 22
36 0 1
17 0 36
18 17 37
26 25 44
37 17 36
45 26 44
19 18 38
25 24 43
38 18 37
44 25 43
20 19 38
24 23 43
21 20 39
23 22 42
39 20 38
43 23 42
22 21 27
27 21 39
22 27 42
27 28 42
28 27 39
42 28 47
28 39 40
36 1 41
46 15 45
41 1 2
29 28 40
28 29 47
41 2 40
47 14 46
2 29 40
29 2 3
30 29 31
29 3 31
33 30 32
34 30 33
32 30 31
35 30 34
31 3 4
4 5 48
5 6 48
48 6 59
6 7 59
59 7 58
56 9 55
58 8 57
57 8 56
7 8 58
8 9 56
31 4 48
49 31 48
50 31 49
53 35 52
32 31 50
35 34 52
33 32 50
34 33 52
51 33 50
52 33 51
49 48 60
50 49 60
61 50 60
51 50 61
52 51 61
61 62 52
53 52 62
54 53 62
55 54 63
56 55 63
64 56 63
57 56 64
64 65 57
58 57 65
59 58 65
65 48 59
# Triangulation 4
# triangulation
28
3
4
36 0 1
17 0 36
18 17 37
37 17 36
19 18 38
38 18 37
20 19 38
36 1 41
41 1 2
29 28 40
41 2 40
2 29 40
29 2 3
30 29 31
29 3 31
31 3 4
4 5 48
5 6 48
48 6 59
6 7 59
59 7 58
58 8 57
7 8 58
31 4 48
49 31 48
50 31 49
51 33 50
51 50 61
# Triangulation 5
# triangulation
90
3
4
23 20 21
23 21 22
36 0 1
15 16 45
17 0 36
16 26 45
18 17 37
26 25 44
37 17 36
45 26 44
19 18 38
25 24 43
38 18 37
44 25 43
20 19 38
24 23 43
21 20 39
23 22 42
39 20 38
43 23 42
22 21 27
27 21 39
22 27 42
27 28 42
28 27 39
42 28 47
28 39 40
36 1 41
46 15 45
41 1 2
14 15 46
29 28 40
28 29 47
41 2 40
47 14 46
2 29 40
29 14 47
29 2 3
13 14 29
35 29 30
29 3 31
35 13 29
33 30 32
34 30 33
32 30 31
35 30 34
31 3 4
12 13 35
4 5 48
11 12 54
5 6 48
10 11 54
48 6 59
55 10 54
6 7 59
9 10 55
59 7 58
56 9 55
58 8 57
57 8 56
7 8 58
8 9 56
31 4 48
54 12 35
49 31 48
54 35 53
50 31 49
53 35 52
32 31 50
35 34 52
33 32 50
34 33 52
51 33 50
52 33 51
49 48 60
50 49 60
61 50 60
51 50 61
52 51 61
61 62 52
53 52 62
54 53 62
55 54 63
56 55 63
64 56 63
57 56 64
64 65 57
58 57 65
59 58 65
65 48 59
# Triangulation 6
# triangulation
77
3
4
23 20 21
23 21 22
15 16 45
16 26 45
18 17 37
26 25 44
37 17 36
45 26 44
19 18 38
25 24 43
38 18 37
44 25 43
20 19 38
24 23 43
21 20 39
23 22 42
39 20 38
43 23 42
22 21 27
27 21 39
22 27 42
27 28 42
28 27 39
42 28 47
28 39 40
36 1 41
46 15 45
14 15 46
29 28 40
28 29 47
41 2 40
47 14 46
29 14 47
13 14 29
35 29 30
35 13 29
33 30 32
34 30 33
32 30 31
35 30 34
12 13 35
11 12 54
10 11 54
55 10 54
9 10 55
59 7 58
56 9 55
58 8 57
57 8 56
7 8 58
8 9 56
54 12 35
54 35 53
50 31 49
53 35 52
32 31 50
35 34 52
33 32 50
34 33 52
51 33 50
52 33 51
49 48 60
50 49 60
61 50 60
51 50 61
52 51 61
61 62 52
53 52 62
54 53 62
55 54 63
56 55 63
64 56 63
57 56 64
64 65 57
58 57 65
59 58 65
65 48 59
# Triangulation 7
# triangulation
28
3
4
15 16 45
16 26 45
26 25 44
45 26 44
25 24 43
44 25 43
24 23 43
46 15 45
14 15 46
28 29 47
47 14 46
29 14 47
13 14 29
35 29 30
35 13 29
12 13 35
11 12 54
10 11 54
55 10 54
9 10 55
56 9 55
57 8 56
8 9 56
54 12 35
54 35 53
53 35 52
52 33 51
52 51 61

BIN
pkg/OpenFace/matlab_version/face_validation/trained/tris_68.mat Normal file
View File

Binary file not shown.

558
pkg/OpenFace/matlab_version/face_validation/trained/tris_68.txt Normal file
View File

@@ -0,0 +1,558 @@
# Number of triangulations
7
# Triangulation 1
# triangulation
97
3
4
58 67 59
60 49 48
58 59 6
34 52 35
44 45 25
39 40 29
37 18 36
27 42 22
23 44 24
41 36 1
50 62 51
56 65 66
57 58 7
64 53 63
28 27 39
52 34 51
54 14 35
29 42 28
19 20 24
35 15 46
37 19 18
36 0 1
18 17 36
37 20 19
38 20 37
21 20 38
21 38 39
24 44 25
30 34 35
21 39 27
28 42 27
39 29 28
29 30 35
31 30 29
30 33 34
31 29 40
36 17 0
41 31 40
31 32 30
31 41 1
49 31 48
48 2 3
67 60 59
4 48 3
5 48 4
6 59 5
59 48 5
60 48 59
7 58 6
61 49 60
58 66 67
31 2 48
31 50 32
1 2 31
61 50 49
52 62 63
50 31 49
34 33 51
51 62 52
32 50 51
50 61 62
63 53 52
54 55 11
57 8 9
66 58 57
8 57 7
56 57 9
66 57 56
10 56 9
55 56 10
53 54 35
53 35 52
12 54 11
55 10 11
65 56 55
64 55 54
65 55 64
54 53 64
12 13 54
14 54 13
15 35 14
47 35 46
33 32 51
30 32 33
29 35 47
15 45 46
22 21 27
20 21 23
43 23 22
29 47 42
23 21 22
24 20 23
22 42 43
23 43 44
45 16 26
15 16 45
25 45 26
# Triangulation 2
# triangulation
97
3
4
58 67 59
60 49 48
58 59 6
34 52 35
44 45 25
39 40 29
37 18 36
27 42 22
23 44 24
41 36 1
50 62 51
56 65 66
57 58 7
64 53 63
28 27 39
52 34 51
54 14 35
29 42 28
19 20 24
35 15 46
37 19 18
36 0 1
18 17 36
37 20 19
38 20 37
21 20 38
21 38 39
24 44 25
30 34 35
21 39 27
28 42 27
39 29 28
29 30 35
31 30 29
30 33 34
31 29 40
36 17 0
41 31 40
31 32 30
31 41 1
49 31 48
48 2 3
67 60 59
4 48 3
5 48 4
6 59 5
59 48 5
60 48 59
7 58 6
61 49 60
58 66 67
31 2 48
31 50 32
1 2 31
61 50 49
52 62 63
50 31 49
34 33 51
51 62 52
32 50 51
50 61 62
63 53 52
54 55 11
57 8 9
66 58 57
8 57 7
56 57 9
66 57 56
10 56 9
55 56 10
53 54 35
53 35 52
12 54 11
55 10 11
65 56 55
64 55 54
65 55 64
54 53 64
12 13 54
14 54 13
15 35 14
47 35 46
33 32 51
30 32 33
29 35 47
15 45 46
22 21 27
20 21 23
43 23 22
29 47 42
23 21 22
24 20 23
22 42 43
23 43 44
45 16 26
15 16 45
25 45 26
# Triangulation 3
# triangulation
84
3
4
58 67 59
60 49 48
58 59 6
34 52 35
44 45 25
39 40 29
37 18 36
27 42 22
23 44 24
41 36 1
50 62 51
56 65 66
57 58 7
64 53 63
28 27 39
52 34 51
29 42 28
19 20 24
37 19 18
36 0 1
18 17 36
37 20 19
38 20 37
21 20 38
21 38 39
24 44 25
30 34 35
21 39 27
28 42 27
39 29 28
31 30 29
30 33 34
31 29 40
36 17 0
41 31 40
31 32 30
31 41 1
49 31 48
48 2 3
67 60 59
4 48 3
5 48 4
6 59 5
59 48 5
60 48 59
7 58 6
61 49 60
58 66 67
31 2 48
31 50 32
1 2 31
61 50 49
52 62 63
50 31 49
34 33 51
51 62 52
32 50 51
50 61 62
63 53 52
57 8 9
66 58 57
8 57 7
56 57 9
66 57 56
55 56 10
53 35 52
65 56 55
64 55 54
65 55 64
54 53 64
47 35 46
33 32 51
30 32 33
29 35 47
15 45 46
22 21 27
20 21 23
43 23 22
29 47 42
23 21 22
24 20 23
22 42 43
23 43 44
25 45 26
# Triangulation 4
# triangulation
31
3
4
58 67 59
58 59 6
37 18 36
41 36 1
50 62 51
57 58 7
37 19 18
36 0 1
18 17 36
37 20 19
38 20 37
31 30 29
31 29 40
36 17 0
41 31 40
31 41 1
49 31 48
48 2 3
4 48 3
5 48 4
6 59 5
59 48 5
7 58 6
58 66 67
31 2 48
1 2 31
61 50 49
50 31 49
50 61 62
66 58 57
8 57 7
# Triangulation 5
# triangulation
97
3
4
58 67 59
60 49 48
58 59 6
34 52 35
44 45 25
39 40 29
37 18 36
27 42 22
23 44 24
41 36 1
50 62 51
56 65 66
57 58 7
64 53 63
28 27 39
52 34 51
54 14 35
29 42 28
19 20 24
35 15 46
37 19 18
36 0 1
18 17 36
37 20 19
38 20 37
21 20 38
21 38 39
24 44 25
30 34 35
21 39 27
28 42 27
39 29 28
29 30 35
31 30 29
30 33 34
31 29 40
36 17 0
41 31 40
31 32 30
31 41 1
49 31 48
48 2 3
67 60 59
4 48 3
5 48 4
6 59 5
59 48 5
60 48 59
7 58 6
61 49 60
58 66 67
31 2 48
31 50 32
1 2 31
61 50 49
52 62 63
50 31 49
34 33 51
51 62 52
32 50 51
50 61 62
63 53 52
54 55 11
57 8 9
66 58 57
8 57 7
56 57 9
66 57 56
10 56 9
55 56 10
53 54 35
53 35 52
12 54 11
55 10 11
65 56 55
64 55 54
65 55 64
54 53 64
12 13 54
14 54 13
15 35 14
47 35 46
33 32 51
30 32 33
29 35 47
15 45 46
22 21 27
20 21 23
43 23 22
29 47 42
23 21 22
24 20 23
22 42 43
23 43 44
45 16 26
15 16 45
25 45 26
# Triangulation 6
# triangulation
84
3
4
58 67 59
60 49 48
58 59 6
34 52 35
44 45 25
39 40 29
37 18 36
27 42 22
23 44 24
41 36 1
50 62 51
56 65 66
57 58 7
64 53 63
28 27 39
52 34 51
54 14 35
29 42 28
19 20 24
35 15 46
37 19 18
18 17 36
37 20 19
38 20 37
21 20 38
21 38 39
24 44 25
30 34 35
21 39 27
28 42 27
39 29 28
29 30 35
30 33 34
31 29 40
41 31 40
31 32 30
67 60 59
60 48 59
61 49 60
58 66 67
31 50 32
61 50 49
52 62 63
50 31 49
34 33 51
51 62 52
32 50 51
50 61 62
63 53 52
54 55 11
57 8 9
66 58 57
8 57 7
56 57 9
66 57 56
10 56 9
55 56 10
53 54 35
53 35 52
12 54 11
55 10 11
65 56 55
64 55 54
65 55 64
54 53 64
12 13 54
14 54 13
15 35 14
47 35 46
33 32 51
30 32 33
29 35 47
15 45 46
22 21 27
20 21 23
43 23 22
29 47 42
23 21 22
24 20 23
22 42 43
23 43 44
45 16 26
15 16 45
25 45 26
# Triangulation 7
# triangulation
31
3
4
44 45 25
23 44 24
56 65 66
54 14 35
35 15 46
24 44 25
29 30 35
52 62 63
51 62 52
63 53 52
54 55 11
57 8 9
56 57 9
66 57 56
10 56 9
55 56 10
53 54 35
53 35 52
12 54 11
55 10 11
65 56 55
12 13 54
14 54 13
15 35 14
47 35 46
29 35 47
15 45 46
23 43 44
45 16 26
15 16 45
25 45 26

BIN
pkg/OpenFace/matlab_version/face_validation/trained/validator_cnn.txt Normal file
View File

Binary file not shown.

6
pkg/OpenFace/matlab_version/face_validation/unQuantizeContinuous.m Normal file
View File

@@ -0,0 +1,6 @@
function [unquantized] = unQuantizeContinuous(values, min_v, max_v, num_bins)
step_size = (max_v - min_v) / num_bins;
unquantized = min_v + step_size/2 + (values-1) * step_size;
end