open source pkg v1

pkg/OpenFace/matlab_version/experiments_JANUS/Collect_JANUS_imgs.m

@@ -0,0 +1,78 @@
+function [images, detections, labels] = Collect_JANUS_imgs(root_test_data)
+
+     dataset_loc = [root_test_data, '/'];
+
+    landmarkLabels = dir([dataset_loc '\*.pts']);
+
+    num_samples = 5;
+    
+    num_imgs = size(landmarkLabels,1);
+
+    images = struct;
+    labels = zeros(num_imgs * num_samples, 68, 2);    
+
+    num_landmarks = 68;
+    
+    rng(0);
+    detections = [];
+
+    ind = 1;
+    
+    for imgs = 1:num_imgs
+
+        [~,name,~] = fileparts(landmarkLabels(imgs).name);
+
+        landmarks = dlmread([dataset_loc, landmarkLabels(imgs).name], ' ', [3,0,num_landmarks+2,1]);
+        tmp = landmarks(:,1);
+        landmarks(:,1) = landmarks(:,2);
+        landmarks(:,2) = tmp;
+                
+        landmarks(landmarks == - 1) = 0;
+        
+        % Swap around some points
+        eyes = landmarks(18:29,:);
+        brow_l = landmarks(30:34,:);
+        nose = landmarks(35:43,:);
+        brow_r = landmarks(44:48,:);
+        
+        landmarks(18:22,:) = brow_l;
+        landmarks(23:27,:) = brow_r;
+        landmarks(28:36,:) = nose;
+        landmarks(37:48,:) = eyes;
+        
+        non_occluded = landmarks(:,1) ~= 0;
+        for s=1:num_samples
+            % Create detections based on 300W noise level - just shifting in x, y
+            % and adding some scaling        
+            scale_x = 1 + randn(1,1) * 0.01;
+            scale_y = 1 + randn(1,1) * 0.015;
+
+            tx = 0.06 * randn(1,1);
+            ty = 0.05 * randn(1,1);
+
+            width_gt = (max(landmarks(non_occluded,1)) - min(landmarks(non_occluded,1)));
+            height_gt = (max(landmarks(non_occluded,2)) - min(landmarks(non_occluded,2)));
+
+            width = width_gt * scale_x;
+            height = height_gt * scale_y;
+
+            x = min(landmarks(non_occluded,1)) + width * tx;
+            y = min(landmarks(non_occluded,2)) + height * ty;
+
+            % Add like 5 more
+            detections = cat(1, detections, [x, y, x+ width, y+height]);
+
+            images(ind).img = [dataset_loc,  name];
+            labels(ind,:,:) = landmarks;
+            
+%             imshow(imread(images(ind).img));
+%             hold on;
+%             rectangle('position', [x, y, width, height]);
+%             hold off;            
+
+            ind = ind + 1;
+        end
+
+    end
+        
+end

pkg/OpenFace/matlab_version/experiments_JANUS/Construct_error_table.m

@@ -0,0 +1,35 @@
+% Run this in order to construct the results table for the CE-CLM paper
+Extract_table_results_68;
+
+file_out = fopen('results/IJB-FL_68.txt', 'w');
+
+fprintf(file_out, 'Errors with outline (68 points)\n');
+fprintf(file_out, '------------------------------\n');
+fprintf(file_out, 'Method\tfrontal\tprofile\n');
+fprintf(file_out, 'CLNF\t%.2f\t%.2f\n', median(clnf_error_frontal)*100, median(clnf_error_profile)*100);
+fprintf(file_out, 'CFAN\t%.2f\t%.2f\n', median(cfan_error_frontal)*100, median(cfan_error_profile)*100);
+fprintf(file_out, 'CFSS\t%.2f\t%.2f\n', median(cfss_error_frontal)*100, median(cfss_error_profile)*100);
+fprintf(file_out, 'TCDCN\t%.2f\t%.2f\n', median(tcdcn_error_frontal)*100, median(tcdcn_error_profile)*100);
+fprintf(file_out, '3DDFA\t%.2f\t%.2f\n', median(error_3ddfa_frontal)*100, median(error_3ddfa_profile)*100);
+fprintf(file_out, '------------------------------\n');
+fprintf(file_out, 'CE-CLM\t%.2f\t%.2f\n', median(ceclm_error_frontal)*100, median(ceclm_error_profile)*100);
+fclose(file_out);
+
+Extract_table_results_49;
+file_out = fopen('results/IJB-FL_49.txt', 'w');
+
+fprintf(file_out, 'Errors without outline (49 points)\n');
+fprintf(file_out, '------------------------------\n');
+fprintf(file_out, 'Method\tfrontal\tprofile\n');
+fprintf(file_out, 'CLNF\t%.2f\t%.2f\n', median(clnf_error_frontal)*100, median(clnf_error_profile)*100);
+fprintf(file_out, 'SDM \t%.2f\t%.2f\n', median(sdm_error_frontal)*100, median(sdm_error_profile)*100);
+fprintf(file_out, 'CFAN\t%.2f\t%.2f\n', median(cfan_error_frontal)*100, median(cfan_error_profile)*100);
+fprintf(file_out, 'DRMF\t%.2f\t%.2f\n', median(drmf_error_frontal)*100, median(drmf_error_profile)*100);
+fprintf(file_out, 'CFSS\t%.2f\t%.2f\n', median(cfss_error_frontal)*100, median(cfss_error_profile)*100);
+fprintf(file_out, 'PO-CR\t%.2f\t%.2f\n', median(pocr_error_frontal)*100, median(pocr_error_profile)*100);
+fprintf(file_out, 'TCDCN\t%.2f\t%.2f\n', median(tcdcn_error_frontal)*100, median(tcdcn_error_profile)*100);
+fprintf(file_out, '3DDFA\t%.2f\t%.2f\n', median(error_3ddfa_frontal)*100, median(error_3ddfa_profile)*100);
+fprintf(file_out, '------------------------------\n');
+fprintf(file_out, 'CE-CLM\t%.2f\t%.2f\n', median(ceclm_error_frontal)*100, median(ceclm_error_profile)*100);
+
+fclose(file_out);

pkg/OpenFace/matlab_version/experiments_JANUS/Display_ceclm_results_49.m

@@ -0,0 +1,53 @@
+clear
+
+Extract_table_results_49
+%% 
+scrsz = get(0,'ScreenSize');
+figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
+
+set(figure1,'Units','Inches');
+pos = get(figure1,'Position');
+set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
+
+% Create axes
+axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
+
+line_width = 6;
+hold on;
+
+[error_x, error_y] = cummErrorCurve(ceclm_error);
+plot(error_x, error_y, 'r','DisplayName', 'OpenFace 2.0', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(clnf_error);
+plot(error_x, error_y, 'DisplayName', 'OpenFace', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfss_error);
+plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(sdm_error);
+plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(pocr_error);
+plot(error_x, error_y,'DisplayName', 'PO-CR', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfan_error);
+plot(error_x, error_y,'DisplayName', 'CFAN', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(tcdcn_error);
+plot(error_x, error_y,'DisplayName', 'TCDCN', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(error_3ddfa);
+plot(error_x, error_y,'DisplayName', '3DDFA', 'LineWidth',line_width);
+
+set(gca,'xtick',[0:0.02:0.10])
+xlim([0.02,0.10]);
+xlabel('Size normalised MSE','FontName','Helvetica');
+ylabel('Proportion of images','FontName','Helvetica');
+grid on
+% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica');
+
+leg = legend('show', 'Location', 'SouthEast');
+set(leg,'FontSize',40)
+
+print -dpdf results/Janus-no-outline.pdf
+print -dpng results/Janus-no-outline.png

pkg/OpenFace/matlab_version/experiments_JANUS/Display_ceclm_results_68.m

@@ -0,0 +1,47 @@
+clear
+
+Extract_table_results_68
+%% 
+scrsz = get(0,'ScreenSize');
+figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
+
+set(figure1,'Units','Inches');
+pos = get(figure1,'Position');
+set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
+
+% Create axes
+axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
+
+line_width = 6;
+hold on;
+
+[error_x, error_y] = cummErrorCurve(ceclm_error);
+plot(error_x, error_y, 'r','DisplayName', 'OpenFace 2.0', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(clnf_error);
+plot(error_x, error_y, 'DisplayName', 'OpenFace', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfss_error);
+plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfan_error);
+plot(error_x, error_y,'DisplayName', 'CFAN', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(tcdcn_error);
+plot(error_x, error_y,'DisplayName', 'TCDCN', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(error_3ddfa);
+plot(error_x, error_y,'DisplayName', '3DDFA', 'LineWidth',line_width);
+
+set(gca,'xtick',[0:0.02:0.10])
+xlim([0.02,0.10]);
+xlabel('Size normalised MSE','FontName','Helvetica');
+ylabel('Proportion of images','FontName','Helvetica');
+grid on
+% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica');
+
+leg = legend('show', 'Location', 'SouthEast');
+set(leg,'FontSize',40)
+
+print -dpdf results/Janus-full.pdf
+print -dpng results/Janus-full.png

pkg/OpenFace/matlab_version/experiments_JANUS/Extract_table_results_49.m

@@ -0,0 +1,69 @@
+clear
+
+load('./results/results_clnf_wild.mat');
+
+[clnf_error, ~,~,frontal_ids] = compute_error_small( experiment.labels,  experiment.shapes-1.0);
+clnf_error_frontal = clnf_error(frontal_ids);
+clnf_error_profile = clnf_error(~frontal_ids);
+
+load('./results/results_ceclm_general.mat');
+
+[ceclm_error,~,~,frontal_ids] = compute_error_small( experiment.labels,  experiment.shapes-1.0);
+ceclm_error_frontal = ceclm_error(frontal_ids);
+ceclm_error_profile = ceclm_error(~frontal_ids);
+labels = experiment.labels;
+
+load('results/CFAN_JANUS.mat');
+
+[cfan_error,~,~,frontal_ids] = compute_error_small( labels_all,  shapes_all-1.0);
+cfan_error_frontal = cfan_error(frontal_ids);
+cfan_error_profile = cfan_error(~frontal_ids);
+
+load('results/JANUS_3DDFA.mat');
+
+[error_3ddfa,~,~,frontal_ids] = compute_error_small( labels_all,  shapes-1.0);
+error_3ddfa_frontal = error_3ddfa(frontal_ids);
+error_3ddfa_profile = error_3ddfa(~frontal_ids);
+
+load('results/JANUS_pocr.mat');
+
+[pocr_error,~,~,frontal_ids] = compute_error_small( labels,  experiments.shapes-0.5);
+pocr_error_frontal = pocr_error(frontal_ids);
+pocr_error_profile = pocr_error(~frontal_ids);
+
+load('results/JANUS_chehra.mat');
+
+[drmf_error,~,~,frontal_ids] = compute_error_small( labels,  shapes);
+drmf_error_frontal = drmf_error(frontal_ids);
+drmf_error_profile = drmf_error(~frontal_ids);
+
+load('results/JANUS_sdm.mat');
+
+[sdm_error,~,~,frontal_ids] = compute_error_small( labels,  experiments.shapes);
+sdm_error_frontal = sdm_error(frontal_ids);
+sdm_error_profile = sdm_error(~frontal_ids);
+
+load('results/JANUS-CFSS.mat');
+shapes = zeros(68,2, size(estimatedPose,1));
+
+for i = 1:size(shapes, 3)
+    shapes(:,1,i) = estimatedPose(i,1:68);
+    shapes(:,2,i) = estimatedPose(i,69:end);
+end
+
+[cfss_error,~,~,frontal_ids] = compute_error_small( experiments.labels, shapes-1.0);
+cfss_error_frontal = cfss_error(frontal_ids);
+cfss_error_profile = cfss_error(~frontal_ids);
+
+load('results/tcdcn_JANUS.mat');
+shapes_c = shapes;
+shapes = zeros(68,2, numel(shapes_c));
+
+for i = 1:size(shapes, 3)
+    shapes(:,1,i) = shapes_c{i}(:,1);
+    shapes(:,2,i) = shapes_c{i}(:,2);
+end
+
+[tcdcn_error,~,~,frontal_ids] = compute_error_small( experiments.labels, shapes-1.0);
+tcdcn_error_frontal = tcdcn_error(frontal_ids);
+tcdcn_error_profile = tcdcn_error(~frontal_ids);

pkg/OpenFace/matlab_version/experiments_JANUS/Extract_table_results_68.m

@@ -0,0 +1,51 @@
+clear
+
+load('./results/results_clnf_wild.mat');
+
+[clnf_error, ~,~,frontal_ids] = compute_error( experiment.labels,  experiment.shapes-1.0);
+clnf_error_frontal = clnf_error(frontal_ids);
+clnf_error_profile = clnf_error(~frontal_ids);
+
+load('./results/results_ceclm_general.mat');
+
+[ceclm_error,~,~,frontal_ids] = compute_error( experiment.labels,  experiment.shapes-1.0);
+ceclm_error_frontal = ceclm_error(frontal_ids);
+ceclm_error_profile = ceclm_error(~frontal_ids);
+
+load('results/CFAN_JANUS.mat');
+
+[cfan_error,~,~,frontal_ids] = compute_error( labels_all,  shapes_all-1.0);
+cfan_error_frontal = cfan_error(frontal_ids);
+cfan_error_profile = cfan_error(~frontal_ids);
+
+load('results/JANUS_3DDFA.mat');
+
+[error_3ddfa,~,~,frontal_ids] = compute_error( labels_all,  shapes-1.0);
+error_3ddfa_frontal = error_3ddfa(frontal_ids);
+error_3ddfa_profile = error_3ddfa(~frontal_ids);
+
+load('results/JANUS-CFSS.mat');
+shapes = zeros(68,2, size(estimatedPose,1));
+
+for i = 1:size(shapes, 3)
+    shapes(:,1,i) = estimatedPose(i,1:68);
+    shapes(:,2,i) = estimatedPose(i,69:end);
+end
+
+[cfss_error,~,~,frontal_ids] = compute_error( experiment.labels, shapes-1.0);
+cfss_error_frontal = cfss_error(frontal_ids);
+cfss_error_profile = cfss_error(~frontal_ids);
+
+%
+load('results/tcdcn_JANUS.mat');
+shapes_c = shapes;
+shapes = zeros(68,2, numel(shapes_c));
+
+for i = 1:size(shapes, 3)
+    shapes(:,1,i) = shapes_c{i}(:,1);
+    shapes(:,2,i) = shapes_c{i}(:,2);
+end
+
+[tcdcn_error,~,~,frontal_ids] = compute_error( experiment.labels, shapes-1);
+tcdcn_error_frontal = tcdcn_error(frontal_ids);
+tcdcn_error_profile = tcdcn_error(~frontal_ids);

pkg/OpenFace/matlab_version/experiments_JANUS/Script_CECLM_general.m

@@ -0,0 +1,97 @@
+function Script_CECLM_general()
+
+addpath(genpath('../'));
+
+% Replace this with the location of the IJB-FL data location
+root_test_data = 'F:\Dropbox\janus_labeled';
+[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
+
+%% loading the CE-CLM model and parameters   
+[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
+views = views * pi/180;                                                                                     
+
+%% Setup recording
+experiment.params = clmParams;
+
+num_points = numel(pdm.M)/3;
+
+shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+lhoods = zeros(numel(images),1);
+all_lmark_lhoods = zeros(num_points, numel(images));
+all_views_used = zeros(numel(images),1);
+
+% Change if you want to visualize the outputs
+verbose = false;
+output_img = false;
+
+if(output_img)
+    output_root = './ceclm_gen_out/';
+    if(~exist(output_root, 'dir'))
+        mkdir(output_root);
+    end
+end
+if(verbose)
+    f = figure;
+end
+
+
+%% Fitting the model to the provided images
+
+tic
+for i=1:numel(images)
+
+    image = imread(images(i).img);
+    image_orig = image;
+    
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+
+    bbox = detections(i,:);                  
+    
+    [shape,~,~,lhood,lmark_lhood,view_used] =...
+        Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params);
+
+    all_lmark_lhoods(:,i) = lmark_lhood;
+    all_views_used(i) = view_used;
+
+    shapes_all(:,:,i) = shape;
+    labels_all(:,:,i) = labels(i,:,:);
+
+    if(mod(i, 200)==0)
+        fprintf('%d done\n', i );
+    end
+
+    lhoods(i) = lhood;
+
+    if(output_img)
+        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
+        DrawFaceOnImg(image_orig, shape, sprintf('%s/%s%d.jpg', output_root, 'fit', i), bbox, v_points);
+    end
+    
+    if(verbose)
+        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
+        DrawFaceOnFig(image_orig, shape, bbox, v_points);
+    end
+end
+toc
+
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
+experiment.lhoods = lhoods;
+experiment.shapes = shapes_all;
+experiment.labels = labels_all;
+experiment.all_lmark_lhoods = all_lmark_lhoods;
+experiment.all_views_used = all_views_used;
+
+fprintf('Done: mean normed error %.3f median normed error %.4f\n', ...
+    mean(experiment.errors_normed), median(experiment.errors_normed));
+
+%%
+output_results = 'results/results_ceclm_general.mat';
+save(output_results, 'experiment');
+    
+end

pkg/OpenFace/matlab_version/experiments_JANUS/Script_CECLM_menpo.m

@@ -0,0 +1,95 @@
+function Script_CECLM_menpo()
+
+addpath(genpath('../'));
+
+% Replace this with the location of the IJB-FL data location
+root_test_data = 'F:\Dropbox\janus_labeled';
+[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
+
+%% loading the CE-CLM model and parameters   
+[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
+views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
+views = views * pi/180;                                                                                     
+
+%% Setup recording
+experiment.params = clmParams;
+
+num_points = numel(pdm.M)/3;
+
+shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+lhoods = zeros(numel(images),1);
+all_lmark_lhoods = zeros(num_points, numel(images));
+all_views_used = zeros(numel(images),1);
+
+% Change if you want to visualize the outputs
+verbose = false;
+output_img = false;
+
+if(output_img)
+    output_root = './ceclm_menpo_out/';
+    if(~exist(output_root, 'dir'))
+        mkdir(output_root);
+    end
+end
+if(verbose)
+    f = figure;
+end
+
+
+%% Fitting the model to the provided images
+
+tic
+for i=1:numel(images)
+
+    image = imread(images(i).img);
+    image_orig = image;
+    
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+
+    bbox = detections(i,:);                  
+    
+    [shape,~,~,lhood,lmark_lhood,view_used] =...
+        Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params);
+
+    all_lmark_lhoods(:,i) = lmark_lhood;
+    all_views_used(i) = view_used;
+
+    shapes_all(:,:,i) = shape;
+    labels_all(:,:,i) = labels(i,:,:);
+
+    if(mod(i, 200)==0)
+        fprintf('%d done\n', i );
+    end
+
+    lhoods(i) = lhood;
+
+    if(output_img)
+        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
+        DrawFaceOnImg(image_orig, shape, sprintf('%s/%s%d.jpg', output_root, 'fit', i), bbox, v_points);
+    end
+    
+    if(verbose)
+        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
+        DrawFaceOnFig(image_orig, shape, bbox, v_points);
+    end
+end
+toc
+
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
+experiment.lhoods = lhoods;
+experiment.shapes = shapes_all;
+experiment.labels = labels_all;
+experiment.all_lmark_lhoods = all_lmark_lhoods;
+experiment.all_views_used = all_views_used;
+
+fprintf('Done: mean normed error %.3f median normed error %.4f\n', ...
+    mean(experiment.errors_normed), median(experiment.errors_normed));
+
+%%
+output_results = 'results/results_ceclm_menpo.mat';
+save(output_results, 'experiment');
+    
+end

pkg/OpenFace/matlab_version/experiments_JANUS/Script_CLNF_general.m

@@ -0,0 +1,72 @@
+function Script_CLNF_general()
+
+addpath(genpath('../'));
+
+% Replace this with the location of the IJB-FL data location
+root_test_data = 'F:\Dropbox\janus_labeled';
+[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
+
+%% loading the patch experts
+   
+[ patches, pdm, clmParams ] = Load_CLNF_general();
+views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
+views = views * pi/180;                                                                                     
+
+[ patches_51, pdm_51, clmParams_51, inds_full, inds_inner ] = Load_CLNF_inner();
+
+shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+lhoods = zeros(numel(images),1);
+
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+verbose = false; % set to true to visualise the fitting
+
+tic
+for i=1:numel(images)
+
+    image = imread(images(i).img);
+    image_orig = image;
+
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+
+    bbox = detections(i,:);                  
+
+    [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views);
+             
+    % Perform inner landmark fitting now
+    [shape, shape_inner] = Fitting_from_bb_hierarch(image, pdm, pdm_51, patches_51, clmParams_51, shape, inds_full, inds_inner);
+
+    shapes_all(:,:,i) = shape;                    
+
+    labels_all(:,:,i) = labels(i,:,:);
+
+    if(mod(i, 200)==0)
+        fprintf('%d done\n', i );
+    end
+
+    lhoods(i) = lhood;
+
+    if(verbose)
+        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
+        DrawFaceOnFig(image_orig, shape, bbox, v_points);
+    end
+
+end
+toc
+
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
+experiment.lhoods = lhoods;
+experiment.shapes = shapes_all;
+experiment.labels = labels_all;
+
+fprintf('Done: mean normed error %.3f median normed error %.4f\n', ...
+    mean(experiment.errors_normed), median(experiment.errors_normed));
+
+%%
+output_results = 'results/results_clnf_general.mat';
+save(output_results, 'experiment');
+    
+end

pkg/OpenFace/matlab_version/experiments_JANUS/Script_CLNF_wild.m

@@ -0,0 +1,69 @@
+function Script_CLNF_wild()
+
+addpath(genpath('../'));
+
+% Replace this with the location of the IJB-FL data location
+root_test_data = 'F:\Dropbox\janus_labeled';
+[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
+
+%% loading the patch experts
+   
+[ patches, pdm, clmParams ] = Load_CLNF_wild();
+views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40];
+views = views * pi/180;                                                                                     
+
+% for recording purposes
+experiment.params = clmParams;
+
+shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
+lhoods = zeros(numel(images),1);
+
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+verbose = false; % set to true to visualise the fitting
+
+tic
+for i=1:numel(images)
+
+    image = imread(images(i).img);
+    image_orig = image;
+
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+
+    bbox = detections(i,:);                  
+
+    [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views);
+
+    shapes_all(:,:,i) = shape;
+    labels_all(:,:,i) = labels(i,:,:);
+
+    if(mod(i, 200)==0)
+        fprintf('%d done\n', i );
+    end
+
+    lhoods(i) = lhood;
+
+    if(verbose)
+        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
+        DrawFaceOnFig(image_orig, shape, bbox, v_points);
+    end
+
+end
+toc
+
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
+experiment.lhoods = lhoods;
+experiment.shapes = shapes_all;
+experiment.labels = labels_all;
+
+fprintf('Done: mean normed error %.3f median normed error %.4f\n', ...
+    mean(experiment.errors_normed), median(experiment.errors_normed));
+
+%%
+output_results = 'results/results_clnf_wild.mat';
+save(output_results, 'experiment');
+    
+end

pkg/OpenFace/matlab_version/experiments_JANUS/compute_error.m

@@ -0,0 +1,48 @@
+function [ error_per_image, err_pp, err_pp_dim,frontal_ids ] = compute_error( ground_truth_all, detected_points_all )
+%compute_error
+%   compute the average point-to-point Euclidean error normalized by the
+%   inter-ocular distance (measured as the Euclidean distance between the
+%   outer corners of the eyes)
+%
+%   Inputs:
+%          grounth_truth_all, size: num_of_points x 2 x num_of_images
+%          detected_points_all, size: num_of_points x 2 x num_of_images
+%   Output:
+%          error_per_image, size: num_of_images x 1
+
+
+num_of_images = size(ground_truth_all,3);
+num_of_points = size(ground_truth_all,1);
+
+error_per_image = zeros(num_of_images,1);
+err_pp = zeros(num_of_images, num_of_points);
+err_pp_dim = zeros(num_of_images, num_of_points, 2);
+frontal_ids = true(num_of_images,1);
+
+for i =1:num_of_images
+    detected_points      = detected_points_all(:,:,i);
+    ground_truth_points  = ground_truth_all(:,:,i);
+    visible = ground_truth_points(:,1) > 0;
+    if(sum(visible) < 55)
+        frontal_ids(i) = false;
+    end
+    
+    normalization_x = max(ground_truth_points(visible,1)) - min(ground_truth_points(visible,1));
+    normalization_y = max(ground_truth_points(visible,2)) - min(ground_truth_points(visible,2));
+    normalization = (normalization_x + normalization_y)/2;
+    
+    sum_c=0;
+    for j=1:num_of_points
+        if(visible(j))
+            sum_c = sum_c+norm(detected_points(j,:)-ground_truth_points(j,:));
+            err_pp(i,j) = norm(detected_points(j,:)-ground_truth_points(j,:));
+            err_pp_dim(i,j,1) = detected_points(j,1)-ground_truth_points(j,1);
+            err_pp_dim(i,j,2) = detected_points(j,2)-ground_truth_points(j,2);
+        end
+    end
+    error_per_image(i) = sum_c/(sum(visible)*normalization);
+    err_pp(i,:) = err_pp(i,:) ./ normalization;
+    err_pp_dim(i,:) = err_pp_dim(i,:) ./ normalization;
+end
+
+end

pkg/OpenFace/matlab_version/experiments_JANUS/compute_error_small.m

@@ -0,0 +1,61 @@
+function [ error_per_image, err_pp, err_pp_dim, frontal_ids ] = compute_error_small( ground_truth_all, detected_points_all )
+%compute_error
+%   compute the average point-to-point Euclidean error normalized by the
+%   inter-ocular distance (measured as the Euclidean distance between the
+%   outer corners of the eyes)
+%
+%   Inputs:
+%          grounth_truth_all, size: num_of_points x 2 x num_of_images
+%          detected_points_all, size: num_of_points x 2 x num_of_images
+%   Output:
+%          error_per_image, size: num_of_images x 1
+
+
+num_of_images = size(ground_truth_all,3);
+ground_truth_all_orig = ground_truth_all;
+
+ground_truth_all = ground_truth_all([1:60,62:64,66:end],:,:);
+ground_truth_all = ground_truth_all(18:end,:,:);
+
+num_of_points = size(ground_truth_all,1);
+frontal_ids = true(num_of_images,1);
+
+if(size(detected_points_all,1) == 68)
+    detected_points_all = detected_points_all([1:60,62:64,66:end],:,:);
+    detected_points_all = detected_points_all(18:end,:,:);
+end
+
+error_per_image = zeros(num_of_images,1);
+err_pp = zeros(num_of_images, num_of_points);
+err_pp_dim = zeros(num_of_images, num_of_points, 2);
+
+for i =1:num_of_images
+    detected_points      = detected_points_all(:,:,i);
+    ground_truth_points  = ground_truth_all(:,:,i);
+    visible = ground_truth_points(:,1) > 0;
+    ground_truth_points_orig = ground_truth_all_orig(:,:,i);
+    visible_orig = ground_truth_points_orig(:,1) > 0;
+    
+    if(sum(visible_orig) < 55)
+        frontal_ids(i) = false;
+    end
+    
+    normalization_x = max(ground_truth_points_orig(visible_orig,1)) - min(ground_truth_points_orig(visible_orig,1));
+    normalization_y = max(ground_truth_points_orig(visible_orig,2)) - min(ground_truth_points_orig(visible_orig,2));
+    normalization = (normalization_x + normalization_y)/2;
+    
+    sum_c=0;
+    for j=1:num_of_points
+        if(visible(j))
+            sum_c = sum_c+norm(detected_points(j,:)-ground_truth_points(j,:));
+            err_pp(i,j) = norm(detected_points(j,:)-ground_truth_points(j,:));
+            err_pp_dim(i,j,1) = detected_points(j,1)-ground_truth_points(j,1);
+            err_pp_dim(i,j,2) = detected_points(j,2)-ground_truth_points(j,2);
+        end
+    end
+    error_per_image(i) = sum_c/(sum(visible)*normalization);
+    err_pp(i,:) = err_pp(i,:) ./ normalization;
+    err_pp_dim(i,:) = err_pp_dim(i,:) ./ normalization;
+end
+
+end

pkg/OpenFace/matlab_version/experiments_JANUS/cummErrorCurve.m

@@ -0,0 +1,19 @@
+function [x, y] = cummErrorCurve( errorVec )
+%CUMMERRORCURVE Summary of this function goes here
+%   Detailed explanation goes here
+
+    
+    spacing = 0.001;       
+    
+    sampling = [0:spacing:max(errorVec)];
+
+    x = sampling;
+    
+    y = zeros(numel(sampling,1));
+    
+    for i=1:numel(sampling)
+    
+        y(i) = sum(errorVec < sampling(i)) / numel(errorVec);
+    end
+end
+

pkg/OpenFace/matlab_version/experiments_JANUS/readme.txt

@@ -0,0 +1,22 @@
+Scripts for evaluating CE-CLM, and CLNF methods on the IJB-FL dataset - http://www.bmva.org/bmvc/2016/papers/paper095/paper095.pdf
+
+To run the models on the data using models trained on 300W:
+- Script_CLNF_wild.m
+
+To run the models on the data using models trained on 300W + MultiPIE:
+- Script_CECLM_general.m
+- Script_CLNF_general.m
+
+To run CE-CLM model trained on 300W + MultiPIE + Menpo:
+- Script CECLM_menpo.m
+
+To visualize the results of the models against baselines (using inner face markup - 49, and full face markup - 68):
+Display_ceclm_results_49.m
+Display_ceclm_results_68.m
+
+To construct the error table:
+Construct_error_table.m
+
+The 300W trained CLNF model performs best so it is used in comparisons, for CE-CLM 300W + MultiPIE works better so it is used. We do not use Menpo trained CE-CLM model for comparison as other models have not been trained on the more extensive Menpo data, but it is the best performing model.
+
+For a fair comparison all the models and baselines are initialized using the bounding boxes generated by adding noise to ground truth bounding boxes.

pkg/OpenFace/matlab_version/experiments_JANUS/results/IJB-FL_49.txt

@@ -0,0 +1,13 @@
+Errors without outline (49 points)
+------------------------------
+Method	frontal	profile
+CLNF	3.82	6.22
+SDM 	3.93	30.79
+CFAN	4.37	22.92
+DRMF	4.55	25.52
+CFSS	3.57	6.79
+PO-CR	3.73	21.22
+TCDCN	4.53	9.09
+3DDFA	4.85	6.48
+------------------------------
+CE-CLM	3.45	5.13

pkg/OpenFace/matlab_version/experiments_JANUS/results/IJB-FL_68.txt

@@ -0,0 +1,10 @@
+Errors with outline (68 points)
+------------------------------
+Method	frontal	profile
+CLNF	4.39	7.73
+CFAN	4.89	20.26
+CFSS	4.16	7.66
+TCDCN	4.91	9.26
+3DDFA	5.90	8.14
+------------------------------
+CE-CLM	4.08	6.30