open source pkg v1

pkg/OpenFace/matlab_version/models/Load_CECLM_Patch_Experts.m

@@ -0,0 +1,44 @@
+function [patches] = Load_CECLM_Patch_Experts( col_patch_dir, col_patch_file)
+%LOAD_PATCH_EXPERTS Summary of this function goes here
+%   Detailed explanation goes here
+   
+    colourPatchFiles = dir([col_patch_dir col_patch_file]);
+    patches = [];
+
+    % load all of the pathes
+    for i=1:numel(colourPatchFiles)
+        
+        load([col_patch_dir, colourPatchFiles(i).name]);
+
+        patch = struct;
+        patch.centers = centers;
+        patch.trainingScale = trainingScale;
+        patch.visibilities = visiIndex; 
+        patch.patch_experts = patch_experts.patch_experts;
+        patch.correlations = patch_experts.correlations;
+        patch.rms_errors = patch_experts.rms_errors;
+        patch.modalities = patch_experts.types;
+        patch.multi_modal_types = patch_experts.types;
+
+        patch.type = 'CEN';
+
+        % Knowing what normalisation was performed during training is
+        % important for fitting
+        patch.normalisationOptionsCol = normalisationOptions;
+
+        if(i==1)
+            patches = patch;
+        else
+            patches = [patches; patch];
+        end
+                     
+    end
+    
+    if(isempty(patches))
+       fprintf("Could not find CEN patch experts, for instructions of how to download them, please visit - https://github.com/TadasBaltrusaitis/OpenFace/wiki/Model-download\n"); 
+       ME = MException('cenExperts:modelError', 'Could not find CEN model at location %s, see - https://github.com/TadasBaltrusaitis/OpenFace/wiki/Model-download',[col_patch_dir col_patch_file]);       
+       throw(ME);
+    end
+    
+end
+

pkg/OpenFace/matlab_version/models/Load_CECLM_general.m

@@ -0,0 +1,38 @@
+function [patches, pdm, clmParams, early_term_params] = Load_CECLM_general()
+%LOAD_PATCH_EXPERTS Summary of this function goes here
+%   Detailed explanation goes here
+   
+    % Load the patch experts/local detectors
+    [patches] = Load_CECLM_Patch_Experts( '../models/cen/', 'cen_patches_*_general.mat');
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    clmParams.regFactor = 0.9*[35, 27, 20, 20];
+    clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+    clmParams.numPatchIters = 4;
+    
+    % As the orientations are not equally reliable reweigh them
+    load('../models/cen/cen_general_mapping.mat');    
+end
+

pkg/OpenFace/matlab_version/models/Load_CECLM_menpo.m

@@ -0,0 +1,38 @@
+function [patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo()
+%LOAD_PATCH_EXPERTS Summary of this function goes here
+%   Detailed explanation goes here
+   
+    % Load the patch experts/local detectors
+    [patches] = Load_CECLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat');
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    clmParams.regFactor = 0.9*[35, 27, 20, 20];
+    clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+    clmParams.numPatchIters = 4;
+    
+    % As the orientations are not equally reliable reweigh them
+    load('../models/cen/cen_menpo_mapping.mat');    
+end
+

pkg/OpenFace/matlab_version/models/Load_CLM_general.m

@@ -0,0 +1,33 @@
+function [ patches, pdm, clmParams ] = Load_CLM_general()
+%LOAD_CLM_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    [patches] = Load_Patch_Experts( '../models/wild/', 'svr_patches_*_wild.mat', [], [], clmParams);
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    % the default model parameters to use
+    clmParams.regFactor = [35, 27, 20, 5];
+    clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.75]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 12.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+
+end
+

pkg/OpenFace/matlab_version/models/Load_CLM_params_eye.m

@@ -0,0 +1,38 @@
+function [ clmParams, pdm_right, pdm_left ] = Load_CLM_params_eye() 
+%LOAD_CLM_PARAMS_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams.window_size = [15,15; 13,13;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/hierarch_pdm/pdm_6_r_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_right = struct;
+    pdm_right.M = double(M);
+    pdm_right.E = double(E);
+    pdm_right.V = double(V);
+
+    pdmLoc = ['../models/hierarch_pdm/pdm_6_l_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_left = struct;
+    pdm_left.M = double(M);
+    pdm_left.E = double(E);
+    pdm_left.V = double(V);
+    
+    % the default model parameters to use
+    clmParams.regFactor = 0.1;               
+    clmParams.sigmaMeanShift = 2;
+    clmParams.tikhonov_factor = 0;
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 5;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.tikhonov_factor = 0;
+end
+

pkg/OpenFace/matlab_version/models/Load_CLM_params_eye_28.m

@@ -0,0 +1,38 @@
+function [ clmParams, pdm_right, pdm_left ] = Load_CLM_params_eye_28() 
+%LOAD_CLM_PARAMS_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams.window_size = [17,17; 15,15; 13,13;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/hierarch_pdm/pdm_28_r_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_right = struct;
+    pdm_right.M = double(M);
+    pdm_right.E = double(E);
+    pdm_right.V = double(V);
+
+    pdmLoc = ['../models/hierarch_pdm/pdm_28_l_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_left = struct;
+    pdm_left.M = double(M);
+    pdm_left.E = double(E);
+    pdm_left.V = double(V);
+    
+    % the default model parameters to use
+    clmParams.regFactor = 2.0;               
+    clmParams.sigmaMeanShift = 1.5;
+    clmParams.tikhonov_factor = 0;
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 0;
+    clmParams.tikhonov_factor = 0;
+end
+

pkg/OpenFace/matlab_version/models/Load_CLM_params_inner.m

@@ -0,0 +1,28 @@
+function [ clmParams, pdm_mouth] = Load_CLM_params_inner() 
+%LOAD_CLM_PARAMS_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams.window_size = [19,19;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/hierarch_pdm/pdm_51_inner.mat'];
+
+    load(pdmLoc);
+
+    pdm_mouth = struct;
+    pdm_mouth.M = double(M);
+    pdm_mouth.E = double(E);
+    pdm_mouth.V = double(V);
+    
+    % the default model parameters to use
+    clmParams.regFactor = 2.5;               
+    clmParams.sigmaMeanShift = 1.75;
+    clmParams.tikhonov_factor = 2.5;
+    
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 5;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+
+end

pkg/OpenFace/matlab_version/models/Load_CLM_params_vid.m

@@ -0,0 +1,29 @@
+function [ clmParams, pdm ] = Load_CLM_params_vid() 
+%LOAD_CLM_PARAMS_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams.window_size = [21,21; 19,19; 17,17;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from Multi-PIE data
+    pdmLoc = ['../models/pdm/pdm_68_multi_pie.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    % the default model parameters to use
+    clmParams.regFactor = 25;               
+    clmParams.sigmaMeanShift = 2;
+    clmParams.tikhonov_factor = 5;
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+
+end
+

pkg/OpenFace/matlab_version/models/Load_CLM_params_wild.m

@@ -0,0 +1,29 @@
+function [ clmParams, pdm ] = Load_CLM_params_wild() 
+%LOAD_CLM_PARAMS_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams.window_size = [25,25; 25,25; 25,25;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    % the default model parameters to use
+    clmParams.regFactor = 25;               
+    clmParams.sigmaMeanShift = 2;
+    clmParams.tikhonov_factor = 5;
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+
+end
+

pkg/OpenFace/matlab_version/models/Load_CLM_wild.m

@@ -0,0 +1,33 @@
+function [ patches, pdm, clmParams ] = Load_CLM_wild()
+%LOAD_CLM_WILD Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21;];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    [patches] = Load_Patch_Experts( '../models/general/', 'svr_patches_*_general.mat', [], [], clmParams);
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    % the default model parameters to use
+    clmParams.regFactor = [35, 27, 20, 5];
+    clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.75]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 12.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+
+end
+

pkg/OpenFace/matlab_version/models/Load_CLNF_general.m

@@ -0,0 +1,33 @@
+function [ patches, pdm, clmParams ] = Load_CLNF_general()
+%LOAD_CLNF_GENERAL Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    [patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams);
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    % the default model parameters to use
+    clmParams.regFactor = [35, 27, 20];
+    clmParams.sigmaMeanShift = [1.25, 1.375, 1.5]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+
+end
+

pkg/OpenFace/matlab_version/models/Load_CLNF_inner.m

@@ -0,0 +1,40 @@
+function [ patches, pdm, clmParams, inds_full, inds_inner ] = Load_CLNF_inner()
+%LOAD_CLNF_Inner Summary of this function goes here
+%   Detailed explanation goes here
+
+    % 51 landmark model of a face (a subset of 68 landmark model without
+    % the face outline)
+    clmParams = struct;
+    clmParams.window_size = [19,19];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/hierarch_pdm/pdm_51_inner.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+    
+    % the default model parameters to use
+    clmParams.regFactor = 2.5;               
+    clmParams.sigmaMeanShift = 1.75;
+    clmParams.tikhonov_factor = 2.5;
+    
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 5;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    
+    [patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*general_no_out.mat', [], [], clmParams);
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+
+    % Corresponding indices between the 68 point and the 51 point version
+    inds_full = 18:68;
+    inds_inner = 1:51;
+    
+end
+

pkg/OpenFace/matlab_version/models/Load_CLNF_wild.m

@@ -0,0 +1,34 @@
+function [ patches, pdm, clmParams ] = Load_CLNF_wild()
+%LOAD_CLNF_GENERAL Summary of this function goes here
+%   Detailed explanation goes here
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21;21,21];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    [patches] = Load_Patch_Experts( '../models/wild/', 'ccnf_patches_*_wild.mat', [], [], clmParams);
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    % the default model parameters to use
+    clmParams.regFactor = [35, 27, 20, 20];
+    clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+
+
+end
+

pkg/OpenFace/matlab_version/models/Load_Patch_Experts.m

@@ -0,0 +1,223 @@
+function [patches] = Load_Patch_Experts( col_patch_dir, col_patch_file, depth_patch_dir, depth_patch_file, clmParams)
+%LOAD_PATCH_EXPERTS Summary of this function goes here
+%   Detailed explanation goes here
+   
+    colourPatchFiles = dir([col_patch_dir col_patch_file]);
+    
+    % load all of the pathes
+    for i=1:numel(colourPatchFiles)
+        
+        load([col_patch_dir, colourPatchFiles(i).name]);
+
+        % determine patch type (slightly hacky but SVR patch experts don't
+        % CCNF ratio variable)
+        if(isfield(normalisationOptions, 'ccnf_ratio'))
+            
+            patch = struct;
+            patch.centers = centers;
+            patch.trainingScale = trainingScale;
+            patch.visibilities = visiIndex; 
+            patch.patch_experts = patch_experts.patch_experts;
+            patch.correlations = patch_experts.correlations;
+            patch.rms_errors = patch_experts.rms_errors;
+            patch.modalities = patch_experts.types;
+            patch.multi_modal_types = patch_experts.types;
+            
+            patch.type = 'CCNF';
+            
+            % Knowing what normalisation was performed during training is
+            % important for fitting
+            patch.normalisationOptionsCol = normalisationOptions;
+
+            % As the similarity inverses will depend on the window size
+            % and alphas and betas, but not actual data, precalculate
+            % them here
+            
+            % create the similarity inverses
+            window_sizes = unique(clmParams.window_size(:));
+            
+            for s=1:size(window_sizes,1)
+                               
+                for view=1:size(patch.patch_experts,1)
+                    for lmk=1:size(patch.patch_experts,2)
+                        if(visiIndex(view, lmk))
+                            num_modalities = size(patch.patch_experts{view,lmk}.thetas,3);
+                            
+                            num_hls = size(patch.patch_experts{view,lmk}.thetas,1);
+                            
+                            patchSize = sqrt(size( patch.patch_experts{view,lmk}.thetas,2)-1);
+                            patchSize = [patchSize, patchSize];
+
+                            % normalisation so that patch expert can be
+                            % applied using convolution
+                            w = cell(num_hls, num_modalities);
+                            norm_w = cell(num_hls, num_modalities);
+
+                            for hl=1:num_hls
+                                for p=1:num_modalities
+
+                                    w_c = patch.patch_experts{view,lmk}.thetas(hl, 2:end, p);
+                                    norm_w_c = norm(w_c);
+                                    w_c = w_c/norm(w_c);
+                                    w_c = reshape(w_c, patchSize);
+                                    w{hl,p} = w_c;
+                                    norm_w{hl,p} = norm_w_c;
+                                end
+                            end
+
+                            patch.patch_experts{view,lmk}.w = w;
+                            patch.patch_experts{view,lmk}.norm_w = norm_w;
+                            
+                            similarities = {};
+                            response_side_length = window_sizes(s) - 11 + 1;
+                            for st=1:size(patch.patch_experts{view,lmk}.similarity_types, 1)
+                                type_sim = patch.patch_experts{view,lmk}.similarity_types{st};
+                                neighFn = @(x) similarity_neighbor_grid(x, response_side_length(1), type_sim);
+                                similarities = [similarities; {neighFn}];
+                            end
+                            
+                            sparsities = {};
+
+                            for st=1:size(patch.patch_experts{view,lmk}.sparsity_types, 1)
+                                spFn = @(x) sparsity_grid(x, response_side_length(1), patch.patch_experts{view,lmk}.sparsity_types(st,1), patch.patch_experts{view,lmk}.sparsity_types(st,2));
+                                sparsities = [sparsities; {spFn}];
+                            end
+                                                            
+                            region_length = response_side_length^2;
+                                
+                            [ ~, ~, PrecalcQ2sFlat, ~ ] = CalculateSimilarities_sparsity( 1, {zeros(region_length,1)}, similarities, sparsities);
+
+                            PrecalcQ2flat = PrecalcQ2sFlat{1};
+                            
+                            SigmaInv = CalcSigmaCCNFflat(patch.patch_experts{view,lmk}.alphas, patch.patch_experts{view,lmk}.betas, region_length, PrecalcQ2flat, eye(region_length), zeros(region_length));
+                            if(s == 1)
+                                patch.patch_experts{view,lmk}.Sigma = {inv(SigmaInv)};
+                            else
+                                patch.patch_experts{view,lmk}.Sigma = cat(1, patch.patch_experts{view,lmk}.Sigma, {inv(SigmaInv)});
+                            end
+                        end
+                    end
+                end
+            end
+            
+            if(i==1)
+                patches = patch;
+            else
+                patches = [patches; patch];
+            end
+            
+            
+        else
+            % creating the struct
+            patch = struct;            
+            
+            patch.centers = centers;
+            patch.trainingScale = trainingScale;
+            patch.visibilities = visiIndex;
+            patch.type = 'SVR';
+            
+            % if the normalisation options present in the loaded patch use
+            % them, if not we use default values
+            if(exist('normalisationOptions', 'var'))
+                patch.normalisationOptionsCol = normalisationOptions;
+            else
+                patch.normalisationOptionsCol = normalisationColour;
+            end
+
+            % default for depth uses normalised-cross-corr per each sample,
+            % rather than in a broad area
+            if(~isfield(patch.normalisationOptionsCol, 'useZeroMeanPerPatch'))
+                patch.normalisationOptionsCol.zscore = 0;
+                patch.normalisationOptionsCol.useNormalisedCrossCorr = 1;
+                patch.normalisationOptionsCol.useZeroMeanPerPatch = 1;
+            end
+
+            % Multi-modal section
+            patch.patch_experts = cell(size(visiIndex,1), size(visiIndex,2));
+
+            for view=1:size(visiIndex,1)
+                for landmark=1:size(visiIndex,2)
+                    patch.patch_experts{view, landmark} = struct;
+                    multi_modal_types = patch_experts.types;
+                    for p=1:numel(patch_experts.types)
+                        patch.patch_experts{view, landmark}(p).type = patch_experts.types(p);
+                        patch.patch_experts{view, landmark}(p).correlations = patch_experts.correlations(p);
+                        patch.patch_experts{view, landmark}(p).rms_errors = patch_experts.rms_errors(p);
+
+                        patch.patch_experts{view,landmark}(p).scaling = patch_experts.patch_experts{p}(view, landmark, 1);
+                        patch.patch_experts{view,landmark}(p).bias = patch_experts.patch_experts{p}(view, landmark, 2);
+
+                        patch.patch_experts{view,landmark}(p).w = reshape(patch_experts.patch_experts{p}(view, landmark, 3:end),11,11);                        
+
+                    end
+                end
+            end
+
+            patch.multi_modal_types = multi_modal_types;
+            
+            if(i==1)
+                patches = patch;
+            else
+                patches = [patches; patch];
+            end
+        
+        end
+        clear 'normalisationOptions, centers, trainingScale, visiIndex, correlations, rmsErrors';
+    end
+
+    
+    if(~isempty(depth_patch_file))
+        
+    depthPatchFiles = dir([depth_patch_dir depth_patch_file]);
+       
+    % load all of the depth patches
+    for i=1:numel(depthPatchFiles)
+        
+        load([depth_patch_dir, depthPatchFiles(i).name]);
+        
+        % assuming that same view seen in depth and intensity
+            
+        % if the normalisation options present in the loaded patch use
+        % them, if not we use default values
+        if(exist('normalisationOptions', 'var'))
+            patches(i).normalisationOptionsDepth = normalisationOptions;
+        else
+            patches(i).normalisationOptionsDepth = normalisationDepth;
+        end
+
+        % Multi-modal section
+        patches(i).patch_experts_depth = cell(size(visiIndex,1), size(visiIndex,2));
+
+        for view=1:size(visiIndex,1)
+            for landmark=1:size(visiIndex,2)
+                patches(i).patch_experts_depth{view, landmark} = struct;
+                if(exist('patch_m', 'var'))
+                    multi_modal_types = patch_m.types;
+                    for p=1:numel(patch_m.types)
+                        patches(i).patch_experts_depth{view, landmark}(p).type = patch_m.types(p);
+                        patches(i).patch_experts_depth{view, landmark}(p).correlations = patch_m.correlations(p);
+                        patches(i).patch_experts_depth{view, landmark}(p).rms_errors = patch_m.rms_errors(p);
+
+                        patches(i).patch_experts_depth{view,landmark}(p).scaling = patch_m.patch_experts{p}(view, landmark, 1);
+                        patches(i).patch_experts_depth{view,landmark}(p).bias = patch_m.patch_experts{p}(view, landmark, 2);
+                        patches(i).patch_experts_depth{view,landmark}(p).w = reshape(patch_m.patch_experts{p}(view, landmark, 3:end),11,11);                        
+                    end
+                else
+                    multi_modal_types = {'reg'};
+                    % for backward compatibility
+                    patches(i).patch_experts_depth{view, landmark}(1).type = {'reg'};
+                    patches(i).patch_experts_depth{view, landmark}(1).correlations = correlations;
+                    patches(i).patch_experts_depth{view, landmark}(1).rms_errors = rmsErrors;
+
+                    patches(i).patch_experts_depth{view,landmark}(1).scaling = patchExperts(view, landmark, 1);
+                    patches(i).patch_experts_depth{view,landmark}(1).bias = patchExperts(view, landmark, 2);
+                    patches(i).patch_experts_depth{view,landmark}(1).w = reshape(patchExperts(view, landmark, 3:end), 11,11);
+                end
+            end
+        end
+
+    clear 'normalisationOptions, centers, trainingScale, visiIndex, correlations, rmsErrors';
+    end
+    end
+end
+

pkg/OpenFace/matlab_version/models/Load_eye_models.m

@@ -0,0 +1,49 @@
+function [ clmParams_eye, pdm_right_eye, pdm_left_eye, ...
+    patches_left_eye, patches_right_eye,...
+    left_eye_inds_in_68, right_eye_inds_in_68,...
+    left_eye_inds_in_28, right_eye_inds_in_28] = Load_eye_models() 
+
+    clmParams_eye.window_size = [17,17; 15,15; 13,13;];
+    clmParams_eye.numPatchIters = size(clmParams_eye.window_size,1);
+    
+    % the PDM created from in the wild data
+    pdmLoc = ['../models/hierarch_pdm/pdm_28_r_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_right_eye = struct;
+    pdm_right_eye.M = double(M);
+    pdm_right_eye.E = double(E);
+    pdm_right_eye.V = double(V);
+
+    pdmLoc = ['../models/hierarch_pdm/pdm_28_l_eye.mat'];
+
+    load(pdmLoc);
+
+    pdm_left_eye = struct;
+    pdm_left_eye.M = double(M);
+    pdm_left_eye.E = double(E);
+    pdm_left_eye.V = double(V);
+    
+    % the default model parameters to use
+    clmParams_eye.regFactor = 2.0;               
+    clmParams_eye.sigmaMeanShift = 1.5;
+    clmParams_eye.tikhonov_factor = 0;
+
+    clmParams_eye.startScale = 1;
+    clmParams_eye.num_RLMS_iter = 10;
+    clmParams_eye.fTol = 0.01;
+    clmParams_eye.useMultiScale = true;
+    clmParams_eye.use_multi_modal = 0;
+    clmParams_eye.tikhonov_factor = 0;
+
+    [patches_right_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_right_eye.mat', [], [], clmParams_eye);
+    [patches_left_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_left_eye.mat', [], [], clmParams_eye);
+    clmParams_eye.multi_modal_types  = patches_right_eye(1).multi_modal_types;
+    right_eye_inds_in_68 = [43,44,45,46,47,48];
+    left_eye_inds_in_68 = [37,38,39,40,41,42];
+
+    right_eye_inds_in_28 = [9 11 13 15 17 19];
+    left_eye_inds_in_28 = [9 11 13 15 17 19];
+
+end

pkg/OpenFace/matlab_version/models/cen/create_cen_experts_OF.m

@@ -0,0 +1,115 @@
+clear;
+%%
+mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+61,65;62,64;68,66];   
+
+mirror_inds = [1:68];
+mirror_inds(mirrorInds(:,1)) = mirrorInds(:,2);
+mirror_inds(mirrorInds(:,2)) = mirrorInds(:,1);
+
+scales = {'0.50', '1.00'};
+
+for s=scales
+
+    gen_experts = load(sprintf('cen_patches_%s_general.mat', s{1}));
+    load(sprintf('cen_patches_%s_menpo.mat', s{1}));
+    
+    for c=1:size(visiIndex,1)
+        for i=1:size(visiIndex,2)
+            % If present in general and not menpo replace
+            if(gen_experts.visiIndex(c,i) && ~visiIndex(c,i))
+                visiIndex(c,i) = 1;
+                patch_experts.correlations(c,i) = gen_experts.patch_experts.correlations(c,i);
+                patch_experts.rms_errors(c,i) = gen_experts.patch_experts.rms_errors(c,i);
+                patch_experts.patch_experts(c,i) = gen_experts.patch_experts.patch_experts(c,i);
+            elseif(~visiIndex(c,i))
+                patch_experts.correlations(c,i) = 0;
+                patch_experts.rms_errors(c,i) = 0;
+                patch_experts.patch_experts(c,i) = {[]}; 
+            end
+                
+        end
+    end
+    
+    trainingScale = str2num(s{1});
+    save(['cen_patches_', s{1} '_of.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+
+    % Work out the frontal view and remove mirror indices for it
+    [~, frontal] = min(mean(abs(bsxfun(@plus, centers, [0,0,0])')));
+
+    % First clean up the frontal view
+    patch_experts.patch_experts(frontal, mirrorInds(:,2)) = {[]};
+
+    % Work out which views have mirrors of each other, and keep only one set
+    % of them
+    n_views = size(visiIndex,1);
+
+    mirror_view = 1:n_views;
+
+    for i = 1:n_views
+        [~, mirror_view(i)] = min(mean(abs(bsxfun(@plus, centers, centers(i,:))')));
+    end
+
+    % Remove a set of mirror indices
+    to_rem = mirror_view < 1:n_views;
+
+    patch_experts.patch_experts(to_rem, :) = {[]};
+
+    trainingScale = str2num(s{1});
+    write_patch_expert_bin_simple(['cen_patches_', s{1} '_of.dat'], trainingScale, centers, visiIndex, patch_experts, mirror_inds - 1, mirror_view - 1);
+
+end
+
+scales = {'0.25', '0.35'};
+
+for s=scales
+
+    gen_experts = load(sprintf('cen_patches_%s_general_model_half.mat', s{1}));
+    load(sprintf('cen_patches_%s_menpo_model_half.mat', s{1}));
+    
+    for c=1:size(visiIndex,1)
+        for i=1:size(visiIndex,2)
+            % If present in general and not menpo replace
+            if(gen_experts.visiIndex(c,i) && ~visiIndex(c,i))
+                visiIndex(c,i) = 1;
+                patch_experts.correlations(c,i) = gen_experts.patch_experts.correlations(c,i);
+                patch_experts.rms_errors(c,i) = gen_experts.patch_experts.rms_errors(c,i);
+                patch_experts.patch_experts(c,i) = gen_experts.patch_experts.patch_experts(c,i);
+            elseif(~visiIndex(c,i))
+                patch_experts.correlations(c,i) = 0;
+                patch_experts.rms_errors(c,i) = 0;
+                patch_experts.patch_experts(c,i) = {[]}; 
+            end
+                
+        end
+    end
+    
+    trainingScale = str2num(s{1});
+    save(['cen_patches_', s{1} '_of.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+
+    % Work out the frontal view and remove mirror indices for it
+    [~, frontal] = min(mean(abs(bsxfun(@plus, centers, [0,0,0])')));
+
+    % First clean up the frontal view
+    patch_experts.patch_experts(frontal, mirrorInds(:,2)) = {[]};
+
+    % Work out which views have mirrors of each other, and keep only one set
+    % of them
+    n_views = size(visiIndex,1);
+
+    mirror_view = 1:n_views;
+
+    for i = 1:n_views
+        [~, mirror_view(i)] = min(mean(abs(bsxfun(@plus, centers, centers(i,:))')));
+    end
+
+    % Remove a set of mirror indices
+    to_rem = mirror_view < 1:n_views;
+
+    patch_experts.patch_experts(to_rem, :) = {[]};
+
+    trainingScale = str2num(s{1});
+    write_patch_expert_bin_simple(['cen_patches_', s{1} '_of.dat'], trainingScale, centers, visiIndex, patch_experts, mirror_inds - 1, mirror_view - 1);
+
+end

pkg/OpenFace/matlab_version/models/cen/create_cen_experts_gen.m

@@ -0,0 +1,113 @@
+clear;
+load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
+
+mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+  32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+  61,65;62,64;68,66];   
+
+% For mirroring
+frontalView = 1;
+profileViewInds = [2,3,4];
+
+% Grab all related experts and mirror them appropriatelly, just need to
+% mirror the first layer
+
+non_mirrored = mirrorInds(:,1);
+normalisationOptions = rmfield(normalisationOptions, 'ccnf_ratio');
+normalisationOptions.dccnf = true;
+
+n_landmarks = size(visiIndex, 2);
+n_views = size(visiIndex, 1);
+
+patch_experts.correlations = zeros(n_views, n_landmarks);
+patch_experts.rms_errors = zeros(n_views, n_landmarks);
+patch_experts.types = {'reg'};
+patch_experts.patch_experts = cell(n_views, n_landmarks);
+
+scales = {'0.25', '0.35', '0.50', '1.00'};
+
+root = 'D:/deep_experts/2017-02-02/rmses/';
+
+for s=scales
+
+    for c=1:n_views
+
+        if(c == frontalView || sum(profileViewInds==c)> 0)
+
+            for i=1:n_landmarks
+
+                if(visiIndex(c,i))
+                    mirror = false;
+                    % Find the relevant file
+                    if(c == frontalView)
+                        rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_frontal_%d_512.mat'], s{1}, i);
+                    else
+                        rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_profile%d_%d_512.mat'], s{1}, c-1, i);                    
+                    end
+                    if(exist(rel_file, 'file'))
+                       load(rel_file); 
+                    else
+                       rel_id = mirrorInds(mirrorInds(:,2)==i,1);
+                       if(isempty(rel_id))
+                           rel_id = mirrorInds(mirrorInds(:,1)==i,2);
+                       end
+                       if(~visiIndex(c, rel_id))
+                           continue;
+                       end
+                       if(c == frontalView)
+                           rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_frontal_%d_512.mat'], s{1}, rel_id);
+
+                           mirror = true;
+                           load(rel_file);
+                       end
+                    end
+                    patch_experts.correlations(c, i) = correlation_2;
+                    patch_experts.rms_errors(c, i) = rmse;
+
+                    if(~mirror)
+                        patch_experts.patch_experts{c, i} = weights;
+                    else
+                        flips = fliplr(reshape([1:121]', 11, 11));
+                        weights_flipped = weights;
+                        weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                        patch_experts.patch_experts{c,i} = weights_flipped;
+                    end
+                end
+
+            end
+        else
+
+            swap_id = find(centers(:,2) == -centers(c,2));
+
+            corr_T = patch_experts.correlations(swap_id,:);
+            % Appending a mirror view instead, based on the profile view               
+            corr_T  = swap(corr_T, mirrorInds(:,1), mirrorInds(:,2));
+            patch_experts.correlations(c,:) = corr_T;
+
+            rmsT = patch_experts.rms_errors(swap_id,:);
+            rmsT = swap(rmsT, mirrorInds(:,1), mirrorInds(:,2));
+            patch_experts.rms_errors(c,:) = rmsT;
+
+            patchExpertMirror = patch_experts.patch_experts(swap_id,:);
+            patchExpertMirrorT1 = patchExpertMirror(1,mirrorInds(:,1),:);
+            patchExpertMirrorT2 = patchExpertMirror(1,mirrorInds(:,2),:);
+            patchExpertMirror(1,mirrorInds(:,2),:) = patchExpertMirrorT1;
+            patchExpertMirror(1,mirrorInds(:,1),:) = patchExpertMirrorT2;
+
+            % To flip a patch expert it        
+            for p=1:size(patchExpertMirror,2)
+                if(visiIndex(c, p))
+                    weights = patchExpertMirror{p};
+                    flips = fliplr(reshape([1:121]', 11, 11));
+                    weights_flipped = weights;
+                    weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                    patch_experts.patch_experts{c,p} = weights_flipped;
+                end
+            end
+
+        end
+    end
+    trainingScale = str2num(s{1});
+    save(['cen_patches_', s{1} '_general.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+
+end

pkg/OpenFace/matlab_version/models/cen/create_cen_experts_menpo.m

@@ -0,0 +1,115 @@
+clear;
+load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
+
+mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+  32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+  61,65;62,64;68,66];   
+
+% For mirroring
+frontalView = 1;
+profileViewInds = [2,3,4];
+
+% Grab all related experts and mirror them appropriatelly, just need to
+% mirror the first layer
+
+non_mirrored = mirrorInds(:,1);
+normalisationOptions = rmfield(normalisationOptions, 'ccnf_ratio');
+normalisationOptions.dccnf = true;
+
+n_landmarks = size(visiIndex, 2);
+n_views = size(visiIndex, 1);
+
+patch_experts.correlations = zeros(n_views, n_landmarks);
+patch_experts.rms_errors = zeros(n_views, n_landmarks);
+patch_experts.types = {'reg'};
+patch_experts.patch_experts = cell(n_views, n_landmarks);
+
+scales = {'0.25', '0.35', '0.50', '1.00'};
+
+visiIndex = zeros(7, 68);
+
+root = 'D:/deep_experts/menpo/rmses/';
+
+for s=scales
+
+    for c=1:n_views
+
+        if(c == frontalView || sum(profileViewInds==c)> 0)
+
+            for i=1:n_landmarks
+
+                mirror = false;
+                % Find the relevant file
+                if(c == frontalView)
+                    rel_file = sprintf([root, '/%s_frontal_%d_512.mat'], s{1}, i);
+                else                    
+                    rel_file = sprintf([root, '/%s_profile%d_%d_512.mat'], s{1}, c-1, i);                    
+                end
+                if(exist(rel_file, 'file'))
+                    visiIndex(c,i) = 1;
+                    load(rel_file); 
+                else
+                   rel_id = mirrorInds(mirrorInds(:,2)==i,1);
+                   if(isempty(rel_id))
+                       rel_id = mirrorInds(mirrorInds(:,1)==i,2);
+                   end
+                   if(c == frontalView)                       
+                       rel_file = sprintf([root, '/%s_frontal_%d_512.mat'], s{1}, rel_id);
+                       mirror = true;
+                       visiIndex(c,i) = 1;
+                       load(rel_file);
+                   end
+                end
+                patch_experts.correlations(c, i) = correlation_2;
+                patch_experts.rms_errors(c, i) = rmse;
+
+                if(~mirror)
+                    patch_experts.patch_experts{c, i} = weights;
+                else
+                    flips = fliplr(reshape([1:121]', 11, 11));
+                    weights_flipped = weights;
+                    weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                    patch_experts.patch_experts{c,i} = weights_flipped;
+                end
+            end
+        else
+
+            swap_id = find(centers(:,2) == -centers(c,2));
+
+            corr_T = patch_experts.correlations(swap_id,:);
+            % Appending a mirror view instead, based on the profile view               
+            corr_T  = swap(corr_T, mirrorInds(:,1), mirrorInds(:,2));
+            patch_experts.correlations(c,:) = corr_T;
+
+            vis_T = visiIndex(swap_id,:);
+            % Appending a mirror view instead, based on the profile view               
+            vis_T  = swap(vis_T, mirrorInds(:,1), mirrorInds(:,2));
+            visiIndex(c,:) = vis_T;
+            
+            rmsT = patch_experts.rms_errors(swap_id,:);
+            rmsT = swap(rmsT, mirrorInds(:,1), mirrorInds(:,2));
+            patch_experts.rms_errors(c,:) = rmsT;
+
+            patchExpertMirror = patch_experts.patch_experts(swap_id,:);
+            patchExpertMirrorT1 = patchExpertMirror(1,mirrorInds(:,1),:);
+            patchExpertMirrorT2 = patchExpertMirror(1,mirrorInds(:,2),:);
+            patchExpertMirror(1,mirrorInds(:,2),:) = patchExpertMirrorT1;
+            patchExpertMirror(1,mirrorInds(:,1),:) = patchExpertMirrorT2;
+
+            % To flip a patch expert it        
+            for p=1:size(patchExpertMirror,2)
+                if(visiIndex(c, p))
+                    weights = patchExpertMirror{p};
+                    flips = fliplr(reshape([1:121]', 11, 11));
+                    weights_flipped = weights;
+                    weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                    patch_experts.patch_experts{c,p} = weights_flipped;
+                end
+            end
+
+        end
+    end
+    trainingScale = str2num(s{1});
+    save(['cen_patches_', s{1} '_menpo.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+    write_patch_expert_bin(['cen_patches_', s{1} '_menpo.dat'], trainingScale, centers, visiIndex, patch_experts);
+end

pkg/OpenFace/matlab_version/models/cen/create_cen_experts_sparse.m

@@ -0,0 +1,140 @@
+clear;
+load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
+
+mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+  32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+  61,65;62,64;68,66];   
+
+% For mirroring
+frontalView = 1;
+profileViewInds = [2,3,4];
+
+% Grab all related experts and mirror them appropriatelly, just need to
+% mirror the first layer
+
+non_mirrored = mirrorInds(:,1);
+normalisationOptions = rmfield(normalisationOptions, 'ccnf_ratio');
+normalisationOptions.dccnf = true;
+
+n_landmarks = size(visiIndex, 2);
+n_views = size(visiIndex, 1);
+
+patch_experts.correlations = zeros(n_views, n_landmarks);
+patch_experts.rms_errors = zeros(n_views, n_landmarks);
+patch_experts.types = {'reg'};
+patch_experts.patch_experts = cell(n_views, n_landmarks);
+
+scales = {'0.25', '0.35', '0.50', '1.00'};
+visiIndex_full = visiIndex;
+
+to_rem_from = [1,2,3,6,7];
+to_rem_1 = [4;68;58;62;51;6;59;20;63;53;25;56;14;64;9;67;2;33;11;37;17;52;26;60;28;34;44;38;29;8;21;15;12;18];
+to_rem_2 = [6;62;50;25;59;20;17;66;64;57;39;14;12;68;41;45;34;43;30;60;4;29;1;61;47;9;65;52;37;22;15;35;54;58];
+to_rem_3 = [66;62;54;60;38;5;30;13;28;59;44;67;41;57;25];
+
+for s=scales
+    
+    visiIndex = visiIndex_full;
+    
+    for c=1:n_views
+
+        if(c == frontalView || sum(profileViewInds==c)> 0)
+
+            for i=1:n_landmarks
+
+                if(visiIndex(c,i))
+                    mirror = false;
+                    % Find the relevant file
+                    if(c == frontalView)
+                        rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, i);
+                    else
+                        rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, i);                    
+                    end
+                    if(exist(rel_file, 'file'))
+                       load(rel_file); 
+                    else
+                       rel_id = mirrorInds(mirrorInds(:,2)==i,1);
+                       if(isempty(rel_id))
+                           rel_id = mirrorInds(mirrorInds(:,1)==i,2);
+                       end
+                       if(~visiIndex(c, rel_id))
+                           break;
+                       end
+                       if(c == frontalView)
+                           rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, rel_id);
+                       else
+                           rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, rel_id);                    
+                       end
+                       mirror = true;
+                       load(rel_file);
+                    end
+                    patch_experts.correlations(c, i) = correlation_2;
+                    patch_experts.rms_errors(c, i) = rmse;
+
+                    if(~mirror)
+                        patch_experts.patch_experts{c, i} = weights;
+                    else
+                        flips = fliplr(reshape([1:121]', 11, 11));
+                        weights_flipped = weights;
+                        weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                        patch_experts.patch_experts{c,i} = weights_flipped;
+                    end
+                end
+
+            end
+        else
+
+            swap_id = find(centers(:,2) == -centers(c,2));
+
+            corr_T = patch_experts.correlations(swap_id,:);
+            % Appending a mirror view instead, based on the profile view               
+            corr_T  = swap(corr_T, mirrorInds(:,1), mirrorInds(:,2));
+            patch_experts.correlations(c,:) = corr_T;
+
+            rmsT = patch_experts.rms_errors(swap_id,:);
+            rmsT = swap(rmsT, mirrorInds(:,1), mirrorInds(:,2));
+            patch_experts.rms_errors(c,:) = rmsT;
+
+            patchExpertMirror = patch_experts.patch_experts(swap_id,:);
+            patchExpertMirrorT1 = patchExpertMirror(1,mirrorInds(:,1),:);
+            patchExpertMirrorT2 = patchExpertMirror(1,mirrorInds(:,2),:);
+            patchExpertMirror(1,mirrorInds(:,2),:) = patchExpertMirrorT1;
+            patchExpertMirror(1,mirrorInds(:,1),:) = patchExpertMirrorT2;
+
+            % To flip a patch expert it        
+            for p=1:size(patchExpertMirror,2)
+                if(visiIndex(c, p))
+                    weights = patchExpertMirror{p};
+                    flips = fliplr(reshape([1:121]', 11, 11));
+                    weights_flipped = weights;
+                    weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                    patch_experts.patch_experts{c,p} = weights_flipped;
+                end
+            end
+
+        end
+    end
+    
+    if(strcmp('0.25', s))
+        visiIndex(to_rem_from, to_rem_1) = 0;
+        patch_experts.correlations(to_rem_from, to_rem_1) = 0;
+        patch_experts.rms_errors(to_rem_from, to_rem_1) = 0;
+        patch_experts.patch_experts(to_rem_from, to_rem_1) = {[]};
+    end
+    if(strcmp('0.35', s))
+        visiIndex(to_rem_from, to_rem_2) = 0;
+        patch_experts.correlations(to_rem_from, to_rem_2) = 0;
+        patch_experts.rms_errors(to_rem_from, to_rem_2) = 0;
+        patch_experts.patch_experts(to_rem_from, to_rem_2) = {[]};
+    end        
+    if(strcmp('0.50', s))
+        visiIndex(to_rem_from, to_rem_3) = 0;
+        patch_experts.correlations(to_rem_from, to_rem_3) = 0;
+        patch_experts.rms_errors(to_rem_from, to_rem_3) = 0;
+        patch_experts.patch_experts(to_rem_from, to_rem_3) = {[]};
+    end 
+    
+    trainingScale = str2num(s{1});
+    save(['cen_patches_', s{1} '_general_sparse.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+    write_patch_expert_bin(['cen_patches_', s{1} '_general_sparse.dat'], trainingScale, centers, visiIndex, patch_experts);
+end

pkg/OpenFace/matlab_version/models/cen/readme.txt

@@ -0,0 +1,12 @@
+Scripts for creating CEN patch experts from already trained models.
+
+1. To create one from 300W + MultiPIE - create_cen_experts_gen.m
+2. To create one from 300W + MultiPIE + Menpo - create_cen_experts_menpo.m
+
+Please note that items 1 and 2 use the output of the code from "OpenFace/model_training/ce-clm_training/cen_training/train_cen.py. The output of this code is fed as "MultiGeneral_arch4general_%s_[frontal,profilex]_%d_512.mat'" in file 1 which can be decoded as follows: we call the training procedure MultiGeneral. It uses arch4 and is trained on 300W + MultiPIE (called general). Hence the architecture and data are represented as arch4general. The first %s is the scale. When generating the data from "OpenFace/model_training/ce-clm_training/patch_generation/" you will have 4 scales of {'0.25', '0.35', '0.50', '1.00'}. The frontal means trained for frontal faces (patches from frontal images) and profilex means from profilex; x can be 1,2,3 or depending on how many different profiles you defined when generating the data. The %d means which landmark number the training has been done for. 512 denotes the minibatch size. If you decide to go with different names please replace the "MultiGeneral_arch4general_%s_[frontal,profilex]_%d_512.mat'" with your desired name. We didn't change it to demonstrate which architecture and configurations we used. This name should match the output of the script "OpenFace/model_training/ce-clm_training/patch_generation/" denoted with parameter <results_dir> where all the results are svaed. All the training epochs will be stored when training ce-clm model, you can simply pick the best one and use it for the scripts in this folder. 
+
+To create one used in OpenFace for C++ code:
+create_cen_experts_OF.m (this uses both the general and menpo experts to create a joint one, with general ones used when menpo unavailable for that view)
+
+To dowload pretrained models, go to:
+https://www.dropbox.com/sh/o8g1530jle17spa/AADRntSHl_jLInmrmSwsX-Qsa?dl=0

pkg/OpenFace/matlab_version/models/cen/swap.m

@@ -0,0 +1,6 @@
+function arr = swap(arr, ind1, ind2)
+    val1 = arr(ind1);
+    val2 = arr(ind2);
+    arr(ind1) = val2;
+    arr(ind2) = val1;
+end

pkg/OpenFace/matlab_version/models/cen/writeMatrixBin.m

@@ -0,0 +1,37 @@
+% for easier readibility write them row by row
+function writeMatrixBin(fileID, M, type)
+
+    % 4 bytes each for the description
+    fwrite(fileID, size(M,1), 'uint');
+    fwrite(fileID, size(M,2), 'uint');
+    fwrite(fileID, type, 'uint');
+    
+    % Convert the matrix to OpenCV format (row minor as opposed to column
+    % minor)
+    M = M';
+    
+    % type 0 - uint8, 1 - int8, 2 - uint16, 3 - int16, 4 - int, 5 -
+    % float32, 6 - float64
+    
+    % Write out the matrix itself    
+    
+    switch type
+        case 0
+            type = 'uint8';
+        case 1
+            type = 'int8';
+        case 2
+            type = 'uint16';
+        case 3
+            type = 'int16';
+        case 4
+            type = 'int';
+        case 5
+            type = 'float32';
+        case 6
+            type = 'float64';            
+        otherwise
+            type = 'float32';
+    end
+    fwrite(fileID, M, type);
+end

pkg/OpenFace/matlab_version/models/cen/write_patch_expert_bin.m

@@ -0,0 +1,73 @@
+function write_patch_expert_bin(location, trainingScale, centers, visiIndex, patch_experts)
+          
+    patches_file = fopen(location, 'w');        
+    
+    [n_views, n_landmarks, ~] = size(patch_experts.correlations);
+
+    % write out the scaling factor as this is what will be used when
+    % fitting on the window
+    fwrite(patches_file, trainingScale, 'float64');
+    
+    fwrite(patches_file, n_views, 'int');
+    
+    % Write out the information about the view's and centers here
+    for i=1:n_views
+        % this indicates that we're writing a 3x1 double matrix
+        writeMatrixBin(patches_file, centers(i,:)', 6);
+    end
+            
+    % Write out the visibilities
+    for i=1:n_views
+        % this indicates that we're writing a 3x1 double matrix
+        writeMatrixBin(patches_file, visiIndex(i,:)', 4);
+    end
+            
+    for i=1:n_views
+        for j=1:n_landmarks
+
+            % Write out that we're writing a CEN patch expert of 11x11 support region
+            fwrite(patches_file, 6, 'int');
+            fwrite(patches_file, 11, 'int');
+            fwrite(patches_file, 11, 'int');
+            
+            if(~visiIndex(i,j))       
+                % Write out that there won't be any neurons for this
+                % landmark
+                fwrite(patches_file, 0, 'int');
+                fwrite(patches_file, 0, 'int');
+            else
+                num_layers = numel(patch_experts.patch_experts{i,j})/2;
+
+                fwrite(patches_file, num_layers, 'int');
+
+                for n=1:num_layers
+                    
+                    % output the actual layer
+                    
+                    % Layer type, bias, weights
+                    
+                    % Type of layer, first two are relu, the final one is a
+                    % sigmoid (0 - sigmoid, 1 - tanh_opt, 2 - ReLU)
+                    if(n < 3)
+                        fwrite(patches_file, 2, 'int');
+                    else
+                        fwrite(patches_file, 0, 'int');
+                    end
+                                                            
+                    bias = patch_experts.patch_experts{i,j}{n*2};
+                    
+                    weights = patch_experts.patch_experts{i,j}{n*2-1};
+
+                    % the actual bias/weight matrix
+                    writeMatrixBin(patches_file, bias, 5);
+                    writeMatrixBin(patches_file, weights, 5);
+                end
+                
+                % finally write out the confidence
+                fwrite(patches_file, patch_experts.correlations(i,j), 'float64');
+                
+            end    
+        end
+    end
+
+    fclose(patches_file);

pkg/OpenFace/matlab_version/models/cen/write_patch_expert_bin_simple.m

@@ -0,0 +1,79 @@
+function write_patch_expert_bin_simple(location, trainingScale, centers, visiIndex, patch_experts, mirror_inds, mirror_view)
+          
+    patches_file = fopen(location, 'w');        
+    
+    [n_views, n_landmarks, ~] = size(patch_experts.correlations);
+
+    % write out the scaling factor as this is what will be used when
+    % fitting on the window
+    fwrite(patches_file, trainingScale, 'float64');
+    
+    fwrite(patches_file, n_views, 'int');
+    
+    % Write out the information about the view's and centers here
+    for i=1:n_views
+        % this indicates that we're writing a 3x1 double matrix
+        writeMatrixBin(patches_file, centers(i,:)', 6);
+    end
+            
+    % Write out the visibilities
+    for i=1:n_views
+        % this indicates that we're writing a 3x1 double matrix
+        writeMatrixBin(patches_file, visiIndex(i,:)', 4);
+    end
+            
+    % Write out the mirror indices
+    writeMatrixBin(patches_file, mirror_inds, 4);
+    
+    % Write out the mirror views
+    writeMatrixBin(patches_file, mirror_view, 4);
+    
+    for i=1:n_views
+        for j=1:n_landmarks
+
+            % Write out that we're writing a CEN patch expert of 11x11 support region
+            fwrite(patches_file, 6, 'int');
+            fwrite(patches_file, 11, 'int');
+            fwrite(patches_file, 11, 'int');
+            
+            if(~visiIndex(i,j))       
+                % Write out that there won't be any neurons for this
+                % landmark
+                fwrite(patches_file, 0, 'int');
+                fwrite(patches_file, patch_experts.correlations(i,j), 'float64'); % also writing out a fake correlation
+            else
+                num_layers = numel(patch_experts.patch_experts{i,j})/2;
+
+                fwrite(patches_file, num_layers, 'int');
+
+                for n=1:num_layers
+                    
+                    % output the actual layer
+                    
+                    % Layer type, bias, weights
+                    
+                    % Type of layer, first two are relu, the final one is a
+                    % sigmoid (0 - sigmoid, 1 - tanh_opt, 2 - ReLU)
+                    if(n < 3)
+                        fwrite(patches_file, 2, 'int');
+                    else
+                        fwrite(patches_file, 0, 'int');
+                    end
+                                                            
+                    bias = patch_experts.patch_experts{i,j}{n*2};
+                    
+                    weights = patch_experts.patch_experts{i,j}{n*2-1};
+
+                    % the actual bias/weight matrix
+                    writeMatrixBin(patches_file, bias, 5);
+                    writeMatrixBin(patches_file, weights', 5);
+                end
+                
+                % finally write out the confidence
+                fwrite(patches_file, patch_experts.correlations(i,j), 'float64');
+                
+            end    
+        end
+    end
+
+    fclose(patches_file);