open source pkg v1

pkg/OpenFace/model_training/ce-clm_training/cen_training/keras2matlab.py

@@ -0,0 +1,71 @@
+import os
+import sys
+import time
+import pickle
+import scipy.io as sio
+import keras
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation
+from keras.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax
+from keras.constraints import maxnorm, nonneg
+import numpy
+import scipy
+from scipy import io
+from itertools import product
+import pickle
+import time
+import uuid
+import os
+import h5py
+import sys
+from keras.models import model_from_json
+from keras.models import model_from_yaml
+
+
+
+
+#accuracies is a list of (rmse_i and correlation_i) for i in range of your epochs - we want to pick the model from somwhere in your  
+
+def dump(keras_weight_file,output_filename,rmse,corr):
+
+        #build the model
+        model=build_model()
+        model.load_weights(keras_weight_file)
+        denses=[layer for layer in model.layers if type(layer) is keras.layers.Dense]
+        #creating the matlab object
+        final=numpy.zeros(len(denses)*2,dtype=numpy.object)
+        #getting all the weights and biases in their correspoding place in final object
+        for i in range(len(denses)):
+                w,b=(denses[i].get_weights())
+                final[i*2]=w
+                final[i*2+1]=b
+        #writing weights,rmse,correlation
+        sio.savemat(output_filename, {'rmse':rmse,'correlation_2':corr,'weights':final})
+
+
+#this is the arch4 - #TODO BY YOU: replace with your architecture - just make sure the loop inside keras2matlab.py is working correctly with your architecture
+
+def build_model():
+
+        model = Sequential()
+        model.add(Dense(500, input_dim=122, activation='relu'))
+        model.add(Dropout(0.3))
+        model.add(Dense(200, activation='relu'))
+        model.add(Dropout(0.3))
+        model.add(Dense(100, activation='sigmoid'))
+        model.add(Dropout(0.3))
+
+        model.add(Dense(1, W_constraint=nonneg(), activation='sigmoid'))
+
+
+        optimizers=[];
+        optimizers.append(SGD(lr=.1, momentum=0.1, decay=0.0))
+        optimizers.append(RMSprop(lr=0.001,rho=0.9, epsilon=1e-06))
+        optimizers.append(Adagrad(lr=0.01, epsilon=1e-06))
+        optimizers.append(Adadelta(lr=1.0, rho=0.95, epsilon=1e-06))
+        optimizers.append(Adam(lr=0.0001/2, beta_1=0.9, beta_2=0.999, epsilon=1e-08))
+        optimizers.append(Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08))
+
+
+        model.compile(loss='mean_squared_error', optimizer=optimizers[4])
+        return model

pkg/OpenFace/model_training/ce-clm_training/cen_training/readme.txt

@@ -0,0 +1,20 @@
+The code in this folder trains CEN patch experts for use with the CE-CLM
+landmark detector. First generate the patches for training, using the code
+in ../patch_generation.
+
+Then run the training code via train_cen.py. Each patch expert is trained for a single scale, view and landmark.
+
+python train_cen.py (location of patches) (model to train) (scale to train) (view to train) (landmark to train) (minibatch size) (folder to save models to) (menpo or general/300W patches)
+
+
+other options:
+--num_epochs : number of epochs to train (default 50)
+--outfile : file to save training history to (default acc.txt)
+--acc_file : file to load training history from to resume (set this flag in order to resume training)
+
+
+e.g.
+python train_cen.py menpo_data/ arch6 0.35 profile3 5 256 model_saves menpo --num_epochs 100 --outfile menpo_acc_120.txt --acc_file menpo_acc_20.txt
+
+
+Subsequently, run the keras2matlab.py for getting your model in matlab format. Each landmark patch detector needs to be converted into a matlab input file using this script. The script takes in an input file which would be one output output of the train_cen code. Then it turns it into matlab object. The code is written for architecture 4, if you are using other architecture you need to copy and paste the architecture to build_model. 

pkg/OpenFace/model_training/ce-clm_training/cen_training/train_cen.py

@@ -0,0 +1,426 @@
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation
+from keras.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam, Adamax 
+from keras.constraints import max_norm, non_neg
+from keras.callbacks import ModelCheckpoint, Callback
+import numpy
+import scipy
+from scipy import io
+from itertools import product
+import pickle
+import time
+import uuid
+import os
+import h5py
+import sys
+from keras.models import model_from_json
+from keras.models import model_from_yaml
+from keras.models import load_model
+
+import keras.backend as K
+
+from datetime import datetime
+import argparse
+
+logfile = None
+final_results = []
+
+def log(message):
+    with open(logfile, 'a') as f:
+        f.write("{}\n".format(message))
+
+def log_init(folder, filename):
+    if not os.path.exists(folder):
+        os.mkdir(folder)
+    
+    global logfile
+    logfile = os.path.join(folder, filename)
+    with open(logfile, 'a') as f:
+        f.write("----------------------\n")
+        f.write("{}\n".format(datetime.now().strftime("%b/%d/%y %H:%M:%S")))
+
+def put_in_format(samples, training_samples_size):
+    samples=samples.reshape(samples.shape[0]/training_samples_size, training_samples_size,samples.shape[1])
+    return numpy.squeeze(samples);
+
+def read_data(folder, scale, view, lm):
+    folder = os.path.join(folder)
+    print("--------------------------------------------------------------")
+    try:
+        #reading from h5
+        h5_file = os.path.join(folder, str(lm), 'data' + scale + '_' + view + '.mat')
+        print('loading patches from ' + h5_file)
+        dataset = h5py.File(h5_file, 'r');
+        print("Landmark " + str(lm))
+    except:
+        print("Landmark " + str(lm) + ' not found!')
+        print("--------------------------------------------------------------")
+        sys.exit()
+
+    train_data = put_in_format(numpy.array(dataset['samples_train']),81)
+    train_labels = put_in_format(numpy.array(dataset['labels_train']).T,81)
+    test_data = put_in_format(numpy.array(dataset['samples_test']),81)
+    test_labels = put_in_format(numpy.array(dataset['labels_test']).T,81)
+    
+    train_data_dnn = train_data.reshape([train_data.shape[0]*train_data.shape[1],122])
+    train_labels_dnn = train_labels.reshape([train_labels.shape[0]*train_labels.shape[1],1])
+    test_data_dnn = test_data.reshape([test_data.shape[0]*test_data.shape[1],122])
+    test_labels_dnn = test_labels.reshape([test_labels.shape[0]*test_labels.shape[1],1])
+
+    print(train_data_dnn.shape)
+    print(train_labels_dnn.shape)
+    print(test_data_dnn.shape)
+    print(test_labels_dnn.shape)
+
+    return train_data_dnn.astype('float32'), train_labels_dnn.flatten().astype('float32'), test_data_dnn.astype('float32'), test_labels_dnn.flatten().astype('float32')
+
+def read_data_menpo(folder, scale, view, lm):
+    train_file = "menpo_train_data{}_{}_{}.mat".format(scale, view, lm)
+    valid_file = "menpo_valid_data{}_{}_{}.mat".format(scale, view, lm)
+    print("training file: {}".format(train_file))
+    print("validation file: {}".format(valid_file))
+    print("--------------------------------------------------------------")
+    try:
+        #reading from h5
+        train = h5py.File(os.path.join(folder, train_file), 'r')
+        valid = h5py.File(os.path.join(folder, valid_file), 'r')
+        print("Landmark " + str(lm))
+    except:
+        print("Landmark " + str(lm) + 'not found!')
+        print("--------------------------------------------------------------")
+        sys.exit()
+
+    train_data = put_in_format(numpy.array(train['samples']),81)
+    train_labels = put_in_format(numpy.array(train['labels']).T,81)
+    train_data_dnn = train_data.reshape([train_data.shape[0]*train_data.shape[1],122])
+    train_labels_dnn = train_labels.reshape([train_labels.shape[0]*train_labels.shape[1],1])
+    print(train_data_dnn.shape)
+    print(train_labels_dnn.shape)
+
+    if 'samples' in valid:
+        valid_data=put_in_format(numpy.array(valid['samples']),81)
+        valid_labels=put_in_format(numpy.array(valid['labels']).T,81)
+        valid_data_dnn = valid_data.reshape([valid_data.shape[0]*valid_data.shape[1],122])
+        valid_labels_dnn = valid_labels.reshape([valid_labels.shape[0]*valid_labels.shape[1],1])
+        print(valid_data_dnn.shape)
+        print(valid_labels_dnn.shape)
+        return train_data_dnn.astype('float32'), train_labels_dnn.flatten().astype('float32'), valid_data_dnn.astype('float32'), valid_labels_dnn.astype('float32')
+    else:
+        print("No validation data")
+        return train_data_dnn.astype('float32'), train_labels_dnn.flatten().astype('float32'), None, None
+
+    train.close()
+    valid.close()
+
+def model_half():
+    model = Sequential()
+    model.add(Dense(300, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(100, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(50, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def model_300():
+    model = Sequential()
+    model.add(Dense(300, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(200, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(100, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch4():
+    model = Sequential()
+    model.add(Dense(500, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(200, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(100, activation='sigmoid'))
+    model.add(Dropout(0.3))
+    
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+    
+    return model
+    
+def arch6():
+    model = Sequential()
+    model.add(Dense(50, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(20, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(10, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch6a():
+    model = Sequential()
+    model.add(Dense(50, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(200, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(100, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch7():
+    model = Sequential()
+    model.add(Dense(128, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(64, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(32, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch7a():
+    model = Sequential()
+    model.add(Dense(100, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(40, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(20, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch7b():
+    model = Sequential()
+    model.add(Dense(150, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(60, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(30, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch7c():
+    model = Sequential()
+    model.add(Dense(200, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(80, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(40, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch8():
+    model = Sequential()
+    model.add(Dense(512, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(32, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def arch9():
+    model = Sequential()
+    model.add(Dense(500, input_dim=122, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(200, activation='relu'))
+    model.add(Dropout(0.3))
+    model.add(Dense(10, activation='sigmoid'))
+    model.add(Dropout(0.3))
+
+    model.add(Dense(1, kernel_constraint=non_neg(), activation='sigmoid'))
+
+    return model
+
+def find_last(epoch_src, acc_file, dataset):
+    filename = os.path.join(epoch_src, acc_file)
+    if not os.path.exists(filename):
+        print("{} not found!".format(filename))
+        sys.exit()
+
+    results = pickle.load(open(filename, 'r'))
+    print("starting from epoch {}".format(len(results['corr'])))
+    filename = '{}_epoch_{}.h5'.format(dataset, len(results['corr']) - 1)
+    print("loading from {}".format(filename))
+    return os.path.join(epoch_src, filename), len(results['corr'])
+
+def load_old_model(epoch_src, acc_file, dataset):
+    # load pre-trained model to continue training
+    filename = None
+    start_epoch = 0
+    if acc_file is not None:
+        filename, start_epoch = find_last(epoch_src, acc_file, dataset)
+
+    return filename, start_epoch
+
+def build_model(model_fn, model_file=None):
+    if model_file is not None:
+        model = load_model(model_file)
+        print(model.get_config())
+        return model
+
+    model = model_fn()
+
+    optimizers=[]
+    optimizers.append(SGD(lr=.1, momentum=0.1, decay=0.0))
+    optimizers.append(RMSprop(lr=0.001,rho=0.9, epsilon=1e-06))
+    optimizers.append(Adagrad(lr=0.01, epsilon=1e-06))
+    optimizers.append(Adadelta(lr=1.0, rho=0.95, epsilon=1e-06))
+    #this is the optimizer that is used - Adam
+    #you can change the lr parameter 
+    #initial: 2
+    lr = 0.0001/2
+    log("Learning rate for Adam: {}".format(lr))
+    optimizers.append(Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08))
+    optimizers.append(Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08))
+    
+    model.compile(loss='mean_squared_error', optimizer=optimizers[4])
+
+    return model
+
+corrs = []
+class EpochCallback(Callback):
+    def __init__(self, valid_data, valid_labels):
+        self.valid_data = valid_data
+        self.valid_labels = valid_labels
+
+    def on_epoch_end(self, epoch, logs={}):
+        prediction = self.model.predict(self.valid_data, batch_size=4096)
+        coeff = numpy.corrcoef(prediction.flatten(), self.valid_labels.flatten())[0, 1]
+
+        print("RMSE: {}\tCorr: {}".format(numpy.sqrt(logs['val_loss']), coeff ** 2))
+
+        global corrs
+        corrs.append(coeff ** 2)
+
+# TODO: make symbolic
+def corr(y_true, y_pred):
+    return K.constant((numpy.corrcoef(y_true.flatten(), y_pred.flatten())[0, 1]) ** 2)
+
+def get_best(history, corrs):
+    history = history.history
+    print("Keys: {}".format(history.keys()))
+
+    best_mse = None
+    best_corr = None
+    for i, (mse, corr) in enumerate(zip(history['val_loss'], corrs)):
+        if best_mse is None or mse < best_mse[1]:
+            best_mse = (i, mse)
+        if best_corr is None or corr > best_corr[1]:
+            best_corr = (i, corr)
+
+    return best_mse, best_corr
+
+MODELS = {
+    'model_half': model_half,
+    'model_300': model_300,
+    'arch4': arch4,
+    'arch6': arch6,
+    'arch7': arch7,
+    'arch8': arch8,
+    'arch6a': arch6a,
+    'arch7a': arch7a,
+    'arch7b': arch7b,
+    'arch7c': arch7c,
+    'arch9': arch9
+}
+
+if __name__=='__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('mat_src', type=str)
+    parser.add_argument('model', type=str, choices=MODELS.keys())
+    parser.add_argument('scale', type=str, help="scale of image")
+    parser.add_argument('view', type=str, help="view of image")
+    parser.add_argument('lm', type=int, help="landmark number")
+    parser.add_argument('minibatch', type=int, help="size of minibatch")
+    parser.add_argument('results_dir', type=str, help="location to save model epochs")
+    parser.add_argument('dataset', choices=['general', 'menpo'], help='dataset to train on')
+    parser.add_argument('--acc_file', type=str, default=None, help='if this option is set, resume training of model based on last epoch in this file (in results_dir)')
+    parser.add_argument('--num_epochs', type=int, default=25, help='number of epochs to train model')
+    parser.add_argument('--outfile', type=str, default='accuracies.txt', help='file to save training history to (in results_dir)')
+
+    args = parser.parse_args()
+
+    logfile = "{}_{}_{}.log".format(args.scale, args.view, args.lm)
+    log_init('./logs', logfile)
+
+    log("""Loading data from: {}
+Model: {}\nScale: {}\t View: {}\tLM: {}\nMinibatch size: {}
+Results saved to: {}\nDataset: {}\nFile to resume from: {}
+# of epochs: {}
+Training history saved to: {}""".format(args.mat_src, args.model, args.scale,
+        args.view, args.lm, args.minibatch, args.results_dir, args.dataset,
+        args.acc_file, args.num_epochs, args.outfile))
+    
+    if not os.path.isdir(args.mat_src):
+        print("Error, mat source {} does not exist".format(args.mat_src))
+        sys.exit()
+
+    model_folder = os.path.join(os.path.abspath(os.path.dirname(__file__)), args.results_dir, args.model)
+    if not os.path.isdir(model_folder):
+        os.makedirs(model_folder)
+    outfolder = os.path.join(model_folder, "{}_{}_{}_{}".format(args.scale, args.view, args.lm, args.minibatch))
+    if not os.path.isdir(outfolder):
+        os.mkdir(outfolder)
+
+    filename, start_epoch = load_old_model(outfolder, args.acc_file, args.dataset)
+
+    model = build_model(MODELS[args.model], model_file=filename)
+
+    if args.dataset == 'general':
+        train_data, train_labels, valid_data, valid_labels = read_data(args.mat_src, args.scale, args.view, args.lm)
+    elif args.dataset == 'menpo':
+        train_data, train_labels, valid_data, valid_labels = read_data_menpo(args.mat_src, args.scale, args.view, args.lm)
+
+    callbacks = [
+            EpochCallback(valid_data, valid_labels),
+            ModelCheckpoint(os.path.join(outfolder, args.dataset + "_epoch_{epoch}.h5"), verbose=1)
+    ]
+
+    history = model.fit(train_data, train_labels, verbose=1,
+            epochs=args.num_epochs+start_epoch, batch_size=args.minibatch,
+            validation_data=(valid_data, valid_labels), callbacks=callbacks,
+            initial_epoch=start_epoch)
+    
+    (mse_index, bestMSE), bestCorr = get_best(history, corrs)
+    log("Number of epochs run: {}".format(args.num_epochs))
+    log("Best RMSE {}.\t Best Corr: {}.".format((mse_index, numpy.sqrt(bestMSE)), bestCorr))
+
+    # append new training stats to old ones, if continuing
+    old_data = None
+    if args.acc_file is not None:
+        with open(os.path.join(outfolder, args.acc_file), 'r') as g:
+            old_data = pickle.load(g)
+
+    with open(os.path.join(outfolder, args.outfile), 'w') as f:
+        rmses = list(numpy.sqrt(history.history['val_loss']))
+
+        if old_data is not None:
+            rmses = old_data['rmse'] + rmses
+            corrs = old_data['corr'] + corrs
+
+        results = {'rmse': rmses, 'corr': corrs}
+        pickle.dump(results, f)
+
+    log("Successfully trained\n--------------------")

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/readme.txt

@@ -0,0 +1,9 @@
+The code here prepares the training images and labels into a format that can be easilly used to train SVR and CCNF regressors.
+
+Just run "scripts/Prepare_data_wild_all.m" for data needed to train patch experts for in-the-wild experiments.(you have to have the relevant datasets, but they are all available online at http://ibug.doc.ic.ac.uk/resources/facial-point-annotations/) 
+
+Run "scripts/Prepare_data_Multi_PIE_all.m" (you have to have the multi-pie dataset and labels)
+
+Run "scripts/Prepare_data_general_all.m" (you have to have both of the datasets, and you have to run "scripts/Prepare_data_wild_all.m" and scripts/Prepare_data_Multi_PIE_all.m" first.
+
+PDM model used is trained on 2D landmark labels using Non-Rigid-Structure for motion (code can be found http://www.cl.cam.ac.uk/~tb346/res/ccnf/pdm_generation.zip)

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AddOrthRow.m

@@ -0,0 +1,9 @@
+function [RotFull] = AddOrthRow(RotSmall)
+
+  % We can work out these values from the small version of the rotation matrix Rx * Ry * Rz (if you plug in values you can work it out, just slightly tedious)
+  RotFull = zeros(3,3);
+  RotFull(1:2, :) = RotSmall;
+  RotFull(3,1) = RotSmall(1, 2) * RotSmall(2, 3) - RotSmall(1, 3) * RotSmall(2, 2);
+  RotFull(3,2) = RotSmall(1, 3) * RotSmall(2, 1) - RotSmall(1, 1) * RotSmall(2, 3);
+  RotFull(3,3) = RotSmall(1, 1) * RotSmall(2, 2) - RotSmall(1, 2) * RotSmall(2, 1);
+  

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AlignShapesKabsch.m

@@ -0,0 +1,22 @@
+function [ R, T ] = AlignShapesKabsch ( alignFrom, alignTo )
+%ALIGN3DSHAPES Summary of this function goes here
+%   Detailed explanation goes here
+
+    dims = size(alignFrom, 2);
+
+    alignFromMean = alignFrom - repmat(mean(alignFrom), size(alignFrom,1),1);
+    alignToMean = alignTo - repmat(mean(alignTo), size(alignTo,1),1);
+
+    [U, ~, V] = svd( alignFromMean' * alignToMean);
+
+    % make sure no reflection is there
+    d = sign(det(V*U'));
+    corr = eye(dims);
+    corr(end,end) = d;
+    
+    R = V*corr*U';
+
+    T = mean(alignTo) - (R * mean(alignFrom)')';
+    T = T';
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AlignShapesWithScale.m

@@ -0,0 +1,30 @@
+function [ A, T, error, alignedShape ] = AlignShapesWithScale( alignFrom, alignTo )
+%ALIGNSHAPESWITHSCALE Summary of this function goes here
+%   Detailed explanation goes here
+
+    numPoints = size(alignFrom,1);
+
+    meanFrom = mean(alignFrom);    
+    meanTo = mean(alignTo);
+    
+    alignFromMeanNormed = bsxfun(@minus, alignFrom, meanFrom);
+    alignToMeanNormed = bsxfun(@minus, alignTo, meanTo);
+
+    % scale now
+    sFrom = sqrt(sum(alignFromMeanNormed(:).^2)/numPoints);
+    sTo = sqrt(sum(alignToMeanNormed(:).^2)/numPoints);
+    
+    s = sTo / sFrom;
+    
+    alignFromMeanNormed = alignFromMeanNormed/sFrom;
+    alignToMeanNormed = alignToMeanNormed/sTo;
+    
+    [R, t] = AlignShapesKabsch(alignFromMeanNormed, alignToMeanNormed);
+    
+    A = s * R;
+    aligned = (A * alignFrom')';
+    T = mean(alignTo - aligned);
+    alignedShape = bsxfun(@plus, aligned, T);
+    error = mean(sum((alignedShape - alignTo).^2,2));
+    
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AxisAngle2Rot.m

@@ -0,0 +1,11 @@
+function [Rot] = AxisAngle2Rot(axisAngle)
+
+    theta = norm(axisAngle, 2);
+    
+    nx = axisAngle / theta;
+    
+    nx = [    0  -nx(3)  nx(2);
+           nx(3)     0  -nx(1);
+          -nx(2)  nx(1)     0 ];
+    
+    Rot = eye(3) + sin(theta) * nx + (1-cos(theta))*nx^2;

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/Euler2Rot.m

@@ -0,0 +1,11 @@
+function [Rot] = Euler2Rot(euler)
+
+	rx = euler(1);
+	ry = euler(2);
+	rz = euler(3);
+
+	Rx = [1 0 0; 0 cos(rx) -sin(rx); 0 sin(rx) cos(rx)];
+	Ry = [cos(ry) 0 sin(ry); 0 1 0; -sin(ry) 0 cos(ry)];
+	Rz = [cos(rz) -sin(rz) 0; sin(rz) cos(rz) 0; 0 0 1];
+	
+	Rot = Rx * Ry * Rz;

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/GetShape3D.m

@@ -0,0 +1,7 @@
+function [shape3D] = GetShape3D(M, V, p)
+
+    shape3D = M + V * p;
+
+    shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
+    
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/GetShapeOrtho.m

@@ -0,0 +1,20 @@
+function [shape2D] = GetShapeOrtho(M, V, p, global_params)
+
+    % M - mean shape vector
+    % V - eigenvectors
+    % p - parameters of non-rigid shape
+    % V_exp
+    % p_exp
+    % global_params includes scale, euler rotation, translation, 
+    % R - rotation matrix
+    % T - translation vector (tx, ty)    
+    
+    R = Euler2Rot(global_params(2:4));    
+    T = [global_params(5:6); 0];
+    a = global_params(1);
+    
+    shape3D = GetShape3D(M, V, p);
+    
+    shape2D = bsxfun(@plus, a * R*shape3D', T);
+    shape2D = shape2D';
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/ProcrustesAnalysis.m

@@ -0,0 +1,103 @@
+function [normX, normY, meanShape, Transform] = ProcrustesAnalysis(x, y, options)
+
+% Translate all elements to origin and scale to 1
+normX = zeros(size(x));
+normY = zeros(size(y));
+
+for i = 1:size(x,1)
+    
+    offsetX = mean(x(i,:));
+    offsetY = mean(y(i,:));
+
+    Transform.offsetX(i) = offsetX;
+    Transform.offsetY(i) = offsetY;
+    
+    normX(i,:) = x(i,:) - offsetX;
+    normY(i,:) = y(i,:) - offsetY;
+
+    % Get the Frobenius norm, to scale the shapes to unit size
+    scale = norm([normX(i,:) normY(i,:)], 'fro');
+    
+    Transform.scale(i) = scale;
+    
+    normX(i,:) = normX(i,:)/scale;
+    normY(i,:) = normY(i,:)/scale;
+    
+end
+
+% Rotate elements untill all of them have the same orientation
+
+% the initial estimate of rotation would be the first element
+% if change is less than 1% stop (shouldn't take more than 2 steps)
+change = 0.1;
+
+meanShape = [ normX(1,:); normY(1,:) ]';
+
+Transform.Rotation = zeros(size(x,1),1);
+
+for i = 1:30
+    
+    % align all of the shapes to the mean shape
+    
+    % remember all orientations to get the mean one
+    orientations = zeros(size(normX,1),1);
+    
+    for j = 1:size(x,1)
+                
+        % do SVD of mean * X'
+        currentShape = [ normX(j,:); normY(j,:) ]';
+        [U, ~, V] = svd( meanShape' * currentShape);
+        rot = V*U';
+        
+        if(asin(rot(2,1)) > 0)
+           orientations(j) = real(acos(rot(1,1)));
+        else
+           orientations(j) = real(-acos(rot(1,1)));
+        end
+        
+        Transform.Rotation(j) = Transform.Rotation(j) + orientations(j);
+        
+        currentShape = currentShape * rot;                
+        
+        normX(j,:) = currentShape(:,1)';
+        normY(j,:) = currentShape(:,2)';
+        
+    end
+    
+    % recalculate the mean shape;
+    oldMean = meanShape;
+    meanShape = [mean(normX); mean(normY)]';
+    
+    % rotate the mean shape to mean rotation
+    meanOrientation = mean(orientations);
+    
+    % Do this only the first time
+    if(i==1)
+        
+        rotM = [ cos(-meanOrientation) -sin(-meanOrientation); sin(-meanOrientation) cos(-meanOrientation) ]; 
+        meanShape = meanShape * rotM;
+    end
+    % scale mean shape to unit
+    meanScale = norm(meanShape, 'fro');
+    meanShape = meanShape*(1/meanScale);
+    
+    % find frobenious norm
+    diff = norm(oldMean - meanShape, 'fro');
+    
+    if(diff/norm(oldMean,'fro') < change)
+        break;
+    end
+    
+end
+
+% transform to tangent space to preserve linearities
+
+% get the scaling factors for each shape
+if(options.TangentSpaceTransform)
+    scaling = [ normX normY ] * [ meanShape(:,1)' meanShape(:,2)']';
+    for i=1:size(x,1)
+        normX(i,:) = normX(i,:) * (1 / scaling(i));
+        normY(i,:) = normY(i,:) * (1 / scaling(i));
+    end
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/ProcrustesAnalysis3D.m

@@ -0,0 +1,139 @@
+function [ normX, normY, normZ, meanShape, Transform ] = ProcrustesAnalysis3D( x, y, z, tangentSpace, meanShape )
+%PROCRUSTESANALYSIS3D Summary of this function goes here
+%   Detailed explanation goes here
+
+meanProvided = false;
+
+if(nargin > 4)
+    meanProvided = true;
+end
+
+% Translate all elements to origin
+normX = zeros(size(x));
+normY = zeros(size(y));
+normZ = zeros(size(z));
+
+for i = 1:size(x,1)
+    
+    offsetX = mean(x(i,:));
+    offsetY = mean(y(i,:));
+    offsetZ = mean(z(i,:));
+
+    Transform.offsetX(i) = offsetX;
+    Transform.offsetY(i) = offsetY;
+    Transform.offsetZ(i) = offsetZ;
+    
+    normX(i,:) = x(i,:) - offsetX;
+    normY(i,:) = y(i,:) - offsetY;
+    normZ(i,:) = z(i,:) - offsetZ;
+    
+end
+
+% Rotate elements untill all of them have the same orientation
+
+% the initial estimate of rotation would be the first element
+% if change is less than 1% stop (shouldn't take more than 2 steps)
+change = 0.1;
+
+if(~meanProvided)
+    meanShape = [ mean(normX); mean(normY); mean(normZ) ]';
+end
+% scale all the shapes to mean shape
+
+% Get the Frobenius norm, to scale the shapes to mean size (still want to
+% retain mm)
+meanScale = norm(meanShape, 'fro');    
+    
+for i = 1:size(x,1)
+    
+    scale = norm([normX(i,:) normY(i,:) normZ(i,:)], 'fro')/meanScale;
+    
+    normX(i,:) = normX(i,:)/scale;
+    normY(i,:) = normY(i,:)/scale;
+    normZ(i,:) = normZ(i,:)/scale;
+    
+end
+
+Transform.RotationX = zeros(size(x,1),1);
+Transform.RotationY = zeros(size(x,1),1);
+Transform.RotationZ = zeros(size(x,1),1);
+
+for i = 1:30
+    
+    % align all of the shapes to the mean shape
+    
+    % remember all orientations to get the mean one (in euler angle form, pitch, yaw roll)
+    orientationsX = zeros(size(normX,1),1);
+    orientationsY = zeros(size(normX,1),1);
+    orientationsZ = zeros(size(normX,1),1);
+    
+    for j = 1:size(x,1)
+                
+        currentShape = [normX(j,:); normY(j,:); normZ(j,:)]';
+        % we want to align the current shape to the mean one
+        [ R, T ] = AlignShapesKabsch(currentShape, meanShape);
+        
+        eulers = Rot2Euler(R);
+
+        orientationsX(j) = eulers(1);
+        orientationsY(j) = eulers(2);
+        orientationsZ(j) = eulers(3);
+
+        Transform.RotationX(j) = eulers(1);
+        Transform.RotationY(j) = eulers(2);
+        Transform.RotationZ(j) = eulers(3);
+        
+        currentShape = R * currentShape';                
+        
+        normX(j,:) = currentShape(1,:);
+        normY(j,:) = currentShape(2,:);
+        normZ(j,:) = currentShape(3,:);
+        
+    end
+    
+    % recalculate the mean shape
+%     if(~meanProvided)
+        oldMean = meanShape;
+        meanShape = [mean(normX); mean(normY); mean(normZ)]';
+        meanScale = norm(meanShape, 'fro');  
+%     end
+    
+    for j = 1:size(x,1)
+    
+        scale = norm([normX(j,:) normY(j,:) normZ(j,:)], 'fro')/meanScale;
+
+        normX(j,:) = normX(j,:)/scale;
+        normY(j,:) = normY(j,:)/scale;
+        normZ(j,:) = normZ(j,:)/scale;
+
+    end
+
+    if(i==1 && ~meanProvided)
+        
+        % rotate the mean shape to mean rotation
+        meanOrientationX = mean(orientationsX);
+        meanOrientationY = mean(orientationsY);
+        meanOrientationZ = mean(orientationsZ);
+
+        R = Euler2Rot([meanOrientationX, meanOrientationY, meanOrientationZ]);
+        meanShape = (R * meanShape')';
+    end
+    
+    % find frobenious norm
+    diff = norm(oldMean - meanShape, 'fro');
+    
+    if(diff/norm(oldMean,'fro') < change)
+        break;
+    end
+    
+end
+
+% transform to tangent space to preserve linearities
+
+% get the scaling factors for each shape
+if(tangentSpace)
+    [ normX, normY, normZ] = TangentSpaceTransform(normX, normY, normZ, meanShape);
+end
+
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/Rot2AxisAngle.m

@@ -0,0 +1,11 @@
+function [ axisAngle ] = Rot2AxisAngle( Rot )
+%ROT2AXISANGLE Summary of this function goes here
+%   Detailed explanation goes here
+
+    theta = acos((trace(Rot) - 1) / 2);
+
+    vec = 1.0/(2*sin(theta));
+    vec = vec * [Rot(3,2) - Rot(2,3), Rot(1,3) - Rot(3,1), Rot(2,1) - Rot(1,2)];
+    axisAngle = vec * theta;
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/Rot2Euler.m

@@ -0,0 +1,12 @@
+function [euler] = Rot2Euler(R)
+
+    q0 = sqrt( 1 + R(1,1) + R(2,2) + R(3,3) ) / 2;
+    q1 = (R(3,2) - R(2,3)) / (4*q0) ;
+    q2 = (R(1,3) - R(3,1)) / (4*q0) ;
+    q3 = (R(2,1) - R(1,2)) / (4*q0) ;
+
+    yaw  = asin(2*(q0*q2 + q1*q3));
+    pitch= atan2(2*(q0*q1-q2*q3), q0*q0-q1*q1-q2*q2+q3*q3); 
+    roll = atan2(2*(q0*q3-q1*q2), q0*q0+q1*q1-q2*q2-q3*q3);
+    
+    euler = [pitch, yaw, roll];

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/TangentSpaceTransform.m

@@ -0,0 +1,17 @@
+function [ transformedX, transformedY, transformedZ ] = TangentSpaceTransform( x, y, z, meanShape )
+%TANGENTSPACETRANSFORM Summary of this function goes here
+%   Detailed explanation goes here
+
+    scaling = [ x y z] * [ meanShape(:,1)' meanShape(:,2)' meanShape(:,3)']';
+    for i=1:size(x,1)
+        x(i,:) = x(i,:) * (1 / scaling(i));
+        y(i,:) = y(i,:) * (1 / scaling(i));
+        z(i,:) = z(i,:) * (1 / scaling(i));
+    end
+    
+    transformedX = x * mean(scaling);
+    transformedY = y * mean(scaling);
+    transformedZ = z * mean(scaling);
+    
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/fit_PDM_ortho_proj_to_2D.m

@@ -0,0 +1,345 @@
+function [ a, R, T, T3D, params, error, shapeOrtho ] = fit_PDM_ortho_proj_to_2D( M, E, V, shape2D, f, cx, cy)
+%FITPDMTO2DSHAPE Summary of this function goes here
+%   Detailed explanation goes here
+
+    params = zeros(size(E));
+
+    hidden = false;
+
+    % if some of the points are unavailable modify M, V, and shape2D (can
+    % later infer the actual shape from this)
+    if(sum(shape2D(:)==0) > 0)        
+
+        hidden = true;
+        % which indices to remove
+        inds_to_rem = shape2D(:,1) == 0 | shape2D(:,2) == 0;
+
+        shape2D = shape2D(~inds_to_rem,:);
+
+        inds_to_rem = repmat(inds_to_rem, 3, 1);
+
+        M_old = M;
+        V_old = V;
+
+        M = M(~inds_to_rem);
+        V = V(~inds_to_rem,:);
+
+    end
+    
+    num_points = numel(M) / 3;
+    
+    m = reshape(M, num_points, 3)';
+    width_model = max(m(1,:)) - min(m(1,:));
+    height_model = max(m(2,:)) - min(m(2,:));
+
+    bounding_box = [min(shape2D(:,1)), min(shape2D(:,2)),...
+                    max(shape2D(:,1)), max(shape2D(:,2))];
+    
+    a = (((bounding_box(3) - bounding_box(1)) / width_model) + ((bounding_box(4) - bounding_box(2))/ height_model)) / 2;
+        
+    tx = (bounding_box(3) + bounding_box(1))/2;
+    ty = (bounding_box(4) + bounding_box(2))/2;
+    
+    % correct it so that the bounding box is just around the minimum
+    % and maximum point in the initialised face
+    tx = tx - a*(min(m(1,:)) + max(m(1,:)))/2;
+    ty = ty - a*(min(m(2,:)) + max(m(2,:)))/2;    
+    
+    R = eye(3); 
+    T = [tx; ty];
+    
+    currShape = getShapeOrtho(M, V, params, R, T, a);
+    
+    currError = getRMSerror(currShape, shape2D);
+    
+    reg_rigid = zeros(6,1);
+    regFactor = 20;
+    regularisations = [reg_rigid; regFactor ./ E]; % the above version, however, does not perform as well
+    regularisations = diag(regularisations)*diag(regularisations);
+    
+    red_in_a_row = 0;
+    
+    for i=1:1000
+                      
+        shape3D = M + V * params;
+        shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
+                
+        % Now find the current residual error        
+        currShape = a * R(1:2,:)*shape3D' + repmat(T, 1, numel(M)/3); 
+        currShape = currShape'; 
+        
+        error_res = shape2D - currShape;
+        
+        eul = Rot2Euler(R);
+        
+        p_global = [a; eul'; T];
+        
+        % get the Jacobians
+        J = CalcJacobian(M, V, params, p_global);
+        
+        % RLMS style update
+        p_delta = (J'*J + regularisations) \ (J'*error_res(:) - regularisations*[p_global;params]);
+                                
+        [params, p_global] = CalcReferenceUpdate(p_delta, params, p_global);
+        
+        a = p_global(1);
+        R = Euler2Rot(p_global(2:4));
+        T = p_global(5:6);
+        
+        shape3D = M + V * params;
+        shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
+        currShape = a * R(1:2,:)*shape3D' + repmat(T, 1, numel(M)/3); 
+        currShape = currShape'; 
+             
+        error = getRMSerror(currShape, shape2D);
+        
+        if(0.999 * currError < error)
+            red_in_a_row = red_in_a_row + 1;
+            if(red_in_a_row == 5)
+                break;
+            end
+        end
+        
+        currError = error;
+        
+    end    
+    
+    if(hidden)
+        shapeOrtho = getShapeOrtho(M_old, V_old, params, R, T, a);
+    else
+        shapeOrtho = currShape;
+    end
+    if(nargin == 7)
+    
+        Zavg = f / a;
+        Xavg = (T(1) - cx) / a;
+        Yavg = (T(2) - cy) / a;
+
+        T3D = [Xavg;Yavg;Zavg];
+    else
+        T3D = [0;0;0];
+    end
+    
+end
+
+function [shape2D] = getShapeOrtho(M, V, p, R, T, a)
+
+    % M - mean shape vector
+    % V - eigenvectors
+    % p - parameters of non-rigid shape
+    % R - rotation matrix
+    % T - translation vector (tx, ty)
+    shape3D = getShape3D(M, V, p);
+    shape2D = a * R(1:2,:)*shape3D' + repmat(T, 1, numel(M)/3);
+    shape2D = shape2D';
+end
+
+function [shape2D] = getShapeOrthoFull(M, V, p, R, T, a)
+
+    % M - mean shape vector
+    % V - eigenvectors
+    % p - parameters of non-rigid shape
+    % R - rotation matrix
+    % T - translation vector (tx, ty)    
+    T = [T; 0];
+    shape3D = getShape3D(M, V, p);
+    shape2D = a * R*shape3D' + repmat(T, 1, numel(M)/3);
+    shape2D = shape2D';
+end
+
+function [shape3D] = getShape3D(M, V, params)
+
+    shape3D = M + V * params;
+    shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
+    
+end
+
+function [error] = getRMSerror(shape2Dv1, shape2Dv2)
+
+    error = sqrt(mean(reshape(shape2Dv1 - shape2Dv2, numel(shape2Dv1), 1).^2));
+
+end
+
+% This calculates the combined rigid with non-rigid Jacobian
+function J = CalcJacobian(M, V, p, p_global)
+
+    n = size(M, 1)/3;
+    
+    non_rigid_modes = size(V,2);
+    
+    J = zeros(n*2, 6 + non_rigid_modes);
+    
+    
+    % now the layour is
+    % ---------- Rigid part -------------------|----Non rigid part--------|
+    % dx_1/ds, dx_1/dr1, ... dx_1/dtx, dx_1/dty dx_1/dp_1 ... dx_1/dp_m
+    % dx_2/ds, dx_2/dr1, ... dx_2/dtx, dx_2/dty dx_2/dp_1 ... dx_2/dp_m
+    % ...
+    % dx_n/ds, dx_n/dr1, ... dx_n/dtx, dx_n/dty dx_n/dp_1 ... dx_n/dp_m
+    % dy_1/ds, dy_1/dr1, ... dy_1/dtx, dy_1/dty dy_1/dp_1 ... dy_1/dp_m
+    % ...
+    % dy_n/ds, dy_n/dr1, ... dy_n/dtx, dy_n/dty dy_n/dp_1 ... dy_n/dp_m
+    
+    % getting the rigid part
+    J(:,1:6) = CalcRigidJacobian(M, V, p, p_global);
+    
+    % constructing the non-rigid part
+    R = Euler2Rot(p_global(2:4));
+    s = p_global(1);
+    
+    % 'rotate' and 'scale' the principal components
+    
+    % First reshape to 3D
+    V_X = V(1:n,:);
+    V_Y = V(n+1:2*n,:);
+    V_Z = V(2*n+1:end,:);
+    
+    J_x_non_rigid = s*(R(1,1)*V_X + R(1,2)*V_Y + R(1,3)*V_Z);
+    J_y_non_rigid = s*(R(2,1)*V_X + R(2,2)*V_Y + R(2,3)*V_Z);
+    
+    J(1:n, 7:end) = J_x_non_rigid;
+    J(n+1:end, 7:end) = J_y_non_rigid;
+    
+end
+
+function J = CalcRigidJacobian(M, V, p, p_global)
+
+ 	n = size(M, 1)/3;
+  
+	% Get the current 3D shape (not affected by global transform, as this
+	% is how the Jacobian was derived (for derivation please see
+	% ../derivations/orthoJacobian
+	shape3D = GetShape3D(M, V, p);
+
+	% Get the rotation matrix corresponding to current global orientation
+	R = Euler2Rot(p_global(2:4));
+	s = p_global(1);
+    
+    % Rigid Jacobian is laid out as follows
+    % dx_1/ds, dx_1/dr1, dx_1/dr2, dx_1/dr3, dx_1/dtx, dx_1/dty
+    % dx_2/ds, dx_2/dr1, dx_2/dr2, dx_2/dr3, dx_2/dtx, dx_2/dty
+    % ...
+    % dx_n/ds, dx_n/dr1, dx_n/dr2, dx_n/dr3, dx_n/dtx, dx_n/dty
+    % dy_1/ds, dy_1/dr1, dy_1/dr2, dy_1/dr3, dy_1/dtx, dy_1/dty
+    % ...
+    % dy_n/ds, dy_n/dr1, dy_n/dr2, dy_n/dr3, dy_n/dtx, dy_n/dty
+    
+    J = zeros(n*2, 6);
+    
+    % dx/ds = X * r11  + Y * r12 + Z * r13
+    % dx/dr1 =  s*(r13 * Y - r12 * Z)
+    % dx/dr2 = -s*(r13 * X - r11 * Z)
+    % dx/dr3 =  s*(r12 * X - r11 * Y)
+    % dx/dtx = 1
+    % dx/dty = 0
+    
+    % dy/ds = X * r21  + Y * r22 + Z * r23
+    % dy/dr1 =  s * (r23 * Y - r22 * Z)
+    % dy/dr2 = -s * (r23 * X - r21 * Z)
+    % dy/dr3 =  s * (r22 * X - r21 * Y)
+    % dy/dtx = 0
+    % dy/dty = 1
+        
+    % set the Jacobian for x's
+    
+    % with respect to scaling factor
+    J(1:n,1) = shape3D * R(1,:)';
+    
+    % with respect to angular rotation around x, y, and z axes
+    
+    % Change of x with respect to change in axis angle rotation
+	dxdR = [      0,  R(1,3), -R(1,2);
+            -R(1,3),       0,  R(1,1);
+             R(1,2), -R(1,1),      0];
+
+    J(1:n,2:4) = s*(dxdR * shape3D')';
+         
+    % with respect to translation
+    J(1:n,5) = 1;
+    J(1:n,6) = 0;
+    
+    % set the Jacobian for y's
+
+    % with respect to scaling factor
+    J(n+1:end,1) = shape3D * R(2,:)';
+
+     % with respect to angular rotation around x, y, and z axes
+    
+    % Change of y with respect to change in axis angle rotation
+    dydR = [      0,  R(2,3), -R(2,2);
+            -R(2,3),       0,  R(2,1);
+             R(2,2), -R(2,1),      0];
+         
+     J(n+1:end,2:4) = s*(dydR * shape3D')';
+
+     % with respect to translation
+     J(n+1:end,5) = 0;
+     J(n+1:end,6) = 1;
+
+end
+
+% This updates the parameters based on the updates from the RLMS
+function [non_rigid, rigid] = CalcReferenceUpdate(params_delta, current_non_rigid, current_global)
+
+
+    rigid = zeros(6, 1);
+    % Same goes for scaling and translation parameters
+    rigid(1) = current_global(1) + params_delta(1);
+    rigid(5) = current_global(5) + params_delta(5);
+    rigid(6) = current_global(6) + params_delta(6);
+    
+    % for rotation however, we want to make sure that the rotation matrix
+    % approximation we have 
+	% R' = [1, -wz, wy
+	%       wz, 1, -wx
+	%       -wy, wx, 1]	
+    % is a legal rotation matrix, and then we combine it with current
+    % rotation (through matrix multiplication) to acquire the new rotation
+    
+	R = Euler2Rot(current_global(2:4));
+
+    wx = params_delta(2);
+    wy = params_delta(3);
+    wz = params_delta(4);
+    
+    R_delta = [1, -wz, wy;
+               wz, 1, -wx;
+               -wy, wx, 1];
+	
+	% Make sure R_delta is orthonormal
+	R_delta = OrthonormaliseRotation(R_delta);
+	
+    % Combine rotations
+	R_final = R * R_delta;
+
+	% Extract euler angle
+	euler = Rot2Euler(R_final);	
+	
+	rigid(2:4) = euler;
+    
+    if(length(params_delta) > 6)
+        % non-rigid parameters can just be added together
+        non_rigid = params_delta(7:end) +  current_non_rigid;
+    else
+        non_rigid = current_non_rigid;
+    end
+    
+end
+
+function R_ortho = OrthonormaliseRotation(R)
+  
+    % U * V' is basically what we want, as it's guaranteed to be
+    % orthonormal
+    [U, ~, V] = svd(R);
+
+    % We also want to make sure no reflection happened
+    
+    % get the orthogonal matrix from the initial rotation matrix
+    X = U*V';
+
+    % This makes sure that the handedness is preserved and no reflection happened
+    % by making sure the determinant is 1 and not -1
+    W = eye(3);
+    W(3,3) = det(X);
+    R_ortho = U*W*V';
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/writeMatrix.m

@@ -0,0 +1,16 @@
+% for easier readibility write them row by row
+function writeMatrix(fileID, M, type)
+
+    fprintf(fileID, '%d\r\n', size(M,1));
+    fprintf(fileID, '%d\r\n', size(M,2));
+    fprintf(fileID, '%d\r\n', type);
+    
+    for i=1:size(M,1)
+        if(type == 4 || type == 0)
+            fprintf(fileID, '%d ', M(i,:));
+        else
+            fprintf(fileID, '%.9f ', M(i,:));
+        end
+        fprintf(fileID, '\r\n');
+    end
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_COFW.m

@@ -0,0 +1,195 @@
+% Convert the training images into a suitable format
+function Prepare_data_COFW()
+
+    % replace with folder where you downloaded and extracted the 300-W challenge data
+    data_root = '../../../../CLM-framework/matlab_version/occlusion_experiments/data/';
+    
+    PrepareTrainingWild(data_root, 0.25);
+    PrepareTrainingWild(data_root, 0.35);
+    PrepareTrainingWild(data_root, 0.5);
+end
+
+function PrepareTrainingWild( data_root, training_scale )
+%PREPARETRAININGIMAGEMPIE Summary of this function goes here
+%   Detailed explanation goes here
+
+    % img size
+
+    imgSize = [400, 400] * training_scale;
+    
+    %%
+    addpath('PDM_helpers/');
+    load 'PDM_helpers/pdm_29_cofw_aligned.mat';
+
+    output_location = '../prepared_data/cofw_';
+    output_location = [output_location num2str(training_scale,3)];
+
+    num_landmarks = 29;
+    
+    % Use mirror images to provide extra training data
+    mirror_inds = [1,2; 3,4; 5,7; 6,8; 9,10; 11,12; 13,15; 14,16; 17,18; 19,20; 23,24];
+
+    % The centres of views we want to extract
+    centres_all = [ 0, 0, 0 ];
+    
+    num_centers = size(centres_all, 1);
+            
+    counter_colour = zeros(num_centers,1);
+
+    load([data_root, '/COFW_train.mat']);
+                 
+    % go through all images, see which centres they match and then add to
+    % the appropriate bin
+    num_imgs = numel(IsTr);
+    scales = zeros(num_imgs, 1);
+    views = zeros(num_imgs, 1);
+    for lbl=1:num_imgs
+       
+        Msm = M;
+        Vsm = V;
+        labels = reshape(phisTr(lbl,1:num_landmarks*2), num_landmarks, 2);
+        
+        % Find the best PDM parameters given the 2D labels
+        [ a, R] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
+        
+        eul = Rot2Euler(R);
+        eul = eul * 180 / pi;
+        
+        % find the closest view
+        [~, view] = min(sum(abs(centres_all - repmat(eul, num_centers, 1)),2));
+        counter_colour(view) = counter_colour(view) + 1;
+        scales(lbl) = a;
+        views(lbl) = view;
+    end
+    
+    % preallocate data
+    allExamplesColourAllViews = cell(size(centres_all,1),1);    
+    landmarkLocationsAllViews = cell(size(centres_all,1),1); 
+    occlusionsAllViews = cell(size(centres_all,1),1); 
+    
+    for r=1:size(centres_all,1)
+
+        allExamplesColourAllViews{r} = zeros(counter_colour(r), imgSize(1), imgSize(2));
+        landmarkLocationsAllViews{r} = zeros(counter_colour(r), num_landmarks, 2);
+        occlusionsAllViews{r} = zeros(counter_colour(r), num_landmarks);
+        actual_imgs_used_all_views{r} = cell(counter_colour(r), 1);
+    end
+    
+    counter_colour = zeros(num_centers,1);
+    
+    % go through all images and add to corresponding container
+    for lbl=1:num_imgs                   
+
+        labels = reshape(phisTr(lbl,1:num_landmarks*2), num_landmarks, 2);
+        occlusions = phisTr(lbl,(num_landmarks*2+1):end);
+        
+        imgCol = IsTr{lbl};
+
+        if(size(imgCol,3) == 3)
+            imgCol = rgb2gray(imgCol);
+        end
+        
+        % resize the image to desired scale
+        scalingFactor = training_scale / scales(lbl);
+
+        resizeColImage = imresize(imgCol, scalingFactor, 'bilinear');
+
+        labels = labels * scalingFactor;
+
+        % we want to crop out the image now
+        meanX = round(mean(labels(:,1)));
+        meanY = round(mean(labels(:,2)));
+        startX = round(meanX-imgSize(1)/2);            
+        startY = round(meanY-imgSize(2)/2);            
+
+        if(startX < 1)
+            startX = 1;
+        end
+        if(startY < 1) 
+            startY = 1;
+        end
+
+        endX = startX + imgSize(1) - 1;
+        endY = startY + imgSize(2) - 1;
+
+        if(endX > size(resizeColImage,2))
+            resizeColImage = cat(2, resizeColImage, zeros(size(resizeColImage,1), endX - size(resizeColImage,2)));
+        end
+
+        if(endY > size(resizeColImage,1))
+            resizeColImage = cat(1, resizeColImage, zeros(endY - size(resizeColImage,1), size(resizeColImage,2)));
+        end
+        resizeColImage = resizeColImage(startY:endY,startX:endX);        
+
+        labels(:,1) = labels(:,1) - startX + 1;
+        labels(:,2) = labels(:,2) - startY + 1;
+        
+        counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
+
+        allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = resizeColImage;
+
+        landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = labels;
+        occlusionsAllViews{views(lbl)}(counter_colour(views(lbl)),:) = occlusions;
+        actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = 'placeholder';
+        
+        if(mod(lbl, 100) == 0)
+            fprintf('%d/%d done\n', lbl, num_imgs);
+        end
+    end
+    
+    % write out the training data for each view
+    for r=1:size(centres_all,1)
+        mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
+        
+        % if the mirrored view already added no need to do it again
+        % so if (0,-20,0) is done no need to compute (0,20,0)
+        if(mirrorIdx < r)
+            continue
+        end
+        
+        mirrorImgs = allExamplesColourAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
+        mirrorLbls = landmarkLocationsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
+        mirrorOccls = occlusionsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:);
+        
+        for i=1:size(mirrorImgs,1)
+           
+            flippedImg = fliplr(squeeze(mirrorImgs(i,:,:)));
+                    
+            flippedLbls = squeeze(mirrorLbls(i,:,:));
+            flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1);
+            
+            tmp1 = flippedLbls(mirror_inds(:,1),:);
+            tmp2 = flippedLbls(mirror_inds(:,2),:);            
+            flippedLbls(mirror_inds(:,2),:) = tmp1;
+            flippedLbls(mirror_inds(:,1),:) = tmp2;   
+        
+            mirrorImgs(i,:,:) = flippedImg;
+            mirrorLbls(i,:,:) = flippedLbls;
+            
+            tmp1 = mirrorOccls(i,mirror_inds(:,1));
+            tmp2 = mirrorOccls(i,mirror_inds(:,2));
+            
+            mirrorOccls(i,mirror_inds(:,1)) = tmp2;
+            mirrorOccls(i,mirror_inds(:,2)) = tmp1;
+        end
+        
+        all_images = cat(1, allExamplesColourAllViews{r}(1:counter_colour(r),:,:), mirrorImgs);
+        
+        all_images = uint8(all_images);
+        
+        landmark_locations = cat(1, landmarkLocationsAllViews{r}(1:counter_colour(r),:,:), mirrorLbls);
+        
+        occlusions = cat(1, occlusionsAllViews{r}(1:counter_colour(r),:), mirrorOccls);
+        
+        actual_imgs_used = actual_imgs_used_all_views{r};
+        actual_imgs_used = cat(1, actual_imgs_used_all_views{r}, actual_imgs_used_all_views{mirrorIdx});
+        
+        centres = centres_all(r,:);
+
+        visiIndex = ones(1,num_landmarks);
+        save([output_location '_' num2str(r) '.mat'], 'all_images', 'occlusions', 'landmark_locations', 'training_scale', 'actual_imgs_used', 'centres', 'visiIndex', '-v7.3');
+
+    end 
+        
+                
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_Multi_PIE_all.m

@@ -0,0 +1,499 @@
+function Prepare_data_Multi_PIE_all()
+
+    % This bit collects all of the multi-pie labels into a single structure for
+    % easy access
+    labels_root = 'C:\Users\tbaltrus\Dropbox\AAM\test data\MultiPI_AAM/';
+    
+    % The location of the Multi-PIE data folder
+    multi_pie_root = 'D:\MultiPIE/Image_Data/';
+    
+    multi_pie_labels = CollectMultiPieLabels(labels_root, multi_pie_root);
+
+    %% 
+    
+    % Make sure same images are used across scales
+    rng(0);    
+    ExtractTrainingMultiPIE(0.25, multi_pie_labels);
+    
+    rng(0);    
+    ExtractTrainingMultiPIE(0.35, multi_pie_labels);
+    
+    rng(0);    
+    ExtractTrainingMultiPIE(0.5, multi_pie_labels);
+        
+    rng(0);    
+    ExtractTrainingMultiPIE(1.0, multi_pie_labels);    
+    
+end
+% Now extract the relevant information
+
+function [multi_pie_labels] = CollectMultiPieLabels(labels_root, multi_pie_root)
+
+    multi_pie_labels = struct;
+    currentLabel = 0;
+    left_to_frontal_map = [1, 28;
+                        2, 29;
+                        3, 30;
+                        4, 31;
+                        5, 34;
+                        6, 36;
+                        7, 27;
+                        8, 26;
+                        9, 25;
+                        10,24;
+                        11,46;
+                        12,45;
+                        13,44;
+                        14,48;
+                        15,47;
+                        16,52;
+                        17,53;
+                        18,54;
+                        19,55;
+                        20,56;
+                        21,57;
+                        22,58;
+                        23,63;
+                        24,64;
+                        26,66;
+                        27,67;
+                        30,9;
+                        31,10;
+                        32,11;
+                        33,12;
+                        34,13;
+                        35,14;
+                        36,15;
+                        37,16;
+                        38,17;];
+
+    right_to_frontal_map = [1, 28;
+                            2, 29;
+                            3, 30;
+                            4, 31;
+                            5, 34;
+                            6, 32;
+                            7 ,18;
+                            8 ,19;
+                            9 ,20;
+                            10,21;
+                            11,37;
+                            12,38;
+                            13,39;
+                            14,41;
+                            15,42;
+                            16,52;
+                            17,51;
+                            18,50;
+                            19,49;
+                            20,60;
+                            21,59;
+                            22,58;
+                            23,63;
+                            24,62;
+                            26,68;
+                            27,67;
+                            30, 9;
+                            31, 8;
+                            32, 7;
+                            33, 6;
+                            34, 5;
+                            35, 4;
+                            36, 3;
+                            37, 2;
+                            38, 1];
+
+    for i=1:4
+
+        labelsDir = sprintf('%sSession%d/', labels_root, i);
+
+        labels = dir([labelsDir '/*.mat']);
+
+        double_label = false;
+
+        for j = 1:numel(labels)
+            if(double_label)
+                double_label = false;
+                continue;
+            end
+
+            load([labelsDir labels(j).name]);
+
+            if(size(pts,1) ~= 68 && size(pts,1) ~= 39)
+                continue;
+            end
+
+            userID = labels(j).name(1:3);        
+            recID = labels(j).name(8:9);        
+            camID = labels(j).name(11:12);        
+            camID2 = labels(j).name(13);
+            viewID = labels(j).name(15:16);
+
+            % doubling labels seem to be odd/incorrect
+            if(j < numel(labels))
+
+                userIDN = labels(j+1).name(1:3);            
+                recIDN = labels(j+1).name(8:9);            
+                camIDN = labels(j+1).name(11:12);            
+                camID2N = labels(j+1).name(13);
+
+                if(strcmp(userIDN, userID) && strcmp(recID, recIDN) && strcmp(camIDN, camID) && strcmp(camID2N, camID2))
+                    double_label = true;
+                    continue;
+                end
+
+            end
+
+            % camera id 08 is from a very different perspective
+            if(strcmp(camID, '08') && strcmp(camID2, '1'))
+                continue;
+            end
+
+            currentLabel = currentLabel + 1;
+
+            multi_pie_labels(currentLabel).label_file = [labelsDir labels(j).name];
+            multi_pie_labels(currentLabel).user_ID = userID;
+            multi_pie_labels(currentLabel).rec_ID = recID;
+            multi_pie_labels(currentLabel).cam_ID = camID;
+            multi_pie_labels(currentLabel).cam_ID2 = camID2;
+
+            multiPieImageDir = sprintf('%s/session0%d/multiview/%s/%s/%s_%s/', multi_pie_root, i,userID,recID,camID,camID2);
+            multiPieImgs = dir([multiPieImageDir '*.png']);
+
+            multi_pie_labels(currentLabel).img_dir = multiPieImageDir;
+    %         multi_pie_labels(currentLabel).img_loc = [labels(j+1).name(1:end-7), '.png'];        
+            actualLabel = dir([multiPieImageDir '/*' viewID '.png']);
+            multi_pie_labels(currentLabel).img_locs = {multiPieImgs.name};
+            multi_pie_labels(currentLabel).actual_img = actualLabel.name;
+            landmark_labels = zeros(68,2);        
+
+            % This is a profile image
+            if(size(pts,1) == 39)
+                % Determine if left or right
+
+                if(pts(4,1) < pts(39,1))
+                    multi_pie_labels(currentLabel).type = 'profile_left';
+                    landmark_labels(left_to_frontal_map(:,2),:) = pts(left_to_frontal_map(:,1),:);
+                else
+                    multi_pie_labels(currentLabel).type = 'profile_right';
+                    landmark_labels(right_to_frontal_map(:,2),:) = pts(right_to_frontal_map(:,1),:);
+                end
+            else
+                landmark_labels = pts;
+            end
+
+            multi_pie_labels(currentLabel).landmark_labels = landmark_labels;
+
+        end
+    end
+end
+
+function ExtractTrainingMultiPIE( training_scale, multi_pie_labels)
+%PREPARETRAININGIMAGEMPIE This function collects a subset of Multi-PIE
+%images at different expressions and lighting conditions to store it in a
+%format suitable for training patch experts
+
+%   Detailed explanation goes here
+
+
+    img_size = [400, 400] * training_scale;
+    
+    %%
+    addpath('PDM_helpers/');
+    load 'PDM_helpers/pdm_68_aligned_wild.mat';
+
+    output_location = '../prepared_data/mpie_';
+    output_location = [output_location num2str(training_scale,3)];
+
+    num_landmarks = 68;
+    
+    % Use mirror images to provide extra training data
+    mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+                  32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+                  61,65;62,64;68,66];
+
+    % The centres of views we want to extract
+    centres_all = [   0,   0,  0
+                      0, -20,  0
+                      0, -45,  0
+                      0, -70,  0                      
+                      0,  20,  0
+                      0,  45,  0
+                      0,  70,  0                                            
+                      ];
+    
+    num_centers = size(centres_all, 1);
+            
+    counter_colour = zeros(num_centers,1);
+
+    % read in all of the labels, together with names of images used
+    
+    landmark_labels = zeros(68,2,numel(multi_pie_labels));
+    img_locations = cell(numel(multi_pie_labels),1);
+    extra_locations = cell(numel(multi_pie_labels),1);
+    img_dirs = cell(numel(multi_pie_labels),1);
+    
+    for i=1:numel(multi_pie_labels)
+               
+        img_locations{i} = [multi_pie_labels(i).img_dir, multi_pie_labels(i).actual_img];
+        extra_locations{i} =  multi_pie_labels(i).img_locs;        
+        img_dirs{i} =  multi_pie_labels(i).img_dir;
+        
+        landmark_labels(:,:,i) = multi_pie_labels(i).landmark_labels;
+        
+    end        
+    
+    % go through all images, see which centres they match and then add to
+    % the appropriate bin
+    num_imgs = size(landmark_labels,3);
+    scales = zeros(num_imgs, 1);
+    views = zeros(num_imgs, 1);
+    for lbl=1:num_imgs
+       
+        Msm = M;
+        Vsm = V;
+        labels = landmark_labels(:,:,lbl);
+        
+        % Find the best PDM parameters given the 2D labels
+        [ a,R ] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
+        
+        eul = Rot2Euler(R);        
+        eul_orig = eul * 180 / pi;
+        
+        if(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '24_0'))
+            eul = [0, -90, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '01_0'))
+            eul = [0, -75, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '20_0'))
+            eul = [0, -60, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '19_0'))
+            eul = [0, -45, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '04_1'))
+            eul = [0, -30, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '05_0'))
+            eul = [0, -15, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '05_1'))
+            eul = [0, 0, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '14_0'))
+            eul = [0, 15, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '13_0'))
+            eul = [0, 30, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '08_0'))            
+            eul = [0, 45, 0];            
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '09_0'))
+            eul = [0, 60, 0];
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '12_0'))
+            eul = [0, 75, 0];            
+        elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '11_0'))
+            eul = [0, 90, 0];
+        end
+        
+        % find the closest view (also mirror img?)
+        [~, view] = min(sum(abs(centres_all - repmat(eul, num_centers, 1)),2));
+        counter_colour(view) = counter_colour(view) + 1;
+        scales(lbl) = a;
+        views(lbl) = view;
+    end
+    
+    % preallocate data
+    allExamplesColourAllViews = cell(size(centres_all,1),1);    
+    landmarkLocationsAllViews = cell(size(centres_all,1),1); 
+    
+    % if we don't have enough labelled original images add ones from diff
+    % lighting conditions (that might not be perfectly labelled, but they
+    % are better than fewer images)
+    images_aim = 8000;
+    extra_factors = ones(size(counter_colour));
+    
+    for r=1:size(centres_all,1)
+
+         % see if extra is needed        
+        mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);        
+        count = counter_colour(r) + counter_colour(mirrorIdx);
+                        
+        if(count < images_aim)
+            extra_factors(r) = images_aim / count;
+        end
+        
+    end
+    
+    for r=1:size(centres_all,1)
+        counter_colour(r) = round(counter_colour(r) * extra_factors(r));
+            
+        allExamplesColourAllViews{r} = uint8(zeros(counter_colour(r), img_size(1), img_size(2)));
+        landmarkLocationsAllViews{r} = zeros(counter_colour(r), num_landmarks, 2);
+
+        actual_imgs_used_all_views{r} = cell(counter_colour(r), 1);                                    
+
+    end
+    
+    counter_colour = zeros(num_centers,1);
+    
+    % The shape fitting is performed in the reference frame of the
+    % patch training scale
+    refGlobal = [training_scale, 0, 0, 0, 0, 0]';
+    
+    % go through all images and add to corresponding container
+    for lbl=1:num_imgs                   
+
+        labels = landmark_labels(:,:,lbl);
+        occluded = labels(:,1) == 0;
+        
+        % Convert the labels to matlab format (we expect 1,1 to represent
+        % the center of the top left pixel)
+        labels = labels + 1;
+        
+        labels(occluded,:) = 0;
+                
+        imgCol = imread(img_locations{lbl});
+
+        if(size(imgCol,3) == 3)
+            imgCol = rgb2gray(imgCol);
+        end
+          
+        % the reference shape
+        [ ~, ~, ~,~, local_params] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
+        refShape = GetShapeOrtho(M, Vsm, local_params, refGlobal);
+        
+        % Create transform using a slightly modified version of Kabsch that
+        % takes scaling into account as well, in essence we get a
+        % similarity transform from current estimate to reference shape
+        [A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(labels(~occluded,:), refShape(~occluded,1:2));
+
+        T_img2ref = T_img2ref + [img_size(1)/2, img_size(2)/2];
+        
+        % Create a transform, from shape in image to reference shape
+        T = affine2d([A_img2ref';T_img2ref]);
+                
+        % transform the current shape to the reference one
+        shape2D_in_ref = bsxfun(@plus, (A_img2ref * labels')', T_img2ref);        
+        
+        % warp the image
+        resizeColImage = imwarp(imgCol, T, 'linear', 'OutputView', imref2d(img_size));             
+        
+        shape2D_in_ref(occluded,1) = 0;
+        shape2D_in_ref(occluded,2) = 0;
+        
+        counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
+
+        allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = resizeColImage;
+
+        landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = shape2D_in_ref;
+        actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = img_locations{lbl};
+        
+        % Here to add extra images if missing and not filled yet
+        if(extra_factors(views(lbl)) > 1 && counter_colour(views(lbl)) < numel(actual_imgs_used_all_views{views(lbl)}))
+           
+            factor = extra_factors(views(lbl));
+            
+            % pick if to use this image or not
+            if(randi(100) > (factor - floor(factor)) * 100)
+                factor = floor(factor);
+            else
+                factor = ceil(factor);
+            end
+            
+            while(factor > 1)
+                
+                lighting_id =  randi(20);
+                [~, img_orig, ext] = fileparts(img_locations{lbl});
+                img_orig = [img_orig, ext];
+                if( strcmp(img_orig, extra_locations{lbl}{lighting_id}))
+                    lighting_id = lighting_id + 1;
+                    if(lighting_id > 20)
+                        lighting_id = 1;
+                    end
+                end
+                img_loc = [img_dirs{lbl}, extra_locations{lbl}{lighting_id}];
+                imgCol = imread(img_loc);
+
+                if(size(imgCol,3) == 3)
+                    imgCol = rgb2gray(imgCol);
+                end
+
+                % resize the image to desired scale
+                resizeColImage = imwarp(imgCol, T, 'linear', 'OutputView', imref2d(img_size));   
+
+                counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
+
+                allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = resizeColImage;
+
+                landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = shape2D_in_ref;
+                actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = img_loc;
+                factor = factor - 1;
+            end
+            
+        end
+        % Make sure same one not added as well
+        
+        if(mod(lbl, 100) == 0)
+            fprintf('%d/%d done\n', lbl, num_imgs);
+        end
+        
+    end
+    
+    % write out the training data for each view
+    for r=1:size(centres_all,1)
+        mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
+        
+        % if the mirrored view already added no need to do it again
+        % so if (0,-20,0) is done no need to compute (0,20,0)
+        if(mirrorIdx < r)
+            continue
+        end
+        
+        % Cap to two thousand images for space reasons, and because more
+        % wouldn't actually be used        
+        max_images = 2000;
+        
+        mirrorImgs = allExamplesColourAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
+        mirrorLbls = landmarkLocationsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
+
+        for i=1:size(mirrorImgs,1)
+
+            flippedImg = fliplr(squeeze(mirrorImgs(i,:,:)));
+
+            flippedLbls = squeeze(mirrorLbls(i,:,:));
+            flippedLbls(:,1) = img_size(1) - flippedLbls(:,1) + 1;
+
+            tmp1 = flippedLbls(mirror_inds(:,1),:);
+            tmp2 = flippedLbls(mirror_inds(:,2),:);            
+            flippedLbls(mirror_inds(:,2),:) = tmp1;
+            flippedLbls(mirror_inds(:,1),:) = tmp2;   
+
+            mirrorImgs(i,:,:) = flippedImg;
+            mirrorLbls(i,:,:) = flippedLbls;
+
+            % Ensure that after mirroring invalid (occluded) feature points
+            % are set to 0 in both x and y dims
+            mirrorLbls(i,flippedLbls(:,2)==0,1) = 0;
+
+        end
+
+        all_images = cat(1, allExamplesColourAllViews{r}(1:counter_colour(r),:,:), mirrorImgs);        
+        landmark_locations = cat(1, landmarkLocationsAllViews{r}(1:counter_colour(r),:,:), mirrorLbls);           
+        actual_imgs_used = cat(1, actual_imgs_used_all_views{r}(1:counter_colour(r)), actual_imgs_used_all_views{mirrorIdx}(1:counter_colour(mirrorIdx)));  
+          
+        centres = centres_all(r,:);
+
+        % identify the visibility of a point
+        num_visible = sum(landmark_locations(:,:,1)~=0);
+        visible_max = max(num_visible);
+
+        visiIndex = ones(1,68);
+        visiIndex(num_visible < 0.5*visible_max) = 0;
+                
+        if(size(all_images,1) > max_images)
+            im_to_select = randperm(size(all_images,1));
+            im_to_select = im_to_select(1:max_images);
+            all_images = all_images(im_to_select,:,:);
+            landmark_locations = landmark_locations(im_to_select,:,:);
+            actual_imgs_used = actual_imgs_used(im_to_select);
+        end
+        save([output_location '_' num2str(r) '.mat'], 'all_images', 'landmark_locations', 'training_scale', 'centres', 'actual_imgs_used', 'visiIndex', '-v7.3');
+
+    end 
+        
+                
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_combined_all.m

@@ -0,0 +1,48 @@
+function Prepare_data_combined_all()
+
+    % Read in in-the-wild data and Multi-PIE data and concatenate them
+    root = '../prepared_data/';
+    Collect_combined_data(root, '0.25');
+    Collect_combined_data(root, '0.35');
+    Collect_combined_data(root, '0.5');
+    Collect_combined_data(root, '1');
+end
+
+function Collect_combined_data(root, scale)
+
+    wild_locs = dir(sprintf('%s/wild_%s*.mat', root, scale));
+    mpie_locs = dir(sprintf('%s/mpie_%s*.mat', root, scale));
+    
+    % For a particular loaded m_pie check if an appropriate in-the-wild
+    % exists
+    for i=1:numel(mpie_locs)
+       
+        load([root, mpie_locs(i).name]);
+        
+        imgs_used_pie = actual_imgs_used;
+        samples_pie = all_images;
+        centres_pie = centres;
+        landmark_locations_pie = landmark_locations;
+        
+        wild_to_use = -1;
+        for j=1:numel(wild_locs)
+            load([root, wild_locs(j).name], 'centres');
+            if(isequal(centres_pie, centres))
+                wild_to_use = j;
+            end                
+        end
+        
+        % Reset the centres
+        centres = centres_pie;
+        
+        if(wild_to_use ~= -1)
+            load([root, wild_locs(wild_to_use).name]);
+            actual_imgs_used = cat(1, actual_imgs_used, imgs_used_pie);
+            all_images = cat(1, all_images, samples_pie);
+            landmark_locations = cat(1, landmark_locations, landmark_locations_pie);
+        end
+        
+        save(sprintf('%s/combined_%s_%d.mat', root, scale, i), 'actual_imgs_used', 'all_images', 'centres', 'landmark_locations', 'training_scale', 'visiIndex');
+    end
+    
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_wild_all.m

@@ -0,0 +1,198 @@
+% Convert the training images into a suitable format
+function Prepare_data_wild_all()
+
+    % replace with folder where you downloaded and extracted the 300-W challenge data
+    data_root = 'C:\Users\tbaltrus\Dropbox\AAM/test data/';
+    
+    PrepareTrainingWild(data_root, 0.25);
+    PrepareTrainingWild(data_root, 0.35);
+    PrepareTrainingWild(data_root, 0.5);
+    PrepareTrainingWild(data_root, 1.0);
+end
+
+function PrepareTrainingWild( data_root, training_scale )
+%PREPARETRAININGIMAGEMPIE Summary of this function goes here
+%   Detailed explanation goes here
+
+    % img size
+
+    imgSize = [400, 400] * training_scale;
+    
+    %%
+    addpath('PDM_helpers/');
+    load 'PDM_helpers/pdm_68_aligned_wild.mat';
+
+    output_location = '../prepared_data/wild_';
+    output_location = [output_location num2str(training_scale,3)];
+
+    num_landmarks = 68;
+    
+    % Use mirror images to provide extra training data
+    mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+                  32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+                  61,65;62,64;68,66];
+
+    % The centres of views we want to extract
+    centres_all = [    0,   0,  0
+                      0, -20,  0
+                      0,  20,  0
+                      ];
+    
+    num_centers = size(centres_all, 1);
+            
+    counter_colour = zeros(num_centers,1);
+
+    % use only 2 of the subsets (others used for testing)
+    % You just need to provide the location in your system
+    dataset_locs = { [data_root, '/helen/trainset/'];
+                     [data_root, '/lfpw/trainset/']};
+    
+    landmark_labels = [];
+    img_locations = {};
+    
+    % read in all of the labels, together with names of images used
+    for i=1:numel(dataset_locs)
+        curr_labels = dir([dataset_locs{i} '\*.pts']);
+        imgs = dir([dataset_locs{i} '\*.jpg']);
+        if(isempty(imgs))
+            imgs = dir([dataset_locs{i} '\*.png']);
+        end
+        
+        for p=1:numel(curr_labels)
+            landmarks = dlmread([dataset_locs{i}, curr_labels(p).name], ' ', [3,0,68+2,1]);
+            landmark_labels = cat(3, landmark_labels, landmarks);
+            img_locations = cat(1, img_locations, [dataset_locs{i} imgs(p).name]);
+        end
+    end
+                 
+    % go through all images, see which centres they match and then add to
+    % the appropriate bin
+    num_imgs = size(landmark_labels,3);
+    scales = zeros(num_imgs, 1);
+    views = zeros(num_imgs, 1);
+    for lbl=1:num_imgs
+       
+        Msm = M;
+        Vsm = V;
+        labels = landmark_labels(:,:,lbl);
+        
+        % Find the best PDM parameters given the 2D labels
+        [ a, R] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
+        
+        eul = Rot2Euler(R);
+        eul = eul * 180 / pi;
+        
+        % find the closest view
+        [~, view] = min(sum(abs(centres_all - repmat(eul, num_centers, 1)),2));
+        counter_colour(view) = counter_colour(view) + 1;
+        scales(lbl) = a;
+        views(lbl) = view;
+    end
+    
+    % preallocate data
+    allExamplesColourAllViews = cell(size(centres_all,1),1);    
+    landmarkLocationsAllViews = cell(size(centres_all,1),1); 
+    
+    for r=1:size(centres_all,1)
+
+        allExamplesColourAllViews{r} = uint8(zeros(counter_colour(r), imgSize(1), imgSize(2)));
+        landmarkLocationsAllViews{r} = zeros(counter_colour(r), num_landmarks, 2);
+
+        actual_imgs_used_all_views{r} = cell(counter_colour(r), 1);        
+    end
+    
+    counter_colour = zeros(num_centers,1);
+    
+    % The shape fitting is performed in the reference frame of the
+    % patch training scale
+    refGlobal = [training_scale, 0, 0, 0, 0, 0]';
+        
+    % go through all images and add to corresponding container
+    for lbl=1:num_imgs                   
+
+        % shift the pixels to be centered on pixel as opposed to top left
+        labels = landmark_labels(:,:,lbl) - 0.5;
+        
+        imgCol = imread(img_locations{lbl});
+
+        if(size(imgCol,3) == 3)
+            imgCol = rgb2gray(imgCol);
+        end        
+        
+        % the reference shape
+        [ ~, ~, ~,~, local_params] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
+        refShape = GetShapeOrtho(M, Vsm, local_params, refGlobal);
+        
+        % Create transform using a slightly modified version of Kabsch that
+        % takes scaling into account as well, in essence we get a
+        % similarity transform from current estimate to reference shape
+        [A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(labels, refShape(:,1:2));
+
+        T_img2ref = T_img2ref + [imgSize(1)/2, imgSize(2)/2];
+        
+        % Create a transform, from shape in image to reference shape
+        T = affine2d([A_img2ref';T_img2ref]);
+                
+        % transform the current shape to the reference one
+        shape2D_in_ref = bsxfun(@plus, (A_img2ref * labels')', T_img2ref);        
+        
+        % warp the image
+        [warped_img] = imwarp(imgCol, T, 'linear', 'OutputView', imref2d(imgSize));
+                
+        counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
+
+        allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = warped_img;
+
+        landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = shape2D_in_ref;
+        actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = img_locations{lbl};
+        if(mod(lbl, 100) == 0)
+            fprintf('%d/%d done\n', lbl, num_imgs);
+        end
+    end
+    
+    % write out the training data for each view
+    for r=1:size(centres_all,1)
+        mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
+        
+        % if the mirrored view already added no need to do it again
+        % so if (0,-20,0) is done no need to compute (0,20,0)
+        if(mirrorIdx < r)
+            continue
+        end
+        
+        mirrorImgs = allExamplesColourAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
+        mirrorLbls = landmarkLocationsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
+        
+        for i=1:size(mirrorImgs,1)
+           
+            flippedImg = fliplr(squeeze(mirrorImgs(i,:,:)));
+                    
+            flippedLbls = squeeze(mirrorLbls(i,:,:));
+            flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1) + 1;
+            
+            tmp1 = flippedLbls(mirror_inds(:,1),:);
+            tmp2 = flippedLbls(mirror_inds(:,2),:);            
+            flippedLbls(mirror_inds(:,2),:) = tmp1;
+            flippedLbls(mirror_inds(:,1),:) = tmp2;   
+        
+            mirrorImgs(i,:,:) = flippedImg;
+            mirrorLbls(i,:,:) = flippedLbls;
+            
+        end
+        
+        all_images = cat(1, allExamplesColourAllViews{r}(1:counter_colour(r),:,:), mirrorImgs);
+        
+        landmark_locations = cat(1, landmarkLocationsAllViews{r}(1:counter_colour(r),:,:), mirrorLbls);
+        
+        actual_imgs_used = actual_imgs_used_all_views{r};
+        actual_imgs_used = cat(1, actual_imgs_used_all_views{r}, actual_imgs_used_all_views{mirrorIdx});
+        
+        centres = centres_all(r,:);
+
+        visiIndex = ones(1,68);
+        save([output_location '_' num2str(r) '.mat'], 'all_images', 'landmark_locations', 'training_scale', 'centres', 'actual_imgs_used', 'visiIndex', '-v7.3');
+
+    end 
+        
+                
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/readme.txt

@@ -0,0 +1,16 @@
+This code is in support of the "Convolutional experts constrained local model 
+for 3d facial landmark detection" paper by Amir Zadeh, Yao Chong Lim,
+Tadas Baltrusaitis, and Louis-Philippe Morency. The code is based on
+support code of the "Constrained Local Neural Fields for robust facial
+landmark detection in the wild" paper by Tadas Baltrusaitis, Peter Robinson, and Louis-Philippe Morency.
+
+This folder provides the code for generating patches used in training the patch experts.
+
+You have to have the relevant datasets, to run the code, the in-the-wild
+datasets can be found at http://ibug.doc.ic.ac.uk/resources/facial-point-annotations/),
+the annotations are provided. The Multi-PIE dataset can be acquired from -
+http://www.multipie.org/, you will need to ask the authors of the dataset for the annotations. 
+
+./data_preparation/ folder contains scripts to prepare data for patch generation
+
+./scripts/ contains the scripts to generate the patches for training.

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/ExtractTrainingSamples.m

@@ -0,0 +1,132 @@
+function [samples, labels, samples_unnormed, imgs_used] = ExtractTrainingSamples(examples, landmarkLoc, img_names, sigma, numSamples, landmark, normalisation_options)
+
+%%
+% for an area of interest of 19x19 and patch support region of 11x11, we
+% would have 9x9=81 samples (9 is the single_input_size, 11 is
+% patch_expert_support_size, 19x19 is normalisation_size, 9 would be the
+% normalisation_side_size)
+evaluation_size = normalisation_options.normalisationRegion;
+patch_expert_support_size = normalisation_options.patchSize;
+
+normalisation_side_size = (evaluation_size - 1)/2;
+single_input_size = evaluation_size - patch_expert_support_size + 1;
+
+% Determine the ratio of images to be sampled (most likely not all of them will be)
+samples_per_img = (numSamples / (size(examples,1) * (1 + normalisation_options.rate_negative))) / (single_input_size(1)^2);
+
+num_samples = int32(samples_per_img * (1 + normalisation_options.rate_negative) * size(examples,1) * (single_input_size(1)^2));
+
+%% Initialise the samples and labels
+samples = zeros(num_samples, patch_expert_support_size(1) * patch_expert_support_size(2));                
+labels = zeros(num_samples, 1);    
+
+%% Initialise the unnormed versions of the images
+
+% This is done in order to assert our use of algorithms for calculating
+% the responses, as for training we might use regular ml procedures,
+% whereas for fitting normalised cross-correlation or just
+% cross-correlation will be used, so keep some unnormed samples
+samples_unnormed = zeros(int32(num_samples/300), evaluation_size(1)^2);
+
+img_size = [size(examples,2), size(examples,3)];
+
+% Extract only images of differing shaped faces to extract more diverse
+% training samples
+to_keep = FindDistantLandmarks(landmarkLoc, landmark, round(samples_per_img*size(examples,1)));
+
+inds_all = 1:size(examples,1);
+samples_to_use = inds_all(to_keep);
+
+% Keep track of how many samples have been computed already
+samples_filled = 1;
+samples_unnormed_filled = 1;
+
+%% parse the image names for reporting purposes
+imgs_used = img_names(samples_to_use);
+for i=1:numel(imgs_used)
+    [~,name,ext] = fileparts(imgs_used{i});
+    imgs_used{i} = [name, ext];
+end
+for i=samples_to_use
+
+    % Do rate_negative negatives and a single positive
+    for p=1:normalisation_options.rate_negative+1
+
+        % create a gaussian
+        corrPoint = landmarkLoc(i,landmark,:);
+        
+        % Ignore occluded points
+        if(corrPoint(1) == 0)
+           break; 
+        end
+                
+        startX = 1 - corrPoint(1);
+        startY = 1 - corrPoint(2);
+
+        patchWidth = img_size(2);
+        patchHeight = img_size(1);
+
+        [X, Y] = meshgrid(startX:patchWidth + startX-1, startY:patchHeight + startY-1);
+
+        response = exp(-0.5*(X.^2+Y.^2)/(sigma^2));
+
+        % Choose positive or negative sample
+        if(p==normalisation_options.rate_negative+1)
+            sample_centre = squeeze(corrPoint) + round(1*randn(2,1));
+        else
+            sample_centre = squeeze(corrPoint) + round(10*randn(2,1));                
+        end
+
+        sample_centre = round(sample_centre);
+
+        sample_centre(sample_centre <= normalisation_side_size(1)) = normalisation_side_size(1) + 1;
+        sample_centre(sample_centre > img_size(1)-normalisation_side_size(1)) = img_size(1) - normalisation_side_size(1) - 1;
+
+        patches = squeeze(examples(i, sample_centre(2) - normalisation_side_size:sample_centre(2) + normalisation_side_size, sample_centre(1) - normalisation_side_size:sample_centre(1) + normalisation_side_size));
+        side = (single_input_size - 1)/2;
+        responses = response(sample_centre(2) - side(2):sample_centre(2) + side(2), sample_centre(1) - side(1):sample_centre(1) + side(1));
+
+        if(samples_unnormed_filled <= size(samples_unnormed,1))
+            % even if correct size is not initialised Matlab will
+            % sort that out (would only happen once anyway)
+            samples_unnormed(samples_unnormed_filled,:) = patches(:);
+            samples_unnormed_filled = samples_unnormed_filled + 1;
+        end
+
+        % if we want to normalise each patch individualy do it here
+
+        patch = im2col(patches, patch_expert_support_size, 'sliding')';
+        response = im2col(responses, [1,1], 'sliding');
+
+        labels(samples_filled:samples_filled+size(patch,1)-1,:) = response;
+
+        samples(samples_filled:samples_filled+size(patch,1)-1,:) = patch;                             
+        samples_filled = samples_filled + size(patch,1);           
+
+    end
+end
+
+if(normalisation_options.useNormalisedCrossCorr == 1)
+
+    mean_curr = mean(samples, 2);
+    patch_normed = samples - repmat(mean_curr,1, patch_expert_support_size(1)*patch_expert_support_size(2));
+
+    % Normalising the patches using the L2 norm
+    scaling = sqrt(sum(patch_normed.^2,2));
+    scaling(scaling == 0) = 1;
+
+    patch_normed = patch_normed ./ repmat(scaling, 1, patch_expert_support_size(1)*patch_expert_support_size(2));
+
+    samples = patch_normed;
+    clear 'patch_normed';
+end
+
+% Only keep the filled samples
+samples = samples(1:samples_filled-1,:);
+labels = labels(1:samples_filled-1,:);
+
+if((samples_filled-1)/(single_input_size(1)*single_input_size(2)) < size(samples_unnormed,1))
+    samples_unnormed = samples_unnormed(1:(samples_filled-1)/(single_input_size(1)*single_input_size(2)),:);
+end
+
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/FindDistantLandmarks.m

@@ -0,0 +1,41 @@
+function [to_keep] = FindDistantLandmarks(landmarkLoc, landmark_num, num_to_keep)
+
+    % First align all of them
+    a = landmarkLoc(:,:,1);
+    b = landmarkLoc(:,:,2);
+
+    offset_x = mean(a,2);
+    offset_y = mean(b,2);
+
+    landmark_loc_off = cat(3, bsxfun(@plus, a, -offset_x), bsxfun(@plus, b, -offset_y));
+    fixed_x = landmark_loc_off(:,:,1);
+    fixed_y = landmark_loc_off(:,:,2);
+
+    % Extract the relevant landmarks
+    fixed_x_l = fixed_x(:,landmark_num);
+    fixed_y_l = fixed_y(:,landmark_num);
+    
+    obs = cat(2, fixed_x_l, fixed_y_l);
+    
+    % Discard landmarks that are very close to each other, so that we only
+    % keep more diverse images
+    D = squareform(pdist(obs));
+    
+    to_keep = true(size(landmarkLoc,1),1);
+    
+    for i = 1:(size(landmarkLoc,1) - num_to_keep)
+       
+        diversity_score = mean(D,2);
+        
+        a = min(diversity_score);
+        
+        lowest = find(diversity_score == a);
+        lowest = lowest(1);
+        
+        to_keep(lowest) = 0;
+        
+        D(:,~to_keep) = 0;
+        D(~to_keep,:) = 200;
+    end
+    
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_cofw.m

@@ -0,0 +1,32 @@
+clear
+% define the root name of database
+root = '../data_preparation/prepared_data/';
+
+% which scales we're doing
+sigma = 1;
+num_samples = 5e5;
+
+scales = [0.25,0.35,0.5];
+frontalView = 1;
+
+profileViewInds = [];
+
+version = 'cofw';
+ratio_neg = 5;
+norm = 1;
+
+data_loc = 'cofw_';
+rng(0);
+
+% where to save generated patches for external training
+% (e.g. for training CEN patches)
+patches_loc = './patches/';
+patch_folder = [patches_loc version '/'];
+
+for s=scales
+    Save_all_patches(root, frontalView, profileViewInds,...
+        s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
+        'num_samples', num_samples, 'data_loc', data_loc,...
+        'normalisation_size', 19);
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_general.m

@@ -0,0 +1,32 @@
+clear
+% define the root name of database
+root = '../data_preparation/prepared_data/';
+
+% which scales we're doing
+sigma = 1;
+num_samples = 10e6;
+
+scales = [0.25,0.35,0.5,1.0];
+frontalView = 1;
+
+profileViewInds = [2,3,4];
+
+version = 'general';
+ratio_neg = 10;
+norm = 1;
+
+data_loc = 'combined_';
+rng(0);
+
+% where to save generated patches for external training
+% (e.g. for training CEN patches)
+patches_loc = './patches/';
+patch_folder = [patches_loc version '/'];
+
+for s=scales
+    Save_all_patches(root, frontalView, profileViewInds,...
+        s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
+        'num_samples', num_samples, 'data_loc', data_loc,...
+        'normalisation_size', 19);
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_multi_pie.m

@@ -0,0 +1,32 @@
+clear
+% define the root name of database
+root = '../data_preparation/prepared_data/';
+
+% which scales we're doing
+sigma = 1;
+num_samples = 5e5;
+
+scales = [0.25,0.35,0.5];
+frontalView = 1;
+
+profileViewInds = [2,3,4];
+
+version = 'multi_pie';
+ratio_neg = 5;
+norm = 1;
+
+data_loc = 'mpie_';
+rng(0);
+
+% where to save generated patches for external training
+% (e.g. for training CEN patches)
+patches_loc = './patches/';
+patch_folder = [patches_loc version '/'];
+
+for s=scales
+    Save_all_patches(root, frontalView, profileViewInds,...
+        s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
+        'num_samples', num_samples, 'data_loc', data_loc,...
+        'normalisation_size', 19);
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_wild.m

@@ -0,0 +1,32 @@
+clear
+% define the root name of database
+root = '../data_preparation/prepared_data/';
+
+% which scales we're doing
+sigma = 1;
+num_samples = 2e6;
+
+scales = [0.25,0.35,0.5,1.0];
+frontalView = 1;
+
+profileViewInds = [2];
+
+version = 'wild';
+ratio_neg = 10;
+norm = 1;
+
+data_loc = 'wild_';
+rng(0);
+
+% where to save generated patches for external training
+% (e.g. for training CEN patches)
+patches_loc = './patches/';
+patch_folder = [patches_loc version '/'];
+
+for s=scales
+    Save_all_patches(root, frontalView, profileViewInds,...
+        s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
+        'num_samples', num_samples, 'data_loc', data_loc,...
+        'normalisation_size', 19);
+end
+

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Save_all_patches.m

@@ -0,0 +1,39 @@
+function Save_all_patches(trainingLoc, frontalView, profile_views, scaling, sigma, version, patches_loc, varargin)
+% need some documentation here
+
+if(sum(strcmp(varargin,'ratio_neg')))
+    ind = find(strcmp(varargin,'ratio_neg')) + 1;
+    ratio_neg = varargin{ind};
+else
+    ratio_neg = 20;
+end
+
+if(sum(strcmp(varargin,'num_samples')))
+    ind = find(strcmp(varargin,'num_samples')) + 1;
+    num_samples = varargin{ind};
+else
+    num_samples = 5e5;
+end
+
+% first do the frontal view
+imgs_used = Save_patches(trainingLoc, frontalView, scaling, sigma, ratio_neg, num_samples, patches_loc, 'frontal', varargin{:});
+
+fprintf('Frontal done\n');
+
+% now do the profile views
+for i=1:numel(profile_views)
+    view_name = sprintf(['profile%s'], num2str(i));
+    imgs_used_profile = ...
+        Save_patches(trainingLoc, profile_views(i), scaling, sigma, ratio_neg, num_samples, patches_loc, view_name, varargin{:});
+    fprintf('Profile %d done\n', i);
+    
+    imgs_used = cat(1, imgs_used, imgs_used_profile);
+
+end
+
+% save the images used
+[status,msg,msgID] = mkdir('generated');
+location_imgs_used = sprintf('generated/imgs_used_%s.mat', version);
+save(location_imgs_used, 'imgs_used');
+
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Save_patches.m

@@ -0,0 +1,128 @@
+function [imgs_used, normalisation_options] = Save_patches(training_loc, view, scale, sigma, ratio_neg, num_samples, patches_loc, view_name, varargin)
+
+    % patch generation options
+    num_samples = num_samples;
+
+    normalisation_options = struct;
+    normalisation_options.patchSize = [11 11];
+    if(sum(strcmp(varargin,'normalisation_size')))
+        ind = find(strcmp(varargin,'normalisation_size')) + 1;
+        normalisation_options.normalisationRegion = [varargin{ind}, varargin{ind}];
+    else
+        normalisation_options.normalisationRegion = [21 21];
+    end
+    normalisation_options.rate_negative = ratio_neg;
+    normalisation_options.useNormalisedCrossCorr = 1;
+
+    train_test_ratio = 0.9;
+
+    
+    if(sum(strcmp(varargin,'data_loc')))       
+        ind = find(strcmp(varargin,'data_loc')) + 1;
+        data_loc = varargin{ind};
+        data_loc = sprintf(['%s/' data_loc '%s_%s.mat'], training_loc, num2str(scale), num2str(view));
+    else        
+        data_loc = sprintf('%s/wild_%s_%s.mat', training_loc, num2str(scale), num2str(view));
+    end    
+    
+    % location to save generated patches to (for training other patch experts
+    % such as CEN)
+
+    fprintf('saving patches to %s\n', patches_loc);
+    patches_filename = sprintf('data%.2f_%s.mat', scale, view_name);
+
+    % data_loc should contain landmark_locations, all_images, actual_imgs_used,
+    % visiIndex, centres
+    load(data_loc);
+    examples = all_images;
+    landmark_loc = landmark_locations;
+    clear 'all_images'    
+    
+    numPoints = size(landmark_loc,2);
+    
+    done = zeros(1, numPoints);
+    
+    for j=1:numPoints
+        [status,msg,msgID] = mkdir(sprintf([patches_loc '/%d'], j));
+    end
+
+    for j=1:numPoints
+        pause(0);
+        % can only do mirroring if there is no yaw
+        if((numPoints == 68 || numPoints == 29 )&& centres(2) == 0)
+            % Do not redo a mirror feature (just flip them)
+            if(numPoints == 68)
+                mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
+                      32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
+                      61,65;62,64;68,66];     
+            else
+                mirrorInds = [1,2; 3,4; 5,7; 6,8; 9,10; 11,12; 13,15; 14,16; 17,18; 19,20; 23,24];                
+            end
+            
+            mirror_idx = j;
+            if(any(mirrorInds(:,1)==j))
+                mirror_idx = mirrorInds(mirrorInds(:,1)==j,2);
+            elseif(any(mirrorInds(:,2)==j))
+                mirror_idx = mirrorInds(mirrorInds(:,2)==j,1);
+            end
+
+            if (mirror_idx ~= j & done(1, mirror_idx) ~= 0)
+
+                fprintf('not generating patches for lm %d, mirrored by %d\n', j, mirror_idx);
+                done(1, j) = done(1, mirror_idx);
+
+                fprintf('Feature %d done\n', j);
+
+                continue;
+
+            end
+        end
+        
+        imgs_used = {};
+        if(visiIndex(j)) 
+            
+            tic;
+            % instead of loading the patches compute them here:
+
+            [samples, labels, unnormed_samples, imgs_used_n] = ExtractTrainingSamples(examples, landmark_loc, actual_imgs_used, sigma, num_samples, j, normalisation_options);
+            imgs_used = union(imgs_used, imgs_used_n);
+            % add the bias term
+            samples = [ones(1,size(samples,1)); samples'];                  
+            
+            region_length = normalisation_options.normalisationRegion - normalisation_options.patchSize + 1;
+            region_length = region_length(1) * region_length(2);
+                        
+            num_examples = size(samples, 2);
+
+            % this part sets the split boundaries for training and test subsets
+            train_ccnf_start = 1;
+            train_ccnf_end = int32(train_test_ratio * num_examples - 1);
+            % make sure we don't split a full training region apart
+            train_ccnf_end = train_ccnf_end - mod(train_ccnf_end, region_length);
+
+            test_start = train_ccnf_end + 1;
+            test_end = size(samples,2);
+
+            samples_train = samples(:,train_ccnf_start:train_ccnf_end);
+            labels_train = labels(train_ccnf_start:train_ccnf_end);
+
+            samples_test = samples(:,test_start:test_end);                                                
+            labels_test = labels(test_start:test_end);
+            
+            filename = sprintf([patches_loc '/%s/' patches_filename], num2str(j));
+            save(filename, 'samples_train', 'labels_train', 'samples_test', 'labels_test', '-v7.3');
+
+            done(1, j) = 1;
+
+            fprintf('Landmark %d done\n', j);
+            clear samples
+            clear samples_test
+            clear samples_train
+            clear labels
+            clear unnormed_samples
+            clear imgs_used_n
+            toc;
+        end
+    end
+    
+end

pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/readme.txt

@@ -0,0 +1,11 @@
+Scripts for generating random patches to train continuous conditional neural field (CCNF, or alternatively Local Neural Field - LNF) patch experts.
+
+This requires data preparation first (not included with the source code), to prepare the data go to '../data_preparation/readme.txt'.
+
+To generate the patches run:
+Generate_Patches_general (using combined data)
+Generate_Patches_wild (using in-the-wild data)
+Generate_Patches_multi_pie (using multi-pie data)
+Generate_Patches_cofw (using cofw data)
+
+By default, the patches will be placed in ./patches/. To change the save location, edit the 'patches_loc' variable in the scripts above.