open source pkg v1

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic.m

@@ -0,0 +1,110 @@
+function [data_train, labels_train, data_devel, labels_devel, raw_devel, PC, means_norm, stds_norm, valid_ids_test] = ...
+    Prepare_HOG_AU_data_generic(train_users, devel_users, au_train, bp4d_dir, features_dir)
+
+%%
+addpath(genpath('../../data extraction/'));
+
+au_other = setdiff([1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23], au_train);
+[ labels_other, ~, ~ ] = extract_BP4D_labels(bp4d_dir, train_users, au_other);
+labels_other = cat(1, labels_other{:});
+
+% First extracting the labels
+[ labels_train, valid_ids_train, vid_ids_train ] = extract_BP4D_labels(bp4d_dir, train_users, au_train);
+
+train_geom_data = Read_geom_files(train_users, features_dir);
+% Reading in the HOG data (of only relevant frames)
+[train_appearance_data, valid_ids_train_hog, vid_ids_train_string] = Read_HOG_files(train_users, features_dir);
+train_appearance_data = cat(2, train_appearance_data, train_geom_data);
+
+% Subsample the data to make training quicker
+labels_train = cat(1, labels_train{:});
+valid_ids_train = logical(cat(1, valid_ids_train{:}));
+reduced_inds = false(size(labels_train,1),1);
+
+if(numel(au_train) == 1)
+    reduced_inds(labels_train == 1) = true;
+else
+    reduced_inds(:) = true; 
+end
+
+% make sure the same number of positive and negative samples is taken
+pos_count = sum(labels_train == 1);
+neg_count = sum(labels_train == 0);
+
+num_other = floor(pos_count / (size(labels_other, 2)));
+
+inds_all = 1:size(labels_train,1);
+
+if(numel(au_train) == 1)
+    for i=1:size(labels_other, 2)+1
+
+        if(i > size(labels_other, 2))
+            % fill the rest with a proportion of neutral
+            inds_other = inds_all(sum(labels_other,2)==0 & ~labels_train );   
+            num_other_i = min(numel(inds_other), pos_count - sum(labels_train(reduced_inds,:)==0));     
+        else
+            % take a proportion of each other AU
+            inds_other = inds_all(labels_other(:, i) & ~labels_train );      
+            num_other_i = min(numel(inds_other), num_other);        
+        end
+        inds_other_to_keep = inds_other(round(linspace(1, numel(inds_other), num_other_i)));
+        reduced_inds(inds_other_to_keep) = true;
+
+    end
+end
+% Remove invalid ids based on CLM failing or AU not being labelled
+reduced_inds(~valid_ids_train) = false;
+reduced_inds(~valid_ids_train_hog) = false;
+
+% labels_other = labels_other(reduced_inds, :);
+labels_train = labels_train(reduced_inds,:);
+train_appearance_data = train_appearance_data(reduced_inds,:);
+vid_ids_train_string = vid_ids_train_string(reduced_inds,:);
+
+%% Extract devel data
+
+% First extracting the labels
+[ labels_devel, valid_ids_devel, vid_ids_devel ] = extract_BP4D_labels(bp4d_dir, devel_users, au_train);
+
+% Reading in the HOG data (of only relevant frames)
+devel_geom_data = Read_geom_files(devel_users, features_dir);
+[devel_appearance_data, valid_ids_devel_hog, vid_ids_devel_string] = Read_HOG_files(devel_users, features_dir);
+devel_appearance_data = cat(2, devel_appearance_data, devel_geom_data);
+
+labels_devel = cat(1, labels_devel{:});
+
+valid_ids_test = valid_ids_devel_hog;
+
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+
+PC_n = zeros(size(PC)+size(train_geom_data, 2));
+PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
+PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(size(train_geom_data, 2));
+PC = PC_n;
+
+means_norm = cat(2, means_norm, zeros(1, size(train_geom_data,2)));
+stds_norm = cat(2, stds_norm, ones(1, size(train_geom_data,2)));
+
+% Grab all data for validation as want good params for all the data
+raw_devel = devel_appearance_data;
+
+devel_appearance_data = bsxfun(@times, bsxfun(@plus, devel_appearance_data, -means_norm), 1./stds_norm);
+train_appearance_data = bsxfun(@times, bsxfun(@plus, train_appearance_data, -means_norm), 1./stds_norm);
+
+data_train = train_appearance_data * PC;
+data_devel = devel_appearance_data * PC;
+
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic_dynamic.m

@@ -0,0 +1,106 @@
+function [data_train, labels_train, data_devel, labels_devel, raw_devel, PC, means_norm, stds_norm, valid_ids_devel] = ...
+    Prepare_HOG_AU_data_generic_dynamic(train_users, devel_users, au_train, bp4d_dir, hog_data_dir, pca_file)
+
+%%
+addpath(genpath('../../data extraction/'));
+
+au_other = setdiff([1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23], au_train);
+[ labels_other, ~, ~ ] = extract_BP4D_labels(bp4d_dir, train_users, au_other);
+labels_other = cat(1, labels_other{:});
+
+% First extracting the labels
+[ labels_train, valid_ids_train, vid_ids_train ] = extract_BP4D_labels(bp4d_dir, train_users, au_train);
+
+train_geom_data = Read_geom_files_dynamic(train_users, hog_data_dir);
+
+% Reading in the HOG data (of only relevant frames)
+[train_appearance_data, valid_ids_train_hog, vid_ids_train_string] = Read_HOG_files_dynamic_pp(train_users, hog_data_dir);
+
+train_appearance_data = cat(2, train_appearance_data, train_geom_data);
+
+% Subsample the data to make training quicker
+labels_train = cat(1, labels_train{:});
+valid_ids_train = logical(cat(1, valid_ids_train{:}));
+
+reduced_inds = false(size(labels_train,1),1);
+reduced_inds(labels_train == 1) = true;
+
+% make sure the same number of positive and negative samples is taken
+pos_count = sum(labels_train == 1);
+neg_count = sum(labels_train == 0);
+
+num_other = floor(pos_count / (size(labels_other, 2)));
+
+inds_all = 1:size(labels_train,1);
+
+for i=1:size(labels_other, 2)+1
+   
+    if(i > size(labels_other, 2))
+        % fill the rest with a proportion of neutral
+        inds_other = inds_all(sum(labels_other,2)==0 & ~labels_train );   
+        num_other_i = min(numel(inds_other), pos_count - sum(labels_train(reduced_inds,:)==0));     
+    else
+        % take a proportion of each other AU
+        inds_other = inds_all(labels_other(:, i) & ~labels_train );      
+        num_other_i = min(numel(inds_other), num_other);        
+    end
+    inds_other_to_keep = inds_other(round(linspace(1, numel(inds_other), num_other_i)));
+    reduced_inds(inds_other_to_keep) = true;
+    
+end
+
+% Remove invalid ids based on CLM failing or AU not being labelled
+reduced_inds(~valid_ids_train) = false;
+reduced_inds(~valid_ids_train_hog) = false;
+
+labels_other = labels_other(reduced_inds, :);
+labels_train = labels_train(reduced_inds,:);
+train_appearance_data = train_appearance_data(reduced_inds,:);
+vid_ids_train_string = vid_ids_train_string(reduced_inds,:);
+
+%% Extract devel data
+
+% First extracting the labels
+[ labels_devel, valid_ids_devel, vid_ids_devel ] = extract_BP4D_labels(bp4d_dir, devel_users, au_train);
+labels_devel = cat(1, labels_devel{:});
+
+% Reading in the HOG data (of only relevant frames)
+devel_geom_data = Read_geom_files_dynamic(devel_users, hog_data_dir);
+[devel_appearance_data, valid_ids_devel_hog, vid_ids_devel_string] = Read_HOG_files_dynamic_pp(devel_users, hog_data_dir);
+devel_appearance_data = cat(2, devel_appearance_data, devel_geom_data);
+
+valid_ids_devel = valid_ids_devel_hog;
+
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+
+PC_n = zeros(size(PC)+size(train_geom_data, 2));
+PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
+PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(size(train_geom_data, 2));
+PC = PC_n;
+
+means_norm = cat(2, means_norm, zeros(1, size(train_geom_data,2)));
+stds_norm = cat(2, stds_norm, ones(1, size(train_geom_data,2)));
+
+% Grab all data for validation as want good params for all the data
+raw_devel = devel_appearance_data;
+
+devel_appearance_data = bsxfun(@times, bsxfun(@plus, devel_appearance_data, -means_norm), 1./stds_norm);
+train_appearance_data = bsxfun(@times, bsxfun(@plus, train_appearance_data, -means_norm), 1./stds_norm);
+
+data_train = train_appearance_data * PC;
+data_devel = devel_appearance_data * PC;
+
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic_intensity.m

@@ -0,0 +1,115 @@
+function [data_train, labels_train, vid_ids_train_string, data_devel, labels_devel, vid_ids_devel_string, raw_devel, PC, means_norm, stds_norm, success_devel] = ...
+    Prepare_HOG_AU_data_generic_intensity(train_users, devel_users, au_train, bp4d_dir, features_dir)
+
+%%
+addpath(genpath('../data extraction/'));
+
+% First extracting the labels
+[ labels_train, valid_ids_train, vid_ids_train ] = extract_BP4D_labels_intensity(bp4d_dir, train_users, au_train);
+au_other = setdiff([6, 10, 12, 14, 17], au_train);
+[ labels_other, ~, ~ ] = extract_BP4D_labels_intensity(bp4d_dir, train_users, au_other);
+labels_other = cat(1, labels_other{:});
+
+train_geom_data = Read_geom_files(train_users, features_dir);
+
+% Reading in the HOG data (of only relevant frames)
+[train_appearance_data, valid_ids_train_hog, vid_ids_train_string] = Read_HOG_files(train_users, features_dir);
+train_appearance_data = cat(2, train_appearance_data, train_geom_data);
+
+% Subsample the data to make training quicker
+labels_train = cat(1, labels_train{:});
+valid_ids_train = logical(cat(1, valid_ids_train{:}));
+
+reduced_inds = false(size(labels_train,1),1);
+reduced_inds(labels_train > 0) = true;
+
+% make sure the same number of positive and negative samples is taken
+pos_count = sum(labels_train > 0);
+neg_count = sum(labels_train == 0);
+
+num_other = floor(pos_count / (size(labels_other, 2)));
+
+inds_all = 1:size(labels_train,1);
+
+if(numel(train_users) > 0)
+    if(numel(au_train) == 1)
+        for i=1:size(labels_other, 2)+1
+
+            if(i > size(labels_other, 2))
+                % fill the rest with a proportion of neutral
+                inds_other = inds_all(sum(labels_other,2)==0 & ~labels_train );   
+                num_other_i = min(numel(inds_other), pos_count - sum(labels_train(reduced_inds,:)==0));     
+            else
+                % take a proportion of each other AU
+                inds_other = inds_all(labels_other(:, i) & ~labels_train );      
+                num_other_i = min(numel(inds_other), num_other);        
+            end
+            inds_other_to_keep = inds_other(round(linspace(1, numel(inds_other), num_other_i)));
+            reduced_inds(inds_other_to_keep) = true;
+
+        end
+    end
+    % Remove invalid ids based on CLM failing or AU not being labelled
+    reduced_inds(~valid_ids_train) = false;
+    reduced_inds(~valid_ids_train_hog) = false;
+
+    labels_other = labels_other(reduced_inds, :);
+    labels_train = labels_train(reduced_inds,:);
+    train_appearance_data = train_appearance_data(reduced_inds,:);
+    vid_ids_train_string = vid_ids_train_string(reduced_inds,:);
+
+end
+
+%% Extract devel data
+
+% First extracting the labels
+[ labels_devel, valid_ids_devel, vid_ids_devel ] = extract_BP4D_labels_intensity(bp4d_dir, devel_users, au_train);
+
+devel_geom_data = Read_geom_files(devel_users, features_dir);
+% Reading in the HOG data (of only relevant frames)
+[devel_appearance_data, valid_ids_devel_hog, vid_ids_devel_string] = Read_HOG_files(devel_users, features_dir);
+devel_appearance_data = cat(2, devel_appearance_data, devel_geom_data);
+
+valid_ids_devel = logical(cat(1, valid_ids_devel{:}));
+
+labels_devel = cat(1, labels_devel{:});
+
+success_devel = valid_ids_devel;
+
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+
+PC_n = zeros(size(PC)+size(devel_geom_data, 2));
+PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
+PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(size(devel_geom_data, 2));
+PC = PC_n;
+
+means_norm = cat(2, means_norm, zeros(1, size(devel_geom_data,2)));
+stds_norm = cat(2, stds_norm, ones(1, size(devel_geom_data,2)));
+
+% Grab all data for validation as want good params for all the data
+raw_devel = devel_appearance_data;
+
+devel_appearance_data = bsxfun(@times, bsxfun(@plus, devel_appearance_data, -means_norm), 1./stds_norm);
+data_devel = devel_appearance_data * PC;
+
+if(numel(train_users) > 0)
+    train_appearance_data = bsxfun(@times, bsxfun(@plus, train_appearance_data, -means_norm), 1./stds_norm);
+    data_train = train_appearance_data * PC;
+else
+    data_train = [];
+end
+
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Prepare_HOG_AU_data_generic_intensity_dynamic.m

@@ -0,0 +1,115 @@
+function [data_train, labels_train, vid_ids_train_string, data_devel, labels_devel, vid_ids_devel_string, raw_devel, PC, means_norm, stds_norm, success_devel] = ...
+    Prepare_HOG_AU_data_generic_intensity_dynamic(train_users, devel_users, au_train, bp4d_dir, features_dir)
+
+%%
+addpath(genpath('../data extraction/'));
+
+% First extracting the labels
+[ labels_train, valid_ids_train, vid_ids_train ] = extract_BP4D_labels_intensity(bp4d_dir, train_users, au_train);
+au_other = setdiff([6, 10, 12, 14, 17], au_train);
+[ labels_other, ~, ~ ] = extract_BP4D_labels_intensity(bp4d_dir, train_users, au_other);
+labels_other = cat(1, labels_other{:});
+
+train_geom_data = Read_geom_files_dynamic(train_users, features_dir);
+
+% Reading in the HOG data (of only relevant frames)
+[train_appearance_data, valid_ids_train_hog, vid_ids_train_string] = Read_HOG_files_dynamic_pp(train_users, features_dir);
+train_appearance_data = cat(2, train_appearance_data, train_geom_data);
+
+% Subsample the data to make training quicker
+labels_train = cat(1, labels_train{:});
+valid_ids_train = logical(cat(1, valid_ids_train{:}));
+
+reduced_inds = false(size(labels_train,1),1);
+reduced_inds(labels_train > 0) = true;
+
+% make sure the same number of positive and negative samples is taken
+pos_count = sum(labels_train > 0);
+neg_count = sum(labels_train == 0);
+
+num_other = floor(pos_count / (size(labels_other, 2)));
+
+inds_all = 1:size(labels_train,1);
+
+if(numel(train_users) > 0)
+    if(numel(au_train) == 1)
+        for i=1:size(labels_other, 2)+1
+
+            if(i > size(labels_other, 2))
+                % fill the rest with a proportion of neutral
+                inds_other = inds_all(sum(labels_other,2)==0 & ~labels_train );   
+                num_other_i = min(numel(inds_other), pos_count - sum(labels_train(reduced_inds,:)==0));     
+            else
+                % take a proportion of each other AU
+                inds_other = inds_all(labels_other(:, i) & ~labels_train );      
+                num_other_i = min(numel(inds_other), num_other);        
+            end
+            inds_other_to_keep = inds_other(round(linspace(1, numel(inds_other), num_other_i)));
+            reduced_inds(inds_other_to_keep) = true;
+
+        end
+    end
+    % Remove invalid ids based on CLM failing or AU not being labelled
+    reduced_inds(~valid_ids_train) = false;
+    reduced_inds(~valid_ids_train_hog) = false;
+
+    labels_other = labels_other(reduced_inds, :);
+    labels_train = labels_train(reduced_inds,:);
+    train_appearance_data = train_appearance_data(reduced_inds,:);
+    vid_ids_train_string = vid_ids_train_string(reduced_inds,:);
+
+end
+
+%% Extract devel data
+
+% First extracting the labels
+[ labels_devel, valid_ids_devel, vid_ids_devel ] = extract_BP4D_labels_intensity(bp4d_dir, devel_users, au_train);
+
+devel_geom_data = Read_geom_files_dynamic(devel_users, features_dir);
+% Reading in the HOG data (of only relevant frames)
+[devel_appearance_data, valid_ids_devel_hog, vid_ids_devel_string] = Read_HOG_files_dynamic_pp(devel_users, features_dir);
+devel_appearance_data = cat(2, devel_appearance_data, devel_geom_data);
+
+valid_ids_devel = logical(cat(1, valid_ids_devel{:}));
+
+labels_devel = cat(1, labels_devel{:});
+
+success_devel = valid_ids_devel;
+
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+
+PC_n = zeros(size(PC)+size(devel_geom_data, 2));
+PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
+PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(size(devel_geom_data, 2));
+PC = PC_n;
+
+means_norm = cat(2, means_norm, zeros(1, size(devel_geom_data,2)));
+stds_norm = cat(2, stds_norm, ones(1, size(devel_geom_data,2)));
+
+% Grab all data for validation as want good params for all the data
+raw_devel = devel_appearance_data;
+
+devel_appearance_data = bsxfun(@times, bsxfun(@plus, devel_appearance_data, -means_norm), 1./stds_norm);
+data_devel = devel_appearance_data * PC;
+
+if(numel(train_users) > 0)
+    train_appearance_data = bsxfun(@times, bsxfun(@plus, train_appearance_data, -means_norm), 1./stds_norm);
+    data_train = train_appearance_data * PC;
+else
+    data_train = [];
+end
+
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Read_HOG_files.m

@@ -0,0 +1,93 @@
+function [hog_data, valid_inds, vid_id] = Read_HOG_files(users, hog_data_dir)
+
+    hog_data = [];
+    vid_id = {};
+    valid_inds = [];
+    
+    feats_filled = 0;
+
+    for i=1:numel(users)
+        
+        hog_files = dir([hog_data_dir, '/train/' users{i} '*.hog']);
+        hog_dir = [hog_data_dir, '/train/'];
+        if(isempty(hog_files))
+            hog_files = dir([hog_data_dir, '/devel/' users{i} '*.hog']);
+            hog_dir = [hog_data_dir, '/devel/'];
+        end
+        
+        for h=1:numel(hog_files)
+            hog_file = [hog_dir, hog_files(h).name];
+            f = fopen(hog_file, 'r');
+
+            curr_data = [];
+            curr_ind = 0;
+
+            while(~feof(f))
+
+                if(curr_ind == 0)
+                    num_cols = fread(f, 1, 'int32');
+                    if(isempty(num_cols))
+                        break;
+                    end
+
+                    num_rows = fread(f, 1, 'int32');
+                    num_chan = fread(f, 1, 'int32');
+
+                    curr_ind = curr_ind + 1;            
+
+                    % preallocate some space
+                    if(curr_ind == 1)
+                        curr_data = zeros(1000, 1 + num_rows * num_cols * num_chan);
+                        num_feats =  1 + num_rows * num_cols * num_chan;
+                    end
+
+                    if(curr_ind > size(curr_data,1))
+                        curr_data = cat(1, curr_data, zeros(1000, 1 + num_rows * num_cols * num_chan));
+                    end
+                    feature_vec = fread(f, [1, 1 + num_rows * num_cols * num_chan], 'float32');
+                    curr_data(curr_ind, :) = feature_vec;
+                else
+
+                    % Reading in batches of 5000
+
+                    feature_vec = fread(f, [4 + num_rows * num_cols * num_chan, 5000], 'float32');
+                    feature_vec = feature_vec(4:end,:)';
+
+                    num_rows_read = size(feature_vec,1);
+
+                    curr_data(curr_ind+1:curr_ind+num_rows_read,:) = feature_vec;
+
+                    curr_ind = curr_ind + size(feature_vec,1);
+
+                end
+
+            end
+
+            fclose(f);
+
+            curr_data = curr_data(1:curr_ind,:);
+            vid_id_curr = cell(curr_ind,1);
+            vid_id_curr(:) = users(i);
+
+            vid_id = cat(1, vid_id, vid_id_curr);
+
+            % Assume same number of frames per video
+            if(i==1 && h == 1)
+                hog_data = zeros(curr_ind * numel(users) * 8, num_feats);
+            end
+
+            if(size(hog_data,1) < feats_filled+curr_ind)
+               hog_data = cat(1, hog_data, zeros(size(hog_data,1), num_feats));
+            end
+
+            hog_data(feats_filled+1:feats_filled+curr_ind,:) = curr_data;
+
+            feats_filled = feats_filled + curr_ind;
+        end
+    end
+    
+    if(~isempty(hog_data))        
+        valid_inds = hog_data(1:feats_filled,1);
+        hog_data = hog_data(1:feats_filled,2:end);
+    end
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Read_HOG_files_dynamic_pp.m

@@ -0,0 +1,101 @@
+function [hog_data, valid_inds, vid_id] = Read_HOG_files_dynamic_pp(users, hog_data_dir)
+
+    hog_data = [];
+    vid_id = {};
+    valid_inds = [];
+    
+    feats_filled = 0;
+
+    for i=1:numel(users)
+        
+        start_person_ind = feats_filled + 1;
+        
+        hog_files = dir([hog_data_dir, '/train/' users{i} '*.hog']);
+        hog_dir = [hog_data_dir, '/train/'];
+        if(isempty(hog_files))
+            hog_files = dir([hog_data_dir, '/devel/' users{i} '*.hog']);
+            hog_dir = [hog_data_dir, '/devel/'];
+        end
+        
+        for h=1:numel(hog_files)
+            hog_file = [hog_dir, hog_files(h).name];
+            f = fopen(hog_file, 'r');
+
+            curr_data = [];
+            curr_ind = 0;
+
+            while(~feof(f))
+
+                if(curr_ind == 0)
+                    num_cols = fread(f, 1, 'int32');
+                    if(isempty(num_cols))
+                        break;
+                    end
+
+                    num_rows = fread(f, 1, 'int32');
+                    num_chan = fread(f, 1, 'int32');
+
+                    curr_ind = curr_ind + 1;            
+
+                    % preallocate some space
+                    if(curr_ind == 1)
+                        curr_data = zeros(1000, 1 + num_rows * num_cols * num_chan);
+                        num_feats =  1 + num_rows * num_cols * num_chan;
+                    end
+
+                    if(curr_ind > size(curr_data,1))
+                        curr_data = cat(1, curr_data, zeros(1000, 1 + num_rows * num_cols * num_chan));
+                    end
+                    feature_vec = fread(f, [1, 1 + num_rows * num_cols * num_chan], 'float32');
+                    curr_data(curr_ind, :) = feature_vec;
+                else
+
+                    % Reading in batches of 5000
+
+                    feature_vec = fread(f, [4 + num_rows * num_cols * num_chan, 5000], 'float32');
+                    feature_vec = feature_vec(4:end,:)';
+
+                    num_rows_read = size(feature_vec,1);
+
+                    curr_data(curr_ind+1:curr_ind+num_rows_read,:) = feature_vec;
+
+                    curr_ind = curr_ind + size(feature_vec,1);
+
+                end
+
+            end
+
+            fclose(f);
+
+            curr_data = curr_data(1:curr_ind,:);
+            vid_id_curr = cell(curr_ind,1);
+            vid_id_curr(:) = users(i);
+
+            vid_id = cat(1, vid_id, vid_id_curr);
+
+            % Assume same number of frames per video
+            if(i==1 && h == 1)
+                hog_data = zeros(curr_ind * numel(users) * 8, num_feats);
+            end
+
+            if(size(hog_data,1) < feats_filled+curr_ind)
+               hog_data = cat(1, hog_data, zeros(size(hog_data,1), num_feats));
+            end
+
+
+            hog_data(feats_filled+1:feats_filled+curr_ind,:) = curr_data;
+
+            feats_filled = feats_filled + curr_ind;
+        end
+        
+        person_ids = start_person_ind:feats_filled;
+        % Do the median normalisation per person here
+        hog_data(person_ids,2:end) = bsxfun(@plus, hog_data(person_ids,2:end), -median(hog_data(person_ids,2:end)));
+
+    end
+    
+    if(~isempty(hog_data))        
+        valid_inds = hog_data(1:feats_filled,1);
+        hog_data = hog_data(1:feats_filled,2:end);
+    end
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Read_geom_files.m

@@ -0,0 +1,52 @@
+function [geom_data, valid_ids] = Read_geom_files(users, hog_data_dir)
+
+    geom_data = [];
+    valid_ids = [];
+
+    load('../../pca_generation/pdm_68_aligned_wild.mat');
+    
+    for i=1:numel(users)
+        
+        geom_files = dir([hog_data_dir, '/train/', users{i} '*.csv']);
+        geom_dir = [hog_data_dir, '/train/'];
+        if(isempty(geom_files))
+            geom_files = dir([hog_data_dir, '/devel/', users{i} '*.csv']);
+            geom_dir = [hog_data_dir, '/devel/'];
+        end
+        
+        for h=1:numel(geom_files)
+            geom_file = [geom_dir, geom_files(h).name];
+            [~, nm, ~] = fileparts(geom_file);
+            m_file = [geom_dir, '/' nm '.params.mat'];
+              
+            if(~exist(m_file, 'file'))
+                if(i == 1)
+                    tab = readtable(geom_file);
+                    column_names = tab.Properties.VariableNames; 
+                    valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'success'));
+                    shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_'));
+                end
+        
+                res = dlmread(geom_file, ',', 1, 0);
+                
+                valid = res(:, valid_ind) > 0.7;        
+                res = res(:, shape_inds);
+                
+                % Do not consider global parameters
+                res = res(:, 7:end);
+                
+                save(m_file, 'res', 'valid');
+            else
+                load(m_file);
+            end
+                    
+            actual_locs = res * V';
+            res = cat(2, actual_locs, res);
+            
+            valid_ids = cat(1, valid_ids, valid);
+            
+            geom_data = cat(1, geom_data, res);
+                
+        end
+    end
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Read_geom_files_dynamic.m

@@ -0,0 +1,57 @@
+function [geom_data, valid_ids] = Read_geom_files_dynamic(users, hog_data_dir)
+
+    geom_data = [];
+    valid_ids = [];
+    
+    load('../../pca_generation/pdm_68_aligned_wild.mat');
+    
+    for i=1:numel(users)
+        
+        geom_files = dir([hog_data_dir, '/train/', users{i} '*.csv']);
+        geom_dir = [hog_data_dir, '/train/'];
+        if(isempty(geom_files))
+            geom_files = dir([hog_data_dir, '/devel/', users{i} '*.csv']);
+            geom_dir = [hog_data_dir, '/devel/'];
+        end
+        
+        geom_data_curr = [];
+        for h=1:numel(geom_files)
+            geom_file = [geom_dir, geom_files(h).name];
+                        
+            [~, nm, ~] = fileparts(geom_file);
+            m_file = [geom_dir, '/' nm '.params.mat'];
+              
+            if(~exist(m_file, 'file'))
+                if(i == 1)
+                    tab = readtable(geom_file);
+                    column_names = tab.Properties.VariableNames; 
+                    valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'success'));
+                    shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_'));
+                end
+        
+                res = dlmread(geom_file, ',', 1, 0);
+                
+                valid = res(:, valid_ind) > 0.7;        
+                res = res(:, shape_inds);
+                
+                % Do not consider global parameters
+                res = res(:, 7:end);
+                
+                save(m_file, 'res', 'valid');
+            else
+                load(m_file);
+            end
+
+            actual_locs = res * V';
+            res = cat(2, actual_locs, res);
+            
+            valid_ids = cat(1, valid_ids, valid);            
+            
+            geom_data_curr = cat(1, geom_data_curr, res);
+        end
+        geom_data_curr = bsxfun(@plus, geom_data_curr, -median(geom_data_curr));
+        
+        geom_data = cat(1, geom_data, geom_data_curr);
+                
+    end
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Script_HOG_SVM_dynamic.m

@@ -0,0 +1,64 @@
+% Change to your downloaded location
+clear
+addpath('C:\liblinear\matlab')
+addpath('../training_code/');
+addpath('../utilities/');
+addpath('../../data extraction/');
+%% load shared definitions and AU data
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:0.5:1);
+hyperparams.e = 10.^(-3);
+
+hyperparams.validate_params = {'c', 'e'};
+
+% Set the training function
+svm_train = @svm_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svm_test = @svm_test_linear;
+
+pca_loc = '../../pca_generation/generic_face_rigid.mat';
+
+hog_data_dir_BP4D = hog_data_dir;
+
+aus = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
+%%
+for a=1:numel(aus)
+        
+    au = aus(a);
+
+    rest_aus = setdiff(all_aus, au);        
+
+    % load the training and testing data for the current fold
+    [train_samples, train_labels, valid_samples, valid_labels, ~, PC, means, scaling] = Prepare_HOG_AU_data_generic_dynamic(train_recs, devel_recs, au, BP4D_dir, hog_data_dir_BP4D);
+
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svm_train, svm_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+    model = svm_train(train_labels, train_samples, best_params);        
+    
+    [~, predictions_all] = svm_test(valid_labels, valid_samples, model);
+    
+    name = sprintf('results_BP4D_devel/AU_%d_dynamic.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( predictions_all, valid_labels );
+    
+    save(name, 'model', 'F1s', 'accuracies', 'predictions_all', 'valid_labels');
+       
+    % Write out the model
+    name = sprintf('models/AU_%d_dynamic.dat', au);
+
+    pos_lbl = model.Label(1);
+    neg_lbl = model.Label(2);
+    
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+    
+    write_lin_dyn_svm(name, means, svs, b, pos_lbl, neg_lbl);
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Script_HOG_SVM_static.m

@@ -0,0 +1,66 @@
+% Change to your downloaded location
+clear
+addpath('C:\liblinear\matlab')
+addpath('../training_code/');
+addpath('../utilities/');
+addpath('../../data extraction/');
+%% load shared definitions and AU data
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:0.5:1);
+hyperparams.e = 10.^(-3);
+
+hyperparams.validate_params = {'c', 'e'};
+
+% Set the training function
+svm_train = @svm_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svm_test = @svm_test_linear;
+
+pca_loc = '../../pca_generation/generic_face_rigid.mat';
+
+hog_data_dir_BP4D = hog_data_dir;
+
+aus = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
+%%
+for a=1:numel(aus)
+        
+    au = aus(a);
+
+    rest_aus = setdiff(all_aus, au);        
+
+    % load the training and testing data for the current fold
+    [train_samples, train_labels, valid_samples, valid_labels, ~, PC, means, scaling] = Prepare_HOG_AU_data_generic(train_recs, devel_recs, au, BP4D_dir, hog_data_dir_BP4D);
+
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svm_train, svm_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+    model = svm_train(train_labels, train_samples, best_params);        
+    
+    [~, predictions_all] = svm_test(valid_labels, valid_samples, model);
+    
+    name = sprintf('results_BP4D_devel/AU_%d_static.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( predictions_all, valid_labels );
+    
+    save(name, 'model', 'F1s', 'accuracies', 'predictions_all', 'valid_labels');
+       
+    % Write out the model
+    name = sprintf('models/AU_%d_static.dat', au);
+
+    pos_lbl = model.Label(1);
+    neg_lbl = model.Label(2);
+    
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+    
+    write_lin_svm(name, means, svs, b, pos_lbl, neg_lbl);
+end
+
+

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Script_HOG_SVR_dynamic_shift.m

@@ -0,0 +1,83 @@
+%% load shared definitions and AU data
+clear
+
+addpath('../../data extraction/');
+addpath('../utilities/');
+addpath('../training_code/');
+
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:1:4);
+hyperparams.p = 10.^(-2);
+
+hyperparams.validate_params = {'c', 'p'};
+
+% Set the training function
+svr_train = @svr_train_linear_shift;
+    
+% Set the test function (the first output will be used for validation)
+svr_test = @svr_test_linear_shift;
+
+pca_loc = '../../pca_generation/generic_face_rigid.mat';
+
+hog_data_dir_BP4D = hog_data_dir;
+
+aus = [6, 10, 12, 14, 17];
+%%
+for a=1:numel(aus)
+    
+    predictions_all = [];
+    test_labels_all = [];
+    
+    au = aus(a);
+
+    rest_aus = setdiff(all_aus, au);        
+    
+    % load the training and testing data for the current fold
+    [train_samples, train_labels, ~, valid_samples, valid_labels, vid_ids_devel, ~, PC, means, scaling, success_devel] = Prepare_HOG_AU_data_generic_intensity(train_recs, devel_recs, au, BP4D_dir_int, hog_data_dir_BP4D);
+    
+    ignore = valid_labels == 9;
+    
+    valid_samples = valid_samples(~ignore, :);
+    valid_labels = valid_labels(~ignore);
+    vid_ids_devel = vid_ids_devel(~ignore);
+    success_devel = success_devel(~ignore);    
+    
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    hyperparams.success = success_devel;
+    hyperparams.valid_samples = valid_samples;
+    hyperparams.valid_labels = valid_labels;
+    hyperparams.vid_ids = vid_ids_devel;
+    
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svr_train, svr_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+    model = svr_train(train_labels, train_samples, best_params);        
+    
+    clear 'train_samples'
+    
+    %% Now test the model
+    model.vid_ids = vid_ids_devel;
+    
+    [~, prediction] = svr_test(valid_labels, valid_samples, model);
+
+    name = sprintf('results_BP4D_devel/AU_%d_static_intensity_shift.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( prediction, valid_labels );
+    
+    save(name, 'model', 'F1s', 'corrs', 'accuracies', 'ccc', 'rms', 'prediction', 'valid_labels');
+        
+        % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_static_intensity_shift.dat', au);
+
+    write_lin_svr(name, means, svs, b);
+    
+end
+

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Script_HOG_SVR_static.m

@@ -0,0 +1,78 @@
+%% load shared definitions and AU data
+clear
+
+addpath('../../data extraction/');
+addpath('../utilities/');
+addpath('../training_code/');
+
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:1:4);
+hyperparams.p = 10.^(-2);
+
+hyperparams.validate_params = {'c', 'p'};
+
+% Set the training function
+svr_train = @svr_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svr_test = @svr_test_linear;
+
+hog_data_dir_BP4D = hog_data_dir;
+
+aus = [6, 10, 12, 14, 17];
+%%
+for a=1:numel(aus)
+    
+    predictions_all = [];
+    test_labels_all = [];
+    
+    au = aus(a);
+
+    rest_aus = setdiff(all_aus, au);        
+    
+    % load the training and testing data for the current fold
+    [train_samples, train_labels, ~, valid_samples, valid_labels, vid_ids_devel, ~, PC, means, scaling, success_devel] = Prepare_HOG_AU_data_generic_intensity(train_recs, devel_recs, au, BP4D_dir_int, hog_data_dir_BP4D);
+    
+    ignore = valid_labels == 9;
+    
+    valid_samples = valid_samples(~ignore, :);
+    valid_labels = valid_labels(~ignore);
+    vid_ids_devel = vid_ids_devel(~ignore);
+    success_devel = success_devel(~ignore);
+    
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    hyperparams.success = success_devel;
+
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svr_train, svr_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+    model = svr_train(train_labels, train_samples, best_params);        
+    
+    clear 'train_samples'
+    
+    %% Now test the model
+    model.vid_ids = vid_ids_devel;
+    
+    [~, prediction] = svr_test(valid_labels, valid_samples, model);
+
+    name = sprintf('results_BP4D_devel/AU_%d_static_intensity.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( prediction, valid_labels );
+    
+    save(name, 'model', 'F1s', 'corrs', 'accuracies', 'ccc', 'rms', 'prediction', 'valid_labels');
+        
+        % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_static_intensity.dat', au);
+
+    write_lin_svr(name, means, svs, b);
+    
+end
+

pkg/OpenFace/model_training/AU_training/experiments/BP4D/Script_HOG_SVR_static_shift.m

@@ -0,0 +1,83 @@
+%% load shared definitions and AU data
+clear
+
+addpath('../../data extraction/');
+addpath('../utilities/');
+addpath('../training_code/');
+
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:1:4);
+hyperparams.p = 10.^(-2);
+
+hyperparams.validate_params = {'c', 'p'};
+
+% Set the training function
+svr_train = @svr_train_linear_shift;
+    
+% Set the test function (the first output will be used for validation)
+svr_test = @svr_test_linear_shift;
+
+pca_loc = '../../pca_generation/generic_face_rigid.mat';
+
+hog_data_dir_BP4D = hog_data_dir;
+
+aus = [6, 10, 12, 14, 17];
+%%
+for a=1:numel(aus)
+    
+    predictions_all = [];
+    test_labels_all = [];
+    
+    au = aus(a);
+
+    rest_aus = setdiff(all_aus, au);        
+    
+    % load the training and testing data for the current fold
+    [train_samples, train_labels, ~, valid_samples, valid_labels, vid_ids_devel, ~, PC, means, scaling, success_devel] = Prepare_HOG_AU_data_generic_intensity(train_recs, devel_recs, au, BP4D_dir_int, hog_data_dir_BP4D);
+    
+    ignore = valid_labels == 9;
+    
+    valid_samples = valid_samples(~ignore, :);
+    valid_labels = valid_labels(~ignore);
+    vid_ids_devel = vid_ids_devel(~ignore);
+    success_devel = success_devel(~ignore);    
+    
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    hyperparams.success = success_devel;
+    hyperparams.valid_samples = valid_samples;
+    hyperparams.valid_labels = valid_labels;
+    hyperparams.vid_ids = vid_ids_devel;
+    
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svr_train, svr_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+    model = svr_train(train_labels, train_samples, best_params);        
+    
+    clear 'train_samples'
+    
+    %% Now test the model
+    model.vid_ids = vid_ids_devel;
+    
+    [~, prediction] = svr_test(valid_labels, valid_samples, model);
+
+    name = sprintf('results_BP4D_devel/AU_%d_static_intensity_shift.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( prediction, valid_labels );
+    
+    save(name, 'model', 'F1s', 'corrs', 'accuracies', 'ccc', 'rms', 'prediction', 'valid_labels');
+        
+        % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_static_intensity_shift.dat', au);
+
+    write_lin_svr(name, means, svs, b);
+    
+end
+

pkg/OpenFace/model_training/AU_training/experiments/BP4D/get_balanced_fold.m

@@ -0,0 +1,21 @@
+function [train_users, dev_users] = get_balanced_fold(BP4D_dir, users, au, prop_test)
+
+    % Extracting the labels
+    [labels, valid_ids, vid_ids, filenames] = extract_BP4D_labels(BP4D_dir, users, au);
+    
+    % the grouping should be done per person
+    
+    for f=1:numel(filenames)
+        filenames{f} = filenames{f}(1:4);
+    end
+    
+    counts = zeros(numel(users),1);
+    for k=1:numel(users)
+        counts(k) = sum(cat(1, labels{strcmp(filenames, users{k})}));
+    end
+
+    [sorted, inds] = sort(counts);
+
+    dev_users = users(inds(1:round(1/prop_test):end));
+    train_users = setdiff(users, dev_users);
+end

pkg/OpenFace/model_training/AU_training/experiments/BP4D/shared_defs.m

@@ -0,0 +1,36 @@
+% this is data defined across the experiments (to make sure all of them have same user conventions)
+
+% Defining which AU's we are extracting (all corrs above 0.5)
+all_aus = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
+aus = [1, 2, 4, 6, 7, 10, 12, 14, 15, 17, 23];
+
+all_aus_int = [6, 10, 12, 14, 17];
+
+% load all of the data together (for efficiency)
+% it will be split up accordingly at later stages
+if(exist('C:\tadas\face_datasets\fera_2015\bp4d\AUCoding/', 'file'))
+    BP4D_dir = 'C:\tadas\face_datasets\fera_2015\bp4d\AUCoding/'; 
+    BP4D_dir_int = 'C:\tadas\face_datasets\fera_2015\bp4d\AU Intensity Codes3.0/';
+elseif(exist('E:\datasets\FERA_2015\BP4D\AUCoding/', 'file'))
+    BP4D_dir = 'E:\datasets\FERA_2015\BP4D\AUCoding/';       
+    BP4D_dir_int = 'E:\datasets\FERA_2015\BP4D\AU Intensity Codes3.0/';
+elseif(exist('D:\datasets\face_datasets\fera_2015\bp4d\AUCoding/','file'))
+    BP4D_dir = 'D:\datasets\face_datasets\fera_2015\bp4d\AUCoding/';
+    BP4D_dir_int = 'D:\datasets\face_datasets\fera_2015\bp4d\AU Intensity Codes3.0/';
+elseif(exist('D:\Datasets\FERA_2015\BP4D\AUCoding/','file'))
+    BP4D_dir = 'D:\Datasets\FERA_2015\BP4D\AUCoding/';
+    BP4D_dir_int = 'D:\Datasets\FERA_2015\BP4D\AU Intensity Codes3.0/';
+elseif(exist('I:\datasets\FERA_2015\BP4D\AUCoding/', 'file'))
+    BP4D_dir = 'I:\datasets\FERA_2015\BP4D\AUCoding/';
+    BP4D_dir_int = 'I:\datasets\FERA_2015\BP4D\AU Intensity Codes3.0/';
+else
+    fprintf('BP4D location not found (or not defined)\n'); 
+end
+
+if(exist('BP4D_dir', 'var'))
+    hog_data_dir = 'E:\datasets\face_datasets_processed\bp4d/';
+end
+
+train_recs = {'F001', 'F003', 'F005', 'F007', 'F009', 'F011', 'F013', 'F015', 'F017', 'F019', 'F021', 'F023', 'M001', 'M003', 'M005', 'M007', 'M009', 'M011', 'M013', 'M015' 'M017'};
+devel_recs = {'F002', 'F004', 'F006', 'F008', 'F010', 'F012', 'F014', 'F016', 'F018', 'F020', 'F022', 'M002', 'M004', 'M006', 'M008', 'M010', 'M012', 'M014', 'M016', 'M018'};
+

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Prepare_HOG_AU_data.m

@@ -0,0 +1,118 @@
+function [data_train, labels_train, data_devel, labels_devel, raw_devel, PC, means_norm, stds_norm, devel_ids, devel_success] = ...
+    Prepare_HOG_AU_data(train_users, devel_users, au_train, rest_aus, Bosphorus_dir, params_data_dir)
+
+%%
+addpath(genpath('../data extraction/'));
+
+% First extracting the labels
+[ labels_train, valid_ids_train, filenames ] = extract_Bosphorus_labels(Bosphorus_dir, train_users, au_train);
+
+[ labels_other, ~, ~ ] = extract_Bosphorus_labels(Bosphorus_dir, train_users, rest_aus);
+
+% Reading in the HOG data (of only relevant frames)
+[train_appearance_data, valid_ids_train_hog, vid_ids_train_string] = Read_HOG_files(filenames, params_data_dir);
+
+[train_geom_data] = Read_geom_files(filenames,  params_data_dir);
+
+% Subsample the data to rebalance it
+if(numel(train_users) > 0)
+    reduced_inds = false(size(labels_train,1),1);
+    reduced_inds(labels_train > 0) = true;
+
+    % make sure the same number of positive and negative samples is taken
+    pos_count = sum(labels_train > 0);
+    neg_count = sum(labels_train == 0);
+
+    num_other = floor( pos_count / (size(labels_other, 2)));
+
+    inds_all = 1:size(labels_train,1);
+
+    for i=1:size(labels_other, 2)+1
+   
+        if(i > size(labels_other, 2))
+            % fill the rest with a proportion of neutral
+            inds_other = inds_all(sum(labels_other,2)==0 & ~labels_train );   
+                num_other_i = min(numel(inds_other), pos_count - sum(labels_train(reduced_inds,:)==0));     
+        else
+                % take a proportion of each other AU
+            inds_other = inds_all(labels_other(:, i) & ~labels_train );      
+            num_other_i = min(numel(inds_other), num_other);        
+        end
+        inds_other_to_keep = inds_other(round(linspace(1, numel(inds_other), num_other_i)));
+        reduced_inds(inds_other_to_keep) = true;
+
+    end
+
+    % Remove invalid ids based on CLM failing or AU not being labelled
+    reduced_inds(~valid_ids_train) = false;
+    reduced_inds(~valid_ids_train_hog) = false;
+
+    labels_other = labels_other(reduced_inds, :);
+    labels_train = labels_train(reduced_inds,:);
+    train_appearance_data = train_appearance_data(reduced_inds,:);
+    train_geom_data = train_geom_data(reduced_inds,:);
+    vid_ids_train_string = vid_ids_train_string(reduced_inds,:);
+end
+%% Extract devel data
+
+% First extracting the labels
+[ labels_devel, valid_ids_devel, filenames_devel ] = extract_Bosphorus_labels(Bosphorus_dir, devel_users, au_train);
+
+% Reading in the HOG data (of only relevant frames)
+[devel_appearance_data, valid_ids_devel_hog, vid_ids_devel_string] = Read_HOG_files(filenames_devel, params_data_dir);
+devel_success = valid_ids_devel_hog;
+devel_ids = vid_ids_devel_string;
+
+[devel_geom_data] = Read_geom_files(filenames_devel, params_data_dir);
+
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+     
+% Grab all data for validation as want good params for all the data
+raw_devel = cat(2, devel_appearance_data, devel_geom_data);
+
+devel_appearance_data = bsxfun(@times, bsxfun(@plus, devel_appearance_data, -means_norm), 1./stds_norm);
+
+data_devel = devel_appearance_data * PC;
+
+data_devel = cat(2, data_devel, devel_geom_data);
+
+valid_ids_devel = valid_ids_devel & devel_success;
+data_devel = data_devel(valid_ids_devel,:);
+labels_devel = labels_devel(valid_ids_devel,:);
+raw_devel = raw_devel(valid_ids_devel,:);
+devel_success = devel_success(valid_ids_devel,:);
+devel_ids = devel_ids(valid_ids_devel);
+
+if(numel(train_users) > 0)
+    train_appearance_data = bsxfun(@times, bsxfun(@plus, train_appearance_data, -means_norm), 1./stds_norm);
+
+    data_train = train_appearance_data * PC;
+    data_train = cat(2, data_train, train_geom_data);
+else
+    data_train = [];
+end
+
+geom_size = max(size(train_geom_data, 2), size(devel_geom_data, 2));
+
+PC_n = zeros(size(PC)+geom_size);
+PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
+PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(geom_size);
+PC = PC_n;
+
+means_norm = cat(2, means_norm, zeros(1, geom_size));
+stds_norm = cat(2, stds_norm, ones(1, geom_size));
+
+end

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Prepare_HOG_AU_data_dynamic.m

@@ -0,0 +1,117 @@
+function [data_train, labels_train, data_devel, labels_devel, raw_devel, PC, means_norm, stds_norm, devel_ids, devel_success] = ...
+    Prepare_HOG_AU_data_dynamic(train_users, devel_users, au_train, rest_aus, Bosphorus_dir, params_data_dir)
+
+%%
+addpath(genpath('../data extraction/'));
+
+% First extracting the labels
+[ labels_train, valid_ids_train, filenames ] = extract_Bosphorus_labels(Bosphorus_dir, train_users, au_train);
+
+[ labels_other, ~, ~ ] = extract_Bosphorus_labels(Bosphorus_dir, train_users, rest_aus);
+
+% Reading in the HOG data (of only relevant frames)
+[train_appearance_data, valid_ids_train_hog, vid_ids_train_string] = Read_HOG_files_dynamic(filenames, params_data_dir);
+
+[train_geom_data] = Read_geom_files_dynamic(filenames,  params_data_dir);
+
+if(numel(train_users) > 0)
+    reduced_inds = false(size(labels_train,1),1);
+    reduced_inds(labels_train > 0) = true;
+
+    % make sure the same number of positive and negative samples is taken
+    pos_count = sum(labels_train > 0);
+    neg_count = sum(labels_train == 0);
+
+    num_other = floor( pos_count / (size(labels_other, 2)));
+
+    inds_all = 1:size(labels_train,1);
+
+    for i=1:size(labels_other, 2)+1
+   
+        if(i > size(labels_other, 2))
+            % fill the rest with a proportion of neutral
+            inds_other = inds_all(sum(labels_other,2)==0 & ~labels_train );   
+                num_other_i = min(numel(inds_other), pos_count - sum(labels_train(reduced_inds,:)==0));     
+        else
+                % take a proportion of each other AU
+            inds_other = inds_all(labels_other(:, i) & ~labels_train );      
+            num_other_i = min(numel(inds_other), num_other);        
+        end
+        inds_other_to_keep = inds_other(round(linspace(1, numel(inds_other), num_other_i)));
+        reduced_inds(inds_other_to_keep) = true;
+
+    end
+
+    % Remove invalid ids based on CLM failing or AU not being labelled
+    reduced_inds(~valid_ids_train) = false;
+    reduced_inds(~valid_ids_train_hog) = false;
+
+    labels_other = labels_other(reduced_inds, :);
+    labels_train = labels_train(reduced_inds,:);
+    train_appearance_data = train_appearance_data(reduced_inds,:);
+    train_geom_data = train_geom_data(reduced_inds,:);
+    vid_ids_train_string = vid_ids_train_string(reduced_inds,:);
+end
+%% Extract devel data
+
+% First extracting the labels
+[ labels_devel, valid_ids_devel, devel_files ] = extract_Bosphorus_labels(Bosphorus_dir, devel_users, au_train);
+
+% Reading in the HOG data (of only relevant frames)
+[devel_appearance_data, valid_ids_devel_hog, vid_ids_devel_string] = Read_HOG_files_dynamic(devel_files, params_data_dir);
+devel_success = valid_ids_devel_hog;
+devel_ids = vid_ids_devel_string;
+
+[devel_geom_data] = Read_geom_files_dynamic(devel_files, params_data_dir);
+
+% Peforming zone specific masking
+if(au_train < 8 || au_train == 43 || au_train == 45) % upper face AUs ignore bottom face
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_upper.mat';
+    load(pca_file);
+elseif(au_train > 9) % lower face AUs ignore upper face and the sides
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_lower.mat';
+    load(pca_file);
+elseif(au_train == 9) % Central face model
+    % normalise the data
+    pca_file = '../../pca_generation/generic_face_rigid.mat';
+    load(pca_file);
+end
+     
+% Grab all data for validation as want good params for all the data
+raw_devel = cat(2, devel_appearance_data, devel_geom_data);
+
+devel_appearance_data = bsxfun(@times, bsxfun(@plus, devel_appearance_data, -means_norm), 1./stds_norm);
+
+data_devel = devel_appearance_data * PC;
+
+data_devel = cat(2, data_devel, devel_geom_data);
+
+valid_ids_devel = valid_ids_devel & devel_success;
+data_devel = data_devel(valid_ids_devel,:);
+labels_devel = labels_devel(valid_ids_devel,:);
+raw_devel = raw_devel(valid_ids_devel,:);
+devel_success = devel_success(valid_ids_devel,:);
+devel_ids = devel_ids(valid_ids_devel);
+
+if(numel(train_users) > 0)
+    train_appearance_data = bsxfun(@times, bsxfun(@plus, train_appearance_data, -means_norm), 1./stds_norm);
+
+    data_train = train_appearance_data * PC;
+    data_train = cat(2, data_train, train_geom_data);
+else
+    data_train = [];
+end
+
+geom_size = max(size(train_geom_data, 2), size(devel_geom_data, 2));
+
+PC_n = zeros(size(PC)+geom_size);
+PC_n(1:size(PC,1), 1:size(PC,2)) = PC;
+PC_n(size(PC,1)+1:end, size(PC,2)+1:end) = eye(geom_size);
+PC = PC_n;
+
+means_norm = cat(2, means_norm, zeros(1, geom_size));
+stds_norm = cat(2, stds_norm, ones(1, geom_size));
+
+end

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Read_HOG_files.m

@@ -0,0 +1,89 @@
+function [hog_data, valid_data, vid_id] = Read_HOG_files(users, hog_data_dir)
+
+    
+    hog_data = [];
+    vid_id = {};
+    valid_data = [];
+    
+    feats_filled = 0;
+
+    for i=1:numel(users)
+        
+        hog_file =[hog_data_dir, '/' users{i} '.hog'];
+        
+        f = fopen(hog_file, 'r');
+
+        curr_data = [];
+        curr_ind = 0;
+
+        while(~feof(f))
+
+            if(curr_ind == 0)
+                num_cols = fread(f, 1, 'int32');
+                if(isempty(num_cols))
+                    break;
+                end
+
+                num_rows = fread(f, 1, 'int32');
+                num_chan = fread(f, 1, 'int32');
+
+                curr_ind = curr_ind + 1;            
+
+                % preallocate some space
+                if(curr_ind == 1)
+                    curr_data = zeros(5000, 1 + num_rows * num_cols * num_chan);
+                    num_feats =  1 + num_rows * num_cols * num_chan;
+                end
+
+                if(curr_ind > size(curr_data,1))
+                    curr_data = cat(1, curr_data, zeros(6000, 1 + num_rows * num_cols * num_chan));
+                end
+                feature_vec = fread(f, [1, 1 + num_rows * num_cols * num_chan], 'float32');
+                curr_data(curr_ind, :) = feature_vec;
+            else
+
+                % Reading in batches of 5000
+                feature_vec = fread(f, [4 + num_rows * num_cols * num_chan, 5000], 'float32');
+                feature_vec = feature_vec(4:end,:)';
+
+                if(isempty(feature_vec))
+                    continue
+                end
+                
+                num_rows_read = size(feature_vec,1);
+
+                curr_data(curr_ind+1:curr_ind+num_rows_read,:) = feature_vec;
+
+                curr_ind = curr_ind + size(feature_vec,1);
+
+            end
+        end
+        fclose(f);
+
+        curr_data = curr_data(1:curr_ind,:);
+        vid_id_curr = cell(curr_ind,1);
+        vid_id_curr(:) = users(i);
+
+        vid_id = cat(1, vid_id, vid_id_curr);
+
+        % Assume same number of frames per video
+        if(i==1)
+            hog_data = zeros(curr_ind*numel(users), num_feats);
+        end
+
+        if(size(hog_data,1) < feats_filled+curr_ind)
+           hog_data = cat(1, hog_data, zeros(size(hog_data,1), num_feats));
+        end
+
+        hog_data(feats_filled+1:feats_filled+curr_ind,:) = curr_data;
+
+        feats_filled = feats_filled + curr_ind;
+
+    end
+    
+    if(numel(users) > 0)
+        valid_data = hog_data(1:feats_filled,1) > 0;
+        hog_data = hog_data(1:feats_filled,2:end);
+    end
+    
+end

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Read_HOG_files_dynamic.m

@@ -0,0 +1,101 @@
+function [hog_data, valid_data, vid_id] = Read_HOG_files_dynamic(users, hog_data_dir)
+
+    
+    hog_data = [];
+    vid_id = {};
+    valid_data = [];
+    
+    user_files = [];
+    
+    feats_filled = 0;
+        
+    for i=1:numel(users)
+        
+        hog_file = [hog_data_dir, '/' users{i} '.hog'];
+        
+        user_files = cat(1, user_files, str2num(users{i}(3:5)));
+        
+        f = fopen(hog_file, 'r');
+
+        curr_data = [];
+        curr_ind = 0;
+
+        while(~feof(f))
+
+            if(curr_ind == 0)
+                num_cols = fread(f, 1, 'int32');
+                if(isempty(num_cols))
+                    break;
+                end
+
+                num_rows = fread(f, 1, 'int32');
+                num_chan = fread(f, 1, 'int32');
+
+                curr_ind = curr_ind + 1;            
+
+                % preallocate some space
+                if(curr_ind == 1)
+                    curr_data = zeros(5000, 1 + num_rows * num_cols * num_chan);
+                    num_feats =  1 + num_rows * num_cols * num_chan;
+                end
+
+                if(curr_ind > size(curr_data,1))
+                    curr_data = cat(1, curr_data, zeros(6000, 1 + num_rows * num_cols * num_chan));
+                end
+                feature_vec = fread(f, [1, 1 + num_rows * num_cols * num_chan], 'float32');
+                curr_data(curr_ind, :) = feature_vec;
+            else
+
+                % Reading in batches of 5000
+
+                feature_vec = fread(f, [4 + num_rows * num_cols * num_chan, 5000], 'float32');
+                feature_vec = feature_vec(4:end,:)';
+
+                if(isempty(feature_vec))
+                    continue;
+                end
+                
+                num_rows_read = size(feature_vec,1);
+
+                curr_data(curr_ind+1:curr_ind+num_rows_read,:) = feature_vec;
+
+                curr_ind = curr_ind + size(feature_vec,1);
+
+            end
+
+        end
+
+        fclose(f);
+
+        curr_data = curr_data(1:curr_ind,:);
+        vid_id_curr = cell(curr_ind,1);
+        vid_id_curr(:) = users(i);
+
+        vid_id = cat(1, vid_id, vid_id_curr);
+
+        % Assume same number of frames per video
+        if(i==1)
+            hog_data = zeros(curr_ind*numel(users), num_feats);
+        end
+
+        if(size(hog_data,1) < feats_filled+curr_ind)
+           hog_data = cat(1, hog_data, zeros(size(hog_data,1), num_feats));
+        end
+
+        hog_data(feats_filled+1:feats_filled+curr_ind,:) = curr_data;
+
+        feats_filled = feats_filled + curr_ind;
+        
+    end
+    
+    if(numel(users) > 0)
+        valid_data = hog_data(:,1) > 0;
+        hog_data = hog_data(:,2:end);
+        
+        uq_ids = unique(user_files)';
+        
+        for u=uq_ids
+            hog_data(user_files==u,:) = bsxfun(@plus, hog_data(user_files==u,:), -median( hog_data(valid_data & user_files==u,:)));        
+        end
+    end
+end

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Read_geom_files.m

@@ -0,0 +1,36 @@
+function [geom_data, valid_ids] = Read_geom_files(users, model_param_data_dir)
+
+    geom_data = [];
+    valid_ids = [];
+    
+    load('../../pca_generation/pdm_68_aligned_wild.mat');
+    
+    for i=1:numel(users)
+    
+        geom_file = [model_param_data_dir, '/' users{i} '.csv'];
+
+        if(i == 1)
+            tab = readtable(geom_file);
+            column_names = tab.Properties.VariableNames; 
+            valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'confidence'));
+            shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_'));
+        end
+                
+        res = dlmread(geom_file, ',', 1, 0);    
+
+        % Check the confidence of detection
+        valid = res(:, valid_ind) > 0.7;        
+        res = res(:, shape_inds);
+        
+        % Do not consider global parameters
+        res = res(:, 7:end);
+        
+        actual_locs = res * V';
+        res = cat(2, actual_locs, res);
+
+        valid_ids = cat(1, valid_ids, valid);
+
+        geom_data = cat(1, geom_data, res);
+
+    end
+end

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Read_geom_files_dynamic.m

@@ -0,0 +1,50 @@
+function [geom_data, valid_ids] = Read_geom_files_dynamic(users, model_param_data_dir)
+
+    geom_data = [];
+    valid_ids = [];
+    
+    load('../../pca_generation/pdm_68_aligned_wild.mat');
+    
+    user_files = [];
+    
+    for i=1:numel(users)
+        
+        geom_file = [model_param_data_dir, '/' users{i} '.csv'];
+
+        user_files = cat(1, user_files, str2num(users{i}(3:5)));
+
+        if(i == 1)
+            tab = readtable(geom_file);
+            column_names = tab.Properties.VariableNames; 
+            valid_ind = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'confidence'));
+            shape_inds = cellfun(@(x) ~isempty(x) && x==1, strfind(column_names, 'p_'));
+        end
+                
+        res = dlmread(geom_file, ',', 1, 0);    
+
+        valid = res(:, valid_ind) > 0.7;     
+        res = res(:, shape_inds);
+        
+        % Do not consider global parameters
+        res = res(:, 7:end);
+        
+        actual_locs = res * V';
+        res = cat(2, actual_locs, res);
+
+        valid_ids = cat(1, valid_ids, valid);
+
+        geom_data_curr_p = res;
+
+        geom_data = cat(1, geom_data, geom_data_curr_p);
+        
+    end
+       
+    if(numel(users) > 0)
+        uq_ids = unique(user_files)';
+        
+        for u=uq_ids
+            geom_data(user_files==u,:) = bsxfun(@plus, geom_data(user_files==u,:), -median( geom_data(valid_ids & user_files==u,:)));        
+        end
+    end    
+    
+end

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Script_HOG_SVM_train.m

@@ -0,0 +1,80 @@
+function Script_HOG_SVM_train()
+
+% Change to your downloaded location
+addpath('C:\liblinear\matlab')
+addpath('../training_code/');
+addpath('../utilities/');
+addpath('../../data extraction/');
+%% load shared definitions and AU data
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-9:0.5:1);
+hyperparams.e = 10.^(-3);
+
+hyperparams.validate_params = {'c', 'e'};
+
+% Set the training function
+svm_train = @svm_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svm_test = @svm_test_linear;
+
+%%
+for a=1:numel(aus)
+    
+    au = aus(a);
+            
+    rest_aus = setdiff(all_aus, au);        
+    
+    [users_train, users_valid] = get_balanced_fold(Bosphorus_dir, all_recs, au, 1/3, 1);
+    
+    % load the training and testing data for the current fold    
+    [train_samples, train_labels, valid_samples, valid_labels, ~, PC, means, scaling] = Prepare_HOG_AU_data(users_train, users_valid, au, rest_aus, Bosphorus_dir, hog_data_dir);
+    
+    % Binarizing the data
+    train_labels(train_labels > 1) = 1;
+    valid_labels(valid_labels > 1) = 1;
+    
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svm_train, svm_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+
+    model = svm_train(train_labels, train_samples, best_params);        
+
+    [prediction, a, actual_vals] = predict(valid_labels, valid_samples, model);
+
+    % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_static.dat', au);
+
+    pos_lbl = model.Label(1);
+    neg_lbl = model.Label(2);
+        
+    write_lin_svm(name, means, svs, b, pos_lbl, neg_lbl);
+
+    name = sprintf('results_Bosphorus_devel/AU_%d_static.mat', au);
+
+    tp = sum(valid_labels == 1 & prediction == 1);
+    fp = sum(valid_labels == 0 & prediction == 1);
+    fn = sum(valid_labels == 1 & prediction == 0);
+    tn = sum(valid_labels == 0 & prediction == 0);
+
+    precision = tp/(tp+fp);
+    recall = tp/(tp+fn);
+
+    f1 = 2 * precision * recall / (precision + recall);    
+    
+    save(name, 'model', 'f1', 'precision', 'recall', 'best_params', 'valid_labels', 'prediction', 'users_valid');
+        
+end
+
+end
+
+

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Script_HOG_SVM_train_dyn.m

@@ -0,0 +1,83 @@
+function Script_HOG_SVM_train_dyn()
+
+% Change to your downloaded location
+addpath('C:\liblinear\matlab')
+addpath('../training_code/');
+addpath('../utilities/');
+addpath('../../data extraction/');
+
+%% load shared definitions and AU data
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-9:0.5:1);
+hyperparams.e = 10.^(-3);
+
+hyperparams.validate_params = {'c', 'e'};
+
+% Set the training function
+svm_train = @svm_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svm_test = @svm_test_linear;
+
+pca_loc = '../../pca_generation/generic_face_rigid.mat';
+
+%%
+for a=1:numel(aus)
+    
+    au = aus(a);
+            
+    rest_aus = setdiff(all_aus, au);        
+
+    [users_train, users_valid] = get_balanced_fold(Bosphorus_dir, all_recs, au, 1/3, 1);
+    
+    % load the training and testing data for the current fold    
+    [train_samples, train_labels, valid_samples, valid_labels, ~, PC, means, scaling] = Prepare_HOG_AU_data_dynamic(users_train, users_valid, au, rest_aus, Bosphorus_dir, hog_data_dir);
+    
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+    
+    % Binarizing the data
+    train_labels(train_labels > 1) = 1;
+    valid_labels(valid_labels > 1) = 1;
+    
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svm_train, svm_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+
+    model = svm_train(train_labels, train_samples, best_params);        
+
+    [prediction, a, actual_vals] = predict(valid_labels, valid_samples, model);
+
+    % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_dyn.dat', au);
+
+    pos_lbl = model.Label(1);
+    neg_lbl = model.Label(2);
+        
+    write_lin_dyn_svm(name, means, svs, b, pos_lbl, neg_lbl);
+
+    name = sprintf('results_Bosphorus_devel/AU_%d_dyn.mat', au);
+
+    tp = sum(valid_labels == 1 & prediction == 1);
+    fp = sum(valid_labels == 0 & prediction == 1);
+    fn = sum(valid_labels == 1 & prediction == 0);
+    tn = sum(valid_labels == 0 & prediction == 0);
+
+    precision = tp/(tp+fp);
+    recall = tp/(tp+fn);
+
+    f1 = 2 * precision * recall / (precision + recall);    
+    
+    save(name, 'model', 'f1', 'precision', 'recall', 'best_params', 'valid_labels', 'prediction', 'users_valid');
+        
+end
+
+end
+
+

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Script_HOG_SVR_train.m

@@ -0,0 +1,68 @@
+function Script_HOG_SVR_train()
+
+% Change to your downloaded location
+addpath('C:\liblinear\matlab')
+addpath('../training_code/');
+addpath('../utilities/');
+addpath('../../data extraction/');
+
+%% load shared definitions and AU data
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:1:4);
+hyperparams.p = 10.^(-2);
+
+hyperparams.validate_params = {'c', 'p'};
+
+% Set the training function
+svr_train = @svr_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svr_test = @svr_test_linear;
+
+%%
+for a=1:numel(aus)
+    
+    au = aus(a);
+            
+    rest_aus = setdiff(all_aus, au);        
+    
+    [users_train, users_valid] = get_balanced_fold(Bosphorus_dir, all_recs, au, 1/3, 1);
+    
+    % load the training and testing data for the current fold    
+    [train_samples, train_labels, valid_samples, valid_labels, ~, PC, means, scaling, valid_ids, valid_success] = Prepare_HOG_AU_data(users_train, users_valid, au, rest_aus, Bosphorus_dir, hog_data_dir);
+        
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    hyperparams.success = valid_success;    
+    
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svr_train, svr_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+
+    model = svr_train(train_labels, train_samples, best_params);        
+
+    [~, prediction] = svr_test(valid_labels, valid_samples, model);
+
+    % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_static_intensity.dat', au);
+
+    write_lin_svr(name, means, svs, b);
+
+    name = sprintf('results_Bosphorus_devel/AU_%d_static_intensity.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( prediction, valid_labels );
+            
+    save(name, 'model', 'F1s', 'corrs', 'accuracies', 'ccc', 'rms', 'prediction', 'valid_labels', 'users_valid');
+        
+end
+
+end
+
+

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/Script_HOG_SVR_train_dyn.m

@@ -0,0 +1,68 @@
+function Script_HOG_SVR_train_dyn()
+
+% Change to your downloaded location
+addpath('C:\liblinear\matlab')
+addpath('../training_code/');
+addpath('../utilities/');
+addpath('../../data extraction/');
+
+%% load shared definitions and AU data
+shared_defs;
+
+% Set up the hyperparameters to be validated
+hyperparams.c = 10.^(-7:1:4);
+hyperparams.p = 10.^(-2);
+
+hyperparams.validate_params = {'c', 'p'};
+
+% Set the training function
+svr_train = @svr_train_linear;
+    
+% Set the test function (the first output will be used for validation)
+svr_test = @svr_test_linear;
+
+%%
+for a=1:numel(aus)
+    
+    au = aus(a);
+            
+    rest_aus = setdiff(all_aus, au);        
+    
+    [users_train, users_valid] = get_balanced_fold(Bosphorus_dir, all_recs, au, 1/3, 1);
+    
+    % load the training and testing data for the current fold    
+    [train_samples, train_labels, valid_samples, valid_labels, ~, PC, means, scaling, valid_ids, valid_success] = Prepare_HOG_AU_data_dynamic(users_train, users_valid, au, rest_aus, Bosphorus_dir, hog_data_dir);
+        
+    train_samples = sparse(train_samples);
+    valid_samples = sparse(valid_samples);
+
+    hyperparams.success = valid_success;    
+    
+    %% Cross-validate here                
+    [ best_params, ~ ] = validate_grid_search_no_par(svr_train, svr_test, false, train_samples, train_labels, valid_samples, valid_labels, hyperparams);
+
+    model = svr_train(train_labels, train_samples, best_params);        
+
+    [~, prediction] = svr_test(valid_labels, valid_samples, model);
+
+    % Go from raw data to the prediction
+    w = model.w(1:end-1)';
+    b = model.w(end);
+
+    svs = bsxfun(@times, PC, 1./scaling') * w;
+
+    name = sprintf('models/AU_%d_static_intensity.dat', au);
+
+    write_lin_dyn_svr(name, means, svs, b, 0);
+
+    name = sprintf('results_Bosphorus_devel/AU_%d_dyn_intensity.mat', au);
+
+    [ accuracies, F1s, corrs, ccc, rms, classes ] = evaluate_regression_results( prediction, valid_labels );
+            
+    save(name, 'model', 'F1s', 'corrs', 'accuracies', 'ccc', 'rms', 'prediction', 'valid_labels', 'users_valid');
+        
+end
+
+end
+
+

pkg/OpenFace/model_training/AU_training/experiments/Bosphorus/get_balanced_fold.m

@@ -0,0 +1,35 @@
+function [train_users, dev_users] = get_balanced_fold(Bosphorus_dir, users, au, prop_test, offset)
+
+
+    [ labels_train, valid_ids_train, filenames ] = extract_Bosphorus_labels(Bosphorus_dir, users, au);
+        
+    labels_train = labels_train(valid_ids_train,:);
+    filenames = filenames(valid_ids_train);
+    
+    % Shorten the filenames to reflect user id
+    for i=1:numel(filenames)
+        filenames{i} = filenames{i}(1:5);
+    end
+    
+    counts = zeros(numel(users),1);
+    for k=1:numel(users)
+        counts(k) = sum(labels_train(strcmp(filenames, users{k}))>0);
+    end
+
+    [sorted, inds] = sort(counts);
+    
+    dev_users = users(inds(offset:round(1/prop_test):end));
+    train_users = setdiff(users, dev_users);
+    
+    count_dev = 0;
+    count_train = 0;
+    for k=1:numel(users)
+        if(any(strcmp(dev_users, users{k})))
+            count_dev = count_dev + counts(k);
+        else
+            count_train = count_train + counts(k);
+        end
+        
+    end
+
+end