open source pkg v1

pkg/OpenFace/model_training/AU_training/pca_generation/Read_HOG_files_small.m

@@ -0,0 +1,107 @@
+function [hog_data, valid_inds, vid_id] = Read_HOG_files_small(hog_files, hog_data_dir, num_samples)
+
+    hog_data = [];
+    vid_id = {};
+    
+    feats_filled = 0;
+
+    curr_data_buff = [];   
+    
+    for i=1:numel(hog_files)
+        
+        hog_file = [hog_data_dir, hog_files(i).name];
+        
+        fprintf('%d %s\n', i, hog_file);
+        
+        f = fopen(hog_file, 'r');
+                         
+        curr_ind = 0;
+        
+        while(~feof(f))
+                        
+            if(i == 1 && curr_ind == 0)
+                num_cols = fread(f, 1, 'int32');
+                if(isempty(num_cols))
+                    break;
+                end
+
+                num_rows = fread(f, 1, 'int32');
+                num_chan = fread(f, 1, 'int32');
+                
+                curr_ind = curr_ind + 1;            
+
+                % preallocate some space
+                if(curr_ind == 1)
+                    curr_data_buff = zeros(5000, 1 + num_rows * num_cols * num_chan);
+                    
+                    num_feats =  1 + num_rows * num_cols * num_chan;
+                end
+
+                if(curr_ind > size(curr_data_buff,1))
+                    curr_data_buff = cat(1, curr_data_buff, zeros(6000, num_rows * num_cols * num_chan));
+                end
+                feature_vec = fread(f, [1, 1 + num_rows * num_cols * num_chan], 'float32');
+                curr_data_buff(curr_ind, :) = feature_vec;
+                
+            else
+            
+                % Reading in batches of 5000
+                
+                feature_vec = fread(f, [4 + num_rows * num_cols * num_chan, 5000], 'float32');
+                feature_vec = feature_vec(4:end,:)';
+                
+                if(~isempty(feature_vec))
+                    num_rows_read = size(feature_vec,1);
+
+                    curr_data_buff(curr_ind+1:curr_ind+num_rows_read,:) = feature_vec;
+                    %valid_data_buff = 
+                    curr_ind = curr_ind + size(feature_vec,1);
+                end
+            end
+                        
+        end
+        
+        fclose(f);
+        
+        curr_data_small = curr_data_buff(1:curr_ind,:);
+        vid_id_curr = cell(curr_ind,1);
+        vid_id_curr(:) = {hog_files(i).name};
+        
+        % Keep up to 20 frames from the whole video (so that it is balanced
+        % per dataset/video/participant)
+        
+        if(nargin > 2)
+            num_instances = num_samples;
+        else
+            num_instances = 20;
+        end
+        
+        increment = round(curr_ind / num_instances);
+        if(increment == 0)
+            increment = 1;
+        end
+        
+        curr_data_small = curr_data_small(1:increment:end,:);
+        vid_id_curr = vid_id_curr(1:increment:end,:);        
+        
+        vid_id = cat(1, vid_id, vid_id_curr);
+        
+        % Assume same number of frames per video
+        if(i==1)
+            hog_data = zeros(10*numel(hog_files), num_feats);
+        end
+        
+        if(size(hog_data,1) < feats_filled+size(curr_data_small,1))
+           hog_data = cat(1, hog_data, zeros(feats_filled + size(curr_data_small,1) - size(hog_data,1), num_feats));
+        end
+        
+        hog_data(feats_filled+1:feats_filled + size(curr_data_small,1),:) = curr_data_small;
+        
+        feats_filled = feats_filled + size(curr_data_small,1);
+        
+    end
+    
+    valid_inds = hog_data(1:feats_filled,1) > 0;
+    hog_data = hog_data(1:feats_filled,2:end);
+    
+end

pkg/OpenFace/model_training/AU_training/pca_generation/createGenericFaceModel.m

@@ -0,0 +1,112 @@
+clear;
+face_processed_dir = 'E:\datasets\face_datasets_processed';
+
+%% CK+
+hog_dir = [face_processed_dir, '/ck+/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data, valid_inds, vid_ids_train] = Read_HOG_files_small(hog_files, hog_dir);
+appearance_data = appearance_data(valid_inds,:);
+vid_ids_train = vid_ids_train(valid_inds,:);
+
+%% Bosphorus
+hog_dir = [face_processed_dir, '/bosph/'];
+hog_files = dir([hog_dir '*.hog']);
+
+% Remove non-frontal
+frontal = true(size(hog_files));
+for i = 1:numel(frontal)
+    if(~isempty(strfind(hog_files(i).name, 'YR')) || ~isempty(strfind(hog_files(i).name, 'PR'))|| ~isempty(strfind(hog_files(i).name, 'CR')))
+        frontal(i) = false;
+    end   
+end
+
+hog_files = hog_files(frontal);
+
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% FERA2011
+hog_dir = [face_processed_dir, '/fera2011/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+ appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% UNBC
+hog_dir = [face_processed_dir, '/unbc/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+
+
+%% DISFA
+hog_dir = [face_processed_dir, '/disfa/'];
+hog_files = dir([hog_dir '*.hog']);
+
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% BP4D train
+hog_dir = [face_processed_dir, '/bp4d/train/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% SEMAINE train
+hog_dir = [face_processed_dir, '/semaine/train/'];
+hog_files = dir([hog_dir '*.hog']);
+
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%%
+means_norm = mean(appearance_data);
+stds_norm = std(appearance_data);
+
+normed_data = bsxfun(@times, bsxfun(@plus, appearance_data, -means_norm), 1./stds_norm);
+
+[PC, score, eigen_vals] = princomp(normed_data, 'econ');
+
+% Keep 95 percent of variability
+total_sum = sum(eigen_vals);
+count = numel(eigen_vals);
+for i=1:numel(eigen_vals)
+   if ((sum(eigen_vals(1:i)) / total_sum) >= 0.95)
+      count = i;
+      break;
+   end
+end
+
+PC = PC(:,1:count);
+
+save('generic_face_rigid.mat', 'PC', 'means_norm', 'stds_norm');    

pkg/OpenFace/model_training/AU_training/pca_generation/createSubFaceModels.m

@@ -0,0 +1,153 @@
+clear;
+face_processed_dir = 'E:\datasets\face_datasets_processed';
+
+%% CK+
+hog_dir = [face_processed_dir, '/ck+/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data, valid_inds, vid_ids_train] = Read_HOG_files_small(hog_files, hog_dir);
+appearance_data = appearance_data(valid_inds,:);
+vid_ids_train = vid_ids_train(valid_inds,:);
+
+%% Bosphorus
+hog_dir = [face_processed_dir, '/bosph/'];
+hog_files = dir([hog_dir '*.hog']);
+
+% Remove non-frontal
+frontal = true(size(hog_files));
+for i = 1:numel(frontal)
+    if(~isempty(strfind(hog_files(i).name, 'YR')) || ~isempty(strfind(hog_files(i).name, 'PR'))|| ~isempty(strfind(hog_files(i).name, 'CR')))
+        frontal(i) = false;
+    end   
+end
+
+hog_files = hog_files(frontal);
+
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% FERA2011
+hog_dir = [face_processed_dir, '/fera2011/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+ appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% UNBC
+hog_dir = [face_processed_dir, '/unbc/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+
+
+%% DISFA
+hog_dir = [face_processed_dir, '/disfa/'];
+hog_files = dir([hog_dir '*.hog']);
+
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% BP4D train
+hog_dir = [face_processed_dir, '/bp4d/train/'];
+hog_files = dir([hog_dir '*.hog']);
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%% SEMAINE train
+hog_dir = [face_processed_dir, '/semaine/train/'];
+hog_files = dir([hog_dir '*.hog']);
+
+[appearance_data_tmp, valid_inds_tmp, vid_ids_train_tmp] = Read_HOG_files_small(hog_files, hog_dir);
+
+appearance_data_tmp = appearance_data_tmp(valid_inds_tmp,:);
+vid_ids_train_tmp = vid_ids_train_tmp(valid_inds_tmp,:);
+
+appearance_data = cat(1,appearance_data, appearance_data_tmp);
+vid_ids_train = cat(1,vid_ids_train, vid_ids_train_tmp);
+
+%%
+means_norm = mean(appearance_data);
+stds_norm = std(appearance_data);
+
+normed_data = bsxfun(@times, bsxfun(@plus, appearance_data, -means_norm), 1./stds_norm);
+
+%% Creating a generic model
+[PC, score, eigen_vals] = princomp(normed_data, 'econ');
+
+% Keep 95 percent of variability
+total_sum = sum(eigen_vals);
+count = numel(eigen_vals);
+for i=1:numel(eigen_vals)
+   if ((sum(eigen_vals(1:i)) / total_sum) >= 0.95)
+      count = i;
+      break;
+   end
+end
+
+PC = PC(:,1:count);
+
+save('generic_face_rigid.mat', 'PC', 'means_norm', 'stds_norm');    
+
+%% Creating a lower face model
+normed_data_lower_face = normed_data;
+normed_data_lower_face(:, 1:5*12*31) = 0;
+
+[PC, score, eigen_vals] = princomp(normed_data_lower_face, 'econ');
+
+% Keep 98 percent of variability
+total_sum = sum(eigen_vals);
+count = numel(eigen_vals);
+for i=1:numel(eigen_vals)
+   if ((sum(eigen_vals(1:i)) / total_sum) >= 0.98)
+      count = i;
+      break;
+   end
+end
+
+PC = PC(:,1:count);
+
+save('generic_face_lower.mat', 'PC', 'means_norm', 'stds_norm');   
+
+%% Creating an upper face model
+normed_data_upper_face = normed_data;
+normed_data_upper_face(:, end-5*12*31+1:end) = 0;
+
+[PC, score, eigen_vals] = princomp(normed_data_upper_face, 'econ');
+
+% Keep 98 percent of variability
+total_sum = sum(eigen_vals);
+count = numel(eigen_vals);
+for i=1:numel(eigen_vals)
+   if ((sum(eigen_vals(1:i)) / total_sum) >= 0.98)
+      count = i;
+      break;
+   end
+end
+
+PC = PC(:,1:count);
+
+save('generic_face_upper.mat', 'PC', 'means_norm', 'stds_norm');