bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
pkg/OpenFace/matlab_version/demo/face_image_demo.m Normal file
View File

@@ -0,0 +1,97 @@
clear
addpath('../PDM_helpers/');
addpath(genpath('../fitting/'));
addpath('../models/');
addpath(genpath('../face_detection'));
addpath('../CCNF/');
%% loading the patch experts
% Default OpenFace landmark model, using CE-CLM patch experts
[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
% faster but less accurate
%[patches, pdm, clmParams] = Load_CLNF_general();
% even faster but even less accurate
%[patches, pdm, clmParams] = Load_CLM_general();
% Using a multi-view approach
views = [0,0,0; 0,-30,0; 0,30,0; 0,0,30; 0,0,-30;];
views = views * pi/180;
% Dependencies for face detection (MatConvNet), remove if not present
setup_mconvnet;
addpath('../face_detection/mtcnn/');
%%
root_dir = '../../samples/';
images = dir([root_dir, '*.jpg']);
verbose = true;
for img=1:numel(images)
image_orig = imread([root_dir images(img).name]);
% Face detectiopn
[bboxs] = detect_faces(image_orig, 'mtcnn');
% If MTCNN detector not available, can use the cascaded regression one
% [bboxs] = detect_faces(image_orig, 'cascade');
if(size(image_orig,3) == 3)
image_gray = rgb2gray(image_orig);
else
image_gray = image_orig;
end
%%
if(verbose)
f = figure;
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
end
for i=1:size(bboxs,1)
% Convert from the initial detected shape to CLM model parameters,
% if shape is available
bbox = bboxs(i,:);
if(exist('early_term_params', 'var'))
[shape,~,~,lhood,lmark_lhood,view_used] =...
Fitting_from_bb_multi_hyp(image_gray, [], bbox, pdm, patches, clmParams, views, early_term_params);
else
[shape,~,~,lhood,lmark_lhood,view_used] =...
Fitting_from_bb_multi_hyp(image_gray, [], bbox, pdm, patches, clmParams, views);
end
% shape correction for matlab format
shape = shape + 1;
if(verbose)
% valid points to draw (not to draw self-occluded ones)
v_points = logical(patches(1).visibilities(view_used,:));
try
plot(shape(v_points,1), shape(v_points',2),'.r','MarkerSize',20);
plot(shape(v_points,1), shape(v_points',2),'.b','MarkerSize',10);
catch warn
end
end
end
hold off;
end

93
pkg/OpenFace/matlab_version/demo/face_image_demo_eyes.m Normal file
View File

@@ -0,0 +1,93 @@
clear
addpath('../PDM_helpers/');
addpath(genpath('../fitting/'));
addpath('../models/');
addpath(genpath('../face_detection'));
addpath('../CCNF/');
%% loading the patch experts
% Default OpenFace landmark model, using CE-CLM patch experts
[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
% faster but less accurate
%[patches, pdm, clmParams] = Load_CLNF_general();
% even faster but even less accurate
%[patches, pdm, clmParams] = Load_CLM_general();
% Using a multi-view approach
views = [0,0,0; 0,-30,0; 0,30,0; 0,0,30; 0,0,-30;];
views = views * pi/180;
% Load the eye landmark models that will be used
[ clmParams_eye, pdm_right_eye, pdm_left_eye, ...
patches_left_eye, patches_right_eye,...
left_eye_inds_in_68, right_eye_inds_in_68,...
left_eye_inds_in_28, right_eye_inds_in_28] = Load_eye_models();
%%
% root_dir = 'C:\Users\Tadas\Dropbox\AAM\test data\gaze_original\p00/';
% images = dir([root_dir, '*.jpg']);
%root_dir = './sample_eye_imgs/';
%images = dir([root_dir, '/*.png']);
root_dir = '../../samples/';
images = dir([root_dir, '*.jpg']);
verbose = true;
for img=1:numel(images)
image_orig = imread([root_dir images(img).name]);
% Face detection
[bboxs] = detect_faces(image_orig, 'mtcnn');
% If MTCNN detector not available, can use the cascaded regression one
% [bboxs] = detect_faces(image_orig, 'cascade');
if(size(image_orig,3) == 3)
image_gray = rgb2gray(image_orig);
else
image_gray = image_orig;
end
%%
if(verbose)
f = figure;
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
end
for i=1:size(bboxs,1)
% Convert from the initial detected shape to CLM model parameters,
% if shape is available
bbox = bboxs(i,:);
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image_gray, [], bbox, pdm, patches, clmParams, views);
% Perform eye fitting now
[shape, shape_r_eye] = Fitting_from_bb_hierarch(image_gray, pdm, pdm_right_eye, patches_right_eye, clmParams_eye, shape, right_eye_inds_in_68, right_eye_inds_in_28);
[shape, shape_l_eye] = Fitting_from_bb_hierarch(image_gray, pdm, pdm_left_eye, patches_left_eye, clmParams_eye, shape, left_eye_inds_in_68, left_eye_inds_in_28);
% Convert it to matlab convention
shape_r_eye = shape_r_eye + 1;
shape_l_eye = shape_l_eye + 1;
plot(shape_l_eye(9:20,1), shape_l_eye(9:20,2), '.g', 'MarkerSize',7);
plot(shape_l_eye(1:8,1), shape_l_eye(1:8,2), '.b', 'MarkerSize',7);
plot(shape_r_eye(9:20,1), shape_r_eye(9:20,2), '.g', 'MarkerSize',7);
plot(shape_r_eye(1:8,1), shape_r_eye(1:8,2), '.b', 'MarkerSize',7);
end
hold off;
end

80
pkg/OpenFace/matlab_version/demo/face_image_depth_demo.m Normal file
View File

@@ -0,0 +1,80 @@
clear
addpath('../PDM_helpers/');
addpath(genpath('../fitting/'));
addpath('../models/');
addpath(genpath('../face_detection'));
addpath('../CCNF/');
%% loading the patch experts
[clmParams, pdm] = Load_CLM_params_66();
% A CLM-Z model trained on Multi-PIE and BU-4DFE
[patches] = Load_Patch_Experts( '../models/clmz/', 'svr_patches_multi_pie_*.mat', '../models/clmz/', 'svr_depth_patches_*.mat', clmParams);
clmParams.multi_modal_types = patches(1).multi_modal_types;
%%
images = {'sample_depth_imgs/1.jpg', 'sample_depth_imgs/2.jpg', 'sample_depth_imgs/3.jpg', 'sample_depth_imgs/4.jpg', 'sample_depth_imgs/5.jpg'};
images_depth = {'sample_depth_imgs/1d.png', 'sample_depth_imgs/2d.png', 'sample_depth_imgs/3d.png', 'sample_depth_imgs/4d.png', 'sample_depth_imgs/5d.png'};
verbose = true;
for img=1:numel(images)
image_orig = imread(images{img});
image_depth = imread(images_depth{img});
% Need to convert from the disparity to depth values, and threshold
image_depth = 10000./(image_depth);
image_depth(image_depth > 300) = 0;
% First attempt to use the Matlab one (fastest but not as accurate, if not present use yu et al.)
[bboxs] = detect_faces(image_orig, {'cascade', 'zhu'});
if(size(image_orig,3) == 3)
image = rgb2gray(image_orig);
end
%%
if(verbose)
f = figure;
if(max(image(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
end
for i=1:size(bboxs,2)
% Convert from the initial detected shape to CLM model parameters
bbox = bboxs(:,i);
% Use the initial global and local params for clm fitting in the image
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, image_depth, bbox, pdm, patches, clmParams);
% shape correction for matlab format
shape = shape + 1;
if(verbose)
% valid points to draw (not to draw self-occluded ones)
v_points = logical(patches(1).visibilities(view_used,:));
try
plot(shape(v_points,1), shape(v_points',2),'.r','MarkerSize',20);
plot(shape(v_points,1), shape(v_points',2),'.b','MarkerSize',10);
catch warn
end
end
end
hold off;
end

202
pkg/OpenFace/matlab_version/demo/face_video_demo.m Normal file
View File

@@ -0,0 +1,202 @@
clear
addpath('../PDM_helpers/');
addpath(genpath('../fitting/'));
addpath('../models/');
addpath(genpath('../face_detection'));
addpath('../CCNF/');
%%
vid_dir = '../../samples/';
vids = cat(1, dir([vid_dir, '*.avi']), dir([vid_dir, '*.wmv']));
%%
verbose = true;
record = true;
%% loading the patch experts
% Default OpenFace landmark model, using CE-CLM patch experts
[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
% faster but less accurate
%[patches, pdm, clmParams] = Load_CLNF_general();
% even faster but even less accurate
%[patches, pdm, clmParams] = Load_CLM_general();
% load the face validator and add its dependency
load('../face_validation/trained/faceCheckers.mat', 'faceCheckers');
addpath(genpath('../face_validation'));
od = cd('../face_validation/');
setup;
cd(od);
% Setup the face detector (remove the setup mconvnet if not using
% MatConvNet)
setup_mconvnet;
addpath('../face_detection/mtcnn/');
%%
for v=1:numel(vids)
% load the video
vr = VideoReader([vid_dir, vids(v).name]);
[~,fname,~] = fileparts(vids(v).name);
if(record)
if(~exist('./tracked_vids', 'file'))
mkdir('tracked_vids');
end
writerObj = VideoWriter(sprintf('./tracked_vids/%s.avi', fname));
open(writerObj);
end
det = false;
initialised = false;
nFrames = vr.NumberOfFrames;
% Read one frame at a time.
all_local_params = zeros(nFrames, numel(pdm.E));
all_global_params = zeros(nFrames,6);
for i = 1 : nFrames
% if this version throws a "Dot name reference on non-scalar structure"
% error change obj.NumberOfFrames to obj(1).NumberOfFrames (in two
% places in read function) or surround it with an empty try catch
% statement
image_orig = read(vr, i);
if((~det && mod(i,4) == 0) || ~initialised)
% Face detection
[bboxs] = detect_faces(image_orig, 'mtcnn');
% If MTCNN detector not available, can use the cascaded regression one
% [bboxs] = detect_faces(image_orig, 'cascade');
if(~isempty(bboxs))
% Pick the biggest face for tracking
[~,ind] = max(bboxs(:,3) - bboxs(:,1));
bbox = bboxs(ind,:);
% Discard overly small detections
if(bbox(3) - bbox(1) > 40)
% Either infer the local and global shape parameters
% from the detected landmarks or just using the
% bounding box
num_points = numel(pdm.M) / 3;
M = reshape(pdm.M, num_points, 3);
width_model = max(M(:,1)) - min(M(:,1));
height_model = max(M(:,2)) - min(M(:,2));
a = (((bbox(3) - bbox(1)) / width_model) + ((bbox(4) - bbox(2))/ height_model)) / 2;
tx = (bbox(3) + bbox(1))/2;
ty = (bbox(4) + bbox(2))/2;
% correct it so that the bounding box is just around the minimum
% and maximum point in the initialised face
tx = tx - a*(min(M(:,1)) + max(M(:,1)))/2;
ty = ty + a*(min(M(:,2)) + max(M(:,2)))/2;
% visualisation
g_param_n = [a, 0, 0, 0, tx, ty]';
l_param_n = zeros(size(pdm.E));
% If tracking has not started trust the detection
if(~initialised)
g_param = g_param_n;
l_param = l_param_n;
det = true;
initialised = true;
else
% If tracking has already started double check the
% detection
shape_new = GetShapeOrtho(pdm.M, pdm.V, params, g_param_n);
dec = face_check_cnn(image, shape_new, g_param, faceCheckers);
if(dec < 0.5)
det = true;
g_param = g_param_n;
l_param = l_param_n;
else
det = false;
end
end
end
end
end
if(size(image_orig,3) == 3)
image = rgb2gray(image_orig);
else
image = image_orig;
end
d_image = [];
if(initialised)
[shape,g_param,l_param,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, d_image, bbox, pdm, patches, clmParams, 'gparam', g_param, 'lparam', l_param);
all_local_params(i,:) = l_param;
all_global_params(i,:) = g_param;
dec = face_check_cnn(image, shape, g_param, faceCheckers);
if(dec < 0.5)
clmParams.window_size = [19,19; 17,17;];
clmParams.numPatchIters = 2;
det = true;
else
clmParams.window_size = [21,21; 19,19; 17,17;];
clmParams.numPatchIters = 3;
det = false;
end
end
if(verbose)
try
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
if(initialised)
plot(shape(:,1), shape(:,2),'.r','MarkerSize',20);
plot(shape(:,1), shape(:,2),'.b','MarkerSize',10);
end
hold off;
drawnow expose;
pause(0.01);
if(record)
frame = getframe;
writeVideo(writerObj,frame);
end
catch warn
fprintf('%s', warn.message);
end
end
end
if(record)
close(writerObj);
end
close all;
experiments.local_params = all_local_params;
experiments.global_params = all_global_params;
end

28
pkg/OpenFace/matlab_version/demo/setup_mconvnet.m Normal file
View File

@@ -0,0 +1,28 @@
function setup(varargin)
try
run D:\soft\matconvnet-master\matconvnet-master\matlab/vl_setupnn ;
addpath D:\soft\matconvnet-master\matconvnet-master\examples ;
opts.useGpu = false ;
opts.verbose = false ;
opts = vl_argparse(opts, varargin) ;
try
vl_nnconv(single(1),single(1),[]) ;
catch
warning('VL_NNCONV() does not seem to be compiled. Trying to compile it now.') ;
vl_compilenn('enableGpu', opts.useGpu, 'verbose', opts.verbose) ;
end
if opts.useGpu
try
vl_nnconv(gpuArray(single(1)),gpuArray(single(1)),[]) ;
catch
vl_compilenn('enableGpu', opts.useGpu, 'verbose', opts.verbose) ;
warning('GPU support does not seem to be compiled in MatConvNet. Trying to compile it now') ;
end
end
catch
fprintf('Could not setup MatConvNet, face detection will be slower, install the library and set the right location for it in setup_mconvnet.m\n');
end

BIN
pkg/OpenFace/matlab_version/demo/tracked_vids/2015-10-15-15-14.avi Normal file
View File

Binary file not shown.