bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
pkg/OpenFace/matlab_version/fitting/CalcJacobian.m Normal file
View File

@@ -0,0 +1,42 @@
% This calculates the combined rigid with non-rigid Jacobian (non-rigid can
% eiher be expression or identity one)
function J = CalcJacobian(M, V, p_local, p_global)
n = size(M, 1)/3;
non_rigid_modes = size(V,2);
J = zeros(n*2, 6 + non_rigid_modes);
% now the layour is
% ---------- Rigid part -------------------|----Non rigid part--------|
% dx_1/ds, dx_1/dr1, ... dx_1/dtx, dx_1/dty dx_1/dp_1 ... dx_1/dp_m
% dx_2/ds, dx_2/dr1, ... dx_2/dtx, dx_2/dty dx_2/dp_1 ... dx_2/dp_m
% ...
% dx_n/ds, dx_n/dr1, ... dx_n/dtx, dx_n/dty dx_n/dp_1 ... dx_n/dp_m
% dy_1/ds, dy_1/dr1, ... dy_1/dtx, dy_1/dty dy_1/dp_1 ... dy_1/dp_m
% ...
% dy_n/ds, dy_n/dr1, ... dy_n/dtx, dy_n/dty dy_n/dp_1 ... dy_n/dp_m
% getting the rigid part
J(:,1:6) = CalcRigidJacobian(M, V, p_local, p_global);
% constructing the non-rigid part
R = Euler2Rot(p_global(2:4));
s = p_global(1);
% 'rotate' and 'scale' the principal components
% First reshape to 3D
V_X = V(1:n,:);
V_Y = V(n+1:2*n,:);
V_Z = V(2*n+1:end,:);
J_x_non_rigid = s*(R(1,1)*V_X + R(1,2)*V_Y + R(1,3)*V_Z);
J_y_non_rigid = s*(R(2,1)*V_X + R(2,2)*V_Y + R(2,3)*V_Z);
J(1:n, 7:end) = J_x_non_rigid;
J(n+1:end, 7:end) = J_y_non_rigid;
end

76
pkg/OpenFace/matlab_version/fitting/CalcRigidJacobian.m Normal file
View File

@@ -0,0 +1,76 @@
function J = CalcRigidJacobian(M, V, p_local, p_global)
n = size(M, 1)/3;
% Get the current 3D shape (not affected by global transform, as this
% is how the Jacobian was derived (for derivation please see
% ../derivations/orthoJacobian
shape3D = GetShape3D(M, V, p_local);
% Get the rotation matrix corresponding to current global orientation
R = Euler2Rot(p_global(2:4));
s = p_global(1);
% Rigid Jacobian is laid out as follows
% dx_1/ds, dx_1/dr1, dx_1/dr2, dx_1/dr3, dx_1/dtx, dx_1/dty
% dx_2/ds, dx_2/dr1, dx_2/dr2, dx_2/dr3, dx_2/dtx, dx_2/dty
% ...
% dx_n/ds, dx_n/dr1, dx_n/dr2, dx_n/dr3, dx_n/dtx, dx_n/dty
% dy_1/ds, dy_1/dr1, dy_1/dr2, dy_1/dr3, dy_1/dtx, dy_1/dty
% ...
% dy_n/ds, dy_n/dr1, dy_n/dr2, dy_n/dr3, dy_n/dtx, dy_n/dty
J = zeros(n*2, 6);
% dx/ds = X * r11 + Y * r12 + Z * r13
% dx/dr1 = s*(r13 * Y - r12 * Z)
% dx/dr2 = -s*(r13 * X - r11 * Z)
% dx/dr3 = s*(r12 * X - r11 * Y)
% dx/dtx = 1
% dx/dty = 0
% dy/ds = X * r21 + Y * r22 + Z * r23
% dy/dr1 = s * (r23 * Y - r22 * Z)
% dy/dr2 = -s * (r23 * X - r21 * Z)
% dy/dr3 = s * (r22 * X - r21 * Y)
% dy/dtx = 0
% dy/dty = 1
% set the Jacobian for x's
% with respect to scaling factor
J(1:n,1) = shape3D * R(1,:)';
% with respect to angular rotation around x, y, and z axes
% Change of x with respect to change in axis angle rotation
dxdR = [ 0, R(1,3), -R(1,2);
-R(1,3), 0, R(1,1);
R(1,2), -R(1,1), 0];
J(1:n,2:4) = s*(dxdR * shape3D')';
% with respect to translation
J(1:n,5) = 1;
J(1:n,6) = 0;
% set the Jacobian for y's
% with respect to scaling factor
J(n+1:end,1) = shape3D * R(2,:)';
% with respect to angular rotation around x, y, and z axes
% Change of y with respect to change in axis angle rotation
dydR = [ 0, R(2,3), -R(2,2);
-R(2,3), 0, R(2,1);
R(2,2), -R(2,1), 0];
J(n+1:end,2:4) = s*(dydR * shape3D')';
% with respect to translation
J(n+1:end,5) = 0;
J(n+1:end,6) = 1;
end

46
pkg/OpenFace/matlab_version/fitting/DrawFaceOnFig.m Normal file
View File

@@ -0,0 +1,46 @@
function DrawFaceOnFig( img, shape, orig_bbox, visibilities)
%DRAWFACEONIMG Summary of this function goes here
% Detailed explanation goes here
if(nargin > 2)
[height_img, width_img,~] = size(img);
width = orig_bbox(3) - orig_bbox(1);
height = orig_bbox(4) - orig_bbox(2);
img_min_x = max(int32(orig_bbox(1) - width/3),1);
img_max_x = min(int32(orig_bbox(3) + width/3),width_img);
img_min_y = max(int32(orig_bbox(2) - height/3),1);
img_max_y = min(int32(orig_bbox(4) + height/3),height_img);
shape(:,1) = shape(:,1) - double(img_min_x);
shape(:,2) = shape(:,2) - double(img_min_y);
img = img(img_min_y:img_max_y, img_min_x:img_max_x, :);
end
% Downsize the image if it is very big
if(size(img,1) > 500 || size(img,2) > 500)
scale = min(500 / size(img,1), 500 / size(img,2));
img = imresize(img, scale);
shape = scale * shape;
end
if(nargin > 3)
% valid points to draw (not to draw
% occluded ones)
v_points = visibilities;
else
v_points = true(size(shape,1),1);
end
if(max(img(:)) > 1)
imshow(double(img)/255, 'Border', 'tight');
else
imshow(double(img), 'Border', 'tight');
end
hold on;
plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20);
plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10);
drawnow expose;
hold off;
end

48
pkg/OpenFace/matlab_version/fitting/DrawFaceOnImg.m Normal file
View File

@@ -0,0 +1,48 @@
function DrawFaceOnImg( img, shape, out_loc, orig_bbox, visibilities)
%DRAWFACEONIMG Summary of this function goes here
% Detailed explanation goes here
f = figure('visible','off','Position', [100, 100, 600, 600]);
if(nargin > 3)
[height_img, width_img,~] = size(img);
width = orig_bbox(3) - orig_bbox(1);
height = orig_bbox(4) - orig_bbox(2);
img_min_x = max(int32(orig_bbox(1) - width/3),1);
img_max_x = min(int32(orig_bbox(3) + width/3),width_img);
img_min_y = max(int32(orig_bbox(2) - height/3),1);
img_max_y = min(int32(orig_bbox(4) + height/3),height_img);
shape(:,1) = shape(:,1) - double(img_min_x);
shape(:,2) = shape(:,2) - double(img_min_y);
img = img(img_min_y:img_max_y, img_min_x:img_max_x, :);
end
% Downsize the image if it is very big
if(size(img,1) > 500 || size(img,2) > 500)
scale = min(500 / size(img,1), 500 / size(img,2));
img = imresize(img, scale);
shape = scale * shape;
end
if(nargin > 4)
% valid points to draw (not to draw
% occluded ones)
v_points = visibilities;
else
v_points = true(size(shape,1),1);
end
if(max(img(:)) > 1)
imshow(double(img)/255, 'Border', 'tight');
else
imshow(double(img), 'Border', 'tight');
end
hold on;
plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20);
plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10);
print(f, '-djpeg', out_loc);
close(f);
end

272
pkg/OpenFace/matlab_version/fitting/Fitting_from_bb.m Normal file
View File

@@ -0,0 +1,272 @@
function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, view_used ] = Fitting_from_bb( Image, DepthImage, bounding_box, PDM, patchExperts, clmParams, varargin)
%FITTING Summary of this function goes here
% Detailed explanation goes here
% the bounding box format is [minX, minY, maxX, maxY];
% the mean model shape
M = PDM.M;
num_points = numel(M) / 3;
if(any(strcmp(varargin,'orientation')))
orientation = varargin{find(strcmp(varargin, 'orientation'))+1};
rot = Euler2Rot(orientation);
else
rot = eye(3);
orientation = [0;0;0];
end
rot_m = rot * reshape(M, num_points, 3)';
width_model = max(rot_m(1,:)) - min(rot_m(1,:));
height_model = max(rot_m(2,:)) - min(rot_m(2,:));
a = (((bounding_box(3) - bounding_box(1)) / width_model) + ((bounding_box(4) - bounding_box(2))/ height_model)) / 2;
tx = (bounding_box(3) + bounding_box(1))/2;
ty = (bounding_box(4) + bounding_box(2))/2;
% correct it so that the bounding box is just around the minimum
% and maximum point in the initialised face
tx = tx - a*(min(rot_m(1,:)) + max(rot_m(1,:)))/2;
ty = ty - a*(min(rot_m(2,:)) + max(rot_m(2,:)))/2;
% visualisation of the initial state
%hold off;imshow(Image);hold on;plot(a*rot_m(1,:)+tx, a*rot_m(2,:)+ty,'.r');hold on;rectangle('Position', [bounding_box(1), bounding_box(2), bounding_box(3)-bounding_box(1), bounding_box(4)-bounding_box(2)]);
global_params = [a, 0, 0, 0, tx, ty]';
global_params(2:4) = orientation;
local_params = zeros(numel(PDM.E), 1);
if(any(strcmp(varargin,'gparam')))
global_params = varargin{find(strcmp(varargin, 'gparam'))+1};
end
if(any(strcmp(varargin,'lparam')))
local_params = varargin{find(strcmp(varargin, 'lparam'))+1};
end
scale = clmParams.startScale;
if(size(Image, 3) == 1)
GrayImage = Image;
else
GrayImage = rgb2gray(Image);
end
[heightImg, widthImg] = size(GrayImage);
% Some predefinitions for faster patch extraction
[xi, yi] = meshgrid(0:widthImg-1,0:heightImg-1);
xi = double(xi);
yi = double(yi);
GrayImageDb = double(GrayImage);
clmParams_old = clmParams;
% In case there are no iterations initialize thes to empty
view = GetView(patchExperts(scale).centers, global_params(2:4));
final_lhood = 0;
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
landmark_lhoods = zeros(size(shape2D,1),1);
% multi iteration refinement using NU-RLMS in each one
for i=1:clmParams.numPatchIters
current_patch_scaling = patchExperts(scale).trainingScale;
visibilities = patchExperts(scale).visibilities;
view = GetView(patchExperts(scale).centers, global_params(2:4));
% The shape fitting is performed in the reference frame of the
% patch training scale
refGlobal = [current_patch_scaling, 0, 0, 0, 0, 0]';
% the reference shape
refShape = GetShapeOrtho(M, PDM.V, local_params, refGlobal);
% shape around which the patch experts will be evaluated in the original image
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
shape2D_img = shape2D(:,1:2);
% Create transform using a slightly modified version of Kabsch that
% takes scaling into account as well, in essence we get a
% similarity transform from current estimate to reference shape
[A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(shape2D_img(:,1:2),refShape(:,1:2));
% Create a transform, from shape in image to reference shape
T = maketform('affine', [A_img2ref;T_img2ref]);
shape_2D_ref = tformfwd(T, shape2D_img);
% transform the current shape to the reference one, so we can
% interpolate
shape2D_in_ref = (A_img2ref * shape2D_img')';
sideSizeX = (clmParams.window_size(i,1) - 1)/2;
sideSizeY = (clmParams.window_size(i,2) - 1)/2;
patches = zeros(size(shape2D_in_ref,1), clmParams.window_size(i,1) * clmParams.window_size(i,2));
Ainv = inv(A_img2ref);
% extract patches on which patch experts will be evaluted
for l=1:size(shape2D_in_ref,1)
if(visibilities(view,l))
xs = (shape2D_in_ref(l,1)-sideSizeX):(shape2D_in_ref(l,1)+sideSizeX);
ys = (shape2D_in_ref(l,2)-sideSizeY):(shape2D_in_ref(l,2)+sideSizeY);
[xs, ys] = meshgrid(xs, ys);
pairs = [xs(:), ys(:)];
actualLocs = (Ainv * pairs')';
actualLocs(actualLocs(:,1) < 0,1) = 0;
actualLocs(actualLocs(:,2) < 0,2) = 0;
actualLocs(actualLocs(:,1) > widthImg - 1,1) = widthImg - 1;
actualLocs(actualLocs(:,2) > heightImg - 1,2) = heightImg - 1;
[t_patch] = interp2_mine(xi, yi, GrayImageDb, actualLocs(:,1), actualLocs(:,2), 'bilinear');
t_patch = reshape(t_patch, size(xs));
patches(l,:) = t_patch(:);
end
end
% Calculate patch responses, either SVR or CCNF
if(strcmp(patchExperts(scale).type, 'SVR'))
responses = PatchResponseSVM_multi_modal( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsCol, clmParams, clmParams.window_size(i,:));
elseif(strcmp(patchExperts(scale).type, 'CCNF'))
responses = PatchResponseCCNF( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
elseif(strcmp(patchExperts(scale).type, 'CEN'))
% responses_o = PatchResponseCEN( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
% A faster (and very slightly less accurate) version of patch
% response computation
responses = PatchResponseCEN_sparse( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), clmParams.window_size(i,:), patchExperts(scale).normalisationOptionsCol.patchSize);
end
% If a depth image is provided compute patch experts around it as
% well (unless it's the final iteration)
if(~isempty(DepthImage) && (i ~= clmParams.numPatchIters))
% Extracting the depth patches here
patches_depth = zeros(size(shape2D_in_ref,1), clmParams.window_size(i,1) * clmParams.window_size(i,2));
% extract patches on which patch experts will be evaluted
for l=1:size(shape2D_in_ref,1)
if(visibilities(view,l))
xs = (shape2D_in_ref(l,1)-sideSizeX):(shape2D_in_ref(l,1)+sideSizeX);
ys = (shape2D_in_ref(l,2)-sideSizeY):(shape2D_in_ref(l,2)+sideSizeY);
[xs, ys] = meshgrid(xs, ys);
pairs = [xs(:), ys(:)];
actualLocs = (Ainv * pairs')';
actualLocs(actualLocs(:,1) < 1,1) = 1;
actualLocs(actualLocs(:,2) < 1,2) = 1;
actualLocs(actualLocs(:,1) > widthImg,1) = widthImg;
actualLocs(actualLocs(:,2) > heightImg,2) = heightImg;
% use nearest neighbour interpolation as bilinear would
% produce artefacts in depth image (when missing data
% is there)
[t_patch] = interp2_mine(xi, yi, DepthImage, actualLocs(:,1), actualLocs(:,2), 'nearest');
t_patch = reshape(t_patch, size(xs));
patches_depth(l,:) = t_patch(:);
end
end
old_mm = clmParams.use_multi_modal;
clmParams.use_multi_modal = 0;
responses_depth = PatchResponseSVM_multi_modal( patches_depth, patchExperts(scale).patch_experts_depth(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsDepth, clmParams, clmParams.window_size(i,:));
clmParams.use_multi_modal = old_mm;
% Combining the patch responses from different channels here
for l=1:size(shape2D_in_ref,1)
responses{l} = responses{l} + responses_depth{l};
end
end
% the better the correlation in training the more reliable the feature
% the reliabilities are independent for every modality in SVR, so
% combine them (also correlation is inverse to variance)
if(strcmp(patchExperts(scale).type, 'SVR'))
if(clmParams.use_multi_modal)
reliabilities = patchExperts(scale).patch_experts{1,1}(end).correlations{1};
else
reliabilities = patchExperts(scale).patch_experts{1,1}(1).correlations{1};
end
else
% for CCNF the modalities work together
reliabilities = patchExperts(scale).correlations;
end
reliabilities = reliabilities(view,:);
% deal with the fact that params might be different for different
% scales
if(numel(clmParams_old.regFactor) > 1)
clmParams.regFactor = clmParams_old.regFactor(scale);
end
if(numel(clmParams_old.sigmaMeanShift) > 1)
clmParams.sigmaMeanShift = clmParams_old.sigmaMeanShift(scale);
end
if(numel(clmParams_old.tikhonov_factor) > 1)
clmParams.tikhonov_factor = clmParams_old.tikhonov_factor(scale);
end
% The actual NU-RLMS step
% first the rigid transform
[global_params, local_params] = NU_RLMS(global_params, local_params, PDM, responses, visibilities, view, reliabilities, shape2D_img, T, true, clmParams, []);
% second the combined transform
[global_params, local_params, final_lhood, landmark_lhoods] = ...
NU_RLMS(global_params, local_params, PDM, responses, visibilities, view, reliabilities, shape2D_img, T, false, clmParams, []);
% Clamp orientation and make sure it doesn't get out of hand
orientation = global_params(2:4);
orientation(orientation < -pi/2) = -pi/2;
orientation(orientation > pi/2) = pi/2;
global_params(2:4) = orientation;
% move up a scale if possible
if(clmParams.useMultiScale && scale ~= numel(patchExperts))
% only go up a scale if we don't need to upsample
if(0.9 * patchExperts(scale+1).trainingScale < global_params(1))
scale = scale + 1;
else
break;
end
end
end
% the view in last iteration
view_used = view;
% See how good the tracking was in the end
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
% Moving to matlab format
shape2D = shape2D(:,1:2) + 1;
end
function [id] = GetView(centers, rotation)
[~,id] = min(sum((centers * pi/180 - repmat(rotation', size(centers,1), 1)).^2,2));
end

28
pkg/OpenFace/matlab_version/fitting/Fitting_from_bb_hierarch.m Normal file
View File

@@ -0,0 +1,28 @@
function [ shape2D_full, shape_hierarch, global_params, local_params, final_lhood, landmark_lhoods, view_used] = Fitting_from_bb_hierarch( Image, pdm_full, pdm_hierarch, patches, clmParams, shape2D_full, inds_full, inds_hierarch)
% Perform hierarchical model fitting
%, TODO this should be a loop?
shape_hierarch = zeros(numel(pdm_hierarch.M)/3, 2);
shape_hierarch(inds_hierarch,:) = shape2D_full(inds_full,:);
% Get the hierarchical model parameters based on current parent
% landmark locations
[ a, R, T, ~, l_params, ~] = fit_PDM_ortho_proj_to_2D(pdm_hierarch.M, pdm_hierarch.E, pdm_hierarch.V, shape_hierarch);
g_param = [a; Rot2Euler(R)'; T];
bbox_hierarch = [min(shape_hierarch(:,1)), min(shape_hierarch(:,2)), max(shape_hierarch(:,1)), max(shape_hierarch(:,2))];
[shape_hierarch, global_params, local_params, final_lhood, landmark_lhoods, view_used] = Fitting_from_bb(Image, [], bbox_hierarch, pdm_hierarch, patches, clmParams, 'gparam', g_param, 'lparam', l_params);
% Now after detections incorporate the subset of the landmarks into the
% main model, but only if we didn't upsample
if(a > 0.9)
shape2D_full(inds_full, :) = shape_hierarch(inds_hierarch,:);
% Make sure the combined version can be expressed by the PDM
[ ~, ~, ~, ~, ~, ~, shape2D_full] = fit_PDM_ortho_proj_to_2D(pdm_full.M, pdm_full.E, pdm_full.V, shape2D_full);
end
end

96
pkg/OpenFace/matlab_version/fitting/Fitting_from_bb_multi_hyp.m Normal file
View File

@@ -0,0 +1,96 @@
function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, view_used, early_term ] = Fitting_from_bb_multi_hyp( Image, DepthImage, bounding_box, PDM, patches, clmParams, views, early_term_params)
num_points = numel(PDM.M)/3;
shapes = zeros(num_points, 2, size(views,1));
ls = zeros(size(views,1),1);
lmark_lhoods = zeros(num_points,size(views,1));
views_used = zeros(size(views,1),1);
globals = cell(size(views,1),1);
locals = cell(size(views,1),1);
% If early termination parameters are defined use clever multi-hypothesis setup
% otherwise evaluate all of them in a simplistic manner
if(nargin > 7)
patches_small = patches(1);
patches_large = patches(2:end);
clmParams_small = clmParams;
clmParams_small.numPatchIters = 1;
clmParams_large = clmParams;
clmParams_large.regFactor = clmParams_large.regFactor(2:end);
clmParams_large.sigmaMeanShift = clmParams_large.sigmaMeanShift(2:end);
clmParams_large.tikhonov_factor = clmParams_large.tikhonov_factor(2:end);
clmParams_large.numPatchIters = 3;
clmParams_large.window_size = clmParams_large.window_size(2:end,:);
early_term = 0;
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),globals{v},locals{v},ls(v),lmark_lhoods(:,v),views_used(v)]...
= Fitting_from_bb(Image, DepthImage, bounding_box, PDM, patches_small, clmParams_small, 'orientation', views(v,:));
ls(v) = ls(v) * early_term_params.weights_scale(views_used(v)) + early_term_params.weights_add(views_used(v));
if(ls(v) > early_term_params.cutoffs(views_used(v)))
early_term = v;
[shape,global_params,local_params,final_lhood,landmark_lhoods,view_used]...
= Fitting_from_bb(Image, DepthImage, bounding_box, PDM, patches_large, clmParams_large, 'gparam', globals{v}, 'lparam', locals{v});
break;
end
end
if(early_term == 0)
% Discard the really unreliable starts
[~, ids] = sort(ls);
to_refine = ids(end-3:end);
% If there are any starts left try refining them in a
% multiscale approach
if(~isempty(to_refine))
shapes = zeros(num_points, 2, numel(to_refine));
ls = zeros(numel(to_refine),1);
lmark_lhoods = zeros(num_points,numel(to_refine));
views_used = zeros(numel(to_refine),1);
globals_n = cell(size(to_refine,1),1);
locals_n = cell(size(to_refine,1),1);
ind = 1;
for v = to_refine'
[shapes(:,:,ind), globals_n{ind}, locals_n{ind},...
ls(ind),lmark_lhoods(:,ind),views_used(ind)] ...
= Fitting_from_bb(Image, DepthImage, bounding_box, PDM, patches_large, clmParams_large, 'gparam', globals{v}, 'lparam', locals{v});
ind = ind+1;
end
end
% TODO aggregation?
[final_lhood, v_ind] = max(ls);
landmark_lhoods = lmark_lhoods(:,v_ind);
global_params = globals_n{v_ind};
local_params = locals_n{v_ind};
shape = shapes(:,:,v_ind);
view_used = views_used(v_ind);
end
shape2D = shape;
else
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),globals{v},locals{v},ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(Image, [], bounding_box, PDM, patches, clmParams, 'orientation', views(v,:));
end
[final_lhood, v_ind] = max(ls);
landmark_lhoods = lmark_lhoods(:,v_ind);
global_params = globals{v_ind};
local_params = locals{v_ind};
shape2D = shapes(:,:,v_ind);
view_used = views_used(v_ind);
end
end

273
pkg/OpenFace/matlab_version/fitting/Fitting_from_bb_precomp.m Normal file
View File

@@ -0,0 +1,273 @@
function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, view_used, precomp_resps ] = Fitting_from_bb_precomp( Image, DepthImage, bounding_box, PDM, patchExperts, clmParams, precomp_resps, varargin)
%FITTING Summary of this function goes here
% Detailed explanation goes here
% the bounding box format is [minX, minY, maxX, maxY];
% the mean model shape
M = PDM.M;
num_points = numel(M) / 3;
if(any(strcmp(varargin,'orientation')))
orientation = varargin{find(strcmp(varargin, 'orientation'))+1};
rot = Euler2Rot(orientation);
else
rot = eye(3);
orientation = [0;0;0];
end
rot_m = rot * reshape(M, num_points, 3)';
width_model = max(rot_m(1,:)) - min(rot_m(1,:));
height_model = max(rot_m(2,:)) - min(rot_m(2,:));
a = (((bounding_box(3) - bounding_box(1)) / width_model) + ((bounding_box(4) - bounding_box(2))/ height_model)) / 2;
tx = (bounding_box(3) + bounding_box(1))/2;
ty = (bounding_box(4) + bounding_box(2))/2;
% correct it so that the bounding box is just around the minimum
% and maximum point in the initialised face
tx = tx - a*(min(rot_m(1,:)) + max(rot_m(1,:)))/2;
ty = ty - a*(min(rot_m(2,:)) + max(rot_m(2,:)))/2;
% visualisation of the initial state
%hold off;imshow(Image);hold on;plot(a*rot_m(1,:)+tx, a*rot_m(2,:)+ty,'.r');hold on;rectangle('Position', [bounding_box(1), bounding_box(2), bounding_box(3)-bounding_box(1), bounding_box(4)-bounding_box(2)]);
global_params = [a, 0, 0, 0, tx, ty]';
global_params(2:4) = orientation;
local_params = zeros(numel(PDM.E), 1);
if(any(strcmp(varargin,'gparam')))
global_params = varargin{find(strcmp(varargin, 'gparam'))+1};
end
if(any(strcmp(varargin,'lparam')))
local_params = varargin{find(strcmp(varargin, 'lparam'))+1};
end
scale = clmParams.startScale;
if(isempty(precomp_resps))
if(size(Image, 3) == 1)
GrayImage = Image;
else
GrayImage = rgb2gray(Image);
end
[heightImg, widthImg] = size(GrayImage);
% Some predefinitions for faster patch extraction
[xi, yi] = meshgrid(0:widthImg-1,0:heightImg-1);
xi = double(xi);
yi = double(yi);
GrayImageDb = double(GrayImage);
end
clmParams_old = clmParams;
% In case there are no iterations initialize thes to empty
view = GetView(patchExperts(scale).centers, global_params(2:4));
final_lhood = 0;
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
landmark_lhoods = zeros(size(shape2D,1),1);
% multi iteration refinement using NU-RLMS in each one
for i=1:clmParams.numPatchIters
current_patch_scaling = patchExperts(scale).trainingScale;
visibilities = patchExperts(scale).visibilities;
view = GetView(patchExperts(scale).centers, global_params(2:4));
% The shape fitting is performed in the reference frame of the
% patch training scale
refGlobal = [current_patch_scaling, 0, 0, 0, 0, 0]';
% the reference shape
refShape = GetShapeOrtho(M, PDM.V, local_params, refGlobal);
% shape around which the patch experts will be evaluated in the original image
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
shape2D_img = shape2D(:,1:2);
% Create transform using a slightly modified version of Kabsch that
% takes scaling into account as well, in essence we get a
% similarity transform from current estimate to reference shape
[A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(shape2D_img(:,1:2),refShape(:,1:2));
% Create a transform, from shape in image to reference shape
T = maketform('affine', [A_img2ref;T_img2ref]);
shape_2D_ref = tformfwd(T, shape2D_img);
% transform the current shape to the reference one, so we can
% interpolate
shape2D_in_ref = (A_img2ref * shape2D_img')';
sideSizeX = (clmParams.window_size(i,1) - 1)/2;
sideSizeY = (clmParams.window_size(i,2) - 1)/2;
if(isempty(precomp_resps))
patches = zeros(size(shape2D_in_ref,1), clmParams.window_size(i,1) * clmParams.window_size(i,2));
Ainv = inv(A_img2ref);
% extract patches on which patch experts will be evaluted
for l=1:size(shape2D_in_ref,1)
if(visibilities(view,l))
xs = (shape2D_in_ref(l,1)-sideSizeX):(shape2D_in_ref(l,1)+sideSizeX);
ys = (shape2D_in_ref(l,2)-sideSizeY):(shape2D_in_ref(l,2)+sideSizeY);
[xs, ys] = meshgrid(xs, ys);
pairs = [xs(:), ys(:)];
actualLocs = (Ainv * pairs')';
actualLocs(actualLocs(:,1) < 0,1) = 0;
actualLocs(actualLocs(:,2) < 0,2) = 0;
actualLocs(actualLocs(:,1) > widthImg - 1,1) = widthImg - 1;
actualLocs(actualLocs(:,2) > heightImg - 1,2) = heightImg - 1;
[t_patch] = interp2_mine(xi, yi, GrayImageDb, actualLocs(:,1), actualLocs(:,2), 'bilinear');
t_patch = reshape(t_patch, size(xs));
patches(l,:) = t_patch(:);
end
end
% Calculate patch responses, either SVR or CCNF
if(strcmp(patchExperts(scale).type, 'SVR'))
responses = PatchResponseSVM_multi_modal( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsCol, clmParams, clmParams.window_size(i,:));
elseif(strcmp(patchExperts(scale).type, 'CCNF'))
responses = PatchResponseCCNF( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
elseif(strcmp(patchExperts(scale).type, 'CEN'))
responses = PatchResponseCEN_sparse( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), clmParams.window_size(i,:), patchExperts(scale).normalisationOptionsCol.patchSize);
end
precomp_resps = responses;
else
responses = precomp_resps;
end
% If a depth image is provided compute patch experts around it as
% well (unless it's the final iteration)
if(~isempty(DepthImage) && (i ~= clmParams.numPatchIters))
% Extracting the depth patches here
patches_depth = zeros(size(shape2D_in_ref,1), clmParams.window_size(i,1) * clmParams.window_size(i,2));
% extract patches on which patch experts will be evaluted
for l=1:size(shape2D_in_ref,1)
if(visibilities(view,l))
xs = (shape2D_in_ref(l,1)-sideSizeX):(shape2D_in_ref(l,1)+sideSizeX);
ys = (shape2D_in_ref(l,2)-sideSizeY):(shape2D_in_ref(l,2)+sideSizeY);
[xs, ys] = meshgrid(xs, ys);
pairs = [xs(:), ys(:)];
actualLocs = (Ainv * pairs')';
actualLocs(actualLocs(:,1) < 1,1) = 1;
actualLocs(actualLocs(:,2) < 1,2) = 1;
actualLocs(actualLocs(:,1) > widthImg,1) = widthImg;
actualLocs(actualLocs(:,2) > heightImg,2) = heightImg;
% use nearest neighbour interpolation as bilinear would
% produce artefacts in depth image (when missing data
% is there)
[t_patch] = interp2_mine(xi, yi, DepthImage, actualLocs(:,1), actualLocs(:,2), 'nearest');
t_patch = reshape(t_patch, size(xs));
patches_depth(l,:) = t_patch(:);
end
end
old_mm = clmParams.use_multi_modal;
clmParams.use_multi_modal = 0;
responses_depth = PatchResponseSVM_multi_modal( patches_depth, patchExperts(scale).patch_experts_depth(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsDepth, clmParams, clmParams.window_size(i,:));
clmParams.use_multi_modal = old_mm;
% Combining the patch responses from different channels here
for l=1:size(shape2D_in_ref,1)
responses{l} = responses{l} + responses_depth{l};
end
end
% the better the correlation in training the more reliable the feature
% the reliabilities are independent for every modality in SVR, so
% combine them (also correlation is inverse to variance)
if(strcmp(patchExperts(scale).type, 'SVR'))
if(clmParams.use_multi_modal)
reliabilities = patchExperts(scale).patch_experts{1,1}(end).correlations{1};
else
reliabilities = patchExperts(scale).patch_experts{1,1}(1).correlations{1};
end
else
% for CCNF the modalities work together
reliabilities = patchExperts(scale).correlations;
end
reliabilities = reliabilities(view,:);
% deal with the fact that params might be different for different
% scales
if(numel(clmParams_old.regFactor) > 1)
clmParams.regFactor = clmParams_old.regFactor(scale);
end
if(numel(clmParams_old.sigmaMeanShift) > 1)
clmParams.sigmaMeanShift = clmParams_old.sigmaMeanShift(scale);
end
if(numel(clmParams_old.tikhonov_factor) > 1)
clmParams.tikhonov_factor = clmParams_old.tikhonov_factor(scale);
end
% The actual NU-RLMS step
% first the rigid transform
[global_params, local_params] = NU_RLMS(global_params, local_params, PDM, responses, visibilities, view, reliabilities, shape2D_img, T, true, clmParams, []);
% second the combined transform
[global_params, local_params, final_lhood, landmark_lhoods] = ...
NU_RLMS(global_params, local_params, PDM, responses, visibilities, view, reliabilities, shape2D_img, T, false, clmParams, []);
% Clamp orientation and make sure it doesn't get out of hand
orientation = global_params(2:4);
orientation(orientation < -pi/2) = -pi/2;
orientation(orientation > pi/2) = pi/2;
global_params(2:4) = orientation;
% move up a scale if possible
if(clmParams.useMultiScale && scale ~= numel(patchExperts))
% only go up a scale if we don't need to upsample
if(0.9 * patchExperts(scale+1).trainingScale < global_params(1))
scale = scale + 1;
else
break;
end
end
end
% the view in last iteration
view_used = view;
% See how good the tracking was in the end
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
% Moving to matlab format
shape2D = shape2D(:,1:2) + 1;
end
function [id] = GetView(centers, rotation)
[~,id] = min(sum((centers * pi/180 - repmat(rotation', size(centers,1), 1)).^2,2));
end

163
pkg/OpenFace/matlab_version/fitting/Fitting_from_bb_vis.m Normal file
View File

@@ -0,0 +1,163 @@
function Fitting_from_bb_vis( Image, DepthImage, bounding_box, PDM, patchExperts, clmParams, out_dir, varargin)
%FITTING Summary of this function goes here
% Detailed explanation goes here
% the bounding box format is [minX, minY, maxX, maxY];
% the mean model shape
M = PDM.M;
num_points = numel(M) / 3;
if(any(strcmp(varargin,'orientation')))
orientation = varargin{find(strcmp(varargin, 'orientation'))+1};
rot = Euler2Rot(orientation);
else
rot = eye(3);
orientation = [0;0;0];
end
rot_m = rot * reshape(M, num_points, 3)';
width_model = max(rot_m(1,:)) - min(rot_m(1,:));
height_model = max(rot_m(2,:)) - min(rot_m(2,:));
a = (((bounding_box(3) - bounding_box(1)) / width_model) + ((bounding_box(4) - bounding_box(2))/ height_model)) / 2;
tx = (bounding_box(3) + bounding_box(1))/2;
ty = (bounding_box(4) + bounding_box(2))/2;
% correct it so that the bounding box is just around the minimum
% and maximum point in the initialised face
tx = tx - a*(min(rot_m(1,:)) + max(rot_m(1,:)))/2;
ty = ty - a*(min(rot_m(2,:)) + max(rot_m(2,:)))/2;
% visualisation of the initial state
%hold off;imshow(Image);hold on;plot(a*rot_m(1,:)+tx, a*rot_m(2,:)+ty,'.r');hold on;rectangle('Position', [bounding_box(1), bounding_box(2), bounding_box(3)-bounding_box(1), bounding_box(4)-bounding_box(2)]);
global_params = [a, 0, 0, 0, tx, ty]';
global_params(2:4) = orientation;
local_params = zeros(numel(PDM.E), 1);
if(any(strcmp(varargin,'gparam')))
global_params = varargin{find(strcmp(varargin, 'gparam'))+1};
end
if(any(strcmp(varargin,'lparam')))
local_params = varargin{find(strcmp(varargin, 'lparam'))+1};
end
scale = clmParams.startScale;
if(size(Image, 3) == 1)
GrayImage = Image;
else
GrayImage = rgb2gray(Image);
end
[heightImg, widthImg] = size(GrayImage);
% Some predefinitions for faster patch extraction
[xi, yi] = meshgrid(0:widthImg-1,0:heightImg-1);
xi = double(xi);
yi = double(yi);
GrayImageDb = double(GrayImage);
% multi iteration refinement using NU-RLMS in each one
i=1;
current_patch_scaling = patchExperts(scale).trainingScale;
visibilities = patchExperts(scale).visibilities;
view = GetView(patchExperts(scale).centers, global_params(2:4));
% The shape fitting is performed in the reference frame of the
% patch training scale
refGlobal = [current_patch_scaling, 0, 0, 0, 0, 0]';
% the reference shape
refShape = GetShapeOrtho(M, PDM.V, local_params, refGlobal);
% shape around which the patch experts will be evaluated in the original image
[shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params);
shape2D_img = shape2D(:,1:2);
% Create transform using a slightly modified version of Kabsch that
% takes scaling into account as well, in essence we get a
% similarity transform from current estimate to reference shape
[A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(shape2D_img(:,1:2),refShape(:,1:2));
% Create a transform, from shape in image to reference shape
T = maketform('affine', [A_img2ref;T_img2ref]);
shape_2D_ref = tformfwd(T, shape2D_img);
% transform the current shape to the reference one, so we can
% interpolate
shape2D_in_ref = (A_img2ref * shape2D_img')';
sideSizeX = (clmParams.window_size(i,1) - 1)/2;
sideSizeY = (clmParams.window_size(i,2) - 1)/2;
patches = zeros(size(shape2D_in_ref,1), clmParams.window_size(i,1) * clmParams.window_size(i,2));
Ainv = inv(A_img2ref);
% extract patches on which patch experts will be evaluted
for l=1:size(shape2D_in_ref,1)
if(visibilities(view,l))
xs = (shape2D_in_ref(l,1)-sideSizeX):(shape2D_in_ref(l,1)+sideSizeX);
ys = (shape2D_in_ref(l,2)-sideSizeY):(shape2D_in_ref(l,2)+sideSizeY);
[xs, ys] = meshgrid(xs, ys);
pairs = [xs(:), ys(:)];
actualLocs = (Ainv * pairs')';
actualLocs(actualLocs(:,1) < 0,1) = 0;
actualLocs(actualLocs(:,2) < 0,2) = 0;
actualLocs(actualLocs(:,1) > widthImg - 1,1) = widthImg - 1;
actualLocs(actualLocs(:,2) > heightImg - 1,2) = heightImg - 1;
[t_patch] = interp2_mine(xi, yi, GrayImageDb, actualLocs(:,1), actualLocs(:,2), 'bilinear');
t_patch = reshape(t_patch, size(xs));
patches(l,:) = t_patch(:);
end
end
% Calculate patch responses, either SVR or CCNF
if(strcmp(patchExperts(scale).type, 'SVR'))
responses = PatchResponseSVM_multi_modal( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsCol, clmParams, clmParams.window_size(i,:));
for r=1:numel(responses)
out_patch = reshape(patches(r,:)/255, size(xs));
imwrite(out_patch, [out_dir, '/', num2str(r), '_a.png']);
imwrite(responses{r}/max(responses{r}(:)), [out_dir, '/', num2str(r), '_svr.png']);
end
elseif(strcmp(patchExperts(scale).type, 'CCNF'))
responses = PatchResponseCCNF( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
for r=1:numel(responses)
imwrite(responses{r}/max(responses{r}(:)), [out_dir, '/', num2str(r), '_lnf.png']);
end
elseif(strcmp(patchExperts(scale).type, 'DNN'))
responses = PatchResponseDNN( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
for r=1:numel(responses)
imwrite(responses{r}/max(responses{r}(:)), [out_dir, '/', num2str(r), '_dnn.png']);
end
end
end
function [id] = GetView(centers, rotation)
[~,id] = min(sum((centers * pi/180 - repmat(rotation', size(centers,1), 1)).^2,2));
end

314
pkg/OpenFace/matlab_version/fitting/NU_RLMS.m Normal file
View File

@@ -0,0 +1,314 @@
function [ final_global, final_local, final_lhood, landmark_lhoods ] = NU_RLMS( ...
init_global, init_local, PDM, patchResponses, visibilities,...
view, reliabilities, baseShape, OrigToRefTransform, rigid, ...
clmParams, gauss_resp)
%RLMS Summary of this function goes here
% Detailed explanation goes here
m = numel(PDM.E);
E = PDM.E;
V = PDM.V;
n = size(V, 1)/3;
current_local = init_local;
current_global = init_global;
pxWidth = size(patchResponses{1},1);
responseSize = size(patchResponses{1});
[iis,jjs] = meshgrid(1:pxWidth, 1:pxWidth);
% Grab all of the patch responses and convert them to single matrix
% representation for speed
patchResponsesFlat = reshape(cat(3,patchResponses{:}), responseSize(1)*responseSize(2), n)';
iisFlat = repmat(iis(:)', n, 1);
jjsFlat = repmat(jjs(:)', n, 1);
% An alternative formulation
reg_rigid = zeros(6,1);
if(rigid)
regularisations = reg_rigid;
regularisations = diag(regularisations);
else
regularisations = [reg_rigid; clmParams.regFactor ./ E]; % the above version, however, does not perform as well
regularisations = diag(regularisations);
end
% For generalised Tikhonov
if(clmParams.tikhonov_factor == 0)
P = eye(n*2);
else
% the worse the reliability the higher the variance of the
% prediction, so inverse variance correlate with reliability
P = clmParams.tikhonov_factor * diag(repmat(reliabilities',2,1));
end
for iter = 1:clmParams.num_RLMS_iter
% get the current estimates of x and y in image
currentShape = GetShapeOrtho(PDM.M, PDM.V, current_local, current_global);
currentShape = currentShape(:,1:2);
if(iter > 1)
% if the shape hasn't changed terminate
if(norm(currentShape - previousShape) < clmParams.fTol)
break;
end
end
previousShape = currentShape;
% calculate the appropriate Jacobians in 2D, even though the actual behaviour is in 3D, using small angle approximation and oriented shape
if(rigid)
J = CalcRigidJacobian(PDM.M, PDM.V, current_local, current_global);
else
J = CalcJacobian(PDM.M, PDM.V, current_local, current_global);
end
% as the mean shift is with reference to the point, we don't care about
% the translation
OrigToRefTransform.tdata.T(3,1:2) = 0;
OrigToRefTransform.tdata.Tinv(3,1:2) = 0;
% distance from center where the response was calculated around
% in reference frame of the patch
offsets = (currentShape - baseShape) * OrigToRefTransform.tdata.T(1:2,1:2)';
% perform the parallel version of the mean shift algorithm
dxs = offsets(:, 1) + (pxWidth-1)/2 + 1;
dys = offsets(:, 2) + (pxWidth-1)/2 + 1;
if(numel(gauss_resp) > 0)
meanShifts = meanShiftParallel_precalc(patchResponsesFlat, dxs, dys, iisFlat, jjsFlat, responseSize, gauss_resp.kd_precalc, gauss_resp.stepSize);
else
meanShifts = meanShiftParallel(patchResponsesFlat, clmParams.sigmaMeanShift, dxs, dys, iisFlat, jjsFlat, responseSize);
end
% invalidate illegal mean shifts
illegal_inds = find(~visibilities(view, :));
J(illegal_inds,:) = 0;
J(illegal_inds + n,:) = 0;
meanShifts(illegal_inds,:) = 0;
% Mean shift's here are calculate in the reference image frame,
% we want to move them back to actual image frame
meanShifts = meanShifts * OrigToRefTransform.tdata.Tinv(1:2,1:2)';
% put it into column format
meanShifts = meanShifts(:);
rigid_params = current_global - init_global;
rigid_params(2:4) = 0;
if(rigid)
params = rigid_params;
else
params = [rigid_params; current_local];
end
params_delta = (J'*P*J + regularisations) \ (J'*P*meanShifts - regularisations*params);
% update the reference
[current_local, current_global] = CalcReferenceUpdate(params_delta, current_local, current_global);
if(~rigid)
% clamp to the local parameters for valid expressions
current_local = ClampPDM(current_local, E);
end
end
if(nargout >= 4)
% get the current estimates of x and y in image
currentShape = GetShapeOrtho(PDM.M, PDM.V, current_local, current_global);
currentShape = currentShape(:,1:2);
% as the mean shift is with reference to the point, we don't care about
% the translation
OrigToRefTransform.tdata.T(3,1:2) = 0;
OrigToRefTransform.tdata.Tinv(3,1:2) = 0;
% distance from center where the response was calculated around
% in reference frame of the patch
offsets = (currentShape - baseShape) * OrigToRefTransform.tdata.T(1:2,1:2)';
% perform the parallel version of the mean shift algorithm
dxs = offsets(:, 1) + (pxWidth-1)/2 + 1;
dys = offsets(:, 2) + (pxWidth-1)/2 + 1;
landmark_lhoods = zeros(n,1);
prob = 0;
for i=1:n
if(visibilities(view, i))
dx = dxs(i);
dy = dys(i);
vxs = (-iis+dx).^2;
vys = (-jjs+dy).^2;
% Calculate the kde per patch
vs = patchResponses{i}.*exp(-0.5*(vxs + vys)/(clmParams.sigmaMeanShift*clmParams.sigmaMeanShift));
kde_est = sum(vs(:));
landmark_lhoods(i) = kde_est;
prob = prob + log(kde_est + 1e-8);
end
end
% Do not add the local parameter prior as it overpowers the
% log-likelihoods
final_lhood = prob / sum(visibilities(view, :));
% - LogPDMprior(current_local, E);
end
final_global = current_global;
final_local = current_local;
end
% This clamps the non-rigid parameters to stay within +- 3 standard
% deviations
function [non_rigid_params] = ClampPDM(non_rigid, E)
stds = sqrt(E);
non_rigid_params = non_rigid;
lower = non_rigid_params < -3 * stds;
non_rigid_params(lower) = -3*stds(lower);
higher = non_rigid_params > 3 * stds;
non_rigid_params(higher) = 3*stds(higher);
end
% This calculate the mean shift based on the kernel density response at dx,
% dy in the patch response, this can be used to find the mode
function [meanShifts] = meanShiftParallel(patchResponses, sigma, dxs, dys, iis, jjs, patchSize)
% Kernel density is
% K(x_i-x) = p(x_i)*exp(-0.5 * ||x_i-x||^2/sigma), so probability weighted
% distance from the center
% Mean shift is then m(x) = sum(K(x_i - dx)*x_i)/sum(K(x_i-dx))
% step_size = 0.1;
% gauss_resp_prec = precalc_kernel_densities(patchSize, sigma, step_size);
%
% %iis are row vectors of the locations of interest, for each patch
% nYs = numel(0:step_size:patchSize(2));
% % calculate the indices needed
%
% dxs2 = dxs - mod(dxs, step_size);
% dys2 = dys - mod(dys, step_size);
%
% xs = round(dxs2 * nYs * 1/step_size + (dys2 /step_size) +1);
%
% gauss_resp_c = gauss_resp_prec(xs,:);
% this part is doing (x_i - dx)^2
vxs = bsxfun(@plus, -iis, dxs);
vxs = vxs.^2;
% this part is doing (y_i - dy)^2
vys = bsxfun(@plus, -jjs, dys);
vys = vys.^2;
a = -0.5/(sigma.^2);
% this part is calculating K(x_i - x)
gauss_resp = exp(a*(vxs + vys));
vs = patchResponses.*gauss_resp;
% this part is calculating K(x_i - dx)*x_i
mxss = vs.*iis;
myss = vs.*jjs;
% this part is caluclating sum(K(x_i - dx)*x_i)
mxs = sum(mxss,2);
mys = sum(myss,2);
sumVs = sum(vs,2);
sumVs(sumVs == 0) = 1;
msx = mxs ./ sumVs - dxs;
msy = mys ./ sumVs - dys;
meanShifts = [msx, msy];
end
% This updates the parameters based on the updates from the RLMS
function [non_rigid, rigid] = CalcReferenceUpdate(params_delta, current_non_rigid, current_global)
rigid = zeros(6, 1);
% Same goes for scaling and translation parameters
rigid(1) = current_global(1) + params_delta(1);
rigid(5) = current_global(5) + params_delta(5);
rigid(6) = current_global(6) + params_delta(6);
% for rotation however, we want to make sure that the rotation matrix
% approximation we have
% R' = [1, -wz, wy
% wz, 1, -wx
% -wy, wx, 1]
% is a legal rotation matrix, and then we combine it with current
% rotation (through matrix multiplication) to acquire the new rotation
R = Euler2Rot(current_global(2:4));
wx = params_delta(2);
wy = params_delta(3);
wz = params_delta(4);
R_delta = [1, -wz, wy;
wz, 1, -wx;
-wy, wx, 1];
% Make sure R_delta is orthonormal
R_delta = OrthonormaliseRotation(R_delta);
% Combine rotations
R_final = R * R_delta;
% Extract euler angle
euler = Rot2Euler(R_final);
rigid(2:4) = euler;
if(length(params_delta) > 6)
% non-rigid parameters can just be added together
non_rigid = params_delta(7:end) + current_non_rigid;
else
non_rigid = current_non_rigid;
end
end
function R_ortho = OrthonormaliseRotation(R)
% U * V' is basically what we want, as it's guaranteed to be
% orthonormal
[U, ~, V] = svd(R);
% We also want to make sure no reflection happened
% get the orthogonal matrix from the initial rotation matrix
X = U*V';
% This makes sure that the handedness is preserved and no reflection happened
% by making sure the determinant is 1 and not -1
W = eye(3);
W(3,3) = det(X);
R_ortho = U*W*V';
end
function [prob] = LogPDMprior(params, E)
cov = diag(E);
prob = 0.5 * params'*inv(cov)*params;
end

77
pkg/OpenFace/matlab_version/fitting/PatchResponseCCNF.m Normal file
View File

@@ -0,0 +1,77 @@
function [ responses ] = PatchResponseCCNF(patches, patch_experts_class, visibilities, patchExperts, window_size)
%PATCHRESPONSESVM Summary of this function goes here
% Detailed explanation goes here
normalisationOptions = patchExperts.normalisationOptionsCol;
patchSize = normalisationOptions.patchSize;
responses = cell(size(patches, 1), 1);
empty = zeros(window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
for i = 1:numel(patches(:,1))
responses{i} = empty;
if visibilities(i)
col_norm = normalisationOptions.useNormalisedCrossCorr == 1;
b = zeros(numel(empty), 1);
num_hl = size(patch_experts_class{i}.thetas,1);
smallRegionVec = patches(i,:);
smallRegion = reshape(smallRegionVec, window_size(1), window_size(2));
for hls = 1:num_hl
w = patch_experts_class{i}.w{hls};
% normalisation needed per each response
norm_w = patch_experts_class{i}.norm_w{hls};
response = -norm_w * SVMresponse(smallRegion, w, col_norm, patchSize) - patch_experts_class{i}.thetas(hls, 1);
% here we include the bias term as well, as it wasn't added
% during the response calculation
h1 = 1./(1 + exp(response(:)));
b = b + (2 * patch_experts_class{i}.alphas(hls) * h1);
end
% Sigma will be dependent on the size of the patch find the
% needed precomputed Sigma
rel_sigma = 1;
if(numel(patch_experts_class{i}.Sigma) > 1)
for sig=1:numel(patch_experts_class{i}.Sigma)
if(size(patch_experts_class{i}.Sigma{sig},2)==numel(b))
rel_sigma = sig;
break;
end
end
end
response = patch_experts_class{i}.Sigma{rel_sigma} * b;
% make sure we have no negative responses
response = response - min(response);
responses{i}(:) = response;
end
end
end
function response = SVMresponse(region, patchExpert, normalise_x_corr,patchSize)
if(normalise_x_corr)
% the fast mex convolution
[response] = normxcorr2_mex(patchExpert, region);
response = response(patchSize(1):end-patchSize(1)+1,patchSize(2):end-patchSize(2)+1);
else
% this assumes that the patch is already normed
template = rot90(patchExpert,2);
response = conv2(region, template, 'valid');
end
end

59
pkg/OpenFace/matlab_version/fitting/PatchResponseCEN.m Normal file
View File

@@ -0,0 +1,59 @@
function [ responses ] = PatchResponseCEN(patches, patch_experts_class, visibilities, patchExperts, window_size)
%PATCHRESPONSESVM Summary of this function goes here
% Detailed explanation goes here
normalisationOptions = patchExperts.normalisationOptionsCol;
patchSize = normalisationOptions.patchSize;
responses = cell(size(patches, 1), 1);
empty = zeros(window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
for i = 1:numel(patches(:,1))
responses{i} = empty;
if visibilities(i)
col_norm = normalisationOptions.useNormalisedCrossCorr == 1;
smallRegionVec = patches(i,:);
smallRegion = reshape(smallRegionVec, window_size(1), window_size(2));
patch = im2col_mine(smallRegion, patchSize)';
% Normalize
if(col_norm)
mean_curr = mean(patch, 2);
patch_normed = patch - repmat(mean_curr, 1, patchSize(1)* patchSize(2));
% Normalising the patches using the L2 norm
scaling = sqrt(sum(patch_normed.^2,2));
scaling(scaling == 0) = 1;
patch_normed = patch_normed ./ repmat(scaling, 1, 11 * 11);
patch = patch_normed;
end
patch = patch';
% Add bias
patch_normed = cat(1, ones(1, size(patch,2)), patch);
weights = patch_experts_class{i};
% Where DNN will happen
for w =1:numel(weights)/2
% mult and bias
patch_normed = weights{(w-1)*2+1}' * patch_normed + repmat(weights{(w-1)*2+2}', 1, size(patch_normed,2));
if w < 3
% patch_normed(patch_normed < 0) = 0;
patch_normed = max(0, patch_normed);
else
patch_normed = 1./(1+exp(-patch_normed));
end
end
responses{i}(:) = reshape(patch_normed', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
end
end
end

84
pkg/OpenFace/matlab_version/fitting/PatchResponseCEN_mirror.m Normal file
View File

@@ -0,0 +1,84 @@
function [ responses ] = PatchResponseCEN_mirror(patches, patch_experts_class, visibilities, patchExperts, window_size)
% As frontal faces are roughly symmetrical can compute the responses for
% two patches at the same time using only one of the landmark patch experts
normalisationOptions = patchExperts.normalisationOptionsCol;
patchSize = normalisationOptions.patchSize;
responses = cell(size(patches, 1), 1);
empty = zeros(window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
% These landmark responses can be computed together
mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
for i = 1:numel(patches(:,1))
if visibilities(i)
% Do it only if not mirrored
if(isempty(find(mirror_inds(:,2)==i, 1)))
responses{i} = empty;
col_norm = normalisationOptions.useNormalisedCrossCorr == 1;
smallRegionVec = patches(i,:);
smallRegion = reshape(smallRegionVec, window_size(1), window_size(2));
patch = im2col_mine(smallRegion, patchSize)';
% Add the mirrored version as well (it will be applied the
% same way)
mirr_id = mirror_inds(find(mirror_inds(:,1)==i,1),2);
if(~isempty(mirr_id))
responses{mirr_id} = empty;
smallRegionVec_mirr = patches(mirr_id,:);
smallRegion_mirr = reshape(smallRegionVec_mirr, window_size(1), window_size(2));
patch_mirr = im2col_mine(fliplr(smallRegion_mirr), patchSize)';
patch = cat(1, patch, patch_mirr);
end
% Normalize
if(col_norm)
mean_curr = mean(patch, 2);
patch_normed = patch - repmat(mean_curr, 1, patchSize(1)* patchSize(2));
% Normalising the patches using the L2 norm
scaling = sqrt(sum(patch_normed.^2,2));
scaling(scaling == 0) = 1;
patch_normed = patch_normed ./ repmat(scaling, 1, 11 * 11);
patch = patch_normed;
end
patch = patch';
% Add bias
patch_normed = cat(1, ones(1, size(patch,2)), patch);
weights = patch_experts_class{i};
% Where DNN will happen
for w =1:numel(weights)/2
% mult and bias
patch_normed = weights{(w-1)*2+1}' * patch_normed + repmat(weights{(w-1)*2+2}', 1, size(patch_normed,2));
if w < 3
% patch_normed(patch_normed < 0) = 0;
patch_normed = max(0, patch_normed);
else
patch_normed = 1./(1+exp(-patch_normed));
end
end
% If no mirroring took place
if(isempty(mirr_id))
responses{i}(:) = reshape(patch_normed', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
else
patch_normed_1 = patch_normed(1:end/2);
patch_normed_2 = patch_normed(end/2+1:end);
responses{i}(:) = reshape(patch_normed_1', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
responses{mirr_id}(:) = fliplr(reshape(patch_normed_2', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1));
end
end
end
end
end

70
pkg/OpenFace/matlab_version/fitting/PatchResponseCEN_sparse.m Normal file
View File

@@ -0,0 +1,70 @@
function [ responses ] = PatchResponseCEN_sparse(rois, patch_experts, visibilities, window_size, support_size)
%PatchResponseCEN_sparse computing response maps on the areas of interest
responses = cell(size(rois, 1), 1);
empty = zeros(window_size(1)-support_size(1)+1, window_size(2)-support_size(2)+1);
% Compute which indices to keep and which to remove
to_rem_ids = [1:2:numel(empty)];
to_keep_ids = setdiff(1:numel(empty), to_rem_ids);
% Prepare the interpolation back to dense response
[x_scattered, y_scattered] = ind2sub(size(empty), to_keep_ids);
F = scatteredInterpolant(cat(2,y_scattered', x_scattered'), zeros(size(x_scattered))', 'natural');
[yq, xq] = ind2sub(size(empty), 1:numel(empty));
% The actual response accross patches
for i = 1:numel(rois(:,1))
responses{i} = empty;
if visibilities(i)
smallRegionVec = rois(i,:);
smallRegion = reshape(smallRegionVec, window_size(1), window_size(2));
roi = im2col_mine(smallRegion, support_size)';
% Normalize
mean_curr = mean(roi, 2);
roi_normed = roi - repmat(mean_curr, 1, support_size(1) * support_size(2));
% Normalising the patches using the L2 norm
scaling = sqrt(sum(roi_normed.^2,2));
scaling(scaling == 0) = 1;
roi_normed = roi_normed ./ repmat(scaling, 1, 11 * 11);
roi = roi_normed;
roi = roi';
% Discard the support regions on which we will not evaluate
roi = roi(:,to_keep_ids);
% Add bias
roi_normed = cat(1, ones(1, size(roi,2)), roi);
output = roi_normed;
weights = patch_experts{i};
% Where forward pass of CEN happens
for w =1:numel(weights)/2
% mult and bias
output = weights{(w-1)*2+1}' * output + repmat(weights{(w-1)*2+2}', 1, size(roi_normed,2));
if w < 3
output = max(0, output);
else
output = 1./(1+exp(-output));
end
end
vals = double(output);
% Now interpolate
F.Values = vals';
responses{i}(:) = F(xq,yq);
end
end
end

132
pkg/OpenFace/matlab_version/fitting/PatchResponseSVM_multi_modal.m Normal file
View File

@@ -0,0 +1,132 @@
function [ responses ] = PatchResponseSVM_multi_modal( patches, patch_experts, visibilities, normalisationOptions, clmParameters, window_size)
%PATCHRESPONSESVM Summary of this function goes here
% Detailed explanation goes here
patchSize = normalisationOptions.patchSize;
responses = cell(size(patches, 1), 1);
empty = zeros(window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1);
% prepare the patches through either turning them to gradients or
if(clmParameters.use_multi_modal)
patches_to_use = cell(numel(clmParameters.multi_modal_types),1);
for t=1:numel(clmParameters.multi_modal_types)
if(strcmp(clmParameters.multi_modal_types{t}, 'reg'))
patches_reg = patches;
if(normalisationOptions.zscore)
meanCurr = mean(patches, 2);
stdCurr = std(double(patches), 0, 2);
stdCurr(stdCurr == 0) = 1;
patches_reg = bsxfun(@minus, patches, meanCurr);
patches_reg = bsxfun(@rdivide, patches_reg, stdCurr);
end
patches_to_use{t} = patches_reg;
elseif(strcmp(clmParameters.multi_modal_types{t}, 'grad'))
v = [1];
h = [-1 0 1];
grad_patches = zeros(size(patches));
for i = 1:numel(patches(:,1))
if visibilities(i)
currSample = reshape(patches(i,:), window_size(1), window_size(2));
edgeX = conv2(conv2(currSample, v, 'same'), h, 'same');
edgeY = conv2(conv2(currSample, v', 'same'), h', 'same');
grad = edgeX.^2 + edgeY.^2;
grad(1,:) = 0;
grad(:,1) = 0;
grad(end,:) = 0;
grad(:,end) = 0;
grad_patches(i,:) = reshape(grad, window_size(1) * window_size(1),1);
end
end
patches_to_use{t} = grad_patches;
end
end
else
patches_reg = patches;
if(normalisationOptions.zscore)
if(normalisationOptions.ignoreInvalidInMeanStd)
% invalid data represented with 0, ignore it when computing
% mean and standard deviation (useful for depth)
for i = 1:size(patches,1)
mask = patches(i,:) ~= 0;
meanCurr = mean(patches(i,mask));
stdCurr = std(patches(i,mask));
patches(i,mask) = patches(i, mask) - meanCurr;
if(stdCurr ~= 0)
patches(i, mask) = patches(i, mask) ./ meanCurr;
end
patches(i, ~mask) = normalisationOptions.setIllegalToPost;
end
patches_reg = patches;
else
meanCurr = mean(patches, 2);
stdCurr = std(double(patches), 0, 2);
stdCurr(stdCurr == 0) = 1;
patches_reg = bsxfun(@minus, patches, meanCurr);
patches_reg = bsxfun(@rdivide, patches_reg, stdCurr);
end
end
patches_to_use = {patches_reg};
end
for i = 1:numel(patches(:,1))
responses{i} = empty;
if visibilities(i)
% responses{i} = ones(size(empty));
colNorm = normalisationOptions.useNormalisedCrossCorr == 1;
for p=1:numel(patches_to_use)
smallRegionVec = patches_to_use{p}(i,:);
smallRegion = reshape(smallRegionVec, window_size(1), window_size(2));
% get the patch response
response = SVMresponse(smallRegion, patch_experts{i}(p).w, colNorm, patchSize);
response = (exp(-(patch_experts{i}(p).scaling*response+patch_experts{i}(p).bias))+1).^-1;
if(p==1)
responses{i} = response;
else
responses{i} = responses{i} .* response;
end
end
normOp = (sum(responses{i}(:)));
if(normOp ~= 0)
responses{i} = responses{i} ./ normOp;
end
end
end
end
function response = SVMresponse(region, patchExpert, normalise_x_corr,patchSize)
if(normalise_x_corr)
% the much faster mex version
[response] = normxcorr2_mex(patchExpert, region);
response = response(patchSize(1):end-patchSize(1)+1,patchSize(2):end-patchSize(2)+1);
else
% this assumes that the patch is already normed
template = rot90(patchExpert,2);
response = conv2(region, template, 'valid');
end
end

80
pkg/OpenFace/matlab_version/fitting/im2col_mine.m Normal file
View File

@@ -0,0 +1,80 @@
function b=im2col_mine(a, block)
%IM2COL Rearrange image blocks into columns.
% B = IM2COL(A,[M N],'distinct') rearranges each distinct
% M-by-N block in the image A into a column of B. IM2COL pads A
% with zeros, if necessary, so its size is an integer multiple
% of M-by-N. If A = [A11 A12; A21 A22], where each Aij is
% M-by-N, then B = [A11(:) A21(:) A12(:) A22(:)].
%
% B = IM2COL(A,[M N],'sliding') converts each sliding M-by-N
% block of A into a column of B, with no zero padding. B has
% M*N rows and will contain as many columns as there are M-by-N
% neighborhoods in A. If the size of A is [MM NN], then the
% size of B is (M*N)-by-((MM-M+1)*(NN-N+1). Each column of B
% contains the neighborhoods of A reshaped as NHOOD(:), where
% NHOOD is a matrix containing an M-by-N neighborhood of
% A. IM2COL orders the columns of B so that they can be
% reshaped to form a matrix in the normal way. For example,
% suppose you use a function, such as SUM(B), that returns a
% scalar for each column of B. You can directly store the
% result in a matrix of size (MM-M+1)-by-(NN-N+1) using these
% calls:
%
% B = im2col(A,[M N],'sliding');
% C = reshape(sum(B),MM-M+1,NN-N+1);
%
% B = IM2COL(A,[M N]) uses the default block type of
% 'sliding'.
%
% B = IM2COL(A,'indexed',...) processes A as an indexed image,
% padding with zeros if the class of A is uint8 or uint16, or
% ones if the class of A is double.
%
% Class Support
% -------------
% The input image A can be numeric or logical. The output matrix
% B is of the same class as the input image.
%
% Example
% -------
% Calculate the local mean using a [2 2] neighborhood with zero padding.
%
% A = reshape(linspace(0,1,16),[4 4])'
% B = im2col(A,[2 2])
% M = mean(B)
% newA = col2im(M,[1 1],[3 3])
%
% See also BLOCKPROC, COL2IM, COLFILT, NLFILTER.
% Copyright 1993-2012 The MathWorks, Inc.
[ma,na] = size(a);
m = block(1); n = block(2);
if any([ma na] < [m n]) % if neighborhood is larger than image
b = zeros(m*n,0);
return
end
% Create Hankel-like indexing sub matrix.
mc = block(1); nc = ma-m+1; nn = na-n+1;
cidx = (0:mc-1)'; ridx = 1:nc;
t = cidx(:,ones(nc,1)) + ridx(ones(mc,1),:); % Hankel Subscripts
tt = zeros(mc*n,nc);
rows = 1:mc;
for i=0:n-1,
tt(i*mc+rows,:) = t+ma*i;
end
ttt = zeros(mc*n,nc*nn);
cols = 1:nc;
for j=0:nn-1,
ttt(:,j*nc+cols) = tt+ma*j;
end
% If a is a row vector, change it to a column vector. This change is
% necessary when A is a row vector and [M N] = size(A).
if ndims(a) == 2 && na > 1 && ma == 1
a = a(:);
end
b = a(ttt);

593
pkg/OpenFace/matlab_version/fitting/interp2_mine.m Normal file
View File

@@ -0,0 +1,593 @@
function zi = interp2_mine(varargin)
%INTERP2 2-D interpolation (table lookup).
% ZI = INTERP2(X,Y,Z,XI,YI) interpolates to find ZI, the values of the
% underlying 2-D function Z at the points in matrices XI and YI.
% Matrices X and Y specify the points at which the data Z is given.
%
% XI can be a row vector, in which case it specifies a matrix with
% constant columns. Similarly, YI can be a column vector and it
% specifies a matrix with constant rows.
%
% ZI = INTERP2(Z,XI,YI) assumes X=1:N and Y=1:M where [M,N]=SIZE(Z).
% ZI = INTERP2(Z,NTIMES) expands Z by interleaving interpolates between
% every element, working recursively for NTIMES. INTERP2(Z) is the
% same as INTERP2(Z,1).
%
% ZI = INTERP2(...,METHOD) specifies alternate methods. The default
% is linear interpolation. Available methods are:
%
% 'nearest' - nearest neighbor interpolation
% 'linear' - bilinear interpolation
% 'spline' - spline interpolation
% 'cubic' - bicubic interpolation as long as the data is
% uniformly spaced, otherwise the same as 'spline'
%
% For faster interpolation when X and Y are equally spaced and monotonic,
% use the syntax ZI = INTERP2(...,*METHOD).
%
% ZI = INTERP2(...,METHOD,EXTRAPVAL) specificies a method and a scalar
% value for ZI outside of the domain created by X and Y. Thus, ZI will
% equal EXTRAPVAL for any value of YI or XI which is not spanned by Y
% or X respectively. A method must be specified for EXTRAPVAL to be used,
% the default method is 'linear'.
%
% All the interpolation methods require that X and Y be monotonic and
% plaid (as if they were created using MESHGRID). If you provide two
% monotonic vectors, interp2 changes them to a plaid internally.
% X and Y can be non-uniformly spaced.
%
% For example, to generate a coarse approximation of PEAKS and
% interpolate over a finer mesh:
% [x,y,z] = peaks(10); [xi,yi] = meshgrid(-3:.1:3,-3:.1:3);
% zi = interp2(x,y,z,xi,yi); mesh(xi,yi,zi)
%
% Class support for inputs X, Y, Z, XI, YI:
% float: double, single
%
% See also INTERP1, INTERP3, INTERPN, MESHGRID, TriScatteredInterp.
% Copyright 1984-2011 The MathWorks, Inc.
% $Revision: 5.33.4.24 $ $Date: 2011/05/17 02:32:27 $
% error(nargchk(1,7,nargin,'struct')); % allowing for an ExtrapVal
bypass = false;
uniform = true;
if (nargin > 1)
if nargin == 7 && ~isnumeric(varargin{end})
error(message('MATLAB:interp2:extrapvalNotNumeric'));
end
if ischar(varargin{end})
narg = nargin-1;
method = [varargin{end} ' ']; % Protect against short string.
if strncmpi(method,'s',1) || strncmpi(method, '*s', 2)
ExtrapVal = 'extrap'; % Splines can extrapolate
else
ExtrapVal = nan; % setting default ExtrapVal as NAN
end
index = 1; %subtract off the elements not in method
elseif ischar(varargin{end-1}) && isnumeric(varargin{end})
narg = nargin-2;
method = [ varargin{end-1} ' '];
ExtrapVal = varargin{end}; % user specified ExtrapVal
index = 2; % subtract off the elements not in method and ExtrapVal
else
narg = nargin;
method = 'linear';
ExtrapVal = nan; % protecting default
index = 0;
end
if strncmpi(method,'*',1) % Direct call bypass.
if (narg ==5 || narg ==3)
xitemp = varargin{end-index - 1};
yitemp = varargin{end-index};
if isrow(xitemp) && iscolumn(yitemp)
varargin{end-index - 1} = repmat(xitemp, [size(yitemp,1), 1]);
varargin{end-index} = repmat(yitemp, [1, size(xitemp,2)]);
elseif iscolumn(xitemp) && isrow(yitemp)
varargin{end-index - 1} = repmat(xitemp', [size(yitemp, 2), 1]);
varargin{end-index} = repmat(yitemp', [1, size(xitemp,1)]);
end
end
if strcmpi(method(2),'l') || strcmpi(method(2:4),'bil')
% bilinear interpolation.
zi = linear(ExtrapVal, varargin{1:end-index});
return
elseif strcmpi(method(2),'c') || strcmpi(method(2:4),'bic')
% bicubic interpolation
zi = cubic(ExtrapVal, varargin{1:end-index});
return
elseif strcmpi(method(2),'n')
% Nearest neighbor interpolation
zi = nearest(ExtrapVal, varargin{1:end-index});
return
elseif strcmpi(method(2),'s')
% spline interpolation
method = 'spline'; bypass = true;
else
error(message('MATLAB:interp2:InvalidMethod', deblank( method )));
end
elseif strncmpi(method,'s',1), % Spline interpolation
method = 'spline'; bypass = true;
end
else
narg = nargin;
method = 'linear';
ExtrapVal = nan; % default ExtrapVal is NaN
end
% if narg==1, % interp2(z), % Expand Z
% [nrows,ncols] = size(varargin{1});
% xi = 1:.5:ncols; yi = (1:.5:nrows)';
% x = 1:ncols; y = 1:nrows;
% [msg,x,y,z,xi,yi] = xyzchk(x,y,varargin{1},xi,yi);
%
% elseif narg==2. % interp2(z,n), Expand Z n times
% [nrows,ncols] = size(varargin{1});
% ntimes = floor(varargin{2}(1));
% xi = 1:1/(2^ntimes):ncols; yi = (1:1/(2^ntimes):nrows)';
% x = 1:ncols; y = 1:nrows;
% [msg,x,y,z,xi,yi] = xyzchk(x,y,varargin{1},xi,yi);
%
% elseif narg==3, % interp2(z,xi,yi)
% [nrows,ncols] = size(varargin{1});
% x = 1:ncols; y = 1:nrows;
% [msg,x,y,z,xi,yi] = xyzchk(x,y,varargin{1:3});
%
% elseif narg==4,
% error(message('MATLAB:interp2:nargin'));
% elseif narg==5, % linear(x,y,z,xi,yi)
% [msg,x,y,z,xi,yi] = xyzchk(varargin{1:5});
%
% end
x = varargin{1};
y = varargin{2};
z = varargin{3};
xi = varargin{4};
yi = varargin{5};
% if ~isempty(msg)
% error(message(msg.identifier));
% end
%
% Check for plaid data.
%
xx = x(1,:); yy = y(:,1);
% if (size(x,2)>1 && ~isequal(repmat(xx,size(x,1),1),x)) || ...
% (size(y,1)>1 && ~isequal(repmat(yy,1,size(y,2)),y)),
% error(message('MATLAB:interp2:meshgrid'));
% end
%
% Check for non-equally spaced data. If so, map (x,y) and
% (xi,yi) to matrix (row,col) coordinate system.
%
if ~bypass,
xx = xx.'; % Make sure it's a column.
dx = diff(xx); dy = diff(yy);
xdiff = max(abs(diff(dx))); if isempty(xdiff), xdiff = 0; end
ydiff = max(abs(diff(dy))); if isempty(ydiff), ydiff = 0; end
if (xdiff > eps(class(xx))*max(abs(xx))) || (ydiff > eps(class(yy))*max(abs(yy)))
if any(dx < 0), % Flip orientation of data so x is increasing.
x = fliplr(x); y = fliplr(y); z = fliplr(z);
xx = flipud(xx); dx = -flipud(dx);
end
if any(dy < 0), % Flip orientation of data so y is increasing.
x = flipud(x); y = flipud(y); z = flipud(z);
yy = flipud(yy); dy = -flipud(dy);
end
if any(dx<=0) || any(dy<=0),
error(message('MATLAB:interp2:XorYNotMonotonic'));
end
% Bypass mapping code for cubic
if ~strncmp(method(1),'c',1)
% Determine the nearest location of xi in x
[xxi,j] = sort(xi(:));
[~,i] = sort([xx;xxi]);
ui(i) = 1:length(i);
ui = (ui(length(xx)+1:end)-(1:length(xxi)))';
ui(j) = ui;
% Map values in xi to index offset (ui) via linear interpolation
ui(ui<1) = 1;
ui(ui>length(xx)-1) = length(xx)-1;
ui = ui + (xi(:)-xx(ui))./(xx(ui+1)-xx(ui));
% Determine the nearest location of yi in y
[yyi,j] = sort(yi(:));
[~,i] = sort([yy;yyi(:)]);
vi(i) = 1:length(i);
vi = (vi(length(yy)+1:end)-(1:length(yyi)))';
vi(j) = vi;
% Map values in yi to index offset (vi) via linear interpolation
vi(vi<1) = 1;
vi(vi>length(yy)-1) = length(yy)-1;
vi = vi + (yi(:)-yy(vi))./(yy(vi+1)-yy(vi));
[x,y] = meshgrid(ones(class(x)):size(x,2),ones(class(y)):size(y,1));
xi(:) = ui; yi(:) = vi;
else
uniform = false;
end
end
end
% Now do the interpolation based on method.
if strncmpi(method,'l',1) || strncmpi(method,'bil',3) % bilinear interpolation.
zi = linear(ExtrapVal,x,y,z,xi,yi);
elseif strncmpi(method,'c',1) || strncmpi(method,'bic',3) % bicubic interpolation
if uniform
zi = cubic(ExtrapVal,x,y,z,xi,yi);
else
zi = spline2(x,y,z,xi,yi,ExtrapVal);
end
elseif strncmpi(method,'n',1) % Nearest neighbor interpolation
zi = nearest(ExtrapVal,x,y,z,xi,yi);
elseif strncmpi(method,'s',1) % Spline interpolation
% A column is removed from z if it contains a NaN.
% Orient to preserve as much data as possible.
[inan, jnan] = find(isnan(z));
ncolnan = length(unique(jnan));
nrownan = length(unique(inan));
if ncolnan > nrownan
zi = spline2(y',x',z',yi,xi,ExtrapVal);
else
zi = spline2(x,y,z,xi,yi,ExtrapVal);
end
else
error(message('MATLAB:interp2:InvalidMethod', deblank( method )));
end
%------------------------------------------------------
function F = linear(ExtrapVal,arg1,arg2,arg3,arg4,arg5)
%LINEAR 2-D bilinear data interpolation.
% ZI = LINEAR(EXTRAPVAL,X,Y,Z,XI,YI) uses bilinear interpolation to
% find ZI, the values of the underlying 2-D function in Z at the points
% in matrices XI and YI. Matrices X and Y specify the points at which
% the data Z is given. X and Y can also be vectors specifying the
% abscissae for the matrix Z as for MESHGRID. In both cases, X
% and Y must be equally spaced and monotonic.
%
% Values of EXTRAPVAL are returned in ZI for values of XI and YI that are
% outside of the range of X and Y.
%
% If XI and YI are vectors, LINEAR returns vector ZI containing
% the interpolated values at the corresponding points (XI,YI).
%
% ZI = LINEAR(EXTRAPVAL,Z,XI,YI) assumes X = 1:N and Y = 1:M, where
% [M,N] = SIZE(Z).
%
% ZI = LINEAR(EXTRAPVAL,Z,NTIMES) returns the matrix Z expanded by
% interleaving bilinear interpolates between every element, working
% recursively for NTIMES. LINEAR(EXTRAPVAL,Z) is the same as
% LINEAR(EXTRAPVAL,Z,1).
%
% See also INTERP2, CUBIC.
if nargin==2 % linear(extrapval,z), Expand Z
[nrows,ncols] = size(arg1);
s = 1:.5:ncols; lengths = length(s);
t = (1:.5:nrows)'; lengtht = length(t);
s = repmat(s,lengtht,1);
t = repmat(t,1,lengths);
elseif nargin==3 % linear(extrapval,z,n), Expand Z n times
[nrows,ncols] = size(arg1);
ntimes = floor(arg2);
s = 1:1/(2^ntimes):ncols; lengths = length(s);
t = (1:1/(2^ntimes):nrows)'; lengtht = length(t);
s = repmat(s,lengtht,1);
t = repmat(t,1,lengths);
elseif nargin==4 % linear(extrapval,z,s,t), No X or Y specified.
[nrows,ncols] = size(arg1);
s = arg2; t = arg3;
elseif nargin==5
error(message('MATLAB:interp2:linear:nargin'));
elseif nargin==6 % linear(extrapval,x,y,z,s,t), X and Y specified.
[nrows,ncols] = size(arg3);
mx = numel(arg1); my = numel(arg2);
if (mx ~= ncols || my ~= nrows) && ~isequal(size(arg1),size(arg2),size(arg3))
error(message('MATLAB:interp2:linear:XYZLengthMismatch'));
end
if nrows < 2 || ncols < 2
error(message('MATLAB:interp2:linear:sizeZ'));
end
s = 1 + (arg4-arg1(1))/(arg1(end)-arg1(1))*(ncols-1);
t = 1 + (arg5-arg2(1))/(arg2(end)-arg2(1))*(nrows-1);
end
if nrows < 2 || ncols < 2
error(message('MATLAB:interp2:linear:sizeZsq'));
end
if ~isequal(size(s),size(t))
error(message('MATLAB:interp2:linear:XIandYISizeMismatch'));
end
% Check for out of range values of s and set to 1
sout = find((s<1)|(s>ncols));
if ~isempty(sout), s(sout) = 1; end
% Check for out of range values of t and set to 1
tout = find((t<1)|(t>nrows));
if ~isempty(tout), t(tout) = 1; end
% Matrix element indexing
ndx = floor(t)+floor(s-1)*nrows;
% Compute intepolation parameters, check for boundary value.
if isempty(s), d = s; else d = find(s==ncols); end
s(:) = (s - floor(s));
if ~isempty(d), s(d) = s(d)+1; ndx(d) = ndx(d)-nrows; end
% Compute intepolation parameters, check for boundary value.
if isempty(t), d = t; else d = find(t==nrows); end
t(:) = (t - floor(t));
if ~isempty(d), t(d) = t(d)+1; ndx(d) = ndx(d)-1; end
% Now interpolate.
onemt = 1-t;
if nargin==6,
F = ( arg3(ndx).*(onemt) + arg3(ndx+1).*t ).*(1-s) + ...
( arg3(ndx+nrows).*(onemt) + arg3(ndx+(nrows+1)).*t ).*s;
else
F = ( arg1(ndx).*(onemt) + arg1(ndx+1).*t ).*(1-s) + ...
( arg1(ndx+nrows).*(onemt) + arg1(ndx+(nrows+1)).*t ).*s;
end
% Now set out of range values to ExtrapVal.
if ~isempty(sout), F(sout) = ExtrapVal; end
if ~isempty(tout), F(tout) = ExtrapVal; end
%------------------------------------------------------
function F = cubic(ExtrapVal,arg1,arg2,arg3,arg4,arg5)
%CUBIC 2-D bicubic data interpolation.
% CUBIC(...) is the same as LINEAR(....) except that it uses
% bicubic interpolation.
%
% This function needs about 7-8 times SIZE(XI) memory to be available.
%
% See also LINEAR.
% Based on "Cubic Convolution Interpolation for Digital Image
% Processing", Robert G. Keys, IEEE Trans. on Acoustics, Speech, and
% Signal Processing, Vol. 29, No. 6, Dec. 1981, pp. 1153-1160.
if nargin==2, % cubic(extrapval,z), Expand Z
[nrows,ncols] = size(arg1);
s = 1:.5:ncols; lengths = length(s);
t = (1:.5:nrows)'; lengtht = length(t);
s = repmat(s,lengtht,1);
t = repmat(t,1,lengths);
elseif nargin==3, % cubic(extrapval,z,n), Expand Z n times
[nrows,ncols] = size(arg1);
ntimes = floor(arg2);
s = 1:1/(2^ntimes):ncols; lengths = length(s);
t = (1:1/(2^ntimes):nrows)'; lengtht = length(t);
s = repmat(s,lengtht,1);
t = repmat(t,1,lengths);
elseif nargin==4, % cubic(extrapval,z,s,t), No X or Y specified.
[nrows,ncols] = size(arg1);
s = arg2; t = arg3;
elseif nargin==5,
error(message('MATLAB:interp2:cubic:nargin'));
elseif nargin==6, % cubic(extrapval,x,y,z,s,t), X and Y specified.
[nrows,ncols] = size(arg3);
mx = numel(arg1); my = numel(arg2);
if (mx ~= ncols || my ~= nrows) && ~isequal(size(arg1),size(arg2),size(arg3))
error(message('MATLAB:interp2:cubic:XYZLengthMismatch'));
end
if nrows < 3 || ncols < 3
error(message('MATLAB:interp2:cubic:sizeZ'));
end
s = 1 + (arg4-arg1(1))/(arg1(end)-arg1(1))*(ncols-1);
t = 1 + (arg5-arg2(1))/(arg2(end)-arg2(1))*(nrows-1);
end
if nrows < 3 || ncols < 3
error(message('MATLAB:interp2:cubic:sizeZsq'));
end
if ~isequal(size(s),size(t)),
error(message('MATLAB:interp2:cubic:XIandYISizeMismatch'));
end
% Check for out of range values of s and set to 1
sout = find((s<1)|(s>ncols));
if ~isempty(sout), s(sout) = 1; end
% Check for out of range values of t and set to 1
tout = find((t<1)|(t>nrows));
if ~isempty(tout), t(tout) = 1; end
% Matrix element indexing
ndx = floor(t)+floor(s-1)*(nrows+2);
% Compute intepolation parameters, check for boundary value.
if isempty(s), d = s; else d = find(s==ncols); end
s(:) = (s - floor(s));
if ~isempty(d), s(d) = s(d)+1; ndx(d) = ndx(d)-nrows-2; end
% Compute intepolation parameters, check for boundary value.
if isempty(t), d = t; else d = find(t==nrows); end
t(:) = (t - floor(t));
if ~isempty(d), t(d) = t(d)+1; ndx(d) = ndx(d)-1; end
if nargin==6,
% Expand z so interpolation is valid at the boundaries.
zz = zeros(size(arg3)+2);
zz(1,2:ncols+1) = 3*arg3(1,:)-3*arg3(2,:)+arg3(3,:);
zz(2:nrows+1,2:ncols+1) = arg3;
zz(nrows+2,2:ncols+1) = 3*arg3(nrows,:)-3*arg3(nrows-1,:)+arg3(nrows-2,:);
zz(:,1) = 3*zz(:,2)-3*zz(:,3)+zz(:,4);
zz(:,ncols+2) = 3*zz(:,ncols+1)-3*zz(:,ncols)+zz(:,ncols-1);
nrows = nrows+2; %also ncols = ncols+2;
else
% Expand z so interpolation is valid at the boundaries.
zz = zeros(size(arg1)+2);
zz(1,2:ncols+1) = 3*arg1(1,:)-3*arg1(2,:)+arg1(3,:);
zz(2:nrows+1,2:ncols+1) = arg1;
zz(nrows+2,2:ncols+1) = 3*arg1(nrows,:)-3*arg1(nrows-1,:)+arg1(nrows-2,:);
zz(:,1) = 3*zz(:,2)-3*zz(:,3)+zz(:,4);
zz(:,ncols+2) = 3*zz(:,ncols+1)-3*zz(:,ncols)+zz(:,ncols-1);
nrows = nrows+2; %also ncols = ncols+2;
end
% Now interpolate using computationally efficient algorithm.
t0 = ((2-t).*t-1).*t;
t1 = (3*t-5).*t.*t+2;
t2 = ((4-3*t).*t+1).*t;
t(:) = (t-1).*t.*t;
F = ( zz(ndx).*t0 + zz(ndx+1).*t1 + zz(ndx+2).*t2 + zz(ndx+3).*t ) ...
.* (((2-s).*s-1).*s);
ndx(:) = ndx + nrows;
F(:) = F + ( zz(ndx).*t0 + zz(ndx+1).*t1 + zz(ndx+2).*t2 + zz(ndx+3).*t ) ...
.* ((3*s-5).*s.*s+2);
ndx(:) = ndx + nrows;
F(:) = F + ( zz(ndx).*t0 + zz(ndx+1).*t1 + zz(ndx+2).*t2 + zz(ndx+3).*t ) ...
.* (((4-3*s).*s+1).*s);
ndx(:) = ndx + nrows;
F(:) = F + ( zz(ndx).*t0 + zz(ndx+1).*t1 + zz(ndx+2).*t2 + zz(ndx+3).*t ) ...
.* ((s-1).*s.*s);
F(:) = F/4;
% Now set out of range values to ExtrapVal.
if ~isempty(sout), F(sout) = ExtrapVal; end
if ~isempty(tout), F(tout) = ExtrapVal; end
%------------------------------------------------------
function F = nearest(ExtrapVal,arg1,arg2,arg3,arg4,arg5)
%NEAREST 2-D Nearest neighbor interpolation.
% ZI = NEAREST(EXTRAPVAL,X,Y,Z,XI,YI) uses nearest neighbor interpolation
% to find ZI, the values of the underlying 2-D function in Z at the points
% in matrices XI and YI. Matrices X and Y specify the points at which
% the data Z is given. X and Y can also be vectors specifying the
% abscissae for the matrix Z as for MESHGRID. In both cases, X
% and Y must be equally spaced and monotonic.
%
% Values of EXTRAPVAL are returned in ZI for values of XI and YI that are
% outside of the range of X and Y.
%
% If XI and YI are vectors, NEAREST returns vector ZI containing
% the interpolated values at the corresponding points (XI,YI).
%
% ZI = NEAREST(EXTRAPVAL,Z,XI,YI) assumes X = 1:N and Y = 1:M, where
% [M,N] = SIZE(Z).
%
% F = NEAREST(EXTRAPVAL,Z,NTIMES) returns the matrix Z expanded by
% interleaving interpolates between every element. NEAREST(EXTRAPVAL,Z)
% is the same as NEAREST(EXTRAPVAL,Z,1).
%
% See also INTERP2, LINEAR, CUBIC.
if nargin==2, % nearest(z), Expand Z
[nrows,ncols] = size(arg1);
u = 1:.5:ncols; lengthu = length(u);
v = (1:.5:nrows)'; lengthv = length(v);
u = repmat(u,lengthv,1);
v = repmat(v,1,lengthu);
elseif nargin==3, % nearest(z,n), Expand Z n times
[nrows,ncols] = size(arg1);
ntimes = floor(arg2);
u = 1:1/(2^ntimes):ncols; lengthu = length(u);
v = (1:1/(2^ntimes):nrows)'; lengthv = length(v);
u = repmat(u,lengthv,1);
v = repmat(v,1,lengthu);
elseif nargin==4, % nearest(z,u,v)
[nrows,ncols] = size(arg1);
u = arg2; v = arg3;
elseif nargin==5,
error(message('MATLAB:interp2:nearest:nargin'));
elseif nargin==6, % nearest(x,y,z,u,v), X and Y specified.
[nrows,ncols] = size(arg3);
mx = numel(arg1); my = numel(arg2);
if (mx ~= ncols || my ~= nrows) && ...
~isequal(size(arg1),size(arg2),size(arg3))
error(message('MATLAB:interp2:nearest:XYZLengthMismatch'));
end
if nrows > 1 && ncols > 1
u = 1 + (arg4-arg1(1))/(arg1(mx)-arg1(1))*(ncols-1);
v = 1 + (arg5-arg2(1))/(arg2(my)-arg2(1))*(nrows-1);
else
u = 1 + (arg4-arg1(1));
v = 1 + (arg5-arg2(1));
end
end
if ~isequal(size(u),size(v))
error(message('MATLAB:interp2:nearest:XIandYISizeMismatch'));
end
% Check for out of range values of u and set to 1
uout = (u<.5)|(u>=ncols+.5);
anyuout = any(uout(:));
if anyuout, u(uout) = 1; end
% Check for out of range values of v and set to 1
vout = (v<.5)|(v>=nrows+.5);
anyvout = any(vout(:));
if anyvout, v(vout) = 1; end
% Interpolation parameters
u = round(u); v = round(v);
% Now interpolate
ndx = v+(u-1)*nrows;
if nargin==6,
F = arg3(ndx);
else
F = arg1(ndx);
end
% Now set out of range values to ExtrapVal.
if anyuout, F(uout) = ExtrapVal; end
if anyvout, F(vout) = ExtrapVal; end
%----------------------------------------------------------
function F = spline2(varargin)
%2-D spline interpolation
% Determine abscissa vectors
varargin{1} = varargin{1}(1,:);
varargin{2} = varargin{2}(:,1).';
%
% Check for plaid data.
%
xi = varargin{4}; yi = varargin{5};
xxi = xi(1,:); yyi = yi(:,1);
if ~isequal(repmat(xxi,size(xi,1),1),xi) || ...
~isequal(repmat(yyi,1,size(yi,2)),yi)
F = splncore(varargin(2:-1:1),varargin{3},varargin(5:-1:4));
else
F = splncore(varargin(2:-1:1),varargin{3},{yyi(:).' xxi},'gridded');
end
ExtrapVal = varargin{6};
% Set out-of-range values to ExtrapVal
if isnumeric(ExtrapVal)
d = xi < min(varargin{1}) | xi > max(varargin{1}) | ...
yi < min(varargin{2}) | yi > max(varargin{2});
F(d) = ExtrapVal;
end

4
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/Makefile Normal file
View File

@@ -0,0 +1,4 @@
all: normxcorr2_mex
normxcorr2_mex: normxcorr2_mex.cpp
mex -O cv_src/*.cpp normxcorr2_mex.cpp -o normxcorr2_mex

18
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/README.txt Normal file
View File

@@ -0,0 +1,18 @@
This software is taken verbatim from the OpenCV (http://sourceforge.net/projects/opencvlibrary/).
I don't know who wrote the routine cvMatchTemplate, but it is exceptional code and deserves recognition. All I've done
is write a Matlab MEX interface, and provide the routine separately from the OpenCV library. My motivation
for doing this was simply to make it more accessible. The OpenCV is bloated and poorly documented -- two negatives
which make it confusing and inconvenient for many people to use, despite the fact it contains many excellent features.
It seems that providing this code meshes well with the OpenCV's aim for widespread use, but if I've
violated the license, please indicate so.
Anyway, I hope this interface to the OpenCV code is beneficial to someone. I was frustrated by the lack of fast
NCC code available on the net -- many claim to have written fast routines, but don't readily share them. In my
experience this far, the OpenCV routine is the fastest general purpose, exact way to do NCC ou there. There are
domain-dependent speed-ups, which I discuss at www.cs.ubc.ca/~deaton/remarks_ncc.html. You may find my remarks there
useful.
I'd be interested to hear about your own NCC experiences,
Daniel Eaton
danieljameseaton@gmail.com

1
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/linuxBuildReadme.txt Normal file
View File

@@ -0,0 +1 @@
Compiled with GCC version 3.3.3 on SuSE Linux (x86).

48
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_demo.m Normal file
View File

@@ -0,0 +1,48 @@
templateSize = 11; % square template
imgWidth = 320; % rectangular image
imgHeight = 240;
template = rand(templateSize); % template with random entries
img = rand(imgHeight, imgWidth); % similarly for the image
% randomly embed the template inside the img
embedX = round(rand*(imgWidth-templateSize-1)+1);
embedY = round(rand*(imgHeight-templateSize-1)+1);
img( embedY:embedY+templateSize-1, embedX:embedX+templateSize-1 ) = template;
% display the input for normxcorr2
H = figure(1); clf;
set(H,'name','Can you find the template?');
subplot(1,2,1);
imagesc(img);
axis('image'); title('image');
subplot(1,2,2);
imagesc(template);
axis('image'); title('template');
colormap(gray);
% perform NCC
% choose 'same' as the output shape -- ie. zero-pad the output so
% that it's the same size as the original image
% this is not necessary, but is used for display
ncc = normxcorr2_mex(template, img, 'same');
ncc1 = normxcorr2_mex(template, img);
ncc2 = normxcorr2(template, img);
% display the ncc
H = figure(2); clf;
set(H,'name','NCC can!');
imagesc(ncc);
axis('image'); title('normalized cross correlation');
colormap(gray);
% output where there template was embedded (centered)
fprintf('template was embedded at (%i,%i)\n', embedY+ceil((templateSize-1)/2), embedX+ceil((templateSize-1)/2));
% compute, then output where ncc 'found' the template (the highest score,
% which in this case is +1, since there is no noise)
[t mi] = max(ncc);
[t mj] = max(t);
mi = mi(mj);
fprintf('ncc found it at (%i,%i)\n', mi, mj);

140
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_mex.cpp Normal file
View File

@@ -0,0 +1,140 @@
#include "mex.h"
#include "cv_src/cv.h"
#include <string.h>
#include <stdio.h>
#include <math.h>
#include <ctype.h>
void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) {
int i, j, oi, oj, k;
int imw, imh;
int tpw, tph;
int resw, resh;
int outw, outh; // real size of output (taking into account 'shape')
int offw, offh;
double* im;
double* tp;
double* res;
IplImage* cvIm;
IplImage* cvTp;
IplImage* cvRes;
int shapeTypeCharSize = 6;
char* shapeTypeChar;
int shapeType = 1;
/* 1, full
2, valid
3, same
*/
float* cvImP;
float* cvTpP;
float* cvResP;
if( nrhs < 2 ) {
mexErrMsgTxt("result = normxcorr2Mex(TEMPLATE, IMAGE)");
} else if( !( mxIsDouble(prhs[0])&&mxIsDouble(prhs[1])) ) {
mexErrMsgTxt("IMAGE and TEMPLATE must be of type double");
} else if( nrhs > 2 && !mxIsChar(prhs[2]) ) {
mexErrMsgTxt("SHAPE parameter must be a character string");
}
shapeTypeChar = (char*)malloc(shapeTypeCharSize);
if( nrhs>2 ) {
mxGetString( prhs[2], shapeTypeChar, shapeTypeCharSize );
for( int ci=0; ci<strlen(shapeTypeChar); ci++ )
shapeTypeChar[ci] = tolower(shapeTypeChar[ci]);
if( !strcmp(shapeTypeChar, "full") )
shapeType = 1;
else if( !strcmp(shapeTypeChar, "valid") )
shapeType = 2;
else if( !strcmp(shapeTypeChar, "same") )
shapeType = 3;
else
shapeType = 0;
}
free(shapeTypeChar);
if( !shapeType )
mexErrMsgTxt("unknown shape type");
im = mxGetPr(prhs[1]);
imw = mxGetN(prhs[1]);
imh = mxGetM(prhs[1]);
tp = mxGetPr(prhs[0]);
tpw = mxGetN(prhs[0]);
tph = mxGetM(prhs[0]);
resw = imw-tpw+1;
resh = imh-tph+1;
if( resw<=0 || resh<=0 ) {
mexErrMsgTxt("size(image) < size(template)");
}
cvIm = cvCreateImage(cvSize(imw,imh), IPL_DEPTH_32F, 1);
cvTp = cvCreateImage(cvSize(tpw,tph), IPL_DEPTH_32F, 1);
cvRes = cvCreateImage(cvSize(resw,resh), IPL_DEPTH_32F, 1);
cvImP = (float*)cvIm->imageData;
cvTpP = (float*)cvTp->imageData;
cvResP = (float*)cvRes->imageData;
if( shapeType==1 ) { //full
outw = imw+tpw-1;
outh = imh+tph-1;
offw = tpw-1;
offh = tph-1;
} else if( shapeType==2 ) { //valid
outw = resw;
outh = resh;
offw = 0;
offh = 0;
} else if( shapeType==3 ) { // same
outw = imw;
outh = imh;
offw = ceil((float)(tpw-1)/2.0f);
offh = ceil((float)(tph-1)/2.0f);
}
plhs[0] = mxCreateDoubleMatrix( outh, outw, mxREAL );
res = mxGetPr(plhs[0]);
// we don't need to worry about alignment since we're using 32f
k = 0;
for( i=0; i<imw; i++ ) {
for( j=0; j<imh; j++ ) {
cvImP[ j*imw+i ] = im[ k ];
k++;
}
}
k = 0;
for( i=0; i<tpw; i++ ) {
for( j=0; j<tph; j++ ) {
cvTpP[ j*tpw+i ] = tp[ k ];
k++;
}
}
cvMatchTemplate( cvIm, cvTp, cvRes, CV_TM_CCOEFF_NORMED );
for( i=0, oi=offw; i<resw; i++, oi++ ) {
for( j=0, oj=offh; j<resh; j++, oj++ ) {
res[ oi*outh + oj ] = cvResP[ j*resw + i ];
}
}
cvReleaseImage(&cvIm);
cvReleaseImage(&cvTp);
cvReleaseImage(&cvRes);
}

BIN
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_mex.dll Normal file
View File

Binary file not shown.

42
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_mex.m Normal file
View File

@@ -0,0 +1,42 @@
% NORMXCORR2_MEX
% RESULT = normxcorr2_mex(TEMPLATE, IMAGE, SHAPE)
%
% TEMPLATE - type double, size <= size of image
% IMAGE - type double
% SHAPE - one of: 'valid', 'same', 'full'. same as conv2 shape parameter
%
% RESULT - type double, values in [-1,1]. size depends on SHAPE
%
% the syntax of this function is identical to Matlab's
% normxcorr2, except for the output size. the formula for the output size
% is:
%
% size(template) = [tp_H, tp_W]
% size(image) = [im_H, im_W]
%
% SHAPE='valid' size(result) = [im_H-tp_H+1, im_W-tp_W+1]
% SHAPE='same' size(result) = [im_H im_W]
% SHAPE='full' size(result) = [im_H+tp_H-1, im_W+tp_W-1]
%
% note:
% all choices of the SHAPE parameter yield the same output. by this I mean
% that the 'same' and 'full' cases just zero-pad the result so that they
% are the correct size (for your convenience). this implementation cannot return partial matching
% results near the boundary.
%
% the following are equivalent:
% result = normxcorr2_mex(template, image, 'full');
% AND
% result = normxcorr2(template, image);
% except that normxcorr2_mex has 0's in the 'invalid' area along the boundary
%
% SEE ALSO CONV2 (for an explanation of SHAPE), or normxcorr2_demo.m (for a demo)
%
% Major NB I want made clear:
% The core code uses the cvMatchTemplate from the OpenCV. I don't know who
% was responsible for this routine, but it is extremely well coded and I want
% to give them credit. All I've done is write the MEX interface.
% For more on this, see the readme.
%
% Daniel Eaton, danieljameseaton@gmail.com, 2005

BIN
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_mex.mexglx Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_mex.mexsol Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/normxcorr2_mex.mexw64 Normal file
View File

Binary file not shown.

21
pkg/OpenFace/matlab_version/fitting/normxcorr2_mex_ALL/sunBuildReadme.txt Normal file
View File

@@ -0,0 +1,21 @@
Machine hardware: sun4u
OS version: 5.9
Processor type: sparc
Hardware: SUNW,Sun-Fire-280R
The following components are installed on your system:
Forte Developer 7
Forte Developer 7 Compilers C
Forte Developer 7 Compilers C++
Forte Developer 7 Tools.h++ 7.1
Forte Developer 7 Standard 64-bit Class Library for C++
Forte Developer 7 Garbage Collector
Forte Developer 7 Compilers Fortran 95
Forte Developer 7 DBX Debugging Tools
Forte Developer 7 Performance Tools
Forte Developer 7 Performance Library
Forte Developer 7 LockLint
Forte Developer 7 Building Software
Forte Developer 7 Documentation Set