bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/readme.txt Normal file
View File

@@ -0,0 +1,9 @@
The code here prepares the training images and labels into a format that can be easilly used to train SVR and CCNF regressors.
Just run "scripts/Prepare_data_wild_all.m" for data needed to train patch experts for in-the-wild experiments.(you have to have the relevant datasets, but they are all available online at http://ibug.doc.ic.ac.uk/resources/facial-point-annotations/)
Run "scripts/Prepare_data_Multi_PIE_all.m" (you have to have the multi-pie dataset and labels)
Run "scripts/Prepare_data_general_all.m" (you have to have both of the datasets, and you have to run "scripts/Prepare_data_wild_all.m" and scripts/Prepare_data_Multi_PIE_all.m" first.
PDM model used is trained on 2D landmark labels using Non-Rigid-Structure for motion (code can be found http://www.cl.cam.ac.uk/~tb346/res/ccnf/pdm_generation.zip)

9
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AddOrthRow.m Normal file
View File

@@ -0,0 +1,9 @@
function [RotFull] = AddOrthRow(RotSmall)
% We can work out these values from the small version of the rotation matrix Rx * Ry * Rz (if you plug in values you can work it out, just slightly tedious)
RotFull = zeros(3,3);
RotFull(1:2, :) = RotSmall;
RotFull(3,1) = RotSmall(1, 2) * RotSmall(2, 3) - RotSmall(1, 3) * RotSmall(2, 2);
RotFull(3,2) = RotSmall(1, 3) * RotSmall(2, 1) - RotSmall(1, 1) * RotSmall(2, 3);
RotFull(3,3) = RotSmall(1, 1) * RotSmall(2, 2) - RotSmall(1, 2) * RotSmall(2, 1);

22
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AlignShapesKabsch.m Normal file
View File

@@ -0,0 +1,22 @@
function [ R, T ] = AlignShapesKabsch ( alignFrom, alignTo )
%ALIGN3DSHAPES Summary of this function goes here
% Detailed explanation goes here
dims = size(alignFrom, 2);
alignFromMean = alignFrom - repmat(mean(alignFrom), size(alignFrom,1),1);
alignToMean = alignTo - repmat(mean(alignTo), size(alignTo,1),1);
[U, ~, V] = svd( alignFromMean' * alignToMean);
% make sure no reflection is there
d = sign(det(V*U'));
corr = eye(dims);
corr(end,end) = d;
R = V*corr*U';
T = mean(alignTo) - (R * mean(alignFrom)')';
T = T';
end

30
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AlignShapesWithScale.m Normal file
View File

@@ -0,0 +1,30 @@
function [ A, T, error, alignedShape ] = AlignShapesWithScale( alignFrom, alignTo )
%ALIGNSHAPESWITHSCALE Summary of this function goes here
% Detailed explanation goes here
numPoints = size(alignFrom,1);
meanFrom = mean(alignFrom);
meanTo = mean(alignTo);
alignFromMeanNormed = bsxfun(@minus, alignFrom, meanFrom);
alignToMeanNormed = bsxfun(@minus, alignTo, meanTo);
% scale now
sFrom = sqrt(sum(alignFromMeanNormed(:).^2)/numPoints);
sTo = sqrt(sum(alignToMeanNormed(:).^2)/numPoints);
s = sTo / sFrom;
alignFromMeanNormed = alignFromMeanNormed/sFrom;
alignToMeanNormed = alignToMeanNormed/sTo;
[R, t] = AlignShapesKabsch(alignFromMeanNormed, alignToMeanNormed);
A = s * R;
aligned = (A * alignFrom')';
T = mean(alignTo - aligned);
alignedShape = bsxfun(@plus, aligned, T);
error = mean(sum((alignedShape - alignTo).^2,2));
end

11
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/AxisAngle2Rot.m Normal file
View File

@@ -0,0 +1,11 @@
function [Rot] = AxisAngle2Rot(axisAngle)
theta = norm(axisAngle, 2);
nx = axisAngle / theta;
nx = [ 0 -nx(3) nx(2);
nx(3) 0 -nx(1);
-nx(2) nx(1) 0 ];
Rot = eye(3) + sin(theta) * nx + (1-cos(theta))*nx^2;

11
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/Euler2Rot.m Normal file
View File

@@ -0,0 +1,11 @@
function [Rot] = Euler2Rot(euler)
rx = euler(1);
ry = euler(2);
rz = euler(3);
Rx = [1 0 0; 0 cos(rx) -sin(rx); 0 sin(rx) cos(rx)];
Ry = [cos(ry) 0 sin(ry); 0 1 0; -sin(ry) 0 cos(ry)];
Rz = [cos(rz) -sin(rz) 0; sin(rz) cos(rz) 0; 0 0 1];
Rot = Rx * Ry * Rz;

7
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/GetShape3D.m Normal file
View File

@@ -0,0 +1,7 @@
function [shape3D] = GetShape3D(M, V, p)
shape3D = M + V * p;
shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
end

20
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/GetShapeOrtho.m Normal file
View File

@@ -0,0 +1,20 @@
function [shape2D] = GetShapeOrtho(M, V, p, global_params)
% M - mean shape vector
% V - eigenvectors
% p - parameters of non-rigid shape
% V_exp
% p_exp
% global_params includes scale, euler rotation, translation,
% R - rotation matrix
% T - translation vector (tx, ty)
R = Euler2Rot(global_params(2:4));
T = [global_params(5:6); 0];
a = global_params(1);
shape3D = GetShape3D(M, V, p);
shape2D = bsxfun(@plus, a * R*shape3D', T);
shape2D = shape2D';
end

103
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/ProcrustesAnalysis.m Normal file
View File

@@ -0,0 +1,103 @@
function [normX, normY, meanShape, Transform] = ProcrustesAnalysis(x, y, options)
% Translate all elements to origin and scale to 1
normX = zeros(size(x));
normY = zeros(size(y));
for i = 1:size(x,1)
offsetX = mean(x(i,:));
offsetY = mean(y(i,:));
Transform.offsetX(i) = offsetX;
Transform.offsetY(i) = offsetY;
normX(i,:) = x(i,:) - offsetX;
normY(i,:) = y(i,:) - offsetY;
% Get the Frobenius norm, to scale the shapes to unit size
scale = norm([normX(i,:) normY(i,:)], 'fro');
Transform.scale(i) = scale;
normX(i,:) = normX(i,:)/scale;
normY(i,:) = normY(i,:)/scale;
end
% Rotate elements untill all of them have the same orientation
% the initial estimate of rotation would be the first element
% if change is less than 1% stop (shouldn't take more than 2 steps)
change = 0.1;
meanShape = [ normX(1,:); normY(1,:) ]';
Transform.Rotation = zeros(size(x,1),1);
for i = 1:30
% align all of the shapes to the mean shape
% remember all orientations to get the mean one
orientations = zeros(size(normX,1),1);
for j = 1:size(x,1)
% do SVD of mean * X'
currentShape = [ normX(j,:); normY(j,:) ]';
[U, ~, V] = svd( meanShape' * currentShape);
rot = V*U';
if(asin(rot(2,1)) > 0)
orientations(j) = real(acos(rot(1,1)));
else
orientations(j) = real(-acos(rot(1,1)));
end
Transform.Rotation(j) = Transform.Rotation(j) + orientations(j);
currentShape = currentShape * rot;
normX(j,:) = currentShape(:,1)';
normY(j,:) = currentShape(:,2)';
end
% recalculate the mean shape;
oldMean = meanShape;
meanShape = [mean(normX); mean(normY)]';
% rotate the mean shape to mean rotation
meanOrientation = mean(orientations);
% Do this only the first time
if(i==1)
rotM = [ cos(-meanOrientation) -sin(-meanOrientation); sin(-meanOrientation) cos(-meanOrientation) ];
meanShape = meanShape * rotM;
end
% scale mean shape to unit
meanScale = norm(meanShape, 'fro');
meanShape = meanShape*(1/meanScale);
% find frobenious norm
diff = norm(oldMean - meanShape, 'fro');
if(diff/norm(oldMean,'fro') < change)
break;
end
end
% transform to tangent space to preserve linearities
% get the scaling factors for each shape
if(options.TangentSpaceTransform)
scaling = [ normX normY ] * [ meanShape(:,1)' meanShape(:,2)']';
for i=1:size(x,1)
normX(i,:) = normX(i,:) * (1 / scaling(i));
normY(i,:) = normY(i,:) * (1 / scaling(i));
end
end

139
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/ProcrustesAnalysis3D.m Normal file
View File

@@ -0,0 +1,139 @@
function [ normX, normY, normZ, meanShape, Transform ] = ProcrustesAnalysis3D( x, y, z, tangentSpace, meanShape )
%PROCRUSTESANALYSIS3D Summary of this function goes here
% Detailed explanation goes here
meanProvided = false;
if(nargin > 4)
meanProvided = true;
end
% Translate all elements to origin
normX = zeros(size(x));
normY = zeros(size(y));
normZ = zeros(size(z));
for i = 1:size(x,1)
offsetX = mean(x(i,:));
offsetY = mean(y(i,:));
offsetZ = mean(z(i,:));
Transform.offsetX(i) = offsetX;
Transform.offsetY(i) = offsetY;
Transform.offsetZ(i) = offsetZ;
normX(i,:) = x(i,:) - offsetX;
normY(i,:) = y(i,:) - offsetY;
normZ(i,:) = z(i,:) - offsetZ;
end
% Rotate elements untill all of them have the same orientation
% the initial estimate of rotation would be the first element
% if change is less than 1% stop (shouldn't take more than 2 steps)
change = 0.1;
if(~meanProvided)
meanShape = [ mean(normX); mean(normY); mean(normZ) ]';
end
% scale all the shapes to mean shape
% Get the Frobenius norm, to scale the shapes to mean size (still want to
% retain mm)
meanScale = norm(meanShape, 'fro');
for i = 1:size(x,1)
scale = norm([normX(i,:) normY(i,:) normZ(i,:)], 'fro')/meanScale;
normX(i,:) = normX(i,:)/scale;
normY(i,:) = normY(i,:)/scale;
normZ(i,:) = normZ(i,:)/scale;
end
Transform.RotationX = zeros(size(x,1),1);
Transform.RotationY = zeros(size(x,1),1);
Transform.RotationZ = zeros(size(x,1),1);
for i = 1:30
% align all of the shapes to the mean shape
% remember all orientations to get the mean one (in euler angle form, pitch, yaw roll)
orientationsX = zeros(size(normX,1),1);
orientationsY = zeros(size(normX,1),1);
orientationsZ = zeros(size(normX,1),1);
for j = 1:size(x,1)
currentShape = [normX(j,:); normY(j,:); normZ(j,:)]';
% we want to align the current shape to the mean one
[ R, T ] = AlignShapesKabsch(currentShape, meanShape);
eulers = Rot2Euler(R);
orientationsX(j) = eulers(1);
orientationsY(j) = eulers(2);
orientationsZ(j) = eulers(3);
Transform.RotationX(j) = eulers(1);
Transform.RotationY(j) = eulers(2);
Transform.RotationZ(j) = eulers(3);
currentShape = R * currentShape';
normX(j,:) = currentShape(1,:);
normY(j,:) = currentShape(2,:);
normZ(j,:) = currentShape(3,:);
end
% recalculate the mean shape
% if(~meanProvided)
oldMean = meanShape;
meanShape = [mean(normX); mean(normY); mean(normZ)]';
meanScale = norm(meanShape, 'fro');
% end
for j = 1:size(x,1)
scale = norm([normX(j,:) normY(j,:) normZ(j,:)], 'fro')/meanScale;
normX(j,:) = normX(j,:)/scale;
normY(j,:) = normY(j,:)/scale;
normZ(j,:) = normZ(j,:)/scale;
end
if(i==1 && ~meanProvided)
% rotate the mean shape to mean rotation
meanOrientationX = mean(orientationsX);
meanOrientationY = mean(orientationsY);
meanOrientationZ = mean(orientationsZ);
R = Euler2Rot([meanOrientationX, meanOrientationY, meanOrientationZ]);
meanShape = (R * meanShape')';
end
% find frobenious norm
diff = norm(oldMean - meanShape, 'fro');
if(diff/norm(oldMean,'fro') < change)
break;
end
end
% transform to tangent space to preserve linearities
% get the scaling factors for each shape
if(tangentSpace)
[ normX, normY, normZ] = TangentSpaceTransform(normX, normY, normZ, meanShape);
end
end

11
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/Rot2AxisAngle.m Normal file
View File

@@ -0,0 +1,11 @@
function [ axisAngle ] = Rot2AxisAngle( Rot )
%ROT2AXISANGLE Summary of this function goes here
% Detailed explanation goes here
theta = acos((trace(Rot) - 1) / 2);
vec = 1.0/(2*sin(theta));
vec = vec * [Rot(3,2) - Rot(2,3), Rot(1,3) - Rot(3,1), Rot(2,1) - Rot(1,2)];
axisAngle = vec * theta;
end

12
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/Rot2Euler.m Normal file
View File

@@ -0,0 +1,12 @@
function [euler] = Rot2Euler(R)
q0 = sqrt( 1 + R(1,1) + R(2,2) + R(3,3) ) / 2;
q1 = (R(3,2) - R(2,3)) / (4*q0) ;
q2 = (R(1,3) - R(3,1)) / (4*q0) ;
q3 = (R(2,1) - R(1,2)) / (4*q0) ;
yaw = asin(2*(q0*q2 + q1*q3));
pitch= atan2(2*(q0*q1-q2*q3), q0*q0-q1*q1-q2*q2+q3*q3);
roll = atan2(2*(q0*q3-q1*q2), q0*q0+q1*q1-q2*q2-q3*q3);
euler = [pitch, yaw, roll];

17
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/TangentSpaceTransform.m Normal file
View File

@@ -0,0 +1,17 @@
function [ transformedX, transformedY, transformedZ ] = TangentSpaceTransform( x, y, z, meanShape )
%TANGENTSPACETRANSFORM Summary of this function goes here
% Detailed explanation goes here
scaling = [ x y z] * [ meanShape(:,1)' meanShape(:,2)' meanShape(:,3)']';
for i=1:size(x,1)
x(i,:) = x(i,:) * (1 / scaling(i));
y(i,:) = y(i,:) * (1 / scaling(i));
z(i,:) = z(i,:) * (1 / scaling(i));
end
transformedX = x * mean(scaling);
transformedY = y * mean(scaling);
transformedZ = z * mean(scaling);
end

345
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/fit_PDM_ortho_proj_to_2D.m Normal file
View File

@@ -0,0 +1,345 @@
function [ a, R, T, T3D, params, error, shapeOrtho ] = fit_PDM_ortho_proj_to_2D( M, E, V, shape2D, f, cx, cy)
%FITPDMTO2DSHAPE Summary of this function goes here
% Detailed explanation goes here
params = zeros(size(E));
hidden = false;
% if some of the points are unavailable modify M, V, and shape2D (can
% later infer the actual shape from this)
if(sum(shape2D(:)==0) > 0)
hidden = true;
% which indices to remove
inds_to_rem = shape2D(:,1) == 0 | shape2D(:,2) == 0;
shape2D = shape2D(~inds_to_rem,:);
inds_to_rem = repmat(inds_to_rem, 3, 1);
M_old = M;
V_old = V;
M = M(~inds_to_rem);
V = V(~inds_to_rem,:);
end
num_points = numel(M) / 3;
m = reshape(M, num_points, 3)';
width_model = max(m(1,:)) - min(m(1,:));
height_model = max(m(2,:)) - min(m(2,:));
bounding_box = [min(shape2D(:,1)), min(shape2D(:,2)),...
max(shape2D(:,1)), max(shape2D(:,2))];
a = (((bounding_box(3) - bounding_box(1)) / width_model) + ((bounding_box(4) - bounding_box(2))/ height_model)) / 2;
tx = (bounding_box(3) + bounding_box(1))/2;
ty = (bounding_box(4) + bounding_box(2))/2;
% correct it so that the bounding box is just around the minimum
% and maximum point in the initialised face
tx = tx - a*(min(m(1,:)) + max(m(1,:)))/2;
ty = ty - a*(min(m(2,:)) + max(m(2,:)))/2;
R = eye(3);
T = [tx; ty];
currShape = getShapeOrtho(M, V, params, R, T, a);
currError = getRMSerror(currShape, shape2D);
reg_rigid = zeros(6,1);
regFactor = 20;
regularisations = [reg_rigid; regFactor ./ E]; % the above version, however, does not perform as well
regularisations = diag(regularisations)*diag(regularisations);
red_in_a_row = 0;
for i=1:1000
shape3D = M + V * params;
shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
% Now find the current residual error
currShape = a * R(1:2,:)*shape3D' + repmat(T, 1, numel(M)/3);
currShape = currShape';
error_res = shape2D - currShape;
eul = Rot2Euler(R);
p_global = [a; eul'; T];
% get the Jacobians
J = CalcJacobian(M, V, params, p_global);
% RLMS style update
p_delta = (J'*J + regularisations) \ (J'*error_res(:) - regularisations*[p_global;params]);
[params, p_global] = CalcReferenceUpdate(p_delta, params, p_global);
a = p_global(1);
R = Euler2Rot(p_global(2:4));
T = p_global(5:6);
shape3D = M + V * params;
shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
currShape = a * R(1:2,:)*shape3D' + repmat(T, 1, numel(M)/3);
currShape = currShape';
error = getRMSerror(currShape, shape2D);
if(0.999 * currError < error)
red_in_a_row = red_in_a_row + 1;
if(red_in_a_row == 5)
break;
end
end
currError = error;
end
if(hidden)
shapeOrtho = getShapeOrtho(M_old, V_old, params, R, T, a);
else
shapeOrtho = currShape;
end
if(nargin == 7)
Zavg = f / a;
Xavg = (T(1) - cx) / a;
Yavg = (T(2) - cy) / a;
T3D = [Xavg;Yavg;Zavg];
else
T3D = [0;0;0];
end
end
function [shape2D] = getShapeOrtho(M, V, p, R, T, a)
% M - mean shape vector
% V - eigenvectors
% p - parameters of non-rigid shape
% R - rotation matrix
% T - translation vector (tx, ty)
shape3D = getShape3D(M, V, p);
shape2D = a * R(1:2,:)*shape3D' + repmat(T, 1, numel(M)/3);
shape2D = shape2D';
end
function [shape2D] = getShapeOrthoFull(M, V, p, R, T, a)
% M - mean shape vector
% V - eigenvectors
% p - parameters of non-rigid shape
% R - rotation matrix
% T - translation vector (tx, ty)
T = [T; 0];
shape3D = getShape3D(M, V, p);
shape2D = a * R*shape3D' + repmat(T, 1, numel(M)/3);
shape2D = shape2D';
end
function [shape3D] = getShape3D(M, V, params)
shape3D = M + V * params;
shape3D = reshape(shape3D, numel(shape3D) / 3, 3);
end
function [error] = getRMSerror(shape2Dv1, shape2Dv2)
error = sqrt(mean(reshape(shape2Dv1 - shape2Dv2, numel(shape2Dv1), 1).^2));
end
% This calculates the combined rigid with non-rigid Jacobian
function J = CalcJacobian(M, V, p, p_global)
n = size(M, 1)/3;
non_rigid_modes = size(V,2);
J = zeros(n*2, 6 + non_rigid_modes);
% now the layour is
% ---------- Rigid part -------------------|----Non rigid part--------|
% dx_1/ds, dx_1/dr1, ... dx_1/dtx, dx_1/dty dx_1/dp_1 ... dx_1/dp_m
% dx_2/ds, dx_2/dr1, ... dx_2/dtx, dx_2/dty dx_2/dp_1 ... dx_2/dp_m
% ...
% dx_n/ds, dx_n/dr1, ... dx_n/dtx, dx_n/dty dx_n/dp_1 ... dx_n/dp_m
% dy_1/ds, dy_1/dr1, ... dy_1/dtx, dy_1/dty dy_1/dp_1 ... dy_1/dp_m
% ...
% dy_n/ds, dy_n/dr1, ... dy_n/dtx, dy_n/dty dy_n/dp_1 ... dy_n/dp_m
% getting the rigid part
J(:,1:6) = CalcRigidJacobian(M, V, p, p_global);
% constructing the non-rigid part
R = Euler2Rot(p_global(2:4));
s = p_global(1);
% 'rotate' and 'scale' the principal components
% First reshape to 3D
V_X = V(1:n,:);
V_Y = V(n+1:2*n,:);
V_Z = V(2*n+1:end,:);
J_x_non_rigid = s*(R(1,1)*V_X + R(1,2)*V_Y + R(1,3)*V_Z);
J_y_non_rigid = s*(R(2,1)*V_X + R(2,2)*V_Y + R(2,3)*V_Z);
J(1:n, 7:end) = J_x_non_rigid;
J(n+1:end, 7:end) = J_y_non_rigid;
end
function J = CalcRigidJacobian(M, V, p, p_global)
n = size(M, 1)/3;
% Get the current 3D shape (not affected by global transform, as this
% is how the Jacobian was derived (for derivation please see
% ../derivations/orthoJacobian
shape3D = GetShape3D(M, V, p);
% Get the rotation matrix corresponding to current global orientation
R = Euler2Rot(p_global(2:4));
s = p_global(1);
% Rigid Jacobian is laid out as follows
% dx_1/ds, dx_1/dr1, dx_1/dr2, dx_1/dr3, dx_1/dtx, dx_1/dty
% dx_2/ds, dx_2/dr1, dx_2/dr2, dx_2/dr3, dx_2/dtx, dx_2/dty
% ...
% dx_n/ds, dx_n/dr1, dx_n/dr2, dx_n/dr3, dx_n/dtx, dx_n/dty
% dy_1/ds, dy_1/dr1, dy_1/dr2, dy_1/dr3, dy_1/dtx, dy_1/dty
% ...
% dy_n/ds, dy_n/dr1, dy_n/dr2, dy_n/dr3, dy_n/dtx, dy_n/dty
J = zeros(n*2, 6);
% dx/ds = X * r11 + Y * r12 + Z * r13
% dx/dr1 = s*(r13 * Y - r12 * Z)
% dx/dr2 = -s*(r13 * X - r11 * Z)
% dx/dr3 = s*(r12 * X - r11 * Y)
% dx/dtx = 1
% dx/dty = 0
% dy/ds = X * r21 + Y * r22 + Z * r23
% dy/dr1 = s * (r23 * Y - r22 * Z)
% dy/dr2 = -s * (r23 * X - r21 * Z)
% dy/dr3 = s * (r22 * X - r21 * Y)
% dy/dtx = 0
% dy/dty = 1
% set the Jacobian for x's
% with respect to scaling factor
J(1:n,1) = shape3D * R(1,:)';
% with respect to angular rotation around x, y, and z axes
% Change of x with respect to change in axis angle rotation
dxdR = [ 0, R(1,3), -R(1,2);
-R(1,3), 0, R(1,1);
R(1,2), -R(1,1), 0];
J(1:n,2:4) = s*(dxdR * shape3D')';
% with respect to translation
J(1:n,5) = 1;
J(1:n,6) = 0;
% set the Jacobian for y's
% with respect to scaling factor
J(n+1:end,1) = shape3D * R(2,:)';
% with respect to angular rotation around x, y, and z axes
% Change of y with respect to change in axis angle rotation
dydR = [ 0, R(2,3), -R(2,2);
-R(2,3), 0, R(2,1);
R(2,2), -R(2,1), 0];
J(n+1:end,2:4) = s*(dydR * shape3D')';
% with respect to translation
J(n+1:end,5) = 0;
J(n+1:end,6) = 1;
end
% This updates the parameters based on the updates from the RLMS
function [non_rigid, rigid] = CalcReferenceUpdate(params_delta, current_non_rigid, current_global)
rigid = zeros(6, 1);
% Same goes for scaling and translation parameters
rigid(1) = current_global(1) + params_delta(1);
rigid(5) = current_global(5) + params_delta(5);
rigid(6) = current_global(6) + params_delta(6);
% for rotation however, we want to make sure that the rotation matrix
% approximation we have
% R' = [1, -wz, wy
% wz, 1, -wx
% -wy, wx, 1]
% is a legal rotation matrix, and then we combine it with current
% rotation (through matrix multiplication) to acquire the new rotation
R = Euler2Rot(current_global(2:4));
wx = params_delta(2);
wy = params_delta(3);
wz = params_delta(4);
R_delta = [1, -wz, wy;
wz, 1, -wx;
-wy, wx, 1];
% Make sure R_delta is orthonormal
R_delta = OrthonormaliseRotation(R_delta);
% Combine rotations
R_final = R * R_delta;
% Extract euler angle
euler = Rot2Euler(R_final);
rigid(2:4) = euler;
if(length(params_delta) > 6)
% non-rigid parameters can just be added together
non_rigid = params_delta(7:end) + current_non_rigid;
else
non_rigid = current_non_rigid;
end
end
function R_ortho = OrthonormaliseRotation(R)
% U * V' is basically what we want, as it's guaranteed to be
% orthonormal
[U, ~, V] = svd(R);
% We also want to make sure no reflection happened
% get the orthogonal matrix from the initial rotation matrix
X = U*V';
% This makes sure that the handedness is preserved and no reflection happened
% by making sure the determinant is 1 and not -1
W = eye(3);
W(3,3) = det(X);
R_ortho = U*W*V';
end

BIN
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/pdm_29_cofw_aligned.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/pdm_68_aligned_wild.mat Normal file
View File

Binary file not shown.

BIN
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/pdm_68_multi_pie.mat Normal file
View File

Binary file not shown.

16
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/PDM_helpers/writeMatrix.m Normal file
View File

@@ -0,0 +1,16 @@
% for easier readibility write them row by row
function writeMatrix(fileID, M, type)
fprintf(fileID, '%d\r\n', size(M,1));
fprintf(fileID, '%d\r\n', size(M,2));
fprintf(fileID, '%d\r\n', type);
for i=1:size(M,1)
if(type == 4 || type == 0)
fprintf(fileID, '%d ', M(i,:));
else
fprintf(fileID, '%.9f ', M(i,:));
end
fprintf(fileID, '\r\n');
end
end

195
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_COFW.m Normal file
View File

@@ -0,0 +1,195 @@
% Convert the training images into a suitable format
function Prepare_data_COFW()
% replace with folder where you downloaded and extracted the 300-W challenge data
data_root = '../../../../CLM-framework/matlab_version/occlusion_experiments/data/';
PrepareTrainingWild(data_root, 0.25);
PrepareTrainingWild(data_root, 0.35);
PrepareTrainingWild(data_root, 0.5);
end
function PrepareTrainingWild( data_root, training_scale )
%PREPARETRAININGIMAGEMPIE Summary of this function goes here
% Detailed explanation goes here
% img size
imgSize = [400, 400] * training_scale;
%%
addpath('PDM_helpers/');
load 'PDM_helpers/pdm_29_cofw_aligned.mat';
output_location = '../prepared_data/cofw_';
output_location = [output_location num2str(training_scale,3)];
num_landmarks = 29;
% Use mirror images to provide extra training data
mirror_inds = [1,2; 3,4; 5,7; 6,8; 9,10; 11,12; 13,15; 14,16; 17,18; 19,20; 23,24];
% The centres of views we want to extract
centres_all = [ 0, 0, 0 ];
num_centers = size(centres_all, 1);
counter_colour = zeros(num_centers,1);
load([data_root, '/COFW_train.mat']);
% go through all images, see which centres they match and then add to
% the appropriate bin
num_imgs = numel(IsTr);
scales = zeros(num_imgs, 1);
views = zeros(num_imgs, 1);
for lbl=1:num_imgs
Msm = M;
Vsm = V;
labels = reshape(phisTr(lbl,1:num_landmarks*2), num_landmarks, 2);
% Find the best PDM parameters given the 2D labels
[ a, R] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
eul = Rot2Euler(R);
eul = eul * 180 / pi;
% find the closest view
[~, view] = min(sum(abs(centres_all - repmat(eul, num_centers, 1)),2));
counter_colour(view) = counter_colour(view) + 1;
scales(lbl) = a;
views(lbl) = view;
end
% preallocate data
allExamplesColourAllViews = cell(size(centres_all,1),1);
landmarkLocationsAllViews = cell(size(centres_all,1),1);
occlusionsAllViews = cell(size(centres_all,1),1);
for r=1:size(centres_all,1)
allExamplesColourAllViews{r} = zeros(counter_colour(r), imgSize(1), imgSize(2));
landmarkLocationsAllViews{r} = zeros(counter_colour(r), num_landmarks, 2);
occlusionsAllViews{r} = zeros(counter_colour(r), num_landmarks);
actual_imgs_used_all_views{r} = cell(counter_colour(r), 1);
end
counter_colour = zeros(num_centers,1);
% go through all images and add to corresponding container
for lbl=1:num_imgs
labels = reshape(phisTr(lbl,1:num_landmarks*2), num_landmarks, 2);
occlusions = phisTr(lbl,(num_landmarks*2+1):end);
imgCol = IsTr{lbl};
if(size(imgCol,3) == 3)
imgCol = rgb2gray(imgCol);
end
% resize the image to desired scale
scalingFactor = training_scale / scales(lbl);
resizeColImage = imresize(imgCol, scalingFactor, 'bilinear');
labels = labels * scalingFactor;
% we want to crop out the image now
meanX = round(mean(labels(:,1)));
meanY = round(mean(labels(:,2)));
startX = round(meanX-imgSize(1)/2);
startY = round(meanY-imgSize(2)/2);
if(startX < 1)
startX = 1;
end
if(startY < 1)
startY = 1;
end
endX = startX + imgSize(1) - 1;
endY = startY + imgSize(2) - 1;
if(endX > size(resizeColImage,2))
resizeColImage = cat(2, resizeColImage, zeros(size(resizeColImage,1), endX - size(resizeColImage,2)));
end
if(endY > size(resizeColImage,1))
resizeColImage = cat(1, resizeColImage, zeros(endY - size(resizeColImage,1), size(resizeColImage,2)));
end
resizeColImage = resizeColImage(startY:endY,startX:endX);
labels(:,1) = labels(:,1) - startX + 1;
labels(:,2) = labels(:,2) - startY + 1;
counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = resizeColImage;
landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = labels;
occlusionsAllViews{views(lbl)}(counter_colour(views(lbl)),:) = occlusions;
actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = 'placeholder';
if(mod(lbl, 100) == 0)
fprintf('%d/%d done\n', lbl, num_imgs);
end
end
% write out the training data for each view
for r=1:size(centres_all,1)
mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
% if the mirrored view already added no need to do it again
% so if (0,-20,0) is done no need to compute (0,20,0)
if(mirrorIdx < r)
continue
end
mirrorImgs = allExamplesColourAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
mirrorLbls = landmarkLocationsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
mirrorOccls = occlusionsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:);
for i=1:size(mirrorImgs,1)
flippedImg = fliplr(squeeze(mirrorImgs(i,:,:)));
flippedLbls = squeeze(mirrorLbls(i,:,:));
flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1);
tmp1 = flippedLbls(mirror_inds(:,1),:);
tmp2 = flippedLbls(mirror_inds(:,2),:);
flippedLbls(mirror_inds(:,2),:) = tmp1;
flippedLbls(mirror_inds(:,1),:) = tmp2;
mirrorImgs(i,:,:) = flippedImg;
mirrorLbls(i,:,:) = flippedLbls;
tmp1 = mirrorOccls(i,mirror_inds(:,1));
tmp2 = mirrorOccls(i,mirror_inds(:,2));
mirrorOccls(i,mirror_inds(:,1)) = tmp2;
mirrorOccls(i,mirror_inds(:,2)) = tmp1;
end
all_images = cat(1, allExamplesColourAllViews{r}(1:counter_colour(r),:,:), mirrorImgs);
all_images = uint8(all_images);
landmark_locations = cat(1, landmarkLocationsAllViews{r}(1:counter_colour(r),:,:), mirrorLbls);
occlusions = cat(1, occlusionsAllViews{r}(1:counter_colour(r),:), mirrorOccls);
actual_imgs_used = actual_imgs_used_all_views{r};
actual_imgs_used = cat(1, actual_imgs_used_all_views{r}, actual_imgs_used_all_views{mirrorIdx});
centres = centres_all(r,:);
visiIndex = ones(1,num_landmarks);
save([output_location '_' num2str(r) '.mat'], 'all_images', 'occlusions', 'landmark_locations', 'training_scale', 'actual_imgs_used', 'centres', 'visiIndex', '-v7.3');
end
end

499
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_Multi_PIE_all.m Normal file
View File

@@ -0,0 +1,499 @@
function Prepare_data_Multi_PIE_all()
% This bit collects all of the multi-pie labels into a single structure for
% easy access
labels_root = 'C:\Users\tbaltrus\Dropbox\AAM\test data\MultiPI_AAM/';
% The location of the Multi-PIE data folder
multi_pie_root = 'D:\MultiPIE/Image_Data/';
multi_pie_labels = CollectMultiPieLabels(labels_root, multi_pie_root);
%%
% Make sure same images are used across scales
rng(0);
ExtractTrainingMultiPIE(0.25, multi_pie_labels);
rng(0);
ExtractTrainingMultiPIE(0.35, multi_pie_labels);
rng(0);
ExtractTrainingMultiPIE(0.5, multi_pie_labels);
rng(0);
ExtractTrainingMultiPIE(1.0, multi_pie_labels);
end
% Now extract the relevant information
function [multi_pie_labels] = CollectMultiPieLabels(labels_root, multi_pie_root)
multi_pie_labels = struct;
currentLabel = 0;
left_to_frontal_map = [1, 28;
2, 29;
3, 30;
4, 31;
5, 34;
6, 36;
7, 27;
8, 26;
9, 25;
10,24;
11,46;
12,45;
13,44;
14,48;
15,47;
16,52;
17,53;
18,54;
19,55;
20,56;
21,57;
22,58;
23,63;
24,64;
26,66;
27,67;
30,9;
31,10;
32,11;
33,12;
34,13;
35,14;
36,15;
37,16;
38,17;];
right_to_frontal_map = [1, 28;
2, 29;
3, 30;
4, 31;
5, 34;
6, 32;
7 ,18;
8 ,19;
9 ,20;
10,21;
11,37;
12,38;
13,39;
14,41;
15,42;
16,52;
17,51;
18,50;
19,49;
20,60;
21,59;
22,58;
23,63;
24,62;
26,68;
27,67;
30, 9;
31, 8;
32, 7;
33, 6;
34, 5;
35, 4;
36, 3;
37, 2;
38, 1];
for i=1:4
labelsDir = sprintf('%sSession%d/', labels_root, i);
labels = dir([labelsDir '/*.mat']);
double_label = false;
for j = 1:numel(labels)
if(double_label)
double_label = false;
continue;
end
load([labelsDir labels(j).name]);
if(size(pts,1) ~= 68 && size(pts,1) ~= 39)
continue;
end
userID = labels(j).name(1:3);
recID = labels(j).name(8:9);
camID = labels(j).name(11:12);
camID2 = labels(j).name(13);
viewID = labels(j).name(15:16);
% doubling labels seem to be odd/incorrect
if(j < numel(labels))
userIDN = labels(j+1).name(1:3);
recIDN = labels(j+1).name(8:9);
camIDN = labels(j+1).name(11:12);
camID2N = labels(j+1).name(13);
if(strcmp(userIDN, userID) && strcmp(recID, recIDN) && strcmp(camIDN, camID) && strcmp(camID2N, camID2))
double_label = true;
continue;
end
end
% camera id 08 is from a very different perspective
if(strcmp(camID, '08') && strcmp(camID2, '1'))
continue;
end
currentLabel = currentLabel + 1;
multi_pie_labels(currentLabel).label_file = [labelsDir labels(j).name];
multi_pie_labels(currentLabel).user_ID = userID;
multi_pie_labels(currentLabel).rec_ID = recID;
multi_pie_labels(currentLabel).cam_ID = camID;
multi_pie_labels(currentLabel).cam_ID2 = camID2;
multiPieImageDir = sprintf('%s/session0%d/multiview/%s/%s/%s_%s/', multi_pie_root, i,userID,recID,camID,camID2);
multiPieImgs = dir([multiPieImageDir '*.png']);
multi_pie_labels(currentLabel).img_dir = multiPieImageDir;
% multi_pie_labels(currentLabel).img_loc = [labels(j+1).name(1:end-7), '.png'];
actualLabel = dir([multiPieImageDir '/*' viewID '.png']);
multi_pie_labels(currentLabel).img_locs = {multiPieImgs.name};
multi_pie_labels(currentLabel).actual_img = actualLabel.name;
landmark_labels = zeros(68,2);
% This is a profile image
if(size(pts,1) == 39)
% Determine if left or right
if(pts(4,1) < pts(39,1))
multi_pie_labels(currentLabel).type = 'profile_left';
landmark_labels(left_to_frontal_map(:,2),:) = pts(left_to_frontal_map(:,1),:);
else
multi_pie_labels(currentLabel).type = 'profile_right';
landmark_labels(right_to_frontal_map(:,2),:) = pts(right_to_frontal_map(:,1),:);
end
else
landmark_labels = pts;
end
multi_pie_labels(currentLabel).landmark_labels = landmark_labels;
end
end
end
function ExtractTrainingMultiPIE( training_scale, multi_pie_labels)
%PREPARETRAININGIMAGEMPIE This function collects a subset of Multi-PIE
%images at different expressions and lighting conditions to store it in a
%format suitable for training patch experts
% Detailed explanation goes here
img_size = [400, 400] * training_scale;
%%
addpath('PDM_helpers/');
load 'PDM_helpers/pdm_68_aligned_wild.mat';
output_location = '../prepared_data/mpie_';
output_location = [output_location num2str(training_scale,3)];
num_landmarks = 68;
% Use mirror images to provide extra training data
mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
% The centres of views we want to extract
centres_all = [ 0, 0, 0
0, -20, 0
0, -45, 0
0, -70, 0
0, 20, 0
0, 45, 0
0, 70, 0
];
num_centers = size(centres_all, 1);
counter_colour = zeros(num_centers,1);
% read in all of the labels, together with names of images used
landmark_labels = zeros(68,2,numel(multi_pie_labels));
img_locations = cell(numel(multi_pie_labels),1);
extra_locations = cell(numel(multi_pie_labels),1);
img_dirs = cell(numel(multi_pie_labels),1);
for i=1:numel(multi_pie_labels)
img_locations{i} = [multi_pie_labels(i).img_dir, multi_pie_labels(i).actual_img];
extra_locations{i} = multi_pie_labels(i).img_locs;
img_dirs{i} = multi_pie_labels(i).img_dir;
landmark_labels(:,:,i) = multi_pie_labels(i).landmark_labels;
end
% go through all images, see which centres they match and then add to
% the appropriate bin
num_imgs = size(landmark_labels,3);
scales = zeros(num_imgs, 1);
views = zeros(num_imgs, 1);
for lbl=1:num_imgs
Msm = M;
Vsm = V;
labels = landmark_labels(:,:,lbl);
% Find the best PDM parameters given the 2D labels
[ a,R ] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
eul = Rot2Euler(R);
eul_orig = eul * 180 / pi;
if(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '24_0'))
eul = [0, -90, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '01_0'))
eul = [0, -75, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '20_0'))
eul = [0, -60, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '19_0'))
eul = [0, -45, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '04_1'))
eul = [0, -30, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '05_0'))
eul = [0, -15, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '05_1'))
eul = [0, 0, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '14_0'))
eul = [0, 15, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '13_0'))
eul = [0, 30, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '08_0'))
eul = [0, 45, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '09_0'))
eul = [0, 60, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '12_0'))
eul = [0, 75, 0];
elseif(strcmp([multi_pie_labels(lbl).cam_ID,'_', multi_pie_labels(lbl).cam_ID2], '11_0'))
eul = [0, 90, 0];
end
% find the closest view (also mirror img?)
[~, view] = min(sum(abs(centres_all - repmat(eul, num_centers, 1)),2));
counter_colour(view) = counter_colour(view) + 1;
scales(lbl) = a;
views(lbl) = view;
end
% preallocate data
allExamplesColourAllViews = cell(size(centres_all,1),1);
landmarkLocationsAllViews = cell(size(centres_all,1),1);
% if we don't have enough labelled original images add ones from diff
% lighting conditions (that might not be perfectly labelled, but they
% are better than fewer images)
images_aim = 8000;
extra_factors = ones(size(counter_colour));
for r=1:size(centres_all,1)
% see if extra is needed
mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
count = counter_colour(r) + counter_colour(mirrorIdx);
if(count < images_aim)
extra_factors(r) = images_aim / count;
end
end
for r=1:size(centres_all,1)
counter_colour(r) = round(counter_colour(r) * extra_factors(r));
allExamplesColourAllViews{r} = uint8(zeros(counter_colour(r), img_size(1), img_size(2)));
landmarkLocationsAllViews{r} = zeros(counter_colour(r), num_landmarks, 2);
actual_imgs_used_all_views{r} = cell(counter_colour(r), 1);
end
counter_colour = zeros(num_centers,1);
% The shape fitting is performed in the reference frame of the
% patch training scale
refGlobal = [training_scale, 0, 0, 0, 0, 0]';
% go through all images and add to corresponding container
for lbl=1:num_imgs
labels = landmark_labels(:,:,lbl);
occluded = labels(:,1) == 0;
% Convert the labels to matlab format (we expect 1,1 to represent
% the center of the top left pixel)
labels = labels + 1;
labels(occluded,:) = 0;
imgCol = imread(img_locations{lbl});
if(size(imgCol,3) == 3)
imgCol = rgb2gray(imgCol);
end
% the reference shape
[ ~, ~, ~,~, local_params] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
refShape = GetShapeOrtho(M, Vsm, local_params, refGlobal);
% Create transform using a slightly modified version of Kabsch that
% takes scaling into account as well, in essence we get a
% similarity transform from current estimate to reference shape
[A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(labels(~occluded,:), refShape(~occluded,1:2));
T_img2ref = T_img2ref + [img_size(1)/2, img_size(2)/2];
% Create a transform, from shape in image to reference shape
T = affine2d([A_img2ref';T_img2ref]);
% transform the current shape to the reference one
shape2D_in_ref = bsxfun(@plus, (A_img2ref * labels')', T_img2ref);
% warp the image
resizeColImage = imwarp(imgCol, T, 'linear', 'OutputView', imref2d(img_size));
shape2D_in_ref(occluded,1) = 0;
shape2D_in_ref(occluded,2) = 0;
counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = resizeColImage;
landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = shape2D_in_ref;
actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = img_locations{lbl};
% Here to add extra images if missing and not filled yet
if(extra_factors(views(lbl)) > 1 && counter_colour(views(lbl)) < numel(actual_imgs_used_all_views{views(lbl)}))
factor = extra_factors(views(lbl));
% pick if to use this image or not
if(randi(100) > (factor - floor(factor)) * 100)
factor = floor(factor);
else
factor = ceil(factor);
end
while(factor > 1)
lighting_id = randi(20);
[~, img_orig, ext] = fileparts(img_locations{lbl});
img_orig = [img_orig, ext];
if( strcmp(img_orig, extra_locations{lbl}{lighting_id}))
lighting_id = lighting_id + 1;
if(lighting_id > 20)
lighting_id = 1;
end
end
img_loc = [img_dirs{lbl}, extra_locations{lbl}{lighting_id}];
imgCol = imread(img_loc);
if(size(imgCol,3) == 3)
imgCol = rgb2gray(imgCol);
end
% resize the image to desired scale
resizeColImage = imwarp(imgCol, T, 'linear', 'OutputView', imref2d(img_size));
counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = resizeColImage;
landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = shape2D_in_ref;
actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = img_loc;
factor = factor - 1;
end
end
% Make sure same one not added as well
if(mod(lbl, 100) == 0)
fprintf('%d/%d done\n', lbl, num_imgs);
end
end
% write out the training data for each view
for r=1:size(centres_all,1)
mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
% if the mirrored view already added no need to do it again
% so if (0,-20,0) is done no need to compute (0,20,0)
if(mirrorIdx < r)
continue
end
% Cap to two thousand images for space reasons, and because more
% wouldn't actually be used
max_images = 2000;
mirrorImgs = allExamplesColourAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
mirrorLbls = landmarkLocationsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
for i=1:size(mirrorImgs,1)
flippedImg = fliplr(squeeze(mirrorImgs(i,:,:)));
flippedLbls = squeeze(mirrorLbls(i,:,:));
flippedLbls(:,1) = img_size(1) - flippedLbls(:,1) + 1;
tmp1 = flippedLbls(mirror_inds(:,1),:);
tmp2 = flippedLbls(mirror_inds(:,2),:);
flippedLbls(mirror_inds(:,2),:) = tmp1;
flippedLbls(mirror_inds(:,1),:) = tmp2;
mirrorImgs(i,:,:) = flippedImg;
mirrorLbls(i,:,:) = flippedLbls;
% Ensure that after mirroring invalid (occluded) feature points
% are set to 0 in both x and y dims
mirrorLbls(i,flippedLbls(:,2)==0,1) = 0;
end
all_images = cat(1, allExamplesColourAllViews{r}(1:counter_colour(r),:,:), mirrorImgs);
landmark_locations = cat(1, landmarkLocationsAllViews{r}(1:counter_colour(r),:,:), mirrorLbls);
actual_imgs_used = cat(1, actual_imgs_used_all_views{r}(1:counter_colour(r)), actual_imgs_used_all_views{mirrorIdx}(1:counter_colour(mirrorIdx)));
centres = centres_all(r,:);
% identify the visibility of a point
num_visible = sum(landmark_locations(:,:,1)~=0);
visible_max = max(num_visible);
visiIndex = ones(1,68);
visiIndex(num_visible < 0.5*visible_max) = 0;
if(size(all_images,1) > max_images)
im_to_select = randperm(size(all_images,1));
im_to_select = im_to_select(1:max_images);
all_images = all_images(im_to_select,:,:);
landmark_locations = landmark_locations(im_to_select,:,:);
actual_imgs_used = actual_imgs_used(im_to_select);
end
save([output_location '_' num2str(r) '.mat'], 'all_images', 'landmark_locations', 'training_scale', 'centres', 'actual_imgs_used', 'visiIndex', '-v7.3');
end
end

48
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_combined_all.m Normal file
View File

@@ -0,0 +1,48 @@
function Prepare_data_combined_all()
% Read in in-the-wild data and Multi-PIE data and concatenate them
root = '../prepared_data/';
Collect_combined_data(root, '0.25');
Collect_combined_data(root, '0.35');
Collect_combined_data(root, '0.5');
Collect_combined_data(root, '1');
end
function Collect_combined_data(root, scale)
wild_locs = dir(sprintf('%s/wild_%s*.mat', root, scale));
mpie_locs = dir(sprintf('%s/mpie_%s*.mat', root, scale));
% For a particular loaded m_pie check if an appropriate in-the-wild
% exists
for i=1:numel(mpie_locs)
load([root, mpie_locs(i).name]);
imgs_used_pie = actual_imgs_used;
samples_pie = all_images;
centres_pie = centres;
landmark_locations_pie = landmark_locations;
wild_to_use = -1;
for j=1:numel(wild_locs)
load([root, wild_locs(j).name], 'centres');
if(isequal(centres_pie, centres))
wild_to_use = j;
end
end
% Reset the centres
centres = centres_pie;
if(wild_to_use ~= -1)
load([root, wild_locs(wild_to_use).name]);
actual_imgs_used = cat(1, actual_imgs_used, imgs_used_pie);
all_images = cat(1, all_images, samples_pie);
landmark_locations = cat(1, landmark_locations, landmark_locations_pie);
end
save(sprintf('%s/combined_%s_%d.mat', root, scale, i), 'actual_imgs_used', 'all_images', 'centres', 'landmark_locations', 'training_scale', 'visiIndex');
end
end

198
pkg/OpenFace/model_training/ce-clm_training/patch_generation/data_preparation/scripts/Prepare_data_wild_all.m Normal file
View File

@@ -0,0 +1,198 @@
% Convert the training images into a suitable format
function Prepare_data_wild_all()
% replace with folder where you downloaded and extracted the 300-W challenge data
data_root = 'C:\Users\tbaltrus\Dropbox\AAM/test data/';
PrepareTrainingWild(data_root, 0.25);
PrepareTrainingWild(data_root, 0.35);
PrepareTrainingWild(data_root, 0.5);
PrepareTrainingWild(data_root, 1.0);
end
function PrepareTrainingWild( data_root, training_scale )
%PREPARETRAININGIMAGEMPIE Summary of this function goes here
% Detailed explanation goes here
% img size
imgSize = [400, 400] * training_scale;
%%
addpath('PDM_helpers/');
load 'PDM_helpers/pdm_68_aligned_wild.mat';
output_location = '../prepared_data/wild_';
output_location = [output_location num2str(training_scale,3)];
num_landmarks = 68;
% Use mirror images to provide extra training data
mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
% The centres of views we want to extract
centres_all = [ 0, 0, 0
0, -20, 0
0, 20, 0
];
num_centers = size(centres_all, 1);
counter_colour = zeros(num_centers,1);
% use only 2 of the subsets (others used for testing)
% You just need to provide the location in your system
dataset_locs = { [data_root, '/helen/trainset/'];
[data_root, '/lfpw/trainset/']};
landmark_labels = [];
img_locations = {};
% read in all of the labels, together with names of images used
for i=1:numel(dataset_locs)
curr_labels = dir([dataset_locs{i} '\*.pts']);
imgs = dir([dataset_locs{i} '\*.jpg']);
if(isempty(imgs))
imgs = dir([dataset_locs{i} '\*.png']);
end
for p=1:numel(curr_labels)
landmarks = dlmread([dataset_locs{i}, curr_labels(p).name], ' ', [3,0,68+2,1]);
landmark_labels = cat(3, landmark_labels, landmarks);
img_locations = cat(1, img_locations, [dataset_locs{i} imgs(p).name]);
end
end
% go through all images, see which centres they match and then add to
% the appropriate bin
num_imgs = size(landmark_labels,3);
scales = zeros(num_imgs, 1);
views = zeros(num_imgs, 1);
for lbl=1:num_imgs
Msm = M;
Vsm = V;
labels = landmark_labels(:,:,lbl);
% Find the best PDM parameters given the 2D labels
[ a, R] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
eul = Rot2Euler(R);
eul = eul * 180 / pi;
% find the closest view
[~, view] = min(sum(abs(centres_all - repmat(eul, num_centers, 1)),2));
counter_colour(view) = counter_colour(view) + 1;
scales(lbl) = a;
views(lbl) = view;
end
% preallocate data
allExamplesColourAllViews = cell(size(centres_all,1),1);
landmarkLocationsAllViews = cell(size(centres_all,1),1);
for r=1:size(centres_all,1)
allExamplesColourAllViews{r} = uint8(zeros(counter_colour(r), imgSize(1), imgSize(2)));
landmarkLocationsAllViews{r} = zeros(counter_colour(r), num_landmarks, 2);
actual_imgs_used_all_views{r} = cell(counter_colour(r), 1);
end
counter_colour = zeros(num_centers,1);
% The shape fitting is performed in the reference frame of the
% patch training scale
refGlobal = [training_scale, 0, 0, 0, 0, 0]';
% go through all images and add to corresponding container
for lbl=1:num_imgs
% shift the pixels to be centered on pixel as opposed to top left
labels = landmark_labels(:,:,lbl) - 0.5;
imgCol = imread(img_locations{lbl});
if(size(imgCol,3) == 3)
imgCol = rgb2gray(imgCol);
end
% the reference shape
[ ~, ~, ~,~, local_params] = fit_PDM_ortho_proj_to_2D( Msm, E, Vsm, labels);
refShape = GetShapeOrtho(M, Vsm, local_params, refGlobal);
% Create transform using a slightly modified version of Kabsch that
% takes scaling into account as well, in essence we get a
% similarity transform from current estimate to reference shape
[A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(labels, refShape(:,1:2));
T_img2ref = T_img2ref + [imgSize(1)/2, imgSize(2)/2];
% Create a transform, from shape in image to reference shape
T = affine2d([A_img2ref';T_img2ref]);
% transform the current shape to the reference one
shape2D_in_ref = bsxfun(@plus, (A_img2ref * labels')', T_img2ref);
% warp the image
[warped_img] = imwarp(imgCol, T, 'linear', 'OutputView', imref2d(imgSize));
counter_colour(views(lbl)) = counter_colour(views(lbl)) + 1;
allExamplesColourAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = warped_img;
landmarkLocationsAllViews{views(lbl)}(counter_colour(views(lbl)),:,:) = shape2D_in_ref;
actual_imgs_used_all_views{views(lbl)}{counter_colour(views(lbl))} = img_locations{lbl};
if(mod(lbl, 100) == 0)
fprintf('%d/%d done\n', lbl, num_imgs);
end
end
% write out the training data for each view
for r=1:size(centres_all,1)
mirrorIdx = find(sum(abs(centres_all - repmat([centres_all(r,1), -centres_all(r,2), -centres_all(r,3)], size(centres_all,1),1)),2)==0);
% if the mirrored view already added no need to do it again
% so if (0,-20,0) is done no need to compute (0,20,0)
if(mirrorIdx < r)
continue
end
mirrorImgs = allExamplesColourAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
mirrorLbls = landmarkLocationsAllViews{mirrorIdx}(1:counter_colour(mirrorIdx),:,:);
for i=1:size(mirrorImgs,1)
flippedImg = fliplr(squeeze(mirrorImgs(i,:,:)));
flippedLbls = squeeze(mirrorLbls(i,:,:));
flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1) + 1;
tmp1 = flippedLbls(mirror_inds(:,1),:);
tmp2 = flippedLbls(mirror_inds(:,2),:);
flippedLbls(mirror_inds(:,2),:) = tmp1;
flippedLbls(mirror_inds(:,1),:) = tmp2;
mirrorImgs(i,:,:) = flippedImg;
mirrorLbls(i,:,:) = flippedLbls;
end
all_images = cat(1, allExamplesColourAllViews{r}(1:counter_colour(r),:,:), mirrorImgs);
landmark_locations = cat(1, landmarkLocationsAllViews{r}(1:counter_colour(r),:,:), mirrorLbls);
actual_imgs_used = actual_imgs_used_all_views{r};
actual_imgs_used = cat(1, actual_imgs_used_all_views{r}, actual_imgs_used_all_views{mirrorIdx});
centres = centres_all(r,:);
visiIndex = ones(1,68);
save([output_location '_' num2str(r) '.mat'], 'all_images', 'landmark_locations', 'training_scale', 'centres', 'actual_imgs_used', 'visiIndex', '-v7.3');
end
end

16
pkg/OpenFace/model_training/ce-clm_training/patch_generation/readme.txt Normal file
View File

@@ -0,0 +1,16 @@
This code is in support of the "Convolutional experts constrained local model
for 3d facial landmark detection" paper by Amir Zadeh, Yao Chong Lim,
Tadas Baltrusaitis, and Louis-Philippe Morency. The code is based on
support code of the "Constrained Local Neural Fields for robust facial
landmark detection in the wild" paper by Tadas Baltrusaitis, Peter Robinson, and Louis-Philippe Morency.
This folder provides the code for generating patches used in training the patch experts.
You have to have the relevant datasets, to run the code, the in-the-wild
datasets can be found at http://ibug.doc.ic.ac.uk/resources/facial-point-annotations/),
the annotations are provided. The Multi-PIE dataset can be acquired from -
http://www.multipie.org/, you will need to ask the authors of the dataset for the annotations.
./data_preparation/ folder contains scripts to prepare data for patch generation
./scripts/ contains the scripts to generate the patches for training.

132
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/ExtractTrainingSamples.m Normal file
View File

@@ -0,0 +1,132 @@
function [samples, labels, samples_unnormed, imgs_used] = ExtractTrainingSamples(examples, landmarkLoc, img_names, sigma, numSamples, landmark, normalisation_options)
%%
% for an area of interest of 19x19 and patch support region of 11x11, we
% would have 9x9=81 samples (9 is the single_input_size, 11 is
% patch_expert_support_size, 19x19 is normalisation_size, 9 would be the
% normalisation_side_size)
evaluation_size = normalisation_options.normalisationRegion;
patch_expert_support_size = normalisation_options.patchSize;
normalisation_side_size = (evaluation_size - 1)/2;
single_input_size = evaluation_size - patch_expert_support_size + 1;
% Determine the ratio of images to be sampled (most likely not all of them will be)
samples_per_img = (numSamples / (size(examples,1) * (1 + normalisation_options.rate_negative))) / (single_input_size(1)^2);
num_samples = int32(samples_per_img * (1 + normalisation_options.rate_negative) * size(examples,1) * (single_input_size(1)^2));
%% Initialise the samples and labels
samples = zeros(num_samples, patch_expert_support_size(1) * patch_expert_support_size(2));
labels = zeros(num_samples, 1);
%% Initialise the unnormed versions of the images
% This is done in order to assert our use of algorithms for calculating
% the responses, as for training we might use regular ml procedures,
% whereas for fitting normalised cross-correlation or just
% cross-correlation will be used, so keep some unnormed samples
samples_unnormed = zeros(int32(num_samples/300), evaluation_size(1)^2);
img_size = [size(examples,2), size(examples,3)];
% Extract only images of differing shaped faces to extract more diverse
% training samples
to_keep = FindDistantLandmarks(landmarkLoc, landmark, round(samples_per_img*size(examples,1)));
inds_all = 1:size(examples,1);
samples_to_use = inds_all(to_keep);
% Keep track of how many samples have been computed already
samples_filled = 1;
samples_unnormed_filled = 1;
%% parse the image names for reporting purposes
imgs_used = img_names(samples_to_use);
for i=1:numel(imgs_used)
[~,name,ext] = fileparts(imgs_used{i});
imgs_used{i} = [name, ext];
end
for i=samples_to_use
% Do rate_negative negatives and a single positive
for p=1:normalisation_options.rate_negative+1
% create a gaussian
corrPoint = landmarkLoc(i,landmark,:);
% Ignore occluded points
if(corrPoint(1) == 0)
break;
end
startX = 1 - corrPoint(1);
startY = 1 - corrPoint(2);
patchWidth = img_size(2);
patchHeight = img_size(1);
[X, Y] = meshgrid(startX:patchWidth + startX-1, startY:patchHeight + startY-1);
response = exp(-0.5*(X.^2+Y.^2)/(sigma^2));
% Choose positive or negative sample
if(p==normalisation_options.rate_negative+1)
sample_centre = squeeze(corrPoint) + round(1*randn(2,1));
else
sample_centre = squeeze(corrPoint) + round(10*randn(2,1));
end
sample_centre = round(sample_centre);
sample_centre(sample_centre <= normalisation_side_size(1)) = normalisation_side_size(1) + 1;
sample_centre(sample_centre > img_size(1)-normalisation_side_size(1)) = img_size(1) - normalisation_side_size(1) - 1;
patches = squeeze(examples(i, sample_centre(2) - normalisation_side_size:sample_centre(2) + normalisation_side_size, sample_centre(1) - normalisation_side_size:sample_centre(1) + normalisation_side_size));
side = (single_input_size - 1)/2;
responses = response(sample_centre(2) - side(2):sample_centre(2) + side(2), sample_centre(1) - side(1):sample_centre(1) + side(1));
if(samples_unnormed_filled <= size(samples_unnormed,1))
% even if correct size is not initialised Matlab will
% sort that out (would only happen once anyway)
samples_unnormed(samples_unnormed_filled,:) = patches(:);
samples_unnormed_filled = samples_unnormed_filled + 1;
end
% if we want to normalise each patch individualy do it here
patch = im2col(patches, patch_expert_support_size, 'sliding')';
response = im2col(responses, [1,1], 'sliding');
labels(samples_filled:samples_filled+size(patch,1)-1,:) = response;
samples(samples_filled:samples_filled+size(patch,1)-1,:) = patch;
samples_filled = samples_filled + size(patch,1);
end
end
if(normalisation_options.useNormalisedCrossCorr == 1)
mean_curr = mean(samples, 2);
patch_normed = samples - repmat(mean_curr,1, patch_expert_support_size(1)*patch_expert_support_size(2));
% Normalising the patches using the L2 norm
scaling = sqrt(sum(patch_normed.^2,2));
scaling(scaling == 0) = 1;
patch_normed = patch_normed ./ repmat(scaling, 1, patch_expert_support_size(1)*patch_expert_support_size(2));
samples = patch_normed;
clear 'patch_normed';
end
% Only keep the filled samples
samples = samples(1:samples_filled-1,:);
labels = labels(1:samples_filled-1,:);
if((samples_filled-1)/(single_input_size(1)*single_input_size(2)) < size(samples_unnormed,1))
samples_unnormed = samples_unnormed(1:(samples_filled-1)/(single_input_size(1)*single_input_size(2)),:);
end
end

41
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/FindDistantLandmarks.m Normal file
View File

@@ -0,0 +1,41 @@
function [to_keep] = FindDistantLandmarks(landmarkLoc, landmark_num, num_to_keep)
% First align all of them
a = landmarkLoc(:,:,1);
b = landmarkLoc(:,:,2);
offset_x = mean(a,2);
offset_y = mean(b,2);
landmark_loc_off = cat(3, bsxfun(@plus, a, -offset_x), bsxfun(@plus, b, -offset_y));
fixed_x = landmark_loc_off(:,:,1);
fixed_y = landmark_loc_off(:,:,2);
% Extract the relevant landmarks
fixed_x_l = fixed_x(:,landmark_num);
fixed_y_l = fixed_y(:,landmark_num);
obs = cat(2, fixed_x_l, fixed_y_l);
% Discard landmarks that are very close to each other, so that we only
% keep more diverse images
D = squareform(pdist(obs));
to_keep = true(size(landmarkLoc,1),1);
for i = 1:(size(landmarkLoc,1) - num_to_keep)
diversity_score = mean(D,2);
a = min(diversity_score);
lowest = find(diversity_score == a);
lowest = lowest(1);
to_keep(lowest) = 0;
D(:,~to_keep) = 0;
D(~to_keep,:) = 200;
end
end

32
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_cofw.m Normal file
View File

@@ -0,0 +1,32 @@
clear
% define the root name of database
root = '../data_preparation/prepared_data/';
% which scales we're doing
sigma = 1;
num_samples = 5e5;
scales = [0.25,0.35,0.5];
frontalView = 1;
profileViewInds = [];
version = 'cofw';
ratio_neg = 5;
norm = 1;
data_loc = 'cofw_';
rng(0);
% where to save generated patches for external training
% (e.g. for training CEN patches)
patches_loc = './patches/';
patch_folder = [patches_loc version '/'];
for s=scales
Save_all_patches(root, frontalView, profileViewInds,...
s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
'num_samples', num_samples, 'data_loc', data_loc,...
'normalisation_size', 19);
end

32
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_general.m Normal file
View File

@@ -0,0 +1,32 @@
clear
% define the root name of database
root = '../data_preparation/prepared_data/';
% which scales we're doing
sigma = 1;
num_samples = 10e6;
scales = [0.25,0.35,0.5,1.0];
frontalView = 1;
profileViewInds = [2,3,4];
version = 'general';
ratio_neg = 10;
norm = 1;
data_loc = 'combined_';
rng(0);
% where to save generated patches for external training
% (e.g. for training CEN patches)
patches_loc = './patches/';
patch_folder = [patches_loc version '/'];
for s=scales
Save_all_patches(root, frontalView, profileViewInds,...
s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
'num_samples', num_samples, 'data_loc', data_loc,...
'normalisation_size', 19);
end

32
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_multi_pie.m Normal file
View File

@@ -0,0 +1,32 @@
clear
% define the root name of database
root = '../data_preparation/prepared_data/';
% which scales we're doing
sigma = 1;
num_samples = 5e5;
scales = [0.25,0.35,0.5];
frontalView = 1;
profileViewInds = [2,3,4];
version = 'multi_pie';
ratio_neg = 5;
norm = 1;
data_loc = 'mpie_';
rng(0);
% where to save generated patches for external training
% (e.g. for training CEN patches)
patches_loc = './patches/';
patch_folder = [patches_loc version '/'];
for s=scales
Save_all_patches(root, frontalView, profileViewInds,...
s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
'num_samples', num_samples, 'data_loc', data_loc,...
'normalisation_size', 19);
end

32
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Generate_Patches_wild.m Normal file
View File

@@ -0,0 +1,32 @@
clear
% define the root name of database
root = '../data_preparation/prepared_data/';
% which scales we're doing
sigma = 1;
num_samples = 2e6;
scales = [0.25,0.35,0.5,1.0];
frontalView = 1;
profileViewInds = [2];
version = 'wild';
ratio_neg = 10;
norm = 1;
data_loc = 'wild_';
rng(0);
% where to save generated patches for external training
% (e.g. for training CEN patches)
patches_loc = './patches/';
patch_folder = [patches_loc version '/'];
for s=scales
Save_all_patches(root, frontalView, profileViewInds,...
s, sigma, version, patch_folder, 'ratio_neg', ratio_neg,...
'num_samples', num_samples, 'data_loc', data_loc,...
'normalisation_size', 19);
end

39
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Save_all_patches.m Normal file
View File

@@ -0,0 +1,39 @@
function Save_all_patches(trainingLoc, frontalView, profile_views, scaling, sigma, version, patches_loc, varargin)
% need some documentation here
if(sum(strcmp(varargin,'ratio_neg')))
ind = find(strcmp(varargin,'ratio_neg')) + 1;
ratio_neg = varargin{ind};
else
ratio_neg = 20;
end
if(sum(strcmp(varargin,'num_samples')))
ind = find(strcmp(varargin,'num_samples')) + 1;
num_samples = varargin{ind};
else
num_samples = 5e5;
end
% first do the frontal view
imgs_used = Save_patches(trainingLoc, frontalView, scaling, sigma, ratio_neg, num_samples, patches_loc, 'frontal', varargin{:});
fprintf('Frontal done\n');
% now do the profile views
for i=1:numel(profile_views)
view_name = sprintf(['profile%s'], num2str(i));
imgs_used_profile = ...
Save_patches(trainingLoc, profile_views(i), scaling, sigma, ratio_neg, num_samples, patches_loc, view_name, varargin{:});
fprintf('Profile %d done\n', i);
imgs_used = cat(1, imgs_used, imgs_used_profile);
end
% save the images used
[status,msg,msgID] = mkdir('generated');
location_imgs_used = sprintf('generated/imgs_used_%s.mat', version);
save(location_imgs_used, 'imgs_used');
end

128
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/Save_patches.m Normal file
View File

@@ -0,0 +1,128 @@
function [imgs_used, normalisation_options] = Save_patches(training_loc, view, scale, sigma, ratio_neg, num_samples, patches_loc, view_name, varargin)
% patch generation options
num_samples = num_samples;
normalisation_options = struct;
normalisation_options.patchSize = [11 11];
if(sum(strcmp(varargin,'normalisation_size')))
ind = find(strcmp(varargin,'normalisation_size')) + 1;
normalisation_options.normalisationRegion = [varargin{ind}, varargin{ind}];
else
normalisation_options.normalisationRegion = [21 21];
end
normalisation_options.rate_negative = ratio_neg;
normalisation_options.useNormalisedCrossCorr = 1;
train_test_ratio = 0.9;
if(sum(strcmp(varargin,'data_loc')))
ind = find(strcmp(varargin,'data_loc')) + 1;
data_loc = varargin{ind};
data_loc = sprintf(['%s/' data_loc '%s_%s.mat'], training_loc, num2str(scale), num2str(view));
else
data_loc = sprintf('%s/wild_%s_%s.mat', training_loc, num2str(scale), num2str(view));
end
% location to save generated patches to (for training other patch experts
% such as CEN)
fprintf('saving patches to %s\n', patches_loc);
patches_filename = sprintf('data%.2f_%s.mat', scale, view_name);
% data_loc should contain landmark_locations, all_images, actual_imgs_used,
% visiIndex, centres
load(data_loc);
examples = all_images;
landmark_loc = landmark_locations;
clear 'all_images'
numPoints = size(landmark_loc,2);
done = zeros(1, numPoints);
for j=1:numPoints
[status,msg,msgID] = mkdir(sprintf([patches_loc '/%d'], j));
end
for j=1:numPoints
pause(0);
% can only do mirroring if there is no yaw
if((numPoints == 68 || numPoints == 29 )&& centres(2) == 0)
% Do not redo a mirror feature (just flip them)
if(numPoints == 68)
mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
61,65;62,64;68,66];
else
mirrorInds = [1,2; 3,4; 5,7; 6,8; 9,10; 11,12; 13,15; 14,16; 17,18; 19,20; 23,24];
end
mirror_idx = j;
if(any(mirrorInds(:,1)==j))
mirror_idx = mirrorInds(mirrorInds(:,1)==j,2);
elseif(any(mirrorInds(:,2)==j))
mirror_idx = mirrorInds(mirrorInds(:,2)==j,1);
end
if (mirror_idx ~= j & done(1, mirror_idx) ~= 0)
fprintf('not generating patches for lm %d, mirrored by %d\n', j, mirror_idx);
done(1, j) = done(1, mirror_idx);
fprintf('Feature %d done\n', j);
continue;
end
end
imgs_used = {};
if(visiIndex(j))
tic;
% instead of loading the patches compute them here:
[samples, labels, unnormed_samples, imgs_used_n] = ExtractTrainingSamples(examples, landmark_loc, actual_imgs_used, sigma, num_samples, j, normalisation_options);
imgs_used = union(imgs_used, imgs_used_n);
% add the bias term
samples = [ones(1,size(samples,1)); samples'];
region_length = normalisation_options.normalisationRegion - normalisation_options.patchSize + 1;
region_length = region_length(1) * region_length(2);
num_examples = size(samples, 2);
% this part sets the split boundaries for training and test subsets
train_ccnf_start = 1;
train_ccnf_end = int32(train_test_ratio * num_examples - 1);
% make sure we don't split a full training region apart
train_ccnf_end = train_ccnf_end - mod(train_ccnf_end, region_length);
test_start = train_ccnf_end + 1;
test_end = size(samples,2);
samples_train = samples(:,train_ccnf_start:train_ccnf_end);
labels_train = labels(train_ccnf_start:train_ccnf_end);
samples_test = samples(:,test_start:test_end);
labels_test = labels(test_start:test_end);
filename = sprintf([patches_loc '/%s/' patches_filename], num2str(j));
save(filename, 'samples_train', 'labels_train', 'samples_test', 'labels_test', '-v7.3');
done(1, j) = 1;
fprintf('Landmark %d done\n', j);
clear samples
clear samples_test
clear samples_train
clear labels
clear unnormed_samples
clear imgs_used_n
toc;
end
end
end

11
pkg/OpenFace/model_training/ce-clm_training/patch_generation/scripts/readme.txt Normal file
View File

@@ -0,0 +1,11 @@
Scripts for generating random patches to train continuous conditional neural field (CCNF, or alternatively Local Neural Field - LNF) patch experts.
This requires data preparation first (not included with the source code), to prepare the data go to '../data_preparation/readme.txt'.
To generate the patches run:
Generate_Patches_general (using combined data)
Generate_Patches_wild (using in-the-wild data)
Generate_Patches_multi_pie (using multi-pie data)
Generate_Patches_cofw (using cofw data)
By default, the patches will be placed in ./patches/. To change the save location, edit the 'patches_loc' variable in the scripts above.