open source pkg v1
This commit is contained in:
Vijay Yadev
@@ -0,0 +1,129 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef CCNF_PATCH_EXPERT_H
|
||||
#define CCNF_PATCH_EXPERT_H
|
||||
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
|
||||
//===========================================================================
|
||||
/**
|
||||
A single Neuron response
|
||||
*/
|
||||
class CCNF_neuron{
|
||||
|
||||
public:
|
||||
|
||||
// Type of patch (0=raw,1=grad,3=depth, other types besides raw are not actually used now)
|
||||
int neuron_type;
|
||||
|
||||
// scaling of weights (needed as the energy of neuron might not be 1)
|
||||
double norm_weights;
|
||||
|
||||
// Weight bias
|
||||
double bias;
|
||||
|
||||
// Neural weights
|
||||
cv::Mat_<float> weights;
|
||||
|
||||
// can have neural weight dfts that are calculated on the go as needed, this allows us not to recompute
|
||||
// the dft of the template each time, improving the speed of tracking
|
||||
std::map<int, cv::Mat_<double> > weights_dfts;
|
||||
|
||||
// the alpha associated with the neuron
|
||||
double alpha;
|
||||
|
||||
// Default constructor
|
||||
CCNF_neuron(){;}
|
||||
|
||||
// Copy constructor
|
||||
CCNF_neuron(const CCNF_neuron& other);
|
||||
|
||||
void Read(std::ifstream &stream);
|
||||
// The im_dft, integral_img, and integral_img_sq are precomputed images for convolution speedups (they get set if passed in empty values)
|
||||
void Response(const cv::Mat_<float> &im, cv::Mat_<double> &im_dft, cv::Mat &integral_img, cv::Mat &integral_img_sq, cv::Mat_<float> &resp);
|
||||
|
||||
};
|
||||
|
||||
//===========================================================================
|
||||
/**
|
||||
A CCNF patch expert
|
||||
*/
|
||||
class CCNF_patch_expert{
|
||||
public:
|
||||
|
||||
// Width and height of the patch expert support region
|
||||
int width;
|
||||
int height;
|
||||
|
||||
// Collection of neurons for this patch expert
|
||||
std::vector<CCNF_neuron> neurons;
|
||||
|
||||
// Information about the vertex features (association potentials)
|
||||
std::vector<int> window_sizes;
|
||||
std::vector<cv::Mat_<float> > Sigmas;
|
||||
std::vector<double> betas;
|
||||
|
||||
// Combined weight matrix from each neuron
|
||||
cv::Mat_<float> weight_matrix;
|
||||
|
||||
// How confident we are in the patch
|
||||
double patch_confidence;
|
||||
|
||||
// Default constructor
|
||||
CCNF_patch_expert(){;}
|
||||
|
||||
// Copy constructor
|
||||
CCNF_patch_expert(const CCNF_patch_expert& other);
|
||||
|
||||
void Read(std::ifstream &stream, std::vector<int> window_sizes, std::vector<std::vector<cv::Mat_<float> > > sigma_components);
|
||||
|
||||
// actual work (can pass in an image and a potential depth image, if the CCNF is trained with depth)
|
||||
void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
|
||||
|
||||
void ResponseOpenBlas(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response, cv::Mat_<float> &im2col_prealloc);
|
||||
|
||||
// Helper function to compute relevant sigmas
|
||||
void ComputeSigmas(std::vector<cv::Mat_<float> > sigma_components, int window_size);
|
||||
|
||||
};
|
||||
//===========================================================================
|
||||
}
|
||||
#endif // CCNF_PATCH_EXPERT_H
|
||||
@@ -0,0 +1,91 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
#ifndef CEN_PATCH_EXPERT_H
|
||||
#define CEN_PATCH_EXPERT_H
|
||||
|
||||
// system includes
|
||||
#include <vector>
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
/**
|
||||
The classes describing the CEN patch experts
|
||||
*/
|
||||
|
||||
class CEN_patch_expert {
|
||||
public:
|
||||
|
||||
// Width and height of the patch expert support area
|
||||
int width_support;
|
||||
int height_support;
|
||||
|
||||
// Neural weights
|
||||
std::vector<cv::Mat_<float>> biases;
|
||||
|
||||
// Neural weights
|
||||
std::vector<cv::Mat_<float>> weights;
|
||||
|
||||
std::vector<int> activation_function;
|
||||
|
||||
// Confidence of the current patch expert (used for NU_RLMS optimisation)
|
||||
double confidence;
|
||||
|
||||
CEN_patch_expert() { ; }
|
||||
|
||||
// A copy constructor
|
||||
CEN_patch_expert(const CEN_patch_expert& other);
|
||||
|
||||
// Reading in the patch expert
|
||||
void Read(std::ifstream &stream);
|
||||
|
||||
// The actual response computation from intensity image
|
||||
void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
|
||||
|
||||
void ResponseInternal(cv::Mat_<float>& response);
|
||||
|
||||
// For frontal faces can apply mirrored and non-mirrored experts at the same time
|
||||
void ResponseSparse(const cv::Mat_<float> &area_of_interest_left, const cv::Mat_<float> &area_of_interest_right, cv::Mat_<float> &response_left, cv::Mat_<float> &response_right, cv::Mat_<float>& mapMatrix, cv::Mat_<float>& im2col_prealloc_left, cv::Mat_<float>& im2col_prealloc_right);
|
||||
|
||||
};
|
||||
|
||||
void interpolationMatrix(cv::Mat_<float>& mapMatrix, int response_height, int response_width, int input_width, int input_height);
|
||||
|
||||
}
|
||||
#endif // CEN_PATCH_EXPERT_H
|
||||
62
pkg/OpenFace/lib/local/LandmarkDetector/include/CNN_utils.h
Normal file
62
pkg/OpenFace/lib/local/LandmarkDetector/include/CNN_utils.h
Normal file
@@ -0,0 +1,62 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef CNN_UTILS_H
|
||||
#define CNN_UTILS_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
// Various CNN layers
|
||||
|
||||
// Parametric ReLU with leaky weights (separate ones per channel)
|
||||
void PReLU(std::vector<cv::Mat_<float> >& input_output_maps, cv::Mat_<float> prelu_weights);
|
||||
|
||||
// The fully connected layer
|
||||
void fully_connected(std::vector<cv::Mat_<float> >& outputs, const std::vector<cv::Mat_<float> >& input_maps, cv::Mat_<float> weights, cv::Mat_<float> biases);
|
||||
|
||||
// Max pooling layer with parametrized stride and kernel sizes
|
||||
void max_pooling(std::vector<cv::Mat_<float> >& outputs, const std::vector<cv::Mat_<float> >& input_maps, int stride_x, int stride_y, int kernel_size_x, int kernel_size_y);
|
||||
|
||||
// Convolution using FFT optimization rather than matrix multiplication, TODO do these still work
|
||||
void convolution_fft2(std::vector<cv::Mat_<float> >& outputs, const std::vector<cv::Mat_<float> >& input_maps,
|
||||
const std::vector<std::vector<cv::Mat_<float> > >& kernels, const std::vector<float >& biases,
|
||||
std::vector<std::map<int, std::vector<cv::Mat_<double> > > >& precomp_dfts);
|
||||
|
||||
// Convolution using matrix multiplication and OpenBLAS optimization, can also provide a pre-allocated im2col result for faster processing
|
||||
void convolution_direct_blas(std::vector<cv::Mat_<float> >& outputs, const std::vector<cv::Mat_<float> >& input_maps, const cv::Mat_<float>& weight_matrix, int height_k, int width_k, cv::Mat_<float>& pre_alloc_im2col);
|
||||
}
|
||||
#endif // CNN_UTILS_H
|
||||
@@ -0,0 +1,134 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef FACE_DETECTOR_MTCNN_H
|
||||
#define FACE_DETECTOR_MTCNN_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
// System includes
|
||||
#include <vector>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
class CNN
|
||||
{
|
||||
public:
|
||||
|
||||
//==========================================
|
||||
|
||||
// Default constructor
|
||||
CNN() { ; }
|
||||
|
||||
// Copy constructor
|
||||
CNN(const CNN& other);
|
||||
|
||||
// Given an image apply a CNN on it, the boolean direct controls if direct convolution is used (through matrix multiplication) or an FFT optimization
|
||||
std::vector<cv::Mat_<float> > Inference(const cv::Mat& input_img, bool direct = true, bool thread_safe = false);
|
||||
|
||||
// Reading in the model
|
||||
void Read(const std::string& location);
|
||||
|
||||
// Clearing precomputed DFTs
|
||||
void ClearPrecomp();
|
||||
|
||||
size_t NumberOfLayers() { return cnn_layer_types.size(); }
|
||||
|
||||
private:
|
||||
//==========================================
|
||||
// Convolutional Neural Network
|
||||
|
||||
// CNN layers
|
||||
// Layer -> Weight matrix
|
||||
std::vector<cv::Mat_<float> > cnn_convolutional_layers_weights;
|
||||
|
||||
// Keeping some pre-allocated im2col data as malloc is a significant time cost (not thread safe though)
|
||||
std::vector<cv::Mat_<float> > conv_layer_pre_alloc_im2col;
|
||||
|
||||
// Layer -> kernel -> input maps
|
||||
std::vector<std::vector<std::vector<cv::Mat_<float> > > > cnn_convolutional_layers;
|
||||
std::vector<std::vector<float > > cnn_convolutional_layers_bias;
|
||||
// Layer matrix + bas
|
||||
std::vector<cv::Mat_<float> > cnn_fully_connected_layers_weights;
|
||||
std::vector<cv::Mat_<float> > cnn_fully_connected_layers_biases;
|
||||
std::vector<cv::Mat_<float> > cnn_prelu_layer_weights;
|
||||
std::vector<std::tuple<int, int, int, int> > cnn_max_pooling_layers;
|
||||
|
||||
// Precomputations for faster convolution
|
||||
std::vector<std::vector<std::map<int, std::vector<cv::Mat_<double> > > > > cnn_convolutional_layers_dft;
|
||||
|
||||
// CNN: 0 - convolutional, 1 - max pooling, 2 - fully connected, 3 - prelu, 4 - sigmoid
|
||||
std::vector<int > cnn_layer_types;
|
||||
};
|
||||
//===========================================================================
|
||||
//
|
||||
// Checking if landmark detection was successful using an SVR regressor
|
||||
// Using multiple validators trained add different views
|
||||
// The regressor outputs -1 for ideal alignment and 1 for worst alignment
|
||||
//===========================================================================
|
||||
class FaceDetectorMTCNN
|
||||
{
|
||||
|
||||
public:
|
||||
|
||||
// Default constructor
|
||||
FaceDetectorMTCNN() { ; }
|
||||
|
||||
FaceDetectorMTCNN(const std::string& location);
|
||||
|
||||
// Copy constructor
|
||||
FaceDetectorMTCNN(const FaceDetectorMTCNN& other);
|
||||
|
||||
// Given an image, orientation and detected landmarks output the result of the appropriate regressor
|
||||
bool DetectFaces(std::vector<cv::Rect_<float> >& o_regions, const cv::Mat& input_img,
|
||||
std::vector<float>& o_confidences, int min_face = 60, float t1 = 0.6, float t2 = 0.7, float t3 = 0.7);
|
||||
|
||||
// Reading in the model
|
||||
void Read(const std::string& location);
|
||||
|
||||
// Indicate if the model has been read in
|
||||
bool empty() { return PNet.NumberOfLayers() == 0 || RNet.NumberOfLayers() == 0 || ONet.NumberOfLayers() == 0; };
|
||||
|
||||
private:
|
||||
//==========================================
|
||||
// Components of the model
|
||||
|
||||
CNN PNet;
|
||||
CNN RNet;
|
||||
CNN ONet;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
#endif // FACE_DETECTOR_MTCNN_H
|
||||
@@ -0,0 +1,44 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
#ifndef LANDMARK_CORE_INCLUDES_H
|
||||
#define LANDMARK_CORE_INCLUDES_H
|
||||
|
||||
#include "LandmarkDetectorModel.h"
|
||||
#include "LandmarkDetectorFunc.h"
|
||||
#include "LandmarkDetectorParameters.h"
|
||||
#include "LandmarkDetectorUtils.h"
|
||||
|
||||
#endif // LANDMARK_CORE_INCLUDES_H
|
||||
@@ -0,0 +1,115 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef LANDMARK_DETECTION_VALIDATOR_H
|
||||
#define LANDMARK_DETECTION_VALIDATOR_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
// System includes
|
||||
#include <vector>
|
||||
|
||||
// Local includes
|
||||
#include "PAW.h"
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
//
|
||||
// Checking if landmark detection was successful using a CNN
|
||||
// Using multiple validators trained add different views
|
||||
// The regressor outputs 1 for ideal alignment and 0 for worst alignment
|
||||
//===========================================================================
|
||||
class DetectionValidator
|
||||
{
|
||||
|
||||
public:
|
||||
|
||||
// The orientations of each of the landmark detection validator
|
||||
std::vector<cv::Vec3d> orientations;
|
||||
|
||||
// Piecewise affine warps to the reference shape (per orientation)
|
||||
std::vector<PAW> paws;
|
||||
|
||||
//==========================================
|
||||
// Convolutional Neural Network
|
||||
|
||||
// CNN layers for each view
|
||||
// view -> layer
|
||||
std::vector<std::vector<std::vector<std::vector<cv::Mat_<float> > > > > cnn_convolutional_layers;
|
||||
std::vector<std::vector<cv::Mat_<float> > > cnn_convolutional_layers_weights;
|
||||
std::vector<std::vector<cv::Mat_<float> > > cnn_convolutional_layers_im2col_precomp;
|
||||
|
||||
std::vector< std::vector<int> > cnn_subsampling_layers;
|
||||
std::vector< std::vector<cv::Mat_<float> > > cnn_fully_connected_layers_weights;
|
||||
std::vector< std::vector<cv::Mat_<float> > > cnn_fully_connected_layers_biases;
|
||||
// NEW CNN: 0 - convolutional, 1 - max pooling (2x2 stride 2), 2 - fully connected, 3 - relu, 4 - sigmoid
|
||||
std::vector<std::vector<int> > cnn_layer_types;
|
||||
|
||||
//==========================================
|
||||
|
||||
// Normalisation for face validation
|
||||
std::vector<cv::Mat_<float> > mean_images;
|
||||
std::vector<cv::Mat_<float> > standard_deviations;
|
||||
|
||||
// Default constructor
|
||||
DetectionValidator(){;}
|
||||
|
||||
// Copy constructor
|
||||
DetectionValidator(const DetectionValidator& other);
|
||||
|
||||
// Given an image, orientation and detected landmarks output the result of the appropriate regressor
|
||||
float Check(const cv::Vec3d& orientation, const cv::Mat_<uchar>& intensity_img, cv::Mat_<float>& detected_landmarks);
|
||||
|
||||
// Reading in the model
|
||||
void Read(std::string location);
|
||||
|
||||
// Getting the closest view center based on orientation
|
||||
int GetViewId(const cv::Vec3d& orientation) const;
|
||||
|
||||
private:
|
||||
|
||||
// The actual regressor application on the image
|
||||
|
||||
// Convolutional Neural Network
|
||||
double CheckCNN(const cv::Mat_<float>& warped_img, int view_id);
|
||||
|
||||
// A normalisation helper
|
||||
void NormaliseWarpedToVector(const cv::Mat_<float>& warped_img, cv::Mat_<float>& feature_vec, int view_id);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
#endif // LANDMARK_DETECTION_VALIDATOR_H
|
||||
@@ -0,0 +1,79 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Header for all external CLM/CLNF/CE-CLM methods of interest to the user
|
||||
//
|
||||
//
|
||||
#ifndef LANDMARK_DETECTOR_FUNC_H
|
||||
#define LANDMARK_DETECTOR_FUNC_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
#include <LandmarkDetectorParameters.h>
|
||||
#include <LandmarkDetectorUtils.h>
|
||||
#include <LandmarkDetectorModel.h>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
|
||||
//================================================================================================================
|
||||
// Landmark detection in videos, need to provide an image and model parameters (default values work well)
|
||||
// Optionally can provide a bounding box from which to start tracking
|
||||
// Can also optionally pass a grayscale image if it has already been computed to speed things up a bit
|
||||
//================================================================================================================
|
||||
bool DetectLandmarksInVideo(const cv::Mat &rgb_image, CLNF& clnf_model, FaceModelParameters& params, cv::Mat &grayscale_image);
|
||||
bool DetectLandmarksInVideo(const cv::Mat &rgb_image, const cv::Rect_<double> bounding_box, CLNF& clnf_model, FaceModelParameters& params, cv::Mat &grayscale_image);
|
||||
|
||||
//================================================================================================================
|
||||
// Landmark detection in image, need to provide an image and optionally CLNF model together with parameters (default values work well)
|
||||
// Optionally can provide a bounding box in which detection is performed (this is useful if multiple faces are to be detected in images)
|
||||
// Can also optionally pass a grayscale image if it has already been computed to speed things up a bit
|
||||
//================================================================================================================
|
||||
bool DetectLandmarksInImage(const cv::Mat &rgb_image, CLNF& clnf_model, FaceModelParameters& params, cv::Mat &grayscale_image);
|
||||
// Providing a bounding box
|
||||
bool DetectLandmarksInImage(const cv::Mat &rgb_image, const cv::Rect_<double> bounding_box, CLNF& clnf_model, FaceModelParameters& params, cv::Mat &grayscale_image);
|
||||
|
||||
//================================================================
|
||||
// Helper function for getting head pose from CLNF parameters
|
||||
|
||||
// Return the current estimate of the head pose in world coordinates with camera at origin (0,0,0)
|
||||
// The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z]
|
||||
cv::Vec6f GetPose(const CLNF& clnf_model, float fx, float fy, float cx, float cy);
|
||||
|
||||
// Return the current estimate of the head pose in world coordinates with camera at origin (0,0,0), but with rotation representing if the head is looking at the camera
|
||||
// The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z]
|
||||
cv::Vec6f GetPoseWRTCamera(const CLNF& clnf_model, float fx, float fy, float cx, float cy);
|
||||
}
|
||||
#endif // LANDMARK_DETECTOR_FUNC_H
|
||||
@@ -0,0 +1,215 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef LANDMARK_DETECTOR_MODEL_H
|
||||
#define LANDMARK_DETECTOR_MODEL_H
|
||||
|
||||
// OpenCV dependencies
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/objdetect.hpp>
|
||||
|
||||
// dlib dependencies for face detection
|
||||
#include <dlib/image_processing/frontal_face_detector.h>
|
||||
#include <dlib/opencv.h>
|
||||
|
||||
#include "PDM.h"
|
||||
#include "Patch_experts.h"
|
||||
#include "LandmarkDetectionValidator.h"
|
||||
#include "LandmarkDetectorParameters.h"
|
||||
#include "FaceDetectorMTCNN.h"
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
|
||||
// A main class containing all the modules required for landmark detection
|
||||
// Face shape model
|
||||
// Patch experts
|
||||
// Optimization techniques
|
||||
class CLNF{
|
||||
|
||||
public:
|
||||
|
||||
//===========================================================================
|
||||
// Member variables that contain the model description
|
||||
|
||||
// The linear 3D Point Distribution Model
|
||||
PDM pdm;
|
||||
// The set of patch experts
|
||||
Patch_experts patch_experts;
|
||||
|
||||
// The local and global parameters describing the current model instance (current landmark detections)
|
||||
|
||||
// Local parameters describing the non-rigid shape
|
||||
cv::Mat_<float> params_local;
|
||||
|
||||
// Global parameters describing the rigid shape [scale, euler_x, euler_y, euler_z, tx, ty]
|
||||
cv::Vec6f params_global;
|
||||
|
||||
// A collection of hierarchical CLNF models that can be used for refinement
|
||||
std::vector<CLNF> hierarchical_models;
|
||||
std::vector<std::string> hierarchical_model_names;
|
||||
std::vector<std::vector<std::pair<int,int>>> hierarchical_mapping;
|
||||
std::vector<FaceModelParameters> hierarchical_params;
|
||||
|
||||
//==================== Helpers for face detection and landmark detection validation =========================================
|
||||
|
||||
// TODO these should be static, and loading should be made easier
|
||||
|
||||
// Haar cascade classifier for face detection
|
||||
cv::CascadeClassifier face_detector_HAAR;
|
||||
std::string haar_face_detector_location;
|
||||
|
||||
// A HOG SVM-struct based face detector
|
||||
dlib::frontal_face_detector face_detector_HOG;
|
||||
|
||||
FaceDetectorMTCNN face_detector_MTCNN;
|
||||
std::string mtcnn_face_detector_location;
|
||||
|
||||
// Validate if the detected landmarks are correct using an SVR regressor
|
||||
DetectionValidator landmark_validator;
|
||||
|
||||
// Indicating if landmark detection succeeded (based on SVR validator)
|
||||
bool detection_success;
|
||||
|
||||
// Indicating if the tracking has been initialised (for video based tracking)
|
||||
bool tracking_initialised;
|
||||
|
||||
// The actual output of the regressor (-1 is perfect detection 1 is worst detection)
|
||||
float detection_certainty;
|
||||
|
||||
// Indicator if eye model is there for eye detection
|
||||
bool eye_model;
|
||||
|
||||
// the triangulation per each view (for drawing purposes only)
|
||||
std::vector<cv::Mat_<int> > triangulations;
|
||||
|
||||
//===========================================================================
|
||||
// Member variables that retain the state of the tracking (reflecting the state of the lastly tracked (detected) image
|
||||
|
||||
// Lastly detect 2D model shape [x1,x2,...xn,y1,...yn]
|
||||
cv::Mat_<float> detected_landmarks;
|
||||
|
||||
// The landmark detection likelihoods (combined and per patch expert)
|
||||
float model_likelihood;
|
||||
cv::Mat_<float> landmark_likelihoods;
|
||||
|
||||
// Keeping track of how many frames the tracker has failed in so far when tracking in videos
|
||||
// This is useful for knowing when to initialise and reinitialise tracking
|
||||
int failures_in_a_row;
|
||||
|
||||
// A template of a face that last succeeded with tracking (useful for large motions in video)
|
||||
cv::Mat_<uchar> face_template;
|
||||
|
||||
// Useful when resetting or initialising the model closer to a specific location (when multiple faces are present)
|
||||
cv::Point_<double> preference_det;
|
||||
|
||||
// Tracking which view was used last
|
||||
int view_used;
|
||||
|
||||
// See if the model was read in correctly
|
||||
bool loaded_successfully;
|
||||
|
||||
// A default constructor
|
||||
CLNF();
|
||||
|
||||
// Constructor from a model file
|
||||
CLNF(std::string fname);
|
||||
|
||||
// Copy constructor (makes a deep copy of the detector)
|
||||
CLNF(const CLNF& other);
|
||||
|
||||
// Assignment operator for lvalues (makes a deep copy of the detector)
|
||||
CLNF & operator= (const CLNF& other);
|
||||
|
||||
// Empty Destructor as the memory of every object will be managed by the corresponding libraries (no pointers)
|
||||
~CLNF(){}
|
||||
|
||||
// Move constructor
|
||||
CLNF(const CLNF&& other);
|
||||
|
||||
// Assignment operator for rvalues
|
||||
CLNF & operator= (const CLNF&& other);
|
||||
|
||||
// Does the actual work - landmark detection
|
||||
bool DetectLandmarks(const cv::Mat_<uchar> &image, FaceModelParameters& params);
|
||||
|
||||
// Gets the shape of the current detected landmarks in camera space (given camera calibration)
|
||||
// Can only be called after a call to DetectLandmarksInVideo or DetectLandmarksInImage
|
||||
cv::Mat_<float> GetShape(float fx, float fy, float cx, float cy) const;
|
||||
|
||||
// A utility bounding box function
|
||||
cv::Rect_<float> GetBoundingBox() const;
|
||||
|
||||
// Get the currently non-self occluded landmarks
|
||||
cv::Mat_<int> GetVisibilities() const;
|
||||
|
||||
// Reset the model (useful if we want to completelly reinitialise, or we want to track another video)
|
||||
void Reset();
|
||||
|
||||
// Reset the model, choosing the face nearest (x,y) where x and y are between 0 and 1.
|
||||
void Reset(double x, double y);
|
||||
|
||||
// Reading the model in
|
||||
void Read(std::string name);
|
||||
|
||||
private:
|
||||
|
||||
// Helper reading function
|
||||
bool Read_CLNF(std::string clnf_location);
|
||||
|
||||
// the speedup of RLMS using precalculated KDE responses (described in Saragih 2011 RLMS paper)
|
||||
std::map<int, cv::Mat_<float> > kde_resp_precalc;
|
||||
|
||||
// The model fitting: patch response computation and optimisation steps
|
||||
bool Fit(const cv::Mat_<float>& intensity_image, const std::vector<int>& window_sizes, const FaceModelParameters& parameters);
|
||||
|
||||
// Mean shift computation that uses precalculated kernel density estimators (the one actually used)
|
||||
void NonVectorisedMeanShift_precalc_kde(cv::Mat_<float>& out_mean_shifts, const std::vector<cv::Mat_<float> >& patch_expert_responses,
|
||||
const cv::Mat_<float> &dxs, const cv::Mat_<float> &dys, int resp_size, float a, int scale, int view_id,
|
||||
std::map<int, cv::Mat_<float> >& mean_shifts);
|
||||
|
||||
// The actual model optimisation (update step), returns the model likelihood
|
||||
float NU_RLMS(cv::Vec6f& final_global, cv::Mat_<float>& final_local, const std::vector<cv::Mat_<float> >& patch_expert_responses,
|
||||
const cv::Vec6f& initial_global, const cv::Mat_<float>& initial_local,
|
||||
const cv::Mat_<float>& base_shape, const cv::Matx22f& sim_img_to_ref,
|
||||
const cv::Matx22f& sim_ref_to_img, int resp_size, int view_idx, bool rigid, int scale,
|
||||
cv::Mat_<float>& landmark_lhoods, const FaceModelParameters& parameters, bool compute_lhood);
|
||||
|
||||
// Generating the weight matrix for the Weighted least squares
|
||||
void GetWeightMatrix(cv::Mat_<float>& WeightMatrix, int scale, int view_id, const FaceModelParameters& parameters);
|
||||
|
||||
};
|
||||
//===========================================================================
|
||||
}
|
||||
#endif // LANDMARK_DETECTOR_MODEL_H
|
||||
@@ -0,0 +1,118 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Parameters of the CE-CLM, CLNF, and CLM trackers
|
||||
#ifndef LANDMARK_DETECTOR_PARAM_H
|
||||
#define LANDMARK_DETECTOR_PARAM_H
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
|
||||
struct FaceModelParameters
|
||||
{
|
||||
|
||||
// A number of RLMS or NU-RLMS iterations
|
||||
int num_optimisation_iteration;
|
||||
|
||||
// Should pose be limited to 180 degrees frontal
|
||||
bool limit_pose;
|
||||
|
||||
// Should face validation be done
|
||||
bool validate_detections;
|
||||
|
||||
// Landmark detection validator boundary for correct detection, the regressor output 1 (perfect alignment) 0 (bad alignment),
|
||||
float validation_boundary;
|
||||
|
||||
// Used when tracking is going well
|
||||
std::vector<int> window_sizes_small;
|
||||
|
||||
// Used when initialising or tracking fails
|
||||
std::vector<int> window_sizes_init;
|
||||
|
||||
// Used for the current frame
|
||||
std::vector<int> window_sizes_current;
|
||||
|
||||
// How big is the tracking template that helps with large motions
|
||||
float face_template_scale;
|
||||
bool use_face_template;
|
||||
|
||||
// Where to load the model from
|
||||
std::string model_location;
|
||||
|
||||
// this is used for the smooting of response maps (KDE sigma)
|
||||
float sigma;
|
||||
|
||||
float reg_factor; // weight put to regularisation
|
||||
float weight_factor; // factor for weighted least squares
|
||||
|
||||
// should multiple views be considered during reinit
|
||||
bool multi_view;
|
||||
|
||||
// Based on model location, this affects the parameter settings
|
||||
enum LandmarkDetector { CLM_DETECTOR, CLNF_DETECTOR, CECLM_DETECTOR };
|
||||
LandmarkDetector curr_landmark_detector;
|
||||
|
||||
// How often should face detection be used to attempt reinitialisation, every n frames (set to negative not to reinit)
|
||||
int reinit_video_every;
|
||||
|
||||
// Determining which face detector to use for (re)initialisation, HAAR is quicker but provides more false positives and is not goot for in-the-wild conditions
|
||||
// Also HAAR detector can detect smaller faces while HOG SVM is only capable of detecting faces at least 70px across
|
||||
// MTCNN detector is much more accurate that the other two, and is even suitable for profile faces, but it is somewhat slower
|
||||
enum FaceDetector{HAAR_DETECTOR, HOG_SVM_DETECTOR, MTCNN_DETECTOR};
|
||||
|
||||
std::string haar_face_detector_location;
|
||||
std::string mtcnn_face_detector_location;
|
||||
FaceDetector curr_face_detector;
|
||||
|
||||
// Should the model be refined hierarchically (if available)
|
||||
bool refine_hierarchical;
|
||||
|
||||
// Should the parameters be refined for different scales
|
||||
bool refine_parameters;
|
||||
|
||||
FaceModelParameters();
|
||||
|
||||
FaceModelParameters(std::vector<std::string> &arguments);
|
||||
|
||||
private:
|
||||
void init();
|
||||
void check_model_path(const std::string& root = "/");
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif // LANDMARK_DETECTOR_PARAM_H
|
||||
@@ -0,0 +1,107 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef LANDMARK_DETECTOR_UTILS_H
|
||||
#define LANDMARK_DETECTOR_UTILS_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
#include "LandmarkDetectorModel.h"
|
||||
|
||||
#include "FaceDetectorMTCNN.h"
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
// Defining a set of useful utility functions to be used within CLNF
|
||||
|
||||
//===========================================================================
|
||||
// Fast patch expert response computation (linear model across a ROI) using normalised cross-correlation
|
||||
//===========================================================================
|
||||
// This is a modified version of openCV code that allows for precomputed dfts of templates and for precomputed dfts of an image
|
||||
// _img is the input img, _img_dft it's dft (optional), _integral_img the images integral image (optional), squared integral image (optional),
|
||||
// templ is the template we are convolving with, templ_dfts it's dfts at varying windows sizes (optional), _result - the output, method the type of convolution
|
||||
void matchTemplate_m(const cv::Mat_<float>& input_img, cv::Mat_<double>& img_dft, cv::Mat& _integral_img, cv::Mat& _integral_img_sq,
|
||||
const cv::Mat_<float>& templ, std::map<int, cv::Mat_<double> >& templ_dfts, cv::Mat_<float>& result, int method);
|
||||
|
||||
// Useful utility for grabing a bounding box around a set of 2D landmarks (as a 1D 2n x 1 vector of xs followed by doubles or as an n x 2 vector)
|
||||
void ExtractBoundingBox(const cv::Mat_<float>& landmarks, float &min_x, float &max_x, float &min_y, float &max_y);
|
||||
|
||||
std::vector<cv::Point2f> CalculateVisibleLandmarks(const cv::Mat_<float>& shape2D, const cv::Mat_<int>& visibilities);
|
||||
std::vector<cv::Point2f> CalculateVisibleLandmarks(const CLNF& clnf_model);
|
||||
std::vector<cv::Point2f> CalculateVisibleEyeLandmarks(const CLNF& clnf_model);
|
||||
|
||||
std::vector<cv::Point2f> CalculateAllLandmarks(const cv::Mat_<float>& shape2D);
|
||||
std::vector<cv::Point2f> CalculateAllLandmarks(const CLNF& clnf_model);
|
||||
std::vector<cv::Point2f> CalculateAllEyeLandmarks(const CLNF& clnf_model);
|
||||
std::vector<cv::Point3f> Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy);
|
||||
|
||||
//============================================================================
|
||||
// Face detection helpers
|
||||
//============================================================================
|
||||
|
||||
// Face detection using Haar cascade classifier
|
||||
bool DetectFaces(std::vector<cv::Rect_<float> >& o_regions, const cv::Mat_<uchar>& intensity, float min_width = -1, cv::Rect_<float> roi = cv::Rect_<float>(0.0, 0.0, 1.0, 1.0));
|
||||
bool DetectFaces(std::vector<cv::Rect_<float> >& o_regions, const cv::Mat_<uchar>& intensity, cv::CascadeClassifier& classifier, float min_width = -1, cv::Rect_<float> roi = cv::Rect_<float>(0.0, 0.0, 1.0, 1.0));
|
||||
// The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen
|
||||
bool DetectSingleFace(cv::Rect_<float>& o_region, const cv::Mat_<uchar>& intensity, cv::CascadeClassifier& classifier, const cv::Point preference = cv::Point(-1, -1), float min_width = -1, cv::Rect_<float> roi = cv::Rect_<float>(0.0, 0.0, 1.0, 1.0));
|
||||
|
||||
// Face detection using HOG-SVM classifier
|
||||
bool DetectFacesHOG(std::vector<cv::Rect_<float> >& o_regions, const cv::Mat_<uchar>& intensity, std::vector<float>& confidences, float min_width = -1, cv::Rect_<float> roi = cv::Rect_<float>(0.0, 0.0, 1.0, 1.0));
|
||||
bool DetectFacesHOG(std::vector<cv::Rect_<float> >& o_regions, const cv::Mat_<uchar>& intensity, dlib::frontal_face_detector& classifier, std::vector<float>& confidences, float min_width = -1, cv::Rect_<float> roi = cv::Rect_<float>(0.0, 0.0, 1.0, 1.0));
|
||||
// The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen
|
||||
bool DetectSingleFaceHOG(cv::Rect_<float>& o_region, const cv::Mat_<uchar>& intensity, dlib::frontal_face_detector& classifier, float& confidence, const cv::Point preference = cv::Point(-1, -1), float min_width = -1, cv::Rect_<float> roi = cv::Rect_<float>(0.0, 0.0, 1.0, 1.0));
|
||||
|
||||
// Face detection using Multi-task Convolutional Neural Network
|
||||
bool DetectFacesMTCNN(std::vector<cv::Rect_<float> >& o_regions, const cv::Mat& image, LandmarkDetector::FaceDetectorMTCNN& detector, std::vector<float>& confidences);
|
||||
// The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen
|
||||
bool DetectSingleFaceMTCNN(cv::Rect_<float>& o_region, const cv::Mat& image, LandmarkDetector::FaceDetectorMTCNN& detector, float& confidence, const cv::Point preference = cv::Point(-1, -1));
|
||||
|
||||
//============================================================================
|
||||
// Matrix reading functionality
|
||||
//============================================================================
|
||||
|
||||
// Reading a matrix written in a binary format
|
||||
void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat);
|
||||
|
||||
// Reading in a matrix from a stream
|
||||
void ReadMat(std::ifstream& stream, cv::Mat& output_matrix);
|
||||
|
||||
// Skipping comments (lines starting with # symbol)
|
||||
void SkipComments(std::ifstream& stream);
|
||||
|
||||
|
||||
}
|
||||
#endif // LANDMARK_DETECTOR_UTILS_H
|
||||
138
pkg/OpenFace/lib/local/LandmarkDetector/include/PAW.h
Normal file
138
pkg/OpenFace/lib/local/LandmarkDetector/include/PAW.h
Normal file
@@ -0,0 +1,138 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef PAW_H
|
||||
#define PAW_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
/**
|
||||
A Piece-wise Affine Warp
|
||||
The ideas for this piece-wise affine triangular warping are taken from the
|
||||
Active appearance models revisited by Iain Matthews and Simon Baker in IJCV 2004
|
||||
This is used for both validation of landmark detection, and for avatar animation
|
||||
|
||||
The code is based on the CLM tracker by Jason Saragih et al.
|
||||
*/
|
||||
|
||||
class PAW {
|
||||
public:
|
||||
// Number of pixels after the warping to neutral shape
|
||||
int number_of_pixels;
|
||||
|
||||
// Minimum x coordinate in destination
|
||||
float min_x;
|
||||
|
||||
// minimum y coordinate in destination
|
||||
float min_y;
|
||||
|
||||
// Destination points (landmarks to be warped to)
|
||||
cv::Mat_<float> destination_landmarks;
|
||||
|
||||
// Destination points (landmarks to be warped from)
|
||||
cv::Mat_<float> source_landmarks;
|
||||
|
||||
// Triangulation, each triangle is warped using an affine transform
|
||||
cv::Mat_<int> triangulation;
|
||||
|
||||
// Triangle index, indicating which triangle each of destination pixels lies in
|
||||
cv::Mat_<int> triangle_id;
|
||||
|
||||
// Indicating if the destination warped pixels is valid (lies within a face)
|
||||
cv::Mat_<uchar> pixel_mask;
|
||||
|
||||
// A number of precomputed coefficients that are helpful for quick warping
|
||||
|
||||
// affine coefficients for all triangles (see Matthews and Baker 2004)
|
||||
// 6 coefficients for each triangle (are computed from alpha and beta)
|
||||
// This is computed during each warp based on source landmarks
|
||||
cv::Mat_<float> coefficients;
|
||||
|
||||
// matrix of (c,x,y) coeffs for alpha
|
||||
cv::Mat_<float> alpha;
|
||||
|
||||
// matrix of (c,x,y) coeffs for alpha
|
||||
cv::Mat_<float> beta;
|
||||
|
||||
// x-source of warped points
|
||||
cv::Mat_<float> map_x;
|
||||
|
||||
// y-source of warped points
|
||||
cv::Mat_<float> map_y;
|
||||
|
||||
// Default constructor
|
||||
PAW() { ; }
|
||||
|
||||
// Construct a warp from a destination shape and triangulation
|
||||
PAW(const cv::Mat_<float>& destination_shape, const cv::Mat_<int>& triangulation);
|
||||
|
||||
// The final optional argument allows for optimisation if the triangle indices from previous frame are known (for tracking in video)
|
||||
PAW(const cv::Mat_<float>& destination_shape, const cv::Mat_<int>& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y);
|
||||
|
||||
// Copy constructor
|
||||
PAW(const PAW& other);
|
||||
|
||||
void Read(std::ifstream &s);
|
||||
|
||||
// The actual warping
|
||||
void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_<float>& landmarks_to_warp);
|
||||
|
||||
// Compute coefficients needed for warping
|
||||
void CalcCoeff();
|
||||
|
||||
// Perform the actual warping
|
||||
void WarpRegion(cv::Mat_<float>& map_x, cv::Mat_<float>& map_y);
|
||||
|
||||
inline int NumberOfLandmarks() const { return destination_landmarks.rows / 2; };
|
||||
inline int NumberOfTriangles() const { return triangulation.rows; };
|
||||
|
||||
// The width and height of the warped image
|
||||
inline int constWidth() const { return pixel_mask.cols; }
|
||||
inline int Height() const { return pixel_mask.rows; }
|
||||
|
||||
private:
|
||||
|
||||
// Helper functions for dealing with triangles
|
||||
static bool sameSide(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3);
|
||||
static bool pointInTriangle(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3);
|
||||
static int findTriangle(const cv::Point_<float>& point, const std::vector<std::vector<float>>& control_points, int guess = -1);
|
||||
|
||||
};
|
||||
//===========================================================================
|
||||
}
|
||||
#endif // PAW_H
|
||||
105
pkg/OpenFace/lib/local/LandmarkDetector/include/PDM.h
Normal file
105
pkg/OpenFace/lib/local/LandmarkDetector/include/PDM.h
Normal file
@@ -0,0 +1,105 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef PDM_H
|
||||
#define PDM_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
#include "LandmarkDetectorParameters.h"
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
// A linear 3D Point Distribution Model (constructed using Non-Rigid structure from motion or PCA)
|
||||
// Only describes the model but does not contain an instance of it (no local or global parameters are stored here)
|
||||
// Contains the utility functions to help manipulate the model
|
||||
|
||||
class PDM{
|
||||
public:
|
||||
|
||||
// The 3D mean shape vector of the PDM [x1,..,xn,y1,...yn,z1,...,zn]
|
||||
cv::Mat_<float> mean_shape;
|
||||
|
||||
// Principal components or variation bases of the model,
|
||||
cv::Mat_<float> princ_comp;
|
||||
|
||||
// Eigenvalues (variances) corresponding to the bases
|
||||
cv::Mat_<float> eigen_values;
|
||||
|
||||
PDM(){;}
|
||||
|
||||
// A copy constructor
|
||||
PDM(const PDM& other);
|
||||
|
||||
bool Read(std::string location);
|
||||
|
||||
// Number of vertices
|
||||
inline int NumberOfPoints() const {return mean_shape.rows/3;}
|
||||
|
||||
// Listing the number of modes of variation
|
||||
inline int NumberOfModes() const {return princ_comp.cols;}
|
||||
|
||||
void Clamp(cv::Mat_<float>& params_local, cv::Vec6f& params_global, const FaceModelParameters& params);
|
||||
|
||||
// Compute shape in object space (3D)
|
||||
void CalcShape3D(cv::Mat_<float>& out_shape, const cv::Mat_<float>& params_local) const;
|
||||
|
||||
// Compute shape in image space (2D)
|
||||
void CalcShape2D(cv::Mat_<float>& out_shape, const cv::Mat_<float>& params_local, const cv::Vec6f& params_global) const;
|
||||
|
||||
// provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box
|
||||
void CalcParams(cv::Vec6f& out_params_global, const cv::Rect_<float>& bounding_box, const cv::Mat_<float>& params_local, const cv::Vec3f rotation = cv::Vec3f(0.0f));
|
||||
|
||||
// Provided the landmark location compute global and local parameters best fitting it (can provide optional rotation for potentially better results)
|
||||
void CalcParams(cv::Vec6f& out_params_global, cv::Mat_<float>& out_params_local, const cv::Mat_<float>& landmark_locations, const cv::Vec3f rotation = cv::Vec3f(0.0f));
|
||||
|
||||
// provided the model parameters, compute the bounding box of a face
|
||||
void CalcBoundingBox(cv::Rect_<float>& out_bounding_box, const cv::Vec6f& params_global, const cv::Mat_<float>& params_local);
|
||||
|
||||
// Helpers for computing Jacobians, and Jacobians with the weight matrix
|
||||
void ComputeRigidJacobian(const cv::Mat_<float>& params_local, const cv::Vec6f& params_global, cv::Mat_<float> &Jacob, const cv::Mat_<float> W, cv::Mat_<float> &Jacob_t_w);
|
||||
void ComputeJacobian(const cv::Mat_<float>& params_local, const cv::Vec6f& params_global, cv::Mat_<float> &Jacobian, const cv::Mat_<float> W, cv::Mat_<float> &Jacob_t_w);
|
||||
|
||||
// Given the current parameters, and the computed delta_p compute the updated parameters
|
||||
void UpdateModelParameters(const cv::Mat_<float>& delta_p, cv::Mat_<float>& params_local, cv::Vec6f& params_global);
|
||||
|
||||
private:
|
||||
// Helper utilities
|
||||
static void Orthonormalise(cv::Matx33f &R);
|
||||
};
|
||||
//===========================================================================
|
||||
}
|
||||
#endif // PDM_H
|
||||
125
pkg/OpenFace/lib/local/LandmarkDetector/include/Patch_experts.h
Normal file
125
pkg/OpenFace/lib/local/LandmarkDetector/include/Patch_experts.h
Normal file
@@ -0,0 +1,125 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef PATCH_EXPERTS_H
|
||||
#define PATCH_EXPERTS_H
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
|
||||
#include "SVR_patch_expert.h"
|
||||
#include "CCNF_patch_expert.h"
|
||||
#include "CEN_patch_expert.h"
|
||||
#include "PDM.h"
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
/**
|
||||
Combined class for all of the patch experts
|
||||
*/
|
||||
class Patch_experts
|
||||
{
|
||||
|
||||
public:
|
||||
|
||||
// The collection of SVR patch experts (for intensity/grayscale images), the experts are laid out scale->view->landmark
|
||||
std::vector<std::vector<std::vector<Multi_SVR_patch_expert> > > svr_expert_intensity;
|
||||
|
||||
// The collection of LNF (CCNF) patch experts (for intensity images), the experts are laid out scale->view->landmark
|
||||
std::vector<std::vector<std::vector<CCNF_patch_expert> > > ccnf_expert_intensity;
|
||||
|
||||
// The node connectivity for CCNF experts, at different window sizes and corresponding to separate edge features
|
||||
std::vector<std::vector<cv::Mat_<float> > > sigma_components;
|
||||
|
||||
// The collection of CEN patch experts (for intensity images), the experts are laid out scale->view->landmark
|
||||
std::vector<std::vector<std::vector<CEN_patch_expert> > > cen_expert_intensity;
|
||||
|
||||
//Useful to pre-allocate data for im2col so that it is not allocated for every iteration and every patch
|
||||
std::vector< std::map<int, cv::Mat_<float> > > preallocated_im2col;
|
||||
|
||||
// The available scales for intensity patch experts
|
||||
std::vector<double> patch_scaling;
|
||||
|
||||
// The available views for the patch experts at every scale (in radians)
|
||||
std::vector<std::vector<cv::Vec3d> > centers;
|
||||
|
||||
// Landmark visibilities for each scale and view
|
||||
std::vector<std::vector<cv::Mat_<int> > > visibilities;
|
||||
|
||||
cv::Mat_<int> mirror_inds;
|
||||
cv::Mat_<int> mirror_views;
|
||||
|
||||
// Early termination calibration values, useful for CE-CLM model to speed up the multi-hypothesis setup
|
||||
std::vector<double> early_term_weights;
|
||||
std::vector<double> early_term_biases;
|
||||
std::vector<double> early_term_cutoffs;
|
||||
|
||||
|
||||
// A default constructor
|
||||
Patch_experts(){;}
|
||||
|
||||
// A copy constructor
|
||||
Patch_experts(const Patch_experts& other);
|
||||
|
||||
// Returns the patch expert responses given a grayscale image.
|
||||
// Additionally returns the transform from the image coordinates to the response coordinates (and vice versa).
|
||||
// The computation also requires the current landmark locations to compute response around, the PDM corresponding to the desired model, and the parameters describing its instance
|
||||
// Also need to provide the size of the area of interest and the desired scale of analysis
|
||||
void Response(std::vector<cv::Mat_<float> >& patch_expert_responses, cv::Matx22f& sim_ref_to_img, cv::Matx22f& sim_img_to_ref, const cv::Mat_<float>& grayscale_image,
|
||||
const PDM& pdm, const cv::Vec6f& params_global, const cv::Mat_<float>& params_local, int window_size, int scale);
|
||||
|
||||
// Getting the best view associated with the current orientation
|
||||
int GetViewIdx(const cv::Vec6f& params_global, int scale) const;
|
||||
|
||||
// The number of views at a particular scale
|
||||
inline int nViews(size_t scale = 0) const { return (int)centers[scale].size(); };
|
||||
|
||||
// Reading in all of the patch experts
|
||||
bool Read(std::vector<std::string> intensity_svr_expert_locations, std::vector<std::string> intensity_ccnf_expert_locations,
|
||||
std::vector<std::string> intensity_cen_expert_locations, std::string early_term_loc = "");
|
||||
|
||||
|
||||
private:
|
||||
bool Read_SVR_patch_experts(std::string expert_location, std::vector<cv::Vec3d>& centers, std::vector<cv::Mat_<int> >& visibility, std::vector<std::vector<Multi_SVR_patch_expert> >& patches, double& scale);
|
||||
bool Read_CCNF_patch_experts(std::string patchesFileLocation, std::vector<cv::Vec3d>& centers, std::vector<cv::Mat_<int> >& visibility, std::vector<std::vector<CCNF_patch_expert> >& patches, double& patchScaling);
|
||||
bool Read_CEN_patch_experts(std::string expert_location, std::vector<cv::Vec3d>& centers, std::vector<cv::Mat_<int> >& visibility, std::vector<std::vector<CEN_patch_expert> >& patches, double& scale);
|
||||
|
||||
// Helper for collecting visibilities
|
||||
std::vector<int> Collect_visible_landmarks(std::vector<std::vector<cv::Mat_<int> > > visibilities, int scale, int view_id, int n);
|
||||
};
|
||||
|
||||
}
|
||||
#endif // PATCH_EXPERTS_H
|
||||
@@ -0,0 +1,114 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
// * Any publications arising from the use of this software, including but
|
||||
// not limited to academic journal and conference publications, technical
|
||||
// reports and manuals, must cite at least one of the following works:
|
||||
//
|
||||
// OpenFace 2.0: Facial Behavior Analysis Toolkit
|
||||
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
|
||||
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
|
||||
//
|
||||
// Convolutional experts constrained local model for facial landmark detection.
|
||||
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
|
||||
// in Computer Vision and Pattern Recognition Workshops, 2017.
|
||||
//
|
||||
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
|
||||
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
|
||||
// in IEEE International. Conference on Computer Vision (ICCV), 2015
|
||||
//
|
||||
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
|
||||
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
|
||||
// in Facial Expression Recognition and Analysis Challenge,
|
||||
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
#ifndef SVR_PATCH_EXPERT_H
|
||||
#define SVR_PATCH_EXPERT_H
|
||||
|
||||
// system includes
|
||||
#include <map>
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
|
||||
namespace LandmarkDetector
|
||||
{
|
||||
//===========================================================================
|
||||
/**
|
||||
The classes describing the SVR patch experts
|
||||
*/
|
||||
|
||||
class SVR_patch_expert{
|
||||
public:
|
||||
|
||||
// Type of data the patch expert operated on (0=raw, 1=grad)
|
||||
int type;
|
||||
|
||||
// Logistic regression slope
|
||||
double scaling;
|
||||
|
||||
// Logistic regression bias
|
||||
double bias;
|
||||
|
||||
// Support vector regression weights
|
||||
cv::Mat_<float> weights;
|
||||
|
||||
// Discrete Fourier Transform of SVR weights, precalculated for speed (at different window sizes)
|
||||
std::map<int, cv::Mat_<double> > weights_dfts;
|
||||
|
||||
// Confidence of the current patch expert (used for NU_RLMS optimisation)
|
||||
double confidence;
|
||||
|
||||
SVR_patch_expert(){;}
|
||||
|
||||
// A copy constructor
|
||||
SVR_patch_expert(const SVR_patch_expert& other);
|
||||
|
||||
// Reading in the patch expert
|
||||
void Read(std::ifstream &stream);
|
||||
|
||||
// The actual response computation from intensity or depth (for CLM-Z)
|
||||
void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
|
||||
void ResponseDepth(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
|
||||
|
||||
};
|
||||
//===========================================================================
|
||||
/**
|
||||
A Multi-patch Expert that can include different patch types. Raw pixel values or image gradients
|
||||
*/
|
||||
class Multi_SVR_patch_expert{
|
||||
public:
|
||||
|
||||
// Width and height of the patch expert support area
|
||||
int width;
|
||||
int height;
|
||||
|
||||
// Vector of all of the patch experts (different modalities) for this particular Multi patch expert
|
||||
std::vector<SVR_patch_expert> svr_patch_experts;
|
||||
|
||||
// Default constructor
|
||||
Multi_SVR_patch_expert(){;}
|
||||
|
||||
// Copy constructor
|
||||
Multi_SVR_patch_expert(const Multi_SVR_patch_expert& other);
|
||||
|
||||
void Read(std::ifstream &stream);
|
||||
|
||||
// actual response computation from intensity of depth (for CLM-Z)
|
||||
void Response(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
|
||||
void ResponseDepth(const cv::Mat_<float> &area_of_interest, cv::Mat_<float> &response);
|
||||
|
||||
};
|
||||
}
|
||||
#endif // SVR_PATCH_EXPERT_H
|
||||
74
pkg/OpenFace/lib/local/LandmarkDetector/include/stdafx.h
Normal file
74
pkg/OpenFace/lib/local/LandmarkDetector/include/stdafx.h
Normal file
@@ -0,0 +1,74 @@
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
|
||||
// all rights reserved.
|
||||
//
|
||||
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
|
||||
//
|
||||
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
|
||||
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
|
||||
//
|
||||
// License can be found in OpenFace-license.txt
|
||||
//
|
||||
|
||||
// Precompiled headers stuff
|
||||
|
||||
#ifndef __STDAFX_h_
|
||||
#define __STDAFX_h_
|
||||
|
||||
// OpenCV includes
|
||||
#include <opencv2/core/core.hpp>
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/objdetect.hpp>
|
||||
#include <opencv2/calib3d.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/highgui/highgui.hpp>
|
||||
|
||||
// dlib dependencies for face detection
|
||||
#include <dlib/image_processing/frontal_face_detector.h>
|
||||
#include <dlib/opencv.h>
|
||||
|
||||
// C++ standard stuff
|
||||
#include <stdio.h>
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <vector>
|
||||
#include <map>
|
||||
|
||||
#define _USE_MATH_DEFINES
|
||||
#include <cmath>
|
||||
|
||||
// Filesystem stuff
|
||||
// It can either be in std filesystem (C++17), or in experimental/filesystem (partial C++17 support) or in boost
|
||||
#if __has_include(<boost/filesystem.hpp>)
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
namespace fs = boost::filesystem;
|
||||
#elif __has_include(<filesystem>)
|
||||
#include <filesystem>
|
||||
namespace fs = std::filesystem;
|
||||
#elif __has_include(<experimental/filesystem>)
|
||||
#include <experimental/filesystem>
|
||||
namespace fs = std::filesystem;
|
||||
#endif
|
||||
|
||||
// OpenBLAS stuff
|
||||
|
||||
#include <openblas_config.h>
|
||||
// Instead of including cblas.h and f77blas.h (the definitions from OpenBLAS and other BLAS libraries differ, declare the required OpenBLAS functionality here)
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
/* Assume C declarations for C++ */
|
||||
#endif /* __cplusplus */
|
||||
|
||||
/*Set the number of threads on runtime.*/
|
||||
void openblas_set_num_threads(int num_threads);
|
||||
|
||||
void sgemm_(char *, char *, blasint *, blasint *, blasint *, float *,
|
||||
float *, blasint *, float *, blasint *, float *, float *, blasint *);
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user