open source pkg v1

pkg/OpenFace/lib/local/Utilities/include/ConcurrentQueue.h

@@ -0,0 +1,101 @@
+//
+// Copyright (c) 2013 Juan Palacios juan.palacios.puyana@gmail.com
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met :
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CONCURRENT_QUEUE_
+#define CONCURRENT_QUEUE_
+
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+
+template <typename T>
+class ConcurrentQueue
+{
+public:
+
+	T pop()
+	{
+		std::unique_lock<std::mutex> mlock(mutex_);
+		while (queue_.empty())
+		{
+			cond_empty_.wait(mlock);
+		}
+		auto val = queue_.front();
+		queue_.pop();
+		mlock.unlock();
+		cond_full_.notify_one();
+		return val;
+	}
+
+	void pop(T& item)
+	{
+		std::unique_lock<std::mutex> mlock(mutex_);
+		while (queue_.empty())
+		{
+			cond_empty_.wait(mlock);
+		}
+		item = queue_.front();
+		queue_.pop();
+		mlock.unlock();
+		cond_full_.notify_one();
+	}
+
+	void push(const T& item)
+	{
+		std::unique_lock<std::mutex> mlock(mutex_);
+
+		while (capacity_ > 0 && queue_.size() >= capacity_)
+		{
+			cond_full_.wait(mlock);
+		}
+		queue_.push(item);
+		mlock.unlock();
+		cond_empty_.notify_one();
+	}
+	
+	void set_capacity(int capacity)
+	{
+		std::unique_lock<std::mutex> mlock(mutex_);
+		capacity_ = capacity;
+	}
+
+	bool empty()
+	{
+		std::unique_lock<std::mutex> mlock(mutex_);
+		return queue_.empty();
+	}
+
+	ConcurrentQueue() = default;
+	ConcurrentQueue(const ConcurrentQueue&) = delete;            // disable copying
+	ConcurrentQueue& operator=(const ConcurrentQueue&) = delete; // disable assignment
+
+private:
+	std::queue<T> queue_;
+	std::mutex mutex_;
+	std::condition_variable cond_empty_;
+	std::condition_variable cond_full_;
+	// If capacity greater than one, the queue will block on push if there are too many elements in it
+	int capacity_ = 0;
+};
+
+#endif

pkg/OpenFace/lib/local/Utilities/include/ImageCapture.h

@@ -0,0 +1,121 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef IMAGE_CAPTURE_H
+#define IMAGE_CAPTURE_H
+
+// System includes
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+namespace Utilities
+{
+
+	//===========================================================================
+	/**
+	A class for capturing sequences from video, webcam, and image directories
+	*/
+	class ImageCapture {
+
+	public:
+
+		// Default constructor
+		ImageCapture() {};
+
+		// Opening based on command line arguments
+		bool Open(std::vector<std::string>& arguments);
+
+		// Direct opening
+
+		// Image sequence in the directory
+		bool OpenDirectory(std::string directory, std::string bbox_directory="", float fx = -1, float fy = -1, float cx = -1, float cy = -1);
+
+		// Video file
+		bool OpenImageFiles(const std::vector<std::string>& image_files, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
+
+		// Getting the next frame
+		cv::Mat GetNextImage();
+
+		// Getting the most recent grayscale frame (need to call GetNextImage first)
+		cv::Mat_<uchar> GetGrayFrame();
+
+		// Return bounding boxes associated with the image (if defined)
+		std::vector<cv::Rect_<float> > GetBoundingBoxes();
+
+		// Parameters describing the sequence and it's progress (what's the proportion of images opened)
+		double GetProgress();
+
+		int image_width;
+		int image_height;
+
+		float fx, fy, cx, cy;
+
+		// Name of the video file, image directory, or the webcam
+		std::string name;
+
+		bool has_bounding_boxes;
+
+	private:
+
+		// Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source
+		ImageCapture & operator= (const ImageCapture& other);
+		ImageCapture & operator= (const ImageCapture&& other);
+		ImageCapture(const ImageCapture&& other);
+		ImageCapture(const ImageCapture& other);
+
+		// Storing the latest captures
+		cv::Mat latest_frame;
+		cv::Mat_<uchar> latest_gray_frame;
+
+		// Keeping track of how many files are read and the filenames
+		size_t  frame_num;
+		std::vector<std::string> image_files;
+
+		// Could optionally read the bounding box locations from files (each image could have multiple bounding boxes)
+		std::vector<std::vector<cv::Rect_<float> > > bounding_boxes;
+
+		void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
+
+		bool image_focal_length_set;
+		bool image_optical_center_set;
+
+		bool no_input_specified;
+
+	};
+}
+#endif // IMAGE_CAPTURE_H

pkg/OpenFace/lib/local/Utilities/include/ImageManipulationHelpers.h

@@ -0,0 +1,91 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef IMAGE_MANIPULATION_HELPERS_H
+#define IMAGE_MANIPULATION_HELPERS_H
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/imgproc.hpp>
+
+namespace Utilities
+{
+	//===========================================================================
+	// Converting between color spaces and bit depths
+	//===========================================================================
+
+	static void ConvertToGrayscale_8bit(const cv::Mat& in, cv::Mat& out)
+	{
+		if (in.channels() == 3)
+		{
+			// Make sure it's in a correct format
+			if (in.depth() == CV_16U)
+			{
+				cv::Mat tmp = in / 256;
+				tmp.convertTo(out, CV_8U);
+				cv::cvtColor(out, out, cv::COLOR_BGR2GRAY);
+			}
+			else
+			{
+				cv::cvtColor(in, out, cv::COLOR_BGR2GRAY);
+			}
+		}
+		else if (in.channels() == 4)
+		{
+			if (in.depth() == CV_16U)
+			{
+				cv::Mat tmp = in / 256;
+				tmp.convertTo(out, CV_8U);
+				cv::cvtColor(out, out, cv::COLOR_BGRA2GRAY);
+			}
+			else
+			{
+				cv::cvtColor(in, out, cv::COLOR_BGRA2GRAY);
+			}
+		}
+		else
+		{
+			if (in.depth() == CV_16U)
+			{
+				cv::Mat tmp = in / 256;
+				tmp.convertTo(out, CV_8U);
+			}
+			else if (in.depth() == CV_8U)
+			{
+				out = in.clone();
+			}
+		}
+	}
+
+
+}
+#endif // IMAGE_MANIPULATION_HELPERS_H

pkg/OpenFace/lib/local/Utilities/include/RecorderCSV.h

@@ -0,0 +1,100 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef RECORDER_CSV_H
+#define RECORDER_CSV_H
+
+// System includes
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+
+namespace Utilities
+{
+
+	//===========================================================================
+	/**
+	A class for recording CSV file from OpenFace
+	*/
+	class RecorderCSV {
+
+	public:
+
+		// The constructor for the recorder, need to specify if we are recording a sequence or not
+		RecorderCSV();
+
+		// Opening the file and preparing the header for it
+		bool Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
+			int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
+
+		bool isOpen() const { return output_file.is_open(); }
+
+		// Closing the file and cleaning up
+		void Close();
+
+		void WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
+			const cv::Mat_<float>& landmarks_2D, const cv::Mat_<float>& landmarks_3D, const cv::Mat_<float>& pdm_model_params, const cv::Vec6f& rigid_shape_params, cv::Vec6f& pose_estimate,
+			const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2f& gaze_angle, const std::vector<cv::Point2f>& eye_landmarks2d, const std::vector<cv::Point3f>& eye_landmarks3d,
+			const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
+
+	private:
+
+		// Blocking copy and move, as it doesn't make sense to read to write to the same file
+		RecorderCSV & operator= (const RecorderCSV& other);
+		RecorderCSV & operator= (const RecorderCSV&& other);
+		RecorderCSV(const RecorderCSV&& other);
+		RecorderCSV(const RecorderCSV& other);
+
+		// The actual output file stream that will be written
+		std::ofstream output_file;
+
+		// If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face
+		bool is_sequence;
+
+		// Keep track of what we are recording
+		bool output_2D_landmarks;
+		bool output_3D_landmarks;
+		bool output_model_params;
+		bool output_pose;
+		bool output_AUs;
+		bool output_gaze;
+
+		std::vector<std::string> au_names_class;
+		std::vector<std::string> au_names_reg;
+
+	};
+}
+#endif // RECORDER_CSV_H

pkg/OpenFace/lib/local/Utilities/include/RecorderHOG.h

@@ -0,0 +1,88 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef RECORDER_HOG_H
+#define RECORDER_HOG_H
+
+// System includes
+#include <vector>
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+
+#include <iostream>
+#include <fstream>
+
+namespace Utilities
+{
+
+	//===========================================================================
+	/**
+	A class for recording CSV file from OpenFace
+	*/
+	class RecorderHOG {
+
+	public:
+
+		// The constructor for the recorder, by default does not do anything
+		RecorderHOG();
+		
+		// Adding observations to the recorder
+		void SetObservationHOG(bool success, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
+
+		void Write();
+
+		bool Open(std::string filename);
+
+		void Close();
+
+	private:
+
+		// Blocking copy and move, as it doesn't make sense to read to write to the same file
+		RecorderHOG & operator= (const RecorderHOG& other);
+		RecorderHOG & operator= (const RecorderHOG&& other);
+		RecorderHOG(const RecorderHOG&& other);
+		RecorderHOG(const RecorderHOG& other);
+
+		std::ofstream hog_file;
+
+		// Internals for recording
+		int num_cols;
+		int num_rows;
+		int num_channels;
+		cv::Mat_<double> hog_descriptor;
+		bool good_frame;
+
+	};
+}
+#endif // RECORDER_HOG_H

pkg/OpenFace/lib/local/Utilities/include/RecorderOpenFace.h

@@ -0,0 +1,192 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef RECORDER_OPENFACE_H
+#define RECORDER_OPENFACE_H
+
+#include "RecorderCSV.h"
+#include "RecorderHOG.h"
+#include "RecorderOpenFaceParameters.h"
+
+// System includes
+#include <vector>
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+#include <thread>
+
+#include <ConcurrentQueue.h>
+
+namespace Utilities
+{
+
+	//===========================================================================
+	/**
+	A class for recording data processed by OpenFace (facial landmarks, head pose, facial action units, aligned face, HOG features, and tracked video
+	*/
+	class RecorderOpenFace {
+
+	public:
+
+		// The constructor for the recorder, need to specify if we are recording a sequence or not, in_filename should be just the name and not contain extensions
+		RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::vector<std::string>& arguments);
+		RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::string output_directory);
+
+		~RecorderOpenFace();
+
+		// Closing and cleaning up the recorder
+		void Close();
+
+		// Adding observations to the recorder
+
+		// Required observations for video/image-sequence
+		void SetObservationTimestamp(double timestamp);
+
+		// Required observations for video/image-sequence
+		void SetObservationFrameNumber(int frame_number);
+
+		// If in multiple face mode, identifying which face was tracked
+		void SetObservationFaceID(int face_id);
+
+		// All observations relevant to facial landmarks
+		void SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, const cv::Mat_<float>& landmarks_3D,
+			const cv::Vec6f& params_global, const cv::Mat_<float>& params_local, double confidence, bool success);
+
+		// Pose related observations
+		void SetObservationPose(const cv::Vec6f& pose);
+
+		// AU related observations
+		void SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities, 
+			const std::vector<std::pair<std::string, double> >& au_occurences);
+
+		// Gaze related observations
+		void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1,
+			const cv::Vec2f& gaze_angle, const std::vector<cv::Point2f>& eye_landmarks2D, const std::vector<cv::Point3f>& eye_landmarks3D);
+
+		// Face alignment related observations
+		void SetObservationFaceAlign(const cv::Mat& aligned_face);
+
+		// HOG feature related observations
+		void SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
+
+		void SetObservationVisualization(const cv::Mat &vis_track);
+
+		// Write out all observations for current face (except for tracked image/video)
+		void WriteObservation();
+
+		// Separate method for writing tracked video observation, this is done because video observation is written once a frame/image, other observations can happen multiple times a frame/image
+		void WriteObservationTracked();
+
+		std::string GetCSVFile() { return csv_filename; }
+
+	private:
+
+		// Blocking copy, assignment and move operators, as it does not make sense to save to the same location
+		RecorderOpenFace & operator= (const RecorderOpenFace& other);
+		RecorderOpenFace & operator= (const RecorderOpenFace&& other);
+		RecorderOpenFace(const RecorderOpenFace&& other);
+		RecorderOpenFace(const RecorderOpenFace& other);
+
+		void PrepareRecording(const std::string& in_filename);
+
+		// A thread that will write image and video output (the slowest parts of output_
+		void VideoWritingTask(bool is_sequence);
+		void AlignedImageWritingTask();
+
+		// Keeping track of what to output and how to output it
+		const RecorderOpenFaceParameters params;
+
+		// Keep track of the file and output root location
+		std::string record_root;
+		std::string default_record_directory = "processed"; // By default we are writing in the processed directory in the working directory, if no output parameters provided
+		std::string out_name; // Short name, based on which other names are constructed
+		std::string csv_filename;
+		std::string aligned_output_directory;
+		std::ofstream metadata_file;
+
+		// The actual output file stream that will be written
+		RecorderCSV csv_recorder;
+		RecorderHOG hog_recorder;
+
+		// The actual temporary storage for the observations
+		
+		double timestamp;
+		int face_id;
+		int frame_number;
+
+		// Facial landmark related observations
+		cv::Mat_<float> landmarks_2D;
+		cv::Mat_<float> landmarks_3D;
+		cv::Vec6f pdm_params_global;
+		cv::Mat_<float> pdm_params_local;
+		double landmark_detection_confidence;
+		bool landmark_detection_success;
+
+		// Head pose related observations
+		cv::Vec6f head_pose;
+
+		// Action Unit related observations
+		std::vector<std::pair<std::string, double> > au_intensities;
+		std::vector<std::pair<std::string, double> > au_occurences;
+
+		// Gaze related observations
+		cv::Point3f gaze_direction0;
+		cv::Point3f gaze_direction1;
+		cv::Vec2f gaze_angle;
+		std::vector<cv::Point2f> eye_landmarks2D;
+		std::vector<cv::Point3f> eye_landmarks3D;
+
+		// For video writing
+		cv::VideoWriter video_writer;
+		std::string media_filename;
+		
+		// Do not exceed 100MB in the concurrent queue
+		const int TRACKED_QUEUE_CAPACITY = 100;
+		bool tracked_writing_thread_started;
+		cv::Mat vis_to_out;
+		ConcurrentQueue<std::pair<std::string, cv::Mat> > vis_to_out_queue;
+
+		// For aligned face writing
+		const int ALIGNED_QUEUE_CAPACITY = 100;
+		bool aligned_writing_thread_started;
+		cv::Mat aligned_face;
+		ConcurrentQueue<std::pair<std::string, cv::Mat> > aligned_face_queue;
+
+		std::thread video_writing_thread;
+		std::thread aligned_writing_thread;
+
+	};
+}
+#endif // RECORDER_OPENFACE_H

pkg/OpenFace/lib/local/Utilities/include/RecorderOpenFaceParameters.h

@@ -0,0 +1,117 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+//  Parameters of the Face analyser
+#ifndef RECORDER_OPENFACE_PARAM_H
+#define RECORDER_OPENFACE_PARAM_H
+
+#include <vector>
+#include <opencv2/core/core.hpp>
+
+namespace Utilities
+{
+
+	class RecorderOpenFaceParameters
+	{
+
+	public:
+
+		// Constructors
+		RecorderOpenFaceParameters(std::vector<std::string> &arguments, bool sequence, bool is_from_webcam, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30);
+		RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks,
+			bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked,
+			bool output_aligned_faces, bool record_bad = true, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30);
+
+		bool isSequence() const { return is_sequence; }
+		bool isFromWebcam() const { return is_from_webcam; }
+		bool output2DLandmarks() const { return output_2D_landmarks; }
+		bool output3DLandmarks() const { return output_3D_landmarks; }
+		bool outputPDMParams() const { return output_model_params; }
+		bool outputPose() const { return output_pose; }
+		bool outputAUs() const { return output_AUs; }
+		bool outputGaze() const { return output_gaze; }
+		bool outputHOG() const { return output_hog; }
+		bool outputTracked() const { return output_tracked; }
+		bool outputAlignedFaces() const { return output_aligned_faces; }
+		std::string outputCodec() const { return output_codec; }
+		std::string imageFormatAligned() const { return image_format_aligned; }
+		std::string imageFormatVisualization() const { return image_format_visualization; }
+		double outputFps() const { return fps_vid_out; }
+
+		bool outputBadAligned() const { return record_aligned_bad; }
+
+		float getFx() const { return fx; }
+		float getFy() const { return fy; }
+		float getCx() const { return cx; }
+		float getCy() const { return cy; }
+
+		void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; }
+		void setOutputGaze(bool output_gaze) { this->output_gaze = output_gaze; }
+
+	private:
+		
+		// If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face
+		bool is_sequence;
+		// If the data is coming from a webcam
+		bool is_from_webcam;
+
+		// Keep track of what we are recording
+		bool output_2D_landmarks;
+		bool output_3D_landmarks;
+		bool output_model_params;
+		bool output_pose;
+		bool output_AUs;
+		bool output_gaze;
+		bool output_hog;
+		bool output_tracked;
+		bool output_aligned_faces;
+		
+		// Should the algined faces be recorded even if the detection failed (blank images)
+		bool record_aligned_bad;
+
+		// Some video recording parameters
+		std::string output_codec;
+		double fps_vid_out;
+
+		// Image recording parameters
+		std::string image_format_aligned;
+		std::string image_format_visualization;
+
+		// Camera parameters for recording in the meta file;
+		float fx, fy, cx, cy;
+
+	};
+
+}
+
+#endif // RECORDER_OPENFACE_PARAM_H

pkg/OpenFace/lib/local/Utilities/include/RotationHelpers.h

@@ -0,0 +1,246 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef ROTATION_HELPERS_H
+#define ROTATION_HELPERS_H
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/calib3d.hpp>
+
+namespace Utilities
+{
+	//===========================================================================
+	// Angle representation conversion helpers
+	//===========================================================================
+
+	// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign
+	static cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles)
+	{
+		cv::Matx33f rotation_matrix;
+
+		float s1 = sin(eulerAngles[0]);
+		float s2 = sin(eulerAngles[1]);
+		float s3 = sin(eulerAngles[2]);
+
+		float c1 = cos(eulerAngles[0]);
+		float c2 = cos(eulerAngles[1]);
+		float c3 = cos(eulerAngles[2]);
+
+		rotation_matrix(0, 0) = c2 * c3;
+		rotation_matrix(0, 1) = -c2 *s3;
+		rotation_matrix(0, 2) = s2;
+		rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2;
+		rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3;
+		rotation_matrix(1, 2) = -c2 * s1;
+		rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2;
+		rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3;
+		rotation_matrix(2, 2) = c1 * c2;
+
+		return rotation_matrix;
+	}
+
+	// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign
+	static cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix)
+	{
+		float q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0f;
+		float q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0f*q0);
+		float q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0f*q0);
+		float q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0f*q0);
+
+		// Slower, but dealing with degenerate cases due to precision
+		float t1 = 2.0f * (q0*q2 + q1*q3);
+		if (t1 > 1) t1 = 1.0f;
+		if (t1 < -1) t1 = -1.0f;
+
+		float yaw = asin(t1);
+		float pitch = atan2(2.0f * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3);
+		float roll = atan2(2.0f * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3);
+
+		return cv::Vec3f(pitch, yaw, roll);
+	}
+
+	static cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler)
+	{
+		cv::Matx33f rotMatrix = Euler2RotationMatrix(euler);
+		cv::Vec3f axis_angle;
+		cv::Rodrigues(rotMatrix, axis_angle);
+		return axis_angle;
+	}
+
+	static cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle)
+	{
+		cv::Matx33f rotation_matrix;
+		cv::Rodrigues(axis_angle, rotation_matrix);
+		return RotationMatrix2Euler(rotation_matrix);
+	}
+
+	static cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle)
+	{
+		cv::Matx33f rotation_matrix;
+		cv::Rodrigues(axis_angle, rotation_matrix);
+		return rotation_matrix;
+	}
+
+	static cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix)
+	{
+		cv::Vec3f axis_angle;
+		cv::Rodrigues(rotation_matrix, axis_angle);
+		return axis_angle;
+	}
+
+	// Generally useful 3D functions
+	static void Project(cv::Mat_<float>& dest, const cv::Mat_<float>& mesh, float fx, float fy, float cx, float cy)
+	{
+		dest = cv::Mat_<float>(mesh.rows, 2, 0.0);
+
+		int num_points = mesh.rows;
+
+		float X, Y, Z;
+
+
+		cv::Mat_<float>::const_iterator mData = mesh.begin();
+		cv::Mat_<float>::iterator projected = dest.begin();
+
+		for (int i = 0; i < num_points; i++)
+		{
+			// Get the points
+			X = *(mData++);
+			Y = *(mData++);
+			Z = *(mData++);
+
+			float x;
+			float y;
+
+			// if depth is 0 the projection is different
+			if (Z != 0)
+			{
+				x = ((X * fx / Z) + cx);
+				y = ((Y * fy / Z) + cy);
+			}
+			else
+			{
+				x = X;
+				y = Y;
+			}
+
+			// Project and store in dest matrix
+			(*projected++) = x;
+			(*projected++) = y;
+		}
+
+	}
+
+	//===========================================================================
+	// Point set and landmark manipulation functions
+	//===========================================================================
+	// Using Kabsch's algorithm for aligning shapes
+	//This assumes that align_from and align_to are already mean normalised
+	static cv::Matx22f AlignShapesKabsch2D(const cv::Mat_<float>& align_from, const cv::Mat_<float>& align_to)
+	{
+
+		cv::SVD svd(align_from.t() * align_to);
+
+		// make sure no reflection is there
+		// corr ensures that we do only rotaitons and not reflections
+		double d = cv::determinant(svd.vt.t() * svd.u.t());
+
+		cv::Matx22f corr = cv::Matx22f::eye();
+		if (d > 0)
+		{
+			corr(1, 1) = 1;
+		}
+		else
+		{
+			corr(1, 1) = -1;
+		}
+
+		cv::Matx22f R;
+		cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R);
+
+		return R;
+	}
+
+	//=============================================================================
+	// Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other
+	static cv::Matx22f AlignShapesWithScale(cv::Mat_<float>& src, cv::Mat_<float> dst)
+	{
+		int n = src.rows;
+
+		// First we mean normalise both src and dst
+		float mean_src_x = (float)cv::mean(src.col(0))[0];
+		float mean_src_y = (float)cv::mean(src.col(1))[0];
+
+		float mean_dst_x = (float)cv::mean(dst.col(0))[0];
+		float mean_dst_y = (float)cv::mean(dst.col(1))[0];
+
+		cv::Mat_<float> src_mean_normed = src.clone();
+		src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x;
+		src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y;
+
+		cv::Mat_<float> dst_mean_normed = dst.clone();
+		dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x;
+		dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y;
+
+		// Find the scaling factor of each
+		cv::Mat src_sq;
+		cv::pow(src_mean_normed, 2, src_sq);
+
+		cv::Mat dst_sq;
+		cv::pow(dst_mean_normed, 2, dst_sq);
+
+		float s_src = (float)sqrt(cv::sum(src_sq)[0] / n);
+		float s_dst = (float)sqrt(cv::sum(dst_sq)[0] / n);
+
+		src_mean_normed = src_mean_normed / s_src;
+		dst_mean_normed = dst_mean_normed / s_dst;
+
+		float s = s_dst / s_src;
+
+		// Get the rotation
+		cv::Matx22f R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed);
+
+		cv::Matx22f	A;
+		cv::Mat(s * R).copyTo(A);
+
+		//cv::Mat_<float> aligned = (cv::Mat(cv::Mat(A) * src.t())).t();
+		//cv::Mat_<float> offset = dst - aligned;
+
+		//float t_x = cv::mean(offset.col(0))[0];
+		//float t_y = cv::mean(offset.col(1))[0];
+
+		return A;
+
+	}
+
+}
+#endif // ROTATION_HELPERS_H

pkg/OpenFace/lib/local/Utilities/include/SequenceCapture.h

@@ -0,0 +1,164 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef SEQUENCE_CAPTURE_H
+#define SEQUENCE_CAPTURE_H
+
+// System includes
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+#include <thread>
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+#include <ConcurrentQueue.h>
+
+namespace Utilities
+{
+
+	//===========================================================================
+	/**
+	A class for capturing sequences from video, webcam, and image directories
+	*/
+	class SequenceCapture {
+
+	public:
+
+		// Default constructor
+		SequenceCapture() {};
+
+		// Destructor
+		~SequenceCapture();
+
+		// Opening based on command line arguments
+		bool Open(std::vector<std::string>& arguments);
+
+		// Direct opening
+
+		// Webcam
+		bool OpenWebcam(int device_id, int image_width = 640, int image_height = 480, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
+
+		// Image sequence in the directory
+		bool OpenImageSequence(std::string directory, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
+
+		// Video file
+		bool OpenVideoFile(std::string video_file, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
+
+		bool IsWebcam() { return is_webcam; }
+
+		// Getting the next frame
+		cv::Mat GetNextFrame();
+
+		// Getting the most recent grayscale frame (need to call GetNextFrame first)
+		cv::Mat_<uchar> GetGrayFrame();
+
+		// Parameters describing the sequence and it's progress
+		double GetProgress();
+
+		size_t GetFrameNumber() { return frame_num; }
+
+		bool IsOpened();
+
+		void Close();
+
+		int frame_width;
+		int frame_height;
+
+		float fx, fy, cx, cy;
+
+		double fps;
+
+		double time_stamp;
+
+		// Name of the video file, image directory, or the webcam
+		std::string name;
+
+		// Allows to differentiate if failed because no input specified or if failed to open a specified input
+		bool no_input_specified;
+
+				// Storing the captured data queue
+		static const int CAPTURE_CAPACITY = 200; // 200 MB
+
+	private:
+
+		// For faster input, multi-thread the capture so it is not waiting for processing to be done
+
+		// Used to keep track if the recording is still going (for the writing threads)
+		bool capturing;
+
+		// For keeping track of tasks
+		std::thread capture_thread;
+
+		// A thread that will write video output, so that the rest of the application does not block on it
+		void CaptureThread();
+
+		// Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source, and this would cause issues, especially with webcams
+		SequenceCapture & operator= (const SequenceCapture& other);
+		SequenceCapture & operator= (const SequenceCapture&& other);
+		SequenceCapture(const SequenceCapture&& other);
+		SequenceCapture(const SequenceCapture& other);
+
+		// Used for capturing webcam and video
+		cv::VideoCapture capture;
+
+		// Storing the latest captures
+		cv::Mat latest_frame;
+		cv::Mat_<uchar> latest_gray_frame;
+		
+		// Storing capture timestamp, RGB image, gray image
+		ConcurrentQueue<std::tuple<double, cv::Mat, cv::Mat_<uchar> > > capture_queue;
+
+		// Keeping track of frame number and the files in the image sequence
+		size_t  frame_num;
+		std::vector<std::string> image_files;
+
+		// Length of video allowing to assess progress
+		size_t vid_length;
+
+		// If using a webcam, helps to keep track of time
+		int64 start_time;
+
+		// Keeping track if we are opening a video, webcam or image sequence
+		bool is_webcam;
+		bool is_image_seq;
+
+		void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
+
+
+	};
+}
+#endif // SEQUENCE_CAPTURE_H

pkg/OpenFace/lib/local/Utilities/include/VisualizationUtils.h

@@ -0,0 +1,70 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef VISUALIZATION_UTILS_H
+#define VISUALIZATION_UTILS_H
+#include <stdafx_ut.h>
+
+namespace Utilities
+{
+
+	// Drawing a bounding box around the face in an image
+	void DrawBox(cv::Mat image, cv::Vec6f pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy);
+	void DrawBox(const std::vector<std::pair<cv::Point2f, cv::Point2f>>& lines, cv::Mat image, cv::Scalar color, int thickness);
+
+	// Computing a bounding box to be drawn
+	std::vector<std::pair<cv::Point2f, cv::Point2f>> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy);
+
+    void Visualise_FHOG(const cv::Mat_<double>& descriptor, int num_rows, int num_cols, cv::Mat& visualisation);
+
+	class FpsTracker
+	{
+	public:
+		
+		double history_length;
+
+		void AddFrame();
+
+		double GetFPS();
+
+		FpsTracker();
+
+	private:
+		std::queue<double> frame_times;
+
+		void DiscardOldFrames();
+
+	};
+
+}
+#endif // VISUALIZATION_UTILS_H

pkg/OpenFace/lib/local/Utilities/include/Visualizer.h

@@ -0,0 +1,112 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018  
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.    
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef VISUALIZER_H
+#define VISUALIZER_H
+
+// System includes
+#include <vector>
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+namespace Utilities
+{
+
+	//===========================================================================
+	/**
+	A class for recording data processed by OpenFace (facial landmarks, head pose, facial action units, aligned face, HOG features, and tracked video
+	*/
+	class Visualizer {
+
+	public:
+
+		// The constructor for the visualizer that specifies what to visualize
+		Visualizer(std::vector<std::string> arguments);
+		Visualizer(bool vis_track, bool vis_hog, bool vis_align, bool vis_aus);
+
+		// Adding observations to the visualizer
+		
+		// Pose related observations
+		void SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy);
+
+		// All observations relevant to facial landmarks (optional visibilities parameter to not display all landmarks)
+		void SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities = cv::Mat_<int>());
+
+		// Pose related observations
+		void SetObservationPose(const cv::Vec6f& pose, double confidence);
+		
+		void SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
+
+		// Gaze related observations
+		void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2f>& eye_landmarks, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence);
+
+		// Face alignment related observations
+		void SetObservationFaceAlign(const cv::Mat& aligned_face);
+
+		// HOG feature related observations
+		void SetObservationHOG(const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows);
+
+		void SetFps(double fps);
+
+		// Return key-press that could have resulted in the open windows
+		char ShowObservation();
+
+		cv::Mat GetVisImage();
+		cv::Mat GetHOGVis();
+
+		// Keeping track of what we're visualizing
+		bool vis_track;
+		bool vis_hog;
+		bool vis_align;
+		bool vis_aus;
+
+		// Can be adjusted to show less confident frames
+		double visualisation_boundary = 0.4;
+
+	private:
+
+		// Temporary variables for visualization
+		cv::Mat captured_image; // out canvas
+		cv::Mat tracked_image;
+		cv::Mat hog_image;
+		cv::Mat aligned_face_image;
+		cv::Mat action_units_image;
+
+		// Useful for drawing 3d
+		float fx, fy, cx, cy;
+
+	};
+}
+#endif

pkg/OpenFace/lib/local/Utilities/include/stdafx_ut.h

@@ -0,0 +1,62 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
+// all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+
+// Precompiled headers stuff
+
+#ifndef __STDAFX_UT_h_
+#define __STDAFX_UT_h_
+
+// OpenCV includes
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+
+// For FHOG visualisation
+#include <dlib/opencv.h>
+#include <dlib/image_processing/frontal_face_detector.h>
+
+// C++ standard stuff
+#include <stdio.h>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <locale>
+#include <iomanip>
+
+#include <vector>
+#include <queue>
+#include <map>
+#include <set>
+
+// For sorting
+#include <algorithm>
+
+// For threading and timing
+#include <chrono>
+#include <ctime>
+
+// Filesystem stuff
+// It can either be in std filesystem (C++17), or in experimental/filesystem (partial C++17 support) or in boost
+#if __has_include(<boost/filesystem.hpp>)
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+namespace fs = boost::filesystem;
+#elif __has_include(<filesystem>)
+#include <filesystem>
+namespace fs = std::filesystem;
+#elif __has_include(<experimental/filesystem>)
+#include <experimental/filesystem>
+namespace fs = std::filesystem;
+#endif
+
+#endif