bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
pkg/OpenFace/lib/local/CppInerop/AssemblyInfo.cpp Normal file
View File

@@ -0,0 +1,72 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
using namespace System;
using namespace System::Reflection;
using namespace System::Runtime::CompilerServices;
using namespace System::Runtime::InteropServices;
using namespace System::Security::Permissions;
//
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
//
[assembly:AssemblyTitleAttribute("CppInterop")];
[assembly:AssemblyDescriptionAttribute("")];
[assembly:AssemblyConfigurationAttribute("")];
[assembly:AssemblyCompanyAttribute("")];
[assembly:AssemblyProductAttribute("CppInterop")];
[assembly:AssemblyCopyrightAttribute("Copyright (c) 2016")];
[assembly:AssemblyTrademarkAttribute("")];
[assembly:AssemblyCultureAttribute("")];
//
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the value or you can default the Revision and Build Numbers
// by using the '*' as shown below:
[assembly:AssemblyVersionAttribute("1.0.*")];
[assembly:ComVisible(false)];
[assembly:CLSCompliantAttribute(true)];
[assembly:SecurityPermission(SecurityAction::RequestMinimum, UnmanagedCode = true)];

220
pkg/OpenFace/lib/local/CppInerop/CppInerop.vcxproj Normal file
View File

@@ -0,0 +1,220 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{78196985-EE54-411F-822B-5A23EDF80642}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>CppInerop</RootNamespace>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;$(SolutionDir)lib\3rdParty\CameraEnumerator;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;$(SolutionDir)lib\3rdParty\CameraEnumerator;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;$(SolutionDir)lib\3rdParty\CameraEnumerator;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_WINDOWS;_USRDLL;CPPINEROP_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./;$(SolutionDir)lib\local\LandmarkDetector\include;$(SolutionDir)lib\local\FaceAnalyser\include;$(SolutionDir)lib\local\GazeAnalyser\include;$(SolutionDir)lib\local\Utilities\include;$(SolutionDir)lib\3rdParty\CameraEnumerator;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="AssemblyInfo.cpp" />
<ClCompile Include="CppInterop.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="FaceAnalyserInterop.h" />
<ClInclude Include="GazeAnalyserInterop.h" />
<ClInclude Include="ImageReader.h" />
<ClInclude Include="LandmarkDetectorInterop.h" />
<ClInclude Include="FaceDetectorInterop.h" />
<ClInclude Include="OpenCVWrappers.h" />
<ClInclude Include="RecorderInterop.h" />
<ClInclude Include="SequenceReader.h" />
<ClInclude Include="VisualizerInterop.h" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\3rdParty\CameraEnumerator\CameraEnumerator.vcxproj">
<Project>{50b7d4bf-e33b-41d0-aa89-76bba57bf5cc}</Project>
</ProjectReference>
<ProjectReference Include="..\FaceAnalyser\FaceAnalyser.vcxproj">
<Project>{0e7fc556-0e80-45ea-a876-dde4c2fedcd7}</Project>
</ProjectReference>
<ProjectReference Include="..\GazeAnalyser\GazeAnalyser.vcxproj">
<Project>{5f915541-f531-434f-9c81-79f5db58012b}</Project>
</ProjectReference>
<ProjectReference Include="..\LandmarkDetector\LandmarkDetector.vcxproj">
<Project>{bdc1d107-de17-4705-8e7b-cdde8bfb2bf8}</Project>
</ProjectReference>
<ProjectReference Include="..\Utilities\Utilities.vcxproj">
<Project>{8e741ea2-9386-4cf2-815e-6f9b08991eac}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<Reference Include="PresentationCore" />
<Reference Include="System.Drawing" />
<Reference Include="WindowsBase">
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.0\WindowsBase.dll</HintPath>
</Reference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

54
pkg/OpenFace/lib/local/CppInerop/CppInerop.vcxproj.filters Normal file
View File

@@ -0,0 +1,54 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Header Files">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
</Filter>
<Filter Include="Resource Files">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="AssemblyInfo.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="CppInterop.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="LandmarkDetectorInterop.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="OpenCVWrappers.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="FaceAnalyserInterop.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="GazeAnalyserInterop.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="ImageReader.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="FaceDetectorInterop.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="RecorderInterop.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="VisualizerInterop.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="SequenceReader.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
</Project>

44
pkg/OpenFace/lib/local/CppInerop/CppInterop.cpp Normal file
View File

@@ -0,0 +1,44 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// This is the main DLL file.
#include "GazeAnalyserInterop.h"
#include "LandmarkDetectorInterop.h"
#include "FaceAnalyserInterop.h"
#include "OpenCVWrappers.h"
#include "ImageReader.h"
#include "FaceDetectorInterop.h"
#include "RecorderInterop.h"
#include "VisualizerInterop.h"
#include "SequenceReader.h"

276
pkg/OpenFace/lib/local/CppInerop/FaceAnalyserInterop.h Normal file
View File

@@ -0,0 +1,276 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// FaceAnalyser_Interop.h
#ifndef FACE_ANALYSER_INTEROP_H
#define FACE_ANALYSER_INTEROP_H
#pragma once
// Include all the unmanaged things we need.
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
#pragma unmanaged
#include <opencv2/opencv.hpp>
#include <OpenCVWrappers.h>
#include <Face_utils.h>
#include <FaceAnalyser.h>
#include <VisualizationUtils.h>
#pragma managed
using namespace System::Collections::Generic;
namespace FaceAnalyser_Interop {
public ref class FaceAnalyserManaged
{
private:
FaceAnalysis::FaceAnalyser* face_analyser;
// The actual descriptors (for visualisation and output)
cv::Mat_<float>* hog_features;
cv::Mat* aligned_face;
int* num_rows;
int* num_cols;
public:
FaceAnalyserManaged(System::String^ root, bool dynamic, int output_width, bool mask_aligned)
{
std::string root_std = msclr::interop::marshal_as<std::string>(root);
FaceAnalysis::FaceAnalyserParameters params(root_std);
if (!dynamic)
{
params.OptimizeForImages();
}
params.setAlignedOutput(output_width, -1.0, mask_aligned);
face_analyser = new FaceAnalysis::FaceAnalyser(params);
hog_features = new cv::Mat_<float>();
aligned_face = new cv::Mat();
num_rows = new int;
num_cols = new int;
}
void PostProcessOutputFile(System::String^ file)
{
face_analyser->PostprocessOutputFile(msclr::interop::marshal_as<std::string>(file));
}
void AddNextFrame(OpenCVWrappers::RawImage^ frame, List<System::Tuple<float, float>^>^ landmarks, bool success, bool online) {
// Construct an OpenCV matric from the landmarks
cv::Mat_<float> landmarks_mat(landmarks->Count * 2, 1, 0.0);
for (int i = 0; i < landmarks->Count; ++i)
{
landmarks_mat.at<float>(i, 0) = landmarks[i]->Item1;
landmarks_mat.at<float>(i + landmarks->Count, 0) = landmarks[i]->Item2;
}
//(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam());
face_analyser->AddNextFrame(frame->Mat, landmarks_mat, success, 0, online);
cv::Mat_<double> hog_d;
face_analyser->GetLatestHOG(hog_d, *num_rows, *num_cols);
hog_d.convertTo(*hog_features, CV_32F);
face_analyser->GetLatestAlignedFace(*aligned_face);
}
// Predicting AUs from a single image
System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>^
PredictStaticAUsAndComputeFeatures(OpenCVWrappers::RawImage^ frame, List<System::Tuple<float, float>^>^ landmarks)
{
// Construct an OpenCV matric from the landmarks
cv::Mat_<float> landmarks_mat(landmarks->Count * 2, 1, 0.0);
for (int i = 0; i < landmarks->Count; ++i)
{
landmarks_mat.at<float>(i, 0) = landmarks[i]->Item1;
landmarks_mat.at<float>(i + landmarks->Count, 0) = landmarks[i]->Item2;
}
face_analyser->PredictStaticAUsAndComputeFeatures(frame->Mat, landmarks_mat);
// Set the computed appearance features
cv::Mat_<double> hog_tmp;
face_analyser->GetLatestHOG(hog_tmp, *num_rows, *num_cols);
hog_tmp.convertTo(*hog_features, CV_32F);
face_analyser->GetLatestAlignedFace(*aligned_face);
// Set the computed AUs
auto AU_predictions_intensity = face_analyser->GetCurrentAUsReg();
auto AU_predictions_occurence = face_analyser->GetCurrentAUsClass();
auto au_intensities = gcnew Dictionary<System::String^, double>();
auto au_occurences = gcnew Dictionary<System::String^, double>();
for (auto p : AU_predictions_intensity)
{
au_intensities->Add(gcnew System::String(p.first.c_str()), p.second);
}
for (auto p : AU_predictions_occurence)
{
au_occurences->Add(gcnew System::String(p.first.c_str()), p.second);
}
return gcnew System::Tuple<Dictionary<System::String^, double>^, Dictionary<System::String^, double>^>(au_intensities, au_occurences);
}
List<System::String^>^ GetClassActionUnitsNames()
{
auto names = face_analyser->GetAUClassNames();
auto names_ret = gcnew List<System::String^>();
for(std::string name : names)
{
names_ret->Add(gcnew System::String(name.c_str()));
}
return names_ret;
}
List<System::String^>^ GetRegActionUnitsNames()
{
auto names = face_analyser->GetAURegNames();
auto names_ret = gcnew List<System::String^>();
for(std::string name : names)
{
names_ret->Add(gcnew System::String(name.c_str()));
}
return names_ret;
}
Dictionary<System::String^, double>^ GetCurrentAUsClass()
{
auto classes = face_analyser->GetCurrentAUsClass();
auto au_classes = gcnew Dictionary<System::String^, double>();
for(auto p: classes)
{
au_classes->Add(gcnew System::String(p.first.c_str()), p.second);
}
return au_classes;
}
Dictionary<System::String^, double>^ GetCurrentAUsReg()
{
auto preds = face_analyser->GetCurrentAUsReg();
auto au_preds = gcnew Dictionary<System::String^, double>();
for(auto p: preds)
{
au_preds->Add(gcnew System::String(p.first.c_str()), p.second);
}
return au_preds;
}
OpenCVWrappers::RawImage^ GetLatestAlignedFace() {
OpenCVWrappers::RawImage^ face_aligned_image = gcnew OpenCVWrappers::RawImage(*aligned_face);
return face_aligned_image;
}
OpenCVWrappers::RawImage^ GetLatestHOGFeature() {
OpenCVWrappers::RawImage^ HOG_feature = gcnew OpenCVWrappers::RawImage(*hog_features);
return HOG_feature;
}
// As the number of HOG rows and columns might not be known in advance, have methods for querying them
int GetHOGRows()
{
return *num_rows;
}
int GetHOGCols()
{
return *num_cols;
}
// The number of channels is always the same
int GetHOGChannels()
{
return 31;
}
void Reset()
{
face_analyser->Reset();
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!FaceAnalyserManaged()
{
delete hog_features;
delete aligned_face;
delete num_cols;
delete num_rows;
delete face_analyser;
}
// Destructor. Called on explicit Dispose() only.
~FaceAnalyserManaged()
{
this->!FaceAnalyserManaged();
}
};
}
#endif // FACE_ANALYSER_INTEROP_H

157
pkg/OpenFace/lib/local/CppInerop/FaceDetectorInterop.h Normal file
View File

@@ -0,0 +1,157 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef FACE_DETECTOR_INTEROP_H
#define FACE_DETECTOR_INTEROP_H
#pragma once
// Include all the unmanaged things we need.
#pragma unmanaged
#undef _M_CEE
#include <opencv2/opencv.hpp>
#include <OpenCVWrappers.h>
#include <LandmarkCoreIncludes.h>
#define _M_CEE
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
using namespace System::Collections::Generic;
namespace FaceDetectorInterop {
public ref class FaceDetector
{
private:
// Where the face detectors are stored
dlib::frontal_face_detector* face_detector_hog;
LandmarkDetector::FaceDetectorMTCNN* face_detector_mtcnn;
cv::CascadeClassifier* face_detector_haar;
public:
// The constructor initializes the dlib face detector
FaceDetector(System::String^ haar_location, System::String^ mtcnn_location)
{
// Initialize all of the detectors (TODO should be done on need only basis)
face_detector_hog = new dlib::frontal_face_detector(dlib::get_frontal_face_detector());
face_detector_mtcnn = new LandmarkDetector::FaceDetectorMTCNN(msclr::interop::marshal_as<std::string>(mtcnn_location));
face_detector_haar = new cv::CascadeClassifier(msclr::interop::marshal_as<std::string>(haar_location));
}
// Face detection using HOG-SVM classifier
void DetectFacesHOG(List<System::Windows::Rect>^ o_regions, OpenCVWrappers::RawImage^ intensity, List<float>^ o_confidences)
{
std::vector<cv::Rect_<float> > regions_ocv;
std::vector<float> confidences_std;
::LandmarkDetector::DetectFacesHOG(regions_ocv, intensity->Mat, *face_detector_hog, confidences_std);
o_regions->Clear();
o_confidences->Clear();
for (size_t i = 0; i < regions_ocv.size(); ++i)
{
o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height));
o_confidences->Add(confidences_std[i]);
}
}
// Face detection using HOG-SVM classifier
void DetectFacesHaar(List<System::Windows::Rect>^ o_regions, OpenCVWrappers::RawImage^ intensity, List<float>^ o_confidences)
{
std::vector<cv::Rect_<float> > regions_ocv;
::LandmarkDetector::DetectFaces(regions_ocv, intensity->Mat, *face_detector_haar);
o_regions->Clear();
o_confidences->Clear();
for(size_t i = 0; i < regions_ocv.size(); ++i)
{
o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height));
// As Haar does not provide confidence, create a fake value
o_confidences->Add(1);
}
}
bool IsMTCNNLoaded()
{
return !face_detector_mtcnn->empty();
}
// Face detection using MTCNN face detector
void DetectFacesMTCNN(List<System::Windows::Rect>^ o_regions, OpenCVWrappers::RawImage^ rgb_image, List<float>^ o_confidences)
{
std::vector<cv::Rect_<float> > regions_ocv;
std::vector<float> confidences_std;
::LandmarkDetector::DetectFacesMTCNN(regions_ocv, rgb_image->Mat, *face_detector_mtcnn, confidences_std);
o_regions->Clear();
o_confidences->Clear();
for (size_t i = 0; i < regions_ocv.size(); ++i)
{
o_regions->Add(System::Windows::Rect(regions_ocv[i].x, regions_ocv[i].y, regions_ocv[i].width, regions_ocv[i].height));
o_confidences->Add(confidences_std[i]);
}
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!FaceDetector()
{
delete face_detector_hog;
delete face_detector_mtcnn;
delete face_detector_haar;
}
// Destructor. Called on explicit Dispose() only.
~FaceDetector()
{
this->!FaceDetector();
}
};
}
#endif // FACE_DETECTOR_INTEROP_H

193
pkg/OpenFace/lib/local/CppInerop/GazeAnalyserInterop.h Normal file
View File

@@ -0,0 +1,193 @@
#pragma once
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef GAZE_ANALYSER_INTEROP_H
#define GAZE_ANALYSER_INTEROP_H
#pragma once
// Include all the unmanaged things we need.
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
#pragma unmanaged
#include <opencv2/opencv.hpp>
#include <OpenCVWrappers.h>
#include <LandmarkDetectorInterop.h>
#include <GazeEstimation.h>
#pragma managed
namespace GazeAnalyser_Interop {
public ref class GazeAnalyserManaged
{
private:
// Variable storing gaze for recording
// Absolute gaze direction
cv::Point3f* gazeDirection0;
cv::Point3f* gazeDirection1;
cv::Vec2f* gazeAngle;
cv::Point3f* pupil_left;
cv::Point3f* pupil_right;
public:
GazeAnalyserManaged()
{
gazeDirection0 = new cv::Point3f();
gazeDirection1 = new cv::Point3f();
gazeAngle = new cv::Vec2f();
pupil_left = new cv::Point3f();
pupil_right = new cv::Point3f();
}
void AddNextFrame(CppInterop::LandmarkDetector::CLNF^ clnf, bool success, float fx, float fy, float cx, float cy) {
// After the AUs have been detected do some gaze estimation as well
GazeAnalysis::EstimateGaze(*clnf->getCLNF(), *gazeDirection0, fx, fy, cx, cy, true);
GazeAnalysis::EstimateGaze(*clnf->getCLNF(), *gazeDirection1, fx, fy, cx, cy, false);
// Estimate the gaze angle WRT to head pose here
System::Collections::Generic::List<float>^ pose_list = gcnew System::Collections::Generic::List<float>();
*gazeAngle = GazeAnalysis::GetGazeAngle(*gazeDirection0, *gazeDirection1);
// Grab pupil locations
int part_left = -1;
int part_right = -1;
for (size_t i = 0; i < clnf->getCLNF()->hierarchical_models.size(); ++i)
{
if (clnf->getCLNF()->hierarchical_model_names[i].compare("left_eye_28") == 0)
{
part_left = i;
}
if (clnf->getCLNF()->hierarchical_model_names[i].compare("right_eye_28") == 0)
{
part_right = i;
}
}
cv::Mat_<float> eyeLdmks3d_left = clnf->getCLNF()->hierarchical_models[part_left].GetShape(fx, fy, cx, cy);
cv::Point3f pupil_left_h = GazeAnalysis::GetPupilPosition(eyeLdmks3d_left);
pupil_left->x = pupil_left_h.x; pupil_left->y = pupil_left_h.y; pupil_left->z = pupil_left_h.z;
cv::Mat_<float> eyeLdmks3d_right = clnf->getCLNF()->hierarchical_models[part_right].GetShape(fx, fy, cx, cy);
cv::Point3f pupil_right_h = GazeAnalysis::GetPupilPosition(eyeLdmks3d_right);
pupil_right->x = pupil_right_h.x; pupil_right->y = pupil_right_h.y; pupil_right->z = pupil_right_h.z;
}
System::Tuple<System::Tuple<float, float, float>^, System::Tuple<float, float, float>^>^ GetGazeCamera()
{
auto gaze0 = gcnew System::Tuple<float, float, float>(gazeDirection0->x, gazeDirection0->y, gazeDirection0->z);
auto gaze1 = gcnew System::Tuple<float, float, float>(gazeDirection1->x, gazeDirection1->y, gazeDirection1->z);
return gcnew System::Tuple<System::Tuple<float, float, float>^, System::Tuple<float, float, float>^>(gaze0, gaze1);
}
System::Tuple<float, float>^ GetGazeAngle()
{
auto gaze_angle = gcnew System::Tuple<float, float>((*gazeAngle)[0], (*gazeAngle)[1]);
return gaze_angle;
}
System::Collections::Generic::List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateGazeLines(float fx, float fy, float cx, float cy)
{
std::vector<cv::Point3f> points_left;
points_left.push_back(cv::Point3f(*pupil_left));
points_left.push_back(cv::Point3f(*pupil_left + *gazeDirection0 * 40.0));
std::vector<cv::Point3f> points_right;
points_right.push_back(cv::Point3f(*pupil_right));
points_right.push_back(cv::Point3f(*pupil_right + *gazeDirection1 * 40.0));
// Perform manual projection of points
std::vector<cv::Point2f> imagePoints_left;
for (size_t i = 0; i < points_left.size(); ++i)
{
float x = points_left[i].x * fx / points_left[i].z + cx;
float y = points_left[i].y * fy / points_left[i].z + cy;
imagePoints_left.push_back(cv::Point2f(x, y));
}
std::vector<cv::Point2f> imagePoints_right;
for (size_t i = 0; i < points_right.size(); ++i)
{
float x = points_right[i].x * fx / points_right[i].z + cx;
float y = points_right[i].y * fy / points_right[i].z + cy;
imagePoints_right.push_back(cv::Point2f(x, y));
}
auto lines = gcnew System::Collections::Generic::List<System::Tuple<System::Windows::Point, System::Windows::Point>^>();
lines->Add(gcnew System::Tuple<System::Windows::Point, System::Windows::Point>(System::Windows::Point(imagePoints_left[0].x, imagePoints_left[0].y), System::Windows::Point(imagePoints_left[1].x, imagePoints_left[1].y)));
lines->Add(gcnew System::Tuple<System::Windows::Point, System::Windows::Point>(System::Windows::Point(imagePoints_right[0].x, imagePoints_right[0].y), System::Windows::Point(imagePoints_right[1].x, imagePoints_right[1].y)));
return lines;
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!GazeAnalyserManaged()
{
delete gazeDirection0;
delete gazeDirection1;
delete gazeAngle;
delete pupil_left;
delete pupil_right;
}
// Destructor. Called on explicit Dispose() only.
~GazeAnalyserManaged()
{
this->!GazeAnalyserManaged();
}
};
}
#endif // GAZE_ANALYSER_INTEROP_H

191
pkg/OpenFace/lib/local/CppInerop/ImageReader.h Normal file
View File

@@ -0,0 +1,191 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#pragma once
#pragma unmanaged
// Include all the unmanaged things we need.
#include <opencv2/opencv.hpp>
#include <vector>
#include <set>
#include <OpenCVWrappers.h>
#include <ImageCapture.h>
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
namespace UtilitiesOF {
public ref class ReadingFailedException : System::Exception
{
public:
ReadingFailedException(System::String^ message) : Exception(message) {}
};
public ref class ImageReader
{
private:
// OpenCV based video capture for reading from files
Utilities::ImageCapture* m_image_capture;
OpenCVWrappers::RawImage^ m_rgb_frame;
OpenCVWrappers::RawImage^ m_gray_frame;
bool* m_is_opened;
public:
// Can provide a directory, or a list of files
ImageReader(System::String^ image_directory, float fx, float fy, float cx, float cy)
{
m_image_capture = new Utilities::ImageCapture();
m_is_opened = new bool;
std::string image_dir_std = msclr::interop::marshal_as<std::string>(image_directory);
*m_is_opened = m_image_capture->OpenDirectory(image_dir_std, "", fx, fy, cx, cy);
if (!*m_is_opened)
{
throw gcnew ReadingFailedException("Failed to open a directory or an image");
}
}
// Can provide a directory, or a list of files
ImageReader(System::Collections::Generic::List<System::String^>^ image_files, float fx, float fy, float cx, float cy)
{
m_image_capture = new Utilities::ImageCapture();
m_is_opened = new bool;
std::vector<std::string> image_files_std;
for (int i = 0; i < image_files->Count; ++i)
{
std::string image_file = msclr::interop::marshal_as<std::string>(image_files[i]);
image_files_std.push_back(image_file);
}
*m_is_opened = m_image_capture->OpenImageFiles(image_files_std, fx, fy, cx, cy);
if (!*m_is_opened)
{
throw gcnew ReadingFailedException("Failed to open a directory or an image");
}
}
OpenCVWrappers::RawImage^ GetNextImage()
{
cv::Mat next_image = m_image_capture->GetNextImage();
m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image);
if (next_image.empty())
{
*m_is_opened = false;
}
return m_rgb_frame;
}
System::String^ GetName()
{
std::string filename = m_image_capture->name;
return gcnew System::String(filename.c_str());
}
double GetProgress()
{
return m_image_capture->GetProgress();
}
float GetFx()
{
return m_image_capture->fx;
}
float GetFy()
{
return m_image_capture->fy;
}
float GetCx()
{
return m_image_capture->cx;
}
float GetCy()
{
return m_image_capture->cy;
}
bool isOpened()
{
return *m_is_opened;
}
OpenCVWrappers::RawImage^ GetCurrentFrameGray() {
cv::Mat next_gray_image = m_image_capture->GetGrayFrame();
m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image);
return m_gray_frame;
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!ImageReader()
{
delete m_image_capture;
delete m_rgb_frame;
delete m_gray_frame;
delete m_is_opened;
}
// Destructor. Called on explicit Dispose() only.
~ImageReader()
{
this->!ImageReader();
}
};
}

466
pkg/OpenFace/lib/local/CppInerop/LandmarkDetectorInterop.h Normal file
View File

@@ -0,0 +1,466 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef LANDMARK_DETECTOR_INTEROP_H
#define LANDMARK_DETECTOR_INTEROP_H
#pragma once
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
#pragma unmanaged
// Include all the unmanaged things we need.
#include <opencv2/opencv.hpp>
#include <OpenCVWrappers.h>
#undef _M_CEE
#include <LandmarkCoreIncludes.h>
#include <Face_utils.h>
#include <FaceAnalyser.h>
#include <VisualizationUtils.h>
#define _M_CEE
using namespace System::Collections::Generic;
#pragma managed
namespace CppInterop {
namespace LandmarkDetector {
public ref class FaceModelParameters
{
public:
::LandmarkDetector::FaceModelParameters* params;
public:
// Initialise the parameters
FaceModelParameters(System::String^ root, bool ceclm, bool clnf, bool clm)
{
std::string root_std = msclr::interop::marshal_as<std::string>(root);
std::vector<std::string> args;
args.push_back(root_std);
std::string model_loc = "model/main_ceclm_general.txt";
if (ceclm)
{
model_loc = "model/main_ceclm_general.txt";
}
else if(clnf)
{
model_loc = "model/main_clnf_general.txt";
}
else if (clm)
{
model_loc = "model/main_clm_general.txt";
}
args.push_back("-mloc");
args.push_back(model_loc);
params = new ::LandmarkDetector::FaceModelParameters(args);
}
// TODO this could have optimize for demo mode (also could appropriately update sigma, reg_factor as well)
void optimiseForVideo()
{
params->window_sizes_small = std::vector<int>(4);
params->window_sizes_init = std::vector<int>(4);
// For fast tracking
params->window_sizes_small[0] = 0;
params->window_sizes_small[1] = 9;
params->window_sizes_small[2] = 7;
params->window_sizes_small[3] = 0;
// Just for initialisation
params->window_sizes_init.at(0) = 11;
params->window_sizes_init.at(1) = 9;
params->window_sizes_init.at(2) = 7;
params->window_sizes_init.at(3) = 5;
// For first frame use the initialisation
params->window_sizes_current = params->window_sizes_init;
params->multi_view = false;
params->num_optimisation_iteration = 5;
params->sigma = 1.5;
params->reg_factor = 25;
params->weight_factor = 0;
// Parameter optimizations for CE-CLM
if (params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::CECLM_DETECTOR)
{
params->sigma = 1.5f * params->sigma;
params->reg_factor = 0.9f * params->reg_factor;
}
}
bool IsCECLM()
{
return params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::CECLM_DETECTOR;
}
bool IsCLNF()
{
return params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::CLNF_DETECTOR;
}
bool IsCLM()
{
return params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::CLM_DETECTOR;
}
System::String^ GetMTCNNLocation()
{
return gcnew System::String(params->mtcnn_face_detector_location.c_str());
}
System::String^ GetHaarLocation()
{
return gcnew System::String(params->haar_face_detector_location.c_str());
}
void SetFaceDetector(bool haar, bool hog, bool cnn)
{
if (cnn)
{
params->curr_face_detector = params->MTCNN_DETECTOR;
}
else if (hog)
{
params->curr_face_detector = params->HOG_SVM_DETECTOR;
}
else if (haar)
{
params->curr_face_detector = params->HAAR_DETECTOR;
}
}
void optimiseForImages()
{
params->window_sizes_init = std::vector<int>(4);
params->window_sizes_init[0] = 15;
params->window_sizes_init[1] = 13;
params->window_sizes_init[2] = 11;
params->window_sizes_init[3] = 11;
params->multi_view = true;
params->sigma = 1.25;
params->reg_factor = 35;
params->weight_factor = 2.5;
params->num_optimisation_iteration = 10;
// Parameter optimizations for CE-CLM
if (params->curr_landmark_detector == ::LandmarkDetector::FaceModelParameters::MTCNN_DETECTOR)
{
params->sigma = 1.5f * params->sigma;
params->reg_factor = 0.9f * params->reg_factor;
}
}
::LandmarkDetector::FaceModelParameters* getParams() {
return params;
}
!FaceModelParameters()
{
delete params;
}
~FaceModelParameters()
{
this->!FaceModelParameters();
}
};
public ref class CLNF
{
public:
// A pointer to the CLNF landmark detector
::LandmarkDetector::CLNF* clnf;
public:
// Wrapper functions for the relevant CLNF functionality
CLNF() : clnf(new ::LandmarkDetector::CLNF()) { }
CLNF(FaceModelParameters^ params)
{
clnf = new ::LandmarkDetector::CLNF(params->getParams()->model_location);
}
!CLNF()
{
delete clnf;
}
~CLNF()
{
this->!CLNF();
}
bool isLoaded()
{
return clnf->loaded_successfully;
}
::LandmarkDetector::CLNF* getCLNF() {
return clnf;
}
void Reset() {
clnf->Reset();
}
void Reset(double x, double y) {
clnf->Reset(x, y);
}
bool HasEyeModel() {
return clnf->eye_model;
}
double GetConfidence()
{
return clnf->detection_certainty;
}
bool DetectLandmarksInVideo(OpenCVWrappers::RawImage^ rgb_image, FaceModelParameters^ modelParams, OpenCVWrappers::RawImage^ gray_image) {
return ::LandmarkDetector::DetectLandmarksInVideo(rgb_image->Mat, *clnf, *modelParams->getParams(), gray_image->Mat);
}
bool DetectLandmarksInVideo(OpenCVWrappers::RawImage^ rgb_image, FaceModelParameters^ modelParams) {
return ::LandmarkDetector::DetectLandmarksInVideo(rgb_image->Mat, *clnf, *modelParams->getParams(), cv::Mat());
}
bool DetectFaceLandmarksInImage(OpenCVWrappers::RawImage^ rgb_image, FaceModelParameters^ modelParams, OpenCVWrappers::RawImage^ gray_image) {
return ::LandmarkDetector::DetectLandmarksInImage(rgb_image->Mat, *clnf, *modelParams->getParams(), gray_image->Mat);
}
bool DetectFaceLandmarksInImage(OpenCVWrappers::RawImage^ rgb_image, FaceModelParameters^ modelParams) {
return ::LandmarkDetector::DetectLandmarksInImage(rgb_image->Mat, *clnf, *modelParams->getParams(), cv::Mat());
}
bool DetectFaceLandmarksInImage(OpenCVWrappers::RawImage^ rgb_image, Rect^ bounding_box, FaceModelParameters^ modelParams, OpenCVWrappers::RawImage^ gray_image) {
cv::Rect_<float> bbox(bounding_box->Left, bounding_box->Top, bounding_box->Width, bounding_box->Height);
return ::LandmarkDetector::DetectLandmarksInImage(rgb_image->Mat, bbox, *clnf, *modelParams->getParams(), gray_image->Mat);
}
bool DetectFaceLandmarksInImage(OpenCVWrappers::RawImage^ rgb_image, Rect^ bounding_box, FaceModelParameters^ modelParams) {
cv::Rect_<float> bbox(bounding_box->Left, bounding_box->Top, bounding_box->Width, bounding_box->Height);
return ::LandmarkDetector::DetectLandmarksInImage(rgb_image->Mat, bbox, *clnf, *modelParams->getParams(), cv::Mat());
}
void GetPoseWRTCamera(List<float>^ pose, float fx, float fy, float cx, float cy) {
auto pose_vec = ::LandmarkDetector::GetPoseWRTCamera(*clnf, fx, fy, cx, cy);
pose->Clear();
for(int i = 0; i < 6; ++i)
{
pose->Add(pose_vec[i]);
}
}
void GetPose(List<float>^ pose, float fx, float fy, float cx, float cy) {
auto pose_vec = ::LandmarkDetector::GetPose(*clnf, fx, fy, cx, cy);
pose->Clear();
for(int i = 0; i < 6; ++i)
{
pose->Add(pose_vec[i]);
}
}
// Get the mask of which landmarks are currently visible (not self-occluded)
List<bool>^ GetVisibilities()
{
cv::Mat_<int> vis = clnf->GetVisibilities();
List<bool>^ visibilities = gcnew List<bool>();
for (auto vis_it = vis.begin(); vis_it != vis.end(); vis_it++)
{
visibilities->Add(*vis_it != 0);
}
return visibilities;
}
List<System::Tuple<float,float>^>^ CalculateVisibleLandmarks() {
std::vector<cv::Point2f> vecLandmarks = ::LandmarkDetector::CalculateVisibleLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<float, float>^>();
for(cv::Point2f p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<float, float>(p.x, p.y));
}
return landmarks;
}
List<System::Tuple<float, float>^>^ CalculateAllLandmarks() {
std::vector<cv::Point2f> vecLandmarks = ::LandmarkDetector::CalculateAllLandmarks(*clnf);
auto landmarks = gcnew List<System::Tuple<float, float>^>();
for (cv::Point2f p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<float, float>(p.x, p.y));
}
return landmarks;
}
List<System::Tuple<float, float>^>^ CalculateAllEyeLandmarks() {
std::vector<cv::Point2f> vecLandmarks = ::LandmarkDetector::CalculateAllEyeLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<float, float>^>();
for (cv::Point2f p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<float, float>(p.x, p.y));
}
return landmarks;
}
List<System::Tuple<float, float, float>^>^ CalculateAllEyeLandmarks3D(float fx, float fy, float cx, float cy) {
std::vector<cv::Point3f> vecLandmarks = ::LandmarkDetector::Calculate3DEyeLandmarks(*clnf, fx, fy, cx, cy);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<float, float, float>^>();
for (cv::Point3f p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<float, float, float>(p.x, p.y, p.z));
}
return landmarks;
}
List<System::Tuple<float, float>^>^ CalculateVisibleEyeLandmarks() {
std::vector<cv::Point2f> vecLandmarks = ::LandmarkDetector::CalculateVisibleEyeLandmarks(*clnf);
auto landmarks = gcnew System::Collections::Generic::List<System::Tuple<float, float>^>();
for (cv::Point2f p : vecLandmarks) {
landmarks->Add(gcnew System::Tuple<float, float>(p.x, p.y));
}
return landmarks;
}
List<System::Tuple<float, float, float>^>^ Calculate3DLandmarks(float fx, float fy, float cx, float cy) {
cv::Mat_<float> shape3D = clnf->GetShape(fx, fy, cx, cy);
auto landmarks_3D = gcnew List<System::Tuple<float, float, float>^>();
for(int i = 0; i < shape3D.cols; ++i)
{
landmarks_3D->Add(gcnew System::Tuple<float, float, float>(shape3D.at<float>(0, i), shape3D.at<float>(1, i), shape3D.at<float>(2, i)));
}
return landmarks_3D;
}
List<System::Tuple<System::Windows::Point, System::Windows::Point>^>^ CalculateBox(float fx, float fy, float cx, float cy) {
cv::Vec6f pose = ::LandmarkDetector::GetPose(*clnf, fx,fy, cx, cy);
std::vector<std::pair<cv::Point2f, cv::Point2f>> vecLines = ::Utilities::CalculateBox(pose, fx, fy, cx, cy);
auto lines = gcnew List<System::Tuple<System::Windows::Point,System::Windows::Point>^>();
for(std::pair<cv::Point2f, cv::Point2f> line : vecLines) {
lines->Add(gcnew System::Tuple<System::Windows::Point, System::Windows::Point>(System::Windows::Point(line.first.x, line.first.y), System::Windows::Point(line.second.x, line.second.y)));
}
return lines;
}
int GetNumPoints()
{
return clnf->pdm.NumberOfPoints();
}
int GetNumModes()
{
return clnf->pdm.NumberOfModes();
}
// Getting the non-rigid shape parameters describing the facial expression
List<float>^ GetNonRigidParams()
{
auto non_rigid_params = gcnew List<float>();
for (int i = 0; i < clnf->params_local.rows; ++i)
{
non_rigid_params->Add(clnf->params_local.at<float>(i));
}
return non_rigid_params;
}
// Getting the rigid shape parameters describing face scale rotation and translation (scale,rotx,roty,rotz,tx,ty)
List<float>^ GetRigidParams()
{
auto rigid_params = gcnew List<float>();
for (size_t i = 0; i < 6; ++i)
{
rigid_params->Add(clnf->params_global[i]);
}
return rigid_params;
}
// Rigid params followed by non-rigid ones
List<float>^ GetParams()
{
auto all_params = GetRigidParams();
all_params->AddRange(GetNonRigidParams());
return all_params;
}
};
}
}
#endif // LANDMARK_DETECTOR_INTEROP_H

284
pkg/OpenFace/lib/local/CppInerop/OpenCVWrappers.h Normal file
View File

@@ -0,0 +1,284 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// OpenCVWrappers.h
#pragma once
#pragma managed
#include <msclr\marshal_cppstd.h>
#pragma unmanaged
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#pragma managed
#pragma make_public(cv::Mat)
using namespace System::Windows;
using namespace System::Windows::Threading;
using namespace System::Windows::Media;
using namespace System::Windows::Media::Imaging;
namespace OpenCVWrappers {
public ref class RawImage : System::IDisposable
{
private:
cv::Mat* mat;
public:
static int PixelFormatToType(PixelFormat fmt)
{
if (fmt == PixelFormats::Gray8)
return CV_8UC1;
else if (fmt == PixelFormats::Bgr24)
return CV_8UC3;
else if (fmt == PixelFormats::Bgra32)
return CV_8UC4;
else if (fmt == PixelFormats::Gray32Float)
return CV_32FC1;
else
throw gcnew System::Exception("Unsupported pixel format");
}
static PixelFormat TypeToPixelFormat(int type)
{
switch (type) {
case CV_8UC1:
return PixelFormats::Gray8;
case CV_8UC3:
return PixelFormats::Bgr24;
case CV_8UC4:
return PixelFormats::Bgra32;
case CV_32FC1:
return PixelFormats::Gray32Float;
default:
throw gcnew System::Exception("Unsupported image type");
}
}
RawImage(const cv::Mat& m)
{
mat = new cv::Mat(m.clone());
}
// Constructing a raw image from bitmap
RawImage(System::Drawing::Bitmap^ bitmap)
{
auto fmt = bitmap->PixelFormat;
if (fmt == System::Drawing::Imaging::PixelFormat::Format24bppRgb)
{
mat = new cv::Mat(cv::Size(bitmap->Width, bitmap->Height), CV_8UC3);
}
else if (fmt == System::Drawing::Imaging::PixelFormat::Format32bppArgb)
{
mat = new cv::Mat(cv::Size(bitmap->Width, bitmap->Height), CV_8UC4);
}
else
{
throw gcnew System::Exception("Unsupported image type");
}
auto rect = System::Drawing::Rectangle(0, 0, bitmap->Width, bitmap->Height);
auto bitmap_data = bitmap->LockBits(rect, System::Drawing::Imaging::ImageLockMode::ReadOnly, bitmap->PixelFormat);
System::IntPtr source_data = bitmap_data->Scan0;
int bytes = bitmap_data->Stride * bitmap_data->Height;
memcpy(mat->data, source_data.ToPointer(), bytes);
//Unlock the bits.
bitmap->UnlockBits(bitmap_data);
if (fmt == System::Drawing::Imaging::PixelFormat::Format32bppArgb)
{
cv::cvtColor(*mat, *mat, cv::COLOR_RGBA2RGB);
}
}
void Mirror()
{
cv::flip(*mat, *mat, 1);
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!RawImage()
{
if (mat)
{
delete mat;
mat = NULL;
}
}
// Destructor. Called on explicit Dispose() only.
~RawImage()
{
this->!RawImage();
}
property int Width
{
int get()
{
return mat->cols;
}
}
property int Height
{
int get()
{
return mat->rows;
}
}
property int Stride
{
int get()
{
return (int) mat->step;
}
}
property PixelFormat Format
{
PixelFormat get()
{
return TypeToPixelFormat(mat->type());
}
}
property cv::Mat& Mat
{
cv::Mat& get()
{
return *mat;
}
}
property bool IsEmpty
{
bool get()
{
return !mat || mat->empty();
}
}
bool UpdateWriteableBitmap(WriteableBitmap^ bitmap)
{
if (bitmap == nullptr || bitmap->PixelWidth != Width || bitmap->PixelHeight != Height || bitmap->Format != Format)
return false;
else {
if (mat->data == NULL) {
cv::Mat zeros(bitmap->PixelHeight, bitmap->PixelWidth, PixelFormatToType(bitmap->Format), 0);
bitmap->WritePixels(Int32Rect(0, 0, Width, Height), System::IntPtr(zeros.data), Stride * Height * (Format.BitsPerPixel / 8), Stride, 0, 0);
}
else {
bitmap->WritePixels(Int32Rect(0, 0, Width, Height), System::IntPtr(mat->data), Stride * Height * (Format.BitsPerPixel / 8), Stride, 0, 0);
}
return true;
}
}
WriteableBitmap^ CreateWriteableBitmap()
{
return gcnew WriteableBitmap(Width, Height, 72, 72, Format, nullptr);
}
};
public ref class VideoWriter
{
private:
// OpenCV based video capture for reading from files
cv::VideoWriter* vc;
public:
VideoWriter(System::String^ location, int width, int height, double fps, bool colour)
{
msclr::interop::marshal_context context;
std::string location_std_string = context.marshal_as<std::string>(location);
vc = new cv::VideoWriter(location_std_string, cv::VideoWriter::fourcc('D', 'I', 'V', 'X'), fps, cv::Size(width, height), colour);
}
// Return success
bool Write(RawImage^ img)
{
if (vc != nullptr && vc->isOpened())
{
vc->write(img->Mat);
return true;
}
else
{
return false;
}
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!VideoWriter()
{
if (vc != nullptr)
{
vc->~VideoWriter();
}
}
// Destructor. Called on explicit Dispose() only.
~VideoWriter()
{
this->!VideoWriter();
}
};
}

255
pkg/OpenFace/lib/local/CppInerop/RecorderInterop.h Normal file
View File

@@ -0,0 +1,255 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#pragma once
#pragma unmanaged
// Include all the unmanaged things we need.
#include <RecorderOpenFace.h>
#include <OpenCVWrappers.h>
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
using namespace System::Collections::Generic;
namespace UtilitiesOF {
public ref class RecorderOpenFaceParameters
{
private:
Utilities::RecorderOpenFaceParameters *m_params;
public:
RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked,
bool output_aligned_faces, bool record_bad_aligned, float fx, float fy, float cx, float cy, double fps_vid_out)
{
m_params = new Utilities::RecorderOpenFaceParameters(sequence, is_from_webcam,
output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs,
output_gaze, output_hog, output_tracked, output_aligned_faces, record_bad_aligned, fx, fy, cx, cy, fps_vid_out);
}
Utilities::RecorderOpenFaceParameters * GetParams()
{
return m_params;
}
!RecorderOpenFaceParameters()
{
delete m_params;
}
// Destructor. Called on explicit Dispose() only.
~RecorderOpenFaceParameters()
{
this->!RecorderOpenFaceParameters();
}
};
public ref class RecorderOpenFace
{
private:
// OpenCV based video capture for reading from files
Utilities::RecorderOpenFace* m_recorder;
public:
// Can provide a directory, or a list of files
RecorderOpenFace(System::String^ in_filename, UtilitiesOF::RecorderOpenFaceParameters^ parameters, System::String^ output_directory)
{
std::string in_filename_std = msclr::interop::marshal_as<std::string>(in_filename);
std::string output_directory_std = msclr::interop::marshal_as<std::string>(output_directory);
m_recorder = new Utilities::RecorderOpenFace(in_filename_std, *parameters->GetParams(), output_directory_std);
}
void Close()
{
m_recorder->Close();
}
void WriteObservation()
{
m_recorder->WriteObservation();
}
void WriteObservationTracked()
{
m_recorder->WriteObservationTracked();
}
void SetObservationGaze(System::Tuple<float, float, float>^ gaze_direction0, System::Tuple<float, float, float>^ gaze_direction1, System::Tuple<float, float>^ gaze_angle,
List<System::Tuple<float, float>^>^ landmarks_2D, List<System::Tuple<float, float, float>^>^ landmarks_3D)
{
cv::Point3f gaze_direction0_cv(gaze_direction0->Item1, gaze_direction0->Item2, gaze_direction0->Item3);
cv::Point3f gaze_direction1_cv(gaze_direction1->Item1, gaze_direction1->Item2, gaze_direction1->Item3);
cv::Vec2f gaze_angle_cv(gaze_angle->Item1, gaze_angle->Item2);
// Construct an OpenCV matrix from the landmarks
std::vector<cv::Point2f> landmarks_2D_cv;
for (int i = 0; i < landmarks_2D->Count; ++i)
{
landmarks_2D_cv.push_back(cv::Point2f(landmarks_2D[i]->Item1, landmarks_2D[i]->Item2));
}
// Construct an OpenCV matrix from the landmarks
std::vector<cv::Point3f> landmarks_3D_cv;
for (int i = 0; i < landmarks_3D->Count; ++i)
{
landmarks_3D_cv.push_back(cv::Point3f(landmarks_3D[i]->Item1, landmarks_3D[i]->Item2, landmarks_3D[i]->Item3));
}
m_recorder->SetObservationGaze(gaze_direction0_cv, gaze_direction1_cv, gaze_angle_cv, landmarks_2D_cv, landmarks_3D_cv);
}
System::String^ GetCSVFile()
{
return gcnew System::String(m_recorder->GetCSVFile().c_str());
}
// Setting the observations
void SetObservationTimestamp(double timestamp)
{
m_recorder->SetObservationTimestamp(timestamp);
}
void SetObservationPose(List<float>^ pose)
{
cv::Vec6f pose_vec(pose[0], pose[1], pose[2], pose[3], pose[4], pose[5]);
m_recorder->SetObservationPose(pose_vec);
}
void SetObservationActionUnits(Dictionary<System::String^, double>^ au_regs, Dictionary<System::String^, double>^ au_class)
{
std::vector<std::pair<std::string, double> > au_regs_std;
auto enum_reg = au_regs->GetEnumerator();
while (enum_reg.MoveNext())
{
std::string au_name = msclr::interop::marshal_as<std::string>(enum_reg.Current.Key);
double value = (double)enum_reg.Current.Value;
au_regs_std.push_back(std::pair<std::string, double>(au_name, value));
}
std::vector<std::pair<std::string, double> > au_class_std;
auto enum_class = au_class->GetEnumerator();
while (enum_class.MoveNext())
{
std::string au_name = msclr::interop::marshal_as<std::string>(enum_class.Current.Key);
double value = (double)enum_class.Current.Value;
au_class_std.push_back(std::pair<std::string, double>(au_name, value));
}
m_recorder->SetObservationActionUnits(au_regs_std, au_class_std);
}
void SetObservationFaceAlign(OpenCVWrappers::RawImage^ aligned_face_image)
{
m_recorder->SetObservationFaceAlign(aligned_face_image->Mat);
}
void SetObservationVisualization(OpenCVWrappers::RawImage^ vis_image)
{
m_recorder->SetObservationVisualization(vis_image->Mat);
}
void SetObservationFaceID(int face_id)
{
m_recorder->SetObservationFaceID(face_id);
}
void SetObservationFrameNumber(int frame_number)
{
m_recorder->SetObservationFrameNumber(frame_number);
}
void SetObservationHOG(bool success, OpenCVWrappers::RawImage^ aligned_face_image, int num_cols, int num_rows, int num_channels)
{
m_recorder->SetObservationHOG(success, aligned_face_image->Mat, num_cols, num_rows, num_channels);
}
void SetObservationLandmarks(List<System::Tuple<float, float>^>^ landmarks_2D, List<System::Tuple<float, float, float>^>^ landmarks_3D, List<float>^ params_global, List<float>^ params_local, double confidence, bool success)
{
// Construct an OpenCV matrix from the landmarks
cv::Mat_<float> landmarks_2D_mat(landmarks_2D->Count * 2, 1, 0.0);
for (int i = 0; i < landmarks_2D->Count; ++i)
{
landmarks_2D_mat.at<float>(i, 0) = landmarks_2D[i]->Item1;
landmarks_2D_mat.at<float>(i + landmarks_2D->Count, 0) = landmarks_2D[i]->Item2;
}
// Construct an OpenCV matrix from the landmarks
cv::Mat_<float> landmarks_3D_mat(landmarks_3D->Count * 3, 1, 0.0);
for (int i = 0; i < landmarks_3D->Count; ++i)
{
landmarks_3D_mat.at<float>(i, 0) = landmarks_3D[i]->Item1;
landmarks_3D_mat.at<float>(i + landmarks_3D->Count, 0) = landmarks_3D[i]->Item2;
landmarks_3D_mat.at<float>(i + 2 * landmarks_3D->Count, 0) = landmarks_3D[i]->Item3;
}
cv::Vec6f params_global_vec(params_global[0], params_global[1], params_global[2], params_global[3], params_global[4], params_global[5]);
cv::Mat_<float> params_local_vec(params_local->Count, 1, 0.0);
for (int i = 0; i < params_local->Count; ++i)
{
params_local_vec.at<float>(i, 0) = params_local[i];
}
m_recorder->SetObservationLandmarks(landmarks_2D_mat, landmarks_3D_mat, params_global_vec, params_local_vec, confidence, success);
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!RecorderOpenFace()
{
delete m_recorder;
}
// Destructor. Called on explicit Dispose() only.
~RecorderOpenFace()
{
this->!RecorderOpenFace();
}
};
}

417
pkg/OpenFace/lib/local/CppInerop/SequenceReader.h Normal file
View File

@@ -0,0 +1,417 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#pragma once
#pragma unmanaged
// Include all the unmanaged things we need.
#include <opencv2/opencv.hpp>
#include <vector>
#include <set>
#include <string>
#include <OpenCVWrappers.h>
#include <ImageReader.h>
#include <DeviceEnumerator.h>
#include <SequenceCapture.h>
#pragma managed
#include <msclr\marshal.h>
#include <msclr\marshal_cppstd.h>
using namespace System::Collections::Generic;
namespace UtilitiesOF {
public ref class SequenceReader
{
private:
// OpenCV based video capture for reading from files
Utilities::SequenceCapture* m_sequence_capture;
OpenCVWrappers::RawImage^ m_rgb_frame;
OpenCVWrappers::RawImage^ m_gray_frame;
public:
// Can provide a directory or a video filename, need to specify which
SequenceReader(System::String^ filename, bool directory)
{
m_sequence_capture = new Utilities::SequenceCapture();
std::string name_std = msclr::interop::marshal_as<std::string>(filename);
bool success;
if(directory)
{
success = m_sequence_capture->OpenImageSequence(name_std);
}
else
{
success = m_sequence_capture->OpenVideoFile(name_std);
}
if (!success)
{
throw gcnew ReadingFailedException("Failed to open an image sequence");
}
}
SequenceReader(System::String^ filename, bool directory, float fx, float fy, float cx, float cy)
{
m_sequence_capture = new Utilities::SequenceCapture();
std::string name_std = msclr::interop::marshal_as<std::string>(filename);
bool success;
if (directory)
{
success = m_sequence_capture->OpenImageSequence(name_std, fx, fy, cx, cy);
}
else
{
success = m_sequence_capture->OpenVideoFile(name_std, fx, fy, cx, cy);
}
if (!success)
{
throw gcnew ReadingFailedException("Failed to open an image sequence");
}
}
// Can provide a webcam id
SequenceReader(int webcam_id, int width, int height)
{
m_sequence_capture = new Utilities::SequenceCapture();
bool success = m_sequence_capture->OpenWebcam(webcam_id, width, height);
if (!success)
{
throw gcnew ReadingFailedException("Failed to open an image sequence");
}
}
SequenceReader(int webcam_id, int width, int height, float fx, float fy, float cx, float cy)
{
m_sequence_capture = new Utilities::SequenceCapture();
bool success = m_sequence_capture->OpenWebcam(webcam_id, width, height, fx, fy, cx, cy);
if (!success)
{
throw gcnew ReadingFailedException("Failed to open an image sequence");
}
}
OpenCVWrappers::RawImage^ GetNextImage()
{
cv::Mat next_image = m_sequence_capture->GetNextFrame();
m_rgb_frame = gcnew OpenCVWrappers::RawImage(next_image);
return m_rgb_frame;
}
OpenCVWrappers::RawImage^ GetCurrentFrameGray() {
cv::Mat_<uchar> next_gray_image = m_sequence_capture->GetGrayFrame();
m_gray_frame = gcnew OpenCVWrappers::RawImage(next_gray_image);
return m_gray_frame;
}
int GetFrameNumber()
{
return m_sequence_capture->GetFrameNumber();
}
double GetTimestamp()
{
return m_sequence_capture->time_stamp;
}
System::String^ GetName()
{
std::string filename = m_sequence_capture->name;
return gcnew System::String(filename.c_str());
}
double GetProgress()
{
return m_sequence_capture->GetProgress();
}
float GetFx()
{
return m_sequence_capture->fx;
}
float GetFy()
{
return m_sequence_capture->fy;
}
float GetCx()
{
return m_sequence_capture->cx;
}
float GetCy()
{
return m_sequence_capture->cy;
}
bool IsOpened()
{
return m_sequence_capture->IsOpened();
}
bool IsWebcam()
{
return m_sequence_capture->IsWebcam();
}
double GetFPS()
{
return m_sequence_capture->fps;
}
void Close() {
m_sequence_capture->Close();
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!SequenceReader()
{
delete m_sequence_capture;
delete m_rgb_frame;
delete m_gray_frame;
}
// Destructor. Called on explicit Dispose() only.
~SequenceReader()
{
this->!SequenceReader();
}
private:
// Static methods for listing cameras and their resolutions
static void split(const std::string &s, char delim, std::vector<std::string> &elems) {
std::stringstream ss;
ss.str(s);
std::string item;
while (std::getline(ss, item, delim)) {
elems.push_back(item);
}
}
// Camera listing is camera name and supported resolutions
static Dictionary<System::String^, List<System::Tuple<int, int>^>^>^ GetListingFromFile(std::string filename)
{
// Check what cameras have been written (using OpenCVs XML packages)
cv::FileStorage fs_read(filename, cv::FileStorage::READ);
auto managed_camera_list_initial = gcnew Dictionary<System::String^, List<System::Tuple<int, int>^>^>();
cv::FileNode camera_node_list = fs_read["cameras"];
// iterate through a sequence using FileNodeIterator
for (size_t idx = 0; idx < camera_node_list.size(); idx++)
{
std::string camera_name = (std::string)camera_node_list[idx]["name"];
cv::FileNode resolution_list = camera_node_list[idx]["resolutions"];
auto resolutions = gcnew System::Collections::Generic::List<System::Tuple<int, int>^>();
for (size_t r_idx = 0; r_idx < resolution_list.size(); r_idx++)
{
std::string res = resolution_list[r_idx]["res"];
std::vector<std::string> elems;
split(res, 'x', elems);
int x = stoi(elems[0]);
int y = stoi(elems[1]);
resolutions->Add(gcnew System::Tuple<int, int>(x, y));
}
managed_camera_list_initial[gcnew System::String(camera_name.c_str())] = resolutions;
}
fs_read.release();
return managed_camera_list_initial;
}
static void WriteCameraListingToFile(System::Collections::Generic::Dictionary<System::String^, System::Collections::Generic::List<System::Tuple<int, int>^>^>^ camera_list, std::string filename)
{
cv::FileStorage fs("camera_list.xml", cv::FileStorage::WRITE);
fs << "cameras" << "[";
for each(System::String^ name_m in camera_list->Keys)
{
std::string name = msclr::interop::marshal_as<std::string>(name_m);
fs << "{:" << "name" << name;
fs << "resolutions" << "[";
auto resolutions = camera_list[name_m];
for (int j = 0; j < resolutions->Count; j++)
{
std::stringstream ss;
ss << resolutions[j]->Item1 << "x" << resolutions[j]->Item2;
fs << "{:" << "res" << ss.str();
fs << "}";
}
fs << "]";
fs << "}";
}
fs << "]";
fs.release();
}
// A utility for listing the currently connected cameras together with their ID, name, subset of supported resolutions and a thumbnail
public:
static List<System::Tuple<int, System::String^, List<System::Tuple<int, int>^>^, OpenCVWrappers::RawImage^>^>^ GetCameras(System::String^ root_directory_m)
{
auto managed_camera_list = gcnew List<System::Tuple<int, System::String^, List<System::Tuple<int, int>^>^, OpenCVWrappers::RawImage^>^>();
DeviceEnumerator de;
// Get a listing of all connected video devices
std::map<int, Device> cameras = de.getVideoDevicesMap();
//std::cout << "Number of cameras found: " << cameras.size() << std::endl;
//// Print information about the devices
//for (auto const &device : cameras) {
// std::cout << "== VIDEO DEVICE (id:" << device.first << ") ==" << std::endl;
// std::cout << "Name: " << device.second.deviceName << std::endl;
// std::cout << "Path: " << device.second.devicePath << std::endl;
//}
size_t num_cameras = cameras.size();
// Pre-load supported camera resolutions if already computed
std::string root_directory = msclr::interop::marshal_as<std::string>(root_directory_m);
auto camera_resolution_list = GetListingFromFile(root_directory + "camera_list.xml");
for (size_t i = 0; i < num_cameras; ++i)
{
// Thumbnail to help with camera selection
cv::Mat sample_img;
auto resolutions = gcnew List<System::Tuple<int, int>^>();
// Before trying the resolutions, check if the resolutions have already been computed for the camera of interest
std::string device_name = cameras[i].deviceName;
System::String^ device_name_m = gcnew System::String(device_name.c_str());
if (camera_resolution_list->ContainsKey(device_name_m))
{
resolutions = camera_resolution_list[device_name_m];
// Grab a thumbnail from mid resolution
cv::VideoCapture cap1(i);
auto resolution = resolutions[(int)(resolutions->Count / 2)];
cap1.set(cv::CAP_PROP_FRAME_WIDTH, resolution->Item1);
cap1.set(cv::CAP_PROP_FRAME_HEIGHT, resolution->Item2);
// Read several frames, as the first one often is over-exposed
for (int k = 0; k < 2; ++k)
cap1.read(sample_img);
}
else
{
// A common set of resolutions for webcams
std::vector<std::pair<int, int>> common_resolutions;
common_resolutions.push_back(std::pair<int, int>(320, 240));
common_resolutions.push_back(std::pair<int, int>(640, 480));
common_resolutions.push_back(std::pair<int, int>(800, 600));
common_resolutions.push_back(std::pair<int, int>(960, 720));
common_resolutions.push_back(std::pair<int, int>(1280, 720));
common_resolutions.push_back(std::pair<int, int>(1280, 960));
common_resolutions.push_back(std::pair<int, int>(1920, 1080));
// Grab some sample images and confirm the resolutions
cv::VideoCapture cap1(i);
// Go through resolutions if they have not been identified
for (size_t i = 0; i < common_resolutions.size(); ++i)
{
auto resolution = gcnew System::Tuple<int, int>(common_resolutions[i].first, common_resolutions[i].second);
cap1.set(cv::CAP_PROP_FRAME_WIDTH, resolution->Item1);
cap1.set(cv::CAP_PROP_FRAME_HEIGHT, resolution->Item2);
// Add only valid resolutions as API sometimes provides wrong ones
int set_width = cap1.get(cv::CAP_PROP_FRAME_WIDTH);
int set_height = cap1.get(cv::CAP_PROP_FRAME_HEIGHT);
// Grab a thumbnail from mid resolution
if (i == (int)common_resolutions.size() / 2)
{
// Read several frames, as the first one often is over-exposed
for (int k = 0; k < 2; ++k)
cap1.read(sample_img);
}
resolution = gcnew System::Tuple<int, int>(set_width, set_height);
if (!resolutions->Contains(resolution))
{
resolutions->Add(resolution);
}
}
cap1.~VideoCapture();
// Ass the resolutions were not on the list, add them now
camera_resolution_list[device_name_m] = resolutions;
WriteCameraListingToFile(camera_resolution_list, root_directory + "camera_list.xml");
}
OpenCVWrappers::RawImage^ sample_img_managed = gcnew OpenCVWrappers::RawImage(sample_img);
managed_camera_list->Add(gcnew System::Tuple<int, System::String^, List<System::Tuple<int, int>^>^, OpenCVWrappers::RawImage^>(i, device_name_m, resolutions, sample_img_managed));
}
return managed_camera_list;
}
};
}

169
pkg/OpenFace/lib/local/CppInerop/VisualizerInterop.h Normal file
View File

@@ -0,0 +1,169 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#pragma once
#pragma unmanaged
// Include all the unmanaged things we need.
#include <Visualizer.h>
#include <OpenCVWrappers.h>
#pragma managed
using System::Collections::Generic::List;
namespace UtilitiesOF {
public ref class Visualizer
{
private:
// OpenCV based video capture for reading from files
Utilities::Visualizer* m_visualizer;
public:
Visualizer(bool vis_track, bool vis_hog, bool vis_aligned, bool vis_aus)
{
m_visualizer = new Utilities::Visualizer(vis_track, vis_hog, vis_aligned, vis_aus);
}
void SetObservationGaze(System::Tuple<float, float, float>^ gaze_direction0, System::Tuple<float, float, float>^ gaze_direction1,
List<System::Tuple<float, float>^>^ landmarks_2D, List<System::Tuple<float, float, float>^>^ landmarks_3D,
double confidence)
{
cv::Point3f gaze_direction0_cv(gaze_direction0->Item1, gaze_direction0->Item2, gaze_direction0->Item3);
cv::Point3f gaze_direction1_cv(gaze_direction1->Item1, gaze_direction1->Item2, gaze_direction1->Item3);
// Construct an OpenCV matrix from the landmarks
std::vector<cv::Point2f> landmarks_2D_cv;
for (int i = 0; i < landmarks_2D->Count; ++i)
{
landmarks_2D_cv.push_back(cv::Point2f(landmarks_2D[i]->Item1, landmarks_2D[i]->Item2));
}
// Construct an OpenCV matrix from the landmarks
std::vector<cv::Point3f> landmarks_3D_cv;
for (int i = 0; i < landmarks_3D->Count; ++i)
{
landmarks_3D_cv.push_back(cv::Point3f(landmarks_3D[i]->Item1, landmarks_3D[i]->Item2, landmarks_3D[i]->Item3));
}
m_visualizer->SetObservationGaze(gaze_direction0_cv, gaze_direction1_cv, landmarks_2D_cv, landmarks_3D_cv, confidence);
}
// Setting the observations
void SetObservationPose(List<float>^ pose, double confidence)
{
cv::Vec6f pose_vec(pose[0], pose[1], pose[2], pose[3], pose[4], pose[5]);
m_visualizer->SetObservationPose(pose_vec, confidence);
}
void SetObservationFaceAlign(OpenCVWrappers::RawImage^ aligned_face_image)
{
m_visualizer->SetObservationFaceAlign(aligned_face_image->Mat);
}
void SetObservationHOG(OpenCVWrappers::RawImage^ observation_HOG, int num_cols, int num_rows)
{
m_visualizer->SetObservationHOG(observation_HOG->Mat, num_cols, num_rows);
}
void SetObservationLandmarks(List<System::Tuple<float, float>^>^ landmarks_2D, double confidence, List<bool>^ visibilities)
{
// Construct an OpenCV matrix from the landmarks
cv::Mat_<float> landmarks_2D_mat(landmarks_2D->Count * 2, 1, 0.0);
for (int i = 0; i < landmarks_2D->Count; ++i)
{
landmarks_2D_mat.at<float>(i, 0) = landmarks_2D[i]->Item1;
landmarks_2D_mat.at<float>(i + landmarks_2D->Count, 0) = landmarks_2D[i]->Item2;
}
// Construct an OpenCV matrix from the landmarks
cv::Mat_<int> visibilities_cv(visibilities->Count, 1, 0);
for (int i = 0; i < visibilities->Count; ++i)
{
if (visibilities[i])
{
visibilities_cv.at<int>(i, 0) = 1;
}
else
{
visibilities_cv.at<int>(i, 0) = 0;
}
}
m_visualizer->SetObservationLandmarks(landmarks_2D_mat, confidence, visibilities_cv);
}
void SetObservationLandmarks(List<System::Tuple<float, float>^>^ landmarks_2D, double confidence)
{
SetObservationLandmarks(landmarks_2D, confidence, gcnew List<bool>());
}
void SetImage(OpenCVWrappers::RawImage^ canvas, float fx, float fy, float cx, float cy)
{
m_visualizer->SetImage(canvas->Mat, fx, fy, cx, cy);
}
OpenCVWrappers::RawImage^ GetHOGVis()
{
OpenCVWrappers::RawImage^ hog_image = gcnew OpenCVWrappers::RawImage(m_visualizer->GetHOGVis());
return hog_image;
}
OpenCVWrappers::RawImage^ GetVisImage()
{
OpenCVWrappers::RawImage^ vis_image = gcnew OpenCVWrappers::RawImage(m_visualizer->GetVisImage());
return vis_image;
}
// Finalizer. Definitely called before Garbage Collection,
// but not automatically called on explicit Dispose().
// May be called multiple times.
!Visualizer()
{
// Automatically closes capture object before freeing memory.
delete m_visualizer;
}
// Destructor. Called on explicit Dispose() only.
~Visualizer()
{
this->!Visualizer();
}
};
}