bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

open source pkg v1

Browse Source
This commit is contained in:
Vijay Yadev
2020-08-04 19:12:31 -04:00
parent bef213dba9
commit c389fc2c47
3708 changed files with 1624220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
pkg/OpenFace/lib/local/Utilities/CMakeLists.txt Normal file
View File

@@ -0,0 +1,47 @@
SET(SOURCE
src/ImageCapture.cpp
src/RecorderCSV.cpp
src/RecorderHOG.cpp
src/RecorderOpenFace.cpp
src/RecorderOpenFaceParameters.cpp
src/SequenceCapture.cpp
src/stdafx_ut.cpp
src/VisualizationUtils.cpp
src/Visualizer.cpp
)
SET(HEADERS
include/ImageCapture.h
include/RecorderCSV.h
include/RecorderHOG.h
include/RecorderOpenFace.h
include/RecorderOpenFaceParameters.h
include/SequenceCapture.h
include/stdafx_ut.h
include/VisualizationUtils.h
include/Visualizer.h
include/ConcurrentQueue.h
)
add_library( Utilities ${SOURCE} ${HEADERS})
add_library( OpenFace::Utilities ALIAS Utilities)
target_include_directories(Utilities PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include/OpenFace>)
target_include_directories(Utilities PRIVATE ${UTILITIES_SOURCE_DIR}/include)
target_include_directories(Utilities PUBLIC ${OpenCV_INCLUDE_DIRS})
target_link_libraries(Utilities PUBLIC ${OpenCV_LIBS})
target_link_libraries(Utilities PUBLIC dlib::dlib)
if(${Boost_FOUND})
target_include_directories(Utilities PUBLIC ${Boost_INCLUDE_DIRS})
target_link_libraries(Utilities PUBLIC ${Boost_LIBRARIES})
else()
target_link_libraries(Utilities PUBLIC stdc++fs)
endif()
install (TARGETS Utilities EXPORT OpenFaceTargets LIBRARY DESTINATION lib ARCHIVE DESTINATION lib)
install (FILES ${HEADERS} DESTINATION include/OpenFace)

193
pkg/OpenFace/lib/local/Utilities/Utilities.vcxproj Normal file
View File

@@ -0,0 +1,193 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{8E741EA2-9386-4CF2-815E-6F9B08991EAC}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>Utilities</RootNamespace>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
<ProjectName>Utilities</ProjectName>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_x86.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="..\..\3rdParty\dlib\dlib.props" />
<Import Project="..\..\3rdParty\OpenCV\openCV.props" />
<Import Project="..\..\3rdParty\OpenBLAS\OpenBLAS_64.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup />
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>Use</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
<PrecompiledHeaderFile>stdafx_ut.h</PrecompiledHeaderFile>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>Use</PrecompiledHeader>
<Optimization>Full</Optimization>
<FunctionLevelLinking>
</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>./include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<EnableEnhancedInstructionSet>
</EnableEnhancedInstructionSet>
<LanguageStandard>stdcpp17</LanguageStandard>
<PrecompiledHeaderFile>stdafx_ut.h</PrecompiledHeaderFile>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="src\ImageCapture.cpp" />
<ClCompile Include="src\RecorderCSV.cpp" />
<ClCompile Include="src\RecorderHOG.cpp" />
<ClCompile Include="src\RecorderOpenFace.cpp" />
<ClCompile Include="src\RecorderOpenFaceParameters.cpp" />
<ClCompile Include="src\SequenceCapture.cpp" />
<ClCompile Include="src\stdafx_ut.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
</ClCompile>
<ClCompile Include="src\VisualizationUtils.cpp" />
<ClCompile Include="src\Visualizer.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="include\ConcurrentQueue.h" />
<ClInclude Include="include\ImageCapture.h" />
<ClInclude Include="include\ImageManipulationHelpers.h" />
<ClInclude Include="include\RecorderCSV.h" />
<ClInclude Include="include\RecorderHOG.h" />
<ClInclude Include="include\RecorderOpenFace.h" />
<ClInclude Include="include\RecorderOpenFaceParameters.h" />
<ClInclude Include="include\RotationHelpers.h" />
<ClInclude Include="include\SequenceCapture.h" />
<ClInclude Include="include\stdafx_ut.h" />
<ClInclude Include="include\VisualizationUtils.h" />
<ClInclude Include="include\Visualizer.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

84
pkg/OpenFace/lib/local/Utilities/Utilities.vcxproj.filters Normal file
View File

@@ -0,0 +1,84 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Header Files">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
</Filter>
<Filter Include="Resource Files">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="src\RecorderCSV.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\RecorderHOG.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\RecorderOpenFace.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\RecorderOpenFaceParameters.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\SequenceCapture.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\VisualizationUtils.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\Visualizer.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\ImageCapture.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\stdafx_ut.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="include\RecorderCSV.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\RecorderHOG.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\RecorderOpenFace.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\RecorderOpenFaceParameters.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\SequenceCapture.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\Visualizer.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\VisualizationUtils.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\RotationHelpers.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\ImageCapture.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\ImageManipulationHelpers.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\ConcurrentQueue.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="include\stdafx_ut.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
</Project>

101
pkg/OpenFace/lib/local/Utilities/include/ConcurrentQueue.h Normal file
View File

@@ -0,0 +1,101 @@
//
// Copyright (c) 2013 Juan Palacios juan.palacios.puyana@gmail.com
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met :
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef CONCURRENT_QUEUE_
#define CONCURRENT_QUEUE_
#include <queue>
#include <thread>
#include <mutex>
#include <condition_variable>
template <typename T>
class ConcurrentQueue
{
public:
T pop()
{
std::unique_lock<std::mutex> mlock(mutex_);
while (queue_.empty())
{
cond_empty_.wait(mlock);
}
auto val = queue_.front();
queue_.pop();
mlock.unlock();
cond_full_.notify_one();
return val;
}
void pop(T& item)
{
std::unique_lock<std::mutex> mlock(mutex_);
while (queue_.empty())
{
cond_empty_.wait(mlock);
}
item = queue_.front();
queue_.pop();
mlock.unlock();
cond_full_.notify_one();
}
void push(const T& item)
{
std::unique_lock<std::mutex> mlock(mutex_);
while (capacity_ > 0 && queue_.size() >= capacity_)
{
cond_full_.wait(mlock);
}
queue_.push(item);
mlock.unlock();
cond_empty_.notify_one();
}
void set_capacity(int capacity)
{
std::unique_lock<std::mutex> mlock(mutex_);
capacity_ = capacity;
}
bool empty()
{
std::unique_lock<std::mutex> mlock(mutex_);
return queue_.empty();
}
ConcurrentQueue() = default;
ConcurrentQueue(const ConcurrentQueue&) = delete; // disable copying
ConcurrentQueue& operator=(const ConcurrentQueue&) = delete; // disable assignment
private:
std::queue<T> queue_;
std::mutex mutex_;
std::condition_variable cond_empty_;
std::condition_variable cond_full_;
// If capacity greater than one, the queue will block on push if there are too many elements in it
int capacity_ = 0;
};
#endif

121
pkg/OpenFace/lib/local/Utilities/include/ImageCapture.h Normal file
View File

@@ -0,0 +1,121 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef IMAGE_CAPTURE_H
#define IMAGE_CAPTURE_H
// System includes
#include <fstream>
#include <sstream>
#include <vector>
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
namespace Utilities
{
//===========================================================================
/**
A class for capturing sequences from video, webcam, and image directories
*/
class ImageCapture {
public:
// Default constructor
ImageCapture() {};
// Opening based on command line arguments
bool Open(std::vector<std::string>& arguments);
// Direct opening
// Image sequence in the directory
bool OpenDirectory(std::string directory, std::string bbox_directory="", float fx = -1, float fy = -1, float cx = -1, float cy = -1);
// Video file
bool OpenImageFiles(const std::vector<std::string>& image_files, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
// Getting the next frame
cv::Mat GetNextImage();
// Getting the most recent grayscale frame (need to call GetNextImage first)
cv::Mat_<uchar> GetGrayFrame();
// Return bounding boxes associated with the image (if defined)
std::vector<cv::Rect_<float> > GetBoundingBoxes();
// Parameters describing the sequence and it's progress (what's the proportion of images opened)
double GetProgress();
int image_width;
int image_height;
float fx, fy, cx, cy;
// Name of the video file, image directory, or the webcam
std::string name;
bool has_bounding_boxes;
private:
// Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source
ImageCapture & operator= (const ImageCapture& other);
ImageCapture & operator= (const ImageCapture&& other);
ImageCapture(const ImageCapture&& other);
ImageCapture(const ImageCapture& other);
// Storing the latest captures
cv::Mat latest_frame;
cv::Mat_<uchar> latest_gray_frame;
// Keeping track of how many files are read and the filenames
size_t frame_num;
std::vector<std::string> image_files;
// Could optionally read the bounding box locations from files (each image could have multiple bounding boxes)
std::vector<std::vector<cv::Rect_<float> > > bounding_boxes;
void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
bool image_focal_length_set;
bool image_optical_center_set;
bool no_input_specified;
};
}
#endif // IMAGE_CAPTURE_H

91
pkg/OpenFace/lib/local/Utilities/include/ImageManipulationHelpers.h Normal file
View File

@@ -0,0 +1,91 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef IMAGE_MANIPULATION_HELPERS_H
#define IMAGE_MANIPULATION_HELPERS_H
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc.hpp>
namespace Utilities
{
//===========================================================================
// Converting between color spaces and bit depths
//===========================================================================
static void ConvertToGrayscale_8bit(const cv::Mat& in, cv::Mat& out)
{
if (in.channels() == 3)
{
// Make sure it's in a correct format
if (in.depth() == CV_16U)
{
cv::Mat tmp = in / 256;
tmp.convertTo(out, CV_8U);
cv::cvtColor(out, out, cv::COLOR_BGR2GRAY);
}
else
{
cv::cvtColor(in, out, cv::COLOR_BGR2GRAY);
}
}
else if (in.channels() == 4)
{
if (in.depth() == CV_16U)
{
cv::Mat tmp = in / 256;
tmp.convertTo(out, CV_8U);
cv::cvtColor(out, out, cv::COLOR_BGRA2GRAY);
}
else
{
cv::cvtColor(in, out, cv::COLOR_BGRA2GRAY);
}
}
else
{
if (in.depth() == CV_16U)
{
cv::Mat tmp = in / 256;
tmp.convertTo(out, CV_8U);
}
else if (in.depth() == CV_8U)
{
out = in.clone();
}
}
}
}
#endif // IMAGE_MANIPULATION_HELPERS_H

100
pkg/OpenFace/lib/local/Utilities/include/RecorderCSV.h Normal file
View File

@@ -0,0 +1,100 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef RECORDER_CSV_H
#define RECORDER_CSV_H
// System includes
#include <fstream>
#include <sstream>
#include <vector>
// OpenCV includes
#include <opencv2/core/core.hpp>
namespace Utilities
{
//===========================================================================
/**
A class for recording CSV file from OpenFace
*/
class RecorderCSV {
public:
// The constructor for the recorder, need to specify if we are recording a sequence or not
RecorderCSV();
// Opening the file and preparing the header for it
bool Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg);
bool isOpen() const { return output_file.is_open(); }
// Closing the file and cleaning up
void Close();
void WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
const cv::Mat_<float>& landmarks_2D, const cv::Mat_<float>& landmarks_3D, const cv::Mat_<float>& pdm_model_params, const cv::Vec6f& rigid_shape_params, cv::Vec6f& pose_estimate,
const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2f& gaze_angle, const std::vector<cv::Point2f>& eye_landmarks2d, const std::vector<cv::Point3f>& eye_landmarks3d,
const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
private:
// Blocking copy and move, as it doesn't make sense to read to write to the same file
RecorderCSV & operator= (const RecorderCSV& other);
RecorderCSV & operator= (const RecorderCSV&& other);
RecorderCSV(const RecorderCSV&& other);
RecorderCSV(const RecorderCSV& other);
// The actual output file stream that will be written
std::ofstream output_file;
// If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face
bool is_sequence;
// Keep track of what we are recording
bool output_2D_landmarks;
bool output_3D_landmarks;
bool output_model_params;
bool output_pose;
bool output_AUs;
bool output_gaze;
std::vector<std::string> au_names_class;
std::vector<std::string> au_names_reg;
};
}
#endif // RECORDER_CSV_H

88
pkg/OpenFace/lib/local/Utilities/include/RecorderHOG.h Normal file
View File

@@ -0,0 +1,88 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef RECORDER_HOG_H
#define RECORDER_HOG_H
// System includes
#include <vector>
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <iostream>
#include <fstream>
namespace Utilities
{
//===========================================================================
/**
A class for recording CSV file from OpenFace
*/
class RecorderHOG {
public:
// The constructor for the recorder, by default does not do anything
RecorderHOG();
// Adding observations to the recorder
void SetObservationHOG(bool success, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
void Write();
bool Open(std::string filename);
void Close();
private:
// Blocking copy and move, as it doesn't make sense to read to write to the same file
RecorderHOG & operator= (const RecorderHOG& other);
RecorderHOG & operator= (const RecorderHOG&& other);
RecorderHOG(const RecorderHOG&& other);
RecorderHOG(const RecorderHOG& other);
std::ofstream hog_file;
// Internals for recording
int num_cols;
int num_rows;
int num_channels;
cv::Mat_<double> hog_descriptor;
bool good_frame;
};
}
#endif // RECORDER_HOG_H

192
pkg/OpenFace/lib/local/Utilities/include/RecorderOpenFace.h Normal file
View File

@@ -0,0 +1,192 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef RECORDER_OPENFACE_H
#define RECORDER_OPENFACE_H
#include "RecorderCSV.h"
#include "RecorderHOG.h"
#include "RecorderOpenFaceParameters.h"
// System includes
#include <vector>
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <thread>
#include <ConcurrentQueue.h>
namespace Utilities
{
//===========================================================================
/**
A class for recording data processed by OpenFace (facial landmarks, head pose, facial action units, aligned face, HOG features, and tracked video
*/
class RecorderOpenFace {
public:
// The constructor for the recorder, need to specify if we are recording a sequence or not, in_filename should be just the name and not contain extensions
RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::vector<std::string>& arguments);
RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::string output_directory);
~RecorderOpenFace();
// Closing and cleaning up the recorder
void Close();
// Adding observations to the recorder
// Required observations for video/image-sequence
void SetObservationTimestamp(double timestamp);
// Required observations for video/image-sequence
void SetObservationFrameNumber(int frame_number);
// If in multiple face mode, identifying which face was tracked
void SetObservationFaceID(int face_id);
// All observations relevant to facial landmarks
void SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, const cv::Mat_<float>& landmarks_3D,
const cv::Vec6f& params_global, const cv::Mat_<float>& params_local, double confidence, bool success);
// Pose related observations
void SetObservationPose(const cv::Vec6f& pose);
// AU related observations
void SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities,
const std::vector<std::pair<std::string, double> >& au_occurences);
// Gaze related observations
void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1,
const cv::Vec2f& gaze_angle, const std::vector<cv::Point2f>& eye_landmarks2D, const std::vector<cv::Point3f>& eye_landmarks3D);
// Face alignment related observations
void SetObservationFaceAlign(const cv::Mat& aligned_face);
// HOG feature related observations
void SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels);
void SetObservationVisualization(const cv::Mat &vis_track);
// Write out all observations for current face (except for tracked image/video)
void WriteObservation();
// Separate method for writing tracked video observation, this is done because video observation is written once a frame/image, other observations can happen multiple times a frame/image
void WriteObservationTracked();
std::string GetCSVFile() { return csv_filename; }
private:
// Blocking copy, assignment and move operators, as it does not make sense to save to the same location
RecorderOpenFace & operator= (const RecorderOpenFace& other);
RecorderOpenFace & operator= (const RecorderOpenFace&& other);
RecorderOpenFace(const RecorderOpenFace&& other);
RecorderOpenFace(const RecorderOpenFace& other);
void PrepareRecording(const std::string& in_filename);
// A thread that will write image and video output (the slowest parts of output_
void VideoWritingTask(bool is_sequence);
void AlignedImageWritingTask();
// Keeping track of what to output and how to output it
const RecorderOpenFaceParameters params;
// Keep track of the file and output root location
std::string record_root;
std::string default_record_directory = "processed"; // By default we are writing in the processed directory in the working directory, if no output parameters provided
std::string out_name; // Short name, based on which other names are constructed
std::string csv_filename;
std::string aligned_output_directory;
std::ofstream metadata_file;
// The actual output file stream that will be written
RecorderCSV csv_recorder;
RecorderHOG hog_recorder;
// The actual temporary storage for the observations
double timestamp;
int face_id;
int frame_number;
// Facial landmark related observations
cv::Mat_<float> landmarks_2D;
cv::Mat_<float> landmarks_3D;
cv::Vec6f pdm_params_global;
cv::Mat_<float> pdm_params_local;
double landmark_detection_confidence;
bool landmark_detection_success;
// Head pose related observations
cv::Vec6f head_pose;
// Action Unit related observations
std::vector<std::pair<std::string, double> > au_intensities;
std::vector<std::pair<std::string, double> > au_occurences;
// Gaze related observations
cv::Point3f gaze_direction0;
cv::Point3f gaze_direction1;
cv::Vec2f gaze_angle;
std::vector<cv::Point2f> eye_landmarks2D;
std::vector<cv::Point3f> eye_landmarks3D;
// For video writing
cv::VideoWriter video_writer;
std::string media_filename;
// Do not exceed 100MB in the concurrent queue
const int TRACKED_QUEUE_CAPACITY = 100;
bool tracked_writing_thread_started;
cv::Mat vis_to_out;
ConcurrentQueue<std::pair<std::string, cv::Mat> > vis_to_out_queue;
// For aligned face writing
const int ALIGNED_QUEUE_CAPACITY = 100;
bool aligned_writing_thread_started;
cv::Mat aligned_face;
ConcurrentQueue<std::pair<std::string, cv::Mat> > aligned_face_queue;
std::thread video_writing_thread;
std::thread aligned_writing_thread;
};
}
#endif // RECORDER_OPENFACE_H

117
pkg/OpenFace/lib/local/Utilities/include/RecorderOpenFaceParameters.h Normal file
View File

@@ -0,0 +1,117 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
// Parameters of the Face analyser
#ifndef RECORDER_OPENFACE_PARAM_H
#define RECORDER_OPENFACE_PARAM_H
#include <vector>
#include <opencv2/core/core.hpp>
namespace Utilities
{
class RecorderOpenFaceParameters
{
public:
// Constructors
RecorderOpenFaceParameters(std::vector<std::string> &arguments, bool sequence, bool is_from_webcam, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30);
RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked,
bool output_aligned_faces, bool record_bad = true, float fx = -1, float fy = -1, float cx = -1, float cy = -1, double fps_vid_out = 30);
bool isSequence() const { return is_sequence; }
bool isFromWebcam() const { return is_from_webcam; }
bool output2DLandmarks() const { return output_2D_landmarks; }
bool output3DLandmarks() const { return output_3D_landmarks; }
bool outputPDMParams() const { return output_model_params; }
bool outputPose() const { return output_pose; }
bool outputAUs() const { return output_AUs; }
bool outputGaze() const { return output_gaze; }
bool outputHOG() const { return output_hog; }
bool outputTracked() const { return output_tracked; }
bool outputAlignedFaces() const { return output_aligned_faces; }
std::string outputCodec() const { return output_codec; }
std::string imageFormatAligned() const { return image_format_aligned; }
std::string imageFormatVisualization() const { return image_format_visualization; }
double outputFps() const { return fps_vid_out; }
bool outputBadAligned() const { return record_aligned_bad; }
float getFx() const { return fx; }
float getFy() const { return fy; }
float getCx() const { return cx; }
float getCy() const { return cy; }
void setOutputAUs(bool output_AUs) { this->output_AUs = output_AUs; }
void setOutputGaze(bool output_gaze) { this->output_gaze = output_gaze; }
private:
// If we are recording results from a sequence each row refers to a frame, if we are recording an image each row is a face
bool is_sequence;
// If the data is coming from a webcam
bool is_from_webcam;
// Keep track of what we are recording
bool output_2D_landmarks;
bool output_3D_landmarks;
bool output_model_params;
bool output_pose;
bool output_AUs;
bool output_gaze;
bool output_hog;
bool output_tracked;
bool output_aligned_faces;
// Should the algined faces be recorded even if the detection failed (blank images)
bool record_aligned_bad;
// Some video recording parameters
std::string output_codec;
double fps_vid_out;
// Image recording parameters
std::string image_format_aligned;
std::string image_format_visualization;
// Camera parameters for recording in the meta file;
float fx, fy, cx, cy;
};
}
#endif // RECORDER_OPENFACE_PARAM_H

246
pkg/OpenFace/lib/local/Utilities/include/RotationHelpers.h Normal file
View File

@@ -0,0 +1,246 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef ROTATION_HELPERS_H
#define ROTATION_HELPERS_H
#include <opencv2/core/core.hpp>
#include <opencv2/calib3d.hpp>
namespace Utilities
{
//===========================================================================
// Angle representation conversion helpers
//===========================================================================
// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign
static cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles)
{
cv::Matx33f rotation_matrix;
float s1 = sin(eulerAngles[0]);
float s2 = sin(eulerAngles[1]);
float s3 = sin(eulerAngles[2]);
float c1 = cos(eulerAngles[0]);
float c2 = cos(eulerAngles[1]);
float c3 = cos(eulerAngles[2]);
rotation_matrix(0, 0) = c2 * c3;
rotation_matrix(0, 1) = -c2 *s3;
rotation_matrix(0, 2) = s2;
rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2;
rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3;
rotation_matrix(1, 2) = -c2 * s1;
rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2;
rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3;
rotation_matrix(2, 2) = c1 * c2;
return rotation_matrix;
}
// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign
static cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix)
{
float q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0f;
float q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0f*q0);
float q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0f*q0);
float q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0f*q0);
// Slower, but dealing with degenerate cases due to precision
float t1 = 2.0f * (q0*q2 + q1*q3);
if (t1 > 1) t1 = 1.0f;
if (t1 < -1) t1 = -1.0f;
float yaw = asin(t1);
float pitch = atan2(2.0f * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3);
float roll = atan2(2.0f * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3);
return cv::Vec3f(pitch, yaw, roll);
}
static cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler)
{
cv::Matx33f rotMatrix = Euler2RotationMatrix(euler);
cv::Vec3f axis_angle;
cv::Rodrigues(rotMatrix, axis_angle);
return axis_angle;
}
static cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle)
{
cv::Matx33f rotation_matrix;
cv::Rodrigues(axis_angle, rotation_matrix);
return RotationMatrix2Euler(rotation_matrix);
}
static cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle)
{
cv::Matx33f rotation_matrix;
cv::Rodrigues(axis_angle, rotation_matrix);
return rotation_matrix;
}
static cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix)
{
cv::Vec3f axis_angle;
cv::Rodrigues(rotation_matrix, axis_angle);
return axis_angle;
}
// Generally useful 3D functions
static void Project(cv::Mat_<float>& dest, const cv::Mat_<float>& mesh, float fx, float fy, float cx, float cy)
{
dest = cv::Mat_<float>(mesh.rows, 2, 0.0);
int num_points = mesh.rows;
float X, Y, Z;
cv::Mat_<float>::const_iterator mData = mesh.begin();
cv::Mat_<float>::iterator projected = dest.begin();
for (int i = 0; i < num_points; i++)
{
// Get the points
X = *(mData++);
Y = *(mData++);
Z = *(mData++);
float x;
float y;
// if depth is 0 the projection is different
if (Z != 0)
{
x = ((X * fx / Z) + cx);
y = ((Y * fy / Z) + cy);
}
else
{
x = X;
y = Y;
}
// Project and store in dest matrix
(*projected++) = x;
(*projected++) = y;
}
}
//===========================================================================
// Point set and landmark manipulation functions
//===========================================================================
// Using Kabsch's algorithm for aligning shapes
//This assumes that align_from and align_to are already mean normalised
static cv::Matx22f AlignShapesKabsch2D(const cv::Mat_<float>& align_from, const cv::Mat_<float>& align_to)
{
cv::SVD svd(align_from.t() * align_to);
// make sure no reflection is there
// corr ensures that we do only rotaitons and not reflections
double d = cv::determinant(svd.vt.t() * svd.u.t());
cv::Matx22f corr = cv::Matx22f::eye();
if (d > 0)
{
corr(1, 1) = 1;
}
else
{
corr(1, 1) = -1;
}
cv::Matx22f R;
cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R);
return R;
}
//=============================================================================
// Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other
static cv::Matx22f AlignShapesWithScale(cv::Mat_<float>& src, cv::Mat_<float> dst)
{
int n = src.rows;
// First we mean normalise both src and dst
float mean_src_x = (float)cv::mean(src.col(0))[0];
float mean_src_y = (float)cv::mean(src.col(1))[0];
float mean_dst_x = (float)cv::mean(dst.col(0))[0];
float mean_dst_y = (float)cv::mean(dst.col(1))[0];
cv::Mat_<float> src_mean_normed = src.clone();
src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x;
src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y;
cv::Mat_<float> dst_mean_normed = dst.clone();
dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x;
dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y;
// Find the scaling factor of each
cv::Mat src_sq;
cv::pow(src_mean_normed, 2, src_sq);
cv::Mat dst_sq;
cv::pow(dst_mean_normed, 2, dst_sq);
float s_src = (float)sqrt(cv::sum(src_sq)[0] / n);
float s_dst = (float)sqrt(cv::sum(dst_sq)[0] / n);
src_mean_normed = src_mean_normed / s_src;
dst_mean_normed = dst_mean_normed / s_dst;
float s = s_dst / s_src;
// Get the rotation
cv::Matx22f R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed);
cv::Matx22f A;
cv::Mat(s * R).copyTo(A);
//cv::Mat_<float> aligned = (cv::Mat(cv::Mat(A) * src.t())).t();
//cv::Mat_<float> offset = dst - aligned;
//float t_x = cv::mean(offset.col(0))[0];
//float t_y = cv::mean(offset.col(1))[0];
return A;
}
}
#endif // ROTATION_HELPERS_H

164
pkg/OpenFace/lib/local/Utilities/include/SequenceCapture.h Normal file
View File

@@ -0,0 +1,164 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef SEQUENCE_CAPTURE_H
#define SEQUENCE_CAPTURE_H
// System includes
#include <fstream>
#include <sstream>
#include <vector>
#include <thread>
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <ConcurrentQueue.h>
namespace Utilities
{
//===========================================================================
/**
A class for capturing sequences from video, webcam, and image directories
*/
class SequenceCapture {
public:
// Default constructor
SequenceCapture() {};
// Destructor
~SequenceCapture();
// Opening based on command line arguments
bool Open(std::vector<std::string>& arguments);
// Direct opening
// Webcam
bool OpenWebcam(int device_id, int image_width = 640, int image_height = 480, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
// Image sequence in the directory
bool OpenImageSequence(std::string directory, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
// Video file
bool OpenVideoFile(std::string video_file, float fx = -1, float fy = -1, float cx = -1, float cy = -1);
bool IsWebcam() { return is_webcam; }
// Getting the next frame
cv::Mat GetNextFrame();
// Getting the most recent grayscale frame (need to call GetNextFrame first)
cv::Mat_<uchar> GetGrayFrame();
// Parameters describing the sequence and it's progress
double GetProgress();
size_t GetFrameNumber() { return frame_num; }
bool IsOpened();
void Close();
int frame_width;
int frame_height;
float fx, fy, cx, cy;
double fps;
double time_stamp;
// Name of the video file, image directory, or the webcam
std::string name;
// Allows to differentiate if failed because no input specified or if failed to open a specified input
bool no_input_specified;
// Storing the captured data queue
static const int CAPTURE_CAPACITY = 200; // 200 MB
private:
// For faster input, multi-thread the capture so it is not waiting for processing to be done
// Used to keep track if the recording is still going (for the writing threads)
bool capturing;
// For keeping track of tasks
std::thread capture_thread;
// A thread that will write video output, so that the rest of the application does not block on it
void CaptureThread();
// Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source, and this would cause issues, especially with webcams
SequenceCapture & operator= (const SequenceCapture& other);
SequenceCapture & operator= (const SequenceCapture&& other);
SequenceCapture(const SequenceCapture&& other);
SequenceCapture(const SequenceCapture& other);
// Used for capturing webcam and video
cv::VideoCapture capture;
// Storing the latest captures
cv::Mat latest_frame;
cv::Mat_<uchar> latest_gray_frame;
// Storing capture timestamp, RGB image, gray image
ConcurrentQueue<std::tuple<double, cv::Mat, cv::Mat_<uchar> > > capture_queue;
// Keeping track of frame number and the files in the image sequence
size_t frame_num;
std::vector<std::string> image_files;
// Length of video allowing to assess progress
size_t vid_length;
// If using a webcam, helps to keep track of time
int64 start_time;
// Keeping track if we are opening a video, webcam or image sequence
bool is_webcam;
bool is_image_seq;
void SetCameraIntrinsics(float fx, float fy, float cx, float cy);
};
}
#endif // SEQUENCE_CAPTURE_H

70
pkg/OpenFace/lib/local/Utilities/include/VisualizationUtils.h Normal file
View File

@@ -0,0 +1,70 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef VISUALIZATION_UTILS_H
#define VISUALIZATION_UTILS_H
#include <stdafx_ut.h>
namespace Utilities
{
// Drawing a bounding box around the face in an image
void DrawBox(cv::Mat image, cv::Vec6f pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy);
void DrawBox(const std::vector<std::pair<cv::Point2f, cv::Point2f>>& lines, cv::Mat image, cv::Scalar color, int thickness);
// Computing a bounding box to be drawn
std::vector<std::pair<cv::Point2f, cv::Point2f>> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy);
void Visualise_FHOG(const cv::Mat_<double>& descriptor, int num_rows, int num_cols, cv::Mat& visualisation);
class FpsTracker
{
public:
double history_length;
void AddFrame();
double GetFPS();
FpsTracker();
private:
std::queue<double> frame_times;
void DiscardOldFrames();
};
}
#endif // VISUALIZATION_UTILS_H

112
pkg/OpenFace/lib/local/Utilities/include/Visualizer.h Normal file
View File

@@ -0,0 +1,112 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#ifndef VISUALIZER_H
#define VISUALIZER_H
// System includes
#include <vector>
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
namespace Utilities
{
//===========================================================================
/**
A class for recording data processed by OpenFace (facial landmarks, head pose, facial action units, aligned face, HOG features, and tracked video
*/
class Visualizer {
public:
// The constructor for the visualizer that specifies what to visualize
Visualizer(std::vector<std::string> arguments);
Visualizer(bool vis_track, bool vis_hog, bool vis_align, bool vis_aus);
// Adding observations to the visualizer
// Pose related observations
void SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy);
// All observations relevant to facial landmarks (optional visibilities parameter to not display all landmarks)
void SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities = cv::Mat_<int>());
// Pose related observations
void SetObservationPose(const cv::Vec6f& pose, double confidence);
void SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
// Gaze related observations
void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2f>& eye_landmarks, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence);
// Face alignment related observations
void SetObservationFaceAlign(const cv::Mat& aligned_face);
// HOG feature related observations
void SetObservationHOG(const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows);
void SetFps(double fps);
// Return key-press that could have resulted in the open windows
char ShowObservation();
cv::Mat GetVisImage();
cv::Mat GetHOGVis();
// Keeping track of what we're visualizing
bool vis_track;
bool vis_hog;
bool vis_align;
bool vis_aus;
// Can be adjusted to show less confident frames
double visualisation_boundary = 0.4;
private:
// Temporary variables for visualization
cv::Mat captured_image; // out canvas
cv::Mat tracked_image;
cv::Mat hog_image;
cv::Mat aligned_face_image;
cv::Mat action_units_image;
// Useful for drawing 3d
float fx, fy, cx, cy;
};
}
#endif

62
pkg/OpenFace/lib/local/Utilities/include/stdafx_ut.h Normal file
View File

@@ -0,0 +1,62 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// Precompiled headers stuff
#ifndef __STDAFX_UT_h_
#define __STDAFX_UT_h_
// OpenCV includes
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc.hpp>
// For FHOG visualisation
#include <dlib/opencv.h>
#include <dlib/image_processing/frontal_face_detector.h>
// C++ standard stuff
#include <stdio.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <locale>
#include <iomanip>
#include <vector>
#include <queue>
#include <map>
#include <set>
// For sorting
#include <algorithm>
// For threading and timing
#include <chrono>
#include <ctime>
// Filesystem stuff
// It can either be in std filesystem (C++17), or in experimental/filesystem (partial C++17 support) or in boost
#if __has_include(<boost/filesystem.hpp>)
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
namespace fs = boost::filesystem;
#elif __has_include(<filesystem>)
#include <filesystem>
namespace fs = std::filesystem;
#elif __has_include(<experimental/filesystem>)
#include <experimental/filesystem>
namespace fs = std::filesystem;
#endif
#endif

415
pkg/OpenFace/lib/local/Utilities/src/ImageCapture.cpp Normal file
View File

@@ -0,0 +1,415 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "ImageCapture.h"
#include "ImageManipulationHelpers.h"
using namespace Utilities;
#define INFO_STREAM( stream ) \
std::cout << stream << std::endl
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
#define ERROR_STREAM( stream ) \
std::cout << "Error: " << stream << std::endl
bool ImageCapture::Open(std::vector<std::string>& arguments)
{
// Consuming the input arguments
bool* valid = new bool[arguments.size()];
for (size_t i = 0; i < arguments.size(); ++i)
{
valid[i] = true;
}
// Some default values
std::string input_root = "";
fx = -1; fy = -1; cx = -1; cy = -1;
std::string separator = std::string(1, fs::path::preferred_separator);
// First check if there is a root argument (so that videos and input directories could be defined more easily)
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-root") == 0)
{
input_root = arguments[i + 1] + separator;
i++;
}
if (arguments[i].compare("-inroot") == 0)
{
input_root = arguments[i + 1] + separator;
i++;
}
}
std::string input_directory;
std::string bbox_directory;
bool directory_found = false;
has_bounding_boxes = false;
std::vector<std::string> input_image_files;
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-f") == 0)
{
input_image_files.push_back(input_root + arguments[i + 1]);
valid[i] = false;
valid[i + 1] = false;
i++;
}
else if (arguments[i].compare("-fdir") == 0)
{
if (directory_found)
{
WARN_STREAM("Input directory already found, using the first one:" + input_directory);
}
else
{
input_directory = (input_root + arguments[i + 1]);
valid[i] = false;
valid[i + 1] = false;
i++;
directory_found = true;
}
}
else if (arguments[i].compare("-bboxdir") == 0)
{
bbox_directory = (input_root + arguments[i + 1]);
valid[i] = false;
valid[i + 1] = false;
has_bounding_boxes = true;
i++;
}
else if (arguments[i].compare("-fx") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> fx;
i++;
}
else if (arguments[i].compare("-fy") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> fy;
i++;
}
else if (arguments[i].compare("-cx") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> cx;
i++;
}
else if (arguments[i].compare("-cy") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> cy;
i++;
}
}
for (int i = (int)arguments.size() - 1; i >= 0; --i)
{
if (!valid[i])
{
arguments.erase(arguments.begin() + i);
}
}
// Based on what was read in open the sequence
if (!input_image_files.empty())
{
return OpenImageFiles(input_image_files, fx, fy, cx, cy);
}
if (!input_directory.empty())
{
return OpenDirectory(input_directory, bbox_directory, fx, fy, cx, cy);
}
// If no input found return false and set a flag for it
no_input_specified = true;
return false;
}
bool ImageCapture::OpenImageFiles(const std::vector<std::string>& image_files, float fx, float fy, float cx, float cy)
{
// Setting some defaults
frame_num = 0;
no_input_specified = false;
latest_frame = cv::Mat();
latest_gray_frame = cv::Mat();
this->image_files = image_files;
// Allow for setting the camera intrinsics, but have to be the same ones for every image
if (fx != -1 && fy != -1 )
{
image_focal_length_set = true;
this->fx = fx;
this->fy = fy;
}
else
{
image_focal_length_set = false;
}
if (cx != -1 && cy != -1)
{
this->cx = cx;
this->cy = cy;
image_optical_center_set = true;
}
else
{
image_optical_center_set = false;
}
return true;
}
bool ImageCapture::OpenDirectory(std::string directory, std::string bbox_directory, float fx, float fy, float cx, float cy)
{
INFO_STREAM("Attempting to read from directory: " << directory);
// Setup some defaults
frame_num = 0;
no_input_specified = false;
image_files.clear();
fs::path image_directory(directory);
std::vector<fs::path> file_in_directory;
copy(fs::directory_iterator(image_directory), fs::directory_iterator(), back_inserter(file_in_directory));
// Sort the images in the directory first
sort(file_in_directory.begin(), file_in_directory.end());
std::vector<std::string> curr_dir_files;
for (std::vector<fs::path>::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator)
{
// Possible image extension .jpg and .png
if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".jpeg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0)
{
curr_dir_files.push_back(file_iterator->string());
// If bounding box directory is specified, read the bounding boxes from it
if (!bbox_directory.empty())
{
fs::path current_file = *file_iterator;
fs::path bbox_file = bbox_directory / current_file.filename().replace_extension("txt");
// If there is a bounding box file push it to the list of bounding boxes
if (fs::exists(bbox_file))
{
std::ifstream in_bbox(bbox_file.string().c_str(), std::ios_base::in);
std::vector<cv::Rect_<float> > bboxes_image;
// Keep reading bounding boxes from a file, stop if empty line or
while (!in_bbox.eof())
{
std::string bbox_string;
std::getline(in_bbox, bbox_string);
if (bbox_string.empty())
continue;
std::stringstream ss(bbox_string);
float min_x, min_y, max_x, max_y;
ss >> min_x >> min_y >> max_x >> max_y;
bboxes_image.push_back(cv::Rect_<float>(min_x, min_y, max_x - min_x, max_y - min_y));
}
in_bbox.close();
bounding_boxes.push_back(bboxes_image);
}
else
{
ERROR_STREAM("Could not find the corresponding bounding box for file:" + file_iterator->string());
exit(1);
}
}
}
}
image_files = curr_dir_files;
if (image_files.empty())
{
std::cout << "No images found in the directory: " << directory << std::endl;
return false;
}
// Allow for setting the camera intrinsics, but have to be the same ones for every image
if (fx != -1 && fy != -1)
{
image_focal_length_set = true;
this->fx = fx;
this->fy = fy;
}
else
{
image_focal_length_set = false;
}
if (cx != -1 && cy != -1)
{
this->cx = cx;
this->cy = cy;
image_optical_center_set = true;
}
else
{
image_optical_center_set = false;
}
return true;
}
void ImageCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy)
{
// If optical centers are not defined just use center of image
if (cx == -1)
{
this->cx = this->image_width / 2.0f;
this->cy = this->image_height / 2.0f;
}
else
{
this->cx = cx;
this->cy = cy;
}
// Use a rough guess-timate of focal length
if (fx == -1)
{
this->fx = 500.0f * (this->image_width / 640.0f);
this->fy = 500.0f * (this->image_height / 480.0f);
this->fx = (this->fx + this->fy) / 2.0f;
this->fy = this->fx;
}
else
{
this->fx = fx;
this->fy = fy;
}
}
// Returns a read image in 3 channel RGB format, also prepares a grayscale frame if needed
cv::Mat ImageCapture::GetNextImage()
{
if (image_files.empty() || frame_num >= image_files.size())
{
// Indicate lack of success by returning an empty image
latest_frame = cv::Mat();
return latest_frame;
}
// Load the image as an 8 bit RGB
latest_frame = cv::imread(image_files[frame_num], cv::IMREAD_COLOR);
if (latest_frame.empty())
{
ERROR_STREAM("Could not open the image: " + image_files[frame_num]);
exit(1);
}
image_height = latest_frame.size().height;
image_width = latest_frame.size().width;
// Reset the intrinsics for every image if they are not set globally
float _fx = -1;
float _fy = -1;
if (image_focal_length_set)
{
_fx = fx;
_fy = fy;
}
float _cx = -1;
float _cy = -1;
if (image_optical_center_set)
{
_cx = cx;
_cy = cy;
}
SetCameraIntrinsics(_fx, _fy, _cx, _cy);
// Set the grayscale frame
ConvertToGrayscale_8bit(latest_frame, latest_gray_frame);
this->name = image_files[frame_num];
frame_num++;
return latest_frame;
}
std::vector<cv::Rect_<float> > ImageCapture::GetBoundingBoxes()
{
if (!bounding_boxes.empty())
{
return bounding_boxes[frame_num - 1];
}
else
{
return std::vector<cv::Rect_<float> >();
}
}
double ImageCapture::GetProgress()
{
return (double)frame_num / (double)image_files.size();
}
cv::Mat_<uchar> ImageCapture::GetGrayFrame()
{
return latest_gray_frame;
}

341
pkg/OpenFace/lib/local/Utilities/src/RecorderCSV.cpp Normal file
View File

@@ -0,0 +1,341 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "RecorderCSV.h"
using namespace Utilities;
// Default constructor initializes the variables
RecorderCSV::RecorderCSV():output_file(){};
// Making sure full stop is used for decimal point separation
struct fullstop : std::numpunct<char> {
char do_decimal_point() const { return '.'; }
};
// Opening the file and preparing the header for it
bool RecorderCSV::Open(std::string output_file_name, bool is_sequence, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze,
int num_face_landmarks, int num_model_modes, int num_eye_landmarks, const std::vector<std::string>& au_names_class, const std::vector<std::string>& au_names_reg)
{
output_file.open(output_file_name, std::ios_base::out);
output_file.imbue(std::locale(output_file.getloc(), new fullstop));
if (!output_file.is_open())
return false;
this->is_sequence = is_sequence;
// Set up what we are recording
this->output_2D_landmarks = output_2D_landmarks;
this->output_3D_landmarks = output_3D_landmarks;
this->output_AUs = output_AUs;
this->output_gaze = output_gaze;
this->output_model_params = output_model_params;
this->output_pose = output_pose;
this->au_names_class = au_names_class;
this->au_names_reg = au_names_reg;
// Different headers if we are writing out the results on a sequence or an individual image
if(this->is_sequence)
{
output_file << "frame, face_id, timestamp, confidence, success";
}
else
{
output_file << "face, confidence";
}
if (output_gaze)
{
output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y";
for (int i = 0; i < num_eye_landmarks; ++i)
{
output_file << ", eye_lmk_x_" << i;
}
for (int i = 0; i < num_eye_landmarks; ++i)
{
output_file << ", eye_lmk_y_" << i;
}
for (int i = 0; i < num_eye_landmarks; ++i)
{
output_file << ", eye_lmk_X_" << i;
}
for (int i = 0; i < num_eye_landmarks; ++i)
{
output_file << ", eye_lmk_Y_" << i;
}
for (int i = 0; i < num_eye_landmarks; ++i)
{
output_file << ", eye_lmk_Z_" << i;
}
}
if (output_pose)
{
output_file << ", pose_Tx, pose_Ty, pose_Tz, pose_Rx, pose_Ry, pose_Rz";
}
if (output_2D_landmarks)
{
for (int i = 0; i < num_face_landmarks; ++i)
{
output_file << ", x_" << i;
}
for (int i = 0; i < num_face_landmarks; ++i)
{
output_file << ", y_" << i;
}
}
if (output_3D_landmarks)
{
for (int i = 0; i < num_face_landmarks; ++i)
{
output_file << ", X_" << i;
}
for (int i = 0; i < num_face_landmarks; ++i)
{
output_file << ", Y_" << i;
}
for (int i = 0; i < num_face_landmarks; ++i)
{
output_file << ", Z_" << i;
}
}
// Outputting model parameters (rigid and non-rigid), the first parameters are the 6 rigid shape parameters, they are followed by the non rigid shape parameters
if (output_model_params)
{
output_file << ", p_scale, p_rx, p_ry, p_rz, p_tx, p_ty";
for (int i = 0; i < num_model_modes; ++i)
{
output_file << ", p_" << i;
}
}
if (output_AUs)
{
std::sort(this->au_names_reg.begin(), this->au_names_reg.end());
for (std::string reg_name : this->au_names_reg)
{
output_file << ", " << reg_name << "_r";
}
std::sort(this->au_names_class.begin(), this->au_names_class.end());
for (std::string class_name : this->au_names_class)
{
output_file << ", " << class_name << "_c";
}
}
output_file << std::endl;
return true;
}
void RecorderCSV::WriteLine(int face_id, int frame_num, double time_stamp, bool landmark_detection_success, double landmark_confidence,
const cv::Mat_<float>& landmarks_2D, const cv::Mat_<float>& landmarks_3D, const cv::Mat_<float>& pdm_model_params, const cv::Vec6f& rigid_shape_params, cv::Vec6f& pose_estimate,
const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2f& gaze_angle, const std::vector<cv::Point2f>& eye_landmarks2d, const std::vector<cv::Point3f>& eye_landmarks3d,
const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences)
{
if (!output_file.is_open())
{
std::cout << "The output CSV file is not open, exiting" << std::endl;
exit(1);
}
// Making sure fixed and not scientific notation is used
output_file << std::fixed;
output_file << std::noshowpoint;
if(is_sequence)
{
output_file << std::setprecision(3);
output_file << frame_num << ", " << face_id << ", " << time_stamp;
output_file << std::setprecision(2);
output_file << ", " << landmark_confidence;
output_file << std::setprecision(0);
output_file << ", " << landmark_detection_success;
}
else
{
output_file << std::setprecision(3);
output_file << face_id << ", " << landmark_confidence;
}
// Output the estimated gaze
if (output_gaze)
{
output_file << std::setprecision(6);
output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z
<< ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z;
// Output gaze angle (same format as head pose angle)
output_file << std::setprecision(3);
output_file << ", " << gaze_angle[0] << ", " << gaze_angle[1];
// Output the 2D eye landmarks
output_file << std::setprecision(1);
for (auto eye_lmk : eye_landmarks2d)
{
output_file << ", " << eye_lmk.x;
}
for (auto eye_lmk : eye_landmarks2d)
{
output_file << ", " << eye_lmk.y;
}
// Output the 3D eye landmarks
for (auto eye_lmk : eye_landmarks3d)
{
output_file << ", " << eye_lmk.x;
}
for (auto eye_lmk : eye_landmarks3d)
{
output_file << ", " << eye_lmk.y;
}
for (auto eye_lmk : eye_landmarks3d)
{
output_file << ", " << eye_lmk.z;
}
}
// Output the estimated head pose
if (output_pose)
{
output_file << std::setprecision(1);
output_file << ", " << pose_estimate[0] << ", " << pose_estimate[1] << ", " << pose_estimate[2];
output_file << std::setprecision(3);
output_file << ", " << pose_estimate[3] << ", " << pose_estimate[4] << ", " << pose_estimate[5];
}
// Output the detected 2D facial landmarks
if (output_2D_landmarks)
{
output_file.precision(1);
// Output the 2D eye landmarks
for (auto lmk : landmarks_2D)
{
output_file << ", " << lmk;
}
}
// Output the detected 3D facial landmarks
if (output_3D_landmarks)
{
output_file.precision(1);
// Output the 2D eye landmarks
for (auto lmk : landmarks_3D)
{
output_file << ", " << lmk;
}
}
if (output_model_params)
{
output_file.precision(3);
for (int i = 0; i < 6; ++i)
{
output_file << ", " << rigid_shape_params[i];
}
// Output the non_rigid shape parameters
for (auto lmk : pdm_model_params)
{
output_file << ", " << lmk;
}
}
if (output_AUs)
{
// write out ar the correct index
output_file.precision(2);
for (std::string au_name : au_names_reg)
{
for (auto au_reg : au_intensities)
{
if (au_name.compare(au_reg.first) == 0)
{
output_file << ", " << au_reg.second;
break;
}
}
}
if (au_intensities.size() == 0)
{
for (size_t p = 0; p < au_names_reg.size(); ++p)
{
output_file << ", 0";
}
}
output_file.precision(1);
// write out ar the correct index
for (std::string au_name : au_names_class)
{
for (auto au_class : au_occurences)
{
if (au_name.compare(au_class.first) == 0)
{
output_file << ", " << au_class.second;
break;
}
}
}
if (au_occurences.size() == 0)
{
for (size_t p = 0; p < au_names_class.size(); ++p)
{
output_file << ", 0";
}
}
}
output_file << std::endl;
}
// Closing the file and cleaning up
void RecorderCSV::Close()
{
output_file.close();
}

102
pkg/OpenFace/lib/local/Utilities/src/RecorderHOG.cpp Normal file
View File

@@ -0,0 +1,102 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "RecorderHOG.h"
using namespace Utilities;
// Default constructor initializes the variables
RecorderHOG::RecorderHOG() :hog_file() {};
// Opening the file and preparing the header for it
bool RecorderHOG::Open(std::string output_file_name)
{
hog_file.open(output_file_name, std::ios_base::out | std::ios_base::binary);
return hog_file.is_open();
}
void RecorderHOG::Close()
{
hog_file.close();
}
void RecorderHOG::Write()
{
hog_file.write((char*)(&num_cols), 4);
hog_file.write((char*)(&num_rows), 4);
hog_file.write((char*)(&num_channels), 4);
// Not the best way to store a bool, but will be much easier to read it
float good_frame_float;
if (good_frame)
good_frame_float = 1;
else
good_frame_float = -1;
hog_file.write((char*)(&good_frame_float), 4);
if(hog_descriptor.isContinuous())
{
cv::Mat_<float> desc;
hog_descriptor.convertTo(desc, CV_32F);
hog_file.write((char*)desc.data, 4 * num_cols * num_rows * 31);
}
else
{
cv::MatConstIterator_<double> descriptor_it = hog_descriptor.begin();
for (int y = 0; y < num_cols; ++y)
{
for (int x = 0; x < num_rows; ++x)
{
for (unsigned int o = 0; o < 31; ++o)
{
float hog_data = (float)(*descriptor_it++);
hog_file.write((char*)&hog_data, 4);
}
}
}
}
}
// Writing to a HOG file
void RecorderHOG::SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels)
{
this->num_cols = num_cols;
this->num_rows = num_rows;
this->num_channels = num_channels;
this->hog_descriptor = hog_descriptor;
this->good_frame = good_frame;
}

525
pkg/OpenFace/lib/local/Utilities/src/RecorderOpenFace.cpp Normal file
View File

@@ -0,0 +1,525 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "RecorderOpenFace.h"
using namespace Utilities;
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
void CreateDirectory(std::string output_path)
{
// Removing trailing separators, as that causes issues with directory creation in unix
while (output_path[output_path.size() - 1] == '/' || output_path[output_path.size() - 1] == '\\')
{
output_path = output_path.substr(0, output_path.size() - 1);
}
// Creating the right directory structure
if (!fs::exists(output_path))
{
bool success = fs::create_directories(output_path);
if (!success)
{
std::cout << "ERROR: failed to create output directory:" << output_path << ", do you have permission to create directory" << std::endl;
exit(1);
}
}
}
void RecorderOpenFace::VideoWritingTask(bool is_sequence)
{
std::pair<std::string, cv::Mat> tracked_data;
while (true)
{
vis_to_out_queue.pop(tracked_data);
// Indicate that the thread should complete
if (tracked_data.second.empty())
{
break;
}
if (is_sequence)
{
if (video_writer.isOpened())
{
video_writer.write(tracked_data.second);
}
}
else
{
bool out_success = cv::imwrite(tracked_data.first, tracked_data.second);
if (!out_success)
{
WARN_STREAM("Could not output tracked image");
}
}
}
}
void RecorderOpenFace::AlignedImageWritingTask()
{
std::pair<std::string, cv::Mat> tracked_data;
while (true)
{
aligned_face_queue.pop(tracked_data);
// Empty frame indicates termination
if (tracked_data.second.empty())
break;
bool write_success = cv::imwrite(tracked_data.first, tracked_data.second);
if (!write_success)
{
WARN_STREAM("Could not output similarity aligned image image");
}
}
}
void RecorderOpenFace::PrepareRecording(const std::string& in_filename)
{
// Construct the directories required for the output
CreateDirectory(record_root);
// Create the filename for the general output file that contains all of the meta information about the recording
fs::path of_det_name(out_name);
of_det_name = fs::path(record_root) / fs::path(out_name + "_of_details.txt");
// Write in the of file what we are outputing what is the input etc.
metadata_file.open(of_det_name.string(), std::ios_base::out);
if (!metadata_file.is_open())
{
std::cout << "ERROR: could not open the output file:" << of_det_name << ", either the path of the output directory is wrong or you do not have the permissions to write to it" << std::endl;
exit(1);
}
// Populate relative and full path names in the meta file, unless it is a webcam
if (!params.isFromWebcam())
{
std::string input_filename_relative = in_filename;
std::string input_filename_full = in_filename;
if (!fs::path(input_filename_full).is_absolute())
{
input_filename_full = fs::canonical(input_filename_relative).string();
}
metadata_file << "Input:" << input_filename_relative << std::endl;
metadata_file << "Input full path:" << input_filename_full << std::endl;
}
else
{
// Populate the metadata file
metadata_file << "Input:webcam" << std::endl;
}
metadata_file << "Camera parameters:" << params.getFx() << "," << params.getFy() << "," << params.getCx() << "," << params.getCy() << std::endl;
// Create the required individual recorders, CSV, HOG, aligned, video
csv_filename = out_name + ".csv";
// Consruct HOG recorder here
if (params.outputHOG())
{
// Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory
std::string hog_filename = out_name + ".hog";
metadata_file << "Output HOG:" << hog_filename << std::endl;
hog_filename = (fs::path(record_root) / hog_filename).string();
hog_recorder.Open(hog_filename);
}
// saving the videos
if (params.outputTracked())
{
if (params.isSequence())
{
// Output the data based on record_root, but do not include record_root in the meta file, as it is also in that directory
this->media_filename = out_name + ".avi";
metadata_file << "Output video:" << this->media_filename << std::endl;
this->media_filename = (fs::path(record_root) / this->media_filename).string();
}
else
{
this->media_filename = out_name + "." + params.imageFormatVisualization();
metadata_file << "Output image:" << this->media_filename << std::endl;
this->media_filename = (fs::path(record_root) / this->media_filename).string();
}
}
// Prepare image recording
if (params.outputAlignedFaces())
{
aligned_output_directory = out_name + "_aligned";
metadata_file << "Output aligned directory:" << this->aligned_output_directory << std::endl;
this->aligned_output_directory = (fs::path(record_root) / this->aligned_output_directory).string();
CreateDirectory(aligned_output_directory);
}
this->frame_number = 0;
this->tracked_writing_thread_started = false;
this->aligned_writing_thread_started = false;
}
RecorderOpenFace::RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::vector<std::string>& arguments):video_writer(), params(parameters)
{
// From the filename, strip out the name without directory and extension
if (fs::is_directory(in_filename))
{
out_name = fs::canonical(in_filename).filename().string();
}
else
{
out_name = fs::path(in_filename).filename().replace_extension("").string();
}
// Consuming the input arguments
bool* valid = new bool[arguments.size()];
for (size_t i = 0; i < arguments.size(); ++i)
{
valid[i] = true;
}
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-out_dir") == 0)
{
record_root = arguments[i + 1];
}
}
// Determine output directory
bool output_found = false;
for (size_t i = 0; i < arguments.size(); ++i)
{
if (!output_found && arguments[i].compare("-of") == 0)
{
record_root = (fs::path(record_root) / fs::path(arguments[i + 1])).remove_filename().string();
out_name = fs::path(fs::path(arguments[i + 1])).replace_extension("").filename().string();
valid[i] = false;
valid[i + 1] = false;
i++;
output_found = true;
}
}
// If recording directory not set, record to default location
if (record_root.empty())
record_root = default_record_directory;
for (int i = (int)arguments.size() - 1; i >= 0; --i)
{
if (!valid[i])
{
arguments.erase(arguments.begin() + i);
}
}
PrepareRecording(in_filename);
}
RecorderOpenFace::RecorderOpenFace(const std::string in_filename, const RecorderOpenFaceParameters& parameters, std::string output_directory):video_writer(), params(parameters)
{
// From the filename, strip out the name without directory and extension
if (fs::is_directory(in_filename))
{
out_name = fs::canonical(fs::path(in_filename)).filename().string();
}
else
{
out_name = fs::path(in_filename).filename().replace_extension("").string();
}
record_root = output_directory;
// If recording directory not set, record to default location
if (record_root.empty())
record_root = default_record_directory;
PrepareRecording(in_filename);
}
void RecorderOpenFace::SetObservationFaceAlign(const cv::Mat& aligned_face)
{
this->aligned_face = aligned_face;
}
void RecorderOpenFace::SetObservationVisualization(const cv::Mat &vis_track)
{
if (params.outputTracked())
{
vis_to_out = vis_track;
}
}
void RecorderOpenFace::WriteObservation()
{
// Write out the CSV file (it will always be there, even if not outputting anything more but frame/face numbers)
if(!csv_recorder.isOpen())
{
// As we are writing out the header, work out some things like number of landmarks, names of AUs etc.
int num_face_landmarks = landmarks_2D.rows / 2;
int num_eye_landmarks = (int)eye_landmarks2D.size();
int num_model_modes = pdm_params_local.rows;
std::vector<std::string> au_names_class;
for (auto au : au_occurences)
{
au_names_class.push_back(au.first);
}
std::sort(au_names_class.begin(), au_names_class.end());
std::vector<std::string> au_names_reg;
for (auto au : au_intensities)
{
au_names_reg.push_back(au.first);
}
std::sort(au_names_reg.begin(), au_names_reg.end());
metadata_file << "Output csv:" << csv_filename << std::endl;
metadata_file << "Gaze: " << params.outputGaze() << std::endl;
metadata_file << "AUs: " << params.outputAUs() << std::endl;
metadata_file << "Landmarks 2D: " << params.output2DLandmarks() << std::endl;
metadata_file << "Landmarks 3D: " << params.output3DLandmarks() << std::endl;
metadata_file << "Pose: " << params.outputPose() << std::endl;
metadata_file << "Shape parameters: " << params.outputPDMParams() << std::endl;
csv_filename = (fs::path(record_root) / csv_filename).string();
csv_recorder.Open(csv_filename, params.isSequence(), params.output2DLandmarks(), params.output3DLandmarks(), params.outputPDMParams(), params.outputPose(),
params.outputAUs(), params.outputGaze(), num_face_landmarks, num_model_modes, num_eye_landmarks, au_names_class, au_names_reg);
}
this->csv_recorder.WriteLine(face_id, frame_number, timestamp, landmark_detection_success,
landmark_detection_confidence, landmarks_2D, landmarks_3D, pdm_params_local, pdm_params_global, head_pose,
gaze_direction0, gaze_direction1, gaze_angle, eye_landmarks2D, eye_landmarks3D, au_intensities, au_occurences);
if(params.outputHOG())
{
this->hog_recorder.Write();
}
// Write aligned faces
if (params.outputAlignedFaces())
{
if (!aligned_writing_thread_started)
{
aligned_writing_thread_started = true;
int capacity = (1024 * 1024 * ALIGNED_QUEUE_CAPACITY) / (aligned_face.size().width *aligned_face.size().height * aligned_face.channels()) + 1;
aligned_face_queue.set_capacity(capacity);
// Start the alignment output thread
aligned_writing_thread = std::thread(&RecorderOpenFace::AlignedImageWritingTask, this);
}
char name[100];
// Filename is based on frame number (TODO stringstream this)
if(params.isSequence())
std::sprintf(name, "frame_det_%02d_%06d.", face_id, frame_number);
else
std::sprintf(name, "face_det_%06d.", face_id);
// Construct the output filename
std::string out_file = (fs::path(aligned_output_directory) / fs::path(std::string(name) + params.imageFormatAligned())).string();
if(params.outputBadAligned() || landmark_detection_success)
{
aligned_face_queue.push(std::pair<std::string, cv::Mat>(out_file, aligned_face));
}
// Clear the image
aligned_face = cv::Mat();
}
}
void RecorderOpenFace::WriteObservationTracked()
{
if (params.outputTracked())
{
if (!tracked_writing_thread_started)
{
tracked_writing_thread_started = true;
// Set up the queue for video writing based on output size
int capacity = (1024 * 1024 * TRACKED_QUEUE_CAPACITY) / (vis_to_out.size().width * vis_to_out.size().height * vis_to_out.channels()) + 1;
vis_to_out_queue.set_capacity(capacity);
// Initialize the video writer if it has not been opened yet
if (params.isSequence())
{
std::string output_codec = params.outputCodec();
try
{
video_writer.open(media_filename, cv::VideoWriter::fourcc(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), params.outputFps(), vis_to_out.size(), true);
if (!video_writer.isOpened())
{
WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN.");
}
}
catch (cv::Exception e)
{
WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)");
}
}
// Start the video and tracked image writing thread
video_writing_thread = std::thread(&RecorderOpenFace::VideoWritingTask, this, params.isSequence());
}
if (vis_to_out.empty())
{
WARN_STREAM("Output tracked video frame is not set");
}
if (params.isSequence())
{
vis_to_out_queue.push(std::pair<std::string, cv::Mat>("", vis_to_out));
}
else
{
vis_to_out_queue.push(std::pair<std::string, cv::Mat>(media_filename, vis_to_out));
}
// Clear the output
vis_to_out = cv::Mat();
}
}
void RecorderOpenFace::SetObservationHOG(bool good_frame, const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows, int num_channels)
{
this->hog_recorder.SetObservationHOG(good_frame, hog_descriptor, num_cols, num_rows, num_channels);
}
void RecorderOpenFace::SetObservationTimestamp(double timestamp)
{
this->timestamp = timestamp;
}
// Required observations for video/image-sequence
void RecorderOpenFace::SetObservationFrameNumber(int frame_number)
{
this->frame_number = frame_number;
}
// If in multiple face mode, identifying which face was tracked
void RecorderOpenFace::SetObservationFaceID(int face_id)
{
this->face_id = face_id;
}
void RecorderOpenFace::SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, const cv::Mat_<float>& landmarks_3D,
const cv::Vec6f& pdm_params_global, const cv::Mat_<float>& pdm_params_local, double confidence, bool success)
{
this->landmarks_2D = landmarks_2D;
this->landmarks_3D = landmarks_3D;
this->pdm_params_global = pdm_params_global;
this->pdm_params_local = pdm_params_local;
this->landmark_detection_confidence = confidence;
this->landmark_detection_success = success;
}
void RecorderOpenFace::SetObservationPose(const cv::Vec6f& pose)
{
this->head_pose = pose;
}
void RecorderOpenFace::SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities,
const std::vector<std::pair<std::string, double> >& au_occurences)
{
this->au_intensities = au_intensities;
this->au_occurences = au_occurences;
}
void RecorderOpenFace::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1,
const cv::Vec2f& gaze_angle, const std::vector<cv::Point2f>& eye_landmarks2D, const std::vector<cv::Point3f>& eye_landmarks3D)
{
this->gaze_direction0 = gaze_direction0;
this->gaze_direction1 = gaze_direction1;
this->gaze_angle = gaze_angle;
this->eye_landmarks2D = eye_landmarks2D;
this->eye_landmarks3D = eye_landmarks3D;
}
RecorderOpenFace::~RecorderOpenFace()
{
this->Close();
}
void RecorderOpenFace::Close()
{
// Insert terminating frames to the queues
vis_to_out_queue.push(std::pair<std::string, cv::Mat>("", cv::Mat()));
aligned_face_queue.push(std::pair<std::string, cv::Mat>("", cv::Mat()));
// Make sure the recording threads complete
if (video_writing_thread.joinable())
video_writing_thread.join();
if (aligned_writing_thread.joinable())
aligned_writing_thread.join();
tracked_writing_thread_started = false;
aligned_writing_thread_started = false;
hog_recorder.Close();
csv_recorder.Close();
video_writer.release();
metadata_file.close();
}

191
pkg/OpenFace/lib/local/Utilities/src/RecorderOpenFaceParameters.cpp Normal file
View File

@@ -0,0 +1,191 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "RecorderOpenFaceParameters.h"
using namespace Utilities;
RecorderOpenFaceParameters::RecorderOpenFaceParameters(std::vector<std::string> &arguments, bool sequence, bool from_webcam, float fx, float fy, float cx, float cy, double fps_vid_out)
{
this->is_sequence = sequence;
this->is_from_webcam = from_webcam;
this->fx = fx;
this->fy = fy;
this->cx = cx;
this->cy = cy;
if(fps_vid_out > 0)
{
this->fps_vid_out = fps_vid_out;
}
else
{
this->fps_vid_out = 30; // If an illegal value for fps provided, default to 30
}
// Default output code
this->output_codec = "DIVX";
this->image_format_aligned = "bmp";
this->image_format_visualization = "jpg";
bool output_set = false;
this->output_2D_landmarks = false;
this->output_3D_landmarks = false;
this->output_model_params = false;
this->output_pose = false;
this->output_AUs = false;
this->output_gaze = false;
this->output_hog = false;
this->output_tracked = false;
this->output_aligned_faces = false;
this->record_aligned_bad = true;
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-format_aligned") == 0)
{
this->image_format_aligned = arguments[i+1];
i++;
}
if (arguments[i].compare("-format_vis_image") == 0)
{
this->image_format_visualization = arguments[i + 1];
i++;
}
if (arguments[i].compare("-nobadaligned") == 0)
{
this->record_aligned_bad = false;
}
if (arguments[i].compare("-simalign") == 0)
{
this->output_aligned_faces = true;
output_set = true;
}
else if (arguments[i].compare("-hogalign") == 0)
{
this->output_hog = true;
output_set = true;
}
else if (arguments[i].compare("-2Dfp") == 0)
{
this->output_2D_landmarks = true;
output_set = true;
}
else if (arguments[i].compare("-3Dfp") == 0)
{
this->output_3D_landmarks = true;
output_set = true;
}
else if (arguments[i].compare("-pdmparams") == 0)
{
this->output_model_params = true;
output_set = true;
}
else if (arguments[i].compare("-pose") == 0)
{
this->output_pose = true;
output_set = true;
}
else if (arguments[i].compare("-aus") == 0)
{
this->output_AUs = true;
output_set = true;
}
else if (arguments[i].compare("-gaze") == 0)
{
this->output_gaze = true;
output_set = true;
}
else if (arguments[i].compare("-tracked") == 0)
{
this->output_tracked = true;
output_set = true;
}
}
// Output everything if nothing has been set
if (!output_set)
{
this->output_2D_landmarks = true;
this->output_3D_landmarks = true;
this->output_model_params = true;
this->output_pose = true;
this->output_AUs = true;
this->output_gaze = true;
this->output_hog = true;
this->output_tracked = true;
this->output_aligned_faces = true;
}
}
RecorderOpenFaceParameters::RecorderOpenFaceParameters(bool sequence, bool is_from_webcam, bool output_2D_landmarks, bool output_3D_landmarks,
bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, bool output_hog, bool output_tracked,
bool output_aligned_faces, bool record_bad, float fx, float fy, float cx, float cy, double fps_vid_out)
{
this->is_sequence = sequence;
this->is_from_webcam = is_from_webcam;
this->fx = fx;
this->fy = fy;
this->cx = cx;
this->cy = cy;
if (fps_vid_out > 0)
{
this->fps_vid_out = fps_vid_out;
}
else
{
this->fps_vid_out = 30; // If an illegal value for fps provided, default to 30
}
// Default output code
this->output_codec = "DIVX";
this->image_format_aligned = "bmp";
this->image_format_visualization = "jpg";
this->output_2D_landmarks = output_2D_landmarks;
this->output_3D_landmarks = output_3D_landmarks;
this->output_model_params = output_model_params;
this->output_pose = output_pose;
this->output_AUs = output_AUs;
this->output_gaze = output_gaze;
this->output_hog = output_hog;
this->output_tracked = output_tracked;
this->output_aligned_faces = output_aligned_faces;
}

540
pkg/OpenFace/lib/local/Utilities/src/SequenceCapture.cpp Normal file
View File

@@ -0,0 +1,540 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "SequenceCapture.h"
#include "ImageManipulationHelpers.h"
using namespace Utilities;
#define INFO_STREAM( stream ) \
std::cout << stream << std::endl
#define WARN_STREAM( stream ) \
std::cout << "Warning: " << stream << std::endl
#define ERROR_STREAM( stream ) \
std::cout << "Error: " << stream << std::endl
bool SequenceCapture::Open(std::vector<std::string>& arguments)
{
// Consuming the input arguments
bool* valid = new bool[arguments.size()];
for (size_t i = 0; i < arguments.size(); ++i)
{
valid[i] = true;
}
// Some default values
std::string input_root = "";
fx = -1; fy = -1; cx = -1; cy = -1;
std::string separator = std::string(1, fs::path::preferred_separator);
// First check if there is a root argument (so that videos and input directories could be defined more easily)
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-root") == 0)
{
input_root = arguments[i + 1] + separator;
i++;
}
if (arguments[i].compare("-inroot") == 0)
{
input_root = arguments[i + 1] + separator;
i++;
}
}
std::string input_video_file;
std::string input_sequence_directory;
int device = -1;
int cam_width = 640;
int cam_height = 480;
bool file_found = false;
for (size_t i = 0; i < arguments.size(); ++i)
{
if (!file_found && arguments[i].compare("-f") == 0)
{
input_video_file = (input_root + arguments[i + 1]);
valid[i] = false;
valid[i + 1] = false;
i++;
file_found = true;
}
else if (!file_found && arguments[i].compare("-fdir") == 0)
{
input_sequence_directory = (input_root + arguments[i + 1]);
valid[i] = false;
valid[i + 1] = false;
i++;
file_found = true;
}
else if (arguments[i].compare("-fx") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> fx;
i++;
}
else if (arguments[i].compare("-fy") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> fy;
i++;
}
else if (arguments[i].compare("-cx") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> cx;
i++;
}
else if (arguments[i].compare("-cy") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> cy;
i++;
}
else if (arguments[i].compare("-device") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> device;
valid[i] = false;
valid[i + 1] = false;
i++;
}
else if (arguments[i].compare("-cam_width") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> cam_width;
valid[i] = false;
valid[i + 1] = false;
i++;
}
else if (arguments[i].compare("-cam_height") == 0)
{
std::stringstream data(arguments[i + 1]);
data >> cam_height;
valid[i] = false;
valid[i + 1] = false;
i++;
}
}
for (int i = (int)arguments.size() - 1; i >= 0; --i)
{
if (!valid[i])
{
arguments.erase(arguments.begin() + i);
}
}
no_input_specified = !file_found;
// Based on what was read in open the sequence
if (device != -1)
{
return OpenWebcam(device, cam_width, cam_height, fx, fy, cx, cy);
}
if (!input_video_file.empty())
{
return OpenVideoFile(input_video_file, fx, fy, cx, cy);
}
if (!input_sequence_directory.empty())
{
return OpenImageSequence(input_sequence_directory, fx, fy, cx, cy);
}
// If no input found return false and set a flag for it
no_input_specified = true;
return false;
}
// Get current date/time, format is YYYY-MM-DD.HH:mm, useful for saving data from webcam
const std::string currentDateTime()
{
time_t rawtime;
struct tm * timeinfo;
char buffer[80];
time(&rawtime);
timeinfo = localtime(&rawtime);
strftime(buffer, sizeof(buffer), "%Y-%m-%d-%H-%M", timeinfo);
return buffer;
}
bool SequenceCapture::OpenWebcam(int device, int image_width, int image_height, float fx, float fy, float cx, float cy)
{
INFO_STREAM("Attempting to read from webcam: " << device);
no_input_specified = false;
frame_num = 0;
time_stamp = 0;
if (device < 0)
{
std::cout << "Specify a valid device" << std::endl;
return false;
}
latest_frame = cv::Mat();
latest_gray_frame = cv::Mat();
capture.open(device);
capture.set(cv::CAP_PROP_FRAME_WIDTH, image_width);
capture.set(cv::CAP_PROP_FRAME_HEIGHT, image_height);
is_webcam = true;
is_image_seq = false;
vid_length = 0;
this->frame_width = (int)capture.get(cv::CAP_PROP_FRAME_WIDTH);
this->frame_height = (int)capture.get(cv::CAP_PROP_FRAME_HEIGHT);
if (!capture.isOpened())
{
std::cout << "Failed to open the webcam" << std::endl;
return false;
}
if (frame_width != image_width || frame_height != image_height)
{
std::cout << "Failed to open the webcam with desired resolution" << std::endl;
std::cout << "Defaulting to " << frame_width << "x" << frame_height << std::endl;
}
this->fps = capture.get(cv::CAP_PROP_FPS);
// Check if fps is nan or less than 0
if (fps != fps || fps <= 0)
{
INFO_STREAM("FPS of the webcam cannot be determined, assuming 30");
fps = 30;
}
SetCameraIntrinsics(fx, fy, cx, cy);
std::string time = currentDateTime();
this->name = "webcam_" + time;
start_time = cv::getTickCount();
capturing = true;
return true;
}
void SequenceCapture::Close()
{
// Close the capturing threads
capturing = false;
// If the queue is full it will be blocked, so need to empty it
while (!capture_queue.empty())
{
capture_queue.pop();
}
if (capture_thread.joinable())
capture_thread.join();
// Release the capture objects
if (capture.isOpened())
capture.release();
}
// Destructor that releases the capture
SequenceCapture::~SequenceCapture()
{
Close();
}
bool SequenceCapture::OpenVideoFile(std::string video_file, float fx, float fy, float cx, float cy)
{
INFO_STREAM("Attempting to read from file: " << video_file);
no_input_specified = false;
frame_num = 0;
time_stamp = 0;
latest_frame = cv::Mat();
latest_gray_frame = cv::Mat();
capture.open(video_file);
if (!capture.isOpened())
{
std::cout << "Failed to open the video file at location: " << video_file << std::endl;
return false;
}
this->fps = capture.get(cv::CAP_PROP_FPS);
// Check if fps is nan or less than 0
if (fps != fps || fps <= 0)
{
WARN_STREAM("FPS of the video file cannot be determined, assuming 30");
fps = 30;
}
is_webcam = false;
is_image_seq = false;
this->frame_width = (int)capture.get(cv::CAP_PROP_FRAME_WIDTH);
this->frame_height = (int)capture.get(cv::CAP_PROP_FRAME_HEIGHT);
vid_length = (int)capture.get(cv::CAP_PROP_FRAME_COUNT);
SetCameraIntrinsics(fx, fy, cx, cy);
this->name = video_file;
capturing = true;
capture_thread = std::thread(&SequenceCapture::CaptureThread, this);
return true;
}
bool SequenceCapture::OpenImageSequence(std::string directory, float fx, float fy, float cx, float cy)
{
INFO_STREAM("Attempting to read from directory: " << directory);
no_input_specified = false;
frame_num = 0;
time_stamp = 0;
image_files.clear();
fs::path image_directory(directory);
if (!fs::exists(image_directory))
{
std::cout << "Provided directory does not exist: " << directory << std::endl;
return false;
}
std::vector<fs::path> file_in_directory;
copy(fs::directory_iterator(image_directory), fs::directory_iterator(), back_inserter(file_in_directory));
// Sort the images in the directory first
sort(file_in_directory.begin(), file_in_directory.end());
std::vector<std::string> curr_dir_files;
for (std::vector<fs::path>::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator)
{
// Possible image extension .jpg and .png
if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".jpeg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0)
{
curr_dir_files.push_back(file_iterator->string());
}
}
image_files = curr_dir_files;
if (image_files.empty())
{
std::cout << "No images found in the directory: " << directory << std::endl;
return false;
}
// Assume all images are same size in an image sequence
cv::Mat tmp = cv::imread(image_files[0], cv::IMREAD_COLOR);
this->frame_height = tmp.size().height;
this->frame_width = tmp.size().width;
SetCameraIntrinsics(fx, fy, cx, cy);
// No fps as we have a sequence
this->fps = 0;
this->name = directory;
is_webcam = false;
is_image_seq = true;
vid_length = image_files.size();
capturing = true;
capture_thread = std::thread(&SequenceCapture::CaptureThread, this);
return true;
}
void SequenceCapture::SetCameraIntrinsics(float fx, float fy, float cx, float cy)
{
// If optical centers are not defined just use center of image
if (cx == -1)
{
this->cx = this->frame_width / 2.0f;
this->cy = this->frame_height / 2.0f;
}
else
{
this->cx = cx;
this->cy = cy;
}
// Use a rough guess-timate of focal length
if (fx == -1)
{
this->fx = 500.0f * (this->frame_width / 640.0f);
this->fy = 500.0f * (this->frame_height / 480.0f);
this->fx = (this->fx + this->fy) / 2.0f;
this->fy = this->fx;
}
else
{
this->fx = fx;
this->fy = fy;
}
}
void SequenceCapture::CaptureThread()
{
int capacity = (CAPTURE_CAPACITY * 1024 * 1024) / (4 * frame_width * frame_height);
capture_queue.set_capacity(capacity);
int frame_num_int = 0;
while(capturing)
{
double timestamp_curr = 0;
cv::Mat tmp_frame;
cv::Mat_<uchar> tmp_gray_frame;
if (!is_image_seq)
{
bool success = capture.read(tmp_frame);
if (!success)
{
// Indicate lack of success by returning an empty image
tmp_frame = cv::Mat();
capturing = false;
}
// Recording the timestamp
timestamp_curr = frame_num_int * (1.0 / fps);
}
else if (is_image_seq)
{
if (image_files.empty() || frame_num_int >= (int)image_files.size())
{
// Indicate lack of success by returning an empty image
tmp_frame = cv::Mat();
capturing = false;
}
else
{
tmp_frame = cv::imread(image_files[frame_num_int], cv::IMREAD_COLOR);
}
timestamp_curr = 0;
}
frame_num_int++;
// Set the grayscale frame
ConvertToGrayscale_8bit(tmp_frame, tmp_gray_frame);
capture_queue.push(std::make_tuple(timestamp_curr, tmp_frame, tmp_gray_frame));
}
}
cv::Mat SequenceCapture::GetNextFrame()
{
if(!is_webcam)
{
std::tuple<double, cv::Mat, cv::Mat_<uchar> > data;
data = capture_queue.pop();
time_stamp = std::get<0>(data);
latest_frame = std::get<1>(data);
latest_gray_frame = std::get<2>(data);
}
else
{
// Webcam does not use the threaded interface
bool success = capture.read(latest_frame);
time_stamp = (cv::getTickCount() - start_time) / cv::getTickFrequency();
if (!success)
{
// Indicate lack of success by returning an empty image
latest_frame = cv::Mat();
}
ConvertToGrayscale_8bit(latest_frame, latest_gray_frame);
}
frame_num++;
return latest_frame;
}
double SequenceCapture::GetProgress()
{
if (is_webcam)
{
return -1.0;
}
else
{
return (double)frame_num / (double)vid_length;
}
}
bool SequenceCapture::IsOpened()
{
if (is_webcam || !is_image_seq)
return capture.isOpened();
else
return (image_files.size() > 0 && frame_num < image_files.size());
}
cv::Mat_<uchar> SequenceCapture::GetGrayFrame()
{
return latest_gray_frame;
}

185
pkg/OpenFace/lib/local/Utilities/src/VisualizationUtils.cpp Normal file
View File

@@ -0,0 +1,185 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "VisualizationUtils.h"
#include "RotationHelpers.h"
namespace Utilities
{
FpsTracker::FpsTracker()
{
// Keep two seconds of history
history_length = 2;
}
void FpsTracker::AddFrame()
{
double current_time = cv::getTickCount() / cv::getTickFrequency();
frame_times.push(current_time);
DiscardOldFrames();
}
double FpsTracker::GetFPS()
{
DiscardOldFrames();
if (frame_times.size() == 0)
return 0;
double current_time = cv::getTickCount() / cv::getTickFrequency();
return ((double)frame_times.size()) / (current_time - frame_times.front());
}
void FpsTracker::DiscardOldFrames()
{
double current_time = cv::getTickCount() / cv::getTickFrequency();
// Remove old history
while (frame_times.size() > 0 && (current_time - frame_times.front()) > history_length)
frame_times.pop();
}
void DrawBox(cv::Mat image, cv::Vec6f pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy)
{
auto edge_lines = CalculateBox(pose, fx, fy, cx, cy);
DrawBox(edge_lines, image, color, thickness);
}
std::vector<std::pair<cv::Point2f, cv::Point2f>> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy)
{
float boxVerts[] = { -1, 1, -1,
1, 1, -1,
1, 1, 1,
-1, 1, 1,
1, -1, 1,
1, -1, -1,
-1, -1, -1,
-1, -1, 1 };
std::vector<std::pair<int, int>> edges;
edges.push_back(std::pair<int, int>(0, 1));
edges.push_back(std::pair<int, int>(1, 2));
edges.push_back(std::pair<int, int>(2, 3));
edges.push_back(std::pair<int, int>(0, 3));
edges.push_back(std::pair<int, int>(2, 4));
edges.push_back(std::pair<int, int>(1, 5));
edges.push_back(std::pair<int, int>(0, 6));
edges.push_back(std::pair<int, int>(3, 7));
edges.push_back(std::pair<int, int>(6, 5));
edges.push_back(std::pair<int, int>(5, 4));
edges.push_back(std::pair<int, int>(4, 7));
edges.push_back(std::pair<int, int>(7, 6));
// The size of the head is roughly 200mm x 200mm x 200mm
cv::Mat_<float> box = cv::Mat(8, 3, CV_32F, boxVerts).clone() * 100.0f;
cv::Matx33f rot = Euler2RotationMatrix(cv::Vec3f(pose[3], pose[4], pose[5]));
cv::Mat_<float> rotBox;
// Rotate the box
rotBox = cv::Mat(rot) * box.t();
rotBox = rotBox.t();
// Move the bounding box to head position
rotBox.col(0) = rotBox.col(0) + pose[0];
rotBox.col(1) = rotBox.col(1) + pose[1];
rotBox.col(2) = rotBox.col(2) + pose[2];
// draw the lines
cv::Mat_<float> rotBoxProj;
Project(rotBoxProj, rotBox, fx, fy, cx, cy);
std::vector<std::pair<cv::Point2f, cv::Point2f>> lines;
for (size_t i = 0; i < edges.size(); ++i)
{
cv::Mat_<float> begin;
cv::Mat_<float> end;
rotBoxProj.row(edges[i].first).copyTo(begin);
rotBoxProj.row(edges[i].second).copyTo(end);
cv::Point2f p1(begin.at<float>(0), begin.at<float>(1));
cv::Point2f p2(end.at<float>(0), end.at<float>(1));
lines.push_back(std::pair<cv::Point2f, cv::Point2f>(p1, p2));
}
return lines;
}
void DrawBox(const std::vector<std::pair<cv::Point2f, cv::Point2f>>& lines, cv::Mat image, cv::Scalar color, int thickness)
{
cv::Rect image_rect(0, 0, image.cols, image.rows);
for (size_t i = 0; i < lines.size(); ++i)
{
cv::Point2f p1 = lines.at(i).first;
cv::Point2f p2 = lines.at(i).second;
// Only draw the line if one of the points is inside the image
if (p1.inside(image_rect) || p2.inside(image_rect))
{
cv::line(image, p1, p2, color, thickness, cv::LINE_AA);
}
}
}
void Visualise_FHOG(const cv::Mat_<double>& descriptor, int num_rows, int num_cols, cv::Mat& visualisation)
{
// First convert to dlib format
dlib::array2d<dlib::matrix<float, 31, 1> > hog(num_rows, num_cols);
cv::MatConstIterator_<double> descriptor_it = descriptor.begin();
for (int y = 0; y < num_cols; ++y)
{
for (int x = 0; x < num_rows; ++x)
{
for (unsigned int o = 0; o < 31; ++o)
{
hog[y][x](o) = *descriptor_it++;
}
}
}
// Draw the FHOG to OpenCV format
auto fhog_vis = dlib::draw_fhog(hog);
visualisation = dlib::toMat(fhog_vis).clone();
}
}

455
pkg/OpenFace/lib/local/Utilities/src/Visualizer.cpp Normal file
View File

@@ -0,0 +1,455 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Tadas Baltrusaitis, all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
// * Any publications arising from the use of this software, including but
// not limited to academic journal and conference publications, technical
// reports and manuals, must cite at least one of the following works:
//
// OpenFace 2.0: Facial Behavior Analysis Toolkit
// Tadas Baltrušaitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
// in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
//
// Convolutional experts constrained local model for facial landmark detection.
// A. Zadeh, T. Baltrušaitis, and Louis-Philippe Morency,
// in Computer Vision and Pattern Recognition Workshops, 2017.
//
// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
// in IEEE International. Conference on Computer Vision (ICCV), 2015
//
// Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
// in Facial Expression Recognition and Analysis Challenge,
// IEEE International Conference on Automatic Face and Gesture Recognition, 2015
//
///////////////////////////////////////////////////////////////////////////////
#include "stdafx_ut.h"
#include "Visualizer.h"
#include "VisualizationUtils.h"
#include "RotationHelpers.h"
#include "ImageManipulationHelpers.h"
using namespace Utilities;
// For subpixel accuracy drawing
const int draw_shiftbits = 4;
const int draw_multiplier = 1 << 4;
const std::map<std::string, std::string> AUS_DESCRIPTION = {
{ "AU01", "Inner Brow Raiser " },
{ "AU02", "Outer Brow Raiser " },
{ "AU04", "Brow Lowerer " },
{ "AU05", "Upper Lid Raiser " },
{ "AU06", "Cheek Raiser " },
{ "AU07", "Lid Tightener " },
{ "AU09", "Nose Wrinkler " },
{ "AU10", "Upper Lip Raiser " },
{ "AU12", "Lip Corner Puller " },
{ "AU14", "Dimpler " },
{ "AU15", "Lip Corner Depressor" },
{ "AU17", "Chin Raiser " },
{ "AU20", "Lip stretcher " },
{ "AU23", "Lip Tightener " },
{ "AU25", "Lips part " },
{ "AU26", "Jaw Drop " },
{ "AU28", "Lip Suck " },
{ "AU45", "Blink " },
};
Visualizer::Visualizer(std::vector<std::string> arguments)
{
// By default not visualizing anything
this->vis_track = false;
this->vis_hog = false;
this->vis_align = false;
this->vis_aus = false;
for (size_t i = 0; i < arguments.size(); ++i)
{
if (arguments[i].compare("-verbose") == 0)
{
this->vis_track = true;
this->vis_align = true;
this->vis_hog = true;
this->vis_aus = true;
}
else if (arguments[i].compare("-vis-align") == 0)
{
this->vis_align = true;
}
else if (arguments[i].compare("-vis-hog") == 0)
{
this->vis_hog = true;
}
else if (arguments[i].compare("-vis-track") == 0)
{
this->vis_track = true;
}
else if (arguments[i].compare("-vis-aus") == 0)
{
this->vis_aus = true;
}
}
}
Visualizer::Visualizer(bool vis_track, bool vis_hog, bool vis_align, bool vis_aus)
{
this->vis_track = vis_track;
this->vis_hog = vis_hog;
this->vis_align = vis_align;
this->vis_aus = vis_aus;
}
// Setting the image on which to draw
void Visualizer::SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy)
{
// Convert the image to 8 bit RGB
captured_image = canvas.clone();
this->fx = fx;
this->fy = fy;
this->cx = cx;
this->cy = cy;
// Clearing other images
hog_image = cv::Mat();
aligned_face_image = cv::Mat();
action_units_image = cv::Mat();
}
void Visualizer::SetObservationFaceAlign(const cv::Mat& aligned_face)
{
if(this->aligned_face_image.empty())
{
this->aligned_face_image = aligned_face;
}
else
{
cv::vconcat(this->aligned_face_image, aligned_face, this->aligned_face_image);
}
}
void Visualizer::SetObservationHOG(const cv::Mat_<double>& hog_descriptor, int num_cols, int num_rows)
{
if(vis_hog)
{
if (this->hog_image.empty())
{
Visualise_FHOG(hog_descriptor, num_rows, num_cols, this->hog_image);
}
else
{
cv::Mat tmp_hog;
Visualise_FHOG(hog_descriptor, num_rows, num_cols, tmp_hog);
cv::vconcat(this->hog_image, tmp_hog, this->hog_image);
}
}
}
void Visualizer::SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities)
{
if(confidence > visualisation_boundary)
{
// Draw 2D landmarks on the image
int n = landmarks_2D.rows / 2;
// Drawing feature points
for (int i = 0; i < n; ++i)
{
if (visibilities.empty() || visibilities.at<int>(i))
{
cv::Point featurePoint(cvRound(landmarks_2D.at<float>(i) * (float)draw_multiplier), cvRound(landmarks_2D.at<float>(i + n) * (float)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = (int)std::ceil(3.0* ((double)captured_image.cols) / 640.0);
int thickness_2 = (int)std::ceil(1.0* ((double)captured_image.cols) / 640.0);
cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, cv::LINE_AA, draw_shiftbits);
cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, cv::LINE_AA, draw_shiftbits);
}
else
{
// Draw a fainter point if the landmark is self occluded
cv::Point featurePoint(cvRound(landmarks_2D.at<float>(i) * (double)draw_multiplier), cvRound(landmarks_2D.at<float>(i + n) * (double)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = (int)std::ceil(2.5* ((double)captured_image.cols) / 640.0);
int thickness_2 = (int)std::ceil(1.0* ((double)captured_image.cols) / 640.0);
cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 155), thickness, cv::LINE_AA, draw_shiftbits);
cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(155, 0, 0), thickness_2, cv::LINE_AA, draw_shiftbits);
}
}
}
}
void Visualizer::SetObservationPose(const cv::Vec6f& pose, double confidence)
{
// Only draw if the reliability is reasonable, the value is slightly ad-hoc
if (confidence > visualisation_boundary)
{
double vis_certainty = confidence;
if (vis_certainty > 1)
vis_certainty = 1;
// Scale from 0 to 1, to allow to indicated by colour how confident we are in the tracking
vis_certainty = (vis_certainty - visualisation_boundary) / (1 - visualisation_boundary);
// A rough heuristic for box around the face width
int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0);
// Draw it in reddish if uncertain, blueish if certain
DrawBox(captured_image, pose, cv::Scalar(vis_certainty*255.0, 0, (1 - vis_certainty) * 255), thickness, fx, fy, cx, cy);
}
}
void Visualizer::SetObservationActionUnits(const std::vector<std::pair<std::string, double> >& au_intensities,
const std::vector<std::pair<std::string, double> >& au_occurences)
{
if (au_intensities.size() > 0 || au_occurences.size() > 0)
{
std::set<std::string> au_names;
std::map<std::string, bool> occurences_map;
std::map<std::string, double> intensities_map;
for (size_t idx = 0; idx < au_intensities.size(); idx++)
{
au_names.insert(au_intensities[idx].first);
intensities_map[au_intensities[idx].first] = au_intensities[idx].second;
}
for (size_t idx = 0; idx < au_occurences.size(); idx++)
{
au_names.insert(au_occurences[idx].first);
occurences_map[au_occurences[idx].first] = au_occurences[idx].second > 0;
}
const int AU_TRACKBAR_LENGTH = 400;
const int AU_TRACKBAR_HEIGHT = 10;
const int MARGIN_X = 185;
const int MARGIN_Y = 10;
const int nb_aus = (int) au_names.size();
// Do not reinitialize
if (action_units_image.empty())
{
action_units_image = cv::Mat(nb_aus * (AU_TRACKBAR_HEIGHT + 10) + MARGIN_Y * 2, AU_TRACKBAR_LENGTH + MARGIN_X, CV_8UC3, cv::Scalar(255, 255, 255));
}
else
{
action_units_image.setTo(255);
}
std::map<std::string, std::pair<bool, double>> aus;
// first, prepare a mapping "AU name" -> { present, intensity }
for (auto au_name : au_names)
{
// Insert the intensity and AU presense (as these do not always overlap check if they exist first)
bool occurence = false;
if (occurences_map.find(au_name) != occurences_map.end())
{
occurence = occurences_map[au_name] != 0;
}
else
{
// If we do not have an occurence label, trust the intensity one
occurence = intensities_map[au_name] > 1;
}
double intensity = 0.0;
if (intensities_map.find(au_name) != intensities_map.end())
{
intensity = intensities_map[au_name];
}
else
{
// If we do not have an intensity label, trust the occurence one
intensity = occurences_map[au_name] == 0 ? 0 : 5;
}
aus[au_name] = std::make_pair(occurence, intensity);
}
// then, build the graph
unsigned int idx = 0;
for (auto& au : aus)
{
std::string name = au.first;
bool present = au.second.first;
double intensity = au.second.second;
int offset = MARGIN_Y + idx * (AU_TRACKBAR_HEIGHT + 10);
std::ostringstream au_i;
au_i << std::setprecision(2) << std::setw(4) << std::fixed << intensity;
cv::putText(action_units_image, name, cv::Point(10, offset + 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(present ? 0 : 200, 0, 0), 1, cv::LINE_AA);
cv::putText(action_units_image, AUS_DESCRIPTION.at(name), cv::Point(55, offset + 10), cv::FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, cv::LINE_AA);
if (present)
{
cv::putText(action_units_image, au_i.str(), cv::Point(160, offset + 10), cv::FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 100, 0), 1, cv::LINE_AA);
cv::rectangle(action_units_image, cv::Point(MARGIN_X, offset),
cv::Point((int)(MARGIN_X + AU_TRACKBAR_LENGTH * intensity / 5.0), offset + AU_TRACKBAR_HEIGHT),
cv::Scalar(128, 128, 128),
cv::FILLED);
}
else
{
cv::putText(action_units_image, "0.00", cv::Point(160, offset + 10), cv::FONT_HERSHEY_SIMPLEX, 0.3, CV_RGB(0, 0, 0), 1, cv::LINE_AA);
}
idx++;
}
}
}
// Eye gaze infomration drawing, first of eye landmarks then of gaze
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector<cv::Point2f>& eye_landmarks2d, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence)
{
if(confidence > visualisation_boundary)
{
if (eye_landmarks2d.size() > 0)
{
// First draw the eye region landmarks
for (size_t i = 0; i < eye_landmarks2d.size(); ++i)
{
cv::Point featurePoint(cvRound(eye_landmarks2d[i].x * (double)draw_multiplier), cvRound(eye_landmarks2d[i].y * (double)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = 1;
int thickness_2 = 1;
size_t next_point = i + 1;
if (i == 7)
next_point = 0;
if (i == 19)
next_point = 8;
if (i == 27)
next_point = 20;
if (i == 7 + 28)
next_point = 0 + 28;
if (i == 19 + 28)
next_point = 8 + 28;
if (i == 27 + 28)
next_point = 20 + 28;
cv::Point nextFeaturePoint(cvRound(eye_landmarks2d[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks2d[next_point].y * (double)draw_multiplier));
if ((i < 28 && (i < 8 || i > 19)) || (i >= 28 && (i < 8 + 28 || i > 19 + 28)))
cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, cv::LINE_AA, draw_shiftbits);
else
cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, cv::LINE_AA, draw_shiftbits);
}
// Now draw the gaze lines themselves
cv::Mat cameraMat = (cv::Mat_<float>(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0);
// Grabbing the pupil location, to draw eye gaze need to know where the pupil is
cv::Point3f pupil_left(0, 0, 0);
cv::Point3f pupil_right(0, 0, 0);
for (size_t i = 0; i < 8; ++i)
{
pupil_left = pupil_left + eye_landmarks3d[i];
pupil_right = pupil_right + eye_landmarks3d[i + eye_landmarks3d.size()/2];
}
pupil_left = pupil_left / 8;
pupil_right = pupil_right / 8;
std::vector<cv::Point3f> points_left;
points_left.push_back(cv::Point3f(pupil_left));
points_left.push_back(cv::Point3f(pupil_left) + cv::Point3f(gaze_direction0)*50.0);
std::vector<cv::Point3f> points_right;
points_right.push_back(cv::Point3f(pupil_right));
points_right.push_back(cv::Point3f(pupil_right) + cv::Point3f(gaze_direction1)*50.0);
cv::Mat_<float> proj_points;
cv::Mat_<float> mesh_0 = (cv::Mat_<float>(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z);
Project(proj_points, mesh_0, fx, fy, cx, cy);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<float>(0, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(0, 1) * (float)draw_multiplier)),
cv::Point(cvRound(proj_points.at<float>(1, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(1, 1) * (float)draw_multiplier)), cv::Scalar(110, 220, 0), 2, cv::LINE_AA, draw_shiftbits);
cv::Mat_<float> mesh_1 = (cv::Mat_<float>(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z);
Project(proj_points, mesh_1, fx, fy, cx, cy);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<float>(0, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(0, 1) * (float)draw_multiplier)),
cv::Point(cvRound(proj_points.at<float>(1, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(1, 1) * (float)draw_multiplier)), cv::Scalar(110, 220, 0), 2, cv::LINE_AA, draw_shiftbits);
}
}
}
void Visualizer::SetFps(double fps)
{
// Write out the framerate on the image before displaying it
char fpsC[255];
std::sprintf(fpsC, "%d", (int)fps);
std::string fpsSt("FPS:");
fpsSt += fpsC;
cv::putText(captured_image, fpsSt, cv::Point(10, 20), cv::FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, cv::LINE_AA);
}
char Visualizer::ShowObservation()
{
bool ovservation_shown = false;
if (vis_align && !aligned_face_image.empty())
{
cv::imshow("sim_warp", aligned_face_image);
ovservation_shown = true;
}
if (vis_hog && !hog_image.empty())
{
cv::imshow("hog", hog_image);
ovservation_shown = true;
}
if (vis_aus && !action_units_image.empty())
{
cv::imshow("action units", action_units_image);
ovservation_shown = true;
}
if (vis_track)
{
cv::imshow("tracking result", captured_image);
ovservation_shown = true;
}
// Only perform waitKey if something was shown
char result = '\0';
if (ovservation_shown)
{
result = cv::waitKey(1);
}
return result;
}
cv::Mat Visualizer::GetVisImage()
{
return captured_image;
}
cv::Mat Visualizer::GetHOGVis()
{
return hog_image;
}

12
pkg/OpenFace/lib/local/Utilities/src/stdafx_ut.cpp Normal file
View File

@@ -0,0 +1,12 @@
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
// all rights reserved.
//
// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
//
// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
//
// License can be found in OpenFace-license.txt
//
#include "stdafx_ut.h"