open source pkg v1

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/input.h

@@ -0,0 +1,808 @@
+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNn_INPUT_H_
+#define DLIB_DNn_INPUT_H_
+
+#include "input_abstract.h"
+#include "../matrix.h"
+#include "../array2d.h"
+#include "../pixel.h"
+#include "../image_processing.h"
+#include <sstream>
+#include <array>
+#include "../cuda/tensor_tools.h"
+
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T>
+    class input
+    {
+        const static bool always_false = sizeof(T)!=sizeof(T); 
+        static_assert(always_false, "Unsupported type given to input<>.  input<> only supports "
+            "dlib::matrix and dlib::array2d objects."); 
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <size_t NR, size_t NC=NR>
+    class input_rgb_image_sized;
+
+    class input_rgb_image
+    {
+    public:
+        typedef matrix<rgb_pixel> input_type;
+
+        input_rgb_image (
+        ) : 
+            avg_red(122.782), 
+            avg_green(117.001),
+            avg_blue(104.298) 
+        {
+        }
+
+        input_rgb_image (
+            float avg_red_,
+            float avg_green_,
+            float avg_blue_
+        ) : avg_red(avg_red_), avg_green(avg_green_), avg_blue(avg_blue_) 
+        {}
+
+        template <size_t NR, size_t NC>
+        inline input_rgb_image (
+            const input_rgb_image_sized<NR,NC>& item
+        ); 
+
+        float get_avg_red()   const { return avg_red; }
+        float get_avg_green() const { return avg_green; }
+        float get_avg_blue()  const { return avg_blue; }
+
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const
+        {
+            DLIB_CASSERT(std::distance(ibegin,iend) > 0);
+            const auto nr = ibegin->nr();
+            const auto nc = ibegin->nc();
+            // make sure all the input matrices have the same dimensions
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                DLIB_CASSERT(i->nr()==nr && i->nc()==nc,
+                    "\t input_rgb_image::to_tensor()"
+                    << "\n\t All matrices given to to_tensor() must have the same dimensions."
+                    << "\n\t nr: " << nr
+                    << "\n\t nc: " << nc
+                    << "\n\t i->nr(): " << i->nr()
+                    << "\n\t i->nc(): " << i->nc()
+                );
+            }
+
+            
+            // initialize data to the right size to contain the stuff in the iterator range.
+            data.set_size(std::distance(ibegin,iend), 3, nr, nc);
+
+
+            const size_t offset = nr*nc;
+            auto ptr = data.host();
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                for (long r = 0; r < nr; ++r)
+                {
+                    for (long c = 0; c < nc; ++c)
+                    {
+                        rgb_pixel temp = (*i)(r,c);
+                        auto p = ptr++;
+                        *p = (temp.red-avg_red)/256.0; 
+                        p += offset;
+                        *p = (temp.green-avg_green)/256.0; 
+                        p += offset;
+                        *p = (temp.blue-avg_blue)/256.0; 
+                        p += offset;
+                    }
+                }
+                ptr += offset*(data.k()-1);
+            }
+
+        }
+
+        friend void serialize(const input_rgb_image& item, std::ostream& out)
+        {
+            serialize("input_rgb_image", out);
+            serialize(item.avg_red, out);
+            serialize(item.avg_green, out);
+            serialize(item.avg_blue, out);
+        }
+
+        friend void deserialize(input_rgb_image& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "input_rgb_image" && version != "input_rgb_image_sized")
+                throw serialization_error("Unexpected version found while deserializing dlib::input_rgb_image.");
+            deserialize(item.avg_red, in);
+            deserialize(item.avg_green, in);
+            deserialize(item.avg_blue, in);
+
+            // read and discard the sizes if this was really a sized input layer.
+            if (version == "input_rgb_image_sized")
+            {
+                size_t nr, nc;
+                deserialize(nr, in);
+                deserialize(nc, in);
+            }
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const input_rgb_image& item)
+        {
+            out << "input_rgb_image("<<item.avg_red<<","<<item.avg_green<<","<<item.avg_blue<<")";
+            return out;
+        }
+
+        friend void to_xml(const input_rgb_image& item, std::ostream& out)
+        {
+            out << "<input_rgb_image r='"<<item.avg_red<<"' g='"<<item.avg_green<<"' b='"<<item.avg_blue<<"'/>";
+        }
+
+    private:
+        float avg_red;
+        float avg_green;
+        float avg_blue;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <size_t NR, size_t NC>
+    class input_rgb_image_sized
+    {
+    public:
+        static_assert(NR != 0 && NC != 0, "The input image can't be empty.");
+
+        typedef matrix<rgb_pixel> input_type;
+
+        input_rgb_image_sized (
+        ) : 
+            avg_red(122.782), 
+            avg_green(117.001),
+            avg_blue(104.298) 
+        {
+        }
+
+        input_rgb_image_sized (
+            const input_rgb_image& item
+        ) : avg_red(item.get_avg_red()),
+            avg_green(item.get_avg_green()),
+            avg_blue(item.get_avg_blue())
+        {}
+
+        input_rgb_image_sized (
+            float avg_red_,
+            float avg_green_,
+            float avg_blue_
+        ) : avg_red(avg_red_), avg_green(avg_green_), avg_blue(avg_blue_) 
+        {}
+
+        float get_avg_red()   const { return avg_red; }
+        float get_avg_green() const { return avg_green; }
+        float get_avg_blue()  const { return avg_blue; }
+
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const
+        {
+            DLIB_CASSERT(std::distance(ibegin,iend) > 0);
+            // make sure all input images have the correct size
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                DLIB_CASSERT(i->nr()==NR && i->nc()==NC,
+                    "\t input_rgb_image_sized::to_tensor()"
+                    << "\n\t All input images must have "<<NR<<" rows and "<<NC<< " columns, but we got one with "<<i->nr()<<" rows and "<<i->nc()<<" columns."
+                );
+            }
+
+            
+            // initialize data to the right size to contain the stuff in the iterator range.
+            data.set_size(std::distance(ibegin,iend), 3, NR, NC);
+
+
+            const size_t offset = NR*NC;
+            auto ptr = data.host();
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                for (size_t r = 0; r < NR; ++r)
+                {
+                    for (size_t c = 0; c < NC; ++c)
+                    {
+                        rgb_pixel temp = (*i)(r,c);
+                        auto p = ptr++;
+                        *p = (temp.red-avg_red)/256.0; 
+                        p += offset;
+                        *p = (temp.green-avg_green)/256.0; 
+                        p += offset;
+                        *p = (temp.blue-avg_blue)/256.0; 
+                        p += offset;
+                    }
+                }
+                ptr += offset*(data.k()-1);
+            }
+
+        }
+
+        friend void serialize(const input_rgb_image_sized& item, std::ostream& out)
+        {
+            serialize("input_rgb_image_sized", out);
+            serialize(item.avg_red, out);
+            serialize(item.avg_green, out);
+            serialize(item.avg_blue, out);
+            serialize(NR, out);
+            serialize(NC, out);
+        }
+
+        friend void deserialize(input_rgb_image_sized& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "input_rgb_image_sized")
+                throw serialization_error("Unexpected version found while deserializing dlib::input_rgb_image_sized.");
+            deserialize(item.avg_red, in);
+            deserialize(item.avg_green, in);
+            deserialize(item.avg_blue, in);
+            size_t nr, nc;
+            deserialize(nr, in);
+            deserialize(nc, in);
+            if (nr != NR || nc != NC)
+            {
+                std::ostringstream sout;
+                sout << "Wrong image dimensions found while deserializing dlib::input_rgb_image_sized.\n";
+                sout << "Expected "<<NR<<" rows and "<<NC<< " columns, but found "<<nr<<" rows and "<<nc<<" columns.";
+                throw serialization_error(sout.str());
+            }
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const input_rgb_image_sized& item)
+        {
+            out << "input_rgb_image_sized("<<item.avg_red<<","<<item.avg_green<<","<<item.avg_blue<<") nr="<<NR<<" nc="<<NC;
+            return out;
+        }
+
+        friend void to_xml(const input_rgb_image_sized& item, std::ostream& out)
+        {
+            out << "<input_rgb_image_sized r='"<<item.avg_red<<"' g='"<<item.avg_green<<"' b='"<<item.avg_blue<<"' nr='"<<NR<<"' nc='"<<NC<<"'/>";
+        }
+
+    private:
+        float avg_red;
+        float avg_green;
+        float avg_blue;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <size_t NR, size_t NC>
+    input_rgb_image::
+    input_rgb_image (
+        const input_rgb_image_sized<NR,NC>& item
+    ) : avg_red(item.get_avg_red()),
+        avg_green(item.get_avg_green()),
+        avg_blue(item.get_avg_blue())
+    {}
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T, long NR, long NC, typename MM, typename L>
+    class input<matrix<T,NR,NC,MM,L>> 
+    {
+    public:
+        typedef matrix<T,NR,NC,MM,L> input_type;
+
+        input() {}
+        input(const input&) {}
+
+        template <typename mm>
+        input(const input<array2d<T,mm>>&) {}
+
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const
+        {
+            DLIB_CASSERT(std::distance(ibegin,iend) > 0);
+            const auto nr = ibegin->nr();
+            const auto nc = ibegin->nc();
+            // make sure all the input matrices have the same dimensions
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                DLIB_CASSERT(i->nr()==nr && i->nc()==nc,
+                    "\t input::to_tensor()"
+                    << "\n\t All matrices given to to_tensor() must have the same dimensions."
+                    << "\n\t nr: " << nr
+                    << "\n\t nc: " << nc
+                    << "\n\t i->nr(): " << i->nr()
+                    << "\n\t i->nc(): " << i->nc()
+                );
+            }
+
+            
+            // initialize data to the right size to contain the stuff in the iterator range.
+            data.set_size(std::distance(ibegin,iend), pixel_traits<T>::num, nr, nc);
+
+            typedef typename pixel_traits<T>::basic_pixel_type bptype;
+
+            const size_t offset = nr*nc;
+            auto ptr = data.host();
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                for (long r = 0; r < nr; ++r)
+                {
+                    for (long c = 0; c < nc; ++c)
+                    {
+                        auto temp = pixel_to_vector<float>((*i)(r,c));
+                        auto p = ptr++;
+                        for (long j = 0; j < temp.size(); ++j)
+                        {
+                            if (is_same_type<bptype,unsigned char>::value)
+                                *p = temp(j)/256.0;
+                            else
+                                *p = temp(j);
+                            p += offset;
+                        }
+                    }
+                }
+                ptr += offset*(data.k()-1);
+            }
+
+        }
+
+        friend void serialize(const input& /*item*/, std::ostream& out)
+        {
+            serialize("input<matrix>", out);
+        }
+
+        friend void deserialize(input& /*item*/, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "input<matrix>")
+                throw serialization_error("Unexpected version found while deserializing dlib::input.");
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const input& /*item*/)
+        {
+            out << "input<matrix>";
+            return out;
+        }
+
+        friend void to_xml(const input& /*item*/, std::ostream& out)
+        {
+            out << "<input/>";
+        }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T, long NR, long NC, typename MM, typename L, size_t K>
+    class input<std::array<matrix<T,NR,NC,MM,L>,K>> 
+    {
+    public:
+        typedef std::array<matrix<T,NR,NC,MM,L>,K> input_type;
+
+        input() {}
+        input(const input&) {}
+
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const
+        {
+            DLIB_CASSERT(std::distance(ibegin,iend) > 0);
+            DLIB_CASSERT(ibegin->size() != 0, "When using std::array<matrix> inputs you can't give 0 sized arrays.");
+            const auto nr = (*ibegin)[0].nr();
+            const auto nc = (*ibegin)[0].nc();
+            // make sure all the input matrices have the same dimensions
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                for (size_t k = 0; k < K; ++k)
+                {
+                    const auto& arr = *i;
+                    DLIB_CASSERT(arr[k].nr()==nr && arr[k].nc()==nc,
+                        "\t input::to_tensor()"
+                        << "\n\t When using std::array<matrix> as input, all matrices in a batch must have the same dimensions."
+                        << "\n\t nr: " << nr
+                        << "\n\t nc: " << nc
+                        << "\n\t k:  " << k 
+                        << "\n\t arr[k].nr(): " << arr[k].nr()
+                        << "\n\t arr[k].nc(): " << arr[k].nc()
+                    );
+                }
+            }
+
+            
+            // initialize data to the right size to contain the stuff in the iterator range.
+            data.set_size(std::distance(ibegin,iend), K, nr, nc);
+
+            auto ptr = data.host();
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                for (size_t k = 0; k < K; ++k)
+                {
+                    for (long r = 0; r < nr; ++r)
+                    {
+                        for (long c = 0; c < nc; ++c)
+                        {
+                            if (is_same_type<T,unsigned char>::value)
+                                *ptr++ = (*i)[k](r,c)/256.0;
+                            else
+                                *ptr++ = (*i)[k](r,c);
+                        }
+                    }
+                }
+            }
+
+        }
+
+        friend void serialize(const input& /*item*/, std::ostream& out)
+        {
+            serialize("input<array<matrix>>", out);
+        }
+
+        friend void deserialize(input& /*item*/, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "input<array<matrix>>")
+                throw serialization_error("Unexpected version found while deserializing dlib::input<array<matrix>>.");
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const input& /*item*/)
+        {
+            out << "input<array<matrix>>";
+            return out;
+        }
+
+        friend void to_xml(const input& /*item*/, std::ostream& out)
+        {
+            out << "<input/>";
+        }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename T, typename MM>
+    class input<array2d<T,MM>> 
+    {
+    public:
+        typedef array2d<T,MM> input_type;
+
+        input() {}
+        input(const input&) {}
+
+        template <long NR, long NC, typename mm, typename L>
+        input(const input<matrix<T,NR,NC,mm,L>>&) {}
+
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const
+        {
+            DLIB_CASSERT(std::distance(ibegin,iend) > 0);
+            const auto nr = ibegin->nr();
+            const auto nc = ibegin->nc();
+            // make sure all the input matrices have the same dimensions
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                DLIB_CASSERT(i->nr()==nr && i->nc()==nc,
+                    "\t input::to_tensor()"
+                    << "\n\t All array2d objects given to to_tensor() must have the same dimensions."
+                    << "\n\t nr: " << nr
+                    << "\n\t nc: " << nc
+                    << "\n\t i->nr(): " << i->nr()
+                    << "\n\t i->nc(): " << i->nc()
+                );
+            }
+
+            
+            // initialize data to the right size to contain the stuff in the iterator range.
+            data.set_size(std::distance(ibegin,iend), pixel_traits<T>::num, nr, nc);
+            typedef typename pixel_traits<T>::basic_pixel_type bptype;
+
+            const size_t offset = nr*nc;
+            auto ptr = data.host();
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                for (long r = 0; r < nr; ++r)
+                {
+                    for (long c = 0; c < nc; ++c)
+                    {
+                        auto temp = pixel_to_vector<float>((*i)[r][c]);
+                        auto p = ptr++;
+                        for (long j = 0; j < temp.size(); ++j)
+                        {
+                            if (is_same_type<bptype,unsigned char>::value)
+                                *p = temp(j)/256.0;
+                            else
+                                *p = temp(j);
+                            p += offset;
+                        }
+                    }
+                }
+                ptr += offset*(data.k()-1);
+            }
+
+        }
+
+        friend void serialize(const input& item, std::ostream& out)
+        {
+            serialize("input<array2d>", out);
+        }
+
+        friend void deserialize(input& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "input<array2d>")
+                throw serialization_error("Unexpected version found while deserializing dlib::input.");
+        }
+        friend std::ostream& operator<<(std::ostream& out, const input& item)
+        {
+            out << "input<array2d>";
+            return out;
+        }
+
+        friend void to_xml(const input& item, std::ostream& out)
+        {
+            out << "<input/>";
+        }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename PYRAMID_TYPE>
+    class input_rgb_image_pyramid
+    {
+    public:
+        typedef matrix<rgb_pixel> input_type;
+        typedef PYRAMID_TYPE pyramid_type;
+
+        input_rgb_image_pyramid (
+        ) : 
+            avg_red(122.782), 
+            avg_green(117.001),
+            avg_blue(104.298) 
+        {
+        }
+
+        input_rgb_image_pyramid (
+            float avg_red_,
+            float avg_green_,
+            float avg_blue_
+        ) : avg_red(avg_red_), avg_green(avg_green_), avg_blue(avg_blue_) 
+        {}
+
+        float get_avg_red()   const { return avg_red; }
+        float get_avg_green() const { return avg_green; }
+        float get_avg_blue()  const { return avg_blue; }
+
+        unsigned long get_pyramid_padding () const { return pyramid_padding; }
+        void set_pyramid_padding (unsigned long value) { pyramid_padding = value; }
+
+        unsigned long get_pyramid_outer_padding () const { return pyramid_outer_padding; }
+        void set_pyramid_outer_padding (unsigned long value) { pyramid_outer_padding = value; }
+
+        bool image_contained_point (
+            const tensor& data,
+            const point& p
+        ) const
+        {
+            auto&& rects = any_cast<std::vector<rectangle>>(data.annotation());
+            DLIB_CASSERT(rects.size() > 0);
+            return rects[0].contains(p+rects[0].tl_corner());
+        }
+
+        drectangle tensor_space_to_image_space (
+            const tensor& data,
+            drectangle r
+        ) const
+        {
+            auto&& rects = any_cast<std::vector<rectangle>>(data.annotation());
+            return tiled_pyramid_to_image<pyramid_type>(rects, r);
+        }
+
+        drectangle image_space_to_tensor_space (
+            const tensor& data,
+            double scale,
+            drectangle r 
+        ) const
+        {
+            DLIB_CASSERT(0 < scale && scale <= 1 , "scale: "<< scale);
+            auto&& rects = any_cast<std::vector<rectangle>>(data.annotation());
+            return image_to_tiled_pyramid<pyramid_type>(rects, scale, r);
+        }
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const
+        {
+            DLIB_CASSERT(std::distance(ibegin,iend) > 0);
+            auto nr = ibegin->nr();
+            auto nc = ibegin->nc();
+            // make sure all the input matrices have the same dimensions
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                DLIB_CASSERT(i->nr()==nr && i->nc()==nc,
+                    "\t input_rgb_image_pyramid::to_tensor()"
+                    << "\n\t All matrices given to to_tensor() must have the same dimensions."
+                    << "\n\t nr: " << nr
+                    << "\n\t nc: " << nc
+                    << "\n\t i->nr(): " << i->nr()
+                    << "\n\t i->nc(): " << i->nc()
+                );
+            }
+
+            long NR, NC;
+            pyramid_type pyr;
+            auto& rects = data.annotation().get<std::vector<rectangle>>();
+            impl::compute_tiled_image_pyramid_details(pyr, nr, nc, pyramid_padding, pyramid_outer_padding, rects, NR, NC);
+
+            // initialize data to the right size to contain the stuff in the iterator range.
+            data.set_size(std::distance(ibegin,iend), 3, NR, NC);
+
+            // We need to zero the image before doing the pyramid, since the pyramid
+            // creation code doesn't write to all parts of the image.  We also take
+            // care to avoid triggering any device to hosts copies.
+            auto ptr = data.host_write_only();
+            for (size_t i = 0; i < data.size(); ++i)
+                ptr[i] = 0;
+
+            if (rects.size() == 0)
+                return;
+
+            // copy the first raw image into the top part of the tiled pyramid.  We need to
+            // do this for each of the input images/samples in the tensor.
+            for (auto i = ibegin; i != iend; ++i)
+            {
+                auto& img = *i;
+                ptr += rects[0].top()*data.nc();
+                for (long r = 0; r < img.nr(); ++r)
+                {
+                    auto p = ptr+rects[0].left();
+                    for (long c = 0; c < img.nc(); ++c)
+                        p[c] = (img(r,c).red-avg_red)/256.0;
+                    ptr += data.nc();
+                }
+                ptr += data.nc()*(data.nr()-rects[0].bottom()-1);
+
+                ptr += rects[0].top()*data.nc();
+                for (long r = 0; r < img.nr(); ++r)
+                {
+                    auto p = ptr+rects[0].left();
+                    for (long c = 0; c < img.nc(); ++c)
+                        p[c] = (img(r,c).green-avg_green)/256.0;
+                    ptr += data.nc();
+                }
+                ptr += data.nc()*(data.nr()-rects[0].bottom()-1);
+
+                ptr += rects[0].top()*data.nc();
+                for (long r = 0; r < img.nr(); ++r)
+                {
+                    auto p = ptr+rects[0].left();
+                    for (long c = 0; c < img.nc(); ++c)
+                        p[c] = (img(r,c).blue-avg_blue)/256.0;
+                    ptr += data.nc();
+                }
+                ptr += data.nc()*(data.nr()-rects[0].bottom()-1);
+            }
+
+            // now build the image pyramid into data.  This does the same thing as
+            // create_tiled_pyramid(), except we use the GPU if one is available. 
+            for (size_t i = 1; i < rects.size(); ++i)
+            {
+                alias_tensor src(data.num_samples(),data.k(),rects[i-1].height(),rects[i-1].width());
+                alias_tensor dest(data.num_samples(),data.k(),rects[i].height(),rects[i].width());
+
+                auto asrc  = src(data, data.nc()*rects[i-1].top() + rects[i-1].left());
+                auto adest = dest(data, data.nc()*rects[i].top() + rects[i].left());
+
+                tt::resize_bilinear(adest, data.nc(), data.nr()*data.nc(), 
+                                    asrc, data.nc(), data.nr()*data.nc());
+            }
+        }
+
+        friend void serialize(const input_rgb_image_pyramid& item, std::ostream& out)
+        {
+            serialize("input_rgb_image_pyramid2", out);
+            serialize(item.avg_red, out);
+            serialize(item.avg_green, out);
+            serialize(item.avg_blue, out);
+            serialize(item.pyramid_padding, out);
+            serialize(item.pyramid_outer_padding, out);
+        }
+
+        friend void deserialize(input_rgb_image_pyramid& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "input_rgb_image_pyramid" && version != "input_rgb_image_pyramid2")
+                throw serialization_error("Unexpected version found while deserializing dlib::input_rgb_image_pyramid.");
+            deserialize(item.avg_red, in);
+            deserialize(item.avg_green, in);
+            deserialize(item.avg_blue, in);
+            if (version == "input_rgb_image_pyramid2")
+            {
+                deserialize(item.pyramid_padding, in);
+                deserialize(item.pyramid_outer_padding, in);
+            }
+            else
+            {
+                item.pyramid_padding = 10;
+                item.pyramid_outer_padding = 11;
+            }
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const input_rgb_image_pyramid& item)
+        {
+            out << "input_rgb_image_pyramid("<<item.avg_red<<","<<item.avg_green<<","<<item.avg_blue<<")";
+            out << " pyramid_padding="<<item.pyramid_padding;
+            out << " pyramid_outer_padding="<<item.pyramid_outer_padding;
+            return out;
+        }
+
+        friend void to_xml(const input_rgb_image_pyramid& item, std::ostream& out)
+        {
+            out << "<input_rgb_image_pyramid r='"<<item.avg_red<<"' g='"<<item.avg_green
+                <<"' b='"<<item.avg_blue
+                <<"' pyramid_padding='"<<item.pyramid_padding
+                <<"' pyramid_outer_padding='"<<item.pyramid_outer_padding
+                <<"'/>";
+        }
+
+    private:
+        float avg_red;
+        float avg_green;
+        float avg_blue;
+        unsigned long pyramid_padding = 10;
+        unsigned long pyramid_outer_padding = 11;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_INPUT_H_
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/input_abstract.h

@@ -0,0 +1,467 @@
+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_DNn_INPUT_ABSTRACT_H_
+#ifdef DLIB_DNn_INPUT_ABSTRACT_H_
+
+#include "../matrix.h"
+#include "../pixel.h"
+
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    class EXAMPLE_INPUT_LAYER
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                Each deep neural network model in dlib begins with an input layer. The job
+                of the input layer is to convert an input_type into a tensor.  Nothing more
+                and nothing less.  
+                
+                Note that there is no dlib::EXAMPLE_INPUT_LAYER type.  It is shown here
+                purely to document the interface that an input layer object must implement.
+                If you are using some kind of image or matrix object as your input_type
+                then you can use the provided dlib::input layer defined below.  Otherwise,
+                you need to define your own custom input layer.
+
+            THREAD SAFETY
+                to_tensor() must be thread safe.  That is, multiple threads must be able to
+                make calls to to_tensor() on a single instance of this object at the same
+                time.
+        !*/
+    public:
+
+        EXAMPLE_INPUT_LAYER(
+        );
+        /*!
+            ensures
+                - Default constructs this object.  This function is not required to do
+                  anything in particular but it must exist, that is, it is required that
+                  layer objects be default constructable. 
+        !*/
+
+        EXAMPLE_INPUT_LAYER (
+            const EXAMPLE_INPUT_LAYER& item
+        );
+        /*!
+            ensures
+                - EXAMPLE_INPUT_LAYER objects are copy constructable
+        !*/
+
+        EXAMPLE_INPUT_LAYER(
+            const some_other_input_layer_type& item
+        );
+        /*!
+            ensures
+                - Constructs this object from item.  This form of constructor is optional
+                  but it allows you to provide a conversion from one input layer type to
+                  another.  For example, the following code is valid only if my_input_layer2 can
+                  be constructed from my_input_layer1:
+                    relu<fc<relu<fc<my_input_layer1>>>> my_dnn1;
+                    relu<fc<relu<fc<my_input_layer2>>>> my_dnn2(my_dnn1);
+                  This kind of pattern is useful if you want to use one type of input layer
+                  during training but a different type of layer during testing since it
+                  allows you to easily convert between related deep neural network types.  
+        !*/
+
+        typedef whatever_type_to_tensor_expects input_type;
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const;
+        /*!
+            requires
+                - [ibegin, iend) is an iterator range over input_type objects.
+                - std::distance(ibegin,iend) > 0
+            ensures
+                - Converts the iterator range into a tensor and stores it into #data.
+                - #data.num_samples()%distance(ibegin,iend) == 0. 
+                  Normally you would have #data.num_samples() == distance(ibegin,iend) but
+                  you can also expand the output by some integer factor so long as the loss
+                  you use can deal with it correctly.
+                - The data in the ith sample of #data corresponds to the input_type object
+                  *(ibegin+i/sample_expansion_factor).
+                  where sample_expansion_factor==#data.num_samples()/distance(ibegin,iend).
+        !*/
+    };
+
+    std::ostream& operator<<(std::ostream& out, const EXAMPLE_INPUT_LAYER& item);
+    /*!
+        print a string describing this layer.
+    !*/
+
+    void to_xml(const EXAMPLE_INPUT_LAYER& item, std::ostream& out);
+    /*!
+        This function is optional, but required if you want to print your networks with
+        net_to_xml().  Therefore, to_xml() prints a layer as XML.
+    !*/
+
+    void serialize(const EXAMPLE_INPUT_LAYER& item, std::ostream& out);
+    void deserialize(EXAMPLE_INPUT_LAYER& item, std::istream& in);
+    /*!
+        provides serialization support  
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename T
+        >
+    class input 
+    {
+        /*!
+            REQUIREMENTS ON T
+                One of the following must be true:
+                    - T is a matrix or array2d object and it must contain some kind of
+                      pixel type.  I.e. pixel_traits<T::type> must be defined.   
+                    - T is a std::array<matrix<U>> where U is any built in scalar type like
+                      float, double, or unsigned char. 
+
+            WHAT THIS OBJECT REPRESENTS
+                This is a basic input layer that simply copies images into a tensor.  
+        !*/
+
+    public:
+        typedef T input_type;
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const;
+        /*!
+            requires
+                - [ibegin, iend) is an iterator range over input_type objects.
+                - std::distance(ibegin,iend) > 0
+                - The input range should contain image objects that all have the same
+                  dimensions.
+            ensures
+                - Converts the iterator range into a tensor and stores it into #data.  In
+                  particular, if the input images have R rows, C columns, and K channels
+                  (where K is given by pixel_traits::num or std::array::size() if
+                  std::array inputs are used) then we will have:
+                    - #data.num_samples() == std::distance(ibegin,iend)
+                    - #data.nr() == R
+                    - #data.nc() == C
+                    - #data.k() == K
+                  For example, a matrix<float,3,3> would turn into a tensor with 3 rows, 3
+                  columns, and k()==1.  Or a matrix<rgb_pixel,4,5> would turn into a tensor
+                  with 4 rows, 5 columns, and k()==3 (since rgb_pixels have 3 channels).
+                  Or a std::array<matrix<float,3,3>,5> would turn into a tensor with 3 rows
+                  and columns, and k()==5 channels.
+                - If the input data contains pixels of type unsigned char, rgb_pixel, or
+                  other pixel types with a basic_pixel_type of unsigned char then each
+                  value written to the output tensor is first divided by 256.0 so that the
+                  resulting outputs are all in the range [0,1].
+        !*/
+
+        // Provided for compatibility with input_rgb_image_pyramid's interface
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class input_rgb_image
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This input layer works with RGB images of type matrix<rgb_pixel>.  It is
+                very similar to the dlib::input layer except that it allows you to subtract
+                the average color value from each color channel when converting an image to
+                a tensor.
+        !*/
+    public:
+        typedef matrix<rgb_pixel> input_type;
+
+        input_rgb_image (
+        );
+        /*!
+            ensures
+                - #get_avg_red()   == 122.782
+                - #get_avg_green() == 117.001
+                - #get_avg_blue()  == 104.298
+        !*/
+
+        input_rgb_image (
+            float avg_red,
+            float avg_green,
+            float avg_blue
+        ); 
+        /*!
+            ensures
+                - #get_avg_red() == avg_red
+                - #get_avg_green() == avg_green
+                - #get_avg_blue() == avg_blue
+        !*/
+
+        float get_avg_red(
+        ) const;
+        /*!
+            ensures
+                - returns the value subtracted from the red color channel.
+        !*/
+
+        float get_avg_green(
+        ) const;
+        /*!
+            ensures
+                - returns the value subtracted from the green color channel.
+        !*/
+
+        float get_avg_blue(
+        ) const;
+        /*!
+            ensures
+                - returns the value subtracted from the blue color channel.
+        !*/
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const;
+        /*!
+            requires
+                - [ibegin, iend) is an iterator range over input_type objects.
+                - std::distance(ibegin,iend) > 0
+                - The input range should contain images that all have the same
+                  dimensions.
+            ensures
+                - Converts the iterator range into a tensor and stores it into #data.  In
+                  particular, if the input images have R rows, C columns then we will have:
+                    - #data.num_samples() == std::distance(ibegin,iend)
+                    - #data.nr() == R
+                    - #data.nc() == C
+                    - #data.k() == 3
+                  Moreover, each color channel is normalized by having its average value
+                  subtracted (according to get_avg_red(), get_avg_green(), or
+                  get_avg_blue()) and then is divided by 256.0.
+        !*/
+
+
+        // Provided for compatibility with input_rgb_image_pyramid's interface
+        bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
+        drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
+        drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <size_t NR, size_t NC=NR>
+    class input_rgb_image_sized 
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This layer has an interface and behavior identical to input_rgb_image
+                except that it requires input images to have NR rows and NC columns.  This
+                is checked by a DLIB_CASSERT inside to_tensor().
+
+                You can also convert between input_rgb_image and input_rgb_image_sized by
+                copy construction or assignment.
+        !*/
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename PYRAMID_TYPE
+        >
+    class input_rgb_image_pyramid
+    {
+        /*!
+            REQUIREMENTS ON PYRAMID_TYPE
+                PYRAMID_TYPE must be an instance of the dlib::pyramid_down template.
+
+            WHAT THIS OBJECT REPRESENTS
+                This input layer works with RGB images of type matrix<rgb_pixel>.  It is
+                identical to input_rgb_image except that it outputs a tensor containing a
+                tiled image pyramid of each input image rather than a simple copy of each
+                image.  The tiled image pyramid is created using create_tiled_pyramid().
+        !*/
+
+    public:
+
+        typedef matrix<rgb_pixel> input_type;
+        typedef PYRAMID_TYPE pyramid_type;
+
+        input_rgb_image_pyramid (
+        );
+        /*!
+            ensures
+                - #get_avg_red()   == 122.782
+                - #get_avg_green() == 117.001
+                - #get_avg_blue()  == 104.298
+                - #get_pyramid_padding() == 10
+                - #get_pyramid_outer_padding() == 11
+        !*/
+
+        input_rgb_image_pyramid (
+            float avg_red,
+            float avg_green,
+            float avg_blue
+        ); 
+        /*!
+            ensures
+                - #get_avg_red() == avg_red
+                - #get_avg_green() == avg_green
+                - #get_avg_blue() == avg_blue
+                - #get_pyramid_padding() == 10
+                - #get_pyramid_outer_padding() == 11
+        !*/
+
+        float get_avg_red(
+        ) const;
+        /*!
+            ensures
+                - returns the value subtracted from the red color channel.
+        !*/
+
+        float get_avg_green(
+        ) const;
+        /*!
+            ensures
+                - returns the value subtracted from the green color channel.
+        !*/
+
+        float get_avg_blue(
+        ) const;
+        /*!
+            ensures
+                - returns the value subtracted from the blue color channel.
+        !*/
+
+        unsigned long get_pyramid_padding (
+        ) const; 
+        /*!
+            ensures
+                - When this object creates a pyramid it will call create_tiled_pyramid() and
+                  set create_tiled_pyramid's pyramid_padding parameter to get_pyramid_padding().
+        !*/
+        void set_pyramid_padding (
+            unsigned long value
+        );
+        /*!
+            ensures
+                - #get_pyramid_padding() == value
+        !*/
+
+        unsigned long get_pyramid_outer_padding (
+        ) const; 
+        /*!
+            ensures
+                - When this object creates a pyramid it will call create_tiled_pyramid()
+                  and set create_tiled_pyramid's pyramid_outer_padding parameter to
+                  get_pyramid_outer_padding().
+        !*/
+        void set_pyramid_outer_padding (
+            unsigned long value
+        );
+        /*!
+            ensures
+                - #get_pyramid_outer_padding() == value
+        !*/
+
+        template <typename forward_iterator>
+        void to_tensor (
+            forward_iterator ibegin,
+            forward_iterator iend,
+            resizable_tensor& data
+        ) const;
+        /*!
+            requires
+                - [ibegin, iend) is an iterator range over input_type objects.
+                - std::distance(ibegin,iend) > 0
+                - The input range should contain images that all have the same
+                  dimensions.
+            ensures
+                - Converts the iterator range into a tensor and stores it into #data.  In
+                  particular, we will have:
+                    - #data.num_samples() == std::distance(ibegin,iend)
+                    - #data.k() == 3
+                    - Each sample in #data contains a tiled image pyramid of the
+                      corresponding input image.  The tiled pyramid is created by
+                      create_tiled_pyramid().
+                  Moreover, each color channel is normalized by having its average value
+                  subtracted (according to get_avg_red(), get_avg_green(), or
+                  get_avg_blue()) and then is divided by 256.0.
+        !*/
+
+        bool image_contained_point (
+            const tensor& data,
+            const point& p
+        ) const;
+        /*!
+            requires
+                - data is a tensor that was produced by this->to_tensor()
+            ensures
+                - Since data is a tensor that is built from a bunch of identically sized
+                  images, we can ask if those images were big enough to contain the point
+                  p.  This function returns the answer to that question.
+        !*/
+
+        drectangle image_space_to_tensor_space (
+            const tensor& data,
+            double scale,
+            drectangle r 
+        ) const;
+        /*!
+            requires
+                - data is a tensor that was produced by this->to_tensor()
+                - 0 < scale <= 1
+            ensures
+                - This function maps from to_tensor()'s input image space to its output
+                  tensor space.  Therefore, given that data is a tensor produced by
+                  to_tensor(), image_space_to_tensor_space() allows you to ask for the
+                  rectangle in data that corresponds to a rectangle in the original image
+                  space.
+
+                  Note that since the output tensor contains an image pyramid, there are
+                  multiple points in the output tensor that correspond to any input
+                  location.  So you must also specify a scale so we know what level of the
+                  pyramid is needed.  So given a rectangle r in an input image, you can
+                  ask, what rectangle in data corresponds to r when things are scale times
+                  smaller?  That rectangle is returned by this function.
+                - A scale of 1 means we don't move anywhere in the pyramid scale space relative
+                  to the input image while smaller values of scale mean we move down the
+                  pyramid.
+        !*/
+
+        drectangle tensor_space_to_image_space (
+            const tensor& data,
+            drectangle r
+        ) const;
+        /*!
+            requires
+                - data is a tensor that was produced by this->to_tensor()
+            ensures
+                - This function maps from to_tensor()'s output tensor space to its input
+                  image space.  Therefore, given that data is a tensor produced by
+                  to_tensor(), tensor_space_to_image_space() allows you to ask for the
+                  rectangle in the input image that corresponds to a rectangle in data.
+                - It should be noted that this function isn't always an inverse of
+                  image_space_to_tensor_space().  This is because you can ask
+                  image_space_to_tensor_space() for the coordinates of points outside the input
+                  image and they will be mapped to somewhere that doesn't have an inverse.
+                  But for points actually inside the input image this function performs an
+                  approximate inverse mapping.  I.e. when image_contained_point(data,center(r))==true 
+                  there is an approximate inverse.
+        !*/
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_INPUT_ABSTRACT_H_
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/solvers.h

@@ -0,0 +1,405 @@
+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNn_SOLVERS_H_
+#define DLIB_DNn_SOLVERS_H_
+
+#include "solvers_abstract.h"
+#include "../cuda/tensor.h"
+#include <iostream>
+#include "layers.h"
+
+namespace dlib
+{
+    class sgd
+    {
+    public:
+
+        explicit sgd(
+            float weight_decay_,
+            float momentum_ = 0.9
+        ) 
+        { 
+            weight_decay = weight_decay_;
+            momentum = momentum_;
+        }
+
+        sgd(
+        ) : sgd(0.0005, 0.9) 
+        { 
+        }
+
+        float get_momentum (
+        ) const { return momentum; }
+
+        float get_weight_decay (
+        ) const { return weight_decay; }
+
+        template <typename layer_type> 
+        const tensor& operator() (
+            const float learning_rate,
+            const layer_type& l,
+            const tensor& params_grad
+        )
+        {
+            const tensor& params = l.get_layer_params();
+
+            DLIB_CASSERT(params.size() != 0);
+            if (v.size() == 0)
+            {
+                v.copy_size(params_grad);
+                v = 0;
+            }
+
+            const double lr = learning_rate*get_learning_rate_multiplier(l);
+            const double wd = weight_decay*get_weight_decay_multiplier(l);
+            
+            //perform: v = momentum*mat(v) - wd*lr*mat(params) - lr*mat(params_grad);
+            tt::affine_transform(v, v, params, params_grad, momentum, -wd*lr, -lr);
+
+            return v;
+        }
+
+        template <unsigned long N>
+        const tensor& operator() (
+            const float learning_rate,
+            const fc_<N,FC_HAS_BIAS>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.get_num_outputs());
+            return v;
+        }
+
+        template <
+            long _num_filters,
+            long _nr,
+            long _nc,
+            int _stride_y,
+            int _stride_x,
+            int _padding_y,
+            int _padding_x
+            >
+        const tensor& operator() (
+            const float learning_rate,
+            const con_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
+            return v;
+        }
+
+        template <
+            long _num_filters,
+            long _nr,
+            long _nc,
+            int _stride_y,
+            int _stride_x,
+            int _padding_y,
+            int _padding_x
+            >
+        const tensor& operator() (
+            const float learning_rate,
+            const cont_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
+            return v;
+        }
+
+        template < layer_mode mode >
+        const tensor& operator() (
+            const float learning_rate,
+            const bn_<mode>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()/2);
+            return v;
+        }
+
+        friend void serialize(const sgd& item, std::ostream& out)
+        {
+            serialize("sgd2", out);
+            serialize(item.v, out);
+            serialize(item.weight_decay, out);
+            serialize(item.momentum, out);
+        }
+
+        friend void deserialize(sgd& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "sgd2")
+                throw serialization_error("Unexpected version found while deserializing dlib::sgd.");
+            deserialize(item.v, in);
+            deserialize(item.weight_decay, in);
+            deserialize(item.momentum, in);
+        }
+
+        friend std::ostream& operator<< (std::ostream& out, const sgd& item)
+        {
+            out << "sgd: weight_decay="<<item.get_weight_decay() << ", momentum="<<item.get_momentum(); 
+            return out;
+        }
+
+    private:
+
+        template <typename layer_type> 
+        void update_considering_bias(
+            const float learning_rate,
+            const layer_type& l,
+            const tensor& params_grad,
+            unsigned long bias_offset
+        )
+        {
+            const tensor& params = l.get_layer_params();
+
+            DLIB_CASSERT(params.size() != 0);
+            if (v.size() == 0)
+            {
+                v.copy_size(params_grad);
+                v = 0;
+            }
+
+            double lr = learning_rate*get_learning_rate_multiplier(l);
+            double wd = weight_decay*get_weight_decay_multiplier(l);
+            
+            //perform: v = momentum*mat(v) - wd*lr*mat(params) - lr*mat(params_grad);
+
+            if (l.get_bias_learning_rate_multiplier() == 1 && l.get_bias_weight_decay_multiplier() == 1)
+            {
+                tt::affine_transform(v, v, params, params_grad, momentum, -wd*lr, -lr);
+            }
+            else
+            {
+
+                tt::affine_transform_range(0, bias_offset, v, v, params, params_grad, momentum, -wd*lr, -lr);
+
+                // now update the biases but apply their multipliers
+                lr *= l.get_bias_learning_rate_multiplier();
+                wd *= l.get_bias_weight_decay_multiplier();
+                tt::affine_transform_range(bias_offset, v.size(), v, v, params, params_grad, momentum, -wd*lr, -lr);
+            }
+        }
+
+        resizable_tensor v;
+        float weight_decay;
+        float momentum;
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    class adam 
+    {
+    public:
+
+        adam(
+            float weight_decay_,
+            float momentum1_, 
+            float momentum2_
+        ) 
+        { 
+            weight_decay = weight_decay_;
+            momentum1 = momentum1_;
+            momentum2 = momentum2_;
+            t = 0;
+        }
+
+        adam(
+        ) : adam(0.0005, 0.9, 0.999) 
+        {}
+
+        float get_momentum1 (
+        ) const { return momentum1; }
+
+        float get_momentum2 (
+        ) const { return momentum2; }
+
+        float get_weight_decay (
+        ) const { return weight_decay; }
+
+        template <typename layer_type>
+        const tensor& operator() (
+            const float learning_rate,
+            const layer_type& l,
+            const tensor& params_grad
+        )
+        {
+            const tensor& params = l.get_layer_params();
+            DLIB_CASSERT(params.size() != 0);
+            if (v.size() == 0)
+            {
+                m.copy_size(params_grad);
+                m = 0;
+                v.copy_size(params_grad);
+                v = 0;
+                s.copy_size(params_grad);
+            }
+
+            ++t;
+
+            
+            tt::compute_adam_update(0, params.size(), s, m, v, t,
+                learning_rate*get_learning_rate_multiplier(l),
+                weight_decay*get_weight_decay_multiplier(l), 
+                momentum1, momentum2, params, params_grad);
+
+            return s;
+        }
+
+        template <unsigned long N>
+        const tensor& operator() (
+            const float learning_rate,
+            const fc_<N,FC_HAS_BIAS>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.get_num_outputs());
+            return s;
+        }
+
+        template <
+            long _num_filters,
+            long _nr,
+            long _nc,
+            int _stride_y,
+            int _stride_x,
+            int _padding_y,
+            int _padding_x
+            >
+        const tensor& operator() (
+            const float learning_rate,
+            const con_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
+            return s;
+        }
+
+        template <
+            long _num_filters,
+            long _nr,
+            long _nc,
+            int _stride_y,
+            int _stride_x,
+            int _padding_y,
+            int _padding_x
+            >
+        const tensor& operator() (
+            const float learning_rate,
+            const cont_<_num_filters,_nr,_nc,_stride_y,_stride_x,_padding_y,_padding_x>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()-l.num_filters());
+            return s;
+        }
+
+        template < layer_mode mode >
+        const tensor& operator() (
+            const float learning_rate,
+            const bn_<mode>& l,
+            const tensor& params_grad
+        )
+        {
+            update_considering_bias(learning_rate, l, params_grad, params_grad.size()/2);
+            return s;
+        }
+
+
+        friend void serialize(const adam& item, std::ostream& out)
+        {
+            serialize("adam2", out);
+            serialize(item.m, out);
+            serialize(item.v, out);
+            serialize(item.s, out);
+            serialize(item.weight_decay, out);
+            serialize(item.momentum1, out);
+            serialize(item.momentum2, out);
+            serialize(item.t, out);
+        }
+
+        friend void deserialize(adam& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "adam2")
+                throw serialization_error("Unexpected version found while deserializing dlib::adam.");
+            deserialize(item.m, in);
+            deserialize(item.v, in);
+            deserialize(item.s, in);
+            deserialize(item.weight_decay, in);
+            deserialize(item.momentum1, in);
+            deserialize(item.momentum2, in);
+            deserialize(item.t, in);
+        }
+
+        friend std::ostream& operator<< (std::ostream& out, const adam& item)
+        {
+            out << "adam: weight_decay="<<item.get_weight_decay() << ", momentum1="<<item.get_momentum1() << ", momentum2="<<item.get_momentum2(); 
+            return out;
+        }
+
+    private:
+
+        template <typename layer_type> 
+        void update_considering_bias(
+            const float learning_rate,
+            const layer_type& l,
+            const tensor& params_grad,
+            unsigned long bias_offset
+        )
+        {
+            const tensor& params = l.get_layer_params();
+            DLIB_CASSERT(params.size() != 0);
+            if (v.size() == 0)
+            {
+                m.copy_size(params_grad);
+                m = 0;
+                v.copy_size(params_grad);
+                v = 0;
+                s.copy_size(params_grad);
+            }
+
+
+            ++t;
+
+            if (l.get_bias_learning_rate_multiplier() == 1 && l.get_bias_weight_decay_multiplier() == 1)
+            {
+                tt::compute_adam_update(0, params.size(), s, m, v, t,
+                    learning_rate*get_learning_rate_multiplier(l),
+                    weight_decay*get_weight_decay_multiplier(l), 
+                    momentum1, momentum2, params, params_grad);
+            }
+            else
+            {
+                tt::compute_adam_update(0, bias_offset, s, m, v, t,
+                    learning_rate*get_learning_rate_multiplier(l),
+                    weight_decay*get_weight_decay_multiplier(l), 
+                    momentum1, momentum2, params, params_grad);
+
+                tt::compute_adam_update(bias_offset, params.size(), s, m, v, t,
+                    learning_rate*get_learning_rate_multiplier(l)*l.get_bias_learning_rate_multiplier(),
+                    weight_decay*get_weight_decay_multiplier(l)*l.get_bias_weight_decay_multiplier(), 
+                    momentum1, momentum2, params, params_grad);
+            }
+        }
+        resizable_tensor m;
+        resizable_tensor v;
+        resizable_tensor s;
+        float weight_decay;
+        float momentum1;
+        float momentum2;
+        float t;
+    };
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_SOLVERS_H_
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/solvers_abstract.h

@@ -0,0 +1,204 @@
+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_DNn_SOLVERS_ABSTRACT_H_
+#ifdef DLIB_DNn_SOLVERS_ABSTRACT_H_
+
+#include "../cuda/tensor_abstract.h"
+#include <iostream>
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    class EXAMPLE_SOLVER 
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                A solver defines the parameter update rule for a single layer in a deep
+                neural network.  It takes a parameter gradient vector and the layer's
+                parameters and tells you how the parameters should be updated.
+                Importantly, each solver instance is used with only one layer in a network.
+                This allows us to define solvers that have per layer state, for example, a
+                solver may keep a momentum term and apply it to its update rule.
+
+                Note that there is no dlib::EXAMPLE_SOLVER type.  It is shown here purely
+                to document the interface a solver object must implement.
+        !*/
+
+    public:
+
+        EXAMPLE_SOLVER(
+        );
+
+        template <typename layer_type>
+        const tensor& operator() (
+            const float learning_rate,
+            const layer_type& l,
+            const tensor& params_grad
+        )
+        /*!
+            requires
+                - l.get_layer_params().size() != 0
+                - have_same_dimensions(l.get_layer_params(), params_grad) == true.
+                - When this function is invoked on a particular solver instance, it is
+                  always supplied with the same layer instance, l.  That is, the solver is
+                  allowed to remember things from one invocation to another and to assume
+                  that it is being serially applied to optimize the same layer's
+                  parameters. 
+            ensures
+                - Returns a step vector V that is intended to be used to update the
+                  parameters by adding V to l.get_layer_params().
+                - This function will use the given "learning rate" to compute V.  How the
+                  learning rate is used is solver dependent.  But in general the learning
+                  rate should be used to select the step size, i.e. to somehow determine
+                  the magnitude of V.
+        !*/
+    };
+
+    void serialize(const EXAMPLE_SOLVER& item, std::ostream& out);
+    void deserialize(EXAMPLE_SOLVER& item, std::istream& in);
+    /*!
+        provides serialization support  
+    !*/
+
+    std::ostream& operator<< (std::ostream& out, const EXAMPLE_SOLVER& item);
+    /*!
+        Prints the solver's name and parameters to out.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------------
+
+    class sgd
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object implements the EXAMPLE_SOLVER interface defined above.  It is a
+                basic stochastic gradient descent solver which uses momentum and weight
+                decay.  In particular, it computes the update vector V according to:
+                    V = momentum*V - weight_decay*learning_rate*l.get_layer_params() - learning_rate*params_grad;
+                Here V is a momentum term that is remembered by the solver from one
+                invocation of operator() to the next.  
+
+
+                Note that the actual learning rate and weight decay used by the solver are
+                multiplied by the per layer multipliers.  That is, the solver will call
+                get_learning_rate_multiplier(l) and get_weight_decay_multiplier(l) and
+                multiply these values with the nominal learning rate and weight decay,
+                respectively, to determine the values it will use during each step.  It is
+                also overloaded to allow additional learning rate multipliers to be applied
+                to fc_ and con_ bias parameters.
+        !*/
+    public:
+
+        sgd(
+        ); 
+        /*!
+            ensures
+                - #get_weight_decay()  == 0.0005 
+                - #get_momentum()      == 0.9 
+        !*/
+
+        explicit sgd(
+            float weight_decay,
+            float momentum = 0.9
+        ); 
+        /*!
+            requires
+                - weight_decay >= 0
+                - momentum >= 0
+            ensures
+                - #get_weight_decay()  == weight_decay 
+                - #get_momentum()      == momentum 
+        !*/
+
+        float get_weight_decay () const;
+        float get_momentum () const; 
+    };
+
+    void serialize(const sgd& item, std::ostream& out);
+    void deserialize(sgd& item, std::istream& in);
+    /*!
+        provides serialization support  
+    !*/
+
+    std::ostream& operator<< (std::ostream& out, const sgd& item);
+    /*!
+        Prints the solver's name and parameters to out.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    class adam
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object implements the EXAMPLE_SOLVER interface defined above.  In
+                particular, it implements the ADAM parameter update method described in the
+                paper:
+                    Kingma, Diederik P., and Jimmy Ba Adam. "A method for stochastic
+                    optimization." International Conference on Learning Representation. 2015.
+
+
+                Note that the actual learning rate and weight decay used by the solver are
+                multiplied by the per layer multipliers.  That is, the solver will call
+                get_learning_rate_multiplier(l) and get_weight_decay_multiplier(l) and
+                multiply these values with the nominal learning rate and weight decay,
+                respectively, to determine the values it will use during each step.  It is
+                also overloaded to allow additional learning rate multipliers to be applied
+                to fc_ and con_ bias parameters.
+        !*/
+
+    public:
+
+        adam(
+        ); 
+        /*!
+            ensures
+                - #get_weight_decay()  == 0.0005 
+                - #get_momentum1()     == 0.9 
+                - #get_momentum2()     == 0.999 
+        !*/
+
+        adam(
+            float weight_decay,
+            float momentum1, 
+            float momentum2 
+        ); 
+        /*!
+            requires
+                - weight_decay >= 0
+                - 0 <= momentum1 < 1
+                - 0 <= momentum2 < 1
+            ensures
+                - #get_weight_decay()  == weight_decay 
+                - #get_momentum1()     == momentum1
+                - #get_momentum2()     == momentum2
+        !*/
+
+        float get_weight_decay () const;
+        float get_momentum1 () const; 
+        float get_momentum2 () const; 
+    };
+
+    void serialize(const adam& item, std::ostream& out);
+    void deserialize(adam& item, std::istream& in);
+    /*!
+        provides serialization support  
+    !*/
+
+    std::ostream& operator<< (std::ostream& out, const adam& item);
+    /*!
+        Prints the solver's name and parameters to out.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_SOLVERS_ABSTRACT_H_
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/trainer_abstract.h

@@ -0,0 +1,765 @@
+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_DNn_TRAINER_ABSTRACT_H_
+#ifdef DLIB_DNn_TRAINER_ABSTRACT_H_
+
+#include "core_abstract.h"
+#include "solvers_abstract.h"
+#include <vector>
+#include <chrono>
+
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    enum class force_flush_to_disk {
+        no = 0,
+        yes = 1
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename net_type, 
+        typename solver_type = sgd
+        >
+    class dnn_trainer
+    {
+        /*!
+            REQUIREMENTS ON net_type
+                - net_type is an add_loss_layer object.
+
+            REQUIREMENTS ON solver_type
+                - solver_type is an implementation of the EXAMPLE_SOLVER interface defined
+                  in solvers_abstract.h
+
+            WHAT THIS OBJECT REPRESENTS
+                This object is a tool training a deep neural network. To use it you supply
+                a neural network type and a solver, then you call train() with your
+                training data and it will output a new network instance that has hopefully
+                learned something useful from your training data.
+
+                If you are compiling with CUDA then this object will use the GPU that is
+                currently selected (i.e. the one indicated by cudaGetDevice()) when
+                dnn_trainer is constructed.  It will continue to use that device even if
+                you later change it by a call to cudaSetDevice().
+
+            EXCEPTIONS
+                If an exception is thrown by any part of the neural network during training
+                then the exception will be propagated out of the trainer to the user.
+                Moreover, the trainer instance will be unusable and should be destroyed.
+        !*/
+
+    public:
+
+        typedef typename net_type::training_label_type training_label_type;
+        typedef typename net_type::input_type input_type;
+        const static size_t num_computational_layers = net_type::num_computational_layers;
+
+        dnn_trainer() = delete;
+        dnn_trainer(const dnn_trainer&) = delete;
+        dnn_trainer& operator=(const dnn_trainer&) = delete;
+
+        dnn_trainer(
+            net_type& net, 
+            const solver_type& solver = solver_type(),
+            const std::vector<int>& cuda_extra_devices = {}
+        ); 
+        /*!
+            requires
+                - for all valid i:
+                    - 0 <= cuda_extra_devices[i] < dlib::cuda::get_num_devices()
+            ensures
+                - &#get_net() == &net 
+                  (i.e. The dnn_trainer holds a reference to net, it does not copy it.
+                  Therefore, you must ensure net has a lifetime at least as long as the
+                  dnn_trainer).
+                - #get_solvers() == a set of solvers that are all initialized with the
+                  provided solver instance.
+                - #get_max_num_epochs() == 10000
+                - #get_mini_batch_size() == 128
+                - #get_learning_rate() == 1e-2 
+                - #get_min_learning_rate() == 1e-5
+                - #get_iterations_without_progress_threshold() == 2000
+                - #get_test_iterations_without_progress_threshold() == 500
+                - #get_learning_rate_shrink_factor() == 0.1
+                - #get_learning_rate_schedule().size() == 0
+                - #get_train_one_step_calls() == 0
+                - #get_test_one_step_calls() == 0
+                - #get_synchronization_file() == ""
+                - if (cuda_extra_devices.size() > 0) then
+                    - This object will use multiple graphics cards to run the learning
+                      algorithms.  In particular, it will always use whatever device is
+                      currently selected on the calling thread (the device indicated by
+                      cudaGetDevice()).  In addition, you can ask to use additional
+                      devices, which you do by putting their device numbers into
+                      cuda_extra_devices.
+        !*/
+
+        net_type& get_net (
+            force_flush_to_disk force_flush = force_flush_to_disk::yes
+        ); 
+        /*!
+            ensures
+                - returns the neural network object used by this trainer.  This is the
+                  network that is optimized when you call train() or train_one_step().
+                  Recall that the dnn_trainer doesn't contain the net_type object but
+                  simply holds a reference to an external network which was provided to the
+                  dnn_trainer's constructor.
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+                - If force_flush is yes, then this function will sync the trainer state to
+                  disk if the current state hasn't already been synced to disk since the
+                  last network modification.
+        !*/
+
+        const std::vector<solver_type>& get_solvers (
+        ) const; 
+        /*!
+            ensures
+                - returns the solvers used to optimize each layer of the neural network
+                  get_net().  In particular, the first layer's solver is
+                  get_solvers()[0], the second layer's solver is
+                  get_solvers()[1], and so on.
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        unsigned long get_mini_batch_size (
+        ) const; 
+        /*!
+            ensures
+                - During training, we call the network's update() routine over and over
+                  with training data.  The number of training samples we give to each call
+                  to update is the "mini-batch size", which is defined by
+                  get_mini_batch_size().
+        !*/
+
+        void set_mini_batch_size (
+            unsigned long batch_size 
+        );
+        /*!
+            requires
+                - batch_size > 0
+            ensures
+                - #get_mini_batch_size() == batch_size
+        !*/
+
+        unsigned long get_max_num_epochs (
+        ) const; 
+        /*!
+            ensures
+                - train() will execute at most get_max_num_epochs() iterations over the
+                  training data before returning.
+        !*/
+
+        void set_max_num_epochs (
+            unsigned long num
+        );
+        /*!
+            requires
+                - num > 0
+            ensures
+                - #get_max_num_epochs() == num
+        !*/
+
+        void set_learning_rate (
+            double lr
+        );
+        /*!
+            requires
+                - lr > 0
+            ensures
+                - #get_learning_rate() == lr
+                - #get_learning_rate_schedule().size() == 0
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        double get_learning_rate(
+        ) const;
+        /*!
+            ensures
+                - During each training step, a solver tells us how to modify the parameters
+                  of each layer in the network.  It does this by outputting a step vector
+                  that, when added to the parameters, will hopefully result in improved
+                  network performance.  The learning rate is one of the inputs to the
+                  solver and influences the size of this step vector.  This function
+                  returns the current learning rate, that is, the learning rate that will
+                  be used during the next training step.
+        !*/
+
+        void set_min_learning_rate (
+            double lr
+        );
+        /*!
+            requires
+                - lr > 0
+            ensures
+                - #get_min_learning_rate() == lr
+                - #get_learning_rate_schedule().size() == 0
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        double get_min_learning_rate (
+        ) const;
+        /*!
+            ensures
+                - During training via this->train(), this object will test if progress is
+                  still being made and if it isn't then it will reduce get_learning_rate()
+                  by setting it to get_learning_rate()*get_learning_rate_shrink_factor().
+                  However, it will not reduce it below get_min_learning_rate().  Once this
+                  minimum learning rate is crossed the training will terminate.
+                - get_min_learning_rate() doesn't apply if you are using train_one_step().  
+                  You can keep calling train_one_step() as many times as you want and the
+                  learning rate will drop infinitely close to 0 if you run long enough.
+        !*/
+
+        template <typename EXP>
+        void set_learning_rate_schedule (
+            const matrix_exp<EXP>& schedule
+        );
+        /*!
+            requires
+                - schedule.size() > 0
+                - min(schedule) > 0
+            ensures
+                - #get_learning_rate_schedule() == reshape_to_column_vector(schedule)
+                - #get_learning_rate() == schedule(0,0)
+                - #get_min_learning_rate() == min(schedule)
+                - #set_learning_rate_shrink_factor() == 1
+        !*/
+
+        const matrix<double,0,1>& get_learning_rate_schedule (
+        ) const;
+        /*!
+            ensures
+                - if (this function returns a non-empty matrix) then
+                    - This trainer will use an explicit learning rate schedule defined by
+                      the learning rate values in get_learning_rate_schedule().  For
+                      example, if get_learning_rate_schedule() returned {0.1, 0.09, 0.08,
+                      0.07, 0.06} then the first training mini-batch would use a learning
+                      rate of 0.1, then the next training mini-batch uses 0.09, and then
+                      0.8, and so on until the end of the schedule is reached.  
+                      
+                      If you continue to run training after the end of the schedule has
+                      been reached then the learning rate will be fixed to 0.99 times the
+                      final value.  So in our example, eventually the learning rate would
+                      be fixed to 0.99*0.06.  This allows you to test if we have reached the
+                      end of the schedule by checking if get_learning_rate() >= 0.06.
+        !*/
+
+        unsigned long get_steps_without_progress (
+        ) const;
+        /*!
+            ensures
+                - if (get_learning_rate_shrink_factor() != 1) then
+                    - returns an estimate of how many mini-batches have executed without us
+                      observing a statistically significant decrease in the training error.
+                - else
+                    - returns 0
+        !*/
+
+        void set_iterations_without_progress_threshold (
+            unsigned long thresh 
+        );
+        /*!
+            ensures
+                - #get_iterations_without_progress_threshold() == thresh
+                - #get_learning_rate_schedule().size() == 0
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        unsigned long get_iterations_without_progress_threshold (
+        ) const;
+        /*!
+            ensures
+                - This object monitors the progress of training and estimates if the
+                  training error is being reduced.  It does this by looking at the previous
+                  get_iterations_without_progress_threshold() mini-batch results and
+                  applying the statistical test defined by the running_gradient object to
+                  see if the training error is getting smaller.  If it isn't being reduced
+                  then get_learning_rate() is made smaller by a factor of get_learning_rate_shrink_factor().
+
+                  Therefore, get_iterations_without_progress_threshold() should always be
+                  set to something sensibly large so that this test can be done with
+                  reasonably high confidence.  Think of this test as saying "if the loss
+                  hasn't decreased for the previous get_iterations_without_progress_threshold() 
+                  then shrink the learning rate".
+        !*/
+
+        void set_learning_rate_shrink_factor (
+            double shrink
+        );
+        /*!
+            requires
+                - 0 < shrink && shrink <= 1
+            ensures
+                - #get_learning_rate_shrink_factor() == shrink
+                - #get_learning_rate_schedule().size() == 0
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        double get_learning_rate_shrink_factor (
+        ) const;
+        /*!
+            ensures
+                - Whenever the training routine thinks it isn't making progress anymore it
+                  will reduce get_learning_rate() by multiplying it by get_learning_rate_shrink_factor().
+                - You can disable the automatic learning rate reduction by setting
+                  get_learning_rate_shrink_factor() to 1.
+        !*/
+
+        unsigned long long get_train_one_step_calls (
+        ) const;
+        /*!
+            ensures
+                - returns the number of times train_one_step() has been called.
+        !*/
+
+        unsigned long long get_test_one_step_calls (
+        ) const;
+        /*!
+            ensures
+                - returns the number of times test_one_step() has been called.
+        !*/
+
+        void be_verbose (
+        );
+        /*!
+            ensures
+                - This object will print status messages to standard out so that a 
+                  user can observe the progress of the algorithm.
+        !*/
+
+        void be_quiet (
+        );
+        /*!
+            ensures
+                - This object will not print anything to standard out
+        !*/
+
+        void set_synchronization_file (
+            const std::string& filename,
+            std::chrono::seconds time_between_syncs = std::chrono::minutes(15)
+        );
+        /*!
+            ensures
+                - #get_synchronization_file() == filename
+                - While training is running, either via train() or repeated calls to
+                  train_one_step(), this object will save its entire state, including the
+                  state of get_net(), to disk in the file named filename every
+                  time_between_syncs seconds.
+                - If the filename file already exists then the state of this trainer will
+                  be loaded from that file by this call to set_synchronization_file().
+                  This allows you to resume a training session which was previously
+                  interrupted.
+                - It should be noted that when saving, the trainer will alternate between
+                  saving to a file called filename and another file called filename+"_".
+                  We do this because it's possible that your computer might crash (not
+                  because of dlib, just in general) before the data is safely saved to
+                  disk.  This way, you will always have a backup file if the write to disk
+                  gets corrupted or is incomplete.  Moreover, when loading, we will always
+                  load from the newest of the two possible files.
+        !*/
+
+        const std::string& get_synchronization_file (
+        );
+        /*!
+            ensures
+                - Returns the name of the file the dnn_trainer will periodically save it's
+                  state to.  If the return value is "" then synchronization is disabled.
+        !*/
+
+        void train (
+            const std::vector<input_type>& data,
+            const std::vector<training_label_type>& labels 
+        ); 
+        /*!
+            requires
+                - data.size() == labels.size()
+                - data.size() > 0
+                - net_type uses a supervised loss.  
+                  i.e. net_type::training_label_type != no_label_type.
+            ensures
+                - Trains a supervised neural network based on the given training data.
+                  The goal of training is to find the network parameters that minimize
+                  get_net().compute_loss(data.begin(), data.end(), labels.begin()). 
+                - The optimizer will run until get_learning_rate() < get_min_learning_rate() 
+                  or get_max_num_epochs() training epochs have been executed. 
+                - Each layer in the network will be optimized by its corresponding solver
+                  in get_solvers().  
+                - Each call to train DOES NOT reinitialize the state of get_net() or
+                  get_solvers().  That is, the existing state of the solvers and network is
+                  the starting point for the optimization each time train() is called.  In
+                  particular, if you use the set_synchronization_file() method you can
+                  resume an interrupted train() call by simply calling train() again and it
+                  will pick up from the last synchronization point.  
+                - You can obtain the average loss value during the final training epoch by
+                  calling get_average_loss().
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        void train (
+            const std::vector<input_type>& data
+        );
+        /*!
+            requires 
+                - data.size() > 0
+                - net_type uses an unsupervised loss.  
+                  i.e. net_type::training_label_type == no_label_type.
+            ensures
+                - Trains an unsupervised neural network based on the given training data.
+                  The goal of training is to find the network parameters that minimize
+                  get_net().compute_loss(data.begin(), data.end()). 
+                - The optimizer will run until get_learning_rate() < get_min_learning_rate() 
+                  or get_max_num_epochs() training epochs have been executed. 
+                - Each layer in the network will be optimized by its corresponding solver
+                  in get_solvers().  
+                - Each call to train DOES NOT reinitialize the state of get_net() or
+                  get_solvers().  That is, the existing state of the solvers and network is
+                  the starting point for the optimization each time train() is called.  In
+                  particular, if you use the set_synchronization_file() method you can
+                  resume an interrupted train() call by simply calling train() again and it
+                  will pick up from the last synchronization point.  
+                - You can obtain the average loss value during the final training epoch by
+                  calling get_average_loss().
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        void train_one_step (
+            const std::vector<input_type>& data,
+            const std::vector<training_label_type>& labels 
+        );
+        /*!
+            requires
+                - data.size() == labels.size()
+                - data.size() > 0
+                - net_type uses a supervised loss.  
+                  i.e. net_type::training_label_type != no_label_type.
+            ensures
+                - Performs one stochastic gradient update step based on the mini-batch of
+                  data and labels supplied to this function.  In particular, calling
+                  train_one_step() in a loop is equivalent to calling the train() method
+                  defined above.  However, train_one_step() allows you to stream data from
+                  disk into the training process while train() requires you to first load
+                  all the training data into RAM.  Otherwise, these training methods are
+                  equivalent.
+                - You can observe the current average loss value by calling get_average_loss().
+                - The network training will happen in another thread.  Therefore, after
+                  calling this function you should call get_net() before you touch the net
+                  object from the calling thread to ensure no other threads are still
+                  accessing the network.
+                - #get_train_one_step_calls() == get_train_one_step_calls() + 1.
+        !*/
+
+        template <
+            typename data_iterator,
+            typename label_iterator
+            >
+        void train_one_step (
+            data_iterator dbegin,
+            data_iterator dend,
+            label_iterator lbegin
+        );
+        /*!
+            requires
+                - std::advance(lbegin, std::distance(dbegin, dend) - 1) is dereferencable
+                - std::distance(dbegin, dend) > 0
+                - net_type uses a supervised loss.  
+                  i.e. net_type::training_label_type != no_label_type.
+            ensures
+                - Performs one stochastic gradient update step based on the mini-batch of
+                  data and labels supplied to this function.  In particular, calling
+                  train_one_step() in a loop is equivalent to calling the train() method
+                  defined above.  However, train_one_step() allows you to stream data from
+                  disk into the training process while train() requires you to first load
+                  all the training data into RAM.  Otherwise, these training methods are
+                  equivalent.
+                - You can observe the current average loss value by calling get_average_loss().
+                - The network training will happen in another thread.  Therefore, after
+                  calling this function you should call get_net() before you touch the net
+                  object from the calling thread to ensure no other threads are still
+                  accessing the network.
+                - #get_train_one_step_calls() == get_train_one_step_calls() + 1.
+        !*/
+
+        void train_one_step (
+            const std::vector<input_type>& data
+        );
+        /*!
+            requires
+                - data.size() > 0
+                - net_type uses an unsupervised loss.  
+                  i.e. net_type::training_label_type == no_label_type.
+            ensures
+                - Performs one stochastic gradient update step based on the mini-batch of
+                  data supplied to this function.  In particular, calling train_one_step()
+                  in a loop is equivalent to calling the train() method defined above.
+                  However, train_one_step() allows you to stream data from disk into the
+                  training process while train() requires you to first load all the
+                  training data into RAM.  Otherwise, these training methods are
+                  equivalent.
+                - You can observe the current average loss value by calling get_average_loss().
+                - The network training will happen in another thread.  Therefore, after
+                  calling this function you should call get_net() before you touch the net
+                  object from the calling thread to ensure no other threads are still
+                  accessing the network.
+                - #get_train_one_step_calls() == get_train_one_step_calls() + 1.
+        !*/
+
+        template <
+            typename data_iterator
+            >
+        void train_one_step (
+            data_iterator dbegin,
+            data_iterator dend
+        );
+        /*!
+            requires
+                - std::distance(dbegin, dend) > 0
+                - net_type uses an unsupervised loss.  
+                  i.e. net_type::training_label_type == no_label_type.
+            ensures
+                - Performs one stochastic gradient update step based on the mini-batch of
+                  data supplied to this function.  In particular, calling train_one_step()
+                  in a loop is equivalent to calling the train() method defined above.
+                  However, train_one_step() allows you to stream data from disk into the
+                  training process while train() requires you to first load all the
+                  training data into RAM.  Otherwise, these training methods are
+                  equivalent.
+                - You can observe the current average loss value by calling get_average_loss().
+                - The network training will happen in another thread.  Therefore, after
+                  calling this function you should call get_net() before you touch the net
+                  object from the calling thread to ensure no other threads are still
+                  accessing the network.
+                - #get_train_one_step_calls() == get_train_one_step_calls() + 1.
+        !*/
+        
+        double get_average_loss (
+        ) const;
+        /*!
+            ensures
+                - returns the average loss value observed during previous calls to
+                  train_one_step() or train().  That is, the average output of
+                  net_type::update() during the previous mini-batch updates.
+                - Note that, if be_verbose() has been called, then this object will
+                  automatically call clear_average_loss() periodically when it logs the
+                  loss to the console.
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        void clear_average_loss (
+        );
+        /*!
+            ensures
+                - #get_average_loss() == 0
+                - get_average_loss() uses a dlib::running_stats object to keep a running
+                  average of the loss values seen during the previous mini-batch updates
+                  applied during training.  Calling clear_average_loss() resets the
+                  running_stats object so it forgets about all previous loss values
+                  observed.
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+    // ----------------------
+
+        double get_average_test_loss (
+        ) const;
+        /*!
+            ensures
+                - returns the average loss value observed during previous calls to
+                  test_one_step(). 
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        void test_one_step (
+            const std::vector<input_type>& data,
+            const std::vector<training_label_type>& labels 
+        );
+        /*!
+            requires
+                - data.size() == labels.size()
+                - data.size() > 0
+                - net_type uses a supervised loss.  
+                  i.e. net_type::training_label_type != no_label_type.
+            ensures
+                - Runs the given data through the network and computes and records the loss.  
+                - This call does not modify network parameters.  The point of
+                  test_one_step() is two fold, to allow you to observe the accuracy of the
+                  network on hold out data during training, and to allow the trainer to
+                  automatically adjust the learning rate when the test loss stops
+                  improving.  It should be noted that you are not required to use
+                  test_one_step() at all, but if you want to do this kind of thing it is
+                  available.
+                - You can observe the current average loss value by calling get_average_test_loss().
+                - The computation will happen in another thread.  Therefore, after calling
+                  this function you should call get_net() before you touch the net object
+                  from the calling thread to ensure no other threads are still accessing
+                  the network.
+                - #get_test_one_step_calls() == get_test_one_step_calls() + 1.
+        !*/
+
+        template <
+            typename data_iterator,
+            typename label_iterator
+            >
+        void test_one_step (
+            data_iterator dbegin, 
+            data_iterator dend,
+            label_iterator lbegin
+        );
+        /*!
+            requires
+                - std::advance(lbegin, std::distance(dbegin, dend) - 1) is dereferencable
+                - std::distance(dbegin, dend) > 0
+                - net_type uses a supervised loss.  
+                  i.e. net_type::training_label_type != no_label_type.
+            ensures
+                - Runs the given data through the network and computes and records the loss.  
+                - This call does not modify network parameters.  The point of
+                  test_one_step() is two fold, to allow you to observe the accuracy of the
+                  network on hold out data during training, and to allow the trainer to
+                  automatically adjust the learning rate when the test loss stops
+                  improving.  It should be noted that you are not required to use
+                  test_one_step() at all, but if you want to do this kind of thing it is
+                  available.
+                - You can observe the current average loss value by calling get_average_test_loss().
+                - The computation will happen in another thread.  Therefore, after calling
+                  this function you should call get_net() before you touch the net object
+                  from the calling thread to ensure no other threads are still accessing
+                  the network.
+                - #get_test_one_step_calls() == get_test_one_step_calls() + 1.
+        !*/
+
+        void test_one_step (
+            const std::vector<input_type>& data
+        );
+        /*!
+            requires
+                - data.size() > 0
+                - net_type uses an unsupervised loss.  
+                  i.e. net_type::training_label_type == no_label_type.
+            ensures
+                - Runs the given data through the network and computes and records the loss.  
+                - This call does not modify network parameters.  The point of
+                  test_one_step() is two fold, to allow you to observe the accuracy of the
+                  network on hold out data during training, and to allow the trainer to
+                  automatically adjust the learning rate when the test loss stops
+                  improving.  It should be noted that you are not required to use
+                  test_one_step() at all, but if you want to do this kind of thing it is
+                  available.
+                - You can observe the current average loss value by calling get_average_test_loss().
+                - The computation will happen in another thread.  Therefore, after calling
+                  this function you should call get_net() before you touch the net object
+                  from the calling thread to ensure no other threads are still accessing
+                  the network.
+                - #get_test_one_step_calls() == get_test_one_step_calls() + 1.
+        !*/
+
+        template <
+            typename data_iterator
+            >
+        void test_one_step (
+            data_iterator dbegin, 
+            data_iterator dend
+        );
+        /*!
+            requires
+                - std::distance(dbegin, dend) > 0
+                - net_type uses an unsupervised loss.  
+                  i.e. net_type::training_label_type == no_label_type.
+            ensures
+                - Runs the given data through the network and computes and records the loss.  
+                - This call does not modify network parameters.  The point of
+                  test_one_step() is two fold, to allow you to observe the accuracy of the
+                  network on hold out data during training, and to allow the trainer to
+                  automatically adjust the learning rate when the test loss stops
+                  improving.  It should be noted that you are not required to use
+                  test_one_step() at all, but if you want to do this kind of thing it is
+                  available.
+                - You can observe the current average loss value by calling get_average_test_loss().
+                - The computation will happen in another thread.  Therefore, after calling
+                  this function you should call get_net() before you touch the net object
+                  from the calling thread to ensure no other threads are still accessing
+                  the network.
+                - #get_test_one_step_calls() == get_test_one_step_calls() + 1.
+        !*/
+
+        void set_test_iterations_without_progress_threshold (
+            unsigned long thresh 
+        );
+        /*!
+            ensures
+                - #get_test_iterations_without_progress_threshold() == thresh
+                - #get_learning_rate_schedule().size() == 0
+                - This function blocks until all threads inside the dnn_trainer have
+                  stopped touching the net. 
+        !*/
+
+        unsigned long get_test_iterations_without_progress_threshold (
+        ) const;
+        /*!
+            ensures
+                - This object monitors the progress of training and estimates if the
+                  testing error is being reduced.  It does this by looking at the previous
+                  get_test_iterations_without_progress_threshold() mini-batch results from
+                  test_one_step() and applying the statistical test defined by the
+                  running_gradient object to see if the testing error is getting smaller.
+                  If it isn't being reduced then get_learning_rate() is made smaller by a
+                  factor of get_learning_rate_shrink_factor().
+
+                  Therefore, get_test_iterations_without_progress_threshold() should always be
+                  set to something sensibly large so that this test can be done with
+                  reasonably high confidence.  Think of this test as saying "if the testing loss
+                  hasn't decreased for the previous get_test_iterations_without_progress_threshold() 
+                  calls to test_one_step() then shrink the learning rate".
+        !*/
+
+        unsigned long get_test_steps_without_progress (
+        ) const;
+        /*!
+            ensures
+                - if (get_learning_rate_shrink_factor() != 1) then
+                    - returns an estimate of how many mini-batches have executed without us
+                      observing a statistically significant decrease in the testing error
+                      (i.e. the error on the data given to the trainer via test_one_step()
+                      calls).
+                - else
+                    - returns 0
+        !*/
+
+    };
+
+// ----------------------------------------------------------------------------------------
+
+    template <
+        typename net_type, 
+        typename solver_type 
+        >
+    std::ostream& operator<< (
+        std::ostream& out,
+        dnn_trainer<net_type,solver_type>& trainer
+    );
+    /*!
+        ensures
+            - Prints a log of the current parameters of trainer to out.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_TRAINER_ABSTRACT_H_ 
+
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/utilities.h

@@ -0,0 +1,281 @@
+// Copyright (C) 2016  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNn_UTILITIES_H_
+#define DLIB_DNn_UTILITIES_H_
+
+#include "core.h"
+#include "utilities_abstract.h"
+#include "../geometry.h"
+#include <fstream>
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    inline double log1pexp(double x)
+    {
+        using std::exp;
+        using namespace std; // Do this instead of using std::log1p because some compilers
+                             // error out otherwise (E.g. gcc 4.9 in cygwin)
+        if (x <= -37)
+            return exp(x);
+        else if (-37 < x && x <= 18)
+            return log1p(exp(x));
+        else if (18 < x && x <= 33.3)
+            return x + exp(-x);
+        else
+            return x;
+    }
+    
+// ----------------------------------------------------------------------------------------
+
+    inline void randomize_parameters (
+        tensor& params,
+        unsigned long num_inputs_and_outputs,
+        dlib::rand& rnd
+    )
+    {
+        for (auto& val : params)
+        {
+            // Draw a random number to initialize the layer according to formula (16)
+            // from Understanding the difficulty of training deep feedforward neural
+            // networks by Xavier Glorot and Yoshua Bengio.
+            val = 2*rnd.get_random_float()-1;
+            val *= std::sqrt(6.0/(num_inputs_and_outputs));
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    namespace impl
+    {
+        class visitor_net_to_xml
+        {
+        public:
+
+            visitor_net_to_xml(std::ostream& out_) : out(out_) {}
+
+            template<typename input_layer_type>
+            void operator()(size_t idx, const input_layer_type& l) 
+            {
+                out << "<layer idx='"<<idx<<"' type='input'>\n";
+                to_xml(l,out);
+                out << "</layer>\n";
+            }
+
+            template <typename T, typename U>
+            void operator()(size_t idx, const add_loss_layer<T,U>& l) 
+            {
+                out << "<layer idx='"<<idx<<"' type='loss'>\n";
+                to_xml(l.loss_details(),out);
+                out << "</layer>\n";
+            }
+
+            template <typename T, typename U, typename E>
+            void operator()(size_t idx, const add_layer<T,U,E>& l) 
+            {
+                out << "<layer idx='"<<idx<<"' type='comp'>\n";
+                to_xml(l.layer_details(),out);
+                out << "</layer>\n";
+            }
+
+            template <unsigned long ID, typename U, typename E>
+            void operator()(size_t idx, const add_tag_layer<ID,U,E>& l) 
+            {
+                out << "<layer idx='"<<idx<<"' type='tag' id='"<<ID<<"'/>\n";
+            }
+
+            template <template<typename> class T, typename U>
+            void operator()(size_t idx, const add_skip_layer<T,U>& l) 
+            {
+                out << "<layer idx='"<<idx<<"' type='skip' id='"<<(tag_id<T>::id)<<"'/>\n";
+            }
+
+        private:
+
+            std::ostream& out;
+        };
+    }
+
+    template <typename net_type>
+    void net_to_xml (
+        const net_type& net,
+        std::ostream& out
+    )
+    {
+        auto old_precision = out.precision(9);
+        out << "<net>\n";
+        visit_layers(net, impl::visitor_net_to_xml(out));
+        out << "</net>\n";
+        // restore the original stream precision.
+        out.precision(old_precision);
+    }
+
+    template <typename net_type>
+    void net_to_xml (
+        const net_type& net,
+        const std::string& filename
+    )
+    {
+        std::ofstream fout(filename);
+        net_to_xml(net, fout);
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    namespace impl
+    {
+
+        class visitor_net_map_input_to_output
+        {
+        public:
+
+            visitor_net_map_input_to_output(dpoint& p_) : p(p_) {}
+
+            dpoint& p;
+
+            template<typename input_layer_type>
+            void operator()(const input_layer_type& net) 
+            {
+            }
+
+            template <typename T, typename U>
+            void operator()(const add_loss_layer<T,U>& net) 
+            {
+                (*this)(net.subnet());
+            }
+
+            template <typename T, typename U, typename E>
+            void operator()(const add_layer<T,U,E>& net) 
+            {
+                (*this)(net.subnet());
+                p = net.layer_details().map_input_to_output(p);
+            }
+            template <bool B, typename T, typename U, typename E>
+            void operator()(const dimpl::subnet_wrapper<add_layer<T,U,E>,B>& net) 
+            {
+                (*this)(net.subnet());
+                p = net.layer_details().map_input_to_output(p);
+            }
+
+
+            template <unsigned long ID, typename U, typename E>
+            void operator()(const add_tag_layer<ID,U,E>& net) 
+            {
+                // tag layers are an identity transform, so do nothing
+                (*this)(net.subnet());
+            }
+            template <bool is_first, unsigned long ID, typename U, typename E>
+            void operator()(const dimpl::subnet_wrapper<add_tag_layer<ID,U,E>,is_first>& net) 
+            {
+                // tag layers are an identity transform, so do nothing
+                (*this)(net.subnet());
+            }
+
+
+            template <template<typename> class TAG_TYPE, typename U>
+            void operator()(const add_skip_layer<TAG_TYPE,U>& net) 
+            {
+                (*this)(layer<TAG_TYPE>(net));
+            }
+            template <bool is_first, template<typename> class TAG_TYPE, typename SUBNET>
+            void operator()(const dimpl::subnet_wrapper<add_skip_layer<TAG_TYPE,SUBNET>,is_first>& net) 
+            {
+                // skip layers are an identity transform, so do nothing
+                (*this)(layer<TAG_TYPE>(net));
+            }
+
+        };
+
+        class visitor_net_map_output_to_input
+        {
+        public:
+            visitor_net_map_output_to_input(dpoint& p_) : p(p_) {}
+
+            dpoint& p;
+
+            template<typename input_layer_type>
+            void operator()(const input_layer_type& net) 
+            {
+            }
+
+            template <typename T, typename U>
+            void operator()(const add_loss_layer<T,U>& net) 
+            {
+                (*this)(net.subnet());
+            }
+
+            template <typename T, typename U, typename E>
+            void operator()(const add_layer<T,U,E>& net) 
+            {
+                p = net.layer_details().map_output_to_input(p);
+                (*this)(net.subnet());
+            }
+            template <bool B, typename T, typename U, typename E>
+            void operator()(const dimpl::subnet_wrapper<add_layer<T,U,E>,B>& net) 
+            {
+                p = net.layer_details().map_output_to_input(p);
+                (*this)(net.subnet());
+            }
+
+
+            template <unsigned long ID, typename U, typename E>
+            void operator()(const add_tag_layer<ID,U,E>& net) 
+            {
+                // tag layers are an identity transform, so do nothing
+                (*this)(net.subnet());
+            }
+            template <bool is_first, unsigned long ID, typename U, typename E>
+            void operator()(const dimpl::subnet_wrapper<add_tag_layer<ID,U,E>,is_first>& net) 
+            {
+                // tag layers are an identity transform, so do nothing
+                (*this)(net.subnet());
+            }
+
+
+            template <template<typename> class TAG_TYPE, typename U>
+            void operator()(const add_skip_layer<TAG_TYPE,U>& net) 
+            {
+                (*this)(layer<TAG_TYPE>(net));
+            }
+            template <bool is_first, template<typename> class TAG_TYPE, typename SUBNET>
+            void operator()(const dimpl::subnet_wrapper<add_skip_layer<TAG_TYPE,SUBNET>,is_first>& net) 
+            {
+                // skip layers are an identity transform, so do nothing
+                (*this)(layer<TAG_TYPE>(net));
+            }
+
+        };
+    }
+
+    template <typename net_type>
+    inline dpoint input_tensor_to_output_tensor(
+        const net_type& net,
+        dpoint p 
+    )
+    {
+        impl::visitor_net_map_input_to_output temp(p);
+        temp(net);
+        return p;
+    }
+
+    template <typename net_type>
+    inline dpoint output_tensor_to_input_tensor(
+        const net_type& net,
+        dpoint p  
+    )
+    {
+        impl::visitor_net_map_output_to_input temp(p);
+        temp(net);
+        return p;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_UTILITIES_H_ 
+
+
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/utilities_abstract.h

@@ -0,0 +1,127 @@
+// Copyright (C) 2016  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#undef DLIB_DNn_UTILITIES_ABSTRACT_H_
+#ifdef DLIB_DNn_UTILITIES_ABSTRACT_H_
+
+#include "core_abstract.h"
+#include "../geometry/vector_abstract.h"
+
+namespace dlib
+{
+
+// ----------------------------------------------------------------------------------------
+
+    double log1pexp(
+        double x
+    );
+    /*!
+        ensures
+            - returns log(1+exp(x))
+              (except computes it using a numerically accurate method)
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    void randomize_parameters (
+        tensor& params,
+        unsigned long num_inputs_and_outputs,
+        dlib::rand& rnd
+    );
+    /*!
+        ensures
+            - This function assigns random values into params based on the given random
+              number generator.  In particular, it uses the parameter initialization method
+              of formula 16 from the paper "Understanding the difficulty of training deep
+              feedforward neural networks" by Xavier Glorot and Yoshua Bengio.
+            - It is assumed that the total number of inputs and outputs from the layer is
+              num_inputs_and_outputs.  That is, you should set num_inputs_and_outputs to
+              the sum of the dimensionalities of the vectors going into and out of the
+              layer that uses params as its parameters.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename net_type>
+    void net_to_xml (
+        const net_type& net,
+        std::ostream& out
+    );
+    /*!
+        requires
+            - net_type is an object of type add_layer, add_loss_layer, add_skip_layer, or
+              add_tag_layer.
+            - All layers in the net must provide to_xml() functions.
+        ensures
+            - Prints the given neural network object as an XML document to the given output
+              stream.
+    !*/
+
+    template <typename net_type>
+    void net_to_xml (
+        const net_type& net,
+        const std::string& filename
+    );
+    /*!
+        requires
+            - net_type is an object of type add_layer, add_loss_layer, add_skip_layer, or
+              add_tag_layer.
+            - All layers in the net must provide to_xml() functions.
+        ensures
+            - This function is just like the above net_to_xml(), except it writes to a file
+              rather than an ostream.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename net_type>
+    dpoint input_tensor_to_output_tensor(
+        const net_type& net,
+        dpoint p 
+    );
+    /*!
+        requires
+            - net_type is an object of type add_layer, add_skip_layer, or add_tag_layer.
+            - All layers in the net must provide map_input_to_output() functions.
+        ensures
+            - Given a dpoint (i.e. a row,column coordinate) in the input tensor given to
+              net, this function returns the corresponding dpoint in the output tensor
+              net.get_output().  This kind of mapping is useful when working with fully
+              convolutional networks as you will often want to know what parts of the
+              output feature maps correspond to what parts of the input.
+            - If the network contains skip layers then any layers skipped over by the skip
+              layer are ignored for the purpose of computing this coordinate mapping.  That
+              is, if you walk the network from the output layer to the input layer, where
+              each time you encounter a skip layer you jump to the layer indicated by the
+              skip layer, you will visit exactly the layers in the network involved in the
+              input_tensor_to_output_tensor() calculation. This behavior is useful since it
+              allows you to compute some auxiliary DNN as a separate branch of computation
+              that is separate from the main network's job of running some kind of fully
+              convolutional network over an image.  For instance, you might want to have a
+              branch in your network that computes some global image level
+              summarization/feature.
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+    template <typename net_type>
+    dpoint output_tensor_to_input_tensor(
+        const net_type& net,
+        dpoint p  
+    );
+    /*!
+        requires
+            - net_type is an object of type add_layer, add_skip_layer, or add_tag_layer.
+            - All layers in the net must provide map_output_to_input() functions.
+        ensures
+            - This function provides the reverse mapping of input_tensor_to_output_tensor().
+              That is, given a dpoint in net.get_output(), what is the corresponding dpoint
+              in the input tensor?
+    !*/
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_UTILITIES_ABSTRACT_H_ 
+
+

pkg/OpenFace/lib/3rdParty/dlib/include/dlib/dnn/validation.h

@@ -0,0 +1,122 @@
+// Copyright (C) 2016  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNn_VALIDATION_H_
+#define DLIB_DNn_VALIDATION_H_
+
+#include "../svm/cross_validate_object_detection_trainer_abstract.h"
+#include "../svm/cross_validate_object_detection_trainer.h"
+#include "layers.h"
+#include <set>
+
+namespace dlib
+{
+    namespace impl
+    {
+        inline std::set<std::string> get_labels (
+            const std::vector<mmod_rect>& rects1,
+            const std::vector<mmod_rect>& rects2
+        )
+        {
+            std::set<std::string> labels;
+            for (auto& rr : rects1)
+                labels.insert(rr.label);
+            for (auto& rr : rects2)
+                labels.insert(rr.label);
+            return labels;
+        }
+    }
+
+    template <
+        typename SUBNET,
+        typename image_array_type
+        >
+    const matrix<double,1,3> test_object_detection_function (
+        loss_mmod<SUBNET>& detector,
+        const image_array_type& images,
+        const std::vector<std::vector<mmod_rect>>& truth_dets,
+        const test_box_overlap& overlap_tester = test_box_overlap(),
+        const double adjust_threshold = 0,
+        const test_box_overlap& overlaps_ignore_tester = test_box_overlap()
+    )
+    {
+        // make sure requires clause is not broken
+        DLIB_CASSERT( is_learning_problem(images,truth_dets) == true , 
+                    "\t matrix test_object_detection_function()"
+                    << "\n\t invalid inputs were given to this function"
+                    << "\n\t is_learning_problem(images,truth_dets): " << is_learning_problem(images,truth_dets)
+                    << "\n\t images.size(): " << images.size() 
+                    );
+
+
+
+        double correct_hits = 0;
+        double total_true_targets = 0;
+
+        std::vector<std::pair<double,bool> > all_dets;
+        unsigned long missing_detections = 0;
+
+        resizable_tensor temp;
+
+        for (unsigned long i = 0; i < images.size(); ++i)
+        {
+            std::vector<mmod_rect> hits; 
+            detector.to_tensor(&images[i], &images[i]+1, temp);
+            detector.subnet().forward(temp);
+            detector.loss_details().to_label(temp, detector.subnet(), &hits, adjust_threshold);
+
+
+            for (auto& label : impl::get_labels(truth_dets[i], hits))
+            {
+                std::vector<full_object_detection> truth_boxes;
+                std::vector<rectangle> ignore;
+                std::vector<std::pair<double,rectangle>> boxes;
+                // copy hits and truth_dets into the above three objects
+                for (auto&& b : truth_dets[i])
+                {
+                    if (b.ignore)
+                    {
+                        ignore.push_back(b);
+                    }
+                    else if (b.label == label)
+                    {
+                        truth_boxes.push_back(full_object_detection(b.rect));
+                        ++total_true_targets;
+                    }
+                }
+                for (auto&& b : hits)
+                {
+                    if (b.label == label)
+                        boxes.push_back(std::make_pair(b.detection_confidence, b.rect));
+                }
+
+                correct_hits += impl::number_of_truth_hits(truth_boxes, ignore, boxes, overlap_tester, all_dets, missing_detections, overlaps_ignore_tester);
+            }
+        }
+
+        std::sort(all_dets.rbegin(), all_dets.rend());
+
+        double precision, recall;
+
+        double total_hits = all_dets.size();
+
+        if (total_hits == 0)
+            precision = 1;
+        else
+            precision = correct_hits / total_hits;
+
+        if (total_true_targets == 0)
+            recall = 1;
+        else
+            recall = correct_hits / total_true_targets;
+
+        matrix<double, 1, 3> res;
+        res = precision, recall, average_precision(all_dets, missing_detections);
+        return res;
+    }
+
+// ----------------------------------------------------------------------------------------
+
+}
+
+#endif // DLIB_DNn_VALIDATION_H_
+