include/facial_landmark_detector.h

   1 // -*- mode: c++ -*-
   2
   3 #ifndef __FACIAL_LANDMARK_DETECTOR_H__
   4 #define __FACIAL_LANDMARK_DETECTOR_H__
   5
   6 /****
   7 Copyright (c) 2020 Adrian I. Lam
   8
   9 Permission is hereby granted, free of charge, to any person obtaining a copy
  10 of this software and associated documentation files (the "Software"), to deal
  11 in the Software without restriction, including without limitation the rights
  12 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  13 copies of the Software, and to permit persons to whom the Software is
  14 furnished to do so, subject to the following conditions:
  15
  16 The above copyright notice and this permission notice shall be included in all
  17 copies or substantial portions of the Software.
  18
  19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  24 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  25 SOFTWARE.
  26 ****/
  27
  28 #include <deque>
  29 #include <string>
  30 #include <opencv2/opencv.hpp>
  31 #include <dlib/image_processing/frontal_face_detector.h>
  32 #include <dlib/image_processing.h>
  33 #include <dlib/gui_widgets.h>
  34
  35 class FacialLandmarkDetector
  36 {
  37 public:
  38     struct Params
  39     {
  40         double leftEyeOpenness;
  41         double rightEyeOpenness;
  42         double leftEyeSmile;
  43         double rightEyeSmile;
  44         double mouthOpenness;
  45         double mouthForm;
  46         double faceXAngle;
  47         double faceYAngle;
  48         double faceZAngle;
  49         // TODO eyebrows currently not supported...
  50         // I'd like to include them, but the dlib detection is very
  51         // noisy and inaccurate (at least for my face).
  52     };
  53
  54     FacialLandmarkDetector(std::string cfgPath);
  55
  56     Params getParams(void) const;
  57
  58     void stop(void);
  59
  60     void mainLoop(void);
  61
  62 private:
  63     enum LeftRight : bool
  64     {
  65         LEFT,
  66         RIGHT
  67     };
  68
  69     cv::VideoCapture webcam;
  70     dlib::image_window win;
  71     dlib::frontal_face_detector detector;
  72     dlib::shape_predictor predictor;
  73     bool m_stop;
  74
  75     double calcEyeAspectRatio(dlib::point& p1, dlib::point& p2,
  76                               dlib::point& p3, dlib::point& p4,
  77                               dlib::point& p5, dlib::point& p6) const;
  78
  79     double calcRightEyeAspectRatio(dlib::full_object_detection& shape) const;
  80     double calcLeftEyeAspectRatio(dlib::full_object_detection& shape) const;
  81
  82     double calcEyeOpenness(LeftRight eye,
  83                            dlib::full_object_detection& shape,
  84                            double faceYAngle) const;
  85
  86     double calcMouthForm(dlib::full_object_detection& shape) const;
  87     double calcMouthOpenness(dlib::full_object_detection& shape, double mouthForm) const;
  88
  89     double calcFaceXAngle(dlib::full_object_detection& shape) const;
  90     double calcFaceYAngle(dlib::full_object_detection& shape, double faceXAngle, double mouthForm) const;
  91     double calcFaceZAngle(dlib::full_object_detection& shape) const;
  92
  93     void populateDefaultConfig(void);
  94     void parseConfig(std::string cfgPath);
  95     void throwConfigError(std::string paramName, std::string expectedType,
  96                           std::string line, unsigned int lineNum);
  97
  98
  99     std::deque<double> m_leftEyeOpenness;
 100     std::deque<double> m_rightEyeOpenness;
 101
 102     std::deque<double> m_mouthOpenness;
 103     std::deque<double> m_mouthForm;
 104
 105     std::deque<double> m_faceXAngle;
 106     std::deque<double> m_faceYAngle;
 107     std::deque<double> m_faceZAngle;
 108
 109     struct Config
 110     {
 111         int cvVideoCaptureId;
 112         std::string predictorPath;
 113         double faceYAngleCorrection;
 114         double eyeSmileEyeOpenThreshold;
 115         double eyeSmileMouthFormThreshold;
 116         double eyeSmileMouthOpenThreshold;
 117         bool showWebcamVideo;
 118         bool renderLandmarksOnVideo;
 119         bool lateralInversion;
 120         std::size_t faceXAngleNumTaps;
 121         std::size_t faceYAngleNumTaps;
 122         std::size_t faceZAngleNumTaps;
 123         std::size_t mouthFormNumTaps;
 124         std::size_t mouthOpenNumTaps;
 125         std::size_t leftEyeOpenNumTaps;
 126         std::size_t rightEyeOpenNumTaps;
 127         int cvWaitKeyMs;
 128         double eyeClosedThreshold;
 129         double eyeOpenThreshold;
 130         double mouthNormalThreshold;
 131         double mouthSmileThreshold;
 132         double mouthClosedThreshold;
 133         double mouthOpenThreshold;
 134         double mouthOpenLaughCorrection;
 135         double faceYAngleXRotCorrection;
 136         double faceYAngleSmileCorrection;
 137         double faceYAngleZeroValue;
 138         double faceYAngleUpThreshold;
 139         double faceYAngleDownThreshold;
 140     } m_cfg;
 141 };
 142
 143 #endif
 144