include/facial_landmark_detector.h

   1 // -*- mode: c++ -*-
   2
   3 #ifndef __FACIAL_LANDMARK_DETECTOR_H__
   4 #define __FACIAL_LANDMARK_DETECTOR_H__
   5
   6 /****
   7 Copyright (c) 2020 Adrian I. Lam
   8
   9 Permission is hereby granted, free of charge, to any person obtaining a copy
  10 of this software and associated documentation files (the "Software"), to deal
  11 in the Software without restriction, including without limitation the rights
  12 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  13 copies of the Software, and to permit persons to whom the Software is
  14 furnished to do so, subject to the following conditions:
  15
  16 The above copyright notice and this permission notice shall be included in all
  17 copies or substantial portions of the Software.
  18
  19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  24 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  25 SOFTWARE.
  26 ****/
  27
  28 #include <deque>
  29 #include <string>
  30 #include <opencv2/opencv.hpp>
  31 #include <dlib/image_processing/frontal_face_detector.h>
  32 #include <dlib/image_processing.h>
  33 #include <dlib/gui_widgets.h>
  34
  35 class FacialLandmarkDetector
  36 {
  37 public:
  38     struct Params
  39     {
  40         double leftEyeOpenness;
  41         double rightEyeOpenness;
  42         double leftEyeSmile;
  43         double rightEyeSmile;
  44         double mouthOpenness;
  45         double mouthForm;
  46         double faceXAngle;
  47         double faceYAngle;
  48         double faceZAngle;
  49         bool autoBlink;
  50         bool autoBreath;
  51         bool randomMotion;
  52         // TODO eyebrows currently not supported...
  53         // I'd like to include them, but the dlib detection is very
  54         // noisy and inaccurate (at least for my face).
  55     };
  56
  57     FacialLandmarkDetector(std::string cfgPath);
  58
  59     Params getParams(void) const;
  60
  61     void stop(void);
  62
  63     void mainLoop(void);
  64
  65 private:
  66     enum LeftRight : bool
  67     {
  68         LEFT,
  69         RIGHT
  70     };
  71
  72     cv::VideoCapture webcam;
  73     dlib::image_window win;
  74     dlib::frontal_face_detector detector;
  75     dlib::shape_predictor predictor;
  76     bool m_stop;
  77
  78     double calcEyeAspectRatio(dlib::point& p1, dlib::point& p2,
  79                               dlib::point& p3, dlib::point& p4,
  80                               dlib::point& p5, dlib::point& p6) const;
  81
  82     double calcRightEyeAspectRatio(dlib::full_object_detection& shape) const;
  83     double calcLeftEyeAspectRatio(dlib::full_object_detection& shape) const;
  84
  85     double calcEyeOpenness(LeftRight eye,
  86                            dlib::full_object_detection& shape,
  87                            double faceYAngle) const;
  88
  89     double calcMouthForm(dlib::full_object_detection& shape) const;
  90     double calcMouthOpenness(dlib::full_object_detection& shape, double mouthForm) const;
  91
  92     double calcFaceXAngle(dlib::full_object_detection& shape) const;
  93     double calcFaceYAngle(dlib::full_object_detection& shape, double faceXAngle, double mouthForm) const;
  94     double calcFaceZAngle(dlib::full_object_detection& shape) const;
  95
  96     void populateDefaultConfig(void);
  97     void parseConfig(std::string cfgPath);
  98     void throwConfigError(std::string paramName, std::string expectedType,
  99                           std::string line, unsigned int lineNum);
 100
 101
 102     std::deque<double> m_leftEyeOpenness;
 103     std::deque<double> m_rightEyeOpenness;
 104
 105     std::deque<double> m_mouthOpenness;
 106     std::deque<double> m_mouthForm;
 107
 108     std::deque<double> m_faceXAngle;
 109     std::deque<double> m_faceYAngle;
 110     std::deque<double> m_faceZAngle;
 111
 112     struct Config
 113     {
 114         int cvVideoCaptureId;
 115         std::string predictorPath;
 116         double faceYAngleCorrection;
 117         double eyeSmileEyeOpenThreshold;
 118         double eyeSmileMouthFormThreshold;
 119         double eyeSmileMouthOpenThreshold;
 120         bool showWebcamVideo;
 121         bool renderLandmarksOnVideo;
 122         bool lateralInversion;
 123         std::size_t faceXAngleNumTaps;
 124         std::size_t faceYAngleNumTaps;
 125         std::size_t faceZAngleNumTaps;
 126         std::size_t mouthFormNumTaps;
 127         std::size_t mouthOpenNumTaps;
 128         std::size_t leftEyeOpenNumTaps;
 129         std::size_t rightEyeOpenNumTaps;
 130         int cvWaitKeyMs;
 131         double eyeClosedThreshold;
 132         double eyeOpenThreshold;
 133         double mouthNormalThreshold;
 134         double mouthSmileThreshold;
 135         double mouthClosedThreshold;
 136         double mouthOpenThreshold;
 137         double mouthOpenLaughCorrection;
 138         double faceYAngleXRotCorrection;
 139         double faceYAngleSmileCorrection;
 140         double faceYAngleZeroValue;
 141         double faceYAngleUpThreshold;
 142         double faceYAngleDownThreshold;
 143         bool autoBlink;
 144         bool autoBreath;
 145         bool randomMotion;
 146     } m_cfg;
 147 };
 148
 149 #endif
 150