include/facial_landmark_detector.h

   1 // -*- mode: c++ -*-
   2
   3 #ifndef FACIAL_LANDMARK_DETECTOR_H
   4 #define FACIAL_LANDMARK_DETECTOR_H
   5
   6 /****
   7 Copyright (c) 2020-2021 Adrian I. Lam
   8
   9 Permission is hereby granted, free of charge, to any person obtaining a copy
  10 of this software and associated documentation files (the "Software"), to deal
  11 in the Software without restriction, including without limitation the rights
  12 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  13 copies of the Software, and to permit persons to whom the Software is
  14 furnished to do so, subject to the following conditions:
  15
  16 The above copyright notice and this permission notice shall be included in all
  17 copies or substantial portions of the Software.
  18
  19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  24 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  25 SOFTWARE.
  26 ****/
  27
  28 #include <deque>
  29 #include <string>
  30
  31 struct Point
  32 {
  33     double x;
  34     double y;
  35
  36     Point(double _x = 0, double _y = 0)
  37     {
  38         x = _x;
  39         y = _y;
  40     }
  41 };
  42
  43 class FacialLandmarkDetector
  44 {
  45 public:
  46     struct Params
  47     {
  48         double leftEyeOpenness;
  49         double rightEyeOpenness;
  50         double leftEyeSmile;
  51         double rightEyeSmile;
  52         double mouthOpenness;
  53         double mouthForm;
  54         double faceXAngle;
  55         double faceYAngle;
  56         double faceZAngle;
  57         bool autoBlink;
  58         bool autoBreath;
  59         bool randomMotion;
  60         // TODO eyebrows currently not supported...
  61         // I'd like to include them, but the dlib / OSF detection is very
  62         // noisy and inaccurate (at least for my face).
  63     };
  64
  65     FacialLandmarkDetector(std::string cfgPath);
  66     ~FacialLandmarkDetector();
  67
  68     Params getParams(void) const;
  69
  70     void stop(void);
  71
  72     void mainLoop(void);
  73
  74 private:
  75     FacialLandmarkDetector(const FacialLandmarkDetector&) = delete;
  76     FacialLandmarkDetector& operator=(const FacialLandmarkDetector &) = delete;
  77
  78     enum LeftRight : bool
  79     {
  80         LEFT,
  81         RIGHT
  82     };
  83
  84     bool m_stop;
  85
  86     int m_sock;
  87     static const int m_faceId = 0; // Only support one face for now
  88
  89     double calcEyeAspectRatio(Point& p1, Point& p2,
  90                               Point& p3, Point& p4,
  91                               Point& p5, Point& p6) const;
  92
  93     double calcRightEyeAspectRatio(Point landmarks[]) const;
  94     double calcLeftEyeAspectRatio(Point landmarks[]) const;
  95
  96     double calcEyeOpenness(LeftRight eye,
  97                            Point landmarks[],
  98                            double faceYAngle) const;
  99
 100     double calcMouthForm(Point landmarks[]) const;
 101     double calcMouthOpenness(Point landmarks[], double mouthForm) const;
 102
 103     double calcFaceXAngle(Point landmarks[]) const;
 104     double calcFaceYAngle(Point landmarks[], double faceXAngle, double mouthForm) const;
 105     double calcFaceZAngle(Point landmarks[]) const;
 106
 107     void populateDefaultConfig(void);
 108     void parseConfig(std::string cfgPath);
 109     void throwConfigError(std::string paramName, std::string expectedType,
 110                           std::string line, unsigned int lineNum);
 111
 112
 113     std::deque<double> m_leftEyeOpenness;
 114     std::deque<double> m_rightEyeOpenness;
 115
 116     std::deque<double> m_mouthOpenness;
 117     std::deque<double> m_mouthForm;
 118
 119     std::deque<double> m_faceXAngle;
 120     std::deque<double> m_faceYAngle;
 121     std::deque<double> m_faceZAngle;
 122
 123     struct Config
 124     {
 125         std::string osfIpAddress;
 126         int osfPort;
 127         double faceYAngleCorrection;
 128         double eyeSmileEyeOpenThreshold;
 129         double eyeSmileMouthFormThreshold;
 130         double eyeSmileMouthOpenThreshold;
 131         std::size_t faceXAngleNumTaps;
 132         std::size_t faceYAngleNumTaps;
 133         std::size_t faceZAngleNumTaps;
 134         std::size_t mouthFormNumTaps;
 135         std::size_t mouthOpenNumTaps;
 136         std::size_t leftEyeOpenNumTaps;
 137         std::size_t rightEyeOpenNumTaps;
 138         double eyeClosedThreshold;
 139         double eyeOpenThreshold;
 140         double mouthNormalThreshold;
 141         double mouthSmileThreshold;
 142         double mouthClosedThreshold;
 143         double mouthOpenThreshold;
 144         double mouthOpenLaughCorrection;
 145         double faceYAngleXRotCorrection;
 146         double faceYAngleSmileCorrection;
 147         double faceYAngleZeroValue;
 148         double faceYAngleUpThreshold;
 149         double faceYAngleDownThreshold;
 150         bool winkEnable;
 151         bool autoBlink;
 152         bool autoBreath;
 153         bool randomMotion;
 154     } m_cfg;
 155 };
 156
 157 #endif
 158