From: Adrian Iain Lam Date: Sun, 27 Sep 2020 16:46:07 +0000 (+0100) Subject: Adding options to enable auto blink, auto breath and random motion X-Git-Url: https://adrianiainlam.tk/git/?a=commitdiff_plain;h=2b1f0c7c63dc30d20c9d55fda5e99b208a9f82b3;p=mouse-tracker-for-cubism.git Adding options to enable auto blink, auto breath and random motion Also updating README to include troubleshooting for SIGILL. Special thanks to fluno for providing feedback! fluno's YouTube channel: https://www.youtube.com/channel/UCAaORfSmJ7CHD1rskBleT2A --- diff --git a/README.md b/README.md index 29e63f2..62c80f5 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Video showing me using the example program: This library was developed and tested only on Ubuntu 18.04 using GCC 7.5.0. However I don't think I've used anything that prevents it from being cross-platform compatible -- it should still work as long as you have a -recent C/C++ compiler. (The library should only require C++11. The Cubism +recent C/C++ compiler. The library should only require C++11. The Cubism SDK requires C++14. I have made use of one C++17 library (``) in the example program, but it should be straightforward to change this if you don't have C++17 support. @@ -31,9 +31,10 @@ environment without a `/bin/sh` shell you may have to run the commands manually. Hereafter, all build instructions will assume a Linux environment where a shell is available. -If your CPU does not support AVX instructions you may want to edit the -relevant build scripts to remove the `-D USE_AVX_INSTRUCTIONS=1` variable -(or change it the SSE3 etc). However there could be a penalty in performance. +If your CPU does not support AVX instructions you may want to edit "build.sh" +and "example/demo.patch" to remove the `-D USE_AVX_INSTRUCTIONS=1` variable +(or change AVX to SSE4 or SSE2). However there could be a penalty in +performance. ## Build instructions @@ -51,7 +52,8 @@ relevant build scripts to remove the `-D USE_AVX_INSTRUCTIONS=1` variable git clone --recurse-submodules https://github.com/adrianiainlam/facial-landmarks-for-cubism.git -3. To build the library only: +3. To build the library only: (Skip this step if you want to build the example + program. It will be done automatically.) cd ./build.sh @@ -123,10 +125,24 @@ Due to the differences in hardware and differences in each person's face, I have decided to make pretty much every parameter tweakable. The file "config.txt" lists and documents all parameters and their default values. You can change the values there and pass it to the example program using -to `-c` argument. If using the library directly, the path to this file +the `-c` argument. If using the library directly, the path to this file should be passed to the constructor (or pass an empty string to use default values). +## Troubleshooting + +1. Example program crashes with SIGILL (Illegal instruction). + + Your CPU probably doesn't support AVX instructions which is used by dlib. + You can confirm this by running + + grep avx /proc/cpuinfo + + If this is the case, try to find out if your CPU supports SSE4 or SSE2, + then edit "build.sh" and "example/demo.patch" to change + `USE_AVX_INSTRUCTIONS=1` to `USE_SSE4_INSTRUCTIONS=1` or + `USE_SSE2_INSTRUCTIONS=1`. + ## License The library itself is provided under the MIT license. By "the library itself" diff --git a/config.txt b/config.txt index 915485e..9c28cee 100644 --- a/config.txt +++ b/config.txt @@ -38,6 +38,14 @@ lateralInversion 1 # to person. The following values seem to work OK for my face, but # your milage may vary. +# Section 2.0: Live2D automatic functionality +# Set 1 to enable, 0 to disable. +# If these are set, the automatic functionality in Live2D will be enabled. +# Note: If you set auto blink, eye control will be disabled. +autoBlink 0 +autoBreath 0 +randomMotion 0 + # Section 2.1: Face Y direction angle (head pointing up/down) # The Y angle is calculated mainly based on the angle formed # by the corners and the tip of the nose (hereafter referred diff --git a/example/demo.patch b/example/demo.patch index 186b8cb..c6b78d0 100644 --- a/example/demo.patch +++ b/example/demo.patch @@ -1,5 +1,5 @@ diff -pruN --exclude build ./demo_clean/CMakeLists.txt ./demo_dev/CMakeLists.txt ---- ./demo_clean/CMakeLists.txt 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/CMakeLists.txt 2020-09-27 17:43:12.069477246 +0100 +++ ./demo_dev/CMakeLists.txt 2020-07-11 22:52:49.099117981 +0100 @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.16) # Set app name. @@ -61,8 +61,8 @@ diff -pruN --exclude build ./demo_clean/CMakeLists.txt ./demo_dev/CMakeLists.txt # # * USE_RENDER_TARGET diff -pruN --exclude build ./demo_clean/scripts/make_gcc ./demo_dev/scripts/make_gcc ---- ./demo_clean/scripts/make_gcc 2020-07-12 16:16:33.999809687 +0100 -+++ ./demo_dev/scripts/make_gcc 2020-07-11 21:22:23.615043956 +0100 +--- ./demo_clean/scripts/make_gcc 2020-09-27 17:43:12.069477246 +0100 ++++ ./demo_dev/scripts/make_gcc 2020-07-14 15:33:09.865020790 +0100 @@ -9,5 +9,6 @@ BUILD_PATH=$SCRIPT_PATH/../build/make_gc # Run CMake. cmake -S "$CMAKE_PATH" \ @@ -73,7 +73,7 @@ diff -pruN --exclude build ./demo_clean/scripts/make_gcc ./demo_dev/scripts/make + -D USE_AVX_INSTRUCTIONS=1 +cd "$BUILD_PATH" && make -j4 diff -pruN --exclude build ./demo_clean/src/CMakeLists.txt ./demo_dev/src/CMakeLists.txt ---- ./demo_clean/src/CMakeLists.txt 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/CMakeLists.txt 2020-09-27 17:43:12.081477263 +0100 +++ ./demo_dev/src/CMakeLists.txt 2020-07-11 17:39:18.358435702 +0100 @@ -19,6 +19,4 @@ target_sources(${APP_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/LAppView.cpp @@ -83,7 +83,7 @@ diff -pruN --exclude build ./demo_clean/src/CMakeLists.txt ./demo_dev/src/CMakeL - ${CMAKE_CURRENT_SOURCE_DIR}/TouchManager.hpp ) diff -pruN --exclude build ./demo_clean/src/LAppDelegate.cpp ./demo_dev/src/LAppDelegate.cpp ---- ./demo_clean/src/LAppDelegate.cpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppDelegate.cpp 2020-09-27 17:43:12.081477263 +0100 +++ ./demo_dev/src/LAppDelegate.cpp 2020-07-11 17:35:02.414902548 +0100 @@ -45,7 +45,8 @@ void LAppDelegate::ReleaseInstance() s_instance = NULL; @@ -213,7 +213,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppDelegate.cpp ./demo_dev/src/LApp Csm::csmVector LAppDelegate::Split(const std::string& baseString, char delimiter) diff -pruN --exclude build ./demo_clean/src/LAppDelegate.hpp ./demo_dev/src/LAppDelegate.hpp ---- ./demo_clean/src/LAppDelegate.hpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppDelegate.hpp 2020-09-27 17:43:12.081477263 +0100 +++ ./demo_dev/src/LAppDelegate.hpp 2020-07-11 17:34:40.778602504 +0100 @@ -40,7 +40,8 @@ public: /** @@ -289,7 +289,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppDelegate.hpp ./demo_dev/src/LApp - -}; diff -pruN --exclude build ./demo_clean/src/LAppLive2DManager.cpp ./demo_dev/src/LAppLive2DManager.cpp ---- ./demo_clean/src/LAppLive2DManager.cpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppLive2DManager.cpp 2020-09-27 17:43:12.081477263 +0100 +++ ./demo_dev/src/LAppLive2DManager.cpp 2020-07-11 23:20:11.548419176 +0100 @@ -52,9 +52,10 @@ void LAppLive2DManager::ReleaseInstance( @@ -394,7 +394,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppLive2DManager.cpp ./demo_dev/src + _translateY = translateY; +} diff -pruN --exclude build ./demo_clean/src/LAppLive2DManager.hpp ./demo_dev/src/LAppLive2DManager.hpp ---- ./demo_clean/src/LAppLive2DManager.hpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppLive2DManager.hpp 2020-09-27 17:43:12.069477246 +0100 +++ ./demo_dev/src/LAppLive2DManager.hpp 2020-07-11 23:21:17.969484538 +0100 @@ -6,12 +6,15 @@ */ @@ -471,8 +471,8 @@ diff -pruN --exclude build ./demo_clean/src/LAppLive2DManager.hpp ./demo_dev/src + float _translateY; }; diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppModel.cpp ---- ./demo_clean/src/LAppModel.cpp 2020-07-12 16:16:33.999809687 +0100 -+++ ./demo_dev/src/LAppModel.cpp 2020-07-11 15:57:43.784019311 +0100 +--- ./demo_clean/src/LAppModel.cpp 2020-09-27 17:43:12.069477246 +0100 ++++ ./demo_dev/src/LAppModel.cpp 2020-09-27 17:40:16.401166244 +0100 @@ -21,6 +21,8 @@ #include "LAppTextureManager.hpp" #include "LAppDelegate.hpp" @@ -513,47 +513,10 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod //Physics if (strcmp(_modelSetting->GetPhysicsFileName(), "") != 0) { -@@ -174,27 +152,6 @@ void LAppModel::SetupModel(ICubismModelS - DeleteBuffer(buffer, path.GetRawString()); - } - -- //EyeBlink -- if (_modelSetting->GetEyeBlinkParameterCount() > 0) -- { -- _eyeBlink = CubismEyeBlink::Create(_modelSetting); -- } -- -- //Breath -- { -- _breath = CubismBreath::Create(); -- -- csmVector breathParameters; -- -- breathParameters.PushBack(CubismBreath::BreathParameterData(_idParamAngleX, 0.0f, 15.0f, 6.5345f, 0.5f)); -- breathParameters.PushBack(CubismBreath::BreathParameterData(_idParamAngleY, 0.0f, 8.0f, 3.5345f, 0.5f)); -- breathParameters.PushBack(CubismBreath::BreathParameterData(_idParamAngleZ, 0.0f, 10.0f, 5.5345f, 0.5f)); -- breathParameters.PushBack(CubismBreath::BreathParameterData(_idParamBodyAngleX, 0.0f, 4.0f, 15.5345f, 0.5f)); -- breathParameters.PushBack(CubismBreath::BreathParameterData(CubismFramework::GetIdManager()->GetId(ParamBreath), 0.5f, 0.5f, 3.2345f, 0.5f)); -- -- _breath->SetParameters(breathParameters); -- } -- - //UserData - if (strcmp(_modelSetting->GetUserDataFile(), "") != 0) - { -@@ -205,24 +162,6 @@ void LAppModel::SetupModel(ICubismModelS - DeleteBuffer(buffer, path.GetRawString()); +@@ -214,15 +192,6 @@ void LAppModel::SetupModel(ICubismModelS + } } -- // EyeBlinkIds -- { -- csmInt32 eyeBlinkIdCount = _modelSetting->GetEyeBlinkParameterCount(); -- for (csmInt32 i = 0; i < eyeBlinkIdCount; ++i) -- { -- _eyeBlinkIds.PushBack(_modelSetting->GetEyeBlinkParameterId(i)); -- } -- } -- - // LipSyncIds - { - csmInt32 lipSyncIdCount = _modelSetting->GetLipSyncParameterCount(); @@ -566,22 +529,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod //Layout csmMap layout; _modelSetting->GetLayoutMap(layout); -@@ -230,14 +169,6 @@ void LAppModel::SetupModel(ICubismModelS - - _model->SaveParameters(); - -- for (csmInt32 i = 0; i < _modelSetting->GetMotionGroupCount(); i++) -- { -- const csmChar* group = _modelSetting->GetMotionGroupName(i); -- PreloadMotionGroup(group); -- } -- -- _motionManager->StopAllMotions(); -- - _updating = false; - _initialized = true; - } -@@ -335,59 +266,29 @@ void LAppModel::Update() +@@ -335,59 +304,57 @@ void LAppModel::Update() const csmFloat32 deltaTimeSeconds = LAppPal::GetDeltaTime(); _userTimeSeconds += deltaTimeSeconds; @@ -600,25 +548,33 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod - StartRandomMotion(MotionGroupIdle, PriorityIdle); - } - else -- { ++ if (_detector) + { - motionUpdated = _motionManager->UpdateMotion(_model, deltaTimeSeconds); // モーションを更新 - } - _model->SaveParameters(); // 状態を保存 - //----------------------------------------------------------------- -- ++ auto idMan = CubismFramework::GetIdManager(); ++ auto params = _detector->getParams(); + - // まばたき - if (!motionUpdated) - { - if (_eyeBlink != NULL) -- { ++ // NOTE: Apparently, this LoadParameters/SaveParameters pair ++ // is needed for auto breath to work. ++ _model->LoadParameters(); // 前回セーブされた状態をロード ++ if (_motionManager->IsFinished() && params.randomMotion) + { - // メインモーションの更新がないとき - _eyeBlink->UpdateParameters(_model, deltaTimeSeconds); // 目パチ -- } ++ // モーションの再生がない場合、待機モーションの中からランダムで再生する ++ StartRandomMotion(MotionGroupIdle, PriorityIdle); + } - } - - if (_expressionManager != NULL) -+ if (_detector) - { +- { - _expressionManager->UpdateMotion(_model, deltaTimeSeconds); // 表情でパラメータ更新（相対変化） - } - @@ -630,21 +586,31 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod - - //ドラッグによる体の向きの調整 - _model->AddParameterValue(_idParamBodyAngleX, _dragX * 10); // -10から10の値を加える -- ++ else ++ { ++ _motionManager->UpdateMotion(_model, deltaTimeSeconds); // モーションを更新 ++ } ++ _model->SaveParameters(); // 状態を保存 + - //ドラッグによる目の向きの調整 - _model->AddParameterValue(_idParamEyeBallX, _dragX); // -1から1の値を加える - _model->AddParameterValue(_idParamEyeBallY, _dragY); -+ auto idMan = CubismFramework::GetIdManager(); -+ auto params = _detector->getParams(); - // 呼吸など - if (_breath != NULL) - { - _breath->UpdateParameters(_model, deltaTimeSeconds); -+ _model->SetParameterValue(idMan->GetId("ParamEyeLOpen"), -+ params.leftEyeOpenness); -+ _model->SetParameterValue(idMan->GetId("ParamEyeROpen"), -+ params.rightEyeOpenness); ++ if (params.autoBlink && _eyeBlink) ++ { ++ _eyeBlink->UpdateParameters(_model, deltaTimeSeconds); ++ } ++ else ++ { ++ _model->SetParameterValue(idMan->GetId("ParamEyeLOpen"), ++ params.leftEyeOpenness); ++ _model->SetParameterValue(idMan->GetId("ParamEyeROpen"), ++ params.rightEyeOpenness); ++ } + _model->SetParameterValue(idMan->GetId("ParamMouthForm"), + params.mouthForm); + _model->SetParameterValue(idMan->GetId("ParamMouthOpenY"), @@ -659,10 +625,16 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod + params.faceYAngle); + _model->SetParameterValue(idMan->GetId("ParamAngleZ"), + params.faceZAngle); ++ if (params.autoBreath && _breath) ++ { ++ // Note: _model->LoadParameters and SaveParameters is needed ++ // before - see above. ++ _breath->UpdateParameters(_model, deltaTimeSeconds); ++ } } // 物理演算の設定 -@@ -396,17 +297,6 @@ void LAppModel::Update() +@@ -396,17 +363,6 @@ void LAppModel::Update() _physics->Evaluate(_model, deltaTimeSeconds); } @@ -680,7 +652,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod // ポーズの設定 if (_pose != NULL) { -@@ -626,3 +516,9 @@ Csm::Rendering::CubismOffscreenFrame_Ope +@@ -626,3 +582,9 @@ Csm::Rendering::CubismOffscreenFrame_Ope { return _renderBuffer; } @@ -691,7 +663,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.cpp ./demo_dev/src/LAppMod +} + diff -pruN --exclude build ./demo_clean/src/LAppModel.hpp ./demo_dev/src/LAppModel.hpp ---- ./demo_clean/src/LAppModel.hpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppModel.hpp 2020-09-27 17:43:12.081477263 +0100 +++ ./demo_dev/src/LAppModel.hpp 2020-07-11 15:40:18.977286166 +0100 @@ -13,6 +13,7 @@ #include @@ -725,7 +697,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppModel.hpp ./demo_dev/src/LAppMod diff -pruN --exclude build ./demo_clean/src/LAppPal.cpp ./demo_dev/src/LAppPal.cpp ---- ./demo_clean/src/LAppPal.cpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppPal.cpp 2020-09-27 17:43:12.081477263 +0100 +++ ./demo_dev/src/LAppPal.cpp 2020-07-11 23:29:09.084910139 +0100 @@ -6,6 +6,7 @@ */ @@ -748,7 +720,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppPal.cpp ./demo_dev/src/LAppPal.c } file.read(buf, size); diff -pruN --exclude build ./demo_clean/src/LAppTextureManager.cpp ./demo_dev/src/LAppTextureManager.cpp ---- ./demo_clean/src/LAppTextureManager.cpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppTextureManager.cpp 2020-09-27 17:43:12.085477268 +0100 +++ ./demo_dev/src/LAppTextureManager.cpp 2020-07-11 22:22:18.004965003 +0100 @@ -96,6 +96,46 @@ LAppTextureManager::TextureInfo* LAppTex @@ -798,7 +770,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppTextureManager.cpp ./demo_dev/sr { for (Csm::csmUint32 i = 0; i < _textures.GetSize(); i++) diff -pruN --exclude build ./demo_clean/src/LAppTextureManager.hpp ./demo_dev/src/LAppTextureManager.hpp ---- ./demo_clean/src/LAppTextureManager.hpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppTextureManager.hpp 2020-09-27 17:43:12.069477246 +0100 +++ ./demo_dev/src/LAppTextureManager.hpp 2020-07-11 17:36:31.180131039 +0100 @@ -72,6 +72,8 @@ public: */ @@ -810,7 +782,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppTextureManager.hpp ./demo_dev/sr * @brief 画像の解放 * diff -pruN --exclude build ./demo_clean/src/LAppView.cpp ./demo_dev/src/LAppView.cpp ---- ./demo_clean/src/LAppView.cpp 2020-07-12 16:16:34.003809759 +0100 +--- ./demo_clean/src/LAppView.cpp 2020-09-27 17:43:12.085477268 +0100 +++ ./demo_dev/src/LAppView.cpp 2020-07-11 17:38:06.905451955 +0100 @@ -13,7 +13,6 @@ #include "LAppLive2DManager.hpp" @@ -987,7 +959,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppView.cpp ./demo_dev/src/LAppView - } } diff -pruN --exclude build ./demo_clean/src/LAppView.hpp ./demo_dev/src/LAppView.hpp ---- ./demo_clean/src/LAppView.hpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/LAppView.hpp 2020-09-27 17:43:12.069477246 +0100 +++ ./demo_dev/src/LAppView.hpp 2020-07-11 17:38:25.541708705 +0100 @@ -14,7 +14,6 @@ #include "CubismFramework.hpp" @@ -1043,7 +1015,7 @@ diff -pruN --exclude build ./demo_clean/src/LAppView.hpp ./demo_dev/src/LAppView // レンダリング先を別ターゲットにする方式の場合に使用 LAppSprite* _renderSprite; ///< モードによっては_renderBufferのテクスチャを描画 diff -pruN --exclude build ./demo_clean/src/main.cpp ./demo_dev/src/main.cpp ---- ./demo_clean/src/main.cpp 2020-07-12 16:16:33.999809687 +0100 +--- ./demo_clean/src/main.cpp 2020-09-27 17:43:12.069477246 +0100 +++ ./demo_dev/src/main.cpp 2020-07-12 15:06:29.194034887 +0100 @@ -5,18 +5,156 @@ * that can be found at https://www.live2d.com/eula/live2d-open-software-license-agreement_en.html. diff --git a/include/facial_landmark_detector.h b/include/facial_landmark_detector.h index 7c6f639..5163fe9 100644 --- a/include/facial_landmark_detector.h +++ b/include/facial_landmark_detector.h @@ -46,6 +46,9 @@ public: double faceXAngle; double faceYAngle; double faceZAngle; + bool autoBlink; + bool autoBreath; + bool randomMotion; // TODO eyebrows currently not supported... // I'd like to include them, but the dlib detection is very // noisy and inaccurate (at least for my face). @@ -137,6 +140,9 @@ private: double faceYAngleZeroValue; double faceYAngleUpThreshold; double faceYAngleDownThreshold; + bool autoBlink; + bool autoBreath; + bool randomMotion; } m_cfg; }; diff --git a/src/facial_landmark_detector.cpp b/src/facial_landmark_detector.cpp index 20ec7c8..fb536b6 100644 --- a/src/facial_landmark_detector.cpp +++ b/src/facial_landmark_detector.cpp @@ -111,6 +111,10 @@ FacialLandmarkDetector::Params FacialLandmarkDetector::getParams(void) const params.rightEyeSmile = 0; } + params.autoBlink = m_cfg.autoBlink; + params.autoBreath = m_cfg.autoBreath; + params.randomMotion = m_cfg.randomMotion; + return params; } @@ -668,6 +672,30 @@ void FacialLandmarkDetector::parseConfig(std::string cfgPath) line, lineNum); } } + else if (paramName == "autoBlink") + { + if (!(ss >> m_cfg.autoBlink)) + { + throwConfigError(paramName, "bool", + line, lineNum); + } + } + else if (paramName == "autoBreath") + { + if (!(ss >> m_cfg.autoBreath)) + { + throwConfigError(paramName, "bool", + line, lineNum); + } + } + else if (paramName == "randomMotion") + { + if (!(ss >> m_cfg.randomMotion)) + { + throwConfigError(paramName, "bool", + line, lineNum); + } + } else { std::ostringstream oss; @@ -714,6 +742,9 @@ void FacialLandmarkDetector::populateDefaultConfig(void) m_cfg.faceYAngleZeroValue = 1.8; m_cfg.faceYAngleDownThreshold = 2.3; m_cfg.faceYAngleUpThreshold = 1.3; + m_cfg.autoBlink = false; + m_cfg.autoBreath = false; + m_cfg.randomMotion = false; } void FacialLandmarkDetector::throwConfigError(std::string paramName,