2 Copyright (c) 2020 Adrian I. Lam
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to deal
6 in the Software without restriction, including without limitation the rights
7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
40 #include "mouse_cursor_tracker.h"
45 #include <pulse/simple.h>
46 #include <string.h> // strdup
51 static double rms(float *buf
, std
::size_t count
)
54 for (std
::size_t i
= 0; i
< count
; i
++)
56 sum
+= buf
[i
] * buf
[i
];
58 return std
::sqrt(sum
/ count
);
61 static std
::vector
<std
::string
> split(std
::string s
)
63 std
::vector
<std
::string
> v
;
66 for (std
::size_t i
= 0; i
< s
.length(); i
++)
90 /* Using readline callback functions means that we need to pass
91 * information from our class to the callback, and the only way
92 * to do so is using globals.
95 // Taken from https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/utils.cpp
96 static std
::string
longest_common_prefix(std
::string s
,
97 const std
::vector
<std
::string
>& candidates
) {
98 assert(candidates
.size() > 0);
99 if (candidates
.size() == 1) {
100 return candidates
[0];
103 std
::string
prefix(s
);
105 // Each iteration of this loop advances to the next location in all the
106 // candidates and sees if they match up to it.
107 size_t nextloc
= prefix
.size();
108 auto i
= candidates
.begin();
109 if (i
->size() <= nextloc
) {
112 char nextchar
= (*(i
++))[nextloc
];
113 for (; i
!= candidates
.end(); ++i
) {
114 if (i
->size() <= nextloc
|| (*i
)[nextloc
] != nextchar
) {
115 // Bail out if there's a mismatch for this candidate.
119 // All candidates have contents[nextloc] == nextchar, so we can safely
120 // extend the prefix.
121 prefix
.append(1, nextchar
);
124 assert(0 && "unreachable");
127 std
::vector
<std
::string
> commands
=
129 "help", "motion", "expression", "set", "clear"
132 std
::vector
<std
::string
> live2dParams
=
133 { // https://docs.live2d.com/cubism-editor-manual/standard-parametor-list/?locale=ja
134 "ParamAngleX", "ParamAngleY", "ParamAngleZ",
135 "ParamEyeLOpen", "ParamEyeLSmile", "ParamEyeROpen", "ParamEyeRSmile",
136 "ParamEyeBallX", "ParamEyeBallY", "ParamEyeBallForm",
137 "ParamBrowLY", "ParamBrowRY", "ParamBrowLX", "ParamBrowRX",
138 "ParamBrowLAngle", "ParamBrowRAngle", "ParamBrowLForm", "ParamBrowRForm",
139 "ParamMouthForm", "ParamMouthOpenY", "ParamTere",
140 "ParamBodyAngleX", "ParamBodyAngleY", "ParamBodyAngleZ", "ParamBreath",
141 "ParamArmLA", "ParamArmRA", "ParamArmLB", "ParamArmRB",
142 "ParamHandL", "ParamHandR",
143 "ParamHairFront", "ParamHairSide", "ParamHairBack", "ParamHairFluffy",
144 "ParamShoulderY", "ParamBustX", "ParamBustY",
145 "ParamBaseX", "ParamBaseY"
147 std
::vector
<std
::pair
<std
::string
, int> > MCT_motions
;
148 std
::vector
<std
::string
> MCT_expressions
;
149 std
::map
<std
::string
, double> *MCT_overrideMap
;
151 static int cliListPossible(char *token
, char ***av
)
153 // Reference: https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/readline-complete-subcommand.cpp
155 std
::string
line(rl_line_buffer
);
157 std
::vector
<std
::string
> cmdline
= split(line
);
159 std
::vector
<std
::string
> constructed
;
160 std
::vector
<std
::string
> *vocab
= nullptr;
162 if (cmdline
.size() == 0 ||
163 (cmdline
.size() == 1 && line
.back() != ' ') ||
164 cmdline
[0] == "help")
168 else if (cmdline
[0] == "motion")
170 for (auto it
= MCT_motions
.begin(); it
!= MCT_motions
.end(); ++it
)
172 if ((cmdline
.size() == 1 && line
.back() == ' ') ||
173 (cmdline
.size() == 2 && line
.back() != ' '))
176 constructed
.push_back(it
->first
);
179 else if ((cmdline
.size() == 2 && line
.back() == ' ') ||
180 (cmdline
.size() == 3 && line
.back() != ' '))
182 if (it
->first
== cmdline
[1])
184 for (int i
= 0; i
< it
->second
; i
++)
186 constructed
.push_back(std
::to_string(i
));
191 else if (cmdline
.size() <= 4)
193 for (int i
= 0; i
< 4; i
++)
195 constructed
.push_back(std
::to_string(i
));
197 break; // No need to loop through the list, all motions have the same set of priorities
201 vocab
= &constructed
;
203 else if (cmdline
[0] == "expression")
205 vocab
= &MCT_expressions
;
207 else if (cmdline
[0] == "set")
209 if ((cmdline
.size() % 2 == 0 && line
.back() != ' ') ||
210 (cmdline
.size() % 2 == 1 && line
.back() == ' '))
212 vocab
= &live2dParams
;
215 else if (cmdline
[0] == "clear")
217 constructed
.push_back("all");
218 for (auto const &entry
: *MCT_overrideMap
)
220 constructed
.push_back(entry
.first
);
222 vocab
= &constructed
;
230 std
::string
text(token
);
231 std
::vector
<std
::string
> matches
;
232 std
::copy_if(vocab
->begin(), vocab
->end(), std
::back_inserter(matches
),
233 [&text
](const std
::string
&s
)
235 return (s
.size() >= text
.size() &&
236 s
.compare(0, text
.size(), text
) == 0);
239 *av
= static_cast<char **>(malloc(matches
.size() * sizeof (**av
)));
241 for (std
::size_t i
= 0; i
< matches
.size(); i
++)
243 (*av
)[i
] = strdup(matches
[i
].c_str());
245 return matches
.size();
248 static char *cliComplete(char *token
, int *match
)
251 int numMatches
= cliListPossible(token
, &av
);
254 if (numMatches
== 0) return nullptr;
256 std
::vector
<std
::string
> list
;
258 for (std
::size_t i
= 0; i
< numMatches
; i
++)
260 list
.push_back(std
::string(av
[i
]));
268 std
::string result
= list
[0] + " ";
269 return strdup(result
.c_str() + strlen(token
));
272 std
::string lcp
= longest_common_prefix(token
, list
);
274 if (lcp
.size() > strlen(token
))
277 return strdup(lcp
.c_str() + strlen(token
));
283 MouseCursorTracker
::MouseCursorTracker(std
::string cfgPath
,
284 std
::vector
<std
::pair
<std
::string
, int> > motions
,
285 std
::vector
<std
::string
> expressions
)
288 m_motionPriority
= MotionPriority
::none
;
292 parseConfig(cfgPath
);
293 m_xdo
= xdo_new(nullptr);
295 const pa_sample_spec ss
=
297 .format
= PA_SAMPLE_FLOAT32NE
,
301 m_pulse
= pa_simple_new(nullptr, "MouseCursorTracker", PA_STREAM_RECORD
,
302 nullptr, "LipSync", &ss
, nullptr, nullptr, nullptr);
305 throw std
::runtime_error("Unable to create pulse");
308 m_gtkapp
= Gtk
::Application
::create();
310 m_getVolumeThread
= std
::thread(&MouseCursorTracker
::audioLoop
, this);
311 m_parseCommandThread
= std
::thread(&MouseCursorTracker
::cliLoop
, this);
312 m_guiThread
= std
::thread(&MouseCursorTracker
::guiLoop
, this);
314 MCT_motions
= motions
;
315 MCT_expressions
= expressions
;
316 MCT_overrideMap
= &m_overrideMap
;
319 void MouseCursorTracker
::audioLoop(void)
321 float *buf
= new float[m_cfg
.audioBufSize
];
323 std
::size_t audioBufByteSize
= m_cfg
.audioBufSize
* sizeof *buf
;
327 if (pa_simple_read(m_pulse
, buf
, audioBufByteSize
, nullptr) < 0)
329 throw std
::runtime_error("Unable to get audio data");
331 m_currentVol
= rms(buf
, m_cfg
.audioBufSize
);
337 void MouseCursorTracker
::cliLoop(void)
339 rl_set_complete_func(&cliComplete
);
340 rl_set_list_possib_func(&cliListPossible
);
343 char *buf
= readline(">> ");
347 std
::string
cmdline(buf
);
349 processCommand(cmdline
);
353 std
::cout
<< "Exiting CLI loop. Use Ctrl+C to exit the whole process." << std
::endl
;
360 void MouseCursorTracker
::processCommand(std
::string cmdline
)
362 auto cmdSplit
= split(cmdline
);
364 if (cmdSplit
.size() > 0)
366 add_history(cmdline
.c_str());
368 if (cmdSplit
[0] == "help")
370 if (cmdSplit
.size() == 1)
372 std
::cout
<< "Available commands: motion expression set clear\n"
373 << "Type \"help <command>\" for more help" << std
::endl
;
375 else if (cmdSplit
[1] == "motion")
377 std
::cout
<< "motion <motionGroup> <motionNumber> [<priority>]\n"
378 << "motionGroup: The motion name in the .model3.json file\n"
379 << "motionNumber: The index of this motion in the .model3.json file, 0-indexed\n"
380 << "priority: 0 = none, 1 = idle, 2 = normal, 3 = force (default normal)" << std
::endl
;
382 else if (cmdSplit
[1] == "expression")
384 std
::cout
<< "expression <expressionName>\n"
385 << "expressionName: Name of expression in the .model3.json file" << std
::endl
;
387 else if (cmdSplit
[1] == "set")
389 std
::cout
<< "set <param1> <value1> [<param2> <value2> ...]\n"
390 << "Set parameter value. Overrides any tracking."
391 << "See live2D documentation for full list of params" << std
::endl
;
393 else if (cmdSplit
[1] == "clear")
395 std
::cout
<< "clear <param1> [<param2> ...]\n"
396 << "Clear parameter value. Re-enables tracking if it was overridden by \"set\"\n"
397 << "You can also use \"clear all\" to clear everything" << std
::endl
;
401 std
::cout
<< "Unrecognized command" << std
::endl
;
404 else if (cmdSplit
[0] == "motion")
406 if (cmdSplit
.size() == 3 || cmdSplit
.size() == 4)
408 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::defer_lock
);
410 m_motionGroup
= cmdSplit
[1];
413 m_motionNumber
= std
::stoi(cmdSplit
[2]);
414 if (cmdSplit
.size() == 4)
416 m_motionPriority
= static_cast<MotionPriority
>(std
::stoi(cmdSplit
[3]));
420 m_motionPriority
= MotionPriority
::normal
;
423 catch (const std
::exception
&e
)
425 std
::cerr
<< "std::stoi failed" << std
::endl
;
431 std
::cerr
<< "Incorrect command, expecting 2 or 3 arguments" << std
::endl
;
432 std
::cerr
<< "motion motionGroup motionNumber [motionPriority]" << std
::endl
;
435 else if (cmdSplit
[0] == "expression")
437 if (cmdSplit
.size() == 2)
439 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::defer_lock
);
441 m_expression
= cmdSplit
[1];
446 std
::cerr
<< "Incorrect command, expecting 1 argument: expressionName" << std
::endl
;
449 else if (cmdSplit
[0] == "set")
451 if (cmdSplit
.size() % 2 != 1)
453 // "set param1 value1 param2 value2 ..."
454 std
::cerr
<< "Incorrect number of arguments for command 'set'" << std
::endl
;
456 for (std
::size_t i
= 1; i
< cmdSplit
.size(); i
+= 2)
460 m_overrideMap
[cmdSplit
[i
]] = std
::stod(cmdSplit
[i
+ 1]);
462 catch (const std
::exception
&e
)
464 std
::cerr
<< "std::stod failed" << std
::endl
;
468 else if (cmdSplit
[0] == "clear")
470 for (std
::size_t i
= 1; i
< cmdSplit
.size(); i
++)
472 if (cmdSplit
[i
] == "all")
474 m_overrideMap
.clear();
477 std
::size_t removed
= m_overrideMap
.erase(cmdSplit
[i
]);
480 std
::cerr
<< "Warning: key " << cmdSplit
[i
] << " not found" << std
::endl
;
486 std
::cerr
<< "Unknown command" << std
::endl
;
491 MouseCursorTracker
::~MouseCursorTracker()
494 m_getVolumeThread
.join();
495 m_parseCommandThread
.join();
496 pa_simple_free(m_pulse
);
499 void MouseCursorTracker
::stop(void)
504 MouseCursorTracker
::Params MouseCursorTracker
::getParams(void)
506 Params params
= Params();
508 int xOffset
= m_curPos
.x
- m_cfg
.origin
.x
;
509 int leftRange
= m_cfg
.origin
.x
- m_cfg
.left
;
510 int rightRange
= m_cfg
.right
- m_cfg
.origin
.x
;
512 if (xOffset
> 0) // i.e. to the right
514 params
.live2d
["ParamAngleX"] = 30.0 * xOffset
/ rightRange
;
518 params
.live2d
["ParamAngleX"] = 30.0 * xOffset
/ leftRange
;
521 int yOffset
= m_curPos
.y
- m_cfg
.origin
.y
;
522 int topRange
= m_cfg
.origin
.y
- m_cfg
.top
;
523 int bottomRange
= m_cfg
.bottom
- m_cfg
.origin
.y
;
525 if (yOffset
> 0) // downwards
527 params
.live2d
["ParamAngleY"] = -30.0 * yOffset
/ bottomRange
;
531 params
.live2d
["ParamAngleY"] = -30.0 * yOffset
/ topRange
;
534 params
.autoBlink
= m_cfg
.autoBlink
;
535 params
.autoBreath
= m_cfg
.autoBreath
;
536 params
.randomIdleMotion
= m_cfg
.randomIdleMotion
;
537 params
.useLipSync
= m_cfg
.useLipSync
;
539 params
.live2d
["ParamMouthForm"] = m_cfg
.mouthForm
;
541 if (m_cfg
.useLipSync
)
543 params
.lipSyncParam
= m_currentVol
* m_cfg
.lipSyncGain
;
544 if (params
.lipSyncParam
< m_cfg
.lipSyncCutOff
)
546 params
.lipSyncParam
= 0;
548 else if (params
.lipSyncParam
> 1)
550 params
.lipSyncParam
= 1;
554 // Don't block in getParams()
555 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::try_to_lock
);
556 if (lock
.owns_lock())
558 if (m_motionPriority
!= MotionPriority
::none
)
560 params
.motionPriority
= m_motionPriority
;
561 params
.motionGroup
= m_motionGroup
;
562 params
.motionNumber
= m_motionNumber
;
564 m_motionPriority
= MotionPriority
::none
;
568 params
.expression
= m_expression
;
573 // Leave everything else as zero
577 for (auto const &x
: m_overrideMap
)
579 std
::string key
= x
.first
;
580 double val
= x
.second
;
582 params
.live2d
[key
] = val
;
589 void MouseCursorTracker
::mainLoop(void)
597 xdo_get_mouse_location(m_xdo
, &x
, &y
, &screenNum
);
599 if (screenNum
== m_cfg
.screen
)
604 // else just silently ignore for now
605 std
::this_thread
::sleep_for(std
::chrono
::milliseconds(m_cfg
.sleepMs
));
609 void MouseCursorTracker
::parseConfig(std
::string cfgPath
)
611 populateDefaultConfig();
615 std
::ifstream
file(cfgPath
);
618 throw std
::runtime_error("Failed to open config file");
621 unsigned int lineNum
= 0;
623 while (std
::getline(file
, line
))
630 std
::istringstream
ss(line
);
631 std
::string paramName
;
635 if (paramName
== "sleep_ms")
637 if (!(ss
>> m_cfg
.sleepMs
))
639 throw std
::runtime_error("Error parsing sleep_ms");
642 else if (paramName
== "autoBlink")
644 if (!(ss
>> m_cfg
.autoBlink
))
646 throw std
::runtime_error("Error parsing autoBlink");
649 else if (paramName
== "autoBreath")
651 if (!(ss
>> m_cfg
.autoBreath
))
653 throw std
::runtime_error("Error parsing autoBreath");
656 else if (paramName
== "randomIdleMotion")
658 if (!(ss
>> m_cfg
.randomIdleMotion
))
660 throw std
::runtime_error("Error parsing randomIdleMotion");
663 else if (paramName
== "useLipSync")
665 if (!(ss
>> m_cfg
.useLipSync
))
667 throw std
::runtime_error("Error parsing useLipSync");
670 else if (paramName
== "lipSyncGain")
672 if (!(ss
>> m_cfg
.lipSyncGain
))
674 throw std
::runtime_error("Error parsing lipSyncGain");
677 else if (paramName
== "lipSyncCutOff")
679 if (!(ss
>> m_cfg
.lipSyncCutOff
))
681 throw std
::runtime_error("Error parsing lipSyncCutOff");
684 else if (paramName
== "audioBufSize")
686 if (!(ss
>> m_cfg
.audioBufSize
))
688 throw std
::runtime_error("Error parsing audioBufSize");
691 else if (paramName
== "mouthForm")
693 if (!(ss
>> m_cfg
.mouthForm
))
695 throw std
::runtime_error("Error parsing mouthForm");
698 else if (paramName
== "screen")
700 if (!(ss
>> m_cfg
.screen
))
702 throw std
::runtime_error("Error parsing screen");
705 else if (paramName
== "origin_x")
707 if (!(ss
>> m_cfg
.origin
.x
))
709 throw std
::runtime_error("Error parsing origin_x");
712 else if (paramName
== "origin_y")
714 if (!(ss
>> m_cfg
.origin
.y
))
716 throw std
::runtime_error("Error parsing origin_y");
719 else if (paramName
== "top")
721 if (!(ss
>> m_cfg
.top
))
723 throw std
::runtime_error("Error parsing top");
726 else if (paramName
== "bottom")
728 if (!(ss
>> m_cfg
.bottom
))
730 throw std
::runtime_error("Error parsing bottom");
733 else if (paramName
== "left")
735 if (!(ss
>> m_cfg
.left
))
737 throw std
::runtime_error("Error parsing left");
740 else if (paramName
== "right")
742 if (!(ss
>> m_cfg
.right
))
744 throw std
::runtime_error("Error parsing right");
749 throw std
::runtime_error("Unrecognized config parameter");
756 void MouseCursorTracker
::populateDefaultConfig(void)
759 m_cfg
.autoBlink
= true;
760 m_cfg
.autoBreath
= true;
761 m_cfg
.randomIdleMotion
= true;
762 m_cfg
.useLipSync
= true;
763 m_cfg
.lipSyncGain
= 10;
764 m_cfg
.lipSyncCutOff
= 0.15;
765 m_cfg
.audioBufSize
= 4096;
770 m_cfg
.right
= 1919; // These will be the full screen for 1920x1080
773 m_cfg
.origin
= {1600, 870}; // Somewhere near the bottom right