2 Copyright (c) 2020 Adrian I. Lam
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to deal
6 in the Software without restriction, including without limitation the rights
7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
43 #include <pulse/simple.h>
44 #include <string.h> // strdup
47 #include "mouse_cursor_tracker.h"
49 static double rms(float *buf
, std
::size_t count
)
52 for (std
::size_t i
= 0; i
< count
; i
++)
54 sum
+= buf
[i
] * buf
[i
];
56 return std
::sqrt(sum
/ count
);
59 static std
::vector
<std
::string
> split(std
::string s
)
61 std
::vector
<std
::string
> v
;
64 for (std
::size_t i
= 0; i
< s
.length(); i
++)
88 /* Using readline callback functions means that we need to pass
89 * information from our class to the callback, and the only way
90 * to do so is using globals.
93 // Taken from https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/utils.cpp
94 static std
::string
longest_common_prefix(std
::string s
,
95 const std
::vector
<std
::string
>& candidates
) {
96 assert(candidates
.size() > 0);
97 if (candidates
.size() == 1) {
101 std
::string
prefix(s
);
103 // Each iteration of this loop advances to the next location in all the
104 // candidates and sees if they match up to it.
105 size_t nextloc
= prefix
.size();
106 auto i
= candidates
.begin();
107 if (i
->size() <= nextloc
) {
110 char nextchar
= (*(i
++))[nextloc
];
111 for (; i
!= candidates
.end(); ++i
) {
112 if (i
->size() <= nextloc
|| (*i
)[nextloc
] != nextchar
) {
113 // Bail out if there's a mismatch for this candidate.
117 // All candidates have contents[nextloc] == nextchar, so we can safely
118 // extend the prefix.
119 prefix
.append(1, nextchar
);
122 assert(0 && "unreachable");
125 std
::vector
<std
::string
> commands
=
127 "help", "motion", "expression", "set", "clear"
130 std
::vector
<std
::string
> live2dParams
=
131 { // https://docs.live2d.com/cubism-editor-manual/standard-parametor-list/?locale=ja
132 "ParamAngleX", "ParamAngleY", "ParamAngleZ",
133 "ParamEyeLOpen", "ParamEyeLSmile", "ParamEyeROpen", "ParamEyeRSmile",
134 "ParamEyeBallX", "ParamEyeBallY", "ParamEyeBallForm",
135 "ParamBrowLY", "ParamBrowRY", "ParamBrowLX", "ParamBrowRX",
136 "ParamBrowLAngle", "ParamBrowRAngle", "ParamBrowLForm", "ParamBrowRForm",
137 "ParamMouthForm", "ParamMouthOpenY", "ParamTere",
138 "ParamBodyAngleX", "ParamBodyAngleY", "ParamBodyAngleZ", "ParamBreath",
139 "ParamArmLA", "ParamArmRA", "ParamArmLB", "ParamArmRB",
140 "ParamHandL", "ParamHandR",
141 "ParamHairFront", "ParamHairSide", "ParamHairBack", "ParamHairFluffy",
142 "ParamShoulderY", "ParamBustX", "ParamBustY",
143 "ParamBaseX", "ParamBaseY"
145 std
::vector
<std
::pair
<std
::string
, int> > MCT_motions
;
146 std
::vector
<std
::string
> MCT_expressions
;
147 std
::map
<std
::string
, double> *MCT_overrideMap
;
149 static int cliListPossible(char *token
, char ***av
)
151 // Reference: https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/readline-complete-subcommand.cpp
153 std
::string
line(rl_line_buffer
);
155 std
::vector
<std
::string
> cmdline
= split(line
);
157 std
::vector
<std
::string
> constructed
;
158 std
::vector
<std
::string
> *vocab
= nullptr;
160 if (cmdline
.size() == 0 ||
161 (cmdline
.size() == 1 && line
.back() != ' ') ||
162 cmdline
[0] == "help")
166 else if (cmdline
[0] == "motion")
168 for (auto it
= MCT_motions
.begin(); it
!= MCT_motions
.end(); ++it
)
170 if ((cmdline
.size() == 1 && line
.back() == ' ') ||
171 (cmdline
.size() == 2 && line
.back() != ' '))
174 constructed
.push_back(it
->first
);
177 else if ((cmdline
.size() == 2 && line
.back() == ' ') ||
178 (cmdline
.size() == 3 && line
.back() != ' '))
180 if (it
->first
== cmdline
[1])
182 for (int i
= 0; i
< it
->second
; i
++)
184 constructed
.push_back(std
::to_string(i
));
189 else if (cmdline
.size() <= 4)
191 for (int i
= 0; i
< 4; i
++)
193 constructed
.push_back(std
::to_string(i
));
195 break; // No need to loop through the list, all motions have the same set of priorities
199 vocab
= &constructed
;
201 else if (cmdline
[0] == "expression")
203 vocab
= &MCT_expressions
;
205 else if (cmdline
[0] == "set")
207 if ((cmdline
.size() % 2 == 0 && line
.back() != ' ') ||
208 (cmdline
.size() % 2 == 1 && line
.back() == ' '))
210 vocab
= &live2dParams
;
213 else if (cmdline
[0] == "clear")
215 constructed
.push_back("all");
216 for (auto const &entry
: *MCT_overrideMap
)
218 constructed
.push_back(entry
.first
);
220 vocab
= &constructed
;
228 std
::string
text(token
);
229 std
::vector
<std
::string
> matches
;
230 std
::copy_if(vocab
->begin(), vocab
->end(), std
::back_inserter(matches
),
231 [&text
](const std
::string
&s
)
233 return (s
.size() >= text
.size() &&
234 s
.compare(0, text
.size(), text
) == 0);
237 *av
= static_cast<char **>(malloc(matches
.size() * sizeof (**av
)));
239 for (std
::size_t i
= 0; i
< matches
.size(); i
++)
241 (*av
)[i
] = strdup(matches
[i
].c_str());
243 return matches
.size();
246 static char *cliComplete(char *token
, int *match
)
249 int numMatches
= cliListPossible(token
, &av
);
252 if (numMatches
== 0) return nullptr;
254 std
::vector
<std
::string
> list
;
256 for (std
::size_t i
= 0; i
< numMatches
; i
++)
258 list
.push_back(std
::string(av
[i
]));
266 std
::string result
= list
[0] + " ";
267 return strdup(result
.c_str() + strlen(token
));
270 std
::string lcp
= longest_common_prefix(token
, list
);
272 if (lcp
.size() > strlen(token
))
275 return strdup(lcp
.c_str() + strlen(token
));
281 MouseCursorTracker
::MouseCursorTracker(std
::string cfgPath
,
282 std
::vector
<std
::pair
<std
::string
, int> > motions
,
283 std
::vector
<std
::string
> expressions
)
286 m_motionPriority
= MotionPriority
::none
;
290 parseConfig(cfgPath
);
291 m_xdo
= xdo_new(nullptr);
293 const pa_sample_spec ss
=
295 .format
= PA_SAMPLE_FLOAT32NE
,
299 m_pulse
= pa_simple_new(nullptr, "MouseCursorTracker", PA_STREAM_RECORD
,
300 nullptr, "LipSync", &ss
, nullptr, nullptr, nullptr);
303 throw std
::runtime_error("Unable to create pulse");
306 m_getVolumeThread
= std
::thread(&MouseCursorTracker
::audioLoop
, this);
307 m_parseCommandThread
= std
::thread(&MouseCursorTracker
::cliLoop
, this);
309 MCT_motions
= motions
;
310 MCT_expressions
= expressions
;
311 MCT_overrideMap
= &m_overrideMap
;
314 void MouseCursorTracker
::audioLoop(void)
316 float *buf
= new float[m_cfg
.audioBufSize
];
318 std
::size_t audioBufByteSize
= m_cfg
.audioBufSize
* sizeof *buf
;
322 if (pa_simple_read(m_pulse
, buf
, audioBufByteSize
, nullptr) < 0)
324 throw std
::runtime_error("Unable to get audio data");
326 m_currentVol
= rms(buf
, m_cfg
.audioBufSize
);
332 void MouseCursorTracker
::cliLoop(void)
334 rl_set_complete_func(&cliComplete
);
335 rl_set_list_possib_func(&cliListPossible
);
338 char *buf
= readline(">> ");
342 std
::string
cmdline(buf
);
344 processCommand(cmdline
);
348 std
::cout
<< "Exiting CLI loop. Use Ctrl+C to exit the whole process." << std
::endl
;
355 void MouseCursorTracker
::processCommand(std
::string cmdline
)
357 auto cmdSplit
= split(cmdline
);
359 if (cmdSplit
.size() > 0)
361 add_history(cmdline
.c_str());
363 if (cmdSplit
[0] == "help")
365 if (cmdSplit
.size() == 1)
367 std
::cout
<< "Available commands: motion set clear\n"
368 << "Type \"help <command>\" for more help" << std
::endl
;
370 else if (cmdSplit
[1] == "motion")
372 std
::cout
<< "motion <motionGroup> <motionNumber> [<priority>]\n"
373 << "motionGroup: The motion name in the .model3.json file\n"
374 << "motionNumber: The index of this motion in the .model3.json file, 0-indexed\n"
375 << "priority: 0 = none, 1 = idle, 2 = normal, 3 = force (default normal)" << std
::endl
;
377 else if (cmdSplit
[1] == "expression")
379 std
::cout
<< "expression <expressionName>\n"
380 << "expressionName: Name of expression in the .model3.json file" << std
::endl
;
382 else if (cmdSplit
[1] == "set")
384 std
::cout
<< "set <param1> <value1> [<param2> <value2> ...]\n"
385 << "Set parameter value. Overrides any tracking."
386 << "See live2D documentation for full list of params" << std
::endl
;
388 else if (cmdSplit
[1] == "clear")
390 std
::cout
<< "clear <param1> [<param2> ...]\n"
391 << "Clear parameter value. Re-enables tracking if it was overridden by \"set\"\n"
392 << "You can also use \"clear all\" to clear everything" << std
::endl
;
396 std
::cout
<< "Unrecognized command" << std
::endl
;
399 else if (cmdSplit
[0] == "motion")
401 if (cmdSplit
.size() == 3 || cmdSplit
.size() == 4)
403 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::defer_lock
);
405 m_motionGroup
= cmdSplit
[1];
408 m_motionNumber
= std
::stoi(cmdSplit
[2]);
409 if (cmdSplit
.size() == 4)
411 m_motionPriority
= static_cast<MotionPriority
>(std
::stoi(cmdSplit
[3]));
415 m_motionPriority
= MotionPriority
::normal
;
418 catch (const std
::exception
&e
)
420 std
::cerr
<< "std::stoi failed" << std
::endl
;
426 std
::cerr
<< "Incorrect command, expecting 2 or 3 arguments" << std
::endl
;
427 std
::cerr
<< "motion motionGroup motionNumber [motionPriority]" << std
::endl
;
430 else if (cmdSplit
[0] == "expression")
432 if (cmdSplit
.size() == 2)
434 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::defer_lock
);
436 m_expression
= cmdSplit
[1];
441 std
::cerr
<< "Incorrect command, expecting 1 argument: expressionName" << std
::endl
;
444 else if (cmdSplit
[0] == "set")
446 if (cmdSplit
.size() % 2 != 1)
448 // "set param1 value1 param2 value2 ..."
449 std
::cerr
<< "Incorrect number of arguments for command 'set'" << std
::endl
;
451 for (std
::size_t i
= 1; i
< cmdSplit
.size(); i
+= 2)
455 m_overrideMap
[cmdSplit
[i
]] = std
::stod(cmdSplit
[i
+ 1]);
457 catch (const std
::exception
&e
)
459 std
::cerr
<< "std::stod failed" << std
::endl
;
463 else if (cmdSplit
[0] == "clear")
465 for (std
::size_t i
= 1; i
< cmdSplit
.size(); i
++)
467 if (cmdSplit
[i
] == "all")
469 m_overrideMap
.clear();
472 std
::size_t removed
= m_overrideMap
.erase(cmdSplit
[i
]);
475 std
::cerr
<< "Warning: key " << cmdSplit
[i
] << " not found" << std
::endl
;
481 std
::cerr
<< "Unknown command" << std
::endl
;
486 MouseCursorTracker
::~MouseCursorTracker()
489 m_getVolumeThread
.join();
490 m_parseCommandThread
.join();
491 pa_simple_free(m_pulse
);
494 void MouseCursorTracker
::stop(void)
499 MouseCursorTracker
::Params MouseCursorTracker
::getParams(void)
501 Params params
= Params();
503 int xOffset
= m_curPos
.x
- m_cfg
.origin
.x
;
504 int leftRange
= m_cfg
.origin
.x
- m_cfg
.left
;
505 int rightRange
= m_cfg
.right
- m_cfg
.origin
.x
;
507 if (xOffset
> 0) // i.e. to the right
509 params
.live2d
["ParamAngleX"] = 30.0 * xOffset
/ rightRange
;
513 params
.live2d
["ParamAngleX"] = 30.0 * xOffset
/ leftRange
;
516 int yOffset
= m_curPos
.y
- m_cfg
.origin
.y
;
517 int topRange
= m_cfg
.origin
.y
- m_cfg
.top
;
518 int bottomRange
= m_cfg
.bottom
- m_cfg
.origin
.y
;
520 if (yOffset
> 0) // downwards
522 params
.live2d
["ParamAngleY"] = -30.0 * yOffset
/ bottomRange
;
526 params
.live2d
["ParamAngleY"] = -30.0 * yOffset
/ topRange
;
529 params
.autoBlink
= m_cfg
.autoBlink
;
530 params
.autoBreath
= m_cfg
.autoBreath
;
531 params
.randomIdleMotion
= m_cfg
.randomIdleMotion
;
532 params
.useLipSync
= m_cfg
.useLipSync
;
534 params
.live2d
["ParamMouthForm"] = m_cfg
.mouthForm
;
536 if (m_cfg
.useLipSync
)
538 params
.lipSyncParam
= m_currentVol
* m_cfg
.lipSyncGain
;
539 if (params
.lipSyncParam
< m_cfg
.lipSyncCutOff
)
541 params
.lipSyncParam
= 0;
543 else if (params
.lipSyncParam
> 1)
545 params
.lipSyncParam
= 1;
549 // Don't block in getParams()
550 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::try_to_lock
);
551 if (lock
.owns_lock())
553 if (m_motionPriority
!= MotionPriority
::none
)
555 params
.motionPriority
= m_motionPriority
;
556 params
.motionGroup
= m_motionGroup
;
557 params
.motionNumber
= m_motionNumber
;
559 m_motionPriority
= MotionPriority
::none
;
563 params
.expression
= m_expression
;
568 // Leave everything else as zero
572 for (auto const &x
: m_overrideMap
)
574 std
::string key
= x
.first
;
575 double val
= x
.second
;
577 params
.live2d
[key
] = val
;
584 void MouseCursorTracker
::mainLoop(void)
592 xdo_get_mouse_location(m_xdo
, &x
, &y
, &screenNum
);
594 if (screenNum
== m_cfg
.screen
)
599 // else just silently ignore for now
600 std
::this_thread
::sleep_for(std
::chrono
::milliseconds(m_cfg
.sleepMs
));
604 void MouseCursorTracker
::parseConfig(std
::string cfgPath
)
606 populateDefaultConfig();
610 std
::ifstream
file(cfgPath
);
613 throw std
::runtime_error("Failed to open config file");
616 unsigned int lineNum
= 0;
618 while (std
::getline(file
, line
))
625 std
::istringstream
ss(line
);
626 std
::string paramName
;
630 if (paramName
== "sleep_ms")
632 if (!(ss
>> m_cfg
.sleepMs
))
634 throw std
::runtime_error("Error parsing sleep_ms");
637 else if (paramName
== "autoBlink")
639 if (!(ss
>> m_cfg
.autoBlink
))
641 throw std
::runtime_error("Error parsing autoBlink");
644 else if (paramName
== "autoBreath")
646 if (!(ss
>> m_cfg
.autoBreath
))
648 throw std
::runtime_error("Error parsing autoBreath");
651 else if (paramName
== "randomIdleMotion")
653 if (!(ss
>> m_cfg
.randomIdleMotion
))
655 throw std
::runtime_error("Error parsing randomIdleMotion");
658 else if (paramName
== "useLipSync")
660 if (!(ss
>> m_cfg
.useLipSync
))
662 throw std
::runtime_error("Error parsing useLipSync");
665 else if (paramName
== "lipSyncGain")
667 if (!(ss
>> m_cfg
.lipSyncGain
))
669 throw std
::runtime_error("Error parsing lipSyncGain");
672 else if (paramName
== "lipSyncCutOff")
674 if (!(ss
>> m_cfg
.lipSyncCutOff
))
676 throw std
::runtime_error("Error parsing lipSyncCutOff");
679 else if (paramName
== "audioBufSize")
681 if (!(ss
>> m_cfg
.audioBufSize
))
683 throw std
::runtime_error("Error parsing audioBufSize");
686 else if (paramName
== "mouthForm")
688 if (!(ss
>> m_cfg
.mouthForm
))
690 throw std
::runtime_error("Error parsing mouthForm");
693 else if (paramName
== "screen")
695 if (!(ss
>> m_cfg
.screen
))
697 throw std
::runtime_error("Error parsing screen");
700 else if (paramName
== "origin_x")
702 if (!(ss
>> m_cfg
.origin
.x
))
704 throw std
::runtime_error("Error parsing origin_x");
707 else if (paramName
== "origin_y")
709 if (!(ss
>> m_cfg
.origin
.y
))
711 throw std
::runtime_error("Error parsing origin_y");
714 else if (paramName
== "top")
716 if (!(ss
>> m_cfg
.top
))
718 throw std
::runtime_error("Error parsing top");
721 else if (paramName
== "bottom")
723 if (!(ss
>> m_cfg
.bottom
))
725 throw std
::runtime_error("Error parsing bottom");
728 else if (paramName
== "left")
730 if (!(ss
>> m_cfg
.left
))
732 throw std
::runtime_error("Error parsing left");
735 else if (paramName
== "right")
737 if (!(ss
>> m_cfg
.right
))
739 throw std
::runtime_error("Error parsing right");
744 throw std
::runtime_error("Unrecognized config parameter");
751 void MouseCursorTracker
::populateDefaultConfig(void)
754 m_cfg
.autoBlink
= true;
755 m_cfg
.autoBreath
= true;
756 m_cfg
.randomIdleMotion
= false;
757 m_cfg
.useLipSync
= true;
758 m_cfg
.lipSyncGain
= 10;
759 m_cfg
.lipSyncCutOff
= 0.15;
760 m_cfg
.audioBufSize
= 4096;
765 m_cfg
.right
= 1919; // These will be the full screen for 1920x1080
768 m_cfg
.origin
= {1600, 870}; // Somewhere near the bottom right