2 Copyright (c) 2020 Adrian I. Lam
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to deal
6 in the Software without restriction, including without limitation the rights
7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
43 #include <pulse/simple.h>
44 #include <readline/readline.h>
45 #include <readline/history.h>
47 #include "mouse_cursor_tracker.h"
49 static double rms(float *buf
, std
::size_t count
)
52 for (std
::size_t i
= 0; i
< count
; i
++)
54 sum
+= buf
[i
] * buf
[i
];
56 return std
::sqrt(sum
/ count
);
59 static std
::vector
<std
::string
> split(std
::string s
)
61 std
::vector
<std
::string
> v
;
64 for (std
::size_t i
= 0; i
< s
.length(); i
++)
88 /* Using readline callback functions means that we need to pass
89 * information from our class to the callback, and the only way
90 * to do so is using globals.
93 // Taken from https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/utils.cpp
94 static std
::string
longest_common_prefix(std
::string s
,
95 const std
::vector
<std
::string
>& candidates
) {
96 assert(candidates
.size() > 0);
97 if (candidates
.size() == 1) {
101 std
::string
prefix(s
);
103 // Each iteration of this loop advances to the next location in all the
104 // candidates and sees if they match up to it.
105 size_t nextloc
= prefix
.size();
106 auto i
= candidates
.begin();
107 if (i
->size() <= nextloc
) {
110 char nextchar
= (*(i
++))[nextloc
];
111 for (; i
!= candidates
.end(); ++i
) {
112 if (i
->size() <= nextloc
|| (*i
)[nextloc
] != nextchar
) {
113 // Bail out if there's a mismatch for this candidate.
117 // All candidates have contents[nextloc] == nextchar, so we can safely
118 // extend the prefix.
119 prefix
.append(1, nextchar
);
122 assert(0 && "unreachable");
125 std
::vector
<std
::string
> commands
=
127 "help", "motion", "expression", "set", "clear"
130 std
::vector
<std
::string
> live2dParams
=
131 { // https://docs.live2d.com/cubism-editor-manual/standard-parametor-list/?locale=ja
132 "ParamAngleX", "ParamAngleY", "ParamAngleZ",
133 "ParamEyeLOpen", "ParamEyeLSmile", "ParamEyeROpen", "ParamEyeRSmile",
134 "ParamEyeBallX", "ParamEyeBallY", "ParamEyeBallForm",
135 "ParamBrowLY", "ParamBrowRY", "ParamBrowLX", "ParamBrowRX",
136 "ParamBrowLAngle", "ParamBrowRAngle", "ParamBrowLForm", "ParamBrowRForm",
137 "ParamMouthForm", "ParamMouthOpenY", "ParamTere",
138 "ParamBodyAngleX", "ParamBodyAngleY", "ParamBodyAngleZ", "ParamBreath",
139 "ParamArmLA", "ParamArmRA", "ParamArmLB", "ParamArmRB",
140 "ParamHandL", "ParamHandR",
141 "ParamHairFront", "ParamHairSide", "ParamHairBack", "ParamHairFluffy",
142 "ParamShoulderY", "ParamBustX", "ParamBustY",
143 "ParamBaseX", "ParamBaseY"
145 std
::vector
<std
::pair
<std
::string
, int> > MCT_motions
;
146 std
::vector
<std
::string
> MCT_expressions
;
147 std
::map
<std
::string
, double> *MCT_overrideMap
;
149 static char **cliCompletionFunction(const char *textCStr
, int start
, int end
)
151 // Reference: https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/readline-complete-subcommand.cpp
152 rl_attempted_completion_over
= 1;
154 std
::string
line(rl_line_buffer
);
156 std
::vector
<std
::string
> cmdline
= split(line
);
158 std
::vector
<std
::string
> constructed
;
159 std
::vector
<std
::string
> *vocab
= nullptr;
161 if (cmdline
.size() == 0 ||
162 (cmdline
.size() == 1 && line
.back() != ' ') ||
163 cmdline
[0] == "help")
167 else if (cmdline
[0] == "motion")
169 for (auto it
= MCT_motions
.begin(); it
!= MCT_motions
.end(); ++it
)
171 if ((cmdline
.size() == 1 && line
.back() == ' ') ||
172 (cmdline
.size() == 2 && line
.back() != ' '))
175 constructed
.push_back(it
->first
);
178 else if ((cmdline
.size() == 2 && line
.back() == ' ') ||
179 (cmdline
.size() == 3 && line
.back() != ' '))
181 if (it
->first
== cmdline
[1])
183 for (int i
= 0; i
< it
->second
; i
++)
185 constructed
.push_back(std
::to_string(i
));
190 else if (cmdline
.size() <= 4)
192 for (int i
= 0; i
< 4; i
++)
194 constructed
.push_back(std
::to_string(i
));
199 vocab
= &constructed
;
201 else if (cmdline
[0] == "expression")
203 vocab
= &MCT_expressions
;
205 else if (cmdline
[0] == "set")
207 if ((cmdline
.size() % 2 == 0 && line
.back() != ' ') ||
208 (cmdline
.size() % 2 == 1 && line
.back() == ' '))
210 vocab
= &live2dParams
;
213 else if (cmdline
[0] == "clear")
215 constructed
.push_back("all");
216 for (auto const &entry
: *MCT_overrideMap
)
218 constructed
.push_back(entry
.first
);
220 vocab
= &constructed
;
228 std
::string
text(textCStr
);
229 std
::vector
<std
::string
> matches
;
230 std
::copy_if(vocab
->begin(), vocab
->end(), std
::back_inserter(matches
),
231 [&text
](const std
::string
&s
)
233 return (s
.size() >= text
.size() &&
234 s
.compare(0, text
.size(), text
) == 0);
243 static_cast<char**>(malloc((2 + matches
.size()) * sizeof(*array
)));
244 array
[0] = strdup(longest_common_prefix(text
, matches
).c_str());
246 for (const auto& m
: matches
) {
247 array
[ptr
++] = strdup(m
.c_str());
249 array
[ptr
] = nullptr;
253 MouseCursorTracker
::MouseCursorTracker(std
::string cfgPath
,
254 std
::vector
<std
::pair
<std
::string
, int> > motions
,
255 std
::vector
<std
::string
> expressions
)
258 m_motionPriority
= MotionPriority
::none
;
262 parseConfig(cfgPath
);
263 m_xdo
= xdo_new(nullptr);
265 const pa_sample_spec ss
=
267 .format
= PA_SAMPLE_FLOAT32NE
,
271 m_pulse
= pa_simple_new(nullptr, "MouseCursorTracker", PA_STREAM_RECORD
,
272 nullptr, "LipSync", &ss
, nullptr, nullptr, nullptr);
275 throw std
::runtime_error("Unable to create pulse");
278 m_getVolumeThread
= std
::thread(&MouseCursorTracker
::audioLoop
, this);
279 m_parseCommandThread
= std
::thread(&MouseCursorTracker
::cliLoop
, this);
281 MCT_motions
= motions
;
282 MCT_expressions
= expressions
;
283 MCT_overrideMap
= &m_overrideMap
;
286 void MouseCursorTracker
::audioLoop(void)
288 float *buf
= new float[m_cfg
.audioBufSize
];
290 std
::size_t audioBufByteSize
= m_cfg
.audioBufSize
* sizeof *buf
;
294 if (pa_simple_read(m_pulse
, buf
, audioBufByteSize
, nullptr) < 0)
296 throw std
::runtime_error("Unable to get audio data");
298 m_currentVol
= rms(buf
, m_cfg
.audioBufSize
);
304 void MouseCursorTracker
::cliLoop(void)
306 rl_catch_signals
= 0;
307 rl_attempted_completion_function
= cliCompletionFunction
;
310 char *buf
= readline(">> ");
314 std
::string
cmdline(buf
);
316 processCommand(cmdline
);
320 std
::cout
<< "Exiting CLI loop. Use Ctrl+C to exit the whole process." << std
::endl
;
327 void MouseCursorTracker
::processCommand(std
::string cmdline
)
329 auto cmdSplit
= split(cmdline
);
331 if (cmdSplit
.size() > 0)
333 add_history(cmdline
.c_str());
335 if (cmdSplit
[0] == "help")
337 if (cmdSplit
.size() == 1)
339 std
::cout
<< "Available commands: motion set clear\n"
340 << "Type \"help <command>\" for more help" << std
::endl
;
342 else if (cmdSplit
[1] == "motion")
344 std
::cout
<< "motion <motionGroup> <motionNumber> [<priority>]\n"
345 << "motionGroup: The motion name in the .model3.json file\n"
346 << "motionNumber: The index of this motion in the .model3.json file, 0-indexed\n"
347 << "priority: 0 = none, 1 = idle, 2 = normal, 3 = force (default normal)" << std
::endl
;
349 else if (cmdSplit
[1] == "expression")
351 std
::cout
<< "expression <expressionName>\n"
352 << "expressionName: Name of expression in the .model3.json file" << std
::endl
;
354 else if (cmdSplit
[1] == "set")
356 std
::cout
<< "set <param1> <value1> [<param2> <value2> ...]\n"
357 << "Set parameter value. Overrides any tracking."
358 << "See live2D documentation for full list of params" << std
::endl
;
360 else if (cmdSplit
[1] == "clear")
362 std
::cout
<< "clear <param1> [<param2> ...]\n"
363 << "Clear parameter value. Re-enables tracking if it was overridden by \"set\"\n"
364 << "You can also use \"clear all\" to clear everything" << std
::endl
;
368 std
::cout
<< "Unrecognized command" << std
::endl
;
371 else if (cmdSplit
[0] == "motion")
373 if (cmdSplit
.size() == 3 || cmdSplit
.size() == 4)
375 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::defer_lock
);
377 m_motionGroup
= cmdSplit
[1];
380 m_motionNumber
= std
::stoi(cmdSplit
[2]);
381 if (cmdSplit
.size() == 4)
383 m_motionPriority
= static_cast<MotionPriority
>(std
::stoi(cmdSplit
[3]));
387 m_motionPriority
= MotionPriority
::normal
;
390 catch (const std
::exception
&e
)
392 std
::cerr
<< "std::stoi failed" << std
::endl
;
398 std
::cerr
<< "Incorrect command, expecting 2 or 3 arguments" << std
::endl
;
399 std
::cerr
<< "motion motionGroup motionNumber [motionPriority]" << std
::endl
;
402 else if (cmdSplit
[0] == "expression")
404 if (cmdSplit
.size() == 2)
406 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::defer_lock
);
408 m_expression
= cmdSplit
[1];
413 std
::cerr
<< "Incorrect command, expecting 1 argument: expressionName" << std
::endl
;
416 else if (cmdSplit
[0] == "set")
418 if (cmdSplit
.size() % 2 != 1)
420 // "set param1 value1 param2 value2 ..."
421 std
::cerr
<< "Incorrect number of arguments for command 'set'" << std
::endl
;
423 for (std
::size_t i
= 1; i
< cmdSplit
.size(); i
+= 2)
427 m_overrideMap
[cmdSplit
[i
]] = std
::stod(cmdSplit
[i
+ 1]);
429 catch (const std
::exception
&e
)
431 std
::cerr
<< "std::stod failed" << std
::endl
;
434 std
::cerr
<< "Debug: setting " << cmdSplit
[i
] << std
::endl
;
437 else if (cmdSplit
[0] == "clear")
439 for (std
::size_t i
= 1; i
< cmdSplit
.size(); i
++)
441 if (cmdSplit
[i
] == "all")
443 m_overrideMap
.clear();
446 std
::size_t removed
= m_overrideMap
.erase(cmdSplit
[i
]);
449 std
::cerr
<< "Warning: key " << cmdSplit
[i
] << " not found" << std
::endl
;
455 std
::cerr
<< "Unknown command" << std
::endl
;
460 MouseCursorTracker
::~MouseCursorTracker()
463 m_getVolumeThread
.join();
464 m_parseCommandThread
.join();
465 pa_simple_free(m_pulse
);
468 void MouseCursorTracker
::stop(void)
473 MouseCursorTracker
::Params MouseCursorTracker
::getParams(void)
475 Params params
= Params();
477 int xOffset
= m_curPos
.x
- m_cfg
.origin
.x
;
478 int leftRange
= m_cfg
.origin
.x
- m_cfg
.left
;
479 int rightRange
= m_cfg
.right
- m_cfg
.origin
.x
;
481 if (xOffset
> 0) // i.e. to the right
483 params
.live2d
["ParamAngleX"] = 30.0 * xOffset
/ rightRange
;
487 params
.live2d
["ParamAngleX"] = 30.0 * xOffset
/ leftRange
;
490 int yOffset
= m_curPos
.y
- m_cfg
.origin
.y
;
491 int topRange
= m_cfg
.origin
.y
- m_cfg
.top
;
492 int bottomRange
= m_cfg
.bottom
- m_cfg
.origin
.y
;
494 if (yOffset
> 0) // downwards
496 params
.live2d
["ParamAngleY"] = -30.0 * yOffset
/ bottomRange
;
500 params
.live2d
["ParamAngleY"] = -30.0 * yOffset
/ topRange
;
503 params
.autoBlink
= m_cfg
.autoBlink
;
504 params
.autoBreath
= m_cfg
.autoBreath
;
505 params
.randomIdleMotion
= m_cfg
.randomIdleMotion
;
506 params
.useLipSync
= m_cfg
.useLipSync
;
508 params
.live2d
["ParamMouthForm"] = m_cfg
.mouthForm
;
510 if (m_cfg
.useLipSync
)
512 params
.lipSyncParam
= m_currentVol
* m_cfg
.lipSyncGain
;
513 if (params
.lipSyncParam
< m_cfg
.lipSyncCutOff
)
515 params
.lipSyncParam
= 0;
517 else if (params
.lipSyncParam
> 1)
519 params
.lipSyncParam
= 1;
523 // Don't block in getParams()
524 std
::unique_lock
<std
::mutex
> lock(m_motionMutex
, std
::try_to_lock
);
525 if (lock
.owns_lock())
527 if (m_motionPriority
!= MotionPriority
::none
)
529 params
.motionPriority
= m_motionPriority
;
530 params
.motionGroup
= m_motionGroup
;
531 params
.motionNumber
= m_motionNumber
;
533 m_motionPriority
= MotionPriority
::none
;
537 params
.expression
= m_expression
;
542 // Leave everything else as zero
546 for (auto const &x
: m_overrideMap
)
548 std
::string key
= x
.first
;
549 double val
= x
.second
;
551 params
.live2d
[key
] = val
;
558 void MouseCursorTracker
::mainLoop(void)
566 xdo_get_mouse_location(m_xdo
, &x
, &y
, &screenNum
);
568 if (screenNum
== m_cfg
.screen
)
573 // else just silently ignore for now
574 std
::this_thread
::sleep_for(std
::chrono
::milliseconds(m_cfg
.sleepMs
));
578 void MouseCursorTracker
::parseConfig(std
::string cfgPath
)
580 populateDefaultConfig();
584 std
::ifstream
file(cfgPath
);
587 throw std
::runtime_error("Failed to open config file");
590 unsigned int lineNum
= 0;
592 while (std
::getline(file
, line
))
599 std
::istringstream
ss(line
);
600 std
::string paramName
;
604 if (paramName
== "sleep_ms")
606 if (!(ss
>> m_cfg
.sleepMs
))
608 throw std
::runtime_error("Error parsing sleep_ms");
611 else if (paramName
== "autoBlink")
613 if (!(ss
>> m_cfg
.autoBlink
))
615 throw std
::runtime_error("Error parsing autoBlink");
618 else if (paramName
== "autoBreath")
620 if (!(ss
>> m_cfg
.autoBreath
))
622 throw std
::runtime_error("Error parsing autoBreath");
625 else if (paramName
== "randomIdleMotion")
627 if (!(ss
>> m_cfg
.randomIdleMotion
))
629 throw std
::runtime_error("Error parsing randomIdleMotion");
632 else if (paramName
== "useLipSync")
634 if (!(ss
>> m_cfg
.useLipSync
))
636 throw std
::runtime_error("Error parsing useLipSync");
639 else if (paramName
== "lipSyncGain")
641 if (!(ss
>> m_cfg
.lipSyncGain
))
643 throw std
::runtime_error("Error parsing lipSyncGain");
646 else if (paramName
== "lipSyncCutOff")
648 if (!(ss
>> m_cfg
.lipSyncCutOff
))
650 throw std
::runtime_error("Error parsing lipSyncCutOff");
653 else if (paramName
== "audioBufSize")
655 if (!(ss
>> m_cfg
.audioBufSize
))
657 throw std
::runtime_error("Error parsing audioBufSize");
660 else if (paramName
== "mouthForm")
662 if (!(ss
>> m_cfg
.mouthForm
))
664 throw std
::runtime_error("Error parsing mouthForm");
667 else if (paramName
== "screen")
669 if (!(ss
>> m_cfg
.screen
))
671 throw std
::runtime_error("Error parsing screen");
674 else if (paramName
== "origin_x")
676 if (!(ss
>> m_cfg
.origin
.x
))
678 throw std
::runtime_error("Error parsing origin_x");
681 else if (paramName
== "origin_y")
683 if (!(ss
>> m_cfg
.origin
.y
))
685 throw std
::runtime_error("Error parsing origin_y");
688 else if (paramName
== "top")
690 if (!(ss
>> m_cfg
.top
))
692 throw std
::runtime_error("Error parsing top");
695 else if (paramName
== "bottom")
697 if (!(ss
>> m_cfg
.bottom
))
699 throw std
::runtime_error("Error parsing bottom");
702 else if (paramName
== "left")
704 if (!(ss
>> m_cfg
.left
))
706 throw std
::runtime_error("Error parsing left");
709 else if (paramName
== "right")
711 if (!(ss
>> m_cfg
.right
))
713 throw std
::runtime_error("Error parsing right");
718 throw std
::runtime_error("Unrecognized config parameter");
725 void MouseCursorTracker
::populateDefaultConfig(void)
728 m_cfg
.autoBlink
= true;
729 m_cfg
.autoBreath
= true;
730 m_cfg
.randomIdleMotion
= false;
731 m_cfg
.useLipSync
= true;
732 m_cfg
.lipSyncGain
= 10;
733 m_cfg
.lipSyncCutOff
= 0.15;
734 m_cfg
.audioBufSize
= 4096;
739 m_cfg
.right
= 1919; // These will be the full screen for 1920x1080
742 m_cfg
.origin
= {1600, 870}; // Somewhere near the bottom right