Add command-line interface to control model
[mouse-tracker-for-cubism.git] / src / mouse_cursor_tracker.cpp
1 /****
2 Copyright (c) 2020 Adrian I. Lam
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to deal
6 in the Software without restriction, including without limitation the rights
7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 SOFTWARE.
21 ****/
22
23 #include <stdexcept>
24 #include <string>
25 #include <chrono>
26 #include <thread>
27 #include <fstream>
28 #include <sstream>
29 #include <vector>
30 #include <utility>
31 #include <algorithm>
32 #include <cstdlib>
33 #include <cmath>
34 #include <cassert>
35
36 #include <iostream>
37 #include <cctype>
38 #include <mutex>
39
40 extern "C"
41 {
42 #include <xdo.h>
43 #include <pulse/simple.h>
44 #include <readline/readline.h>
45 #include <readline/history.h>
46 }
47 #include "mouse_cursor_tracker.h"
48
49 static double rms(float *buf, std::size_t count)
50 {
51 double sum = 0;
52 for (std::size_t i = 0; i < count; i++)
53 {
54 sum += buf[i] * buf[i];
55 }
56 return std::sqrt(sum / count);
57 }
58
59 static std::vector<std::string> split(std::string s)
60 {
61 std::vector<std::string> v;
62 std::string tmp;
63
64 for (std::size_t i = 0; i < s.length(); i++)
65 {
66 char c = s[i];
67 if (std::isspace(c))
68 {
69 if (tmp != "")
70 {
71 v.push_back(tmp);
72 tmp = "";
73 }
74 }
75 else
76 {
77 tmp += c;
78 }
79 }
80 if (tmp != "")
81 {
82 v.push_back(tmp);
83 }
84 return v;
85 }
86
87
88 /* Using readline callback functions means that we need to pass
89 * information from our class to the callback, and the only way
90 * to do so is using globals.
91 */
92
93 // Taken from https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/utils.cpp
94 static std::string longest_common_prefix(std::string s,
95 const std::vector<std::string>& candidates) {
96 assert(candidates.size() > 0);
97 if (candidates.size() == 1) {
98 return candidates[0];
99 }
100
101 std::string prefix(s);
102 while (true) {
103 // Each iteration of this loop advances to the next location in all the
104 // candidates and sees if they match up to it.
105 size_t nextloc = prefix.size();
106 auto i = candidates.begin();
107 if (i->size() <= nextloc) {
108 return prefix;
109 }
110 char nextchar = (*(i++))[nextloc];
111 for (; i != candidates.end(); ++i) {
112 if (i->size() <= nextloc || (*i)[nextloc] != nextchar) {
113 // Bail out if there's a mismatch for this candidate.
114 return prefix;
115 }
116 }
117 // All candidates have contents[nextloc] == nextchar, so we can safely
118 // extend the prefix.
119 prefix.append(1, nextchar);
120 }
121
122 assert(0 && "unreachable");
123 }
124
125 std::vector<std::string> commands =
126 {
127 "help", "motion", "expression", "set", "clear"
128 };
129
130 std::vector<std::string> live2dParams =
131 { // https://docs.live2d.com/cubism-editor-manual/standard-parametor-list/?locale=ja
132 "ParamAngleX", "ParamAngleY", "ParamAngleZ",
133 "ParamEyeLOpen", "ParamEyeLSmile", "ParamEyeROpen", "ParamEyeRSmile",
134 "ParamEyeBallX", "ParamEyeBallY", "ParamEyeBallForm",
135 "ParamBrowLY", "ParamBrowRY", "ParamBrowLX", "ParamBrowRX",
136 "ParamBrowLAngle", "ParamBrowRAngle", "ParamBrowLForm", "ParamBrowRForm",
137 "ParamMouthForm", "ParamMouthOpenY", "ParamTere",
138 "ParamBodyAngleX", "ParamBodyAngleY", "ParamBodyAngleZ", "ParamBreath",
139 "ParamArmLA", "ParamArmRA", "ParamArmLB", "ParamArmRB",
140 "ParamHandL", "ParamHandR",
141 "ParamHairFront", "ParamHairSide", "ParamHairBack", "ParamHairFluffy",
142 "ParamShoulderY", "ParamBustX", "ParamBustY",
143 "ParamBaseX", "ParamBaseY"
144 };
145 std::vector<std::pair<std::string, int> > MCT_motions;
146 std::vector<std::string> MCT_expressions;
147 std::map<std::string, double> *MCT_overrideMap;
148
149 static char **cliCompletionFunction(const char *textCStr, int start, int end)
150 {
151 // Reference: https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/readline-complete-subcommand.cpp
152 rl_attempted_completion_over = 1;
153
154 std::string line(rl_line_buffer);
155
156 std::vector<std::string> cmdline = split(line);
157
158 std::vector<std::string> constructed;
159 std::vector<std::string> *vocab = nullptr;
160
161 if (cmdline.size() == 0 ||
162 (cmdline.size() == 1 && line.back() != ' ') ||
163 cmdline[0] == "help")
164 {
165 vocab = &commands;
166 }
167 else if (cmdline[0] == "motion")
168 {
169 for (auto it = MCT_motions.begin(); it != MCT_motions.end(); ++it)
170 {
171 if ((cmdline.size() == 1 && line.back() == ' ') ||
172 (cmdline.size() == 2 && line.back() != ' '))
173 { // motionGroup
174 {
175 constructed.push_back(it->first);
176 }
177 }
178 else if ((cmdline.size() == 2 && line.back() == ' ') ||
179 (cmdline.size() == 3 && line.back() != ' '))
180 { // motionNumber
181 if (it->first == cmdline[1])
182 {
183 for (int i = 0; i < it->second; i++)
184 {
185 constructed.push_back(std::to_string(i));
186 }
187 break;
188 }
189 }
190 else if (cmdline.size() <= 4)
191 { // priority
192 for (int i = 0; i < 4; i++)
193 {
194 constructed.push_back(std::to_string(i));
195 }
196 }
197 }
198
199 vocab = &constructed;
200 }
201 else if (cmdline[0] == "expression")
202 {
203 vocab = &MCT_expressions;
204 }
205 else if (cmdline[0] == "set")
206 {
207 if ((cmdline.size() % 2 == 0 && line.back() != ' ') ||
208 (cmdline.size() % 2 == 1 && line.back() == ' '))
209 {
210 vocab = &live2dParams;
211 }
212 }
213 else if (cmdline[0] == "clear")
214 {
215 constructed.push_back("all");
216 for (auto const &entry : *MCT_overrideMap)
217 {
218 constructed.push_back(entry.first);
219 }
220 vocab = &constructed;
221 }
222
223 if (!vocab)
224 {
225 return nullptr;
226 }
227
228 std::string text(textCStr);
229 std::vector<std::string> matches;
230 std::copy_if(vocab->begin(), vocab->end(), std::back_inserter(matches),
231 [&text](const std::string &s)
232 {
233 return (s.size() >= text.size() &&
234 s.compare(0, text.size(), text) == 0);
235 });
236
237 if (matches.empty())
238 {
239 return nullptr;
240 }
241
242 char** array =
243 static_cast<char**>(malloc((2 + matches.size()) * sizeof(*array)));
244 array[0] = strdup(longest_common_prefix(text, matches).c_str());
245 size_t ptr = 1;
246 for (const auto& m : matches) {
247 array[ptr++] = strdup(m.c_str());
248 }
249 array[ptr] = nullptr;
250 return array;
251 }
252
253 MouseCursorTracker::MouseCursorTracker(std::string cfgPath,
254 std::vector<std::pair<std::string, int> > motions,
255 std::vector<std::string> expressions)
256 : m_stop(false)
257 {
258 m_motionPriority = MotionPriority::none;
259 m_motionNumber = 0;
260
261
262 parseConfig(cfgPath);
263 m_xdo = xdo_new(nullptr);
264
265 const pa_sample_spec ss =
266 {
267 .format = PA_SAMPLE_FLOAT32NE,
268 .rate = 44100,
269 .channels = 2
270 };
271 m_pulse = pa_simple_new(nullptr, "MouseCursorTracker", PA_STREAM_RECORD,
272 nullptr, "LipSync", &ss, nullptr, nullptr, nullptr);
273 if (!m_pulse)
274 {
275 throw std::runtime_error("Unable to create pulse");
276 }
277
278 m_getVolumeThread = std::thread(&MouseCursorTracker::audioLoop, this);
279 m_parseCommandThread = std::thread(&MouseCursorTracker::cliLoop, this);
280
281 MCT_motions = motions;
282 MCT_expressions = expressions;
283 MCT_overrideMap = &m_overrideMap;
284 }
285
286 void MouseCursorTracker::audioLoop(void)
287 {
288 float *buf = new float[m_cfg.audioBufSize];
289
290 std::size_t audioBufByteSize = m_cfg.audioBufSize * sizeof *buf;
291
292 while (!m_stop)
293 {
294 if (pa_simple_read(m_pulse, buf, audioBufByteSize, nullptr) < 0)
295 {
296 throw std::runtime_error("Unable to get audio data");
297 }
298 m_currentVol = rms(buf, m_cfg.audioBufSize);
299 }
300
301 delete[] buf;
302 }
303
304 void MouseCursorTracker::cliLoop(void)
305 {
306 rl_catch_signals = 0;
307 rl_attempted_completion_function = cliCompletionFunction;
308 while (!m_stop)
309 {
310 char *buf = readline(">> ");
311
312 if (buf)
313 {
314 std::string cmdline(buf);
315 free(buf);
316 processCommand(cmdline);
317 }
318 else
319 {
320 std::cout << "Exiting CLI loop. Use Ctrl+C to exit the whole process." << std::endl;
321 stop();
322 }
323 }
324 }
325
326
327 void MouseCursorTracker::processCommand(std::string cmdline)
328 {
329 auto cmdSplit = split(cmdline);
330
331 if (cmdSplit.size() > 0)
332 {
333 add_history(cmdline.c_str());
334
335 if (cmdSplit[0] == "help")
336 {
337 if (cmdSplit.size() == 1)
338 {
339 std::cout << "Available commands: motion set clear\n"
340 << "Type \"help <command>\" for more help" << std::endl;
341 }
342 else if (cmdSplit[1] == "motion")
343 {
344 std::cout << "motion <motionGroup> <motionNumber> [<priority>]\n"
345 << "motionGroup: The motion name in the .model3.json file\n"
346 << "motionNumber: The index of this motion in the .model3.json file, 0-indexed\n"
347 << "priority: 0 = none, 1 = idle, 2 = normal, 3 = force (default normal)" << std::endl;
348 }
349 else if (cmdSplit[1] == "expression")
350 {
351 std::cout << "expression <expressionName>\n"
352 << "expressionName: Name of expression in the .model3.json file" << std::endl;
353 }
354 else if (cmdSplit[1] == "set")
355 {
356 std::cout << "set <param1> <value1> [<param2> <value2> ...]\n"
357 << "Set parameter value. Overrides any tracking."
358 << "See live2D documentation for full list of params" << std::endl;
359 }
360 else if (cmdSplit[1] == "clear")
361 {
362 std::cout << "clear <param1> [<param2> ...]\n"
363 << "Clear parameter value. Re-enables tracking if it was overridden by \"set\"\n"
364 << "You can also use \"clear all\" to clear everything" << std::endl;
365 }
366 else
367 {
368 std::cout << "Unrecognized command" << std::endl;
369 }
370 }
371 else if (cmdSplit[0] == "motion")
372 {
373 if (cmdSplit.size() == 3 || cmdSplit.size() == 4)
374 {
375 std::unique_lock<std::mutex> lock(m_motionMutex, std::defer_lock);
376 lock.lock();
377 m_motionGroup = cmdSplit[1];
378 try
379 {
380 m_motionNumber = std::stoi(cmdSplit[2]);
381 if (cmdSplit.size() == 4)
382 {
383 m_motionPriority = static_cast<MotionPriority>(std::stoi(cmdSplit[3]));
384 }
385 else
386 {
387 m_motionPriority = MotionPriority::normal;
388 }
389 }
390 catch (const std::exception &e)
391 {
392 std::cerr << "std::stoi failed" << std::endl;
393 }
394 lock.unlock();
395 }
396 else
397 {
398 std::cerr << "Incorrect command, expecting 2 or 3 arguments" << std::endl;
399 std::cerr << "motion motionGroup motionNumber [motionPriority]" << std::endl;
400 }
401 }
402 else if (cmdSplit[0] == "expression")
403 {
404 if (cmdSplit.size() == 2)
405 {
406 std::unique_lock<std::mutex> lock(m_motionMutex, std::defer_lock);
407 lock.lock();
408 m_expression = cmdSplit[1];
409 lock.unlock();
410 }
411 else
412 {
413 std::cerr << "Incorrect command, expecting 1 argument: expressionName" << std::endl;
414 }
415 }
416 else if (cmdSplit[0] == "set")
417 {
418 if (cmdSplit.size() % 2 != 1)
419 {
420 // "set param1 value1 param2 value2 ..."
421 std::cerr << "Incorrect number of arguments for command 'set'" << std::endl;
422 }
423 for (std::size_t i = 1; i < cmdSplit.size(); i += 2)
424 {
425 try
426 {
427 m_overrideMap[cmdSplit[i]] = std::stod(cmdSplit[i + 1]);
428 }
429 catch (const std::exception &e)
430 {
431 std::cerr << "std::stod failed" << std::endl;
432 }
433
434 std::cerr << "Debug: setting " << cmdSplit[i] << std::endl;
435 }
436 }
437 else if (cmdSplit[0] == "clear")
438 {
439 for (std::size_t i = 1; i < cmdSplit.size(); i++)
440 {
441 if (cmdSplit[i] == "all")
442 {
443 m_overrideMap.clear();
444 break;
445 }
446 std::size_t removed = m_overrideMap.erase(cmdSplit[i]);
447 if (removed == 0)
448 {
449 std::cerr << "Warning: key " << cmdSplit[i] << " not found" << std::endl;
450 }
451 }
452 }
453 else
454 {
455 std::cerr << "Unknown command" << std::endl;
456 }
457 }
458 }
459
460 MouseCursorTracker::~MouseCursorTracker()
461 {
462 xdo_free(m_xdo);
463 m_getVolumeThread.join();
464 m_parseCommandThread.join();
465 pa_simple_free(m_pulse);
466 }
467
468 void MouseCursorTracker::stop(void)
469 {
470 m_stop = true;
471 }
472
473 MouseCursorTracker::Params MouseCursorTracker::getParams(void)
474 {
475 Params params = Params();
476
477 int xOffset = m_curPos.x - m_cfg.origin.x;
478 int leftRange = m_cfg.origin.x - m_cfg.left;
479 int rightRange = m_cfg.right - m_cfg.origin.x;
480
481 if (xOffset > 0) // i.e. to the right
482 {
483 params.live2d["ParamAngleX"] = 30.0 * xOffset / rightRange;
484 }
485 else // to the left
486 {
487 params.live2d["ParamAngleX"] = 30.0 * xOffset / leftRange;
488 }
489
490 int yOffset = m_curPos.y - m_cfg.origin.y;
491 int topRange = m_cfg.origin.y - m_cfg.top;
492 int bottomRange = m_cfg.bottom - m_cfg.origin.y;
493
494 if (yOffset > 0) // downwards
495 {
496 params.live2d["ParamAngleY"] = -30.0 * yOffset / bottomRange;
497 }
498 else // upwards
499 {
500 params.live2d["ParamAngleY"] = -30.0 * yOffset / topRange;
501 }
502
503 params.autoBlink = m_cfg.autoBlink;
504 params.autoBreath = m_cfg.autoBreath;
505 params.randomIdleMotion = m_cfg.randomIdleMotion;
506 params.useLipSync = m_cfg.useLipSync;
507
508 params.live2d["ParamMouthForm"] = m_cfg.mouthForm;
509
510 if (m_cfg.useLipSync)
511 {
512 params.lipSyncParam = m_currentVol * m_cfg.lipSyncGain;
513 if (params.lipSyncParam < m_cfg.lipSyncCutOff)
514 {
515 params.lipSyncParam = 0;
516 }
517 else if (params.lipSyncParam > 1)
518 {
519 params.lipSyncParam = 1;
520 }
521 }
522
523 // Don't block in getParams()
524 std::unique_lock<std::mutex> lock(m_motionMutex, std::try_to_lock);
525 if (lock.owns_lock())
526 {
527 if (m_motionPriority != MotionPriority::none)
528 {
529 params.motionPriority = m_motionPriority;
530 params.motionGroup = m_motionGroup;
531 params.motionNumber = m_motionNumber;
532
533 m_motionPriority = MotionPriority::none;
534 m_motionGroup = "";
535 m_motionNumber = 0;
536 }
537 params.expression = m_expression;
538 m_expression = "";
539 lock.unlock();
540 }
541
542 // Leave everything else as zero
543
544
545 // Process overrides
546 for (auto const &x : m_overrideMap)
547 {
548 std::string key = x.first;
549 double val = x.second;
550
551 params.live2d[key] = val;
552 }
553
554
555 return params;
556 }
557
558 void MouseCursorTracker::mainLoop(void)
559 {
560 while (!m_stop)
561 {
562 int x;
563 int y;
564 int screenNum;
565
566 xdo_get_mouse_location(m_xdo, &x, &y, &screenNum);
567
568 if (screenNum == m_cfg.screen)
569 {
570 m_curPos.x = x;
571 m_curPos.y = y;
572 }
573 // else just silently ignore for now
574 std::this_thread::sleep_for(std::chrono::milliseconds(m_cfg.sleepMs));
575 }
576 }
577
578 void MouseCursorTracker::parseConfig(std::string cfgPath)
579 {
580 populateDefaultConfig();
581
582 if (cfgPath != "")
583 {
584 std::ifstream file(cfgPath);
585 if (!file)
586 {
587 throw std::runtime_error("Failed to open config file");
588 }
589 std::string line;
590 unsigned int lineNum = 0;
591
592 while (std::getline(file, line))
593 {
594 if (line[0] == '#')
595 {
596 continue;
597 }
598
599 std::istringstream ss(line);
600 std::string paramName;
601
602 if (ss >> paramName)
603 {
604 if (paramName == "sleep_ms")
605 {
606 if (!(ss >> m_cfg.sleepMs))
607 {
608 throw std::runtime_error("Error parsing sleep_ms");
609 }
610 }
611 else if (paramName == "autoBlink")
612 {
613 if (!(ss >> m_cfg.autoBlink))
614 {
615 throw std::runtime_error("Error parsing autoBlink");
616 }
617 }
618 else if (paramName == "autoBreath")
619 {
620 if (!(ss >> m_cfg.autoBreath))
621 {
622 throw std::runtime_error("Error parsing autoBreath");
623 }
624 }
625 else if (paramName == "randomIdleMotion")
626 {
627 if (!(ss >> m_cfg.randomIdleMotion))
628 {
629 throw std::runtime_error("Error parsing randomIdleMotion");
630 }
631 }
632 else if (paramName == "useLipSync")
633 {
634 if (!(ss >> m_cfg.useLipSync))
635 {
636 throw std::runtime_error("Error parsing useLipSync");
637 }
638 }
639 else if (paramName == "lipSyncGain")
640 {
641 if (!(ss >> m_cfg.lipSyncGain))
642 {
643 throw std::runtime_error("Error parsing lipSyncGain");
644 }
645 }
646 else if (paramName == "lipSyncCutOff")
647 {
648 if (!(ss >> m_cfg.lipSyncCutOff))
649 {
650 throw std::runtime_error("Error parsing lipSyncCutOff");
651 }
652 }
653 else if (paramName == "audioBufSize")
654 {
655 if (!(ss >> m_cfg.audioBufSize))
656 {
657 throw std::runtime_error("Error parsing audioBufSize");
658 }
659 }
660 else if (paramName == "mouthForm")
661 {
662 if (!(ss >> m_cfg.mouthForm))
663 {
664 throw std::runtime_error("Error parsing mouthForm");
665 }
666 }
667 else if (paramName == "screen")
668 {
669 if (!(ss >> m_cfg.screen))
670 {
671 throw std::runtime_error("Error parsing screen");
672 }
673 }
674 else if (paramName == "origin_x")
675 {
676 if (!(ss >> m_cfg.origin.x))
677 {
678 throw std::runtime_error("Error parsing origin_x");
679 }
680 }
681 else if (paramName == "origin_y")
682 {
683 if (!(ss >> m_cfg.origin.y))
684 {
685 throw std::runtime_error("Error parsing origin_y");
686 }
687 }
688 else if (paramName == "top")
689 {
690 if (!(ss >> m_cfg.top))
691 {
692 throw std::runtime_error("Error parsing top");
693 }
694 }
695 else if (paramName == "bottom")
696 {
697 if (!(ss >> m_cfg.bottom))
698 {
699 throw std::runtime_error("Error parsing bottom");
700 }
701 }
702 else if (paramName == "left")
703 {
704 if (!(ss >> m_cfg.left))
705 {
706 throw std::runtime_error("Error parsing left");
707 }
708 }
709 else if (paramName == "right")
710 {
711 if (!(ss >> m_cfg.right))
712 {
713 throw std::runtime_error("Error parsing right");
714 }
715 }
716 else
717 {
718 throw std::runtime_error("Unrecognized config parameter");
719 }
720 }
721 }
722 }
723 }
724
725 void MouseCursorTracker::populateDefaultConfig(void)
726 {
727 m_cfg.sleepMs = 5;
728 m_cfg.autoBlink = true;
729 m_cfg.autoBreath = true;
730 m_cfg.randomIdleMotion = false;
731 m_cfg.useLipSync = true;
732 m_cfg.lipSyncGain = 10;
733 m_cfg.lipSyncCutOff = 0.15;
734 m_cfg.audioBufSize = 4096;
735 m_cfg.mouthForm = 0;
736 m_cfg.top = 0;
737 m_cfg.bottom = 1079;
738 m_cfg.left = 0;
739 m_cfg.right = 1919; // These will be the full screen for 1920x1080
740
741 m_cfg.screen = 0;
742 m_cfg.origin = {1600, 870}; // Somewhere near the bottom right
743 }
744