Add GUI control panel (first draft)
[mouse-tracker-for-cubism.git] / src / mouse_cursor_tracker.cpp
1 /****
2 Copyright (c) 2020 Adrian I. Lam
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to deal
6 in the Software without restriction, including without limitation the rights
7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 SOFTWARE.
21 ****/
22
23 #include <stdexcept>
24 #include <string>
25 #include <chrono>
26 #include <thread>
27 #include <fstream>
28 #include <sstream>
29 #include <vector>
30 #include <utility>
31 #include <algorithm>
32 #include <cstdlib>
33 #include <cmath>
34 #include <cassert>
35
36 #include <iostream>
37 #include <cctype>
38 #include <mutex>
39
40 #include "mouse_cursor_tracker.h"
41
42 extern "C"
43 {
44 #include <xdo.h>
45 #include <pulse/simple.h>
46 #include <string.h> // strdup
47 #include "editline.h"
48 }
49
50
51 static double rms(float *buf, std::size_t count)
52 {
53 double sum = 0;
54 for (std::size_t i = 0; i < count; i++)
55 {
56 sum += buf[i] * buf[i];
57 }
58 return std::sqrt(sum / count);
59 }
60
61 static std::vector<std::string> split(std::string s)
62 {
63 std::vector<std::string> v;
64 std::string tmp;
65
66 for (std::size_t i = 0; i < s.length(); i++)
67 {
68 char c = s[i];
69 if (std::isspace(c))
70 {
71 if (tmp != "")
72 {
73 v.push_back(tmp);
74 tmp = "";
75 }
76 }
77 else
78 {
79 tmp += c;
80 }
81 }
82 if (tmp != "")
83 {
84 v.push_back(tmp);
85 }
86 return v;
87 }
88
89
90 /* Using readline callback functions means that we need to pass
91 * information from our class to the callback, and the only way
92 * to do so is using globals.
93 */
94
95 // Taken from https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/utils.cpp
96 static std::string longest_common_prefix(std::string s,
97 const std::vector<std::string>& candidates) {
98 assert(candidates.size() > 0);
99 if (candidates.size() == 1) {
100 return candidates[0];
101 }
102
103 std::string prefix(s);
104 while (true) {
105 // Each iteration of this loop advances to the next location in all the
106 // candidates and sees if they match up to it.
107 size_t nextloc = prefix.size();
108 auto i = candidates.begin();
109 if (i->size() <= nextloc) {
110 return prefix;
111 }
112 char nextchar = (*(i++))[nextloc];
113 for (; i != candidates.end(); ++i) {
114 if (i->size() <= nextloc || (*i)[nextloc] != nextchar) {
115 // Bail out if there's a mismatch for this candidate.
116 return prefix;
117 }
118 }
119 // All candidates have contents[nextloc] == nextchar, so we can safely
120 // extend the prefix.
121 prefix.append(1, nextchar);
122 }
123
124 assert(0 && "unreachable");
125 }
126
127 std::vector<std::string> commands =
128 {
129 "help", "motion", "expression", "set", "clear"
130 };
131
132 std::vector<std::string> live2dParams =
133 { // https://docs.live2d.com/cubism-editor-manual/standard-parametor-list/?locale=ja
134 "ParamAngleX", "ParamAngleY", "ParamAngleZ",
135 "ParamEyeLOpen", "ParamEyeLSmile", "ParamEyeROpen", "ParamEyeRSmile",
136 "ParamEyeBallX", "ParamEyeBallY", "ParamEyeBallForm",
137 "ParamBrowLY", "ParamBrowRY", "ParamBrowLX", "ParamBrowRX",
138 "ParamBrowLAngle", "ParamBrowRAngle", "ParamBrowLForm", "ParamBrowRForm",
139 "ParamMouthForm", "ParamMouthOpenY", "ParamTere",
140 "ParamBodyAngleX", "ParamBodyAngleY", "ParamBodyAngleZ", "ParamBreath",
141 "ParamArmLA", "ParamArmRA", "ParamArmLB", "ParamArmRB",
142 "ParamHandL", "ParamHandR",
143 "ParamHairFront", "ParamHairSide", "ParamHairBack", "ParamHairFluffy",
144 "ParamShoulderY", "ParamBustX", "ParamBustY",
145 "ParamBaseX", "ParamBaseY"
146 };
147 std::vector<std::pair<std::string, int> > MCT_motions;
148 std::vector<std::string> MCT_expressions;
149 std::map<std::string, double> *MCT_overrideMap;
150
151 static int cliListPossible(char *token, char ***av)
152 {
153 // Reference: https://github.com/eliben/code-for-blog/blob/master/2016/readline-samples/readline-complete-subcommand.cpp
154
155 std::string line(rl_line_buffer);
156
157 std::vector<std::string> cmdline = split(line);
158
159 std::vector<std::string> constructed;
160 std::vector<std::string> *vocab = nullptr;
161
162 if (cmdline.size() == 0 ||
163 (cmdline.size() == 1 && line.back() != ' ') ||
164 cmdline[0] == "help")
165 {
166 vocab = &commands;
167 }
168 else if (cmdline[0] == "motion")
169 {
170 for (auto it = MCT_motions.begin(); it != MCT_motions.end(); ++it)
171 {
172 if ((cmdline.size() == 1 && line.back() == ' ') ||
173 (cmdline.size() == 2 && line.back() != ' '))
174 { // motionGroup
175 {
176 constructed.push_back(it->first);
177 }
178 }
179 else if ((cmdline.size() == 2 && line.back() == ' ') ||
180 (cmdline.size() == 3 && line.back() != ' '))
181 { // motionNumber
182 if (it->first == cmdline[1])
183 {
184 for (int i = 0; i < it->second; i++)
185 {
186 constructed.push_back(std::to_string(i));
187 }
188 break;
189 }
190 }
191 else if (cmdline.size() <= 4)
192 { // priority
193 for (int i = 0; i < 4; i++)
194 {
195 constructed.push_back(std::to_string(i));
196 }
197 break; // No need to loop through the list, all motions have the same set of priorities
198 }
199 }
200
201 vocab = &constructed;
202 }
203 else if (cmdline[0] == "expression")
204 {
205 vocab = &MCT_expressions;
206 }
207 else if (cmdline[0] == "set")
208 {
209 if ((cmdline.size() % 2 == 0 && line.back() != ' ') ||
210 (cmdline.size() % 2 == 1 && line.back() == ' '))
211 {
212 vocab = &live2dParams;
213 }
214 }
215 else if (cmdline[0] == "clear")
216 {
217 constructed.push_back("all");
218 for (auto const &entry : *MCT_overrideMap)
219 {
220 constructed.push_back(entry.first);
221 }
222 vocab = &constructed;
223 }
224
225 if (!vocab)
226 {
227 return 0;
228 }
229
230 std::string text(token);
231 std::vector<std::string> matches;
232 std::copy_if(vocab->begin(), vocab->end(), std::back_inserter(matches),
233 [&text](const std::string &s)
234 {
235 return (s.size() >= text.size() &&
236 s.compare(0, text.size(), text) == 0);
237 });
238
239 *av = static_cast<char **>(malloc(matches.size() * sizeof (**av)));
240 if (!*av) return 0;
241 for (std::size_t i = 0; i < matches.size(); i++)
242 {
243 (*av)[i] = strdup(matches[i].c_str());
244 }
245 return matches.size();
246 }
247
248 static char *cliComplete(char *token, int *match)
249 {
250 char **av;
251 int numMatches = cliListPossible(token, &av);
252
253 *match = 0;
254 if (numMatches == 0) return nullptr;
255
256 std::vector<std::string> list;
257
258 for (std::size_t i = 0; i < numMatches; i++)
259 {
260 list.push_back(std::string(av[i]));
261 free(av[i]);
262 }
263 free(av);
264
265 if (numMatches == 1)
266 {
267 *match = 1;
268 std::string result = list[0] + " ";
269 return strdup(result.c_str() + strlen(token));
270 }
271
272 std::string lcp = longest_common_prefix(token, list);
273
274 if (lcp.size() > strlen(token))
275 {
276 *match = 1;
277 return strdup(lcp.c_str() + strlen(token));
278 }
279
280 return nullptr;
281 }
282
283 MouseCursorTracker::MouseCursorTracker(std::string cfgPath,
284 std::vector<std::pair<std::string, int> > motions,
285 std::vector<std::string> expressions)
286 : m_stop(false)
287 {
288 m_motionPriority = MotionPriority::none;
289 m_motionNumber = 0;
290
291
292 parseConfig(cfgPath);
293 m_xdo = xdo_new(nullptr);
294
295 const pa_sample_spec ss =
296 {
297 .format = PA_SAMPLE_FLOAT32NE,
298 .rate = 44100,
299 .channels = 2
300 };
301 m_pulse = pa_simple_new(nullptr, "MouseCursorTracker", PA_STREAM_RECORD,
302 nullptr, "LipSync", &ss, nullptr, nullptr, nullptr);
303 if (!m_pulse)
304 {
305 throw std::runtime_error("Unable to create pulse");
306 }
307
308 m_gtkapp = Gtk::Application::create();
309
310 m_getVolumeThread = std::thread(&MouseCursorTracker::audioLoop, this);
311 m_parseCommandThread = std::thread(&MouseCursorTracker::cliLoop, this);
312 m_guiThread = std::thread(&MouseCursorTracker::guiLoop, this);
313
314 MCT_motions = motions;
315 MCT_expressions = expressions;
316 MCT_overrideMap = &m_overrideMap;
317 }
318
319 void MouseCursorTracker::audioLoop(void)
320 {
321 float *buf = new float[m_cfg.audioBufSize];
322
323 std::size_t audioBufByteSize = m_cfg.audioBufSize * sizeof *buf;
324
325 while (!m_stop)
326 {
327 if (pa_simple_read(m_pulse, buf, audioBufByteSize, nullptr) < 0)
328 {
329 throw std::runtime_error("Unable to get audio data");
330 }
331 m_currentVol = rms(buf, m_cfg.audioBufSize);
332 }
333
334 delete[] buf;
335 }
336
337 void MouseCursorTracker::cliLoop(void)
338 {
339 rl_set_complete_func(&cliComplete);
340 rl_set_list_possib_func(&cliListPossible);
341 while (!m_stop)
342 {
343 char *buf = readline(">> ");
344
345 if (buf)
346 {
347 std::string cmdline(buf);
348 free(buf);
349 processCommand(cmdline);
350 }
351 else
352 {
353 std::cout << "Exiting CLI loop. Use Ctrl+C to exit the whole process." << std::endl;
354 stop();
355 }
356 }
357 }
358
359
360 void MouseCursorTracker::processCommand(std::string cmdline)
361 {
362 auto cmdSplit = split(cmdline);
363
364 if (cmdSplit.size() > 0)
365 {
366 add_history(cmdline.c_str());
367
368 if (cmdSplit[0] == "help")
369 {
370 if (cmdSplit.size() == 1)
371 {
372 std::cout << "Available commands: motion expression set clear\n"
373 << "Type \"help <command>\" for more help" << std::endl;
374 }
375 else if (cmdSplit[1] == "motion")
376 {
377 std::cout << "motion <motionGroup> <motionNumber> [<priority>]\n"
378 << "motionGroup: The motion name in the .model3.json file\n"
379 << "motionNumber: The index of this motion in the .model3.json file, 0-indexed\n"
380 << "priority: 0 = none, 1 = idle, 2 = normal, 3 = force (default normal)" << std::endl;
381 }
382 else if (cmdSplit[1] == "expression")
383 {
384 std::cout << "expression <expressionName>\n"
385 << "expressionName: Name of expression in the .model3.json file" << std::endl;
386 }
387 else if (cmdSplit[1] == "set")
388 {
389 std::cout << "set <param1> <value1> [<param2> <value2> ...]\n"
390 << "Set parameter value. Overrides any tracking."
391 << "See live2D documentation for full list of params" << std::endl;
392 }
393 else if (cmdSplit[1] == "clear")
394 {
395 std::cout << "clear <param1> [<param2> ...]\n"
396 << "Clear parameter value. Re-enables tracking if it was overridden by \"set\"\n"
397 << "You can also use \"clear all\" to clear everything" << std::endl;
398 }
399 else
400 {
401 std::cout << "Unrecognized command" << std::endl;
402 }
403 }
404 else if (cmdSplit[0] == "motion")
405 {
406 if (cmdSplit.size() == 3 || cmdSplit.size() == 4)
407 {
408 std::unique_lock<std::mutex> lock(m_motionMutex, std::defer_lock);
409 lock.lock();
410 m_motionGroup = cmdSplit[1];
411 try
412 {
413 m_motionNumber = std::stoi(cmdSplit[2]);
414 if (cmdSplit.size() == 4)
415 {
416 m_motionPriority = static_cast<MotionPriority>(std::stoi(cmdSplit[3]));
417 }
418 else
419 {
420 m_motionPriority = MotionPriority::normal;
421 }
422 }
423 catch (const std::exception &e)
424 {
425 std::cerr << "std::stoi failed" << std::endl;
426 }
427 lock.unlock();
428 }
429 else
430 {
431 std::cerr << "Incorrect command, expecting 2 or 3 arguments" << std::endl;
432 std::cerr << "motion motionGroup motionNumber [motionPriority]" << std::endl;
433 }
434 }
435 else if (cmdSplit[0] == "expression")
436 {
437 if (cmdSplit.size() == 2)
438 {
439 std::unique_lock<std::mutex> lock(m_motionMutex, std::defer_lock);
440 lock.lock();
441 m_expression = cmdSplit[1];
442 lock.unlock();
443 }
444 else
445 {
446 std::cerr << "Incorrect command, expecting 1 argument: expressionName" << std::endl;
447 }
448 }
449 else if (cmdSplit[0] == "set")
450 {
451 if (cmdSplit.size() % 2 != 1)
452 {
453 // "set param1 value1 param2 value2 ..."
454 std::cerr << "Incorrect number of arguments for command 'set'" << std::endl;
455 }
456 for (std::size_t i = 1; i < cmdSplit.size(); i += 2)
457 {
458 try
459 {
460 m_overrideMap[cmdSplit[i]] = std::stod(cmdSplit[i + 1]);
461 }
462 catch (const std::exception &e)
463 {
464 std::cerr << "std::stod failed" << std::endl;
465 }
466 }
467 }
468 else if (cmdSplit[0] == "clear")
469 {
470 for (std::size_t i = 1; i < cmdSplit.size(); i++)
471 {
472 if (cmdSplit[i] == "all")
473 {
474 m_overrideMap.clear();
475 break;
476 }
477 std::size_t removed = m_overrideMap.erase(cmdSplit[i]);
478 if (removed == 0)
479 {
480 std::cerr << "Warning: key " << cmdSplit[i] << " not found" << std::endl;
481 }
482 }
483 }
484 else
485 {
486 std::cerr << "Unknown command" << std::endl;
487 }
488 }
489 }
490
491 MouseCursorTracker::~MouseCursorTracker()
492 {
493 xdo_free(m_xdo);
494 m_getVolumeThread.join();
495 m_parseCommandThread.join();
496 pa_simple_free(m_pulse);
497 }
498
499 void MouseCursorTracker::stop(void)
500 {
501 m_stop = true;
502 }
503
504 MouseCursorTracker::Params MouseCursorTracker::getParams(void)
505 {
506 Params params = Params();
507
508 int xOffset = m_curPos.x - m_cfg.origin.x;
509 int leftRange = m_cfg.origin.x - m_cfg.left;
510 int rightRange = m_cfg.right - m_cfg.origin.x;
511
512 if (xOffset > 0) // i.e. to the right
513 {
514 params.live2d["ParamAngleX"] = 30.0 * xOffset / rightRange;
515 }
516 else // to the left
517 {
518 params.live2d["ParamAngleX"] = 30.0 * xOffset / leftRange;
519 }
520
521 int yOffset = m_curPos.y - m_cfg.origin.y;
522 int topRange = m_cfg.origin.y - m_cfg.top;
523 int bottomRange = m_cfg.bottom - m_cfg.origin.y;
524
525 if (yOffset > 0) // downwards
526 {
527 params.live2d["ParamAngleY"] = -30.0 * yOffset / bottomRange;
528 }
529 else // upwards
530 {
531 params.live2d["ParamAngleY"] = -30.0 * yOffset / topRange;
532 }
533
534 params.autoBlink = m_cfg.autoBlink;
535 params.autoBreath = m_cfg.autoBreath;
536 params.randomIdleMotion = m_cfg.randomIdleMotion;
537 params.useLipSync = m_cfg.useLipSync;
538
539 params.live2d["ParamMouthForm"] = m_cfg.mouthForm;
540
541 if (m_cfg.useLipSync)
542 {
543 params.lipSyncParam = m_currentVol * m_cfg.lipSyncGain;
544 if (params.lipSyncParam < m_cfg.lipSyncCutOff)
545 {
546 params.lipSyncParam = 0;
547 }
548 else if (params.lipSyncParam > 1)
549 {
550 params.lipSyncParam = 1;
551 }
552 }
553
554 // Don't block in getParams()
555 std::unique_lock<std::mutex> lock(m_motionMutex, std::try_to_lock);
556 if (lock.owns_lock())
557 {
558 if (m_motionPriority != MotionPriority::none)
559 {
560 params.motionPriority = m_motionPriority;
561 params.motionGroup = m_motionGroup;
562 params.motionNumber = m_motionNumber;
563
564 m_motionPriority = MotionPriority::none;
565 m_motionGroup = "";
566 m_motionNumber = 0;
567 }
568 params.expression = m_expression;
569 m_expression = "";
570 lock.unlock();
571 }
572
573 // Leave everything else as zero
574
575
576 // Process overrides
577 for (auto const &x : m_overrideMap)
578 {
579 std::string key = x.first;
580 double val = x.second;
581
582 params.live2d[key] = val;
583 }
584
585
586 return params;
587 }
588
589 void MouseCursorTracker::mainLoop(void)
590 {
591 while (!m_stop)
592 {
593 int x;
594 int y;
595 int screenNum;
596
597 xdo_get_mouse_location(m_xdo, &x, &y, &screenNum);
598
599 if (screenNum == m_cfg.screen)
600 {
601 m_curPos.x = x;
602 m_curPos.y = y;
603 }
604 // else just silently ignore for now
605 std::this_thread::sleep_for(std::chrono::milliseconds(m_cfg.sleepMs));
606 }
607 }
608
609 void MouseCursorTracker::parseConfig(std::string cfgPath)
610 {
611 populateDefaultConfig();
612
613 if (cfgPath != "")
614 {
615 std::ifstream file(cfgPath);
616 if (!file)
617 {
618 throw std::runtime_error("Failed to open config file");
619 }
620 std::string line;
621 unsigned int lineNum = 0;
622
623 while (std::getline(file, line))
624 {
625 if (line[0] == '#')
626 {
627 continue;
628 }
629
630 std::istringstream ss(line);
631 std::string paramName;
632
633 if (ss >> paramName)
634 {
635 if (paramName == "sleep_ms")
636 {
637 if (!(ss >> m_cfg.sleepMs))
638 {
639 throw std::runtime_error("Error parsing sleep_ms");
640 }
641 }
642 else if (paramName == "autoBlink")
643 {
644 if (!(ss >> m_cfg.autoBlink))
645 {
646 throw std::runtime_error("Error parsing autoBlink");
647 }
648 }
649 else if (paramName == "autoBreath")
650 {
651 if (!(ss >> m_cfg.autoBreath))
652 {
653 throw std::runtime_error("Error parsing autoBreath");
654 }
655 }
656 else if (paramName == "randomIdleMotion")
657 {
658 if (!(ss >> m_cfg.randomIdleMotion))
659 {
660 throw std::runtime_error("Error parsing randomIdleMotion");
661 }
662 }
663 else if (paramName == "useLipSync")
664 {
665 if (!(ss >> m_cfg.useLipSync))
666 {
667 throw std::runtime_error("Error parsing useLipSync");
668 }
669 }
670 else if (paramName == "lipSyncGain")
671 {
672 if (!(ss >> m_cfg.lipSyncGain))
673 {
674 throw std::runtime_error("Error parsing lipSyncGain");
675 }
676 }
677 else if (paramName == "lipSyncCutOff")
678 {
679 if (!(ss >> m_cfg.lipSyncCutOff))
680 {
681 throw std::runtime_error("Error parsing lipSyncCutOff");
682 }
683 }
684 else if (paramName == "audioBufSize")
685 {
686 if (!(ss >> m_cfg.audioBufSize))
687 {
688 throw std::runtime_error("Error parsing audioBufSize");
689 }
690 }
691 else if (paramName == "mouthForm")
692 {
693 if (!(ss >> m_cfg.mouthForm))
694 {
695 throw std::runtime_error("Error parsing mouthForm");
696 }
697 }
698 else if (paramName == "screen")
699 {
700 if (!(ss >> m_cfg.screen))
701 {
702 throw std::runtime_error("Error parsing screen");
703 }
704 }
705 else if (paramName == "origin_x")
706 {
707 if (!(ss >> m_cfg.origin.x))
708 {
709 throw std::runtime_error("Error parsing origin_x");
710 }
711 }
712 else if (paramName == "origin_y")
713 {
714 if (!(ss >> m_cfg.origin.y))
715 {
716 throw std::runtime_error("Error parsing origin_y");
717 }
718 }
719 else if (paramName == "top")
720 {
721 if (!(ss >> m_cfg.top))
722 {
723 throw std::runtime_error("Error parsing top");
724 }
725 }
726 else if (paramName == "bottom")
727 {
728 if (!(ss >> m_cfg.bottom))
729 {
730 throw std::runtime_error("Error parsing bottom");
731 }
732 }
733 else if (paramName == "left")
734 {
735 if (!(ss >> m_cfg.left))
736 {
737 throw std::runtime_error("Error parsing left");
738 }
739 }
740 else if (paramName == "right")
741 {
742 if (!(ss >> m_cfg.right))
743 {
744 throw std::runtime_error("Error parsing right");
745 }
746 }
747 else
748 {
749 throw std::runtime_error("Unrecognized config parameter");
750 }
751 }
752 }
753 }
754 }
755
756 void MouseCursorTracker::populateDefaultConfig(void)
757 {
758 m_cfg.sleepMs = 5;
759 m_cfg.autoBlink = true;
760 m_cfg.autoBreath = true;
761 m_cfg.randomIdleMotion = true;
762 m_cfg.useLipSync = true;
763 m_cfg.lipSyncGain = 10;
764 m_cfg.lipSyncCutOff = 0.15;
765 m_cfg.audioBufSize = 4096;
766 m_cfg.mouthForm = 0;
767 m_cfg.top = 0;
768 m_cfg.bottom = 1079;
769 m_cfg.left = 0;
770 m_cfg.right = 1919; // These will be the full screen for 1920x1080
771
772 m_cfg.screen = 0;
773 m_cfg.origin = {1600, 870}; // Somewhere near the bottom right
774 }
775