PlayerReplay.cs 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356
  1. using System;
  2. using System.Collections;
  3. using System.Collections.Generic;
  4. using UnityEngine;
  5. using Kinect = Windows.Kinect;
  6. public class PlayerReplay : MonoBehaviour
  7. {
  8. // Joints already given the values from BodySourceView.cs that keeps updating
  9. public Transform[] joints = new Transform[25];
  10. public List<Vector3[]> jointsSequence = new List<Vector3[]>();
  11. public List<float> recordingTimes = new List<float>();
  12. public Material transparentMat;
  13. public Visualizer_FadeIn vfi;
  14. public Visualizer_FadeInSeries vfis;
  15. public BodyComparer bc;
  16. public StartStepPreview startStepPreview;
  17. public ModeController mc;
  18. public GameObject textFinish;
  19. public void AddJoints(float recordingTime)
  20. {
  21. Vector3[] positions = new Vector3[25];
  22. for (int i = 0; i < joints.Length; i++)
  23. {
  24. positions[i] = joints[i].position;
  25. }
  26. jointsSequence.Add(positions);
  27. recordingTimes.Add(recordingTime);
  28. }
  29. public void ResetRecording()
  30. {
  31. jointsSequence.Clear();
  32. recordingTimes.Clear();
  33. }
  34. public void Save()
  35. {
  36. if (jointsSequence.Count == 0)
  37. {
  38. Debug.Log("jointsSequence is empty");
  39. return;
  40. }
  41. string path = Application.dataPath + "/Recordings/" + mc.complexity.ToString() + "_" + mc.direction.ToString() + ".sav";
  42. SaveSystem.Save(path, jointsSequence, recordingTimes);
  43. Debug.Log("Save success");
  44. }
  45. public IEnumerator Load()
  46. {
  47. if (mc.testing)
  48. {
  49. string path = Application.dataPath + "/Recordings/" + mc.complexity.ToString() + "_" + mc.direction.ToString() + ".sav";
  50. JointsDataSequence data = SaveSystem.Load(path);
  51. if (data == null)
  52. {
  53. Debug.Log("Load failed");
  54. //return;
  55. yield break;
  56. }
  57. yield return StartCoroutine(ShowJoints(data));
  58. }
  59. else
  60. {
  61. string[] complexity = { "OneArm", "TwoArms", "TwoArmsPlusLeg" };
  62. string[] direction = { "Sideways", "Forward", "Backward" };
  63. string[] feedback = { "None", "Color", "Haptic" };
  64. string[] speed = { "Slow", "Fast" };
  65. List<string> config = new List<string>();
  66. for (int i = 0; i < complexity.Length; i++)
  67. {
  68. for (int j = 0; j < direction.Length; j++)
  69. {
  70. for (int k = 0; k < feedback.Length; k++)
  71. {
  72. for (int l = 0; l < speed.Length; l++)
  73. {
  74. config.Add(complexity[i] + ";" + direction[j] + ";" + feedback[k] + ";" + speed[l]);
  75. }
  76. }
  77. }
  78. }
  79. // Shuffle config list
  80. System.Random rand = new System.Random();
  81. int n = config.Count;
  82. while (n > 1)
  83. {
  84. n--;
  85. int k = rand.Next(n + 1);
  86. string value = config[k];
  87. config[k] = config[n];
  88. config[n] = value;
  89. }
  90. foreach (string s in config)
  91. {
  92. string[] subs = s.Split(';');
  93. string path = Application.dataPath + "/Recordings/" + subs[0] + "_" + subs[1] + ".sav";
  94. if (subs[0] == "OneArm")
  95. mc.complexity = ModeController.Complexity.OneArm;
  96. else if (subs[0] == "TwoArms")
  97. mc.complexity = ModeController.Complexity.TwoArms;
  98. else
  99. mc.complexity = ModeController.Complexity.TwoArmsPlusLeg;
  100. if (subs[1] == "Sideways")
  101. mc.direction = ModeController.Direction.Sideways;
  102. else if (subs[1] == "Forward")
  103. mc.direction = ModeController.Direction.Forward;
  104. else
  105. mc.direction = ModeController.Direction.Backward;
  106. if (subs[2] == "None")
  107. mc.feedback = ModeController.Feedback.None;
  108. else if (subs[2] == "Color")
  109. mc.feedback = ModeController.Feedback.ColorFeedback;
  110. else
  111. mc.feedback = ModeController.Feedback.HapticFeedback;
  112. mc.speed = subs[3] == "Slow" ? ModeController.Speed.Slow : ModeController.Speed.Fast;
  113. JointsDataSequence data = SaveSystem.Load(path);
  114. if (data == null)
  115. {
  116. Debug.Log("Load failed");
  117. //return;
  118. yield break;
  119. }
  120. yield return StartCoroutine(ShowJoints(data));
  121. }
  122. }
  123. yield return StartCoroutine(End());
  124. //textFinish.SetActive(true);
  125. //Debug.Log("Load finish");
  126. //ViveInput.StopPlaying();
  127. }
  128. private IEnumerator ShowJoints(JointsDataSequence data)
  129. {
  130. List<JointsData> jointsData = data.jointsDataSequence;
  131. List<float> recordingTimes = data.recordingTimes;
  132. yield return StartCoroutine(VisualizeFadeInSeries(jointsData, recordingTimes));
  133. }
  134. private IEnumerator Visualize(List<JointsData> newJointsData, List<float> newRecordingTimes)
  135. {
  136. for (int i = 0; i < newJointsData.Count; i++)
  137. {
  138. // Create GameObject body
  139. GameObject body = new GameObject("Recorded Body Demo " + i);
  140. // TODO: Convert JointsData to Vector3[] and rescale it
  141. JointsData jd = newJointsData[i];
  142. Vector3[] joints = HelperScript.ConvertJointsDataToVector3Array(jd);
  143. Vector3[] rescaledJoints = HelperScript.RescaleJoints(joints);
  144. for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++)
  145. {
  146. // Skip these joints
  147. if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight
  148. || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight
  149. || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight)
  150. continue;
  151. // Create GameObject cubes for joints
  152. GameObject jointObj = GameObject.CreatePrimitive(PrimitiveType.Cube);
  153. LineRenderer lr = jointObj.AddComponent<LineRenderer>();
  154. lr.positionCount = 2;
  155. lr.material = new Material(Shader.Find("Sprites/Default"))
  156. {
  157. color = new Color(1, 0.8f, 0.6f)
  158. };
  159. lr.startWidth = 0.05f;
  160. lr.endWidth = 0.05f;
  161. jointObj.transform.localScale = new Vector3(0.05f, 0.05f, 0.05f);
  162. jointObj.name = jt.ToString();
  163. jointObj.transform.position = rescaledJoints[(int)jt];
  164. jointObj.transform.parent = body.transform;
  165. // Remove LineRenderer component from neck
  166. if (jt == Kinect.JointType.Neck)
  167. {
  168. Destroy(jointObj.GetComponent<LineRenderer>());
  169. }
  170. }
  171. // Connect the joints with LineRenderer
  172. for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++)
  173. {
  174. // Skip if dictionary not contains the joint or other these joints
  175. if (!HelperScript._BoneMap.ContainsKey(jt) || jt == Kinect.JointType.Neck
  176. || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight
  177. || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight
  178. || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight)
  179. continue;
  180. Transform jointObj = body.transform.Find(jt.ToString());
  181. Transform targetJoint = body.transform.Find(HelperScript._BoneMap[jt].ToString());
  182. LineRenderer lr = jointObj.GetComponent<LineRenderer>();
  183. lr.SetPosition(0, jointObj.localPosition);
  184. lr.SetPosition(1, targetJoint.localPosition);
  185. }
  186. float waitTime = 1;
  187. if (i < newRecordingTimes.Count - 1)
  188. {
  189. waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i];
  190. }
  191. if (mc.speed == ModeController.Speed.Slow)
  192. waitTime *= 2;
  193. yield return new WaitForSeconds(waitTime);
  194. Destroy(body);
  195. }
  196. }
  197. private IEnumerator VisualizeFadeIn(List<JointsData> newJointsData, List<float> newRecordingTimes)
  198. {
  199. for (int i = 0; i < newJointsData.Count; i++)
  200. {
  201. float waitTime = 1;
  202. if (i < newRecordingTimes.Count - 1)
  203. {
  204. waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i];
  205. }
  206. WaitForSeconds wait = new WaitForSeconds(waitTime);
  207. Visualizer_FadeIn body = Instantiate(vfi) as Visualizer_FadeIn;
  208. body.SetData(newJointsData[i]);
  209. body.ShowBody();
  210. yield return wait;
  211. }
  212. }
  213. private IEnumerator VisualizeFadeInSeries(List<JointsData> jointsData, List<float> recordingTimes)
  214. {
  215. // Visualization demo
  216. yield return Visualize(jointsData, recordingTimes);
  217. bc.SetDataDemo(jointsData, recordingTimes);
  218. // Visualization steps/series
  219. // Filtered datas according to distance
  220. List<JointsData> filteredJointsData = new List<JointsData>();
  221. List<float> filteredRecordingTimes = new List<float>();
  222. JointsData prevJd = jointsData[0];
  223. // Add first pose
  224. filteredJointsData.Add(jointsData[0]);
  225. filteredRecordingTimes.Add(recordingTimes[0]);
  226. for (int i = 1; i < jointsData.Count; i++)
  227. {
  228. JointsData jd = jointsData[i];
  229. Vector3[] prevJoints = HelperScript.ConvertJointsDataToVector3Array(jd);
  230. Vector3[] prevRescaledJoints = HelperScript.RescaleJoints(prevJoints);
  231. Vector3[] joints = HelperScript.ConvertJointsDataToVector3Array(prevJd);
  232. Vector3[] rescaledJoints = HelperScript.RescaleJoints(joints);
  233. for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++)
  234. {
  235. // Skip these joints
  236. if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight
  237. || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight
  238. || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight)
  239. continue;
  240. Vector3 prevJointPosition = prevRescaledJoints[(int)jt];
  241. Vector3 jointPosition = rescaledJoints[(int)jt];
  242. float distance = Vector3.Distance(prevJointPosition, jointPosition);
  243. // If a joint is bigger than a certain distance, add it to the replay, else ignore it
  244. if (distance >= 0.15f)
  245. {
  246. filteredJointsData.Add(jd);
  247. filteredRecordingTimes.Add(recordingTimes[i]);
  248. prevJd = jd;
  249. continue;
  250. }
  251. }
  252. }
  253. // Add last step of demo
  254. filteredJointsData.Add(jointsData[jointsData.Count - 1]);
  255. // Show start position of steps
  256. startStepPreview.SetData(filteredJointsData[0]);
  257. startStepPreview.ShowBody();
  258. // Wait for input
  259. yield return ViveInput.WaitForControllerPress();
  260. // Delete start position of steps
  261. startStepPreview.DeleteBody();
  262. Visualizer_FadeInSeries body = Instantiate(vfis) as Visualizer_FadeInSeries;
  263. body.transform.parent = gameObject.transform;
  264. body.SetData(filteredJointsData, filteredRecordingTimes);
  265. body.ShowBody();
  266. // Compare visualization demo with body
  267. yield return bc.StartCompare();
  268. // Input for end steps. Wait for input if finish comparing. Don't wait for input if comparing is interupted.
  269. if (!bc.endStepsPressed)
  270. {
  271. // Unknown reason, need 2 times (not working if only 1)
  272. yield return ViveInput.WaitForControllerPress();
  273. yield return ViveInput.WaitForControllerPress();
  274. }
  275. else
  276. {
  277. // Reset
  278. bc.endStepsPressed = false;
  279. }
  280. Destroy(body);
  281. // Write positions to csv
  282. bc.WriteCSV();
  283. // Wait for input
  284. // Unknown reason, need 2 times (not working if only 1)
  285. yield return ViveInput.WaitForControllerPress();
  286. yield return ViveInput.WaitForControllerPress();
  287. }
  288. private IEnumerator End()
  289. {
  290. textFinish.SetActive(true);
  291. yield return new WaitForSeconds(3);
  292. textFinish.SetActive(false);
  293. Debug.Log("Load finish");
  294. ViveInput.StopPlaying();
  295. }
  296. }