PlayerReplay.cs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323
  1. using System;
  2. using System.Collections;
  3. using System.Collections.Generic;
  4. using UnityEngine;
  5. using Kinect = Windows.Kinect;
  6. public class PlayerReplay : MonoBehaviour
  7. {
  8. // Joints already given the values from BodySourceView.cs that keeps updating
  9. public Transform[] joints = new Transform[25];
  10. public List<Vector3[]> jointsSequence = new List<Vector3[]>();
  11. public List<float> recordingTimes = new List<float>();
  12. public Material transparentMat;
  13. public Visualizer_FadeIn vfi;
  14. public Visualizer_FadeInSeries vfis;
  15. public BodyComparer bc;
  16. public StartStepPreview startStepPreview;
  17. public GameObject textFinish;
  18. private Dictionary<Kinect.JointType, Kinect.JointType> _BoneMap = new Dictionary<Kinect.JointType, Kinect.JointType>()
  19. {
  20. { Kinect.JointType.FootLeft, Kinect.JointType.AnkleLeft },
  21. { Kinect.JointType.AnkleLeft, Kinect.JointType.KneeLeft },
  22. { Kinect.JointType.KneeLeft, Kinect.JointType.HipLeft },
  23. { Kinect.JointType.HipLeft, Kinect.JointType.SpineBase },
  24. { Kinect.JointType.FootRight, Kinect.JointType.AnkleRight },
  25. { Kinect.JointType.AnkleRight, Kinect.JointType.KneeRight },
  26. { Kinect.JointType.KneeRight, Kinect.JointType.HipRight },
  27. { Kinect.JointType.HipRight, Kinect.JointType.SpineBase },
  28. { Kinect.JointType.HandTipLeft, Kinect.JointType.HandLeft },
  29. { Kinect.JointType.ThumbLeft, Kinect.JointType.HandLeft },
  30. { Kinect.JointType.HandLeft, Kinect.JointType.WristLeft },
  31. { Kinect.JointType.WristLeft, Kinect.JointType.ElbowLeft },
  32. { Kinect.JointType.ElbowLeft, Kinect.JointType.ShoulderLeft },
  33. { Kinect.JointType.ShoulderLeft, Kinect.JointType.SpineShoulder },
  34. { Kinect.JointType.HandTipRight, Kinect.JointType.HandRight },
  35. { Kinect.JointType.ThumbRight, Kinect.JointType.HandRight },
  36. { Kinect.JointType.HandRight, Kinect.JointType.WristRight },
  37. { Kinect.JointType.WristRight, Kinect.JointType.ElbowRight },
  38. { Kinect.JointType.ElbowRight, Kinect.JointType.ShoulderRight },
  39. { Kinect.JointType.ShoulderRight, Kinect.JointType.SpineShoulder },
  40. { Kinect.JointType.SpineBase, Kinect.JointType.SpineMid },
  41. { Kinect.JointType.SpineMid, Kinect.JointType.SpineShoulder },
  42. { Kinect.JointType.SpineShoulder, Kinect.JointType.Neck },
  43. { Kinect.JointType.Neck, Kinect.JointType.Head },
  44. };
  45. public void AddJoints(float recordingTime)
  46. {
  47. Vector3[] positions = new Vector3[25];
  48. for (int i = 0; i < joints.Length; i++)
  49. {
  50. positions[i] = joints[i].position;
  51. }
  52. jointsSequence.Add(positions);
  53. recordingTimes.Add(recordingTime);
  54. }
  55. public void ResetRecording()
  56. {
  57. jointsSequence.Clear();
  58. recordingTimes.Clear();
  59. }
  60. public void Save()
  61. {
  62. if (jointsSequence.Count == 0)
  63. {
  64. Debug.Log("jointsSequence is empty");
  65. return;
  66. }
  67. SaveSystem.Save(jointsSequence, recordingTimes);
  68. Debug.Log("Save success");
  69. }
  70. public void Load()
  71. {
  72. JointsDataSequence data = SaveSystem.Load();
  73. if (data == null)
  74. {
  75. Debug.Log("Load failed");
  76. return;
  77. }
  78. ShowJoints(data);
  79. Debug.Log("Load success");
  80. }
  81. private void ShowJoints(JointsDataSequence data)
  82. {
  83. List<JointsData> jointsData = data.jointsDataSequence;
  84. List<float> recordingTimes = data.recordingTimes;
  85. StartCoroutine(VisualizeFadeInSeries(jointsData, recordingTimes));
  86. }
  87. private IEnumerator Visualize(List<JointsData> newJointsData, List<float> newRecordingTimes)
  88. {
  89. for (int i = 0; i < newJointsData.Count; i++)
  90. {
  91. float waitTime = 1;
  92. if (i < newRecordingTimes.Count - 1)
  93. {
  94. waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i];
  95. }
  96. WaitForSeconds wait = new WaitForSeconds(waitTime);
  97. // Create GameObject body
  98. GameObject body = new GameObject("Recorded Body Demo " + i);
  99. for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++)
  100. {
  101. // Skip these joints
  102. if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight
  103. || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight
  104. || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight)
  105. continue;
  106. // Create GameObject cubes for joints
  107. GameObject jointObj = GameObject.CreatePrimitive(PrimitiveType.Cube);
  108. LineRenderer lr = jointObj.AddComponent<LineRenderer>();
  109. lr.positionCount = 2;
  110. lr.material = new Material(Shader.Find("Sprites/Default"))
  111. {
  112. color = new Color(1, 0.8f, 0.6f)
  113. };
  114. lr.startWidth = 0.05f;
  115. lr.endWidth = 0.05f;
  116. jointObj.transform.localScale = new Vector3(0.05f, 0.05f, 0.05f);
  117. jointObj.name = jt.ToString();
  118. JointsData jd = newJointsData[i];
  119. jointObj.transform.position = new Vector3(jd.jointsPositionsX[(int)jt], jd.jointsPositionsY[(int)jt], jd.jointsPositionsZ[(int)jt]);
  120. jointObj.transform.parent = body.transform;
  121. // Remove LineRenderer component from neck
  122. if (jt == Kinect.JointType.Neck)
  123. {
  124. Destroy(jointObj.GetComponent<LineRenderer>());
  125. }
  126. }
  127. // Connect the joints with LineRenderer
  128. for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++)
  129. {
  130. // Skip if dictionary not contains the joint or other these joints
  131. if (!_BoneMap.ContainsKey(jt) || jt == Kinect.JointType.Neck
  132. || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight
  133. || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight
  134. || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight)
  135. continue;
  136. Transform jointObj = body.transform.Find(jt.ToString());
  137. Transform targetJoint = body.transform.Find(_BoneMap[jt].ToString());
  138. LineRenderer lr = jointObj.GetComponent<LineRenderer>();
  139. lr.SetPosition(0, jointObj.localPosition);
  140. lr.SetPosition(1, targetJoint.localPosition);
  141. }
  142. yield return wait;
  143. Destroy(body);
  144. }
  145. }
  146. private IEnumerator VisualizeFadeIn(List<JointsData> newJointsData, List<float> newRecordingTimes)
  147. {
  148. for (int i = 0; i < newJointsData.Count; i++)
  149. {
  150. float waitTime = 1;
  151. if (i < newRecordingTimes.Count - 1)
  152. {
  153. waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i];
  154. }
  155. WaitForSeconds wait = new WaitForSeconds(waitTime);
  156. Visualizer_FadeIn body = Instantiate(vfi) as Visualizer_FadeIn;
  157. body.SetData(newJointsData[i]);
  158. body.ShowBody();
  159. yield return wait;
  160. }
  161. }
  162. private IEnumerator VisualizeFadeInSeries(List<JointsData> jointsData, List<float> recordingTimes)
  163. {
  164. float waitTime = 3;
  165. List<JointsData> jointsDataDemo = new List<JointsData>();
  166. List<float> recordingTimesDemo = new List<float>();
  167. int indexDemo = 0;
  168. bool finish = false;
  169. while (true)
  170. {
  171. for (int i = indexDemo; i < jointsData.Count; i++)
  172. {
  173. if (recordingTimes[i] < waitTime)
  174. {
  175. jointsDataDemo.Add(jointsData[i]);
  176. recordingTimesDemo.Add(recordingTimes[i]);
  177. finish = true;
  178. }
  179. else
  180. {
  181. indexDemo = i;
  182. waitTime += 3;
  183. finish = false;
  184. break;
  185. }
  186. }
  187. // Visualization demo
  188. yield return Visualize(jointsDataDemo, recordingTimesDemo);
  189. bc.SetDataDemo(jointsDataDemo, recordingTimesDemo);
  190. // Visualization steps/series
  191. // Filtered datas according to distance
  192. List<JointsData> filteredJointsData = new List<JointsData>();
  193. List<float> filteredRecordingTimes = new List<float>();
  194. JointsData prevJd = jointsDataDemo[0];
  195. // Add first pose
  196. filteredJointsData.Add(jointsDataDemo[0]);
  197. filteredRecordingTimes.Add(recordingTimesDemo[0]);
  198. for (int i = 1; i < jointsDataDemo.Count; i++)
  199. {
  200. JointsData jd = jointsDataDemo[i];
  201. for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++)
  202. {
  203. // Skip these joints
  204. if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight
  205. || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight
  206. || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight)
  207. continue;
  208. Vector3 prevJointPosition = new Vector3(prevJd.jointsPositionsX[(int)jt], prevJd.jointsPositionsY[(int)jt], prevJd.jointsPositionsZ[(int)jt]);
  209. Vector3 jointPosition = new Vector3(jd.jointsPositionsX[(int)jt], jd.jointsPositionsY[(int)jt], jd.jointsPositionsZ[(int)jt]);
  210. float distance = Vector3.Distance(prevJointPosition, jointPosition);
  211. // If a joint is bigger than a certain distance, add it to the replay, else ignore it
  212. if (distance >= 0.15f)
  213. {
  214. filteredJointsData.Add(jd);
  215. filteredRecordingTimes.Add(recordingTimesDemo[i]);
  216. prevJd = jd;
  217. continue;
  218. }
  219. }
  220. }
  221. // Add last step of demo
  222. filteredJointsData.Add(jointsDataDemo[jointsDataDemo.Count - 1]);
  223. // Show start position of steps
  224. startStepPreview.SetData(filteredJointsData[0]);
  225. startStepPreview.ShowBody();
  226. // Wait for input
  227. yield return ViveInput.WaitForControllerPress();
  228. // Delete start position of steps
  229. startStepPreview.DeleteBody();
  230. Visualizer_FadeInSeries body = Instantiate(vfis) as Visualizer_FadeInSeries;
  231. body.transform.parent = gameObject.transform;
  232. body.SetData(filteredJointsData, filteredRecordingTimes);
  233. body.ShowBody();
  234. // Compare visualization demo with body
  235. yield return bc.StartCompare();
  236. // Clear list
  237. jointsDataDemo.Clear();
  238. recordingTimesDemo.Clear();
  239. // Input for end steps. Wait for input if finish comparing. Don't wait for input if comparing is interupted.
  240. if (!bc.endStepsPressed)
  241. {
  242. // Unknown reason, need 2 times (not working if only 1)
  243. yield return ViveInput.WaitForControllerPress();
  244. yield return ViveInput.WaitForControllerPress();
  245. }
  246. else
  247. {
  248. // Reset
  249. bc.endStepsPressed = false;
  250. }
  251. // TODO: maybe need to check if body is null or not (after 6 seconds body is destroyed, destroy a destroyed gameobject error?)
  252. Destroy(body);
  253. // Wait for input
  254. // Unknown reason, need 2 times (not working if only 1)
  255. yield return ViveInput.WaitForControllerPress();
  256. yield return ViveInput.WaitForControllerPress();
  257. if (finish)
  258. break;
  259. }
  260. textFinish.SetActive(true);
  261. yield return new WaitForSeconds(3);
  262. textFinish.SetActive(false);
  263. // Write positions to csv
  264. bc.WriteCSV();
  265. ViveInput.StopPlaying();
  266. }
  267. }