using System; using System.Collections; using System.Collections.Generic; using UnityEngine; using Kinect = Windows.Kinect; public class PlayerReplay : MonoBehaviour { // Joints already given the values from BodySourceView.cs that keeps updating public Transform[] joints = new Transform[25]; public List jointsSequence = new List(); public List recordingTimes = new List(); public Material transparentMat; public Visualizer_FadeIn vfi; public Visualizer_FadeInSeries vfis; public BodyComparer bc; public StartStepPreview startStepPreview; public GameObject textFinish; private Dictionary _BoneMap = new Dictionary() { { Kinect.JointType.FootLeft, Kinect.JointType.AnkleLeft }, { Kinect.JointType.AnkleLeft, Kinect.JointType.KneeLeft }, { Kinect.JointType.KneeLeft, Kinect.JointType.HipLeft }, { Kinect.JointType.HipLeft, Kinect.JointType.SpineBase }, { Kinect.JointType.FootRight, Kinect.JointType.AnkleRight }, { Kinect.JointType.AnkleRight, Kinect.JointType.KneeRight }, { Kinect.JointType.KneeRight, Kinect.JointType.HipRight }, { Kinect.JointType.HipRight, Kinect.JointType.SpineBase }, { Kinect.JointType.HandTipLeft, Kinect.JointType.HandLeft }, { Kinect.JointType.ThumbLeft, Kinect.JointType.HandLeft }, { Kinect.JointType.HandLeft, Kinect.JointType.WristLeft }, { Kinect.JointType.WristLeft, Kinect.JointType.ElbowLeft }, { Kinect.JointType.ElbowLeft, Kinect.JointType.ShoulderLeft }, { Kinect.JointType.ShoulderLeft, Kinect.JointType.SpineShoulder }, { Kinect.JointType.HandTipRight, Kinect.JointType.HandRight }, { Kinect.JointType.ThumbRight, Kinect.JointType.HandRight }, { Kinect.JointType.HandRight, Kinect.JointType.WristRight }, { Kinect.JointType.WristRight, Kinect.JointType.ElbowRight }, { Kinect.JointType.ElbowRight, Kinect.JointType.ShoulderRight }, { Kinect.JointType.ShoulderRight, Kinect.JointType.SpineShoulder }, { Kinect.JointType.SpineBase, Kinect.JointType.SpineMid }, { Kinect.JointType.SpineMid, Kinect.JointType.SpineShoulder }, { Kinect.JointType.SpineShoulder, Kinect.JointType.Neck }, { Kinect.JointType.Neck, Kinect.JointType.Head }, }; public void AddJoints(float recordingTime) { Vector3[] positions = new Vector3[25]; for (int i = 0; i < joints.Length; i++) { positions[i] = joints[i].position; } jointsSequence.Add(positions); recordingTimes.Add(recordingTime); } public void ResetRecording() { jointsSequence.Clear(); recordingTimes.Clear(); } public void Save() { if (jointsSequence.Count == 0) { Debug.Log("jointsSequence is empty"); return; } SaveSystem.Save(jointsSequence, recordingTimes); Debug.Log("Save success"); } public void Load() { JointsDataSequence data = SaveSystem.Load(); if (data == null) { Debug.Log("Load failed"); return; } ShowJoints(data); Debug.Log("Load success"); } private void ShowJoints(JointsDataSequence data) { List jointsData = data.jointsDataSequence; List recordingTimes = data.recordingTimes; // Filtered datas according to distance List newJointsData = new List(); List newRecordingTimes = new List(); JointsData prevJd = jointsData[0]; // Add first pose newJointsData.Add(jointsData[0]); newRecordingTimes.Add(recordingTimes[0]); for (int i = 1; i < jointsData.Count; i++) { JointsData jd = jointsData[i]; for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++) { // Skip these joints if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight) continue; Vector3 prevJointPosition = new Vector3(prevJd.jointsPositionsX[(int)jt], prevJd.jointsPositionsY[(int)jt], prevJd.jointsPositionsZ[(int)jt]); Vector3 jointPosition = new Vector3(jd.jointsPositionsX[(int)jt], jd.jointsPositionsY[(int)jt], jd.jointsPositionsZ[(int)jt]); float distance = Vector3.Distance(prevJointPosition, jointPosition); // If a joint is bigger than a certain distance, add it to the replay, else ignore it // TODO: maybe distance need to be adjusted if (distance >= 0.15f) { newJointsData.Add(jd); newRecordingTimes.Add(recordingTimes[i]); prevJd = jd; continue; } } } StartCoroutine(VisualizeFadeInSeries(jointsData, recordingTimes, newJointsData, newRecordingTimes)); } private IEnumerator Visualize(List newJointsData, List newRecordingTimes) { for (int i = 0; i < newJointsData.Count; i++) { float waitTime = 1; if (i < newRecordingTimes.Count - 1) { waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i]; } WaitForSeconds wait = new WaitForSeconds(waitTime); // Create GameObject body GameObject body = new GameObject("Recorded Body Demo " + i); for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++) { // Skip these joints if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight) continue; // Create GameObject cubes for joints GameObject jointObj = GameObject.CreatePrimitive(PrimitiveType.Cube); LineRenderer lr = jointObj.AddComponent(); lr.positionCount = 2; lr.material = new Material(Shader.Find("Sprites/Default")); lr.material.color = new Color(1, 0.8f, 0.6f); lr.startWidth = 0.05f; lr.endWidth = 0.05f; jointObj.transform.localScale = new Vector3(0.05f, 0.05f, 0.05f); jointObj.name = jt.ToString(); JointsData jd = newJointsData[i]; jointObj.transform.position = new Vector3(jd.jointsPositionsX[(int)jt], jd.jointsPositionsY[(int)jt], jd.jointsPositionsZ[(int)jt]); jointObj.transform.parent = body.transform; // Remove LineRenderer component from neck if (jt == Kinect.JointType.Neck) { Destroy(jointObj.GetComponent()); } } // Connect the joints with LineRenderer for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++) { // Skip if dictionary not contains the joint or other these joints if (!_BoneMap.ContainsKey(jt) || jt == Kinect.JointType.Neck || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight) continue; Transform jointObj = body.transform.Find(jt.ToString()); Transform targetJoint = body.transform.Find(_BoneMap[jt].ToString()); LineRenderer lr = jointObj.GetComponent(); lr.SetPosition(0, jointObj.localPosition); lr.SetPosition(1, targetJoint.localPosition); } yield return wait; Destroy(body); } } private IEnumerator VisualizeFadeIn(List newJointsData, List newRecordingTimes) { for (int i = 0; i < newJointsData.Count; i++) { float waitTime = 1; if (i < newRecordingTimes.Count - 1) { waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i]; } WaitForSeconds wait = new WaitForSeconds(waitTime); Visualizer_FadeIn body = Instantiate(vfi) as Visualizer_FadeIn; body.SetData(newJointsData[i]); body.ShowBody(); yield return wait; } } private IEnumerator VisualizeFadeInSeries(List jointsData, List recordingTimes, List newJointsData, List newRecordingTimes) { float waitTime = 3; List jointsDataDemo = new List(); List recordingTimesDemo = new List(); List jointsDataSeries = new List(); List recordingTimesSeries = new List(); int indexDemo = 0; int indexSeries = 0; while (true) { // Visualization demo for (int i = indexDemo; i < jointsData.Count; i++) { if (recordingTimes[i] < waitTime) { jointsDataDemo.Add(jointsData[i]); recordingTimesDemo.Add(recordingTimes[i]); } else { indexDemo = i; yield return Visualize(jointsDataDemo, recordingTimesDemo); bc.SetDataDemo(jointsDataDemo, recordingTimesDemo); break; } } // Visualization steps/series for (int i = indexSeries; i < newJointsData.Count; i++) { if (newRecordingTimes[i] < waitTime) { jointsDataSeries.Add(newJointsData[i]); recordingTimesSeries.Add(newRecordingTimes[i]); } else { // Show start position of steps startStepPreview.SetData(jointsDataSeries[0]); startStepPreview.ShowBody(); // Wait for input yield return ViveInput.WaitForControllerPress(); // Delete start position of steps startStepPreview.DeleteBody(); indexSeries = i; waitTime += 3; Visualizer_FadeInSeries body = Instantiate(vfis) as Visualizer_FadeInSeries; body.transform.parent = gameObject.transform; body.SetData(jointsDataSeries, recordingTimesSeries); body.ShowBody(); // Compare visualization demo with body yield return bc.StartCompare(); jointsDataDemo.Clear(); recordingTimesDemo.Clear(); jointsDataSeries.Clear(); recordingTimesSeries.Clear(); // Wait for input yield return ViveInput.WaitForControllerPress(); break; } } if (jointsDataSeries.Count != 0) break; } // Last visualization yield return Visualize(jointsDataDemo, recordingTimesDemo); bc.SetDataDemo(jointsDataDemo, recordingTimesDemo); // Show start position of steps startStepPreview.SetData(jointsDataSeries[0]); startStepPreview.ShowBody(); // Wait for input yield return ViveInput.WaitForControllerPress(); // Delete start position of steps startStepPreview.DeleteBody(); Visualizer_FadeInSeries bodyEnd = Instantiate(vfis) as Visualizer_FadeInSeries; bodyEnd.transform.parent = gameObject.transform; bodyEnd.SetData(jointsDataSeries, recordingTimesSeries); bodyEnd.ShowBody(); // Compare visualization demo with body yield return bc.StartCompare(); jointsDataDemo.Clear(); recordingTimesDemo.Clear(); jointsDataSeries.Clear(); recordingTimesSeries.Clear(); textFinish.SetActive(true); yield return new WaitForSeconds(3); textFinish.SetActive(false); } }