using System; using System.Collections; using System.Collections.Generic; using UnityEngine; using Kinect = Windows.Kinect; public class PlayerReplay : MonoBehaviour { // Joints already given the values from BodySourceView.cs that keeps updating public Transform[] joints = new Transform[25]; public List jointsSequence = new List(); public List recordingTimes = new List(); public Material transparentMat; public Visualizer_FadeIn vfi; public Visualizer_FadeInSeries vfis; public BodyComparer bc; public StartStepPreview startStepPreview; public GameObject textFinish; private Dictionary _BoneMap = new Dictionary() { { Kinect.JointType.FootLeft, Kinect.JointType.AnkleLeft }, { Kinect.JointType.AnkleLeft, Kinect.JointType.KneeLeft }, { Kinect.JointType.KneeLeft, Kinect.JointType.HipLeft }, { Kinect.JointType.HipLeft, Kinect.JointType.SpineBase }, { Kinect.JointType.FootRight, Kinect.JointType.AnkleRight }, { Kinect.JointType.AnkleRight, Kinect.JointType.KneeRight }, { Kinect.JointType.KneeRight, Kinect.JointType.HipRight }, { Kinect.JointType.HipRight, Kinect.JointType.SpineBase }, { Kinect.JointType.HandTipLeft, Kinect.JointType.HandLeft }, { Kinect.JointType.ThumbLeft, Kinect.JointType.HandLeft }, { Kinect.JointType.HandLeft, Kinect.JointType.WristLeft }, { Kinect.JointType.WristLeft, Kinect.JointType.ElbowLeft }, { Kinect.JointType.ElbowLeft, Kinect.JointType.ShoulderLeft }, { Kinect.JointType.ShoulderLeft, Kinect.JointType.SpineShoulder }, { Kinect.JointType.HandTipRight, Kinect.JointType.HandRight }, { Kinect.JointType.ThumbRight, Kinect.JointType.HandRight }, { Kinect.JointType.HandRight, Kinect.JointType.WristRight }, { Kinect.JointType.WristRight, Kinect.JointType.ElbowRight }, { Kinect.JointType.ElbowRight, Kinect.JointType.ShoulderRight }, { Kinect.JointType.ShoulderRight, Kinect.JointType.SpineShoulder }, { Kinect.JointType.SpineBase, Kinect.JointType.SpineMid }, { Kinect.JointType.SpineMid, Kinect.JointType.SpineShoulder }, { Kinect.JointType.SpineShoulder, Kinect.JointType.Neck }, { Kinect.JointType.Neck, Kinect.JointType.Head }, }; public void AddJoints(float recordingTime) { Vector3[] positions = new Vector3[25]; for (int i = 0; i < joints.Length; i++) { positions[i] = joints[i].position; } jointsSequence.Add(positions); recordingTimes.Add(recordingTime); } public void ResetRecording() { jointsSequence.Clear(); recordingTimes.Clear(); } public void Save() { if (jointsSequence.Count == 0) { Debug.Log("jointsSequence is empty"); return; } SaveSystem.Save(jointsSequence, recordingTimes); Debug.Log("Save success"); } public void Load() { JointsDataSequence data = SaveSystem.Load(); if (data == null) { Debug.Log("Load failed"); return; } ShowJoints(data); Debug.Log("Load success"); } private void ShowJoints(JointsDataSequence data) { List jointsData = data.jointsDataSequence; List recordingTimes = data.recordingTimes; StartCoroutine(VisualizeFadeInSeries(jointsData, recordingTimes)); } private IEnumerator Visualize(List newJointsData, List newRecordingTimes) { for (int i = 0; i < newJointsData.Count; i++) { float waitTime = 1; if (i < newRecordingTimes.Count - 1) { waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i]; } WaitForSeconds wait = new WaitForSeconds(waitTime); // Create GameObject body GameObject body = new GameObject("Recorded Body Demo " + i); for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++) { // Skip these joints if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight) continue; // Create GameObject cubes for joints GameObject jointObj = GameObject.CreatePrimitive(PrimitiveType.Cube); LineRenderer lr = jointObj.AddComponent(); lr.positionCount = 2; lr.material = new Material(Shader.Find("Sprites/Default")) { color = new Color(1, 0.8f, 0.6f) }; lr.startWidth = 0.05f; lr.endWidth = 0.05f; jointObj.transform.localScale = new Vector3(0.05f, 0.05f, 0.05f); jointObj.name = jt.ToString(); JointsData jd = newJointsData[i]; jointObj.transform.position = new Vector3(jd.jointsPositionsX[(int)jt], jd.jointsPositionsY[(int)jt], jd.jointsPositionsZ[(int)jt]); jointObj.transform.parent = body.transform; // Remove LineRenderer component from neck if (jt == Kinect.JointType.Neck) { Destroy(jointObj.GetComponent()); } } // Connect the joints with LineRenderer for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++) { // Skip if dictionary not contains the joint or other these joints if (!_BoneMap.ContainsKey(jt) || jt == Kinect.JointType.Neck || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight) continue; Transform jointObj = body.transform.Find(jt.ToString()); Transform targetJoint = body.transform.Find(_BoneMap[jt].ToString()); LineRenderer lr = jointObj.GetComponent(); lr.SetPosition(0, jointObj.localPosition); lr.SetPosition(1, targetJoint.localPosition); } yield return wait; Destroy(body); } } private IEnumerator VisualizeFadeIn(List newJointsData, List newRecordingTimes) { for (int i = 0; i < newJointsData.Count; i++) { float waitTime = 1; if (i < newRecordingTimes.Count - 1) { waitTime = newRecordingTimes[i + 1] - newRecordingTimes[i]; } WaitForSeconds wait = new WaitForSeconds(waitTime); Visualizer_FadeIn body = Instantiate(vfi) as Visualizer_FadeIn; body.SetData(newJointsData[i]); body.ShowBody(); yield return wait; } } private IEnumerator VisualizeFadeInSeries(List jointsData, List recordingTimes) { float waitTime = 3; List jointsDataDemo = new List(); List recordingTimesDemo = new List(); int indexDemo = 0; bool finish = false; while (true) { for (int i = indexDemo; i < jointsData.Count; i++) { if (recordingTimes[i] < waitTime) { jointsDataDemo.Add(jointsData[i]); recordingTimesDemo.Add(recordingTimes[i]); finish = true; } else { indexDemo = i; waitTime += 3; finish = false; break; } } // Visualization demo yield return Visualize(jointsDataDemo, recordingTimesDemo); bc.SetDataDemo(jointsDataDemo, recordingTimesDemo); // Visualization steps/series // Filtered datas according to distance List filteredJointsData = new List(); List filteredRecordingTimes = new List(); JointsData prevJd = jointsDataDemo[0]; // Add first pose filteredJointsData.Add(jointsDataDemo[0]); filteredRecordingTimes.Add(recordingTimesDemo[0]); for (int i = 1; i < jointsDataDemo.Count; i++) { JointsData jd = jointsDataDemo[i]; for (Kinect.JointType jt = Kinect.JointType.SpineBase; jt <= Kinect.JointType.ThumbRight; jt++) { // Skip these joints if (jt == Kinect.JointType.Head || jt == Kinect.JointType.ThumbLeft || jt == Kinect.JointType.ThumbRight || jt == Kinect.JointType.HandLeft || jt == Kinect.JointType.HandRight || jt == Kinect.JointType.HandTipLeft || jt == Kinect.JointType.HandTipRight) continue; Vector3 prevJointPosition = new Vector3(prevJd.jointsPositionsX[(int)jt], prevJd.jointsPositionsY[(int)jt], prevJd.jointsPositionsZ[(int)jt]); Vector3 jointPosition = new Vector3(jd.jointsPositionsX[(int)jt], jd.jointsPositionsY[(int)jt], jd.jointsPositionsZ[(int)jt]); float distance = Vector3.Distance(prevJointPosition, jointPosition); // If a joint is bigger than a certain distance, add it to the replay, else ignore it if (distance >= 0.15f) { filteredJointsData.Add(jd); filteredRecordingTimes.Add(recordingTimesDemo[i]); prevJd = jd; continue; } } } // Add last step of demo filteredJointsData.Add(jointsDataDemo[jointsDataDemo.Count - 1]); // Show start position of steps startStepPreview.SetData(filteredJointsData[0]); startStepPreview.ShowBody(); // Wait for input yield return ViveInput.WaitForControllerPress(); // Delete start position of steps startStepPreview.DeleteBody(); Visualizer_FadeInSeries body = Instantiate(vfis) as Visualizer_FadeInSeries; body.transform.parent = gameObject.transform; body.SetData(filteredJointsData, filteredRecordingTimes); body.ShowBody(); // Compare visualization demo with body yield return bc.StartCompare(); // Clear list jointsDataDemo.Clear(); recordingTimesDemo.Clear(); // Input for end steps. Wait for input if finish comparing. Don't wait for input if comparing is interupted. if (!bc.endStepsPressed) { // Unknown reason, need 2 times (not working if only 1) yield return ViveInput.WaitForControllerPress(); yield return ViveInput.WaitForControllerPress(); } else { // Reset bc.endStepsPressed = false; } // TODO: maybe need to check if body is null or not (after 6 seconds body is destroyed, destroy a destroyed gameobject error?) Destroy(body); // Wait for input // Unknown reason, need 2 times (not working if only 1) yield return ViveInput.WaitForControllerPress(); yield return ViveInput.WaitForControllerPress(); if (finish) break; } textFinish.SetActive(true); yield return new WaitForSeconds(3); textFinish.SetActive(false); // Write positions to csv bc.WriteCSV(); ViveInput.StopPlaying(); } }