//======= Copyright (c) Stereolabs Corporation, All rights reserved. =============== using System.Collections.Generic; using UnityEngine; using UnityEngine.Rendering; #if UNITY_EDITOR using UnityEditor; #endif /// /// Creates a garbage matte mask from its position and apply it on the pipeline. /// It's created by GreenScreenManager as a 3D object, that you can create from the Inspector by placing the bounds of the mesh. /// The garbage matte represents a region of the screen where no real pixels will be rendered, regardless of depth. /// This can be used to extend the bounds of a greenscreen when the physical screen isn't large enough to fill the background. /// [RequireComponent(typeof(GreenScreenManager))] [RequireComponent(typeof(Camera))] public class GarbageMatte { /// /// Reference to the ZEDCamera. /// private sl.ZEDCamera zed; /// /// Position in the render queue used by Unity's renderer to render a mesh transparent. This gets appled to the shader. /// private const int QUEUE_TRANSPARENT_VALUE = 3000; /// /// List of 3D points used to make the matte mesh, eg the "corners". /// private List points = new List(); /// /// The current camera looking at the scene, used to transform ScreenPosition to worldPosition when placing the boundary points. /// private Camera cam; /// /// List of the gameObjects used by the mesh. /// private List go = null; /// /// List of the meshes. /// private List meshFilters; /// /// Triangles of the current mesh. /// private List triangles = new List(); /// /// The sphere objects the user places via GreenScreenManager to define the bounds of the matte object. /// private List borderspheres = new List(); /// /// The ZED greenscreen material. Usually Mat_ZED_GreenScreen. /// private Material shader_greenScreen; /// /// The material used on the spheres that the user places via GreenScreenManager to define the bounds of the matte object. /// Usually Mat_ZED_Outlined. /// private Material outlineMaterial; /// /// Whether or not the maatte is currently being edited. /// private bool isClosed = false; /// /// The index of meshFilters that refers to the plane mesh we're currently editing, if applicable. /// private int currentPlaneIndex; /// /// The Unity layer where spheres exist, for visibility reasons. /// private int sphereLayer = 21; /// /// The Unity CommandBuffer that gets applied to the camera, which results in the matte getting rendered. /// private CommandBuffer commandBuffer; [SerializeField] [HideInInspector] public string garbageMattePath = "garbageMatte.cfg"; [SerializeField] [HideInInspector] public bool editMode = true; [SerializeField] [HideInInspector] public bool loadAtStart = false; private Transform target; private bool isInit = false; public bool IsInit { get { return isInit; } } /// /// Constructor that sets up the garbage matte for the desired camera in the desired place. /// /// Camera in which to apply the matte effect /// Material reference, usually Mat_ZED_Greenscreen /// Center location of the matte effect /// Optional reference to another garbage matte, used to copy its current edit mode. public GarbageMatte(ZEDManager camManager, Material greenScreenMaterial, Transform target, GarbageMatte matte) { this.target = target; currentPlaneIndex = 0; zed = camManager.zedCamera; this.cam = camManager.GetComponentInChildren(); points.Clear(); outlineMaterial = Resources.Load("Materials/Mat_ZED_Outlined") as Material; go = new List(); meshFilters = new List(); shader_greenScreen = greenScreenMaterial; ResetPoints(false); if (matte != null) { editMode = matte.editMode; } if (commandBuffer == null) { //Create a command buffer to clear the depth and stencil commandBuffer = new CommandBuffer(); commandBuffer.name = "GarbageMatte"; commandBuffer.SetRenderTarget(BuiltinRenderTextureType.CurrentActive, BuiltinRenderTextureType.Depth); //Remove the previous command buffer to set the garbage matte first CommandBuffer[] cmd = cam.GetCommandBuffers(CameraEvent.BeforeDepthTexture); cam.RemoveCommandBuffers(CameraEvent.BeforeDepthTexture); if(cmd.Length > 0) { cam.AddCommandBuffer(CameraEvent.BeforeDepthTexture, commandBuffer); for(int i = 0; i < cmd.Length; ++i) { cam.AddCommandBuffer(CameraEvent.BeforeDepthTexture, cmd[i]); } } } if (loadAtStart && Load()) { Debug.Log("Config garbage matte found, and loaded ( " + garbageMattePath + " )"); ApplyGarbageMatte(); editMode = false; } isInit = true; } /// /// Constructor to create a dummy garbage matte that does nothing. Should be used only as a cache in memory /// public GarbageMatte() { isInit = false; } private List indexSelected = new List(); private List currentGOSelected = new List(); private List meshFilterSelected = new List(); private List planeSelectedIndex = new List(); private int numberSpheresSelected = -1; /// /// Update the garbage matte and manage the movement of the spheres. /// public void Update() { if (editMode) { // if at least a sphere is selected if (numberSpheresSelected != -1) { if (zed.IsCameraReady) { Vector3 vec = cam.ScreenToWorldPoint(new Vector4(Input.mousePosition.x, Input.mousePosition.y, zed.GetDepthValue(Input.mousePosition))); // For each sphere selected move their position with the mouse for (int i = 0; i < currentGOSelected.Count; ++i) { currentGOSelected[i].transform.position = vec; } } } if (zed != null && zed.IsCameraReady) { //If left mouse is clicked, add a sphere if (Input.GetMouseButtonDown(0)) { //Add a new plane if needed if (go.Count - 1 < currentPlaneIndex) { go.Add(CreateGameObject()); go[currentPlaneIndex].GetComponent().material.renderQueue = QUEUE_TRANSPARENT_VALUE + 5; meshFilters[currentPlaneIndex] = go[currentPlaneIndex].GetComponent(); meshFilters[currentPlaneIndex].sharedMesh = CreateMesh(); meshFilters[currentPlaneIndex].sharedMesh.MarkDynamic(); } if (numberSpheresSelected != -1) { //Remove outline from the sphere cause a sphere was selected //Clear the meshes and spheres selected for (int i = 0; i < currentGOSelected.Count; ++i) { currentGOSelected[i].GetComponent().material.SetFloat("_Outline", 0.00f); currentGOSelected[i] = null; } currentGOSelected.Clear(); for (int i = 0; i < meshFilterSelected.Count; ++i) { meshFilterSelected[i].mesh.Clear(); } meshFilterSelected.Clear(); //Create the planes if needed for (int i = 0; i < planeSelectedIndex.Count; ++i) { if ((borderspheres.Count - planeSelectedIndex[i] * 4) < 4) { numberSpheresSelected = -1; planeSelectedIndex.Clear(); return; } List triangles = new List(); points = new List(); for (int j = planeSelectedIndex[i] * 4; j < (planeSelectedIndex[i] + 1) * 4; j++) { points.Add(borderspheres[j].transform.position); } CloseShape(triangles, points, planeSelectedIndex[i]); } numberSpheresSelected = -1; return; } // Add a sphere else if (points.Count < 100 && !isClosed) { Vector3 vec = cam.ScreenToWorldPoint(new Vector4(Input.mousePosition.x, Input.mousePosition.y, zed.GetDepthValue(Input.mousePosition))); RaycastHit hit; if (Physics.Raycast(target.position, (vec - target.position), out hit, 10, (1 << sphereLayer))) { int hitIndex = borderspheres.IndexOf(hit.transform.gameObject); vec = borderspheres[hitIndex].transform.position; } points.Add(vec); GameObject sphere = GameObject.CreatePrimitive(PrimitiveType.Sphere); sphere.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f); sphere.hideFlags = HideFlags.HideInHierarchy; sphere.tag = "HelpObject"; sphere.GetComponent().material = outlineMaterial; sphere.GetComponent().material.SetFloat("_Outline", 0.02f); sphere.transform.position = points[points.Count - 1]; sphere.layer = sphereLayer; borderspheres.Add(sphere); if (borderspheres.Count >= 2) { borderspheres[borderspheres.Count - 2].GetComponent().material.SetFloat("_Outline", 0.00f); } if (borderspheres.Count % 4 == 0) { points = new List(); for (int i = currentPlaneIndex * 4; i < (currentPlaneIndex + 1) * 4; i++) { points.Add(borderspheres[i].transform.position); } CloseShape(triangles, points, currentPlaneIndex); EndPlane(); } } } //Select the sphere, to move them if (Input.GetMouseButtonDown(1)) { if (numberSpheresSelected != -1) return; Vector3 vec = cam.ScreenToWorldPoint(new Vector4(Input.mousePosition.x, Input.mousePosition.y, zed.GetDepthValue(Input.mousePosition))); RaycastHit[] hits; hits = Physics.RaycastAll(target.position, (vec - target.position), 10, (1 << sphereLayer)); if (hits.Length > 0) { indexSelected.Clear(); currentGOSelected.Clear(); planeSelectedIndex.Clear(); meshFilterSelected.Clear(); for (int i = 0; i < hits.Length; ++i) { int hitIndex = borderspheres.IndexOf(hits[i].transform.gameObject); indexSelected.Add(hitIndex); currentGOSelected.Add(borderspheres[hitIndex]); planeSelectedIndex.Add(hitIndex / 4); meshFilterSelected.Add(meshFilters[planeSelectedIndex[planeSelectedIndex.Count - 1]]); borderspheres[hitIndex].GetComponent().material.SetFloat("_Outline", 0.02f); } numberSpheresSelected = hits.Length; borderspheres[borderspheres.Count - 1].GetComponent().material.SetFloat("_Outline", 0.00f); } else { numberSpheresSelected = -1; } } } } } /// /// Finishes the current plane and increases the index of the plane /// private void EndPlane() { currentPlaneIndex++; ResetDataCurrentPlane(); } /// /// Enables editing the matte. /// public void EnterEditMode() { if (isClosed) { foreach (GameObject s in borderspheres) { s.SetActive(true); } if (shader_greenScreen != null) { Shader.SetGlobalInt("_ZEDStencilComp", 0); } for (int i = 0; i < go.Count; i++) { if (go[i] == null) continue; go[i].GetComponent().sharedMaterial.SetFloat("alpha", 0.5f); go[i].GetComponent().sharedMaterial.renderQueue = QUEUE_TRANSPARENT_VALUE + 5; } isClosed = false; } } /// /// Removes the last sphere the user placed while defining the matte object's boundaries. /// public void RemoveLastPoint() { //Prevent to remove and move a sphere at the same time if (numberSpheresSelected != -1) return; if (isClosed) { foreach (GameObject s in borderspheres) { s.SetActive(true); } if (shader_greenScreen != null) { Shader.SetGlobalInt("_ZEDStencilComp", 0); } for (int i = 0; i < go.Count; i++) { if (go[i] == null) continue; go[i].GetComponent().sharedMaterial.SetFloat("alpha", 0.5f); go[i].GetComponent().sharedMaterial.renderQueue = QUEUE_TRANSPARENT_VALUE + 5; } isClosed = false; } if (borderspheres.Count % 4 == 0 && currentPlaneIndex > 0) { GameObject.Destroy(go[currentPlaneIndex - 1]); go.RemoveAll(item => item == null); meshFilters.RemoveAll(item => item == null); meshFilters[currentPlaneIndex - 1].sharedMesh.Clear(); currentPlaneIndex--; } if (borderspheres != null && borderspheres.Count > 0) { GameObject.DestroyImmediate(borderspheres[borderspheres.Count - 1]); borderspheres.RemoveAt(borderspheres.Count - 1); if (borderspheres.Count % 4 == 0 && borderspheres.Count > 0) { borderspheres[borderspheres.Count - 1].GetComponent().material.SetFloat("_Outline", 0.02f); } } } /// /// Clears the boundary points and triangles. Used before making a new plane, or when resetting all data. /// private void ResetDataCurrentPlane() { points.Clear(); triangles.Clear(); } /// /// Removes existing sphere objects used to define bounds. /// Used to ensure they're properly cleaned up when not being used to edit the garbage matte. /// public void CleanSpheres() { GameObject[] remain_sphere2 = GameObject.FindGameObjectsWithTag ("HelpObject"); if (remain_sphere2.Length > 0) { foreach (GameObject sph in remain_sphere2) GameObject.DestroyImmediate (sph); } } /// /// Destroys all planes and spheres used to edit the matte to start from scratch. /// /// public void ResetPoints(bool cleansphere) { if (cleansphere) { GameObject[] remain_sphere2 = GameObject.FindGameObjectsWithTag ("HelpObject"); if (remain_sphere2.Length > 0) { foreach (GameObject sph in remain_sphere2) GameObject.Destroy (sph); } } Shader.SetGlobalInt("_ZEDStencilComp", 0); if (go == null) return; isClosed = false; currentPlaneIndex = 0; for (int i = 0; i < go.Count; i++) { GameObject.DestroyImmediate(go[i]); } go.Clear(); meshFilters.Clear(); ResetDataCurrentPlane(); if (borderspheres != null) { foreach (GameObject s in borderspheres) { GameObject.DestroyImmediate(s); } } borderspheres.Clear(); if (commandBuffer != null) { commandBuffer.Clear(); } points.Clear (); } /// /// Helper function to determine a point's orientation along the plane. /// Used by OrderPoints to sort vertices. /// /// /// /// /// /// /// 1 if it should be higher on the list, 0 if it should be lower. private static int Orientation(Vector3 p1, Vector3 p2, Vector3 p, Vector3 X, Vector3 Y) { return (Vector3.Dot(p2, X) - Vector3.Dot(p1, X)) * (Vector3.Dot(p, Y) - Vector3.Dot(p1, Y)) - (Vector3.Dot(p, X) - Vector3.Dot(p1, X)) * (Vector3.Dot(p2, Y) - Vector3.Dot(p1, Y)) > 0 ? 1 : 0; } /// /// Orders the points in the points list in an order proper for drawing a mesh. /// Points need to appear in the list in clockwise order within a triangle being drawn with them, around the plane's normal. /// /// private List OrderPoints(List points) { Vector3 normal = Vector3.Cross((points[1] - points[0]), (points[2] - points[0])); normal.Normalize(); Vector3 X = new Vector3(-normal.y, normal.x, 0); X.Normalize(); Vector3 Y = Vector3.Cross(X, normal); Y.Normalize(); List orderedIndex = new List(); List convexHull = new List(); float minX = Vector3.Dot(points[0], X); Vector3 p = points[0]; for (int i = 0; i < points.Count; i++) { if (Vector3.Dot(points[i], X) < minX) { minX = Vector3.Dot(points[i], X); p = points[i]; } } Vector3 currentTestPoint; for (int i = 0; i < 4; i++) { convexHull.Add(p); orderedIndex.Add(points.IndexOf(p)); currentTestPoint = points[0]; for (int j = 0; j < points.Count; j++) { if ((currentTestPoint == p) || (Orientation(p, currentTestPoint, points[j], X, Y) == 1)) { currentTestPoint = points[j]; } } p = currentTestPoint; } return orderedIndex; } /// /// Finish off the last quad mesh. /// public void CloseShape(List triangles, List points, int currentPlaneIndex) { triangles.Clear(); List indexOrder = OrderPoints(points); triangles.Add(indexOrder[0]); triangles.Add(indexOrder[1]); triangles.Add(indexOrder[2]); triangles.Add(indexOrder[0]); triangles.Add(indexOrder[2]); triangles.Add(indexOrder[3]); if (go[currentPlaneIndex] == null) { go[currentPlaneIndex] = CreateGameObject(); meshFilters[currentPlaneIndex] = go[currentPlaneIndex].GetComponent(); meshFilters[currentPlaneIndex].sharedMesh = CreateMesh(); meshFilters[currentPlaneIndex].sharedMesh.MarkDynamic(); } go[currentPlaneIndex].GetComponent().sharedMesh.Clear(); go[currentPlaneIndex].GetComponent().sharedMesh.vertices = points.ToArray(); go[currentPlaneIndex].GetComponent().sharedMesh.triangles = triangles.ToArray(); borderspheres[borderspheres.Count - 1].GetComponent().material.SetFloat("_Outline", 0.00f); } /// /// Apply the garbage matte by rendering into the stencil buffer. /// public void ApplyGarbageMatte() { if (currentPlaneIndex <= 0) { editMode = false; ResetPoints(false); return; } if (shader_greenScreen != null) { isClosed = true; foreach (GameObject s in borderspheres) { s.SetActive(false); } Shader.SetGlobalInt("_ZEDStencilComp", 3); for (int i = 0; i < go.Count; i++) { if (go[i] == null) continue; go[i].GetComponent().sharedMaterial.SetFloat("alpha", 0.0f); go[i].GetComponent().sharedMaterial.renderQueue = QUEUE_TRANSPARENT_VALUE - 5; } } commandBuffer.Clear(); for (int i = 0; i < go.Count; ++i) { if (go[i] != null) { commandBuffer.DrawMesh(go[i].GetComponent().mesh, go[i].transform.localToWorldMatrix, go[i].GetComponent().material); } } editMode = false; } private void OnApplicationQuit() { ResetPoints(true); } /// /// Create a hidden GameObject used to hold the editing components. /// /// private GameObject CreateGameObject() { GameObject plane = new GameObject("PlaneTest"); plane.hideFlags = HideFlags.HideInHierarchy; meshFilters.Add((MeshFilter)plane.AddComponent(typeof(MeshFilter))); MeshRenderer renderer = plane.AddComponent(typeof(MeshRenderer)) as MeshRenderer; renderer.sharedMaterial = new Material(Resources.Load("Materials/Mat_ZED_Mask_Quad") as Material); renderer.sharedMaterial.SetFloat("alpha", 0.5f); return plane; } private Mesh CreateMesh() { Mesh m = new Mesh(); m.name = "ScriptedMesh"; triangles.Add(0); triangles.Add(1); triangles.Add(2); return m; } /// /// Represents a single plane to be made into a mesh, then a matte. /// [System.Serializable] public struct Plane { public int numberVertices; public List vertices; } /// /// Holds all planes to be turned into mattes and the total number of meshes. /// [System.Serializable] public struct GarbageMatteData { public int numberMeshes; public List planes; } /// /// Packages the current garbage matte into GarbageMatteData, which can be serialized/saved by GreenScreenEditor. /// /// Data ready to be serialized. public GarbageMatteData RegisterData() { GarbageMatteData garbageMatteData = new GarbageMatteData(); if (meshFilters == null) return garbageMatteData; garbageMatteData.numberMeshes = meshFilters.Count; garbageMatteData.planes = new List(); for (int i = 0; i < meshFilters.Count ; i++) { Vector3[] vertices = meshFilters[i].mesh.vertices; Plane p = new Plane(); p.numberVertices = vertices.Length; p.vertices = new List(vertices); garbageMatteData.planes.Add(p); //garbageMatteData.plane.ad } return garbageMatteData; } /// /// Loads a serialized GarbageMatteData instance to be used/viewed/edited. /// /// /// True if there was actual data to load (at least one plane). public bool LoadData(GarbageMatteData garbageMatteData) { int nbMesh = garbageMatteData.numberMeshes; if (nbMesh < 0) return false; currentPlaneIndex = 0; ResetPoints(false); for (int i = 0; i < nbMesh; i++) { points.Clear(); triangles.Clear(); go.Add(CreateGameObject()); go[currentPlaneIndex].GetComponent().material.renderQueue = QUEUE_TRANSPARENT_VALUE + 5; meshFilters[currentPlaneIndex] = go[currentPlaneIndex].GetComponent(); meshFilters[currentPlaneIndex].sharedMesh = CreateMesh(); meshFilters[currentPlaneIndex].sharedMesh.MarkDynamic(); Plane p = garbageMatteData.planes[i]; for (int j = 0; j < p.numberVertices; j++) { points.Add(p.vertices[j]); GameObject sphere = GameObject.CreatePrimitive(PrimitiveType.Sphere); sphere.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f); sphere.tag = "HelpObject"; sphere.hideFlags = HideFlags.HideInHierarchy; outlineMaterial.SetFloat("_Outline", 0.00f); sphere.GetComponent().material = outlineMaterial; sphere.transform.position = points[points.Count - 1]; sphere.layer = sphereLayer; borderspheres.Add(sphere); } if (go.Count == 0) return false; CloseShape(triangles, points, currentPlaneIndex); EndPlane(); } return true; } /// /// Save the points into a file. /// public void Save() { List meshes = new List(); meshes.Add((meshFilters.Count - 1).ToString()); for (int i = 0; i < meshFilters.Count - 1; i++) { Vector3[] vertices = meshFilters[i].mesh.vertices; int[] tri = meshFilters[i].mesh.triangles; meshes.Add("v#" + vertices.Length); for (int j = 0; j < vertices.Length; j++) { meshes.Add(vertices[j].x + " " + vertices[j].y + " " + vertices[j].z); } } System.IO.File.WriteAllLines(garbageMattePath, meshes.ToArray()); } /// /// Load the current shape /// public bool Load() { if (!System.IO.File.Exists(garbageMattePath)) return false; string[] meshes = System.IO.File.ReadAllLines(garbageMattePath); if (meshes == null) return false; int nbMesh = int.Parse(meshes[0]); if (nbMesh < 0) return false; currentPlaneIndex = 0; ResetPoints(false); int lineCount = 1; string[] splittedLine; for (int i = 0; i < nbMesh; i++) { points.Clear(); triangles.Clear(); go.Add(CreateGameObject()); go[currentPlaneIndex].GetComponent().material.renderQueue = QUEUE_TRANSPARENT_VALUE + 5; meshFilters[currentPlaneIndex] = go[currentPlaneIndex].GetComponent(); meshFilters[currentPlaneIndex].sharedMesh = CreateMesh(); meshFilters[currentPlaneIndex].sharedMesh.MarkDynamic(); splittedLine = meshes[lineCount].Split('#'); lineCount++; int nbVertices = int.Parse(splittedLine[1]); for (int j = 0; j < nbVertices; j++) { splittedLine = meshes[lineCount].Split(' '); lineCount++; float x = float.Parse(splittedLine[0]), y = float.Parse(splittedLine[1]), z = float.Parse(splittedLine[2]); points.Add(new Vector3(x, y, z)); GameObject sphere = GameObject.CreatePrimitive(PrimitiveType.Sphere); sphere.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f); sphere.hideFlags = HideFlags.HideInHierarchy; sphere.tag = "HelpObject"; sphere.transform.position = points[points.Count - 1]; sphere.layer = sphereLayer; borderspheres.Add(sphere); } if (go.Count == 0) return false; CloseShape(triangles, points, currentPlaneIndex); EndPlane(); } return true; } }