using System; using System.Collections; using System.Collections.Generic; using System.Linq; using UnityEngine; using UnityEngine.UI; /// /// For the ZED 2D Object Detection sample. /// Listens for new object detections (via the ZEDManager.OnObjectDetection event) and moves + resizes canvas prefabs /// to represent them. /// Works by instantiating a pool of prefabs, and each frame going through the DetectedFrame received from the event /// to make sure each detected object has a representative GameObject. Also disables GameObjects whose objects are no /// longer visible and returns them to the pool. /// public class ZED2DObjectVisualizer : MonoBehaviour { /// /// The scene's ZEDManager. /// If you want to visualize detections from multiple ZEDs at once you will need multiple ZED3DObjectVisualizer commponents in the scene. /// [Tooltip("The scene's ZEDManager.\r\n" + "If you want to visualize detections from multiple ZEDs at once you will need multiple ZED3DObjectVisualizer commponents in the scene. ")] public ZEDManager zedManager; /// /// The scene's canvas. This will be adjusted to have required settings/components so that the bounding boxes /// will line up properly with the ZED video feed. /// [Tooltip("The scene's canvas. This will be adjusted to have required settings/components so that the bounding boxes " + "will line up properly with the ZED video feed.")] public Canvas canvas; /// /// If true, the ZED Object Detection manual will be started as soon as the ZED is initiated. /// This avoids having to press the Start Object Detection button in ZEDManager's Inspector. /// [Tooltip("If true, the ZED Object Detection manual will be started as soon as the ZED is initiated. " + "This avoids having to press the Start Object Detection button in ZEDManager's Inspector.")] public bool startObjectDetectionAutomatically = true; /// /// Prefab object that's instantiated to represent detected objects. /// This should ideally be the 2D Bounding Box prefab. But otherwise, it expects the object to have a BBox2DHandler script in the root object, /// and the RectTransform should be bottom-left-aligned (pivot set to 0, 0). /// [Space(5)] [Header("Box Appearance")] [Tooltip("Prefab object that's instantiated to represent detected objects. " + "This class expects the object to have the default Unity cube as a mesh - otherwise, it may be scaled incorrectly.\r\n" + "It also expects a BBox3DHandler component in the root object, but you won't have errors if it lacks one. ")] public GameObject boundingBoxPrefab; /// /// The colors that will be cycled through when assigning colors to new bounding boxes. /// [Tooltip("The colors that will be cycled through when assigning colors to new bounding boxes. ")] //[ColorUsage(true, true)] //Uncomment to enable HDR colors in versions of Unity that support it. public List boxColors = new List() { new Color(.231f, .909f, .69f, 1), new Color(.098f, .686f, .816f, 1), new Color(.412f, .4f, .804f, 1), new Color(1, .725f, 0f, 1), new Color(.989f, .388f, .419f, 1) }; /// /// When a detected object is first given a box and assigned a color, we store it so that if the object /// disappears and appears again later, it's assigned the same color. /// This is also solvable by making the color a function of the ID number itself, but then you can get /// repeat colors under certain conditions. /// private Dictionary idColorDict = new Dictionary(); /// /// If true, draws a 2D mask over where the SDK believes the detected object is. /// [Space(5)] [Header("Mask")] public bool showObjectMask = true; /// /// Used to warn the user only once if they enable the mask but the mask was not enabled when object detection was initialized. See OnValidate. /// private bool lastShowObjectMaskValue; /// /// Display bounding boxes of objects that are actively being tracked by object tracking, where valid positions are known. /// [Space(5)] [Header("Filters")] [Tooltip("Display bounding boxes of objects that are actively being tracked by object tracking, where valid positions are known. ")] public bool showONTracked = true; /// /// Display bounding boxes of objects that were actively being tracked by object tracking, but that were lost very recently. /// [Tooltip("Display bounding boxes of objects that were actively being tracked by object tracking, but that were lost very recently.")] public bool showSEARCHINGTracked = false; /// /// Display bounding boxes of objects that are visible but not actively being tracked by object tracking (usually because object tracking is disabled in ZEDManager). /// [Tooltip("Display bounding boxes of objects that are visible but not actively being tracked by object tracking (usually because object tracking is disabled in ZEDManager).")] public bool showOFFTracked = false; /// /// Used to know which of the available colors will be assigned to the next bounding box to be used. /// private int nextColorIndex = 0; /// /// Pre-instantiated bbox prefabs currently not in use. /// private Stack bboxPool = new Stack(); /// /// All active RectTransforms within GameObjects that were instantiated to the prefab and that currently represent a detected object. /// Key is the object's objectID. /// private Dictionary liveBBoxes = new Dictionary(); /// /// List of all 2D masks created in a frame. Used so that they can all be disposed of in the frame afterward. /// private List lastFrameMasks = new List(); private void Start() { if (!zedManager) { zedManager = FindObjectOfType(); } zedManager.OnObjectDetection += Visualize2DBoundingBoxes; zedManager.OnZEDReady += OnZEDReady; if (!canvas) //If we don't have a canvas in the scene, we need one. { GameObject canvasgo = new GameObject("Canvas - " + zedManager.name); canvas = canvasgo.AddComponent(); } lastShowObjectMaskValue = showObjectMask; } private void OnZEDReady() { if (startObjectDetectionAutomatically && !zedManager.IsObjectDetectionRunning) { zedManager.StartObjectDetection(); } //Enforce some specific settings on the canvas that are needed for things to line up. canvas.renderMode = RenderMode.ScreenSpaceCamera; canvas.worldCamera = zedManager.GetLeftCamera(); //Canvas needs to have its plane distance set within the camera's view frustum. canvas.planeDistance = 1; CanvasScaler scaler = canvas.GetComponent(); if (!scaler) { scaler = canvas.gameObject.AddComponent(); } scaler.uiScaleMode = CanvasScaler.ScaleMode.ScaleWithScreenSize; scaler.referenceResolution = new Vector2(zedManager.zedCamera.ImageWidth, zedManager.zedCamera.ImageHeight); } //TEST private void Update() { //zedManager.GetLeftCamera().ResetProjectionMatrix(); } /// /// Given a frame of object detections, positions a canvas object to represent every visible object /// to encompass the object within the 2D image from the ZED. /// Called from ZEDManager.OnObjectDetection each time there's a new detection frame available. /// public void Visualize2DBoundingBoxes(DetectionFrame dframe) { //Clear any masks that were displayed last frame, to avoid memory leaks. DestroyLastFrameMaskTextures(); //Debug.Log("Received frame with " + dframe.detectedObjects.Count + " objects."); //Get a list of all active IDs from last frame, and we'll remove each box that's visible this frame. //At the end, we'll clear the remaining boxes, as those are objects no longer visible to the ZED. List activeids = liveBBoxes.Keys.ToList(); List newobjects = dframe.GetFilteredObjectList(showONTracked, showSEARCHINGTracked, showOFFTracked); //Test just setting box to first available. foreach (DetectedObject dobj in newobjects) { //Remove the ID from the list we'll use to clear no-longer-visible boxes. if (activeids.Contains(dobj.id)) activeids.Remove(dobj.id); //Get the relevant box. This function will create a new one if it wasn't designated yet. RectTransform bbox = GetBBoxForObject(dobj); BBox2DHandler idtext = bbox.GetComponentInChildren(); if (idtext) { float disttobox = Vector3.Distance(dobj.detectingZEDManager.GetLeftCameraTransform().position, dobj.Get3DWorldPosition()); idtext.SetDistance(disttobox); } #if UNITY_2018_3_OR_NEWER float xmod = canvas.GetComponent().rect.width / zedManager.zedCamera.ImageWidth; Rect objrect = dobj.Get2DBoundingBoxRect(xmod); #else Rect objrect = dobj.Get2DBoundingBoxRect(); #endif //Adjust the size of the RectTransform to encompass the object. bbox.sizeDelta = new Vector2(objrect.width, objrect.height); bbox.anchoredPosition = new Vector2(objrect.x, objrect.y); /* #if UNITY_2018_3_OR_NEWER float xmod = canvas.GetComponent().rect.width / zedManager.zedCamera.ImageWidth; bbox.anchoredPosition = new Vector2(bbox.anchoredPosition.x * xmod, bbox.anchoredPosition.y); bbox.sizeDelta *= xmod; #endif */ //Apply the mask. if (showObjectMask) { //Make a new image for this new mask. Texture2D maskimage; if (dobj.GetMaskTexture(out maskimage, false)) { idtext.SetMaskImage(maskimage); //Apply to 2D bbox. lastFrameMasks.Add(maskimage); //Cache the texture so it's deleted next time we update our objects. } } } //Remove boxes for objects that the ZED can no longer see. foreach (int id in activeids) { ReturnBoxToPool(id, liveBBoxes[id]); } SortActiveObjectsByDepth(); //Sort all object transforms so that ones with further depth appear behind objects that are closer. } /// /// Returs the RectTransform within the GameObject (instantiated from boundingBoxPrefab) that represents the provided DetectedObject. /// If none exists, it retrieves one from the pool (or instantiates a new one if none is available) and /// sets it up with the proper ID and colors. /// private RectTransform GetBBoxForObject(DetectedObject dobj) { if (!liveBBoxes.ContainsKey(dobj.id)) { GameObject newbox = GetAvailableBBox(); newbox.transform.SetParent(canvas.transform, false); newbox.name = "Object #" + dobj.id; Color col; if (idColorDict.ContainsKey(dobj.id)) { col = idColorDict[dobj.id]; } else { col = GetNextColor(); idColorDict.Add(dobj.id, col); } BBox2DHandler boxhandler = newbox.GetComponent(); if (boxhandler) { boxhandler.SetColor(col); boxhandler.SetID(dobj.id); } RectTransform newrecttrans = newbox.GetComponent(); if (!newrecttrans) { Debug.LogError("BBox prefab needs a RectTransform in the root object."); return null; } liveBBoxes[dobj.id] = newrecttrans; return newrecttrans; } else return liveBBoxes[dobj.id]; } /// /// Gets an available GameObject (instantiated from boundingBoxPrefab) from the pool, /// or instantiates a new one if none are available. /// /// private GameObject GetAvailableBBox() { if (bboxPool.Count == 0) { GameObject newbbox = Instantiate(boundingBoxPrefab); newbbox.transform.SetParent(transform, false); bboxPool.Push(newbbox); } GameObject bbox = bboxPool.Pop(); bbox.SetActive(true); return bbox; } /// /// Disables a RectTransform's GameObject that was being used to represent an object (of the given id) and /// puts it back into the pool for later use. /// private void ReturnBoxToPool(int id, RectTransform bbox) { bbox.gameObject.SetActive(false); bbox.name = "Unused"; bboxPool.Push(bbox.gameObject); if (liveBBoxes.ContainsKey(id)) { liveBBoxes.Remove(id); } else { Debug.LogError("Tried to remove object ID " + id + " from active bboxes, but it wasn't in the dictionary."); } } /// /// Returns a color from the boxColors list. /// Colors are returned sequentially in order of their appearance in that list. /// /// private Color GetNextColor() { if (boxColors.Count == 0) { return new Color(.043f, .808f, .435f, 1); } if (nextColorIndex >= boxColors.Count) { nextColorIndex = 0; } Color returncol = boxColors[nextColorIndex]; nextColorIndex++; return returncol; } /// /// Sorts all objects in the canvas based on their distance from the camera, so that closer objects overlap further objects. /// private void SortActiveObjectsByDepth() { List handlers = new List(); foreach (Transform child in canvas.transform) { BBox2DHandler handler = child.GetComponent(); if (handler) handlers.Add(handler); } handlers.Sort((x, y) => y.currentDistance.CompareTo(x.currentDistance)); for (int i = 0; i < handlers.Count; i++) { handlers[i].transform.SetSiblingIndex(i); } } /// /// Destroys all textures added to the lastFrameMasks the last time Object Detection was called. /// Called when we're done using them (before updating with new data) to avoid memory leaks. /// private void DestroyLastFrameMaskTextures() { if (lastFrameMasks.Count > 0) { for (int i = 0; i < lastFrameMasks.Count; i++) { Destroy(lastFrameMasks[i]); } lastFrameMasks.Clear(); } } private void OnValidate() { //If the user changes the showObjectMask setting to true, warn them if its ZEDManager has objectDetection2DMask set to false, because masks won't show up. if (Application.isPlaying && showObjectMask != lastShowObjectMaskValue) { lastShowObjectMaskValue = showObjectMask; if (!zedManager) zedManager = ZEDManager.GetInstance(sl.ZED_CAMERA_ID.CAMERA_ID_01); if(showObjectMask == true && zedManager != null && zedManager.objectDetection2DMask == false) { Debug.LogError("ZED2DObjectVisualizer has showObjectMask enabled, but its ZEDManager has objectDetection2DMask disabled. " + "objectDetection2DMask must be enabled when Object Detection is started or masks will not be visible."); } } } private void OnDestroy() { if (zedManager) { zedManager.OnObjectDetection -= Visualize2DBoundingBoxes; zedManager.OnZEDReady -= OnZEDReady; } DestroyLastFrameMaskTextures(); } }