Merge branch 'fignertracker'

Conflicts:
	bbiwarg/bbiwarg.csproj

bbiwarg/Constants.cs

@@ -39,6 +39,7 @@ namespace bbiwarg
         public static readonly int FingerMinSize = 5;
         public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
         public static readonly int FingerNumFramesUntilTracked = 2;
+        public static readonly int FingerNumFramesUntilLost = 3;
         public static readonly int FingerOutSliceFactor = 10;
         public static readonly int FingerContourMargin = 2;
         public static readonly int FingerSliceOverlapFactor = 2;

bbiwarg/Recognition/FingerRecognition/FingerHistory.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bbiwarg.Recognition.FingerRecognition
+{
+    public enum TrackingState
+    {
+        None,
+        Detected,
+        Tracked,
+        Lost,
+    }
+
+    class FingerHistory
+    {
+        public List<Finger> Fingers { get; private set; }
+        public List<TrackingState> States { get; private set; }
+        public Finger LastFinger { get; private set; }
+        public Finger CurrentFinger { get { return Fingers[Fingers.Count - 1]; } }
+        public TrackingState CurrentState { get { return States[States.Count - 1]; } }
+
+
+        public FingerHistory(Finger finger)
+        {
+            Fingers = new List<Finger>();
+            States = new List<TrackingState>();
+
+            Fingers.Add(finger);
+            States.Add(TrackingState.Detected);
+            LastFinger = finger;
+        }
+
+        public void addFinger(Finger finger)
+        {
+            TrackingState previousState = CurrentState;
+            TrackingState newState = TrackingState.None;
+            int numFramesInCurrentState = getNumFramesInCurrentState();
+
+            if (finger == null)
+                newState = TrackingState.Lost;
+            else if (previousState == TrackingState.Tracked)
+                newState = TrackingState.Tracked;
+            else if (previousState == TrackingState.Detected)
+            {
+                if (numFramesInCurrentState == Constants.FingerNumFramesUntilTracked)
+                    newState = TrackingState.Tracked;
+                else
+                    newState = TrackingState.Detected;
+            }
+
+            Fingers.Add(finger);
+            States.Add(newState);
+            if (finger != null)
+                LastFinger = finger;
+        }
+
+        public int getNumFramesInCurrentState()
+        {
+            TrackingState currentState = CurrentState;
+            int count = 0;
+            for (int i = States.Count - 1; i >= 0; i++)
+            {
+                if (States[i] == currentState)
+                    count++;
+                else
+                    break;
+            }
+            return count;
+        }
+    }
+}

bbiwarg/Recognition/FingerRecognition/FingerTracker.cs

@@ -9,79 +9,129 @@ using bbiwarg.Graphics;
 
 namespace bbiwarg.Recognition.FingerRecognition
 {
+    struct FingerSimilarity
+    {
+        public float similarity;
+        public FingerHistory fingerHistory;
+        public Finger fingerDetected;
+    }
+
     class FingerTracker
     {
-        private OutputImage outputImage;
-        private List<Finger>[] detectedFingers;
-        private int framesUntilTracked;
-        public List<Finger> TrackedFingers { get; private set; }
+        public List<FingerHistory> FingerHistories { get; private set; }
+        public List<Finger> Fingers { get; private set; }
+        private List<FingerSimilarity> similarities;
 
         public FingerTracker()
         {
-            framesUntilTracked = Constants.FingerNumFramesUntilTracked;
-            detectedFingers = new List<Finger>[framesUntilTracked];
-            TrackedFingers = new List<Finger>();
+            FingerHistories = new List<FingerHistory>();
+        }
 
-            for (int i = 0; i < framesUntilTracked; i++)
+        public void updateTrackedFingers(List<Finger> detectedFingers)
+        {
+            if (FingerHistories.Count == 0)
+            {
+                foreach (Finger finger in detectedFingers)
+                    FingerHistories.Add(new FingerHistory(finger));
+            }
+            else
             {
-                detectedFingers[i] = new List<Finger>();
+                updateHistories(detectedFingers);
+                removeLongLostFingers();
             }
         }
 
-        public void setDetectedTouchEventsThisFrame(List<Finger> detectedFingersThisFrame, OutputImage outputImage)
+        private void updateHistories(List<Finger> detectedFingers)
         {
-            this.outputImage = outputImage;
+            Fingers = new List<Finger>();
+
+            List<FingerHistory> unasignedFingerHistories = new List<FingerHistory>();
+            foreach (FingerHistory fh in FingerHistories)
+                unasignedFingerHistories.Add(fh);
 
-            for (int i = (framesUntilTracked - 1); i > 0; i--)
+            List<Finger> unasignedDetectedFingers = new List<Finger>();
+            foreach (Finger f in detectedFingers)
+                unasignedDetectedFingers.Add(f);
+
+            createSimilarities(detectedFingers);
+
+            while (similarities.Count > 0)
+            {
+                FingerSimilarity maxSimilarity = similarities[0];
+                FingerHistory fingerHistory = maxSimilarity.fingerHistory;
+                Finger fingerDetected = maxSimilarity.fingerDetected;
+                fingerHistory.addFinger(fingerDetected);
+
+                Fingers.Add(fingerDetected);
+                unasignedFingerHistories.Remove(fingerHistory);
+                unasignedDetectedFingers.Remove(fingerDetected);
+                removeConcurringFingerSimilarities(maxSimilarity);
+            }
+
+            //add new fingerHistory for each new unmatched finger
+            foreach (Finger f in unasignedDetectedFingers)
             {
-                detectedFingers[i] = detectedFingers[i - 1];
+                FingerHistories.Add(new FingerHistory(f));
             }
-            detectedFingers[0] = detectedFingersThisFrame;
 
+            //add null finger to TrackedFingers that haven't found a match in this frame (status tracked->lost)
+            foreach (FingerHistory fh in unasignedFingerHistories)
+            {
+                fh.addFinger(null);
+            }
 
-            findTrackedFingers();
         }
 
-        private void findTrackedFingers()
+        private void createSimilarities(List<Finger> detectedFingers)
         {
-            TrackedFingers = new List<Finger>();
+            similarities = new List<FingerSimilarity>();
 
-            foreach (Finger finger in detectedFingers[0])
+            foreach (FingerHistory fh in FingerHistories)
             {
-                bool tracked = true;
-                Finger mostSimilarLastFrame = finger;
-                for (int i = 1; i < framesUntilTracked; i++)
+                foreach (Finger fingerDetected in detectedFingers)
                 {
-                    Finger mostSimilarThisFrame = getMostSimilarFingerInFrame(mostSimilarLastFrame, i);
-                    if (mostSimilarThisFrame == null || mostSimilarLastFrame.getSimilarity(mostSimilarThisFrame) < Constants.FingerMinSimilarityForTracking)
+                    float similarity = fingerDetected.getSimilarity(fh.LastFinger);
+                    if (similarity > Constants.FingerMinSimilarityForTracking)
                     {
-                        tracked = false;
-                        break;
-
+                        FingerSimilarity fs = new FingerSimilarity();
+                        fs.similarity = similarity;
+                        fs.fingerHistory = fh;
+                        fs.fingerDetected = fingerDetected;
+                        similarities.Add(fs);
                     }
                 }
-                if (tracked)
-                {
-                    outputImage.drawLineSegment(finger.LineSegment, Constants.FingerTrackedColor);
-                    TrackedFingers.Add(finger);
-                }
             }
+
+            similarities.Sort((fs1, fs2) => fs1.similarity.CompareTo(fs2.similarity));
         }
 
-        private Finger getMostSimilarFingerInFrame(Finger finger, int frame)
+        private void removeConcurringFingerSimilarities(FingerSimilarity removeSimilarity)
         {
-            Finger mostSimilarFinger = null;
-            float maxSimilarity = 0;
-            foreach (Finger f in detectedFingers[frame])
+            for (int i = similarities.Count - 1; i >= 0; i--)
             {
-                float similarity = finger.getSimilarity(f);
-                if (similarity > maxSimilarity)
-                {
-                    mostSimilarFinger = f;
-                    maxSimilarity = similarity;
-                }
+                FingerSimilarity currentSimilarity = similarities[i];
+                if (currentSimilarity.fingerHistory == removeSimilarity.fingerHistory || currentSimilarity.fingerDetected == removeSimilarity.fingerDetected)
+                    similarities.RemoveAt(i);
+            }
+        }
+
+        private void removeFingerDetectedFromSimilarities(Finger removeFinger)
+        {
+            for (int i = similarities.Count - 1; i >= 0; i--)
+            {
+                if (similarities[i].fingerDetected == removeFinger)
+                    similarities.RemoveAt(i);
+            }
+        }
+
+        private void removeLongLostFingers()
+        {
+            for (int i = FingerHistories.Count - 1; i >= 0; i--)
+            {
+                FingerHistory tf = FingerHistories[i];
+                if (tf.CurrentState == TrackingState.Lost && tf.getNumFramesInCurrentState() >= Constants.FingerNumFramesUntilLost)
+                    FingerHistories.RemoveAt(i);
             }
-            return mostSimilarFinger;
         }
     }
 }

bbiwarg/VideoHandle.cs

@@ -177,7 +177,7 @@ namespace bbiwarg
 
             //track fingers
             Timer.start("fingerTracking");
-            fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, OutputImages[1]);
+            fingerTracker.updateTrackedFingers(fingerDetector.Fingers);
             Timer.stop("fingerTracking");
 
             //detect hands
@@ -199,7 +199,7 @@ namespace bbiwarg
 
             //detect touchEvents
             Timer.start("touchDetection");
-            touchDetector = new TouchDetector(fingerTracker.TrackedFingers, depthImage, OutputImages[0]);
+            touchDetector = new TouchDetector(fingerTracker.Fingers, depthImage, OutputImages[0]);
             if (palmDetector.PalmQuad != null)
                 palmTouchDetector = new PalmTouchDetector(touchDetector.TouchEvents, palmDetector.PalmQuad);
             Timer.stop("touchDetection");