removed FingerTracker and made Tracker generic

bbiwarg/Constants.cs

@@ -15,7 +15,7 @@ namespace bbiwarg
         public static readonly Logger.LogSubject LogLevel =
             Logger.LogSubject.FingerTracker;
         
-        // BBIWARG
+        // Output
         public static readonly int OutputNumImages = 5;
         public static readonly int OutputNumImagesPerRow = 3;
         public static readonly float OutputScaleFactor = 1f; // output window size is scaled by this factor (from necessary size for images)

bbiwarg/Recognition/FingerRecognition/Finger.cs

@@ -8,11 +8,12 @@ using bbiwarg.Images;
 using bbiwarg.Utility;
 using bbiwarg.Graphics;
 using bbiwarg.Recognition.HandRecognition;
+using bbiwarg.Recognition.Tracking;
 using Emgu.CV;
 
 namespace bbiwarg.Recognition.FingerRecognition
 {
-    class Finger
+    class Finger : TrackableObject
     {
         public Vector2D TipPoint { get { return SliceTrail.StartSlice.Mid; } }
         public Vector2D HandPoint { get { return SliceTrail.EndSlice.Mid; } }
@@ -24,13 +25,15 @@ namespace bbiwarg.Recognition.FingerRecognition
         public FingerSliceTrail SliceTrail { get; private set; }
         public Contour<Point> Contour { get { return SliceTrail.getContour(); } }
 
-        public Finger(FingerSliceTrail sliceTrail)
+        public Finger(FingerSliceTrail sliceTrail) : base()
         {
             SliceTrail = sliceTrail;
         }
 
-        public float getSimilarity(Finger compareFinger)
+        public override float getSimilarity(TrackableObject compareObject)
         {
+            Finger compareFinger = (Finger)compareObject;
+
             LineSegment2D compareLineSegment = compareFinger.LineSegment;
             Line2D compareLine = compareLineSegment.Line;
 

bbiwarg/Recognition/FingerRecognition/FingerHistory.cs

@@ -1,94 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace bbiwarg.Recognition.FingerRecognition
-{
-    public enum TrackingState
-    {
-        None,
-        Detected,
-        Tracked,
-        Lost,
-    }
-
-    class FingerHistory
-    {
-        private static List<int> usedIDs = new List<int>();
-        public int ID { get; private set; }
-        public List<Finger> Fingers { get; private set; }
-        public List<TrackingState> States { get; private set; }
-        public Finger LastFinger { get; private set; }
-        public Finger CurrentFinger { get { return Fingers[Fingers.Count - 1]; } }
-        public TrackingState CurrentState { get { return States[States.Count - 1]; } }
-
-
-        public FingerHistory(Finger finger)
-        {
-            ID = FingerHistory.getNextUnusedID();
-
-            Fingers = new List<Finger>();
-            States = new List<TrackingState>();
-
-            Fingers.Add(finger);
-            States.Add(TrackingState.Detected);
-            LastFinger = finger;
-        }
-
-        ~FingerHistory() {
-            FingerHistory.setIDUnused(ID);
-        }
-
-        public void addFinger(Finger finger)
-        {
-            TrackingState previousState = CurrentState;
-            TrackingState newState = TrackingState.None;
-            int numFramesInCurrentState = getNumFramesInCurrentState();
-
-            if (finger == null)
-                newState = TrackingState.Lost;
-            else if (previousState == TrackingState.Tracked)
-                newState = TrackingState.Tracked;
-            else if (previousState == TrackingState.Detected)
-            {
-                if (numFramesInCurrentState == Constants.FingerNumFramesUntilTracked)
-                    newState = TrackingState.Tracked;
-                else
-                    newState = TrackingState.Detected;
-            }
-
-            Fingers.Add(finger);
-            States.Add(newState);
-            if (finger != null)
-                LastFinger = finger;
-        }
-
-        public int getNumFramesInCurrentState()
-        {
-            TrackingState currentState = CurrentState;
-            int count = 0;
-            for (int i = States.Count - 1; i >= 0; i--)
-            {
-                if (States[i] == currentState)
-                    count++;
-                else
-                    break;
-            }
-            return count;
-        }
-
-        public static int getNextUnusedID() {
-            int id = 1;
-            while (usedIDs.Contains(id))
-                id++;
-            usedIDs.Add(id);
-            return id;
-        }
-
-        public static void setIDUnused(int id) {
-            usedIDs.Remove(id);
-        }
-    }
-}

bbiwarg/Recognition/FingerRecognition/FingerTracker.cs

@@ -1,179 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using bbiwarg.Images;
-using bbiwarg.Graphics;
-
-using bbiwarg.Utility;
-
-namespace bbiwarg.Recognition.FingerRecognition
-{
-    struct FingerSimilarity
-    {
-        public float similarity;
-        public FingerHistory fingerHistory;
-        public Finger fingerDetected;
-    }
-
-    class FingerTracker
-    {
-        public List<FingerHistory> FingerHistories { get; private set; }
-        public List<Finger> Fingers { get; private set; }
-        private List<FingerSimilarity> similarities;
-
-        public FingerTracker()
-        {
-            reset();
-        }
-
-        public void reset()
-        {
-            FingerHistories = new List<FingerHistory>();
-            Fingers = new List<Finger>();
-        }
-
-        public void updateTrackedFingers(List<Finger> detectedFingers, OutputImage outputImage)
-        {
-            if (FingerHistories.Count == 0)
-            {
-                foreach (Finger finger in detectedFingers)
-                {
-                    FingerHistory fh = new FingerHistory(finger);
-                    FingerHistories.Add(fh);
-                    Logger.log("Finger #" + fh.ID + " detected", Logger.LogSubject.FingerTracker);
-                }
-            }
-            else
-            {
-                updateHistories(detectedFingers);
-                removeLongLostFingers();
-            }
-
-            drawFingers(outputImage);
-            drawFingerIDs(outputImage);
-        }
-
-        private void updateHistories(List<Finger> detectedFingers)
-        {
-            Fingers = new List<Finger>();
-
-            List<FingerHistory> unasignedFingerHistories = new List<FingerHistory>();
-            foreach (FingerHistory fh in FingerHistories)
-                unasignedFingerHistories.Add(fh);
-
-            List<Finger> unasignedDetectedFingers = new List<Finger>();
-            foreach (Finger f in detectedFingers)
-                unasignedDetectedFingers.Add(f);
-
-            createSimilarities(detectedFingers);
-
-            while (similarities.Count > 0)
-            {
-                FingerSimilarity maxSimilarity = similarities[0];
-                FingerHistory fingerHistory = maxSimilarity.fingerHistory;
-                Finger fingerDetected = maxSimilarity.fingerDetected;
-                fingerHistory.addFinger(fingerDetected);
-
-                Fingers.Add(fingerDetected);
-                unasignedFingerHistories.Remove(fingerHistory);
-                unasignedDetectedFingers.Remove(fingerDetected);
-                removeConcurringFingerSimilarities(maxSimilarity);
-            }
-
-            //add new fingerHistory for each new unmatched finger
-            foreach (Finger finger in unasignedDetectedFingers)
-            {
-                FingerHistory fh = new FingerHistory(finger);
-                FingerHistories.Add(fh);
-                Logger.log("Finger #" + fh.ID + " detected", Logger.LogSubject.FingerTracker);
-            }
-
-            //add null finger to TrackedFingers that haven't found a match in this frame (status tracked->lost)
-            foreach (FingerHistory fh in unasignedFingerHistories)
-            {
-                if (fh.CurrentState != TrackingState.Lost)
-                    Logger.log("Finger #" + fh.ID + " lost", Logger.LogSubject.FingerTracker);
-                fh.addFinger(null);
-            }
-
-        }
-
-        private void createSimilarities(List<Finger> detectedFingers)
-        {
-            similarities = new List<FingerSimilarity>();
-
-            foreach (FingerHistory fh in FingerHistories)
-            {
-                foreach (Finger fingerDetected in detectedFingers)
-                {
-                    float similarity = fingerDetected.getSimilarity(fh.LastFinger);
-                    if (similarity > Constants.FingerMinSimilarityForTracking)
-                    {
-                        FingerSimilarity fs = new FingerSimilarity();
-                        fs.similarity = similarity;
-                        fs.fingerHistory = fh;
-                        fs.fingerDetected = fingerDetected;
-                        similarities.Add(fs);
-                    }
-                }
-            }
-
-            similarities.Sort((fs1, fs2) => fs2.similarity.CompareTo(fs1.similarity));
-        }
-
-        private void removeConcurringFingerSimilarities(FingerSimilarity removeSimilarity)
-        {
-            for (int i = similarities.Count - 1; i >= 0; i--)
-            {
-                FingerSimilarity currentSimilarity = similarities[i];
-                if (currentSimilarity.fingerHistory == removeSimilarity.fingerHistory || currentSimilarity.fingerDetected == removeSimilarity.fingerDetected)
-                    similarities.RemoveAt(i);
-            }
-        }
-
-        private void removeFingerDetectedFromSimilarities(Finger removeFinger)
-        {
-            for (int i = similarities.Count - 1; i >= 0; i--)
-            {
-                if (similarities[i].fingerDetected == removeFinger)
-                    similarities.RemoveAt(i);
-            }
-        }
-
-        private void removeLongLostFingers()
-        {
-            for (int i = FingerHistories.Count - 1; i >= 0; i--)
-            {
-                FingerHistory fh = FingerHistories[i];
-                if (fh.CurrentState == TrackingState.Lost && fh.getNumFramesInCurrentState() >= Constants.FingerNumFramesUntilLost)
-                {
-                    FingerHistories.RemoveAt(i);
-                    Logger.log("Finger #" + fh.ID + " deleted", Logger.LogSubject.FingerTracker);
-                }
-            }
-        }
-
-        private void drawFingers(OutputImage outputImage)
-        {
-            foreach (Finger finger in Fingers)
-            {
-                finger.draw(outputImage, true);
-            }
-        }
-
-        private void drawFingerIDs(OutputImage outputImage)
-        {
-            foreach (FingerHistory fh in FingerHistories)
-            {
-                if (fh.CurrentFinger != null)
-                {
-                    Finger f = fh.CurrentFinger;
-                    outputImage.drawText(f.MidPoint.IntX, f.MidPoint.IntY, fh.ID.ToString(), Constants.FingerIDColor);
-                }
-            }
-        }
-    }
-}

bbiwarg/Recognition/Tracking/TrackableObject.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bbiwarg.Recognition.Tracking
+{
+    abstract class TrackableObject
+    {
+        public bool IsTracked { get; private set; }
+        public TrackableObjectHistory<TrackableObject> History { get; private set; }
+        public int TrackID { get { return (IsTracked) ? History.ID : 0; } }
+
+        public TrackableObject() {
+            IsTracked = false;
+        }
+
+        abstract public float getSimilarity(TrackableObject compareObject);
+
+        public void setTracked(TrackableObjectHistory<TrackableObject> history) {
+            IsTracked = true;
+            History = history;
+        }
+    }
+}

bbiwarg/Recognition/Tracking/TrackableObjectHistory.cs

@@ -0,0 +1,77 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bbiwarg.Recognition.Tracking
+{
+    class TrackableObjectHistory<T> where T : TrackableObject
+    {
+        private static List<int> usedIDs = new List<int>();
+        public int ID { get; private set; }
+        public List<T> TrackedObjects { get; private set; }
+        public List<TrackingState> States { get; private set; }
+        public TrackableObject LastObject { get; private set; }
+        public TrackableObject CurrentObject { get { return TrackedObjects[TrackedObjects.Count - 1]; } }
+        public TrackingState CurrentState { get { return States[States.Count - 1]; } }
+        public int NumFramesInCurrentState { get; private set; }
+        private int numFramesUntilTracked;
+
+        public TrackableObjectHistory(int numFramesUntilTracked) {
+            ID = getNextUnusedID();
+            TrackedObjects = new List<T>();
+            States = new List<TrackingState>();
+            TrackedObjects.Add(null);
+            States.Add(TrackingState.None);
+            NumFramesInCurrentState = 1;
+            this.numFramesUntilTracked = numFramesUntilTracked;
+        }
+
+        ~TrackableObjectHistory() {
+            setIDUnused(ID);
+        }
+
+        public void addObjectToHistory(T detectedObject) {
+            TrackingState previousState = CurrentState;
+            TrackingState newState = TrackingState.None;
+
+            if (detectedObject != null)
+            {
+                LastObject = detectedObject;
+                if (previousState == TrackingState.Tracked)
+                    newState = TrackingState.Tracked;
+                else if (previousState == TrackingState.Detected)
+                {
+                    if (NumFramesInCurrentState == numFramesUntilTracked)
+                        newState = TrackingState.Tracked;
+                    else
+                        newState = TrackingState.Detected;
+                }
+            }
+            else 
+                newState = TrackingState.Lost;
+
+            if (newState == previousState)
+                NumFramesInCurrentState++;
+
+            TrackedObjects.Add(detectedObject);
+            States.Add(newState);
+
+        }
+
+        public static int getNextUnusedID()
+        {
+            int id = 1;
+            while (usedIDs.Contains(id))
+                id++;
+            usedIDs.Add(id);
+            return id;
+        }
+
+        public static void setIDUnused(int id)
+        {
+            usedIDs.Remove(id);
+        }
+    }
+}

bbiwarg/Recognition/Tracking/Tracker.cs

@@ -0,0 +1,170 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bbiwarg.Graphics;
+using bbiwarg.Utility;
+
+namespace bbiwarg.Recognition.Tracking
+{
+    public enum TrackingState
+    {
+        None,
+        Detected,
+        Tracked,
+        Lost,
+    }
+
+    struct SimilarityContainer<T> where T : TrackableObject
+    {
+        public float similarity;
+        public TrackableObjectHistory<T> history;
+        public T detectedObject;
+    }
+
+    class Tracker<T> where T : TrackableObject
+    {
+        public List<TrackableObjectHistory<T>> Histories { get; private set; }
+        public List<T> TrackedObjects { get; private set; }
+        private List<SimilarityContainer<T>> similarities;
+        private String className;
+        private Logger.LogSubject logSubject;
+        private int numFramesUntilTracked;
+        private int numFramesUntilLost;
+        private float minSimilarityForTracking;
+
+        public Tracker(Logger.LogSubject logSubject, int numFramesUntilTracked, int numFramesUntilLost, float minSimilarityForTracking)
+        {
+            this.className = typeof(T).Name;
+            this.logSubject = logSubject;
+            this.numFramesUntilTracked = numFramesUntilTracked;
+            this.numFramesUntilLost = numFramesUntilLost;
+            this.minSimilarityForTracking = minSimilarityForTracking;
+
+            reset();
+        }
+
+        public void reset()
+        {
+            Histories = new List<TrackableObjectHistory<T>>();
+            TrackedObjects = new List<T>();
+        }
+
+        public void updateFrame(List<T> detectedObjects)
+        {
+            if (Histories.Count == 0)
+            {
+                foreach (T detectedObject in detectedObjects)
+                {
+                    TrackableObjectHistory<T> history = new TrackableObjectHistory<T>(numFramesUntilTracked);
+                    history.addObjectToHistory(detectedObject);
+                    addToHistories(history);
+                }
+            }
+            else
+            {
+                updateHistories(detectedObjects);
+                removeLongLostObjects();
+            }
+        }
+
+        private void updateHistories(List<T> detectedObjects)
+        {
+            TrackedObjects = new List<T>();
+
+            List<TrackableObjectHistory<T>> unasignedHistories = new List<TrackableObjectHistory<T>>();
+            foreach (TrackableObjectHistory<T> history in Histories)
+                unasignedHistories.Add(history);
+
+            List<TrackableObject> unasignedDetectedObjects = new List<TrackableObject>();
+            foreach (T detectedObject in detectedObjects)
+                unasignedDetectedObjects.Add(detectedObject);
+
+
+            //asign previously tracked objects to their best fits of detectedObjects
+            createSimilarities(detectedObjects);
+            while (similarities.Count > 0)
+            {
+                SimilarityContainer<T> maxSimilarity = similarities[0];
+                TrackableObjectHistory<T> history = maxSimilarity.history;
+                T detectedObject = maxSimilarity.detectedObject;
+                history.addObjectToHistory(detectedObject);
+
+                TrackedObjects.Add(detectedObject);
+                unasignedHistories.Remove(history);
+                unasignedDetectedObjects.Remove(detectedObject);
+                removeConcurringSimilarities(maxSimilarity);
+            }
+            //add new history for each new unasigned detectedObject
+            foreach (T unasignedDetectedObject in unasignedDetectedObjects)
+            {
+                TrackableObjectHistory<T> newHistory = new TrackableObjectHistory<T>(numFramesUntilTracked);
+                newHistory.addObjectToHistory(unasignedDetectedObject);
+                addToHistories(newHistory);
+            }
+
+            //add null-object to each unasigned history (didn't find a best fit during this frame -> lost track)
+            foreach (TrackableObjectHistory<T> unasignedHistory in unasignedHistories)
+            {
+                if (unasignedHistory.CurrentState != TrackingState.Lost)
+                    Logger.log(className + " #" + unasignedHistory.ID + " lost", logSubject);
+                unasignedHistory.addObjectToHistory(null);
+            }
+
+        }
+
+        private void createSimilarities(List<T> detectedObjects)
+        {
+            similarities = new List<SimilarityContainer<T>>();
+
+            foreach (TrackableObjectHistory<T> history in Histories)
+            {
+                foreach (T detectedObject in detectedObjects)
+                {
+                    float similarity = detectedObject.getSimilarity(history.LastObject);
+                    if (similarity > minSimilarityForTracking)
+                    {
+                        SimilarityContainer<T> similarityContainer = new SimilarityContainer<T>();
+                        similarityContainer.similarity = similarity;
+                        similarityContainer.history = history;
+                        similarityContainer.detectedObject = detectedObject;
+                        similarities.Add(similarityContainer);
+                    }
+                }
+            }
+
+            // sort depending on similarity-value
+            similarities.Sort((fs1, fs2) => fs2.similarity.CompareTo(fs1.similarity));
+        }
+
+        private void removeConcurringSimilarities(SimilarityContainer<T> removeSimilarity)
+        {
+            for (int i = similarities.Count - 1; i >= 0; i--)
+            {
+                SimilarityContainer<T> similarity = similarities[i];
+                if (similarity.history == removeSimilarity.history || similarity.detectedObject == removeSimilarity.detectedObject)
+                    similarities.RemoveAt(i);
+            }
+        }
+
+        private void removeLongLostObjects()
+        {
+            for (int i = Histories.Count - 1; i >= 0; i--)
+            {
+                TrackableObjectHistory<T> history = Histories[i];
+                if (history.CurrentState == TrackingState.Lost && history.NumFramesInCurrentState >= numFramesUntilLost)
+                {
+                    Histories.RemoveAt(i);
+                    Logger.log(className + " #" + history.ID + " deleted", logSubject);
+                }
+            }
+        }
+
+        private void addToHistories(TrackableObjectHistory<T> history)
+        {
+            Histories.Add(history);
+            Logger.log(className + " #" + history.ID + " detected", logSubject);
+        }
+    }
+}

bbiwarg/VideoHandle.cs

@@ -10,6 +10,7 @@ using bbiwarg.Recognition.FingerRecognition;
 using bbiwarg.Recognition.PalmRecognition;
 using bbiwarg.Recognition.TouchRecognition;
 using bbiwarg.Recognition.HandRecognition;
+using bbiwarg.Recognition.Tracking;
 using bbiwarg.Images;
 using bbiwarg.InputProviders;
 using Emgu.CV;
@@ -54,7 +55,7 @@ namespace bbiwarg
         private TouchDetector touchDetector;
         private PalmTouchDetector palmTouchDetector;
 
-        private FingerTracker fingerTracker;
+        private Tracker<Finger> fingerTracker;
         private TouchTracker touchTracker;
 
         private TouchEventVisualizer touchEventVisualizer;
@@ -121,7 +122,7 @@ namespace bbiwarg
 
         private void initTrackers() {
             touchTracker = new TouchTracker();
-            fingerTracker = new FingerTracker();
+            fingerTracker = new Tracker<Finger>(Logger.LogSubject.FingerTracker, Constants.FingerNumFramesUntilTracked, Constants.FingerNumFramesUntilLost, Constants.FingerMinSimilarityForTracking);
         }
 
         private void processFrameUpdate()
@@ -130,7 +131,7 @@ namespace bbiwarg
             Timer.start("processFrameUpdate");
 
             if (newStarted) {
-                //initialize trackers
+                //reset trackers
                 touchTracker.reset();
                 fingerTracker.reset();
             }
@@ -178,7 +179,7 @@ namespace bbiwarg
 
             //track fingers
             Timer.start("fingerTracking");
-            fingerTracker.updateTrackedFingers(fingerDetector.Fingers, OutputImages[1]);
+            fingerTracker.updateFrame(fingerDetector.Fingers);
             Timer.stop("fingerTracking");
 
             //detect hands
@@ -200,7 +201,7 @@ namespace bbiwarg
 
             //detect touchEvents
             Timer.start("touchDetection");
-            touchDetector = new TouchDetector(fingerTracker.Fingers, depthImage, OutputImages[0]);
+            touchDetector = new TouchDetector(fingerTracker.TrackedObjects, depthImage, OutputImages[0]);
             if (palmDetector.PalmQuad != null)
                 palmTouchDetector = new PalmTouchDetector(touchDetector.TouchEvents, palmDetector.PalmQuad, palmDetector.PalmHandSide);
             Timer.stop("touchDetection");

bbiwarg/bbiwarg.csproj

@@ -69,10 +69,8 @@
     <Compile Include="Constants.cs" />
     <Compile Include="Recognition\FingerRecognition\Finger.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerDetector.cs" />
-    <Compile Include="Recognition\FingerRecognition\FingerHistory.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerSliceTrail.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerSlice.cs" />
-    <Compile Include="Recognition\FingerRecognition\FingerTracker.cs" />
     <Compile Include="Recognition\HandRecognition\Hand.cs" />
     <Compile Include="Recognition\HandRecognition\HandDetector.cs" />
     <Compile Include="Recognition\PalmRecognition\PalmDetector.cs" />
@@ -92,6 +90,9 @@
     <Compile Include="InputProvider\IisuInputProvider.cs" />
     <Compile Include="MainBBWIWARG.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Recognition\Tracking\TrackableObject.cs" />
+    <Compile Include="Recognition\Tracking\TrackableObjectHistory.cs" />
+    <Compile Include="Recognition\Tracking\Tracker.cs" />
     <Compile Include="Server\OSC.NET\OSCBundle.cs" />
     <Compile Include="Server\OSC.NET\OSCMessage.cs" />
     <Compile Include="Server\OSC.NET\OSCPacket.cs" />