Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

Conflicts:
	bbiwarg/Constants.cs
	bbiwarg/Server/TUIO/TuioServer.cs

bbiwarg/Constants.cs

@@ -18,6 +18,11 @@ namespace bbiwarg
         public static readonly int OutputNumImagesPerRow = 3;
         public static readonly float OutputScaleFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
 
+        // debug
+        public static readonly bool OutputTimerEnabled = false;
+        public static readonly bool VerboseTUIO = false;
+        public static readonly bool VerboseFingerTracker = true;
+
         // confidence image
         public static readonly int ConfidenceImageMinThreshold = 500;
 
@@ -40,13 +45,14 @@ namespace bbiwarg
         public static readonly int FingerMaxSize = 30;
         public static readonly int FingerMinSize = 5;
         public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
-        public static readonly int FingerNumFramesUntilTracked = 2;
         public static readonly int FingerOutSliceFactor = 10;
         public static readonly int FingerContourMargin = 2;
         public static readonly int FingerSliceOverlapFactor = 2;
         public static readonly int FingerCrippleOutFactor = 8;
         public static readonly int FingerCrippleOutMinDifference = 15;
-        public static readonly float FingerMinSimilarityForTracking = 0.75f;
+        public static readonly int FingerNumFramesUntilTracked = 5;
+        public static readonly int FingerNumFramesUntilLost = 20;
+        public static readonly float FingerMinSimilarityForTracking = 0.1f;
 
         // hand detection
         public static readonly float HandMaxSize = 0.7f;
@@ -81,8 +87,8 @@ namespace bbiwarg
         public static readonly float TouchProcessNoise = 3.0e-4f;
 
         // colors
-        public static readonly Color ColorDetected = Color.Green;
-        public static readonly Color ColorTracked = Color.LightGoldenrodYellow;
+        public static readonly Color ColorDetected = Color.Turquoise;
+        public static readonly Color ColorTracked = Color.Yellow;
 
         //public static readonly Color EdgeColor = Color.Blue; // edgeImage draw direct to blue chanel from outputImage
 
@@ -92,6 +98,7 @@ namespace bbiwarg
         public static readonly Color FingerTipOutSliceColor = Color.Gray;
         public static readonly Color FingerHandOutSliceColor = Color.DarkSlateGray;
         public static readonly Color FingerContourColor = Color.Red;
+        public static readonly Color FingerIDColor = Color.Orange;
 
         public static readonly Color TouchEventDetectedColor = ColorDetected;
         public static readonly Color TouchEventTrackedColor = ColorTracked;

bbiwarg/Graphics/OutputImage.cs

@@ -60,5 +60,10 @@ namespace bbiwarg.Graphics
         {
             Image.Draw(new Rectangle(x, y, width, height), new Rgb(color), thichness);
         }
+
+        public void drawText(int x, int y, String text, Color color) {
+            MCvFont font = new MCvFont(Emgu.CV.CvEnum.FONT.CV_FONT_HERSHEY_PLAIN, 1, 1);
+            Image.Draw(text, ref font, new Point(x, y), new Rgb(color));
+        }
     }
 }

bbiwarg/Recognition/FingerRecognition/Finger.cs

@@ -6,6 +6,7 @@ using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Utility;
+using bbiwarg.Graphics;
 using Emgu.CV;
 
 namespace bbiwarg.Recognition.FingerRecognition
@@ -53,5 +54,22 @@ namespace bbiwarg.Recognition.FingerRecognition
 
             return (angleSimilarity + parallelDistanceSimilarity + verticalDistanceSimilarity) / 3;
         }
+
+        public void draw(OutputImage outputImage, bool tracked)
+        {
+            if (tracked)
+            {
+                outputImage.drawLineSegment(LineSegment, Constants.FingerTrackedColor);
+            }
+            else
+            {
+                for (int i = 0; i < SliceTrail.NumSlices; i++)
+                {
+                    outputImage.drawLineSegment(SliceTrail[i].LineSegment, Constants.FingerSliceColor);
+                }
+                outputImage.drawLineSegment(LineSegment, Constants.FingerDetectedColor);
+                outputImage.drawContour(Contour, Constants.FingerContourColor, 1);
+            }
+        }
     }
 }

bbiwarg/Recognition/FingerRecognition/FingerDetector.cs

@@ -275,19 +275,8 @@ namespace bbiwarg.Recognition.FingerRecognition
         {
             foreach (Finger finger in Fingers)
             {
-                drawFinger(finger);
+                finger.draw(outputImage, false);
             }
         }
-
-        private void drawFinger(Finger finger)
-        {
-            FingerSliceTrail trail = finger.SliceTrail;
-            for (int i = 0; i < trail.NumSlices; i++)
-            {
-                outputImage.drawLineSegment(trail[i].LineSegment, Constants.FingerSliceColor);
-            }
-            outputImage.drawLineSegment(finger.LineSegment, Constants.FingerDetectedColor);
-            outputImage.drawContour(finger.Contour, Constants.FingerContourColor, 1);
-        }
     }
 }

bbiwarg/Recognition/FingerRecognition/FingerHistory.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bbiwarg.Recognition.FingerRecognition
+{
+    public enum TrackingState
+    {
+        None,
+        Detected,
+        Tracked,
+        Lost,
+    }
+
+    class FingerHistory
+    {
+        private static List<int> usedIDs = new List<int>();
+        public int ID { get; private set; }
+        public List<Finger> Fingers { get; private set; }
+        public List<TrackingState> States { get; private set; }
+        public Finger LastFinger { get; private set; }
+        public Finger CurrentFinger { get { return Fingers[Fingers.Count - 1]; } }
+        public TrackingState CurrentState { get { return States[States.Count - 1]; } }
+
+
+        public FingerHistory(Finger finger)
+        {
+            ID = FingerHistory.getNextUnusedID();
+
+            Fingers = new List<Finger>();
+            States = new List<TrackingState>();
+
+            Fingers.Add(finger);
+            States.Add(TrackingState.Detected);
+            LastFinger = finger;
+        }
+
+        ~FingerHistory() {
+            FingerHistory.setIDUnused(ID);
+        }
+
+        public void addFinger(Finger finger)
+        {
+            TrackingState previousState = CurrentState;
+            TrackingState newState = TrackingState.None;
+            int numFramesInCurrentState = getNumFramesInCurrentState();
+
+            if (finger == null)
+                newState = TrackingState.Lost;
+            else if (previousState == TrackingState.Tracked)
+                newState = TrackingState.Tracked;
+            else if (previousState == TrackingState.Detected)
+            {
+                if (numFramesInCurrentState == Constants.FingerNumFramesUntilTracked)
+                    newState = TrackingState.Tracked;
+                else
+                    newState = TrackingState.Detected;
+            }
+
+            Fingers.Add(finger);
+            States.Add(newState);
+            if (finger != null)
+                LastFinger = finger;
+        }
+
+        public int getNumFramesInCurrentState()
+        {
+            TrackingState currentState = CurrentState;
+            int count = 0;
+            for (int i = States.Count - 1; i >= 0; i--)
+            {
+                if (States[i] == currentState)
+                    count++;
+                else
+                    break;
+            }
+            return count;
+        }
+
+        public static int getNextUnusedID() {
+            int id = 1;
+            while (usedIDs.Contains(id))
+                id++;
+            usedIDs.Add(id);
+            return id;
+        }
+
+        public static void setIDUnused(int id) {
+            usedIDs.Remove(id);
+        }
+    }
+}

bbiwarg/Recognition/FingerRecognition/FingerTracker.cs

@@ -9,79 +9,162 @@ using bbiwarg.Graphics;
 
 namespace bbiwarg.Recognition.FingerRecognition
 {
+    struct FingerSimilarity
+    {
+        public float similarity;
+        public FingerHistory fingerHistory;
+        public Finger fingerDetected;
+    }
+
     class FingerTracker
     {
-        private OutputImage outputImage;
-        private List<Finger>[] detectedFingers;
-        private int framesUntilTracked;
-        public List<Finger> TrackedFingers { get; private set; }
+        public List<FingerHistory> FingerHistories { get; private set; }
+        public List<Finger> Fingers { get; private set; }
+        private List<FingerSimilarity> similarities;
 
         public FingerTracker()
         {
-            framesUntilTracked = Constants.FingerNumFramesUntilTracked;
-            detectedFingers = new List<Finger>[framesUntilTracked];
-            TrackedFingers = new List<Finger>();
+            FingerHistories = new List<FingerHistory>();
+            Fingers = new List<Finger>();
+        }
 
-            for (int i = 0; i < framesUntilTracked; i++)
+        public void updateTrackedFingers(List<Finger> detectedFingers, OutputImage outputImage)
+        {
+            if (FingerHistories.Count == 0)
             {
-                detectedFingers[i] = new List<Finger>();
+                foreach (Finger finger in detectedFingers)
+                {
+                    FingerHistory fh = new FingerHistory(finger);
+                    FingerHistories.Add(fh);
+                    if (Constants.VerboseFingerTracker) Console.WriteLine("Finger #" + fh.ID + " detected");
+                }
             }
+            else
+            {
+                updateHistories(detectedFingers);
+                removeLongLostFingers();
+            }
+
+            drawFingers(outputImage);
+            drawFingerIDs(outputImage);
         }
 
-        public void setDetectedTouchEventsThisFrame(List<Finger> detectedFingersThisFrame, OutputImage outputImage)
+        private void updateHistories(List<Finger> detectedFingers)
         {
-            this.outputImage = outputImage;
+            Fingers = new List<Finger>();
 
-            for (int i = (framesUntilTracked - 1); i > 0; i--)
+            List<FingerHistory> unasignedFingerHistories = new List<FingerHistory>();
+            foreach (FingerHistory fh in FingerHistories)
+                unasignedFingerHistories.Add(fh);
+
+            List<Finger> unasignedDetectedFingers = new List<Finger>();
+            foreach (Finger f in detectedFingers)
+                unasignedDetectedFingers.Add(f);
+
+            createSimilarities(detectedFingers);
+
+            while (similarities.Count > 0)
+            {
+                FingerSimilarity maxSimilarity = similarities[0];
+                FingerHistory fingerHistory = maxSimilarity.fingerHistory;
+                Finger fingerDetected = maxSimilarity.fingerDetected;
+                fingerHistory.addFinger(fingerDetected);
+
+                Fingers.Add(fingerDetected);
+                unasignedFingerHistories.Remove(fingerHistory);
+                unasignedDetectedFingers.Remove(fingerDetected);
+                removeConcurringFingerSimilarities(maxSimilarity);
+            }
+
+            //add new fingerHistory for each new unmatched finger
+            foreach (Finger finger in unasignedDetectedFingers)
             {
-                detectedFingers[i] = detectedFingers[i - 1];
+                FingerHistory fh = new FingerHistory(finger);
+                FingerHistories.Add(fh);
+                if (Constants.VerboseFingerTracker) Console.WriteLine("Finger #" + fh.ID + " detected");
             }
-            detectedFingers[0] = detectedFingersThisFrame;
 
+            //add null finger to TrackedFingers that haven't found a match in this frame (status tracked->lost)
+            foreach (FingerHistory fh in unasignedFingerHistories)
+            {
+                if (Constants.VerboseFingerTracker && fh.CurrentState != TrackingState.Lost) Console.WriteLine("Finger #" + fh.ID + " lost");
+                fh.addFinger(null);
+            }
 
-            findTrackedFingers();
         }
 
-        private void findTrackedFingers()
+        private void createSimilarities(List<Finger> detectedFingers)
         {
-            TrackedFingers = new List<Finger>();
+            similarities = new List<FingerSimilarity>();
 
-            foreach (Finger finger in detectedFingers[0])
+            foreach (FingerHistory fh in FingerHistories)
             {
-                bool tracked = true;
-                Finger mostSimilarLastFrame = finger;
-                for (int i = 1; i < framesUntilTracked; i++)
+                foreach (Finger fingerDetected in detectedFingers)
                 {
-                    Finger mostSimilarThisFrame = getMostSimilarFingerInFrame(mostSimilarLastFrame, i);
-                    if (mostSimilarThisFrame == null || mostSimilarLastFrame.getSimilarity(mostSimilarThisFrame) < Constants.FingerMinSimilarityForTracking)
+                    float similarity = fingerDetected.getSimilarity(fh.LastFinger);
+                    if (similarity > Constants.FingerMinSimilarityForTracking)
                     {
-                        tracked = false;
-                        break;
-
+                        FingerSimilarity fs = new FingerSimilarity();
+                        fs.similarity = similarity;
+                        fs.fingerHistory = fh;
+                        fs.fingerDetected = fingerDetected;
+                        similarities.Add(fs);
                     }
                 }
-                if (tracked)
+            }
+
+            similarities.Sort((fs1, fs2) => fs2.similarity.CompareTo(fs1.similarity));
+        }
+
+        private void removeConcurringFingerSimilarities(FingerSimilarity removeSimilarity)
+        {
+            for (int i = similarities.Count - 1; i >= 0; i--)
+            {
+                FingerSimilarity currentSimilarity = similarities[i];
+                if (currentSimilarity.fingerHistory == removeSimilarity.fingerHistory || currentSimilarity.fingerDetected == removeSimilarity.fingerDetected)
+                    similarities.RemoveAt(i);
+            }
+        }
+
+        private void removeFingerDetectedFromSimilarities(Finger removeFinger)
+        {
+            for (int i = similarities.Count - 1; i >= 0; i--)
+            {
+                if (similarities[i].fingerDetected == removeFinger)
+                    similarities.RemoveAt(i);
+            }
+        }
+
+        private void removeLongLostFingers()
+        {
+            for (int i = FingerHistories.Count - 1; i >= 0; i--)
+            {
+                FingerHistory fh = FingerHistories[i];
+                if (fh.CurrentState == TrackingState.Lost && fh.getNumFramesInCurrentState() >= Constants.FingerNumFramesUntilLost)
                 {
-                    outputImage.drawLineSegment(finger.LineSegment, Constants.FingerTrackedColor);
-                    TrackedFingers.Add(finger);
+                    FingerHistories.RemoveAt(i);
+                    if (Constants.VerboseFingerTracker) Console.WriteLine("Finger #" + fh.ID + " deleted");
                 }
             }
         }
 
-        private Finger getMostSimilarFingerInFrame(Finger finger, int frame)
+        private void drawFingers(OutputImage outputImage)
         {
-            Finger mostSimilarFinger = null;
-            float maxSimilarity = 0;
-            foreach (Finger f in detectedFingers[frame])
+            foreach (Finger finger in Fingers)
+            {
+                finger.draw(outputImage, true);
+            }
+        }
+
+        private void drawFingerIDs(OutputImage outputImage) {
+            foreach (FingerHistory fh in FingerHistories)
             {
-                float similarity = finger.getSimilarity(f);
-                if (similarity > maxSimilarity)
+                if (fh.CurrentFinger != null)
                 {
-                    mostSimilarFinger = f;
-                    maxSimilarity = similarity;
+                    Finger f = fh.CurrentFinger;
+                    outputImage.drawText(f.TipPoint.IntX, f.TipPoint.IntY, fh.ID.ToString(), Constants.FingerIDColor);
                 }
             }
-            return mostSimilarFinger;
         }
     }
 }

bbiwarg/Server/TUIO/TuioServer.cs

@@ -135,6 +135,7 @@ namespace TUIO
                 packet.Append(currentMessage);
 
                 Logger.log("Send Cursor "+cursor.getSessionID()+" at time "+TuioTime.getSystemTime().ToString(), Logger.LogSubject.TUIOServer);
+
             }
             currentMessage = new OSCMessage("/tuio/2Dcur");
             currentMessage.Append("fseq");

bbiwarg/VideoHandle.cs

@@ -177,7 +177,7 @@ namespace bbiwarg
 
             //track fingers
             Timer.start("fingerTracking");
-            fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, OutputImages[1]);
+            fingerTracker.updateTrackedFingers(fingerDetector.Fingers, OutputImages[1]);
             Timer.stop("fingerTracking");
 
             //detect hands
@@ -199,7 +199,7 @@ namespace bbiwarg
 
             //detect touchEvents
             Timer.start("touchDetection");
-            touchDetector = new TouchDetector(fingerTracker.TrackedFingers, depthImage, OutputImages[0]);
+            touchDetector = new TouchDetector(fingerTracker.Fingers, depthImage, OutputImages[0]);
             if (palmDetector.PalmQuad != null)
                 palmTouchDetector = new PalmTouchDetector(touchDetector.TouchEvents, palmDetector.PalmQuad, palmDetector.PalmHandSide);
             Timer.stop("touchDetection");

bbiwarg/bbiwarg.csproj

@@ -69,6 +69,7 @@
     <Compile Include="Constants.cs" />
     <Compile Include="Recognition\FingerRecognition\Finger.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerDetector.cs" />
+    <Compile Include="Recognition\FingerRecognition\FingerHistory.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerSliceTrail.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerSlice.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerTracker.cs" />