completly rewritten fingerDetection using FingerSliceTrails (much better performance and even better results)

bbiwarg/Detectors/Fingers/Finger.cs

@@ -13,25 +13,15 @@ namespace bbiwarg.Detectors.Fingers
 {
     class Finger
     {
-        private Line2D line;
-        public Line2D Line { get { return line; } }
-
-        public Finger(Line2D line) {
-            this.line = line;
-        }
-
-        public Vector2D getTipPoint() {
-            if (line.P1.Y < line.P2.Y)
-                return line.P1;
-            else
-                return line.P2;
-        }
-
-        public Vector2D getHandPoint() {
-            if (line.P1.Y < line.P2.Y)
-                return line.P2;
-            else
-                return line.P1;
+        private FingerSliceTrail sliceTrail;
+        public Vector2D Tip { get { return sliceTrail.Start.Mid; } }
+        public Vector2D Hand { get { return sliceTrail.End.Mid; } }
+        public Line2D Line { get {return sliceTrail.Line;}}
+        public FingerSliceTrail SliceTrail { get { return sliceTrail; } private set { sliceTrail = value; } }
+
+        public Finger(FingerSliceTrail sliceTrail)
+        {
+            SliceTrail = sliceTrail;
         }
 
         public float getSimilarity(Finger compareFinger) {
@@ -56,6 +46,5 @@ namespace bbiwarg.Detectors.Fingers
 
             return (angleSimilarity + parallelDistanceSimilarity + verticalDistanceSimilarity) / 3;
         }
-
     }
 }

bbiwarg/Detectors/Fingers/FingerDetector.cs

@@ -4,11 +4,10 @@ using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-using System.Drawing;
-using Emgu.CV;
-using Emgu.CV.Structure;
 using bbiwarg.Images;
 using bbiwarg.Utility;
+using Emgu.CV.Structure;
+using Emgu.CV;
 
 namespace bbiwarg.Detectors.Fingers
 {
@@ -17,169 +16,203 @@ namespace bbiwarg.Detectors.Fingers
         private DepthImage depthImage;
         private EdgeImage edgeImage;
         private FingerImage fingerImage;
-        private Image<Gray, byte> fingerPointsImage;
-        private List<Line2D> fingerLines;
+        private List<FingerSliceTrail> horizontalFingerSliceTrails;
+        private List<FingerSliceTrail> verticalFingerSliceTrails;
         private List<Finger> fingers;
+        public List<Finger> Fingers { get { return fingers; } private set { fingers = value; } }
 
 
-        public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, FingerImage fingerImage) {
+        public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, FingerImage fingerImage)
+        {
             this.depthImage = depthImage;
             this.edgeImage = edgeImage;
             this.fingerImage = fingerImage;
 
-            Stopwatch sw = new Stopwatch();
-            sw.Start();
-
-            findFingerPoints();
-
-            sw.Stop();
-            Console.WriteLine("findFingerPoints:" + sw.ElapsedMilliseconds);
-            sw.Restart();
+            findHorizontalFingerSliceTrails();
+            findVerticalFingerSliceTrails();
+            transformFingerSliceTrailsToFingers();
+        }
 
-            findFingerLines();
+        private void findHorizontalFingerSliceTrails()
+        {
 
-            sw.Stop();
-            Console.WriteLine("findFingerLines:" + sw.ElapsedMilliseconds);
-            sw.Restart();
+            horizontalFingerSliceTrails = new List<FingerSliceTrail>();
 
+            int width = depthImage.getWidth();
+            int height = depthImage.getHeight();
 
-            findFingers();
+            int minFingerSize = 10;
+            int maxFingerSize = 30;
 
-            sw.Stop();
-            Console.WriteLine("findFingers:" + sw.ElapsedMilliseconds);
+            //search horizontal finger-slices
+            for (int y = 0; y < height; y++)
+            {
+                int x = 0;
+                while (x <= width - minFingerSize)
+                {
+                    if (edgeImage.isEdgeAt(x, y) && depthImage.getDepthAt(x, y) > depthImage.getDepthAt(x + 1, y))
+                    {
+                        int sliceX1 = x;
+                        int maxSliceX2 = Math.Min(sliceX1 + maxFingerSize, width - 1);
+                        x++;
+                        while (x <= maxSliceX2)
+                        {
+                            if (edgeImage.isEdgeAt(x, y))
+                            {
+                                int sliceX2 = x;
+                                Vector2D sliceStart = new Vector2D(Math.Max(sliceX1 - 1, 0), y);
+                                Vector2D sliceEnd = new Vector2D(Math.Min(sliceX2 + 1, width - 1), y);
+                                FingerSlice fingerSlice = new FingerSlice(sliceStart, sliceEnd);
+                                if (fingerSlice.Size > minFingerSize && fingerSliceDepthTest(fingerSlice))
+                                {
+                                    addHorizontalFingerSlice(fingerSlice);
+                                    fingerImage.drawLine(fingerSlice.Line, FingerImageState.possibleFingerSlice);
+                                }
+                                break;
+                            }
+                            else x++;
+                        }
+                    }
+                    else
+                        x++;
 
+                }
+            }
         }
 
-        public List<Finger> getFingers() {
-            return fingers;
-        }
+        private void findVerticalFingerSliceTrails()
+        {
+            verticalFingerSliceTrails = new List<FingerSliceTrail>();
 
-        private void findFingerPoints() {
             int width = depthImage.getWidth();
             int height = depthImage.getHeight();
 
-            fingerPointsImage = new Image<Gray, byte>(width, height);
-            
-            for (int y = 0; y < height; y++) {
-                for (int x = 0; x < width; x++) {
-                    if (edgeImage.isEdgeAt(x, y))
+            int minFingerSize = 10;
+            int maxFingerSize = 30;
+
+            //search vertical finger-slices
+            for (int x = 0; x < width; x++)
+            {
+                int y = 0;
+                while (y <= height - minFingerSize)
+                {
+                    if (edgeImage.isEdgeAt(x, y) && depthImage.getDepthAt(x, y) > depthImage.getDepthAt(x, y + 1))
                     {
-                        Vector2D startPoint = new Vector2D(x, y);
-                        searchFingerPoint(startPoint, new Vector2D(1, 0), fingerPointsImage);
-                        searchFingerPoint(startPoint, new Vector2D(0, 1), fingerPointsImage);
-                        //searchFingerPoint(startPoint, new Vector2D(1, -1).normalize(), fingerPointsImage);
-                        //searchFingerPoint(startPoint, new Vector2D(1, 1).normalize(), fingerPointsImage);
+                        int sliceY1 = y;
+                        int maxSliceY2 = Math.Min(sliceY1 + maxFingerSize, height - 1);
+                        y++;
+                        while (y <= maxSliceY2)
+                        {
+                            if (edgeImage.isEdgeAt(x, y))
+                            {
+                                int sliceY2 = y;
+                                Vector2D sliceStart = new Vector2D(x, Math.Max(sliceY1 - 1, 0));
+                                Vector2D sliceEnd = new Vector2D(x, Math.Min(sliceY2 + 1, height - 1));
+                                FingerSlice fingerSlice = new FingerSlice(sliceStart, sliceEnd);
+                                if (fingerSlice.Size > minFingerSize && fingerSliceDepthTest(fingerSlice))
+                                {
+                                    addVerticalFingerSlice(fingerSlice);
+                                    fingerImage.drawLine(fingerSlice.Line, FingerImageState.possibleFingerSlice);
+                                }
+                                break;
+                            }
+                            else y++;
+                        }
                     }
+                    else
+                        y++;
+
                 }
             }
         }
 
-        private void searchFingerPoint(Vector2D start, Vector2D direction, Image<Gray, byte> fingerPointsImage) {
-            int width = fingerPointsImage.Width;
-            int height = fingerPointsImage.Height;
-            float minFingerSize = 10;
-            float maxFingerSize = 30;
-            direction = direction.normalize();
-
-
-            bool edgeFound = false;
-            Vector2D edge = start + minFingerSize * direction;
-            float index = minFingerSize;
-            float maxXIndex = (direction.X == 0) ? float.MaxValue : ((direction.X > 0) ? ((width - start.X) / direction.X) : (-start.X / direction.X));
-            float maxYIndex = (direction.Y == 0) ? float.MaxValue : ((direction.Y > 0) ? ((height - start.Y) / direction.Y) : (-start.Y / direction.Y));
+        private void transformFingerSliceTrailsToFingers()
+        {
+            int minNumSlices = 20;
 
-            float maxIndex = Math.Min(maxFingerSize, Math.Min(maxXIndex, maxYIndex));
+            Fingers = new List<Finger>();
+            List<FingerSliceTrail> fingerSliceTrails = new List<FingerSliceTrail>();
+            fingerSliceTrails.AddRange(horizontalFingerSliceTrails);
+            fingerSliceTrails.AddRange(verticalFingerSliceTrails);
 
-            if (index < maxIndex)
+            foreach (FingerSliceTrail trail in fingerSliceTrails)
             {
-                Int16 depthStart = depthImage.getDepthAt((int)start.X, (int)start.Y);
-                Int16 depthEdge = depthImage.getDepthAt((int)edge.X, (int)edge.Y);
-
-                if (depthStart > depthEdge)
+                if (trail.NumSlices >= minNumSlices)
                 {
-                    while (!edgeFound && index < maxIndex)
+                    Finger finger = new Finger(trail);
+
+                    //filter double hits (horizontal+vertical finger)
+                    bool addFinger = true;
+                    for (int i = fingers.Count - 1; i >= 0; i--)
                     {
-                        if (edgeImage.isEdgeAt((int)edge.X, (int)edge.Y))
-                            edgeFound = true;
-                        else
+                        Finger f = fingers[i];
+                        if (finger.getSimilarity(f) > 0.8)
                         {
-                            index++;
-                            edge += direction;
+                            if (finger.SliceTrail.NumSlices > f.SliceTrail.NumSlices)
+                                fingers.RemoveAt(i);
+                            else
+                                addFinger = false;
                         }
                     }
+
+                    if (addFinger)
+                    {
+                        fingers.Add(finger);
+                        fingerImage.drawFinger(finger, FingerImageState.fingerDetected);
+                    }
+
                 }
+
             }
-            
-            if (edgeFound && fingerDepthTest(start, edge))
-            {
-                Vector2D mid = 0.5f * (start + edge);
-                fingerPointsImage.Data[(int)mid.Y, (int)mid.X, 0] = byte.MaxValue;
-                fingerImage.setFingerAt((int)mid.X, (int)mid.Y, FingerImageState.possibleFinger);
-            }
-        }
 
-        private bool fingerDepthTest(Vector2D p1, Vector2D p2) {
-            Vector2D mid = 0.5f * (p1 + p2);
-            Int16 depthP1 = depthImage.getDepthAt((int)p1.X, (int)p1.Y);
-            Int16 depthMid = depthImage.getDepthAt((int)mid.X, (int)mid.Y);
-            Int16 depthP2 = depthImage.getDepthAt((int)p2.X, (int)p2.Y);
-            return (depthP1 > depthMid && depthMid < depthP2);
         }
 
-        private void findFingerLines() {
-            float maxCombinableAngle = (float) (Math.PI * 20 / 180);
-            float maxCombinableParallelDistance = 10;
-            float maxCombinableVerticalDistance = 10;
+        private bool fingerSliceDepthTest(FingerSlice fingerSlice)
+        {
+            Int16 depthStart = depthImage.getDepthAt((int)fingerSlice.Start.X, (int)fingerSlice.Start.Y);
+            Int16 depthMid = depthImage.getDepthAt((int)fingerSlice.Mid.X, (int)fingerSlice.Mid.Y);
+            Int16 depthEnd = depthImage.getDepthAt((int)fingerSlice.End.X, (int)fingerSlice.End.Y);
+            return (depthStart > depthMid && depthMid < depthEnd);
+        }
 
-            fingerPointsImage = fingerPointsImage.Dilate(1);
-            LineSegment2D[] lineSegments = fingerPointsImage.HoughLinesBinary(1, Math.PI / 90, 10, 20, 2)[0];
+        private void addHorizontalFingerSlice(FingerSlice slice)
+        {
+            int maxYGap = 5;
+            int maxXDifference = 5;
 
-            fingerLines = new List<Line2D>();
-            foreach (LineSegment2D lineSegment in lineSegments)
+            bool assigned = false;
+            foreach (FingerSliceTrail trail in horizontalFingerSliceTrails)
             {
-                Line2D line = new Line2D(new Vector2D(lineSegment.P1), new Vector2D(lineSegment.P2));
-
-                List<Line2D> combineableLines = new List<Line2D>();
-                foreach(Line2D fingerLine in fingerLines) {
-                    float angle = line.getAngleBetween(fingerLine);
-                    if (angle <= maxCombinableAngle) {
-                        float parallelDistance = line.getParallelDistanceTo(fingerLine);
-                        if (parallelDistance <= maxCombinableParallelDistance) {
-                            float verticalDistance = line.getVerticalDistanceTo(fingerLine);
-                            if (verticalDistance <= maxCombinableVerticalDistance) {
-                                combineableLines.Add(fingerLine);
-                            }
-                        }
-                    }
-                }
-                
-                if (combineableLines.Count == 0) {
-                    fingerLines.Add(line);
-                }
-                else {
-                    foreach (Line2D mergableLine in combineableLines) {
-                        fingerLines.Remove(mergableLine);
-                    }
-                    combineableLines.Add(line);
-                    fingerLines.Add(new Line2D(combineableLines));
+                FingerSlice trailEnd = trail.End;
+                if (slice.Mid.Y - trailEnd.Mid.Y <= maxYGap && Math.Abs(trailEnd.Mid.X - slice.Mid.X) <= maxXDifference)
+                {
+                    trail.addSlice(slice);
+                    assigned = true;
                 }
             }
+
+            if (!assigned)
+                horizontalFingerSliceTrails.Add(new FingerSliceTrail(slice));
         }
 
-        private void findFingers()
+        private void addVerticalFingerSlice(FingerSlice slice)
         {
-            float minLength = 20;
-            float minNumCombinedLines = 2;
+            int maxXGap = 4;
+            int maxYDifference = 4;
 
-            fingers = new List<Finger>();
-            foreach (Line2D line in fingerLines) {
-                if (line.Length >= minLength && line.NumCombinedLines >= minNumCombinedLines)
+            bool assigned = false;
+            foreach (FingerSliceTrail trail in verticalFingerSliceTrails)
+            {
+                FingerSlice trailEnd = trail.End;
+                if (slice.Mid.X - trailEnd.Mid.X <= maxXGap && Math.Abs(trailEnd.Mid.Y - slice.Mid.Y) <= maxYDifference)
                 {
-                    fingers.Add(new Finger(line));
-                    fingerImage.drawLine(line, FingerImageState.fingerDetected);
+                    trail.addSlice(slice);
+                    assigned = true;
                 }
             }
 
+            if (!assigned)
+                verticalFingerSliceTrails.Add(new FingerSliceTrail(slice));
         }
     }
 }

bbiwarg/Detectors/Fingers/FingerSlice.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bbiwarg.Utility;
+
+namespace bbiwarg.Detectors.Fingers
+{
+    class FingerSlice
+    {
+        private Vector2D start;
+        private Vector2D mid;
+        private Vector2D end;
+        public Vector2D Start { get { return start; } private set { start = value; } }
+        public Vector2D Mid { get { return mid; } private set { mid = value; } }
+        public Vector2D End { get { return end; } private set { end = value; } }
+        public Line2D Line { get { return new Line2D(Start, End); } }
+        public float Size { get { return Start.getDistanceTo(End); } }
+
+        public FingerSlice(Vector2D start, Vector2D end) {
+            Start = start;
+            End = end;
+            Mid = (start + end) / 2;
+        }
+
+    }
+}

bbiwarg/Detectors/Fingers/FingerSliceTrail.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bbiwarg.Utility;
+
+namespace bbiwarg.Detectors.Fingers
+{
+    class FingerSliceTrail
+    {
+        private List<FingerSlice> slices;
+        public FingerSlice Start { get { return slices[0]; } }
+        public FingerSlice End { get { return slices[slices.Count - 1]; } }
+        public FingerSlice this[int index] { get { return slices[index]; } }
+        public int NumSlices { get { return slices.Count; } }
+        public Line2D Line { get { return new Line2D(Start.Mid, End.Mid); } }
+
+        public FingerSliceTrail(FingerSlice slice)
+        {
+            slices = new List<FingerSlice>();
+            slices.Add(slice);
+        }
+
+        public void addSlice(FingerSlice slice)
+        {
+            slices.Add(slice);
+        }
+
+
+    }
+}

bbiwarg/Detectors/Touch/TouchDetector.cs

@@ -27,20 +27,17 @@ namespace bbiwarg.Detectors.Touch
             float floodValueThreshold = 0.5f;
 
             foreach (Finger finger in fingers) {
-                 Vector2D tipPoint = finger.getTipPoint();
+                 Vector2D tipPoint = finger.Tip;
 
                 float floodValue = getFloodValue((int)tipPoint.X, (int)tipPoint.Y);
                 if (floodValue > floodValueThreshold)
                 {
-                    /* disabled -> direction doesn't along finger
                     //correct touchEvent position
                     Vector2D direction = finger.Line.Direction;
-                    float directionFactor = -10;
+                    float directionFactor = 10;
                     float x = Math.Min(Math.Max(tipPoint.X + directionFactor * direction.X, 0), depthImage.getWidth()-1);
                     float y = Math.Min(Math.Max(tipPoint.Y + directionFactor * direction.Y, 0), depthImage.getHeight()-1);
                     Vector2D tep = new Vector2D(x,y);
-                     */
-                    Vector2D tep = tipPoint;
 
                     touchImage.setTouchAt((int)tep.X, (int)tep.Y, TouchImageState.touchDetected);
                     TouchEvent touchEvent = new TouchEvent((int)tep.X, (int)tep.Y, floodValue, finger);

bbiwarg/Graphics/OutputWindow.cs

@@ -132,8 +132,9 @@ namespace bbiwarg.Graphics
                     red = green = blue = 0;
                     if (videoHandle.isEdgeAt(x, y)) blue = Int16.MaxValue;
                     else if (fis == FingerImageState.fingerTracked) green = Int16.MaxValue;
-                    else if (fis == FingerImageState.fingerDetected) red = blue = Int16.MaxValue;
-                    else if (fis == FingerImageState.possibleFinger) red = Int16.MaxValue;
+                    else if (fis == FingerImageState.fingerDetected) red = Int16.MaxValue;
+                    else if (fis == FingerImageState.fingerSlice) red = blue = Int16.MaxValue;
+                    else if (fis == FingerImageState.possibleFingerSlice) red = Int16.MaxValue;
 
                     edgeTextureData[index] = red;
                     edgeTextureData[index + 1] = green;

bbiwarg/Images/FingerImage.cs

@@ -13,9 +13,10 @@ namespace bbiwarg.Images
 {
     public enum FingerImageState {
         none = 0,
-        possibleFinger = 1,
-        fingerDetected = 2,
-        fingerTracked = 3
+        possibleFingerSlice = 1,
+        fingerSlice = 2,
+        fingerDetected = 3,
+        fingerTracked = 4
     }
 
     class FingerImage
@@ -34,6 +35,14 @@ namespace bbiwarg.Images
             return (FingerImageState)image.Data[y, x, 0];
         }
 
+        public void drawFinger(Finger finger, FingerImageState state) {
+            FingerSliceTrail trail = finger.SliceTrail;
+            for (int i = 0; i < trail.NumSlices; i++) {
+                drawLine(trail[i].Line, FingerImageState.fingerSlice);
+            }
+            drawLine(finger.Line, state);
+        }
+
         public void drawLine(Line2D line, FingerImageState state)
         {
             Vector2D start = line.P1;

bbiwarg/Images/PalmImage.cs

@@ -27,8 +27,8 @@ namespace bbiwarg.Images
             
             foreach (Finger f in fingerTracker.getFingers())
             {
-                Vector<float> tip = new Vector<float>(new float[] { f.getTipPoint().X, f.getTipPoint().Y });
-                Vector<float> hand = new Vector<float>(new float[] { f.getHandPoint().X, f.getHandPoint().Y });
+                Vector<float> tip = new Vector<float>(new float[] { f.Tip.X, f.Tip.Y });
+                Vector<float> hand = new Vector<float>(new float[] { f.Hand.X, f.Hand.Y });
 
 
                 for (int x = 0; x < width; ++x)

bbiwarg/VideoHandle.cs

@@ -122,7 +122,7 @@ namespace bbiwarg
 
             //detect+track fingers
             fingerDetector = new FingerDetector(depthImage, edgeImage, fingerImage);
-            fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.getFingers(), fingerImage);
+            fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, fingerImage);
 
             //detect+track touchEvents
             touchDetector = new TouchDetector(fingerTracker.getFingers(), depthImage, touchImage);

bbiwarg/bbiwarg.csproj

@@ -68,6 +68,8 @@
   <ItemGroup>
     <Compile Include="Detectors\Fingers\Finger.cs" />
     <Compile Include="Detectors\Fingers\FingerDetector.cs" />
+    <Compile Include="Detectors\Fingers\FingerSliceTrail.cs" />
+    <Compile Include="Detectors\Fingers\FingerSlice.cs" />
     <Compile Include="Detectors\Fingers\FingerTracker.cs" />
     <Compile Include="Detectors\Touch\TouchDetector.cs" />
     <Compile Include="Detectors\Touch\TouchEvent.cs" />