Various improvements for palm detection.

bbiwarg/Constants.cs

@@ -36,14 +36,14 @@ namespace bbiwarg
 
         // palm detection
         public static readonly float PalmMinDefectMidFingerLineDistance = 20; // defects with mid point ((start + end) / 2) closer than this to a finger line are removed 
-        public static readonly float PalmMaxThumbDefectAngle = 100; // degree
-
+        public static readonly float PalmMaxThumbDefectAngle = 110; // degree
+        
         //palm Grid
         public static readonly int PalmGridRows = 4;
         public static readonly int PalmGridColumns = 3;
 
         // output window
-        public static readonly int NumImagesPerRow = 2;
+        public static readonly int NumImagesPerRow = 4;
         public static readonly float WindwoSizeFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
     }
 }

bbiwarg/Detectors/Fingers/Finger.cs

@@ -20,7 +20,7 @@ namespace bbiwarg.Detectors.Fingers
         {
             //check direction
             if (sliceTrail.Start.Length > sliceTrail.End.Length)
-                sliceTrail.Slices.Reverse();
+                ;// sliceTrail.Slices.Reverse();
 
             SliceTrail = sliceTrail;
         }
@@ -50,7 +50,7 @@ namespace bbiwarg.Detectors.Fingers
 
             //thresholds
             float thresholdMaxAngle = (float)(30 * Math.PI / 180); // 30°
-            float thresholdMaxParallelDistance = 40;
+            float thresholdMaxParallelDistance = 80;
             float thresholdMaxVerticalDistance = 40;
 
             //check angle

bbiwarg/Detectors/Fingers/FingerDetector.cs

@@ -39,7 +39,7 @@ namespace bbiwarg.Detectors.Fingers
 
             Fingers = new List<Finger>();
 
-            for (int y = 1; y < maxY; y+=5) //y++ for 100% coverage, but y++ for 99% coverage and 3 times better perfomance
+            for (int y = 1; y < maxY; y+=5) //y++ for 100% coverage, but y+=5 for 99% coverage and 3 times better perfomance
             {
                 for (int x = 1; x < maxX; x++)
                 {
@@ -131,7 +131,7 @@ namespace bbiwarg.Detectors.Fingers
             while (currentPosition.isWithin(0, 0, maxX, maxY) && gapCounter <= Math.Min(numSlices, 10))
             {
                 nextSlice = findFingerSliceFromMid(currentPosition, currentDirection);
-                if (nextSlice != null && (nextSlice.Length < lastSlice.Length + 5 && nextSlice.Length > lastSlice.Length - 5))
+                if (nextSlice != null && (nextSlice.Length < lastSlice.Length + 8 && nextSlice.Length > lastSlice.Length - 8))
                 {
                     gapCounter = 0;
                     numSlices++;

bbiwarg/Detectors/Palm/PalmDetector.cs

@@ -30,11 +30,10 @@ namespace bbiwarg.Detectors.Palm
         private List<MCvConvexityDefect> convexityDefects;
         private Vector2D thumbDefectStart;
         private Vector2D thumbDefectEnd;
-        private Vector2D thumbDefectDepth;
+        private Vector2D thumbDefectDepth, lastThumbDefectDepth;
         
         private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
 
-        private bool valid = false;
         private Vector2D topLeft;
         private Vector2D topRight;
         private Vector2D bottomLeft;
@@ -55,36 +54,46 @@ namespace bbiwarg.Detectors.Palm
             this.width = depthImage.Width;
             this.height = depthImage.Height;
 
-            this.edgeImage = edgeImage;
-            this.outputImage = outputImage;
-
-            // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
-            handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
-
-            fingers = getFingersWithoutThumb(trackedFingers);
-            buildPointingHandMask();
-            handImage = handImage.And(pointingHandMask);
+            i1 = new OutputImage(width, height);
+            i2 = new OutputImage(width, height);
+            i3 = new OutputImage(width, height);
+            i4 = new OutputImage(width, height);
+            i5 = new OutputImage(width, height);
+            i6 = new OutputImage(width, height);
+            i7 = new OutputImage(width, height);
+            i8 = new OutputImage(width, height);
+            i9 = new OutputImage(width, height);
 
-            findLongestPalmContour();
-            if (palmContour != null)
+            if (trackedFingers.Count >= 2)
             {
-                findConvexityDefectsSortedByDepth();
-                removeConvexityDefectsCausedByFingers();
+                this.edgeImage = edgeImage;
+                this.outputImage = outputImage;
+
+                // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
+                handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
 
-                findHandPoints();
+                fingers = getFingersWithoutThumb(trackedFingers);
+                buildPointingHandMask();
+                handImage = handImage.And(pointingHandMask);
 
-                if (valid)
+                findLongestPalmContour();
+                if (palmContour != null)
                 {
-                    draw();
+                    findConvexityDefectsSortedByDepth();
+                    removeConvexityDefectsCausedByFingers();
+
+                    findHandPoints();
                 }
             }
+            draw();
         }
 
-        public void resetFilters()
+        public void reset()
         {
             thumbDefectDepthFilter.reset();
             thumbDefectStartFilter.reset();
             thumbDefectEndFilter.reset();
+            lastThumbDefectDepth = null;
         }
 
         private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
@@ -114,9 +123,9 @@ namespace bbiwarg.Detectors.Palm
         {
             foreach (Finger f in fingers)
             {
-                foreach (FingerSlice s in f.SliceTrail.Slices)
+                //foreach (FingerSlice s in f.SliceTrail.Slices)
                 {
-                    image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
+                    image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1);
                 }
             }
         }
@@ -143,34 +152,30 @@ namespace bbiwarg.Detectors.Palm
             if (finger == null)
                 return new Point(0, 0);
             Vector2D direction = (finger.Hand - finger.Tip).normalize();
-            Vector2D pos = finger.Hand + 20 * direction;
+            Vector2D pos = finger.Hand + direction;
+            
+            while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
+                pos += direction;
+
+            i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red);
+
             return new Point(HelperFunctions.thresholdRange<int>(0, width - 1, pos.IntX), HelperFunctions.thresholdRange<int>(0, height - 1, pos.IntY));
         }
 
         public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9;
         private void buildPointingHandMask()
         {
-            /*i1 = new OutputImage(width, height);
-            i2 = new OutputImage(width, height);
-            i3 = new OutputImage(width, height);
-            i4 = new OutputImage(width, height);
-            i5 = new OutputImage(width, height);
-            i6 = new OutputImage(width, height);
-            i7 = new OutputImage(width, height);
-            i8 = new OutputImage(width, height);
-            i9 = new OutputImage(width, height);*/
-
             pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
 
             // dst = (src > 0) ? 1 : 0;
             pointingHandMask = pointingHandMask.Or(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(1)));
-            //i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
+            i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
 
-            pointingHandMask = pointingHandMask.Dilate(4);
-            //i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
+            pointingHandMask = pointingHandMask.Dilate(2);
+            i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
            
             fillFingerSlices(pointingHandMask, 1);
-            //i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
+            i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
 
             //pointingHandMask = pointingHandMask.Dilate(1);
             //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask;
@@ -181,29 +186,23 @@ namespace bbiwarg.Detectors.Palm
 
             // dst = (src > 1) ? 0 : 1    (src > 1 <-> src == 2 <-> src filled by flood fill)
             pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1));
-            //i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
+            i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
             
-            pointingHandMask = pointingHandMask.Erode(8);
-            //i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
+            pointingHandMask = pointingHandMask.Erode(6);
+            i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
 
             fillFingerSlices(pointingHandMask, 0);
-            //i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
+            i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
             
             pointingHandMask = pointingHandMask.Erode(2);
-            //i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
+            i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
             
             // only debug
-            //i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
+            i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
         }
 
         private void findLongestPalmContour()
         {
-            i1 = new OutputImage(width, height);
-            i1.Image[0] = i1.Image[1] = i1.Image[2] = handImage * 255;
-
-            i2 = new OutputImage(width, height);
-            i2.Image[0] = i2.Image[1] = i2.Image[2] = pointingHandMask * 255;
-
             Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
             palmContour = contour;
@@ -237,17 +236,25 @@ namespace bbiwarg.Detectors.Palm
             List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
             foreach (MCvConvexityDefect d in convexityDefects)
             {
+                bool remove = false;
                 float minFingerLineDist = float.MaxValue;
                 foreach (Finger f in fingers)
                 {
-                    Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
+                    Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
+                    Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
+                    if (intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2))
+                    {
+                        remove = true;
+                        break;
+                    }
                     
+                    Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
                     float dist = f.LineSegment.getDistanceTo(mid);
                     if (dist < minFingerLineDist)
                         minFingerLineDist = dist;
                 }
                 
-                if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance)
+                if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !remove)
                     newDefects.Add(d);
             }
             convexityDefects = newDefects;
@@ -264,7 +271,9 @@ namespace bbiwarg.Detectors.Palm
                 float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
 
                 if (angle <= Constants.PalmMaxThumbDefectAngle)
+                {
                     return d;
+                }
             }
             return null;
         }
@@ -288,12 +297,6 @@ namespace bbiwarg.Detectors.Palm
                     thumbDefectStartFilter.setInitialPosition(thumbDefectStart);
                     thumbDefectEndFilter.setInitialPosition(thumbDefectEnd);
                 }
-                else if (thumbDefect == null)
-                {
-                    thumbDefectDepth = thumbDefectDepthFilter.getPrediction();
-                    thumbDefectStart = thumbDefectStartFilter.getPrediction();
-                    thumbDefectEnd = thumbDefectEndFilter.getPrediction();
-                }
                 else
                 {
                     thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth);
@@ -301,6 +304,8 @@ namespace bbiwarg.Detectors.Palm
                     thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd);
                 }
 
+                lastThumbDefectDepth = thumbDefectDepth;
+
                 Vector2D handLength, handWidth;
                 if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd))
                 {
@@ -324,29 +329,26 @@ namespace bbiwarg.Detectors.Palm
                 }
 
                 PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight);
-
-                valid = true;
-            }
-            else
-            {
-                PalmQuad = null;
-                valid = false;
             }
         }
 
         private void draw()
         {
-            outputImage.drawContour(palmContour, Constants.PalmConturColor);
-            outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
+
+
+            if (palmContour != null && palmContour.Count<Point>() > 0) {
+                outputImage.drawContour(palmContour, Constants.PalmConturColor);
+                outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
+            }
 
             if (PalmQuad != null)
             {
+                outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red);
+                outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red);
                 outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red);
-                outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Green);
-                outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Blue);
 
-                //outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
-                
+                outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
+
                 Vector2D[] vertices = PalmQuad.Vertices;
                 for (int i = 0; i < 4; ++i)
                     outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor);
@@ -357,20 +359,17 @@ namespace bbiwarg.Detectors.Palm
 
         private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d)
         {
-            int numRows = 4;
-            int numColumns = 3;
-
-            Vector2D relAB = (b - a) / numRows;
-            Vector2D relDC = (c - d) / numRows;
-            Vector2D relBC = (c - b) / numColumns;
-            Vector2D relAD = (d - a) / numColumns;
+            Vector2D relAB = (b - a) / Constants.PalmGridRows;
+            Vector2D relDC = (c - d) / Constants.PalmGridRows;
+            Vector2D relBC = (c - b) / Constants.PalmGridColumns;
+            Vector2D relAD = (d - a) / Constants.PalmGridColumns;
 
-            for (int i = 1; i < numRows; i++)
+            for (int i = 1; i < Constants.PalmGridRows; i++)
             {
                 outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor);
             }
 
-            for (int i = 1; i < numColumns; i++)
+            for (int i = 1; i < Constants.PalmGridColumns; i++)
             {
                 outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor);
             }

bbiwarg/Images/DepthImage.cs

@@ -14,6 +14,7 @@ namespace bbiwarg.Images
 {
     class DepthImage
     {
+        private OutputImage outputImage;
         public Image<Gray, byte> Image { get; private set; }
         public Image<Gray, byte> BackgroundMask { get; private set; }
         public int Width { get; private set; }
@@ -21,19 +22,23 @@ namespace bbiwarg.Images
         public Int16 MinDepth { get; private set; }
         public Int16 MaxDepth { get; private set; }
 
-        public DepthImage(Image<Gray, Int16> image)
+        public DepthImage(Image<Gray, Int16> image, OutputImage outputImage)
         {
+            this.outputImage = outputImage;
+
             Width = image.Width;
             Height = image.Height;
 
-            image = image.SmoothMedian(3);
+            image = image.SmoothMedian(5);
 
             //threshold min&maxDepth
             MinDepth = findMinDepth(image);
             MaxDepth = (Int16)(MinDepth + 200); // max = minDepth+255 (else it can't fit whole range in byte image)
 
             //smooth+threshold (dst = (src > (MaxDepth - MinDepth)) ? MaxDepth - MinDepth : src)
-            Image = image.Sub(new Gray(MinDepth)).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
+            Image = (image- MinDepth).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
+
+            Image = Image.SmoothMedian(5);
         }
 
         public Int16 getDepthAt(Point point)
@@ -56,17 +61,6 @@ namespace bbiwarg.Images
             Image.Data[y, x, 0] = (byte)(depth - MinDepth);
         }
 
-        public float getRelativeDepthAt(Point point)
-        {
-            return getRelativeDepthAt(point.X, point.Y);
-        }
-
-        public float getRelativeDepthAt(int x, int y)
-        {
-            float minMaxInterval = Math.Max(MaxDepth - MinDepth, 1);
-            return (getDepthAt(x, y) - MinDepth) / minMaxInterval;
-        }
-
         private Int16 findMinDepth(Image<Gray, Int16> image)
         {
             // min and max values
@@ -84,14 +78,16 @@ namespace bbiwarg.Images
             MCvConnectedComp comp = new MCvConnectedComp();
 
             foreach (Finger finger in fingers) {
-                Vector2D mid = (finger.Tip + finger.Hand) / 2;
-                if(BackgroundMask.Data[mid.IntY, mid.IntX,0] != 0)
-                    CvInvoke.cvFloodFill(BackgroundMask, mid, new MCvScalar(0), new MCvScalar(5), new MCvScalar(5), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
+                Vector2D mid = finger.SliceTrail.Slices[finger.SliceTrail.NumSlices / 2].Mid;
+
+                outputImage.fillCircle(mid.IntX, mid.IntY, 3, Color.Blue);
+
+                if (true) //(BackgroundMask.Data[mid.IntY, mid.IntX,0] != 0)
+                    CvInvoke.cvFloodFill(BackgroundMask, mid, new MCvScalar(0), new MCvScalar(3), new MCvScalar(3), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
             }
 
-            BackgroundMask = BackgroundMask.ThresholdBinary(new Gray(0), new Gray(255));
+            BackgroundMask = BackgroundMask.Erode(3).Dilate(3).ThresholdBinary(new Gray(0), new Gray(255));
             Image = Image.Or(BackgroundMask);
-
         }
     }
 }

bbiwarg/VideoHandle.cs

@@ -109,7 +109,7 @@ namespace bbiwarg
             {
                 inputProvider.stop();
             }
-            ++videoFrame;
+            videoFrame++;
         }
 
         public List<PalmTouchEvent> getPalmTouchEvents()
@@ -141,7 +141,7 @@ namespace bbiwarg
             //create depthImage
             Timer.start("createDepthImage");
             Image<Gray, Int16> image = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawDepthData);
-            depthImage = new DepthImage(image);
+            depthImage = new DepthImage(image, OutputImages[0]);
             Image<Gray, byte> tmpDepth = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
             OutputImages[0].Image[0] = OutputImages[0].Image[1] = OutputImages[0].Image[2] = tmpDepth;
             Timer.stop("createDepthImage");
@@ -164,17 +164,18 @@ namespace bbiwarg
 
             //remove background noise
             Timer.start("removeBackground");
-            depthImage.removeBackground(fingerTracker.TrackedFingers);
+            depthImage.removeBackground(fingerDetector.Fingers);
             edgeImage = new EdgeImage(depthImage);
             OutputImages[2].Image[0] = OutputImages[2].Image[1] = OutputImages[2].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
             Timer.stop("removeBackground");
 
+            OutputImage palm = new OutputImage(Width, Height);
+
             //detect palm
             Timer.start("palmDetection");
-            if (fingerTracker.TrackedFingers.Count >= 2)
-                palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[2], fingerTracker.TrackedFingers);
             if (CurrentFrame == 0)
-                palmDetector.resetFilters();
+                palmDetector.reset();
+            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[2], fingerDetector.Fingers);
             Timer.stop("palmDetection");
 
             //detect touchEvents
@@ -209,6 +210,9 @@ namespace bbiwarg
 
             }
 
+            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], palmDetector.i1, palmDetector.i2,
+            palmDetector.i3, palmDetector.i5, palmDetector.i6, palmDetector.i7, palmDetector.i8, palmDetector.i9};
+
             Timer.stop("processFrameUpdate");
         }
     }