Improved contour detection (using depthImage instead of edgeImage and floodfill to remove pointing hand).

bbiwarg/Images/PalmImage.cs

@@ -15,17 +15,35 @@ namespace bbiwarg.Images
 {
     class PalmImage
     {
-        private Image<Gray, Byte> image;
-        private Image<Gray, Byte> fingerSliceMask;
-        private Image<Gray, Byte> output;
+        DepthImage depthImage;
+        EdgeImage edgeImage;
+
+        private Image<Gray, Byte> handImage;
+        private Image<Gray, Byte> pointingHandMask;
+        private Image<Gray, Byte> outputImage;
+
+        private List<Finger> fingers;
+        
         private int width, height;
         
-        public PalmImage(DepthImage depthImage, FingerDetector fingerDetector)
+        public PalmImage(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector)
         {
-            image = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte) 0 : (byte) 1; });
-            width = image.Width;
-            height = image.Height;
+            this.depthImage = depthImage;
+            this.edgeImage = edgeImage;
+
+            handImage = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte) 0 : (byte) 1; });
+            width = depthImage.getWidth();
+            height = depthImage.getHeight();
+
+            fingers = getFingersWithoutThumb(fingerDetector);
+            buildPointingHandMask();
+            handImage = handImage.And(pointingHandMask);
 
+            findPalmContour();
+        }
+
+        private List<Finger> getFingersWithoutThumb(FingerDetector fingerDetector)
+        {
             List<Finger> detectedFingers = fingerDetector.Fingers;
             Finger leftMost = null;
             float minX = float.MaxValue;
@@ -39,39 +57,71 @@ namespace bbiwarg.Images
                 }
             }
 
-            fingerSliceMask = new Image<Gray, byte>(width, height, new Gray(1));
+            List<Finger> result = new List<Finger>();
             foreach (Finger f in detectedFingers)
             {
                 if (f != leftMost)
+                    result.Add(f);
+            }
+            return result;
+        }
+
+        private void fillFingerSlices(Image<Gray, Byte> image, byte val)
+        {
+            foreach (Finger f in fingers)
+            {
+                foreach (FingerSlice s in f.SliceTrail.getSlices())
                 {
-                    foreach (FingerSlice s in f.SliceTrail.getSlices())
-                    {
-                        Vector2D start = s.Start, end = s.End;
-                        if ((int)start.Y == (int)end.Y)
-                        {
-                            int y = (int)start.Y;
-                            for (int x = (int)start.X; x <= (int)end.X; ++x)
-                                fingerSliceMask.Data[y, x, 0] = 0;
-                        }
-                        else
-                        {
-                            int x = (int)start.X;
-                            for (int y = (int)start.Y; y <= (int)end.Y; ++y)
-                                fingerSliceMask.Data[y, x, 0] = 0;
-                        }
-                    }
+                    image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
+                }            
+            }
+        }
+
+        private Finger getLongestFinger()
+        {
+            float maxLength = 0;
+            Finger longest = null;
+            foreach (Finger f in fingers)
+            {
+                
+                if (f.Line.Length > maxLength)
+                {
+                    maxLength = f.Line.Length;
+                    longest = f;
                 }
             }
+            return longest;
+        }
 
-            fingerSliceMask = fingerSliceMask.Erode(2);
-            image = image.And(fingerSliceMask);
+        private Point getPointInPointingHand()
+        {
+            Finger finger = getLongestFinger();
+            if (finger == null)
+                return new Point(0, 0);
+            Vector2D direction = (finger.Hand - finger.Tip).normalize();
+            Vector2D pos = finger.Hand + 20 * direction;
+            return new Point(Math.Min(width - 1, Math.Max(0, (int)pos.X)), Math.Min(height - 1, Math.Max(0, (int)pos.Y)));
+        }
 
-            findContour();
+        private void buildPointingHandMask()
+        {
+            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
+            fillFingerSlices(pointingHandMask, 1);
+            pointingHandMask = pointingHandMask.Dilate(1);
+            pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert<Byte>(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; }));
+            pointingHandMask = pointingHandMask.Dilate(1);
+
+            MCvConnectedComp tmp = new MCvConnectedComp();
+            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
+            pointingHandMask = pointingHandMask.Convert<Byte>(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; });
+            pointingHandMask = pointingHandMask.Erode(1);
+            fillFingerSlices(pointingHandMask, 0);
+            pointingHandMask = pointingHandMask.Erode(2);
         }
 
-        private void findContour()
+        private void findPalmContour()
         {
-            Contour<Point> contour = image.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
+            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
             Contour<Point> maxContour = contour;
             double maxPerimeter = 0;
@@ -84,28 +134,44 @@ namespace bbiwarg.Images
                 }
                 contour = contour.HNext;
             }
-        
-            Contour<Point> contourPoly = maxContour; //.ApproxPoly(0);
-            Seq<Point> hull = contourPoly.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE);
-            Seq<MCvConvexityDefect> defects = contourPoly.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE);
-
-            output = new Image<Gray, byte>(width, height, new Gray(0));
-            output.Draw(contourPoly, new Gray(1), 1);
-            output.Draw(hull, new Gray(1), 1);
 
-            foreach (MCvConvexityDefect defect in defects)
+            if (maxContour != null)
             {
-                Point s = defect.StartPoint;
-                Point e = defect.EndPoint;
-                Point d = defect.DepthPoint;
+                Contour<Point> contourPoly = maxContour; //.ApproxPoly(0);
+                Seq<Point> hull = contourPoly.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE);
+                Seq<MCvConvexityDefect> defects = contourPoly.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE);
 
-                output.Draw(new LineSegment2DF(d, new Point((s.X + e.X) / 2, (s.Y + e.Y) / 2)), new Gray(1), 1);
+                outputImage = new Image<Gray, byte>(width, height, new Gray(0));
+                outputImage.Draw(contourPoly, new Gray(1), 1);
+                outputImage.Draw(hull, new Gray(1), 1);
+
+                foreach (MCvConvexityDefect defect in defects)
+                {
+                    Vector2D center = (new Vector2D(defect.StartPoint) + new Vector2D(defect.EndPoint)) / 2;
+
+                    float minFingerDist = float.MaxValue;
+                    foreach (Finger f in fingers)
+                    {
+                        float dist = f.Tip.getDistanceTo(new Vector2D(defect.DepthPoint));
+                        if (dist < minFingerDist)
+                            minFingerDist = dist;
+                    }
+
+                    if (minFingerDist > 20)
+                    {
+                        //Console.WriteLine(minFingerDist);
+                        outputImage.Draw(new LineSegment2DF(defect.DepthPoint, center), new Gray(1), 1);
+                    }
+                }
+                //Console.WriteLine();
             }
         }
 
         public bool belongsToPalm(int x, int y)
         {
-            return output.Data[y, x, 0] == 1;
+            if (outputImage != null)
+                return outputImage.Data[y, x, 0] == 1;
+            return false;
         }
     }
 }

bbiwarg/Utility/Vector2D.cs

@@ -24,6 +24,11 @@ namespace bbiwarg.Utility
             setXY(point.X, point.Y);
         }
 
+        public static implicit operator PointF(Vector2D vec)
+        {
+            return new PointF(vec.X, vec.Y);
+        }
+
         private void setXY(float x, float y) {
             X = x;
             Y = y;

bbiwarg/VideoHandle.cs

@@ -148,7 +148,7 @@ namespace bbiwarg
             touchDetector = new TouchDetector(fingerTracker.getFingers(), depthImage, touchImage);
             touchTracker.setDetectedTouchEventsThisFrame(touchDetector.getTouchEvents(), touchImage);
 
-            palmImage = new PalmImage(depthImage, fingerDetector);
+            palmImage = new PalmImage(depthImage, edgeImage, fingerDetector);
         }
     }
 }