-added HandDetection

-improved fingerDetection with roughEdges
-moved constants for edge+depthimage

bbiwarg/Constants.cs

@@ -36,6 +36,16 @@ namespace bbiwarg
         public static readonly Color PalmConvexHullColor = Color.Green;
         public static readonly Color PalmThumbDefectColor = Color.Lime;
 
+        // depth image
+        public static readonly int DepthImageMedianSize = 5;
+        public static readonly int DepthImageDepthRange = 200; // <255
+
+        // edge image
+        public static readonly int EdgeImageCannyStartThreshold = 100;
+        public static readonly int EdgeImageCannyLinkingThreshold = 75;
+        public static readonly int EdgeImageCannySize = 3;
+        public static readonly int EdgeImageRoughNumDilationIterations = 1;
+
         // finger detection
         public static readonly int FingerStepSize = 2;
         public static readonly int FingerMinNumSlices = 10;
@@ -46,9 +56,12 @@ namespace bbiwarg
         public static readonly int FingerMinSize = 5;
         public static readonly int FingerNumSlicesForRelativeDirection = 5;
         public static readonly int FingerNumFramesUntilTracked = 2;
-        public static readonly int FingerOutSliceFactor = 4;
+        public static readonly int FingerOutSliceFactor = 5;
         public static readonly float FingerMinSimilarityForTracking = 0.75f;
 
+        // hand detection
+        public static readonly float HandMaxSize = 0.7f;
+
         // palm detection
         public static readonly float PalmMinDefectMidFingerLineDistance = 20; // defects with mid point ((start + end) / 2) closer than this to a finger line are removed 
         public static readonly float PalmMaxThumbDefectAngle = 110; // degree
@@ -68,7 +81,7 @@ namespace bbiwarg
         public static readonly int TouchEventNumFramesUntilTracked = 2;
 
         // output window
-        public static readonly int NumImagesPerRow = 2;
+        public static readonly int NumImagesPerRow = 3;
         public static readonly float WindwoSizeFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
     }
 }

bbiwarg/Detectors/FingerDetection/Finger.cs

@@ -11,8 +11,8 @@ namespace bbiwarg.Detectors.FingerDetection
 {
     class Finger
     {
-        public Vector2D Tip { get { return SliceTrail.Start.Mid; } }
-        public Vector2D Hand { get { return SliceTrail.End.Mid; } }
+        public Vector2D TipPoint { get { return SliceTrail.Start.Mid; } }
+        public Vector2D HandPoint { get { return SliceTrail.End.Mid; } }
         public LineSegment2D LineSegment { get { return SliceTrail.LineSegment; } }
         public FingerSliceTrail SliceTrail { get; private set; }
 
@@ -25,7 +25,7 @@ namespace bbiwarg.Detectors.FingerDetection
             SliceTrail = sliceTrail;
         }
 
-        public Point[] getPolygon()
+        public Point[] getBoundingPolygon()
         {
             List<Point> pointsA = new List<Point>();
             List<Point> pointsB = new List<Point>();
@@ -33,8 +33,9 @@ namespace bbiwarg.Detectors.FingerDetection
             //for (int i = 0; i < numSlices; i++)
             foreach (FingerSlice slice in SliceTrail.Slices)
             {
-                pointsA.Add(slice.Start);
-                pointsB.Add(slice.End);
+                Vector2D direction = (slice.End - slice.Start).normalize();
+                pointsA.Add(slice.Start-2*direction);
+                pointsB.Add(slice.End+2*direction);
             }
 
             pointsB.Reverse();

bbiwarg/Detectors/FingerDetection/FingerDetector.cs

@@ -16,14 +16,16 @@ namespace bbiwarg.Detectors.FingerDetection
     class FingerDetector
     {
         private DepthImage depthImage;
-        private EdgeImage edgeImage;
+        private EdgeImage edgeImageOriginal;
+        private EdgeImage edgeImageAdapted;
         private OutputImage outputImage;
         public List<Finger> Fingers { get; private set; }
 
         public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage)
         {
             this.depthImage = depthImage;
-            this.edgeImage = edgeImage.copy();
+            this.edgeImageOriginal = edgeImage;
+            this.edgeImageAdapted = edgeImage.copy();
             this.outputImage = outputImage;
 
             findFingers();
@@ -40,7 +42,7 @@ namespace bbiwarg.Detectors.FingerDetection
             {
                 for (int x = 1; x < maxX; x++)
                 {
-                    if (edgeImage.isEdgeAt(x, y))
+                    if (edgeImageAdapted.isEdgeAt(x, y))
                     {
                         Vector2D edgePoint = new Vector2D(x, y);
                         Vector2D edgeDirection = getEdgeDirection(edgePoint);
@@ -68,10 +70,10 @@ namespace bbiwarg.Detectors.FingerDetection
             int x = edgePoint.IntX;
             int y = edgePoint.IntY;
 
-            if (edgeImage.isEdgeAt(x, y - 1) && edgeImage.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
-            else if (edgeImage.isEdgeAt(x - 1, y) && edgeImage.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
-            else if (edgeImage.isEdgeAt(x - 1, y - 1) && edgeImage.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
-            else if (edgeImage.isEdgeAt(x + 1, y - 1) && edgeImage.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
+            if (edgeImageAdapted.isEdgeAt(x, y - 1) && edgeImageAdapted.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
+            else if (edgeImageAdapted.isEdgeAt(x - 1, y) && edgeImageAdapted.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
+            else if (edgeImageAdapted.isEdgeAt(x - 1, y - 1) && edgeImageAdapted.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
+            else if (edgeImageAdapted.isEdgeAt(x + 1, y - 1) && edgeImageAdapted.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
             else return null;
         }
 
@@ -148,7 +150,7 @@ namespace bbiwarg.Detectors.FingerDetection
 
         private FingerSlice findFingerSliceFromMid(Vector2D position, Vector2D direction)
         {
-            if (edgeImage.isEdgeAt(position)) return null;
+            if (edgeImageAdapted.isRoughEdgeAt(position)) return null;
 
             Vector2D dirStart = direction.getOrthogonal(true);
             Vector2D dirEnd = direction.getOrthogonal(false);
@@ -163,13 +165,13 @@ namespace bbiwarg.Detectors.FingerDetection
         }
         private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction)
         {
-            Vector2D end = findNextEdge(start, direction);
+            Vector2D end = findNextEdge(start+3*direction, direction);
             if (end == null) return null;
 
             return getFingerSlice(start, end);
         }
 
-        private Vector2D findNextEdge(Vector2D start, Vector2D direction, bool stopAtMaxFingerSize = true, bool returnBoundIfNoEdge = false)
+        private Vector2D findNextEdge(Vector2D start, Vector2D direction, bool adaptedEdgeImage = true, bool stopAtMaxFingerSize = true, bool returnBoundIfNoEdge = false)
         {
             int maxX = depthImage.Width - 1;
             int maxY = depthImage.Height - 1;
@@ -200,7 +202,7 @@ namespace bbiwarg.Detectors.FingerDetection
             {
                 end += direction;
 
-                if (edgeImage.isEdgeAt(end))
+                if ((adaptedEdgeImage && edgeImageAdapted.isRoughEdgeAt(end)) || (!adaptedEdgeImage && edgeImageOriginal.isRoughEdgeAt(end)))
                 {
                     return end;
                 }
@@ -251,7 +253,8 @@ namespace bbiwarg.Detectors.FingerDetection
             drawDetectedFinger(finger);
 
             //remove edges around detected finger to improve performance
-            edgeImage.removeFingerEdges(finger);
+            Point[] polygon = finger.getBoundingPolygon();
+            edgeImageAdapted.removeEdgesInsidePolygon(polygon);
         }
 
         private FingerSlice findOutSlice(Vector2D start, Vector2D direction)
@@ -263,8 +266,8 @@ namespace bbiwarg.Detectors.FingerDetection
             Vector2D dirOrth2 = direction.getOrthogonal(false);
 
             Vector2D outPoint = (start + Constants.FingerOutSliceFactor * Constants.FingerStepSize * direction).moveInBound(0, 0, maxX, maxY);
-            Vector2D p1 = findNextEdge(outPoint, dirOrth1, false, true);
-            Vector2D p2 = findNextEdge(outPoint, dirOrth2, false, true);
+            Vector2D p1 = findNextEdge(outPoint, dirOrth1, false, false, true);
+            Vector2D p2 = findNextEdge(outPoint, dirOrth2, false, false, true);
 
             FingerSlice slice = new FingerSlice(p1, p2);
 

bbiwarg/Detectors/HandDetection/Hand.cs

@@ -3,10 +3,29 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Emgu.CV;
+using Emgu.CV.Structure;
+using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Utility;
 
 namespace bbiwarg.Detectors.HandDetection
 {
     class Hand
     {
+        public Image<Gray, byte> Mask { get; private set; }
+        public List<Finger> Fingers { get; private set; }
+
+        public Hand(Image<Gray, byte> mask) {
+            Mask = mask;
+            Fingers = new List<Finger>();
+        }
+
+        public bool isInside(Vector2D point) {
+            return (Mask.Data[point.IntY, point.IntX, 0] == 1);
+        }
+
+        public void addFinger(Finger finger) {
+            Fingers.Add(finger);
+        }
     }
 }

bbiwarg/Detectors/HandDetection/HandDetector.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Drawing;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Graphics;
+using bbiwarg.Utility;
 using Emgu.CV;
 using Emgu.CV.Structure;
 
@@ -16,29 +19,64 @@ namespace bbiwarg.Detectors.HandDetection
         private EdgeImage edgeImage;
         private List<Finger> fingers;
         public List<Hand> Hands { get; private set; }
+        public OutputImage outputImage;
 
-        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers) {
+        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers, OutputImage outputImage)
+        {
             this.depthImage = depthImage;
             this.edgeImage = edgeImage;
             this.fingers = fingers;
+            this.outputImage = outputImage;
 
             detectHands();
         }
 
-        private void detectHands() {
-            Image<Gray, byte> image = edgeImage.Image.Copy().Mul(255);
+        private void detectHands()
+        {
+            int width = depthImage.Width;
+            int height = depthImage.Height;
+            int maxArea = width * height;
+            Image<Gray, byte> image = edgeImage.Image.Copy().Dilate(2).Erode(2).Mul(255);
 
             //draw top finger slice
-            foreach (Finger finger in fingers) {
-                FingerSlice topSlice = finger.SliceTrail.Start;
-                image.Draw(new LineSegment2D(topSlice.Start, topSlice.End), new Gray(255), 2);
+            foreach (Finger finger in fingers)
+            {
+                FingerSlice slice = finger.SliceTrail.Start;
+                image.Draw(new Emgu.CV.Structure.LineSegment2D(slice.Start, slice.End), new Gray(255), 1);
             }
 
-            int numHands = 0;
-            foreach (Finger finger in fingers) {
-                
+            Hands = new List<Hand>();
+
+            foreach (Finger finger in fingers)
+            {
+                bool newHand = true;
+                foreach (Hand hand in Hands)
+                {
+                    if (hand.isInside(finger.HandPoint))
+                    {
+                        hand.addFinger(finger);
+                        newHand = false;
+                    }
+                }
+
+                if (newHand)
+                {
+                    Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
+                    MCvConnectedComp comp = new MCvConnectedComp();
+                    CvInvoke.cvFloodFill(image, finger.HandPoint, new MCvScalar(255), new MCvScalar(1), new MCvScalar(1), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
+                    if (comp.area < maxArea * Constants.HandMaxSize)
+                    {
+                        Hand hand = new Hand(mask.Copy(new Rectangle(1, 1, width, height)));
+                        hand.addFinger(finger);
+                        Hands.Add(hand);
+                    }
+                }
             }
 
+            for (int i = 0; i < Math.Min(3, Hands.Count); i++)
+            {
+                outputImage.Image[i] = Hands[i].Mask.Mul(255);
+            }
         }
     }
 }

bbiwarg/Detectors/PalmDetection/PalmDetector.cs

@@ -102,7 +102,7 @@ namespace bbiwarg.Detectors.PalmDetection
             float minX = float.MaxValue;
             foreach (Finger f in detectedFingers)
             {
-                float midX = ((f.Hand + f.Tip) / 2).X;
+                float midX = ((f.HandPoint + f.TipPoint) / 2).X;
                 if (midX < minX)
                 {
                     minX = midX;
@@ -132,8 +132,8 @@ namespace bbiwarg.Detectors.PalmDetection
             Finger finger = fingers[0];
             if (finger == null)
                 return new Point(0, 0);
-            Vector2D direction = (finger.Hand - finger.Tip).normalize();
-            Vector2D pos = finger.Hand + direction;
+            Vector2D direction = (finger.HandPoint - finger.TipPoint).normalize();
+            Vector2D pos = finger.HandPoint + direction;
             
             while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
                 pos += direction;
@@ -265,7 +265,7 @@ namespace bbiwarg.Detectors.PalmDetection
             }
 
 
-            Console.WriteLine("no palm defect found (" + i + ")");
+            //Console.WriteLine("no palm defect found (" + i + ")");
             foreach (MCvConvexityDefect d in convexityDefects)
             {
                 Vector2D depth = new Vector2D(d.DepthPoint);
@@ -278,7 +278,7 @@ namespace bbiwarg.Detectors.PalmDetection
 
                 float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
 
-                Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
+                //Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
             }
 
             ++i;

bbiwarg/Detectors/TouchDetection/TouchDetector.cs

@@ -30,7 +30,7 @@ namespace bbiwarg.Detectors.TouchDetection
 
             foreach (Finger finger in fingers)
             {
-                Vector2D tipPoint = finger.Tip;
+                Vector2D tipPoint = finger.TipPoint;
 
                 outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
 

bbiwarg/Images/DepthImage.cs

@@ -29,16 +29,16 @@ namespace bbiwarg.Images
             Width = image.Width;
             Height = image.Height;
 
-            image = image.SmoothMedian(5);
+            image = image.SmoothMedian(Constants.DepthImageMedianSize);
 
             //threshold min&maxDepth
             MinDepth = findMinDepth(image);
-            MaxDepth = (Int16)(MinDepth + 200); // max = minDepth+255 (else it can't fit whole range in byte image)
+            MaxDepth = (Int16)(MinDepth + Constants.DepthImageDepthRange);
 
             //smooth+threshold (dst = (src > (MaxDepth - MinDepth)) ? MaxDepth - MinDepth : src)
             Image = (image- MinDepth).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
 
-            Image = Image.SmoothMedian(5);
+            Image = Image.SmoothMedian(Constants.DepthImageMedianSize);
         }
 
         public Int16 getDepthAt(Point point)

bbiwarg/Images/EdgeImage.cs

@@ -17,15 +17,18 @@ namespace bbiwarg.Images
     class EdgeImage
     {
         public Image<Gray, Byte> Image { get; private set; }
+        public Image<Gray, byte> RoughImage { get; private set; }
 
         public EdgeImage(DepthImage depthImage)
         {
-            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(100, 75, 3).ThresholdBinary(new Gray(0), new Gray(1));
+            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(Constants.EdgeImageCannyStartThreshold, Constants.EdgeImageCannyLinkingThreshold, Constants.EdgeImageCannySize).ThresholdBinary(new Gray(0), new Gray(1));
+            RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }
 
         public EdgeImage(Image<Gray, Byte> edgeImage)
         {
             Image = edgeImage;
+            RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }
 
         public bool isEdgeAt(Point point)
@@ -38,10 +41,18 @@ namespace bbiwarg.Images
             return (Image.Data[y, x, 0] > 0);
         }
 
-        public void removeFingerEdges(Finger finger)
+        public bool isRoughEdgeAt(Point point)
         {
-            Point[] polygon = finger.getPolygon();
+            return isRoughEdgeAt(point.X, point.Y);
+        }
+
+        public bool isRoughEdgeAt(int x, int y)
+        {
+            return (RoughImage.Data[y, x, 0] > 0);
+        }
 
+        public void removeEdgesInsidePolygon(Point[] polygon)
+        {
             Image.FillConvexPoly(polygon, new Gray(0));
         }
 
@@ -49,5 +60,9 @@ namespace bbiwarg.Images
         {
             return new EdgeImage(Image.Copy());
         }
+
+        public EdgeImage getRoughEdgeImage(int numDilateIterations) {
+            return new EdgeImage(Image.Dilate(numDilateIterations));
+        }
     }
 }

bbiwarg/VideoHandle.cs

@@ -132,7 +132,7 @@ namespace bbiwarg
 
             //create output images
             Timer.start("createOtherImages");
-            int numImages = 4;
+            int numImages = 5;
             OutputImages = new OutputImage[numImages];
             for (int i = 0; i < numImages; i++) {
                 OutputImages[i] = new OutputImage(Width, Height);
@@ -166,23 +166,21 @@ namespace bbiwarg
 
             //detect hands
             Timer.start("handDetection");
-            handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.TrackedFingers);
+            handDetector = new HandDetector(depthImage, edgeImage, fingerDetector.Fingers, OutputImages[2]);
             Timer.stop("handDetection");
 
             //remove background noise
             Timer.start("removeBackground");
-            depthImage.removeBackground(fingerDetector.Fingers);
-            edgeImage = new EdgeImage(depthImage);
-            OutputImages[2].Image[0] = OutputImages[2].Image[1] = OutputImages[2].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
+            //depthImage.removeBackground(fingerDetector.Fingers);
+            //edgeImage = new EdgeImage(depthImage);
+            OutputImages[3].Image[0] = OutputImages[3].Image[1] = OutputImages[3].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
             Timer.stop("removeBackground");
 
-            OutputImage palm = new OutputImage(Width, Height);
-
             //detect palm
             Timer.start("palmDetection");
             if (CurrentFrame == 0)
                 palmDetector.reset();
-            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[2], fingerDetector.Fingers);
+            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[3], fingerDetector.Fingers);
             Timer.stop("palmDetection");
 
             //detect touchEvents
@@ -194,7 +192,7 @@ namespace bbiwarg
 
             //track touchEvents
             Timer.start("touchTracking");
-            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[2]);
+            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[3]);
             Timer.stop("touchTracking");
 
             // touch event visualizer
@@ -207,7 +205,7 @@ namespace bbiwarg
                 foreach (PalmTouchEvent e in palmTouchDetector.PalmTouchEvents)
                     touchEventVisualizer.addPalmTouchEvent(e, CurrentFrame);
                 touchEventVisualizer.updateImage();
-                OutputImages[3] = touchEventVisualizer.OutputImage;
+                OutputImages[4] = touchEventVisualizer.OutputImage;
             }
 
             // add borders
@@ -217,7 +215,7 @@ namespace bbiwarg
 
             }
 
-            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], OutputImages[3]};
+            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], OutputImages[3], OutputImages[4]};
             
             //palmDetector.i1, palmDetector.i2, palmDetector.i3, palmDetector.i5, palmDetector.i6, palmDetector.i7, palmDetector.i8, palmDetector.i9};