Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

bbiwarg/Constants.cs

@@ -9,6 +9,62 @@ namespace bbiwarg
 {
     class Constants
     {
+        // BBIWARG
+        public static readonly bool TimerOutputAll = true;
+
+        // depth image
+        public static readonly int DepthImageMedianSize = 3;
+        public static readonly int DepthImageDepthRange = 200; // <255
+
+        // edge image
+        public static readonly int EdgeImageCannyStartThreshold = 100;
+        public static readonly int EdgeImageCannyLinkingThreshold = 75;
+        public static readonly int EdgeImageCannySize = 3;
+        public static readonly int EdgeImageRoughNumDilationIterations = 1;
+
+        // finger detection
+        public static readonly int FingerStepSize = 2;
+        public static readonly int FingerMinNumSlices = 30 / FingerStepSize;
+        public static readonly int FingerRemoveNumSlicesForCorrection = 5;
+        public static readonly int FingerMaxGapCounter = 10;
+        public static readonly int FingerMaxSliceDifferencePerStep = 5;
+        public static readonly int FingerMaxSize = 30;
+        public static readonly int FingerMinSize = 5;
+        public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
+        public static readonly int FingerNumFramesUntilTracked = 2;
+        public static readonly int FingerOutSliceFactor = 10;
+        public static readonly int FingerContourMargin = 2;
+        public static readonly int FingerSliceOverlapFactor = 2;
+        public static readonly float FingerMinSimilarityForTracking = 0.75f;
+
+        // hand detection
+        public static readonly float HandMaxSize = 0.7f;
+
+        // palm detection
+        public static readonly float PalmMinDefectMidFingerLineDistance = 20; // defects with mid point ((start + end) / 2) closer than this to a finger line are removed 
+        public static readonly float PalmMaxThumbDefectAngle = 110; // degree
+        public static readonly float PalmMaxThumbDefectLengthQuotient = 2.2f;
+        public static readonly float PalmMinThumbDefectLengthQuotient = 1.2f;
+        public static readonly float PalmMinTumbDefectDepth = 30.0f;
+        
+        //palm Grid
+        public static readonly int PalmGridRows = 4;
+        public static readonly int PalmGridColumns = 3;
+
+        // touch detection
+        public static readonly float TouchEventMinFloodValue = 0.2f;
+        public static readonly int TouchEventTipCorrectionFactor = 7;
+        public static readonly int TouchEventAreaSize = 30;
+        public static readonly int TouchEventNumFramesUntilTracked = 2;
+
+        // output window
+        public static readonly int NumImagesPerRow = 3;
+        public static readonly float WindwoSizeFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
+
+
+
+
+
         // colors
         public static readonly Color ColorDetected = Color.Green;
         public static readonly Color ColorTracked = Color.LightGoldenrodYellow;
@@ -18,6 +74,8 @@ namespace bbiwarg
         public static readonly Color FingerSliceColor = Color.Magenta;
         public static readonly Color FingerDetectedColor = ColorDetected;
         public static readonly Color FingerTrackedColor = ColorTracked;
+        public static readonly Color FingerTipOutSliceColor = Color.Gray;
+        public static readonly Color FingerHandOutSliceColor = Color.DarkSlateGray;
 
         public static readonly Color TouchEventDetectedColor = ColorDetected;
         public static readonly Color TouchEventTrackedColor = ColorTracked;
@@ -34,37 +92,5 @@ namespace bbiwarg
         public static readonly Color PalmConturColor = Color.Red;
         public static readonly Color PalmConvexHullColor = Color.Green;
         public static readonly Color PalmThumbDefectColor = Color.Lime;
-
-        // finger detection
-        public static readonly int FingerStepSize = 2;
-        public static readonly int FingerMinNumSlices = 7;
-        public static readonly int FingerRemoveNumSlicesForCorrection = 3;
-        public static readonly int FingerMaxGapCounter = 10;
-        public static readonly int FingerMaxSliceDifferencePerStep = 8;
-        public static readonly int FingerMaxSize = 35;
-        public static readonly int FingerMinSize = 5;
-        public static readonly int FingerNumSlicesForRelativeDirection = 5;
-        public static readonly int FingerNumFramesUntilTracked = 2;
-        public static readonly float FingerMinSimilarityForTracking = 0.75f;
-
-        // palm detection
-        public static readonly float PalmMinDefectMidFingerLineDistance = 20; // defects with mid point ((start + end) / 2) closer than this to a finger line are removed 
-        public static readonly float PalmMaxThumbDefectAngle = 110; // degree
-        
-        //palm Grid
-        public static readonly int PalmGridRows = 4;
-        public static readonly int PalmGridColumns = 3;
-
-        // touch detection
-        public static readonly float TouchEventMinFloodValue = 0.33f;
-        public static readonly int TouchEventTipCorrectionFactor = 5;
-        public static readonly int TouchEventAreaSize = 30;
-        public static readonly int TouchEventMaxDepthDifference = 20;
-        public static readonly int TouchEventFingerDiameter = 5;
-        public static readonly int TouchEventNumFramesUntilTracked = 2;
-
-        // output window
-        public static readonly int NumImagesPerRow = 2;
-        public static readonly float WindwoSizeFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
     }
 }

bbiwarg/Detectors/FingerDetection/Finger.cs

@@ -6,13 +6,14 @@ using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Utility;
+using Emgu.CV;
 
 namespace bbiwarg.Detectors.FingerDetection
 {
     class Finger
     {
-        public Vector2D Tip { get { return SliceTrail.Start.Mid; } }
-        public Vector2D Hand { get { return SliceTrail.End.Mid; } }
+        public Vector2D TipPoint { get { return SliceTrail.Start.Mid; } }
+        public Vector2D HandPoint { get { return SliceTrail.End.Mid; } }
         public LineSegment2D LineSegment { get { return SliceTrail.LineSegment; } }
         public FingerSliceTrail SliceTrail { get; private set; }
 
@@ -25,21 +26,26 @@ namespace bbiwarg.Detectors.FingerDetection
             SliceTrail = sliceTrail;
         }
 
-        public Point[] getPolygon()
+        public Contour<Point> getContour()
         {
             List<Point> pointsA = new List<Point>();
             List<Point> pointsB = new List<Point>();
-            //int numSlices = SliceTrail.NumSlices;
-            //for (int i = 0; i < numSlices; i++)
             foreach (FingerSlice slice in SliceTrail.Slices)
             {
-                pointsA.Add(slice.Start);
-                pointsB.Add(slice.End);
+                Vector2D direction = (slice.End - slice.Start).normalize();
+                pointsA.Add(slice.Start - Constants.FingerContourMargin * direction);
+                pointsB.Add(slice.End + Constants.FingerContourMargin * direction);
             }
 
-            pointsB.Reverse();
+            pointsA.Reverse();
             pointsA.AddRange(pointsB);
-            return pointsA.ToArray();
+
+            Contour<Point> contour = new Contour<Point>(new MemStorage());
+            foreach (Point p in pointsA)
+            {
+                contour.Push(p);
+            }
+            return contour;
         }
 
         //TODO: redo (not use similarity but actual distances and angle instead)

bbiwarg/Detectors/FingerDetection/FingerDetector.cs

@@ -16,14 +16,16 @@ namespace bbiwarg.Detectors.FingerDetection
     class FingerDetector
     {
         private DepthImage depthImage;
-        private EdgeImage edgeImage;
+        private EdgeImage edgeImageOriginal;
+        private EdgeImage edgeImageAdapted;
         private OutputImage outputImage;
         public List<Finger> Fingers { get; private set; }
 
         public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage)
         {
             this.depthImage = depthImage;
-            this.edgeImage = edgeImage.copy();
+            this.edgeImageOriginal = edgeImage;
+            this.edgeImageAdapted = edgeImage.copy();
             this.outputImage = outputImage;
 
             findFingers();
@@ -36,11 +38,11 @@ namespace bbiwarg.Detectors.FingerDetection
 
             Fingers = new List<Finger>();
 
-            for (int y = 1; y < maxY; y+=5) //y++ for 100% coverage, but y+=5 for 99% coverage and 3 times better perfomance
+            for (int y = 1; y < maxY; y += 5)
             {
                 for (int x = 1; x < maxX; x++)
                 {
-                    if (edgeImage.isEdgeAt(x, y))
+                    if (edgeImageAdapted.isEdgeAt(x, y))
                     {
                         Vector2D edgePoint = new Vector2D(x, y);
                         Vector2D edgeDirection = getEdgeDirection(edgePoint);
@@ -55,7 +57,9 @@ namespace bbiwarg.Detectors.FingerDetection
                             {
                                 FingerSliceTrail trail = findFingerSliceTrail(slice, edgeDirection);
                                 if (trail != null && trail.NumSlices > Constants.FingerMinNumSlices)
+                                {
                                     createFingerFromTrail(trail);
+                                }
                             }
                         }
                     }
@@ -68,10 +72,10 @@ namespace bbiwarg.Detectors.FingerDetection
             int x = edgePoint.IntX;
             int y = edgePoint.IntY;
 
-            if (edgeImage.isEdgeAt(x, y - 1) && edgeImage.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
-            else if (edgeImage.isEdgeAt(x - 1, y) && edgeImage.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
-            else if (edgeImage.isEdgeAt(x - 1, y - 1) && edgeImage.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
-            else if (edgeImage.isEdgeAt(x + 1, y - 1) && edgeImage.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
+            if (edgeImageAdapted.isEdgeAt(x, y - 1) && edgeImageAdapted.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
+            else if (edgeImageAdapted.isEdgeAt(x - 1, y) && edgeImageAdapted.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
+            else if (edgeImageAdapted.isEdgeAt(x - 1, y - 1) && edgeImageAdapted.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
+            else if (edgeImageAdapted.isEdgeAt(x + 1, y - 1) && edgeImageAdapted.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
             else return null;
         }
 
@@ -93,11 +97,11 @@ namespace bbiwarg.Detectors.FingerDetection
                     trail.addSlice(nextSlice);
                     trail = expandTrail(trail);
 
-                    if (trail.NumSlices > Constants.FingerMinNumSlices)
+                    if (trail.NumSlices > Constants.FingerMinNumSlices / 2)
                     {
                         trail.Slices.RemoveRange(0, Constants.FingerRemoveNumSlicesForCorrection);
                         trail.Slices.Reverse();
-                        trail = expandTrail(trail);
+                        trail = expandTrail(trail, true);
                         trail.Slices.Reverse();
                         return trail;
                     }
@@ -108,7 +112,7 @@ namespace bbiwarg.Detectors.FingerDetection
 
         }
 
-        private FingerSliceTrail expandTrail(FingerSliceTrail trail)
+        private FingerSliceTrail expandTrail(FingerSliceTrail trail, bool reversed = false)
         {
             int maxX = depthImage.Width - 1;
             int maxY = depthImage.Height - 1;
@@ -124,7 +128,10 @@ namespace bbiwarg.Detectors.FingerDetection
 
             while (currentPosition.isWithin(0, 0, maxX, maxY) && gapCounter <= Math.Min(numSlices, Constants.FingerMaxGapCounter))
             {
-                nextSlice = findFingerSliceFromMid(currentPosition, currentDirection);
+                if (reversed)
+                    nextSlice = findFingerSliceFromMid(currentPosition, currentDirection.getInverse());
+                else
+                    nextSlice = findFingerSliceFromMid(currentPosition, currentDirection);
 
                 if (nextSlice != null && Math.Abs(nextSlice.Length - lastSlice.Length) <= Constants.FingerMaxSliceDifferencePerStep)
                 {
@@ -148,7 +155,7 @@ namespace bbiwarg.Detectors.FingerDetection
 
         private FingerSlice findFingerSliceFromMid(Vector2D position, Vector2D direction)
         {
-            if (edgeImage.isEdgeAt(position)) return null;
+            if (edgeImageAdapted.isRoughEdgeAt(position)) return null;
 
             Vector2D dirStart = direction.getOrthogonal(true);
             Vector2D dirEnd = direction.getOrthogonal(false);
@@ -163,13 +170,13 @@ namespace bbiwarg.Detectors.FingerDetection
         }
         private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction)
         {
-            Vector2D end = findNextEdge(start, direction);
+            Vector2D end = findNextEdge(start + Constants.FingerSliceOverlapFactor * direction, direction);
             if (end == null) return null;
 
             return getFingerSlice(start, end);
         }
 
-        private Vector2D findNextEdge(Vector2D start, Vector2D direction)
+        private Vector2D findNextEdge(Vector2D start, Vector2D direction, bool adaptedEdgeImage = true, bool stopAtMaxFingerSize = true, bool returnBoundIfNoEdge = false)
         {
             int maxX = depthImage.Width - 1;
             int maxY = depthImage.Height - 1;
@@ -190,21 +197,26 @@ namespace bbiwarg.Detectors.FingerDetection
             else
                 maxStepsY = int.MaxValue;
 
-            int maxStepsLength = (int)(Constants.FingerMaxSize / direction.Length);
+            int maxSteps = Math.Min(maxStepsX, maxStepsY);
 
-            int maxSteps = Math.Min(maxStepsLength, Math.Min(maxStepsX, maxStepsY));
+            if (stopAtMaxFingerSize)
+                maxSteps = Math.Min(maxSteps, (int)(Constants.FingerMaxSize / direction.Length));
 
             Vector2D end = new Vector2D(start);
             for (int i = 0; i < maxSteps; i++)
             {
                 end += direction;
 
-                if (edgeImage.isEdgeAt(end))
+                if ((adaptedEdgeImage && edgeImageAdapted.isRoughEdgeAt(end)) || (!adaptedEdgeImage && edgeImageOriginal.isRoughEdgeAt(end)))
                 {
                     return end;
                 }
             }
-            return null;
+
+            if (returnBoundIfNoEdge)
+                return end;
+            else
+                return null;
         }
 
         private FingerSlice getFingerSlice(Vector2D start, Vector2D end)
@@ -213,8 +225,8 @@ namespace bbiwarg.Detectors.FingerDetection
             int maxY = depthImage.Height - 1;
 
             Vector2D direction = (end - start).normalize();
-            Vector2D beforeStart = (start - direction).moveInBound(0, 0, maxX, maxY);
-            Vector2D behindEnd = (end + direction).moveInBound(0, 0, maxY, maxY);
+            Vector2D beforeStart = (start - Constants.FingerSliceOverlapFactor * direction).moveInBound(0, 0, maxX, maxY);
+            Vector2D behindEnd = (end + Constants.FingerSliceOverlapFactor * direction).moveInBound(0, 0, maxY, maxY);
 
             FingerSlice slice = new FingerSlice(beforeStart, behindEnd);
             if (slice.Length >= Constants.FingerMinSize && slice.Length <= Constants.FingerMaxSize && fingerSliceDepthTest(slice))
@@ -233,6 +245,10 @@ namespace bbiwarg.Detectors.FingerDetection
 
         private void createFingerFromTrail(FingerSliceTrail trail)
         {
+            //bring finger in correct direction Tip->Hand
+            trail = orderTrailTipToHand(trail);
+
+            //create finger
             Finger finger = new Finger(trail);
 
             //add finger
@@ -242,7 +258,61 @@ namespace bbiwarg.Detectors.FingerDetection
             drawDetectedFinger(finger);
 
             //remove edges around detected finger to improve performance
-            edgeImage.removeFingerEdges(finger);
+            edgeImageAdapted.removeEdgesInsidePolygon(finger.getContour().ToArray());
+        }
+
+        private FingerSlice findOutSlice(Vector2D start, Vector2D direction)
+        {
+            int maxX = depthImage.Width - 1;
+            int maxY = depthImage.Height - 1;
+
+            Vector2D dirOrth1 = direction.getOrthogonal(true);
+            Vector2D dirOrth2 = direction.getOrthogonal(false);
+
+            Vector2D outPoint = (start + Constants.FingerOutSliceFactor * direction).moveInBound(0, 0, maxX, maxY);
+            Vector2D p1 = findNextEdge(outPoint, dirOrth1, false, false, true);
+            Vector2D p2 = findNextEdge(outPoint, dirOrth2, false, false, true);
+
+            FingerSlice slice = new FingerSlice(p1, p2);
+
+            return slice;
+        }
+
+        private FingerSliceTrail orderTrailTipToHand(FingerSliceTrail trail)
+        {
+            int maxX = depthImage.Width - 2;
+            int maxY = depthImage.Height - 2;
+
+            FingerSlice start = trail.Start;
+            FingerSlice end = trail.End;
+
+            Vector2D direction = (end.Mid - start.Mid).normalize();
+
+            FingerSlice startOutSlice = findOutSlice(start.Mid, direction.getInverse());
+            FingerSlice endOutSlice = findOutSlice(end.Mid, direction);
+
+            float startOutLength = float.MaxValue;
+            float endOutLength = float.MaxValue;
+
+            if (startOutSlice.Start.isWithin(1, 1, maxX, maxY) && startOutSlice.End.isWithin(1, 1, maxX, maxY))
+                startOutLength = startOutSlice.Length;
+
+            if (endOutSlice.Start.isWithin(1, 1, maxX, maxY) && endOutSlice.End.isWithin(1, 1, maxX, maxY))
+                endOutLength = endOutSlice.Length;
+
+            if (startOutLength <= endOutLength)
+            {
+                trail.Slices.Reverse();
+                outputImage.drawLineSegment(startOutSlice.LineSegment, Constants.FingerHandOutSliceColor);
+                outputImage.drawLineSegment(endOutSlice.LineSegment, Constants.FingerTipOutSliceColor);
+            }
+            else
+            {
+                outputImage.drawLineSegment(startOutSlice.LineSegment, Constants.FingerTipOutSliceColor);
+                outputImage.drawLineSegment(endOutSlice.LineSegment, Constants.FingerHandOutSliceColor);
+            }
+
+            return trail;
         }
 
         private void drawDetectedFinger(Finger finger)
@@ -253,6 +323,7 @@ namespace bbiwarg.Detectors.FingerDetection
                 outputImage.drawLineSegment(trail.Slices[i].LineSegment, Constants.FingerSliceColor);
             }
             outputImage.drawLineSegment(finger.LineSegment, Constants.FingerDetectedColor);
+            outputImage.drawContour(finger.getContour(), Color.Red, 1);
         }
     }
 }

bbiwarg/Detectors/HandDetection/Hand.cs

@@ -3,10 +3,44 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Emgu.CV;
+using Emgu.CV.Structure;
+using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Utility;
 
 namespace bbiwarg.Detectors.HandDetection
 {
     class Hand
     {
+        public enum HandSide
+        {
+            Left,
+            Right
+        }
+
+        public Vector2D Centroid { private set; get; }
+        public HandSide Side { get; set; }
+        public Image<Gray, byte> Mask { get; private set; }
+        public List<Finger> Fingers { get; private set; }
+
+        public Hand(Image<Gray, byte> mask) {
+            Mask = mask;
+            Centroid = getCentroid();
+            Fingers = new List<Finger>();
+        }
+
+        public bool isInside(Vector2D point) {
+            return (Mask.Data[point.IntY, point.IntX, 0] != 0);
+        }
+
+        public void addFinger(Finger finger) {
+            Fingers.Add(finger);
+        }
+
+        private Vector2D getCentroid()
+        {
+            MCvPoint2D64f gravityCenter = Mask.GetMoments(true).GravityCenter;
+            return new Vector2D((float)gravityCenter.x, (float)gravityCenter.y);
+        }
     }
 }

bbiwarg/Detectors/HandDetection/HandDetector.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Drawing;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Graphics;
+using bbiwarg.Utility;
 using Emgu.CV;
 using Emgu.CV.Structure;
 
@@ -16,29 +19,104 @@ namespace bbiwarg.Detectors.HandDetection
         private EdgeImage edgeImage;
         private List<Finger> fingers;
         public List<Hand> Hands { get; private set; }
+        public OutputImage outputImage;
 
-        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers) {
+        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers, OutputImage outputImage)
+        {
             this.depthImage = depthImage;
             this.edgeImage = edgeImage;
             this.fingers = fingers;
+            this.outputImage = outputImage;
 
             detectHands();
+            drawHands();
         }
 
-        private void detectHands() {
-            Image<Gray, byte> image = edgeImage.Image.Copy();
+        private void detectHands()
+        {
+            int width = depthImage.Width;
+            int height = depthImage.Height;
+            int maxArea = width * height;
+            Image<Gray, byte> image = edgeImage.Image.Copy().Dilate(2).Erode(2).Mul(255);
 
             //draw top finger slice
-            foreach (Finger finger in fingers) {
-                FingerSlice topSlice = finger.SliceTrail.Start;
-                image.Draw(new LineSegment2D(topSlice.Start, topSlice.End), new Gray(255), 2);
-            }
+            foreach (Finger finger in fingers)
+            {
+                // TODO: connect contour with other edges
+                Contour<Point> contour = finger.getContour();
+                image.FillConvexPoly(contour.ToArray(), new Gray(0));
+                image.DrawPolyline(finger.getContour().ToArray(), true, new Gray(255), 1);
 
-            int numHands = 0;
-            foreach (Finger finger in fingers) {
+                FingerSlice slice = finger.SliceTrail.End;
+                Vector2D direction = slice.LineSegment.Line.Direction;
+                Vector2D start = slice.Start+(Constants.FingerContourMargin+Constants.FingerSliceOverlapFactor)*direction;
+                Vector2D end = slice.End-(Constants.FingerContourMargin+Constants.FingerSliceOverlapFactor)*direction;
+                image.Draw(new Emgu.CV.Structure.LineSegment2D(start, end), new Gray(0), 2);
                 
+                //FingerSlice slice = finger.SliceTrail.Slices[1];
+                //image.Draw(new Emgu.CV.Structure.LineSegment2D(slice.Start, slice.End), new Gray(255), 2);
+            }
+
+            Hands = new List<Hand>();
+
+            foreach (Finger finger in fingers)
+            {
+                bool newHand = true;
+                foreach (Hand hand in Hands)
+                {
+                    if (hand.isInside(finger.HandPoint))
+                    {
+                        hand.addFinger(finger);
+                        newHand = false;
+                        break;
+                    }
+                }
+
+                if (newHand)
+                {
+                    Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
+                    MCvConnectedComp comp = new MCvConnectedComp();
+                    CvInvoke.cvFloodFill(image, finger.HandPoint, new MCvScalar(255), new MCvScalar(1), new MCvScalar(1), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
+                    if (comp.area < maxArea * Constants.HandMaxSize)
+                    {
+                        Hand hand = new Hand(mask.Copy(new Rectangle(1, 1, width, height)).Mul(255));
+                        hand.addFinger(finger);
+                        Hands.Add(hand);
+                    }
+                }
+            }
+
+            float minX = float.MaxValue;
+            foreach (Hand hand in Hands)
+            {
+                if (hand.Centroid.X < minX)
+                    minX = hand.Centroid.X;
+            }
+
+            foreach (Hand hand in Hands)
+            {
+                if (hand.Centroid.X == minX)
+                    hand.Side = Hand.HandSide.Left;
+                else
+                    hand.Side = Hand.HandSide.Right;
             }
+        }
 
+        private void drawHands() {
+            if (Hands.Count == 1)
+            {
+                outputImage.Image[0] = outputImage.Image[1] = outputImage.Image[2] = Hands[0].Mask;
+            }
+            else if (Hands.Count > 1)
+            {
+                for (int i = 0; i < 2; ++i)
+                {
+                    if (Hands[i].Side == Hand.HandSide.Left)
+                        outputImage.Image[0] = outputImage.Image[0] | (Hands[i].Mask);
+                    else
+                        outputImage.Image[1] = outputImage.Image[1] | (Hands[i].Mask);
+                }
+            }
         }
     }
 }

bbiwarg/Detectors/PalmDetection/PalmDetector.cs

@@ -11,21 +11,16 @@ using Emgu.CV.Structure;
 using bbiwarg.Utility;
 using bbiwarg.Images;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Detectors.HandDetection;
 using bbiwarg.Graphics;
 
 namespace bbiwarg.Detectors.PalmDetection
 {
     class PalmDetector
     {
-        private int width, height;
         private OutputImage outputImage;
-        private EdgeImage edgeImage;
-
-        private Image<Gray, Byte> handImage;
-        private Image<Gray, Byte> pointingHandMask;
-
-        private List<Finger> fingers;
 
+        private Hand palmHand, pointingHand;
         private Contour<Point> palmContour;
         private List<MCvConvexityDefect> convexityDefects;
         private Vector2D thumbDefectStart;
@@ -34,176 +29,65 @@ namespace bbiwarg.Detectors.PalmDetection
         
         private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
 
-        private Vector2D topLeft;
-        private Vector2D topRight;
-        private Vector2D bottomLeft;
-        private Vector2D bottomRight;
-
         public Quadrangle PalmQuad { get; private set; }
 
         public PalmDetector()
         {
-            // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand
             thumbDefectDepthFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
             thumbDefectStartFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
             thumbDefectEndFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
         }
 
-        public void findPalmQuad(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage, List<Finger> trackedFingers)
+        public void findPalmQuad(OutputImage outputImage, List<Hand> hands)
         {
-            this.width = depthImage.Width;
-            this.height = depthImage.Height;
-
-            i1 = new OutputImage(width, height);
-            i2 = new OutputImage(width, height);
-            i3 = new OutputImage(width, height);
-            i4 = new OutputImage(width, height);
-            i5 = new OutputImage(width, height);
-            i6 = new OutputImage(width, height);
-            i7 = new OutputImage(width, height);
-            i8 = new OutputImage(width, height);
-            i9 = new OutputImage(width, height);
-
-            if (trackedFingers.Count >= 2)
-            {
-                this.edgeImage = edgeImage;
-                this.outputImage = outputImage;
-
-                // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
-                handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
+            this.outputImage = outputImage;
 
-                fingers = getFingersWithoutThumb(trackedFingers);
-                buildPointingHandMask();
-                handImage = handImage.And(pointingHandMask);
-
-                findLongestPalmContour();
-                if (palmContour != null)
+            if (hands.Count == 2)
+            {
+                if (hands[0].Fingers.Count > 1)
                 {
-                    findConvexityDefectsSortedByDepth();
-                    removeConvexityDefectsCausedByFingers();
-
-                    findHandPoints();
+                    pointingHand = hands[0];
+                    palmHand = hands[1];
                 }
-            }
-            draw();
-        }
-
-        public void reset()
-        {
-            thumbDefectDepthFilter.reset();
-            thumbDefectStartFilter.reset();
-            thumbDefectEndFilter.reset();
-            lastThumbDefectDepth = null;
-        }
-
-        private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
-        {
-            Finger leftMost = null;
-            float minX = float.MaxValue;
-            foreach (Finger f in detectedFingers)
-            {
-                float midX = ((f.Hand + f.Tip) / 2).X;
-                if (midX < minX)
+                else if (hands[1].Fingers.Count > 1)
                 {
-                    minX = midX;
-                    leftMost = f;
+                    pointingHand = hands[1];
+                    palmHand = hands[0];
                 }
-            }
-
-            List<Finger> result = new List<Finger>();
-            foreach (Finger f in detectedFingers)
-            {
-                if (f != leftMost)
-                    result.Add(f);
-            }
-            return result;
-        }
-
-        private void fillFingerSlices(Image<Gray, Byte> image, byte val)
-        {
-            foreach (Finger f in fingers)
-            {
-                //foreach (FingerSlice s in f.SliceTrail.Slices)
+                else if (hands[0].Fingers[0].LineSegment.Length > hands[1].Fingers[0].LineSegment.Length)
                 {
-                    image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1);
+                    pointingHand = hands[0];
+                    palmHand = hands[1];
                 }
-            }
-        }
-
-        private Finger getLongestFinger()
-        {
-            float maxLength = 0;
-            Finger longest = null;
-            foreach (Finger f in fingers)
-            {
-
-                if (f.LineSegment.Length > maxLength)
+                else
                 {
-                    maxLength = f.LineSegment.Length;
-                    longest = f;
+                    pointingHand = hands[1];
+                    palmHand = hands[0];
                 }
-            }
-            return longest;
-        }
 
-        private Point getPointInPointingHand()
-        {
-            Finger finger = getLongestFinger();
-            if (finger == null)
-                return new Point(0, 0);
-            Vector2D direction = (finger.Hand - finger.Tip).normalize();
-            Vector2D pos = finger.Hand + direction;
+                findLongestPalmContour();
+                findConvexityDefectsSortedByDepth();
+                removeConvexityDefectsCausedByFingers();
+                findHandPoints();
+            }
             
-            while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
-                pos += direction;
-
-            i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red);
-
-            return pos;
+            draw();
         }
 
-        public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9;
-        private void buildPointingHandMask()
+        public void reset()
         {
-            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
-
-            // dst = (src > 0) ? 1 : 0;
-            pointingHandMask = pointingHandMask.Or(edgeImage.Image);
-            i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
-
-            pointingHandMask = pointingHandMask.Dilate(2);
-            i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
-           
-            fillFingerSlices(pointingHandMask, 1);
-            i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
-
-            //pointingHandMask = pointingHandMask.Dilate(1);
-            //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask;
-
-            MCvConnectedComp tmp = new MCvConnectedComp();
-            // fill with value 2
-            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
-
-            // dst = (src > 1) ? 0 : 1    (src > 1 <-> src == 2 <-> src filled by flood fill)
-            pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1));
-            i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
-            
-            pointingHandMask = pointingHandMask.Erode(6);
-            i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
-
-            fillFingerSlices(pointingHandMask, 0);
-            i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
-            
-            pointingHandMask = pointingHandMask.Erode(2);
-            i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
-            
-            // only debug
-            i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
+            thumbDefectDepthFilter.reset();
+            thumbDefectStartFilter.reset();
+            thumbDefectEndFilter.reset();
+            lastThumbDefectDepth = null;
+            palmContour = null;
+            PalmQuad = null;
         }
 
         private void findLongestPalmContour()
         {
-            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
+            Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, 
+                                                                Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
             palmContour = contour;
             double maxPerimeter = 0;
@@ -236,15 +120,16 @@ namespace bbiwarg.Detectors.PalmDetection
             List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
             foreach (MCvConvexityDefect d in convexityDefects)
             {
-                bool remove = false;
+                bool fingerIntersectsStartEnd = false;
                 float minFingerLineDist = float.MaxValue;
-                foreach (Finger f in fingers)
+                foreach (Finger f in pointingHand.Fingers)
                 {
                     Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
                     Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
-                    if (intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2))
+                    if (intersection != null &&
+                        intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2))
                     {
-                        remove = true;
+                        fingerIntersectsStartEnd = true;
                         break;
                     }
                     
@@ -254,7 +139,7 @@ namespace bbiwarg.Detectors.PalmDetection
                         minFingerLineDist = dist;
                 }
                 
-                if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !remove)
+                if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !fingerIntersectsStartEnd)
                     newDefects.Add(d);
             }
             convexityDefects = newDefects;
@@ -268,13 +153,20 @@ namespace bbiwarg.Detectors.PalmDetection
                 Vector2D start = new Vector2D(d.StartPoint);
                 Vector2D end = new Vector2D(d.EndPoint);
 
+                float l1 = (depth - start).Length;
+                float l2 = (depth - end).Length;
+                float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2);
+
                 float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
 
-                if (angle <= Constants.PalmMaxThumbDefectAngle)
+                if (angle <= Constants.PalmMaxThumbDefectAngle && 
+                    lengthQuotient >= Constants.PalmMinThumbDefectLengthQuotient && lengthQuotient <= Constants.PalmMaxThumbDefectLengthQuotient &&
+                    d.Depth >= Constants.PalmMinTumbDefectDepth)
                 {
                     return d;
                 }
             }
+
             return null;
         }
 
@@ -282,14 +174,11 @@ namespace bbiwarg.Detectors.PalmDetection
         {
             MCvConvexityDefect? thumbDefect = findThumbDefect();
 
-            if (convexityDefects.Count > 0 && (thumbDefect != null || thumbDefectDepthFilter.Initialized))
+            if (thumbDefect != null)
             {
-                if (thumbDefect != null)
-                {
-                    thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
-                    thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
-                    thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
-                }
+                thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
+                thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
+                thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
                 
                 if (!thumbDefectDepthFilter.Initialized)
                 {
@@ -306,26 +195,26 @@ namespace bbiwarg.Detectors.PalmDetection
 
                 lastThumbDefectDepth = thumbDefectDepth;
 
-                Vector2D handLength, handWidth;
-                if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd))
+                Vector2D handLength, handWidth, topLeft, bottomLeft, bottomRight, topRight;
+                if (palmHand.Side == Hand.HandSide.Left)
                 {
-                    //right hand
+                    //left hand
                     handLength = thumbDefectStart - thumbDefectDepth;
-                    handWidth = 0.8f * new Vector2D(-handLength.Y, handLength.X);
-                    topLeft = thumbDefectStart;
+                    handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
+                    topLeft = thumbDefectStart+0.15f*handLength;
                     bottomLeft = thumbDefectDepth - 0.4f * handLength;
                     bottomRight = bottomLeft + handWidth;
                     topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
                 }
                 else
                 {
-                    //left hand
+                    //right hand
                     handLength = thumbDefectEnd - thumbDefectDepth;
-                    handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X);
-                    topRight = thumbDefectEnd;
+                    handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
+                    topRight = thumbDefectEnd+0.15f*handLength;
                     bottomRight = thumbDefectDepth - 0.4f * handLength;
                     bottomLeft = bottomRight + handWidth;
-                    topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth;
+                    topLeft = bottomLeft + 1.15f * handLength - 0.35f * handWidth;
                 }
 
                 PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight);
@@ -334,8 +223,6 @@ namespace bbiwarg.Detectors.PalmDetection
 
         private void draw()
         {
-
-
             if (palmContour != null && palmContour.Count<Point>() > 0) {
                 outputImage.drawContour(palmContour, Constants.PalmConturColor);
                 outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);

bbiwarg/Detectors/TouchDetection/TouchDetector.cs

@@ -30,8 +30,7 @@ namespace bbiwarg.Detectors.TouchDetection
 
             foreach (Finger finger in fingers)
             {
-                Vector2D tipPoint = finger.Tip;
-
+                Vector2D tipPoint = finger.TipPoint;
                 outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
 
                 float floodValue = getFloodValue(tipPoint);
@@ -50,12 +49,18 @@ namespace bbiwarg.Detectors.TouchDetection
 
         private float getFloodValue(Point touchPoint)
         {
-            Int16 depthAtTouch = (Int16)(depthImage.getDepthAt(touchPoint) + Constants.TouchEventFingerDiameter);
+            int minX = Math.Max(touchPoint.X - 2*Constants.TouchEventAreaSize / 3, 0);
+            int maxX = Math.Min(touchPoint.X + Constants.TouchEventAreaSize / 3, depthImage.Width-1);
+            int minY = Math.Max(touchPoint.Y - 2*Constants.TouchEventAreaSize / 3, 0);
+            int maxY = Math.Min(touchPoint.Y + Constants.TouchEventAreaSize / 3, depthImage.Height-1);
+
+
+            Vector2D relTouch = new Vector2D(touchPoint.X - minX, touchPoint.Y - minY);
+            Rectangle rect = new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1);
+            Image<Gray, byte> touchArea = depthImage.Image.Copy(rect);
 
-            int minX = Math.Max(touchPoint.X - Constants.TouchEventAreaSize / 2, 0);
-            int maxX = Math.Min(touchPoint.X + Constants.TouchEventAreaSize / 2, depthImage.Width-1);
-            int minY = Math.Max(touchPoint.Y - Constants.TouchEventAreaSize / 2, 0);
-            int maxY = Math.Min(touchPoint.Y + Constants.TouchEventAreaSize / 2, depthImage.Height-1);
+            MCvConnectedComp comp = new MCvConnectedComp();
+            CvInvoke.cvFloodFill(touchArea, relTouch, new MCvScalar(255), new MCvScalar(0), new MCvScalar(2), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
 
             int matchedPixels = 0;
             int countedPixels = 0;
@@ -63,10 +68,10 @@ namespace bbiwarg.Detectors.TouchDetection
             {
                 for (int y = minY; y <= maxY; y++)
                 {
-                    Int16 depth = depthImage.getDepthAt(x, y);
                     Color color = outputImage.getColotAt(x, y);
+                    byte depth = touchArea.Data[y - minY, x - minX, 0];
                     Color subtractColor;
-                    if (depthAtTouch < depth && Math.Abs(depthAtTouch - depth) < Constants.TouchEventMaxDepthDifference)
+                    if(depth == 255)
                     {
                         matchedPixels++;
                         subtractColor = Constants.TouchEventAreaMatchedSubtractColor;
@@ -75,7 +80,7 @@ namespace bbiwarg.Detectors.TouchDetection
                     {
                         subtractColor = Constants.TouchEventAreaNonMatchedSubtractColor;
                     }
-                    Color newColor = Color.FromArgb(Math.Max(color.R - subtractColor.R, 0), Math.Max(color.G - subtractColor.G, 0), Math.Max(color.B - subtractColor.B, 0));
+                    Color newColor = subtractColor;//Color.FromArgb(Math.Max(color.R - subtractColor.R, 0), Math.Max(color.G - subtractColor.G, 0), Math.Max(color.B - subtractColor.B, 0));
                     outputImage.drawPixel(x, y, newColor);
                     countedPixels++;
                 }

bbiwarg/Graphics/OutputImage.cs

@@ -20,14 +20,15 @@ namespace bbiwarg.Graphics
             Image = new Image<Rgb, byte>(width, height);
         }
 
-        public Color getColotAt(int x, int y) {
-            
+        public Color getColotAt(int x, int y)
+        {
+
             byte red = Image.Data[y, x, 0];
             byte green = Image.Data[y, x, 1];
             byte blue = Image.Data[y, x, 2];
-            return Color.FromArgb(red,green,blue);
+            return Color.FromArgb(red, green, blue);
         }
-        
+
         public void drawLineSegment(bbiwarg.Utility.LineSegment2D lineSegment, Color color, int thickness = 1)
         {
             Image.Draw(new LineSegment2D(lineSegment.P1, lineSegment.P2), new Rgb(color), thickness);

bbiwarg/Graphics/OutputWindow.cs

@@ -165,7 +165,8 @@ namespace bbiwarg.Graphics
 
             Timer.stop("onRenderFrame");
             
-            Timer.outputAll();
+            if(Constants.TimerOutputAll)
+                Timer.outputAll();
         }
 
     }

bbiwarg/Images/DepthImage.cs

@@ -8,6 +8,7 @@ using Emgu.CV;
 using Emgu.CV.Structure;
 using bbiwarg.Graphics;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Detectors.HandDetection;
 using bbiwarg.Utility;
 
 namespace bbiwarg.Images
@@ -29,16 +30,16 @@ namespace bbiwarg.Images
             Width = image.Width;
             Height = image.Height;
 
-            image = image.SmoothMedian(5);
+            image = image.SmoothMedian(Constants.DepthImageMedianSize);
 
             //threshold min&maxDepth
             MinDepth = findMinDepth(image);
-            MaxDepth = (Int16)(MinDepth + 200); // max = minDepth+255 (else it can't fit whole range in byte image)
+            MaxDepth = (Int16)(MinDepth + Constants.DepthImageDepthRange);
 
             //smooth+threshold (dst = (src > (MaxDepth - MinDepth)) ? MaxDepth - MinDepth : src)
             Image = (image- MinDepth).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
 
-            Image = Image.SmoothMedian(5);
+            Image = Image.SmoothMedian(Constants.DepthImageMedianSize);
         }
 
         public Int16 getDepthAt(Point point)
@@ -73,21 +74,14 @@ namespace bbiwarg.Images
             return (Int16)min[0];
         }
 
-        public void removeBackground(List<Finger> fingers) {
-            BackgroundMask = Image.Copy();
-            MCvConnectedComp comp = new MCvConnectedComp();
-
-            foreach (Finger finger in fingers) {
-                Vector2D mid = finger.SliceTrail.Slices[finger.SliceTrail.NumSlices / 2].Mid;
-
-                outputImage.fillCircle(mid.IntX, mid.IntY, 3, Color.Blue);
-
-                if (true) //(BackgroundMask.Data[mid.IntY, mid.IntX,0] != 0)
-                    CvInvoke.cvFloodFill(BackgroundMask, mid, new MCvScalar(0), new MCvScalar(3), new MCvScalar(3), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
+        public void removeBackground(List<Hand> hands) {
+            Image<Gray, byte> mask = Image.CopyBlank();
+            foreach (Hand hand in hands) {
+                mask = mask.Or(hand.Mask);
             }
 
-            BackgroundMask = BackgroundMask.Erode(3).Dilate(3).ThresholdBinary(new Gray(0), new Gray(255));
-            Image = Image.Or(BackgroundMask);
+
+            Image = Image.Or(255-mask);
         }
     }
 }

bbiwarg/Images/EdgeImage.cs

@@ -17,15 +17,18 @@ namespace bbiwarg.Images
     class EdgeImage
     {
         public Image<Gray, Byte> Image { get; private set; }
+        public Image<Gray, byte> RoughImage { get; private set; }
 
         public EdgeImage(DepthImage depthImage)
         {
-            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(100, 75, 3).ThresholdBinary(new Gray(0), new Gray(1));
+            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(Constants.EdgeImageCannyStartThreshold, Constants.EdgeImageCannyLinkingThreshold, Constants.EdgeImageCannySize).ThresholdBinary(new Gray(0), new Gray(1));
+            RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }
 
         public EdgeImage(Image<Gray, Byte> edgeImage)
         {
             Image = edgeImage;
+            RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }
 
         public bool isEdgeAt(Point point)
@@ -38,11 +41,20 @@ namespace bbiwarg.Images
             return (Image.Data[y, x, 0] > 0);
         }
 
-        public void removeFingerEdges(Finger finger)
+        public bool isRoughEdgeAt(Point point)
         {
-            Point[] polygon = finger.getPolygon();
+            return isRoughEdgeAt(point.X, point.Y);
+        }
 
+        public bool isRoughEdgeAt(int x, int y)
+        {
+            return (RoughImage.Data[y, x, 0] > 0);
+        }
+
+        public void removeEdgesInsidePolygon(Point[] polygon)
+        {
             Image.FillConvexPoly(polygon, new Gray(0));
+            RoughImage.FillConvexPoly(polygon, new Gray(0));
         }
 
         public EdgeImage copy()

bbiwarg/Utility/Vector2D.cs

@@ -93,6 +93,10 @@ namespace bbiwarg.Utility
             return new Vector2D(x, y);
         }
 
+        public bool isOnBound(float minX, float minY, float maxX, float maxY) {
+            return (IntX == minX || IntX == maxX || IntY == minY || IntY == maxY);
+        }
+
         public Vector2D getOrthogonal(bool side)
         {
             if (side)

bbiwarg/VideoHandle.cs

@@ -10,6 +10,7 @@ using bbiwarg.Detectors.FingerDetection;
 using bbiwarg.Detectors.PalmDetection;
 using bbiwarg.Detectors.TouchDetection;
 using bbiwarg.Detectors.HandDetection;
+using bbiwarg.Detectors.HandDetection;
 using bbiwarg.Images;
 using bbiwarg.InputProviders;
 using Emgu.CV;
@@ -131,14 +132,14 @@ namespace bbiwarg
             Timer.stop("readInputData");
 
             //create output images
-            Timer.start("createOtherImages");
-            int numImages = 4;
+            Timer.start("createOutputImages");
+            int numImages = 5;
             OutputImages = new OutputImage[numImages];
             for (int i = 0; i < numImages; i++) {
                 OutputImages[i] = new OutputImage(Width, Height);
                     
             }
-            Timer.stop("createOtherImages");
+            Timer.stop("createOutputImages");
 
             //create depthImage
             Timer.start("createDepthImage");
@@ -166,23 +167,21 @@ namespace bbiwarg
 
             //detect hands
             Timer.start("handDetection");
-            handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.TrackedFingers);
+            handDetector = new HandDetector(depthImage, edgeImage, fingerDetector.Fingers, OutputImages[2]);
             Timer.stop("handDetection");
 
             //remove background noise
             Timer.start("removeBackground");
-            depthImage.removeBackground(fingerDetector.Fingers);
-            edgeImage = new EdgeImage(depthImage);
-            OutputImages[2].Image[0] = OutputImages[2].Image[1] = OutputImages[2].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
+            //depthImage.removeBackground(handDetector.Hands);
+            //edgeImage = new EdgeImage(depthImage);
+            OutputImages[3].Image[0] = OutputImages[3].Image[1] = OutputImages[3].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
             Timer.stop("removeBackground");
 
-            OutputImage palm = new OutputImage(Width, Height);
-
             //detect palm
             Timer.start("palmDetection");
             if (CurrentFrame == 0)
                 palmDetector.reset();
-            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[2], fingerDetector.Fingers);
+            palmDetector.findPalmQuad(OutputImages[3], handDetector.Hands);
             Timer.stop("palmDetection");
 
             //detect touchEvents
@@ -194,7 +193,7 @@ namespace bbiwarg
 
             //track touchEvents
             Timer.start("touchTracking");
-            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[2]);
+            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[3]);
             Timer.stop("touchTracking");
 
             // touch event visualizer
@@ -207,7 +206,7 @@ namespace bbiwarg
                 foreach (PalmTouchEvent e in palmTouchDetector.PalmTouchEvents)
                     touchEventVisualizer.addPalmTouchEvent(e, CurrentFrame);
                 touchEventVisualizer.updateImage();
-                OutputImages[3] = touchEventVisualizer.OutputImage;
+                OutputImages[4] = touchEventVisualizer.OutputImage;
             }
 
             // add borders
@@ -217,7 +216,7 @@ namespace bbiwarg
 
             }
 
-            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], OutputImages[3]};
+            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], OutputImages[3], OutputImages[4]};
             
             //palmDetector.i1, palmDetector.i2, palmDetector.i3, palmDetector.i5, palmDetector.i6, palmDetector.i7, palmDetector.i8, palmDetector.i9};