Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

bbiwarg/Constants.cs

@@ -9,46 +9,37 @@ namespace bbiwarg
 {
     class Constants
     {
-        // colors
-        public static readonly Color ColorDetected = Color.White;
-        public static readonly Color ColorTracked = Color.Yellow;
-
-        //public static readonly Color EdgeColor = Color.Blue; // edgeImage draw direct to blue chanel from outputImage
-
-        public static readonly Color FingerSliceColor = Color.Magenta;
-        public static readonly Color FingerDetectedColor = ColorDetected;
-        public static readonly Color FingerTrackedColor = ColorTracked;
-        public static readonly Color FingerOutSliceColor = Color.DarkSlateGray;
-
-        public static readonly Color TouchEventDetectedColor = ColorDetected;
-        public static readonly Color TouchEventTrackedColor = ColorTracked;
-        public static readonly Color TouchEventTipColor = Color.CornflowerBlue;
-        public static readonly Color TouchEventAreaMatchedSubtractColor = Color.DarkOrange;
-        public static readonly Color TouchEventAreaNonMatchedSubtractColor = Color.DarkSlateGray;
-        public static readonly Color TouchEventStatusBarColor = Color.Green;
+        // BBIWARG
+        public static readonly bool TimerOutputAll = true;
 
-        public static readonly Color TouchEventVisualizerLineColor = Color.White;
-        public static readonly Color TouchEventVisualizerPointColor = Color.Red;
+        // depth image
+        public static readonly int DepthImageMedianSize = 5;
+        public static readonly int DepthImageDepthRange = 200; // <255
 
-        public static readonly Color PalmQuadColor = Color.Blue;
-        public static readonly Color PalmGridColor = Color.CornflowerBlue;
-        public static readonly Color PalmConturColor = Color.Red;
-        public static readonly Color PalmConvexHullColor = Color.Green;
-        public static readonly Color PalmThumbDefectColor = Color.Lime;
+        // edge image
+        public static readonly int EdgeImageCannyStartThreshold = 100;
+        public static readonly int EdgeImageCannyLinkingThreshold = 75;
+        public static readonly int EdgeImageCannySize = 3;
+        public static readonly int EdgeImageRoughNumDilationIterations = 1;
 
         // finger detection
         public static readonly int FingerStepSize = 2;
-        public static readonly int FingerMinNumSlices = 10;
-        public static readonly int FingerRemoveNumSlicesForCorrection = 3;
+        public static readonly int FingerMinNumSlices = 30 / FingerStepSize;
+        public static readonly int FingerRemoveNumSlicesForCorrection = 5;
         public static readonly int FingerMaxGapCounter = 10;
         public static readonly int FingerMaxSliceDifferencePerStep = 5;
-        public static readonly int FingerMaxSize = 35;
+        public static readonly int FingerMaxSize = 30;
         public static readonly int FingerMinSize = 5;
-        public static readonly int FingerNumSlicesForRelativeDirection = 5;
+        public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
         public static readonly int FingerNumFramesUntilTracked = 2;
-        public static readonly int FingerOutSliceFactor = 4;
+        public static readonly int FingerOutSliceFactor = 5;
+        public static readonly int FingerContourMargin = 2;
+        public static readonly int FingerSliceOverlapFactor = 2;
         public static readonly float FingerMinSimilarityForTracking = 0.75f;
 
+        // hand detection
+        public static readonly float HandMaxSize = 0.7f;
+
         // palm detection
         public static readonly float PalmMinDefectMidFingerLineDistance = 20; // defects with mid point ((start + end) / 2) closer than this to a finger line are removed 
         public static readonly float PalmMaxThumbDefectAngle = 110; // degree
@@ -60,15 +51,44 @@ namespace bbiwarg
         public static readonly int PalmGridColumns = 3;
 
         // touch detection
-        public static readonly float TouchEventMinFloodValue = 0.33f;
-        public static readonly int TouchEventTipCorrectionFactor = 5;
+        public static readonly float TouchEventMinFloodValue = 0.2f;
+        public static readonly int TouchEventTipCorrectionFactor = 7;
         public static readonly int TouchEventAreaSize = 30;
-        public static readonly int TouchEventMaxDepthDifference = 20;
-        public static readonly int TouchEventFingerDiameter = 5;
         public static readonly int TouchEventNumFramesUntilTracked = 2;
 
         // output window
-        public static readonly int NumImagesPerRow = 2;
+        public static readonly int NumImagesPerRow = 3;
         public static readonly float WindwoSizeFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
+
+
+
+
+
+        // colors
+        public static readonly Color ColorDetected = Color.White;
+        public static readonly Color ColorTracked = Color.Yellow;
+
+        //public static readonly Color EdgeColor = Color.Blue; // edgeImage draw direct to blue chanel from outputImage
+
+        public static readonly Color FingerSliceColor = Color.Magenta;
+        public static readonly Color FingerDetectedColor = ColorDetected;
+        public static readonly Color FingerTrackedColor = ColorTracked;
+        public static readonly Color FingerOutSliceColor = Color.DarkSlateGray;
+
+        public static readonly Color TouchEventDetectedColor = ColorDetected;
+        public static readonly Color TouchEventTrackedColor = ColorTracked;
+        public static readonly Color TouchEventTipColor = Color.CornflowerBlue;
+        public static readonly Color TouchEventAreaMatchedSubtractColor = Color.DarkOrange;
+        public static readonly Color TouchEventAreaNonMatchedSubtractColor = Color.DarkSlateGray;
+        public static readonly Color TouchEventStatusBarColor = Color.Green;
+
+        public static readonly Color TouchEventVisualizerLineColor = Color.White;
+        public static readonly Color TouchEventVisualizerPointColor = Color.Red;
+
+        public static readonly Color PalmQuadColor = Color.Blue;
+        public static readonly Color PalmGridColor = Color.CornflowerBlue;
+        public static readonly Color PalmConturColor = Color.Red;
+        public static readonly Color PalmConvexHullColor = Color.Green;
+        public static readonly Color PalmThumbDefectColor = Color.Lime;
     }
 }

bbiwarg/Detectors/FingerDetection/Finger.cs

@@ -6,13 +6,14 @@ using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Utility;
+using Emgu.CV;
 
 namespace bbiwarg.Detectors.FingerDetection
 {
     class Finger
     {
-        public Vector2D Tip { get { return SliceTrail.Start.Mid; } }
-        public Vector2D Hand { get { return SliceTrail.End.Mid; } }
+        public Vector2D TipPoint { get { return SliceTrail.Start.Mid; } }
+        public Vector2D HandPoint { get { return SliceTrail.End.Mid; } }
         public LineSegment2D LineSegment { get { return SliceTrail.LineSegment; } }
         public FingerSliceTrail SliceTrail { get; private set; }
 
@@ -25,21 +26,26 @@ namespace bbiwarg.Detectors.FingerDetection
             SliceTrail = sliceTrail;
         }
 
-        public Point[] getPolygon()
+        public Contour<Point> getContour()
         {
             List<Point> pointsA = new List<Point>();
             List<Point> pointsB = new List<Point>();
-            //int numSlices = SliceTrail.NumSlices;
-            //for (int i = 0; i < numSlices; i++)
             foreach (FingerSlice slice in SliceTrail.Slices)
             {
-                pointsA.Add(slice.Start);
-                pointsB.Add(slice.End);
+                Vector2D direction = (slice.End - slice.Start).normalize();
+                pointsA.Add(slice.Start - Constants.FingerContourMargin * direction);
+                pointsB.Add(slice.End + Constants.FingerContourMargin * direction);
             }
 
-            pointsB.Reverse();
+            pointsA.Reverse();
             pointsA.AddRange(pointsB);
-            return pointsA.ToArray();
+
+            Contour<Point> contour = new Contour<Point>(new MemStorage());
+            foreach (Point p in pointsA)
+            {
+                contour.Push(p);
+            }
+            return contour;
         }
 
         //TODO: redo (not use similarity but actual distances and angle instead)

bbiwarg/Detectors/FingerDetection/FingerDetector.cs

@@ -16,14 +16,16 @@ namespace bbiwarg.Detectors.FingerDetection
     class FingerDetector
     {
         private DepthImage depthImage;
-        private EdgeImage edgeImage;
+        private EdgeImage edgeImageOriginal;
+        private EdgeImage edgeImageAdapted;
         private OutputImage outputImage;
         public List<Finger> Fingers { get; private set; }
 
         public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage)
         {
             this.depthImage = depthImage;
-            this.edgeImage = edgeImage.copy();
+            this.edgeImageOriginal = edgeImage;
+            this.edgeImageAdapted = edgeImage.copy();
             this.outputImage = outputImage;
 
             findFingers();
@@ -36,11 +38,11 @@ namespace bbiwarg.Detectors.FingerDetection
 
             Fingers = new List<Finger>();
 
-            for (int y = 1; y < maxY; y += 5) //y++ for 100% coverage, but y+=5 for 99% coverage and 3 times better perfomance
+            for (int y = 1; y < maxY; y += 5)
             {
                 for (int x = 1; x < maxX; x++)
                 {
-                    if (edgeImage.isEdgeAt(x, y))
+                    if (edgeImageAdapted.isEdgeAt(x, y))
                     {
                         Vector2D edgePoint = new Vector2D(x, y);
                         Vector2D edgeDirection = getEdgeDirection(edgePoint);
@@ -55,7 +57,9 @@ namespace bbiwarg.Detectors.FingerDetection
                             {
                                 FingerSliceTrail trail = findFingerSliceTrail(slice, edgeDirection);
                                 if (trail != null && trail.NumSlices > Constants.FingerMinNumSlices)
+                                {
                                     createFingerFromTrail(trail);
+                                }
                             }
                         }
                     }
@@ -68,10 +72,10 @@ namespace bbiwarg.Detectors.FingerDetection
             int x = edgePoint.IntX;
             int y = edgePoint.IntY;
 
-            if (edgeImage.isEdgeAt(x, y - 1) && edgeImage.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
-            else if (edgeImage.isEdgeAt(x - 1, y) && edgeImage.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
-            else if (edgeImage.isEdgeAt(x - 1, y - 1) && edgeImage.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
-            else if (edgeImage.isEdgeAt(x + 1, y - 1) && edgeImage.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
+            if (edgeImageAdapted.isEdgeAt(x, y - 1) && edgeImageAdapted.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
+            else if (edgeImageAdapted.isEdgeAt(x - 1, y) && edgeImageAdapted.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
+            else if (edgeImageAdapted.isEdgeAt(x - 1, y - 1) && edgeImageAdapted.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
+            else if (edgeImageAdapted.isEdgeAt(x + 1, y - 1) && edgeImageAdapted.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
             else return null;
         }
 
@@ -93,11 +97,11 @@ namespace bbiwarg.Detectors.FingerDetection
                     trail.addSlice(nextSlice);
                     trail = expandTrail(trail);
 
-                    if (trail.NumSlices > Constants.FingerMinNumSlices)
+                    if (trail.NumSlices > Constants.FingerMinNumSlices / 2)
                     {
                         trail.Slices.RemoveRange(0, Constants.FingerRemoveNumSlicesForCorrection);
                         trail.Slices.Reverse();
-                        trail = expandTrail(trail);
+                        trail = expandTrail(trail, true);
                         trail.Slices.Reverse();
                         return trail;
                     }
@@ -108,7 +112,7 @@ namespace bbiwarg.Detectors.FingerDetection
 
         }
 
-        private FingerSliceTrail expandTrail(FingerSliceTrail trail)
+        private FingerSliceTrail expandTrail(FingerSliceTrail trail, bool reversed = false)
         {
             int maxX = depthImage.Width - 1;
             int maxY = depthImage.Height - 1;
@@ -124,7 +128,10 @@ namespace bbiwarg.Detectors.FingerDetection
 
             while (currentPosition.isWithin(0, 0, maxX, maxY) && gapCounter <= Math.Min(numSlices, Constants.FingerMaxGapCounter))
             {
-                nextSlice = findFingerSliceFromMid(currentPosition, currentDirection);
+                if (reversed)
+                    nextSlice = findFingerSliceFromMid(currentPosition, currentDirection.getInverse());
+                else
+                    nextSlice = findFingerSliceFromMid(currentPosition, currentDirection);
 
                 if (nextSlice != null && Math.Abs(nextSlice.Length - lastSlice.Length) <= Constants.FingerMaxSliceDifferencePerStep)
                 {
@@ -148,7 +155,7 @@ namespace bbiwarg.Detectors.FingerDetection
 
         private FingerSlice findFingerSliceFromMid(Vector2D position, Vector2D direction)
         {
-            if (edgeImage.isEdgeAt(position)) return null;
+            if (edgeImageAdapted.isRoughEdgeAt(position)) return null;
 
             Vector2D dirStart = direction.getOrthogonal(true);
             Vector2D dirEnd = direction.getOrthogonal(false);
@@ -163,13 +170,13 @@ namespace bbiwarg.Detectors.FingerDetection
         }
         private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction)
         {
-            Vector2D end = findNextEdge(start, direction);
+            Vector2D end = findNextEdge(start + Constants.FingerSliceOverlapFactor * direction, direction);
             if (end == null) return null;
 
             return getFingerSlice(start, end);
         }
 
-        private Vector2D findNextEdge(Vector2D start, Vector2D direction, bool stopAtMaxFingerSize = true, bool returnBoundIfNoEdge = false)
+        private Vector2D findNextEdge(Vector2D start, Vector2D direction, bool adaptedEdgeImage = true, bool stopAtMaxFingerSize = true, bool returnBoundIfNoEdge = false)
         {
             int maxX = depthImage.Width - 1;
             int maxY = depthImage.Height - 1;
@@ -200,7 +207,7 @@ namespace bbiwarg.Detectors.FingerDetection
             {
                 end += direction;
 
-                if (edgeImage.isEdgeAt(end))
+                if ((adaptedEdgeImage && edgeImageAdapted.isRoughEdgeAt(end)) || (!adaptedEdgeImage && edgeImageOriginal.isRoughEdgeAt(end)))
                 {
                     return end;
                 }
@@ -218,8 +225,8 @@ namespace bbiwarg.Detectors.FingerDetection
             int maxY = depthImage.Height - 1;
 
             Vector2D direction = (end - start).normalize();
-            Vector2D beforeStart = (start - direction).moveInBound(0, 0, maxX, maxY);
-            Vector2D behindEnd = (end + direction).moveInBound(0, 0, maxY, maxY);
+            Vector2D beforeStart = (start - Constants.FingerSliceOverlapFactor * direction).moveInBound(0, 0, maxX, maxY);
+            Vector2D behindEnd = (end + Constants.FingerSliceOverlapFactor * direction).moveInBound(0, 0, maxY, maxY);
 
             FingerSlice slice = new FingerSlice(beforeStart, behindEnd);
             if (slice.Length >= Constants.FingerMinSize && slice.Length <= Constants.FingerMaxSize && fingerSliceDepthTest(slice))
@@ -251,7 +258,7 @@ namespace bbiwarg.Detectors.FingerDetection
             drawDetectedFinger(finger);
 
             //remove edges around detected finger to improve performance
-            edgeImage.removeFingerEdges(finger);
+            edgeImageAdapted.removeEdgesInsidePolygon(finger.getContour().ToArray());
         }
 
         private FingerSlice findOutSlice(Vector2D start, Vector2D direction)
@@ -262,9 +269,9 @@ namespace bbiwarg.Detectors.FingerDetection
             Vector2D dirOrth1 = direction.getOrthogonal(true);
             Vector2D dirOrth2 = direction.getOrthogonal(false);
 
-            Vector2D outPoint = (start + Constants.FingerOutSliceFactor * Constants.FingerStepSize * direction).moveInBound(0, 0, maxX, maxY);
-            Vector2D p1 = findNextEdge(outPoint, dirOrth1, false, true);
-            Vector2D p2 = findNextEdge(outPoint, dirOrth2, false, true);
+            Vector2D outPoint = (start + Constants.FingerOutSliceFactor * direction).moveInBound(0, 0, maxX, maxY);
+            Vector2D p1 = findNextEdge(outPoint, dirOrth1, false, false, true);
+            Vector2D p2 = findNextEdge(outPoint, dirOrth2, false, false, true);
 
             FingerSlice slice = new FingerSlice(p1, p2);
 
@@ -283,7 +290,7 @@ namespace bbiwarg.Detectors.FingerDetection
 
             Vector2D direction = (end.Mid - start.Mid).normalize();
 
-            FingerSlice startOutSlice = findOutSlice(start.Mid, -1 * direction);
+            FingerSlice startOutSlice = findOutSlice(start.Mid, direction.getInverse());
             FingerSlice endOutSlice = findOutSlice(end.Mid, direction);
 
             float startOutLength = float.MaxValue;
@@ -309,6 +316,7 @@ namespace bbiwarg.Detectors.FingerDetection
                 outputImage.drawLineSegment(trail.Slices[i].LineSegment, Constants.FingerSliceColor);
             }
             outputImage.drawLineSegment(finger.LineSegment, Constants.FingerDetectedColor);
+            outputImage.drawContour(finger.getContour(), Color.Red, 1);
         }
     }
 }

bbiwarg/Detectors/HandDetection/Hand.cs

@@ -3,10 +3,29 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Emgu.CV;
+using Emgu.CV.Structure;
+using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Utility;
 
 namespace bbiwarg.Detectors.HandDetection
 {
     class Hand
     {
+        public Image<Gray, byte> Mask { get; private set; }
+        public List<Finger> Fingers { get; private set; }
+
+        public Hand(Image<Gray, byte> mask) {
+            Mask = mask;
+            Fingers = new List<Finger>();
+        }
+
+        public bool isInside(Vector2D point) {
+            return (Mask.Data[point.IntY, point.IntX, 0] == 1);
+        }
+
+        public void addFinger(Finger finger) {
+            Fingers.Add(finger);
+        }
     }
 }

bbiwarg/Detectors/HandDetection/HandDetector.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Drawing;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Graphics;
+using bbiwarg.Utility;
 using Emgu.CV;
 using Emgu.CV.Structure;
 
@@ -16,29 +19,73 @@ namespace bbiwarg.Detectors.HandDetection
         private EdgeImage edgeImage;
         private List<Finger> fingers;
         public List<Hand> Hands { get; private set; }
+        public OutputImage outputImage;
 
-        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers) {
+        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers, OutputImage outputImage)
+        {
             this.depthImage = depthImage;
             this.edgeImage = edgeImage;
             this.fingers = fingers;
+            this.outputImage = outputImage;
 
             detectHands();
+            drawHands();
         }
 
-        private void detectHands() {
-            Image<Gray, byte> image = edgeImage.Image.Copy().Mul(255);
+        private void detectHands()
+        {
+            int width = depthImage.Width;
+            int height = depthImage.Height;
+            int maxArea = width * height;
+            Image<Gray, byte> image = edgeImage.Image.Copy().Dilate(2).Erode(2).Mul(255);
 
             //draw top finger slice
-            foreach (Finger finger in fingers) {
-                FingerSlice topSlice = finger.SliceTrail.Start;
-                image.Draw(new LineSegment2D(topSlice.Start, topSlice.End), new Gray(255), 2);
+            foreach (Finger finger in fingers)
+            {
+                // TODO: connect contour with other edges
+                //Contour<Point> contour = finger.getContour();
+                //image.FillConvexPoly(contour.ToArray(), new Gray(0));
+                //image.DrawPolyline(finger.getContour().ToArray(), false, new Gray(255), 1);
+
+                FingerSlice slice = finger.SliceTrail.Slices[1];
+                image.Draw(new Emgu.CV.Structure.LineSegment2D(slice.Start, slice.End), new Gray(255), 2);
             }
 
-            int numHands = 0;
-            foreach (Finger finger in fingers) {
-                
+            Hands = new List<Hand>();
+
+            foreach (Finger finger in fingers)
+            {
+                bool newHand = true;
+                foreach (Hand hand in Hands)
+                {
+                    if (hand.isInside(finger.HandPoint))
+                    {
+                        hand.addFinger(finger);
+                        newHand = false;
+                    }
+                }
+
+                if (newHand)
+                {
+                    Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
+                    MCvConnectedComp comp = new MCvConnectedComp();
+                    CvInvoke.cvFloodFill(image, finger.HandPoint, new MCvScalar(255), new MCvScalar(1), new MCvScalar(1), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
+                    if (comp.area < maxArea * Constants.HandMaxSize)
+                    {
+                        Hand hand = new Hand(mask.Copy(new Rectangle(1, 1, width, height)));
+                        hand.addFinger(finger);
+                        Hands.Add(hand);
+                    }
+                }
             }
+        }
 
+        private void drawHands() {
+            int maxIndex = Math.Min(3, Hands.Count);
+            for (int i = 0; i < maxIndex ; i++)
+            {
+                outputImage.Image[i] = Hands[i].Mask.Mul(255);
+            }
         }
     }
 }

bbiwarg/Detectors/PalmDetection/PalmDetector.cs

@@ -102,7 +102,7 @@ namespace bbiwarg.Detectors.PalmDetection
             float minX = float.MaxValue;
             foreach (Finger f in detectedFingers)
             {
-                float midX = ((f.Hand + f.Tip) / 2).X;
+                float midX = ((f.HandPoint + f.TipPoint) / 2).X;
                 if (midX < minX)
                 {
                     minX = midX;
@@ -132,8 +132,8 @@ namespace bbiwarg.Detectors.PalmDetection
             Finger finger = fingers[0];
             if (finger == null)
                 return new Point(0, 0);
-            Vector2D direction = (finger.Hand - finger.Tip).normalize();
-            Vector2D pos = finger.Hand + direction;
+            Vector2D direction = (finger.HandPoint - finger.TipPoint).normalize();
+            Vector2D pos = finger.HandPoint + direction;
             
             while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
                 pos += direction;
@@ -265,7 +265,7 @@ namespace bbiwarg.Detectors.PalmDetection
             }
 
 
-            Console.WriteLine("no palm defect found (" + i + ")");
+            //Console.WriteLine("no palm defect found (" + i + ")");
             foreach (MCvConvexityDefect d in convexityDefects)
             {
                 Vector2D depth = new Vector2D(d.DepthPoint);
@@ -278,7 +278,7 @@ namespace bbiwarg.Detectors.PalmDetection
 
                 float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
 
-                Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
+                //Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
             }
 
             ++i;
@@ -318,8 +318,8 @@ namespace bbiwarg.Detectors.PalmDetection
                 {
                     //right hand
                     handLength = thumbDefectStart - thumbDefectDepth;
-                    handWidth = 0.8f * new Vector2D(-handLength.Y, handLength.X);
-                    topLeft = thumbDefectStart;
+                    handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
+                    topLeft = thumbDefectStart+0.15f*handLength;
                     bottomLeft = thumbDefectDepth - 0.4f * handLength;
                     bottomRight = bottomLeft + handWidth;
                     topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
@@ -328,11 +328,11 @@ namespace bbiwarg.Detectors.PalmDetection
                 {
                     //left hand
                     handLength = thumbDefectEnd - thumbDefectDepth;
-                    handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X);
-                    topRight = thumbDefectEnd;
+                    handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
+                    topRight = thumbDefectEnd+0.15f*handLength;
                     bottomRight = thumbDefectDepth - 0.4f * handLength;
                     bottomLeft = bottomRight + handWidth;
-                    topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth;
+                    topLeft = bottomLeft + 1.15f * handLength - 0.35f * handWidth;
                 }
 
                 PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight);

bbiwarg/Detectors/TouchDetection/TouchDetector.cs

@@ -30,7 +30,7 @@ namespace bbiwarg.Detectors.TouchDetection
 
             foreach (Finger finger in fingers)
             {
-                Vector2D tipPoint = finger.Tip;
+                Vector2D tipPoint = finger.TipPoint;
 
                 outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
 
@@ -50,12 +50,18 @@ namespace bbiwarg.Detectors.TouchDetection
 
         private float getFloodValue(Point touchPoint)
         {
-            Int16 depthAtTouch = (Int16)(depthImage.getDepthAt(touchPoint) + Constants.TouchEventFingerDiameter);
+            int minX = Math.Max(touchPoint.X - 2*Constants.TouchEventAreaSize / 3, 0);
+            int maxX = Math.Min(touchPoint.X + Constants.TouchEventAreaSize / 3, depthImage.Width-1);
+            int minY = Math.Max(touchPoint.Y - 2*Constants.TouchEventAreaSize / 3, 0);
+            int maxY = Math.Min(touchPoint.Y + Constants.TouchEventAreaSize / 3, depthImage.Height-1);
 
-            int minX = Math.Max(touchPoint.X - Constants.TouchEventAreaSize / 2, 0);
-            int maxX = Math.Min(touchPoint.X + Constants.TouchEventAreaSize / 2, depthImage.Width-1);
-            int minY = Math.Max(touchPoint.Y - Constants.TouchEventAreaSize / 2, 0);
-            int maxY = Math.Min(touchPoint.Y + Constants.TouchEventAreaSize / 2, depthImage.Height-1);
+
+            Vector2D relTouch = new Vector2D(touchPoint.X - minX, touchPoint.Y - minY);
+            Rectangle rect = new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1);
+            Image<Gray, byte> touchArea = depthImage.Image.Copy(rect);
+
+            MCvConnectedComp comp = new MCvConnectedComp();
+            CvInvoke.cvFloodFill(touchArea, relTouch, new MCvScalar(255), new MCvScalar(0), new MCvScalar(1), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
 
             int matchedPixels = 0;
             int countedPixels = 0;
@@ -63,10 +69,10 @@ namespace bbiwarg.Detectors.TouchDetection
             {
                 for (int y = minY; y <= maxY; y++)
                 {
-                    Int16 depth = depthImage.getDepthAt(x, y);
                     Color color = outputImage.getColotAt(x, y);
+                    byte depth = touchArea.Data[y - minY, x - minX, 0];
                     Color subtractColor;
-                    if (depthAtTouch < depth && Math.Abs(depthAtTouch - depth) < Constants.TouchEventMaxDepthDifference)
+                    if(depth == 255)
                     {
                         matchedPixels++;
                         subtractColor = Constants.TouchEventAreaMatchedSubtractColor;

bbiwarg/Graphics/OutputImage.cs

@@ -20,14 +20,15 @@ namespace bbiwarg.Graphics
             Image = new Image<Rgb, byte>(width, height);
         }
 
-        public Color getColotAt(int x, int y) {
-            
+        public Color getColotAt(int x, int y)
+        {
+
             byte red = Image.Data[y, x, 0];
             byte green = Image.Data[y, x, 1];
             byte blue = Image.Data[y, x, 2];
-            return Color.FromArgb(red,green,blue);
+            return Color.FromArgb(red, green, blue);
         }
-        
+
         public void drawLineSegment(bbiwarg.Utility.LineSegment2D lineSegment, Color color, int thickness = 1)
         {
             Image.Draw(new LineSegment2D(lineSegment.P1, lineSegment.P2), new Rgb(color), thickness);

bbiwarg/Graphics/OutputWindow.cs

@@ -165,7 +165,8 @@ namespace bbiwarg.Graphics
 
             Timer.stop("onRenderFrame");
             
-            Timer.outputAll();
+            if(Constants.TimerOutputAll)
+                Timer.outputAll();
         }
 
     }

bbiwarg/Images/DepthImage.cs

@@ -29,16 +29,16 @@ namespace bbiwarg.Images
             Width = image.Width;
             Height = image.Height;
 
-            image = image.SmoothMedian(5);
+            image = image.SmoothMedian(Constants.DepthImageMedianSize);
 
             //threshold min&maxDepth
             MinDepth = findMinDepth(image);
-            MaxDepth = (Int16)(MinDepth + 200); // max = minDepth+255 (else it can't fit whole range in byte image)
+            MaxDepth = (Int16)(MinDepth + Constants.DepthImageDepthRange);
 
             //smooth+threshold (dst = (src > (MaxDepth - MinDepth)) ? MaxDepth - MinDepth : src)
             Image = (image- MinDepth).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
 
-            Image = Image.SmoothMedian(5);
+            Image = Image.SmoothMedian(Constants.DepthImageMedianSize);
         }
 
         public Int16 getDepthAt(Point point)

bbiwarg/Images/EdgeImage.cs

@@ -17,15 +17,18 @@ namespace bbiwarg.Images
     class EdgeImage
     {
         public Image<Gray, Byte> Image { get; private set; }
+        public Image<Gray, byte> RoughImage { get; private set; }
 
         public EdgeImage(DepthImage depthImage)
         {
-            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(100, 75, 3).ThresholdBinary(new Gray(0), new Gray(1));
+            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(Constants.EdgeImageCannyStartThreshold, Constants.EdgeImageCannyLinkingThreshold, Constants.EdgeImageCannySize).ThresholdBinary(new Gray(0), new Gray(1));
+            RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }
 
         public EdgeImage(Image<Gray, Byte> edgeImage)
         {
             Image = edgeImage;
+            RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }
 
         public bool isEdgeAt(Point point)
@@ -38,11 +41,20 @@ namespace bbiwarg.Images
             return (Image.Data[y, x, 0] > 0);
         }
 
-        public void removeFingerEdges(Finger finger)
+        public bool isRoughEdgeAt(Point point)
         {
-            Point[] polygon = finger.getPolygon();
+            return isRoughEdgeAt(point.X, point.Y);
+        }
 
+        public bool isRoughEdgeAt(int x, int y)
+        {
+            return (RoughImage.Data[y, x, 0] > 0);
+        }
+
+        public void removeEdgesInsidePolygon(Point[] polygon)
+        {
             Image.FillConvexPoly(polygon, new Gray(0));
+            RoughImage.FillConvexPoly(polygon, new Gray(0));
         }
 
         public EdgeImage copy()

bbiwarg/VideoHandle.cs

@@ -10,6 +10,7 @@ using bbiwarg.Detectors.FingerDetection;
 using bbiwarg.Detectors.PalmDetection;
 using bbiwarg.Detectors.TouchDetection;
 using bbiwarg.Detectors.HandDetection;
+using bbiwarg.Detectors.HandDetection;
 using bbiwarg.Images;
 using bbiwarg.InputProviders;
 using Emgu.CV;
@@ -131,14 +132,14 @@ namespace bbiwarg
             Timer.stop("readInputData");
 
             //create output images
-            Timer.start("createOtherImages");
-            int numImages = 4;
+            Timer.start("createOutputImages");
+            int numImages = 5;
             OutputImages = new OutputImage[numImages];
             for (int i = 0; i < numImages; i++) {
                 OutputImages[i] = new OutputImage(Width, Height);
                     
             }
-            Timer.stop("createOtherImages");
+            Timer.stop("createOutputImages");
 
             //create depthImage
             Timer.start("createDepthImage");
@@ -166,23 +167,21 @@ namespace bbiwarg
 
             //detect hands
             Timer.start("handDetection");
-            handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.TrackedFingers);
+            handDetector = new HandDetector(depthImage, edgeImage, fingerDetector.Fingers, OutputImages[2]);
             Timer.stop("handDetection");
 
             //remove background noise
             Timer.start("removeBackground");
-            depthImage.removeBackground(fingerDetector.Fingers);
-            edgeImage = new EdgeImage(depthImage);
-            OutputImages[2].Image[0] = OutputImages[2].Image[1] = OutputImages[2].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
+            //depthImage.removeBackground(fingerDetector.Fingers);
+            //edgeImage = new EdgeImage(depthImage);
+            OutputImages[3].Image[0] = OutputImages[3].Image[1] = OutputImages[3].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
             Timer.stop("removeBackground");
 
-            OutputImage palm = new OutputImage(Width, Height);
-
             //detect palm
             Timer.start("palmDetection");
             if (CurrentFrame == 0)
                 palmDetector.reset();
-            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[2], fingerDetector.Fingers);
+            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[3], fingerDetector.Fingers);
             Timer.stop("palmDetection");
 
             //detect touchEvents
@@ -194,7 +193,7 @@ namespace bbiwarg
 
             //track touchEvents
             Timer.start("touchTracking");
-            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[2]);
+            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[3]);
             Timer.stop("touchTracking");
 
             // touch event visualizer
@@ -207,7 +206,7 @@ namespace bbiwarg
                 foreach (PalmTouchEvent e in palmTouchDetector.PalmTouchEvents)
                     touchEventVisualizer.addPalmTouchEvent(e, CurrentFrame);
                 touchEventVisualizer.updateImage();
-                OutputImages[3] = touchEventVisualizer.OutputImage;
+                OutputImages[4] = touchEventVisualizer.OutputImage;
             }
 
             // add borders
@@ -217,7 +216,7 @@ namespace bbiwarg
 
             }
 
-            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], OutputImages[3]};
+            OutputImages = new OutputImage[] {OutputImages[0], OutputImages[1], OutputImages[2], OutputImages[3], OutputImages[4]};
             
             //palmDetector.i1, palmDetector.i2, palmDetector.i3, palmDetector.i5, palmDetector.i6, palmDetector.i7, palmDetector.i8, palmDetector.i9};