Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

Conflicts:
	bbiwarg/Detectors/HandDetection/HandDetector.cs

bbiwarg/Constants.cs

@@ -45,6 +45,7 @@ namespace bbiwarg
         public static readonly float PalmMaxThumbDefectAngle = 110; // degree
         public static readonly float PalmMaxThumbDefectLengthQuotient = 2.2f;
         public static readonly float PalmMinThumbDefectLengthQuotient = 1.2f;
+        public static readonly float PalmMinTumbDefectDepth = 30.0f;
         
         //palm Grid
         public static readonly int PalmGridRows = 4;

bbiwarg/Detectors/HandDetection/Hand.cs

@@ -12,11 +12,20 @@ namespace bbiwarg.Detectors.HandDetection
 {
     class Hand
     {
+        public enum HandSide
+        {
+            Left,
+            Right
+        }
+
+        public Vector2D Centroid { private set; get; }
+        public HandSide Side { get; set; }
         public Image<Gray, byte> Mask { get; private set; }
         public List<Finger> Fingers { get; private set; }
 
         public Hand(Image<Gray, byte> mask) {
             Mask = mask;
+            Centroid = getCentroid();
             Fingers = new List<Finger>();
         }
 
@@ -27,5 +36,11 @@ namespace bbiwarg.Detectors.HandDetection
         public void addFinger(Finger finger) {
             Fingers.Add(finger);
         }
+
+        private Vector2D getCentroid()
+        {
+            MCvPoint2D64f gravityCenter = Mask.GetMoments(true).GravityCenter;
+            return new Vector2D((float)gravityCenter.x, (float)gravityCenter.y);
+        }
     }
 }

bbiwarg/Detectors/HandDetection/HandDetector.cs

@@ -85,14 +85,37 @@ namespace bbiwarg.Detectors.HandDetection
                     }
                 }
             }
+
+            float minX = float.MaxValue;
+            foreach (Hand hand in Hands)
+            {
+                if (hand.Centroid.X < minX)
+                    minX = hand.Centroid.X;
+            }
+
+            foreach (Hand hand in Hands)
+            {
+                if (hand.Centroid.X == minX)
+                    hand.Side = Hand.HandSide.Left;
+                else
+                    hand.Side = Hand.HandSide.Right;
+            }
         }
 
-        private void drawHands()
-        {
-            int maxIndex = Math.Min(3, Hands.Count);
-            for (int i = 0; i < maxIndex; i++)
+        private void drawHands() {
+            if (Hands.Count == 1)
+            {
+                outputImage.Image[0] = outputImage.Image[1] = outputImage.Image[2] = Hands[0].Mask;
+            }
+            else if (Hands.Count > 1)
             {
-                outputImage.Image[i] = Hands[i].Mask;
+                for (int i = 0; i < 2; ++i)
+                {
+                    if (Hands[i].Side == Hand.HandSide.Left)
+                        outputImage.Image[0] = outputImage.Image[0] | (Hands[i].Mask);
+                    else
+                        outputImage.Image[1] = outputImage.Image[1] | (Hands[i].Mask);
+                }
             }
         }
     }

bbiwarg/Detectors/PalmDetection/PalmDetector.cs

@@ -11,21 +11,16 @@ using Emgu.CV.Structure;
 using bbiwarg.Utility;
 using bbiwarg.Images;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Detectors.HandDetection;
 using bbiwarg.Graphics;
 
 namespace bbiwarg.Detectors.PalmDetection
 {
     class PalmDetector
     {
-        private int width, height;
         private OutputImage outputImage;
-        private EdgeImage edgeImage;
-
-        private Image<Gray, Byte> handImage;
-        private Image<Gray, Byte> pointingHandMask;
-
-        private List<Finger> fingers;
 
+        private Hand palmHand, pointingHand;
         private Contour<Point> palmContour;
         private List<MCvConvexityDefect> convexityDefects;
         private Vector2D thumbDefectStart;
@@ -34,57 +29,48 @@ namespace bbiwarg.Detectors.PalmDetection
         
         private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
 
-        private Vector2D topLeft;
-        private Vector2D topRight;
-        private Vector2D bottomLeft;
-        private Vector2D bottomRight;
-
         public Quadrangle PalmQuad { get; private set; }
 
         public PalmDetector()
         {
-            // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand
             thumbDefectDepthFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
             thumbDefectStartFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
             thumbDefectEndFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
         }
 
-        public void findPalmQuad(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage, List<Finger> trackedFingers)
+        public void findPalmQuad(OutputImage outputImage, List<Hand> hands)
         {
-            this.width = depthImage.Width;
-            this.height = depthImage.Height;
-
-            i1 = new OutputImage(width, height);
-            i2 = new OutputImage(width, height);
-            i3 = new OutputImage(width, height);
-            i4 = new OutputImage(width, height);
-            i5 = new OutputImage(width, height);
-            i6 = new OutputImage(width, height);
-            i7 = new OutputImage(width, height);
-            i8 = new OutputImage(width, height);
-            i9 = new OutputImage(width, height);
-
-            if (trackedFingers.Count >= 2)
-            {
-                this.edgeImage = edgeImage;
-                this.outputImage = outputImage;
-
-                // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
-                handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
+            this.outputImage = outputImage;
 
-                fingers = getFingersWithoutThumb(trackedFingers);
-                buildPointingHandMask();
-                handImage = handImage.And(pointingHandMask);
-
-                findLongestPalmContour();
-                if (palmContour != null)
+            if (hands.Count == 2)
+            {
+                if (hands[0].Fingers.Count > 1)
                 {
-                    findConvexityDefectsSortedByDepth();
-                    removeConvexityDefectsCausedByFingers();
-
-                    findHandPoints();
+                    pointingHand = hands[0];
+                    palmHand = hands[1];
                 }
+                else if (hands[1].Fingers.Count > 1)
+                {
+                    pointingHand = hands[1];
+                    palmHand = hands[0];
+                }
+                else if (hands[0].Fingers[0].LineSegment.Length > hands[1].Fingers[0].LineSegment.Length)
+                {
+                    pointingHand = hands[0];
+                    palmHand = hands[1];
+                }
+                else
+                {
+                    pointingHand = hands[1];
+                    palmHand = hands[0];
+                }
+
+                findLongestPalmContour();
+                findConvexityDefectsSortedByDepth();
+                removeConvexityDefectsCausedByFingers();
+                findHandPoints();
             }
+            
             draw();
         }
 
@@ -94,97 +80,14 @@ namespace bbiwarg.Detectors.PalmDetection
             thumbDefectStartFilter.reset();
             thumbDefectEndFilter.reset();
             lastThumbDefectDepth = null;
-        }
-
-        private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
-        {
-            Finger leftMost = null;
-            float minX = float.MaxValue;
-            foreach (Finger f in detectedFingers)
-            {
-                float midX = ((f.HandPoint + f.TipPoint) / 2).X;
-                if (midX < minX)
-                {
-                    minX = midX;
-                    leftMost = f;
-                }
-            }
-
-            List<Finger> result = new List<Finger>();
-            foreach (Finger f in detectedFingers)
-            {
-                if (f != leftMost)
-                    result.Add(f);
-            }
-            return result;
-        }
-
-        private void fillFirstFingerSlices(Image<Gray, Byte> image, byte val)
-        {
-            foreach (Finger f in fingers)
-            {
-                    image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1);
-            }
-        }
-
-        private Point getPointInPointingHand()
-        {
-            Finger finger = fingers[0];
-            if (finger == null)
-                return new Point(0, 0);
-            Vector2D direction = (finger.HandPoint - finger.TipPoint).normalize();
-            Vector2D pos = finger.HandPoint + direction;
-            
-            while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
-                pos += direction;
-
-            i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red);
-
-            return pos;
-        }
-
-        public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9;
-        private void buildPointingHandMask()
-        {
-            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
-
-            // dst = (src > 0) ? 1 : 0;
-            pointingHandMask = pointingHandMask.Or(edgeImage.Image);
-            i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
-
-            pointingHandMask = pointingHandMask.Dilate(1);
-            i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
-           
-            fillFirstFingerSlices(pointingHandMask, 1);
-            i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
-
-            //pointingHandMask = pointingHandMask.Dilate(1);
-            //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask;
-
-            MCvConnectedComp tmp = new MCvConnectedComp();
-            // fill with value 2
-            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
-
-            // dst = (src > 1) ? 0 : 1    (src > 1 <-> src == 2 <-> src filled by flood fill)
-            pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1));
-            i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
-            
-            pointingHandMask = pointingHandMask.Erode(6);
-            i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
-
-            fillFirstFingerSlices(pointingHandMask, 0);
-            i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
-            
-            pointingHandMask = pointingHandMask.Erode(2);
-            i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
-            
-            // only debug
-            i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
+            palmContour = null;
+            PalmQuad = null;
         }
 
         private void findLongestPalmContour()
         {
-            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
+            Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, 
+                                                                Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
             palmContour = contour;
             double maxPerimeter = 0;
@@ -219,7 +122,7 @@ namespace bbiwarg.Detectors.PalmDetection
             {
                 bool fingerIntersectsStartEnd = false;
                 float minFingerLineDist = float.MaxValue;
-                foreach (Finger f in fingers)
+                foreach (Finger f in pointingHand.Fingers)
                 {
                     Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
                     Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
@@ -242,7 +145,6 @@ namespace bbiwarg.Detectors.PalmDetection
             convexityDefects = newDefects;
         }
 
-        private int i = 0;
         private MCvConvexityDefect? findThumbDefect()
         {
             foreach (MCvConvexityDefect d in convexityDefects)
@@ -258,30 +160,13 @@ namespace bbiwarg.Detectors.PalmDetection
                 float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
 
                 if (angle <= Constants.PalmMaxThumbDefectAngle && 
-                    lengthQuotient >= Constants.PalmMinThumbDefectLengthQuotient && lengthQuotient <= Constants.PalmMaxThumbDefectLengthQuotient)
+                    lengthQuotient >= Constants.PalmMinThumbDefectLengthQuotient && lengthQuotient <= Constants.PalmMaxThumbDefectLengthQuotient &&
+                    d.Depth >= Constants.PalmMinTumbDefectDepth)
                 {
                     return d;
                 }
             }
 
-
-            //Console.WriteLine("no palm defect found (" + i + ")");
-            foreach (MCvConvexityDefect d in convexityDefects)
-            {
-                Vector2D depth = new Vector2D(d.DepthPoint);
-                Vector2D start = new Vector2D(d.StartPoint);
-                Vector2D end = new Vector2D(d.EndPoint);
-
-                float l1 = (depth - start).Length;
-                float l2 = (depth - end).Length;
-                float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2);
-
-                float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
-
-                //Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
-            }
-
-            ++i;
             return null;
         }
 
@@ -289,14 +174,11 @@ namespace bbiwarg.Detectors.PalmDetection
         {
             MCvConvexityDefect? thumbDefect = findThumbDefect();
 
-            if (convexityDefects.Count > 0 && (thumbDefect != null || thumbDefectDepthFilter.Initialized))
+            if (thumbDefect != null)
             {
-                if (thumbDefect != null)
-                {
-                    thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
-                    thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
-                    thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
-                }
+                thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
+                thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
+                thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
                 
                 if (!thumbDefectDepthFilter.Initialized)
                 {
@@ -313,10 +195,10 @@ namespace bbiwarg.Detectors.PalmDetection
 
                 lastThumbDefectDepth = thumbDefectDepth;
 
-                Vector2D handLength, handWidth;
-                if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd))
+                Vector2D handLength, handWidth, topLeft, bottomLeft, bottomRight, topRight;
+                if (palmHand.Side == Hand.HandSide.Left)
                 {
-                    //right hand
+                    //left hand
                     handLength = thumbDefectStart - thumbDefectDepth;
                     handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
                     topLeft = thumbDefectStart+0.15f*handLength;
@@ -326,7 +208,7 @@ namespace bbiwarg.Detectors.PalmDetection
                 }
                 else
                 {
-                    //left hand
+                    //right hand
                     handLength = thumbDefectEnd - thumbDefectDepth;
                     handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
                     topRight = thumbDefectEnd+0.15f*handLength;

bbiwarg/VideoHandle.cs

@@ -181,7 +181,7 @@ namespace bbiwarg
             Timer.start("palmDetection");
             if (CurrentFrame == 0)
                 palmDetector.reset();
-            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[3], fingerDetector.Fingers);
+            palmDetector.findPalmQuad(OutputImages[3], handDetector.Hands);
             Timer.stop("palmDetection");
 
             //detect touchEvents