Changed PalmDetector to use Hands.

bbiwarg/Detectors/PalmDetection/PalmDetector.cs

@@ -11,21 +11,16 @@ using Emgu.CV.Structure;
 using bbiwarg.Utility;
 using bbiwarg.Images;
 using bbiwarg.Detectors.FingerDetection;
+using bbiwarg.Detectors.HandDetection;
 using bbiwarg.Graphics;
 
 namespace bbiwarg.Detectors.PalmDetection
 {
     class PalmDetector
     {
-        private int width, height;
         private OutputImage outputImage;
-        private EdgeImage edgeImage;
-
-        private Image<Gray, Byte> handImage;
-        private Image<Gray, Byte> pointingHandMask;
-
-        private List<Finger> fingers;
 
+        private Hand palmHand, pointingHand;
         private Contour<Point> palmContour;
         private List<MCvConvexityDefect> convexityDefects;
         private Vector2D thumbDefectStart;
@@ -34,57 +29,48 @@ namespace bbiwarg.Detectors.PalmDetection
         
         private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
 
-        private Vector2D topLeft;
-        private Vector2D topRight;
-        private Vector2D bottomLeft;
-        private Vector2D bottomRight;
-
         public Quadrangle PalmQuad { get; private set; }
 
         public PalmDetector()
         {
-            // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand
             thumbDefectDepthFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
             thumbDefectStartFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
             thumbDefectEndFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
         }
 
-        public void findPalmQuad(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage, List<Finger> trackedFingers)
+        public void findPalmQuad(OutputImage outputImage, List<Hand> hands)
         {
-            this.width = depthImage.Width;
-            this.height = depthImage.Height;
-
-            i1 = new OutputImage(width, height);
-            i2 = new OutputImage(width, height);
-            i3 = new OutputImage(width, height);
-            i4 = new OutputImage(width, height);
-            i5 = new OutputImage(width, height);
-            i6 = new OutputImage(width, height);
-            i7 = new OutputImage(width, height);
-            i8 = new OutputImage(width, height);
-            i9 = new OutputImage(width, height);
-
-            if (trackedFingers.Count >= 2)
-            {
-                this.edgeImage = edgeImage;
-                this.outputImage = outputImage;
-
-                // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
-                handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
+            this.outputImage = outputImage;
 
-                fingers = getFingersWithoutThumb(trackedFingers);
-                buildPointingHandMask();
-                handImage = handImage.And(pointingHandMask);
-
-                findLongestPalmContour();
-                if (palmContour != null)
+            if (hands.Count == 2)
+            {
+                if (hands[0].Fingers.Count > 1)
                 {
-                    findConvexityDefectsSortedByDepth();
-                    removeConvexityDefectsCausedByFingers();
-
-                    findHandPoints();
+                    pointingHand = hands[0];
+                    palmHand = hands[1];
                 }
+                else if (hands[1].Fingers.Count > 1)
+                {
+                    pointingHand = hands[1];
+                    palmHand = hands[0];
+                }
+                else if (hands[0].Fingers[0].LineSegment.Length > hands[1].Fingers[0].LineSegment.Length)
+                {
+                    pointingHand = hands[0];
+                    palmHand = hands[1];
+                }
+                else
+                {
+                    pointingHand = hands[1];
+                    palmHand = hands[0];
+                }
+
+                findLongestPalmContour();
+                findConvexityDefectsSortedByDepth();
+                removeConvexityDefectsCausedByFingers();
+                findHandPoints();
             }
+            
             draw();
         }
 
@@ -94,97 +80,14 @@ namespace bbiwarg.Detectors.PalmDetection
             thumbDefectStartFilter.reset();
             thumbDefectEndFilter.reset();
             lastThumbDefectDepth = null;
-        }
-
-        private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
-        {
-            Finger leftMost = null;
-            float minX = float.MaxValue;
-            foreach (Finger f in detectedFingers)
-            {
-                float midX = ((f.HandPoint + f.TipPoint) / 2).X;
-                if (midX < minX)
-                {
-                    minX = midX;
-                    leftMost = f;
-                }
-            }
-
-            List<Finger> result = new List<Finger>();
-            foreach (Finger f in detectedFingers)
-            {
-                if (f != leftMost)
-                    result.Add(f);
-            }
-            return result;
-        }
-
-        private void fillFirstFingerSlices(Image<Gray, Byte> image, byte val)
-        {
-            foreach (Finger f in fingers)
-            {
-                    image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1);
-            }
-        }
-
-        private Point getPointInPointingHand()
-        {
-            Finger finger = fingers[0];
-            if (finger == null)
-                return new Point(0, 0);
-            Vector2D direction = (finger.HandPoint - finger.TipPoint).normalize();
-            Vector2D pos = finger.HandPoint + direction;
-            
-            while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
-                pos += direction;
-
-            i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red);
-
-            return pos;
-        }
-
-        public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9;
-        private void buildPointingHandMask()
-        {
-            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
-
-            // dst = (src > 0) ? 1 : 0;
-            pointingHandMask = pointingHandMask.Or(edgeImage.Image);
-            i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
-
-            pointingHandMask = pointingHandMask.Dilate(1);
-            i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
-           
-            fillFirstFingerSlices(pointingHandMask, 1);
-            i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
-
-            //pointingHandMask = pointingHandMask.Dilate(1);
-            //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask;
-
-            MCvConnectedComp tmp = new MCvConnectedComp();
-            // fill with value 2
-            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
-
-            // dst = (src > 1) ? 0 : 1    (src > 1 <-> src == 2 <-> src filled by flood fill)
-            pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1));
-            i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
-            
-            pointingHandMask = pointingHandMask.Erode(6);
-            i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
-
-            fillFirstFingerSlices(pointingHandMask, 0);
-            i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
-            
-            pointingHandMask = pointingHandMask.Erode(2);
-            i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
-            
-            // only debug
-            i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
+            palmContour = null;
+            PalmQuad = null;
         }
 
         private void findLongestPalmContour()
         {
-            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
+            Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, 
+                                                                Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
             palmContour = contour;
             double maxPerimeter = 0;
@@ -219,7 +122,7 @@ namespace bbiwarg.Detectors.PalmDetection
             {
                 bool fingerIntersectsStartEnd = false;
                 float minFingerLineDist = float.MaxValue;
-                foreach (Finger f in fingers)
+                foreach (Finger f in pointingHand.Fingers)
                 {
                     Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
                     Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
@@ -242,7 +145,6 @@ namespace bbiwarg.Detectors.PalmDetection
             convexityDefects = newDefects;
         }
 
-        private int i = 0;
         private MCvConvexityDefect? findThumbDefect()
         {
             foreach (MCvConvexityDefect d in convexityDefects)
@@ -260,28 +162,11 @@ namespace bbiwarg.Detectors.PalmDetection
                 if (angle <= Constants.PalmMaxThumbDefectAngle && 
                     lengthQuotient >= Constants.PalmMinThumbDefectLengthQuotient && lengthQuotient <= Constants.PalmMaxThumbDefectLengthQuotient)
                 {
+                    Console.WriteLine(d.Depth);
                     return d;
                 }
             }
 
-
-            //Console.WriteLine("no palm defect found (" + i + ")");
-            foreach (MCvConvexityDefect d in convexityDefects)
-            {
-                Vector2D depth = new Vector2D(d.DepthPoint);
-                Vector2D start = new Vector2D(d.StartPoint);
-                Vector2D end = new Vector2D(d.EndPoint);
-
-                float l1 = (depth - start).Length;
-                float l2 = (depth - end).Length;
-                float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2);
-
-                float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
-
-                //Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
-            }
-
-            ++i;
             return null;
         }
 
@@ -289,14 +174,11 @@ namespace bbiwarg.Detectors.PalmDetection
         {
             MCvConvexityDefect? thumbDefect = findThumbDefect();
 
-            if (convexityDefects.Count > 0 && (thumbDefect != null || thumbDefectDepthFilter.Initialized))
+            if (thumbDefect != null)
             {
-                if (thumbDefect != null)
-                {
-                    thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
-                    thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
-                    thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
-                }
+                thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
+                thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
+                thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
                 
                 if (!thumbDefectDepthFilter.Initialized)
                 {
@@ -313,10 +195,10 @@ namespace bbiwarg.Detectors.PalmDetection
 
                 lastThumbDefectDepth = thumbDefectDepth;
 
-                Vector2D handLength, handWidth;
-                if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd))
+                Vector2D handLength, handWidth, topLeft, bottomLeft, bottomRight, topRight;
+                if (palmHand.Side == Hand.HandSide.Left)
                 {
-                    //right hand
+                    //left hand
                     handLength = thumbDefectStart - thumbDefectDepth;
                     handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
                     topLeft = thumbDefectStart+0.15f*handLength;
@@ -326,7 +208,7 @@ namespace bbiwarg.Detectors.PalmDetection
                 }
                 else
                 {
-                    //left hand
+                    //right hand
                     handLength = thumbDefectEnd - thumbDefectDepth;
                     handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
                     topRight = thumbDefectEnd+0.15f*handLength;

bbiwarg/VideoHandle.cs

@@ -181,7 +181,7 @@ namespace bbiwarg
             Timer.start("palmDetection");
             if (CurrentFrame == 0)
                 palmDetector.reset();
-            palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[3], fingerDetector.Fingers);
+            palmDetector.findPalmQuad(OutputImages[3], handDetector.Hands);
             Timer.stop("palmDetection");
 
             //detect touchEvents