fixed bugs in hand+palm

bbiwarg/Constants.cs

@@ -63,7 +63,7 @@ namespace bbiwarg
         public static readonly int FingerMinNumSlices = 25 / FingerStepSize;
         public static readonly int FingerNumSlicesForDirectionDetection = FingerMinNumSlices / 4;
         public static readonly int FingerMaxWidth = 30;
-        public static readonly int FingerMinWidth = 5;
+        public static readonly int FingerMinWidth = 2;
         public static readonly int FingerRemoveNumSlicesForCorrection = 5;
         public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
         public static readonly int FingerOutMargin = 8;
@@ -74,9 +74,9 @@ namespace bbiwarg
         public static readonly float FingerTrackerMinSimilarityForTracking = 0.8f;
         public static readonly int FingerTrackerNumFramesDetectedUntilTracked = 5;
         public static readonly int FingerTrackerNumFramesLostUntilDeleted = 10;
-        public static readonly float FingermXX = 0.0005f;
-        public static readonly float FingermXY = 0.0005f;
-        public static readonly float FingermYY = 0.0005f;
+        public static readonly float FingermXX = 0.00005f;
+        public static readonly float FingermXY = 0.0f;
+        public static readonly float FingermYY = 0.00005f;
         public static readonly int FingerTrackerNumDirectionsForReverseCheck = 20;
 
         // hand detection
@@ -84,6 +84,7 @@ namespace bbiwarg
         public static readonly int HandFloodFillDownDiff = 2;
         public static readonly int HandFloodFillUpDiff = 2;
         public static readonly float HandMaxSize = 0.6f;
+        public static readonly float HandMinSize = 0.01f;
         public static readonly float HandThumbDefectMaxDistanceToThumb = FingerMaxWidth;
         public static readonly float HandThumbDefectMinThumbShortLengthRatio = 0.75f;
         public static readonly float HandThumbDefectMaxThumbShortLengthRatio = 1.1f;
@@ -96,7 +97,7 @@ namespace bbiwarg
         public static readonly int HandTrackerNumFramesDetectedUntilTracked = 2;
         public static readonly int HandTrackerNumFramesLostUntilDeleted = 5;
         public static readonly float HandmXX = 0.0005f;
-        public static readonly float HandmXY = 0.0005f;
+        public static readonly float HandmXY = 0.0f;
         public static readonly float HandmYY = 0.0005f;
 
         // palm detection
@@ -106,7 +107,7 @@ namespace bbiwarg
         public static readonly int PalmTrackerNumFramesDetectedUntilTracked = 2;
         public static readonly int PalmTrackerNumFramesLostUntilDeleted = 5;
         public static readonly float PalmmXX = 0.00005f;
-        public static readonly float PalmmXY = 0.00005f;
+        public static readonly float PalmmXY = 0.0f;
         public static readonly float PalmmYY = 0.00005f;
 
         //palm Grid
@@ -122,7 +123,7 @@ namespace bbiwarg
         public static readonly int TouchTipOutsideFactor = 7;
 
         // touch tracking
-        public static readonly float TouchTrackerMinSimilarity = 0.7f;
+        public static readonly float TouchTrackerMinSimilarity = 0.3f;
         public static readonly int TouchTrackerNumFramesDetectedUntilTracked = 1;
         public static readonly int TouchTrackerNumFramesLostUntilDeleted = 5;
         public static readonly float TouchmXX = 0.003f;

bbiwarg/InputHandler.cs

@@ -281,18 +281,11 @@ namespace bbiwarg
                 if (h.ThumbDefect != null)
                     OutputImages[2].drawDefect(h.ThumbDefect, Color.CornflowerBlue, Color.Yellow);        
             }
-            foreach(Finger finger in fingerTracker.Fingers) {
-                Vector2D midOut1 = finger.SliceTrail.MidSlice.OutStart;
-                Vector2D midOut2 = finger.SliceTrail.MidSlice.OutEnd;
-                OutputImages[2].fillCircle(midOut1.IntX, midOut1.IntY, 3, Color.Yellow);
-                OutputImages[2].fillCircle(midOut2.IntX, midOut2.IntY, 3, Color.Yellow);
-
-            }
 
             //image3
             OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.ThresholdBinary(new Gray(0), new Gray(255))), Constants.DepthImageColor);
             foreach (TouchEvent te in touchTracker.TouchEvents)
-                OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
+                OutputImages[3].fillCircle(te.AbsolutePosition.IntX, te.AbsolutePosition.IntY, 5, Constants.TouchEventTrackedColor);
             foreach (Palm p in palmTracker.OptimizedPalms)
             {
                 OutputImages[3].drawQuadrangleGrid(p.Quad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns);

bbiwarg/Recognition/FingerRecognition/FingerSliceTrail.cs

@@ -14,9 +14,11 @@ namespace bbiwarg.Recognition.FingerRecognition
         private LineSegment2D lineSegment;
         private Contour<Point> contour;
         private Contour<Point> innerContour;
+        private Contour<Point> outerContour;
         private bool lineSegmentUpToDate;
         private bool contourUpToDate;
         private bool innerContourUpToDate;
+        private bool outerContourUpToDate;
 
         public List<FingerSlice> Slices { get; private set; }
         public FingerSlice StartSlice { get { return Slices[0]; } }
@@ -27,6 +29,7 @@ namespace bbiwarg.Recognition.FingerRecognition
         public LineSegment2D LineSegment { get { if (!lineSegmentUpToDate) updateLineSegment(); return lineSegment; } }
         public Contour<Point> Contour { get { if (!contourUpToDate) updateContour(); return contour; } }
         public Contour<Point> InnerContour { get { if (!innerContourUpToDate) updateInnerContour(); return innerContour; } }
+        public Contour<Point> OuterContour { get { if (!outerContourUpToDate) updateOuterContour(); return outerContour; } }
 
         public FingerSliceTrail(FingerSlice slice)
         {
@@ -119,6 +122,28 @@ namespace bbiwarg.Recognition.FingerRecognition
             innerContourUpToDate = true;
         }
 
+        private void updateOuterContour()
+        {
+            List<Point> pointsA = new List<Point>();
+            List<Point> pointsB = new List<Point>();
+
+            foreach (FingerSlice slice in Slices)
+            {
+                pointsA.Add(slice.OutStart);
+                pointsB.Add(slice.OutEnd);
+            }
+
+            pointsA.Reverse();
+            pointsA.AddRange(pointsB);
+
+            outerContour = new Contour<Point>(new MemStorage());
+            foreach (Point p in pointsA)
+            {
+                outerContour.Push(p);
+            }
+            outerContourUpToDate = true;
+        }
+
 
 
 

bbiwarg/Recognition/HandRecognition/Hand.cs

@@ -13,7 +13,8 @@ using bbiwarg.Recognition.PalmRecognition;
 
 namespace bbiwarg.Recognition.HandRecognition
 {
-    public enum HandSide {
+    public enum HandSide
+    {
         Undefined = 0,
         Right = 1,
         Left = 2
@@ -55,31 +56,46 @@ namespace bbiwarg.Recognition.HandRecognition
             ZIndex = zIndex;
         }
 
-        public void mergeWith(Hand mergeHand) {
+        public void mergeWith(Hand mergeHand)
+        {
             extendMask(mergeHand.Mask);
-            foreach (Finger f2 in mergeHand.Fingers)
+            Fingers.AddRange(mergeHand.Fingers);
+            foreach (Finger finger in Fingers)
             {
-                addFinger(f2);
-                f2.setHand(this);
+                finger.setHand(this);
             }
             findCentroids();
         }
 
-        public void extendMask(Image<Gray, byte> extendMask) {
+        public void extendMask(Image<Gray, byte> extendMask)
+        {
             Mask = Mask.Or(extendMask);
             findCentroids();
         }
 
-        public void fixDefectsFromOverlappingFingers(List<Finger> otherFingers)
+        public void fillOverlappingFingers(List<Finger> otherFingers)
         {
-            foreach (Finger finger in otherFingers) {
-                foreach (FingerSlice slice in finger.SliceTrail.Slices) {
+            foreach (Finger finger in otherFingers)
+            {
+                FingerSliceTrail trail = null;
+                foreach (FingerSlice slice in finger.SliceTrail.Slices)
+                {
                     Vector2D out1 = slice.OutStart;
                     Vector2D out2 = slice.OutEnd;
 
                     if (isInside(out1) && isInside(out2))
-                        Mask.Draw(new Emgu.CV.Structure.LineSegment2D(out1, out2), new Gray(1), 1);
+                    {
+                        if (trail == null)
+                            trail = new FingerSliceTrail(slice);
+                        else
+                            trail.addSlice(slice);
+                    }
                 }
+
+                if (trail != null)
+                    Mask.FillConvexPoly(trail.OuterContour.ToArray(), new Gray(1));
+
+                Mask = Mask.Dilate(1);
             }
         }
 
@@ -107,7 +123,7 @@ namespace bbiwarg.Recognition.HandRecognition
         {
             Contour<Point> contour = Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
             List<MCvConvexityDefect> mcvConvexityDefects = new List<MCvConvexityDefect>(contour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
-            
+
             ConvexityDefects = new List<ConvexityDefect>();
             foreach (MCvConvexityDefect defect in mcvConvexityDefects)
                 ConvexityDefects.Add(new ConvexityDefect(defect));
@@ -132,7 +148,8 @@ namespace bbiwarg.Recognition.HandRecognition
             }
         }
 
-        private void determineHandSide() {
+        private void determineHandSide()
+        {
             if (ThumbDefect != null)
             {
                 if (ThumbDefect.VectorShort.crossProduct(ThumbDefect.VectorLong) > 0)
@@ -157,7 +174,7 @@ namespace bbiwarg.Recognition.HandRecognition
             {
                 Finger finger = Fingers[0];
                 Vector2D direction = (finger.HandPoint - Centroid).normalize();
-                CentroidInHand = Centroid + direction;
+                CentroidInHand = Centroid;
                 while (!isInside(CentroidInHand))
                     CentroidInHand += direction;
             }

bbiwarg/Recognition/HandRecognition/HandDetector.cs

@@ -21,7 +21,6 @@ namespace bbiwarg.Recognition.HandRecognition
         private EdgeImage edgeImage;
         private Image<Gray, byte> modifiedHandDepthImage;
         private List<Finger> fingers;
-        private List<Hand> ignoreMergedHands;
 
         public List<Hand> Hands { get; private set; }
         public Image<Gray, byte> HandMask { get; private set; }
@@ -75,7 +74,8 @@ namespace bbiwarg.Recognition.HandRecognition
                 if (newHand)
                 {
                     Image<Gray, byte> handMask = getHandMask(finger.HandPoint);
-                    if (handMask.CountNonzero()[0] < maxArea * Constants.HandMaxSize)
+                    int area = handMask.CountNonzero()[0];
+                    if (area < maxArea * Constants.HandMaxSize && area > maxArea * Constants.HandMinSize)
                     {
                         Hand hand = new Hand(handMask, finger);
                         Hands.Add(hand);
@@ -107,7 +107,7 @@ namespace bbiwarg.Recognition.HandRecognition
         private void fixOverlappingFingers()
         {
             extendOrMergeThroughOverlappingFingers();
-            fixDefectsFromOverlappingFingers();
+            fillOverlappingFingers();
         }
 
         private void findThumbDefects()
@@ -163,7 +163,7 @@ namespace bbiwarg.Recognition.HandRecognition
                                 bool merge = false;
                                 foreach (Hand hand in Hands)
                                 {
-                                    if (hand.isInside(p2) && !ignoreMergedHands.Contains(hand))
+                                    if (hand.isInside(p2) && !mergeHands.Keys.Contains(hand))
                                     {
                                         mergeHands.Add(underlyingHand, overlappingHand);
                                         merge = true;
@@ -173,7 +173,7 @@ namespace bbiwarg.Recognition.HandRecognition
 
                                 //if no merge, extend hand
                                 if (!merge)
-                                    underlyingHand.extendMask(getHandMask(p2));
+                                    extendHand(underlyingHand, p2);
                             }
                         }
                     }
@@ -187,7 +187,19 @@ namespace bbiwarg.Recognition.HandRecognition
             }
         }
 
-        private void fixDefectsFromOverlappingFingers()
+        private void extendHand(Hand hand, Vector2D p) {
+            int width = edgeImage.Width;
+            int height = edgeImage.Height;
+            int maxArea = width * height;
+            
+            Image<Gray, byte> extendMask = getHandMask(p);
+            int area = extendMask.CountNonzero()[0];
+
+            if(area <= 0.5f*Constants.HandMaxSize * maxArea)
+                hand.extendMask(extendMask);
+        }
+
+        private void fillOverlappingFingers()
         {
             foreach (Hand hand in Hands)
             {
@@ -199,7 +211,7 @@ namespace bbiwarg.Recognition.HandRecognition
                         otherFingers.AddRange(otherHand.Fingers);
                 }
 
-                hand.fixDefectsFromOverlappingFingers(otherFingers);
+                hand.fillOverlappingFingers(otherFingers);
             }
         }
     }

bbiwarg/Recognition/PalmRecognition/PalmDetector.cs

@@ -43,13 +43,46 @@ namespace bbiwarg.Recognition.PalmRecognition
             {
                 ConvexityDefect thumbDefect = palmHand.ThumbDefect;
 
-                Vector2D handLength = 1.5f*thumbDefect.VectorLong;
-                Vector2D handWidth = 0.45f*handLength.getOrthogonal(palmHand.Side == HandSide.Right);
+                Vector2D handLength = 1.5f * thumbDefect.VectorLong;
+                Vector2D handWidth = 0.45f * handLength.getOrthogonal(palmHand.Side == HandSide.Right);
+
+                Vector2D fingersUpperOld = thumbDefect.Inner + 0.8f * handLength;
+                Vector2D wristUpperOld = thumbDefect.Inner - 0.2f * handLength;
+                Vector2D wristLowerOld = wristUpperOld + handWidth;
+                Vector2D fingersLowerOld = wristLowerOld + 0.75f * handLength - 0.3f * handWidth;
+
+                Vector2D directionWristFinger = thumbDefect.VectorLong.normalize();
+                Vector2D directionFingerWrist = directionWristFinger.getInverse();
+                Vector2D directionUpperLower = directionWristFinger.getOrthogonal(palmHand.Side == HandSide.Right);
+                Vector2D directionLowerUpper = directionUpperLower.getInverse();
+
+                //fingersUpper
+                Vector2D fingersUpper = thumbDefect.OuterLong;
+                bool handBelow = true;
+                while (handBelow && fingersUpper.isInBound(Vector2D.Zero, Constants.MaxPixel))
+                {
+                    fingersUpper += directionWristFinger;
+                    Vector2D below = fingersUpper.copy();
+                    bool handBelowFound = false;
+                    while (!handBelowFound && below.getDistanceTo(fingersUpper) < 2*Constants.FingerMaxWidth && below.isInBound(Vector2D.Zero, Constants.MaxPixel))
+                    {
+                        below += directionUpperLower;
+                        handBelowFound = palmHand.isInside(below);
+                    }
+                    handBelow = handBelowFound;
+                }
+
+                //wristUpper
+                Vector2D wristUpper = thumbDefect.Inner;
+                while (wristUpper.isInBound(Vector2D.Zero, Constants.MaxPixel) && palmHand.isInside(wristUpper))
+                    wristUpper += directionFingerWrist;
+
+                //wristLower
+                Vector2D wristLower = wristLowerOld;
+
+                //fingersLower
+                Vector2D fingersLower = fingersLowerOld;
 
-                Vector2D fingersUpper = thumbDefect.Inner + 0.75f*handLength;
-                Vector2D wristUpper = thumbDefect.Inner - 0.25f*handLength;
-                Vector2D wristLower = wristUpper + handWidth;
-                Vector2D fingersLower = wristLower + 0.75f * handLength - 0.2f * handWidth;
 
                 Palm palm = new Palm(palmHand, wristUpper, wristLower, fingersLower, fingersUpper);
                 Palms.Add(palm);

bbiwarg/Recognition/TouchRecognition/TouchEvent.cs

@@ -14,8 +14,8 @@ namespace bbiwarg.Recognition.TouchRecognition
 {
     class TouchEvent : TrackableObject
     {
-        public Vector2D Position { get; private set; }
-        public Vector2D RelativePalmPosition { get; private set; }
+        public Vector2D AbsolutePosition { get; private set; }
+        public Vector2D RelativePosition { get; private set; }
         public Image<Gray, byte> TouchMask { get; private set; }
         public Finger Finger { get; private set; }
         public Palm Palm { get; private set; }
@@ -23,8 +23,8 @@ namespace bbiwarg.Recognition.TouchRecognition
 
         public TouchEvent(Vector2D absolutePosition, Image<Gray, byte> touchMask, Finger finger, Palm palm)
         {
-            Position = absolutePosition;
-            RelativePalmPosition = palm.getRelativePosition(absolutePosition);
+            AbsolutePosition = absolutePosition;
+            RelativePosition = palm.getRelativePosition(absolutePosition);
             TouchMask = touchMask;
             Finger = finger;
             Palm = palm;

bbiwarg/Recognition/TouchRecognition/TrackedTouchEvent.cs

@@ -41,10 +41,10 @@ namespace bbiwarg.Recognition.TouchRecognition
             : base(id, detectedTouchEvent, numFramesDetectedUntilTracked, numFramesLostUntilDeleted)
         {
             absolutePositionKalman = new Kalman2DPositionFilter(Constants.TouchmXX, Constants.TouchmXY, Constants.TouchmYY);
-            absolutePositionKalman.setInitialPosition(detectedTouchEvent.Position);
+            absolutePositionKalman.setInitialPosition(detectedTouchEvent.AbsolutePosition);
 
             relativePositionKalman = new Kalman2DPositionFilter(Constants.TouchmXX, Constants.TouchmXY, Constants.TouchmYY);
-            relativePositionKalman.setInitialPosition(detectedTouchEvent.RelativePalmPosition);
+            relativePositionKalman.setInitialPosition(detectedTouchEvent.RelativePosition);
 
             fingerID = detectedTouchEvent.Finger.TrackID;
             isTouchActive = false;
@@ -61,8 +61,8 @@ namespace bbiwarg.Recognition.TouchRecognition
 
             if (detectedTouchEvent != null)
             {
-                Vector2D correctedAbsolutePosition = absolutePositionKalman.getCorrectedPosition(detectedTouchEvent.Position);
-                Vector2D correctedRelativePosition = relativePositionKalman.getCorrectedPosition(detectedTouchEvent.RelativePalmPosition);
+                Vector2D correctedAbsolutePosition = absolutePositionKalman.getCorrectedPosition(detectedTouchEvent.AbsolutePosition);
+                Vector2D correctedRelativePosition = relativePositionKalman.getCorrectedPosition(detectedTouchEvent.RelativePosition);
 
                 if (CurrentState == TrackingState.Tracked)
                 {
@@ -82,11 +82,15 @@ namespace bbiwarg.Recognition.TouchRecognition
             //finger similarity
             float fingerSimilarity = (fingerID == detectedTouchEvent.Finger.TrackID) ? 1 : 0;
 
-            //position similarity
-            float distance = detectedTouchEvent.Position.getDistanceTo(AbsolutePositionPrediction);
-            float distanceSimilarity = Math.Max(0, 1 - distance / Constants.ImageDiagonalLength);
+            //absolute position similarity
+            float absoluteDistance = detectedTouchEvent.AbsolutePosition.getDistanceTo(AbsolutePositionPrediction);
+            float absoluteSimilarity = Math.Max(0, 1 - absoluteDistance / Constants.ImageDiagonalLength);
+            
+            //relative position similarity
+            float relativeDistance = detectedTouchEvent.RelativePosition.getDistanceTo(RelativePositionPrediction);
+            float relativeSimilarity = Math.Max(0, 1 - relativeDistance / 1f);
 
-            float similarity = fingerSimilarity * distanceSimilarity;
+            float similarity = fingerSimilarity * absoluteSimilarity * relativeSimilarity;
 
             return similarity;
         }