-updated constants

-fixed finger overlay i nhands

bbiwarg/Constants.cs

@@ -18,7 +18,7 @@ namespace bbiwarg
     class Constants
     {
         // input
-        public static readonly InputType InputSource = InputType.Movie;
+        public static readonly InputType InputSource = InputType.Camera;
         public static readonly String InputMoviePath = "..\\..\\videos\\touch\\4.skv";
 
         // Logger
@@ -66,9 +66,9 @@ namespace bbiwarg
         public static readonly int FingerMinWidth = 5;
         public static readonly int FingerRemoveNumSlicesForCorrection = 5;
         public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
-        public static readonly int FingerCrippleCheckMargin = 8;
+        public static readonly int FingerOutMargin = 8;
         public static readonly int FingerCrippleMinDifference = 20;
-        public static readonly int FingerSliceOutMargin = 4;
+        public static readonly int FingerContourMargin = 4;
 
         // finger tracking
         public static readonly float FingerTrackerMinSimilarityForTracking = 0.8f;
@@ -84,11 +84,12 @@ namespace bbiwarg
         public static readonly int HandFloodFillDownDiff = 2;
         public static readonly int HandFloodFillUpDiff = 2;
         public static readonly float HandMaxSize = 0.6f;
-        public static readonly float HandThumbDefectMaxDistanceToThumb = 2 * FingerMaxWidth;
+        public static readonly float HandThumbDefectMaxDistanceToThumb = FingerMaxWidth;
         public static readonly float HandThumbDefectMinThumbShortLengthRatio = 0.75f;
         public static readonly float HandThumbDefectMaxThumbShortLengthRatio = 1.1f;
         public static readonly float HandThumbDefectMinShortLongLengthRatio = 0.3f;
         public static readonly float HandThumbDefectMaxShortLongLengthRatio = 0.7f;
+        public static readonly int HandExtendMaxDifference = 40;
 
         // hand tracker
         public static readonly float HandTrackerMinSimilarity = 0.7f;
@@ -124,9 +125,9 @@ namespace bbiwarg
         public static readonly float TouchTrackerMinSimilarity = 0.7f;
         public static readonly int TouchTrackerNumFramesDetectedUntilTracked = 1;
         public static readonly int TouchTrackerNumFramesLostUntilDeleted = 5;
-        public static readonly float TouchmXX = 0.0065f;
+        public static readonly float TouchmXX = 0.003f;
         public static readonly float TouchmXY = 0.0f;
-        public static readonly float TouchmYY = 0.0065f;
+        public static readonly float TouchmYY = 0.003f;
         public static readonly float TouchProcessNoise = 3.0e-4f;
 
         // touchEventVisualizer

bbiwarg/Graphics/OutputImage.cs

@@ -67,6 +67,16 @@ namespace bbiwarg.Graphics
             Image.Draw(text, ref font, new Point(x, y), new Rgb(color));
         }
 
+        public void drawDefect(ConvexityDefect defect, Color pointColor, Color lineColor) {
+            drawLineSegment(new Utility.LineSegment2D(defect.OuterShort, defect.Inner), lineColor);
+            drawLineSegment(new Utility.LineSegment2D(defect.OuterLong, defect.Inner), lineColor);
+            
+            fillCircle(defect.Inner.IntX, defect.Inner.IntY, 2, pointColor);
+            fillCircle(defect.OuterShort.IntX, defect.OuterShort.IntY, 2, pointColor);
+            fillCircle(defect.OuterLong.IntX, defect.OuterLong.IntY, 2, pointColor);
+                
+        }
+
         public void drawImage(Image<Gray, byte> image, Color color)
         {
             if (color.R != 0)

bbiwarg/InputHandler.cs

@@ -279,11 +279,14 @@ namespace bbiwarg
                 OutputImages[2].drawText(h.Centroid.IntX, h.Centroid.IntY, h.TrackID.ToString(), Constants.HandIDColor);
 
                 if (h.ThumbDefect != null)
-                {
-                    OutputImages[2].fillCircle(h.ThumbDefect.Inner.IntX, h.ThumbDefect.Inner.IntY, 4, Color.CornflowerBlue);
-                    OutputImages[2].fillCircle(h.ThumbDefect.OuterShort.IntX, h.ThumbDefect.OuterShort.IntY, 4, Color.CornflowerBlue);
-                    OutputImages[2].fillCircle(h.ThumbDefect.OuterLong.IntX, h.ThumbDefect.OuterLong.IntY, 4, Color.CornflowerBlue);
-                }
+                    OutputImages[2].drawDefect(h.ThumbDefect, Color.CornflowerBlue, Color.Yellow);        
+            }
+            foreach(Finger finger in fingerTracker.Fingers) {
+                Vector2D midOut1 = finger.SliceTrail.MidSlice.OutStart;
+                Vector2D midOut2 = finger.SliceTrail.MidSlice.OutEnd;
+                OutputImages[2].fillCircle(midOut1.IntX, midOut1.IntY, 3, Color.Yellow);
+                OutputImages[2].fillCircle(midOut2.IntX, midOut2.IntY, 3, Color.Yellow);
+
             }
 
             //image3

bbiwarg/Recognition/FingerRecognition/FingerDetector.cs

@@ -160,7 +160,7 @@ namespace bbiwarg.Recognition.FingerRecognition
         }
         private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction)
         {
-            Vector2D searchStart = start + Constants.FingerSliceOutMargin * direction;
+            Vector2D searchStart = start + Constants.FingerContourMargin * direction;
             Vector2D end = edgeImageAdapted.findNextEdge(searchStart, direction, Constants.FingerMaxWidth);
             if (end == null) 
                 return null;
@@ -179,9 +179,9 @@ namespace bbiwarg.Recognition.FingerRecognition
 
         private bool fingerSliceDepthTest(FingerSlice fingerSlice)
         {
-            Int16 depthStart = depthImage.getDepthAt(fingerSlice.StartOuter);
+            Int16 depthStart = depthImage.getDepthAt(fingerSlice.ContourStart);
             Int16 depthMid = depthImage.getDepthAt(fingerSlice.Mid);
-            Int16 depthEnd = depthImage.getDepthAt(fingerSlice.EndOuter);
+            Int16 depthEnd = depthImage.getDepthAt(fingerSlice.ContourEnd);
             return (depthStart > depthMid && depthMid < depthEnd);
         }
 
@@ -203,15 +203,9 @@ namespace bbiwarg.Recognition.FingerRecognition
 
         private bool isCrippleFinger(Finger finger)
         {
-            FingerSlice midSlice = finger.SliceTrail.MidSlice;
-            Vector2D direction = midSlice.Direction;
-            Vector2D directionInv = direction.getInverse();
-            Vector2D out1 = (midSlice.Start + Constants.FingerCrippleCheckMargin * directionInv).moveInBound(Vector2D.Zero, depthImage.BottomRight, direction);
-            Vector2D out2 = (midSlice.End + Constants.FingerCrippleCheckMargin * direction).moveInBound(Vector2D.Zero, depthImage.BottomRight, directionInv);
-
-            Int16 depthAtFinger = depthImage.getDepthAt(midSlice.Mid);
-            Int16 depthAtOut1 = depthImage.getDepthAt(out1);
-            Int16 depthAtOut2 = depthImage.getDepthAt(out2);
+            Int16 depthAtFinger = depthImage.getDepthAt(finger.MidPoint);
+            Int16 depthAtOut1 = depthImage.getDepthAt(finger.SliceTrail.MidSlice.OutStart);
+            Int16 depthAtOut2 = depthImage.getDepthAt(finger.SliceTrail.MidSlice.OutEnd);
             int minDepthDifference = Math.Min(Math.Abs(depthAtFinger - depthAtOut1), Math.Abs(depthAtFinger - depthAtOut2));
 
             return (minDepthDifference < Constants.FingerCrippleMinDifference);

bbiwarg/Recognition/FingerRecognition/FingerSlice.cs

@@ -14,9 +14,11 @@ namespace bbiwarg.Recognition.FingerRecognition
 
         public Vector2D Mid { get; private set; }
         public Vector2D Start { get; private set; }
-        public Vector2D StartOuter { get; private set; }
         public Vector2D End { get; private set; }
-        public Vector2D EndOuter { get; private set; }
+        public Vector2D ContourStart { get; private set; }
+        public Vector2D ContourEnd { get; private set; }
+        public Vector2D OutStart { get; private set; }
+        public Vector2D OutEnd { get; private set; }
         public LineSegment2D LineSegment { get { if (!lineSegmentUpToDate) updateLineSegment(); return lineSegment; } }
         public Vector2D Direction { get { return LineSegment.Direction; } }
         public float Length { get { return LineSegment.Length; } }
@@ -27,8 +29,10 @@ namespace bbiwarg.Recognition.FingerRecognition
             End = end;
             Mid = (start + end) / 2;
             updateLineSegment();
-            StartOuter = (Start + Constants.FingerSliceOutMargin * Direction.getInverse()).moveInBound(Vector2D.Zero, Constants.MaxPixel, Direction);
-            EndOuter = (End + Constants.FingerSliceOutMargin * Direction).moveInBound(Vector2D.Zero, Constants.MaxPixel, Direction.getInverse());
+            ContourStart = (Start + Constants.FingerContourMargin * Direction.getInverse()).moveInBound(Vector2D.Zero, Constants.MaxPixel, Direction);
+            ContourEnd = (End + Constants.FingerContourMargin * Direction).moveInBound(Vector2D.Zero, Constants.MaxPixel, Direction.getInverse());
+            OutStart = (Start + Constants.FingerOutMargin * Direction.getInverse()).moveInBound(Vector2D.Zero, Constants.MaxPixel, Direction);
+            OutEnd = (End + Constants.FingerOutMargin * Direction).moveInBound(Vector2D.Zero, Constants.MaxPixel, Direction.getInverse());
         }
 
         private void updateLineSegment()

bbiwarg/Recognition/FingerRecognition/FingerSliceTrail.cs

@@ -11,7 +11,6 @@ namespace bbiwarg.Recognition.FingerRecognition
 {
     class FingerSliceTrail
     {
-        private List<FingerSlice> slices;
         private LineSegment2D lineSegment;
         private Contour<Point> contour;
         private Contour<Point> innerContour;
@@ -19,18 +18,19 @@ namespace bbiwarg.Recognition.FingerRecognition
         private bool contourUpToDate;
         private bool innerContourUpToDate;
 
-        public FingerSlice StartSlice { get { return slices[0]; } }
-        public FingerSlice MidSlice { get { return slices[NumSlices / 2]; } }
-        public FingerSlice EndSlice { get { return slices[slices.Count - 1]; } }
-        public FingerSlice this[int index] { get { return slices[index]; } }
-        public int NumSlices { get { return slices.Count; } }
+        public List<FingerSlice> Slices { get; private set; }
+        public FingerSlice StartSlice { get { return Slices[0]; } }
+        public FingerSlice MidSlice { get { return Slices[NumSlices / 2]; } }
+        public FingerSlice EndSlice { get { return Slices[Slices.Count - 1]; } }
+        public FingerSlice this[int index] { get { return Slices[index]; } }
+        public int NumSlices { get { return Slices.Count; } }
         public LineSegment2D LineSegment { get { if (!lineSegmentUpToDate) updateLineSegment(); return lineSegment; } }
         public Contour<Point> Contour { get { if (!contourUpToDate) updateContour(); return contour; } }
         public Contour<Point> InnerContour { get { if (!innerContourUpToDate) updateInnerContour(); return innerContour; } }
 
         public FingerSliceTrail(FingerSlice slice)
         {
-            slices = new List<FingerSlice>();
+            Slices = new List<FingerSlice>();
             addSlice(slice);
             lineSegmentUpToDate = false;
             contourUpToDate = false;
@@ -38,7 +38,7 @@ namespace bbiwarg.Recognition.FingerRecognition
 
         public void addSlice(FingerSlice slice)
         {
-            slices.Add(slice);
+            Slices.Add(slice);
             lineSegmentUpToDate = false;
             contourUpToDate = false;
         }
@@ -46,25 +46,25 @@ namespace bbiwarg.Recognition.FingerRecognition
         public Vector2D getStartDirection()
         {
             int innerStartIndex = Math.Min(NumSlices, Constants.FingerNumSlicesForRelativeDirection);
-            return (StartSlice.Mid - slices[innerStartIndex].Mid).normalize();
+            return (StartSlice.Mid - Slices[innerStartIndex].Mid).normalize();
         }
 
         public Vector2D getEndDirection()
         {
             int innerEndIndex = Math.Max(0, NumSlices - Constants.FingerNumSlicesForRelativeDirection);
-            return (EndSlice.Mid - slices[innerEndIndex].Mid).normalize();
+            return (EndSlice.Mid - Slices[innerEndIndex].Mid).normalize();
         }
 
         public void removeFirstSlices(int numSlices)
         {
-            slices.RemoveRange(0, numSlices);
+            Slices.RemoveRange(0, numSlices);
             lineSegmentUpToDate = false;
             contourUpToDate = false;
         }
 
         public void reverse()
         {
-            slices.Reverse();
+            Slices.Reverse();
             lineSegmentUpToDate = false;
             contourUpToDate = false;
         }
@@ -80,10 +80,10 @@ namespace bbiwarg.Recognition.FingerRecognition
             List<Point> pointsA = new List<Point>();
             List<Point> pointsB = new List<Point>();
 
-            foreach (FingerSlice slice in slices)
+            foreach (FingerSlice slice in Slices)
             {
-                pointsA.Add(slice.StartOuter);
-                pointsB.Add(slice.EndOuter);
+                pointsA.Add(slice.ContourStart);
+                pointsB.Add(slice.ContourEnd);
             }
 
             pointsA.Reverse();
@@ -102,7 +102,7 @@ namespace bbiwarg.Recognition.FingerRecognition
             List<Point> pointsA = new List<Point>();
             List<Point> pointsB = new List<Point>();
 
-            foreach (FingerSlice slice in slices)
+            foreach (FingerSlice slice in Slices)
             {
                 pointsA.Add(slice.Start);
                 pointsB.Add(slice.End);

bbiwarg/Recognition/HandRecognition/Hand.cs

@@ -21,8 +21,7 @@ namespace bbiwarg.Recognition.HandRecognition
 
     class Hand : TrackableObject
     {
-        private List<ConvexityDefect> convexityDefects;
-
+        public List<ConvexityDefect> ConvexityDefects { get; private set; }
         public int ZIndex { get; private set; }
         public Vector2D Centroid { get; private set; }
         public Vector2D CentroidInHand { get; private set; }
@@ -56,6 +55,34 @@ namespace bbiwarg.Recognition.HandRecognition
             ZIndex = zIndex;
         }
 
+        public void mergeWith(Hand mergeHand) {
+            extendMask(mergeHand.Mask);
+            foreach (Finger f2 in mergeHand.Fingers)
+            {
+                addFinger(f2);
+                f2.setHand(this);
+            }
+            findCentroids();
+        }
+
+        public void extendMask(Image<Gray, byte> extendMask) {
+            Mask = Mask.Or(extendMask);
+            findCentroids();
+        }
+
+        public void fixDefectsFromOverlappingFingers(List<Finger> otherFingers)
+        {
+            foreach (Finger finger in otherFingers) {
+                foreach (FingerSlice slice in finger.SliceTrail.Slices) {
+                    Vector2D out1 = slice.OutStart;
+                    Vector2D out2 = slice.OutEnd;
+
+                    if (isInside(out1) && isInside(out2))
+                        Mask.Draw(new Emgu.CV.Structure.LineSegment2D(out1, out2), new Gray(1), 1);
+                }
+            }
+        }
+
         public void findThumbDefect()
         {
             if (isThumbOnly())
@@ -81,20 +108,20 @@ namespace bbiwarg.Recognition.HandRecognition
             Contour<Point> contour = Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
             List<MCvConvexityDefect> mcvConvexityDefects = new List<MCvConvexityDefect>(contour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
             
-            convexityDefects = new List<ConvexityDefect>();
+            ConvexityDefects = new List<ConvexityDefect>();
             foreach (MCvConvexityDefect defect in mcvConvexityDefects)
-                convexityDefects.Add(new ConvexityDefect(defect));
+                ConvexityDefects.Add(new ConvexityDefect(defect));
         }
 
         private void filterThumbDefect()
         {
-            if (convexityDefects.Count > 0)
+            if (ConvexityDefects.Count > 0)
             {
-                convexityDefects.Sort((cd1, cd2) => (cd2.Depth.CompareTo(cd1.Depth)));
+                ConvexityDefects.Sort((cd1, cd2) => (cd2.Depth.CompareTo(cd1.Depth)));
 
                 Finger thumb = Fingers[0];
 
-                foreach (ConvexityDefect defect in convexityDefects)
+                foreach (ConvexityDefect defect in ConvexityDefects)
                 {
                     if (defect.isPossibleThumbDefect(thumb))
                     {

bbiwarg/Recognition/HandRecognition/HandDetector.cs

@@ -21,6 +21,7 @@ namespace bbiwarg.Recognition.HandRecognition
         private EdgeImage edgeImage;
         private Image<Gray, byte> modifiedHandDepthImage;
         private List<Finger> fingers;
+        private List<Hand> ignoreMergedHands;
 
         public List<Hand> Hands { get; private set; }
         public Image<Gray, byte> HandMask { get; private set; }
@@ -33,6 +34,7 @@ namespace bbiwarg.Recognition.HandRecognition
 
             createModifiedHandEdgeImage();
             findHands();
+            fixOverlappingFingers();
             setZIndexes();
             createHandMask();
             findThumbDefects();
@@ -72,13 +74,10 @@ namespace bbiwarg.Recognition.HandRecognition
 
                 if (newHand)
                 {
-                    Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
-                    MCvConnectedComp comp = new MCvConnectedComp();
-                    CvInvoke.cvFloodFill(modifiedHandDepthImage, finger.HandPoint, new MCvScalar(255), new MCvScalar(Constants.HandFloodFillDownDiff), new MCvScalar(Constants.HandFloodFillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
-                    if (comp.area < maxArea * Constants.HandMaxSize)
+                    Image<Gray, byte> handMask = getHandMask(finger.HandPoint);
+                    if (handMask.CountNonzero()[0] < maxArea * Constants.HandMaxSize)
                     {
-                        Image<Gray, byte> cropedMask = mask.Copy(new Rectangle(1, 1, width, height));
-                        Hand hand = new Hand(cropedMask, finger);
+                        Hand hand = new Hand(handMask, finger);
                         Hands.Add(hand);
                         finger.setHand(hand);
                     }
@@ -86,6 +85,16 @@ namespace bbiwarg.Recognition.HandRecognition
             }
         }
 
+        private Image<Gray, byte> getHandMask(Vector2D p)
+        {
+            int width = edgeImage.Width;
+            int height = edgeImage.Height;
+            Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
+            MCvConnectedComp comp = new MCvConnectedComp();
+            CvInvoke.cvFloodFill(modifiedHandDepthImage, p, new MCvScalar(255), new MCvScalar(Constants.HandFloodFillDownDiff), new MCvScalar(Constants.HandFloodFillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
+            return mask.Copy(new Rectangle(1, 1, width, height));
+        }
+
         private void setZIndexes()
         {
             //sort depending on depth of centroid (far->near)
@@ -95,6 +104,18 @@ namespace bbiwarg.Recognition.HandRecognition
                 Hands[i].setZIndex(i);
         }
 
+        private void fixOverlappingFingers()
+        {
+            extendOrMergeThroughOverlappingFingers();
+            fixDefectsFromOverlappingFingers();
+        }
+
+        private void findThumbDefects()
+        {
+            foreach (Hand hand in Hands)
+                hand.findThumbDefect();
+        }
+
         private void createHandMask()
         {
             HandMask = new Image<Gray, byte>(depthImage.Width, depthImage.Height);
@@ -102,10 +123,84 @@ namespace bbiwarg.Recognition.HandRecognition
                 HandMask = HandMask.Or(hand.Mask);
         }
 
-        private void findThumbDefects()
+        private void extendOrMergeThroughOverlappingFingers()
+        {
+            Dictionary<Hand, Hand> mergeHands = new Dictionary<Hand, Hand>();
+
+            foreach (Hand overlappingHand in Hands)
+            {
+                foreach (Hand underlyingHand in Hands)
+                {
+                    if (!mergeHands.Keys.Contains(underlyingHand))
+                    {
+                        foreach (Finger overlappingFinger in overlappingHand.Fingers)
+                        {
+                            Vector2D midOut1 = overlappingFinger.SliceTrail.MidSlice.OutStart;
+                            Vector2D midOut2 = overlappingFinger.SliceTrail.MidSlice.OutEnd;
+
+                            Int16 depthAtMidOut1 = depthImage.getDepthAt(midOut1);
+                            Int16 depthAtMidOut2 = depthImage.getDepthAt(midOut2);
+
+                            bool midOut1InHand = underlyingHand.isInside(midOut1);
+                            bool midOut2InHand = underlyingHand.isInside(midOut2);
+
+                            Int16 maxDepth = depthImage.MaxDepth;
+                            if (midOut1InHand != midOut2InHand && depthAtMidOut1 != maxDepth && depthAtMidOut2 != maxDepth && Math.Abs(depthAtMidOut1 - depthAtMidOut2) < Constants.HandExtendMaxDifference)
+                            {
+                                Vector2D p1, p2;
+                                if (midOut1InHand)
+                                {
+                                    p1 = midOut1;
+                                    p2 = midOut2;
+                                }
+                                else
+                                {
+                                    p1 = midOut2;
+                                    p2 = midOut1;
+                                }
+
+                                //check if p2 is in other hand (if so -> merge)
+                                bool merge = false;
+                                foreach (Hand hand in Hands)
+                                {
+                                    if (hand.isInside(p2) && !ignoreMergedHands.Contains(hand))
+                                    {
+                                        mergeHands.Add(underlyingHand, overlappingHand);
+                                        merge = true;
+                                        break;
+                                    }
+                                }
+
+                                //if no merge, extend hand
+                                if (!merge)
+                                    underlyingHand.extendMask(getHandMask(p2));
+                            }
+                        }
+                    }
+                }
+            }
+
+            foreach (Hand removeHand in mergeHands.Keys)
+            {
+                mergeHands[removeHand].mergeWith(removeHand);
+                Hands.Remove(removeHand);
+            }
+        }
+
+        private void fixDefectsFromOverlappingFingers()
         {
             foreach (Hand hand in Hands)
-                hand.findThumbDefect();
+            {
+                //get other hands fingers
+                List<Finger> otherFingers = new List<Finger>();
+                foreach (Hand otherHand in Hands)
+                {
+                    if (hand != otherHand)
+                        otherFingers.AddRange(otherHand.Fingers);
+                }
+
+                hand.fixDefectsFromOverlappingFingers(otherFingers);
+            }
         }
     }
 }

bbiwarg/Utility/ConvexityDefect.cs

@@ -50,5 +50,9 @@ namespace bbiwarg.Utility
                     thumbShortLengthRatio <= Constants.HandThumbDefectMaxThumbShortLengthRatio && thumbShortLengthRatio >= Constants.HandThumbDefectMinThumbShortLengthRatio &&
                     shortLongLengthRatio <= Constants.HandThumbDefectMaxShortLongLengthRatio && shortLongLengthRatio >= Constants.HandThumbDefectMinShortLongLengthRatio);
         }
+
+        public bool isCausedByFinger(Finger finger) {
+            return (OuterLineSegment.intersectsWith(finger.LineSegment));
+        }
     }
 }