fixed hand merging

bbiwarg/Parameters.cs

@@ -23,7 +23,7 @@ namespace bbiwarg
         public static readonly String InputMoviePath = "..\\..\\videos\\touch\\4.skv";
 
         // Logger
-        public static readonly LogSubject LogLevel = LogSubject.Timer;
+        public static readonly LogSubject LogLevel = LogSubject.None;
         public static readonly int ConsoleWidth = 90;
         public static readonly int ConsoleHeight = 30;
 
@@ -77,8 +77,8 @@ namespace bbiwarg
         // finger tracking
         public static readonly int FingerTrackerNumFramesDetectedUntilTracked = 5;
         public static readonly int FingerTrackerNumFramesLostUntilDeleted = 10;
-        public static readonly float FingerTrackerMaxTipPointMove = 0.25f*ImageDiagonalLength;
-        public static readonly float FingerTrackerMaxHandPointMove = 0.25f*ImageDiagonalLength;
+        public static readonly float FingerTrackerMaxTipPointMove = 0.25f * ImageDiagonalLength;
+        public static readonly float FingerTrackerMaxHandPointMove = 0.25f * ImageDiagonalLength;
         public static readonly float FingermXX = 0.00005f;
         public static readonly float FingermXY = 0.0f;
         public static readonly float FingermYY = 0.00005f;
@@ -91,15 +91,15 @@ namespace bbiwarg
         public static readonly float HandMaxSize = 0.6f;
         public static readonly float HandMinSize = 0.01f;
         public static readonly float HandExtensionMaxSize = 0.5f * HandMaxSize;
+        public static readonly int HandExtendMaxDifference = 40;
         public static readonly float HandThumbDefectMaxDistanceToThumb = FingerMaxWidth;
         public static readonly float HandThumbDefectMinThumbShortLengthRatio = 0.75f;
         public static readonly float HandThumbDefectMaxThumbShortLengthRatio = 1.1f;
         public static readonly float HandThumbDefectMinShortLongLengthRatio = 0.3f;
         public static readonly float HandThumbDefectMaxShortLongLengthRatio = 0.7f;
-        public static readonly int HandExtendMaxDifference = 40;
 
         // hand tracker
-        public static readonly int HandTrackerNumFramesDetectedUntilTracked = 2;
+        public static readonly int HandTrackerNumFramesDetectedUntilTracked = 5;
         public static readonly int HandTrackerNumFramesLostUntilDeleted = 5;
         public static readonly float HandTrackerMaxCentroidMove = 0.25f * ImageDiagonalLength;
         public static readonly float HandmXX = 0.0005f;

bbiwarg/Recognition/HandRecognition/HandDetector.cs

@@ -21,6 +21,7 @@ namespace bbiwarg.Recognition.HandRecognition
         private EdgeImage edgeImage;
         private Image<Gray, byte> modifiedHandDepthImage;
         private List<Finger> fingers;
+        private Dictionary<Hand, List<Finger>> otherHandsFingers;
 
         public List<Hand> Hands { get; private set; }
         public Image<Gray, byte> HandMask { get; private set; }
@@ -33,6 +34,7 @@ namespace bbiwarg.Recognition.HandRecognition
 
             createModifiedHandEdgeImage();
             findHands();
+            findOtherHandsFingers();
             fixOverlappingFingers();
             findThumbDefects();
             createHandMask();
@@ -82,6 +84,21 @@ namespace bbiwarg.Recognition.HandRecognition
             }
         }
 
+        private void findOtherHandsFingers()
+        {
+            otherHandsFingers = new Dictionary<Hand, List<Finger>>();
+            foreach (Hand hand in Hands)
+            {
+                List<Finger> otherFingers = new List<Finger>();
+                foreach (Finger finger in fingers)
+                {
+                    if (!hand.Fingers.Contains(finger))
+                        otherFingers.Add(finger);
+                }
+                otherHandsFingers.Add(hand, otherFingers);
+            }
+        }
+
         private Image<Gray, byte> getHandMask(Vector2D p)
         {
             Image<Gray, byte> mask = new Image<Gray, byte>(Parameters.ImageWidth + 2, Parameters.ImageHeight + 2);
@@ -111,73 +128,79 @@ namespace bbiwarg.Recognition.HandRecognition
 
         private void extendOrMergeThroughOverlappingFingers()
         {
-            Dictionary<Hand, Hand> mergeHands = new Dictionary<Hand, Hand>();
+            List<Hand> mergedHands = new List<Hand>();
 
-            foreach (Hand overlappingHand in Hands)
+            foreach (Hand hand in Hands)
             {
-                foreach (Hand underlyingHand in Hands)
+                if (!mergedHands.Contains(hand))
                 {
-                    if (!mergeHands.Keys.Contains(underlyingHand))
+                    List<Hand> mergeHands = new List<Hand>();
+                    foreach (Finger overlappingFinger in otherHandsFingers[hand])
                     {
-                        foreach (Finger overlappingFinger in overlappingHand.Fingers)
+                        Vector2D midOut1 = overlappingFinger.SliceTrail.MidSlice.OutStart;
+                        Vector2D midOut2 = overlappingFinger.SliceTrail.MidSlice.OutEnd;
+                        Int16 depthAtMidOut1 = depthImage.getDepthAt(midOut1);
+                        Int16 depthAtMidOut2 = depthImage.getDepthAt(midOut2);
+                        bool midOut1InHand = hand.isInside(midOut1);
+                        bool midOut2InHand = hand.isInside(midOut2);
+
+                        Int16 maxDepth = depthImage.MaxDepth;
+                        if (midOut1InHand != midOut2InHand && depthAtMidOut1 != maxDepth && depthAtMidOut2 != maxDepth && Math.Abs(depthAtMidOut1 - depthAtMidOut2) < Parameters.HandExtendMaxDifference)
                         {
-                            Vector2D midOut1 = overlappingFinger.SliceTrail.MidSlice.OutStart;
-                            Vector2D midOut2 = overlappingFinger.SliceTrail.MidSlice.OutEnd;
-
-                            Int16 depthAtMidOut1 = depthImage.getDepthAt(midOut1);
-                            Int16 depthAtMidOut2 = depthImage.getDepthAt(midOut2);
-
-                            bool midOut1InHand = underlyingHand.isInside(midOut1);
-                            bool midOut2InHand = underlyingHand.isInside(midOut2);
-
-                            Int16 maxDepth = depthImage.MaxDepth;
-                            if (midOut1InHand != midOut2InHand && depthAtMidOut1 != maxDepth && depthAtMidOut2 != maxDepth && Math.Abs(depthAtMidOut1 - depthAtMidOut2) < Parameters.HandExtendMaxDifference)
+                            Vector2D pHand, pHandExtension;
+                            if (midOut1InHand)
                             {
-                                Vector2D p1, p2;
-                                if (midOut1InHand)
-                                {
-                                    p1 = midOut1;
-                                    p2 = midOut2;
-                                }
-                                else
-                                {
-                                    p1 = midOut2;
-                                    p2 = midOut1;
-                                }
+                                pHand = midOut1;
+                                pHandExtension = midOut2;
+                            }
+                            else
+                            {
+                                pHand = midOut2;
+                                pHandExtension = midOut1;
+                            }
 
-                                //check if p2 is in other hand (if so -> merge)
-                                bool merge = false;
-                                foreach (Hand hand in Hands)
+                            //check if pHandExtension is in other hand (if so -> merge with hand)
+                            bool merge = false;
+                            foreach (Hand mergeHand in Hands)
+                            {
+                                if (mergeHand.isInside(pHandExtension) && !mergedHands.Contains(mergeHand))
                                 {
-                                    if (hand.isInside(p2) && !mergeHands.Keys.Contains(hand))
-                                    {
-                                        mergeHands.Add(underlyingHand, overlappingHand);
-                                        merge = true;
-                                        break;
-                                    }
+                                    mergeHands.Add(mergeHand);
+                                    merge = true;
+                                    break;
                                 }
-
-                                //if no merge, extend hand
-                                if (!merge)
-                                    extendHand(underlyingHand, p2);
                             }
+
+                            //if no merge, extend hand
+                            if (!merge)
+                                extendToHand(hand, pHandExtension);
                         }
                     }
+                    foreach (Hand mergeHand in mergeHands)
+                    {
+                        mergeToHand(hand, mergeHand);
+                        mergedHands.Add(mergeHand);
+                    }
                 }
             }
+            foreach (Hand mergedHand in mergedHands)
+                Hands.Remove(mergedHand);
+        }
 
-            foreach (Hand removeHand in mergeHands.Keys)
-            {
-                mergeHands[removeHand].mergeWith(removeHand);
-                Hands.Remove(removeHand);
-            }
+        private void mergeToHand(Hand hand, Hand mergeHand)
+        {
+            hand.mergeWith(mergeHand);
+
+            foreach (Finger finger in mergeHand.Fingers)
+                otherHandsFingers[hand].Remove(finger);
         }
 
-        private void extendHand(Hand hand, Vector2D p) {
+        private void extendToHand(Hand hand, Vector2D p)
+        {
             Image<Gray, byte> extendMask = getHandMask(p);
             int numPixels = extendMask.CountNonzero()[0];
 
-            if(numPixels <= Parameters.HandExtensionMaxSize * Parameters.ImageNumPixels)
+            if (numPixels <= Parameters.HandExtensionMaxSize * Parameters.ImageNumPixels)
                 hand.extendMask(extendMask);
         }
 
@@ -185,15 +208,7 @@ namespace bbiwarg.Recognition.HandRecognition
         {
             foreach (Hand hand in Hands)
             {
-                //get other hands fingers
-                List<Finger> otherFingers = new List<Finger>();
-                foreach (Finger finger in fingers)
-                {
-                    if (!hand.Fingers.Contains(finger))
-                        otherFingers.Add(finger);
-                }
-
-                hand.fillOverlappingFingers(otherFingers);
+                hand.fillOverlappingFingers(otherHandsFingers[hand]);
             }
         }
     }