using System; using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Text; using System.Threading.Tasks; using bbiwarg.Images; using bbiwarg.Recognition.FingerRecognition; using bbiwarg.Graphics; using bbiwarg.Utility; using Emgu.CV; using Emgu.CV.Structure; namespace bbiwarg.Recognition.HandRecognition { class HandDetector { private DepthImage depthImage; private EdgeImage edgeImage; private Image modifiedHandDepthImage; private List fingers; public List Hands { get; private set; } public Image HandMask { get; private set; } public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List fingers) { this.depthImage = depthImage; this.edgeImage = edgeImage; this.fingers = fingers; createModifiedHandEdgeImage(); findHands(); fixOverlappingFingers(); setZIndexes(); createHandMask(); findThumbDefects(); } private void createModifiedHandEdgeImage() { modifiedHandDepthImage = depthImage.Image.Copy(); foreach (Finger finger in fingers) { Point[] contour = finger.InnerContour.ToArray(); modifiedHandDepthImage.DrawPolyline(contour, false, new Gray(0), 2); } } private void findHands() { Hands = new List(); foreach (Finger finger in fingers) { bool newHand = true; foreach (Hand hand in Hands) { if (hand.isInside(finger.HandPoint)) { hand.addFinger(finger); finger.setHand(hand); newHand = false; break; } } if (newHand) { Image handMask = getHandMask(finger.HandPoint); int numPixels = handMask.CountNonzero()[0]; if (numPixels < Parameters.HandMaxSize * Parameters.ImageNumPixels && numPixels > Parameters.HandMinSize * Parameters.ImageNumPixels) { Hand hand = new Hand(handMask, finger); Hands.Add(hand); finger.setHand(hand); } } } } private Image getHandMask(Vector2D p) { Image mask = new Image(Parameters.ImageWidth + 2, Parameters.ImageHeight + 2); MCvConnectedComp comp = new MCvConnectedComp(); CvInvoke.cvFloodFill(modifiedHandDepthImage, p, new MCvScalar(255), new MCvScalar(Parameters.HandFloodFillDownDiff), new MCvScalar(Parameters.HandFloodFillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask); return mask.Copy(new Rectangle(1, 1, Parameters.ImageWidth, Parameters.ImageHeight)); } private void setZIndexes() { //sort depending on depth of centroid (far->near) Hands.Sort((h1, h2) => depthImage.getDepthAt(h2.CentroidInHand).CompareTo(depthImage.getDepthAt(h1.CentroidInHand))); for (int i = 0; i < Hands.Count; i++) Hands[i].setZIndex(i); } private void fixOverlappingFingers() { extendOrMergeThroughOverlappingFingers(); fillOverlappingFingers(); } private void findThumbDefects() { foreach (Hand hand in Hands) hand.findThumbDefect(); } private void createHandMask() { HandMask = new Image(Parameters.ImageWidth, Parameters.ImageHeight); foreach (Hand hand in Hands) HandMask = HandMask.Or(hand.Mask); } private void extendOrMergeThroughOverlappingFingers() { Dictionary mergeHands = new Dictionary(); foreach (Hand overlappingHand in Hands) { foreach (Hand underlyingHand in Hands) { if (!mergeHands.Keys.Contains(underlyingHand)) { foreach (Finger overlappingFinger in overlappingHand.Fingers) { Vector2D midOut1 = overlappingFinger.SliceTrail.MidSlice.OutStart; Vector2D midOut2 = overlappingFinger.SliceTrail.MidSlice.OutEnd; Int16 depthAtMidOut1 = depthImage.getDepthAt(midOut1); Int16 depthAtMidOut2 = depthImage.getDepthAt(midOut2); bool midOut1InHand = underlyingHand.isInside(midOut1); bool midOut2InHand = underlyingHand.isInside(midOut2); Int16 maxDepth = depthImage.MaxDepth; if (midOut1InHand != midOut2InHand && depthAtMidOut1 != maxDepth && depthAtMidOut2 != maxDepth && Math.Abs(depthAtMidOut1 - depthAtMidOut2) < Parameters.HandExtendMaxDifference) { Vector2D p1, p2; if (midOut1InHand) { p1 = midOut1; p2 = midOut2; } else { p1 = midOut2; p2 = midOut1; } //check if p2 is in other hand (if so -> merge) bool merge = false; foreach (Hand hand in Hands) { if (hand.isInside(p2) && !mergeHands.Keys.Contains(hand)) { mergeHands.Add(underlyingHand, overlappingHand); merge = true; break; } } //if no merge, extend hand if (!merge) extendHand(underlyingHand, p2); } } } } } foreach (Hand removeHand in mergeHands.Keys) { mergeHands[removeHand].mergeWith(removeHand); Hands.Remove(removeHand); } } private void extendHand(Hand hand, Vector2D p) { Image extendMask = getHandMask(p); int numPixels = extendMask.CountNonzero()[0]; if(numPixels <= Parameters.HandExtensionMaxSize * Parameters.ImageNumPixels) hand.extendMask(extendMask); } private void fillOverlappingFingers() { foreach (Hand hand in Hands) { //get other hands fingers List otherFingers = new List(); foreach (Finger finger in fingers) { if (!hand.Fingers.Contains(finger)) otherFingers.Add(finger); } hand.fillOverlappingFingers(otherFingers); } } } }