using bbiwarg.Images; using bbiwarg.Input.InputHandling; using bbiwarg.Recognition.FingerRecognition; using bbiwarg.Recognition.HandRecognition; using bbiwarg.Utility; using Emgu.CV; using Emgu.CV.Structure; using System; using System.Collections.Generic; using System.Drawing; namespace bbiwarg.Recognition.PalmRecognition { /// /// Detects palms by iterating over each hand, if the hand has exactly one finger (possible thumb) it calculates its convexity defects and checks if there is a convexity defect that matches the requirements for a thumb defect. If a thumb defect is found, the four palm points are generated and a new palm is created. /// internal class PalmDetector { /// /// the depth image of the current frame /// private DepthImage depthImage; /// /// the hands in the current frame /// private List hands; /// /// the palms in the current frames /// private List palms; /// /// Detects palms in the current frame and stores them in frameData.detectedPalms /// /// the current frame public void detectPalms(FrameData frameData) { depthImage = frameData.DepthImage; hands = frameData.TrackedHands; findPalms(); frameData.DetectedPalms = palms; } /// /// Finds the four palm points and creates a new palm /// /// the hand of the palm /// the convexity defect of the thumb /// a new palm private Palm createPalm(Hand hand, ConvexityDefect thumbDefect) { HandSide side = determineHandSide(thumbDefect); Vector2D wristUpper = findWristUpper(hand, thumbDefect); Vector2D fingersUpper = findFingersUpper(hand, thumbDefect, side); float palmWidth = findPalmWidth(hand, thumbDefect, side); float palmLength = wristUpper.getDistanceTo(fingersUpper); Vector2D directionWristFingers = thumbDefect.VectorLong.normalize(); Vector2D directionUpperLower = thumbDefect.VectorLong.getOrthogonal(side == HandSide.Right).normalize(); Vector2D wristLower = wristUpper + palmWidth * directionUpperLower; Vector2D fingersLower = wristUpper + 0.75f * palmLength * directionWristFingers + 0.75f * palmWidth * directionUpperLower; return new Palm(hand, thumbDefect, side, wristUpper, fingersUpper, fingersLower, wristLower); } /// /// Determines the handedness of the palm's hand /// /// the convexity defect of the thumb /// the handedness of the palm's hand private HandSide determineHandSide(ConvexityDefect thumbDefect) { if (thumbDefect.VectorShort.crossProduct(thumbDefect.VectorLong) < 0) return HandSide.Right; else return HandSide.Left; } /// /// Gets the convexity Defects from a hand mask. /// /// the hand /// the conveixty defects private List findConvexityDefects(Hand hand) { List convexityDefects; using (MemStorage ms = new MemStorage()) { Contour contour = hand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); List mcvConvexityDefects = new List(contour.GetConvexityDefacts(ms, Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects = new List(); foreach (MCvConvexityDefect defect in mcvConvexityDefects) convexityDefects.Add(new ConvexityDefect(defect)); } return convexityDefects; } /// /// Finds the upper finger end by walking from the defect.OuterLong in the finger direction, until it reaches the end of the middle finger below. /// /// the hand /// the convexity defect of the thumb /// the handedness /// the position of the upper fingers end private Vector2D findFingersUpper(Hand hand, ConvexityDefect thumbDefect, HandSide side) { Vector2D fingersDirection = thumbDefect.VectorLong.normalize(); Vector2D lowerDirection = fingersDirection.getOrthogonal(side == HandSide.Right).normalize(); Vector2D fingersUpper = thumbDefect.OuterLong; Vector2D fingersUpperNext = fingersUpper + fingersDirection; bool handBelow = true; while (handBelow && fingersUpperNext.isInBound(depthImage.Size)) { fingersUpper = fingersUpperNext; fingersUpperNext += fingersDirection; Vector2D below = fingersUpper.copy(); handBelow = false; int distance = 0; while (!handBelow && distance < Parameters.FingerMaxWidth2D && below.isInBound(depthImage.Size)) { handBelow = hand.isInside(below); below += lowerDirection; } } return fingersUpper; } /// /// Find Palms by checking each hand with only one finger, if that finger is a thumb. /// private void findPalms() { palms = new List(); foreach (Hand hand in hands) { if (hand.Fingers.Count == 1) { List convexityDefects = findConvexityDefects(hand); ConvexityDefect thumbDefect = findThumbDefect(hand.Fingers[0], convexityDefects); if (thumbDefect != null) { Palm palm = createPalm(hand, thumbDefect); palms.Add(palm); } } } } /// /// Gets the palm width by checking for the maximum orthogonal length withing the hand along multiple positions on the index finger (defect.inner<->defect.outerLong) /// /// the hand /// the convexity defect of the thumb /// the handedness /// the palm width private float findPalmWidth(Hand hand, ConvexityDefect thumbDefect, HandSide side) { Vector2D lowerDirection = thumbDefect.VectorLong.getOrthogonal(side == HandSide.Right).normalize(); Vector2D current = thumbDefect.Inner; Vector2D step = thumbDefect.VectorLong / Parameters.PalmNumPositionsForPalmWidth; float maxWidth = float.MinValue; for (int i = 0; i < Parameters.PalmNumPositionsForPalmWidth; i++) { Vector2D lower = current; Vector2D lowerNext = lower + 10 * lowerDirection; while (lowerNext.isInBound(depthImage.Size) && hand.isInside(lowerNext)) { lower = lowerNext; lowerNext += lowerDirection; } maxWidth = Math.Max(maxWidth, current.getDistanceTo(lower)); current += step; } return maxWidth; } /// /// Checks all convexity defects sorted by length if they match the thumb defect requirements and returns the first match or null if none match. /// /// the possible thumb /// the convexity defects /// private ConvexityDefect findThumbDefect(Finger thumb, List convexityDefects) { convexityDefects.Sort((cd1, cd2) => (cd2.Depth.CompareTo(cd1.Depth))); foreach (ConvexityDefect defect in convexityDefects) { if (defect.isPossibleThumbDefect(thumb)) return defect; } return null; } /// /// Finds the upper wrist end by walking from the defect.Inner towards the wrist direction, until it reaches the hand boundaries. /// /// the hand /// the convexity defect of the thumb /// the position of the upper wrist end private Vector2D findWristUpper(Hand hand, ConvexityDefect thumbDefect) { Vector2D wristDirection = thumbDefect.VectorLong.getInverse().normalize(); Vector2D wristUpper = thumbDefect.Inner; Vector2D wristUpperNext = wristUpper + 5 * wristDirection; while (wristUpperNext.isInBound(depthImage.Size) && hand.isInside(wristUpperNext)) { wristUpper = wristUpperNext; wristUpperNext += wristDirection; } return wristUpper; } } }