using BBIWARG.Images;
using BBIWARG.Input.InputHandling;
using BBIWARG.Recognition.FingerRecognition;
using BBIWARG.Recognition.HandRecognition;
using BBIWARG.Utility;
using Emgu.CV;
using Emgu.CV.Structure;
using System;
using System.Collections.Generic;
using System.Drawing;
namespace BBIWARG.Recognition.PalmRecognition
{
///
/// Detects palms by iterating over each hand, if the hand has exactly one finger (possible thumb) it calculates its convexity defects and checks if there is a convexity defect that matches the requirements for a thumb defect. If a thumb defect is found, the four palm points are generated and a new palm is created.
///
internal class PalmDetector
{
///
/// the depth image of the current frame
///
private DepthImage depthImage;
///
/// the hands in the current frame
///
private List hands;
///
/// the palms in the current frames
///
private List palms;
///
/// Detects palms in the current frame and stores them in frameData.detectedPalms
///
/// the current frame
public void detectPalms(FrameData frameData)
{
depthImage = frameData.DepthImage;
hands = frameData.TrackedHands;
findPalms();
frameData.DetectedPalms = palms;
}
///
/// Finds the four palm points and creates a new palm
///
/// the hand of the palm
/// the convexity defect of the thumb
/// a new palm
private Palm createPalm(Hand hand, ConvexityDefect thumbDefect)
{
HandSide side = determineHandSide(thumbDefect);
Vector2D wristUpper = findWristUpper(hand, thumbDefect);
Vector2D fingersUpper = findFingersUpper(hand, thumbDefect, side);
float palmWidth = findPalmWidth(hand, thumbDefect, side);
float palmLength = wristUpper.getDistanceTo(fingersUpper);
Vector2D directionWristFingers = thumbDefect.VectorLong.normalize();
Vector2D directionUpperLower = thumbDefect.VectorLong.getOrthogonal(side == HandSide.Right).normalize();
Vector2D wristLower = wristUpper + palmWidth * directionUpperLower;
Vector2D fingersLower = wristUpper + 0.75f * palmLength * directionWristFingers + 0.75f * palmWidth * directionUpperLower;
return new Palm(hand, thumbDefect, side, wristUpper, fingersUpper, fingersLower, wristLower);
}
///
/// Determines the handedness of the palm's hand
///
/// the convexity defect of the thumb
/// the handedness of the palm's hand
private HandSide determineHandSide(ConvexityDefect thumbDefect)
{
if (thumbDefect.VectorShort.crossProduct(thumbDefect.VectorLong) < 0)
return HandSide.Right;
else
return HandSide.Left;
}
///
/// Gets the convexity Defects from a hand mask.
///
/// the hand
/// the convexity defects
private List findConvexityDefects(Hand hand)
{
List convexityDefects;
using (MemStorage ms = new MemStorage())
{
Contour contour = hand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
List mcvConvexityDefects = new List(contour.GetConvexityDefacts(ms, Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
convexityDefects = new List();
foreach (MCvConvexityDefect defect in mcvConvexityDefects)
convexityDefects.Add(new ConvexityDefect(defect));
}
return convexityDefects;
}
///
/// Finds the upper finger end by walking from the defect.OuterLong in the finger direction, until it reaches the end of the middle finger below.
///
/// the hand
/// the convexity defect of the thumb
/// the handedness
/// the position of the upper fingers end
private Vector2D findFingersUpper(Hand hand, ConvexityDefect thumbDefect, HandSide side)
{
Vector2D fingersDirection = thumbDefect.VectorLong.normalize();
Vector2D lowerDirection = fingersDirection.getOrthogonal(side == HandSide.Right).normalize();
Vector2D fingersUpper = thumbDefect.OuterLong;
Vector2D fingersUpperNext = fingersUpper + fingersDirection;
bool handBelow = true;
while (handBelow && fingersUpperNext.isInBound(depthImage.Size))
{
fingersUpper = fingersUpperNext;
fingersUpperNext += fingersDirection;
Vector2D below = fingersUpper.copy();
handBelow = false;
int distance = 0;
while (!handBelow && distance < Parameters.FingerMaxWidth2D && below.isInBound(depthImage.Size))
{
handBelow = hand.isInside(below);
below += lowerDirection;
}
}
return fingersUpper;
}
///
/// Find Palms by checking each hand with only one finger, if that finger is a thumb.
///
private void findPalms()
{
palms = new List();
foreach (Hand hand in hands)
{
if (hand.Fingers.Count == 1)
{
List convexityDefects = findConvexityDefects(hand);
ConvexityDefect thumbDefect = findThumbDefect(hand.Fingers[0], convexityDefects);
if (thumbDefect != null)
{
Palm palm = createPalm(hand, thumbDefect);
palms.Add(palm);
}
}
}
}
///
/// Gets the palm width by checking for the maximum orthogonal length within the hand along multiple positions on the index finger (defect.inner to defect.outerLong)
///
/// the hand
/// the convexity defect of the thumb
/// the handedness
/// the palm width
private float findPalmWidth(Hand hand, ConvexityDefect thumbDefect, HandSide side)
{
Vector2D lowerDirection = thumbDefect.VectorLong.getOrthogonal(side == HandSide.Right).normalize();
Vector2D current = thumbDefect.Inner;
Vector2D step = thumbDefect.VectorLong / Parameters.PalmNumPositionsForPalmWidth;
float maxWidth = float.MinValue;
for (int i = 0; i < Parameters.PalmNumPositionsForPalmWidth; i++)
{
Vector2D lower = current;
Vector2D lowerNext = lower + 10 * lowerDirection;
while (lowerNext.isInBound(depthImage.Size) && hand.isInside(lowerNext))
{
lower = lowerNext;
lowerNext += lowerDirection;
}
maxWidth = Math.Max(maxWidth, current.getDistanceTo(lower));
current += step;
}
return maxWidth;
}
///
/// Checks all convexity defects sorted by length if they match the thumb defect requirements and returns the first match or null if none match.
///
/// the possible thumb
/// the convexity defects
/// the thumb defect or null
private ConvexityDefect findThumbDefect(Finger thumb, List convexityDefects)
{
convexityDefects.Sort((cd1, cd2) => cd2.Depth.CompareTo(cd1.Depth));
foreach (ConvexityDefect defect in convexityDefects)
{
if (defect.isPossibleThumbDefect(thumb))
return defect;
}
return null;
}
///
/// Finds the upper wrist end by walking from the defect.Inner towards the wrist direction, until it reaches the hand boundaries.
///
/// the hand
/// the convexity defect of the thumb
/// the position of the upper wrist end
private Vector2D findWristUpper(Hand hand, ConvexityDefect thumbDefect)
{
Vector2D wristDirection = thumbDefect.VectorLong.getInverse().normalize();
Vector2D wristUpper = thumbDefect.Inner;
Vector2D wristUpperNext = wristUpper + 5 * wristDirection;
while (wristUpperNext.isInBound(depthImage.Size) && hand.isInside(wristUpperNext))
{
wristUpper = wristUpperNext;
wristUpperNext += wristDirection;
}
return wristUpper;
}
}
}