|
@@ -21,6 +21,7 @@ namespace bbiwarg.Recognition.HandRecognition
|
|
|
private EdgeImage edgeImage;
|
|
|
private Image<Gray, byte> modifiedHandDepthImage;
|
|
|
private List<Finger> fingers;
|
|
|
+ private List<Hand> ignoreMergedHands;
|
|
|
|
|
|
public List<Hand> Hands { get; private set; }
|
|
|
public Image<Gray, byte> HandMask { get; private set; }
|
|
@@ -33,6 +34,7 @@ namespace bbiwarg.Recognition.HandRecognition
|
|
|
|
|
|
createModifiedHandEdgeImage();
|
|
|
findHands();
|
|
|
+ fixOverlappingFingers();
|
|
|
setZIndexes();
|
|
|
createHandMask();
|
|
|
findThumbDefects();
|
|
@@ -72,13 +74,10 @@ namespace bbiwarg.Recognition.HandRecognition
|
|
|
|
|
|
if (newHand)
|
|
|
{
|
|
|
- Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
|
|
|
- MCvConnectedComp comp = new MCvConnectedComp();
|
|
|
- CvInvoke.cvFloodFill(modifiedHandDepthImage, finger.HandPoint, new MCvScalar(255), new MCvScalar(Constants.HandFloodFillDownDiff), new MCvScalar(Constants.HandFloodFillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
|
|
|
- if (comp.area < maxArea * Constants.HandMaxSize)
|
|
|
+ Image<Gray, byte> handMask = getHandMask(finger.HandPoint);
|
|
|
+ if (handMask.CountNonzero()[0] < maxArea * Constants.HandMaxSize)
|
|
|
{
|
|
|
- Image<Gray, byte> cropedMask = mask.Copy(new Rectangle(1, 1, width, height));
|
|
|
- Hand hand = new Hand(cropedMask, finger);
|
|
|
+ Hand hand = new Hand(handMask, finger);
|
|
|
Hands.Add(hand);
|
|
|
finger.setHand(hand);
|
|
|
}
|
|
@@ -86,6 +85,16 @@ namespace bbiwarg.Recognition.HandRecognition
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private Image<Gray, byte> getHandMask(Vector2D p)
|
|
|
+ {
|
|
|
+ int width = edgeImage.Width;
|
|
|
+ int height = edgeImage.Height;
|
|
|
+ Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
|
|
|
+ MCvConnectedComp comp = new MCvConnectedComp();
|
|
|
+ CvInvoke.cvFloodFill(modifiedHandDepthImage, p, new MCvScalar(255), new MCvScalar(Constants.HandFloodFillDownDiff), new MCvScalar(Constants.HandFloodFillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
|
|
|
+ return mask.Copy(new Rectangle(1, 1, width, height));
|
|
|
+ }
|
|
|
+
|
|
|
private void setZIndexes()
|
|
|
{
|
|
|
//sort depending on depth of centroid (far->near)
|
|
@@ -95,6 +104,18 @@ namespace bbiwarg.Recognition.HandRecognition
|
|
|
Hands[i].setZIndex(i);
|
|
|
}
|
|
|
|
|
|
+ private void fixOverlappingFingers()
|
|
|
+ {
|
|
|
+ extendOrMergeThroughOverlappingFingers();
|
|
|
+ fixDefectsFromOverlappingFingers();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void findThumbDefects()
|
|
|
+ {
|
|
|
+ foreach (Hand hand in Hands)
|
|
|
+ hand.findThumbDefect();
|
|
|
+ }
|
|
|
+
|
|
|
private void createHandMask()
|
|
|
{
|
|
|
HandMask = new Image<Gray, byte>(depthImage.Width, depthImage.Height);
|
|
@@ -102,10 +123,84 @@ namespace bbiwarg.Recognition.HandRecognition
|
|
|
HandMask = HandMask.Or(hand.Mask);
|
|
|
}
|
|
|
|
|
|
- private void findThumbDefects()
|
|
|
+ private void extendOrMergeThroughOverlappingFingers()
|
|
|
+ {
|
|
|
+ Dictionary<Hand, Hand> mergeHands = new Dictionary<Hand, Hand>();
|
|
|
+
|
|
|
+ foreach (Hand overlappingHand in Hands)
|
|
|
+ {
|
|
|
+ foreach (Hand underlyingHand in Hands)
|
|
|
+ {
|
|
|
+ if (!mergeHands.Keys.Contains(underlyingHand))
|
|
|
+ {
|
|
|
+ foreach (Finger overlappingFinger in overlappingHand.Fingers)
|
|
|
+ {
|
|
|
+ Vector2D midOut1 = overlappingFinger.SliceTrail.MidSlice.OutStart;
|
|
|
+ Vector2D midOut2 = overlappingFinger.SliceTrail.MidSlice.OutEnd;
|
|
|
+
|
|
|
+ Int16 depthAtMidOut1 = depthImage.getDepthAt(midOut1);
|
|
|
+ Int16 depthAtMidOut2 = depthImage.getDepthAt(midOut2);
|
|
|
+
|
|
|
+ bool midOut1InHand = underlyingHand.isInside(midOut1);
|
|
|
+ bool midOut2InHand = underlyingHand.isInside(midOut2);
|
|
|
+
|
|
|
+ Int16 maxDepth = depthImage.MaxDepth;
|
|
|
+ if (midOut1InHand != midOut2InHand && depthAtMidOut1 != maxDepth && depthAtMidOut2 != maxDepth && Math.Abs(depthAtMidOut1 - depthAtMidOut2) < Constants.HandExtendMaxDifference)
|
|
|
+ {
|
|
|
+ Vector2D p1, p2;
|
|
|
+ if (midOut1InHand)
|
|
|
+ {
|
|
|
+ p1 = midOut1;
|
|
|
+ p2 = midOut2;
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ p1 = midOut2;
|
|
|
+ p2 = midOut1;
|
|
|
+ }
|
|
|
+
|
|
|
+ //check if p2 is in other hand (if so -> merge)
|
|
|
+ bool merge = false;
|
|
|
+ foreach (Hand hand in Hands)
|
|
|
+ {
|
|
|
+ if (hand.isInside(p2) && !ignoreMergedHands.Contains(hand))
|
|
|
+ {
|
|
|
+ mergeHands.Add(underlyingHand, overlappingHand);
|
|
|
+ merge = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //if no merge, extend hand
|
|
|
+ if (!merge)
|
|
|
+ underlyingHand.extendMask(getHandMask(p2));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ foreach (Hand removeHand in mergeHands.Keys)
|
|
|
+ {
|
|
|
+ mergeHands[removeHand].mergeWith(removeHand);
|
|
|
+ Hands.Remove(removeHand);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void fixDefectsFromOverlappingFingers()
|
|
|
{
|
|
|
foreach (Hand hand in Hands)
|
|
|
- hand.findThumbDefect();
|
|
|
+ {
|
|
|
+ //get other hands fingers
|
|
|
+ List<Finger> otherFingers = new List<Finger>();
|
|
|
+ foreach (Hand otherHand in Hands)
|
|
|
+ {
|
|
|
+ if (hand != otherHand)
|
|
|
+ otherFingers.AddRange(otherHand.Fingers);
|
|
|
+ }
|
|
|
+
|
|
|
+ hand.fixDefectsFromOverlappingFingers(otherFingers);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|