bbiwarg-original/bbiwarg/Recognition/HandRecognition/HandDetector.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using bbiwarg.Images;
using bbiwarg.Recognition.FingerRecognition;
using bbiwarg.Graphics;
using bbiwarg.Utility;
using Emgu.CV;
using Emgu.CV.Structure;



namespace bbiwarg.Recognition.HandRecognition
{
    class HandDetector
    {
        private DepthImage depthImage;
        private EdgeImage edgeImage;
        private Image<Gray, byte> modifiedHandDepthImage;
        private List<Finger> fingers;

        public List<Hand> Hands { get; private set; }
        public Image<Gray, byte> HandMask { get; private set; }

        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers)
        {
            this.depthImage = depthImage;
            this.edgeImage = edgeImage;
            this.fingers = fingers;

            createModifiedHandEdgeImage();
            findHands();
            fixOverlappingFingers();
            setZIndexes();
            createHandMask();
            findThumbDefects();
        }

        private void createModifiedHandEdgeImage()
        {
            modifiedHandDepthImage = depthImage.Image.Copy();

            foreach (Finger finger in fingers)
            {
                Point[] contour = finger.InnerContour.ToArray();
                modifiedHandDepthImage.DrawPolyline(contour, false, new Gray(0), 2);
            }
        }

        private void findHands()
        {
            int width = edgeImage.Width;
            int height = edgeImage.Height;
            int maxArea = width * height;
            Hands = new List<Hand>();

            foreach (Finger finger in fingers)
            {
                bool newHand = true;
                foreach (Hand hand in Hands)
                {
                    if (hand.isInside(finger.HandPoint))
                    {
                        hand.addFinger(finger);
                        finger.setHand(hand);
                        newHand = false;
                        break;
                    }
                }

                if (newHand)
                {
                    Image<Gray, byte> handMask = getHandMask(finger.HandPoint);
                    int area = handMask.CountNonzero()[0];
                    if (area < maxArea * Constants.HandMaxSize && area > maxArea * Constants.HandMinSize)
                    {
                        Hand hand = new Hand(handMask, finger);
                        Hands.Add(hand);
                        finger.setHand(hand);
                    }
                }
            }
        }

        private Image<Gray, byte> getHandMask(Vector2D p)
        {
            int width = edgeImage.Width;
            int height = edgeImage.Height;
            Image<Gray, byte> mask = new Image<Gray, byte>(width + 2, height + 2);
            MCvConnectedComp comp = new MCvConnectedComp();
            CvInvoke.cvFloodFill(modifiedHandDepthImage, p, new MCvScalar(255), new MCvScalar(Constants.HandFloodFillDownDiff), new MCvScalar(Constants.HandFloodFillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
            return mask.Copy(new Rectangle(1, 1, width, height));
        }

        private void setZIndexes()
        {
            //sort depending on depth of centroid (far->near)
            Hands.Sort((h1, h2) => depthImage.getDepthAt(h2.CentroidInHand).CompareTo(depthImage.getDepthAt(h1.CentroidInHand)));

            for (int i = 0; i < Hands.Count; i++)
                Hands[i].setZIndex(i);
        }

        private void fixOverlappingFingers()
        {
            extendOrMergeThroughOverlappingFingers();
            fillOverlappingFingers();
        }

        private void findThumbDefects()
        {
            foreach (Hand hand in Hands)
                hand.findThumbDefect();
        }

        private void createHandMask()
        {
            HandMask = new Image<Gray, byte>(depthImage.Width, depthImage.Height);
            foreach (Hand hand in Hands)
                HandMask = HandMask.Or(hand.Mask);
        }

        private void extendOrMergeThroughOverlappingFingers()
        {
            Dictionary<Hand, Hand> mergeHands = new Dictionary<Hand, Hand>();

            foreach (Hand overlappingHand in Hands)
            {
                foreach (Hand underlyingHand in Hands)
                {
                    if (!mergeHands.Keys.Contains(underlyingHand))
                    {
                        foreach (Finger overlappingFinger in overlappingHand.Fingers)
                        {
                            Vector2D midOut1 = overlappingFinger.SliceTrail.MidSlice.OutStart;
                            Vector2D midOut2 = overlappingFinger.SliceTrail.MidSlice.OutEnd;

                            Int16 depthAtMidOut1 = depthImage.getDepthAt(midOut1);
                            Int16 depthAtMidOut2 = depthImage.getDepthAt(midOut2);

                            bool midOut1InHand = underlyingHand.isInside(midOut1);
                            bool midOut2InHand = underlyingHand.isInside(midOut2);

                            Int16 maxDepth = depthImage.MaxDepth;
                            if (midOut1InHand != midOut2InHand && depthAtMidOut1 != maxDepth && depthAtMidOut2 != maxDepth && Math.Abs(depthAtMidOut1 - depthAtMidOut2) < Constants.HandExtendMaxDifference)
                            {
                                Vector2D p1, p2;
                                if (midOut1InHand)
                                {
                                    p1 = midOut1;
                                    p2 = midOut2;
                                }
                                else
                                {
                                    p1 = midOut2;
                                    p2 = midOut1;
                                }

                                //check if p2 is in other hand (if so -> merge)
                                bool merge = false;
                                foreach (Hand hand in Hands)
                                {
                                    if (hand.isInside(p2) && !mergeHands.Keys.Contains(hand))
                                    {
                                        mergeHands.Add(underlyingHand, overlappingHand);
                                        merge = true;
                                        break;
                                    }
                                }

                                //if no merge, extend hand
                                if (!merge)
                                    extendHand(underlyingHand, p2);
                            }
                        }
                    }
                }
            }

            foreach (Hand removeHand in mergeHands.Keys)
            {
                mergeHands[removeHand].mergeWith(removeHand);
                Hands.Remove(removeHand);
            }
        }

        private void extendHand(Hand hand, Vector2D p) {
            int width = edgeImage.Width;
            int height = edgeImage.Height;
            int maxArea = width * height;
            
            Image<Gray, byte> extendMask = getHandMask(p);
            int area = extendMask.CountNonzero()[0];

            if(area <= 0.5f*Constants.HandMaxSize * maxArea)
                hand.extendMask(extendMask);
        }

        private void fillOverlappingFingers()
        {
            foreach (Hand hand in Hands)
            {
                //get other hands fingers
                List<Finger> otherFingers = new List<Finger>();
                foreach (Hand otherHand in Hands)
                {
                    if (hand != otherHand)
                        otherFingers.AddRange(otherHand.Fingers);
                }

                hand.fillOverlappingFingers(otherFingers);
            }
        }
    }
}