bbiwarg-original/bbiwarg/Recognition/PalmRecognition/PalmDetector.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Drawing;

using Emgu.CV;
using Emgu.CV.Structure;

using bbiwarg.Utility;
using bbiwarg.Images;
using bbiwarg.Recognition.FingerRecognition;
using bbiwarg.Recognition.HandRecognition;
using bbiwarg.Graphics;

namespace bbiwarg.Recognition.PalmRecognition
{
    class PalmDetector
    {
        private OutputImage outputImage;

        private Hand palmHand, pointingHand;
        private Contour<Point> palmContour;
        private List<MCvConvexityDefect> convexityDefects;
        private Vector2D thumbDefectStart;
        private Vector2D thumbDefectEnd;
        private Vector2D thumbDefectDepth;
        private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
        private int numFramesNoHandFound;
        private Quadrangle lastPalmQuad;
        private List<Hand> hands;

        public Quadrangle PalmQuad { get; private set; }
        public Hand.HandSide PalmHandSide { get; private set; }

        public PalmDetector()
        {
            thumbDefectDepthFilter = new Kalman2DPositionFilter(Constants.PalmThumbDefectmXX, Constants.PalmThumbDefectmXY,
                                                                Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise);
            thumbDefectStartFilter = new Kalman2DPositionFilter(Constants.PalmThumbDefectmXX, Constants.PalmThumbDefectmXY,
                                                                Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise);
            thumbDefectEndFilter = new Kalman2DPositionFilter(Constants.PalmThumbDefectmXX, Constants.PalmThumbDefectmXY,
                                                                Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise);
        }

        public void findPalmQuad(OutputImage outputImage, List<Hand> hands, Image<Gray, Byte> foregroundMask)
        {
            this.outputImage = outputImage;
            this.hands = hands;

            if (hands.Count == 1 && hands[0].Fingers.Count == 1)
            {
                palmHand = hands[0];
                pointingHand = null;
            }
            else if (hands.Count == 2)
            {
                if (hands[0].Fingers.Count > 1)
                {
                    pointingHand = hands[0];
                    palmHand = hands[1];
                }
                else if (hands[1].Fingers.Count > 1)
                {
                    pointingHand = hands[1];
                    palmHand = hands[0];
                }
                else if (hands[0].Fingers[0].LineSegment.Length > hands[1].Fingers[0].LineSegment.Length)
                {
                    pointingHand = hands[0];
                    palmHand = hands[1];
                }
                else
                {
                    pointingHand = hands[1];
                    palmHand = hands[0];
                }
            }

            if ((hands.Count == 1 && hands[0].Fingers.Count == 1) || hands.Count == 2)
            {
                findLongestPalmContour();
                findConvexityDefectsSortedByDepth();
                if (pointingHand != null)
                    removeConvexityDefectsCausedByFingers();
                findHandPoints(foregroundMask);
            }

            if (hands.Count == 0)
            {
                ++numFramesNoHandFound;
                if (numFramesNoHandFound == Constants.PalmNumFramesNoHandReset)
                    reset();
            }
            else
            {
                numFramesNoHandFound = 0;
            }

            draw();
        }

        public void reset()
        {
            thumbDefectDepthFilter.reset();
            thumbDefectStartFilter.reset();
            thumbDefectEndFilter.reset();
            palmContour = null;
            PalmQuad = null;
            PalmHandSide = Hand.HandSide.Left;
            numFramesNoHandFound = 0;
            lastPalmQuad = null;
        }

        private void findLongestPalmContour()
        {
            Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
                                                                Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);

            palmContour = contour;
            double maxPerimeter = 0;
            while (contour != null)
            {
                if (contour.Perimeter > maxPerimeter)
                {
                    maxPerimeter = contour.Perimeter;
                    palmContour = contour;
                }
                contour = contour.HNext;
            }
        }

        private void findConvexityDefectsSortedByDepth()
        {
            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
            convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2)
            {
                if (d1.Depth < d2.Depth)
                    return 1;
                else if (d1.Depth > d2.Depth)
                    return -1;
                return 0;
            });
        }

        private void removeConvexityDefectsCausedByFingers()
        {
            List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                bool fingerIntersectsStartEnd = false;
                float minFingerLineDist = float.MaxValue;
                foreach (Finger f in pointingHand.Fingers)
                {
                    Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
                    Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
                    if (intersection != null &&
                        intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2))
                    {
                        fingerIntersectsStartEnd = true;
                        break;
                    }

                    Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
                    float dist = f.LineSegment.getDistanceTo(mid);
                    if (dist < minFingerLineDist)
                        minFingerLineDist = dist;
                }

                if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !fingerIntersectsStartEnd)
                    newDefects.Add(d);
            }
            convexityDefects = newDefects;
        }

        private MCvConvexityDefect? findThumbDefect()
        {
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                Vector2D depth = new Vector2D(d.DepthPoint);
                Vector2D start = new Vector2D(d.StartPoint);
                Vector2D end = new Vector2D(d.EndPoint);

                float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);

                float l1 = (depth - start).Length;
                float l2 = (depth - end).Length;
                float startEndLengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2);

                float depthThumbLengthQuotient = d.Depth / palmHand.Fingers[0].LineSegment.Length;

                if (angle <= Constants.PalmMaxThumbDefectAngle &&
                    startEndLengthQuotient >= Constants.PalmMinThumbDefectStartEndLengthQuotient &&
                    startEndLengthQuotient <= Constants.PalmMaxThumbDefectStartEndLengthQuotient &&
                    depthThumbLengthQuotient >= Constants.PalmMinTumbDefectDepthThumbLengthQuotient &&
                    depthThumbLengthQuotient <= Constants.PalmMaxTumbDefectDepthThumbLengthQuotient)
                {
                    return d;
                }
            }

            return null;
        }

        float getForegroundPixelPercentage(Quadrangle quad, Image<Gray, Byte> foregroundMask)
        {
            float numPixelsInMask = quad.Mask.CountNonzero()[0];
            float numPixelsInMaskInForeground = (quad.Mask & foregroundMask).CountNonzero()[0];

            return numPixelsInMaskInForeground / numPixelsInMask;
        }

        private void findHandPoints(Image<Gray, Byte> foregroundMask)
        {
            MCvConvexityDefect? thumbDefect = findThumbDefect();

            if (thumbDefect != null)
            {
                thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
                thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
                thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);

                if (!thumbDefectDepthFilter.Initialized)
                {
                    thumbDefectDepthFilter.setInitialPosition(thumbDefectDepth);
                    thumbDefectStartFilter.setInitialPosition(thumbDefectStart);
                    thumbDefectEndFilter.setInitialPosition(thumbDefectEnd);
                }
                else
                {
                    thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth);
                    thumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(thumbDefectStart);
                    thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd);
                }

                Vector2D handLength, handWidth, longestLineEndpoint, topLeft, bottomLeft, bottomRight, topRight;

                Vector2D startDepth = thumbDefectStart - thumbDefectDepth;
                Vector2D endDepth = thumbDefectEnd - thumbDefectDepth;

                if (startDepth.Length > endDepth.Length)
                {
                    handLength = startDepth;
                    longestLineEndpoint = thumbDefectStart;
                    if (hands.Count == 1)
                    {
                        if (thumbDefectStart.X > thumbDefectDepth.X)
                            hands[0].Side = Hand.HandSide.Left;
                        else
                            hands[0].Side = Hand.HandSide.Right;
                    }
                }
                else
                {
                    handLength = endDepth;
                    longestLineEndpoint = thumbDefectEnd;
                    if (hands.Count == 1)
                    {
                        if (thumbDefectEnd.X > thumbDefectDepth.X)
                            hands[0].Side = Hand.HandSide.Left;
                        else
                            hands[0].Side = Hand.HandSide.Right;
                    }
                }

                if (palmHand.Side == Hand.HandSide.Left)
                {
                    handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
                    topLeft = longestLineEndpoint + 0.15f * handLength;
                    bottomLeft = thumbDefectDepth - 0.4f * handLength;
                    bottomRight = bottomLeft + handWidth;
                    topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
                }
                else
                {
                    handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
                    topRight = longestLineEndpoint + 0.15f * handLength;
                    bottomRight = thumbDefectDepth - 0.4f * handLength;
                    bottomLeft = bottomRight + handWidth;
                    topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth;
                }


                Quadrangle quad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight, foregroundMask.Width, foregroundMask.Height);

                if ((lastPalmQuad == null ||
                    (quad.Area / lastPalmQuad.Area >= Constants.PalmMinAreaQuotient && quad.Area / lastPalmQuad.Area <= Constants.PalmMaxAreaQuotient)) &&
                    getForegroundPixelPercentage(quad, foregroundMask) >= Constants.PalmMinPrecentageQuadForeground)
                {
                    PalmQuad = quad;
                    PalmHandSide = palmHand.Side;
                    lastPalmQuad = PalmQuad;
                }
            }

            if (lastPalmQuad != null && getForegroundPixelPercentage(lastPalmQuad, foregroundMask) <= Constants.PalmMaxPrecentageQuadForegroundReset)
                reset();
        }

        private void draw()
        {
            if (palmContour != null && palmContour.Count<Point>() > 0)
            {
                outputImage.drawContour(palmContour, Constants.PalmConturColor);
                outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
            }

            if (PalmQuad != null)
            {
                outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red);
                outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red);
                outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red);

                outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);

                Vector2D[] vertices = PalmQuad.Vertices;
                for (int i = 0; i < 4; ++i)
                    outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor);

                drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3]));
            }
        }

        private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d)
        {
            Vector2D relAB = (b - a) / Constants.PalmGridRows;
            Vector2D relDC = (c - d) / Constants.PalmGridRows;
            Vector2D relBC = (c - b) / Constants.PalmGridColumns;
            Vector2D relAD = (d - a) / Constants.PalmGridColumns;

            for (int i = 1; i < Constants.PalmGridRows; i++)
            {
                outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor);
            }

            for (int i = 1; i < Constants.PalmGridColumns; i++)
            {
                outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor);
            }
        }
    }
}