bbiwarg-original/bbiwarg/Detectors/PalmDetection/PalmDetector.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Drawing;

using Emgu.CV;
using Emgu.CV.Structure;

using bbiwarg.Utility;
using bbiwarg.Images;
using bbiwarg.Detectors.FingerDetection;
using bbiwarg.Graphics;

namespace bbiwarg.Detectors.PalmDetection
{
    class PalmDetector
    {
        private int width, height;
        private OutputImage outputImage;
        private EdgeImage edgeImage;

        private Image<Gray, Byte> handImage;
        private Image<Gray, Byte> pointingHandMask;

        private List<Finger> fingers;

        private Contour<Point> palmContour;
        private List<MCvConvexityDefect> convexityDefects;
        private Vector2D thumbDefectStart;
        private Vector2D thumbDefectEnd;
        private Vector2D thumbDefectDepth, lastThumbDefectDepth;
        
        private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;

        private Vector2D topLeft;
        private Vector2D topRight;
        private Vector2D bottomLeft;
        private Vector2D bottomRight;

        public Quadrangle PalmQuad { get; private set; }

        public PalmDetector()
        {
            // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand
            thumbDefectDepthFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
            thumbDefectStartFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
            thumbDefectEndFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f);
        }

        public void findPalmQuad(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage, List<Finger> trackedFingers)
        {
            this.width = depthImage.Width;
            this.height = depthImage.Height;

            i1 = new OutputImage(width, height);
            i2 = new OutputImage(width, height);
            i3 = new OutputImage(width, height);
            i4 = new OutputImage(width, height);
            i5 = new OutputImage(width, height);
            i6 = new OutputImage(width, height);
            i7 = new OutputImage(width, height);
            i8 = new OutputImage(width, height);
            i9 = new OutputImage(width, height);

            if (trackedFingers.Count >= 2)
            {
                this.edgeImage = edgeImage;
                this.outputImage = outputImage;

                // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
                handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();

                fingers = getFingersWithoutThumb(trackedFingers);
                buildPointingHandMask();
                handImage = handImage.And(pointingHandMask);

                findLongestPalmContour();
                if (palmContour != null)
                {
                    findConvexityDefectsSortedByDepth();
                    removeConvexityDefectsCausedByFingers();

                    findHandPoints();
                }
            }
            draw();
        }

        public void reset()
        {
            thumbDefectDepthFilter.reset();
            thumbDefectStartFilter.reset();
            thumbDefectEndFilter.reset();
            lastThumbDefectDepth = null;
        }

        private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
        {
            Finger leftMost = null;
            float minX = float.MaxValue;
            foreach (Finger f in detectedFingers)
            {
                float midX = ((f.HandPoint + f.TipPoint) / 2).X;
                if (midX < minX)
                {
                    minX = midX;
                    leftMost = f;
                }
            }

            List<Finger> result = new List<Finger>();
            foreach (Finger f in detectedFingers)
            {
                if (f != leftMost)
                    result.Add(f);
            }
            return result;
        }

        private void fillFirstFingerSlices(Image<Gray, Byte> image, byte val)
        {
            foreach (Finger f in fingers)
            {
                    image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1);
            }
        }

        private Point getPointInPointingHand()
        {
            Finger finger = fingers[0];
            if (finger == null)
                return new Point(0, 0);
            Vector2D direction = (finger.HandPoint - finger.TipPoint).normalize();
            Vector2D pos = finger.HandPoint + direction;
            
            while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
                pos += direction;

            i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red);

            return pos;
        }

        public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9;
        private void buildPointingHandMask()
        {
            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));

            // dst = (src > 0) ? 1 : 0;
            pointingHandMask = pointingHandMask.Or(edgeImage.Image);
            i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;

            pointingHandMask = pointingHandMask.Dilate(1);
            i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
           
            fillFirstFingerSlices(pointingHandMask, 1);
            i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;

            //pointingHandMask = pointingHandMask.Dilate(1);
            //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask;

            MCvConnectedComp tmp = new MCvConnectedComp();
            // fill with value 2
            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);

            // dst = (src > 1) ? 0 : 1    (src > 1 <-> src == 2 <-> src filled by flood fill)
            pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1));
            i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
            
            pointingHandMask = pointingHandMask.Erode(6);
            i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;

            fillFirstFingerSlices(pointingHandMask, 0);
            i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
            
            pointingHandMask = pointingHandMask.Erode(2);
            i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
            
            // only debug
            i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
        }

        private void findLongestPalmContour()
        {
            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);

            palmContour = contour;
            double maxPerimeter = 0;
            while (contour != null)
            {
                if (contour.Perimeter > maxPerimeter)
                {
                    maxPerimeter = contour.Perimeter;
                    palmContour = contour;
                }
                contour = contour.HNext;
            }
        }

        private void findConvexityDefectsSortedByDepth()
        {
            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
            convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2)
            {
                if (d1.Depth < d2.Depth)
                    return 1;
                else if (d1.Depth > d2.Depth)
                    return -1;
                return 0;
            });
        }

        private void removeConvexityDefectsCausedByFingers()
        {
            List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                bool fingerIntersectsStartEnd = false;
                float minFingerLineDist = float.MaxValue;
                foreach (Finger f in fingers)
                {
                    Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
                    Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
                    if (intersection != null &&
                        intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2))
                    {
                        fingerIntersectsStartEnd = true;
                        break;
                    }
                    
                    Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
                    float dist = f.LineSegment.getDistanceTo(mid);
                    if (dist < minFingerLineDist)
                        minFingerLineDist = dist;
                }
                
                if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !fingerIntersectsStartEnd)
                    newDefects.Add(d);
            }
            convexityDefects = newDefects;
        }

        private int i = 0;
        private MCvConvexityDefect? findThumbDefect()
        {
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                Vector2D depth = new Vector2D(d.DepthPoint);
                Vector2D start = new Vector2D(d.StartPoint);
                Vector2D end = new Vector2D(d.EndPoint);

                float l1 = (depth - start).Length;
                float l2 = (depth - end).Length;
                float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2);

                float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);

                if (angle <= Constants.PalmMaxThumbDefectAngle && 
                    lengthQuotient >= Constants.PalmMinThumbDefectLengthQuotient && lengthQuotient <= Constants.PalmMaxThumbDefectLengthQuotient)
                {
                    return d;
                }
            }


            //Console.WriteLine("no palm defect found (" + i + ")");
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                Vector2D depth = new Vector2D(d.DepthPoint);
                Vector2D start = new Vector2D(d.StartPoint);
                Vector2D end = new Vector2D(d.EndPoint);

                float l1 = (depth - start).Length;
                float l2 = (depth - end).Length;
                float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2);

                float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);

                //Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient);
            }

            ++i;
            return null;
        }

        private void findHandPoints()
        {
            MCvConvexityDefect? thumbDefect = findThumbDefect();

            if (convexityDefects.Count > 0 && (thumbDefect != null || thumbDefectDepthFilter.Initialized))
            {
                if (thumbDefect != null)
                {
                    thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
                    thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
                    thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
                }
                
                if (!thumbDefectDepthFilter.Initialized)
                {
                    thumbDefectDepthFilter.setInitialPosition(thumbDefectDepth);
                    thumbDefectStartFilter.setInitialPosition(thumbDefectStart);
                    thumbDefectEndFilter.setInitialPosition(thumbDefectEnd);
                }
                else
                {
                    thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth);
                    thumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(thumbDefectStart);
                    thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd);
                }

                lastThumbDefectDepth = thumbDefectDepth;

                Vector2D handLength, handWidth;
                if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd))
                {
                    //right hand
                    handLength = thumbDefectStart - thumbDefectDepth;
                    handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
                    topLeft = thumbDefectStart+0.15f*handLength;
                    bottomLeft = thumbDefectDepth - 0.4f * handLength;
                    bottomRight = bottomLeft + handWidth;
                    topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
                }
                else
                {
                    //left hand
                    handLength = thumbDefectEnd - thumbDefectDepth;
                    handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
                    topRight = thumbDefectEnd+0.15f*handLength;
                    bottomRight = thumbDefectDepth - 0.4f * handLength;
                    bottomLeft = bottomRight + handWidth;
                    topLeft = bottomLeft + 1.15f * handLength - 0.35f * handWidth;
                }

                PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight);
            }
        }

        private void draw()
        {
            if (palmContour != null && palmContour.Count<Point>() > 0) {
                outputImage.drawContour(palmContour, Constants.PalmConturColor);
                outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
            }

            if (PalmQuad != null)
            {
                outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red);
                outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red);
                outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red);

                outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);

                Vector2D[] vertices = PalmQuad.Vertices;
                for (int i = 0; i < 4; ++i)
                    outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor);

                drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3]));
            }
        }

        private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d)
        {
            Vector2D relAB = (b - a) / Constants.PalmGridRows;
            Vector2D relDC = (c - d) / Constants.PalmGridRows;
            Vector2D relBC = (c - b) / Constants.PalmGridColumns;
            Vector2D relAD = (d - a) / Constants.PalmGridColumns;

            for (int i = 1; i < Constants.PalmGridRows; i++)
            {
                outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor);
            }

            for (int i = 1; i < Constants.PalmGridColumns; i++)
            {
                outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor);
            }
        }
    }
}