bbiwarg-original/bbiwarg/Detectors/Palm/PalmDetector.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Drawing;

using Emgu.CV;
using Emgu.CV.Structure;

using bbiwarg.Utility;

using bbiwarg.Images;

using bbiwarg.Detectors.Fingers;

namespace bbiwarg.Detectors.Palm
{
    class PalmDetector
    {
        private int width, height;
        private DepthImage depthImage;
        private EdgeImage edgeImage;
        private PalmImage palmImage;

        private Image<Gray, Byte> handImage;
        private Image<Gray, Byte> pointingHandMask;

        private List<Finger> fingers;

        private Contour<Point> palmContour;
        private List<MCvConvexityDefect> convexityDefects;
        private Vector2D wristPoint, wristDirection;
        private LineSegment2DF wristLine, thumbLine;
        private Quadrangle palmQuad;

        private bool valid = false;
        private Vector2D topLeft;
        private Vector2D topRight;
        private Vector2D bottomLeft;
        private Vector2D bottomRight;

        public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector, PalmImage palmImage)
        {
            width = depthImage.getWidth();
            height = depthImage.getHeight();
            this.depthImage = depthImage;
            this.edgeImage = edgeImage;
            this.palmImage = palmImage;

            handImage = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte)0 : (byte)1; });
            
            fingers = getFingersWithoutThumb(fingerDetector);
            buildPointingHandMask();
            handImage = handImage.And(pointingHandMask);

            findLongestPalmContour();
            if (palmContour != null)
            {
                findConvexityDefactsSortedByDepth();
                removeConvexityDefectsNearFingerTips();

                findHandPoints();

                if (valid)
                {
                    removePointsFromContour(wristLine, 1);
                    removePointsFromContour(thumbLine, 1);

                    draw();
                }
            }
        }

        public Quadrangle getPalmQuad()
        {
            return palmQuad;
        }

        private List<Finger> getFingersWithoutThumb(FingerDetector fingerDetector)
        {
            List<Finger> detectedFingers = fingerDetector.Fingers;
            Finger leftMost = null;
            float minX = float.MaxValue;
            foreach (Finger f in detectedFingers)
            {
                float midX = ((f.Hand + f.Tip) / 2).X;
                if (midX < minX)
                {
                    minX = midX;
                    leftMost = f;
                }
            }

            List<Finger> result = new List<Finger>();
            foreach (Finger f in detectedFingers)
            {
                if (f != leftMost)
                    result.Add(f);
            }
            return result;
        }

        private void fillFingerSlices(Image<Gray, Byte> image, byte val)
        {
            foreach (Finger f in fingers)
            {
                foreach (FingerSlice s in f.SliceTrail.getSlices())
                {
                    image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
                }
            }
        }

        private Finger getLongestFinger()
        {
            float maxLength = 0;
            Finger longest = null;
            foreach (Finger f in fingers)
            {

                if (f.Line.Length > maxLength)
                {
                    maxLength = f.Line.Length;
                    longest = f;
                }
            }
            return longest;
        }

        private Point getPointInPointingHand()
        {
            Finger finger = getLongestFinger();
            if (finger == null)
                return new Point(0, 0);
            Vector2D direction = (finger.Hand - finger.Tip).normalize();
            Vector2D pos = finger.Hand + 20 * direction;
            return new Point(HelperFunctions.thresholdRange<int>(0, width - 1, (int) pos.X), HelperFunctions.thresholdRange<int>(0, height - 1, (int) pos.Y));
        }

        private void buildPointingHandMask()
        {
            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
            fillFingerSlices(pointingHandMask, 1);
            pointingHandMask = pointingHandMask.Dilate(1);
            pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert<Byte>(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; }));
            pointingHandMask = pointingHandMask.Dilate(1);

            MCvConnectedComp tmp = new MCvConnectedComp();
            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
            pointingHandMask = pointingHandMask.Convert<Byte>(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; });
            pointingHandMask = pointingHandMask.Erode(1);
            fillFingerSlices(pointingHandMask, 0);
            pointingHandMask = pointingHandMask.Erode(2);
        }

        private void findLongestPalmContour()
        {
            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);

            palmContour = contour;
            double maxPerimeter = 0;
            while (contour != null)
            {
                if (contour.Perimeter > maxPerimeter)
                {
                    maxPerimeter = contour.Perimeter;
                    palmContour = contour;
                }
                contour = contour.HNext;
            }
        }

        private void findConvexityDefactsSortedByDepth()
        {
            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
            convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2)
            {
                if (d1.Depth < d2.Depth)
                    return 1;
                else if (d1.Depth > d2.Depth)
                    return -1;
                return 0;
            });
        }

        private void removeConvexityDefectsNearFingerTips()
        {
            List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                float minFingerTipDist = float.MaxValue;
                foreach (Finger f in fingers)
                {
                    float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint));
                    if (dist < minFingerTipDist)
                        minFingerTipDist = dist;
                }
                if (minFingerTipDist > 20)
                    newDefects.Add(d);
            }
            convexityDefects = newDefects;
        }

        private void findHandPoints() {
            if (convexityDefects.Count > 0)
            {
                MCvConvexityDefect thumbDefect = convexityDefects[0];
                Vector2D thumb = new Vector2D(thumbDefect.DepthPoint);
                Vector2D thumbDefectStart = new Vector2D(thumbDefect.StartPoint);

                Vector2D handLength = thumbDefectStart - thumb;
                Vector2D handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X);

                topLeft = thumbDefectStart;
                bottomLeft = thumb - 0.3f * handLength;
                bottomRight = bottomLeft + handWidth;
                topRight = bottomRight + 1.5f * handLength;

                wristLine = new LineSegment2DF(bottomLeft - 1000 * handWidth, bottomRight + 1000 * handWidth);
                thumbLine = new LineSegment2DF(topLeft + 1000 * handLength, bottomLeft - 1000 * handLength);

                palmQuad = new Quadrangle(new Vector2D[] {bottomLeft, topLeft, topRight, bottomRight});

                valid = true;
            }
            else {
                palmQuad = null;
                valid = false;
            }
        }

        private void removePointsFromContour(LineSegment2DF line, int sideToRemove)
        {
            Contour<Point> newContour = new Contour<Point>(new MemStorage());
            int index = 0;
            foreach (Point p in palmContour)
            {
                if (line.Side(p) != sideToRemove)
                {
                    newContour.Insert(index, p);
                    ++index;
                }
            }
            palmContour = newContour;
        }

        private void draw()
        {
            palmImage.drawContour(palmContour);

            palmImage.drawLine(wristLine, PalmImageState.wristLine);
            palmImage.drawLine(thumbLine, PalmImageState.thumbLine);

            if (palmQuad != null)
            {
                Vector2D[] vertices = palmQuad.Vertices;
                for (int i = 0; i < 4; ++i)
                    palmImage.drawLine(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), PalmImageState.palmRect);


                palmImage.drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3]));
            }
        }
    }
}