bbiwarg-original/bbiwarg/Images/PalmImage.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using Emgu.CV;
using Emgu.CV.Structure;
using System.Drawing;

using bbiwarg.Detectors.Fingers;
using bbiwarg.Utility;

namespace bbiwarg.Images
{
    class PalmImage
    {
        DepthImage depthImage;
        EdgeImage edgeImage;

        private Image<Gray, Byte> handImage;
        private Image<Gray, Byte> pointingHandMask;
        private Image<Gray, Byte> outputImage;

        private List<Finger> fingers;

        private Contour<Point> palmContour;
        private List<MCvConvexityDefect> convexityDefects;
        private Vector2D wristPoint, wristDirection;
        private LineSegment2DF wristLine, thumbLine;
        private MCvBox2D palmRect;

        private int width, height;
        
        public PalmImage(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector)
        {
            this.depthImage = depthImage;
            this.edgeImage = edgeImage;

            handImage = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte) 0 : (byte) 1; });
            width = depthImage.getWidth();
            height = depthImage.getHeight();

            outputImage = new Image<Gray, byte>(width, height, new Gray(0));

            fingers = getFingersWithoutThumb(fingerDetector);
            buildPointingHandMask();
            handImage = handImage.And(pointingHandMask);

            findLongestPalmContour();
            if (palmContour != null)
            {
                findConvexityDefactsSortedByDepth();
                removeConvexityDefectsNearFingerTips();

                findWristLine();
                findThumbLine();

                removePointsFromContour(wristLine, 1);
                removePointsFromContour(thumbLine, 1);

                findPalmRect();

                draw();
            }
        }

        private List<Finger> getFingersWithoutThumb(FingerDetector fingerDetector)
        {
            List<Finger> detectedFingers = fingerDetector.Fingers;
            Finger leftMost = null;
            float minX = float.MaxValue;
            foreach (Finger f in detectedFingers)
            {
                float midX = ((f.Hand + f.Tip) / 2).X;
                if (midX < minX)
                {
                    minX = midX;
                    leftMost = f;
                }
            }

            List<Finger> result = new List<Finger>();
            foreach (Finger f in detectedFingers)
            {
                if (f != leftMost)
                    result.Add(f);
            }
            return result;
        }

        private void fillFingerSlices(Image<Gray, Byte> image, byte val)
        {
            foreach (Finger f in fingers)
            {
                foreach (FingerSlice s in f.SliceTrail.getSlices())
                {
                    image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
                }            
            }
        }

        private Finger getLongestFinger()
        {
            float maxLength = 0;
            Finger longest = null;
            foreach (Finger f in fingers)
            {
                
                if (f.Line.Length > maxLength)
                {
                    maxLength = f.Line.Length;
                    longest = f;
                }
            }
            return longest;
        }

        private Point getPointInPointingHand()
        {
            Finger finger = getLongestFinger();
            if (finger == null)
                return new Point(0, 0);
            Vector2D direction = (finger.Hand - finger.Tip).normalize();
            Vector2D pos = finger.Hand + 20 * direction;
            return new Point(Math.Min(width - 1, Math.Max(0, (int)pos.X)), Math.Min(height - 1, Math.Max(0, (int)pos.Y)));
        }

        private void buildPointingHandMask()
        {
            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
            fillFingerSlices(pointingHandMask, 1);
            pointingHandMask = pointingHandMask.Dilate(1);
            pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert<Byte>(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; }));
            pointingHandMask = pointingHandMask.Dilate(1);

            MCvConnectedComp tmp = new MCvConnectedComp();
            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
            pointingHandMask = pointingHandMask.Convert<Byte>(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; });
            pointingHandMask = pointingHandMask.Erode(1);
            fillFingerSlices(pointingHandMask, 0);
            pointingHandMask = pointingHandMask.Erode(2);
        }

        private void findLongestPalmContour()
        {
            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);

            palmContour = contour;
            double maxPerimeter = 0;
            while (contour != null)
            {
                if (contour.Perimeter > maxPerimeter)
                {
                    maxPerimeter = contour.Perimeter;
                    palmContour = contour;
                }
                contour = contour.HNext;
            }
        }

        private void findWristDirection()
        {
            PointF[] points = new PointF[palmContour.Count<Point>()];
            int index = 0;
            foreach (Point p in palmContour)
            {
                points[index] = new PointF(p.X, p.Y);
                ++index;
            }
            PointF direction, tmp;
            PointCollection.Line2DFitting(points, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out direction, out tmp);
            wristDirection = new Vector2D(-direction.Y, direction.X);
        }

        private void findConvexityDefactsSortedByDepth()
        {
            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
            convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) {
                if (d1.Depth < d2.Depth)
                    return 1;
                else if (d1.Depth > d2.Depth)
                    return -1;
                return 0;
            });
        }

        private void removeConvexityDefectsNearFingerTips()
        {
            List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
            foreach (MCvConvexityDefect d in convexityDefects)
            {
                float minFingerTipDist = float.MaxValue;
                foreach (Finger f in fingers)
                {
                    float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint));
                    if (dist < minFingerTipDist)
                        minFingerTipDist = dist;
                }
                if (minFingerTipDist > 20)
                    newDefects.Add(d);
            }
            convexityDefects = newDefects;
        }

        private void findWristPoint()
        {
            if (convexityDefects.Count > 1)
                wristPoint = new Vector2D(convexityDefects[1].DepthPoint);
            else
                wristPoint = new Vector2D(-1, -1);
        }

        private void findWristLine()
        {
            findWristPoint();
            findWristDirection();
            wristLine = new LineSegment2DF(wristPoint - 1000 * wristDirection, wristPoint + 1000 * wristDirection);
        }

        private void findThumbLine()
        {
            if (convexityDefects.Count > 0)
            {
                MCvConvexityDefect thumbDefect = convexityDefects[0];
                Vector2D p1 = new Vector2D(thumbDefect.DepthPoint);
                Vector2D p2 = new Vector2D(thumbDefect.StartPoint);
                Vector2D direction = (p1 - p2).normalize();
                thumbLine = new LineSegment2DF(p1 - 1000 * direction, p1 + 1000 * direction);
            }
            else
            {
                thumbLine = new LineSegment2DF(new PointF(-1, -1), new PointF(-1, -1));
            }
        }

        private void removePointsFromContour(LineSegment2DF line, int sideToRemove)
        {
            Contour<Point> newContour = new Contour<Point>(new MemStorage());
            int index = 0;
            foreach (Point p in palmContour)
            {
                if (line.Side(p) != sideToRemove)
                {
                    newContour.Insert(index, p);
                    ++index;
                }
            }
            palmContour = newContour;
        }

        private void findPalmRect()
        {
            palmRect = palmContour.GetMinAreaRect();
        }

        private void draw()
        {
            //outputImage.Draw(palmContour, new Gray(1), 1);
            PointF[] vertices = palmRect.GetVertices();
            for (int i = 0; i < 4; ++i)
                outputImage.Draw(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), new Gray(1), 1);

            outputImage.Draw(wristLine, new Gray(1), 1);
            outputImage.Draw(thumbLine, new Gray(1), 1);
        }

        public bool belongsToPalm(int x, int y)
        {
            return outputImage.Data[y, x, 0] == 1;
        }
    }
}