using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using Emgu.CV; using Emgu.CV.Structure; using System.Drawing; using bbiwarg.Detectors.Fingers; using bbiwarg.Utility; namespace bbiwarg.Images { class PalmImage { DepthImage depthImage; EdgeImage edgeImage; private Image handImage; private Image pointingHandMask; private Image outputImage; private List fingers; private Contour palmContour; private List convexityDefects; private Vector2D wristPoint, wristDirection; private LineSegment2DF wristLine, thumbLine; private MCvBox2D palmRect; private int width, height; public PalmImage(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector) { this.depthImage = depthImage; this.edgeImage = edgeImage; handImage = depthImage.getImage().Convert(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte) 0 : (byte) 1; }); width = depthImage.getWidth(); height = depthImage.getHeight(); outputImage = new Image(width, height, new Gray(0)); fingers = getFingersWithoutThumb(fingerDetector); buildPointingHandMask(); handImage = handImage.And(pointingHandMask); findLongestPalmContour(); if (palmContour != null) { findConvexityDefactsSortedByDepth(); removeConvexityDefectsNearFingerTips(); findWristLine(); findThumbLine(); removePointsFromContour(wristLine, 1); removePointsFromContour(thumbLine, 1); findPalmRect(); draw(); } } private List getFingersWithoutThumb(FingerDetector fingerDetector) { List detectedFingers = fingerDetector.Fingers; Finger leftMost = null; float minX = float.MaxValue; foreach (Finger f in detectedFingers) { float midX = ((f.Hand + f.Tip) / 2).X; if (midX < minX) { minX = midX; leftMost = f; } } List result = new List(); foreach (Finger f in detectedFingers) { if (f != leftMost) result.Add(f); } return result; } private void fillFingerSlices(Image image, byte val) { foreach (Finger f in fingers) { foreach (FingerSlice s in f.SliceTrail.getSlices()) { image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1); } } } private Finger getLongestFinger() { float maxLength = 0; Finger longest = null; foreach (Finger f in fingers) { if (f.Line.Length > maxLength) { maxLength = f.Line.Length; longest = f; } } return longest; } private Point getPointInPointingHand() { Finger finger = getLongestFinger(); if (finger == null) return new Point(0, 0); Vector2D direction = (finger.Hand - finger.Tip).normalize(); Vector2D pos = finger.Hand + 20 * direction; return new Point(Math.Min(width - 1, Math.Max(0, (int)pos.X)), Math.Min(height - 1, Math.Max(0, (int)pos.Y))); } private void buildPointingHandMask() { pointingHandMask = new Image(width, height, new Gray(0)); fillFingerSlices(pointingHandMask, 1); pointingHandMask = pointingHandMask.Dilate(1); pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; })); pointingHandMask = pointingHandMask.Dilate(1); MCvConnectedComp tmp = new MCvConnectedComp(); CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero); pointingHandMask = pointingHandMask.Convert(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; }); pointingHandMask = pointingHandMask.Erode(1); fillFingerSlices(pointingHandMask, 0); pointingHandMask = pointingHandMask.Erode(2); } private void findLongestPalmContour() { Contour contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findWristDirection() { PointF[] points = new PointF[palmContour.Count()]; int index = 0; foreach (Point p in palmContour) { points[index] = new PointF(p.X, p.Y); ++index; } PointF direction, tmp; PointCollection.Line2DFitting(points, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out direction, out tmp); wristDirection = new Vector2D(-direction.Y, direction.X); } private void findConvexityDefactsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsNearFingerTips() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { float minFingerTipDist = float.MaxValue; foreach (Finger f in fingers) { float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint)); if (dist < minFingerTipDist) minFingerTipDist = dist; } if (minFingerTipDist > 20) newDefects.Add(d); } convexityDefects = newDefects; } private void findWristPoint() { if (convexityDefects.Count > 1) wristPoint = new Vector2D(convexityDefects[1].DepthPoint); else wristPoint = new Vector2D(-1, -1); } private void findWristLine() { findWristPoint(); findWristDirection(); wristLine = new LineSegment2DF(wristPoint - 1000 * wristDirection, wristPoint + 1000 * wristDirection); } private void findThumbLine() { if (convexityDefects.Count > 0) { MCvConvexityDefect thumbDefect = convexityDefects[0]; Vector2D p1 = new Vector2D(thumbDefect.DepthPoint); Vector2D p2 = new Vector2D(thumbDefect.StartPoint); Vector2D direction = (p1 - p2).normalize(); thumbLine = new LineSegment2DF(p1 - 1000 * direction, p1 + 1000 * direction); } else { thumbLine = new LineSegment2DF(new PointF(-1, -1), new PointF(-1, -1)); } } private void removePointsFromContour(LineSegment2DF line, int sideToRemove) { Contour newContour = new Contour(new MemStorage()); int index = 0; foreach (Point p in palmContour) { if (line.Side(p) != sideToRemove) { newContour.Insert(index, p); ++index; } } palmContour = newContour; } private void findPalmRect() { palmRect = palmContour.GetMinAreaRect(); } private void draw() { //outputImage.Draw(palmContour, new Gray(1), 1); PointF[] vertices = palmRect.GetVertices(); for (int i = 0; i < 4; ++i) outputImage.Draw(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), new Gray(1), 1); outputImage.Draw(wristLine, new Gray(1), 1); outputImage.Draw(thumbLine, new Gray(1), 1); } public bool belongsToPalm(int x, int y) { return outputImage.Data[y, x, 0] == 1; } } }