using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Drawing; using Emgu.CV; using Emgu.CV.Structure; using bbiwarg.Utility; using bbiwarg.Images; using bbiwarg.Detectors.Fingers; namespace bbiwarg.Detectors.Palm { class PalmDetector { private int width, height; private DepthImage depthImage; private EdgeImage edgeImage; private PalmImage palmImage; private Image handImage; private Image pointingHandMask; private List fingers; private Contour palmContour; private List convexityDefects; private Vector2D wristPoint, wristDirection; private LineSegment2DF wristLine, thumbLine; private Quad2D palmRect; public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector, PalmImage palmImage) { width = depthImage.getWidth(); height = depthImage.getHeight(); this.depthImage = depthImage; this.edgeImage = edgeImage; this.palmImage = palmImage; handImage = depthImage.getImage().Convert(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte)0 : (byte)1; }); fingers = getFingersWithoutThumb(fingerDetector); buildPointingHandMask(); handImage = handImage.And(pointingHandMask); findLongestPalmContour(); if (palmContour != null) { findConvexityDefactsSortedByDepth(); removeConvexityDefectsNearFingerTips(); findThumbLine(); findWristLine(); removePointsFromContour(wristLine, 1); removePointsFromContour(thumbLine, 1); if (palmContour.Count() != 0) { findPalmRect(); draw(); } } } public PalmRect getPalm() { return new PalmRect(palmRect); } private List getFingersWithoutThumb(FingerDetector fingerDetector) { List detectedFingers = fingerDetector.Fingers; Finger leftMost = null; float minX = float.MaxValue; foreach (Finger f in detectedFingers) { float midX = ((f.Hand + f.Tip) / 2).X; if (midX < minX) { minX = midX; leftMost = f; } } List result = new List(); foreach (Finger f in detectedFingers) { if (f != leftMost) result.Add(f); } return result; } private void fillFingerSlices(Image image, byte val) { foreach (Finger f in fingers) { foreach (FingerSlice s in f.SliceTrail.getSlices()) { image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1); } } } private Finger getLongestFinger() { float maxLength = 0; Finger longest = null; foreach (Finger f in fingers) { if (f.Line.Length > maxLength) { maxLength = f.Line.Length; longest = f; } } return longest; } private Point getPointInPointingHand() { Finger finger = getLongestFinger(); if (finger == null) return new Point(0, 0); Vector2D direction = (finger.Hand - finger.Tip).normalize(); Vector2D pos = finger.Hand + 20 * direction; return new Point(HelperFunctions.thresholdRange(0, width - 1, (int) pos.X), HelperFunctions.thresholdRange(0, height - 1, (int) pos.Y)); } private void buildPointingHandMask() { pointingHandMask = new Image(width, height, new Gray(0)); fillFingerSlices(pointingHandMask, 1); pointingHandMask = pointingHandMask.Dilate(1); pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; })); pointingHandMask = pointingHandMask.Dilate(1); MCvConnectedComp tmp = new MCvConnectedComp(); CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero); pointingHandMask = pointingHandMask.Convert(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; }); pointingHandMask = pointingHandMask.Erode(1); fillFingerSlices(pointingHandMask, 0); pointingHandMask = pointingHandMask.Erode(2); } private void findLongestPalmContour() { Contour contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findWristDirection() { PointF[] points = new PointF[palmContour.Count()]; int index = 0; foreach (Point p in palmContour) { points[index] = new PointF(p.X, p.Y); ++index; } PointF direction, tmp; PointCollection.Line2DFitting(points, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out direction, out tmp); //Vector2D direction = new Vector2D(thumbLine.Direction); wristDirection = new Vector2D(-direction.Y, direction.X); } private void findConvexityDefactsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsNearFingerTips() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { float minFingerTipDist = float.MaxValue; foreach (Finger f in fingers) { float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint)); if (dist < minFingerTipDist) minFingerTipDist = dist; } if (minFingerTipDist > 20) newDefects.Add(d); } convexityDefects = newDefects; } private void findWristPoint() { if (convexityDefects.Count > 1) wristPoint = new Vector2D(convexityDefects[1].DepthPoint); else wristPoint = new Vector2D(-1, -1); } private void findWristLine() { findWristPoint(); findWristDirection(); wristLine = new LineSegment2DF(wristPoint - 1000 * wristDirection, wristPoint + 1000 * wristDirection); } private void findThumbLine() { if (convexityDefects.Count > 0) { MCvConvexityDefect thumbDefect = convexityDefects[0]; Vector2D p1 = new Vector2D(thumbDefect.DepthPoint); Vector2D p2 = new Vector2D(thumbDefect.StartPoint); Vector2D direction = (p1 - p2).normalize(); thumbLine = new LineSegment2DF(p1 - 1000 * direction, p1 + 1000 * direction); } else { thumbLine = new LineSegment2DF(new PointF(-1, -1), new PointF(-1, -1)); } } private void removePointsFromContour(LineSegment2DF line, int sideToRemove) { Contour newContour = new Contour(new MemStorage()); int index = 0; foreach (Point p in palmContour) { if (line.Side(p) != sideToRemove) { newContour.Insert(index, p); ++index; } } palmContour = newContour; } private void findPalmRect() { //palmRect = palmContour.GetMinAreaRect(); Line2D thumbLine2 = new Line2D(new Vector2D(thumbLine.P1), new Vector2D(thumbLine.P2)); Line2D wristLine2 = new Line2D(new Vector2D(wristLine.P1), new Vector2D(wristLine.P2)); Vector2D intersection = thumbLine2.intersection(wristLine2); if (intersection != null) { Vector2D maxThumb = new Vector2D(0, 0), maxWrist = new Vector2D(0, 0); float maxDistanceThumb = 0, maxDistanceWrist = 0; foreach (Point p in palmContour) { Vector2D v = new Vector2D(p); Vector2D projected = thumbLine2.projectToLine(v); float dist = projected.getDistanceTo(intersection); if (dist > maxDistanceThumb) { maxDistanceThumb = dist; maxThumb = projected; } projected = wristLine2.projectToLine(v); dist = projected.getDistanceTo(intersection); if (dist > maxDistanceWrist) { maxDistanceWrist = dist; maxWrist = projected; } } Vector2D origin = intersection; palmRect = new Quad2D(origin, (maxThumb - origin), (maxWrist - origin)); } else { palmRect = null; } } private void draw() { palmImage.drawContour(palmContour); palmImage.drawLine(wristLine, PalmImageState.wristLine); palmImage.drawLine(thumbLine, PalmImageState.thumbLine); if (palmRect != null) { PointF[] vertices = palmRect.getVertices(); for (int i = 0; i < 4; ++i) palmImage.drawLine(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), PalmImageState.palmRect); palmImage.drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3])); } } } }