using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Drawing; using Emgu.CV; using Emgu.CV.Structure; using bbiwarg.Utility; using bbiwarg.Images; using bbiwarg.Detectors.Fingers; using bbiwarg.Graphics; namespace bbiwarg.Detectors.Palm { class PalmDetector { private int width, height; private DepthImage depthImage; private EdgeImage edgeImage; private OutputImage outputImage; private Image handImage; private Image pointingHandMask; private List fingers; private Contour palmContour; private List convexityDefects; private static Kalman2DPositionFilter thumbDefactFilter; private bool valid = false; private Vector2D topLeft; private Vector2D topRight; private Vector2D bottomLeft; private Vector2D bottomRight; public Quadrangle PalmQuad; public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, List detectedFingers, OutputImage outputImage) { // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand width = depthImage.Width; height = depthImage.Height; this.depthImage = depthImage; this.edgeImage = edgeImage; this.outputImage = outputImage; // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1 handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert(); fingers = getFingersWithoutThumb(detectedFingers); buildPointingHandMask(); handImage = handImage.And(pointingHandMask); findLongestPalmContour(); if (palmContour != null) { findConvexityDefactsSortedByDepth(); removeConvexityDefectsNearFingerTips(); findHandPoints(); if (valid) { draw(); } } } public static void resetFilter() { thumbDefactFilter = null; } private List getFingersWithoutThumb(List detectedFingers) { Finger leftMost = null; float minX = float.MaxValue; foreach (Finger f in detectedFingers) { float midX = ((f.Hand + f.Tip) / 2).X; if (midX < minX) { minX = midX; leftMost = f; } } List result = new List(); foreach (Finger f in detectedFingers) { if (f != leftMost) result.Add(f); } return result; } private void fillFingerSlices(Image image, byte val) { foreach (Finger f in fingers) { foreach (FingerSlice s in f.SliceTrail.Slices) { image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1); } } } private Finger getLongestFinger() { float maxLength = 0; Finger longest = null; foreach (Finger f in fingers) { if (f.LineSegment.Length > maxLength) { maxLength = f.LineSegment.Length; longest = f; } } return longest; } private Point getPointInPointingHand() { Finger finger = getLongestFinger(); if (finger == null) return new Point(0, 0); Vector2D direction = (finger.Hand - finger.Tip).normalize(); Vector2D pos = finger.Hand + 20 * direction; return new Point(HelperFunctions.thresholdRange(0, width - 1, pos.IntX), HelperFunctions.thresholdRange(0, height - 1, pos.IntY)); } private void buildPointingHandMask() { pointingHandMask = new Image(width, height, new Gray(0)); fillFingerSlices(pointingHandMask, 1); pointingHandMask = pointingHandMask.Dilate(1); // dst = (src > 0) ? 1 : 0; pointingHandMask = pointingHandMask.Or(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(1))); pointingHandMask = pointingHandMask.Dilate(1); MCvConnectedComp tmp = new MCvConnectedComp(); // fill with value 2 CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero); // dst = (src > 1) ? 0 : 1 (src > 1 <-> src == 2 <-> src filled by flood fill) pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1)); pointingHandMask = pointingHandMask.Erode(1); fillFingerSlices(pointingHandMask, 0); pointingHandMask = pointingHandMask.Erode(2); } private void findLongestPalmContour() { Contour contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findConvexityDefactsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsNearFingerTips() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { float minFingerTipDist = float.MaxValue; foreach (Finger f in fingers) { float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint)); if (dist < minFingerTipDist) minFingerTipDist = dist; } if (minFingerTipDist > 20) newDefects.Add(d); } convexityDefects = newDefects; } private void findHandPoints() { if (convexityDefects.Count > 0) { MCvConvexityDefect thumbDefect = convexityDefects[0]; Vector2D thumb; if (thumbDefactFilter == null) { thumb = new Vector2D(thumbDefect.DepthPoint); thumbDefactFilter = new Kalman2DPositionFilter(thumb, 1.0e-2f, 1.0e-2f); } else { thumb = thumbDefactFilter.getCorrectedPosition(new Vector2D(thumbDefect.DepthPoint)); } Vector2D thumbDefectStart = new Vector2D(thumbDefect.StartPoint); Vector2D thumbDefectEnd = new Vector2D(thumbDefect.EndPoint); Vector2D handLength, handWidth; if (thumb.getDistanceTo(thumbDefectStart) > thumb.getDistanceTo(thumbDefectEnd)) { //right hand handLength = thumbDefectStart - thumb; handWidth = 0.8f * new Vector2D(-handLength.Y, handLength.X); topLeft = thumbDefectStart; bottomLeft = thumb - 0.4f * handLength; bottomRight = bottomLeft + handWidth; topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth; } else { //left hand handLength = thumbDefectEnd - thumb; handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X); topRight = thumbDefectEnd; bottomRight = thumb - 0.4f * handLength; bottomLeft = bottomRight + handWidth; topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth; } PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight); valid = true; } else { PalmQuad = null; valid = false; } } private void draw() { outputImage.drawContour(palmContour, 255, 0, 0); outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), 0, 255, 0); if (PalmQuad != null) { Vector2D[] vertices = PalmQuad.Vertices; for (int i = 0; i < 4; ++i) outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), 0, 0, 255); drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3])); } } private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d) { int numRows = 4; int numColumns = 3; Vector2D relAB = (b - a) / numRows; Vector2D relDC = (c - d) / numRows; Vector2D relBC = (c - b) / numColumns; Vector2D relAD = (d - a) / numColumns; for (int i = 1; i < numRows; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), 80, 80, 255); } for (int i = 1; i < numColumns; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), 80, 80, 255); } } } }