using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Drawing; using Emgu.CV; using Emgu.CV.Structure; using bbiwarg.Utility; using bbiwarg.Images; using bbiwarg.Detectors.Fingers; namespace bbiwarg.Detectors.Palm { class PalmDetector { private int width, height; private DepthImage depthImage; private EdgeImage edgeImage; private PalmImage palmImage; private Image handImage; private Image pointingHandMask; private List fingers; private Contour palmContour; private List convexityDefects; private Quadrangle palmQuad; private bool valid = false; private Vector2D topLeft; private Vector2D topRight; private Vector2D bottomLeft; private Vector2D bottomRight; public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, List detectedFingers, PalmImage palmImage) { width = depthImage.getWidth(); height = depthImage.getHeight(); this.depthImage = depthImage; this.edgeImage = edgeImage; this.palmImage = palmImage; handImage = depthImage.getImage().Convert(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte)0 : (byte)1; }); fingers = getFingersWithoutThumb(detectedFingers); buildPointingHandMask(); handImage = handImage.And(pointingHandMask); findLongestPalmContour(); if (palmContour != null) { findConvexityDefactsSortedByDepth(); removeConvexityDefectsNearFingerTips(); findHandPoints(); if (valid) { draw(); } } } public Quadrangle getPalmQuad() { return palmQuad; } private List getFingersWithoutThumb(List detectedFingers) { Finger leftMost = null; float minX = float.MaxValue; foreach (Finger f in detectedFingers) { float midX = ((f.Hand + f.Tip) / 2).X; if (midX < minX) { minX = midX; leftMost = f; } } List result = new List(); foreach (Finger f in detectedFingers) { if (f != leftMost) result.Add(f); } return result; } private void fillFingerSlices(Image image, byte val) { foreach (Finger f in fingers) { foreach (FingerSlice s in f.SliceTrail.getSlices()) { image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1); } } } private Finger getLongestFinger() { float maxLength = 0; Finger longest = null; foreach (Finger f in fingers) { if (f.LineSegment.Length > maxLength) { maxLength = f.LineSegment.Length; longest = f; } } return longest; } private Point getPointInPointingHand() { Finger finger = getLongestFinger(); if (finger == null) return new Point(0, 0); Vector2D direction = (finger.Hand - finger.Tip).normalize(); Vector2D pos = finger.Hand + 20 * direction; return new Point(HelperFunctions.thresholdRange(0, width - 1, pos.IntX), HelperFunctions.thresholdRange(0, height - 1, pos.IntY)); } private void buildPointingHandMask() { pointingHandMask = new Image(width, height, new Gray(0)); fillFingerSlices(pointingHandMask, 1); pointingHandMask = pointingHandMask.Dilate(1); pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; })); pointingHandMask = pointingHandMask.Dilate(1); MCvConnectedComp tmp = new MCvConnectedComp(); CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero); pointingHandMask = pointingHandMask.Convert(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; }); pointingHandMask = pointingHandMask.Erode(1); fillFingerSlices(pointingHandMask, 0); pointingHandMask = pointingHandMask.Erode(2); } private void findLongestPalmContour() { Contour contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findConvexityDefactsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsNearFingerTips() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { float minFingerTipDist = float.MaxValue; foreach (Finger f in fingers) { float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint)); if (dist < minFingerTipDist) minFingerTipDist = dist; } if (minFingerTipDist > 20) newDefects.Add(d); } convexityDefects = newDefects; } private void findHandPoints() { if (convexityDefects.Count > 0) { MCvConvexityDefect thumbDefect = convexityDefects[0]; Vector2D thumb = new Vector2D(thumbDefect.DepthPoint); Vector2D thumbDefectStart = new Vector2D(thumbDefect.StartPoint); Vector2D thumbDefectEnd = new Vector2D(thumbDefect.EndPoint); Vector2D handLength, handWidth; if (thumb.getDistanceTo(thumbDefectStart) > thumb.getDistanceTo(thumbDefectEnd)) { //right hand handLength = thumbDefectStart - thumb; handWidth = 0.8f * new Vector2D(-handLength.Y, handLength.X); topLeft = thumbDefectStart; bottomLeft = thumb - 0.4f * handLength; bottomRight = bottomLeft + handWidth; topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth; } else { //left hand handLength = thumbDefectEnd - thumb; handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X); topRight = thumbDefectEnd; bottomRight = thumb - 0.4f * handLength; bottomLeft = bottomRight + handWidth; topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth; } palmQuad = new Quadrangle(new Vector2D[] { bottomLeft, topLeft, topRight, bottomRight }); valid = true; } else { palmQuad = null; valid = false; } } private void draw() { palmImage.drawContour(palmContour); if (palmQuad != null) { Vector2D[] vertices = palmQuad.Vertices; for (int i = 0; i < 4; ++i) palmImage.drawLine(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), PalmImageState.palmRect); palmImage.drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3])); } } } }