using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Drawing; using Emgu.CV; using Emgu.CV.Structure; using bbiwarg.Utility; using bbiwarg.Images; using bbiwarg.Detectors.Fingers; using bbiwarg.Graphics; namespace bbiwarg.Detectors.Palm { class PalmDetector { private int width, height; private OutputImage outputImage; private EdgeImage edgeImage; private Image handImage; private Image pointingHandMask; private List fingers; private Contour palmContour; private List convexityDefects; private Vector2D thumbDefectStart; private Vector2D thumbDefectEnd; private Vector2D thumbDefectDepth; private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter; private bool valid = false; private Vector2D topLeft; private Vector2D topRight; private Vector2D bottomLeft; private Vector2D bottomRight; public Quadrangle PalmQuad { get; private set; } public PalmDetector() { // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand thumbDefectDepthFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f); thumbDefectStartFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f); thumbDefectEndFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f); } public void findPalmQuad(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage, List trackedFingers) { this.width = depthImage.Width; this.height = depthImage.Height; this.edgeImage = edgeImage; this.outputImage = outputImage; // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1 handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert(); fingers = getFingersWithoutThumb(trackedFingers); buildPointingHandMask(); handImage = handImage.And(pointingHandMask); findLongestPalmContour(); if (palmContour != null) { findConvexityDefectsSortedByDepth(); removeConvexityDefectsCausedByFingers(); findHandPoints(); if (valid) { draw(); } } } public void resetFilters() { thumbDefectDepthFilter.reset(); thumbDefectStartFilter.reset(); thumbDefectEndFilter.reset(); } private List getFingersWithoutThumb(List detectedFingers) { Finger leftMost = null; float minX = float.MaxValue; foreach (Finger f in detectedFingers) { float midX = ((f.Hand + f.Tip) / 2).X; if (midX < minX) { minX = midX; leftMost = f; } } List result = new List(); foreach (Finger f in detectedFingers) { if (f != leftMost) result.Add(f); } return result; } private void fillFingerSlices(Image image, byte val) { foreach (Finger f in fingers) { foreach (FingerSlice s in f.SliceTrail.Slices) { image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1); } } } private Finger getLongestFinger() { float maxLength = 0; Finger longest = null; foreach (Finger f in fingers) { if (f.LineSegment.Length > maxLength) { maxLength = f.LineSegment.Length; longest = f; } } return longest; } private Point getPointInPointingHand() { Finger finger = getLongestFinger(); if (finger == null) return new Point(0, 0); Vector2D direction = (finger.Hand - finger.Tip).normalize(); Vector2D pos = (finger.Hand + 20 * direction).moveInBound(0,0,width-1,height-1); return pos; } public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9; private void buildPointingHandMask() { /*i1 = new OutputImage(width, height); i2 = new OutputImage(width, height); i3 = new OutputImage(width, height); i4 = new OutputImage(width, height); i5 = new OutputImage(width, height); i6 = new OutputImage(width, height); i7 = new OutputImage(width, height); i8 = new OutputImage(width, height); i9 = new OutputImage(width, height);*/ pointingHandMask = new Image(width, height, new Gray(0)); // dst = (src > 0) ? 1 : 0; pointingHandMask = pointingHandMask.Or(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(1))); //i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask; pointingHandMask = pointingHandMask.Dilate(4); //i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask; fillFingerSlices(pointingHandMask, 1); //i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask; //pointingHandMask = pointingHandMask.Dilate(1); //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask; MCvConnectedComp tmp = new MCvConnectedComp(); // fill with value 2 CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero); // dst = (src > 1) ? 0 : 1 (src > 1 <-> src == 2 <-> src filled by flood fill) pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1)); //i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask; pointingHandMask = pointingHandMask.Erode(8); //i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask; fillFingerSlices(pointingHandMask, 0); //i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask; pointingHandMask = pointingHandMask.Erode(2); //i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask; // only debug //i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask); } private void findLongestPalmContour() { i1 = new OutputImage(width, height); i1.Image[0] = i1.Image[1] = i1.Image[2] = handImage * 255; i2 = new OutputImage(width, height); i2.Image[0] = i2.Image[1] = i2.Image[2] = pointingHandMask * 255; Contour contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findConvexityDefectsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsCausedByFingers() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { float minFingerLineDist = float.MaxValue; foreach (Finger f in fingers) { Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f; float dist = f.LineSegment.getDistanceTo(mid); if (dist < minFingerLineDist) minFingerLineDist = dist; } if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance) newDefects.Add(d); } convexityDefects = newDefects; } private MCvConvexityDefect? findThumbDefect() { foreach (MCvConvexityDefect d in convexityDefects) { Vector2D depth = new Vector2D(d.DepthPoint); Vector2D start = new Vector2D(d.StartPoint); Vector2D end = new Vector2D(d.EndPoint); float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI); if (angle <= Constants.PalmMaxThumbDefectAngle) return d; } return null; } private void findHandPoints() { MCvConvexityDefect? thumbDefect = findThumbDefect(); if (convexityDefects.Count > 0 && (thumbDefect != null || thumbDefectDepthFilter.Initialized)) { if (thumbDefect != null) { thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint); thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint); thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint); } if (!thumbDefectDepthFilter.Initialized) { thumbDefectDepthFilter.setInitialPosition(thumbDefectDepth); thumbDefectStartFilter.setInitialPosition(thumbDefectStart); thumbDefectEndFilter.setInitialPosition(thumbDefectEnd); } else if (thumbDefect == null) { thumbDefectDepth = thumbDefectDepthFilter.getPrediction(); thumbDefectStart = thumbDefectStartFilter.getPrediction(); thumbDefectEnd = thumbDefectEndFilter.getPrediction(); } else { thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth); thumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(thumbDefectStart); thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd); } Vector2D handLength, handWidth; if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd)) { //right hand handLength = thumbDefectStart - thumbDefectDepth; handWidth = 0.8f * new Vector2D(-handLength.Y, handLength.X); topLeft = thumbDefectStart; bottomLeft = thumbDefectDepth - 0.4f * handLength; bottomRight = bottomLeft + handWidth; topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth; } else { //left hand handLength = thumbDefectEnd - thumbDefectDepth; handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X); topRight = thumbDefectEnd; bottomRight = thumbDefectDepth - 0.4f * handLength; bottomLeft = bottomRight + handWidth; topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth; } PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight); valid = true; } else { PalmQuad = null; valid = false; } } private void draw() { outputImage.drawContour(palmContour, Constants.PalmConturColor); outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor); if (PalmQuad != null) { outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red); outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Green); outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Blue); //outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1); Vector2D[] vertices = PalmQuad.Vertices; for (int i = 0; i < 4; ++i) outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor); drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3])); } } private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d) { int numRows = 4; int numColumns = 3; Vector2D relAB = (b - a) / numRows; Vector2D relDC = (c - d) / numRows; Vector2D relBC = (c - b) / numColumns; Vector2D relAD = (d - a) / numColumns; for (int i = 1; i < numRows; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor); } for (int i = 1; i < numColumns; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor); } } } }