using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Drawing; using Emgu.CV; using Emgu.CV.Structure; using bbiwarg.Utility; using bbiwarg.Images; using bbiwarg.Detectors.FingerDetection; using bbiwarg.Graphics; namespace bbiwarg.Detectors.PalmDetection { class PalmDetector { private int width, height; private OutputImage outputImage; private EdgeImage edgeImage; private Image handImage; private Image pointingHandMask; private List fingers; private Contour palmContour; private List convexityDefects; private Vector2D thumbDefectStart; private Vector2D thumbDefectEnd; private Vector2D thumbDefectDepth, lastThumbDefectDepth; private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter; private Vector2D topLeft; private Vector2D topRight; private Vector2D bottomLeft; private Vector2D bottomRight; public Quadrangle PalmQuad { get; private set; } public PalmDetector() { // TODO: determine which fingers are index or thumb-fingers and detect palm in thumb-hand thumbDefectDepthFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f); thumbDefectStartFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f); thumbDefectEndFilter = new Kalman2DPositionFilter(1.0e-2f, 1.0e-2f); } public void findPalmQuad(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage, List trackedFingers) { this.width = depthImage.Width; this.height = depthImage.Height; i1 = new OutputImage(width, height); i2 = new OutputImage(width, height); i3 = new OutputImage(width, height); i4 = new OutputImage(width, height); i5 = new OutputImage(width, height); i6 = new OutputImage(width, height); i7 = new OutputImage(width, height); i8 = new OutputImage(width, height); i9 = new OutputImage(width, height); if (trackedFingers.Count >= 2) { this.edgeImage = edgeImage; this.outputImage = outputImage; // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1 handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert(); fingers = getFingersWithoutThumb(trackedFingers); buildPointingHandMask(); handImage = handImage.And(pointingHandMask); findLongestPalmContour(); if (palmContour != null) { findConvexityDefectsSortedByDepth(); removeConvexityDefectsCausedByFingers(); findHandPoints(); } } draw(); } public void reset() { thumbDefectDepthFilter.reset(); thumbDefectStartFilter.reset(); thumbDefectEndFilter.reset(); lastThumbDefectDepth = null; } private List getFingersWithoutThumb(List detectedFingers) { Finger leftMost = null; float minX = float.MaxValue; foreach (Finger f in detectedFingers) { float midX = ((f.HandPoint + f.TipPoint) / 2).X; if (midX < minX) { minX = midX; leftMost = f; } } List result = new List(); foreach (Finger f in detectedFingers) { if (f != leftMost) result.Add(f); } return result; } private void fillFirstFingerSlices(Image image, byte val) { foreach (Finger f in fingers) { image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1); } } private Point getPointInPointingHand() { Finger finger = fingers[0]; if (finger == null) return new Point(0, 0); Vector2D direction = (finger.HandPoint - finger.TipPoint).normalize(); Vector2D pos = finger.HandPoint + direction; while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0) pos += direction; i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red); return pos; } public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9; private void buildPointingHandMask() { pointingHandMask = new Image(width, height, new Gray(0)); // dst = (src > 0) ? 1 : 0; pointingHandMask = pointingHandMask.Or(edgeImage.Image); i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask; pointingHandMask = pointingHandMask.Dilate(1); i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask; fillFirstFingerSlices(pointingHandMask, 1); i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask; //pointingHandMask = pointingHandMask.Dilate(1); //i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask; MCvConnectedComp tmp = new MCvConnectedComp(); // fill with value 2 CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero); // dst = (src > 1) ? 0 : 1 (src > 1 <-> src == 2 <-> src filled by flood fill) pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1)); i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask; pointingHandMask = pointingHandMask.Erode(6); i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask; fillFirstFingerSlices(pointingHandMask, 0); i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask; pointingHandMask = pointingHandMask.Erode(2); i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask; // only debug i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask); } private void findLongestPalmContour() { Contour contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findConvexityDefectsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsCausedByFingers() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { bool fingerIntersectsStartEnd = false; float minFingerLineDist = float.MaxValue; foreach (Finger f in fingers) { Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint)); Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line); if (intersection != null && intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2)) { fingerIntersectsStartEnd = true; break; } Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f; float dist = f.LineSegment.getDistanceTo(mid); if (dist < minFingerLineDist) minFingerLineDist = dist; } if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !fingerIntersectsStartEnd) newDefects.Add(d); } convexityDefects = newDefects; } private int i = 0; private MCvConvexityDefect? findThumbDefect() { foreach (MCvConvexityDefect d in convexityDefects) { Vector2D depth = new Vector2D(d.DepthPoint); Vector2D start = new Vector2D(d.StartPoint); Vector2D end = new Vector2D(d.EndPoint); float l1 = (depth - start).Length; float l2 = (depth - end).Length; float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2); float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI); if (angle <= Constants.PalmMaxThumbDefectAngle && lengthQuotient >= Constants.PalmMinThumbDefectLengthQuotient && lengthQuotient <= Constants.PalmMaxThumbDefectLengthQuotient) { return d; } } //Console.WriteLine("no palm defect found (" + i + ")"); foreach (MCvConvexityDefect d in convexityDefects) { Vector2D depth = new Vector2D(d.DepthPoint); Vector2D start = new Vector2D(d.StartPoint); Vector2D end = new Vector2D(d.EndPoint); float l1 = (depth - start).Length; float l2 = (depth - end).Length; float lengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2); float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI); //Console.WriteLine("angle: " + angle + " quotient: " + lengthQuotient); } ++i; return null; } private void findHandPoints() { MCvConvexityDefect? thumbDefect = findThumbDefect(); if (convexityDefects.Count > 0 && (thumbDefect != null || thumbDefectDepthFilter.Initialized)) { if (thumbDefect != null) { thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint); thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint); thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint); } if (!thumbDefectDepthFilter.Initialized) { thumbDefectDepthFilter.setInitialPosition(thumbDefectDepth); thumbDefectStartFilter.setInitialPosition(thumbDefectStart); thumbDefectEndFilter.setInitialPosition(thumbDefectEnd); } else { thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth); thumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(thumbDefectStart); thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd); } lastThumbDefectDepth = thumbDefectDepth; Vector2D handLength, handWidth; if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd)) { //right hand handLength = thumbDefectStart - thumbDefectDepth; handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X); topLeft = thumbDefectStart+0.15f*handLength; bottomLeft = thumbDefectDepth - 0.4f * handLength; bottomRight = bottomLeft + handWidth; topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth; } else { //left hand handLength = thumbDefectEnd - thumbDefectDepth; handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X); topRight = thumbDefectEnd+0.15f*handLength; bottomRight = thumbDefectDepth - 0.4f * handLength; bottomLeft = bottomRight + handWidth; topLeft = bottomLeft + 1.15f * handLength - 0.35f * handWidth; } PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight); } } private void draw() { if (palmContour != null && palmContour.Count() > 0) { outputImage.drawContour(palmContour, Constants.PalmConturColor); outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor); } if (PalmQuad != null) { outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red); outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red); outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red); outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1); Vector2D[] vertices = PalmQuad.Vertices; for (int i = 0; i < 4; ++i) outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor); drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3])); } } private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d) { Vector2D relAB = (b - a) / Constants.PalmGridRows; Vector2D relDC = (c - d) / Constants.PalmGridRows; Vector2D relBC = (c - b) / Constants.PalmGridColumns; Vector2D relAD = (d - a) / Constants.PalmGridColumns; for (int i = 1; i < Constants.PalmGridRows; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor); } for (int i = 1; i < Constants.PalmGridColumns; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor); } } } }