using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Drawing; using Emgu.CV; using Emgu.CV.Structure; using bbiwarg.Utility; using bbiwarg.Images; using bbiwarg.Recognition.FingerRecognition; using bbiwarg.Recognition.HandRecognition; using bbiwarg.Graphics; namespace bbiwarg.Recognition.PalmRecognition { class PalmDetector { private OutputImage outputImage; private Hand palmHand, pointingHand; private Contour palmContour; private List convexityDefects; private Vector2D thumbDefectStart; private Vector2D thumbDefectEnd; private Vector2D thumbDefectDepth; private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter; private int numFramesNoHandFound; private Quadrangle lastPalmQuad; private List hands; public Quadrangle PalmQuad { get; private set; } public Hand.HandSide PalmHandSide { get; private set; } public PalmDetector() { thumbDefectDepthFilter = new Kalman2DPositionFilter(Constants.PalmThumbDefectmXX, Constants.PalmThumbDefectmXY, Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise); thumbDefectStartFilter = new Kalman2DPositionFilter(Constants.PalmThumbDefectmXX, Constants.PalmThumbDefectmXY, Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise); thumbDefectEndFilter = new Kalman2DPositionFilter(Constants.PalmThumbDefectmXX, Constants.PalmThumbDefectmXY, Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise); } public void findPalmQuad(OutputImage outputImage, List hands, Image foregroundMask) { this.outputImage = outputImage; this.hands = hands; if (hands.Count == 1 && hands[0].Fingers.Count == 1) { palmHand = hands[0]; pointingHand = null; } else if (hands.Count == 2) { if (hands[0].Fingers.Count > 1) { pointingHand = hands[0]; palmHand = hands[1]; } else if (hands[1].Fingers.Count > 1) { pointingHand = hands[1]; palmHand = hands[0]; } else if (hands[0].Fingers[0].LineSegment.Length > hands[1].Fingers[0].LineSegment.Length) { pointingHand = hands[0]; palmHand = hands[1]; } else { pointingHand = hands[1]; palmHand = hands[0]; } } if ((hands.Count == 1 && hands[0].Fingers.Count == 1) || hands.Count == 2) { findLongestPalmContour(); findConvexityDefectsSortedByDepth(); if (pointingHand != null) removeConvexityDefectsCausedByFingers(); findHandPoints(foregroundMask); } if (hands.Count == 0) { ++numFramesNoHandFound; if (numFramesNoHandFound == Constants.PalmNumFramesNoHandReset) reset(); } else { numFramesNoHandFound = 0; } draw(); } public void reset() { thumbDefectDepthFilter.reset(); thumbDefectStartFilter.reset(); thumbDefectEndFilter.reset(); palmContour = null; PalmQuad = null; PalmHandSide = Hand.HandSide.Left; numFramesNoHandFound = 0; lastPalmQuad = null; } private void findLongestPalmContour() { Contour contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL); palmContour = contour; double maxPerimeter = 0; while (contour != null) { if (contour.Perimeter > maxPerimeter) { maxPerimeter = contour.Perimeter; palmContour = contour; } contour = contour.HNext; } } private void findConvexityDefectsSortedByDepth() { convexityDefects = new List(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE)); convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) { if (d1.Depth < d2.Depth) return 1; else if (d1.Depth > d2.Depth) return -1; return 0; }); } private void removeConvexityDefectsCausedByFingers() { List newDefects = new List(); foreach (MCvConvexityDefect d in convexityDefects) { bool fingerIntersectsStartEnd = false; float minFingerLineDist = float.MaxValue; foreach (Finger f in pointingHand.Fingers) { Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint)); Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line); if (intersection != null && intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2)) { fingerIntersectsStartEnd = true; break; } Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f; float dist = f.LineSegment.getDistanceTo(mid); if (dist < minFingerLineDist) minFingerLineDist = dist; } if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !fingerIntersectsStartEnd) newDefects.Add(d); } convexityDefects = newDefects; } private MCvConvexityDefect? findThumbDefect() { foreach (MCvConvexityDefect d in convexityDefects) { Vector2D depth = new Vector2D(d.DepthPoint); Vector2D start = new Vector2D(d.StartPoint); Vector2D end = new Vector2D(d.EndPoint); float angle = (float)((depth - start).getAngleBetween(depth - end) * 180 / Math.PI); float l1 = (depth - start).Length; float l2 = (depth - end).Length; float startEndLengthQuotient = Math.Max(l1, l2) / Math.Min(l1, l2); float depthThumbLengthQuotient = d.Depth / palmHand.Fingers[0].LineSegment.Length; if (angle <= Constants.PalmMaxThumbDefectAngle && startEndLengthQuotient >= Constants.PalmMinThumbDefectStartEndLengthQuotient && startEndLengthQuotient <= Constants.PalmMaxThumbDefectStartEndLengthQuotient && depthThumbLengthQuotient >= Constants.PalmMinTumbDefectDepthThumbLengthQuotient && depthThumbLengthQuotient <= Constants.PalmMaxTumbDefectDepthThumbLengthQuotient) { return d; } } return null; } float getForegroundPixelPercentage(Quadrangle quad, Image foregroundMask) { float numPixelsInMask = quad.Mask.CountNonzero()[0]; float numPixelsInMaskInForeground = (quad.Mask & foregroundMask).CountNonzero()[0]; return numPixelsInMaskInForeground / numPixelsInMask; } private void findHandPoints(Image foregroundMask) { MCvConvexityDefect? thumbDefect = findThumbDefect(); if (thumbDefect != null) { thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint); thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint); thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint); if (!thumbDefectDepthFilter.Initialized) { thumbDefectDepthFilter.setInitialPosition(thumbDefectDepth); thumbDefectStartFilter.setInitialPosition(thumbDefectStart); thumbDefectEndFilter.setInitialPosition(thumbDefectEnd); } else { thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth); thumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(thumbDefectStart); thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd); } Vector2D handLength, handWidth, longestLineEndpoint, topLeft, bottomLeft, bottomRight, topRight; Vector2D startDepth = thumbDefectStart - thumbDefectDepth; Vector2D endDepth = thumbDefectEnd - thumbDefectDepth; if (startDepth.Length > endDepth.Length) { handLength = startDepth; longestLineEndpoint = thumbDefectStart; if (hands.Count == 1) { if (thumbDefectStart.X > thumbDefectDepth.X) hands[0].Side = Hand.HandSide.Left; else hands[0].Side = Hand.HandSide.Right; } } else { handLength = endDepth; longestLineEndpoint = thumbDefectEnd; if (hands.Count == 1) { if (thumbDefectEnd.X > thumbDefectDepth.X) hands[0].Side = Hand.HandSide.Left; else hands[0].Side = Hand.HandSide.Right; } } if (palmHand.Side == Hand.HandSide.Left) { handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X); topLeft = longestLineEndpoint + 0.15f * handLength; bottomLeft = thumbDefectDepth - 0.4f * handLength; bottomRight = bottomLeft + handWidth; topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth; } else { handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X); topRight = longestLineEndpoint + 0.15f * handLength; bottomRight = thumbDefectDepth - 0.4f * handLength; bottomLeft = bottomRight + handWidth; topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth; } Quadrangle quad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight, foregroundMask.Width, foregroundMask.Height); if ((lastPalmQuad == null || (quad.Area / lastPalmQuad.Area >= Constants.PalmMinAreaQuotient && quad.Area / lastPalmQuad.Area <= Constants.PalmMaxAreaQuotient)) && getForegroundPixelPercentage(quad, foregroundMask) >= Constants.PalmMinPrecentageQuadForeground) { PalmQuad = quad; PalmHandSide = palmHand.Side; lastPalmQuad = PalmQuad; } } if (lastPalmQuad != null && getForegroundPixelPercentage(lastPalmQuad, foregroundMask) <= Constants.PalmMaxPrecentageQuadForegroundReset) reset(); } private void draw() { if (palmContour != null && palmContour.Count() > 0) { outputImage.drawContour(palmContour, Constants.PalmConturColor); outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor); } if (PalmQuad != null) { outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red); outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red); outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red); outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1); Vector2D[] vertices = PalmQuad.Vertices; for (int i = 0; i < 4; ++i) outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor); drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3])); } } private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d) { Vector2D relAB = (b - a) / Constants.PalmGridRows; Vector2D relDC = (c - d) / Constants.PalmGridRows; Vector2D relBC = (c - b) / Constants.PalmGridColumns; Vector2D relAD = (d - a) / Constants.PalmGridColumns; for (int i = 1; i < Constants.PalmGridRows; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor); } for (int i = 1; i < Constants.PalmGridColumns; i++) { outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor); } } } }