using System; using System.Collections.Generic; using System.Diagnostics; using System.Drawing; using System.Linq; using System.Text; using System.Threading.Tasks; using bbiwarg.Images; using bbiwarg.Utility; using Emgu.CV.Structure; using Emgu.CV; using bbiwarg.Graphics; namespace bbiwarg.Detectors.FingerDetection { class FingerDetector { private DepthImage depthImage; private EdgeImage edgeImageOriginal; private EdgeImage edgeImageAdapted; private OutputImage outputImage; public List Fingers { get; private set; } public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage) { this.depthImage = depthImage; this.edgeImageOriginal = edgeImage; this.edgeImageAdapted = edgeImage.copy(); this.outputImage = outputImage; findFingers(); } private void findFingers() { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; Fingers = new List(); for (int y = 1; y < maxY; y += 5) //y++ for 100% coverage, but y+=5 for 99% coverage and 3 times better perfomance { for (int x = 1; x < maxX; x++) { if (edgeImageAdapted.isEdgeAt(x, y)) { Vector2D edgePoint = new Vector2D(x, y); Vector2D edgeDirection = getEdgeDirection(edgePoint); if (edgeDirection != null) { Vector2D dir = edgeDirection.getOrthogonal(true); if (depthImage.getDepthAt(edgePoint - dir) < depthImage.getDepthAt(edgePoint + dir)) dir = dir.getInverse(); FingerSlice slice = findFingerSliceFromStartEdge(edgePoint, dir); if (slice != null) { FingerSliceTrail trail = findFingerSliceTrail(slice, edgeDirection); if (trail != null && trail.NumSlices > Constants.FingerMinNumSlices) createFingerFromTrail(trail); } } } } } } private Vector2D getEdgeDirection(Vector2D edgePoint) { int x = edgePoint.IntX; int y = edgePoint.IntY; if (edgeImageAdapted.isEdgeAt(x, y - 1) && edgeImageAdapted.isEdgeAt(x, y + 1)) return new Vector2D(0, 1); else if (edgeImageAdapted.isEdgeAt(x - 1, y) && edgeImageAdapted.isEdgeAt(x + 1, y)) return new Vector2D(1, 0); else if (edgeImageAdapted.isEdgeAt(x - 1, y - 1) && edgeImageAdapted.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize(); else if (edgeImageAdapted.isEdgeAt(x + 1, y - 1) && edgeImageAdapted.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize(); else return null; } private FingerSliceTrail findFingerSliceTrail(FingerSlice startSlice, Vector2D startDirection) { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; FingerSliceTrail trail = new FingerSliceTrail(startSlice); Vector2D direction = startDirection; Vector2D position = startSlice.Mid + Constants.FingerStepSize * direction; if (position.isWithin(0, 0, maxX, maxY)) { FingerSlice nextSlice = findFingerSliceFromMid(position, direction); if (nextSlice != null) { trail.addSlice(nextSlice); trail = expandTrail(trail); if (trail.NumSlices > Constants.FingerMinNumSlices) { trail.Slices.RemoveRange(0, Constants.FingerRemoveNumSlicesForCorrection); trail.Slices.Reverse(); trail = expandTrail(trail); trail.Slices.Reverse(); return trail; } } } return null; } private FingerSliceTrail expandTrail(FingerSliceTrail trail) { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; Vector2D currentDirection = trail.getEndDirection(); Vector2D currentPosition = trail.End.Mid + Constants.FingerStepSize * currentDirection; int gapCounter = 0; int numSlices = trail.NumSlices; FingerSlice lastSlice = trail.End; FingerSlice nextSlice; while (currentPosition.isWithin(0, 0, maxX, maxY) && gapCounter <= Math.Min(numSlices, Constants.FingerMaxGapCounter)) { nextSlice = findFingerSliceFromMid(currentPosition, currentDirection); if (nextSlice != null && Math.Abs(nextSlice.Length - lastSlice.Length) <= Constants.FingerMaxSliceDifferencePerStep) { gapCounter = 0; numSlices++; trail.addSlice(nextSlice); currentDirection = trail.getEndDirection(); currentPosition = nextSlice.Mid + Constants.FingerStepSize * currentDirection; lastSlice = nextSlice; } else { gapCounter++; currentPosition += currentDirection; } } return trail; } private FingerSlice findFingerSliceFromMid(Vector2D position, Vector2D direction) { if (edgeImageAdapted.isRoughEdgeAt(position)) return null; Vector2D dirStart = direction.getOrthogonal(true); Vector2D dirEnd = direction.getOrthogonal(false); Vector2D start = findNextEdge(position, dirStart); if (start == null) return null; Vector2D end = findNextEdge(position, dirEnd); if (end == null) return null; return getFingerSlice(start, end); } private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction) { Vector2D end = findNextEdge(start+3*direction, direction); if (end == null) return null; return getFingerSlice(start, end); } private Vector2D findNextEdge(Vector2D start, Vector2D direction, bool adaptedEdgeImage = true, bool stopAtMaxFingerSize = true, bool returnBoundIfNoEdge = false) { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; int maxStepsX; if (direction.X > 0) maxStepsX = (int)((maxX - start.X) / direction.X); else if (direction.X < 0) maxStepsX = (int)(start.X / Math.Abs(direction.X)); else maxStepsX = int.MaxValue; int maxStepsY; if (direction.Y > 0) maxStepsY = (int)((maxY - start.Y) / direction.Y); else if (direction.Y < 0) maxStepsY = (int)(start.Y / Math.Abs(direction.Y)); else maxStepsY = int.MaxValue; int maxSteps = Math.Min(maxStepsX, maxStepsY); if (stopAtMaxFingerSize) maxSteps = Math.Min(maxSteps, (int)(Constants.FingerMaxSize / direction.Length)); Vector2D end = new Vector2D(start); for (int i = 0; i < maxSteps; i++) { end += direction; if ((adaptedEdgeImage && edgeImageAdapted.isRoughEdgeAt(end)) || (!adaptedEdgeImage && edgeImageOriginal.isRoughEdgeAt(end))) { return end; } } if (returnBoundIfNoEdge) return end; else return null; } private FingerSlice getFingerSlice(Vector2D start, Vector2D end) { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; Vector2D direction = (end - start).normalize(); Vector2D beforeStart = (start - direction).moveInBound(0, 0, maxX, maxY); Vector2D behindEnd = (end + direction).moveInBound(0, 0, maxY, maxY); FingerSlice slice = new FingerSlice(beforeStart, behindEnd); if (slice.Length >= Constants.FingerMinSize && slice.Length <= Constants.FingerMaxSize && fingerSliceDepthTest(slice)) return slice; return null; } private bool fingerSliceDepthTest(FingerSlice fingerSlice) { Int16 depthStart = depthImage.getDepthAt(fingerSlice.Start); Int16 depthMid = depthImage.getDepthAt(fingerSlice.Mid); Int16 depthEnd = depthImage.getDepthAt(fingerSlice.End); return (depthStart > depthMid && depthMid < depthEnd); } private void createFingerFromTrail(FingerSliceTrail trail) { //bring finger in correct direction Tip->Hand trail = orderTrailTipToHand(trail); //create finger Finger finger = new Finger(trail); //add finger Fingers.Add(finger); //draw finger drawDetectedFinger(finger); //remove edges around detected finger to improve performance Point[] polygon = finger.getBoundingPolygon(); edgeImageAdapted.removeEdgesInsidePolygon(polygon); } private FingerSlice findOutSlice(Vector2D start, Vector2D direction) { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; Vector2D dirOrth1 = direction.getOrthogonal(true); Vector2D dirOrth2 = direction.getOrthogonal(false); Vector2D outPoint = (start + Constants.FingerOutSliceFactor * Constants.FingerStepSize * direction).moveInBound(0, 0, maxX, maxY); Vector2D p1 = findNextEdge(outPoint, dirOrth1, false, false, true); Vector2D p2 = findNextEdge(outPoint, dirOrth2, false, false, true); FingerSlice slice = new FingerSlice(p1, p2); outputImage.drawLineSegment(slice.LineSegment, Constants.FingerOutSliceColor); return slice; } private FingerSliceTrail orderTrailTipToHand(FingerSliceTrail trail) { int maxX = depthImage.Width - 1; int maxY = depthImage.Height - 1; FingerSlice start = trail.Start; FingerSlice end = trail.End; Vector2D direction = (end.Mid - start.Mid).normalize(); FingerSlice startOutSlice = findOutSlice(start.Mid, -1 * direction); FingerSlice endOutSlice = findOutSlice(end.Mid, direction); float startOutLength = float.MaxValue; float endOutLength = float.MaxValue; if (!startOutSlice.Start.isOnBound(0, 0, maxX, maxY) && startOutSlice.End.isOnBound(0, 0, maxX, maxY)) startOutLength = startOutSlice.Length; if (!endOutSlice.Start.isOnBound(0, 0, maxX, maxY) && endOutSlice.End.isOnBound(0, 0, maxX, maxY)) endOutLength = endOutSlice.Length; if (startOutLength < endOutLength) trail.Slices.Reverse(); return trail; } private void drawDetectedFinger(Finger finger) { FingerSliceTrail trail = finger.SliceTrail; for (int i = 0; i < trail.NumSlices; i++) { outputImage.drawLineSegment(trail.Slices[i].LineSegment, Constants.FingerSliceColor); } outputImage.drawLineSegment(finger.LineSegment, Constants.FingerDetectedColor); } } }