FingerDetector.cs 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Diagnostics;
  4. using System.Drawing;
  5. using System.Linq;
  6. using System.Text;
  7. using System.Threading.Tasks;
  8. using bbiwarg.Images;
  9. using bbiwarg.Utility;
  10. using Emgu.CV.Structure;
  11. using Emgu.CV;
  12. using bbiwarg.Graphics;
  13. namespace bbiwarg.Recognition.FingerRecognition
  14. {
  15. class FingerDetector
  16. {
  17. private DepthImage depthImage;
  18. private EdgeImage edgeImageOriginal;
  19. private EdgeImage edgeImageAdapted;
  20. public List<Finger> Fingers { get; private set; }
  21. public FingerDetector(DepthImage depthImage, EdgeImage edgeImage)
  22. {
  23. this.depthImage = depthImage;
  24. this.edgeImageOriginal = edgeImage;
  25. this.edgeImageAdapted = edgeImage.copy();
  26. detectFingers();
  27. }
  28. private void detectFingers()
  29. {
  30. Vector2D maxPixel = Parameters.ImageMaxPixel;
  31. int maxX = maxPixel.IntX;
  32. int maxY = maxPixel.IntY;
  33. Fingers = new List<Finger>();
  34. for (int y = 1; y < maxY; y += 4)
  35. {
  36. for (int x = 1; x < maxX; x += 2)
  37. {
  38. if (edgeImageAdapted.isEdgeAt(x, y))
  39. {
  40. Vector2D edgePoint = new Vector2D(x, y);
  41. Vector2D edgeDirection = getEdgeDirection(edgePoint);
  42. if (edgeDirection != null)
  43. {
  44. Vector2D dir = edgeDirection.getOrthogonal();
  45. if (depthImage.getDepthAt(edgePoint - dir) < depthImage.getDepthAt(edgePoint + dir))
  46. dir = dir.getInverse();
  47. FingerSlice slice = findFingerSliceFromStartEdge(edgePoint, dir);
  48. if (slice != null)
  49. {
  50. FingerSliceTrail trail = findFingerSliceTrail(slice, edgeDirection);
  51. if (trail != null && trail.NumSlices > Parameters.FingerMinNumSlices)
  52. {
  53. createFingerFromTrail(trail);
  54. }
  55. }
  56. }
  57. }
  58. }
  59. }
  60. }
  61. private Vector2D getEdgeDirection(Vector2D edgePoint)
  62. {
  63. int x = edgePoint.IntX;
  64. int y = edgePoint.IntY;
  65. if (edgeImageAdapted.isEdgeAt(x, y - 1) && edgeImageAdapted.isEdgeAt(x, y + 1)) return new Vector2D(0, 1);
  66. else if (edgeImageAdapted.isEdgeAt(x - 1, y) && edgeImageAdapted.isEdgeAt(x + 1, y)) return new Vector2D(1, 0);
  67. else if (edgeImageAdapted.isEdgeAt(x - 1, y - 1) && edgeImageAdapted.isEdgeAt(x + 1, y + 1)) return new Vector2D(1, 1).normalize();
  68. else if (edgeImageAdapted.isEdgeAt(x + 1, y - 1) && edgeImageAdapted.isEdgeAt(x - 1, y + 1)) return new Vector2D(1, -1).normalize();
  69. else return null;
  70. }
  71. private FingerSliceTrail findFingerSliceTrail(FingerSlice startSlice, Vector2D startDirection)
  72. {
  73. FingerSliceTrail trail = new FingerSliceTrail(startSlice);
  74. Vector2D direction = startDirection;
  75. Vector2D position = startSlice.Mid + Parameters.FingerStepSize * direction;
  76. if (position.isInBound())
  77. {
  78. FingerSlice nextSlice = findFingerSliceFromMid(position, direction);
  79. if (nextSlice != null)
  80. {
  81. trail.addSlice(nextSlice);
  82. trail = expandTrail(trail);
  83. if (trail.NumSlices > Parameters.FingerMinNumSlices / 2)
  84. {
  85. trail.removeFirstSlices(Parameters.FingerRemoveNumSlicesForCorrection);
  86. trail = expandTrail(trail, true);
  87. return trail;
  88. }
  89. }
  90. }
  91. return null;
  92. }
  93. private FingerSliceTrail expandTrail(FingerSliceTrail trail, bool reversed = false)
  94. {
  95. if (reversed)
  96. trail.reverse();
  97. Vector2D currentDirection = trail.getEndDirection();
  98. Vector2D currentPosition = trail.EndSlice.Mid + Parameters.FingerStepSize * currentDirection;
  99. int gapCounter = 0;
  100. int numSlices = trail.NumSlices;
  101. FingerSlice lastSlice = trail.EndSlice;
  102. FingerSlice nextSlice;
  103. while (currentPosition.isInBound() && gapCounter < Math.Min(numSlices, Parameters.FingerMaxGapCounter))
  104. {
  105. nextSlice = findFingerSliceFromMid(currentPosition, currentDirection, reversed);
  106. if (nextSlice != null && Math.Abs(nextSlice.Length - lastSlice.Length) <= Parameters.FingerMaxSliceDifferencePerStep)
  107. {
  108. gapCounter = 0;
  109. numSlices++;
  110. trail.addSlice(nextSlice);
  111. currentDirection = trail.getEndDirection();
  112. currentPosition = nextSlice.Mid + Parameters.FingerStepSize * currentDirection;
  113. lastSlice = nextSlice;
  114. }
  115. else
  116. {
  117. gapCounter++;
  118. currentPosition += currentDirection;
  119. }
  120. }
  121. if (reversed)
  122. trail.reverse();
  123. return trail;
  124. }
  125. private FingerSlice findFingerSliceFromMid(Vector2D position, Vector2D direction, bool reversed = false)
  126. {
  127. if (edgeImageAdapted.isRoughEdgeAt(position)) return null;
  128. Vector2D dirStart = direction.getOrthogonal(reversed);
  129. Vector2D dirEnd = direction.getOrthogonal(!reversed);
  130. Vector2D start = edgeImageAdapted.findNextRoughEdge(position, dirStart, Parameters.FingerMaxWidth);
  131. if (start == null) return null;
  132. Vector2D end = edgeImageAdapted.findNextRoughEdge(position, dirEnd, Parameters.FingerMaxWidth);
  133. if (end == null) return null;
  134. return getFingerSlice(start, end);
  135. }
  136. private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction)
  137. {
  138. Vector2D searchStart = start + Parameters.FingerContourMargin * direction;
  139. Vector2D end = edgeImageAdapted.findNextRoughEdge(searchStart, direction, Parameters.FingerMaxWidth);
  140. if (end == null)
  141. return null;
  142. return getFingerSlice(start, end);
  143. }
  144. private FingerSlice getFingerSlice(Vector2D start, Vector2D end)
  145. {
  146. FingerSlice slice = new FingerSlice(start, end);
  147. if (slice.Length >= Parameters.FingerMinWidth && slice.Length <= Parameters.FingerMaxWidth && fingerSliceDepthTest(slice))
  148. return slice;
  149. return null;
  150. }
  151. private bool fingerSliceDepthTest(FingerSlice slice)
  152. {
  153. Int16 depthStart = depthImage.getDepthAt(slice.Start.moveWithinBound(slice.Direction.getInverse(), Parameters.FingerContourMargin));
  154. Int16 depthMid = depthImage.getDepthAt(slice.Mid);
  155. Int16 depthEnd = depthImage.getDepthAt(slice.End.moveWithinBound(slice.Direction, Parameters.FingerContourMargin));
  156. return (depthStart > depthMid && depthMid < depthEnd);
  157. }
  158. private void createFingerFromTrail(FingerSliceTrail trail)
  159. {
  160. //bring finger in correct direction Tip->Hand
  161. trail = orderTrailTipToHand(trail);
  162. //create finger
  163. Finger finger = new Finger(trail);
  164. //add finger
  165. if (!isCrippleFinger(finger))
  166. Fingers.Add(finger);
  167. //remove edges around detected finger to improve performance
  168. edgeImageAdapted.removeEdgesInsidePolygon(finger.getContour(Parameters.FingerContourMargin).ToArray());
  169. }
  170. private bool isCrippleFinger(Finger finger)
  171. {
  172. FingerSlice midSlice = finger.SliceTrail.MidSlice;
  173. Vector2D out1 = midSlice.Start.moveWithinBound(midSlice.Direction.getInverse(), Parameters.FingerOutMargin);
  174. Vector2D out2 = midSlice.End.moveWithinBound(midSlice.Direction, Parameters.FingerOutMargin);
  175. Int16 depthAtFinger = depthImage.getDepthAt(finger.MidPoint);
  176. Int16 depthAtOut1 = depthImage.getDepthAt(out1);
  177. Int16 depthAtOut2 = depthImage.getDepthAt(out2);
  178. int minDepthDifference = Math.Min(Math.Abs(depthAtFinger - depthAtOut1), Math.Abs(depthAtFinger - depthAtOut2));
  179. return (minDepthDifference < Parameters.FingerMaxCrippleDifference);
  180. }
  181. private FingerSliceTrail orderTrailTipToHand(FingerSliceTrail trail)
  182. {
  183. float startLength = trail.StartSlice.Length;
  184. float endLength = trail.EndSlice.Length;
  185. if (startLength > endLength)
  186. trail.reverse();
  187. return trail;
  188. }
  189. }
  190. }