InputHandler.cs 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Drawing;
  4. using System.Linq;
  5. using System.Text;
  6. using System.Threading.Tasks;
  7. using bbiwarg.InputProviders;
  8. using bbiwarg.Images;
  9. using bbiwarg.Recognition.FingerRecognition;
  10. using bbiwarg.Recognition.HandRecognition;
  11. using bbiwarg.Recognition.PalmRecognition;
  12. using bbiwarg.Recognition.TouchRecognition;
  13. using bbiwarg.Recognition.Tracking;
  14. using bbiwarg.Server;
  15. using bbiwarg.Graphics;
  16. using bbiwarg.Utility;
  17. using Emgu.CV;
  18. using Emgu.CV.Structure;
  19. namespace bbiwarg
  20. {
  21. class InputHandler
  22. {
  23. public OutputImage[] OutputImages { get; private set; }
  24. private InputProvider inputProvider;
  25. private InputFrame currentInputFrame;
  26. private int currentFrameID;
  27. private int lastFrameID;
  28. private DepthImage depthImage;
  29. private EdgeImage edgeImage;
  30. private ConfidenceImage confidenceImage;
  31. private FingerDetector fingerDetector;
  32. private HandDetector handDetector;
  33. private PalmDetector palmDetector;
  34. private TouchDetector touchDetector;
  35. private FingerTracker fingerTracker;
  36. private HandTracker handTracker;
  37. private PalmTracker palmTracker;
  38. private TouchTracker touchTracker;
  39. private TouchEventVisualizer touchEventVisualizer;
  40. private TuioCommunicator tuioCommunicator;
  41. public InputHandler(InputProvider inputProvider)
  42. {
  43. this.inputProvider = inputProvider;
  44. this.currentFrameID = inputProvider.CurrentFrameID;
  45. this.lastFrameID = this.currentFrameID - 1;
  46. initializeConsistentObjects();
  47. }
  48. private void initializeConsistentObjects()
  49. {
  50. fingerTracker = new FingerTracker();
  51. handTracker = new HandTracker();
  52. palmTracker = new PalmTracker();
  53. touchTracker = new TouchTracker();
  54. touchEventVisualizer = new TouchEventVisualizer(Parameters.ImageWidth, Parameters.ImageHeight);
  55. touchTracker.TouchDown += touchEventVisualizer.touchDown;
  56. touchTracker.TouchMove += touchEventVisualizer.touchMove;
  57. touchTracker.TouchUp += touchEventVisualizer.touchUp;
  58. if (Parameters.TuioEnabled)
  59. {
  60. tuioCommunicator = new TuioCommunicator(Parameters.TuioIP.ToString(), Parameters.TuioPort);
  61. touchTracker.TouchDown += tuioCommunicator.touchDown;
  62. touchTracker.TouchMove += tuioCommunicator.touchMove;
  63. touchTracker.TouchUp += tuioCommunicator.touchUp;
  64. }
  65. }
  66. private void resetConsistentObjects()
  67. {
  68. touchTracker.reset();
  69. handTracker.reset();
  70. palmTracker.reset();
  71. fingerTracker.reset();
  72. touchEventVisualizer.reset();
  73. if (Parameters.TuioEnabled)
  74. tuioCommunicator.reset();
  75. }
  76. public void updateFrame()
  77. {
  78. Timer.start("updateFrame");
  79. if (currentFrameID != inputProvider.CurrentFrameID)
  80. {
  81. beforeUpdateFrame();
  82. if (lastFrameID - currentFrameID > 20)
  83. resetConsistentObjects();
  84. createConfidenceImage();
  85. createDepthImage();
  86. createEdgeImage();
  87. detectFingers();
  88. trackFingers();
  89. detectHands();
  90. trackHands();
  91. detectPalm();
  92. trackPalm();
  93. detectTouchEvents();
  94. trackTouchEvents();
  95. afterUpdateFrame();
  96. }
  97. Timer.stop("updateFrame");
  98. }
  99. private void beforeUpdateFrame()
  100. {
  101. Timer.start("beforeUpdateFrame");
  102. currentInputFrame = inputProvider.CurrentFrame;
  103. lastFrameID = currentFrameID;
  104. currentFrameID = currentInputFrame.FrameID;
  105. Logger.CurrentFrame = currentFrameID;
  106. if (Parameters.TuioEnabled)
  107. tuioCommunicator.initFrame();
  108. Timer.stop("beforeUpdateFrame");
  109. }
  110. private void afterUpdateFrame()
  111. {
  112. Timer.start("afterUpdateFrame");
  113. if (Parameters.TuioEnabled)
  114. tuioCommunicator.commitFrame();
  115. createOutputImages();
  116. Timer.stop("afterUpdateFrame");
  117. }
  118. private void createConfidenceImage()
  119. {
  120. Timer.start("createCnfdncImg");
  121. Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(Parameters.ImageWidth, Parameters.ImageHeight, Parameters.ImageWidth * 2, currentInputFrame.RawConfidenceData);
  122. confidenceImage = new ConfidenceImage(rawConfidenceImage);
  123. Timer.stop("createCnfdncImg");
  124. }
  125. private void createDepthImage()
  126. {
  127. Timer.start("createDepthImage");
  128. Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(Parameters.ImageWidth, Parameters.ImageHeight, Parameters.ImageWidth * 2, currentInputFrame.RawDepthData);
  129. rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
  130. depthImage = new DepthImage(rawDepthImage);
  131. Timer.stop("createDepthImage");
  132. }
  133. private void createEdgeImage()
  134. {
  135. Timer.start("createEdgeImage");
  136. edgeImage = new EdgeImage(depthImage);
  137. Timer.stop("createEdgeImage");
  138. }
  139. private void detectFingers()
  140. {
  141. Timer.start("detectFingers");
  142. fingerDetector = new FingerDetector(depthImage, edgeImage);
  143. Timer.stop("detectFingers");
  144. }
  145. private void trackFingers()
  146. {
  147. Timer.start("trackFingers");
  148. fingerTracker.updateFrame(fingerDetector.Fingers);
  149. Timer.stop("trackFingers");
  150. }
  151. private void detectHands()
  152. {
  153. Timer.start("detectHands");
  154. handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.Fingers);
  155. Timer.stop("detectHands");
  156. }
  157. private void trackHands()
  158. {
  159. Timer.start("trackHands\t");
  160. handTracker.updateFrame(handDetector.Hands);
  161. Timer.stop("trackHands\t");
  162. }
  163. private void detectPalm()
  164. {
  165. Timer.start("detectPalm\t");
  166. palmDetector = new PalmDetector(handTracker.Hands);
  167. Timer.stop("detectPalm\t");
  168. }
  169. private void trackPalm()
  170. {
  171. Timer.start("trackPalm\t");
  172. palmTracker.updateFrame(palmDetector.Palms);
  173. Timer.stop("trackPalm\t");
  174. }
  175. private void detectTouchEvents()
  176. {
  177. Timer.start("detectTouchEvents");
  178. touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmTracker.OptimizedPalms);
  179. Timer.stop("detectTouchEvents");
  180. }
  181. private void trackTouchEvents()
  182. {
  183. Timer.start("trackTouchEvents");
  184. touchTracker.updateFrame(touchDetector.TouchEvents);
  185. Timer.stop("trackTouchEvents");
  186. }
  187. private void createOutputImages()
  188. {
  189. Timer.start("createOutputImages");
  190. int numImages = Parameters.OutputNumImages;
  191. OutputImages = new OutputImage[numImages];
  192. for (int i = 0; i < numImages; i++)
  193. {
  194. OutputImages[i] = new OutputImage(Parameters.ImageWidth, Parameters.ImageHeight);
  195. }
  196. //image0
  197. OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Parameters.DepthImageColor);
  198. foreach (Finger f in fingerTracker.Fingers)
  199. {
  200. OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 5, Parameters.FingerTipColor);
  201. }
  202. foreach (TrackedFinger tf in fingerTracker.TrackedObjects)
  203. {
  204. if (tf.CurrentState == TrackingState.Tracked)
  205. {
  206. OutputImages[0].fillCircle(tf.TipPointPrediction.IntX, tf.TipPointPrediction.IntY, 3, Parameters.FingerPointsPredictionColor);
  207. OutputImages[0].fillCircle(tf.HandPointPrediction.IntX, tf.HandPointPrediction.IntY, 3, Parameters.FingerPointsPredictionColor);
  208. Vector2D mid = (tf.TipPointPrediction+tf.HandPointPrediction) /2 ;
  209. OutputImages[0].drawLineSegment(new Utility.LineSegment2D(mid, mid + Parameters.FingerTrackerNumDirectionsForMeanDirection*tf.MeanDirection), Parameters.FingerMeanDirectionColor);
  210. }
  211. }
  212. //image1
  213. OutputImages[1].drawImage(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(255)), Parameters.EdgeImageColor);
  214. foreach (Finger f in fingerTracker.Fingers)
  215. {
  216. for (int i = 0; i < f.SliceTrail.NumSlices; i++)
  217. OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Parameters.FingerSliceColor);
  218. OutputImages[1].drawContour(f.Contour, Parameters.FingerContourColor);
  219. OutputImages[1].drawLineSegment(f.LineSegment, Parameters.FingerTrackedColor);
  220. OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Parameters.FingerIDColor);
  221. }
  222. //image2
  223. foreach (Hand h in handTracker.Hands)
  224. {
  225. OutputImages[2].drawImage(h.Mask.ThresholdBinary(new Gray(0), new Gray(255)), Parameters.HandColor[h.ZIndex % Parameters.HandNumColors]);
  226. OutputImages[2].fillCircle(h.Centroid.IntX, h.Centroid.IntY, 5, Parameters.HandCentroidColor);
  227. OutputImages[2].drawText(h.Centroid.IntX, h.Centroid.IntY, h.TrackID.ToString(), Parameters.HandIDColor);
  228. if (h.ThumbDefect != null)
  229. OutputImages[2].drawDefect(h.ThumbDefect, Parameters.HandThumbDefectPointColor, Parameters.HandThumbDefectLineColor);
  230. }
  231. //image3
  232. OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.ThresholdBinary(new Gray(0), new Gray(255))), Parameters.DepthImageColor);
  233. foreach (TrackedTouchEvent tte in touchTracker.TrackedObjects)
  234. {
  235. Vector2D position = tte.AbsolutePositionPrediction;
  236. OutputImages[3].fillCircle(position.IntX, position.IntY, 5, Parameters.TouchEventTrackedColor);
  237. }
  238. foreach (Palm p in palmTracker.OptimizedPalms)
  239. {
  240. OutputImages[3].drawQuadrangleGrid(p.Quad, Parameters.PalmQuadColor, Parameters.PalmGridColor, Parameters.PalmGridNumRows, Parameters.PalmGridNumColumns);
  241. }
  242. //image4
  243. touchEventVisualizer.updateImage();
  244. OutputImages[4] = touchEventVisualizer.OutputImage;
  245. //borders
  246. for (int i = 0; i < numImages; i++)
  247. {
  248. OutputImages[i].drawRectangle(0, 0, Parameters.ImageWidth - 1, Parameters.ImageHeight - 1, Parameters.OutputImageBorderColor);
  249. }
  250. Timer.stop("createOutputImages");
  251. }
  252. }
  253. }