InputHandler.cs 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Drawing;
  4. using System.Linq;
  5. using System.Text;
  6. using System.Threading.Tasks;
  7. using bbiwarg.InputProviders;
  8. using bbiwarg.Images;
  9. using bbiwarg.Recognition.FingerRecognition;
  10. using bbiwarg.Recognition.HandRecognition;
  11. using bbiwarg.Recognition.PalmRecognition;
  12. using bbiwarg.Recognition.TouchRecognition;
  13. using bbiwarg.Recognition.Tracking;
  14. using bbiwarg.Server;
  15. using bbiwarg.Graphics;
  16. using bbiwarg.Utility;
  17. using Emgu.CV;
  18. using Emgu.CV.Structure;
  19. namespace bbiwarg
  20. {
  21. class InputHandler
  22. {
  23. public OutputImage[] OutputImages { get; private set; }
  24. private InputProvider inputProvider;
  25. private InputFrame currentInputFrame;
  26. private int imageWidth;
  27. private int imageHeight;
  28. private int currentFrameID;
  29. private int lastFrameID;
  30. private DepthImage depthImage;
  31. private EdgeImage edgeImage;
  32. private ConfidenceImage confidenceImage;
  33. private FingerDetector fingerDetector;
  34. private HandDetector handDetector;
  35. private PalmDetector palmDetector;
  36. private TouchDetector touchDetector;
  37. private FingerTracker fingerTracker;
  38. private HandTracker handTracker;
  39. private PalmTracker palmTracker;
  40. private TouchTracker touchTracker;
  41. private TouchEventVisualizer touchEventVisualizer;
  42. private TuioCommunicator tuioCommunicator;
  43. public InputHandler(InputProvider inputProvider)
  44. {
  45. this.inputProvider = inputProvider;
  46. this.imageWidth = inputProvider.ImageWidth;
  47. this.imageHeight = inputProvider.ImageHeight;
  48. this.currentFrameID = inputProvider.CurrentFrameID;
  49. this.lastFrameID = this.currentFrameID - 1;
  50. initializeConsistentObjects();
  51. }
  52. private void initializeConsistentObjects()
  53. {
  54. fingerTracker = new FingerTracker();
  55. handTracker = new HandTracker();
  56. palmTracker = new PalmTracker();
  57. touchTracker = new TouchTracker();
  58. touchEventVisualizer = new TouchEventVisualizer(imageWidth, imageHeight);
  59. touchTracker.TouchDown += touchEventVisualizer.touchDown;
  60. touchTracker.TouchMove += touchEventVisualizer.touchMove;
  61. touchTracker.TouchUp += touchEventVisualizer.touchUp;
  62. if (Constants.TuioEnabled)
  63. {
  64. tuioCommunicator = new TuioCommunicator(Constants.TuioIP, Constants.TuioPort);
  65. touchTracker.TouchDown += tuioCommunicator.touchDown;
  66. touchTracker.TouchMove += tuioCommunicator.touchMove;
  67. touchTracker.TouchUp += tuioCommunicator.touchUp;
  68. }
  69. }
  70. private void resetConsistentObjects()
  71. {
  72. touchTracker.reset();
  73. handTracker.reset();
  74. palmTracker.reset();
  75. fingerTracker.reset();
  76. touchEventVisualizer.reset();
  77. if (Constants.TuioEnabled)
  78. tuioCommunicator.reset();
  79. }
  80. public void updateFrame()
  81. {
  82. Timer.start("updateFrame");
  83. if (currentFrameID != inputProvider.CurrentFrameID)
  84. {
  85. beforeUpdateFrame();
  86. if (lastFrameID - currentFrameID > 20)
  87. resetConsistentObjects();
  88. createConfidenceImage();
  89. createDepthImage();
  90. createEdgeImage();
  91. detectFingers();
  92. trackFingers();
  93. detectHands();
  94. trackHands();
  95. detectPalm();
  96. trackPalm();
  97. detectTouchEvents();
  98. trackTouchEvents();
  99. afterUpdateFrame();
  100. }
  101. Timer.stop("updateFrame");
  102. }
  103. private void beforeUpdateFrame()
  104. {
  105. Timer.start("beforeUpdateFrame");
  106. currentInputFrame = inputProvider.CurrentFrame;
  107. lastFrameID = currentFrameID;
  108. currentFrameID = currentInputFrame.FrameID;
  109. Logger.CurrentFrame = currentFrameID;
  110. if (Constants.TuioEnabled)
  111. tuioCommunicator.initFrame();
  112. Timer.stop("beforeUpdateFrame");
  113. }
  114. private void afterUpdateFrame()
  115. {
  116. Timer.start("afterUpdateFrame");
  117. if (Constants.TuioEnabled)
  118. tuioCommunicator.commitFrame();
  119. createOutputImages();
  120. Timer.stop("afterUpdateFrame");
  121. }
  122. private void createConfidenceImage()
  123. {
  124. Timer.start("createCnfdncImg");
  125. Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawConfidenceData);
  126. confidenceImage = new ConfidenceImage(rawConfidenceImage);
  127. Timer.stop("createCnfdncImg");
  128. }
  129. private void createDepthImage()
  130. {
  131. Timer.start("createDepthImage");
  132. Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawDepthData);
  133. rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
  134. depthImage = new DepthImage(rawDepthImage);
  135. Timer.stop("createDepthImage");
  136. }
  137. private void createEdgeImage()
  138. {
  139. Timer.start("createEdgeImage");
  140. edgeImage = new EdgeImage(depthImage);
  141. Timer.stop("createEdgeImage");
  142. }
  143. private void detectFingers()
  144. {
  145. Timer.start("detectFingers");
  146. fingerDetector = new FingerDetector(depthImage, edgeImage);
  147. Timer.stop("detectFingers");
  148. }
  149. private void trackFingers()
  150. {
  151. Timer.start("trackFingers");
  152. fingerTracker.updateFrame(fingerDetector.Fingers);
  153. Timer.stop("trackFingers");
  154. }
  155. private void detectHands()
  156. {
  157. Timer.start("detectHands");
  158. handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.Fingers);
  159. Timer.stop("detectHands");
  160. }
  161. private void trackHands()
  162. {
  163. Timer.start("trackHands\t");
  164. handTracker.updateFrame(handDetector.Hands);
  165. Timer.stop("trackHands\t");
  166. }
  167. private void detectPalm()
  168. {
  169. Timer.start("detectPalm\t");
  170. palmDetector = new PalmDetector(handTracker.Hands);
  171. Timer.stop("detectPalm\t");
  172. }
  173. private void trackPalm()
  174. {
  175. Timer.start("trackPalm\t");
  176. palmTracker.updateFrame(palmDetector.Palms);
  177. Timer.stop("trackPalm\t");
  178. }
  179. private void detectTouchEvents()
  180. {
  181. Timer.start("detectTouchEvents");
  182. touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmTracker.OptimizedPalms);
  183. Timer.stop("detectTouchEvents");
  184. }
  185. private void trackTouchEvents()
  186. {
  187. Timer.start("trackTouchEvents");
  188. touchTracker.updateFrame(touchDetector.TouchEvents);
  189. Timer.stop("trackTouchEvents");
  190. }
  191. private void createOutputImages()
  192. {
  193. Timer.start("createOutputImages");
  194. int numImages = Constants.OutputNumImages;
  195. OutputImages = new OutputImage[numImages];
  196. for (int i = 0; i < numImages; i++)
  197. {
  198. OutputImages[i] = new OutputImage(imageWidth, imageHeight);
  199. }
  200. //image0
  201. OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Constants.DepthImageColor);
  202. foreach (Finger f in fingerTracker.Fingers)
  203. {
  204. OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 4, Constants.FingerTipColor);
  205. }
  206. foreach (TrackedFinger tf in fingerTracker.TrackedObjects)
  207. {
  208. if (tf.CurrentState == TrackingState.Tracked)
  209. {
  210. OutputImages[0].fillCircle(tf.TipPointPrediction.IntX, tf.TipPointPrediction.IntY, 3, Constants.FingerPointsPredictionColor);
  211. OutputImages[0].fillCircle(tf.HandPointPrediction.IntX, tf.HandPointPrediction.IntY, 3, Constants.FingerPointsPredictionColor);
  212. }
  213. }
  214. //image1
  215. OutputImages[1].drawImage(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(255)), Constants.EdgeImageColor);
  216. foreach (Finger f in fingerTracker.Fingers)
  217. {
  218. for (int i = 0; i < f.SliceTrail.NumSlices; i++)
  219. OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Constants.FingerSliceColor);
  220. OutputImages[1].drawContour(f.Contour, Constants.FingerContourColor);
  221. OutputImages[1].drawLineSegment(f.LineSegment, Constants.FingerTrackedColor);
  222. OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
  223. }
  224. //image2
  225. foreach (Hand h in handTracker.Hands)
  226. {
  227. OutputImages[2].drawImage(h.Mask.ThresholdBinary(new Gray(0), new Gray(255)), Constants.HandColor[h.ZIndex % Constants.HandNumColors]);
  228. OutputImages[2].fillCircle(h.Centroid.IntX, h.Centroid.IntY, 5, Constants.HandCentroidColor);
  229. OutputImages[2].drawText(h.Centroid.IntX, h.Centroid.IntY, h.TrackID.ToString(), Constants.HandIDColor);
  230. if (h.ThumbDefect != null)
  231. OutputImages[2].drawDefect(h.ThumbDefect, Color.CornflowerBlue, Color.Yellow);
  232. }
  233. //image3
  234. OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.ThresholdBinary(new Gray(0), new Gray(255))), Constants.DepthImageColor);
  235. foreach (TrackedTouchEvent tte in touchTracker.TrackedObjects)
  236. {
  237. Vector2D position = tte.AbsolutePositionPrediction;
  238. OutputImages[3].fillCircle(position.IntX, position.IntY, 5, Constants.TouchEventTrackedColor);
  239. }
  240. foreach (Palm p in palmTracker.OptimizedPalms)
  241. {
  242. OutputImages[3].drawQuadrangleGrid(p.Quad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns);
  243. }
  244. //image4
  245. touchEventVisualizer.updateImage();
  246. OutputImages[4] = touchEventVisualizer.OutputImage;
  247. //borders
  248. for (int i = 0; i < numImages; i++)
  249. {
  250. OutputImages[i].drawRectangle(0, 0, imageWidth - 1, imageHeight - 1, Constants.OutputImageBorderColor);
  251. }
  252. Timer.stop("createOutputImages");
  253. }
  254. }
  255. }