InputHandler.cs 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Drawing;
  4. using System.Linq;
  5. using System.Text;
  6. using System.Threading.Tasks;
  7. using bbiwarg.InputProviders;
  8. using bbiwarg.Images;
  9. using bbiwarg.Recognition.FingerRecognition;
  10. using bbiwarg.Recognition.HandRecognition;
  11. using bbiwarg.Recognition.PalmRecognition;
  12. using bbiwarg.Recognition.TouchRecognition;
  13. using bbiwarg.Server;
  14. using bbiwarg.Graphics;
  15. using bbiwarg.Utility;
  16. using Emgu.CV;
  17. using Emgu.CV.Structure;
  18. namespace bbiwarg
  19. {
  20. class InputHandler
  21. {
  22. public OutputImage[] OutputImages { get; private set; }
  23. private InputProvider inputProvider;
  24. private InputFrame currentInputFrame;
  25. private int imageWidth;
  26. private int imageHeight;
  27. private int currentFrame;
  28. private int lastFrame;
  29. private DepthImage depthImage;
  30. private EdgeImage edgeImage;
  31. private ConfidenceImage confidenceImage;
  32. private FingerDetector fingerDetector;
  33. private HandDetector handDetector;
  34. private PalmDetector palmDetector;
  35. private TouchDetector touchDetector;
  36. private FingerTracker fingerTracker;
  37. private TouchTracker touchTracker;
  38. private TouchEventVisualizer touchEventVisualizer;
  39. private TuioCommunicator tuioCommunicator;
  40. public InputHandler(InputProvider inputProvider)
  41. {
  42. this.inputProvider = inputProvider;
  43. this.imageWidth = inputProvider.ImageWidth;
  44. this.imageHeight = inputProvider.ImageHeight;
  45. this.currentFrame = inputProvider.CurrentFrame;
  46. this.lastFrame = this.currentFrame - 1;
  47. initializeConsistentObjects();
  48. }
  49. private void initializeConsistentObjects()
  50. {
  51. palmDetector = new PalmDetector();
  52. fingerTracker = new FingerTracker();
  53. touchTracker = new TouchTracker();
  54. if (Constants.OutputEnabled)
  55. {
  56. touchEventVisualizer = new TouchEventVisualizer(imageWidth, imageHeight);
  57. //register touchEventVisualizer to touchTracker
  58. touchTracker.PalmTouchDown += touchEventVisualizer.touchDown;
  59. touchTracker.PalmTouchMove += touchEventVisualizer.touchMove;
  60. touchTracker.PalmTouchUp += touchEventVisualizer.touchUp;
  61. }
  62. if (Constants.TuioEnabled)
  63. {
  64. tuioCommunicator = new TuioCommunicator(Constants.TuioIP, Constants.TuioPort);
  65. //register tuiCommunicator to touchTracker
  66. touchTracker.PalmTouchDown += tuioCommunicator.touchDown;
  67. touchTracker.PalmTouchMove += tuioCommunicator.touchMove;
  68. touchTracker.PalmTouchUp += tuioCommunicator.touchUp;
  69. }
  70. }
  71. private void resetConsistentObjects()
  72. {
  73. palmDetector.reset();
  74. touchTracker.reset();
  75. fingerTracker.reset();
  76. if (Constants.OutputEnabled)
  77. touchEventVisualizer.reset();
  78. if (Constants.TuioEnabled)
  79. tuioCommunicator.reset();
  80. }
  81. public void updateFrame()
  82. {
  83. Timer.start("updateFrame");
  84. beforeUpdateFrame();
  85. if (lastFrame-currentFrame > 20)
  86. resetConsistentObjects();
  87. createConfidenceImage();
  88. createDepthImage();
  89. createEdgeImage();
  90. detectFingers();
  91. trackFingers();
  92. detectHands();
  93. detectPalm();
  94. detectTouchEvents();
  95. trackTouchEvents();
  96. afterUpdateFrame();
  97. Timer.stop("updateFrame");
  98. }
  99. private void beforeUpdateFrame()
  100. {
  101. currentInputFrame = inputProvider.getInputFrame();
  102. lastFrame = currentFrame;
  103. currentFrame = currentInputFrame.FrameNumber;
  104. Logger.CurrentFrame = currentFrame;
  105. if (Constants.TuioEnabled)
  106. tuioCommunicator.initFrame();
  107. }
  108. private void afterUpdateFrame()
  109. {
  110. if (Constants.TuioEnabled)
  111. tuioCommunicator.commitFrame();
  112. if (Constants.OutputEnabled)
  113. createOutputImages();
  114. }
  115. private void createConfidenceImage()
  116. {
  117. Timer.start("createConfidenceImage");
  118. Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawConfidenceData);
  119. confidenceImage = new ConfidenceImage(rawConfidenceImage);
  120. Timer.stop("createConfidenceImage");
  121. }
  122. private void createDepthImage()
  123. {
  124. Timer.start("createDepthImage");
  125. Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawDepthData);
  126. rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
  127. depthImage = new DepthImage(rawDepthImage);
  128. Timer.stop("createDepthImage");
  129. }
  130. private void createEdgeImage()
  131. {
  132. Timer.start("createEdgeImage");
  133. edgeImage = new EdgeImage(depthImage);
  134. Timer.stop("createEdgeImage");
  135. }
  136. private void detectFingers()
  137. {
  138. Timer.start("detectFingers");
  139. fingerDetector = new FingerDetector(depthImage, edgeImage);
  140. Timer.stop("detectFingers");
  141. }
  142. private void trackFingers()
  143. {
  144. Timer.start("trackFingers");
  145. fingerTracker.updateFrame(fingerDetector.Fingers);
  146. Timer.stop("trackFingers");
  147. }
  148. private void detectHands()
  149. {
  150. Timer.start("detectHands");
  151. handDetector = new HandDetector(edgeImage, fingerDetector.Fingers);
  152. Timer.stop("detectHands");
  153. }
  154. private void detectPalm()
  155. {
  156. Timer.start("detectPalm");
  157. palmDetector.findPalmQuad(handDetector.Hands, handDetector.HandMask);
  158. Timer.stop("detectPalm");
  159. }
  160. private void detectTouchEvents()
  161. {
  162. Timer.start("detectTouchEvents");
  163. touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmDetector.PalmQuad);
  164. Timer.stop("detectTouchEvents");
  165. }
  166. private void trackTouchEvents()
  167. {
  168. Timer.start("trackTouchEvents");
  169. touchTracker.updateFrame(touchDetector.TouchEvents);
  170. Timer.stop("trackTouchEvents");
  171. }
  172. private void createOutputImages()
  173. {
  174. Timer.start("createOutputImages");
  175. int numImages = Constants.OutputNumImages;
  176. OutputImages = new OutputImage[numImages];
  177. for (int i = 0; i < numImages; i++)
  178. {
  179. OutputImages[i] = new OutputImage(imageWidth, imageHeight);
  180. }
  181. //image0
  182. OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Constants.DepthImageColor);
  183. foreach (Finger f in fingerTracker.Fingers)
  184. {
  185. OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 3, Constants.FingerTipColor);
  186. if (f.TouchEvent != null)
  187. OutputImages[0].fillCircle(f.TouchEvent.Position.IntX, f.TouchEvent.Position.IntY, 5, Constants.TouchEventDetectedColor);
  188. }
  189. //image1
  190. OutputImages[1].drawImage(edgeImage.Image.Mul(255), Constants.EdgeImageColor);
  191. foreach (Finger f in fingerTracker.Fingers)
  192. {
  193. for (int i = 0; i < f.SliceTrail.NumSlices; i++)
  194. OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Constants.FingerSliceColor);
  195. OutputImages[1].drawContour(f.Contour, Constants.FingerContourColor);
  196. OutputImages[1].drawLineSegment(f.LineSegment, Constants.FingerTrackedColor);
  197. OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
  198. }
  199. //image2
  200. foreach (Hand h in handDetector.Hands)
  201. {
  202. if (h.Side == Hand.HandSide.Right)
  203. OutputImages[2].drawImage(h.Mask.Mul(255), Constants.HandRightColor);
  204. else
  205. OutputImages[2].drawImage(h.Mask.Mul(255), Constants.HandLeftColor);
  206. }
  207. //image3
  208. OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.Mul(255)), Constants.DepthImageColor);
  209. foreach (TouchEvent te in touchTracker.TouchEvents)
  210. OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
  211. if (palmDetector.PalmContour != null && palmDetector.PalmContour.Count<Point>() > 0)
  212. {
  213. OutputImages[3].drawContour(palmDetector.PalmContour, Constants.PalmConturColor);
  214. OutputImages[3].drawPoints(palmDetector.PalmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
  215. }
  216. if (palmDetector.PalmQuad != null) {
  217. OutputImages[3].fillCircle(palmDetector.ThumbDefectStart.IntX, palmDetector.ThumbDefectStart.IntY, 3, Color.Red);
  218. OutputImages[3].fillCircle(palmDetector.ThumbDefectEnd.IntX, palmDetector.ThumbDefectEnd.IntY, 3, Color.Red);
  219. OutputImages[3].fillCircle(palmDetector.ThumbDefectDepth.IntX, palmDetector.ThumbDefectDepth.IntY, 3, Color.Red);
  220. OutputImages[3].drawLineSegment(new Utility.LineSegment2D(palmDetector.ThumbDefectDepth, (palmDetector.ThumbDefectStart + palmDetector.ThumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
  221. OutputImages[3].drawQuadrangleGrid(palmDetector.PalmQuad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns);
  222. }
  223. //image4
  224. touchEventVisualizer.updateImage();
  225. OutputImages[4] = touchEventVisualizer.OutputImage;
  226. //borders
  227. for (int i = 0; i < numImages; i++)
  228. {
  229. OutputImages[i].drawRectangle(0, 0, imageWidth - 1, imageHeight - 1, Constants.OutputImageBorderColor);
  230. }
  231. Timer.stop("createOutputImages");
  232. }
  233. }
  234. }