using System; using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Text; using System.Threading.Tasks; using bbiwarg.InputProviders; using bbiwarg.Images; using bbiwarg.Recognition.FingerRecognition; using bbiwarg.Recognition.HandRecognition; using bbiwarg.Recognition.PalmRecognition; using bbiwarg.Recognition.TouchRecognition; using bbiwarg.Recognition.Tracking; using bbiwarg.Server; using bbiwarg.Graphics; using bbiwarg.Utility; using Emgu.CV; using Emgu.CV.Structure; namespace bbiwarg { class InputHandler { public OutputImage[] OutputImages { get; private set; } private InputProvider inputProvider; private InputFrame currentInputFrame; private int imageWidth; private int imageHeight; private int currentFrameID; private int lastFrameID; private DepthImage depthImage; private EdgeImage edgeImage; private ConfidenceImage confidenceImage; private FingerDetector fingerDetector; private HandDetector handDetector; private PalmDetector palmDetector; private TouchDetector touchDetector; private FingerTracker fingerTracker; private HandTracker handTracker; private PalmTracker palmTracker; private TouchTracker touchTracker; private TouchEventVisualizer touchEventVisualizer; private TuioCommunicator tuioCommunicator; public InputHandler(InputProvider inputProvider) { this.inputProvider = inputProvider; this.imageWidth = inputProvider.ImageWidth; this.imageHeight = inputProvider.ImageHeight; this.currentFrameID = inputProvider.CurrentFrameID; this.lastFrameID = this.currentFrameID - 1; initializeConsistentObjects(); } private void initializeConsistentObjects() { fingerTracker = new FingerTracker(); handTracker = new HandTracker(); palmTracker = new PalmTracker(); touchTracker = new TouchTracker(); touchEventVisualizer = new TouchEventVisualizer(imageWidth, imageHeight); touchTracker.TouchDown += touchEventVisualizer.touchDown; touchTracker.TouchMove += touchEventVisualizer.touchMove; touchTracker.TouchUp += touchEventVisualizer.touchUp; if (Constants.TuioEnabled) { tuioCommunicator = new TuioCommunicator(Constants.TuioIP.ToString(), Constants.TuioPort); touchTracker.TouchDown += tuioCommunicator.touchDown; touchTracker.TouchMove += tuioCommunicator.touchMove; touchTracker.TouchUp += tuioCommunicator.touchUp; } } private void resetConsistentObjects() { touchTracker.reset(); handTracker.reset(); palmTracker.reset(); fingerTracker.reset(); touchEventVisualizer.reset(); if (Constants.TuioEnabled) tuioCommunicator.reset(); } public void updateFrame() { Timer.start("updateFrame"); if (currentFrameID != inputProvider.CurrentFrameID) { beforeUpdateFrame(); if (lastFrameID - currentFrameID > 20) resetConsistentObjects(); createConfidenceImage(); createDepthImage(); createEdgeImage(); detectFingers(); trackFingers(); detectHands(); trackHands(); detectPalm(); trackPalm(); detectTouchEvents(); trackTouchEvents(); afterUpdateFrame(); } Timer.stop("updateFrame"); } private void beforeUpdateFrame() { Timer.start("beforeUpdateFrame"); currentInputFrame = inputProvider.CurrentFrame; lastFrameID = currentFrameID; currentFrameID = currentInputFrame.FrameID; Logger.CurrentFrame = currentFrameID; if (Constants.TuioEnabled) tuioCommunicator.initFrame(); Timer.stop("beforeUpdateFrame"); } private void afterUpdateFrame() { Timer.start("afterUpdateFrame"); if (Constants.TuioEnabled) tuioCommunicator.commitFrame(); createOutputImages(); Timer.stop("afterUpdateFrame"); } private void createConfidenceImage() { Timer.start("createCnfdncImg"); Image rawConfidenceImage = new Image(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawConfidenceData); confidenceImage = new ConfidenceImage(rawConfidenceImage); Timer.stop("createCnfdncImg"); } private void createDepthImage() { Timer.start("createDepthImage"); Image rawDepthImage = new Image(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawDepthData); rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert().Mul(Int16.MaxValue)); depthImage = new DepthImage(rawDepthImage); Timer.stop("createDepthImage"); } private void createEdgeImage() { Timer.start("createEdgeImage"); edgeImage = new EdgeImage(depthImage); Timer.stop("createEdgeImage"); } private void detectFingers() { Timer.start("detectFingers"); fingerDetector = new FingerDetector(depthImage, edgeImage); Timer.stop("detectFingers"); } private void trackFingers() { Timer.start("trackFingers"); fingerTracker.updateFrame(fingerDetector.Fingers); Timer.stop("trackFingers"); } private void detectHands() { Timer.start("detectHands"); handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.Fingers); Timer.stop("detectHands"); } private void trackHands() { Timer.start("trackHands\t"); handTracker.updateFrame(handDetector.Hands); Timer.stop("trackHands\t"); } private void detectPalm() { Timer.start("detectPalm\t"); palmDetector = new PalmDetector(handTracker.Hands); Timer.stop("detectPalm\t"); } private void trackPalm() { Timer.start("trackPalm\t"); palmTracker.updateFrame(palmDetector.Palms); Timer.stop("trackPalm\t"); } private void detectTouchEvents() { Timer.start("detectTouchEvents"); touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmTracker.OptimizedPalms); Timer.stop("detectTouchEvents"); } private void trackTouchEvents() { Timer.start("trackTouchEvents"); touchTracker.updateFrame(touchDetector.TouchEvents); Timer.stop("trackTouchEvents"); } private void createOutputImages() { Timer.start("createOutputImages"); int numImages = Constants.OutputNumImages; OutputImages = new OutputImage[numImages]; for (int i = 0; i < numImages; i++) { OutputImages[i] = new OutputImage(imageWidth, imageHeight); } //image0 OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Constants.DepthImageColor); foreach (Finger f in fingerTracker.Fingers) { OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 4, Constants.FingerTipColor); } foreach (TrackedFinger tf in fingerTracker.TrackedObjects) { if (tf.CurrentState == TrackingState.Tracked) { OutputImages[0].fillCircle(tf.TipPointPrediction.IntX, tf.TipPointPrediction.IntY, 3, Constants.FingerPointsPredictionColor); OutputImages[0].fillCircle(tf.HandPointPrediction.IntX, tf.HandPointPrediction.IntY, 3, Constants.FingerPointsPredictionColor); } } //image1 OutputImages[1].drawImage(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(255)), Constants.EdgeImageColor); foreach (Finger f in fingerTracker.Fingers) { for (int i = 0; i < f.SliceTrail.NumSlices; i++) OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Constants.FingerSliceColor); OutputImages[1].drawContour(f.Contour, Constants.FingerContourColor); OutputImages[1].drawLineSegment(f.LineSegment, Constants.FingerTrackedColor); OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor); } //image2 foreach (Hand h in handTracker.Hands) { OutputImages[2].drawImage(h.Mask.ThresholdBinary(new Gray(0), new Gray(255)), Constants.HandColor[h.ZIndex % Constants.HandNumColors]); OutputImages[2].fillCircle(h.Centroid.IntX, h.Centroid.IntY, 5, Constants.HandCentroidColor); OutputImages[2].drawText(h.Centroid.IntX, h.Centroid.IntY, h.TrackID.ToString(), Constants.HandIDColor); if (h.ThumbDefect != null) OutputImages[2].drawDefect(h.ThumbDefect, Color.CornflowerBlue, Color.Yellow); } //image3 OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.ThresholdBinary(new Gray(0), new Gray(255))), Constants.DepthImageColor); foreach (TrackedTouchEvent tte in touchTracker.TrackedObjects) { Vector2D position = tte.AbsolutePositionPrediction; OutputImages[3].fillCircle(position.IntX, position.IntY, 5, Constants.TouchEventTrackedColor); } foreach (Palm p in palmTracker.OptimizedPalms) { OutputImages[3].drawQuadrangleGrid(p.Quad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns); } //image4 touchEventVisualizer.updateImage(); OutputImages[4] = touchEventVisualizer.OutputImage; //borders for (int i = 0; i < numImages; i++) { OutputImages[i].drawRectangle(0, 0, imageWidth - 1, imageHeight - 1, Constants.OutputImageBorderColor); } Timer.stop("createOutputImages"); } } }