|
@@ -0,0 +1,286 @@
|
|
|
+using System;
|
|
|
+using System.Collections.Generic;
|
|
|
+using System.Drawing;
|
|
|
+using System.Linq;
|
|
|
+using System.Text;
|
|
|
+using System.Threading.Tasks;
|
|
|
+using bbiwarg.InputProviders;
|
|
|
+using bbiwarg.Images;
|
|
|
+using bbiwarg.Recognition.FingerRecognition;
|
|
|
+using bbiwarg.Recognition.HandRecognition;
|
|
|
+using bbiwarg.Recognition.PalmRecognition;
|
|
|
+using bbiwarg.Recognition.TouchRecognition;
|
|
|
+using bbiwarg.Server;
|
|
|
+using bbiwarg.Graphics;
|
|
|
+using bbiwarg.Utility;
|
|
|
+using Emgu.CV;
|
|
|
+using Emgu.CV.Structure;
|
|
|
+
|
|
|
+namespace bbiwarg
|
|
|
+{
|
|
|
+ class InputHandler
|
|
|
+ {
|
|
|
+ public OutputImage[] OutputImages { get; private set; }
|
|
|
+
|
|
|
+ private InputProvider inputProvider;
|
|
|
+ private InputFrame currentInputFrame;
|
|
|
+ private int imageWidth;
|
|
|
+ private int imageHeight;
|
|
|
+ private int currentFrame;
|
|
|
+ private int lastFrame;
|
|
|
+
|
|
|
+ private DepthImage depthImage;
|
|
|
+ private EdgeImage edgeImage;
|
|
|
+ private ConfidenceImage confidenceImage;
|
|
|
+
|
|
|
+ private FingerDetector fingerDetector;
|
|
|
+ private HandDetector handDetector;
|
|
|
+ private PalmDetector palmDetector;
|
|
|
+ private TouchDetector touchDetector;
|
|
|
+
|
|
|
+ private FingerTracker fingerTracker;
|
|
|
+ private TouchTracker touchTracker;
|
|
|
+
|
|
|
+ private TouchEventVisualizer touchEventVisualizer;
|
|
|
+ private TuioCommunicator tuioCommunicator;
|
|
|
+
|
|
|
+ public InputHandler(InputProvider inputProvider)
|
|
|
+ {
|
|
|
+ this.inputProvider = inputProvider;
|
|
|
+ this.imageWidth = inputProvider.ImageWidth;
|
|
|
+ this.imageHeight = inputProvider.ImageHeight;
|
|
|
+ this.currentFrame = inputProvider.CurrentFrame;
|
|
|
+ this.lastFrame = this.currentFrame - 1;
|
|
|
+
|
|
|
+ initializeConsistentObjects();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void initializeConsistentObjects()
|
|
|
+ {
|
|
|
+ palmDetector = new PalmDetector();
|
|
|
+
|
|
|
+ fingerTracker = new FingerTracker();
|
|
|
+ touchTracker = new TouchTracker();
|
|
|
+
|
|
|
+ if (Constants.OutputEnabled)
|
|
|
+ {
|
|
|
+ touchEventVisualizer = new TouchEventVisualizer(imageWidth, imageHeight);
|
|
|
+ //register touchEventVisualizer to touchTracker
|
|
|
+ touchTracker.PalmTouchDown += touchEventVisualizer.touchDown;
|
|
|
+ touchTracker.PalmTouchMove += touchEventVisualizer.touchMove;
|
|
|
+ touchTracker.PalmTouchUp += touchEventVisualizer.touchUp;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (Constants.TuioEnabled)
|
|
|
+ {
|
|
|
+ tuioCommunicator = new TuioCommunicator(Constants.TuioIP, Constants.TuioPort);
|
|
|
+
|
|
|
+ //register tuiCommunicator to touchTracker
|
|
|
+ touchTracker.PalmTouchDown += tuioCommunicator.touchDown;
|
|
|
+ touchTracker.PalmTouchMove += tuioCommunicator.touchMove;
|
|
|
+ touchTracker.PalmTouchUp += tuioCommunicator.touchUp;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void resetConsistentObjects()
|
|
|
+ {
|
|
|
+ palmDetector.reset();
|
|
|
+
|
|
|
+ touchTracker.reset();
|
|
|
+ fingerTracker.reset();
|
|
|
+
|
|
|
+ if (Constants.OutputEnabled)
|
|
|
+ touchEventVisualizer.reset();
|
|
|
+
|
|
|
+ if (Constants.TuioEnabled)
|
|
|
+ tuioCommunicator.reset();
|
|
|
+ }
|
|
|
+
|
|
|
+ public void updateFrame()
|
|
|
+ {
|
|
|
+ Timer.start("updateFrame");
|
|
|
+
|
|
|
+ beforeUpdateFrame();
|
|
|
+
|
|
|
+ if (lastFrame-currentFrame > 20)
|
|
|
+ resetConsistentObjects();
|
|
|
+
|
|
|
+ createConfidenceImage();
|
|
|
+ createDepthImage();
|
|
|
+ createEdgeImage();
|
|
|
+
|
|
|
+ detectFingers();
|
|
|
+ trackFingers();
|
|
|
+
|
|
|
+ detectHands();
|
|
|
+ detectPalm();
|
|
|
+
|
|
|
+ detectTouchEvents();
|
|
|
+ trackTouchEvents();
|
|
|
+
|
|
|
+ afterUpdateFrame();
|
|
|
+
|
|
|
+ Timer.stop("updateFrame");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void beforeUpdateFrame()
|
|
|
+ {
|
|
|
+ currentInputFrame = inputProvider.getInputFrame();
|
|
|
+
|
|
|
+ lastFrame = currentFrame;
|
|
|
+ currentFrame = currentInputFrame.FrameNumber;
|
|
|
+ Logger.CurrentFrame = currentFrame;
|
|
|
+
|
|
|
+ if (Constants.TuioEnabled)
|
|
|
+ tuioCommunicator.initFrame();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void afterUpdateFrame()
|
|
|
+ {
|
|
|
+ if (Constants.TuioEnabled)
|
|
|
+ tuioCommunicator.commitFrame();
|
|
|
+
|
|
|
+ if (Constants.OutputEnabled)
|
|
|
+ createOutputImages();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void createConfidenceImage()
|
|
|
+ {
|
|
|
+ Timer.start("createConfidenceImage");
|
|
|
+ Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawConfidenceData);
|
|
|
+ confidenceImage = new ConfidenceImage(rawConfidenceImage);
|
|
|
+ Timer.stop("createConfidenceImage");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void createDepthImage()
|
|
|
+ {
|
|
|
+ Timer.start("createDepthImage");
|
|
|
+ Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawDepthData);
|
|
|
+ rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
|
|
|
+ depthImage = new DepthImage(rawDepthImage);
|
|
|
+ Timer.stop("createDepthImage");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void createEdgeImage()
|
|
|
+ {
|
|
|
+ Timer.start("createEdgeImage");
|
|
|
+ edgeImage = new EdgeImage(depthImage);
|
|
|
+ Timer.stop("createEdgeImage");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void detectFingers()
|
|
|
+ {
|
|
|
+ Timer.start("detectFingers");
|
|
|
+ fingerDetector = new FingerDetector(depthImage, edgeImage);
|
|
|
+ Timer.stop("detectFingers");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void trackFingers()
|
|
|
+ {
|
|
|
+ Timer.start("trackFingers");
|
|
|
+ fingerTracker.updateFrame(fingerDetector.Fingers);
|
|
|
+ Timer.stop("trackFingers");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void detectHands()
|
|
|
+ {
|
|
|
+ Timer.start("detectHands");
|
|
|
+ handDetector = new HandDetector(edgeImage, fingerDetector.Fingers);
|
|
|
+ Timer.stop("detectHands");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void detectPalm()
|
|
|
+ {
|
|
|
+ Timer.start("detectPalm");
|
|
|
+ palmDetector.findPalmQuad(handDetector.Hands, handDetector.HandMask);
|
|
|
+ Timer.stop("detectPalm");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void detectTouchEvents()
|
|
|
+ {
|
|
|
+ Timer.start("detectTouchEvents");
|
|
|
+ touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmDetector.PalmQuad);
|
|
|
+ Timer.stop("detectTouchEvents");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void trackTouchEvents()
|
|
|
+ {
|
|
|
+ Timer.start("trackTouchEvents");
|
|
|
+ touchTracker.updateFrame(touchDetector.TouchEvents);
|
|
|
+ Timer.stop("trackTouchEvents");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void createOutputImages()
|
|
|
+ {
|
|
|
+ Timer.start("createOutputImages");
|
|
|
+ int numImages = Constants.OutputNumImages;
|
|
|
+ OutputImages = new OutputImage[numImages];
|
|
|
+ for (int i = 0; i < numImages; i++)
|
|
|
+ {
|
|
|
+ OutputImages[i] = new OutputImage(imageWidth, imageHeight);
|
|
|
+ }
|
|
|
+
|
|
|
+ //image0
|
|
|
+ OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Constants.DepthImageColor);
|
|
|
+ foreach (Finger f in fingerTracker.Fingers)
|
|
|
+ {
|
|
|
+ OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 3, Constants.FingerTipColor);
|
|
|
+ if (f.TouchEvent != null)
|
|
|
+ OutputImages[0].fillCircle(f.TouchEvent.Position.IntX, f.TouchEvent.Position.IntY, 5, Constants.TouchEventDetectedColor);
|
|
|
+ }
|
|
|
+
|
|
|
+ //image1
|
|
|
+ OutputImages[1].drawImage(edgeImage.Image.Mul(255), Constants.EdgeImageColor);
|
|
|
+ foreach (Finger f in fingerTracker.Fingers)
|
|
|
+ {
|
|
|
+ for (int i = 0; i < f.SliceTrail.NumSlices; i++)
|
|
|
+ OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Constants.FingerSliceColor);
|
|
|
+ OutputImages[1].drawContour(f.Contour, Constants.FingerContourColor);
|
|
|
+ OutputImages[1].drawLineSegment(f.LineSegment, Constants.FingerTrackedColor);
|
|
|
+ OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
|
|
|
+ }
|
|
|
+
|
|
|
+ //image2
|
|
|
+ foreach (Hand h in handDetector.Hands)
|
|
|
+ {
|
|
|
+ if (h.Side == Hand.HandSide.Right)
|
|
|
+ OutputImages[2].drawImage(h.Mask.Mul(255), Constants.HandRightColor);
|
|
|
+ else
|
|
|
+ OutputImages[2].drawImage(h.Mask.Mul(255), Constants.HandLeftColor);
|
|
|
+ }
|
|
|
+
|
|
|
+ //image3
|
|
|
+ OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.Mul(255)), Constants.DepthImageColor);
|
|
|
+ foreach (TouchEvent te in touchTracker.TouchEvents)
|
|
|
+ OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
|
|
|
+ if (palmDetector.PalmContour != null && palmDetector.PalmContour.Count<Point>() > 0)
|
|
|
+ {
|
|
|
+ OutputImages[3].drawContour(palmDetector.PalmContour, Constants.PalmConturColor);
|
|
|
+ OutputImages[3].drawPoints(palmDetector.PalmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
|
|
|
+ }
|
|
|
+ if (palmDetector.PalmQuad != null) {
|
|
|
+ OutputImages[3].fillCircle(palmDetector.ThumbDefectStart.IntX, palmDetector.ThumbDefectStart.IntY, 3, Color.Red);
|
|
|
+ OutputImages[3].fillCircle(palmDetector.ThumbDefectEnd.IntX, palmDetector.ThumbDefectEnd.IntY, 3, Color.Red);
|
|
|
+ OutputImages[3].fillCircle(palmDetector.ThumbDefectDepth.IntX, palmDetector.ThumbDefectDepth.IntY, 3, Color.Red);
|
|
|
+
|
|
|
+ OutputImages[3].drawLineSegment(new Utility.LineSegment2D(palmDetector.ThumbDefectDepth, (palmDetector.ThumbDefectStart + palmDetector.ThumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
|
|
|
+
|
|
|
+ OutputImages[3].drawQuadrangleGrid(palmDetector.PalmQuad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns);
|
|
|
+ }
|
|
|
+
|
|
|
+ //image4
|
|
|
+ touchEventVisualizer.updateImage();
|
|
|
+ OutputImages[4] = touchEventVisualizer.OutputImage;
|
|
|
+
|
|
|
+
|
|
|
+ //borders
|
|
|
+ for (int i = 0; i < numImages; i++)
|
|
|
+ {
|
|
|
+ OutputImages[i].drawRectangle(0, 0, imageWidth - 1, imageHeight - 1, Constants.OutputImageBorderColor);
|
|
|
+ }
|
|
|
+
|
|
|
+ Timer.stop("createOutputImages");
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+}
|