bbiwarg-original/bbiwarg/InputHandler.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using bbiwarg.InputProviders;
using bbiwarg.Images;
using bbiwarg.Recognition.FingerRecognition;
using bbiwarg.Recognition.HandRecognition;
using bbiwarg.Recognition.PalmRecognition;
using bbiwarg.Recognition.TouchRecognition;
using bbiwarg.Recognition.Tracking;
using bbiwarg.Server;
using bbiwarg.Graphics;
using bbiwarg.Utility;
using Emgu.CV;
using Emgu.CV.Structure;

namespace bbiwarg
{
    class InputHandler
    {
        public OutputImage[] OutputImages { get; private set; }

        private InputProvider inputProvider;
        private InputFrame currentInputFrame;
        private int imageWidth;
        private int imageHeight;
        private int currentFrameID;
        private int lastFrameID;

        private DepthImage depthImage;
        private EdgeImage edgeImage;
        private ConfidenceImage confidenceImage;

        private FingerDetector fingerDetector;
        private HandDetector handDetector;
        private PalmDetector palmDetector;
        private TouchDetector touchDetector;

        private FingerTracker fingerTracker;
        private HandTracker handTracker;
        private PalmTracker palmTracker;
        private TouchTracker touchTracker;

        private TouchEventVisualizer touchEventVisualizer;
        private TuioCommunicator tuioCommunicator;

        public InputHandler(InputProvider inputProvider)
        {
            this.inputProvider = inputProvider;
            this.imageWidth = inputProvider.ImageWidth;
            this.imageHeight = inputProvider.ImageHeight;
            this.currentFrameID = inputProvider.CurrentFrameID;
            this.lastFrameID = this.currentFrameID - 1;

            initializeConsistentObjects();
        }

        private void initializeConsistentObjects()
        {
            fingerTracker = new FingerTracker();
            handTracker = new HandTracker();
            palmTracker = new PalmTracker();
            touchTracker = new TouchTracker();

            if (Constants.OutputEnabled)
            {
                touchEventVisualizer = new TouchEventVisualizer(imageWidth, imageHeight);
                //register touchEventVisualizer to touchTracker
                touchTracker.TouchDown += touchEventVisualizer.touchDown;
                touchTracker.TouchMove += touchEventVisualizer.touchMove;
                touchTracker.TouchUp += touchEventVisualizer.touchUp;
            }

            if (Constants.TuioEnabled)
            {
                tuioCommunicator = new TuioCommunicator(Constants.TuioIP, Constants.TuioPort);

                //register tuiCommunicator to touchTracker
                touchTracker.TouchDown += tuioCommunicator.touchDown;
                touchTracker.TouchMove += tuioCommunicator.touchMove;
                touchTracker.TouchUp += tuioCommunicator.touchUp;
            }
        }

        private void resetConsistentObjects()
        {
            touchTracker.reset();
            handTracker.reset();
            palmTracker.reset();
            fingerTracker.reset();

            if (Constants.OutputEnabled)
                touchEventVisualizer.reset();

            if (Constants.TuioEnabled)
                tuioCommunicator.reset();
        }

        public void updateFrame()
        {
            Timer.start("updateFrame");

            if (currentFrameID != inputProvider.CurrentFrameID)
            {
                beforeUpdateFrame();

                if (lastFrameID - currentFrameID > 20)
                    resetConsistentObjects();

                createConfidenceImage();
                createDepthImage();
                createEdgeImage();

                detectFingers();
                trackFingers();

                detectHands();
                trackHands();

                detectPalm();
                trackPalm();

                detectTouchEvents();
                trackTouchEvents();

                afterUpdateFrame();
            }

            Timer.stop("updateFrame");
        }

        private void beforeUpdateFrame()
        {
            Timer.start("beforeUpdateFrame");

            currentInputFrame = inputProvider.CurrentFrame;

            lastFrameID = currentFrameID;
            currentFrameID = currentInputFrame.FrameID;
            Logger.CurrentFrame = currentFrameID;

            if (Constants.TuioEnabled)
                tuioCommunicator.initFrame();

            Timer.stop("beforeUpdateFrame");
        }

        private void afterUpdateFrame()
        {
            Timer.start("afterUpdateFrame");

            if (Constants.TuioEnabled)
                tuioCommunicator.commitFrame();

            if (Constants.OutputEnabled)
                createOutputImages();

            Timer.stop("afterUpdateFrame");
        }

        private void createConfidenceImage()
        {
            Timer.start("createCnfdncImg");
            Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawConfidenceData);
            confidenceImage = new ConfidenceImage(rawConfidenceImage);
            Timer.stop("createCnfdncImg");
        }

        private void createDepthImage()
        {
            Timer.start("createDepthImage");
            Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawDepthData);
            rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
            depthImage = new DepthImage(rawDepthImage);
            Timer.stop("createDepthImage");
        }

        private void createEdgeImage()
        {
            Timer.start("createEdgeImage");
            edgeImage = new EdgeImage(depthImage);
            Timer.stop("createEdgeImage");
        }

        private void detectFingers()
        {
            Timer.start("detectFingers");
            fingerDetector = new FingerDetector(depthImage, edgeImage);
            Timer.stop("detectFingers");
        }

        private void trackFingers()
        {
            Timer.start("trackFingers");
            fingerTracker.updateFrame(fingerDetector.Fingers);
            Timer.stop("trackFingers");
        }

        private void detectHands()
        {
            Timer.start("detectHands");
            handDetector = new HandDetector(depthImage, edgeImage, fingerTracker.Fingers);
            Timer.stop("detectHands");
        }

        private void trackHands()
        {
            Timer.start("trackHands\t");
            handTracker.updateFrame(handDetector.Hands);
            Timer.stop("trackHands\t");
        }

        private void detectPalm()
        {
            Timer.start("detectPalm\t");
            palmDetector = new PalmDetector(handTracker.Hands);
            Timer.stop("detectPalm\t");
        }

        private void trackPalm()
        {
            Timer.start("trackPalm\t");
            palmTracker.updateFrame(palmDetector.Palms);
            Timer.stop("trackPalm\t");
        }

        private void detectTouchEvents()
        {
            Timer.start("detectTouchEvents");
            touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmTracker.OptimizedPalms);
            Timer.stop("detectTouchEvents");
        }

        private void trackTouchEvents()
        {
            Timer.start("trackTouchEvents");
            touchTracker.updateFrame(touchDetector.TouchEvents);
            Timer.stop("trackTouchEvents");
        }

        private void createOutputImages()
        {
            Timer.start("createOutputImages");
            int numImages = Constants.OutputNumImages;
            OutputImages = new OutputImage[numImages];
            for (int i = 0; i < numImages; i++)
            {
                OutputImages[i] = new OutputImage(imageWidth, imageHeight);
            }

            //image0
            OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Constants.DepthImageColor);
            foreach (Finger f in fingerTracker.Fingers)
            {
                OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 4, Constants.FingerTipColor);
            }

            foreach (TrackedFinger tf in fingerTracker.TrackedObjects)
            {
                if (tf.CurrentState == TrackingState.Tracked)
                {
                    OutputImages[0].fillCircle(tf.TipPointPrediction.IntX, tf.TipPointPrediction.IntY, 3, Constants.FingerPointsPredictionColor);
                    OutputImages[0].fillCircle(tf.HandPointPrediction.IntX, tf.HandPointPrediction.IntY, 3, Constants.FingerPointsPredictionColor);
                }
            }

            //image1
            OutputImages[1].drawImage(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(255)), Constants.EdgeImageColor);
            foreach (Finger f in fingerTracker.Fingers)
            {
                for (int i = 0; i < f.SliceTrail.NumSlices; i++)
                    OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Constants.FingerSliceColor);
                OutputImages[1].drawContour(f.Contour, Constants.FingerContourColor);
                OutputImages[1].drawLineSegment(f.LineSegment, Constants.FingerTrackedColor);
                OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
            }

            //image2
            foreach (Hand h in handTracker.Hands)
            {
                OutputImages[2].drawImage(h.Mask.ThresholdBinary(new Gray(0), new Gray(255)), Constants.HandColor[h.ZIndex % Constants.HandNumColors]);
                OutputImages[2].fillCircle(h.Centroid.IntX, h.Centroid.IntY, 5, Constants.HandCentroidColor);
                OutputImages[2].drawText(h.Centroid.IntX, h.Centroid.IntY, h.TrackID.ToString(), Constants.HandIDColor);

                if (h.ThumbDefect != null)
                {
                    OutputImages[2].fillCircle(h.ThumbDefect.Inner.IntX, h.ThumbDefect.Inner.IntY, 4, Color.CornflowerBlue);
                    OutputImages[2].fillCircle(h.ThumbDefect.OuterShort.IntX, h.ThumbDefect.OuterShort.IntY, 4, Color.CornflowerBlue);
                    OutputImages[2].fillCircle(h.ThumbDefect.OuterLong.IntX, h.ThumbDefect.OuterLong.IntY, 4, Color.CornflowerBlue);
                }
            }

            //image3
            OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.ThresholdBinary(new Gray(0), new Gray(255))), Constants.DepthImageColor);
            foreach (TouchEvent te in touchTracker.TouchEvents)
                OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
            foreach (Palm p in palmTracker.OptimizedPalms) {
                OutputImages[3].drawQuadrangleGrid(p.Quad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns);
            }

            //image4
            touchEventVisualizer.updateImage();
            OutputImages[4] = touchEventVisualizer.OutputImage;


            //borders
            for (int i = 0; i < numImages; i++)
            {
                OutputImages[i].drawRectangle(0, 0, imageWidth - 1, imageHeight - 1, Constants.OutputImageBorderColor);
            }

            Timer.stop("createOutputImages");
        }

    }
}