bbiwarg-original/bbiwarg/VideoHandle.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Diagnostics;
using bbiwarg.Utility;
using bbiwarg.Recognition.FingerRecognition;
using bbiwarg.Recognition.PalmRecognition;
using bbiwarg.Recognition.TouchRecognition;
using bbiwarg.Recognition.HandRecognition;
using bbiwarg.Recognition.Tracking;
using bbiwarg.Images;
using bbiwarg.InputProviders;
using Emgu.CV;
using Emgu.CV.Structure;
using bbiwarg.Graphics;
using TUIO;

namespace bbiwarg
{
    class VideoHandle
    {
        private IInputProvider inputProvider;
        private InputFrame inputFrame;

        private TuioServer tuioServer;
        private TuioCursor currentTouchevent;

        public int Width { get; private set; }
        public int Height { get; private set; }

        public int CurrentFrame
        {
            get
            {
                if (sourceIsMovie())
                    return inputProvider.getCurrentMovieFrame();
                else
                    return videoFrame;
            }
        }
        private int lastFrame = int.MaxValue;

        public OutputImage[] OutputImages { get; private set; }
        
        private DepthImage depthImage;
        private EdgeImage edgeImage;
        private ConfidenceImage confidenceImage;
        
        private FingerDetector fingerDetector;
        private HandDetector handDetector;
        private PalmDetector palmDetector;
        private TouchDetector touchDetector;

        private FingerTracker fingerTracker;
        private TouchTracker touchTracker;

        private TouchEventVisualizer touchEventVisualizer;

        private int videoFrame = 0;

        public VideoHandle(IInputProvider inputProvider)
        {
            this.inputProvider = inputProvider;
        }

        public void start()
        {
            inputProvider.init();
            inputProvider.start();
            inputProvider.updateFrame();
            initTrackers();
            processFrameUpdate();
        }

        public void stop()
        {
            inputProvider.stop();
            tuioServer.close();
        }

        public bool sourceIsMovie()
        {
            return inputProvider.sourceIsMovie();
        }

        public void reversePlay()
        {
            inputProvider.reversePlay();
        }

        public void pauseMovie()
        {
            inputProvider.pauseMovie();
        }

        public void unpauseMovie()
        {
            inputProvider.unpauseMovie();
        }

        public void nextFrame()
        {
            if (inputProvider.isActive())
            {
                inputProvider.releaseFrame();
                inputProvider.updateFrame();
                processFrameUpdate();
            }
            else
            {
                inputProvider.stop();
            }
            videoFrame++;
        }

        private void initTrackers()
        {
            palmDetector = new PalmDetector();
            tuioServer = new TuioServer(Constants.TuioIP, Constants.TuioPort);
            touchEventVisualizer = new TouchEventVisualizer(Constants.ImageWidth, Constants.ImageHeight);

            fingerTracker = new FingerTracker();
            touchTracker = new TouchTracker(tuioServer, touchEventVisualizer);
        }

        private void processFrameUpdate()
        {
            bool newStarted = (lastFrame - CurrentFrame >= 20);
            Timer.start("processFrameUpdate");

            if (newStarted) {
                //reset trackers
                touchTracker.reset();
                fingerTracker.reset();
            }

            //read data from inputProvider
            Timer.start("readInputData");
            inputFrame = inputProvider.getInputFrame();
            Width = inputFrame.Width;
            Height = inputFrame.Height;
            Timer.stop("readInputData");

            //create output images
            Timer.start("createOutputImages");
            OutputImages = new OutputImage[Constants.OutputNumImages];
            for (int i = 0; i < Constants.OutputNumImages; i++) {
                OutputImages[i] = new OutputImage(Width, Height);
                    
            }
            Timer.stop("createOutputImages");

            //create confidenceImage
            Timer.start("createConfidenceImage");
            Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawConfidenceData);
            confidenceImage = new ConfidenceImage(rawConfidenceImage);
            Timer.stop("createConfidenceImage");

            //create depthImage
            Timer.start("createDepthImage");
            Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawDepthData);
            rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
            depthImage = new DepthImage(rawDepthImage);
            OutputImages[0].Image[0] = OutputImages[0].Image[1] = OutputImages[0].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
            Timer.stop("createDepthImage");

            // create edge image
            Timer.start("createEdgeImage");
            edgeImage = new EdgeImage(depthImage);
            OutputImages[1].Image[2] = edgeImage.Image.Mul(255);
            Timer.stop("createEdgeImage");

            //detect fingers
            Timer.start("fingerDetection");
            fingerDetector = new FingerDetector(depthImage, edgeImage, OutputImages[1]);
            Timer.stop("fingerDetection");

            //track fingers
            Timer.start("fingerTracking");
            fingerTracker.updateFrame(fingerDetector.Fingers);
            foreach (Finger f in fingerTracker.Fingers) {
                OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
            }
            Timer.stop("fingerTracking");

            //detect hands
            Timer.start("handDetection");
            handDetector = new HandDetector(depthImage, edgeImage, fingerDetector.Fingers, OutputImages[2]);
            Timer.stop("handDetection");

            //remove background noise
            Timer.start("removeBackground");
            OutputImages[3].Image[0] = OutputImages[3].Image[1] = OutputImages[3].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255-handDetector.HandMask);
            Timer.stop("removeBackground");

            //detect palm
            Timer.start("palmDetection");
            if (newStarted)
                palmDetector.reset();
            palmDetector.findPalmQuad(OutputImages[3], handDetector.Hands, handDetector.HandMask);
            Timer.stop("palmDetection");

            //detect touchEvents
            Timer.start("touchDetection");
            touchDetector = new TouchDetector(fingerTracker.Fingers, palmDetector.PalmQuad, depthImage, OutputImages[0]);
            Timer.stop("touchDetection");

            //track touchEvents
            Timer.start("touchTracking");
            touchTracker.updateFrame(touchDetector.TouchEvents);
            foreach(TouchEvent te in touchTracker.TouchEvents)
            {
                OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
            }

            OutputImages[4] = touchEventVisualizer.OutputImage;
            Timer.stop("touchTracking");

           
            // add borders
            for (int i = 0; i < Constants.OutputNumImages; i++)
            {
                OutputImages[i].drawRectangle(0, 0, Width - 1, Height - 1, Color.White);

            }

            Timer.stop("processFrameUpdate");
            lastFrame = CurrentFrame;
        }
    }
}