bbiwarg-original/bbiwarg/VideoHandle.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Diagnostics;
using bbiwarg.Utility;
using bbiwarg.Recognition.FingerRecognition;
using bbiwarg.Recognition.PalmRecognition;
using bbiwarg.Recognition.TouchRecognition;
using bbiwarg.Recognition.HandRecognition;
using bbiwarg.Images;
using bbiwarg.InputProviders;
using Emgu.CV;
using Emgu.CV.Structure;
using bbiwarg.Graphics;
using TUIO;

namespace bbiwarg
{
    class VideoHandle
    {
        private IInputProvider inputProvider;
        private InputFrame inputFrame;

        private TuioServer server;
        private TuioCursor currentTouchevent;

        public int Width { get; private set; }
        public int Height { get; private set; }

        public int CurrentFrame
        {
            get
            {
                if (sourceIsMovie())
                    return inputProvider.getCurrentMovieFrame();
                else
                    return videoFrame;
            }
        }
        private int lastFrame = int.MaxValue;

        public OutputImage[] OutputImages { get; private set; }
        
        private DepthImage depthImage;
        private EdgeImage edgeImage;
        private ConfidenceImage confidenceImage;
        
        private FingerDetector fingerDetector;
        private HandDetector handDetector;
        private PalmDetector palmDetector;
        private TouchDetector touchDetector;
        private PalmTouchDetector palmTouchDetector;

        private FingerTracker fingerTracker;
        private TouchTracker touchTracker;

        private TouchEventVisualizer touchEventVisualizer;

        private int videoFrame = 0;

        public VideoHandle(IInputProvider inputProvider)
        {
            this.inputProvider = inputProvider;
        }

        public void start()
        {
            palmDetector = new PalmDetector();
            server = new TuioServer("127.0.0.1", 3333);

            inputProvider.init();
            inputProvider.start();
            inputProvider.updateFrame();
            processFrameUpdate();
        }

        public void stop()
        {
            inputProvider.stop();
            server.close();
        }

        public bool sourceIsMovie()
        {
            return inputProvider.sourceIsMovie();
        }

        public void reversePlay()
        {
            inputProvider.reversePlay();
        }

        public void pauseMovie()
        {
            inputProvider.pauseMovie();
        }

        public void unpauseMovie()
        {
            inputProvider.unpauseMovie();
        }

        public void nextFrame()
        {
            if (inputProvider.isActive())
            {
                inputProvider.releaseFrame();
                inputProvider.updateFrame();
                processFrameUpdate();
            }
            else
            {
                inputProvider.stop();
            }
            videoFrame++;
        }

        public List<PalmTouchEvent> getPalmTouchEvents()
        {
            return palmTouchDetector.PalmTouchEvents;
        }

        private void processFrameUpdate()
        {
            bool newStarted = (lastFrame - CurrentFrame >= 20);
            Timer.start("processFrameUpdate");

            if (newStarted) {
                //initialize trackers
                touchTracker = new TouchTracker();
                fingerTracker = new FingerTracker();
            }

            //read data from inputProvider
            Timer.start("readInputData");
            inputFrame = inputProvider.getInputFrame();
            Width = inputFrame.Width;
            Height = inputFrame.Height;
            Timer.stop("readInputData");

            //create output images
            Timer.start("createOutputImages");
            OutputImages = new OutputImage[Constants.OutputNumImages];
            for (int i = 0; i < Constants.OutputNumImages; i++) {
                OutputImages[i] = new OutputImage(Width, Height);
                    
            }
            Timer.stop("createOutputImages");

            //create confidenceImage
            Timer.start("createConfidenceImage");
            Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawConfidenceData);
            confidenceImage = new ConfidenceImage(rawConfidenceImage);
            Timer.stop("createConfidenceImage");

            //create depthImage
            Timer.start("createDepthImage");
            Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawDepthData);
            rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
            depthImage = new DepthImage(rawDepthImage);
            OutputImages[0].Image[0] = OutputImages[0].Image[1] = OutputImages[0].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
            Timer.stop("createDepthImage");

            // create edge image
            Timer.start("createEdgeImage");
            edgeImage = new EdgeImage(depthImage);
            OutputImages[1].Image[2] = edgeImage.Image.Mul(255);
            Timer.stop("createEdgeImage");

            //detect fingers
            Timer.start("fingerDetection");
            fingerDetector = new FingerDetector(depthImage, edgeImage, OutputImages[1]);
            Timer.stop("fingerDetection");

            //track fingers
            Timer.start("fingerTracking");
            fingerTracker.updateTrackedFingers(fingerDetector.Fingers, OutputImages[1]);
            Timer.stop("fingerTracking");

            //detect hands
            Timer.start("handDetection");
            handDetector = new HandDetector(depthImage, edgeImage, fingerDetector.Fingers, OutputImages[2]);
            Timer.stop("handDetection");

            //remove background noise
            Timer.start("removeBackground");
            OutputImages[3].Image[0] = OutputImages[3].Image[1] = OutputImages[3].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255-handDetector.HandMask);
            Timer.stop("removeBackground");

            //detect palm
            Timer.start("palmDetection");
            if (newStarted)
                palmDetector.reset();
            palmDetector.findPalmQuad(OutputImages[3], handDetector.Hands);
            Timer.stop("palmDetection");

            //detect touchEvents
            Timer.start("touchDetection");
            touchDetector = new TouchDetector(fingerTracker.Fingers, depthImage, OutputImages[0]);
            if (palmDetector.PalmQuad != null)
                palmTouchDetector = new PalmTouchDetector(touchDetector.TouchEvents, palmDetector.PalmQuad, palmDetector.PalmHandSide);
            Timer.stop("touchDetection");

            //track touchEvents
            Timer.start("touchTracking");
            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[3]);
            Timer.stop("touchTracking");

            Timer.start("sending touchEvents");
            //send touchevent
            if (palmTouchDetector != null)
            {
                server.initFrame();
                if (palmTouchDetector.PalmTouchEvents.Count == 1)
                {
                    PalmTouchEvent touchEvent = palmTouchDetector.PalmTouchEvents[0];
                    if (currentTouchevent == null)
                        currentTouchevent = server.addTuioCursor(touchEvent.RelativePalmPosition.X, touchEvent.RelativePalmPosition.Y);
                    else
                        server.updateTuioCursor(currentTouchevent, touchEvent.RelativePalmPosition.X, touchEvent.RelativePalmPosition.Y);
                }
                else if (currentTouchevent != null)
                {
                    server.removeTuioCursor(currentTouchevent);
                    currentTouchevent = null;
                }
                server.commitFrame();
            }
            Timer.stop("sending touchEvents");


            // touch event visualizer
            if (touchEventVisualizer == null)
                touchEventVisualizer = new TouchEventVisualizer(Width, Height);
            if (newStarted)
                touchEventVisualizer.Reset();
            if (palmTouchDetector != null)
            {
                foreach (PalmTouchEvent e in palmTouchDetector.PalmTouchEvents)
                {
                touchEventVisualizer.addPalmTouchEvent(e, CurrentFrame);
                }
                touchEventVisualizer.updateImage();
                OutputImages[4] = touchEventVisualizer.OutputImage;
            }
            
            // add borders
            for (int i = 0; i < Constants.OutputNumImages; i++)
            {
                OutputImages[i].drawRectangle(0, 0, Width - 1, Height - 1, Color.White);

            }

            Timer.stop("processFrameUpdate");
            lastFrame = CurrentFrame;
        }
    }
}