bbiwarg-original/bbiwarg/Output/GlassesOutput/GlassesWindow.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using bbiwarg.Input.InputHandling;
using bbiwarg.Input.InputProviding;
using bbiwarg.Utility;
using bbiwarg.Recognition.FingerRecognition;
using bbiwarg.Recognition.PalmRecognition;
using Emgu.CV.UI;

namespace bbiwarg.Output.GlassesOutput
{
    /// <summary>
    /// A Windows Form which displays a fullscreen window to be shown on augmented reality glasses.
    /// </summary>
    public partial class GlassesWindow : Form
    {
        /// <summary>
        /// the input provider
        /// </summary>
        private InputProvider inputProvider;

        /// <summary>
        /// the input handler
        /// </summary>
        private InputHandler inputHandler;

        /// <summary>
        /// timer to periodically update the window
        /// </summary>
        private System.Windows.Forms.Timer timer;

        /// <summary>
        /// id of the current frame
        /// </summary>
        private int currentFrameID;

        /// <summary>
        /// true iff the window is showing the latest data
        /// </summary>
        private bool guiUpToDate;

        /// <summary>
        /// size of the input images
        /// </summary>
        private ImageSize inputSize;

        /// <summary>
        /// size of the image show on the glasses
        /// </summary>
        private ImageSize outputSize;

        /// <summary>
        /// the image shown on the glasses
        /// </summary>
        private OutputImage image;

        /// <summary>
        /// projection from the input image plane to a plane which lies in front of the wearer of the glasses (output plane)
        /// </summary>
        private Projection2DTo2D projection;

        /// <summary>
        /// position of the current calibration point in the output plane
        /// </summary>
        private Vector2D currentCalibrationPoint;

        /// <summary>
        /// true iff the calibration image has the latest data
        /// </summary>
        private bool calibrationImageUpToDate;

        /// <summary>
        /// random number generator
        /// </summary>
        private Random rand;


        /// <summary>
        /// Creates a GlassesWindow.
        /// </summary>
        /// <param name="inputProvider">input provider</param>
        /// <param name="inputHandler">input handler</param>
        /// <param name="name">title of the window</param>
        /// <param name="screen">the screen this window is shown on</param>
        /// <param name="updateInterval">the update interval for the window in ms</param>
        public GlassesWindow(InputProvider inputProvider, InputHandler inputHandler, String name, Screen screen, int updateInterval)
        {
            InitializeComponent();

            this.inputProvider = inputProvider;
            this.inputHandler = inputHandler;
            this.inputSize = inputHandler.ImageSize;
            this.outputSize = new ImageSize(screen.Bounds.Width, screen.Bounds.Height);
            guiUpToDate = false;
            calibrationImageUpToDate = false;

            rand = new Random();
            currentCalibrationPoint = getRandomOutputPoint();
            projection = new Projection2DTo2D(inputSize, outputSize, Parameters.GlassesWindowNumCalibrationPoints);

            Name = name;
            Text = name;

            timer = new System.Windows.Forms.Timer();
            timer.Interval = updateInterval;
            timer.Tick += update;
            timer.Start();

            KeyPreview = true;

            //fullscreen
            FormBorderStyle = System.Windows.Forms.FormBorderStyle.None;
            Location = screen.Bounds.Location;
            Size = screen.Bounds.Size;
        }

        /// <summary>
        /// Stops the input provider when closing the window.
        /// </summary>
        /// <param name="e">event arguments</param>
        protected override void OnClosing(CancelEventArgs e)
        {
            base.OnClosing(e);
            inputProvider.stop();
        }

        /// <summary>
        /// Updates the window.
        /// </summary>
        /// <param name="sender">the event sender</param>
        /// <param name="e">the event arguments</param>
        private void update(object sender, EventArgs e)
        {
            Utility.Timer.start("GlassesWindow.update");

            if (!inputProvider.IsActive)
                Close();

            if (projection.IsCalibrated)
            {
                FrameData frameData = inputHandler.FrameData;
                if (frameData != null)
                {
                    lock (frameData)
                    {
                        if (currentFrameID != frameData.FrameID)
                        {
                            currentFrameID = frameData.FrameID;
                            Utility.Timer.start("GlassesWindow.update::updateImage");
                            updateImage(frameData);
                            Utility.Timer.stop("GlassesWindow.update::updateImage");
                        }
                    }
                }
            }
            else
            {
                if (!calibrationImageUpToDate)
                    updateCalibrationImage();
            }

            if (!guiUpToDate)
            {
                Utility.Timer.start("GlassesWindow.update::updateGUI");
                updateGUI();
                Utility.Timer.stop("GlassesWindow.update::updateGUI");
            }

            Utility.Timer.stop("GlassesWindow.update");
        }

        /// <summary>
        /// Updates the glasses image.
        /// </summary>
        /// <param name="frameData">data for the new frame</param>
        private void updateImage(FrameData frameData)
        {
            guiUpToDate = false;

            if (image != null)
                image.Dispose();

            image = new OutputImage(outputSize);
            foreach (Palm palm in frameData.TrackedPalms)
            {
                Quadrangle quadInput = palm.Quad;
                Vector2D a = projection.projectPoint(quadInput.TopLeft);
                Vector2D b = projection.projectPoint(quadInput.TopRight);
                Vector2D c = projection.projectPoint(quadInput.BottomRight);
                Vector2D d = projection.projectPoint(quadInput.BottomLeft);
                Quadrangle quadOutput = new Quadrangle(a, b, c, d);
                image.drawQuadrangleGrid(quadOutput, Color.Yellow, Color.Orange, 3, 4);
            }

            foreach (Finger finger in frameData.TrackedFingers)
            {
                Vector2D tipProjected = projection.projectPoint(finger.TipPoint);
                image.fillCircle(tipProjected, 10, Color.Yellow);
            }

        }

        /// <summary>
        /// Updates the calibration image.
        /// </summary>
        private void updateCalibrationImage()
        {
            guiUpToDate = false;

            if (image != null)
                image.Dispose();

            image = new OutputImage(outputSize);
            image.fillCircle(currentCalibrationPoint, 25, Color.Orange);
        }

        /// <summary>
        /// Updates the gui elements.
        /// </summary>
        private void updateGUI()
        {
            // update image boxes
            imageBox.Image = image;

            guiUpToDate = true;
        }

        /// <summary>
        /// Handle key down events by showing the next point or finishing or resetting the calibration.
        /// </summary>
        /// <param name="sender">event sender</param>
        /// <param name="e">event arguments</param>
        private void GlassesWindow_OnKeyDown(object sender, KeyEventArgs e)
        {
            if (e.KeyCode == Keys.K)
            {
                FrameData frameData = inputHandler.FrameData;
                if (frameData != null)
                {
                    lock (frameData)
                    {
                        if (frameData.TrackedFingers.Count == 1)
                        {
                            Vector2D pointOutput = currentCalibrationPoint;
                            Vector2D pointInput = frameData.TrackedFingers[0].TipPoint;
                            
                            projection.addCalibrationPoints(pointInput, pointOutput);

                            currentCalibrationPoint = getRandomOutputPoint();
                        }
                    }
                }
            }
            else if (e.KeyCode == Keys.R)
            {
                projection.reset();
                currentCalibrationPoint = getRandomOutputPoint();
            }
        }

        /// <summary>
        /// Returns a random point in the glasses image.
        /// </summary>
        /// <returns></returns>
        private Vector2D getRandomOutputPoint()
        {
            return outputSize.getAbsolutePoint(new Vector2D((float)rand.NextDouble(), (float)rand.NextDouble()));
        }
    }
}