bbiwarg-original/bbiwarg/Graphics/OutputWindow.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using OpenTK;
using OpenTK.Graphics.OpenGL;
using bbiwarg.Images;
using bbiwarg.Detectors.Touch;
using bbiwarg.Utility;

using Emgu.CV;

namespace bbiwarg.Graphics
{
    class OutputWindow : GameWindow
    {
        private VideoHandle videoHandle;
        private uint depthTextureID;
        private uint edgeTextureID;
        private uint kalmanDemoTextureID;
        private bool paused = false;
        private long timeSpacePressed, timeLeftPressed, timeRightPressed;
        private Stopwatch watch;
        private KalmanDemo kalmanDemo;

        public OutputWindow(VideoHandle videoHandle)
            : base(4 * videoHandle.Width, 2 * videoHandle.Height)
        {
            this.videoHandle = videoHandle;
            watch = new Stopwatch();
            watch.Start();
            kalmanDemo = new KalmanDemo();
        }

        protected override void OnLoad(EventArgs e)
        {
            base.OnLoad(e);
            Title = "BBIWARG - Output";
            GL.ClearColor(Color.Black);

            // transparency
            GL.Enable(EnableCap.Blend);
            GL.BlendEquation(BlendEquationMode.Max);

            //Depth Test 
            GL.Enable(EnableCap.DepthTest);

            // Textures
            GL.Enable(EnableCap.Texture2D);

            //depthTexture
            GL.GenTextures(1, out depthTextureID);
            GL.BindTexture(TextureTarget.Texture2D, depthTextureID);
            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

            //edgetexture
            GL.GenTextures(1, out edgeTextureID);
            GL.BindTexture(TextureTarget.Texture2D, edgeTextureID);
            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

            //kalman demo texture
            GL.GenTextures(1, out kalmanDemoTextureID);
            GL.BindTexture(TextureTarget.Texture2D, kalmanDemoTextureID);
            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

        }

        protected override void OnResize(EventArgs e)
        {
            base.OnResize(e);
            GL.Viewport(ClientRectangle.X, ClientRectangle.Y, ClientRectangle.Width, ClientRectangle.Height);
            Matrix4 projection = Matrix4.CreatePerspectiveFieldOfView((float)Math.PI / 3, Width / (float)Height, 0.01f, 3.0f);
            GL.MatrixMode(MatrixMode.Projection);
            GL.LoadMatrix(ref projection);
        }

        protected override void OnRenderFrame(FrameEventArgs e)
        {
            Timer.start("onRenderFrame");

            base.OnRenderFrame(e);
            GL.Clear(ClearBufferMask.ColorBufferBit | ClearBufferMask.DepthBufferBit);
            Matrix4 modelview = Matrix4.LookAt(Vector3.Zero, -Vector3.UnitZ, Vector3.UnitY);
            GL.MatrixMode(MatrixMode.Modelview);
            GL.LoadMatrix(ref modelview);

            bool changedFrame = false;

            const int autoRepeatDelay = 100; // ms
            if (videoHandle.sourceIsMovie())
            {
                // pause and unpause with space
                long elapsed = watch.ElapsedMilliseconds;
                if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Space) && (elapsed - timeSpacePressed) >= autoRepeatDelay)
                {
                    timeSpacePressed = elapsed;
                    if (paused)
                        videoHandle.unpauseMovie();
                    else
                        videoHandle.pauseMovie();
                    paused = !paused;
                }

                // when paused go to next / previous frame with right / left keys
                if (paused)
                {
                    if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Right) && (elapsed - timeRightPressed) >= autoRepeatDelay)
                    {
                        timeRightPressed = elapsed;
                        videoHandle.unpauseMovie();
                        videoHandle.nextFrame();
                        videoHandle.pauseMovie();
                        changedFrame = true;
                    }
                    else if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Left) && (elapsed - timeLeftPressed) >= autoRepeatDelay)
                    {
                        timeLeftPressed = elapsed;
                        videoHandle.unpauseMovie();
                        videoHandle.reversePlay();
                        videoHandle.nextFrame();
                        videoHandle.reversePlay();
                        videoHandle.pauseMovie();
                        changedFrame = true;
                    }
                }
                else
                {
                    videoHandle.nextFrame();
                    changedFrame = true;
                }
            }
            else
            {
                videoHandle.nextFrame();
            }

            if (changedFrame || !videoHandle.sourceIsMovie())
            {
                /*foreach (PalmTouchEvent ev in videoHandle.getTouchEvents())
                {
                    Console.WriteLine("touch at " + ev.Position + " -> " + ev.RelativePalmPosition);
                    //touchVisualizer.addTouchEvent(ev);
                }*/
            }

            // kalman demo
            int mouseX = Mouse.X - (2 * Width) / 3 - 30;
            int mouseY = Mouse.Y - 85;
            if (mouseX >= 0 && mouseX <= videoHandle.Width && mouseY >= 0 && mouseY <= Height)
                kalmanDemo.addPoint(new Point(mouseX, mouseY));
            // reset with right click
            if (OpenTK.Input.Mouse.GetState().IsButtonDown(OpenTK.Input.MouseButton.Right))
                kalmanDemo.reset();


            Timer.start("outputTextures");
            //draw textures
            Int16[] depthTextureData = new Int16[3 * videoHandle.Width * videoHandle.Height];
            Int16[] edgeTextureData = new Int16[3 * videoHandle.Width * videoHandle.Height];
            int index = 0;

            for (int y = 0; y < videoHandle.Height; ++y)
            {
                for (int x = 0; x < videoHandle.Width; ++x)
                {
                    Int16 red = 0;
                    Int16 green = 0;
                    Int16 blue = 0;

                    //depthTexture
                    float relDepth = videoHandle.getRelativeDepth(x, y);
                    red = green = blue = (Int16)((1.0f - videoHandle.getRelativeDepth(x, y)) * Int16.MaxValue);

                    // palm
                    switch (videoHandle.getPalmImageStateAt(x, y))
                    {
                        case PalmImageState.palmContour:
                            red = Int16.MaxValue;
                            blue = green = 0;
                            break;
                        case PalmImageState.palmRect:
                            blue = Int16.MaxValue;
                            red = green = 0;
                            break;
                        case PalmImageState.thumbLine:
                        case PalmImageState.wristLine:
                            green = Int16.MaxValue;
                            blue = red = 0;
                            break;
                        case PalmImageState.palmGrid:
                            green = (Int16)(green / 2 + Int16.MaxValue / 2);
                            blue = (Int16)(blue / 2 + Int16.MaxValue / 2);
                            red = (Int16)(red / 2 + Int16.MaxValue / 2);
                            break;


                    }

                    // touch
                    switch (videoHandle.getTouchImageStateAt(x, y))
                    {
                        case TouchImageState.touchArea:
                            red = (Int16)(red / 2);
                            green = (Int16)(green / 2);
                            blue = (Int16)(blue / 2);
                            break;
                        case TouchImageState.touchAreaMatched:
                            red = (Int16)(red / 2);
                            green = (Int16)(green / 2);
                            blue = Int16.MaxValue;
                            break;
                        case TouchImageState.touchAreaStatusBar:
                            red = (Int16)(red / 2);
                            green = Int16.MaxValue;
                            blue = (Int16)(blue / 2);
                            break;
                        case TouchImageState.touchDetected:
                            red = (Int16)(red / 2);
                            blue = (Int16)(blue / 2);
                            green = (Int16)(green / 2 + Int16.MaxValue / 2);
                            break;
                        case TouchImageState.touchTracked:
                            red = (Int16)(red / 2 + Int16.MaxValue / 2);
                            green = (Int16)(green / 2);
                            blue = (Int16)(blue / 2);
                            break;
                    }

                    depthTextureData[index] = red;
                    depthTextureData[index + 1] = green;
                    depthTextureData[index + 2] = blue;

                    //edgeTexture
                    FingerImageState fis = videoHandle.getFingerImageStateAt(x, y);
                    red = green = blue = 0;
                    if (videoHandle.isEdgeAt(x, y)) blue = Int16.MaxValue;
                    else if (fis == FingerImageState.fingerTracked) green = Int16.MaxValue;
                    else if (fis == FingerImageState.fingerDetected) red = Int16.MaxValue;
                    else if (fis == FingerImageState.fingerSlice) red = blue = Int16.MaxValue;
                    else if (fis == FingerImageState.possibleFingerSlice) red = Int16.MaxValue;

                    edgeTextureData[index] = red;
                    edgeTextureData[index + 1] = green;
                    edgeTextureData[index + 2] = blue;

                    index += 3;
                }
            }

            float size = 0.5f;
            float size_2 = size / 2.0f;
            float size_1_5 = 1.5f * size;
            float gap = 0.0f;

            GL.Enable(EnableCap.Texture2D);
            GL.BindTexture(TextureTarget.Texture2D, edgeTextureID);
            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.Width, videoHandle.Height, 0, PixelFormat.Rgb, PixelType.Short, edgeTextureData);
            GL.Begin(PrimitiveType.Quads);
            GL.Color3(1.0, 1.0, 1.0);
            GL.TexCoord2(0.0, 0.0); GL.Vertex3(-size_1_5 - gap, size_2, -0.5);
            GL.TexCoord2(1.0, 0.0); GL.Vertex3(-size_2 - gap, size_2, -0.5);
            GL.TexCoord2(1.0, 1.0); GL.Vertex3(-size_2 - gap, -size_2, -0.5);
            GL.TexCoord2(0.0, 1.0); GL.Vertex3(-size_1_5 - gap, -size_2, -0.5);
            GL.End();

            GL.Enable(EnableCap.Texture2D);
            GL.BindTexture(TextureTarget.Texture2D, depthTextureID);
            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.Width, videoHandle.Height, 0, PixelFormat.Rgb, PixelType.Short, depthTextureData);
            GL.Begin(PrimitiveType.Quads);
            GL.Color3(1.0, 1.0, 1.0);
            GL.TexCoord2(0.0, 0.0); GL.Vertex3(-size_2, size_2, -0.5);
            GL.TexCoord2(1.0, 0.0); GL.Vertex3(size_2, size_2, -0.5);
            GL.TexCoord2(1.0, 1.0); GL.Vertex3(size_2, -size_2, -0.5);
            GL.TexCoord2(0.0, 1.0); GL.Vertex3(-size_2, -size_2, -0.5);
            GL.End();

            GL.Enable(EnableCap.Texture2D);
            GL.BindTexture(TextureTarget.Texture2D, kalmanDemoTextureID);
            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.Width, videoHandle.Height, 0, PixelFormat.Rgb, PixelType.Short,
                kalmanDemo.getTextureData(videoHandle.Width, videoHandle.Height));
            GL.Begin(PrimitiveType.Quads);
            GL.Color3(1.0, 1.0, 1.0);
            GL.TexCoord2(0.0, 0.0); GL.Vertex3(size_2 + gap, size_2, -0.5);
            GL.TexCoord2(1.0, 0.0); GL.Vertex3(size_1_5 + gap, size_2, -0.5);
            GL.TexCoord2(1.0, 1.0); GL.Vertex3(size_1_5 + gap, -size_2, -0.5);
            GL.TexCoord2(0.0, 1.0); GL.Vertex3(size_2 + gap, -size_2, -0.5);
            GL.End();

            SwapBuffers();
            Timer.stop("outputTextures");

            Timer.stop("onRenderFrame");

            Timer.outputAll();
        }

    }
}