-rewritten videoHandle+Inputprovider (videohandle->inputHandler)

-split up output and data/recognition

bbiwarg/Constants.cs

@@ -12,23 +12,24 @@ namespace bbiwarg
     class Constants
     {
         // Logger
-        public static readonly LogSubject LogLevel = LogSubject.TUIOServer;
+        public static readonly LogSubject LogLevel = LogSubject.Timer;
             //LogSubject.Timer;
         
         // Output
+        public static readonly bool OutputEnabled = true;
+        public static readonly int OutputFrameRate = 30;
         public static readonly int OutputNumImages = 5;
         public static readonly int OutputNumImagesPerRow = 3;
         public static readonly float OutputScaleFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
-        
+        public static readonly String OutputTitle = "BBIWARG - Output";
+
         // TUIO
-        public static readonly String TuioIP = "172.20.10.6";
+        public static readonly bool TuioEnabled = true;
+        public static readonly String TuioIP = "127.0.0.1";
         public static readonly int TuioPort = 3333;
 
         // image
-        public static readonly int ImageWidth = 320;
-        public static readonly int ImageHeight = 240;
-        public static readonly Vector2D ImageMaxPixel = new Vector2D(ImageWidth - 1, ImageHeight - 1);
-        public static readonly float ImageDiagonalLength = ImageMaxPixel.Length;
+        public static readonly float ImageDiagonalLength = new Vector2D(319,239).Length;
 
         // confidence image
         public static readonly int ConfidenceImageMinThreshold = 500;
@@ -87,8 +88,8 @@ namespace bbiwarg
         public static readonly float PalmMaxPrecentageQuadForegroundReset = 0.5f;
 
         //palm Grid
-        public static readonly int PalmGridRows = 3;
-        public static readonly int PalmGridColumns = 3;
+        public static readonly int PalmGridNumRows = 3;
+        public static readonly int PalmGridNumColumns = 3;
 
         // touch detection
         public static readonly float TouchEventMinTouchValue = 0.3f;
@@ -111,19 +112,19 @@ namespace bbiwarg
         public static readonly Color ColorDetected = Color.Turquoise;
         public static readonly Color ColorTracked = Color.Yellow;
 
-        //public static readonly Color EdgeColor = Color.Blue; // edgeImage draw direct to blue chanel from outputImage
+        public static readonly Color DepthImageColor = Color.White;
+        public static readonly Color EdgeImageColor = Color.Blue;
+        public static readonly Color OutputImageBorderColor = Color.White;
 
         public static readonly Color FingerSliceColor = Color.Magenta;
         public static readonly Color FingerDetectedColor = ColorDetected;
         public static readonly Color FingerTrackedColor = ColorTracked;
-        public static readonly Color FingerTipOutSliceColor = Color.Gray;
-        public static readonly Color FingerHandOutSliceColor = Color.DarkSlateGray;
+        public static readonly Color FingerTipColor = Color.Blue;
         public static readonly Color FingerContourColor = Color.Red;
         public static readonly Color FingerIDColor = Color.White;
 
         public static readonly Color TouchEventDetectedColor = ColorDetected;
         public static readonly Color TouchEventTrackedColor = ColorTracked;
-        public static readonly Color TouchEventTipColor = Color.CornflowerBlue;
         public static readonly Color TouchEventAreaMatchedSubtractColor = Color.DarkOrange;
         public static readonly Color TouchEventAreaNonMatchedSubtractColor = Color.DarkSlateGray;
         public static readonly Color TouchEventStatusBarColor = Color.Green;
@@ -137,5 +138,8 @@ namespace bbiwarg
         public static readonly Color PalmConturColor = Color.Red;
         public static readonly Color PalmConvexHullColor = Color.Green;
         public static readonly Color PalmThumbDefectColor = Color.Lime;
+
+        public static readonly Color HandRightColor = Color.Red;
+        public static readonly Color HandLeftColor = Color.Blue;
     }
 }

bbiwarg/Graphics/OutputImage.cs

@@ -3,8 +3,8 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-
 using System.Drawing;
+using bbiwarg.Utility;
 
 using Emgu.CV;
 using Emgu.CV.Structure;
@@ -31,7 +31,7 @@ namespace bbiwarg.Graphics
 
         public void drawLineSegment(bbiwarg.Utility.LineSegment2D lineSegment, Color color, int thickness = 1)
         {
-            Image.Draw(new LineSegment2D(lineSegment.P1, lineSegment.P2), new Rgb(color), thickness);
+            Image.Draw(new Emgu.CV.Structure.LineSegment2D(lineSegment.P1, lineSegment.P2), new Rgb(color), thickness);
         }
 
         public void drawContour(Contour<Point> contour, Color color, int thickness = 1)
@@ -65,5 +65,40 @@ namespace bbiwarg.Graphics
             MCvFont font = new MCvFont(Emgu.CV.CvEnum.FONT.CV_FONT_HERSHEY_PLAIN, 1, 1);
             Image.Draw(text, ref font, new Point(x, y), new Rgb(color));
         }
+
+        public void drawImage(Image<Gray, byte> image, Color color)
+        {
+            Image[0] = Image[0].Or(image.Mul((float)color.R / byte.MaxValue));
+            Image[1] = Image[1].Or(image.Mul((float)color.G / byte.MaxValue));
+            Image[2] = Image[2].Or(image.Mul((float)color.B / byte.MaxValue));
+        }
+
+        public void drawQuadrangleGrid(Quadrangle quad, Color borderColor, Color gridColor, int numRows, int numCols) {
+
+            Vector2D a = quad.BottomLeft;
+            Vector2D b = quad.TopLeft;
+            Vector2D c = quad.TopRight;
+            Vector2D d = quad.BottomRight;
+
+            Vector2D relAB = (b - a) / numRows;
+            Vector2D relDC = (c - d) / numRows;
+            Vector2D relBC = (c - b) / numCols;
+            Vector2D relAD = (d - a) / numCols;
+
+            for (int i = 1; i < numRows; i++)
+            {
+                drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), gridColor);
+            }
+
+            for (int i = 1; i < numCols; i++)
+            {
+                drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), gridColor);
+            }
+
+            drawLineSegment(new bbiwarg.Utility.LineSegment2D(a, b), borderColor);
+            drawLineSegment(new bbiwarg.Utility.LineSegment2D(b, c), borderColor);
+            drawLineSegment(new bbiwarg.Utility.LineSegment2D(c, d), borderColor);
+            drawLineSegment(new bbiwarg.Utility.LineSegment2D(d, a), borderColor);
+        }
     }
 }

bbiwarg/Graphics/OutputWindow.cs

@@ -1,76 +1,75 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Drawing;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using OpenTK;
+using OpenTK.Input;
 using OpenTK.Graphics.OpenGL;
-using bbiwarg.Images;
-using bbiwarg.Recognition.TouchRecognition;
+using bbiwarg.InputProviders;
 using bbiwarg.Utility;
 
-using Emgu.CV;
-using Emgu.CV.Structure;
-
 namespace bbiwarg.Graphics
 {
     class OutputWindow : GameWindow
     {
-        private VideoHandle videoHandle;
-        private uint textureId;
-        private bool paused = false;
-        private long timeSpacePressed, timeLeftPressed, timeRightPressed;
-        private Stopwatch watch;
-
-        public OutputWindow(VideoHandle videoHandle)
-            : base((int) (Constants.OutputScaleFactor * Math.Max(1, Math.Min(Constants.OutputNumImagesPerRow, videoHandle.OutputImages.Length)) * videoHandle.Width), 
-                   (int) (Constants.OutputScaleFactor * (1 + (videoHandle.OutputImages.Length - 1) / Constants.OutputNumImagesPerRow) * videoHandle.Height))
+        private InputHandler inputHandler;
+        private InputProvider inputProvider;
+
+        private uint textureID;
+
+        public OutputWindow(InputProvider inputProvider, InputHandler inputHandler)
+            : base((int)(Constants.OutputScaleFactor * Math.Max(1, Math.Min(Constants.OutputNumImagesPerRow, Constants.OutputNumImages)) * inputProvider.ImageWidth),
+                   (int)(Constants.OutputScaleFactor * (1 + (Constants.OutputNumImages - 1) / Constants.OutputNumImagesPerRow) * inputProvider.ImageHeight))
         {
-            this.videoHandle = videoHandle;
-            watch = new Stopwatch();
-            watch.Start();
+            this.inputProvider = inputProvider;
+            this.inputHandler = inputHandler;
         }
 
         protected override void OnLoad(EventArgs e)
         {
             base.OnLoad(e);
-            Title = "BBIWARG - Output";
+            Title = Constants.OutputTitle;
             GL.ClearColor(Color.Black);
 
             // transparency
             GL.Enable(EnableCap.Blend);
             GL.BlendEquation(BlendEquationMode.Max);
 
-            //Depth Test 
+            // Depth Test 
             GL.Enable(EnableCap.DepthTest);
 
             // Texture
             GL.Enable(EnableCap.Texture2D);
-
-            GL.GenTextures(1, out textureId);
-            GL.BindTexture(TextureTarget.Texture2D, textureId);
+            GL.GenTextures(1, out textureID);
+            GL.BindTexture(TextureTarget.Texture2D, textureID);
             GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
             GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
+
+            // video controls
+            if (inputProvider is VideoInputProvider)
+            {
+                Keyboard.KeyDown += HandleKeyDown;
+                Console.WriteLine("video controls enabled (space=pause, right=nextFrame, left=previousFrame)");
+            }
         }
 
         protected override void OnResize(EventArgs e)
         {
             base.OnResize(e);
-            
+
             int screenWidth = ClientRectangle.Width;
             int screenHeight = ClientRectangle.Height;
 
-            int imageWidth = videoHandle.Width;
-            int imageHeight = videoHandle.Height;
+            int imageWidth = inputProvider.ImageWidth;
+            int imageHeight = inputProvider.ImageHeight;
             float imageAspectRatio = (float)imageWidth / (float)imageHeight;
 
-            int numImages = videoHandle.OutputImages.Length;
-            int numRows = 1 + (numImages - 1) / Constants.OutputNumImagesPerRow;
-            int numCols = Math.Min(videoHandle.OutputImages.Length, Constants.OutputNumImagesPerRow);
+            int numRows = 1 + (Constants.OutputNumImages - 1) / Constants.OutputNumImagesPerRow;
+            int numCols = Math.Min(Constants.OutputNumImages, Constants.OutputNumImagesPerRow);
             int heightForWidth = (int)((float)screenWidth / ((float)numCols * imageAspectRatio) * (float)numRows);
-            
+
             GL.Viewport(0, (screenHeight - heightForWidth) / 2, screenWidth, heightForWidth);
 
             // top left at (0,0) every image from (i, j) to (i + 1, j + 1)
@@ -82,71 +81,27 @@ namespace bbiwarg.Graphics
         protected override void OnRenderFrame(FrameEventArgs e)
         {
             Timer.start("onRenderFrame");
-   
+
             base.OnRenderFrame(e);
             GL.Clear(ClearBufferMask.ColorBufferBit | ClearBufferMask.DepthBufferBit);
             Matrix4 modelview = Matrix4.LookAt(Vector3.Zero, -Vector3.UnitZ, Vector3.UnitY);
             GL.MatrixMode(MatrixMode.Modelview);
             GL.LoadMatrix(ref modelview);
 
-            const int autoRepeatDelay = 100; // ms
-            if (videoHandle.sourceIsMovie())
-            {
-                // pause and unpause with space
-                long elapsed = watch.ElapsedMilliseconds;
-                if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Space) && (elapsed - timeSpacePressed) >= autoRepeatDelay)
-                {
-                    timeSpacePressed = elapsed;
-                    if (paused)
-                        videoHandle.unpauseMovie();
-                    else
-                        videoHandle.pauseMovie();
-                    paused = !paused;
-                }
-
-                // when paused go to next / previous frame with right / left keys
-                if (paused)
-                {
-                    if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Right) && (elapsed - timeRightPressed) >= autoRepeatDelay)
-                    {
-                        timeRightPressed = elapsed;
-                        videoHandle.unpauseMovie();
-                        videoHandle.nextFrame();
-                        videoHandle.pauseMovie();
-                    }
-                    else if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Left) && (elapsed - timeLeftPressed) >= autoRepeatDelay)
-                    {
-                        timeLeftPressed = elapsed;
-                        videoHandle.unpauseMovie();
-                        videoHandle.reversePlay();
-                        videoHandle.nextFrame();
-                        videoHandle.reversePlay();
-                        videoHandle.pauseMovie();
-                    }
-                }
-                else
-                {
-                    videoHandle.nextFrame();
-                }
-            }
-            else
-            {
-                videoHandle.nextFrame();
-            }
+            inputHandler.updateFrame();
 
-            if (videoHandle.sourceIsMovie())
-                Title = "BBIWARG - Output (Frame " + videoHandle.CurrentFrame + ")";
+            Title = Constants.OutputTitle + " (Frame: " + inputProvider.CurrentFrame + ")";
 
             Timer.start("buildTextures");
             GL.Enable(EnableCap.Texture2D);
-                
+
             int imageIndex = 0;
-            foreach (OutputImage image in videoHandle.OutputImages)
+            foreach (OutputImage image in inputHandler.OutputImages)
             {
                 int x = imageIndex % Constants.OutputNumImagesPerRow;
                 int y = imageIndex / Constants.OutputNumImagesPerRow;
-                GL.BindTexture(TextureTarget.Texture2D, textureId);
-                GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.Width, videoHandle.Height, 0, PixelFormat.Rgb, PixelType.UnsignedByte, image.Image.MIplImage.imageData);
+                GL.BindTexture(TextureTarget.Texture2D, textureID);
+                GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, inputProvider.ImageWidth, inputProvider.ImageHeight, 0, PixelFormat.Rgb, PixelType.UnsignedByte, image.Image.MIplImage.imageData);
                 GL.Begin(PrimitiveType.Quads);
                 GL.Color3(1.0, 1.0, 1.0);
                 GL.TexCoord2(0, 0); GL.Vertex3(0 + x, 0 + y, -1);
@@ -163,10 +118,32 @@ namespace bbiwarg.Graphics
             SwapBuffers();
             Timer.stop("swapBuffers");
 
+            Timer.stop("buildTextures");
             Timer.stop("onRenderFrame");
-            
+
             Timer.outputAll();
         }
 
+        private void HandleKeyDown(object sender, KeyboardKeyEventArgs e)
+        {
+            VideoInputProvider vip = (VideoInputProvider)inputProvider;
+            switch (e.Key)
+            {
+                case Key.Space:
+                    if (vip.IsPaused)
+                        vip.play();
+                    else
+                        vip.pause();
+                    break;
+                case Key.Right:
+                    if (vip.IsPaused)
+                        vip.nextFrame();
+                    break;
+                case Key.Left:
+                    if (vip.IsPaused)
+                        vip.previousFrame();
+                    break;
+            }
+        }
     }
 }

bbiwarg/Graphics/TouchEventVisualizer.cs

@@ -54,8 +54,8 @@ namespace bbiwarg.Graphics
             OutputImage.drawRectangle(0, 0, width - 1, height - 1, Constants.TouchEventVisualizerGridColor);
 
             //draw grid
-            int numRows = Constants.PalmGridRows;
-            int numColumns = Constants.PalmGridColumns;
+            int numRows = Constants.PalmGridNumRows;
+            int numColumns = Constants.PalmGridNumColumns;
             int widthPerColumn = width / numColumns;
             int heightPerRow = height / numRows;
 
@@ -69,7 +69,7 @@ namespace bbiwarg.Graphics
                 OutputImage.drawLineSegment(new LineSegment2D(new Vector2D(0,i*heightPerRow), new Vector2D(width-1,i*heightPerRow)), Constants.TouchEventVisualizerGridColor);
             }
 
-            Vector2D maxPixel = Constants.ImageMaxPixel;
+            Vector2D maxPixel = new Vector2D(width - 1, height - 1);
             foreach (int id in positions.Keys) {
                 int numPositions = positions[id].Count;
                 for (int i = 1; i < numPositions; i++) {

bbiwarg/Images/EdgeImage.cs

@@ -18,9 +18,13 @@ namespace bbiwarg.Images
     {
         public Image<Gray, Byte> Image { get; private set; }
         public Image<Gray, byte> RoughImage { get; private set; }
+        public int Width { get; private set; }
+        public int Height { get; private set; }
 
         public EdgeImage(DepthImage depthImage)
         {
+            Width = depthImage.Width;
+            Height = depthImage.Height;
             Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(Constants.EdgeImageCannyStartThreshold, Constants.EdgeImageCannyLinkingThreshold, Constants.EdgeImageCannySize).ThresholdBinary(new Gray(0), new Gray(1));
             RoughImage = Image.Dilate(Constants.EdgeImageRoughNumDilationIterations);
         }

bbiwarg/InputHandler.cs

@@ -0,0 +1,286 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bbiwarg.InputProviders;
+using bbiwarg.Images;
+using bbiwarg.Recognition.FingerRecognition;
+using bbiwarg.Recognition.HandRecognition;
+using bbiwarg.Recognition.PalmRecognition;
+using bbiwarg.Recognition.TouchRecognition;
+using bbiwarg.Server;
+using bbiwarg.Graphics;
+using bbiwarg.Utility;
+using Emgu.CV;
+using Emgu.CV.Structure;
+
+namespace bbiwarg
+{
+    class InputHandler
+    {
+        public OutputImage[] OutputImages { get; private set; }
+
+        private InputProvider inputProvider;
+        private InputFrame currentInputFrame;
+        private int imageWidth;
+        private int imageHeight;
+        private int currentFrame;
+        private int lastFrame;
+
+        private DepthImage depthImage;
+        private EdgeImage edgeImage;
+        private ConfidenceImage confidenceImage;
+
+        private FingerDetector fingerDetector;
+        private HandDetector handDetector;
+        private PalmDetector palmDetector;
+        private TouchDetector touchDetector;
+
+        private FingerTracker fingerTracker;
+        private TouchTracker touchTracker;
+
+        private TouchEventVisualizer touchEventVisualizer;
+        private TuioCommunicator tuioCommunicator;
+
+        public InputHandler(InputProvider inputProvider)
+        {
+            this.inputProvider = inputProvider;
+            this.imageWidth = inputProvider.ImageWidth;
+            this.imageHeight = inputProvider.ImageHeight;
+            this.currentFrame = inputProvider.CurrentFrame;
+            this.lastFrame = this.currentFrame - 1;
+
+            initializeConsistentObjects();
+        }
+
+        private void initializeConsistentObjects()
+        {
+            palmDetector = new PalmDetector();
+
+            fingerTracker = new FingerTracker();
+            touchTracker = new TouchTracker();
+
+            if (Constants.OutputEnabled)
+            {
+                touchEventVisualizer = new TouchEventVisualizer(imageWidth, imageHeight);
+                //register touchEventVisualizer to touchTracker
+                touchTracker.PalmTouchDown += touchEventVisualizer.touchDown;
+                touchTracker.PalmTouchMove += touchEventVisualizer.touchMove;
+                touchTracker.PalmTouchUp += touchEventVisualizer.touchUp;
+            }
+
+            if (Constants.TuioEnabled)
+            {
+                tuioCommunicator = new TuioCommunicator(Constants.TuioIP, Constants.TuioPort);
+
+                //register tuiCommunicator to touchTracker
+                touchTracker.PalmTouchDown += tuioCommunicator.touchDown;
+                touchTracker.PalmTouchMove += tuioCommunicator.touchMove;
+                touchTracker.PalmTouchUp += tuioCommunicator.touchUp;
+            }
+        }
+
+        private void resetConsistentObjects()
+        {
+            palmDetector.reset();
+
+            touchTracker.reset();
+            fingerTracker.reset();
+
+            if (Constants.OutputEnabled)
+                touchEventVisualizer.reset();
+
+            if (Constants.TuioEnabled)
+                tuioCommunicator.reset();
+        }
+
+        public void updateFrame()
+        {
+            Timer.start("updateFrame");
+
+            beforeUpdateFrame();
+
+            if (lastFrame-currentFrame > 20)
+                resetConsistentObjects();
+
+            createConfidenceImage();
+            createDepthImage();
+            createEdgeImage();
+
+            detectFingers();
+            trackFingers();
+
+            detectHands();
+            detectPalm();
+
+            detectTouchEvents();
+            trackTouchEvents();
+
+            afterUpdateFrame();
+
+            Timer.stop("updateFrame");
+        }
+
+        private void beforeUpdateFrame()
+        {
+            currentInputFrame = inputProvider.getInputFrame();
+
+            lastFrame = currentFrame;
+            currentFrame = currentInputFrame.FrameNumber;
+            Logger.CurrentFrame = currentFrame;
+
+            if (Constants.TuioEnabled)
+                tuioCommunicator.initFrame();
+        }
+
+        private void afterUpdateFrame()
+        {
+            if (Constants.TuioEnabled)
+                tuioCommunicator.commitFrame();
+
+            if (Constants.OutputEnabled)
+                createOutputImages();
+        }
+
+        private void createConfidenceImage()
+        {
+            Timer.start("createConfidenceImage");
+            Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawConfidenceData);
+            confidenceImage = new ConfidenceImage(rawConfidenceImage);
+            Timer.stop("createConfidenceImage");
+        }
+
+        private void createDepthImage()
+        {
+            Timer.start("createDepthImage");
+            Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(imageWidth, imageHeight, imageWidth * 2, currentInputFrame.RawDepthData);
+            rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
+            depthImage = new DepthImage(rawDepthImage);
+            Timer.stop("createDepthImage");
+        }
+
+        private void createEdgeImage()
+        {
+            Timer.start("createEdgeImage");
+            edgeImage = new EdgeImage(depthImage);
+            Timer.stop("createEdgeImage");
+        }
+
+        private void detectFingers()
+        {
+            Timer.start("detectFingers");
+            fingerDetector = new FingerDetector(depthImage, edgeImage);
+            Timer.stop("detectFingers");
+        }
+
+        private void trackFingers()
+        {
+            Timer.start("trackFingers");
+            fingerTracker.updateFrame(fingerDetector.Fingers);
+            Timer.stop("trackFingers");
+        }
+
+        private void detectHands()
+        {
+            Timer.start("detectHands");
+            handDetector = new HandDetector(edgeImage, fingerDetector.Fingers);
+            Timer.stop("detectHands");
+        }
+
+        private void detectPalm()
+        {
+            Timer.start("detectPalm");
+            palmDetector.findPalmQuad(handDetector.Hands, handDetector.HandMask);
+            Timer.stop("detectPalm");
+        }
+
+        private void detectTouchEvents()
+        {
+            Timer.start("detectTouchEvents");
+            touchDetector = new TouchDetector(depthImage, fingerTracker.Fingers, palmDetector.PalmQuad);
+            Timer.stop("detectTouchEvents");
+        }
+
+        private void trackTouchEvents()
+        {
+            Timer.start("trackTouchEvents");
+            touchTracker.updateFrame(touchDetector.TouchEvents);
+            Timer.stop("trackTouchEvents");
+        }
+
+        private void createOutputImages()
+        {
+            Timer.start("createOutputImages");
+            int numImages = Constants.OutputNumImages;
+            OutputImages = new OutputImage[numImages];
+            for (int i = 0; i < numImages; i++)
+            {
+                OutputImages[i] = new OutputImage(imageWidth, imageHeight);
+            }
+
+            //image0
+            OutputImages[0].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image, Constants.DepthImageColor);
+            foreach (Finger f in fingerTracker.Fingers)
+            {
+                OutputImages[0].fillCircle(f.TipPoint.IntX, f.TipPoint.IntY, 3, Constants.FingerTipColor);
+                if (f.TouchEvent != null)
+                    OutputImages[0].fillCircle(f.TouchEvent.Position.IntX, f.TouchEvent.Position.IntY, 5, Constants.TouchEventDetectedColor);
+            }
+
+            //image1
+            OutputImages[1].drawImage(edgeImage.Image.Mul(255), Constants.EdgeImageColor);
+            foreach (Finger f in fingerTracker.Fingers)
+            {
+                for (int i = 0; i < f.SliceTrail.NumSlices; i++)
+                    OutputImages[1].drawLineSegment(f.SliceTrail[i].LineSegment, Constants.FingerSliceColor);
+                OutputImages[1].drawContour(f.Contour, Constants.FingerContourColor);
+                OutputImages[1].drawLineSegment(f.LineSegment, Constants.FingerTrackedColor);
+                OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
+            }
+
+            //image2
+            foreach (Hand h in handDetector.Hands)
+            {
+                if (h.Side == Hand.HandSide.Right)
+                    OutputImages[2].drawImage(h.Mask.Mul(255), Constants.HandRightColor);
+                else
+                    OutputImages[2].drawImage(h.Mask.Mul(255), Constants.HandLeftColor);
+            }
+
+            //image3
+            OutputImages[3].drawImage((depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255 - handDetector.HandMask.Mul(255)), Constants.DepthImageColor);
+            foreach (TouchEvent te in touchTracker.TouchEvents)
+                OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
+            if (palmDetector.PalmContour != null && palmDetector.PalmContour.Count<Point>() > 0)
+            {
+                OutputImages[3].drawContour(palmDetector.PalmContour, Constants.PalmConturColor);
+                OutputImages[3].drawPoints(palmDetector.PalmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
+            }
+            if (palmDetector.PalmQuad != null) {
+                OutputImages[3].fillCircle(palmDetector.ThumbDefectStart.IntX, palmDetector.ThumbDefectStart.IntY, 3, Color.Red);
+                OutputImages[3].fillCircle(palmDetector.ThumbDefectEnd.IntX, palmDetector.ThumbDefectEnd.IntY, 3, Color.Red);
+                OutputImages[3].fillCircle(palmDetector.ThumbDefectDepth.IntX, palmDetector.ThumbDefectDepth.IntY, 3, Color.Red);
+
+                OutputImages[3].drawLineSegment(new Utility.LineSegment2D(palmDetector.ThumbDefectDepth, (palmDetector.ThumbDefectStart + palmDetector.ThumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
+
+                OutputImages[3].drawQuadrangleGrid(palmDetector.PalmQuad, Constants.PalmQuadColor, Constants.PalmGridColor, Constants.PalmGridNumRows, Constants.PalmGridNumColumns);
+            }
+
+            //image4
+            touchEventVisualizer.updateImage();
+            OutputImages[4] = touchEventVisualizer.OutputImage;
+
+
+            //borders
+            for (int i = 0; i < numImages; i++)
+            {
+                OutputImages[i].drawRectangle(0, 0, imageWidth - 1, imageHeight - 1, Constants.OutputImageBorderColor);
+            }
+
+            Timer.stop("createOutputImages");
+        }
+
+    }
+}

bbiwarg/InputProvider/IInputProvider.cs

@@ -1,31 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.InputProviders
-{
-    interface IInputProvider
-    {
-        void init();
-        void start();
-        void stop();
-        void updateFrame();
-        void releaseFrame();
-
-        float getFrameRate();
-        int getCurrentMovieFrame();
-        float getHFOV();
-        float getVFOV();
-        bool isActive();
-
-        InputFrame getInputFrame();
-
-        bool sourceIsMovie();
-        void pauseMovie();
-        void unpauseMovie();
-        void reversePlay();
-    }
-}

bbiwarg/InputProvider/IisuInputProvider.cs

@@ -1,167 +0,0 @@
-using System;
-using System.Drawing;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-using Iisu;
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.InputProviders
-{
-    class IisuInputProvider : IInputProvider
-    {
-        private IHandle handle;
-        private IDevice device;
-        private string moviePath;
-        private bool active;
-        private bool sourceIsMovie;
-
-        //parameters
-        private IParameterHandle<float> frameRate;
-        private IParameterHandle<int> currentMovieFrame;
-        private IParameterHandle<float> hfov;
-        private IParameterHandle<float> vfov;
-        private IParameterHandle<int> playStep;
-
-        //data
-        private IDataHandle<Iisu.Data.IImageData> depthImage;
-        private IDataHandle<Iisu.Data.IImageData> confidenceImage;
-
-        public IisuInputProvider(String moviePath = "")
-        {
-            if (moviePath.Length != 0)
-            {
-                this.moviePath = moviePath;
-                sourceIsMovie = true;
-            }
-            else
-            {
-                sourceIsMovie = false;
-            }
-            active = false;
-        }
-
-        // control-methodes
-
-        public void init()
-        {
-            handle = Iisu.Iisu.Context.CreateHandle();
-
-            IDeviceConfiguration conf = handle.CreateDeviceConfiguration();
-            if (sourceIsMovie)
-                conf.MoviePath = moviePath;
-
-            device = handle.InitializeDevice(conf);
-
-            // parameters
-            if (sourceIsMovie)
-            {
-                //device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayMode").Value = 0; // playMode = once
-                device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayMode").Value = 1; // playMode = loop
-                //device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayMode").Value = 2; // playMode = ping-pong
-                playStep = device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayStep");
-                currentMovieFrame = device.RegisterParameterHandle<int>("SOURCE.MOVIE.CurrentFrame");
-            }
-
-            frameRate = device.RegisterParameterHandle<float>("SOURCE.FrameRate");
-            hfov = device.RegisterParameterHandle<float>("SOURCE.CAMERA.DEPTH.HFOV");
-            vfov = device.RegisterParameterHandle<float>("SOURCE.CAMERA.DEPTH.VFOV");
-
-            // events
-            device.EventManager.RegisterEventListener("DEVICE.Status", new Iisu.EventDelegates.Device.Status(onDeviceStatusChanged));
-
-            // data
-            depthImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.DEPTH.Image");
-            confidenceImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.CONFIDENCE.Image");
-        }
-
-        private void onDeviceStatusChanged(string eventName, DeviceStatus status)
-        {
-            active = status.HasFlag(Iisu.DeviceStatus.Playing);
-        }
-
-        public void start()
-        {
-            device.Start();
-        }
-
-        public void stop()
-        {
-            device.Stop(true);
-        }
-
-        public void updateFrame()
-        {
-            device.UpdateFrame(true);
-        }
-
-        public void releaseFrame()
-        {
-            device.ReleaseFrame();
-        }
-
-        // Parameter-methodes
-
-        public float getFrameRate()
-        {
-            return frameRate.Value;
-        }
-
-        public float getHFOV()
-        {
-            return hfov.Value;
-        }
-
-        public float getVFOV()
-        {
-            return vfov.Value;
-        }
-
-        public bool isActive()
-        {
-            return active;
-        }
-
-        // Data-methodes
-        public InputFrame getInputFrame() {
-            //depthData
-            Iisu.Data.IImageInfos imageInfos = depthImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            IntPtr rawDepthData = depthImage.Value.Raw;
-            IntPtr rawConfidenceData = confidenceImage.Value.Raw;
-
-            return new InputFrame(width, height, rawDepthData, rawConfidenceData);
-        }
-
-        // other
-        public void pauseMovie()
-        {
-            if (sourceIsMovie)
-                playStep.Value = 0;
-        }
-
-        public void unpauseMovie()
-        {
-            if (sourceIsMovie)
-                playStep.Value = 1;
-        }
-
-        public int getCurrentMovieFrame()
-        {
-            if (sourceIsMovie)
-                return currentMovieFrame.Value;
-            return 0;
-        }
-
-        bool IInputProvider.sourceIsMovie()
-        {
-            return sourceIsMovie;
-        }
-
-        public void reversePlay()
-        {
-            if (sourceIsMovie)
-                playStep.Value = -playStep.Value;
-        }
-    }
-}

bbiwarg/InputProvider/InputFrame.cs → bbiwarg/InputProviders/InputFrame.cs

@@ -1,26 +1,25 @@
 using System;
 using System.Collections.Generic;
-using System.Drawing;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-using Emgu.CV;
-using Emgu.CV.Structure;
 
 namespace bbiwarg.InputProviders
 {
     class InputFrame
     {
+        public int FrameNumber { get; private set; }
         public int Width { get; private set; }
         public int Height { get; private set; }
         public IntPtr RawDepthData { get; private set; }
         public IntPtr RawConfidenceData { get; private set; }
 
-        public InputFrame(int width, int height, IntPtr rawDepthData, IntPtr rawConfidenceData) {
-            this.Width = width;
-            this.Height = height;
-            this.RawDepthData = rawDepthData;
-            this.RawConfidenceData = rawConfidenceData;
+        public InputFrame(int frameNumber, int width, int height, IntPtr rawDepthData, IntPtr rawConfidenceData) {
+            FrameNumber = frameNumber;
+            Width = width;
+            Height = height;
+            RawDepthData = rawDepthData;
+            RawConfidenceData = rawConfidenceData;
         }
     }
 }

bbiwarg/InputProviders/InputProvider.cs

@@ -0,0 +1,72 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Iisu;
+
+namespace bbiwarg.InputProviders
+{
+    class InputProvider
+    {
+        public int ImageWidth { get; private set; }
+        public int ImageHeight { get; private set; }
+        public virtual int CurrentFrame { get { return frameCounter; }}
+
+        protected int frameCounter = 0;
+
+        protected IHandle handle;
+        protected IDevice device;
+
+        protected IDataHandle<Iisu.Data.IImageData> depthImage;
+        protected IDataHandle<Iisu.Data.IImageData> confidenceImage;
+
+
+        ~InputProvider()
+        {
+            device.Stop(true);
+        }
+
+        public void initialize()
+        {
+            createDevice();
+
+            registerHandles();
+
+            device.Start();
+            device.UpdateFrame(true);
+            ImageWidth = (int)depthImage.Value.ImageInfos.Width;
+            ImageHeight = (int)depthImage.Value.ImageInfos.Height;
+            device.ReleaseFrame();
+        }
+
+        protected void createDevice()
+        {
+            handle = Iisu.Iisu.Context.CreateHandle();
+            IDeviceConfiguration conf = createDeviceConfiguration();
+            device = handle.InitializeDevice(conf);
+        }
+
+        protected virtual IDeviceConfiguration createDeviceConfiguration()
+        {
+            return handle.CreateDeviceConfiguration();
+        }
+
+        protected virtual void registerHandles()
+        {
+            depthImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.DEPTH.Image");
+            confidenceImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.CONFIDENCE.Image");
+        }
+
+        public InputFrame getInputFrame()
+        {
+            device.UpdateFrame(true);
+            frameCounter++;
+            IntPtr rawDepthData = depthImage.Value.Raw;
+            IntPtr rawConfidenceData = confidenceImage.Value.Raw;
+            device.ReleaseFrame();
+
+            return new InputFrame(CurrentFrame, ImageWidth, ImageHeight, rawDepthData, rawConfidenceData);
+        }
+    }
+}

bbiwarg/InputProviders/VideoInputProvider.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Iisu;
+
+namespace bbiwarg.InputProviders
+{
+    class VideoInputProvider : InputProvider
+    {
+        public override int CurrentFrame { get { return currentMovieFrame.Value; } }
+        public String MoviePath { get; private set; }
+        public bool IsPaused { get { return (playStep.Value == 0); } }
+
+        private IParameterHandle<int> currentMovieFrame;
+        private IParameterHandle<int> playStep;
+
+        public VideoInputProvider(String moviePath)
+        {
+            MoviePath = moviePath;
+        }
+
+        protected override IDeviceConfiguration createDeviceConfiguration()
+        {
+            IDeviceConfiguration conf = base.createDeviceConfiguration();
+            conf.MoviePath = MoviePath;
+            return conf;
+        }
+
+        protected override void registerHandles()
+        {
+            base.registerHandles();
+
+            device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayMode").Value = 1; // 0=once, 1=loop, 2=pingPong
+            currentMovieFrame = device.RegisterParameterHandle<int>("SOURCE.MOVIE.CurrentFrame");
+            playStep = device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayStep");
+        }
+
+        public void pause()
+        {
+            playStep.Value = 0;
+            Console.WriteLine("pause");
+        }
+
+        public void play()
+        {
+            playStep.Value = 1;
+            Console.WriteLine("play");
+        }
+
+        public void nextFrame()
+        {
+            playStep.Value = 1;
+            int nextFrame = currentMovieFrame.Value + 1;
+            while (currentMovieFrame.Value != nextFrame)
+            {
+                device.UpdateFrame(false);
+                device.ReleaseFrame();
+            }
+            playStep.Value = 0;
+            Console.WriteLine("nextFrame");
+        }
+
+        public void previousFrame()
+        {
+            playStep.Value = -1;
+            int previousFrame = currentMovieFrame.Value - 1;
+            while (currentMovieFrame.Value != previousFrame)
+            {
+                device.UpdateFrame(false);
+                device.ReleaseFrame();
+            }
+            playStep.Value = 0;
+            Console.WriteLine("previousFrame");
+        }
+    }
+}

bbiwarg/MainBBWIWARG.cs

@@ -12,14 +12,21 @@ namespace bbiwarg
     {
         static void Main(string[] args)
         {
-            IInputProvider inputProvider = new IisuInputProvider();//"..\\..\\videos\\touch\\4.skv");
-            VideoHandle videoHandle = new VideoHandle(inputProvider);
-            videoHandle.start();
+            //InputProvider inputProvider = new InputProvider();
+            InputProvider inputProvider = new VideoInputProvider("..\\..\\videos\\touch\\4.skv");
+            inputProvider.initialize();
 
-            OutputWindow output = new OutputWindow(videoHandle);
-            output.Run(30);
 
-            videoHandle.stop();
+            InputHandler inputHandler = new InputHandler(inputProvider);
+
+            if (Constants.OutputEnabled)
+            {
+                OutputWindow outputWindow = new OutputWindow(inputProvider, inputHandler);
+                outputWindow.Run(Constants.OutputFrameRate);
+            }
+            else
+                throw new NotImplementedException("non-video mode not supported yet");
+
         }
     }
 }

bbiwarg/Recognition/FingerRecognition/Finger.cs

@@ -75,22 +75,5 @@ namespace bbiwarg.Recognition.FingerRecognition
         public void setTouchEvent(TouchEvent touchEvent) {
             TouchEvent = touchEvent;
         }
-
-        public void draw(OutputImage outputImage, bool tracked)
-        {
-            if (tracked)
-            {
-                outputImage.drawLineSegment(LineSegment, Constants.FingerTrackedColor);
-            }
-            else
-            {
-                for (int i = 0; i < SliceTrail.NumSlices; i++)
-                {
-                    outputImage.drawLineSegment(SliceTrail[i].LineSegment, Constants.FingerSliceColor);
-                }
-                outputImage.drawLineSegment(LineSegment, Constants.FingerDetectedColor);
-                outputImage.drawContour(Contour, Constants.FingerContourColor, 1);
-            }
-        }
     }
 }

bbiwarg/Recognition/FingerRecognition/FingerDetector.cs

@@ -18,18 +18,15 @@ namespace bbiwarg.Recognition.FingerRecognition
         private DepthImage depthImage;
         private EdgeImage edgeImageOriginal;
         private EdgeImage edgeImageAdapted;
-        private OutputImage outputImage;
         public List<Finger> Fingers { get; private set; }
 
-        public FingerDetector(DepthImage depthImage, EdgeImage edgeImage, OutputImage outputImage)
+        public FingerDetector(DepthImage depthImage, EdgeImage edgeImage)
         {
             this.depthImage = depthImage;
             this.edgeImageOriginal = edgeImage;
             this.edgeImageAdapted = edgeImage.copy();
-            this.outputImage = outputImage;
 
             detectFingers();
-            drawFingers();
         }
 
         private void detectFingers()
@@ -246,13 +243,5 @@ namespace bbiwarg.Recognition.FingerRecognition
 
             return trail;
         }
-        
-        private void drawFingers()
-        {
-            foreach (Finger finger in Fingers)
-            {
-                finger.draw(outputImage, false);
-            }
-        }
     }
 }

bbiwarg/Recognition/FingerRecognition/FingerTracker.cs

@@ -15,7 +15,6 @@ namespace bbiwarg.Recognition.FingerRecognition
         public FingerTracker()
             : base(Constants.FingerNumFramesDetectedUntilTracked, Constants.FingerNumFramesLostUntilDeleted, Constants.FingerMinSimilarityForTracking)
         {
-
         }
 
         protected override void onDetect(object sender, EventArgs e)

bbiwarg/Recognition/HandRecognition/HandDetector.cs

@@ -15,28 +15,23 @@ namespace bbiwarg.Recognition.HandRecognition
 {
     class HandDetector
     {
-        private DepthImage depthImage;
         private EdgeImage edgeImage;
         private List<Finger> fingers;
         public List<Hand> Hands { get; private set; }
         public Image<Gray, byte> HandMask { get; private set; }
-        public OutputImage outputImage;
 
-        public HandDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> fingers, OutputImage outputImage)
+        public HandDetector(EdgeImage edgeImage, List<Finger> fingers)
         {
-            this.depthImage = depthImage;
             this.edgeImage = edgeImage;
             this.fingers = fingers;
-            this.outputImage = outputImage;
 
             detectHands();
-            drawHands();
         }
 
         private void detectHands()
         {
-            int width = depthImage.Width;
-            int height = depthImage.Height;
+            int width = edgeImage.Width;
+            int height = edgeImage.Height;
             int maxArea = width * height;
             Image<Gray, byte> image = edgeImage.Image.Copy().Dilate(2).Erode(2).Mul(255);
             HandMask = image.CopyBlank();
@@ -44,7 +39,6 @@ namespace bbiwarg.Recognition.HandRecognition
             //draw top finger slice
             foreach (Finger finger in fingers)
             {
-                // TODO: connect contour with other edges
                 Contour<Point> contour = finger.Contour;
                 image.FillConvexPoly(contour.ToArray(), new Gray(0));
                 image.DrawPolyline(finger.Contour.ToArray(), true, new Gray(255), 1);
@@ -82,7 +76,7 @@ namespace bbiwarg.Recognition.HandRecognition
                     CvInvoke.cvFloodFill(image, finger.HandPoint, new MCvScalar(255), new MCvScalar(1), new MCvScalar(1), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, mask);
                     if (comp.area < maxArea * Constants.HandMaxSize)
                     {
-                        Image<Gray, byte> cropedMask = mask.Copy(new Rectangle(1, 1, width, height)).Mul(255).Dilate(1);
+                        Image<Gray, byte> cropedMask = mask.Copy(new Rectangle(1, 1, width, height)).Dilate(1);
                         Hand hand = new Hand(cropedMask);
                         Hands.Add(hand);
                         hand.addFinger(finger);
@@ -116,22 +110,5 @@ namespace bbiwarg.Recognition.HandRecognition
                     Hands[0].Side = Hand.HandSide.Left;
             }
         }
-
-        private void drawHands() {
-            if (Hands.Count == 1)
-            {
-                outputImage.Image[0] = outputImage.Image[1] = outputImage.Image[2] = Hands[0].Mask;
-            }
-            else if (Hands.Count > 1)
-            {
-                for (int i = 0; i < 2; ++i)
-                {
-                    if (Hands[i].Side == Hand.HandSide.Left)
-                        outputImage.Image[0] = outputImage.Image[0] | (Hands[i].Mask);
-                    else
-                        outputImage.Image[1] = outputImage.Image[1] | (Hands[i].Mask);
-                }
-            }
-        }
     }
 }

bbiwarg/Recognition/PalmRecognition/PalmDetector.cs

@@ -18,14 +18,12 @@ namespace bbiwarg.Recognition.PalmRecognition
 {
     class PalmDetector
     {
-        private OutputImage outputImage;
-
+        public Contour<Point> PalmContour { get; private set; }
+        public Vector2D ThumbDefectStart { get; private set; }
+        public Vector2D ThumbDefectEnd { get; private set; }
+        public Vector2D ThumbDefectDepth { get; private set; }
         private Hand palmHand, pointingHand;
-        private Contour<Point> palmContour;
         private List<MCvConvexityDefect> convexityDefects;
-        private Vector2D thumbDefectStart;
-        private Vector2D thumbDefectEnd;
-        private Vector2D thumbDefectDepth;
         private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
         private int numFramesNoHandFound;
         private Quadrangle lastPalmQuad;
@@ -44,9 +42,8 @@ namespace bbiwarg.Recognition.PalmRecognition
                                                                 Constants.PalmThumbDefectmYY, Constants.PalmThumbDefectProcessNoise);
         }
 
-        public void findPalmQuad(OutputImage outputImage, List<Hand> hands, Image<Gray, Byte> foregroundMask)
+        public void findPalmQuad(List<Hand> hands, Image<Gray, Byte> foregroundMask)
         {
-            this.outputImage = outputImage;
             this.hands = hands;
 
             if (hands.Count == 1 && hands[0].Fingers.Count == 1)
@@ -97,8 +94,6 @@ namespace bbiwarg.Recognition.PalmRecognition
             {
                 numFramesNoHandFound = 0;
             }
-
-            draw();
         }
 
         public void reset()
@@ -106,7 +101,7 @@ namespace bbiwarg.Recognition.PalmRecognition
             thumbDefectDepthFilter.reset();
             thumbDefectStartFilter.reset();
             thumbDefectEndFilter.reset();
-            palmContour = null;
+            PalmContour = null;
             PalmQuad = null;
             PalmHandSide = Hand.HandSide.Left;
             numFramesNoHandFound = 0;
@@ -118,14 +113,14 @@ namespace bbiwarg.Recognition.PalmRecognition
             Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
                                                                 Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
-            palmContour = contour;
+            PalmContour = contour;
             double maxPerimeter = 0;
             while (contour != null)
             {
                 if (contour.Perimeter > maxPerimeter)
                 {
                     maxPerimeter = contour.Perimeter;
-                    palmContour = contour;
+                    PalmContour = contour;
                 }
                 contour = contour.HNext;
             }
@@ -133,7 +128,7 @@ namespace bbiwarg.Recognition.PalmRecognition
 
         private void findConvexityDefectsSortedByDepth()
         {
-            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
+            convexityDefects = new List<MCvConvexityDefect>(PalmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
             convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2)
             {
                 if (d1.Depth < d2.Depth)
@@ -217,35 +212,35 @@ namespace bbiwarg.Recognition.PalmRecognition
 
             if (thumbDefect != null)
             {
-                thumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
-                thumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
-                thumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
+                ThumbDefectDepth = new Vector2D(thumbDefect.Value.DepthPoint);
+                ThumbDefectStart = new Vector2D(thumbDefect.Value.StartPoint);
+                ThumbDefectEnd = new Vector2D(thumbDefect.Value.EndPoint);
 
                 if (!thumbDefectDepthFilter.Initialized)
                 {
-                    thumbDefectDepthFilter.setInitialPosition(thumbDefectDepth);
-                    thumbDefectStartFilter.setInitialPosition(thumbDefectStart);
-                    thumbDefectEndFilter.setInitialPosition(thumbDefectEnd);
+                    thumbDefectDepthFilter.setInitialPosition(ThumbDefectDepth);
+                    thumbDefectStartFilter.setInitialPosition(ThumbDefectStart);
+                    thumbDefectEndFilter.setInitialPosition(ThumbDefectEnd);
                 }
                 else
                 {
-                    thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth);
-                    thumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(thumbDefectStart);
-                    thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd);
+                    ThumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(ThumbDefectDepth);
+                    ThumbDefectStart = thumbDefectStartFilter.getCorrectedPosition(ThumbDefectStart);
+                    ThumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(ThumbDefectEnd);
                 }
 
                 Vector2D handLength, handWidth, longestLineEndpoint, topLeft, bottomLeft, bottomRight, topRight;
 
-                Vector2D startDepth = thumbDefectStart - thumbDefectDepth;
-                Vector2D endDepth = thumbDefectEnd - thumbDefectDepth;
+                Vector2D startDepth = ThumbDefectStart - ThumbDefectDepth;
+                Vector2D endDepth = ThumbDefectEnd - ThumbDefectDepth;
 
                 if (startDepth.Length > endDepth.Length)
                 {
                     handLength = startDepth;
-                    longestLineEndpoint = thumbDefectStart;
+                    longestLineEndpoint = ThumbDefectStart;
                     if (hands.Count == 1)
                     {
-                        if (thumbDefectStart.X > thumbDefectDepth.X)
+                        if (ThumbDefectStart.X > ThumbDefectDepth.X)
                             hands[0].Side = Hand.HandSide.Left;
                         else
                             hands[0].Side = Hand.HandSide.Right;
@@ -254,10 +249,10 @@ namespace bbiwarg.Recognition.PalmRecognition
                 else
                 {
                     handLength = endDepth;
-                    longestLineEndpoint = thumbDefectEnd;
+                    longestLineEndpoint = ThumbDefectEnd;
                     if (hands.Count == 1)
                     {
-                        if (thumbDefectEnd.X > thumbDefectDepth.X)
+                        if (ThumbDefectEnd.X > ThumbDefectDepth.X)
                             hands[0].Side = Hand.HandSide.Left;
                         else
                             hands[0].Side = Hand.HandSide.Right;
@@ -268,7 +263,7 @@ namespace bbiwarg.Recognition.PalmRecognition
                 {
                     handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
                     topLeft = longestLineEndpoint + 0.15f * handLength;
-                    bottomLeft = thumbDefectDepth - 0.4f * handLength;
+                    bottomLeft = ThumbDefectDepth - 0.4f * handLength;
                     bottomRight = bottomLeft + handWidth;
                     topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
                 }
@@ -276,7 +271,7 @@ namespace bbiwarg.Recognition.PalmRecognition
                 {
                     handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
                     topRight = longestLineEndpoint + 0.15f * handLength;
-                    bottomRight = thumbDefectDepth - 0.4f * handLength;
+                    bottomRight = ThumbDefectDepth - 0.4f * handLength;
                     bottomLeft = bottomRight + handWidth;
                     topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth;
                 }
@@ -297,47 +292,5 @@ namespace bbiwarg.Recognition.PalmRecognition
             if (lastPalmQuad != null && getForegroundPixelPercentage(lastPalmQuad, foregroundMask) <= Constants.PalmMaxPrecentageQuadForegroundReset)
                 reset();
         }
-
-        private void draw()
-        {
-            if (palmContour != null && palmContour.Count<Point>() > 0)
-            {
-                outputImage.drawContour(palmContour, Constants.PalmConturColor);
-                outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
-            }
-
-            if (PalmQuad != null)
-            {
-                outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red);
-                outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red);
-                outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red);
-
-                outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
-
-                Vector2D[] vertices = PalmQuad.Vertices;
-                for (int i = 0; i < 4; ++i)
-                    outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor);
-
-                drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3]));
-            }
-        }
-
-        private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d)
-        {
-            Vector2D relAB = (b - a) / Constants.PalmGridRows;
-            Vector2D relDC = (c - d) / Constants.PalmGridRows;
-            Vector2D relBC = (c - b) / Constants.PalmGridColumns;
-            Vector2D relAD = (d - a) / Constants.PalmGridColumns;
-
-            for (int i = 1; i < Constants.PalmGridRows; i++)
-            {
-                outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor);
-            }
-
-            for (int i = 1; i < Constants.PalmGridColumns; i++)
-            {
-                outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor);
-            }
-        }
     }
 }

bbiwarg/Recognition/TouchRecognition/TouchDetector.cs

@@ -17,14 +17,12 @@ namespace bbiwarg.Recognition.TouchRecognition
     class TouchDetector
     {
         private DepthImage depthImage;
-        private OutputImage outputImage;
         private List<Finger> fingers;
         public List<TouchEvent> TouchEvents { get; private set; }
 
-        public TouchDetector(List<Finger> fingers, Quadrangle palmQuad, DepthImage depthImage, OutputImage outputImage)
+        public TouchDetector(DepthImage depthImage, List<Finger> fingers, Quadrangle palmQuad)
         {
             this.depthImage = depthImage;
-            this.outputImage = outputImage;
             this.fingers = fingers;
             this.TouchEvents = new List<TouchEvent>();
 
@@ -44,20 +42,15 @@ namespace bbiwarg.Recognition.TouchRecognition
 
                 if (touchValue > Constants.TouchEventMinTouchValue)
                 {
-                    
+
                     if (palmQuad != null && palmQuad.isInside(tipPointOutside))
                     {
-                        outputImage.fillCircle(tipPointOutside.IntX, tipPointOutside.IntY, 5, Constants.TouchEventDetectedColor);
-
                         TouchEvent touchEvent = new PalmTouchEvent(tipPointOutside, touchMask, finger, palmQuad);
                         TouchEvents.Add(touchEvent);
                         finger.setTouchEvent(touchEvent);
                     }
 
                 }
-
-                outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
-                outputImage.drawText(tipPoint.IntX, tipPoint.IntY,String.Format("{0:0.00}", touchValue), Color.Red);
             }
         }
 
@@ -80,54 +73,5 @@ namespace bbiwarg.Recognition.TouchRecognition
             Rectangle cropRect = new Rectangle(1, 1, rect.Width, rect.Height);
             return touchMask.Copy(cropRect);
         }
-
-
-        /*
-        private float getTouchValue(Point touchPoint)
-        {
-            int floodValue = 255;
-
-            int minX = Math.Max(touchPoint.X - 2*Constants.TouchEventAreaSize / 3, 0);
-            int maxX = Math.Min(touchPoint.X + Constants.TouchEventAreaSize / 3, depthImage.Width-1);
-            int minY = Math.Max(touchPoint.Y - 2*Constants.TouchEventAreaSize / 3, 0);
-            int maxY = Math.Min(touchPoint.Y + Constants.TouchEventAreaSize / 3, depthImage.Height-1);
-
-            Vector2D relTouch = new Vector2D(touchPoint.X - minX, touchPoint.Y - minY);
-            Rectangle rect = new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1);
-            Image<Gray, byte> touchArea = depthImage.Image.Copy(rect);
-
-            MCvConnectedComp comp = new MCvConnectedComp();
-            CvInvoke.cvFloodFill(touchArea, relTouch, new MCvScalar(floodValue), new MCvScalar(Constants.TouchEventFloodfillLowDiff), new MCvScalar(Constants.TouchEventFloodfillHighDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
-
-            int matchedPixels = 0;
-            int countedPixels = 0;
-            for (int x = minX; x <= maxX; x++)
-            {
-                for (int y = minY; y <= maxY; y++)
-                {
-                    Color color = outputImage.getColotAt(x, y);
-                    byte depth = touchArea.Data[y - minY, x - minX, 0];
-                    Color subtractColor;
-                    if(depth == floodValue)
-                    {
-                        matchedPixels++;
-                        subtractColor = Constants.TouchEventAreaMatchedSubtractColor;
-                    }
-                    else
-                    {
-                        subtractColor = Constants.TouchEventAreaNonMatchedSubtractColor;
-                    }
-                    Color newColor = Color.FromArgb(Math.Max(color.R - subtractColor.R, 0), Math.Max(color.G - subtractColor.G, 0), Math.Max(color.B - subtractColor.B, 0));
-                    outputImage.drawPixel(x, y, newColor);
-                    countedPixels++;
-                }
-            }
-
-            float rel = (float)matchedPixels / (float)countedPixels;
-
-            outputImage.drawLineSegment(new Utility.LineSegment2D(new Vector2D(minX, maxY), new Vector2D(minX + rel * Constants.TouchEventAreaSize, maxY)), Constants.TouchEventStatusBarColor, 2);
-            
-            return rel;
-        }*/
     }
 }

bbiwarg/VideoHandle.cs

@@ -1,254 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Diagnostics;
-using bbiwarg.Utility;
-using bbiwarg.Recognition.FingerRecognition;
-using bbiwarg.Recognition.PalmRecognition;
-using bbiwarg.Recognition.TouchRecognition;
-using bbiwarg.Recognition.HandRecognition;
-using bbiwarg.Recognition.Tracking;
-using bbiwarg.Images;
-using bbiwarg.InputProviders;
-using Emgu.CV;
-using Emgu.CV.Structure;
-using bbiwarg.Graphics;
-using bbiwarg.Server;
-
-namespace bbiwarg
-{
-    class VideoHandle
-    {
-        private IInputProvider inputProvider;
-        private InputFrame inputFrame;
-
-
-        public int Width { get; private set; }
-        public int Height { get; private set; }
-
-        public int CurrentFrame
-        {
-            get
-            {
-                if (sourceIsMovie())
-                    return inputProvider.getCurrentMovieFrame();
-                else
-                    return videoFrame;
-            }
-        }
-        private int lastFrame = int.MaxValue;
-
-        public OutputImage[] OutputImages { get; private set; }
-        
-        private DepthImage depthImage;
-        private EdgeImage edgeImage;
-        private ConfidenceImage confidenceImage;
-        
-        private FingerDetector fingerDetector;
-        private HandDetector handDetector;
-        private PalmDetector palmDetector;
-        private TouchDetector touchDetector;
-
-        private FingerTracker fingerTracker;
-        private TouchTracker touchTracker;
-
-        private TouchEventVisualizer touchEventVisualizer;
-        private TuioCommunicator tuioCommunicator;
-
-        private int videoFrame = 0;
-
-        public VideoHandle(IInputProvider inputProvider)
-        {
-            this.inputProvider = inputProvider;
-        }
-
-        public void start()
-        {
-            inputProvider.init();
-            inputProvider.start();
-            inputProvider.updateFrame();
-            initTrackers();
-            processFrameUpdate();
-        }
-
-        public void stop()
-        {
-            inputProvider.stop();
-            tuioCommunicator.close();
-        }
-
-        public bool sourceIsMovie()
-        {
-            return inputProvider.sourceIsMovie();
-        }
-
-        public void reversePlay()
-        {
-            inputProvider.reversePlay();
-        }
-
-        public void pauseMovie()
-        {
-            inputProvider.pauseMovie();
-        }
-
-        public void unpauseMovie()
-        {
-            inputProvider.unpauseMovie();
-        }
-
-        public void nextFrame()
-        {
-            if (inputProvider.isActive())
-            {
-                inputProvider.releaseFrame();
-                inputProvider.updateFrame();
-                processFrameUpdate();
-            }
-            else
-            {
-                inputProvider.stop();
-            }
-            videoFrame++;
-        }
-
-        private void initTrackers()
-        {
-            palmDetector = new PalmDetector();
-
-            fingerTracker = new FingerTracker();
-            touchTracker = new TouchTracker();
-
-            //register touchEventVisualizer to touchTracker
-            touchEventVisualizer = new TouchEventVisualizer(Constants.ImageWidth, Constants.ImageHeight);
-            touchTracker.PalmTouchDown += touchEventVisualizer.touchDown;
-            touchTracker.PalmTouchMove += touchEventVisualizer.touchMove;
-            touchTracker.PalmTouchUp += touchEventVisualizer.touchUp;
-
-            //register tuiCommunicator to touchTracker
-            tuioCommunicator = new TuioCommunicator(Constants.TuioIP, Constants.TuioPort);
-            touchTracker.PalmTouchDown += tuioCommunicator.touchDown;
-            touchTracker.PalmTouchMove += tuioCommunicator.touchMove;
-            touchTracker.PalmTouchUp += tuioCommunicator.touchUp;
-
-            //register 
-        }
-
-        private void processFrameUpdate()
-        {
-            Timer.start("processFrameUpdate");
-
-            Logger.CurrentFrame = CurrentFrame;
-            bool newStarted = (lastFrame - CurrentFrame >= 20);
-            tuioCommunicator.initFrame();
-
-            if (newStarted) {
-                //reset trackers
-                touchTracker.reset();
-                fingerTracker.reset();
-                touchEventVisualizer.reset();
-                tuioCommunicator.reset();
-            }
-
-            //read data from inputProvider
-            Timer.start("readInputData");
-            inputFrame = inputProvider.getInputFrame();
-            Width = inputFrame.Width;
-            Height = inputFrame.Height;
-            Timer.stop("readInputData");
-
-            //create output images
-            Timer.start("createOutputImages");
-            OutputImages = new OutputImage[Constants.OutputNumImages];
-            for (int i = 0; i < Constants.OutputNumImages; i++) {
-                OutputImages[i] = new OutputImage(Width, Height);
-                    
-            }
-            Timer.stop("createOutputImages");
-
-            //create confidenceImage
-            Timer.start("createConfidenceImage");
-            Image<Gray, Int16> rawConfidenceImage = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawConfidenceData);
-            confidenceImage = new ConfidenceImage(rawConfidenceImage);
-            Timer.stop("createConfidenceImage");
-
-            //create depthImage
-            Timer.start("createDepthImage");
-            Image<Gray, Int16> rawDepthImage = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawDepthData);
-            rawDepthImage = rawDepthImage.Or((1 - confidenceImage.Mask).Convert<Gray, Int16>().Mul(Int16.MaxValue));
-            depthImage = new DepthImage(rawDepthImage);
-            OutputImages[0].Image[0] = OutputImages[0].Image[1] = OutputImages[0].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
-            Timer.stop("createDepthImage");
-
-            // create edge image
-            Timer.start("createEdgeImage");
-            edgeImage = new EdgeImage(depthImage);
-            OutputImages[1].Image[2] = edgeImage.Image.Mul(255);
-            Timer.stop("createEdgeImage");
-
-            //detect fingers
-            Timer.start("fingerDetection");
-            fingerDetector = new FingerDetector(depthImage, edgeImage, OutputImages[1]);
-            Timer.stop("fingerDetection");
-
-            //track fingers
-            Timer.start("fingerTracking");
-            fingerTracker.updateFrame(fingerDetector.Fingers);
-            foreach (Finger f in fingerTracker.Fingers) {
-                OutputImages[1].drawText(f.MidPoint.IntX, f.MidPoint.IntY, f.TrackID.ToString(), Constants.FingerIDColor);
-            }
-            Timer.stop("fingerTracking");
-
-            //detect hands
-            Timer.start("handDetection");
-            handDetector = new HandDetector(depthImage, edgeImage, fingerDetector.Fingers, OutputImages[2]);
-            Timer.stop("handDetection");
-
-            //remove background noise
-            Timer.start("removeBackground");
-            OutputImages[3].Image[0] = OutputImages[3].Image[1] = OutputImages[3].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image.Or(255-handDetector.HandMask);
-            Timer.stop("removeBackground");
-
-            //detect palm
-            Timer.start("palmDetection");
-            if (newStarted)
-                palmDetector.reset();
-            palmDetector.findPalmQuad(OutputImages[3], handDetector.Hands, handDetector.HandMask);
-            Timer.stop("palmDetection");
-
-            //detect touchEvents
-            Timer.start("touchDetection");
-            touchDetector = new TouchDetector(fingerTracker.Fingers, palmDetector.PalmQuad, depthImage, OutputImages[0]);
-            Timer.stop("touchDetection");
-
-            //track touchEvents
-            Timer.start("touchTracking");
-            touchTracker.updateFrame(touchDetector.TouchEvents);
-            foreach(TouchEvent te in touchTracker.TouchEvents)
-            {
-                OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
-            }
-            Timer.stop("touchTracking");
-
-            //touchEventVisualizer
-            touchEventVisualizer.updateImage();
-            OutputImages[4] = touchEventVisualizer.OutputImage;
-
-
-            // add borders
-            for (int i = 0; i < Constants.OutputNumImages; i++)
-            {
-                OutputImages[i].drawRectangle(0, 0, Width - 1, Height - 1, Color.White);
-
-            }
-
-
-            lastFrame = CurrentFrame;
-            tuioCommunicator.commitFrame();
-            Timer.stop("processFrameUpdate");
-        }
-    }
-}

bbiwarg/bbiwarg.csproj

@@ -67,6 +67,11 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Constants.cs" />
+    <Compile Include="Graphics\OutputWindow.cs" />
+    <Compile Include="InputHandler.cs" />
+    <Compile Include="InputProviders\InputFrame.cs" />
+    <Compile Include="InputProviders\InputProvider.cs" />
+    <Compile Include="InputProviders\VideoInputProvider.cs" />
     <Compile Include="Recognition\FingerRecognition\Finger.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerDetector.cs" />
     <Compile Include="Recognition\FingerRecognition\FingerSliceTrail.cs" />
@@ -79,15 +84,11 @@
     <Compile Include="Recognition\TouchRecognition\TouchDetector.cs" />
     <Compile Include="Recognition\TouchRecognition\TouchEvent.cs" />
     <Compile Include="Recognition\TouchRecognition\TouchTracker.cs" />
-    <Compile Include="Graphics\OutputWindow.cs" />
     <Compile Include="Graphics\TouchEventVisualizer.cs" />
     <Compile Include="Images\ConfidenceImage.cs" />
     <Compile Include="Images\DepthImage.cs" />
     <Compile Include="Images\EdgeImage.cs" />
     <Compile Include="Graphics\OutputImage.cs" />
-    <Compile Include="InputProvider\InputFrame.cs" />
-    <Compile Include="InputProvider\IInputProvider.cs" />
-    <Compile Include="InputProvider\IisuInputProvider.cs" />
     <Compile Include="MainBBWIWARG.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Recognition\Tracking\SimilarityContainer.cs" />
@@ -111,7 +112,6 @@
     <Compile Include="Utility\Quadrangle.cs" />
     <Compile Include="Utility\Timer.cs" />
     <Compile Include="Utility\Vector2D.cs" />
-    <Compile Include="VideoHandle.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />