-cleanup

-implemented first steps of omniTouch-paper

bbiwarg/DepthImage.cs

@@ -0,0 +1,179 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Emgu.CV;
+using Emgu.CV.Structure;
+
+namespace bbiwarg
+{
+    class DepthImage
+    {
+        private int width;
+        private int height;
+        private Image<Gray, Int16> image;
+        private Image<Gray, Int16> edges;
+        private bool[,] fingerPoints;
+
+        private Int16 minDepth;
+        private Int16 maxDepth;
+
+        public DepthImage(int width, int height, Image<Gray, Int16> image) {
+            this.width = width;
+            this.height = height;
+            this.image = image;
+
+            this.image = this.image.SmoothMedian(3);
+            //minDepth
+            findMinDepth();
+
+            //maxDepth
+            maxDepth = (Int16) (minDepth + 200);
+            thresholdDepth(minDepth, maxDepth);
+
+            //edges
+            calculateEdges();
+
+            //findFingerPoints
+            findFingerPoints();
+        }
+
+        public int getWidth() {
+            return width;
+        }
+
+        public int getHeight() {
+            return height;
+        }
+
+        public Int16 getMinDepth() {
+            return minDepth;
+        }
+
+        public Int16 getMaxDepth() {
+            return maxDepth;
+        }
+
+        public Int16 getDepthAt(int x, int y) {
+            return image.Data[y, x, 0];
+        }
+
+        public Int16 getEdgeAt(int x, int y)
+        {
+            return edges.Data[y, x, 0];
+        }
+
+        public bool isFingerPoint(int x, int y) {
+            return fingerPoints[x,y];
+        }
+
+        private void setDepthAt(int x, int y, Int16 value) {
+            image.Data[y, x, 0] = value;
+        }
+
+        private void findMinDepth() {
+            minDepth = Int16.MaxValue;
+            for (int x = 0; x < width; ++x)
+            {
+                for (int y = 0; y < height; ++y)
+                {
+                    Int16 depth = getDepthAt(x,y);
+                    if (depth < minDepth)
+                        minDepth = depth;
+                }
+            }
+        }
+
+        private void thresholdDepth(int min, int max)
+        {
+            image = image.ThresholdToZero(new Gray(min));
+            image = image.ThresholdTrunc(new Gray(max));
+
+            for (int x = 0; x < width; ++x)
+            {
+                for (int y = 0; y < height; ++y)
+                {
+                    if (getDepthAt(x, y) == 0)
+                        setDepthAt(x, y, (short)max);
+                }
+            }
+        }
+
+        private void calculateEdges() {
+            Image<Gray, Byte> tmp = image.Convert<Byte>(delegate(Int16 depth) { return (byte)(((depth - minDepth) * Byte.MaxValue) / (maxDepth - minDepth)); });
+            Image<Gray, byte> tmp2 = tmp.Canny(100, 75, 3);
+            edges = tmp2.Convert<Int16>(delegate(byte depth) { if(depth > 0) {return Int16.MaxValue;} else {return 0;}});//((depth * Int16.MaxValue) / Byte.MaxValue); });
+        }
+
+        private void findFingerPoints()
+        {
+            int maxFingerSize = 30;
+            int minFingerSize = 10;
+            fingerPoints = new bool[width, height];
+            bool[,] pixelsCheckedHorizontal = new bool[width, height];
+            bool[,] pixelsCheckedVertical = new bool[width, height];
+
+            for (int x = 0; x < width; x++) {
+                for (int y = 0; y < height; y++) {
+                    if (pixelsCheckedHorizontal[x,y] == false && getDepthAt(x, y) < maxDepth && getEdgeAt(x,y) == 0) {
+                        pixelsCheckedHorizontal[x, y] = true;
+                        pixelsCheckedVertical[x, y] = true;
+
+                        //check horizontal
+                        int edgeLeftX = x-1;
+                        int edgeRightX = x+1;
+                        bool edgeLeftFound = false;
+                        bool edgeRightFound = false;
+                        while (edgeLeftFound == false && (x-edgeLeftX) < maxFingerSize && edgeLeftX > 0)
+                        {
+                            if (getEdgeAt(edgeLeftX, y) > 0) 
+                                edgeLeftFound = true;
+                            edgeLeftX--;
+                        }
+                        while (edgeRightFound == false && (edgeRightX - x) < maxFingerSize && edgeRightX < width)
+                        {
+                            if (getEdgeAt(edgeRightX, y) > 0)
+                                edgeRightFound = true;
+                            edgeRightX++;
+                        }
+
+                        if (edgeLeftFound && edgeRightFound && (edgeRightX - edgeLeftX) < maxFingerSize && (edgeRightX - edgeLeftX) > minFingerSize) {
+                            for (int i = edgeLeftX; i < edgeRightX; i++) {
+                                pixelsCheckedHorizontal[i, y] = true;
+                            }
+                            fingerPoints[(edgeLeftX+edgeRightX)/2, y] = true;
+                        }
+
+                        //check vertical
+                        int edgeTopY = y-1;
+                        int edgeBottomY = y+1;
+                        bool edgeTopFound = false;
+                        bool edgeBottomFound = false;
+                        while (edgeTopFound == false && (y-edgeTopY) < maxFingerSize && edgeTopY > 0)
+                        {
+                            if (getEdgeAt(x, edgeTopY) > 0)
+                                edgeTopFound = true;
+                            edgeTopY--;
+                        }
+                        while (edgeBottomFound == false && (edgeBottomY-y) < maxFingerSize && edgeBottomY < height)
+                        {
+                            if (getEdgeAt(x, edgeBottomY) > 0)
+                                edgeBottomFound = true;
+                            edgeBottomY++;
+                        }
+                        if (edgeBottomFound && edgeTopFound && (edgeBottomY - edgeTopY) < maxFingerSize && (edgeBottomY - edgeTopY) > minFingerSize)
+                        {
+                            for (int i = edgeTopY; i < edgeBottomY; i++)
+                            {
+                                pixelsCheckedVertical[x, i] = true;
+                            }
+                            fingerPoints[x, (edgeTopY+edgeBottomY)/2] = true;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

bbiwarg/Detectors/CollisionDetection.cs

@@ -1,18 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Detectors
-{
-    class CollisionDetection
-    {
-        public static float getDistance(Vector origin, Vector normal, Vector fingerPoint)
-        {
-            return Math.Abs((fingerPoint - origin).DotProduct(normal));
-        }
-    }
-}

bbiwarg/Detectors/ForeFingerDetection.cs

@@ -1,124 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.InputProviders;
-using bbiwarg.VideoHandles;
-
-
-namespace bbiwarg.Detectors
-{
-    class ForeFingerDetection
-    {
-
-        /*
-         * Returns the forefinger's position
-         * all fingers (except der forefinger) have to be inactive
-         * params:
-         *  handIndex: which hand's forefinger's position
-         */
-
-        public const int TOLERANCE_Z = 3;
-        public const int TOLERANCE_2D = 3;
-
-        private IInputProvider inputProvider;
-        private VideoHandle videoHandle;
-
-        private List<Vector> points;
-        private HashSet<Vector> seenPoints;
-        private Vector palmPosition;
-        private Vector foreFingerPosition;
-        private float foreFingerDistance;
-
-        private int pixelWidth;
-        private int pixelHeight;
-
-
-        public ForeFingerDetection(IInputProvider inputProvider, VideoHandle videoHandle)
-        {
-            this.inputProvider = inputProvider;
-            this.videoHandle = videoHandle;
-        }
-
-        public Vector getForeFingerPosition3D(uint handIndex)
-        {
-            pixelWidth = videoHandle.getWidth();
-            pixelHeight = videoHandle.getHeight();
-
-            Vector palmPosition2D = inputProvider.getPalmPosition2D(handIndex);
-            palmPosition = new DenseVector(3);
-            palmPosition[0] = palmPosition2D[0];
-            palmPosition[1] = palmPosition2D[1];
-            palmPosition[2] = videoHandle.getDepth((int)palmPosition[0], (int)palmPosition[1]);
-
-            points = new List<Vector>();
-            seenPoints = new HashSet<Vector>();
-            foreFingerPosition = palmPosition;
-            foreFingerDistance = 0;
-
-            points.Add(palmPosition);
-            seenPoints.Add(palmPosition);
-            for (int i = 0; i < points.Count(); i++)
-            {
-                checkPoint(points[i]);
-            }
-
-
-            if (points.Count() <= 10)
-            {
-                Console.WriteLine(palmPosition[2]);
-            }
-
-            if (seenPoints.Count() <= 100)
-            {
-
-            }
-            return videoHandle.pixel2VertexPosition(foreFingerPosition);
-        }
-
-
-        private void checkPoint(Vector point)
-        {
-            Vector distanceVector = new DenseVector(3);
-            videoHandle.pixel2VertexPosition(palmPosition).Subtract(videoHandle.pixel2VertexPosition(point), distanceVector);
-
-            float currentPointDistance = distanceVector.Norm(2);
-            if (currentPointDistance >= foreFingerDistance)
-            {
-                foreFingerPosition = point;
-                foreFingerDistance = currentPointDistance;
-            }
-            addNeighbours(point);
-        }
-
-        private void addNeighbours(Vector point)
-        {
-            Vector currentPoint;
-            int x_start = -(int)Math.Min(TOLERANCE_2D, point[0]);
-            for (int y = -(int)Math.Min(TOLERANCE_2D, point[1]); y <= TOLERANCE_2D && point[1] + y < pixelHeight; y++)
-            {
-                for (int x = x_start; x <= TOLERANCE_2D && point[0] + x < pixelWidth; x++)
-                {
-                    currentPoint = new DenseVector(3);
-                    currentPoint[0] = point[0] + x;
-                    currentPoint[1] = point[1] + y;
-                    currentPoint[2] = videoHandle.getDepth((int)currentPoint[0], (int)currentPoint[1]);
-                    if ((!seenPoints.Contains(currentPoint) && point[1] <= palmPosition[1] && Math.Abs(currentPoint[2] - point[2]) <= TOLERANCE_Z)
-                        || point == palmPosition)
-                    {
-                        points.Add(currentPoint);
-                    }
-                    seenPoints.Add(currentPoint);
-                }
-            }
-        }
-
-        public List<Vector> getHandPoints()
-        {
-            return points;
-        }
-    }
-}

bbiwarg/Detectors/Palm.cs

@@ -1,48 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Detectors
-{
-    class Palm
-    {
-        private Vector upperLeft, upperRight, lowerRight, lowerLeft;
-
-        public Palm(Vector upperLeft, Vector upperRight, Vector lowerRight, Vector lowerLeft)
-        {
-            this.upperLeft = upperLeft;
-            this.upperRight = upperRight;
-            this.lowerRight = lowerRight;
-            this.lowerLeft = lowerLeft;
-        }
-
-        public Vector getUpperLeft()
-        {
-            return upperLeft;
-        }
-
-        public Vector getUpperRight()
-        {
-            return upperRight;
-        }
-
-        public Vector getLowerRight()
-        {
-            return lowerRight;
-        }
-
-        public Vector getLowerLeft()
-        {
-            return lowerLeft;
-        }
-
-        public Vector[] getCorners()
-        {
-            return new Vector[4] { upperLeft, upperRight, lowerRight, lowerLeft };
-        }
-    }
-}

bbiwarg/Detectors/PalmDetection.cs

@@ -1,44 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.InputProviders;
-using bbiwarg.VideoHandles;
-using bbiwarg.Helpers;
-
-namespace bbiwarg.Detectors
-{
-    class PalmDetection
-    {
-        private IInputProvider input;
-        private VideoHandle videoHandle;
-        private ForeFingerDetection foreFingerDetection;
-
-        public PalmDetection(IInputProvider input, VideoHandle videoHandle)
-        {
-            this.input = input;
-            this.videoHandle = videoHandle;
-            this.foreFingerDetection = new ForeFingerDetection(input, videoHandle);
-        }
-
-        public Palm getPalm(uint handIndex)
-        {
-            DenseVector palmMiddle = (DenseVector) input.getPalmPosition3D(handIndex);
-            DenseVector tipPosition = (DenseVector)input.getTipPosition3D(handIndex);
-            DenseVector palmNormal = (DenseVector)input.getPalmNormal3D(handIndex);
-            //DenseVector thumbPosition = (DenseVector) input.getFingerTipPositions3D(handIndex)[0];
-
-            //DenseVector palmToThumb_2 = (thumbPosition - palmMiddle) / 2.0f;
-            DenseVector tipToForeFinger = tipPosition - palmMiddle;
-            DenseVector palmToThumb_2 = (DenseVector) tipToForeFinger.Cross(palmNormal) / 3.0f;
-           
-
-            return new Palm(palmMiddle + palmToThumb_2 + tipToForeFinger,
-                            palmMiddle - palmToThumb_2 + tipToForeFinger,
-                            palmMiddle - palmToThumb_2 - tipToForeFinger / 2.0f,
-                            palmMiddle + palmToThumb_2 - tipToForeFinger / 2.0f);
-        }
-    }
-}

bbiwarg/Graphics/GraphicElements2D/IGraphicElement2D.cs

@@ -1,15 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Graphics.GraphicElements2D
-{
-    interface IGraphicElement2D
-    {
-        void draw(short[] textureData, int width);
-    }
-}

bbiwarg/Graphics/GraphicElements2D/Point2D.cs

@@ -1,32 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OpenTK.Graphics.OpenGL;
-using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.Helpers;
-
-namespace bbiwarg.Graphics.GraphicElements2D
-{
-    class Point2D : IGraphicElement2D
-    {
-        private Vector position;
-        private Color color;
-
-        public Point2D(Vector position, Color color)
-        {
-            this.position = position;
-            this.color = color;
-        }
-
-        public void draw(short[] textureData, int width)
-        {
-            int index = (3 * ((int)position.y() * width + (int)position.x()));
-            textureData[index + 0] = (short) ((short.MaxValue / byte.MaxValue) * color.R);
-            textureData[index + 1] = (short) ((short.MaxValue / byte.MaxValue) * color.G);
-            textureData[index + 2] = (short) ((short.MaxValue / byte.MaxValue) * color.B);
-        }
-    }
-}

bbiwarg/Graphics/GraphicElements3D/IGraphicElement3D.cs

@@ -1,15 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Graphics.GraphicElements3D
-{
-    interface IGraphicElement3D
-    {
-        void draw();
-    }
-}

bbiwarg/Graphics/GraphicElements3D/Point3D.cs

@@ -1,37 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OpenTK.Graphics.OpenGL;
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Graphics.GraphicElements3D
-{
-    class Point3D : IGraphicElement3D
-    {
-        private Vector position;
-        private Color color;
-        private float size;
-
-        public Point3D(Vector position, Color color, float size)
-        {
-            this.position = position;
-            this.color = color;
-            this.size = size;
-        }
-
-        public void draw()
-        {
-            GL.Color4(color);
-
-            GL.Begin(BeginMode.Polygon);
-            GL.Vertex3(position[0] - size/2, position[1] + size/2, -position[2]);
-            GL.Vertex3(position[0] + size/2, position[1] + size/2, -position[2]);
-            GL.Vertex3(position[0] + size/2, position[1] - size/2, -position[2]);
-            GL.Vertex3(position[0] - size/2, position[1] - size/2, -position[2]);
-            GL.End();
-        }
-    }
-}

bbiwarg/Graphics/GraphicElements3D/Rectangle3D.cs

@@ -1,35 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OpenTK.Graphics.OpenGL;
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Graphics.GraphicElements3D
-{
-    class Rectangle3D : IGraphicElement3D
-    {
-        private Vector[] corners;
-        private Color color;
-
-        public Rectangle3D(Vector[] corners, Color color)
-        {
-            this.corners = corners;
-            this.color = color;
-        }
-
-        public void draw()
-        {
-            GL.Color4(color);
-
-            GL.Begin(BeginMode.Quads);
-            GL.Vertex3(corners[0][0], corners[0][1], -corners[0][2]);
-            GL.Vertex3(corners[1][0], corners[1][1], -corners[1][2]);
-            GL.Vertex3(corners[2][0], corners[2][1], -corners[2][2]);
-            GL.Vertex3(corners[3][0], corners[3][1], -corners[3][2]);
-            GL.End();
-        }
-    }
-}

bbiwarg/Graphics/Output2D.cs

@@ -1,138 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OpenTK;
-using OpenTK.Graphics.OpenGL;
-using System.Drawing;
-using System.Diagnostics;
-using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.Graphics.GraphicElements2D;
-using bbiwarg.InputProviders;
-using bbiwarg.Images;
-using bbiwarg.Helpers;
-using bbiwarg.VideoHandles;
-
-namespace bbiwarg.Graphics
-{
-    class Output2D: GameWindow
-    {
-        private VideoHandle videoHandle;
-        private uint textureId;
-
-        public Output2D(VideoHandle videoHandle): base(3 * 320, 3 * 240)
-        {
-            this.videoHandle = videoHandle;
-        }
-
-        protected override void OnLoad(EventArgs e)
-        {
-            base.OnLoad(e);
-            Title = "OutputTest";
-            GL.ClearColor(Color.Black);
-
-            // transparency
-            GL.Enable(EnableCap.Blend);
-            GL.BlendEquation(BlendEquationMode.Max);
-
-            //Depth Test 
-            GL.Enable(EnableCap.DepthTest);
-
-            // Textures
-            GL.Enable(EnableCap.Texture2D);
-
-            GL.GenTextures(1, out textureId);
-            GL.BindTexture(TextureTarget.Texture2D, textureId);
-            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
-            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
-        }
-
-        protected override void OnRenderFrame(FrameEventArgs e)
-        {
-            base.OnRenderFrame(e);
-            GL.Clear(ClearBufferMask.ColorBufferBit | ClearBufferMask.DepthBufferBit);
-            Matrix4 modelview = Matrix4.LookAt(Vector3.Zero, -Vector3.UnitZ, Vector3.UnitY);
-            GL.MatrixMode(MatrixMode.Modelview);
-            GL.LoadMatrix(ref modelview);
-
-            videoHandle.nextFrame();
-
-            Stopwatch sw = new Stopwatch();
-            sw.Start();
-
-            short[] textureData = new short[3 * videoHandle.getWidth() * videoHandle.getHeight()];
-            int index = 0;
-            short minDepth = videoHandle.getHandImage().getMinDepth();
-            short maxDepth = videoHandle.getHandImage().getMaxDepth();
-            
-            for (int y = 0; y < videoHandle.getHeight(); ++y)
-            {
-                for (int x = 0; x < videoHandle.getWidth(); ++x)
-                {
-                    short depth = videoHandle.getHandImage().getDepth(x, y);
-                    short diff = (short) Math.Max(maxDepth - minDepth, 1);
-                    short scaledDepth = (short) (Int16.MaxValue - (((int) depth - minDepth) * (int) Int16.MaxValue / diff));
-                    textureData[index] = scaledDepth;
-                    textureData[index + 1] = scaledDepth;
-                    textureData[index + 2] = scaledDepth;
-                    index += 3;
-                }
-            }
-
-            // draw histogram
-            GL.Disable(EnableCap.Texture2D);
-            int[] histogram = videoHandle.getSmoothedHistogram();
-            int maxValue = 0;
-            for (int i = 0; i < histogram.Length; ++i)
-            {
-                if (histogram[i] > maxValue)
-                    maxValue = histogram[i];
-            }
-
-            Int16[] segmentationDepth = videoHandle.getSegementationDepth();
-
-            GL.Begin(PrimitiveType.LineStrip);
-            GL.Color3(0, 0, 1.0);
-            GL.LineWidth(5.0f);
-            for (int i = 0; i < histogram.Length; ++i)
-            {
-                if (i > segmentationDepth[0]) GL.Color3(Color.Yellow);
-                if (i > segmentationDepth[1]) GL.Color3(Color.Red);
-                if (i > segmentationDepth[2]) GL.Color3(Color.Silver);
-                
-                GL.Vertex3(-0.25 + i * (0.5 / histogram.Length), -0.25 + histogram[i] * (0.5 / (maxValue - 5)), -0.5);
-            }
-            GL.End();
-
-            // draw texture
-            GL.Enable(EnableCap.Texture2D);
-            GL.BindTexture(TextureTarget.Texture2D, textureId);
-            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.getWidth(), videoHandle.getHeight(), 0,
-                          PixelFormat.Rgb, PixelType.Short, textureData);
-
-            float size_2 = 0.5f / 2.0f;
-            GL.Begin(PrimitiveType.Quads);
-            GL.Color3(1.0, 1.0, 1.0);
-            GL.TexCoord2(0.0, 0.0); GL.Vertex3(-size_2,  size_2, -0.5);
-            GL.TexCoord2(1.0, 0.0); GL.Vertex3( size_2,  size_2, -0.5);
-            GL.TexCoord2(1.0, 1.0); GL.Vertex3( size_2, -size_2, -0.5);
-            GL.TexCoord2(0.0, 1.0); GL.Vertex3(-size_2, -size_2, -0.5);
-            GL.End();
-
-            sw.Stop();
-            Console.WriteLine(sw.ElapsedMilliseconds);
-
-            SwapBuffers();
-        }
-
-        protected override void OnResize(EventArgs e)
-        {
-            base.OnResize(e);
-            GL.Viewport(ClientRectangle.X, ClientRectangle.Y, ClientRectangle.Width, ClientRectangle.Height);
-            Matrix4 projection = Matrix4.CreatePerspectiveFieldOfView((float)Math.PI / 3, Width / (float)Height, 0.01f, 3.0f);
-            GL.MatrixMode(MatrixMode.Projection);
-            GL.LoadMatrix(ref projection);
-        }
-    }
-}

bbiwarg/Graphics/Output3D.cs

@@ -1,146 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OpenTK;
-using OpenTK.Graphics.OpenGL;
-using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.Detectors;
-using bbiwarg.Graphics.GraphicElements3D;
-using bbiwarg.VideoHandles;
-using bbiwarg.InputProviders;
-
-
-namespace bbiwarg.Graphics
-{
-    class Output3D : GameWindow
-    {
-        private VideoHandle videoHandle;
-
-        private uint imageBufferId;
-
-        public Output3D(VideoHandle videoHandle)
-        {
-            this.videoHandle = videoHandle;
-        }
-
-        protected override void OnLoad(EventArgs e)
-        {
-            base.OnLoad(e);
-            Title = "OutputTest";
-            GL.ClearColor(Color.Black);
-
-            // transparency
-            GL.Enable(EnableCap.Blend);
-            GL.BlendEquation(BlendEquationMode.Max);
-
-            //Depth Test 
-            GL.Enable(EnableCap.DepthTest);
-
-
-            initBuffers();
-
-            GL.Enable(EnableCap.Blend);
-            GL.BlendFunc(BlendingFactorSrc.SrcAlpha, BlendingFactorDest.OneMinusSrcAlpha);
-        }
-
-        protected override void OnRenderFrame(FrameEventArgs e)
-        {
-            base.OnRenderFrame(e);
-            GL.Clear(ClearBufferMask.ColorBufferBit | ClearBufferMask.DepthBufferBit);
-            Matrix4 modelview = Matrix4.LookAt(Vector3.Zero, -Vector3.UnitZ, Vector3.UnitY);
-            GL.MatrixMode(MatrixMode.Modelview);
-            GL.LoadMatrix(ref modelview);
-
-            videoHandle.nextFrame();
-
-            Stopwatch sw = new Stopwatch();
-            sw.Start();
-
-            drawDepthImage();
-
-            // palm (iisu)
-            Vector palmPosition = videoHandle.getPalmPosition3D(1);
-            Point3D palmPoint = new Point3D(palmPosition, Color.Yellow, 0.005f);
-            palmPoint.draw();
-
-            // foreFinger
-            Vector foreFingerPosition = videoHandle.getForeFingerPosition3D(1);
-            Point3D foreFingerPoint = new Point3D(foreFingerPosition, Color.Red, 0.05f);
-            foreFingerPoint.draw();
-
-            //handPoints
-            List<Vector> handPoints = videoHandle.getHandPoints();
-            for (int i = 0; i < handPoints.Count(); i++)
-            {
-                new Point3D(videoHandle.pixel2VertexPosition(handPoints[i]), Color.Yellow, 0.001f).draw();
-            }
-
-            // foreArm (iisu)
-            Vector foreArmPosition = videoHandle.getForearmPosition3D(1);
-            Point3D foreArmPoint = new Point3D(foreArmPosition, Color.Yellow, 0.005f);
-            foreArmPoint.draw();
-
-            // finger (iisu)
-            Vector[] fingerPositions = videoHandle.getFingerTipPositions3D(1);
-            DetectionStatus[] fingerStatus = videoHandle.getFingerStatus(1);
-            for (int i = 0; i < fingerStatus.Length; ++i)
-            {
-                if (fingerStatus[i] == DetectionStatus.Detected || fingerStatus[i] == DetectionStatus.Tracked)
-                {
-                    Point3D fingerPoint = new Point3D(fingerPositions[i], Color.Yellow, 0.005f);
-                    fingerPoint.draw();
-                }
-            }
-
-            // palm
-            Palm palm = videoHandle.getPalm(1);
-            Rectangle3D palmRect = new Rectangle3D(palm.getCorners(), Color.FromArgb(128, Color.Blue));
-            palmRect.draw();
-
-            sw.Stop();
-            Console.WriteLine(sw.ElapsedMilliseconds);
-
-            SwapBuffers();
-        }
-
-        protected override void OnResize(EventArgs e)
-        {
-            base.OnResize(e);
-            GL.Viewport(ClientRectangle.X, ClientRectangle.Y, ClientRectangle.Width, ClientRectangle.Height);
-            Matrix4 projection = Matrix4.CreatePerspectiveFieldOfView((float)Math.PI / 3, Width / (float)Height, 0.01f, 3.0f);
-            GL.MatrixMode(MatrixMode.Projection);
-            GL.LoadMatrix(ref projection);
-        }
-
-        private void initBuffers()
-        {
-            GL.GenBuffers(1, out imageBufferId);
-
-            GL.BindBuffer(BufferTarget.ArrayBuffer, imageBufferId);
-            GL.BufferData(BufferTarget.ArrayBuffer, 
-                          (IntPtr)((3 * sizeof(float) + 4  * sizeof(byte)) * videoHandle.getWidth() * videoHandle.getHeight()), 
-                          IntPtr.Zero, BufferUsageHint.StreamDraw);
-        }
-
-        private void drawDepthImage()
-        {
-            GL.EnableClientState(ArrayCap.VertexArray);
-            GL.EnableClientState(ArrayCap.ColorArray);
-
-            GL.BindBuffer(BufferTarget.ArrayBuffer, imageBufferId);
-            videoHandle.createVertexArray(GL.MapBuffer(BufferTarget.ArrayBuffer, BufferAccess.WriteOnly));
-            GL.UnmapBuffer(BufferTarget.ArrayBuffer);
-
-            GL.VertexPointer(3, VertexPointerType.Float, 3 * sizeof(float) + 4 * sizeof(byte), IntPtr.Zero);
-            GL.ColorPointer(4, ColorPointerType.UnsignedByte, 3 * sizeof(float) + 4 * sizeof(byte), 3 * sizeof(float));
-
-            GL.PointSize(2.0f);
-            GL.DrawArrays(PrimitiveType.Points, 0, videoHandle.getWidth() * videoHandle.getHeight());
-           
-        }
-    }
-}

bbiwarg/Graphics/OutputWindow.cs

@@ -0,0 +1,134 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using OpenTK;
+using OpenTK.Graphics.OpenGL;
+
+namespace bbiwarg.Graphics
+{
+    class OutputWindow : GameWindow
+    {
+        private VideoHandle videoHandle;
+        private uint depthTextureID;
+        private uint edgeTextureID;
+
+        public OutputWindow(VideoHandle videoHandle): base(3 * videoHandle.getWidth(), 2 * videoHandle.getHeight())
+        {
+            this.videoHandle = videoHandle;
+        }
+
+        protected override void OnLoad(EventArgs e)
+        {
+            base.OnLoad(e);
+            Title = "BBIWARG - Output";
+            GL.ClearColor(Color.Black);
+
+            // transparency
+            GL.Enable(EnableCap.Blend);
+            GL.BlendEquation(BlendEquationMode.Max);
+
+            //Depth Test 
+            GL.Enable(EnableCap.DepthTest);
+
+            // Textures
+            GL.Enable(EnableCap.Texture2D);
+
+            //depthTexture
+            GL.GenTextures(1, out depthTextureID);
+            GL.BindTexture(TextureTarget.Texture2D, depthTextureID);
+            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
+            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
+
+            //edgetexture
+            GL.GenTextures(1, out edgeTextureID);
+            GL.BindTexture(TextureTarget.Texture2D, edgeTextureID);
+            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
+            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
+        }
+
+        protected override void OnResize(EventArgs e)
+        {
+            base.OnResize(e);
+            GL.Viewport(ClientRectangle.X, ClientRectangle.Y, ClientRectangle.Width, ClientRectangle.Height);
+            Matrix4 projection = Matrix4.CreatePerspectiveFieldOfView((float)Math.PI / 3, Width / (float)Height, 0.01f, 3.0f);
+            GL.MatrixMode(MatrixMode.Projection);
+            GL.LoadMatrix(ref projection);
+        }
+
+        protected override void OnRenderFrame(FrameEventArgs e)
+        {
+            base.OnRenderFrame(e);
+            GL.Clear(ClearBufferMask.ColorBufferBit | ClearBufferMask.DepthBufferBit);
+            Matrix4 modelview = Matrix4.LookAt(Vector3.Zero, -Vector3.UnitZ, Vector3.UnitY);
+            GL.MatrixMode(MatrixMode.Modelview);
+            GL.LoadMatrix(ref modelview);
+            
+            videoHandle.nextFrame();
+
+            //draw textures
+            DepthImage depthImage = videoHandle.getDepthImage();
+            Int16 minDepth = 0;// depthImage.getMinDepth();
+            Int16 maxDepth = 400;// depthImage.getMaxDerivative();
+            Int64 minMaxInterval = (Int16) Math.Max(maxDepth - minDepth, 1);
+            Int16[] depthTextureData = new Int16[3 * depthImage.getWidth() * depthImage.getHeight()];
+            Int16[] edgeTextureData = new Int16[3 * depthImage.getWidth() * depthImage.getHeight()];
+            int index = 0;
+
+            for (int y = 0; y < depthImage.getHeight(); ++y)
+            {
+                for (int x = 0; x < depthImage.getWidth(); ++x)
+                {
+                    //depthTexture
+                    Int16 depth = Math.Min(depthImage.getDepthAt(x, y), maxDepth);
+                    Int16 scaledDepth =  (Int16)(Int16.MaxValue - (((int)depth - minDepth) * (int)Int16.MaxValue / minMaxInterval));
+                    depthTextureData[index] = scaledDepth;
+                    depthTextureData[index + 1] = scaledDepth;
+                    depthTextureData[index + 2] = scaledDepth;
+
+                    //edgeTexture
+                    Int16 edge = depthImage.getEdgeAt(x, y);
+
+                    if (depthImage.isFingerPoint(x, y)) edge = Int16.MaxValue / 2;
+
+                    edgeTextureData[index] = edge;
+                    edgeTextureData[index + 1] = edge;
+                    edgeTextureData[index + 2] = edge;
+
+                    index += 3;
+                }
+            }
+
+            GL.Enable(EnableCap.Texture2D);
+            GL.BindTexture(TextureTarget.Texture2D, depthTextureID);
+            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.getWidth(), videoHandle.getHeight(), 0, PixelFormat.Rgb, PixelType.Short, depthTextureData);
+            float size = 0.5f;
+            float size_2 = (float) (size / 2.0f);
+            GL.Begin(PrimitiveType.Quads);
+            GL.Color3(1.0, 1.0, 1.0);
+            GL.TexCoord2(0.0, 0.0); GL.Vertex3(0, size_2, -0.5);
+            GL.TexCoord2(1.0, 0.0); GL.Vertex3(size, size_2, -0.5);
+            GL.TexCoord2(1.0, 1.0); GL.Vertex3(size, -size_2, -0.5);
+            GL.TexCoord2(0.0, 1.0); GL.Vertex3(0, -size_2, -0.5);
+            GL.End();
+
+            GL.Enable(EnableCap.Texture2D);
+            GL.BindTexture(TextureTarget.Texture2D, edgeTextureID);
+            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.getWidth(), videoHandle.getHeight(), 0, PixelFormat.Rgb, PixelType.Short, edgeTextureData);
+            GL.Begin(PrimitiveType.Quads);
+            GL.Color3(1.0, 1.0, 1.0);
+            GL.TexCoord2(0.0, 0.0); GL.Vertex3(0, -size/2.0, -0.5);
+            GL.TexCoord2(-1.0, 0.0); GL.Vertex3(-size, -size/2.0, -0.5);
+            GL.TexCoord2(-1.0, -1.0); GL.Vertex3(-size, size/2.0, -0.5);
+            GL.TexCoord2(0.0, -1.0); GL.Vertex3(0, size/2.0, -0.5);
+            GL.End();
+
+
+            SwapBuffers();
+        }
+
+    }
+}

bbiwarg/Helpers/IVideoDataSource.cs

@@ -1,86 +0,0 @@
-using System;
-
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.DataSource
-{
-    public enum DetectionStatus
-    {
-        Inactive,
-        Detected,
-        Tracked
-    }
-
-    public enum HandSide
-    {
-        Unknown,
-        Left,
-        Right
-    }
-
-    /*
-     * Interface get data from a camera or a video file.
-     * Completely independent of iisu.
-     */
-    interface IVideoDataSource
-    {
-        /*
-         * Initializes the data source.
-         */
-        void init();
-
-        /*
-         * Starts the recording of data.
-         */
-        void start();
-        /*
-         * Stops the recording of data.
-         */
-        void stop();
-
-        /*
-         * Updates the data for the current frame.
-         * Needs to be called before any method to read data.
-         */
-        void updateFrame();
-        /*
-         * Lets the data source process the next frame.
-         */
-        void releaseFrame();
-
-        /*
-         * Returns true iff new data is generated.
-         * (e.g camera is running or video hasn't ended)
-         */
-        bool isActive();
-        int getFrameRate();
-        float getHFOV();
-        float getVFOV();
-        /**
-         * The depth is given by the distance to the camera in millimeters. 
-         */
-        DepthImage getDepthImage();
-        ColorImage getColorImage();
-        ConfidenceImage getConfidenceImage();
-        UVImage getUVImage();
-        ImageData getImageData();
-        /*
-         * all handIndices have to be 1 or 2
-         */
-        bool isHandOpen(uint handIndex);
-        Vector getPalmPosition3D(uint handIndex);
-        Vector getPalmPosition2D(uint handIndex);
-        Vector getTipPosition3D(uint handIndex);
-        Vector getForearmPosition3D(uint handIndex);
-        Vector getPalmNormal3D(uint handIndex);
-        DetectionStatus[] getFingerStatus(uint handIndex);
-        DetectionStatus getHandStatus(uint handIndex);
-        Vector[] getFingerTipPositions3D(uint handIndex);
-        Vector[] getFingerTipPositions2D(uint handIndex);
-        HandSide getHandSide(uint handIndex);
-        void createVertexArray();
-        
-        //TODO: implement properly
-        void setVertexBuffer(IntPtr vertexBuffer);
-    }
-}

bbiwarg/Helpers/VectorExtender.cs

@@ -1,43 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Helpers
-{
-    public static class VectorExtender
-    {
-        public static Vector Cross(this Vector v, Vector other)
-        {
-            if ((v.Count != 3 || other.Count != 3))
-            {
-                string message = "Vectors must have a length of 3.";
-                throw new Exception(message);
-            }
-            Vector result = new DenseVector(3);
-            result[0] = v[1] * other[2] - v[2] * other[1];
-            result[1] = -v[0] * other[2] + v[2] * other[0];
-            result[2] = v[0] * other[1] - v[1] * other[0];
-
-            return result;
-        }
-        
-        public static float x(this Vector v)
-        {
-            return v[0];
-        }
-
-        public static float y(this Vector v)
-        {
-            return v[1];
-        }
-
-        public static float z(this Vector v)
-        {
-            return v[2];
-        }
-    }
-}

bbiwarg/Helpers/VertexArray.cs

@@ -1,38 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using bbiwarg.Graphics;
-
-using MathNet.Numerics.LinearAlgebra.Single;
-
-namespace bbiwarg.Helpers
-{
-    class VertexArray
-    {
-        private float[] vertexData;
-        private Color[] colors;
-
-        public VertexArray(float[] vertexData, Color[] colors)
-        {
-            this.vertexData = vertexData;
-            this.colors = colors;
-        }
-
-        public int getNumVertices() 
-        {
-            return vertexData.Length / 3;
-        }
-
-        public float[] getVertexData()
-        {
-            return vertexData;
-        }
-
-        public Color getColor(int i) {
-            return colors[i];
-        }
-    }
-}

bbiwarg/Images/ColorImage.cs

@@ -1,43 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Drawing;
-
-namespace bbiwarg.Images
-{
-    class ColorImage
-    {
-        private int width, height;
-        private byte[] data;
-
-        public ColorImage(int width, int height, byte[] data)
-        {
-            this.width = width;
-            this.height = height;
-            this.data = data;
-        }
-
-        public int getWidth()
-        {
-            return width;
-        }
-
-        public int getHeight()
-        {
-            return height;
-        }
-
-        public Color getColor(int x, int y)
-        {
-            int offset = 4 * (y * width + x);
-            byte alpha = data[offset + 3];
-            byte red = data[offset + 2];
-            byte green = data[offset + 1];
-            byte blue = data[offset + 0];
-
-            return Color.FromArgb(alpha, red, green, blue);
-        }
-    }
-}

bbiwarg/Images/ConfidenceImage.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace bbiwarg.Images
-{
-    /**
-     * Represents an confidence image where the depth is given in distance to the camera in millimeters. 
-     */
-    class ConfidenceImage
-    {
-        private int width, height;
-        private short[] data;
-
-        public ConfidenceImage(int width, int height, short[] data)
-        {
-            this.width = width;
-            this.height = height;
-            this.data = data;
-        }
-
-        public int getWidth()
-        {
-            return width;
-        }
-
-        public int getHeight()
-        {
-            return height;
-        }
-
-        public short getConfidence(int x, int y)
-        {
-            return data[y * width + x];
-        }
-    }
-}

bbiwarg/Images/DepthImage.cs

@@ -1,231 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-using MathNet.Numerics.LinearAlgebra.Single;
-
-using System.Drawing;
-using Emgu.CV;
-using Emgu.CV.Structure;
-
-namespace bbiwarg.Images
-{
-    /**
-     * Represents an depth image where the depth is given in distance to the camera in millimeters. 
-     */
-    class DepthImage
-    {
-        private int width, height;
-        private Image<Gray, short> image;
-        private int[] histogram;
-
-        public DepthImage(int width, int height, short[] data)
-        {
-            this.width = width;
-            this.height = height;
-            
-            image = new Image<Gray, short>(width, height);
-            for (int i = 0; i < data.Length; ++i)
-                setDepth(i % width, i / width, data[i]);
-        }
-
-        public int getWidth()
-        {
-            return width;
-        }
-
-        public int getHeight()
-        {
-            return height;
-        }
-
-        public short getDepth(int x, int y)
-        {
-            return image.Data[y, x, 0];
-        }
-
-        private void setDepth(int x, int y, short depth)
-        {
-            image.Data[y, x, 0] = depth;
-        }
-
-        public void filterMedian(int filterSize)
-        {
-            image = image.SmoothMedian(filterSize);
-        }
-
-        public void thresholdDepth(int min, int max)
-        {
-            image = image.ThresholdToZero(new Gray(min));
-            image = image.ThresholdTrunc(new Gray(max));
-
-            for (int x = 0; x < width; ++x)
-            {
-                for (int y = 0; y < height; ++y)
-                {
-                    if (getDepth(x, y) == 0)
-                        setDepth(x, y, (short) max);
-                }
-            }
-        }
-
-        public void thresholdBinary(Int16 thresholdDepth)
-        {
-            Int16 maxDepth = getMaxDepth();
-            for (int x = 0; x < width; ++x)
-            {
-                for (int y = 0; y < height; ++y)
-                {
-                    Int16 depth = getDepth(x, y);
-                    if (depth <= thresholdDepth)
-                        setDepth(x, y, 0);
-                    else if (depth != maxDepth)
-                        setDepth(x, y, (short) (maxDepth / 2));
-                }
-            }
-        }
-
-        public int[] getSmoothedHistogram()
-        {
-            Int16 minDepth = getMinDepth();
-            Int16 maxDepth = getMaxDepth();
-
-            histogram = new int[maxDepth - minDepth];
-            for (int x = 0; x < width; ++x)
-            {
-                for (int y = 0; y < height; ++y)
-                {
-                    int depth = getDepth(x, y);
-                    if (depth != maxDepth)
-                        histogram[depth - minDepth]++;
-                }
-            }
-
-            smoothHistogram();
-            return histogram;
-        }
-
-        private void smoothHistogram()
-        {
-            histogram[0] = (histogram[0] + histogram[1]) / 2;
-            histogram[histogram.Length - 1] = (histogram[histogram.Length - 2] + histogram[histogram.Length - 1]) / 2;
-
-            for (int i = 1; i < histogram.Length - 1; ++i)
-            {
-                histogram[i] = (histogram[i - 1] + histogram[i] + histogram[i + 1]) / 3;
-            }
-        }
-
-        public Int16[] getSegmentationDepth()
-        {
-            //get first peak
-            Int16 firstMaxIndex = 0;
-
-            for (Int16 i = 0; i < histogram.Length; i++) {
-                if (histogram[i] > histogram[firstMaxIndex]) {
-                    firstMaxIndex = i;
-                }
-            }
-
-            // get second peak
-            Int16 minDistance = 20; // mm
-            Int16 secondMaxIndex = 0;
-            for (Int16 i = 0; i < histogram.Length; i++) {
-                if (Math.Abs(firstMaxIndex - i) > minDistance && histogram[i] > histogram[secondMaxIndex]) {
-                    secondMaxIndex = i;
-                }
-            }
-
-            if (secondMaxIndex < firstMaxIndex) {
-                Int16 dummy = firstMaxIndex;
-                firstMaxIndex = secondMaxIndex;
-                secondMaxIndex = dummy;
-            }
-
-            // get min in between
-            Int16 minIndex = firstMaxIndex;
-            for (Int16 i = firstMaxIndex; i < secondMaxIndex; i++) {
-                if (histogram[i] < histogram[minIndex]) {
-                    minIndex = i;
-                }
-            }
-
-            Int16 minDepth = getMinDepth();
-
-            Int16[] result = new Int16[3];
-            result[0] = firstMaxIndex;
-            result[1] = minIndex;
-            result[2] = secondMaxIndex;
-
-            return result;
-        }
-
-        public Int16 getPeakDepth()
-        {
-            Int16 minDepth = getMinDepth();
-
-            Int16 peak = -1;
-            int maxValue = 0;
-            for (int i = 0; i < histogram.Length; ++i)
-            {
-                if (histogram[i] > maxValue)
-                {
-                    maxValue = histogram[i];
-                    peak = (Int16) (i + minDepth);
-                }
-            }
-            return peak;
-        }
-
-        public Int16 getMinDepth()
-        {
-            Int16 minDepth = Int16.MaxValue;
-            for (int x = 0; x < width; ++x)
-            {
-                for (int y = 0; y < height; ++y)
-                {
-                    Int16 depth = getDepth(x, y);
-                    if (depth < minDepth)
-                        minDepth = depth;
-                }
-            }
-            return minDepth;
-        }
-
-        public Int16 getMaxDepth()
-        {
-            Int16 maxDepth = Int16.MinValue;
-            for (int x = 0; x < width; ++x)
-            {
-                for (int y = 0; y < height; ++y)
-                {
-                    Int16 depth = getDepth(x, y);
-                    if (depth > maxDepth)
-                        maxDepth = depth;
-                }
-            }
-            return maxDepth;
-        }
-
-        public void floodFill(int x, int y, int val)
-        {
-            Image<Gray, Byte> tmp = image.Convert<Byte>(delegate(short s) { return (byte) ((int) s * Byte.MaxValue / 2000); });
-
-            Emgu.CV.Structure.MCvConnectedComp comp = new MCvConnectedComp();
-            Emgu.CV.CvInvoke.cvFloodFill(tmp.Ptr, new Point(x, y), new MCvScalar(val), new MCvScalar(30), new MCvScalar(30), out comp, 0, IntPtr.Zero);
-
-            for (int i = 0; i < width; ++i)
-            {
-                for (int j = 0; j < height; ++j)
-                {
-                    if (tmp.Data[j, i, 0] == 255)
-                        setDepth(i, j, 2000);
-                    else
-                        setDepth(i, j, 0);
-                }
-            }
-        }
-    }
-}

bbiwarg/Images/UVImage.cs

@@ -1,41 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace bbiwarg.Images
-{
-    class UVImage
-    {
-        private int width, height;
-        private float[] data;
-
-        public UVImage(int width, int height, float[] data)
-        {
-            this.width = width;
-            this.height = height;
-            this.data = data;
-        }
-
-        public int getWidth()
-        {
-            return width;
-        }
-
-        public int getHeight()
-        {
-            return height;
-        }
-
-        public float getU(int x, int y)
-        {
-            return data[2 * (y * width + x) + 0];
-        }
-
-        public float getV(int x, int y)
-        {
-            return data[2 * (y * width + x) + 1];
-        }
-    }
-}

bbiwarg/InputProvider/IInputProvider.cs

@@ -4,7 +4,6 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.Images;
 
 namespace bbiwarg.InputProviders
 {
@@ -33,14 +32,9 @@ namespace bbiwarg.InputProviders
         float getFrameRate();
         float getHFOV();
         float getVFOV();
-        int getConfidenceThreshold();
-        void setConfidenceThreshold(int value);
         bool isActive();
 
-        DepthImage getDepthImage();
-        ConfidenceImage getConfidenceImage();
-        UVImage getUVImage();
-        ColorImage getColorImage();
+        InputFrame getInputFrame();
 
         /*
          * all handIndices have to be 1 or 2

bbiwarg/InputProvider/IisuInputProvider.cs

@@ -4,7 +4,6 @@ using System.Diagnostics;
 using System.Runtime.InteropServices;
 using Iisu;
 using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.Images;
 
 namespace bbiwarg.InputProviders
 {
@@ -20,7 +19,6 @@ namespace bbiwarg.InputProviders
         private IParameterHandle<float> frameRate;
         private IParameterHandle<float> hfov;
         private IParameterHandle<float> vfov;
-        private int confidenceThreshold;
 
         //data
         private IDataHandle<int>[] handStatus = new IDataHandle<int>[2];
@@ -51,7 +49,6 @@ namespace bbiwarg.InputProviders
                 sourceIsMovie = false;
             }
             active = false;
-            confidenceThreshold = 100;
         }
 
         // control-methodes
@@ -75,7 +72,7 @@ namespace bbiwarg.InputProviders
             }
             else
             {
-                device.RegisterParameterHandle<int>("SOURCE.DEPTHSENSE.AmplitudeThreshold").Value = confidenceThreshold; // confidence-threshhold
+                device.RegisterParameterHandle<int>("SOURCE.DEPTHSENSE.AmplitudeThreshold").Value = 100; // confidence-threshhold
             }
 
             frameRate = device.RegisterParameterHandle<float>("SOURCE.FrameRate");
@@ -167,16 +164,6 @@ namespace bbiwarg.InputProviders
             return vfov.Value;
         }
 
-        public int getConfidenceThreshold()
-        {
-            return confidenceThreshold;
-        }
-
-        public void setConfidenceThreshold(int value)
-        {
-            confidenceThreshold = value;
-        }
-
         public bool isActive()
         {
             return active;
@@ -298,65 +285,40 @@ namespace bbiwarg.InputProviders
             return results;
         }
 
-
-        public DepthImage getDepthImage()
-        {
+        public InputFrame getInputFrame() {
+            //depthData
             Iisu.Data.IImageInfos imageInfos = depthImage.Value.ImageInfos;
             int width = (int)imageInfos.Width;
             int height = (int)imageInfos.Height;
             int numBytes = (int)imageInfos.BytesRaw;
-
             IntPtr imageData = depthImage.Value.Raw;
-
             short[] depthData = new short[width * height];
             Marshal.Copy(imageData, depthData, 0, width * height);
 
-            return new DepthImage(width, height, depthData);
-        }
-
-        public ConfidenceImage getConfidenceImage()
-        {
-            Iisu.Data.IImageInfos imageInfos = confidenceImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            int numBytes = (int)imageInfos.BytesRaw;
-
-            IntPtr imageData = confidenceImage.Value.Raw;
-
+            //confidenceData
+            imageInfos = confidenceImage.Value.ImageInfos;
+            numBytes = (int)imageInfos.BytesRaw;
+            imageData = confidenceImage.Value.Raw;
             short[] confidenceData = new short[width * height];
             Marshal.Copy(imageData, confidenceData, 0, width * height);
 
-            return new ConfidenceImage(width, height, confidenceData);
-        }
-
-        public UVImage getUVImage()
-        {
-            Iisu.Data.IImageInfos imageInfos = uvImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            int numBytes = (int)imageInfos.BytesRaw;
-
-            IntPtr imageData = uvImage.Value.Raw;
-
+            //uvData
+            imageInfos = uvImage.Value.ImageInfos;
+            numBytes = (int)imageInfos.BytesRaw;
+            imageData = uvImage.Value.Raw;
             float[] uvData = new float[2 * width * height];
             Marshal.Copy(imageData, uvData, 0, 2 * width * height);
 
-            return new UVImage(width, height, uvData);
-        }
-
-        public ColorImage getColorImage()
-        {
-            Iisu.Data.IImageInfos imageInfos = colorImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            int numBytes = (int)imageInfos.BytesRaw;
-
-            IntPtr imageData = colorImage.Value.Raw;
-
+            //colorData
+            imageInfos = colorImage.Value.ImageInfos;
+            int widthRawColor = (int)imageInfos.Width;
+            int heightRawColor = (int)imageInfos.Height;
+            numBytes = (int)imageInfos.BytesRaw;
+            imageData = colorImage.Value.Raw;
             byte[] colorData = new byte[numBytes];
             Marshal.Copy(imageData, colorData, 0, numBytes);
 
-            return new ColorImage(width, height, colorData);
+            return new InputFrame(width, height, widthRawColor, heightRawColor, depthData, confidenceData, uvData, colorData);
         }
 
         // other

bbiwarg/InputProvider/InputFrame.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Emgu.CV;
+using Emgu.CV.Structure;
+
+namespace bbiwarg.InputProviders
+{
+    class InputFrame
+    {
+        private int width;
+        private int height;
+        private int widthRawColor;
+        private int heightRawColor;
+        private Int16[] depthData;
+        private Int16[] confidenceData;
+        private float[] uvData;
+        private byte[] rawColorData;
+
+        public InputFrame(int width, int height, int widthRawColor, int heightRawColor, Int16[] depthData, Int16[] confidenceData, float[] uvData, byte[] rawColorData) {
+            this.width = width;
+            this.height = height;
+            this.widthRawColor = widthRawColor;
+            this.heightRawColor = heightRawColor;
+            this.depthData = depthData;
+            this.confidenceData = confidenceData;
+            this.uvData = uvData;
+            this.rawColorData = rawColorData;
+        }
+
+        public int getWidth() {
+            return width;
+        }
+
+        public int getHeight() {
+            return height;
+        }
+
+        public Int16 getDepthAt(int x, int y) {
+            return depthData[y * width + x];
+        }
+
+        public Int16 getConfidenceAt(int x, int y) {
+            return confidenceData[y * width + x];
+        }
+
+        public float getUAt(int x, int y)
+        {
+            return uvData[2 * (y * width + x) + 0];
+        }
+
+        public float getVAt(int x, int y)
+        {
+            return uvData[2 * (y * width + x) + 1];
+        }
+
+        public Color getColorAt(int x, int y) {
+            float u = getUAt(x, y);
+            float v = getVAt(x, y);
+
+            if (u < 0 || v < 0)
+                return Color.Black;
+
+            int xInColorImage = (int)(u * widthRawColor) % widthRawColor;
+            int yInColorImage = (int)(v * heightRawColor) % heightRawColor;
+            return getRawColorAt(x,y);
+        }
+
+        public Color getRawColorAt(int x, int y) {
+            int offset = 4 * (y * widthRawColor + x);
+            byte alpha = rawColorData[offset + 3];
+            byte red = rawColorData[offset + 2];
+            byte green = rawColorData[offset + 1];
+            byte blue = rawColorData[offset + 0];
+
+            return Color.FromArgb(alpha, red, green, blue);
+        }
+
+        public DepthImage getDepthImage() {
+            Image<Gray, Int16> image = new Image<Gray, Int16>(width, height);
+            for (int i = 0; i < depthData.Length; ++i)
+            {
+                image.Data[i / width, i % width, 0] = depthData[i];
+            }
+            return new DepthImage(width, height, image);
+        }
+    }
+}

bbiwarg/Main/OutputTest.cs → bbiwarg/MainBBWIWARG.cs

@@ -5,18 +5,17 @@ using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Graphics;
 using bbiwarg.InputProviders;
-using bbiwarg.VideoHandles;
 
-namespace bbiwarg.Main
+namespace bbiwarg
 {
-    class OutputTest
+    class MainBBIWARG
     {
         static void Main(string[] args)
         {
-            IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\touch\\3.skv");
+            IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\touch\\4.skv");
             VideoHandle videoHandle = new VideoHandle(inputProvider);
 
-            Output2D output = new Output2D(videoHandle);
+            OutputWindow output = new OutputWindow(videoHandle);
             output.Run(30);
         }
     }

bbiwarg/VideoHandle.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bbiwarg.InputProviders;
+
+namespace bbiwarg
+{
+    class VideoHandle
+    {
+        private IInputProvider inputProvider;
+        private InputFrame inputFrame;
+        private DepthImage depthImage;
+
+        public VideoHandle(IInputProvider inputProvider) {
+            this.inputProvider = inputProvider;
+
+            inputProvider.init();
+            inputProvider.start();
+            inputProvider.updateFrame();
+            inputFrame = inputProvider.getInputFrame();
+            depthImage = inputFrame.getDepthImage();
+
+        }
+
+        public void nextFrame()
+        {
+            if (inputProvider.isActive())
+            {
+                inputProvider.releaseFrame();
+                inputProvider.updateFrame();
+                inputFrame = inputProvider.getInputFrame();
+                depthImage = inputFrame.getDepthImage();
+            }
+            else
+            {
+                inputProvider.stop();
+            }
+        }
+
+        public int getWidth()
+        {
+            return inputFrame.getWidth();
+        }
+
+        public int getHeight()
+        {
+            return inputFrame.getHeight();
+        }
+
+        public DepthImage getDepthImage() {
+            return depthImage;
+        }
+    }
+}

bbiwarg/VideoHandle/VideoHandle.cs

@@ -1,246 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MathNet.Numerics.LinearAlgebra.Single;
-using bbiwarg.Images;
-using bbiwarg.InputProviders;
-using bbiwarg.Detectors;
-using bbiwarg.Helpers;
-
-namespace bbiwarg.VideoHandles
-{
-    class VideoHandle
-    {
-        private IInputProvider inputProvider;
-
-        private ForeFingerDetection foreFingerDetection;
-        private PalmDetection palmDetection;
-
-        private DepthImage depthImage;
-        private ConfidenceImage confidenceImage;
-        private UVImage uvImage;
-        private ColorImage colorImage;
-
-        private DepthImage handImage;
-        private Int16[] segmentationDepth;
-        private int[] histogram;
-
-        private float maxU;
-        private float maxV;
-
-        public VideoHandle(IInputProvider inputProvider) {
-            this.inputProvider = inputProvider;
-            inputProvider.init();
-            inputProvider.start();
-            inputProvider.updateFrame();
-
-            foreFingerDetection = new ForeFingerDetection(inputProvider, this);
-            palmDetection = new PalmDetection(inputProvider, this);
-            createImageData();
-
-            maxU = (float)(Math.Tan(inputProvider.getHFOV() / 2f));
-            maxV = (float)(Math.Tan(inputProvider.getVFOV() / 2f));
-        }
-
-        ~VideoHandle() 
-        {
-            //inputProvider.stop();
-        }
-
-        public void nextFrame() 
-        {
-            if (inputProvider.isActive())
-            {
-                inputProvider.releaseFrame();
-                inputProvider.updateFrame();
-                createImageData();
-            }
-            else {
-                inputProvider.stop();
-            }
-        }
-
-        public int getWidth()
-        {
-            return depthImage.getWidth();
-        }
-        public int getHeight()
-        {
-            return depthImage.getHeight();
-        }
-
-        private void createImageData() 
-        {
-            depthImage = inputProvider.getDepthImage();
-            handImage = inputProvider.getDepthImage();
-            confidenceImage = inputProvider.getConfidenceImage();
-            uvImage = inputProvider.getUVImage();
-            colorImage = inputProvider.getColorImage();
-
-            handImage.filterMedian(3);
-            Int16 minDepth = handImage.getMinDepth();
-            handImage.thresholdDepth(minDepth, minDepth + 200);
-
-            histogram = handImage.getSmoothedHistogram();
-            segmentationDepth = handImage.getSegmentationDepth();
-
-            handImage.thresholdBinary((Int16) (segmentationDepth[1]+minDepth));
-        }
-
-        public Int16[] getSegementationDepth()
-        {
-            return segmentationDepth;
-        }
-
-        public int[] getSmoothedHistogram()
-        {
-            return histogram;
-        }
-
-        public short getDepth(int x, int y)
-        {
-            return depthImage.getDepth(x, y);
-        }
-
-        public short getConfidence(int x, int y)
-        {
-            return confidenceImage.getConfidence(x, y);
-        }
-
-        public Color getColor(int x, int y)
-        {
-            float u = uvImage.getU(x, y);
-            float v = uvImage.getV(x, y);
-
-            if (u < 0 || v < 0)
-                return Color.Black;
-
-            int colorImageWidth = colorImage.getWidth();
-            int colorImageHeight = colorImage.getHeight();
-
-            int xInColorImage = (int)(u * colorImageWidth) % colorImageWidth;
-            int yInColorImage = (int)(v * colorImageHeight) % colorImageHeight;
-            return colorImage.getColor(xInColorImage, yInColorImage);
-        }
-
-        public DepthImage getHandImage()
-        {
-            return handImage;
-        }
-
-        public DetectionStatus[] getFingerStatus(uint handIndex)
-        {
-            return inputProvider.getFingerStatus(handIndex);
-        }
-
-        public Vector[] getFingerTipPositions3D(uint handIndex)
-        {
-            return inputProvider.getFingerTipPositions3D(handIndex);
-        }
-
-        public Vector getPalmPosition3D(uint handIndex)
-        {
-            return inputProvider.getPalmPosition3D(handIndex);
-        }
-
-        public Vector getPalmNormal3D(uint handIndex)
-        {
-            return inputProvider.getPalmNormal3D(handIndex);
-        }
-
-        public Vector getForearmPosition3D(uint handIndex)
-        {
-            return inputProvider.getForearmPosition3D(handIndex);
-        }
-
-        public Vector getForeFingerPosition3D(uint handIndex)
-        {
-            return foreFingerDetection.getForeFingerPosition3D(handIndex);
-        }
-
-        public List<Vector> getHandPoints()
-        {
-            return foreFingerDetection.getHandPoints();
-        }
-
-        public Palm getPalm(uint handIndex)
-        {
-            return palmDetection.getPalm(handIndex);
-        }
-
-        // TODO
-        public void createVertexArray(IntPtr vertexBuffer)
-        {
-            int width = depthImage.getWidth();
-            int height = depthImage.getHeight();
-            int confidenceThreshold = inputProvider.getConfidenceThreshold();
-
-            int index = 0;
-            for (int x = 0; x < width; x++)
-            {
-                for (int y = 0; y < height; y++)
-                {
-                    if (confidenceImage.getConfidence(x, y) > confidenceThreshold)
-                    {
-                        int depth = depthImage.getDepth(x, y);
-                        Color c = getColor(x, y);
-                        create3DVertexFrom2D(x, y, depth, c, index, vertexBuffer);
-
-                        index++;
-                    }
-                }
-            }
-        }
-
-        public Vector pixel2VertexPosition(Vector pixelPosition)
-        {
-            int width = depthImage.getWidth();
-            int height = depthImage.getHeight();
-
-            float convertedDepth = pixelPosition[2] / 1000f; // mm into m
-
-            float u = (pixelPosition[0] / (float)width - 0.5f) * 2f;
-            float v = ((1 - pixelPosition[1] / (float)height) - 0.5f) * 2f;
-
-            float relX = (u * maxU);
-            float relY = (v * maxV);
-
-            Vector result = new DenseVector(3);
-            result[2] = convertedDepth / (float)Math.Sqrt(1 + relX * relX + relY * relY);
-            result[0] = relX * result[2];
-            result[1] = relY * result[2];
-
-            return result;
-        }
-        private void create3DVertexFrom2D(float pixelX, float pixelY, int depth, Color c, int index, IntPtr vertexBuffer)
-        {
-            Vector pixelPosition = new DenseVector(3);
-            pixelPosition[0] = pixelX;
-            pixelPosition[1] = pixelY;
-            pixelPosition[2] = depth;
-            Vector vertexPosition = pixel2VertexPosition(pixelPosition);
-
-
-            int i4 = (3 * sizeof(float) + 4 * sizeof(byte)) / sizeof(float) * index;
-            int i16 = (3 * sizeof(float) + 4 * sizeof(byte)) * index;
-            unsafe
-            {
-                byte* vertexArrayB = (byte*)vertexBuffer.ToPointer();
-                float* vertexArrayF = (float*)vertexBuffer.ToPointer();
-
-                vertexArrayF[i4 + 0] = vertexPosition[0];
-                vertexArrayF[i4 + 1] = vertexPosition[1];
-                vertexArrayF[i4 + 2] = -vertexPosition[2];
-
-                vertexArrayB[i16 + 12] = c.R;
-                vertexArrayB[i16 + 13] = c.G;
-                vertexArrayB[i16 + 14] = c.B;
-                vertexArrayB[i16 + 15] = c.A;
-            }
-
-        }
-    }
-}

bbiwarg/bbiwarg.csproj

@@ -34,7 +34,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <PropertyGroup>
-    <StartupObject>bbiwarg.Main.OutputTest</StartupObject>
+    <StartupObject>bbiwarg.MainBBIWARG</StartupObject>
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="Emgu.CV, Version=2.4.2.1777, Culture=neutral, PublicKeyToken=7281126722ab4438, processorArchitecture=MSIL">
@@ -66,28 +66,14 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="Detectors\CollisionDetection.cs" />
-    <Compile Include="Detectors\ForeFingerDetection.cs" />
+    <Compile Include="DepthImage.cs" />
+    <Compile Include="Graphics\OutputWindow.cs" />
+    <Compile Include="InputProvider\InputFrame.cs" />
     <Compile Include="InputProvider\IInputProvider.cs" />
-    <Compile Include="Detectors\Palm.cs" />
-    <Compile Include="Detectors\PalmDetection.cs" />
-    <Compile Include="Images\UVImage.cs" />
-    <Compile Include="Images\ColorImage.cs" />
-    <Compile Include="Images\ConfidenceImage.cs" />
-    <Compile Include="Images\DepthImage.cs" />
     <Compile Include="InputProvider\IisuInputProvider.cs" />
-    <Compile Include="Helpers\VectorExtender.cs" />
-    <Compile Include="Helpers\VertexArray.cs" />
-    <Compile Include="VideoHandle\VideoHandle.cs" />
-    <Compile Include="Graphics\GraphicElements2D\IGraphicElement2D.cs" />
-    <Compile Include="Graphics\GraphicElements2D\Point2D.cs" />
-    <Compile Include="Graphics\Output2D.cs" />
-    <Compile Include="Graphics\GraphicElements3D\Rectangle3D.cs" />
-    <Compile Include="Graphics\GraphicElements3D\IGraphicElement3D.cs" />
-    <Compile Include="Graphics\GraphicElements3D\Point3D.cs" />
-    <Compile Include="Main\OutputTest.cs" />
-    <Compile Include="Graphics\Output3D.cs" />
+    <Compile Include="MainBBWIWARG.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="VideoHandle.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
@@ -103,7 +89,6 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
-  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.