Added opencv to DepthImage.

Added OutputDepthImage to draw DepthImage as texture.

bbiwarg/DataSource/CollisionDetection.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using MathNet.Numerics.LinearAlgebra.Single;
+
+namespace bbiwarg.DataSource
+{
+    class CollisionDetection
+    {
+        public static float getDistance(Vector origin, Vector normal, Vector fingerPoint)
+        {
+            return Math.Abs((fingerPoint - origin).DotProduct(normal));
+        }
+    }
+}

bbiwarg/DataSource/DepthImage.cs

@@ -4,6 +4,12 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
+using MathNet.Numerics.LinearAlgebra.Single;
+
+using System.Drawing;
+using Emgu.CV;
+using Emgu.CV.Structure;
+
 namespace bbiwarg.DataSource
 {
     /**
@@ -12,13 +18,16 @@ namespace bbiwarg.DataSource
     class DepthImage
     {
         private int width, height;
-        private short[] data;
+        private Image<Gray, Int16> image;
 
         public DepthImage(int width, int height, short[] data)
         {
             this.width = width;
             this.height = height;
-            this.data = data;
+            
+            image = new Image<Gray, Int16>(width, height);
+            for (int i = 0; i < data.Length; ++i)
+                setDepth(i % width, i / width, data[i]);
         }
 
         public int getWidth()
@@ -33,7 +42,51 @@ namespace bbiwarg.DataSource
 
         public short getDepth(int x, int y)
         {
-            return data[y * width + x];
+            return image.Data[y, x, 0];
+        }
+
+        private void setDepth(int x, int y, short depth)
+        {
+            image.Data[y, x, 0] = depth;
+        }
+
+        public void filterMedian(int filterSize)
+        {
+            image = image.SmoothMedian(3);
+        }
+
+        public void thresholdDepth(int min, int max)
+        {
+            image = image.ThresholdToZero(new Gray(min));
+            image = image.ThresholdBinaryInv(new Gray(max), new Gray(2000));
+         }
+
+        public void thresholdPosition(int minX, int maxX, int minY, int maxY)
+        {
+            for (int x = 0; x < width; ++x)
+            {
+                for (int y = 0; y < height; ++y)
+                {
+                    if (x < minX || x > maxX || y < minY || y > maxY)
+                        setDepth(x, y, 0);
+                }
+            }
+        }
+
+        public List<Vector> getRectPoints()
+        {
+            Image<Gray, Byte> tmp = image.Convert<Byte>(delegate(short s) { return (s == 2000) ? (byte) 1 : (byte) 0; });
+
+            Contour<Point> contours = tmp.FindContours();
+            MCvBox2D box = contours.GetMinAreaRect();
+            Seq<Point> points = contours.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE);
+
+            List<Vector> result = new List<Vector>();
+            foreach (Point p in points)
+            {
+                result.Add(new DenseVector(new float[]{p.X, p.Y}));
+            }
+            return result;
         }
     }
 }

bbiwarg/DataSource/IVideoHandle.cs

@@ -34,6 +34,7 @@ namespace bbiwarg.DataSource
         DetectionStatus[] getFingerStatus(uint handIndex);
         Vector[] getFingerTipPositions3D(uint handIndex);
         Vector getPalmPosition3D(uint handIndex);
+        Vector getPalmNormal3D(uint handIndex);
         Vector getForearmPosition3D(uint handIndex);
         Vector getForeFingerPosition3D(uint handIndex);
         List<Vector> getHandPoints();

bbiwarg/DataSource/VideoHandle.cs

@@ -113,6 +113,11 @@ namespace bbiwarg.DataSource
             return inputProvider.getPalmPosition3D(handIndex);
         }
 
+        public Vector getPalmNormal3D(uint handIndex)
+        {
+            return inputProvider.getPalmNormal3D(handIndex);
+        }
+
         public Vector getForearmPosition3D(uint handIndex)
         {
             return inputProvider.getForearmPosition3D(handIndex);

bbiwarg/Graphics/OutputDepthImage.cs

@@ -0,0 +1,145 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using OpenTK;
+using OpenTK.Graphics.OpenGL;
+using System.Drawing;
+using System.Diagnostics;
+
+using MathNet.Numerics.LinearAlgebra.Single;
+using bbiwarg.DataSource;
+
+namespace bbiwarg.Graphics
+{
+    class OutputDepthImage: GameWindow
+    {
+        private IInputProvider input;
+        private uint textureId;
+
+        public OutputDepthImage(IInputProvider input): base(3 * 320, 3 * 240)
+        {
+            this.input = input;
+        }
+
+        protected override void OnLoad(EventArgs e)
+        {
+            base.OnLoad(e);
+            Title = "OutputTest";
+            GL.ClearColor(Color.Black);
+
+            // transparency
+            GL.Enable(EnableCap.Blend);
+            GL.BlendEquation(BlendEquationMode.Max);
+
+            //Depth Test 
+            GL.Enable(EnableCap.DepthTest);
+
+            // Textures
+            GL.Enable(EnableCap.Texture2D);
+
+            GL.GenTextures(1, out textureId);
+            GL.BindTexture(TextureTarget.Texture2D, textureId);
+            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
+            GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
+        }
+
+        protected override void OnRenderFrame(FrameEventArgs e)
+        {
+            base.OnRenderFrame(e);
+            GL.Clear(ClearBufferMask.ColorBufferBit | ClearBufferMask.DepthBufferBit);
+            Matrix4 modelview = Matrix4.LookAt(Vector3.Zero, -Vector3.UnitZ, Vector3.UnitY);
+            GL.MatrixMode(MatrixMode.Modelview);
+            GL.LoadMatrix(ref modelview);
+
+            input.releaseFrame();
+            input.updateFrame();
+
+            Stopwatch sw = new Stopwatch();
+            sw.Start();
+
+            DepthImage depthImage = input.getDepthImage();
+            depthImage.filterMedian(3);
+
+            Vector palmOrigin = input.getPalmPosition2D(1);
+            int palmX = (int) palmOrigin.x();
+            int palmY = (int) palmOrigin.y();
+            int palmDepth = depthImage.getDepth(palmX, palmY);
+
+            int maxHandSize = 150; // mm Wieviel Pixel sind das?
+            List<Vector> points = new List<Vector>();
+            if (palmX != 0 && palmY != 0)
+            {
+                depthImage.thresholdDepth(palmDepth - 10, palmDepth + 30);
+                depthImage.thresholdPosition(palmX - maxHandSize / 2, palmX + maxHandSize / 2, palmY - maxHandSize / 2, palmY + maxHandSize / 2);
+                points = depthImage.getRectPoints();
+            }
+
+            short[] textureData = new short[3 * depthImage.getWidth() * depthImage.getHeight()];
+            int index = 0;
+            for (int y = 0; y < depthImage.getHeight(); ++y) {
+                for (int x = 0; x < depthImage.getWidth(); ++x) {
+                    // 0 --> 0 / 2000 -> Int16.MaxValue
+                    int d = depthImage.getDepth(x, y);
+                    short depth = (short) (d * Int16.MaxValue / 2000);
+                    textureData[index] = textureData[index + 1] = textureData[index + 2] = depth;
+                    index += 3;
+                }
+            }
+
+            // draw palm origin
+            index = 3 * ((int) palmOrigin.y() * depthImage.getWidth() + (int) palmOrigin.x());
+            textureData[index + 0] = Int16.MaxValue;
+            textureData[index + 1] = 0;
+            textureData[index + 2] = 0;
+
+            // draw rect points
+            foreach (Vector v in points)
+            {
+                int x = (int)v.x() % depthImage.getWidth();
+                int y = (int)v.y() % depthImage.getHeight();
+
+                for (int i = -1; i < 2; ++i)
+                {
+                    for (int j = -1; j < 2; ++j)
+                    {
+                        int px = Math.Min(depthImage.getWidth() - 1, Math.Max(0, x + i));
+                        int py = Math.Min(depthImage.getHeight() - 1, Math.Max(0, y + j));
+                        index = 3 * (py * depthImage.getWidth() + px);
+                        textureData[index + 0] = 0;
+                        textureData[index + 1] = 0;
+                        textureData[index + 2] = Int16.MaxValue;
+            
+                    }
+                }
+            }
+
+            GL.BindTexture(TextureTarget.Texture2D, textureId);
+            GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, depthImage.getWidth(), depthImage.getHeight(), 0,
+                          PixelFormat.Rgb, PixelType.Short, textureData);
+
+            float size_2 = 0.5f / 2.0f;
+            GL.Begin(PrimitiveType.Quads);
+            GL.TexCoord2(0.0, 0.0); GL.Vertex3(-size_2,  size_2, -0.5);
+            GL.TexCoord2(1.0, 0.0); GL.Vertex3( size_2,  size_2, -0.5);
+            GL.TexCoord2(1.0, 1.0); GL.Vertex3( size_2, -size_2, -0.5);
+            GL.TexCoord2(0.0, 1.0); GL.Vertex3(-size_2, -size_2, -0.5);
+            GL.End();
+
+            sw.Stop();
+            Console.WriteLine(sw.ElapsedMilliseconds);
+
+            SwapBuffers();
+        }
+
+        protected override void OnResize(EventArgs e)
+        {
+            base.OnResize(e);
+            GL.Viewport(ClientRectangle.X, ClientRectangle.Y, ClientRectangle.Width, ClientRectangle.Height);
+            Matrix4 projection = Matrix4.CreatePerspectiveFieldOfView((float)Math.PI / 3, Width / (float)Height, 0.01f, 3.0f);
+            GL.MatrixMode(MatrixMode.Projection);
+            GL.LoadMatrix(ref projection);
+        }
+    }
+}

bbiwarg/Main/OutputTest.cs

@@ -15,7 +15,7 @@ namespace bbiwarg.Main
             IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\1.skv");
             IVideoHandle videoHandle = new VideoHandle(inputProvider);
 
-            Output output = new Output(videoHandle);
+            OutputDepthImage output = new OutputDepthImage(inputProvider);
             output.Run(30);
         }
     }

bbiwarg/bbiwarg.csproj

@@ -13,7 +13,7 @@
     <FileAlignment>512</FileAlignment>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <PlatformTarget>AnyCPU</PlatformTarget>
+    <PlatformTarget>x86</PlatformTarget>
     <DebugSymbols>true</DebugSymbols>
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
@@ -37,6 +37,14 @@
     <StartupObject>bbiwarg.Main.OutputTest</StartupObject>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="Emgu.CV, Version=2.4.2.1777, Culture=neutral, PublicKeyToken=7281126722ab4438, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>lib\Emgu.CV.dll</HintPath>
+    </Reference>
+    <Reference Include="Emgu.Util, Version=2.4.2.1777, Culture=neutral, PublicKeyToken=7281126722ab4438, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>lib\Emgu.Util.dll</HintPath>
+    </Reference>
     <Reference Include="iisuNet, Version=3.0.0.0, Culture=neutral, processorArchitecture=x86">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>lib\iisuNet.dll</HintPath>
@@ -58,6 +66,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="DataSource\CollisionDetection.cs" />
     <Compile Include="DataSource\ForeFingerDetection.cs" />
     <Compile Include="DataSource\IInputProvider.cs" />
     <Compile Include="DataSource\Palm.cs" />
@@ -71,6 +80,7 @@
     <Compile Include="DataSource\VectorExtender.cs" />
     <Compile Include="DataSource\VertexArray.cs" />
     <Compile Include="DataSource\VideoHandle.cs" />
+    <Compile Include="Graphics\OutputDepthImage.cs" />
     <Compile Include="Graphics\Rectangle.cs" />
     <Compile Include="Graphics\IGraphicElement.cs" />
     <Compile Include="Graphics\Point.cs" />
@@ -81,6 +91,14 @@
   <ItemGroup>
     <None Include="App.config" />
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="opencv_core290.dll">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+    <Content Include="opencv_imgproc290.dll">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.