Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

Output uses IVideoHandle where possible.
VideoHandle creates VertexArray + Colors in mapped gpu buffer.

Conflicts:
	bbiwarg/DataSource/IVideoDataSource.cs
	bbiwarg/DataSource/IisuInputProvider.cs
	bbiwarg/Graphics/Output.cs

bbiwarg/DataSource/ForeFingerDetection.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MathNet.Numerics.LinearAlgebra.Single;
+
+using System.Drawing;
+
+
+namespace bbiwarg.DataSource
+{
+    class ForeFingerDetection
+    {
+        /*
+         * Returns the forefinger's position
+         * all fingers (except der forefinger) have to be inactive
+         * params:
+         *  handIndex: which hand's forefinger's position
+         */
+
+        public const int TOLERANCE_Z = 5;
+        public const int TOLERANCE_2D = 5;
+
+        private IVideoHandle source;
+        private HashSet<Vector> seenPoints;
+        private Vector palmPosition;
+        private Vector foreFingerPosition;
+        private float foreFingerDistance;
+
+
+
+        public ForeFingerDetection(IVideoHandle source)
+        {
+            this.source = source;
+        }
+
+        public Vector getForeFingerPosition3D(uint handIndex)
+        {
+            seenPoints = new HashSet<Vector>();
+            Vector palmPosition2D = source.getPalmPosition2D(handIndex);
+            palmPosition = new DenseVector(3);
+            palmPosition[0] = palmPosition2D[0];
+            palmPosition[1] = palmPosition2D[1];
+            palmPosition[2] = source.getDepth((int)palmPosition[0], (int)palmPosition[1]);
+
+            foreFingerPosition = palmPosition;
+            checkPoint(palmPosition);
+            return foreFingerPosition;
+        }
+
+        private void checkNeighbours(Vector point)
+        {
+            for (int y = -(int)Math.Min(TOLERANCE_2D, point[1]); y <= TOLERANCE_2D && point[1] + y < source.getHeight(); y++)
+            {
+                for (int x = -(int)Math.Min(TOLERANCE_2D, point[0]); x <= TOLERANCE_2D && point[0] + x < source.getWidth(); x++)
+                {
+                    Vector currentPoint = new DenseVector(3);
+                    currentPoint[0] = point[0] + x;
+                    currentPoint[1] = point[1] + y;
+                    currentPoint[2] = source.getDepth((int) currentPoint[0], (int) currentPoint[1]);
+                    if(Math.Abs(currentPoint[2]-point[2]) <= TOLERANCE_Z)
+                      checkPoint(currentPoint);
+                }
+            }
+        }
+        private void checkPoint(Vector point)
+        {
+            if (!seenPoints.Contains<Vector>(point))
+            {
+                seenPoints.Add(point);
+                Vector distanceVector = new DenseVector(3);
+                palmPosition.Subtract(point, distanceVector);
+                float currentPointDistance = distanceVector.Norm(2);
+                if (currentPointDistance >= foreFingerDistance)
+                {
+                    foreFingerPosition = point;
+                    foreFingerDistance = currentPointDistance;
+                    checkNeighbours(point);
+                }
+            }
+        }
+    }
+}

bbiwarg/DataSource/IInputProvider.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using MathNet.Numerics.LinearAlgebra.Single;
+
+namespace bbiwarg.DataSource
+{
+    interface IInputProvider
+    {
+        void init();
+        void start();
+        void stop();
+        void updateFrame();
+        void releaseFrame();
+        float getFrameRate();
+        float getHFOV();
+        float getVFOV();
+        bool isActive();
+
+        DepthImage getDepthImage();
+        ConfidenceImage getConfidenceImage();
+        UVImage getUVImage();
+        ColorImage getColorImage();
+
+        /*
+         * all handIndices have to be 1 or 2
+         */
+        bool isHandOpen(uint handIndex);
+        Vector getPalmPosition3D(uint handIndex);
+        Vector getPalmPosition2D(uint handIndex);
+        Vector getTipPosition3D(uint handIndex);
+        Vector getForearmPosition3D(uint handIndex);
+        Vector getPalmNormal3D(uint handIndex);
+        DetectionStatus[] getFingerStatus(uint handIndex);
+        DetectionStatus getHandStatus(uint handIndex);
+        Vector[] getFingerTipPositions3D(uint handIndex);
+        Vector[] getFingerTipPositions2D(uint handIndex);
+        HandSide getHandSide(uint handIndex);
+    }
+}

bbiwarg/DataSource/IVideoHandle.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Drawing;
+using MathNet.Numerics.LinearAlgebra.Single;
+
+namespace bbiwarg.DataSource
+{
+    public enum DetectionStatus
+    {
+        Inactive,
+        Detected,
+        Tracked
+    }
+
+    public enum HandSide
+    {
+        Unknown,
+        Left,
+        Right
+    }
+
+    interface IVideoHandle
+    {
+        void nextFrame();
+
+        short getDepth(int x, int y);
+        short getConfidence(int x, int y);
+        Color getColor(int x, int y);
+
+        // TODO: implement properly
+        void createVertexArray(IntPtr vertexBuffer);
+        //convert2Dto3D(float x, float y, int depth)
+
+        Vector getPalmPosition2D(uint handIndex);
+        Vector getPalmPosition3D(uint handIndex);
+        int getWidth();
+        int getHeight();
+    }
+}

bbiwarg/DataSource/IisuDataSource.cs → bbiwarg/DataSource/IisuInputProvider.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Drawing;
 using System.Diagnostics;
 using System.Runtime.InteropServices;
@@ -8,68 +8,64 @@ using bbiwarg.Graphics;
 
 namespace bbiwarg.DataSource
 {
-    class IIsuDataSource: IVideoDataSource
+    class IisuInputProvider : IInputProvider
     {
         private IHandle handle;
         private IDevice device;
         private string moviePath;
-        bool active;
+        private bool active;
+        private bool sourceIsMovie;
 
-        // parameters
+        //parameters
         private IParameterHandle<float> frameRate;
+        private IParameterHandle<float> hfov;
+        private IParameterHandle<float> vfov;
 
-        // data
+        //data
+        private IDataHandle<int>[] handStatus = new IDataHandle<int>[2];
         private IDataHandle<bool>[] handOpen = new IDataHandle<bool>[2];
+        private IDataHandle<int>[] handSides = new IDataHandle<int>[2];
+        private IDataHandle<int[]>[] fingerStatus = new IDataHandle<int[]>[2];
         private IDataHandle<Iisu.Data.Vector3>[] palmPositions3D = new IDataHandle<Iisu.Data.Vector3>[2];
         private IDataHandle<Iisu.Data.Vector2>[] palmPositions2D = new IDataHandle<Iisu.Data.Vector2>[2];
         private IDataHandle<Iisu.Data.Vector3>[] tipPositions3D = new IDataHandle<Iisu.Data.Vector3>[2];
         private IDataHandle<Iisu.Data.Vector3>[] forearmPositions3D = new IDataHandle<Iisu.Data.Vector3>[2];
         private IDataHandle<Iisu.Data.Vector3>[] palmNormals3D = new IDataHandle<Iisu.Data.Vector3>[2];
-        private IDataHandle<int[]>[] fingerStatus = new IDataHandle<int[]>[2];
-        private IDataHandle<int>[] handStatus = new IDataHandle<int>[2];
         private IDataHandle<Iisu.Data.Vector3[]>[] fingerTipPositions3D = new IDataHandle<Iisu.Data.Vector3[]>[2];
         private IDataHandle<Iisu.Data.Vector2[]>[] fingerTipPositions2D = new IDataHandle<Iisu.Data.Vector2[]>[2];
-        private IDataHandle<int>[] handSides = new IDataHandle<int>[2];
         private IDataHandle<Iisu.Data.IImageData> depthImage;
-        private IDataHandle<Iisu.Data.IImageData> colorImage;
         private IDataHandle<Iisu.Data.IImageData> confidenceImage;
         private IDataHandle<Iisu.Data.IImageData> uvImage;
-        private IParameterHandle<float> hfov;
-        private IParameterHandle<float> vfov;
+        private IDataHandle<Iisu.Data.IImageData> colorImage;
 
-        private ImageData currentImage;
-        private float maxU;
-        private float maxV;
-        private int width;
-        private int height;
-
-        //TODO
-        private IntPtr vertexBuffer;
-
-        /*
-         * Creates an Iisu data source.
-         * params:
-         *  moviePath: path to movie to be used as source
-         *             if empty the camera is used
-         */
-        public IIsuDataSource(string moviePath = "")
+        public IisuInputProvider(String moviePath = "") 
         {
-            this.moviePath = moviePath;
+            if (moviePath.Length != 0)
+            {
+                this.moviePath = moviePath;
+                sourceIsMovie = true;
+            }
+            else
+            {
+                sourceIsMovie = false;
+            }
             active = false;
         }
 
+        // control-methodes
+
         public void init()
         {
             handle = Iisu.Iisu.Context.CreateHandle();
-            
+
             IDeviceConfiguration conf = handle.CreateDeviceConfiguration();
-            if (moviePath.Length != 0)
+            if (sourceIsMovie)
                 conf.MoviePath = moviePath;
 
             device = handle.InitializeDevice(conf);
 
             // parameters
-            if (moviePath.Length != 0)
+            if (sourceIsMovie)
                 device.RegisterParameterHandle<int>("SOURCE.MOVIE.PlayMode").Value = 0; // playMode = once
             else
                 device.RegisterParameterHandle<int>("SOURCE.DEPTHSENSE.AmplitudeThreshold").Value = 100; // confidence-threshhold
@@ -77,21 +73,23 @@ namespace bbiwarg.DataSource
             frameRate = device.RegisterParameterHandle<float>("SOURCE.FrameRate");
             hfov = device.RegisterParameterHandle<float>("SOURCE.CAMERA.DEPTH.HFOV");
             vfov = device.RegisterParameterHandle<float>("SOURCE.CAMERA.DEPTH.VFOV");
-            maxU = (float)Math.Tan(hfov.Value / 2f);
-            maxV = (float)Math.Tan(vfov.Value / 2f);
 
             // events
             device.EventManager.RegisterEventListener("DEVICE.Status", new Iisu.EventDelegates.Device.Status(onDeviceStatusChanged));
-            
+
             // data
-            depthImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.DEPTH.Image");
-            colorImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.COLOR.Image");
-            confidenceImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.CONFIDENCE.Image");
-            uvImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.COLOR.REGISTRATION.UV.Image");
+            handStatus[0] = device.RegisterDataHandle<int>("CI.HAND1.Status");
+            handStatus[1] = device.RegisterDataHandle<int>("CI.HAND2.Status");
 
             handOpen[0] = device.RegisterDataHandle<bool>("CI.HAND1.IsOpen");
             handOpen[1] = device.RegisterDataHandle<bool>("CI.HAND2.IsOpen");
 
+            handSides[0] = device.RegisterDataHandle<int>("CI.HAND1.Side");
+            handSides[1] = device.RegisterDataHandle<int>("CI.HAND1.Side");
+            
+            fingerStatus[0] = device.RegisterDataHandle<int[]>("CI.HAND1.FingerStatus");
+            fingerStatus[1] = device.RegisterDataHandle<int[]>("CI.HAND2.FingerStatus");
+
             palmPositions3D[0] = device.RegisterDataHandle<Iisu.Data.Vector3>("CI.HAND1.PalmPosition3D");
             palmPositions3D[1] = device.RegisterDataHandle<Iisu.Data.Vector3>("CI.HAND2.PalmPosition3D");
 
@@ -107,20 +105,16 @@ namespace bbiwarg.DataSource
             palmNormals3D[0] = device.RegisterDataHandle<Iisu.Data.Vector3>("CI.HAND1.PalmNormal3D");
             palmNormals3D[1] = device.RegisterDataHandle<Iisu.Data.Vector3>("CI.HAND2.PalmNormal3D");
 
-            fingerStatus[0] = device.RegisterDataHandle<int[]>("CI.HAND1.FingerStatus");
-            fingerStatus[1] = device.RegisterDataHandle<int[]>("CI.HAND2.FingerStatus");
-
-            handStatus[0] = device.RegisterDataHandle<int>("CI.HAND1.Status");
-            handStatus[1] = device.RegisterDataHandle<int>("CI.HAND2.Status");
-
             fingerTipPositions3D[0] = device.RegisterDataHandle<Iisu.Data.Vector3[]>("CI.HAND1.FingerTipPositions3D");
             fingerTipPositions3D[1] = device.RegisterDataHandle<Iisu.Data.Vector3[]>("CI.HAND2.FingerTipPositions3D");
-            
+
             fingerTipPositions2D[0] = device.RegisterDataHandle<Iisu.Data.Vector2[]>("CI.HAND1.FingerTipPositions2D");
             fingerTipPositions2D[1] = device.RegisterDataHandle<Iisu.Data.Vector2[]>("CI.HAND2.FingerTipPositions2D");
-
-            handSides[0] = device.RegisterDataHandle<int>("CI.HAND1.Side");
-            handSides[1] = device.RegisterDataHandle<int>("CI.HAND1.Side");
+            
+            depthImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.DEPTH.Image");
+            colorImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.COLOR.Image");
+            confidenceImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.CONFIDENCE.Image");
+            uvImage = device.RegisterDataHandle<Iisu.Data.IImageData>("SOURCE.CAMERA.COLOR.REGISTRATION.UV.Image");
         }
 
         private void onDeviceStatusChanged(string eventName, DeviceStatus status)
@@ -131,8 +125,7 @@ namespace bbiwarg.DataSource
         public void start()
         {
             device.Start();
-            updateFrame();
-         }
+        }
 
         public void stop()
         {
@@ -142,7 +135,6 @@ namespace bbiwarg.DataSource
         public void updateFrame()
         {
             device.UpdateFrame(true);
-            createImageData();
         }
 
         public void releaseFrame()
@@ -150,99 +142,48 @@ namespace bbiwarg.DataSource
             device.ReleaseFrame();
         }
 
-        public bool isActive()
-        {
-            return active;
-        }
+        // getParameter-methodes
 
-        public int getFrameRate()
+        public float getFrameRate()
         {
-            return (int) frameRate.Value;
-        }
-
-        public float getHFOV() {
-            return hfov.Value;
-        }
-
-        public float getVFOV() {
-            return vfov.Value;
-        }
-
-        public DepthImage getDepthImage()
-        {
-            Iisu.Data.IImageInfos imageInfos = depthImage.Value.ImageInfos;
-            int width = (int) imageInfos.Width;
-            int height = (int) imageInfos.Height;
-            int numBytes = (int) imageInfos.BytesRaw;
-
-            IntPtr imageData = depthImage.Value.Raw;
-            
-            short[] depthData = new short[width * height];
-            Marshal.Copy(imageData, depthData, 0, width * height);
-
-            return new DepthImage(width, height, depthData);
+            return frameRate.Value;
         }
 
-        public ColorImage getColorImage()
+        public float getHFOV()
         {
-            Iisu.Data.IImageInfos imageInfos = colorImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            int numBytes = (int)imageInfos.BytesRaw;
-
-            IntPtr imageData = colorImage.Value.Raw;
-
-            byte[] colorData = new byte[numBytes];
-            Marshal.Copy(imageData, colorData, 0, numBytes);
-
-            return new ColorImage(width, height, colorData);
+            return hfov.Value;
         }
 
-        public ConfidenceImage getConfidenceImage()
+        public float getVFOV()
         {
-            Iisu.Data.IImageInfos imageInfos = confidenceImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            int numBytes = (int)imageInfos.BytesRaw;
-
-            IntPtr imageData = confidenceImage.Value.Raw;
-
-            short[] confidenceData = new short[width * height];
-            Marshal.Copy(imageData, confidenceData, 0, width * height);
-
-            return new ConfidenceImage(width, height, confidenceData);
+            return vfov.Value;
         }
 
-        public UVImage getUVImage()
-        {
-            Iisu.Data.IImageInfos imageInfos = uvImage.Value.ImageInfos;
-            int width = (int)imageInfos.Width;
-            int height = (int)imageInfos.Height;
-            int numBytes = (int)imageInfos.BytesRaw;
-
-            IntPtr imageData = uvImage.Value.Raw;
-
-            float[] uvData = new float[2 * width * height];
-            Marshal.Copy(imageData, uvData, 0, 2 * width * height);
-
-            return new UVImage(width, height, uvData);
-        }
+        // getData-methodes
 
-        public ImageData getImageData()
+        public bool isActive()
         {
-            return currentImage;
-        }
-
-        private void createImageData() {
-            currentImage = new ImageData(getDepthImage(), getConfidenceImage(), getColorImage(), getUVImage());
-            width = currentImage.getWidth();
-            height = currentImage.getHeight();
+            return active;
         }
 
-        private void checkHandIndex(uint handIndex) 
+        public DetectionStatus getHandStatus(uint handIndex)
         {
-            if (handIndex < 1 || handIndex > 2)
-                throw new ArgumentOutOfRangeException("handIndex is out of range [0,1]");
+            checkHandIndex(handIndex);
+            int status = handStatus[handIndex - 1].Value;
+            DetectionStatus result = new DetectionStatus();
+            switch (status)
+            {
+                case 0:
+                    result = DetectionStatus.Inactive;
+                    break;
+                case 1:
+                    result = DetectionStatus.Detected;
+                    break;
+                case 2:
+                    result = DetectionStatus.Tracked;
+                    break;
+            }
+            return result;
         }
 
         public bool isHandOpen(uint handIndex)
@@ -251,34 +192,19 @@ namespace bbiwarg.DataSource
             return handOpen[handIndex - 1].Value;
         }
 
-        public Vector getPalmPosition3D(uint handIndex)
-        {
-            checkHandIndex(handIndex);
-            return new DenseVector(palmPositions3D[handIndex - 1].Value.ToArray());
-        }
-
-        public Vector getPalmPosition2D(uint handIndex)
-        {
-            checkHandIndex(handIndex);
-            return new DenseVector(palmPositions2D[handIndex - 1].Value.ToArray());
-        }
-
-        public Vector getTipPosition3D(uint handIndex)
-        {
-            checkHandIndex(handIndex);
-            return new DenseVector(tipPositions3D[handIndex - 1].Value.ToArray());
-        }
-
-        public Vector getForearmPosition3D(uint handIndex)
-        {
-            checkHandIndex(handIndex);
-            return new DenseVector(forearmPositions3D[handIndex - 1].Value.ToArray());
-        }
-
-        public Vector getPalmNormal3D(uint handIndex)
+        public HandSide getHandSide(uint handIndex)
         {
             checkHandIndex(handIndex);
-            return new DenseVector(palmNormals3D[handIndex - 1].Value.ToArray());
+            int side = handSides[handIndex - 1].Value;
+            switch (side)
+            {
+                case 1:
+                    return HandSide.Left;
+                case 2:
+                    return HandSide.Right;
+                default:
+                    return HandSide.Unknown;
+            }
         }
 
         public DetectionStatus[] getFingerStatus(uint handIndex)
@@ -304,24 +230,34 @@ namespace bbiwarg.DataSource
             return result;
         }
 
-        public DetectionStatus getHandStatus(uint handIndex)
+        public Vector getPalmPosition3D(uint handIndex)
         {
             checkHandIndex(handIndex);
-            int status = handStatus[handIndex - 1].Value;
-            DetectionStatus result = new DetectionStatus();
-            switch (status)
-            {
-                case 0:
-                    result = DetectionStatus.Inactive;
-                    break;
-                case 1:
-                    result = DetectionStatus.Detected;
-                    break;
-                case 2:
-                    result = DetectionStatus.Tracked;
-                    break;
-            }
-            return result;
+            return new DenseVector(swapXZYtoXYZ(palmPositions3D[handIndex - 1].Value.ToArray()));
+        }
+
+        public Vector getPalmPosition2D(uint handIndex)
+        {
+            checkHandIndex(handIndex);
+            return new DenseVector(palmPositions2D[handIndex - 1].Value.ToArray());
+        }
+
+        public Vector getTipPosition3D(uint handIndex)
+        {
+            checkHandIndex(handIndex);
+            return new DenseVector(swapXZYtoXYZ(tipPositions3D[handIndex - 1].Value.ToArray()));
+        }
+
+        public Vector getForearmPosition3D(uint handIndex)
+        {
+            checkHandIndex(handIndex);
+            return new DenseVector(swapXZYtoXYZ(forearmPositions3D[handIndex - 1].Value.ToArray()));
+        }
+
+        public Vector getPalmNormal3D(uint handIndex)
+        {
+            checkHandIndex(handIndex);
+            return new DenseVector(swapXZYtoXYZ(palmNormals3D[handIndex - 1].Value.ToArray()));
         }
 
         public Vector[] getFingerTipPositions3D(uint handIndex)
@@ -330,7 +266,7 @@ namespace bbiwarg.DataSource
             Iisu.Data.Vector3[] positions = fingerTipPositions3D[handIndex - 1].Value;
             Vector[] results = new DenseVector[positions.Length];
             for (int i = 0; i < positions.Length; ++i)
-                results[i] = new DenseVector(positions[i].ToArray());
+                results[i] = new DenseVector(swapXZYtoXYZ(positions[i].ToArray()));
             return results;
         }
 
@@ -344,70 +280,82 @@ namespace bbiwarg.DataSource
             return results;
         }
 
-        public HandSide getHandSide(uint handIndex)
+
+        public DepthImage getDepthImage()
         {
-            checkHandIndex(handIndex);
-            int side = handSides[handIndex - 1].Value;
-            switch (side) 
-            {
-                case 1:
-                    return HandSide.Left;
-                case 2:
-                    return HandSide.Right;
-                default:
-                    return HandSide.Unknown;
-            }
+            Iisu.Data.IImageInfos imageInfos = depthImage.Value.ImageInfos;
+            int width = (int)imageInfos.Width;
+            int height = (int)imageInfos.Height;
+            int numBytes = (int)imageInfos.BytesRaw;
+
+            IntPtr imageData = depthImage.Value.Raw;
+
+            short[] depthData = new short[width * height];
+            Marshal.Copy(imageData, depthData, 0, width * height);
+
+            return new DepthImage(width, height, depthData);
         }
 
-        public void createVertexArray()
+        public ConfidenceImage getConfidenceImage()
         {
-            int index = 0;
-            for (int x = 0; x < width; x++)
-            {
-                for (int y = 0; y < height; y++)
-                {
-                    int depth = currentImage.getDepth(x, y);
-                    create3DVertexFrom2D(x, y, depth, index, currentImage.getColor(x, y));
-                    index++;
-                }
-            }
+            Iisu.Data.IImageInfos imageInfos = confidenceImage.Value.ImageInfos;
+            int width = (int)imageInfos.Width;
+            int height = (int)imageInfos.Height;
+            int numBytes = (int)imageInfos.BytesRaw;
+
+            IntPtr imageData = confidenceImage.Value.Raw;
+
+            short[] confidenceData = new short[width * height];
+            Marshal.Copy(imageData, confidenceData, 0, width * height);
+
+            return new ConfidenceImage(width, height, confidenceData);
         }
 
-        private void create3DVertexFrom2D(float pixelX, float pixelY, int depth, int index, Color c)
+        public UVImage getUVImage()
         {
-            float convertedDepth = depth / 1000f; // mm into m
+            Iisu.Data.IImageInfos imageInfos = uvImage.Value.ImageInfos;
+            int width = (int)imageInfos.Width;
+            int height = (int)imageInfos.Height;
+            int numBytes = (int)imageInfos.BytesRaw;
 
-            float u = (pixelX / (float)width - 0.5f) * 2f;
-            float v = ((1 - pixelY / (float)height) - 0.5f) * 2f;
+            IntPtr imageData = uvImage.Value.Raw;
 
-            float relX = (u * maxU);
-            float relY = (v * maxV);
+            float[] uvData = new float[2 * width * height];
+            Marshal.Copy(imageData, uvData, 0, 2 * width * height);
 
-            float z = convertedDepth / (float)Math.Sqrt(1 + relX * relX + relY * relY);
-            float x = relX * z;
-            float y = relY * z;
+            return new UVImage(width, height, uvData);
+        }
 
-            int i4 = (3 * sizeof(float) + 4 * sizeof(byte)) / sizeof(float) * index;
-            int i16 = (3 * sizeof(float) + 4 * sizeof(byte)) * index;
-            unsafe
-            {
-                byte* vertexArrayB = (byte*)vertexBuffer.ToPointer();
-                float* vertexArrayF = (float*)vertexBuffer.ToPointer();
+        public ColorImage getColorImage()
+        {
+            Iisu.Data.IImageInfos imageInfos = colorImage.Value.ImageInfos;
+            int width = (int)imageInfos.Width;
+            int height = (int)imageInfos.Height;
+            int numBytes = (int)imageInfos.BytesRaw;
 
-                vertexArrayF[i4 + 0] = x;
-                vertexArrayF[i4 + 1] = y;
-                vertexArrayF[i4 + 2] = -z;
+            IntPtr imageData = colorImage.Value.Raw;
 
-                vertexArrayB[i16 + 12] = c.R;
-                vertexArrayB[i16 + 13] = c.G;
-                vertexArrayB[i16 + 14] = c.B; 
-                vertexArrayB[i16 + 15] = c.A; 
-            }
+            byte[] colorData = new byte[numBytes];
+            Marshal.Copy(imageData, colorData, 0, numBytes);
+
+            return new ColorImage(width, height, colorData);
+        }
+
+        // other
+
+        private void checkHandIndex(uint handIndex)
+        {
+            if (handIndex < 1 || handIndex > 2)
+                throw new ArgumentOutOfRangeException("handIndex is out of range [0,1]");
         }
 
-        public void setVertexBuffer(IntPtr vertexBuffer)
+        private float[] swapXZYtoXYZ(float[] vector) 
         {
-            this.vertexBuffer = vertexBuffer;
+            float dummy = vector[1];
+            vector[1] = vector[2];
+            vector[2] = dummy;
+            return vector;
         }
+
     }
-}
+}

bbiwarg/DataSource/ImageData.cs

@@ -1,64 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Drawing;
-
-namespace bbiwarg.DataSource
-{
-    /**
-     * Gives access to all important data in depth image coordinates.
-     */
-    class ImageData
-    {
-        private DepthImage depthImage;
-        private ConfidenceImage confidenceImage;
-        private ColorImage colorImage;
-        private UVImage uvImage;
-
-        public ImageData(DepthImage depthImage, ConfidenceImage confidenceImage, ColorImage colorImage, UVImage uvImage)
-        {
-            this.depthImage = depthImage;
-            this.confidenceImage = confidenceImage;
-            this.colorImage = colorImage;
-            this.uvImage = uvImage;
-        }
-
-        public int getWidth()
-        {
-            return depthImage.getWidth();
-        }
-
-        public int getHeight()
-        {
-            return depthImage.getHeight();
-        }
-
-        public short getDepth(int x, int y)
-        {
-            return depthImage.getDepth(x, y);
-        }
-
-        public short getConfidence(int x, int y)
-        {
-            return confidenceImage.getConfidence(x, y);
-        }
-
-        public Color getColor(int x, int y)
-        {
-            float u = uvImage.getU(x, y);
-            float v = uvImage.getV(x, y);
-
-            if (u < 0 || v < 0)
-                return Color.Black;
-
-            int colorImageWidth = colorImage.getWidth();
-            int colorImageHeight = colorImage.getHeight();
-
-            int xInColorImage = (int) (u * colorImageWidth) % colorImageWidth;
-            int yInColorImage = (int) (v * colorImageHeight) % colorImageHeight;
-            return colorImage.getColor(xInColorImage, yInColorImage);
-        }
-    }
-}

bbiwarg/DataSource/VideoHandle.cs

@@ -0,0 +1,157 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MathNet.Numerics.LinearAlgebra.Single;
+using bbiwarg.Graphics;
+
+namespace bbiwarg.DataSource
+{
+    class VideoHandle : IVideoHandle
+    {
+        private IInputProvider inputProvider;
+        private DepthImage depthImage;
+        private ConfidenceImage confidenceImage;
+        private UVImage uvImage;
+        private ColorImage colorImage;
+
+        private float maxU;
+        private float maxV;
+
+        public VideoHandle(IInputProvider inputProvider) {
+            this.inputProvider = inputProvider;
+            inputProvider.init();
+            inputProvider.start();
+            inputProvider.updateFrame();
+            createImageData();
+
+            maxU = (float)(Math.Tan(inputProvider.getHFOV() / 2f));
+            maxV = (float)(Math.Tan(inputProvider.getVFOV() / 2f));
+        }
+
+        ~VideoHandle() 
+        {
+            inputProvider.stop();
+        }
+
+        public void nextFrame() 
+        {
+            inputProvider.releaseFrame();
+            inputProvider.updateFrame();
+            createImageData();
+        }
+
+        private void createImageData() 
+        {
+            depthImage = inputProvider.getDepthImage();
+            confidenceImage = inputProvider.getConfidenceImage();
+            uvImage = inputProvider.getUVImage();
+            colorImage = inputProvider.getColorImage();
+        }
+
+        public short getDepth(int x, int y)
+        {
+            return depthImage.getDepth(x, y);
+        }
+
+        public short getConfidence(int x, int y)
+        {
+            return confidenceImage.getConfidence(x, y);
+        }
+
+        public Color getColor(int x, int y)
+        {
+            float u = uvImage.getU(x, y);
+            float v = uvImage.getV(x, y);
+
+            if (u < 0 || v < 0)
+                return Color.Black;
+
+            int colorImageWidth = colorImage.getWidth();
+            int colorImageHeight = colorImage.getHeight();
+
+            int xInColorImage = (int)(u * colorImageWidth) % colorImageWidth;
+            int yInColorImage = (int)(v * colorImageHeight) % colorImageHeight;
+            return colorImage.getColor(xInColorImage, yInColorImage);
+        }
+
+        public Vector getPalmPosition2D(uint handIndex)
+        {
+            //improved palm recognition or just return the value provided by inputprovider
+            return inputProvider.getPalmPosition2D(handIndex);
+        }
+
+        public Vector getPalmPosition3D(uint handIndex)
+        {
+            //improved palm recognition or just return the value provided by inputprovider
+            return inputProvider.getPalmPosition3D(handIndex);
+        }
+
+        // TODO
+        public void createVertexArray(IntPtr vertexBuffer)
+        {
+            int width = depthImage.getWidth();
+            int height = depthImage.getHeight();
+
+            int index = 0;
+            for (int x = 0; x < width; x++)
+            {
+                for (int y = 0; y < height; y++)
+                {
+                    int depth = depthImage.getDepth(x, y);
+                    Color c = getColor(x, y);
+                    create3DVertexFrom2D(x, y, depth, c, index, vertexBuffer);
+                    
+                    index++;
+                }
+            }
+        }
+
+        private void create3DVertexFrom2D(float pixelX, float pixelY, int depth, Color c, int index, IntPtr vertexBuffer)
+        {
+            int width = depthImage.getWidth();
+            int height = depthImage.getHeight();
+
+            float convertedDepth = depth / 1000f; // mm into m
+
+            float u = (pixelX / (float)width - 0.5f) * 2f;
+            float v = ((1 - pixelY / (float)height) - 0.5f) * 2f;
+
+            float relX = (u * maxU);
+            float relY = (v * maxV);
+
+            float z = convertedDepth / (float)Math.Sqrt(1 + relX * relX + relY * relY);
+            float x = relX * z;
+            float y = relY * z;
+
+            int i4 = (3 * sizeof(float) + 4 * sizeof(byte)) / sizeof(float) * index;
+            int i16 = (3 * sizeof(float) + 4 * sizeof(byte)) * index;
+            unsafe
+            {
+                byte* vertexArrayB = (byte*)vertexBuffer.ToPointer();
+                float* vertexArrayF = (float*)vertexBuffer.ToPointer();
+
+                vertexArrayF[i4 + 0] = x;
+                vertexArrayF[i4 + 1] = y;
+                vertexArrayF[i4 + 2] = -z;
+
+                vertexArrayB[i16 + 12] = c.R;
+                vertexArrayB[i16 + 13] = c.G;
+                vertexArrayB[i16 + 14] = c.B;
+                vertexArrayB[i16 + 15] = c.A;
+            }
+
+        }
+
+        public int getWidth()
+        {
+            return depthImage.getWidth();
+        }
+        public int getHeight()
+        {
+            return depthImage.getHeight();
+        }
+    }
+}

bbiwarg/Graphics/Output.cs

@@ -15,13 +15,15 @@ namespace bbiwarg.Graphics
 {
     class Output : GameWindow
     {
-        private IVideoDataSource source;
+        private IInputProvider inputProvider;
+        private IVideoHandle videoHandle;
         
         private uint imageBufferId, pointBufferId;
 
-        public Output(IVideoDataSource source)
+        public Output(IInputProvider inputProvider, IVideoHandle videoHandle)
         {
-            this.source = source;
+            this.inputProvider = inputProvider;
+            this.videoHandle = videoHandle;
         }
 
         protected override void OnLoad(EventArgs e)
@@ -41,8 +43,7 @@ namespace bbiwarg.Graphics
             GL.MatrixMode(MatrixMode.Modelview);
             GL.LoadMatrix(ref modelview);
 
-            source.releaseFrame();
-            source.updateFrame();
+            videoHandle.nextFrame();
 
             Stopwatch sw = new Stopwatch();
             sw.Start();
@@ -51,23 +52,19 @@ namespace bbiwarg.Graphics
             GL.EnableClientState(ArrayCap.ColorArray);
 
             GL.BindBuffer(BufferTarget.ArrayBuffer, imageBufferId);
-            source.setVertexBuffer(GL.MapBuffer(BufferTarget.ArrayBuffer, BufferAccess.WriteOnly));
-            source.createVertexArray();
+            videoHandle.createVertexArray(GL.MapBuffer(BufferTarget.ArrayBuffer, BufferAccess.WriteOnly));
             GL.UnmapBuffer(BufferTarget.ArrayBuffer);
 
             GL.VertexPointer(3, VertexPointerType.Float, 3 * sizeof(float) + 4 * sizeof(byte), IntPtr.Zero);
             GL.ColorPointer(4, ColorPointerType.UnsignedByte, 3 * sizeof(float) + 4 * sizeof(byte), 3 * sizeof(float));
 
-            ImageData data = source.getImageData();
-
             GL.PointSize(3.0f);
-            GL.DrawArrays(PrimitiveType.Points, 0, data.getHeight() * data.getWidth());
-
+            GL.DrawArrays(PrimitiveType.Points, 0, videoHandle.getWidth() * videoHandle.getHeight());
 
             // draw points
             float[] pointData;
             
-            DetectionStatus[] fingerStatus = source.getFingerStatus(1);
+            DetectionStatus[] fingerStatus = inputProvider.getFingerStatus(1);
             int numFingersDetected = 0;
             for (int i = 0; i < fingerStatus.Length; ++i)
             {
@@ -77,11 +74,11 @@ namespace bbiwarg.Graphics
 
             pointData = new float[(4 + 3) * (1 + numFingersDetected)];
             Color y = Color.Yellow;
-            Vector palmPosition = source.getPalmPosition3D(1);
+            Vector palmPosition = videoHandle.getPalmPosition3D(1);
 
             pointData[0] = palmPosition[0];
-            pointData[1] = palmPosition[2];
-            pointData[2] = -palmPosition[1];
+            pointData[1] = palmPosition[1];
+            pointData[2] = -palmPosition[2];
 
             pointData[3] = y.R / 255.0f;
             pointData[4] = y.G / 255.0f;
@@ -89,14 +86,14 @@ namespace bbiwarg.Graphics
             pointData[6] = y.A / 255.0f;
 
             int index = 7;
-            Vector[] fingerPositions = source.getFingerTipPositions3D(1);
+            Vector[] fingerPositions = inputProvider.getFingerTipPositions3D(1);
             for (int i = 0; i < fingerStatus.Length; ++i)
             {
                 if (fingerStatus[i] == DetectionStatus.Detected || fingerStatus[i] == DetectionStatus.Tracked)
                 {
                     pointData[index + 0] = fingerPositions[i][0];
-                    pointData[index + 1] = fingerPositions[i][2];
-                    pointData[index + 2] = -fingerPositions[i][1];
+                    pointData[index + 1] = fingerPositions[i][1];
+                    pointData[index + 2] = -fingerPositions[i][2];
 
                     pointData[index + 3] = y.R / 255.0f;
                     pointData[index + 4] = y.G / 255.0f;
@@ -135,11 +132,9 @@ namespace bbiwarg.Graphics
             GL.GenBuffers(1, out imageBufferId);
             GL.GenBuffers(1, out pointBufferId);
 
-            ImageData data = source.getImageData();
-
             GL.BindBuffer(BufferTarget.ArrayBuffer, imageBufferId);
             GL.BufferData(BufferTarget.ArrayBuffer, 
-                          (IntPtr)((3 * sizeof(float) + 4  * sizeof(byte)) * data.getWidth() * data.getHeight()), 
+                          (IntPtr)((3 * sizeof(float) + 4  * sizeof(byte)) * videoHandle.getWidth() * videoHandle.getHeight()), 
                           IntPtr.Zero, BufferUsageHint.StreamDraw);
         } 
     }

bbiwarg/Main/OutputTest.cs

@@ -12,14 +12,11 @@ namespace bbiwarg.Main
     {
         static void Main(string[] args)
         {
-            IVideoDataSource source = new IIsuDataSource("..\\..\\videos\\6.skv");
-            source.init();
-            source.start();
+            IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\1.skv");
+            IVideoHandle videoHandle = new VideoHandle(inputProvider);
 
-            Output output = new Output(source);
+            Output output = new Output(inputProvider, videoHandle);
             output.Run(30);
-
-            source.stop();
         }
     }
 }

bbiwarg/bbiwarg.csproj

@@ -57,15 +57,17 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="DataSource\ImageData.cs" />
+    <Compile Include="DataSource\ForeFingerDetection.cs" />
+    <Compile Include="DataSource\IInputProvider.cs" />
     <Compile Include="DataSource\UVImage.cs" />
     <Compile Include="DataSource\ColorImage.cs" />
     <Compile Include="DataSource\ConfidenceImage.cs" />
     <Compile Include="DataSource\DepthImage.cs" />
-    <Compile Include="DataSource\IisuDataSource.cs" />
-    <Compile Include="DataSource\IVideoDataSource.cs" />
+    <Compile Include="DataSource\IisuInputProvider.cs" />
+    <Compile Include="DataSource\IVideoHandle.cs" />
     <Compile Include="DataSource\VectorExtender.cs" />
     <Compile Include="DataSource\VertexArray.cs" />
+    <Compile Include="DataSource\VideoHandle.cs" />
     <Compile Include="Graphics\Rectangle.cs" />
     <Compile Include="Graphics\IGraphicElement.cs" />
     <Compile Include="Graphics\Point.cs" />