Started implementation of paper.

bbiwarg/Graphics/Output2D.cs

@@ -63,73 +63,29 @@ namespace bbiwarg.Graphics
 
             short[] textureData = new short[3 * videoHandle.getWidth() * videoHandle.getHeight()];
             int index = 0;
+            short minDepth = videoHandle.getHandImage().getMinDepth();
+            short maxDepth = videoHandle.getHandImage().getMaxDepth();
+            
             for (int y = 0; y < videoHandle.getHeight(); ++y)
             {
                 for (int x = 0; x < videoHandle.getWidth(); ++x)
                 {
-                    // 0 --> 0 / 2000 -> short.MaxValue
-                    int depthHand1 = videoHandle.getHand1Image().getDepth(x, y);
-                    int depthHand2 = videoHandle.getHand2Image().getDepth(x, y);
-                    textureData[index] = (short) (depthHand1 * short.MaxValue / 2000);
-                    textureData[index + 1] = (short)(depthHand2 * short.MaxValue / 2000);
-                    textureData[index + 2] = 0;
+                    short depth = videoHandle.getHandImage().getDepth(x, y);
+                    short diff = (short) Math.Max(maxDepth - minDepth, 1);
+                    short scaledDepth = (short) (Int16.MaxValue - (((int) depth - minDepth) * (int) Int16.MaxValue / diff));
+                    textureData[index] = scaledDepth;
+                    textureData[index + 1] = scaledDepth;
+                    textureData[index + 2] = scaledDepth;
                     index += 3;
                 }
             }
 
-            /*
-            DepthImage depthImage = input.getDepthImage();
-            depthImage.filterMedian(3);
-
-            Vector palmOrigin = input.getPalmPosition2D(1);
-            int palmX = (int) palmOrigin.x();
-            int palmY = (int) palmOrigin.y();
-            int palmDepth = depthImage.getDepth(palmX, palmY);
-
-            int maxHandSize = 150; // mm Wieviel Pixel sind das?
-            List<Vector> points = new List<Vector>();
-            if (palmX != 0 && palmY != 0)
+            int[] histogram = videoHandle.getHistogram();
+            for (int i = 0; i < histogram.Length; ++i)
             {
-                depthImage.thresholdDepth(palmDepth - 10, palmDepth + 30);
-                depthImage.thresholdPosition(palmX - maxHandSize / 2, palmX + maxHandSize / 2, palmY - maxHandSize / 2, palmY + maxHandSize / 2);
-                points = depthImage.getRectPoints();
-            }
-
-            short[] textureData = new short[3 * depthImage.getWidth() * depthImage.getHeight()];
-            int index = 0;
-            for (int y = 0; y < depthImage.getHeight(); ++y) {
-                for (int x = 0; x < depthImage.getWidth(); ++x) {
-                    // 0 --> 0 / 2000 -> short.MaxValue
-                    int d = depthImage.getDepth(x, y);
-                    short depth = (short) (d * short.MaxValue / 2000);
-                    textureData[index] = textureData[index + 1] = textureData[index + 2] = depth;
-                    index += 3;
-                }
+                //...
             }
 
-            // draw palm origin
-            Point2D palmPoint = new Point2D(palmOrigin, Color.Yellow);
-            palmPoint.draw(textureData, depthImage.getWidth());
-
-            // draw rect points
-            foreach (Vector v in points)
-            {
-                int x = (int)v.x() % depthImage.getWidth();
-                int y = (int)v.y() % depthImage.getHeight();
-
-                for (int i = -1; i < 2; ++i)
-                {
-                    for (int j = -1; j < 2; ++j)
-                    {
-                        int px = Math.Min(depthImage.getWidth() - 1, Math.Max(0, x + i));
-                        int py = Math.Min(depthImage.getHeight() - 1, Math.Max(0, y + j));
-                        Vector positon = new DenseVector(new float[] { px, py });
-                        Point2D point = new Point2D(positon, Color.Blue);
-                        point.draw(textureData, depthImage.getWidth());
-                    }
-                }
-            }
-            */
             GL.BindTexture(TextureTarget.Texture2D, textureId);
             GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgb, videoHandle.getWidth(), videoHandle.getHeight(), 0,
                           PixelFormat.Rgb, PixelType.Short, textureData);

bbiwarg/Images/DepthImage.cs

@@ -19,6 +19,7 @@ namespace bbiwarg.Images
     {
         private int width, height;
         private Image<Gray, short> image;
+        private int[] histogram;
 
         public DepthImage(int width, int height, short[] data)
         {
@@ -52,47 +53,107 @@ namespace bbiwarg.Images
 
         public void filterMedian(int filterSize)
         {
-            image = image.SmoothMedian(3);
+            image = image.SmoothMedian(filterSize);
         }
 
         public void thresholdDepth(int min, int max)
         {
             image = image.ThresholdToZero(new Gray(min));
-            image = image.ThresholdBinaryInv(new Gray(max), new Gray(2000));
-         }
+            image = image.ThresholdTrunc(new Gray(max));
 
-        public void thresholdPosition(int minX, int maxX, int minY, int maxY)
+            for (int x = 0; x < width; ++x)
+            {
+                for (int y = 0; y < height; ++y)
+                {
+                    if (getDepth(x, y) == 0)
+                        setDepth(x, y, (short) max);
+                }
+            }
+        }
+
+        public int[] getHistogram()
         {
+            Int16 minDepth = getMinDepth();
+            Int16 maxDepth = getMaxDepth();
+
+            histogram = new int[maxDepth - minDepth];
             for (int x = 0; x < width; ++x)
             {
                 for (int y = 0; y < height; ++y)
                 {
-                    if (x < minX || x > maxX || y < minY || y > maxY)
-                        setDepth(x, y, 0);
+                    int depth = getDepth(x, y);
+                    if (depth != maxDepth)
+                        histogram[depth - minDepth]++;
                 }
             }
+
+            return histogram;
         }
 
-        public List<Vector> getRectPoints()
+        public Int16 getPeakDepth()
         {
-            Image<Gray, Byte> tmp = image.Convert<Byte>(delegate(short s) { return (s == 2000) ? (byte) 1 : (byte) 0; });
+            Int16 minDepth = getMinDepth();
 
-            Contour<Point> contours = tmp.FindContours();
-            MCvBox2D box = contours.GetMinAreaRect();
-            Seq<Point> points = contours.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE);
+            Int16 peak = -1;
+            int maxValue = 0;
+            for (int i = 0; i < histogram.Length; ++i)
+            {
+                if (histogram[i] > maxValue)
+                {
+                    maxValue = histogram[i];
+                    peak = (Int16) (i + minDepth);
+                }
+            }
+            return peak;
+        }
 
-            List<Vector> result = new List<Vector>();
-            foreach (Point p in points)
+        public Int16 getMinDepth()
+        {
+            Int16 minDepth = Int16.MaxValue;
+            for (int x = 0; x < width; ++x)
             {
-                result.Add(new DenseVector(new float[]{p.X, p.Y}));
+                for (int y = 0; y < height; ++y)
+                {
+                    Int16 depth = getDepth(x, y);
+                    if (depth < minDepth)
+                        minDepth = depth;
+                }
             }
-            return result;
+            return minDepth;
         }
 
-        public void floodFill(int x, int y)
+        public Int16 getMaxDepth()
         {
+            Int16 maxDepth = Int16.MinValue;
+            for (int x = 0; x < width; ++x)
+            {
+                for (int y = 0; y < height; ++y)
+                {
+                    Int16 depth = getDepth(x, y);
+                    if (depth > maxDepth)
+                        maxDepth = depth;
+                }
+            }
+            return maxDepth;
+        }
+
+        public void floodFill(int x, int y, int val)
+        {
+            Image<Gray, Byte> tmp = image.Convert<Byte>(delegate(short s) { return (byte) ((int) s * Byte.MaxValue / 2000); });
+
             Emgu.CV.Structure.MCvConnectedComp comp = new MCvConnectedComp();
-            Emgu.CV.CvInvoke.cvFloodFill(image.Ptr, new Point(x, y), new MCvScalar(155), new MCvScalar(5), new MCvScalar(5), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, 0, IntPtr.Zero); //, out comp,Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT,mask.Ptr)
+            Emgu.CV.CvInvoke.cvFloodFill(tmp.Ptr, new Point(x, y), new MCvScalar(val), new MCvScalar(30), new MCvScalar(30), out comp, 0, IntPtr.Zero);
+
+            for (int i = 0; i < width; ++i)
+            {
+                for (int j = 0; j < height; ++j)
+                {
+                    if (tmp.Data[j, i, 0] == 255)
+                        setDepth(i, j, 2000);
+                    else
+                        setDepth(i, j, 0);
+                }
+            }
         }
     }
 }

bbiwarg/Main/OutputTest.cs

@@ -13,7 +13,7 @@ namespace bbiwarg.Main
     {
         static void Main(string[] args)
         {
-            IInputProvider inputProvider = new IisuInputProvider();//"..\\..\\videos\\4.skv");
+            IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\1.skv");
             IVideoHandle videoHandle = new VideoHandle(inputProvider);
 
             Output2D output = new Output2D(videoHandle);

bbiwarg/VideoHandle/IVideoHandle.cs

@@ -16,8 +16,7 @@ namespace bbiwarg.VideoHandles
         short getConfidence(int x, int y);
         Color getColor(int x, int y);
 
-        DepthImage getHand1Image();
-        DepthImage getHand2Image();
+        DepthImage getHandImage();
 
         // TODO: implement properly
         void createVertexArray(IntPtr vertexBuffer);
@@ -33,5 +32,8 @@ namespace bbiwarg.VideoHandles
         Palm getPalm(uint handIndex);
         int getWidth();
         int getHeight();
+
+        Int16 getPeakDepth();
+        int[] getHistogram();
     }
 }

bbiwarg/VideoHandle/VideoHandle.cs

@@ -24,8 +24,9 @@ namespace bbiwarg.VideoHandles
         private UVImage uvImage;
         private ColorImage colorImage;
 
-        private DepthImage hand1Image;
-        private DepthImage hand2Image;
+        private DepthImage handImage;
+        private Int16 peakDepth;
+        private int[] histogram;
 
         private float maxU;
         private float maxV;
@@ -74,39 +75,27 @@ namespace bbiwarg.VideoHandles
         private void createImageData() 
         {
             depthImage = inputProvider.getDepthImage();
+            handImage = inputProvider.getDepthImage();
             confidenceImage = inputProvider.getConfidenceImage();
             uvImage = inputProvider.getUVImage();
             colorImage = inputProvider.getColorImage();
 
-            hand1Image = inputProvider.getDepthImage();
-            hand2Image = inputProvider.getDepthImage();
+            handImage.filterMedian(3);
+            int minDepth = handImage.getMinDepth();
+            handImage.thresholdDepth(minDepth, minDepth + 100);
 
-            int maxHandSize = 150; // mm Wieviel Pixel sind das?
-
-            //hand1
-            Vector palm1Origin = inputProvider.getPalmPosition2D(1);
-            int palmX = (int)palm1Origin.x();
-            int palmY = (int)palm1Origin.y();
-            int palmDepth = depthImage.getDepth(palmX, palmY);
-
-            if (palmX != 0 && palmY != 0)
-            {
-                //hand1Image.floodFill(palmX, palmY);
-                hand1Image.thresholdDepth(palmDepth - 10, palmDepth + 30);
-                hand1Image.thresholdPosition(palmX - maxHandSize / 2, palmX + maxHandSize / 2, palmY - maxHandSize / 2, palmY + maxHandSize / 2);
-            }
+            histogram = handImage.getHistogram();
+            peakDepth = handImage.getPeakDepth();
+        }
 
-            //hand2
-            Vector palm2Origin = inputProvider.getPalmPosition2D(2);
-            palmX = (int)palm2Origin.x();
-            palmY = (int)palm2Origin.y();
-            palmDepth = depthImage.getDepth(palmX, palmY);
+        public Int16 getPeakDepth()
+        {
+            return peakDepth;
+        }
 
-            if (palmX != 0 && palmY != 0)
-            {
-                hand2Image.thresholdDepth(palmDepth - 10, palmDepth + 30);
-                hand2Image.thresholdPosition(palmX - maxHandSize / 2, palmX + maxHandSize / 2, palmY - maxHandSize / 2, palmY + maxHandSize / 2);
-            }
+        public int[] getHistogram()
+        {
+            return histogram;
         }
 
         public short getDepth(int x, int y)
@@ -135,14 +124,9 @@ namespace bbiwarg.VideoHandles
             return colorImage.getColor(xInColorImage, yInColorImage);
         }
 
-        public DepthImage getHand1Image()
-        {
-            return hand1Image;
-        }
-
-        public DepthImage getHand2Image()
+        public DepthImage getHandImage()
         {
-            return hand2Image;
+            return handImage;
         }
 
         public DetectionStatus[] getFingerStatus(uint handIndex)