Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

bbiwarg/Detectors/Palm/PalmDetector.cs

@@ -30,11 +30,16 @@ namespace bbiwarg.Detectors.Palm
 
         private Contour<Point> palmContour;
         private List<MCvConvexityDefect> convexityDefects;
-        private Vector2D wristPoint, wristDirection;
         private LineSegment2DF wristLine, thumbLine;
-        private MCvBox2D palmRect;
+        private Quad2D palmRect;
 
-        public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector, PalmImage palmImage)
+        private bool valid = false;
+        private Vector2D topLeft;
+        private Vector2D topRight;
+        private Vector2D bottomLeft;
+        private Vector2D bottomRight;
+
+        public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, List<Finger> detectedFingers, PalmImage palmImage)
         {
             width = depthImage.getWidth();
             height = depthImage.getHeight();
@@ -44,7 +49,7 @@ namespace bbiwarg.Detectors.Palm
 
             handImage = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte)0 : (byte)1; });
             
-            fingers = getFingersWithoutThumb(fingerDetector);
+            fingers = getFingersWithoutThumb(detectedFingers);
             buildPointingHandMask();
             handImage = handImage.And(pointingHandMask);
 
@@ -54,15 +59,12 @@ namespace bbiwarg.Detectors.Palm
                 findConvexityDefactsSortedByDepth();
                 removeConvexityDefectsNearFingerTips();
 
-                findWristLine();
-                findThumbLine();
-
-                removePointsFromContour(wristLine, 1);
-                removePointsFromContour(thumbLine, 1);
+                findHandPoints();
 
-                if (palmContour.Count<Point>() != 0)
+                if (valid)
                 {
-                    findPalmRect();
+                    removePointsFromContour(wristLine, 1);
+                    removePointsFromContour(thumbLine, 1);
 
                     draw();
                 }
@@ -74,9 +76,8 @@ namespace bbiwarg.Detectors.Palm
             return new PalmRect(palmRect);
         }
 
-        private List<Finger> getFingersWithoutThumb(FingerDetector fingerDetector)
+        private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
         {
-            List<Finger> detectedFingers = fingerDetector.Fingers;
             Finger leftMost = null;
             float minX = float.MaxValue;
             foreach (Finger f in detectedFingers)
@@ -168,20 +169,6 @@ namespace bbiwarg.Detectors.Palm
             }
         }
 
-        private void findWristDirection()
-        {
-            PointF[] points = new PointF[palmContour.Count<Point>()];
-            int index = 0;
-            foreach (Point p in palmContour)
-            {
-                points[index] = new PointF(p.X, p.Y);
-                ++index;
-            }
-            PointF direction, tmp;
-            PointCollection.Line2DFitting(points, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out direction, out tmp);
-            wristDirection = new Vector2D(-direction.Y, direction.X);
-        }
-
         private void findConvexityDefactsSortedByDepth()
         {
             convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
@@ -213,34 +200,31 @@ namespace bbiwarg.Detectors.Palm
             convexityDefects = newDefects;
         }
 
-        private void findWristPoint()
-        {
-            if (convexityDefects.Count > 1)
-                wristPoint = new Vector2D(convexityDefects[1].DepthPoint);
-            else
-                wristPoint = new Vector2D(-1, -1);
-        }
-
-        private void findWristLine()
-        {
-            findWristPoint();
-            findWristDirection();
-            wristLine = new LineSegment2DF(wristPoint - 1000 * wristDirection, wristPoint + 1000 * wristDirection);
-        }
-
-        private void findThumbLine()
-        {
+        private void findHandPoints() {
             if (convexityDefects.Count > 0)
             {
                 MCvConvexityDefect thumbDefect = convexityDefects[0];
-                Vector2D p1 = new Vector2D(thumbDefect.DepthPoint);
-                Vector2D p2 = new Vector2D(thumbDefect.StartPoint);
-                Vector2D direction = (p1 - p2).normalize();
-                thumbLine = new LineSegment2DF(p1 - 1000 * direction, p1 + 1000 * direction);
+                Vector2D thumb = new Vector2D(thumbDefect.DepthPoint);
+                Vector2D thumbDefectStart = new Vector2D(thumbDefect.StartPoint);
+
+                Vector2D handLength = thumbDefectStart - thumb;
+                Vector2D handWidth = 0.8f * new Vector2D(handLength.Y, -handLength.X);
+
+                topLeft = thumbDefectStart;
+                bottomLeft = thumb - 0.3f * handLength;
+                bottomRight = bottomLeft + handWidth;
+                topRight = bottomRight + 1.5f * handLength;
+
+                wristLine = new LineSegment2DF(bottomLeft - 1000 * handWidth, bottomRight + 1000 * handWidth);
+                thumbLine = new LineSegment2DF(topLeft + 1000 * handLength, bottomLeft - 1000 * handLength);
+
+                palmRect = new Quad2D(bottomLeft, topLeft, topRight, bottomRight);
+
+                valid = true;
             }
-            else
-            {
-                thumbLine = new LineSegment2DF(new PointF(-1, -1), new PointF(-1, -1));
+            else {
+                palmRect = null;
+                valid = false;
             }
         }
 
@@ -259,11 +243,6 @@ namespace bbiwarg.Detectors.Palm
             palmContour = newContour;
         }
 
-        private void findPalmRect()
-        {
-            palmRect = palmContour.GetMinAreaRect();
-        }
-
         private void draw()
         {
             palmImage.drawContour(palmContour);
@@ -271,11 +250,15 @@ namespace bbiwarg.Detectors.Palm
             palmImage.drawLine(wristLine, PalmImageState.wristLine);
             palmImage.drawLine(thumbLine, PalmImageState.thumbLine);
 
-            PointF[] vertices = palmRect.GetVertices();
-            for (int i = 0; i < 4; ++i)
-                palmImage.drawLine(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), PalmImageState.palmRect);
+            if (palmRect != null)
+            {
+                PointF[] vertices = palmRect.getVertices();
+                for (int i = 0; i < 4; ++i)
+                    palmImage.drawLine(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), PalmImageState.palmRect);
+
 
-            palmImage.drawGrid(new Vector2D(vertices[0]),new Vector2D(vertices[1]),new Vector2D(vertices[2]),new Vector2D(vertices[3]));
+                palmImage.drawGrid(new Vector2D(vertices[0]), new Vector2D(vertices[1]), new Vector2D(vertices[2]), new Vector2D(vertices[3]));
+            }
         }
     }
 }

bbiwarg/Detectors/Palm/PalmRect.cs

@@ -16,14 +16,20 @@ namespace bbiwarg.Detectors.Palm
     {
         private Vector2D origin;
         private Matrix<float> transformationMatrix;
+        bool valid = true;
 
-        public PalmRect(MCvBox2D palmRect)
+        public PalmRect(Quad2D palmRect)
         {
-            PointF[] vertices = palmRect.GetVertices();
+            if (palmRect == null)
+            {
+                valid = false;
+                return;
+            }
 
-            origin = new Vector2D(vertices[1]);
-            PointF v0 = vertices[0];
-            PointF v2 = vertices[2];
+            //TODO
+            origin = palmRect.Origin;
+            PointF v0 = palmRect.Origin + palmRect.DirWidth;
+            PointF v2 = palmRect.Origin + palmRect.DirLength;
 
             Matrix<float> tmp = new Matrix<float>(new float[,] { { v0.X - origin.X, v2.X - origin.X }, { v0.Y - origin.Y, v2.Y - origin.Y } });
             transformationMatrix = new Matrix<float>(2, 2);
@@ -32,11 +38,17 @@ namespace bbiwarg.Detectors.Palm
 
         public bool isWithinMargin(Vector2D position)
         {
+            if (!valid)
+                return false;
+            
             Vector2D relativePosition = getRelativePosition(position);
             return (relativePosition.X >= -0.1 && relativePosition.X <= 1.1 && relativePosition.Y >= -0.1 && relativePosition.Y <= 1.1);
         }
         public Vector2D getRelativePosition(Vector2D absolutePosition)
         {
+            if (!valid)
+                return null;
+            
             Vector2D v = absolutePosition - origin;
             Matrix<float> coordsMatrix = transformationMatrix.Mul(new Matrix<float>(new float[,] { { v.X }, { v.Y } }));
             return new Vector2D(coordsMatrix.Data[0, 0], coordsMatrix.Data[1, 0]);

bbiwarg/Images/PalmImage.cs

@@ -40,7 +40,8 @@ namespace bbiwarg.Images
 
         public void drawContour(Contour<Point> contour)
         {
-            image.Draw(contour, new Gray((byte)PalmImageState.palmContour), 1);
+            image.Draw(contour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), new Gray((byte)PalmImageState.palmContour), 1);
+            //image.Draw(contour, new Gray((byte)PalmImageState.palmContour), 1);
         }
 
         public void drawLine(LineSegment2DF line, PalmImageState state)

bbiwarg/Preprocessing/BackgroundSubtractor.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using bbiwarg.Images;
+
+using Emgu.CV;
+using Emgu.CV.Structure;
+using Emgu.CV.VideoSurveillance;
+
+namespace bbiwarg.Preprocessing
+{
+    class BackgroundSubtractor
+    {
+        private BackgroundSubtractorMOG subtractor;
+
+        public BackgroundSubtractor()
+        {
+            subtractor = new BackgroundSubtractorMOG(0, 0, 0.0, 0.0); // defaults
+        }
+
+        public void update(Image<Bgr, byte> currentColorImage)
+        {
+            subtractor.Update(currentColorImage);
+        }
+
+        public Image<Gray, short> getForeground(Image<Gray, short> image)
+        {
+            Image<Gray, short> foreground = image.Clone();
+            foreground = foreground.And(subtractor.ForegroundMask.Convert<short>(delegate(byte b) { if (b == 0) return 0; else return short.MaxValue; }));
+
+            int numNonZero = 0;
+            for (int i = 0; i < image.Width; ++i)
+                for (int j = 0; j < image.Height; ++j)
+                    if (subtractor.ForegroundMask.Data[j, i, 0] != 0)
+                        ++numNonZero;
+            Console.WriteLine("non zero: " + numNonZero);
+
+            return subtractor.ForegroundMask.Convert<short>(delegate(byte b) { return (b == 0) ? (short) 0 : (short) 100; });
+        }
+    }
+}

bbiwarg/Utility/Line2D.cs

@@ -30,47 +30,6 @@ namespace bbiwarg.Utility
             NumCombinedLines = 1;
         }
 
-        public Line2D(List<Line2D> lines) {
-            //list all points
-            List<Vector2D> points = new List<Vector2D>();
-            List<Vector2D> directions = new List<Vector2D>();
-            NumCombinedLines = 0;
-            foreach (Line2D line in lines) {
-                points.Add(line.P1);
-                points.Add(line.P2);
-                directions.Add(line.Direction);
-                NumCombinedLines += line.NumCombinedLines;
-            }
-
-            //find two points with max-distance
-            Vector2D a = points[0];
-            Vector2D b = points[0];
-            float length = 0;
-            foreach (Vector2D p in points) {
-                float distanceToA = p.getDistanceTo(a);
-                float distanceToB = p.getDistanceTo(b);
-
-                if (distanceToA > length && distanceToA >= distanceToB) {
-                    b = p;
-                    length = distanceToA;
-                }
-                else if (distanceToB > length && distanceToB > distanceToA) {
-                    a = p;
-                    length = distanceToA;
-                }
-            }
-
-            //project a and b on mean-line
-            Vector2D meanPoint = Vector2D.mean(points);
-            Vector2D meanDirection = Vector2D.mean(directions);
-
-            Line2D meanLine = new Line2D(meanPoint, meanPoint + meanDirection);
-            Vector2D meanA = meanLine.projectToLine(a);
-            Vector2D meanB = meanLine.projectToLine(b);
-
-            setPoints(meanA, meanB);
-        }
-
         private void setPoints(Vector2D p1, Vector2D p2) {
             //endpoints
             P1 = p1;
@@ -136,6 +95,23 @@ namespace bbiwarg.Utility
             return new Vector2D(newX, newY);
         }
 
+        public Vector2D intersection(Line2D line)
+        {
+            Vector2D p = P1;
+            Vector2D r = P2 - P1;
+            Vector2D q = line.P1;
+            Vector2D s = line.P2 - line.P1;
+
+            float r_cross_s = r.cross(s);
+            float q_p_cross_s = (q - p).cross(s);
+
+            if (r_cross_s == 0.0)
+                return null;
+
+            float t = q_p_cross_s  / r_cross_s;
+            return p + t * r;
+        }
+
         public override string ToString()
         {
             return (int)P1.X + "|" + (int)P1.Y + " --- " + (int)P2.X + "|" + (int)P2.Y;

bbiwarg/Utility/Quad2D.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+using System.Drawing;
+
+namespace bbiwarg.Utility
+{
+    class Quad2D
+    {
+        public Vector2D TopLeft {private set; get;}
+        public Vector2D TopRight { private set; get; }
+        public Vector2D BottomLeft { private set; get; }
+        public Vector2D BottomRight { private set; get; }
+
+        public Quad2D(Vector2D bottomLeft, Vector2D topLeft, Vector2D topRight, Vector2D bottomRight)
+        {
+            TopLeft = topLeft;
+            TopRight = topRight;
+            BottomLeft = bottomLeft;
+            BottomRight = BottomRight;
+        }
+
+        public PointF[] getVertices()
+        {
+            return new PointF[] {TopLeft,TopRight,BottomRight,BottomLeft};
+        }
+    }
+}

bbiwarg/Utility/Vector2D.cs

@@ -94,6 +94,11 @@ namespace bbiwarg.Utility
             return new Vector2D(vector1.X - vector2.X, vector1.Y - vector2.Y);
         }
 
+        public float cross(Vector2D v)
+        {
+            return X * v.Y - Y * v.X;
+        }
+
         public static Vector2D mean(List<Vector2D> vectors) {
             Vector2D meanVector = new Vector2D(0,0);
             foreach (Vector2D vector in vectors) {

bbiwarg/VideoHandle.cs

@@ -150,7 +150,7 @@ namespace bbiwarg
             fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, fingerImage);
 
             //detect palm
-            palmDetector = new PalmDetector(depthImage, edgeImage, fingerDetector, palmImage);
+            palmDetector = new PalmDetector(depthImage, edgeImage, fingerDetector.Fingers, palmImage);
 
             //detect+track touchEvents
             touchDetector = new TouchDetector(fingerTracker.getFingers(), depthImage, touchImage);

bbiwarg/bbiwarg.csproj

@@ -91,6 +91,7 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Utility\HelperFunctions.cs" />
     <Compile Include="Utility\Line2D.cs" />
+    <Compile Include="Utility\Quad2D.cs" />
     <Compile Include="Utility\Vector.cs" />
     <Compile Include="Utility\Vector2D.cs" />
     <Compile Include="VideoHandle.cs" />