Did some refactoring. (Added Palm and PalmDetector / PalmImage only saves state for display (like FingerImage))

bbiwarg/Detectors/Palm/Palm.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Drawing;
+
+using bbiwarg.Utility;
+
+using Emgu.CV;
+using Emgu.CV.Structure;
+
+namespace bbiwarg.Detectors.Palm
+{
+    class Palm
+    {
+        private Vector2D origin;
+        private Matrix<float> transformationMatrix;
+        
+        public Palm(MCvBox2D palmRect)
+        {
+            PointF[] vertices = palmRect.GetVertices();
+            
+            origin = new Vector2D(vertices[1]);
+            PointF v0 = vertices[0];
+            PointF v2 = vertices[2];
+
+            Matrix<float> tmp = new Matrix<float>(new float[,] { { v0.X - origin.X, v2.X - origin.X }, { v0.Y - origin.Y, v2.Y - origin.Y } });
+            transformationMatrix = new Matrix<float>(2, 2);
+            CvInvoke.cvInvert(tmp.Ptr, transformationMatrix.Ptr, Emgu.CV.CvEnum.SOLVE_METHOD.CV_LU);
+        }
+
+        public Vector2D getRelativePosition(Vector2D absolutePosition)
+        {
+            Vector2D v = absolutePosition - origin;
+            Matrix<float> coordsMatrix = transformationMatrix.Mul(new Matrix<float>(new float[,] { { v.X }, { v.Y } }));
+            return new Vector2D(coordsMatrix.Data[0, 0], coordsMatrix.Data[1, 0]);
+        }
+    }
+}

bbiwarg/Detectors/Palm/PalmDetector.cs

@@ -0,0 +1,276 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Drawing;
+
+using Emgu.CV;
+using Emgu.CV.Structure;
+
+using bbiwarg.Utility;
+
+using bbiwarg.Images;
+
+using bbiwarg.Detectors.Fingers;
+
+namespace bbiwarg.Detectors.Palm
+{
+    class PalmDetector
+    {
+        private int width, height;
+        private DepthImage depthImage;
+        private EdgeImage edgeImage;
+        private PalmImage palmImage;
+
+        private Image<Gray, Byte> handImage;
+        private Image<Gray, Byte> pointingHandMask;
+
+        private List<Finger> fingers;
+
+        private Contour<Point> palmContour;
+        private List<MCvConvexityDefect> convexityDefects;
+        private Vector2D wristPoint, wristDirection;
+        private LineSegment2DF wristLine, thumbLine;
+        private MCvBox2D palmRect;
+
+        public PalmDetector(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector, PalmImage palmImage)
+        {
+            width = depthImage.getWidth();
+            height = depthImage.getHeight();
+            this.depthImage = depthImage;
+            this.edgeImage = edgeImage;
+            this.palmImage = palmImage;
+
+            handImage = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte)0 : (byte)1; });
+            
+            fingers = getFingersWithoutThumb(fingerDetector);
+            buildPointingHandMask();
+            handImage = handImage.And(pointingHandMask);
+
+            findLongestPalmContour();
+            if (palmContour != null)
+            {
+                findConvexityDefactsSortedByDepth();
+                removeConvexityDefectsNearFingerTips();
+
+                findWristLine();
+                findThumbLine();
+
+                removePointsFromContour(wristLine, 1);
+                removePointsFromContour(thumbLine, 1);
+
+                findPalmRect();
+
+                draw();
+            }
+        }
+
+        public Palm getPalm()
+        {
+            return new Palm(palmRect);
+        }
+
+        private List<Finger> getFingersWithoutThumb(FingerDetector fingerDetector)
+        {
+            List<Finger> detectedFingers = fingerDetector.Fingers;
+            Finger leftMost = null;
+            float minX = float.MaxValue;
+            foreach (Finger f in detectedFingers)
+            {
+                float midX = ((f.Hand + f.Tip) / 2).X;
+                if (midX < minX)
+                {
+                    minX = midX;
+                    leftMost = f;
+                }
+            }
+
+            List<Finger> result = new List<Finger>();
+            foreach (Finger f in detectedFingers)
+            {
+                if (f != leftMost)
+                    result.Add(f);
+            }
+            return result;
+        }
+
+        private void fillFingerSlices(Image<Gray, Byte> image, byte val)
+        {
+            foreach (Finger f in fingers)
+            {
+                foreach (FingerSlice s in f.SliceTrail.getSlices())
+                {
+                    image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
+                }
+            }
+        }
+
+        private Finger getLongestFinger()
+        {
+            float maxLength = 0;
+            Finger longest = null;
+            foreach (Finger f in fingers)
+            {
+
+                if (f.Line.Length > maxLength)
+                {
+                    maxLength = f.Line.Length;
+                    longest = f;
+                }
+            }
+            return longest;
+        }
+
+        private Point getPointInPointingHand()
+        {
+            Finger finger = getLongestFinger();
+            if (finger == null)
+                return new Point(0, 0);
+            Vector2D direction = (finger.Hand - finger.Tip).normalize();
+            Vector2D pos = finger.Hand + 20 * direction;
+            return new Point(HelperFunctions.thresholdRange<int>(0, width - 1, (int) pos.X), HelperFunctions.thresholdRange<int>(0, height - 1, (int) pos.Y));
+        }
+
+        private void buildPointingHandMask()
+        {
+            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
+            fillFingerSlices(pointingHandMask, 1);
+            pointingHandMask = pointingHandMask.Dilate(1);
+            pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert<Byte>(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; }));
+            pointingHandMask = pointingHandMask.Dilate(1);
+
+            MCvConnectedComp tmp = new MCvConnectedComp();
+            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
+            pointingHandMask = pointingHandMask.Convert<Byte>(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; });
+            pointingHandMask = pointingHandMask.Erode(1);
+            fillFingerSlices(pointingHandMask, 0);
+            pointingHandMask = pointingHandMask.Erode(2);
+        }
+
+        private void findLongestPalmContour()
+        {
+            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
+
+            palmContour = contour;
+            double maxPerimeter = 0;
+            while (contour != null)
+            {
+                if (contour.Perimeter > maxPerimeter)
+                {
+                    maxPerimeter = contour.Perimeter;
+                    palmContour = contour;
+                }
+                contour = contour.HNext;
+            }
+        }
+
+        private void findWristDirection()
+        {
+            PointF[] points = new PointF[palmContour.Count<Point>()];
+            int index = 0;
+            foreach (Point p in palmContour)
+            {
+                points[index] = new PointF(p.X, p.Y);
+                ++index;
+            }
+            PointF direction, tmp;
+            PointCollection.Line2DFitting(points, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out direction, out tmp);
+            wristDirection = new Vector2D(-direction.Y, direction.X);
+        }
+
+        private void findConvexityDefactsSortedByDepth()
+        {
+            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
+            convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2)
+            {
+                if (d1.Depth < d2.Depth)
+                    return 1;
+                else if (d1.Depth > d2.Depth)
+                    return -1;
+                return 0;
+            });
+        }
+
+        private void removeConvexityDefectsNearFingerTips()
+        {
+            List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
+            foreach (MCvConvexityDefect d in convexityDefects)
+            {
+                float minFingerTipDist = float.MaxValue;
+                foreach (Finger f in fingers)
+                {
+                    float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint));
+                    if (dist < minFingerTipDist)
+                        minFingerTipDist = dist;
+                }
+                if (minFingerTipDist > 20)
+                    newDefects.Add(d);
+            }
+            convexityDefects = newDefects;
+        }
+
+        private void findWristPoint()
+        {
+            if (convexityDefects.Count > 1)
+                wristPoint = new Vector2D(convexityDefects[1].DepthPoint);
+            else
+                wristPoint = new Vector2D(-1, -1);
+        }
+
+        private void findWristLine()
+        {
+            findWristPoint();
+            findWristDirection();
+            wristLine = new LineSegment2DF(wristPoint - 1000 * wristDirection, wristPoint + 1000 * wristDirection);
+        }
+
+        private void findThumbLine()
+        {
+            if (convexityDefects.Count > 0)
+            {
+                MCvConvexityDefect thumbDefect = convexityDefects[0];
+                Vector2D p1 = new Vector2D(thumbDefect.DepthPoint);
+                Vector2D p2 = new Vector2D(thumbDefect.StartPoint);
+                Vector2D direction = (p1 - p2).normalize();
+                thumbLine = new LineSegment2DF(p1 - 1000 * direction, p1 + 1000 * direction);
+            }
+            else
+            {
+                thumbLine = new LineSegment2DF(new PointF(-1, -1), new PointF(-1, -1));
+            }
+        }
+
+        private void removePointsFromContour(LineSegment2DF line, int sideToRemove)
+        {
+            Contour<Point> newContour = new Contour<Point>(new MemStorage());
+            int index = 0;
+            foreach (Point p in palmContour)
+            {
+                if (line.Side(p) != sideToRemove)
+                {
+                    newContour.Insert(index, p);
+                    ++index;
+                }
+            }
+            palmContour = newContour;
+        }
+
+        private void findPalmRect()
+        {
+            palmRect = palmContour.GetMinAreaRect();
+        }
+
+        private void draw()
+        {
+            palmImage.drawContour(palmContour);
+
+            palmImage.drawLine(wristLine, PalmImageState.wristLine);
+            palmImage.drawLine(thumbLine, PalmImageState.thumbLine);
+
+            PointF[] vertices = palmRect.GetVertices();
+            for (int i = 0; i < 4; ++i)
+                palmImage.drawLine(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), PalmImageState.palmRect);
+        }
+    }
+}

bbiwarg/Detectors/Touch/TouchDetector.cs

@@ -8,6 +8,7 @@ using Emgu.CV;
 using Emgu.CV.Structure;
 using bbiwarg.Images;
 using bbiwarg.Detectors.Fingers;
+using bbiwarg.Detectors.Palm;
 using bbiwarg.Utility;
 
 namespace bbiwarg.Detectors.Touch
@@ -19,7 +20,7 @@ namespace bbiwarg.Detectors.Touch
         private List<Finger> fingers;
         private List<TouchEvent> touchEvents;
 
-        public TouchDetector(List<Finger> fingers, DepthImage depthImage, TouchImage touchImage) {
+        public TouchDetector(List<Finger> fingers, DepthImage depthImage, TouchImage touchImage, Palm.Palm palm) {
             this.depthImage = depthImage;
             this.touchImage = touchImage;
             this.fingers = fingers;
@@ -35,13 +36,20 @@ namespace bbiwarg.Detectors.Touch
                     //correct touchEvent position
                     Vector2D direction = finger.Line.Direction;
                     float directionFactor = 10;
-                    float x = Math.Min(Math.Max(tipPoint.X + directionFactor * direction.X, 0), depthImage.getWidth()-1);
-                    float y = Math.Min(Math.Max(tipPoint.Y + directionFactor * direction.Y, 0), depthImage.getHeight()-1);
+                    float x = HelperFunctions.thresholdRange<float>(0, depthImage.getWidth() - 1, tipPoint.X + directionFactor * direction.X);
+                    float y = HelperFunctions.thresholdRange<float>(0, depthImage.getHeight() - 1, tipPoint.Y + directionFactor * direction.Y);
                     Vector2D tep = new Vector2D(x,y);
+                    Vector2D relativePosition = palm.getRelativePosition(tep);
 
-                    touchImage.setTouchAt((int)tep.X, (int)tep.Y, TouchImageState.touchDetected);
-                    TouchEvent touchEvent = new TouchEvent((int)tep.X, (int)tep.Y, floodValue, finger);
-                    touchEvents.Add(touchEvent);
+                    // detect touch event if positon is in 10% margin of palm rect (threshold coordinates to range [0,1])
+                    if (relativePosition.X >= -0.1 && relativePosition.X <= 1.1 && relativePosition.Y >= -0.1 && relativePosition.Y <= 1.1)
+                    {
+                        relativePosition = new Vector2D(HelperFunctions.thresholdRange<float>(0.0f, 1.0f, relativePosition.X), 
+                                                        HelperFunctions.thresholdRange<float>(0.0f, 1.0f, relativePosition.Y));
+                        touchImage.setTouchAt((int)tep.X, (int)tep.Y, TouchImageState.touchDetected);
+                        TouchEvent touchEvent = new TouchEvent((int)tep.X, (int)tep.Y, relativePosition.X, relativePosition.Y, floodValue, finger);
+                        touchEvents.Add(touchEvent);
+                    }
                 }
             }
         }

bbiwarg/Detectors/Touch/TouchEvent.cs

@@ -9,24 +9,40 @@ namespace bbiwarg.Detectors.Touch
 {
     class TouchEvent
     {
-        private int x;
-        private int y;
+        private int absoluteX;
+        private int absoluteY;
+        private float relativeX;
+        private float relativeY;
         private float floodValue;
         private Finger finger;
 
-        public TouchEvent(int x, int y, float floodValue, Finger finger) {
-            this.x = x;
-            this.y = y;
+        public TouchEvent(int absoluteX, int absoluteY, float relativeX, float relativeY, float floodValue, Finger finger) {
+            this.absoluteX = absoluteX;
+            this.absoluteY = absoluteY;
+
+            this.relativeX = relativeX;
+            this.relativeY = relativeY;
+
             this.floodValue = floodValue;
             this.finger = finger;
         }
 
-        public int getX() {
-            return x;
+        public int getAbsoluteX() {
+            return absoluteX;
+        }
+
+        public int getAbsoluteY() {
+            return absoluteY;
+        }
+
+        public float getRelativeX()
+        {
+            return relativeX;
         }
 
-        public int getY() {
-            return y;
+        public float getRelativeY()
+        {
+            return relativeY;
         }
 
         public float getFloodValue() {
@@ -34,8 +50,8 @@ namespace bbiwarg.Detectors.Touch
         }
 
         public float getDistanceTo(TouchEvent te) {
-            int xDiff = x - te.getX();
-            int yDiff = y - te.getY();
+            int xDiff = absoluteX - te.getAbsoluteX();
+            int yDiff = absoluteY - te.getAbsoluteY();
 
             float distance = (float)Math.Sqrt(xDiff * xDiff + yDiff * yDiff);
             return distance;

bbiwarg/Detectors/Touch/TouchTracker.cs

@@ -49,7 +49,7 @@ namespace bbiwarg.Detectors.Touch
                 }
                 if (tracked)
                 {
-                    touchImage.setTouchAt(te.getX(), te.getY(), TouchImageState.touchTracked);
+                    touchImage.setTouchAt(te.getAbsoluteX(), te.getAbsoluteY(), TouchImageState.touchTracked);
                     trackedTouchEvents.Add(te);
                     //Console.WriteLine("touch tracked at x:" + te.getX() + " y:" + te.getY() + " [floodValue:" + te.getFloodValue() + "]");
                 }

bbiwarg/Graphics/OutputWindow.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using OpenTK;
 using OpenTK.Graphics.OpenGL;
 using bbiwarg.Images;
+using bbiwarg.Detectors.Touch;
 
 namespace bbiwarg.Graphics
 {
@@ -74,6 +75,8 @@ namespace bbiwarg.Graphics
             GL.MatrixMode(MatrixMode.Modelview);
             GL.LoadMatrix(ref modelview);
 
+            bool changedFrame = false;
+
             const int auotRepeatDelay = 100; // ms
             if (videoHandle.sourceIsMovie())
             {
@@ -98,6 +101,7 @@ namespace bbiwarg.Graphics
                         videoHandle.unpauseMovie();
                         videoHandle.nextFrame();
                         videoHandle.pauseMovie();
+                        changedFrame = true;
                     }
                     else if (OpenTK.Input.Keyboard.GetState().IsKeyDown(OpenTK.Input.Key.Left) && (elapsed - timeLeftPressed) >= auotRepeatDelay)
                     {
@@ -107,11 +111,13 @@ namespace bbiwarg.Graphics
                         videoHandle.nextFrame();
                         videoHandle.reversePlay();
                         videoHandle.pauseMovie();
+                        changedFrame = true;
                     }
                 }
                 else
                 {
                     videoHandle.nextFrame();
+                    changedFrame = true;
                 }
             }
             else
@@ -119,6 +125,15 @@ namespace bbiwarg.Graphics
                 videoHandle.nextFrame();
             }
 
+            if (changedFrame)
+            {
+                //print tracked touch events
+                foreach (TouchEvent ev in videoHandle.getTrackedTouchEvents())
+                {
+                    Console.WriteLine("Touch event tracked at (" + ev.getAbsoluteX() + ", " + ev.getAbsoluteY() + ") --> (" + ev.getRelativeX() + ", " + ev.getRelativeY() + ")");
+                }
+            }
+
             //draw textures
             Int16[] depthTextureData = new Int16[3 * videoHandle.getWidth() * videoHandle.getHeight()];
             Int16[] edgeTextureData = new Int16[3 * videoHandle.getWidth() * videoHandle.getHeight()];
@@ -136,42 +151,50 @@ namespace bbiwarg.Graphics
                     float relDepth = videoHandle.getRelativeDepth(x,y);
                     red = green = blue = (Int16)((1.0f - videoHandle.getRelativeDepth(x, y)) * Int16.MaxValue);
                     
-                    TouchImageState tis = videoHandle.getTouchImageStateAt(x,y);
-                    if (tis == TouchImageState.touchArea)
-                    {
-                        red = (Int16) (red/2);
-                        green = (Int16) (green / 2);
-                        blue = (Int16) (blue / 2);
-                    }
-                    else if (tis == TouchImageState.touchAreaMatched)
-                    {
-                        red = (Int16) (red / 2);
-                        green = (Int16)(green / 2);
-                        blue = Int16.MaxValue;
-                        
-                    }
-                    else if (tis == TouchImageState.touchAreaStatusBar)
-                    {
-                        red = (Int16)(red / 2);
-                        green = Int16.MaxValue;
-                        blue = (Int16)(blue / 2); ;
-                    }
-                    else if (tis == TouchImageState.touchDetected)
-                    {
-                        red = blue = 0;
-                        green = Int16.MaxValue;
-                    }
-                    else if (tis == TouchImageState.touchTracked)
+                    // palm
+                    switch (videoHandle.getPalmImageStateAt(x, y))
                     {
-                        red = Int16.MaxValue;
-                        green = blue = 0;
+                        case PalmImageState.palmContour:
+                            red = Int16.MaxValue;
+                            blue = green = 0;
+                            break;
+                        case PalmImageState.palmRect:
+                            blue = Int16.MaxValue;
+                            red = green = 0;
+                            break;
+                        case PalmImageState.thumbLine:
+                        case PalmImageState.wristLine:
+                            green = Int16.MaxValue;
+                            blue = red = 0;
+                            break;
                     }
 
-                    // show palm contour
-                    if (videoHandle.isPalmPointAt(x, y))
+                    // touch
+                    switch (videoHandle.getTouchImageStateAt(x, y))
                     {
-                        red = Int16.MaxValue;
-                        green = blue = 0;
+                        case TouchImageState.touchArea:
+                            red = (Int16)(red / 2);
+                            green = (Int16)(green / 2);
+                            blue = (Int16)(blue / 2);
+                            break;
+                        case TouchImageState.touchAreaMatched:
+                            red = (Int16)(red / 2);
+                            green = (Int16)(green / 2);
+                            blue = Int16.MaxValue;
+                            break;
+                        case TouchImageState.touchAreaStatusBar:
+                            red = (Int16)(red / 2);
+                            green = Int16.MaxValue;
+                            blue = (Int16)(blue / 2);
+                            break;
+                        case TouchImageState.touchDetected:
+                            red = blue = 0;
+                            green = Int16.MaxValue;
+                            break;
+                        case TouchImageState.touchTracked:
+                            red = Int16.MaxValue;
+                            green = blue = 0;
+                            break;
                     }
 
                     depthTextureData[index] = red;

bbiwarg/Images/FingerImage.cs

@@ -27,10 +27,6 @@ namespace bbiwarg.Images
             image = new Image<Gray, byte>(width, height);
         }
 
-        public void setFingerAt(int x, int y, FingerImageState fis) {
-            image.Data[y, x, 0] = (byte) fis;
-        }
-
         public FingerImageState getStateAt(int x, int y) {
             return (FingerImageState)image.Data[y, x, 0];
         }

bbiwarg/Images/PalmImage.cs

@@ -14,282 +14,37 @@ using bbiwarg.Detectors.Touch;
 
 namespace bbiwarg.Images
 {
-    class PalmImage
+    public enum PalmImageState
     {
-        DepthImage depthImage;
-        EdgeImage edgeImage;
-
-        TouchDetector touchDetector;
-
-        private Image<Gray, Byte> handImage;
-        private Image<Gray, Byte> pointingHandMask;
-        private Image<Gray, Byte> outputImage;
-
-        private List<Finger> fingers;
-
-        private Contour<Point> palmContour;
-        private List<MCvConvexityDefect> convexityDefects;
-        private Vector2D wristPoint, wristDirection;
-        private LineSegment2DF wristLine, thumbLine;
-        private MCvBox2D palmRect;
-
-        private int width, height;
-        
-        public PalmImage(DepthImage depthImage, EdgeImage edgeImage, FingerDetector fingerDetector, TouchDetector touchDetector)
-        {
-            this.depthImage = depthImage;
-            this.edgeImage = edgeImage;
-
-            this.touchDetector = touchDetector;
-
-            handImage = depthImage.getImage().Convert<Byte>(delegate(short s) { return (s == depthImage.getMaxDepth()) ? (byte) 0 : (byte) 1; });
-            width = depthImage.getWidth();
-            height = depthImage.getHeight();
-
-            outputImage = new Image<Gray, byte>(width, height, new Gray(0));
-
-            fingers = getFingersWithoutThumb(fingerDetector);
-            buildPointingHandMask();
-            handImage = handImage.And(pointingHandMask);
-
-            findLongestPalmContour();
-            if (palmContour != null)
-            {
-                findConvexityDefactsSortedByDepth();
-                removeConvexityDefectsNearFingerTips();
-
-                findWristLine();
-                findThumbLine();
-
-                removePointsFromContour(wristLine, 1);
-                removePointsFromContour(thumbLine, 1);
-
-                findPalmRect();
-
-                draw();
-            }
-        }
-
-        private List<Finger> getFingersWithoutThumb(FingerDetector fingerDetector)
-        {
-            List<Finger> detectedFingers = fingerDetector.Fingers;
-            Finger leftMost = null;
-            float minX = float.MaxValue;
-            foreach (Finger f in detectedFingers)
-            {
-                float midX = ((f.Hand + f.Tip) / 2).X;
-                if (midX < minX)
-                {
-                    minX = midX;
-                    leftMost = f;
-                }
-            }
-
-            List<Finger> result = new List<Finger>();
-            foreach (Finger f in detectedFingers)
-            {
-                if (f != leftMost)
-                    result.Add(f);
-            }
-            return result;
-        }
-
-        private void fillFingerSlices(Image<Gray, Byte> image, byte val)
-        {
-            foreach (Finger f in fingers)
-            {
-                foreach (FingerSlice s in f.SliceTrail.getSlices())
-                {
-                    image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
-                }            
-            }
-        }
-
-        private Finger getLongestFinger()
-        {
-            float maxLength = 0;
-            Finger longest = null;
-            foreach (Finger f in fingers)
-            {
-                
-                if (f.Line.Length > maxLength)
-                {
-                    maxLength = f.Line.Length;
-                    longest = f;
-                }
-            }
-            return longest;
-        }
-
-        private Point getPointInPointingHand()
-        {
-            Finger finger = getLongestFinger();
-            if (finger == null)
-                return new Point(0, 0);
-            Vector2D direction = (finger.Hand - finger.Tip).normalize();
-            Vector2D pos = finger.Hand + 20 * direction;
-            return new Point(Math.Min(width - 1, Math.Max(0, (int)pos.X)), Math.Min(height - 1, Math.Max(0, (int)pos.Y)));
-        }
-
-        private void buildPointingHandMask()
-        {
-            pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
-            fillFingerSlices(pointingHandMask, 1);
-            pointingHandMask = pointingHandMask.Dilate(1);
-            pointingHandMask = pointingHandMask.Or(edgeImage.getImage().Convert<Byte>(delegate(byte b) { return (b == 0) ? (byte)0 : (byte)1; }));
-            pointingHandMask = pointingHandMask.Dilate(1);
-
-            MCvConnectedComp tmp = new MCvConnectedComp();
-            CvInvoke.cvFloodFill(pointingHandMask.Ptr, getPointInPointingHand(), new MCvScalar(2), new MCvScalar(0), new MCvScalar(0), out tmp, 0, IntPtr.Zero);
-            pointingHandMask = pointingHandMask.Convert<Byte>(delegate(byte b) { return (b == 2) ? (byte)0 : (byte)1; });
-            pointingHandMask = pointingHandMask.Erode(1);
-            fillFingerSlices(pointingHandMask, 0);
-            pointingHandMask = pointingHandMask.Erode(2);
-        }
-
-        private void findLongestPalmContour()
-        {
-            Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
-
-            palmContour = contour;
-            double maxPerimeter = 0;
-            while (contour != null)
-            {
-                if (contour.Perimeter > maxPerimeter)
-                {
-                    maxPerimeter = contour.Perimeter;
-                    palmContour = contour;
-                }
-                contour = contour.HNext;
-            }
-        }
-
-        private void findWristDirection()
-        {
-            PointF[] points = new PointF[palmContour.Count<Point>()];
-            int index = 0;
-            foreach (Point p in palmContour)
-            {
-                points[index] = new PointF(p.X, p.Y);
-                ++index;
-            }
-            PointF direction, tmp;
-            PointCollection.Line2DFitting(points, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out direction, out tmp);
-            wristDirection = new Vector2D(-direction.Y, direction.X);
-        }
-
-        private void findConvexityDefactsSortedByDepth()
-        {
-            convexityDefects = new List<MCvConvexityDefect>(palmContour.GetConvexityDefacts(new MemStorage(), Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE));
-            convexityDefects.Sort(delegate(MCvConvexityDefect d1, MCvConvexityDefect d2) {
-                if (d1.Depth < d2.Depth)
-                    return 1;
-                else if (d1.Depth > d2.Depth)
-                    return -1;
-                return 0;
-            });
-        }
-
-        private void removeConvexityDefectsNearFingerTips()
-        {
-            List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
-            foreach (MCvConvexityDefect d in convexityDefects)
-            {
-                float minFingerTipDist = float.MaxValue;
-                foreach (Finger f in fingers)
-                {
-                    float dist = f.Tip.getDistanceTo(new Vector2D(d.DepthPoint));
-                    if (dist < minFingerTipDist)
-                        minFingerTipDist = dist;
-                }
-                if (minFingerTipDist > 20)
-                    newDefects.Add(d);
-            }
-            convexityDefects = newDefects;
-        }
-
-        private void findWristPoint()
-        {
-            if (convexityDefects.Count > 1)
-                wristPoint = new Vector2D(convexityDefects[1].DepthPoint);
-            else
-                wristPoint = new Vector2D(-1, -1);
-        }
-
-        private void findWristLine()
-        {
-            findWristPoint();
-            findWristDirection();
-            wristLine = new LineSegment2DF(wristPoint - 1000 * wristDirection, wristPoint + 1000 * wristDirection);
-        }
+        none = 0,
+        palmContour = 1,
+        wristLine = 2,
+        thumbLine = 3,
+        palmRect = 4
+    }
 
-        private void findThumbLine()
-        {
-            if (convexityDefects.Count > 0)
-            {
-                MCvConvexityDefect thumbDefect = convexityDefects[0];
-                Vector2D p1 = new Vector2D(thumbDefect.DepthPoint);
-                Vector2D p2 = new Vector2D(thumbDefect.StartPoint);
-                Vector2D direction = (p1 - p2).normalize();
-                thumbLine = new LineSegment2DF(p1 - 1000 * direction, p1 + 1000 * direction);
-            }
-            else
-            {
-                thumbLine = new LineSegment2DF(new PointF(-1, -1), new PointF(-1, -1));
-            }
-        }
+    class PalmImage
+    {
+        private Image<Gray, Byte> image;
 
-        private void removePointsFromContour(LineSegment2DF line, int sideToRemove)
+        public PalmImage(int width, int height)
         {
-            Contour<Point> newContour = new Contour<Point>(new MemStorage());
-            int index = 0;
-            foreach (Point p in palmContour)
-            {
-                if (line.Side(p) != sideToRemove)
-                {
-                    newContour.Insert(index, p);
-                    ++index;
-                }
-            }
-            palmContour = newContour;
+            image = new Image<Gray, byte>(width, height);
         }
 
-        private void findPalmRect()
+        public PalmImageState getStateAt(int x, int y)
         {
-            palmRect = palmContour.GetMinAreaRect();
+            return (PalmImageState)image.Data[y, x, 0];
         }
 
-        private void draw()
+        public void drawContour(Contour<Point> contour)
         {
-            //outputImage.Draw(palmContour, new Gray(1), 1);
-            PointF[] vertices = palmRect.GetVertices();
-            for (int i = 0; i < 4; ++i)
-                outputImage.Draw(new LineSegment2DF(vertices[i], vertices[(i + 1) % 4]), new Gray(1), 1);
-
-            Vector2D origin = new Vector2D(vertices[1]);
-            
-            Matrix<float> tmp = new Matrix<float>(new float[,] {{vertices[0].X - origin.X, vertices[2].X - origin.X}, {vertices[0].Y - origin.Y, vertices[2].Y - origin.Y}});
-            Matrix<float> T = new Matrix<float>(2, 2);
-            CvInvoke.cvInvert(tmp.Ptr, T.Ptr, Emgu.CV.CvEnum.SOLVE_METHOD.CV_LU);
-
-            foreach (TouchEvent e in touchDetector.getTouchEvents())
-            {
-                Vector2D point = new Vector2D(e.getX(), e.getY());
-                Matrix<float> res = T.Mul(new Matrix<float>(new float[,] { { point.X - origin.X }, { point.Y - origin.Y } }));
-                Vector2D resVector = new Vector2D(res.Data[0, 0], res.Data[1, 0]);
-
-                Console.WriteLine("touch at " + resVector);
-                outputImage.Draw(new Cross2DF(point, 3, 3), new Gray(1), 1);
-            }
-            Console.WriteLine();
-
-            outputImage.Draw(wristLine, new Gray(1), 1);
-            outputImage.Draw(thumbLine, new Gray(1), 1);
+            image.Draw(contour, new Gray((byte)PalmImageState.palmContour), 1);
         }
 
-        public bool belongsToPalm(int x, int y)
+        public void drawLine(LineSegment2DF line, PalmImageState state)
         {
-            return outputImage.Data[y, x, 0] == 1;
+            image.Draw(line, new Gray((byte)state), 1);
         }
     }
 }

bbiwarg/Utility/HelperFunctions.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bbiwarg.Utility
+{
+    public static class HelperFunctions
+    {
+        public static T thresholdRange<T>(T min, T max, T value)
+        {
+            if (Comparer<T>.Default.Compare(min, value) > 0)
+                return min;
+            if (Comparer<T>.Default.Compare(value, max) > 0)
+                return max;
+            return value;
+        }
+    }
+}

bbiwarg/VideoHandle.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.Threading.Tasks;
 using System.Diagnostics;
 using bbiwarg.Detectors.Fingers;
+using bbiwarg.Detectors.Palm;
 using bbiwarg.Detectors.Touch;
 using bbiwarg.Images;
 using bbiwarg.InputProviders;
@@ -22,11 +23,12 @@ namespace bbiwarg
         private int height;
         private DepthImage depthImage;
         private EdgeImage edgeImage;
-        private TouchImage touchImage;
         private PalmImage palmImage;
+        private TouchImage touchImage;
         private FingerImage fingerImage;
 
         private FingerDetector fingerDetector;
+        private PalmDetector palmDetector;
         private TouchDetector touchDetector;
 
         private FingerTracker fingerTracker;
@@ -111,14 +113,20 @@ namespace bbiwarg
             return fingerImage.getStateAt(x, y);
         }
 
-        public bool isPalmPointAt(int x, int y) {
-            return palmImage.belongsToPalm(x, y);
+        public PalmImageState getPalmImageStateAt(int x, int y)
+        {
+            return palmImage.getStateAt(x, y);
         }
 
         public TouchImageState getTouchImageStateAt(int x, int y) {
             return touchImage.getStateAt(x, y);
         }
 
+        public List<TouchEvent> getTrackedTouchEvents()
+        {
+            return touchTracker.getTouchEvents();
+        }
+
         private void processFrameUpdate()
         {
             //read data from inputProvider
@@ -139,16 +147,18 @@ namespace bbiwarg
             edgeImage = new EdgeImage(depthImage);
             touchImage = new TouchImage(width, height);
             fingerImage = new FingerImage(width, height);
+            palmImage = new PalmImage(width, height);
 
             //detect+track fingers
             fingerDetector = new FingerDetector(depthImage, edgeImage, fingerImage);
             fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, fingerImage);
 
+            //detect palm
+            palmDetector = new PalmDetector(depthImage, edgeImage, fingerDetector, palmImage);
+
             //detect+track touchEvents
-            touchDetector = new TouchDetector(fingerTracker.getFingers(), depthImage, touchImage);
+            touchDetector = new TouchDetector(fingerTracker.getFingers(), depthImage, touchImage, palmDetector.getPalm());
             touchTracker.setDetectedTouchEventsThisFrame(touchDetector.getTouchEvents(), touchImage);
-
-            palmImage = new PalmImage(depthImage, edgeImage, fingerDetector, touchDetector);
         }
     }
 }

bbiwarg/bbiwarg.csproj

@@ -71,6 +71,8 @@
     <Compile Include="Detectors\Fingers\FingerSliceTrail.cs" />
     <Compile Include="Detectors\Fingers\FingerSlice.cs" />
     <Compile Include="Detectors\Fingers\FingerTracker.cs" />
+    <Compile Include="Detectors\Palm\Palm.cs" />
+    <Compile Include="Detectors\Palm\PalmDetector.cs" />
     <Compile Include="Detectors\Touch\TouchDetector.cs" />
     <Compile Include="Detectors\Touch\TouchEvent.cs" />
     <Compile Include="Detectors\Touch\TouchTracker.cs" />
@@ -85,6 +87,7 @@
     <Compile Include="InputProvider\IisuInputProvider.cs" />
     <Compile Include="MainBBWIWARG.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Utility\HelperFunctions.cs" />
     <Compile Include="Utility\Line2D.cs" />
     <Compile Include="Utility\Vector.cs" />
     <Compile Include="Utility\Vector2D.cs" />