improved fingerDetection + changed vector distance to float

bbiwarg/Detectors/Fingers/Finger.cs

@@ -14,31 +14,41 @@ namespace bbiwarg.Detectors.Fingers
     class Finger
     {
         private List<Vector<int>> fingerPoints;
+        private bool fingerEndPointsUpToDate = false;
         private bool lineUpToDate = false;
+        private float length;
+        private Vector<int> tipPoint;
+        private Vector<int> handPoint;
         private Vector<float> direction;
         private Vector<float> pointOnLine;
         private Vector<float> lineEndPoint1;
         private Vector<float> lineEndPoint2;
-        private Vector<int> tipPoint;
-        private Vector<int> handPoint;
-        private float length;
-        
-        public Finger(Vector<int> fingerPoint)
-        {
-            fingerPoints = new List<Vector<int>>();
-            addFingerPoint(fingerPoint);
+
+        public Finger(List<Vector<int>> fingerPoints) {
+            this.fingerPoints = fingerPoints;
         }
 
-        public Vector<float> getLineEndPoint1()
-        {
-            if (!lineUpToDate) updateLine();
-            return lineEndPoint1;
+        public float getLength() {
+            if (!fingerEndPointsUpToDate) updateFingerEndPoints();
+            return length;
         }
 
-        public Vector<float> getLineEndPoint2()
-        {
-            if (!lineUpToDate) updateLine();
-            return lineEndPoint2;
+        public int getNumFingerPoints() {
+            return fingerPoints.Count;
+        }
+
+        public List<Vector<int>> getFingerPoints() {
+            return fingerPoints;
+        }
+
+        public Vector<int> getTipPoint() {
+            if (!fingerEndPointsUpToDate) updateFingerEndPoints();
+            return tipPoint;
+        }
+
+        public Vector<int> getHandPoint() {
+            if (!fingerEndPointsUpToDate) updateFingerEndPoints();
+            return handPoint;
         }
 
         public Vector<float> getDirection()
@@ -47,40 +57,37 @@ namespace bbiwarg.Detectors.Fingers
             return direction;
         }
 
-        public Vector<int> getTipPoint()
+        public Vector<float> getLineEndPoint1()
         {
             if (!lineUpToDate) updateLine();
-            return tipPoint;
+            return lineEndPoint1;
         }
 
-        public float getLength()
+        public Vector<float> getLineEndPoint2()
         {
             if (!lineUpToDate) updateLine();
-            return length;
+            return lineEndPoint2;
         }
 
-        public void addFingerPoint(Vector<int> fingerPoint)
-        {
+        public void addFingerPoint(Vector<int> fingerPoint) {
             fingerPoints.Add(fingerPoint);
+            fingerEndPointsUpToDate = false;
             lineUpToDate = false;
         }
 
-        public float getMinDistance(Vector<int> fingerPoint)
-        {
-            float minDinstance = float.MaxValue;
-            foreach (Vector<int> fp in fingerPoints)
-            {
-                float distance = fingerPoint.subDistance(fp, 2);
-                if (distance < minDinstance)
-                {
-                    minDinstance = distance;
-                }
+        public bool isWithinDistance(Vector<int> point, float maxDistance) {
+            foreach (Vector<int> fp in fingerPoints) {
+                if (fp.subDistance(point, 2) <= maxDistance)
+                    return true;
             }
-
-            return minDinstance;
+            return false;
         }
 
-        public float getSimilarity(Finger compareFinger) {
+        public float getSimilarity(Finger compareFinger)
+        {
+            if (!fingerEndPointsUpToDate) updateFingerEndPoints();
+            if (!lineUpToDate) updateLine();
+
             //startDistance
             float maxStartDistance = 100;
             float xDiffStart = lineEndPoint1.x - compareFinger.getLineEndPoint1().x;
@@ -101,40 +108,24 @@ namespace bbiwarg.Detectors.Fingers
             float lengthCompareDirection = compareFinger.getDirection().norm();
             float directionSimilarity = Math.Abs(scalaProduct / (lengthDirection * lengthCompareDirection));
 
-            //Console.WriteLine(Math.Round(directionSimilarity, 2) + "###" + Math.Round(startSimilarity, 2) + "###" + Math.Round(endSimilarity, 2));
-
             return (startSimilarity + endSimilarity + directionSimilarity) / 3;
-            
-        }
 
-        private void updateLine() {
-            //update direction+pointonline
-            PointF[] pointArray = new PointF[fingerPoints.Count];
-            int i = 0;
-            foreach (Vector<int> fp in fingerPoints)
-            {
-                pointArray[i] = new PointF(fp.x, fp.y);
-                ++i;
-            }
-            PointF tempDirection;
-            PointF tempPointOnLine;
-            PointCollection.Line2DFitting(pointArray, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out tempDirection, out tempPointOnLine);
-            direction = new Vector<float>(new float[2]{tempDirection.X, tempDirection.Y});
-            pointOnLine = new Vector<float>(new float[2] { tempPointOnLine.X, tempPointOnLine.Y });
+        }
 
+        private void updateFingerEndPoints() {
             Vector<int> fp1 = fingerPoints[0];
             Vector<int> fp2 = fingerPoints[0];
-            length = 0.0f;
+            length = float.MinValue;
+
             foreach (Vector<int> fp in fingerPoints) {
-                float distanceToFP1 = fp.subDistance(fp1,2);
-                float distanceToFP2 = fp.subDistance(fp2,2);
-                if (length < distanceToFP1 && distanceToFP1 >= distanceToFP2) 
-                {
+                float distanceToFP1 = fp.subDistance(fp1, 2);
+                float distanceToFP2 = fp.subDistance(fp2, 2);
+
+                if (distanceToFP1 > length && distanceToFP1 >= distanceToFP2) {
                     fp2 = fp;
                     length = distanceToFP1;
                 }
-                else if (length < distanceToFP2 && distanceToFP2 > distanceToFP1)
-                {
+                else if (distanceToFP2 > length && distanceToFP2 > distanceToFP1) {
                     fp1 = fp;
                     length = distanceToFP2;
                 }
@@ -145,33 +136,39 @@ namespace bbiwarg.Detectors.Fingers
                 tipPoint = fp1;
                 handPoint = fp2;
             }
-            else 
+            else
             {
                 tipPoint = fp2;
                 handPoint = fp1;
             }
 
-            //update start+end
-            lineEndPoint1 = projectToLine(tipPoint);
-            lineEndPoint2 = projectToLine(handPoint);
-
-            lineUpToDate = true;
+            fingerEndPointsUpToDate = true;
         }
 
-        private Vector<float> projectToLine(Vector<int> p)
-        {
-            float px = p.x, py = p.y, dx = direction.x, dy = direction.y, ox = pointOnLine.x, oy = pointOnLine.y;
-            float diffx = px - ox;
-            float diffy = py - oy;
+        private void updateLine() {
+
+            if (!fingerEndPointsUpToDate) updateFingerEndPoints();
 
-            float diff_d = (diffx * dx + diffy * dy);
-            float d_d = (dx * dx + dy * dy);
-            float q = diff_d / d_d;
+            //update direction+pointOnLine
+            PointF[] pointArray = new PointF[fingerPoints.Count];
+            int i = 0;
+            foreach (Vector<int> fp in fingerPoints)
+            {
+                pointArray[i] = new PointF(fp.x, fp.y);
+                ++i;
+            }
+            PointF tempDirection;
+            PointF tempPointOnLine;
+            PointCollection.Line2DFitting(pointArray, Emgu.CV.CvEnum.DIST_TYPE.CV_DIST_L2, out tempDirection, out tempPointOnLine);
+            direction = new Vector<float>(tempDirection.X, tempDirection.Y);
+            pointOnLine = new Vector<float>(tempPointOnLine.X, tempPointOnLine.Y);
 
-            float newX = ox + q * dx;
-            float newY = oy + q * dy;
+            //update start+end
+            lineEndPoint1 = Vector<float>.projectToLine(tipPoint, direction, pointOnLine);
+            lineEndPoint2 = Vector<float>.projectToLine(handPoint, direction, pointOnLine);
 
-            return new Vector<float>(new float[2]{newX, newY});
+            lineUpToDate = true;
         }
+
     }
 }

bbiwarg/Detectors/Fingers/FingerDetector.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
@@ -16,6 +17,7 @@ namespace bbiwarg.Detectors.Fingers
         private DepthImage depthImage;
         private EdgeImage edgeImage;
         private FingerImage fingerImage;
+        private List<Vector<int>> possibleFingerPoints;
         private List<Finger> possibleFingers;
         private List<Finger> fingers;
 
@@ -40,6 +42,7 @@ namespace bbiwarg.Detectors.Fingers
             int height = depthImage.getHeight();
             int maxFingerSize = 30;
             int minFingerSize = 10;
+            possibleFingerPoints = new List<Vector<int>>();
 
             for (int y = 0; y < height; y++) {
                 for (int x = 0; x < width; x++) {
@@ -61,6 +64,7 @@ namespace bbiwarg.Detectors.Fingers
                             Int16 depthRight = depthImage.getDepthAt(edgeRightX, y);
 
                             if ((edgeRightX - x) < maxFingerSize && depthLeft > depthMid && depthMid < depthRight) {
+                                possibleFingerPoints.Add(new Vector<int>(midX, y, depthMid)); 
                                 fingerImage.setFingerAt(midX, y, FingerImageState.possibleFinger);
                             } 
                         }
@@ -82,6 +86,7 @@ namespace bbiwarg.Detectors.Fingers
                             Int16 depthBottom = depthImage.getDepthAt(x, edgeBottomY);
 
                             if ((edgeBottomY - y) < maxFingerSize && depthTop > depthMid && depthMid < depthBottom) {
+                                possibleFingerPoints.Add(new Vector<int>(x, midY, depthMid));
                                 fingerImage.setFingerAt(x, midY, FingerImageState.possibleFinger);
                             }
 
@@ -95,38 +100,37 @@ namespace bbiwarg.Detectors.Fingers
         {
             int width = depthImage.getWidth();
             int height = depthImage.getHeight();
-            float maxDistanceTogether = 5.0f;
+            float fingerCylinderRadius = 5f;
             possibleFingers = new List<Finger>();
 
-            for (int y = 0; y < height; y++)
-            {
-                for (int x = 0; x < width; x++)
+            foreach (Vector<int> pfp in possibleFingerPoints) {
+                //find nearest finger
+                List<Finger> nearFingerClusters = new List<Finger>();
+
+                for (int i = 0; i < possibleFingers.Count; i++)
                 {
-                    if (fingerImage.getStateAt(x,y) == FingerImageState.possibleFinger)
-                    {
-                        Int16 depth = depthImage.getDepthAt(x, y);
-                        Vector<int> fingerPoint = new Vector<int>(new int[3]{x,y,depth});
-                        float minDistanceValue = float.MaxValue;
-                        int minDistanceIndex = 0;
-                        for (int i = 0; i < possibleFingers.Count; i++)
-                        {
-                            float distance = possibleFingers[i].getMinDistance(fingerPoint);
-                            if (distance < minDistanceValue)
-                            {
-                                minDistanceValue = distance;
-                                minDistanceIndex = i;
-                            }
-                        }
-                        if (minDistanceValue < maxDistanceTogether)
-                        {
-                            possibleFingers[minDistanceIndex].addFingerPoint(fingerPoint);
-                        }
-                        else
-                        {
-                            possibleFingers.Add(new Finger(fingerPoint));
-                        }
+                    if (possibleFingers[i].isWithinDistance(pfp, fingerCylinderRadius)) {
+                        nearFingerClusters.Add(possibleFingers[i]);
                     }
                 }
+
+                if (nearFingerClusters.Count == 0) {
+                    List<Vector<int>> fingerPoints = new List<Vector<int>>();
+                    fingerPoints.Add(pfp);
+                    possibleFingers.Add(new Finger(fingerPoints));
+                }
+                else if (nearFingerClusters.Count == 1)
+                    nearFingerClusters[0].addFingerPoint(pfp);
+                else {
+                    //merge fingers
+                    List<Vector<int>> fingerPoints = new List<Vector<int>>();
+                    fingerPoints.Add(pfp);
+                    for (int j = 0; j < nearFingerClusters.Count; j++) {
+                        fingerPoints.Concat<Vector<int>>(nearFingerClusters[j].getFingerPoints());
+                        possibleFingers.Remove(nearFingerClusters[j]);
+                    }
+                    possibleFingers.Add(new Finger(fingerPoints));
+                }
             }
         }
 
@@ -134,13 +138,15 @@ namespace bbiwarg.Detectors.Fingers
         {   
             int width = depthImage.getWidth();
             int height = depthImage.getHeight();
-            float minFingerLength = 20.0f;
+            float minFingerLength = 15.0f;
+            int minNumFingerPoints = 15;
             fingers = new List<Finger>();
 
             foreach (Finger finger in possibleFingers)
             {
                 float length = finger.getLength();
-                if (length > minFingerLength)
+                int numFingerPoints = finger.getNumFingerPoints();
+                if (length > minFingerLength && numFingerPoints > minNumFingerPoints)
                 {
                     fingers.Add(finger);
                 }
@@ -148,9 +154,7 @@ namespace bbiwarg.Detectors.Fingers
         }
 
         private void setFingerPoints() {
-            int width = depthImage.getWidth();
-            int height = depthImage.getHeight();
-
+            //fingerImage.setFingers(fingers, FingerImageState.fingerDetected);
             foreach (Finger finger in fingers) {
                 Vector<float> lineEndPoint1 = finger.getLineEndPoint1();
                 Vector<float> lineEndPoint2 = finger.getLineEndPoint2();

bbiwarg/Detectors/Touch/TouchDetector.cs

@@ -35,8 +35,8 @@ namespace bbiwarg.Detectors.Touch
                     //correct touchEvent position
                     Vector<float> direction = finger.getDirection();
                     float directionFactor = -10;
-                    float x = Math.Min(Math.Max(tipPoint.x + directionFactor * direction.x, 0), depthImage.getWidth());
-                    float y = Math.Min(Math.Max(tipPoint.y + directionFactor * direction.y, 0), depthImage.getHeight());
+                    float x = Math.Min(Math.Max(tipPoint.x + directionFactor * direction.x, 0), depthImage.getWidth()-1);
+                    float y = Math.Min(Math.Max(tipPoint.y + directionFactor * direction.y, 0), depthImage.getHeight()-1);
                     Vector<float> tep = new Vector<float>(new float[2]{x,y});
 
                     touchImage.setTouchAt((int)tep.x, (int)tep.y, TouchImageState.touchDetected);

bbiwarg/Images/FingerImage.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using Emgu.CV;
 using Emgu.CV.Structure;
 using bbiwarg.Utility;
+using bbiwarg.Detectors.Fingers;
 
 namespace bbiwarg.Images
 {
@@ -19,20 +20,28 @@ namespace bbiwarg.Images
 
     class FingerImage
     {
-        private Image<Gray, Int16> image;
+        private Image<Gray, byte> image;
 
-        public FingerImage(DepthImage depthImage) {
-            this.image = depthImage.getImage().CopyBlank();
+        public FingerImage(int width, int height) {
+            image = new Image<Gray, byte>(width, height);
         }
 
         public void setFingerAt(int x, int y, FingerImageState fis) {
-            image.Data[y, x, 0] = (Int16) fis;
+            image.Data[y, x, 0] = (byte) fis;
         }
 
         public FingerImageState getStateAt(int x, int y) {
             return (FingerImageState)image.Data[y, x, 0];
         }
 
+        public void setFingers(List<Finger> fingers, FingerImageState state) {
+            foreach (Finger finger in fingers) {
+                foreach (Vector<int> fp in finger.getFingerPoints()) {
+                    setFingerAt(fp.x, fp.y, state);
+                }
+            }
+        }
+
         public void drawLine(Vector<float> start, Vector<float> end, FingerImageState state)
         {
             int width = image.Width;

bbiwarg/Images/PalmImage.cs

@@ -48,7 +48,7 @@ namespace bbiwarg.Images
                 contour = contour.HNext;
             }
 
-            Console.WriteLine("numContours: " + count + "\nmaxPerimeter = " + maxPerimeter);
+            //Console.WriteLine("numContours: " + count + "\nmaxPerimeter = " + maxPerimeter);
 
             inContour = new bool[width, height];
             foreach (Point p in maxContour)

bbiwarg/Images/TouchImage.cs

@@ -18,14 +18,14 @@ namespace bbiwarg.Images
 
     class TouchImage
     {
-        private Image<Gray, Int16> image;
+        private Image<Gray, byte> image;
 
-        public TouchImage(DepthImage depthImage) {
-            image = depthImage.getImage().CopyBlank();
+        public TouchImage(int width, int height) {
+            image = new Image<Gray, byte>(width, height);
         }
 
         public void setTouchAt(int x, int y, TouchImageState tis) {
-            image.Data[y, x, 0] = (Int16) tis;
+            image.Data[y, x, 0] = (byte) tis;
         }
 
         public TouchImageState getStateAt(int x, int y) {

bbiwarg/MainBBWIWARG.cs

@@ -12,7 +12,7 @@ namespace bbiwarg
     {
         static void Main(string[] args)
         {
-            IInputProvider inputProvider = new IisuInputProvider();//"..\\..\\videos\\touch\\4.skv");
+            IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\touch\\4.skv");
             VideoHandle videoHandle = new VideoHandle(inputProvider);
             videoHandle.start();
 

bbiwarg/Utility/Vector.cs

@@ -10,38 +10,23 @@ namespace bbiwarg.Utility
     {
         private T[] elements;
 
-        public T x
+        public T x { get { return elements[0]; } set { elements[0] = value; } }
+        public T y { get { return elements[1]; } set { elements[1] = value; } }
+        public T z { get { return elements[2]; } set { elements[2] = value; } }
+
+        public Vector(T x, T y)
         {
-            get
-            {
-                return elements[0];
-            }
-            set
-            {
-                elements[0] = value;
-            }
+            elements = new T[2];
+            elements[0] = x;
+            elements[1] = y;
         }
-        public T y
+
+        public Vector(T x, T y, T z)
         {
-            get
-            {
-                return elements[1];
-            }
-            set
-            {
-                elements[1] = value;
-            }
-        }
-        public T z
-        {
-            get
-            {
-                return elements[2];
-            }
-            set
-            {
-                elements[2] = value;
-            }
+            elements = new T[3];
+            elements[0] = x;
+            elements[1] = y;
+            elements[2] = z;
         }
 
         public Vector(int numberOfElements)
@@ -96,6 +81,7 @@ namespace bbiwarg.Utility
                 elements[i] = (dynamic)elements[i] - subtrahend[i];
             }
         }
+
         public void multiply(T scalar)
         {
             for (int i = 0; i < elements.Length; i++)
@@ -103,23 +89,32 @@ namespace bbiwarg.Utility
                 elements[i] = (dynamic)scalar * elements[i];
             }
         }
-        public T norm()
+
+        public float norm()
         {
             return subNorm(elements.Length);
         }
-        public T subNorm(int subLength)
+
+        public float subNorm(int subLength)
         {
             T result = (dynamic)0;
             for (int i = 0; i < subLength; i++)
             {
                 result += (dynamic)elements[i] * elements[i];
             }
-            return (T)Math.Sqrt((dynamic)result);
+            return (float)Math.Sqrt((dynamic)result);
         }
-        public T distance(Vector<T> vector)
+
+        public float distance(Vector<T> vector)
         {
             return (vector - this).norm();
         }
+
+        public float subDistance(Vector<T> vector, int subLength)
+        {
+            return (vector - this).subNorm(subLength);
+        }
+
         public T sum()
         {
             T result = (dynamic)0;
@@ -131,7 +126,7 @@ namespace bbiwarg.Utility
         }
         public Vector<T> normalize() 
         {
-            T norm = this.norm();
+            float norm = this.norm();
             Vector<T> result = new Vector<T>(this);
             for (int i = 0; i < result.length(); i++)
             {
@@ -140,11 +135,6 @@ namespace bbiwarg.Utility
             return result;
         }
 
-        public T subDistance(Vector<T> vector, int subLength)
-        {
-            return (vector - this).subNorm(subLength);
-        }
-
 
         private static void checkLength(Vector<T> vector1, Vector<T> vector2)
         {
@@ -249,5 +239,21 @@ namespace bbiwarg.Utility
             return distance;
 
         }
+
+        public static Vector<float> projectToLine(Vector<int> p, Vector<float> direction, Vector<float> pointOnLine)
+        {
+            float px = p.x, py = p.y, dx = direction.x, dy = direction.y, ox = pointOnLine.x, oy = pointOnLine.y;
+            float diffx = px - ox;
+            float diffy = py - oy;
+
+            float diff_d = (diffx * dx + diffy * dy);
+            float d_d = (dx * dx + dy * dy);
+            float q = diff_d / d_d;
+
+            float newX = ox + q * dx;
+            float newY = oy + q * dy;
+
+            return new Vector<float>(newX, newY);
+        }
     }
 }

bbiwarg/VideoHandle.cs

@@ -117,8 +117,9 @@ namespace bbiwarg
 
             //create images
             edgeImage = new EdgeImage(depthImage);
-            touchImage = new TouchImage(depthImage);
-            fingerImage = new FingerImage(depthImage);
+            palmImage = new PalmImage(edgeImage);
+            touchImage = new TouchImage(width, height);
+            fingerImage = new FingerImage(width, height);
 
             //detect+track fingers
             fingerDetector = new FingerDetector(depthImage, edgeImage, fingerImage);
@@ -126,10 +127,7 @@ namespace bbiwarg
 
             //detect+track touchEvents
             touchDetector = new TouchDetector(fingerTracker.getFingers(), depthImage, touchImage);
-            //touchDetector = new TouchDetector(fingerDetector.getFingers(), depthImage, touchImage);
             touchTracker.setDetectedTouchEventsThisFrame(touchDetector.getTouchEvents(), touchImage);
-
-            palmImage = new PalmImage(edgeImage);
         }
     }
 }