cleanup, refactoring, parameter tuning

bbiwarg/Constants.cs

@@ -39,21 +39,25 @@ namespace bbiwarg
 
         // finger detection
         public static readonly int FingerStepSize = 2;
-        public static readonly int FingerMinNumSlices = 30 / FingerStepSize;
-        public static readonly int FingerRemoveNumSlicesForCorrection = 5;
         public static readonly int FingerMaxGapCounter = 5;
         public static readonly int FingerMaxSliceDifferencePerStep = 5;
-        public static readonly int FingerMaxSize = 30;
-        public static readonly int FingerMinSize = 5;
+        public static readonly int FingerMinNumSlices = 30 / FingerStepSize;
+        public static readonly int FingerMaxWidth = 30;
+        public static readonly int FingerMinWidth = 5;
+        public static readonly int FingerRemoveNumSlicesForCorrection = 5;
         public static readonly int FingerNumSlicesForRelativeDirection = FingerRemoveNumSlicesForCorrection;
-        public static readonly int FingerOutSliceFactor = 10;
         public static readonly int FingerContourMargin = 2;
         public static readonly int FingerSliceOverlapFactor = 2;
         public static readonly int FingerCrippleOutFactor = 8;
         public static readonly int FingerCrippleOutMinDifference = 15;
+
+        // finger tracking
         public static readonly int FingerNumFramesUntilTracked = 5;
         public static readonly int FingerNumFramesUntilLost = 10;
         public static readonly float FingerMinSimilarityForTracking = 0.1f;
+        public static readonly float FingerSimilarityMaxAngle = (float)(30 * Math.PI / 180); // 30°
+        public static readonly float FingerSimilarityMaxParallelDistance = 80;
+        public static readonly float FingerSimilarityMaxVerticalDistance = 20;
 
         // hand detection
         public static readonly float HandMaxSize = 0.7f;
@@ -78,15 +82,20 @@ namespace bbiwarg
         public static readonly int PalmGridColumns = 3;
 
         // touch detection
-        public static readonly float TouchEventMinFloodValue = 0.2f;
-        public static readonly int TouchEventTipCorrectionFactor = 7;
-        public static readonly int TouchEventAreaSize = 25;
-        public static readonly int TouchEventNumFramesUntilTracked = 2;
+        public static readonly float TouchEventMinTouchValue = 0.4f;
+        public static readonly int TouchEventAreaSize = 30;
+        public static readonly int TouchEventFloodfillLowDiff = 1;
+        public static readonly int TouchEventFloodfillHighDiff = 2;
+        public static readonly int TouchEventTipInsideFactor = 2;
+        public static readonly int TouchEventTipOutsideFactor = 7;
         public static readonly float TouchmXX = 0.005481396f;
         public static readonly float TouchmXY = 0.0003090923f;
         public static readonly float TouchmYY = 0.0005702336f;
         public static readonly float TouchProcessNoise = 3.0e-4f;
 
+        // touch tracking
+        public static readonly int TouchEventNumFramesUntilTracked = 2;
+
         // colors
         public static readonly Color ColorDetected = Color.Turquoise;
         public static readonly Color ColorTracked = Color.Yellow;

bbiwarg/Recognition/FingerRecognition/Finger.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Utility;
 using bbiwarg.Graphics;
+using bbiwarg.Recognition.HandRecognition;
 using Emgu.CV;
 
 namespace bbiwarg.Recognition.FingerRecognition
@@ -17,41 +18,33 @@ namespace bbiwarg.Recognition.FingerRecognition
         public Vector2D HandPoint { get { return SliceTrail.EndSlice.Mid; } }
         public Vector2D MidPoint { get { return SliceTrail.MidSlice.Mid; } }
         public Vector2D Direction { get { return LineSegment.Direction; } }
+        public Vector2D TipDirection { get { return SliceTrail.getStartDirection(); } }
+        public Vector2D HandDirection { get { return SliceTrail.getEndDirection(); } }
         public LineSegment2D LineSegment { get { return SliceTrail.LineSegment; } }
         public FingerSliceTrail SliceTrail { get; private set; }
         public Contour<Point> Contour { get { return SliceTrail.getContour(); } }
 
         public Finger(FingerSliceTrail sliceTrail)
         {
-            //check direction
-            if (sliceTrail.StartSlice.Length > sliceTrail.EndSlice.Length)
-                sliceTrail.reverse();
-
             SliceTrail = sliceTrail;
         }
 
-        //TODO: redo (not use similarity but actual distances and angle instead)
         public float getSimilarity(Finger compareFinger)
         {
             LineSegment2D compareLineSegment = compareFinger.LineSegment;
             Line2D compareLine = compareLineSegment.Line;
 
-            //thresholds
-            float thresholdMaxAngle = (float)(30 * Math.PI / 180); // 30°
-            float thresholdMaxParallelDistance = 80;
-            float thresholdMaxVerticalDistance = 40;
-
             //check angle
             float angle = LineSegment.Line.getAngleBetween(compareLine);
-            float angleSimilarity = Math.Max(1 - angle / thresholdMaxAngle, 0);
+            float angleSimilarity = Math.Max(1 - angle / Constants.FingerSimilarityMaxAngle, 0);
 
             //check parallel distance
             float parallelDistance = LineSegment.getParallelDistanceTo(compareLineSegment);
-            float parallelDistanceSimilarity = Math.Max(1 - parallelDistance / thresholdMaxParallelDistance, 0);
+            float parallelDistanceSimilarity = Math.Max(1 - parallelDistance / Constants.FingerSimilarityMaxParallelDistance, 0);
 
             //check vertical distance
             float verticalDistance = LineSegment.getVerticalDistanceTo(compareLineSegment);
-            float verticalDistanceSimilarity = Math.Max(1 - verticalDistance / thresholdMaxVerticalDistance, 0);
+            float verticalDistanceSimilarity = Math.Max(1 - verticalDistance / Constants.FingerSimilarityMaxVerticalDistance, 0);
 
             return (angleSimilarity + parallelDistanceSimilarity + verticalDistanceSimilarity) / 3;
         }

bbiwarg/Recognition/FingerRecognition/FingerDetector.cs

@@ -152,10 +152,10 @@ namespace bbiwarg.Recognition.FingerRecognition
             Vector2D dirStart = direction.getOrthogonal(true);
             Vector2D dirEnd = direction.getOrthogonal(false);
 
-            Vector2D start = edgeImageAdapted.findNextEdge(position, dirStart, Constants.FingerMaxSize);
+            Vector2D start = edgeImageAdapted.findNextEdge(position, dirStart, Constants.FingerMaxWidth);
             if (start == null) return null;
 
-            Vector2D end = edgeImageAdapted.findNextEdge(position, dirEnd, Constants.FingerMaxSize);
+            Vector2D end = edgeImageAdapted.findNextEdge(position, dirEnd, Constants.FingerMaxWidth);
             if (end == null) return null;
 
             return getFingerSlice(start, end);
@@ -163,7 +163,7 @@ namespace bbiwarg.Recognition.FingerRecognition
         private FingerSlice findFingerSliceFromStartEdge(Vector2D start, Vector2D direction)
         {
             Vector2D searchStart = start + Constants.FingerSliceOverlapFactor * direction;
-            Vector2D end = edgeImageAdapted.findNextEdge(searchStart, direction, Constants.FingerMaxSize);
+            Vector2D end = edgeImageAdapted.findNextEdge(searchStart, direction, Constants.FingerMaxWidth);
             if (end == null) return null;
 
             return getFingerSlice(start, end);
@@ -177,7 +177,7 @@ namespace bbiwarg.Recognition.FingerRecognition
             Vector2D behindEnd = (end + Constants.FingerSliceOverlapFactor * direction).moveInBound(Vector2D.Zero, depthImage.BottomRight, directionInv);
 
             FingerSlice slice = new FingerSlice(beforeStart, behindEnd);
-            if (slice.Length >= Constants.FingerMinSize && slice.Length <= Constants.FingerMaxSize && fingerSliceDepthTest(slice))
+            if (slice.Length >= Constants.FingerMinWidth && slice.Length <= Constants.FingerMaxWidth && fingerSliceDepthTest(slice))
                 return slice;
 
             return null;
@@ -223,54 +223,16 @@ namespace bbiwarg.Recognition.FingerRecognition
             return (minDepthDifference < Constants.FingerCrippleOutMinDifference);
         }
 
-        private FingerSlice findOutSlice(Vector2D start, Vector2D direction)
-        {
-            Vector2D dirOrth1 = direction.getOrthogonal(true);
-            Vector2D dirOrth2 = direction.getOrthogonal(false);
-
-            Vector2D outPoint = (start + Constants.FingerOutSliceFactor * direction).moveInBound(Vector2D.Zero, depthImage.BottomRight, direction.getInverse());
-            Vector2D p1 = edgeImageOriginal.findNextEdge(outPoint, dirOrth1, 0, true, false);
-            Vector2D p2 = edgeImageOriginal.findNextEdge(outPoint, dirOrth2, 0, true, false);
-
-            FingerSlice slice = new FingerSlice(p1, p2);
-
-            return slice;
-        }
-
+        
         private FingerSliceTrail orderTrailTipToHand(FingerSliceTrail trail)
         {
-            FingerSlice start = trail.StartSlice;
-            FingerSlice end = trail.EndSlice;
-
-            Vector2D direction = (end.Mid - start.Mid).normalize();
-
-            FingerSlice startOutSlice = findOutSlice(start.Mid, direction.getInverse());
-            FingerSlice endOutSlice = findOutSlice(end.Mid, direction);
-
-            float startOutLength = float.MaxValue;
-            float endOutLength = float.MaxValue;
-
-            if (startOutSlice.Start.isInBound(Vector2D.Zero, depthImage.BottomRight) && startOutSlice.End.isInBound(Vector2D.Zero, depthImage.BottomRight))
-                startOutLength = startOutSlice.Length;
-
-            if (endOutSlice.Start.isInBound(Vector2D.Zero, depthImage.BottomRight) && endOutSlice.End.isInBound(Vector2D.Zero, depthImage.BottomRight))
-                endOutLength = endOutSlice.Length;
-
-            if (startOutLength <= endOutLength)
-            {
+            //check direction
+            if (trail.StartSlice.Length > trail.EndSlice.Length)
                 trail.reverse();
-                outputImage.drawLineSegment(startOutSlice.LineSegment, Constants.FingerHandOutSliceColor);
-                outputImage.drawLineSegment(endOutSlice.LineSegment, Constants.FingerTipOutSliceColor);
-            }
-            else
-            {
-                outputImage.drawLineSegment(startOutSlice.LineSegment, Constants.FingerTipOutSliceColor);
-                outputImage.drawLineSegment(endOutSlice.LineSegment, Constants.FingerHandOutSliceColor);
-            }
 
             return trail;
         }
-
+        
         private void drawFingers()
         {
             foreach (Finger finger in Fingers)

bbiwarg/Recognition/FingerRecognition/FingerSliceTrail.cs

@@ -32,10 +32,15 @@ namespace bbiwarg.Recognition.FingerRecognition
         public void addSlice(FingerSlice slice)
         {
             slices.Add(slice);
-            pointsA.Add(slice.Start - Constants.FingerContourMargin * slice.Direction);
+            pointsA.Add(slice.Start + Constants.FingerContourMargin * slice.Direction.getInverse());
             pointsB.Add(slice.End + Constants.FingerContourMargin * slice.Direction);
             pointsA.Add(slice.Start);
-            LineSegment = new LineSegment2D(StartSlice.Mid, slice.Mid);
+            createLineSegment();
+        }
+
+        private void createLineSegment()
+        {
+            LineSegment = new LineSegment2D(EndSlice.Mid, StartSlice.Mid);
         }
 
         public Vector2D getStartDirection()
@@ -55,7 +60,7 @@ namespace bbiwarg.Recognition.FingerRecognition
             slices.RemoveRange(0, numSlices);
             pointsA.RemoveRange(0, numSlices);
             pointsB.RemoveRange(0, numSlices);
-            LineSegment = new LineSegment2D(StartSlice.Mid, EndSlice.Mid);
+            createLineSegment();
         }
 
         public void reverse() {
@@ -67,7 +72,7 @@ namespace bbiwarg.Recognition.FingerRecognition
             pointsA = pointsB;
             pointsB = dummy;
 
-            LineSegment = new LineSegment2D(StartSlice.Mid, EndSlice.Mid);
+            createLineSegment();
         }
 
 

bbiwarg/Recognition/TouchRecognition/TouchDetector.cs

@@ -31,36 +31,38 @@ namespace bbiwarg.Recognition.TouchRecognition
             foreach (Finger finger in fingers)
             {
                 Vector2D tipPoint = finger.TipPoint;
+                Vector2D direction = finger.TipDirection;
+                Vector2D directionInv = direction.getInverse();
+                Vector2D tipPointInside = (tipPoint + Constants.TouchEventTipInsideFactor * directionInv).moveInBound(Vector2D.Zero, depthImage.BottomRight, direction);
+                Vector2D tipPointOutside = (tipPoint + Constants.TouchEventTipOutsideFactor * direction).moveInBound(Vector2D.Zero, depthImage.BottomRight, directionInv);
+
                 outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
 
-                float floodValue = getFloodValue(tipPoint);
-                if (floodValue > Constants.TouchEventMinFloodValue)
+                float floodValue = getTouchValue(tipPointInside);
+                if (floodValue > Constants.TouchEventMinTouchValue)
                 {
-                    //correct touchEvent position
-                    Vector2D direction = finger.Direction;
-                    Vector2D tep = (tipPoint + Constants.TouchEventTipCorrectionFactor * direction.getInverse()).moveInBound(Vector2D.Zero, depthImage.BottomRight, direction);
-
-                    outputImage.fillCircle(tep.IntX, tep.IntY, 5, Constants.TouchEventDetectedColor);
-                    TouchEvent touchEvent = new TouchEvent(tep, floodValue, finger);
+                    outputImage.fillCircle(tipPointOutside.IntX, tipPointOutside.IntY, 5, Constants.TouchEventDetectedColor);
+                    TouchEvent touchEvent = new TouchEvent(tipPointOutside, floodValue, finger);
                     TouchEvents.Add(touchEvent);
                 }
             }
         }
 
-        private float getFloodValue(Point touchPoint)
+        private float getTouchValue(Point touchPoint)
         {
+            int floodValue = 255;
+
             int minX = Math.Max(touchPoint.X - 2*Constants.TouchEventAreaSize / 3, 0);
             int maxX = Math.Min(touchPoint.X + Constants.TouchEventAreaSize / 3, depthImage.Width-1);
             int minY = Math.Max(touchPoint.Y - 2*Constants.TouchEventAreaSize / 3, 0);
             int maxY = Math.Min(touchPoint.Y + Constants.TouchEventAreaSize / 3, depthImage.Height-1);
 
-
             Vector2D relTouch = new Vector2D(touchPoint.X - minX, touchPoint.Y - minY);
             Rectangle rect = new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1);
             Image<Gray, byte> touchArea = depthImage.Image.Copy(rect);
 
             MCvConnectedComp comp = new MCvConnectedComp();
-            CvInvoke.cvFloodFill(touchArea, relTouch, new MCvScalar(255), new MCvScalar(0), new MCvScalar(2), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
+            CvInvoke.cvFloodFill(touchArea, relTouch, new MCvScalar(floodValue), new MCvScalar(Constants.TouchEventFloodfillLowDiff), new MCvScalar(Constants.TouchEventFloodfillHighDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
 
             int matchedPixels = 0;
             int countedPixels = 0;
@@ -71,7 +73,7 @@ namespace bbiwarg.Recognition.TouchRecognition
                     Color color = outputImage.getColotAt(x, y);
                     byte depth = touchArea.Data[y - minY, x - minX, 0];
                     Color subtractColor;
-                    if(depth == 255)
+                    if(depth == floodValue)
                     {
                         matchedPixels++;
                         subtractColor = Constants.TouchEventAreaMatchedSubtractColor;
@@ -88,8 +90,7 @@ namespace bbiwarg.Recognition.TouchRecognition
 
             float rel = (float)matchedPixels / (float)countedPixels;
 
-            outputImage.drawLineSegment(new Utility.LineSegment2D(new Vector2D(minX, maxY), new Vector2D(minX + rel * Constants.TouchEventAreaSize, maxY)), Constants.TouchEventStatusBarColor);
-            outputImage.drawLineSegment(new Utility.LineSegment2D(new Vector2D(minX, maxY-1), new Vector2D(minX + rel * Constants.TouchEventAreaSize, maxY-1)), Constants.TouchEventStatusBarColor);
+            outputImage.drawLineSegment(new Utility.LineSegment2D(new Vector2D(minX, maxY), new Vector2D(minX + rel * Constants.TouchEventAreaSize, maxY)), Constants.TouchEventStatusBarColor, 2);
             
             return rel;
         }

bbiwarg/Utility/Vector2D.cs

@@ -73,15 +73,10 @@ namespace bbiwarg.Utility
             return (X >= topLeft.X && X <= bottomRight.X && Y >= topLeft.Y && Y <= bottomRight.Y);
         }
 
-        /*public bool isOnBound(float minX, float minY, float maxX, float maxY)
-        {
-            return (IntX == minX || IntX == maxX || IntY == minY || IntY == maxY);
-        }*/
-
-        public Vector2D moveInBound(Vector2D topLeft, Vector2D bottomRight, Vector2D direction) {
+        public Vector2D moveInBound(Vector2D topLeft, Vector2D bottomRight, Vector2D inBoundDirection) {
             Vector2D pos = new Vector2D(X, Y);
             while (!pos.isInBound(topLeft, bottomRight)) {
-                pos += direction;
+                pos += inBoundDirection;
             }
             return pos;
         }