-TouchTracker now uses generic tracker

-added events to generic tracker
-moved tuio-server communication to touchTracker
-moved touchEventVisualizer-updates to touchTracker
-fingers now have hands+touchevents

bbiwarg/Constants.cs

@@ -12,7 +12,7 @@ namespace bbiwarg
     class Constants
     {
         // Logger
-        public static readonly LogSubject LogLevel = 0;
+        public static readonly LogSubject LogLevel = LogSubject.TouchTracker;
             //LogSubject.Timer;
         
         // Output
@@ -24,6 +24,12 @@ namespace bbiwarg
         public static readonly String TuioIP = "127.0.0.1";
         public static readonly int TuioPort = 3333;
 
+        // image
+        public static readonly int ImageWidth = 320;
+        public static readonly int ImageHeight = 240;
+        public static readonly Vector2D ImageMaxPixel = new Vector2D(ImageWidth - 1, ImageHeight - 1);
+        public static readonly float ImageDiagonalLength = ImageMaxPixel.Length;
+
         // confidence image
         public static readonly int ConfidenceImageMinThreshold = 500;
 
@@ -53,12 +59,9 @@ namespace bbiwarg
         public static readonly int FingerCrippleOutMinDifference = 15;
 
         // finger tracking
-        private static Vector2D max = new Vector2D(320, 240);
-        public static readonly int FingerNumFramesUntilTracked = 5;
-        public static readonly int FingerNumFramesUntilLost = 10;
+        public static readonly int FingerNumFramesDetectedUntilTracked = 5;
+        public static readonly int FingerNumFramesLostUntilDeleted = 10;
         public static readonly float FingerMinSimilarityForTracking = 0.7f;
-        public static readonly float FingerSimilarityMaxTipPointDistance = max.Length;
-        public static readonly float FingerSimilarityMaxHandPointDistance = max.Length;
         //public static readonly float FingerSimilarityMaxAngle = (float)(45 * Math.PI / 180); // 45°
         //public static readonly float FingerSimilarityMaxParallelDistance = 100;
         //public static readonly float FingerSimilarityMaxVerticalDistance = 20;
@@ -100,7 +103,9 @@ namespace bbiwarg
         public static readonly float TouchProcessNoise = 3.0e-4f;
 
         // touch tracking
-        public static readonly int TouchEventNumFramesUntilTracked = 2;
+        public static readonly int TouchEventNumFramesDetectedUntilTracked = 1;
+        public static readonly int TouchEventNumFramesLostUntilDeleted = 1;
+        public static readonly float TouchEventMinSimilarityForTracking = 0.7f;
 
         // colors
         public static readonly Color ColorDetected = Color.Turquoise;
@@ -123,8 +128,9 @@ namespace bbiwarg
         public static readonly Color TouchEventAreaNonMatchedSubtractColor = Color.DarkSlateGray;
         public static readonly Color TouchEventStatusBarColor = Color.Green;
 
-        public static readonly Color TouchEventVisualizerLineColor = Color.White;
+        public static readonly Color TouchEventVisualizerLineColor = Color.Blue;
         public static readonly Color TouchEventVisualizerPointColor = Color.Red;
+        public static readonly Color TouchEventVisualizerGridColor = Color.White;
 
         public static readonly Color PalmQuadColor = Color.Blue;
         public static readonly Color PalmGridColor = Color.CornflowerBlue;

bbiwarg/Graphics/TouchEventVisualizer.cs

@@ -13,21 +13,73 @@ namespace bbiwarg.Graphics
     {
         public OutputImage OutputImage { get; private set; }
 
-        private int lastUpdated = -1;
         private int width, height;
 
-        private List<Vector2D> lastTouchPositions;
-        private List<Vector2D> currentTouchPositions;
-        private Kalman2DPositionFilter kalmanFilter;
+        private Dictionary<int, List<Vector2D>> positions;
+        private Dictionary<int, Kalman2DPositionFilter> kalmanFilters;
 
         public TouchEventVisualizer(int width, int height)
         {
             this.width = width;
             this.height = height;
 
-            OutputImage = new Graphics.OutputImage(width, height);
+            this.positions = new Dictionary<int, List<Vector2D>>();
+            this.kalmanFilters = new Dictionary<int, Kalman2DPositionFilter>();
         }
 
+        public void addTouchEvent(int trackID, Vector2D relPos) {
+            positions.Add(trackID, new List<Vector2D>());
+            positions[trackID].Add(relPos);
+
+            Kalman2DPositionFilter kalmanFilter = new Kalman2DPositionFilter(Constants.TouchmXX, Constants.TouchmXY, Constants.TouchmYY, Constants.TouchProcessNoise);
+            kalmanFilter.setInitialPosition(relPos);
+            kalmanFilters.Add(trackID, kalmanFilter);
+        }
+
+        public void updateTouchEvent(int trackID, Vector2D relPos) {
+            positions[trackID].Add(kalmanFilters[trackID].getCorrectedPosition(relPos));
+        }
+
+        public void removeTouchEvent(int trackID) {
+            positions.Remove(trackID);
+            kalmanFilters.Remove(trackID);
+        }
+
+        public void updateImage() {
+            OutputImage = new OutputImage(width, height);
+
+            //border
+            OutputImage.drawRectangle(0, 0, width - 1, height - 1, Constants.TouchEventVisualizerGridColor);
+
+            //draw grid
+            int numRows = Constants.PalmGridRows;
+            int numColumns = Constants.PalmGridColumns;
+            int widthPerColumn = width / numColumns;
+            int heightPerRow = height / numRows;
+
+            for (int i = 0; i <= numColumns; i++)
+            {
+                OutputImage.drawLineSegment(new LineSegment2D(new Vector2D(i * widthPerColumn, 0), new Vector2D(i * widthPerColumn, height - 1)), Constants.TouchEventVisualizerGridColor);
+            }
+
+            for (int i = 0; i <= numRows; i++)
+            {
+                OutputImage.drawLineSegment(new LineSegment2D(new Vector2D(0,i*heightPerRow), new Vector2D(width-1,i*heightPerRow)), Constants.TouchEventVisualizerGridColor);
+            }
+
+            Vector2D maxPixel = Constants.ImageMaxPixel;
+            foreach (int id in positions.Keys) {
+                int numPositions = positions[id].Count;
+                for (int i = 1; i < numPositions; i++) {
+                    OutputImage.drawLineSegment(new LineSegment2D(positions[id][i - 1].scale(maxPixel), positions[id][i].scale(maxPixel)), Constants.TouchEventVisualizerLineColor);
+                }
+                Vector2D lastPos = positions[id][numPositions - 1].scale(maxPixel);
+                OutputImage.fillCircle(lastPos.IntX, lastPos.IntY, 3, Constants.TouchEventVisualizerPointColor);
+            }
+        }
+
+
+        /*
         public void addPalmTouchEvent(PalmTouchEvent e, int currentFrame)
         {
             Vector2D pos;
@@ -54,7 +106,7 @@ namespace bbiwarg.Graphics
         public void updateImage()
         {
             OutputImage = new OutputImage(width, height);
-            bool touched = currentTouchPositions != null ? true : false;
+            bool touched = currentTouchPositions != null;
             int touchX = touched ? currentTouchPositions.Last<Vector2D>().IntX : -1;
             int touchY = touched ? currentTouchPositions.Last<Vector2D>().IntY : -1;
 
@@ -121,13 +173,12 @@ namespace bbiwarg.Graphics
             }
 
         }
-
+        */
         public void Reset()
         {
-            lastTouchPositions = null;
-            currentTouchPositions = null;
             OutputImage = new OutputImage(width, height);
-            lastUpdated = -1;
+            positions.Clear();
+            kalmanFilters.Clear();
         }
     }
 }

bbiwarg/Recognition/FingerRecognition/Finger.cs

@@ -8,6 +8,7 @@ using bbiwarg.Images;
 using bbiwarg.Utility;
 using bbiwarg.Graphics;
 using bbiwarg.Recognition.HandRecognition;
+using bbiwarg.Recognition.TouchRecognition;
 using bbiwarg.Recognition.Tracking;
 using Emgu.CV;
 
@@ -24,6 +25,8 @@ namespace bbiwarg.Recognition.FingerRecognition
         public LineSegment2D LineSegment { get { return SliceTrail.LineSegment; } }
         public FingerSliceTrail SliceTrail { get; private set; }
         public Contour<Point> Contour { get { return SliceTrail.getContour(); } }
+        public Hand Hand { get; private set; }
+        public TouchEvent TouchEvent { get; private set; }
 
         public Finger(FingerSliceTrail sliceTrail)
             : base()
@@ -39,11 +42,11 @@ namespace bbiwarg.Recognition.FingerRecognition
             
             //tip position
             float tipPointDistance = TipPoint.getDistanceTo(compareFinger.TipPoint);
-            float tipPointSimilarity = Math.Max(0, 1 - tipPointDistance / Constants.FingerSimilarityMaxTipPointDistance);
+            float tipPointSimilarity = Math.Max(0, 1 - tipPointDistance / Constants.ImageDiagonalLength);
 
             //hand position
             float handPointDistance = HandPoint.getDistanceTo(compareFinger.HandPoint);
-            float handPointSimilarity = Math.Max(0, 1 - handPointDistance / Constants.FingerSimilarityMaxHandPointDistance);
+            float handPointSimilarity = Math.Max(0, 1 - handPointDistance / Constants.ImageDiagonalLength);
             /*
             //check angle
             float angle = LineSegment.Line.getAngleBetween(compareLine);
@@ -65,6 +68,14 @@ namespace bbiwarg.Recognition.FingerRecognition
             return similarity;
         }
 
+        public void setHand(Hand hand) {
+            Hand = hand;
+        }
+
+        public void setTouchEvent(TouchEvent touchEvent) {
+            TouchEvent = touchEvent;
+        }
+
         public void draw(OutputImage outputImage, bool tracked)
         {
             if (tracked)

bbiwarg/Recognition/FingerRecognition/FingerTracker.cs

@@ -13,8 +13,34 @@ namespace bbiwarg.Recognition.FingerRecognition
         public List<Finger> Fingers { get { return TrackedObjects; } }
 
         public FingerTracker()
-            : base(LogSubject.FingerTracker, Constants.FingerNumFramesUntilTracked, Constants.FingerNumFramesUntilLost, Constants.FingerMinSimilarityForTracking) {
-        
+            : base(Constants.FingerNumFramesDetectedUntilTracked, Constants.FingerNumFramesLostUntilDeleted, Constants.FingerMinSimilarityForTracking)
+        {
+
+        }
+
+        protected override void onStateChanged(object sender, TrackingStateChangeEventArgs e)
+        {
+            TrackableObjectHistory<Finger> history = (TrackableObjectHistory<Finger>)sender;
+
+            //log
+            if (history.NumFramesInCurrentState == 1)
+            {
+                switch (e.NextState)
+                {
+                    case TrackingState.Detected:
+                        Logger.log("Finger #" + history.ID.ToString() + " detected", LogSubject.FingerTracker);
+                        break;
+                    case TrackingState.Tracked:
+                        Logger.log("Finger #" + history.ID.ToString() + " tracked", LogSubject.FingerTracker);
+                        break;
+                    case TrackingState.Lost:
+                        Logger.log("Finger #" + history.ID.ToString() + " lost", LogSubject.FingerTracker);
+                        break;
+                    case TrackingState.Delete:
+                        Logger.log("Finger #" + history.ID.ToString() + " deleted", LogSubject.FingerTracker);
+                        break;
+                }
+            }
         }
     }
 }

bbiwarg/Recognition/HandRecognition/Hand.cs

@@ -23,11 +23,10 @@ namespace bbiwarg.Recognition.HandRecognition
         public Image<Gray, byte> Mask { get; private set; }
         public List<Finger> Fingers { get; private set; }
 
-        public Hand(Image<Gray, byte> mask, Finger finger) {
+        public Hand(Image<Gray, byte> mask) {
             Mask = mask;
             Centroid = getCentroid();
             Fingers = new List<Finger>();
-            addFinger(finger);
         }
 
         public bool isInside(Vector2D point) {

bbiwarg/Recognition/HandRecognition/HandDetector.cs

@@ -69,6 +69,7 @@ namespace bbiwarg.Recognition.HandRecognition
                     if (hand.isInside(finger.HandPoint))
                     {
                         hand.addFinger(finger);
+                        finger.setHand(hand);
                         newHand = false;
                         break;
                     }
@@ -82,8 +83,11 @@ namespace bbiwarg.Recognition.HandRecognition
                     if (comp.area < maxArea * Constants.HandMaxSize)
                     {
                         Image<Gray, byte> cropedMask = mask.Copy(new Rectangle(1, 1, width, height)).Mul(255).Dilate(1);
-                        Hand hand = new Hand(cropedMask, finger);
+                        Hand hand = new Hand(cropedMask);
                         Hands.Add(hand);
+                        hand.addFinger(finger);
+                        finger.setHand(hand);
+
                         HandMask = HandMask.Or(cropedMask);
                     }
                 }

bbiwarg/Recognition/PalmRecognition/PalmDetector.cs

@@ -32,7 +32,7 @@ namespace bbiwarg.Recognition.PalmRecognition
         private List<Hand> hands;
 
         public Quadrangle PalmQuad { get; private set; }
-        public  Hand.HandSide PalmHandSide {get; private set; }
+        public Hand.HandSide PalmHandSide { get; private set; }
 
         public PalmDetector()
         {
@@ -115,7 +115,7 @@ namespace bbiwarg.Recognition.PalmRecognition
 
         private void findLongestPalmContour()
         {
-            Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, 
+            Contour<Point> contour = palmHand.Mask.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
                                                                 Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
 
             palmContour = contour;
@@ -161,13 +161,13 @@ namespace bbiwarg.Recognition.PalmRecognition
                         fingerIntersectsStartEnd = true;
                         break;
                     }
-                    
+
                     Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
                     float dist = f.LineSegment.getDistanceTo(mid);
                     if (dist < minFingerLineDist)
                         minFingerLineDist = dist;
                 }
-                
+
                 if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !fingerIntersectsStartEnd)
                     newDefects.Add(d);
             }
@@ -190,8 +190,8 @@ namespace bbiwarg.Recognition.PalmRecognition
 
                 float depthThumbLengthQuotient = d.Depth / palmHand.Fingers[0].LineSegment.Length;
 
-                if (angle <= Constants.PalmMaxThumbDefectAngle && 
-                    startEndLengthQuotient >= Constants.PalmMinThumbDefectStartEndLengthQuotient && 
+                if (angle <= Constants.PalmMaxThumbDefectAngle &&
+                    startEndLengthQuotient >= Constants.PalmMinThumbDefectStartEndLengthQuotient &&
                     startEndLengthQuotient <= Constants.PalmMaxThumbDefectStartEndLengthQuotient &&
                     depthThumbLengthQuotient >= Constants.PalmMinTumbDefectDepthThumbLengthQuotient &&
                     depthThumbLengthQuotient <= Constants.PalmMaxTumbDefectDepthThumbLengthQuotient)
@@ -265,14 +265,22 @@ namespace bbiwarg.Recognition.PalmRecognition
                 }
 
                 if (palmHand.Side == Hand.HandSide.Left)
+                {
                     handWidth = 0.85f * new Vector2D(-handLength.Y, handLength.X);
+                    topLeft = longestLineEndpoint + 0.15f * handLength;
+                    bottomLeft = thumbDefectDepth - 0.4f * handLength;
+                    bottomRight = bottomLeft + handWidth;
+                    topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
+                }
                 else
+                {
                     handWidth = 0.85f * new Vector2D(handLength.Y, -handLength.X);
+                    topRight = longestLineEndpoint + 0.15f * handLength;
+                    bottomRight = thumbDefectDepth - 0.4f * handLength;
+                    bottomLeft = bottomRight + handWidth;
+                    topLeft = bottomLeft + 1.2f * handLength - 0.3f * handWidth;
+                }
 
-                topLeft = longestLineEndpoint + 0.15f * handLength;
-                bottomLeft = thumbDefectDepth - 0.4f * handLength;
-                bottomRight = bottomLeft + handWidth;
-                topRight = bottomRight + 1.2f * handLength - 0.3f * handWidth;
 
                 Quadrangle quad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight, foregroundMask.Width, foregroundMask.Height);
 
@@ -292,7 +300,8 @@ namespace bbiwarg.Recognition.PalmRecognition
 
         private void draw()
         {
-            if (palmContour != null && palmContour.Count<Point>() > 0) {
+            if (palmContour != null && palmContour.Count<Point>() > 0)
+            {
                 outputImage.drawContour(palmContour, Constants.PalmConturColor);
                 outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
             }

bbiwarg/Recognition/TouchRecognition/PalmTouchDetector.cs

@@ -1,34 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using bbiwarg.Recognition.FingerRecognition;
-using bbiwarg.Recognition.PalmRecognition;
-using bbiwarg.Utility;
-
-using bbiwarg.Recognition.HandRecognition;
-
-namespace bbiwarg.Recognition.TouchRecognition
-{
-    class PalmTouchDetector
-    {
-        public List<PalmTouchEvent> PalmTouchEvents { get; private set; }
-
-        public PalmTouchDetector(List<TouchEvent> touchEvents, Quadrangle palmQuad, Hand.HandSide side)
-        {
-            PalmTouchEvents = new List<PalmTouchEvent>();
-
-            foreach (TouchEvent touchEvent in touchEvents)
-            {
-                Vector2D relativePos = palmQuad.getRelativePosition(touchEvent.Position, side);
-                if (relativePos.X >= 0 && relativePos.X <= 1.0 && relativePos.Y >= 0 && relativePos.Y <= 1.0)
-                {
-                    PalmTouchEvent pte = new PalmTouchEvent(touchEvent.Position, relativePos, touchEvent.FloodValue, touchEvent.Finger);
-                    PalmTouchEvents.Add(pte);
-                }
-            }
-
-        }
-    }
-}

bbiwarg/Recognition/TouchRecognition/PalmTouchEvent.cs

@@ -6,6 +6,8 @@ using System.Threading.Tasks;
 using bbiwarg.Utility;
 using bbiwarg.Recognition.FingerRecognition;
 using bbiwarg.Recognition.PalmRecognition;
+using Emgu.CV;
+using Emgu.CV.Structure;
 
 namespace bbiwarg.Recognition.TouchRecognition
 {
@@ -13,10 +15,10 @@ namespace bbiwarg.Recognition.TouchRecognition
     {
         public Vector2D RelativePalmPosition { get; private set; }
 
-        public PalmTouchEvent(Vector2D absolutePosition, Vector2D relativePalmPosition, float floodValue, Finger finger)
-            : base(absolutePosition, floodValue, finger)
+        public PalmTouchEvent(Vector2D absolutePosition, Image<Gray, byte> touchMask, Finger finger, Quadrangle palmQuad)
+            : base(absolutePosition, touchMask, finger)
         {
-            RelativePalmPosition = relativePalmPosition;
+            RelativePalmPosition = palmQuad.getRelativePosition(absolutePosition);
         }
     }
 }

bbiwarg/Recognition/TouchRecognition/TouchDetector.cs

@@ -21,7 +21,7 @@ namespace bbiwarg.Recognition.TouchRecognition
         private List<Finger> fingers;
         public List<TouchEvent> TouchEvents { get; private set; }
 
-        public TouchDetector(List<Finger> fingers, DepthImage depthImage, OutputImage outputImage)
+        public TouchDetector(List<Finger> fingers, Quadrangle palmQuad, DepthImage depthImage, OutputImage outputImage)
         {
             this.depthImage = depthImage;
             this.outputImage = outputImage;
@@ -36,18 +36,53 @@ namespace bbiwarg.Recognition.TouchRecognition
                 Vector2D tipPointInside = (tipPoint + Constants.TouchEventTipInsideFactor * directionInv).moveInBound(Vector2D.Zero, depthImage.BottomRight, direction);
                 Vector2D tipPointOutside = (tipPoint + Constants.TouchEventTipOutsideFactor * direction).moveInBound(Vector2D.Zero, depthImage.BottomRight, directionInv);
 
-                outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
+                Image<Gray, byte> touchMask = getTouchMask(tipPointInside);
+
+                int touchPixels = touchMask.CountNonzero()[0];
+                int numPixels = touchMask.Width * touchMask.Height;
+                float touchValue = touchPixels / (float)numPixels;
 
-                float floodValue = getTouchValue(tipPointInside);
-                if (floodValue > Constants.TouchEventMinTouchValue)
+                if (touchValue > Constants.TouchEventMinTouchValue)
                 {
                     outputImage.fillCircle(tipPointOutside.IntX, tipPointOutside.IntY, 5, Constants.TouchEventDetectedColor);
-                    TouchEvent touchEvent = new TouchEvent(tipPointOutside, floodValue, finger);
+
+                    TouchEvent touchEvent;
+                    if (palmQuad != null && palmQuad.isInside(tipPointOutside))
+                        touchEvent = new PalmTouchEvent(tipPointOutside, touchMask, finger, palmQuad);
+
+                    else
+                        touchEvent = new TouchEvent(tipPointOutside, touchMask, finger);
+
                     TouchEvents.Add(touchEvent);
+                    finger.setTouchEvent(touchEvent);
                 }
+
+                outputImage.fillCircle(tipPoint.IntX, tipPoint.IntY, 3, Constants.TouchEventTipColor);
             }
         }
 
+        private Image<Gray, byte> getTouchMask(Vector2D touchPoint)
+        {
+
+            int minX = Math.Max(touchPoint.IntX - 2 * Constants.TouchEventAreaSize / 3, 0);
+            int maxX = Math.Min(touchPoint.IntX + Constants.TouchEventAreaSize / 3, depthImage.Width - 1);
+            int minY = Math.Max(touchPoint.IntY - 2 * Constants.TouchEventAreaSize / 3, 0);
+            int maxY = Math.Min(touchPoint.IntY + Constants.TouchEventAreaSize / 3, depthImage.Height - 1);
+
+            Vector2D relTouchPoint = new Vector2D(touchPoint.IntX - minX, touchPoint.IntY - minY);
+            Rectangle rect = new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1);
+            Image<Gray, byte> touchArea = depthImage.Image.Copy(rect);
+            Image<Gray, byte> touchMask = new Image<Gray, byte>(rect.Width + 2, rect.Height + 2);
+
+            MCvConnectedComp comp = new MCvConnectedComp();
+            CvInvoke.cvFloodFill(touchArea, relTouchPoint, new MCvScalar(255), new MCvScalar(Constants.TouchEventFloodfillLowDiff), new MCvScalar(Constants.TouchEventFloodfillHighDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, touchMask);
+
+            Rectangle cropRect = new Rectangle(1, 1, rect.Width, rect.Height);
+            return touchMask.Copy(cropRect);
+        }
+
+
+        /*
         private float getTouchValue(Point touchPoint)
         {
             int floodValue = 255;
@@ -93,6 +128,6 @@ namespace bbiwarg.Recognition.TouchRecognition
             outputImage.drawLineSegment(new Utility.LineSegment2D(new Vector2D(minX, maxY), new Vector2D(minX + rel * Constants.TouchEventAreaSize, maxY)), Constants.TouchEventStatusBarColor, 2);
             
             return rel;
-        }
+        }*/
     }
 }

bbiwarg/Recognition/TouchRecognition/TouchEvent.cs

@@ -4,28 +4,48 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Recognition.FingerRecognition;
+using bbiwarg.Recognition.Tracking;
 using bbiwarg.Utility;
+using Emgu.CV;
+using Emgu.CV.Structure;
 
 namespace bbiwarg.Recognition.TouchRecognition
 {
-    class TouchEvent
+    class TouchEvent : TrackableObject
     {
         public Vector2D Position { get; private set; }
-        public float FloodValue { get; private set; }
+        public Image<Gray, byte> TouchMask { get; private set; }
         public Finger Finger { get; private set; }
 
-        public TouchEvent(Vector2D position, float floodValue, Finger finger)
+        public TouchEvent(Vector2D position, Image<Gray, byte> touchMask, Finger finger)
         {
             Position = position;
-            FloodValue = floodValue;
+            TouchMask = touchMask;
             Finger = finger;
         }
 
-        public bool isSimilarTo(TouchEvent compareTouchEvent)
+        public float getTouchValue()
         {
-            float maxDistance = 20;
+            int touchPixels = TouchMask.CountNonzero()[0];
+            int numPixels = TouchMask.Width * TouchMask.Height;
+            float touchValue = touchPixels / (float)numPixels;
+            return touchValue;
+        }
+
+        public override float getSimilarity(TrackableObject compareObject)
+        {
+            TouchEvent compareTouchEvent = (TouchEvent)compareObject;
+
+            //finger similarity
+            float fingerSimilarity = (Finger.TrackID == compareTouchEvent.Finger.TrackID) ? 1 : 0;
+
+            //position similarity
             float distance = Position.getDistanceTo(compareTouchEvent.Position);
-            return (distance < maxDistance);
+            float distanceSimilarity = Math.Max(0, 1 - distance / Constants.ImageDiagonalLength);
+
+            float similarity = fingerSimilarity * distanceSimilarity;
+
+            return similarity;
         }
     }
 }

bbiwarg/Recognition/TouchRecognition/TouchTracker.cs

@@ -6,73 +6,79 @@ using System.Text;
 using System.Threading.Tasks;
 using bbiwarg.Images;
 using bbiwarg.Graphics;
+using bbiwarg.Recognition.Tracking;
+using bbiwarg.Utility;
+using TUIO;
 
 namespace bbiwarg.Recognition.TouchRecognition
 {
-    class TouchTracker
+    class TouchTracker : Tracker<TouchEvent>
     {
-        private OutputImage outputImage;
-        private List<TouchEvent>[] detectedTouchEvents;
-        private int framesUntilTracked;
-        public List<TouchEvent> TrackedTouchEvents;
+        public List<TouchEvent> TouchEvents { get { return TrackedObjects; } }
+        private TuioServer tuioServer;
+        private Dictionary<int, TuioCursor> tuioCursors;
+        private TouchEventVisualizer touchEventVisualizer;
 
-        public TouchTracker()
+        public TouchTracker(TuioServer tuioServer, TouchEventVisualizer touchEventVisualizer)
+            : base(Constants.TouchEventNumFramesDetectedUntilTracked, Constants.TouchEventNumFramesLostUntilDeleted, Constants.TouchEventMinSimilarityForTracking)
         {
-            reset();
+            this.tuioServer = tuioServer;
+            this.tuioCursors = new Dictionary<int, TuioCursor>();
+            this.touchEventVisualizer = touchEventVisualizer;
         }
 
-        public void reset() {
-            framesUntilTracked = Constants.TouchEventNumFramesUntilTracked;
-            detectedTouchEvents = new List<TouchEvent>[framesUntilTracked];
-
-            for (int i = 0; i < framesUntilTracked; i++)
-            {
-                detectedTouchEvents[i] = new List<TouchEvent>();
-            }
+        public new void reset() {
+            tuioCursors.Clear();
+            touchEventVisualizer.Reset();
+            base.reset();
         }
 
-        public void setDetectedTouchEventsThisFrame(List<TouchEvent> touchEventsThisFrame, OutputImage outputImage)
+        public new void updateFrame(List<TouchEvent> detectedObjects)
         {
-            this.outputImage = outputImage;
-
-            for (int i = (framesUntilTracked - 1); i > 0; i--)
-            {
-                detectedTouchEvents[i] = detectedTouchEvents[i - 1];
-            }
-            detectedTouchEvents[0] = touchEventsThisFrame;
-
-
-            findTrackedTouches();
+            tuioServer.initFrame();
+            base.updateFrame(detectedObjects);
+            tuioServer.commitFrame();
+            touchEventVisualizer.updateImage();
         }
 
-        private void findTrackedTouches()
+        protected override void onStateChanged(object sender, TrackingStateChangeEventArgs e)
         {
-            TrackedTouchEvents = new List<TouchEvent>();
+            TrackableObjectHistory<TouchEvent> history = (TrackableObjectHistory<TouchEvent>)sender;
 
-            foreach (TouchEvent te in detectedTouchEvents[0])
+            TouchEvent currentTouchEvent = history.LastObject;
+            if (currentTouchEvent is PalmTouchEvent)
             {
-                bool tracked = true;
-                for (int i = 1; i < framesUntilTracked; i++)
+                PalmTouchEvent currentPalmTouchEvent = (PalmTouchEvent)currentTouchEvent;
+                Vector2D currentRelativePosition = currentPalmTouchEvent.RelativePalmPosition;
+                TuioCursor tuioCursor;
+                switch (e.NextState)
                 {
-                    if (!hasSimilarTouchEventInFrame(te, i)) tracked = false;
+                    case TrackingState.Tracked:
+                        if (tuioCursors.TryGetValue(history.ID, out tuioCursor))
+                        {
+                            tuioServer.updateTuioCursor(tuioCursor, currentRelativePosition.X, currentRelativePosition.Y);
+                            touchEventVisualizer.updateTouchEvent(history.ID, currentRelativePosition);
+                            Logger.log("TouchEvent #" + history.ID.ToString() + " touchMove to (" + currentRelativePosition.X + ", " + currentRelativePosition.Y + ")", LogSubject.TouchTracker);
+                        }
+                        else
+                        {
+                            TuioCursor newTuioCursor = tuioServer.addTuioCursor(currentRelativePosition.X, currentRelativePosition.Y);
+                            tuioCursors.Add(history.ID, newTuioCursor);
+                            touchEventVisualizer.addTouchEvent(history.ID, currentRelativePosition);
+                            Logger.log("TouchEvent #" + history.ID.ToString() + " touchDown at (" + currentRelativePosition.X + ", " + currentRelativePosition.Y + ")", LogSubject.TouchTracker);
+                        }
+                        break;
+                    case TrackingState.Delete:
+                        if (tuioCursors.TryGetValue(history.ID, out tuioCursor))
+                        {
+                            tuioServer.removeTuioCursor(tuioCursor);
+                            tuioCursors.Remove(history.ID);
+                            touchEventVisualizer.removeTouchEvent(history.ID);
+                            Logger.log("TouchEvent #" + history.ID.ToString() + " touchUp at (" + currentRelativePosition.X + ", " + currentRelativePosition.Y + ")", LogSubject.TouchTracker);
+                        }
+                        break;
                 }
-                if (tracked)
-                {
-                    outputImage.fillCircle(te.Position.IntX, te.Position.IntY, 5, Color.Yellow);
-                    TrackedTouchEvents.Add(te);
-                }
-            }
-        }
-
-        private bool hasSimilarTouchEventInFrame(TouchEvent touchEvent, int frame)
-        {
-            foreach (TouchEvent te in detectedTouchEvents[frame])
-            {
-                if (touchEvent.isSimilarTo(te))
-                    return true;
             }
-
-            return false;
         }
     }
 }

bbiwarg/Recognition/Tracking/TrackableObject.cs

@@ -8,18 +8,15 @@ namespace bbiwarg.Recognition.Tracking
 {
     abstract class TrackableObject
     {
-        public bool IsTracked { get; private set; }
         public int TrackID { get; private set; }
 
         public TrackableObject() {
-            IsTracked = false;
             TrackID = 0;
         }
 
         abstract public float getSimilarity(TrackableObject compareObject);
 
-        public void setTracked(int id) {
-            IsTracked = true;
+        public void setTrackID(int id) {
             TrackID = id;
         }
     }

bbiwarg/Recognition/Tracking/TrackableObjectHistory.cs

@@ -7,31 +7,47 @@ using bbiwarg.Utility;
 
 namespace bbiwarg.Recognition.Tracking
 {
+    public class TrackingStateChangeEventArgs : EventArgs
+    {
+        public TrackingState NextState { get; private set; }
+        public TrackingState PreviousState { get; private set; }
+
+        public TrackingStateChangeEventArgs(TrackingState nextState, TrackingState previousState)
+        {
+            NextState = nextState;
+            PreviousState = previousState;
+        }
+    }
+
+    public delegate void TrackingStateChangedEventHandler(object sender, TrackingStateChangeEventArgs e);
+
     class TrackableObjectHistory<T> where T : TrackableObject
     {
         private static List<int> usedIDs = new List<int>();
         public int ID { get; private set; }
         public List<T> TrackedObjects { get; private set; }
         public List<TrackingState> States { get; private set; }
-        public TrackableObject LastObject { get; private set; }
-        public TrackableObject CurrentObject { get { return TrackedObjects[TrackedObjects.Count - 1]; } }
+        public T LastObject { get; private set; }
+        public T CurrentObject { get { return TrackedObjects[TrackedObjects.Count - 1]; } }
         public TrackingState CurrentState { get { return States[States.Count - 1]; } }
         public int NumFramesInCurrentState { get; private set; }
-        private String className;
-        private LogSubject logSubject;
-        private int numFramesUntilTracked;
+        private int numFramesDetectedUntilTracked;
+        private int numFramesLostUntilDeleted;
+
+        public event TrackingStateChangedEventHandler StateChanged;
+
+        protected virtual void OnStateChanged(TrackingStateChangeEventArgs e) { if (StateChanged != null) StateChanged(this, e); }
 
-        public TrackableObjectHistory(LogSubject logSubject, int numFramesUntilTracked)
+        public TrackableObjectHistory(int numFramesDetectedUntilTracked, int numFramesLostUntilDeleted)
         {
             ID = getNextUnusedID();
             TrackedObjects = new List<T>();
-            States = new List<TrackingState>();
             TrackedObjects.Add(null);
+            States = new List<TrackingState>();
             States.Add(TrackingState.None);
             NumFramesInCurrentState = 1;
-            this.className = typeof(T).Name;
-            this.logSubject = logSubject;
-            this.numFramesUntilTracked = numFramesUntilTracked;
+            this.numFramesDetectedUntilTracked = numFramesDetectedUntilTracked;
+            this.numFramesLostUntilDeleted = numFramesLostUntilDeleted;
         }
 
         ~TrackableObjectHistory()
@@ -44,43 +60,44 @@ namespace bbiwarg.Recognition.Tracking
             TrackingState previousState = CurrentState;
             TrackingState newState = TrackingState.None;
 
+            //get newState
             if (detectedObject != null)
             {
                 if (previousState == TrackingState.None)
-                {
                     newState = TrackingState.Detected;
-                    Logger.log(className + " #" + ID + " detected", logSubject);
-                }
                 else if (previousState == TrackingState.Lost)
-                {
                     newState = TrackingState.Tracked;
-                    Logger.log(className + " #" + ID.ToString() + " re-tracked", logSubject);
-                }
                 else if (previousState == TrackingState.Tracked)
                     newState = TrackingState.Tracked;
                 else if (previousState == TrackingState.Detected)
                 {
-                    if (NumFramesInCurrentState == numFramesUntilTracked)
-                    {
+                    if (NumFramesInCurrentState == numFramesDetectedUntilTracked)
                         newState = TrackingState.Tracked;
-                        Logger.log(className + " #" + ID.ToString() + " tracked", logSubject);
-                    }
                     else
                         newState = TrackingState.Detected;
                 }
-
-                LastObject = detectedObject;
-                detectedObject.setTracked(ID);
             }
+            else if (previousState == TrackingState.Lost && NumFramesInCurrentState == numFramesLostUntilDeleted)
+                newState = TrackingState.Delete;
             else
                 newState = TrackingState.Lost;
 
+            //update numFramesInCurrentState
             if (newState == previousState)
                 NumFramesInCurrentState++;
+            else
+                NumFramesInCurrentState = 1;
+
+            //update lastObject
+            if (detectedObject != null)
+            {
+                LastObject = detectedObject;
+                detectedObject.setTrackID(ID);
+            }
 
             TrackedObjects.Add(detectedObject);
             States.Add(newState);
-
+            OnStateChanged(new TrackingStateChangeEventArgs(newState, previousState));
         }
 
         public static int getNextUnusedID()

bbiwarg/Recognition/Tracking/Tracker.cs

@@ -14,25 +14,22 @@ namespace bbiwarg.Recognition.Tracking
         Detected = 1,
         Tracked = 2,
         Lost = 3,
+        Delete = 4
     }
 
-    class Tracker<T> where T : TrackableObject
+    abstract class Tracker<T> where T : TrackableObject
     {
         public List<TrackableObjectHistory<T>> Histories { get; private set; }
         public List<T> TrackedObjects { get; private set; }
         private List<SimilarityContainer<T>> similarities;
-        private String className;
-        private LogSubject logSubject;
-        private int numFramesUntilTracked;
-        private int numFramesUntilLost;
+        private int numFramesDetectedUntilTracked;
+        private int numFramesLostUntilDeleted;
         private float minSimilarityForTracking;
 
-        public Tracker(LogSubject logSubject, int numFramesUntilTracked, int numFramesUntilLost, float minSimilarityForTracking)
+        public Tracker(int numFramesDetectedUntilTracked, int numFramesLostUntilDeleted, float minSimilarityForTracking)
         {
-            this.className = typeof(T).Name;
-            this.logSubject = logSubject;
-            this.numFramesUntilTracked = numFramesUntilTracked;
-            this.numFramesUntilLost = numFramesUntilLost;
+            this.numFramesDetectedUntilTracked = numFramesDetectedUntilTracked;
+            this.numFramesLostUntilDeleted = numFramesLostUntilDeleted;
             this.minSimilarityForTracking = minSimilarityForTracking;
 
             reset();
@@ -50,7 +47,7 @@ namespace bbiwarg.Recognition.Tracking
             {
                 foreach (T detectedObject in detectedObjects)
                 {
-                    TrackableObjectHistory<T> history = new TrackableObjectHistory<T>(logSubject, numFramesUntilTracked);
+                    TrackableObjectHistory<T> history = createNewHistory();
                     history.addObjectToHistory(detectedObject);
                     Histories.Add(history);
                 }
@@ -92,7 +89,7 @@ namespace bbiwarg.Recognition.Tracking
             //add new history for each new unasigned detectedObject
             foreach (T unasignedDetectedObject in unasignedDetectedObjects)
             {
-                TrackableObjectHistory<T> newHistory = new TrackableObjectHistory<T>(logSubject, numFramesUntilTracked);
+                TrackableObjectHistory<T> newHistory = createNewHistory();
                 newHistory.addObjectToHistory(unasignedDetectedObject);
                 Histories.Add(newHistory);
             }
@@ -100,8 +97,6 @@ namespace bbiwarg.Recognition.Tracking
             //add null-object to each unasigned history (didn't find a best fit during this frame -> lost track)
             foreach (TrackableObjectHistory<T> unasignedHistory in unasignedHistories)
             {
-                if (unasignedHistory.CurrentState != TrackingState.Lost)
-                    Logger.log(className + " #" + unasignedHistory.ID + " lost", logSubject);
                 unasignedHistory.addObjectToHistory(null);
             }
 
@@ -142,12 +137,20 @@ namespace bbiwarg.Recognition.Tracking
             for (int i = Histories.Count - 1; i >= 0; i--)
             {
                 TrackableObjectHistory<T> history = Histories[i];
-                if (history.CurrentState == TrackingState.Lost && history.NumFramesInCurrentState >= numFramesUntilLost)
+                if (history.CurrentState == TrackingState.Delete)
                 {
                     Histories.RemoveAt(i);
-                    Logger.log(className + " #" + history.ID + " deleted", logSubject);
                 }
             }
         }
+
+        private TrackableObjectHistory<T> createNewHistory()
+        {
+            TrackableObjectHistory<T> history = new TrackableObjectHistory<T>(numFramesDetectedUntilTracked, numFramesLostUntilDeleted);
+            history.StateChanged += onStateChanged;
+            return history;
+        }
+
+        protected abstract void onStateChanged(object sender, TrackingStateChangeEventArgs e);
     }
 }

bbiwarg/Utility/Quadrangle.cs

@@ -20,7 +20,7 @@ namespace bbiwarg.Utility
         public Vector2D TopLeft { get; private set; }
         public Vector2D TopRight { get; private set; }
         public Vector2D BottomRight { get; private set; }
-        public float Area {get; private set;}
+        public float Area { get; private set; }
         public Image<Gray, Byte> Mask { get; private set; }
 
         public Quadrangle(Vector2D bottomLeft, Vector2D topLeft, Vector2D topRight, Vector2D bottomRight, int width, int height)
@@ -35,7 +35,7 @@ namespace bbiwarg.Utility
             contourPoints.Push(TopLeft);
             contourPoints.Push(TopRight);
             contourPoints.Push(BottomRight);
-            Area = (float) contourPoints.Area;
+            Area = (float)contourPoints.Area;
 
             Mask = new Image<Gray, byte>(width, height);
 
@@ -48,24 +48,14 @@ namespace bbiwarg.Utility
         }
 
 
-        public Vector2D getRelativePosition(Vector2D p, Hand.HandSide side)
+        public Vector2D getRelativePosition(Vector2D p)
         {
             Vector2D a, b, c, d;
 
-            if (side == Hand.HandSide.Left)
-            {
-                a = BottomLeft;
-                b = TopLeft;
-                c = TopRight;
-                d = BottomRight;
-            }
-            else
-            {
-                a = BottomRight;
-                b = TopRight;
-                c = TopLeft;
-                d = BottomLeft;
-            }
+            a = BottomLeft;
+            b = TopLeft;
+            c = TopRight;
+            d = BottomRight;
 
             float C = (a.Y - p.Y) * (d.X - p.X) - (a.X - p.X) * (d.Y - p.Y);
             float B = (a.Y - p.Y) * (c.X - d.X) + (b.Y - a.Y) * (d.X - p.X) - (a.X - p.X) * (c.Y - d.Y) - (b.X - a.X) * (d.Y - p.Y);
@@ -81,7 +71,13 @@ namespace bbiwarg.Utility
 
             float v = (px - p1x) / (p2x - p1x);
 
-            return new Vector2D(v, u);
+            return new Vector2D(u, v);
+        }
+
+        public bool isInside(Vector2D point)
+        {
+            Vector2D relativePos = getRelativePosition(point);
+            return (relativePos.X >= 0 && relativePos.X <= 1.0 && relativePos.Y >= 0 && relativePos.Y <= 1.0);
         }
     }
 }

bbiwarg/Utility/Vector2D.cs

@@ -60,6 +60,10 @@ namespace bbiwarg.Utility
             return X * v.Y - Y * v.X;
         }
 
+        public Vector2D scale(Vector2D v) {
+            return new Vector2D(X * v.X, Y * v.Y);
+        }
+
         public bool isInBox(Vector2D corner1, Vector2D corner2)
         {
             float minX = Math.Min(corner1.X, corner2.X);

bbiwarg/VideoHandle.cs

@@ -25,7 +25,7 @@ namespace bbiwarg
         private IInputProvider inputProvider;
         private InputFrame inputFrame;
 
-        private TuioServer server;
+        private TuioServer tuioServer;
         private TuioCursor currentTouchevent;
 
         public int Width { get; private set; }
@@ -53,7 +53,6 @@ namespace bbiwarg
         private HandDetector handDetector;
         private PalmDetector palmDetector;
         private TouchDetector touchDetector;
-        private PalmTouchDetector palmTouchDetector;
 
         private FingerTracker fingerTracker;
         private TouchTracker touchTracker;
@@ -69,9 +68,6 @@ namespace bbiwarg
 
         public void start()
         {
-            palmDetector = new PalmDetector();
-            server = new TuioServer(Constants.TuioIP, Constants.TuioPort);
-
             inputProvider.init();
             inputProvider.start();
             inputProvider.updateFrame();
@@ -82,7 +78,7 @@ namespace bbiwarg
         public void stop()
         {
             inputProvider.stop();
-            server.close();
+            tuioServer.close();
         }
 
         public bool sourceIsMovie()
@@ -120,9 +116,14 @@ namespace bbiwarg
             videoFrame++;
         }
 
-        private void initTrackers() {
-            touchTracker = new TouchTracker();
+        private void initTrackers()
+        {
+            palmDetector = new PalmDetector();
+            tuioServer = new TuioServer(Constants.TuioIP, Constants.TuioPort);
+            touchEventVisualizer = new TouchEventVisualizer(Constants.ImageWidth, Constants.ImageHeight);
+
             fingerTracker = new FingerTracker();
+            touchTracker = new TouchTracker(tuioServer, touchEventVisualizer);
         }
 
         private void processFrameUpdate()
@@ -204,54 +205,21 @@ namespace bbiwarg
 
             //detect touchEvents
             Timer.start("touchDetection");
-            touchDetector = new TouchDetector(fingerTracker.Fingers, depthImage, OutputImages[0]);
-            if (palmDetector.PalmQuad != null)
-                palmTouchDetector = new PalmTouchDetector(touchDetector.TouchEvents, palmDetector.PalmQuad, palmDetector.PalmHandSide);
+            touchDetector = new TouchDetector(fingerTracker.Fingers, palmDetector.PalmQuad, depthImage, OutputImages[0]);
             Timer.stop("touchDetection");
 
             //track touchEvents
             Timer.start("touchTracking");
-            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[3]);
-            Timer.stop("touchTracking");
-
-            Timer.start("sending touchEvents");
-            //send touchevent
-            if (palmTouchDetector != null)
+            touchTracker.updateFrame(touchDetector.TouchEvents);
+            foreach(TouchEvent te in touchTracker.TouchEvents)
             {
-                server.initFrame();
-                if (palmTouchDetector.PalmTouchEvents.Count == 1)
-                {
-                    PalmTouchEvent touchEvent = palmTouchDetector.PalmTouchEvents[0];
-                    if (currentTouchevent == null)
-                        currentTouchevent = server.addTuioCursor(touchEvent.RelativePalmPosition.X, touchEvent.RelativePalmPosition.Y);
-                    else
-                        server.updateTuioCursor(currentTouchevent, touchEvent.RelativePalmPosition.X, touchEvent.RelativePalmPosition.Y);
-                }
-                else if (currentTouchevent != null)
-                {
-                    server.removeTuioCursor(currentTouchevent);
-                    currentTouchevent = null;
-                }
-                server.commitFrame();
+                OutputImages[3].fillCircle(te.Position.IntX, te.Position.IntY, 5, Constants.TouchEventTrackedColor);
             }
-            Timer.stop("sending touchEvents");
 
+            OutputImages[4] = touchEventVisualizer.OutputImage;
+            Timer.stop("touchTracking");
 
-            // touch event visualizer
-            if (touchEventVisualizer == null)
-                touchEventVisualizer = new TouchEventVisualizer(Width, Height);
-            if (newStarted)
-                touchEventVisualizer.Reset();
-            if (palmTouchDetector != null)
-            {
-                foreach (PalmTouchEvent e in palmTouchDetector.PalmTouchEvents)
-                {
-                touchEventVisualizer.addPalmTouchEvent(e, CurrentFrame);
-                }
-                touchEventVisualizer.updateImage();
-                OutputImages[4] = touchEventVisualizer.OutputImage;
-            }
-            
+           
             // add borders
             for (int i = 0; i < Constants.OutputNumImages; i++)
             {

bbiwarg/bbiwarg.csproj

@@ -75,7 +75,6 @@
     <Compile Include="Recognition\HandRecognition\Hand.cs" />
     <Compile Include="Recognition\HandRecognition\HandDetector.cs" />
     <Compile Include="Recognition\PalmRecognition\PalmDetector.cs" />
-    <Compile Include="Recognition\TouchRecognition\PalmTouchDetector.cs" />
     <Compile Include="Recognition\TouchRecognition\PalmTouchEvent.cs" />
     <Compile Include="Recognition\TouchRecognition\TouchDetector.cs" />
     <Compile Include="Recognition\TouchRecognition\TouchEvent.cs" />