Merge branch 'master' of https://git.tk.informatik.tu-darmstadt.de/etri-smartspaces

Conflicts:
	bbiwarg/Constants.cs
	bbiwarg/Graphics/TouchEventVisualizer.cs
	bbiwarg/VideoHandle.cs

bbiwarg/Constants.cs

@@ -37,9 +37,13 @@ namespace bbiwarg
         // palm detection
         public static readonly float PalmMinDefectMidFingerLineDistance = 20; // defects with mid point ((start + end) / 2) closer than this to a finger line are removed 
         public static readonly float PalmMaxThumbDefectAngle = 100; // degree
-        
+
+        //palm Grid
+        public static readonly int PalmGridRows = 4;
+        public static readonly int PalmGridColumns = 3;
+
         // output window
-        public static readonly int NumImagesPerRow = 3;
+        public static readonly int NumImagesPerRow = 2;
         public static readonly float WindwoSizeFactor = 1f; // output window size is scaled by this factor (from necessary size for images)
     }
 }

bbiwarg/Detectors/Fingers/Finger.cs

@@ -18,6 +18,10 @@ namespace bbiwarg.Detectors.Fingers
 
         public Finger(FingerSliceTrail sliceTrail)
         {
+            //check direction
+            if (sliceTrail.Start.Length > sliceTrail.End.Length)
+                sliceTrail.Slices.Reverse();
+
             SliceTrail = sliceTrail;
         }
 

bbiwarg/Detectors/Fingers/FingerDetector.cs

@@ -88,8 +88,8 @@ namespace bbiwarg.Detectors.Fingers
 
             FingerSliceTrail trail = new FingerSliceTrail(startSlice);
 
-            Vector2D position = startSlice.Mid + startDirection;
             Vector2D direction = startDirection;
+            Vector2D position = startSlice.Mid + 2*direction;
 
             if (position.isWithin(0, 0, maxX, maxY))
             {
@@ -128,7 +128,7 @@ namespace bbiwarg.Detectors.Fingers
             FingerSlice lastSlice = trail.End;
             FingerSlice nextSlice;
 
-            while (currentPosition.isWithin(0, 0, maxX, maxY) && gapCounter <= Math.Min(numSlices / 2, 10))
+            while (currentPosition.isWithin(0, 0, maxX, maxY) && gapCounter <= Math.Min(numSlices, 10))
             {
                 nextSlice = findFingerSliceFromMid(currentPosition, currentDirection);
                 if (nextSlice != null && (nextSlice.Length < lastSlice.Length + 5 && nextSlice.Length > lastSlice.Length - 5))
@@ -180,7 +180,7 @@ namespace bbiwarg.Detectors.Fingers
             int maxX = depthImage.Width - 1;
             int maxY = depthImage.Height - 1;
 
-            int maxFingerSize = 30;
+            int maxFingerSize = 35;
 
             int maxStepsX;
             if (direction.X > 0)
@@ -221,7 +221,7 @@ namespace bbiwarg.Detectors.Fingers
             int maxY = depthImage.Height - 1;
 
             int minFingerSize = 5;
-            int maxFingerSize = 30;
+            int maxFingerSize = 35;
 
             Vector2D direction = (end - start).normalize();
             Vector2D beforeStart = start - direction;

bbiwarg/Detectors/Touch/TouchDetector.cs

@@ -27,7 +27,7 @@ namespace bbiwarg.Detectors.Touch
             this.outputImage = outputImage;
             this.fingers = fingers;
             this.TouchEvents = new List<TouchEvent>();
-            float floodValueThreshold = 0.5f;
+            float floodValueThreshold = 0.33f;
 
             foreach (Finger finger in fingers)
             {
@@ -71,7 +71,7 @@ namespace bbiwarg.Detectors.Touch
                     Int16 depth = depthImage.getDepthAt(x, y);
                     Color color = outputImage.getColotAt(x, y);
                     Color subtractColor;
-                    if (Math.Abs(depthAtTouch - depth) < maxDepthDifference)
+                    if (depthAtTouch < depth && Math.Abs(depthAtTouch - depth) < maxDepthDifference)
                     {
                         matchedPixels++;
                         subtractColor = Constants.TouchEventAreaMatchedSubtractColor;

bbiwarg/Graphics/TouchEventVisualizer.cs

@@ -13,85 +13,93 @@ namespace bbiwarg.Graphics
     {
         public OutputImage OutputImage { get; private set; }
 
-        private List<Kalman2DPositionFilter> kalmanFilters;
-        private List<List<Vector2D>> touchPositions;
         private int lastUpdated = -1;
         private int width, height;
 
+        private List<Vector2D> lastTouchPositions;
+        private List<Vector2D> currentTouchPositions;
+        private Kalman2DPositionFilter kalmanFilter;
+
         public TouchEventVisualizer(int width, int height)
         {
             this.width = width;
             this.height = height;
 
             OutputImage = new Graphics.OutputImage(width, height);
-            touchPositions = new List<List<Vector2D>>();
-            kalmanFilters = new List<Kalman2DPositionFilter>();
         }
 
         public void addPalmTouchEvent(PalmTouchEvent e, int currentFrame)
         {
-            List<Vector2D> currentList;
-            Kalman2DPositionFilter currentFilter;
-
-            bool addList = false;
-            if (lastUpdated == -1 || (currentFrame - lastUpdated) > 5)
-            {
-                currentList = new List<Vector2D>();
-                currentFilter = new Kalman2DPositionFilter();
-                currentFilter.setInitialPosition(e.RelativePalmPosition);
-                addList = true;
-            }
-            else
-            {
-                currentList = touchPositions.Last<List<Vector2D>>();
-                currentFilter = kalmanFilters.Last<Kalman2DPositionFilter>();
-            }
-
             Vector2D pos;
-            if (currentFilter == null)
+            if (lastUpdated == -1 || (currentFrame - lastUpdated) > 5 || currentTouchPositions == null || kalmanFilter == null)
             {
-                currentFilter = new Kalman2DPositionFilter();
-                currentFilter.setInitialPosition(e.RelativePalmPosition);
+                lastTouchPositions = currentTouchPositions;
+                currentTouchPositions = new List<Vector2D>();
+                kalmanFilter = new Kalman2DPositionFilter();
+                kalmanFilter.setInitialPosition(e.RelativePalmPosition);
                 pos = e.RelativePalmPosition;
             }
-            else 
+            else
             {
-                pos = currentFilter.getCorrectedPosition(e.RelativePalmPosition);
+                pos = kalmanFilter.getCorrectedPosition(e.RelativePalmPosition);
             }
 
+
             int size = Math.Min(width, height);
-            currentList.Add(new Vector2D((int)(pos.X * size), (int)(size - pos.Y * size)));
-            if (addList)
-            {
-                touchPositions.Add(currentList);
-                kalmanFilters.Add(currentFilter);
-            }
+            currentTouchPositions.Add(new Vector2D((int)(pos.X * size), (int)(size - pos.Y * size)));
 
             lastUpdated = currentFrame;
         }
 
         public void updateImage()
         {
-            OutputImage = new OutputImage(width, height);            
-            
+            OutputImage = new OutputImage(width, height);
+
+            //-- Border --
             int size = Math.Min(width, height);
             OutputImage.drawRectangle(0, 0, size - 1, size - 1, Color.White);
 
-            foreach (List<Vector2D> positions in touchPositions)
+
+            //-- Grid --
+            int numRows = Constants.PalmGridRows;
+            int numColumns = Constants.PalmGridColumns;
+
+            for (int i = 1; i < numRows; i++)
+            {
+                int tmp = size / numRows * i;
+                OutputImage.drawLineSegment(new LineSegment2D(new Vector2D(0, tmp), new Vector2D(size - 2, tmp)), Constants.PalmGridColor);
+            }
+            for (int i = 1; i < numColumns; i++)
+            {
+                int tmp = size / numColumns * i;
+                OutputImage.drawLineSegment(new LineSegment2D(new Vector2D(tmp, 0), new Vector2D(tmp, size - 2)), Constants.PalmGridColor);
+            }
+
+            //-- Current Touch Event --
+            if (currentTouchPositions != null)
             {
-                for (int i = 1; i < positions.Count; ++i)
+                for (int i = 1; i < currentTouchPositions.Count; ++i)
                 {
-                    OutputImage.drawLineSegment(new LineSegment2D(positions[i - 1], positions[i]), Constants.TouchEventVisualizerLineColor);
+                    OutputImage.drawLineSegment(new LineSegment2D(currentTouchPositions[i - 1], currentTouchPositions[i]), Constants.TouchEventVisualizerLineColor);
                 }
-
-                if (touchPositions.Count != 0)
-                    OutputImage.fillCircle(positions.Last<Vector2D>().IntX, positions.Last<Vector2D>().IntY, 3, Constants.TouchEventVisualizerPointColor);
+                OutputImage.fillCircle(currentTouchPositions.Last<Vector2D>().IntX, currentTouchPositions.Last<Vector2D>().IntY, 3, Constants.TouchEventVisualizerPointColor);
             }
+            //-- Last Touch Event --
+            if (lastTouchPositions != null)
+            {
+                for (int i = 1; i < lastTouchPositions.Count; ++i)
+                {
+                    OutputImage.drawLineSegment(new LineSegment2D(lastTouchPositions[i - 1], lastTouchPositions[i]), Constants.TouchEventVisualizerLineColor);
+                }
+                OutputImage.fillCircle(lastTouchPositions.Last<Vector2D>().IntX, lastTouchPositions.Last<Vector2D>().IntY, 3, Constants.TouchEventVisualizerPointColor);
+            }
+            
         }
 
         public void Reset()
         {
-            touchPositions.Clear();
+            lastTouchPositions = null;
+            currentTouchPositions = null;
             OutputImage = new OutputImage(width, height);
             lastUpdated = -1;
         }

bbiwarg/Images/DepthImage.cs

@@ -7,32 +7,33 @@ using System.Threading.Tasks;
 using Emgu.CV;
 using Emgu.CV.Structure;
 using bbiwarg.Graphics;
+using bbiwarg.Detectors.Fingers;
+using bbiwarg.Utility;
 
 namespace bbiwarg.Images
 {
     class DepthImage
     {
         public Image<Gray, byte> Image { get; private set; }
+        public Image<Gray, byte> BackgroundMask { get; private set; }
         public int Width { get; private set; }
         public int Height { get; private set; }
         public Int16 MinDepth { get; private set; }
         public Int16 MaxDepth { get; private set; }
 
-        public DepthImage(Image<Gray, Int16> image, OutputImage outputImage)
+        public DepthImage(Image<Gray, Int16> image)
         {
             Width = image.Width;
             Height = image.Height;
 
+            image = image.SmoothMedian(3);
+
             //threshold min&maxDepth
             MinDepth = findMinDepth(image);
             MaxDepth = (Int16)(MinDepth + 200); // max = minDepth+255 (else it can't fit whole range in byte image)
 
             //smooth+threshold (dst = (src > (MaxDepth - MinDepth)) ? MaxDepth - MinDepth : src)
-            Image = image.SmoothMedian(3).Sub(new Gray(MinDepth)).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
-
-            // draw depth
-            Image<Gray, byte> tmpDepth = (MaxDepth - MinDepth) - Image;
-            outputImage.Image[0] = outputImage.Image[1] = outputImage.Image[2] = tmpDepth;
+            Image = image.Sub(new Gray(MinDepth)).ThresholdTrunc(new Gray(MaxDepth - MinDepth)).Convert<Gray, byte>();
         }
 
         public Int16 getDepthAt(Point point)
@@ -77,5 +78,20 @@ namespace bbiwarg.Images
 
             return (Int16)min[0];
         }
+
+        public void removeBackground(List<Finger> fingers) {
+            BackgroundMask = Image.Copy();
+            MCvConnectedComp comp = new MCvConnectedComp();
+
+            foreach (Finger finger in fingers) {
+                Vector2D mid = (finger.Tip + finger.Hand) / 2;
+                if(BackgroundMask.Data[mid.IntY, mid.IntX,0] != 0)
+                    CvInvoke.cvFloodFill(BackgroundMask, mid, new MCvScalar(0), new MCvScalar(5), new MCvScalar(5), out comp, Emgu.CV.CvEnum.CONNECTIVITY.FOUR_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, IntPtr.Zero);
+            }
+
+            BackgroundMask = BackgroundMask.ThresholdBinary(new Gray(0), new Gray(255));
+            Image = Image.Or(BackgroundMask);
+
+        }
     }
 }

bbiwarg/Images/EdgeImage.cs

@@ -18,14 +18,9 @@ namespace bbiwarg.Images
     {
         public Image<Gray, Byte> Image { get; private set; }
 
-        public EdgeImage(DepthImage depthImage, OutputImage outputImage)
+        public EdgeImage(DepthImage depthImage)
         {
-            Image<Gray, byte> dimg = depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth));
-            
-            Image = dimg.Canny(100, 75, 3);
-
-            // draw blue edges in outputImage
-            outputImage.Image[2] = Image.ThresholdBinary(new Gray(0), new Gray(1)).Mul(255);
+            Image = (depthImage.Image * (255.0f / (float)(depthImage.MaxDepth - depthImage.MinDepth))).Canny(100, 75, 3);
         }
 
         public EdgeImage(Image<Gray, Byte> edgeImage)

bbiwarg/MainBBWIWARG.cs

@@ -12,7 +12,7 @@ namespace bbiwarg
     {
         static void Main(string[] args)
         {
-            IInputProvider inputProvider = new IisuInputProvider(""); //..\\..\\videos\\touch\\4.skv");
+            IInputProvider inputProvider = new IisuInputProvider("..\\..\\videos\\touch\\4.skv");
             VideoHandle videoHandle = new VideoHandle(inputProvider);
             videoHandle.start();
 

bbiwarg/Utility/Vector2D.cs

@@ -74,6 +74,13 @@ namespace bbiwarg.Utility
             return new Vector2D(X / length, Y / length);
         }
 
+        public Vector2D normalizeComponentOne() {
+            if (Math.Abs(X) > Math.Abs(Y))
+                return new Vector2D(1, Y / X);
+            else
+                return new Vector2D(X / Y, 1);
+        } 
+
         public bool isWithin(float minX, float minY, float maxX, float maxY)
         {
             return (X >= minX && Y >= minY && X <= maxX && Y <= maxY);

bbiwarg/VideoHandle.cs

@@ -49,8 +49,6 @@ namespace bbiwarg
         private FingerTracker fingerTracker;
         private TouchTracker touchTracker;
 
-        private OutputImage edgeFingerOutputImage;
-        private OutputImage depthPalmTouchOutputImage;
         private TouchEventVisualizer touchEventVisualizer;
 
         private int videoFrame = 0;
@@ -132,49 +130,63 @@ namespace bbiwarg
 
             //create output images
             Timer.start("createOtherImages");
-            edgeFingerOutputImage = new OutputImage(Width, Height);
-            depthPalmTouchOutputImage = new OutputImage(Width, Height);
+            int numImages = 4;
+            OutputImages = new OutputImage[numImages];
+            for (int i = 0; i < numImages; i++) {
+                OutputImages[i] = new OutputImage(Width, Height);
+                    
+            }
             Timer.stop("createOtherImages");
 
             //create depthImage
             Timer.start("createDepthImage");
             Image<Gray, Int16> image = new Image<Gray, Int16>(Width, Height, Width * 2, inputFrame.RawDepthData);
-            depthImage = new DepthImage(image, depthPalmTouchOutputImage);
+            depthImage = new DepthImage(image);
+            Image<Gray, byte> tmpDepth = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
+            OutputImages[0].Image[0] = OutputImages[0].Image[1] = OutputImages[0].Image[2] = tmpDepth;
             Timer.stop("createDepthImage");
 
             // create edge image
             Timer.start("createEdgeImage");
-            edgeImage = new EdgeImage(depthImage, edgeFingerOutputImage);
+            edgeImage = new EdgeImage(depthImage);
+            OutputImages[1].Image[2] = edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(1)).Mul(255);
             Timer.stop("createEdgeImage");
 
             //detect fingers
             Timer.start("fingerDetection");
-            fingerDetector = new FingerDetector(depthImage, edgeImage, edgeFingerOutputImage);
+            fingerDetector = new FingerDetector(depthImage, edgeImage, OutputImages[1]);
             Timer.stop("fingerDetection");
 
             //track fingers
             Timer.start("fingerTracking");
-            fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, edgeFingerOutputImage);
+            fingerTracker.setDetectedTouchEventsThisFrame(fingerDetector.Fingers, OutputImages[1]);
             Timer.stop("fingerTracking");
 
+            //remove background noise
+            Timer.start("removeBackground");
+            depthImage.removeBackground(fingerTracker.TrackedFingers);
+            edgeImage = new EdgeImage(depthImage);
+            OutputImages[2].Image[0] = OutputImages[2].Image[1] = OutputImages[2].Image[2] = (depthImage.MaxDepth - depthImage.MinDepth) - depthImage.Image;
+            Timer.stop("removeBackground");
+
             //detect palm
             Timer.start("palmDetection");
             if (fingerTracker.TrackedFingers.Count >= 2)
-                palmDetector.findPalmQuad(depthImage, edgeImage, depthPalmTouchOutputImage, fingerTracker.TrackedFingers);
+                palmDetector.findPalmQuad(depthImage, edgeImage, OutputImages[2], fingerTracker.TrackedFingers);
             if (CurrentFrame == 0)
                 palmDetector.resetFilters();
             Timer.stop("palmDetection");
 
             //detect touchEvents
             Timer.start("touchDetection");
-            touchDetector = new TouchDetector(fingerTracker.TrackedFingers, depthImage, depthPalmTouchOutputImage);
+            touchDetector = new TouchDetector(fingerTracker.TrackedFingers, depthImage, OutputImages[2]);
             if (palmDetector.PalmQuad != null)
                 palmTouchDetector = new PalmTouchDetector(touchDetector.TouchEvents, palmDetector.PalmQuad);
             Timer.stop("touchDetection");
 
             //track touchEvents
             Timer.start("touchTracking");
-            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, depthPalmTouchOutputImage);
+            touchTracker.setDetectedTouchEventsThisFrame(touchDetector.TouchEvents, OutputImages[2]);
             Timer.stop("touchTracking");
 
             // touch event visualizer
@@ -187,12 +199,16 @@ namespace bbiwarg
                 foreach (PalmTouchEvent e in palmTouchDetector.PalmTouchEvents)
                     touchEventVisualizer.addPalmTouchEvent(e, CurrentFrame);
                 touchEventVisualizer.updateImage();
+                OutputImages[3] = touchEventVisualizer.OutputImage;
             }
 
             // add borders
-            edgeFingerOutputImage.drawRectangle(0, 0, Width - 1, Height - 1, Color.White);
-            depthPalmTouchOutputImage.drawRectangle(0, 0, Width - 1, Height - 1, Color.White);
-            OutputImages = new OutputImage[] { edgeFingerOutputImage, depthPalmTouchOutputImage, touchEventVisualizer.OutputImage};
+            for (int i = 0; i < numImages; i++)
+            {
+                OutputImages[i].drawRectangle(0, 0, Width - 1, Height - 1, Color.White);
+
+            }
+
             Timer.stop("processFrameUpdate");
         }
     }