bbiwarg-original/bbiwarg/Recognition/TouchRecognition/TouchDetector.cs

using BBIWARG.Images;
using BBIWARG.Input.InputHandling;
using BBIWARG.Recognition.FingerRecognition;
using BBIWARG.Recognition.PalmRecognition;
using BBIWARG.Utility;
using Emgu.CV;
using Emgu.CV.Structure;
using Emgu.CV.UI;
using System;
using System.Collections.Generic;
using System.Drawing;

namespace BBIWARG.Recognition.TouchRecognition
{
    /// <summary>
    /// Detects touches by flood filling around a small area around each finger tip and counting the number of affected pixels. If the finger is touching or slightly hovering above something, the flood fill spreads into the touched object and the number of affected pixels is higher.
    /// </summary>
    internal class TouchDetector
    {
        /// <summary>
        /// the depth image of the current frame
        /// </summary>
        private DepthImage depthImage;

        /// <summary>
        /// the fingers of the current frame
        /// </summary>
        private List<Finger> fingers;

        /// <summary>
        /// the palms of the current frame
        /// </summary>
        private List<Palm> palms;

        /// <summary>
        /// the detected touches in the current frame
        /// </summary>
        private List<Touch> touches;

        /// <summary>
        /// Detects touches in the current frame and stores them in frameData.detectedTouchEvents
        /// </summary>
        /// <param name="frameData">the current frame</param>
        public void detectTouches(FrameData frameData)
        {
            depthImage = frameData.DepthImage;
            fingers = frameData.TrackedFingers;
            palms = frameData.TrackedPalms;
            touches = new List<Touch>();

            if (palms.Count > 0)
            {
                foreach (Finger finger in fingers)
                {
                    Touch touch = detectTouch(finger);
                    if (touch != null)
                        touches.Add(touch);
                }
            }

            frameData.DetectedTouches = touches;
        }

        /// <summary>
        /// Detects if a finger is touching a palm and either returns a new Touch or null
        /// </summary>
        /// <param name="finger">the fingers</param>
        /// <returns>a new Touch or null</returns>
        private Touch detectTouch(Finger finger)
        {
            Vector2D tipPoint = finger.TipPoint;
            Vector2D direction = finger.Direction;
            Vector2D tipPointInside = tipPoint.moveWithinBound(depthImage.Size, direction.getInverse(), Parameters.TouchTipInsideFactor);
            Vector2D tipPointOutside = tipPoint.moveWithinBound(depthImage.Size, direction, Parameters.TouchTipOutsideFactor);

            foreach (Palm palm in palms)
            {
                if (palm.isInside(tipPointOutside))
                {
                    Image<Gray, byte> touchMask = getTouchMask(tipPointInside);
                    
                    int touchPixels = touchMask.CountNonzero()[0];
                    int numPixels = touchMask.Width * touchMask.Height;
                    float touchValue = touchPixels / (float)numPixels;
                    
                    if (touchValue > Parameters.TouchMinTouchValue)
                    {
                        Touch touch = new Touch(tipPointOutside, finger, palm);
                        return touch;
                    }

                }
            }


            return null;
        }

        /// <summary>
        /// Gets an image of a small area around the desired touch point (copied from the depth image)
        /// </summary>
        /// <param name="touchPoint">the touch position</param>
        /// <returns>image of the touch area around the touch position</returns>
        private Image<Gray, byte> getTouchMask(Vector2D touchPoint)
        {
            int minX = Math.Max(touchPoint.IntX - 2 * Parameters.TouchAreaSize / 3, 0);
            int maxX = Math.Min(touchPoint.IntX + Parameters.TouchAreaSize / 3, depthImage.Size.Width - 1);
            int minY = Math.Max(touchPoint.IntY - 2 * Parameters.TouchAreaSize / 3, 0);
            int maxY = Math.Min(touchPoint.IntY + Parameters.TouchAreaSize / 3, depthImage.Size.Height - 1);

            Vector2D relTouchPoint = new Vector2D(touchPoint.IntX - minX, touchPoint.IntY - minY);
            Rectangle rect = new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1);
            Image<Gray, byte> touchArea = depthImage.Image.Copy(rect);
            Image<Gray, byte> touchMask = new Image<Gray, byte>(rect.Width + 2, rect.Height + 2);

            MCvConnectedComp comp = new MCvConnectedComp();
            CvInvoke.cvFloodFill(touchArea, relTouchPoint, new MCvScalar(255), new MCvScalar(Parameters.TouchFloodfillDownDiff), new MCvScalar(Parameters.TouchFloodfillUpDiff), out comp, Emgu.CV.CvEnum.CONNECTIVITY.EIGHT_CONNECTED, Emgu.CV.CvEnum.FLOODFILL_FLAG.DEFAULT, touchMask);

            Rectangle cropRect = new Rectangle(1, 1, rect.Width, rect.Height);
            return touchMask.Copy(cropRect);
        }
    }
}