|
@@ -30,11 +30,10 @@ namespace bbiwarg.Detectors.Palm
|
|
|
private List<MCvConvexityDefect> convexityDefects;
|
|
|
private Vector2D thumbDefectStart;
|
|
|
private Vector2D thumbDefectEnd;
|
|
|
- private Vector2D thumbDefectDepth;
|
|
|
+ private Vector2D thumbDefectDepth, lastThumbDefectDepth;
|
|
|
|
|
|
private Kalman2DPositionFilter thumbDefectDepthFilter, thumbDefectStartFilter, thumbDefectEndFilter;
|
|
|
|
|
|
- private bool valid = false;
|
|
|
private Vector2D topLeft;
|
|
|
private Vector2D topRight;
|
|
|
private Vector2D bottomLeft;
|
|
@@ -55,36 +54,46 @@ namespace bbiwarg.Detectors.Palm
|
|
|
this.width = depthImage.Width;
|
|
|
this.height = depthImage.Height;
|
|
|
|
|
|
- this.edgeImage = edgeImage;
|
|
|
- this.outputImage = outputImage;
|
|
|
-
|
|
|
- // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
|
|
|
- handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
|
|
|
-
|
|
|
- fingers = getFingersWithoutThumb(trackedFingers);
|
|
|
- buildPointingHandMask();
|
|
|
- handImage = handImage.And(pointingHandMask);
|
|
|
+ i1 = new OutputImage(width, height);
|
|
|
+ i2 = new OutputImage(width, height);
|
|
|
+ i3 = new OutputImage(width, height);
|
|
|
+ i4 = new OutputImage(width, height);
|
|
|
+ i5 = new OutputImage(width, height);
|
|
|
+ i6 = new OutputImage(width, height);
|
|
|
+ i7 = new OutputImage(width, height);
|
|
|
+ i8 = new OutputImage(width, height);
|
|
|
+ i9 = new OutputImage(width, height);
|
|
|
|
|
|
- findLongestPalmContour();
|
|
|
- if (palmContour != null)
|
|
|
+ if (trackedFingers.Count >= 2)
|
|
|
{
|
|
|
- findConvexityDefectsSortedByDepth();
|
|
|
- removeConvexityDefectsCausedByFingers();
|
|
|
+ this.edgeImage = edgeImage;
|
|
|
+ this.outputImage = outputImage;
|
|
|
+
|
|
|
+ // dst = (src > (MaxDepth - MinDepth)) ? 0 : 1
|
|
|
+ handImage = depthImage.Image.ThresholdBinaryInv(new Gray(depthImage.MaxDepth - depthImage.MinDepth - 1), new Gray(1)).Convert<Gray, Byte>();
|
|
|
|
|
|
- findHandPoints();
|
|
|
+ fingers = getFingersWithoutThumb(trackedFingers);
|
|
|
+ buildPointingHandMask();
|
|
|
+ handImage = handImage.And(pointingHandMask);
|
|
|
|
|
|
- if (valid)
|
|
|
+ findLongestPalmContour();
|
|
|
+ if (palmContour != null)
|
|
|
{
|
|
|
- draw();
|
|
|
+ findConvexityDefectsSortedByDepth();
|
|
|
+ removeConvexityDefectsCausedByFingers();
|
|
|
+
|
|
|
+ findHandPoints();
|
|
|
}
|
|
|
}
|
|
|
+ draw();
|
|
|
}
|
|
|
|
|
|
- public void resetFilters()
|
|
|
+ public void reset()
|
|
|
{
|
|
|
thumbDefectDepthFilter.reset();
|
|
|
thumbDefectStartFilter.reset();
|
|
|
thumbDefectEndFilter.reset();
|
|
|
+ lastThumbDefectDepth = null;
|
|
|
}
|
|
|
|
|
|
private List<Finger> getFingersWithoutThumb(List<Finger> detectedFingers)
|
|
@@ -114,9 +123,9 @@ namespace bbiwarg.Detectors.Palm
|
|
|
{
|
|
|
foreach (Finger f in fingers)
|
|
|
{
|
|
|
- foreach (FingerSlice s in f.SliceTrail.Slices)
|
|
|
+ //foreach (FingerSlice s in f.SliceTrail.Slices)
|
|
|
{
|
|
|
- image.Draw(new LineSegment2DF(s.Start, s.End), new Gray(val), 1);
|
|
|
+ image.Draw(new LineSegment2DF(f.SliceTrail.Slices[0].Start, f.SliceTrail.Slices[0].End), new Gray(val), 1);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
@@ -143,34 +152,30 @@ namespace bbiwarg.Detectors.Palm
|
|
|
if (finger == null)
|
|
|
return new Point(0, 0);
|
|
|
Vector2D direction = (finger.Hand - finger.Tip).normalize();
|
|
|
- Vector2D pos = (finger.Hand + 20 * direction).moveInBound(0,0,width-1,height-1);
|
|
|
+ Vector2D pos = finger.Hand + direction;
|
|
|
+
|
|
|
+ while (pos.isWithin(0, 0, width - 1, height - 1) && pointingHandMask.Data[pos.IntY, pos.IntX, 0] != 0)
|
|
|
+ pos += direction;
|
|
|
+
|
|
|
+ i3.fillCircle(pos.IntX, pos.IntY, 3, Color.Red);
|
|
|
+
|
|
|
return pos;
|
|
|
}
|
|
|
|
|
|
public OutputImage i1, i2, i3, i4, i5, i6, i7, i8, i9;
|
|
|
private void buildPointingHandMask()
|
|
|
{
|
|
|
- /*i1 = new OutputImage(width, height);
|
|
|
- i2 = new OutputImage(width, height);
|
|
|
- i3 = new OutputImage(width, height);
|
|
|
- i4 = new OutputImage(width, height);
|
|
|
- i5 = new OutputImage(width, height);
|
|
|
- i6 = new OutputImage(width, height);
|
|
|
- i7 = new OutputImage(width, height);
|
|
|
- i8 = new OutputImage(width, height);
|
|
|
- i9 = new OutputImage(width, height);*/
|
|
|
-
|
|
|
pointingHandMask = new Image<Gray, byte>(width, height, new Gray(0));
|
|
|
|
|
|
// dst = (src > 0) ? 1 : 0;
|
|
|
pointingHandMask = pointingHandMask.Or(edgeImage.Image.ThresholdBinary(new Gray(0), new Gray(1)));
|
|
|
- //i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
|
|
|
+ i1.Image[0] = i1.Image[1] = i1.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
- pointingHandMask = pointingHandMask.Dilate(4);
|
|
|
- //i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
|
|
|
+ pointingHandMask = pointingHandMask.Dilate(2);
|
|
|
+ i2.Image[0] = i2.Image[1] = i2.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
fillFingerSlices(pointingHandMask, 1);
|
|
|
- //i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
|
|
|
+ i3.Image[0] = i3.Image[1] = i3.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
//pointingHandMask = pointingHandMask.Dilate(1);
|
|
|
//i4.Image[0] = i4.Image[1] = i4.Image[2] = 255 * pointingHandMask;
|
|
@@ -181,29 +186,23 @@ namespace bbiwarg.Detectors.Palm
|
|
|
|
|
|
// dst = (src > 1) ? 0 : 1 (src > 1 <-> src == 2 <-> src filled by flood fill)
|
|
|
pointingHandMask = pointingHandMask.ThresholdBinaryInv(new Gray(1), new Gray(1));
|
|
|
- //i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
|
|
|
+ i5.Image[0] = i5.Image[1] = i5.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
- pointingHandMask = pointingHandMask.Erode(8);
|
|
|
- //i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
|
|
|
+ pointingHandMask = pointingHandMask.Erode(6);
|
|
|
+ i6.Image[0] = i6.Image[1] = i6.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
fillFingerSlices(pointingHandMask, 0);
|
|
|
- //i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
|
|
|
+ i7.Image[0] = i7.Image[1] = i7.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
pointingHandMask = pointingHandMask.Erode(2);
|
|
|
- //i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
|
|
|
+ i8.Image[0] = i8.Image[1] = i8.Image[2] = 255 * pointingHandMask;
|
|
|
|
|
|
// only debug
|
|
|
- //i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
|
|
|
+ i9.Image[0] = i9.Image[1] = i9.Image[2] = 255 * handImage.And(pointingHandMask);
|
|
|
}
|
|
|
|
|
|
private void findLongestPalmContour()
|
|
|
{
|
|
|
- i1 = new OutputImage(width, height);
|
|
|
- i1.Image[0] = i1.Image[1] = i1.Image[2] = handImage * 255;
|
|
|
-
|
|
|
- i2 = new OutputImage(width, height);
|
|
|
- i2.Image[0] = i2.Image[1] = i2.Image[2] = pointingHandMask * 255;
|
|
|
-
|
|
|
Contour<Point> contour = handImage.FindContours(Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL);
|
|
|
|
|
|
palmContour = contour;
|
|
@@ -237,17 +236,25 @@ namespace bbiwarg.Detectors.Palm
|
|
|
List<MCvConvexityDefect> newDefects = new List<MCvConvexityDefect>();
|
|
|
foreach (MCvConvexityDefect d in convexityDefects)
|
|
|
{
|
|
|
+ bool remove = false;
|
|
|
float minFingerLineDist = float.MaxValue;
|
|
|
foreach (Finger f in fingers)
|
|
|
{
|
|
|
- Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
|
|
|
+ Utility.LineSegment2D defectLine = new Utility.LineSegment2D(new Vector2D(d.StartPoint), new Vector2D(d.EndPoint));
|
|
|
+ Vector2D intersection = defectLine.Line.getIntersection(f.LineSegment.Line);
|
|
|
+ if (intersection.isInBox(defectLine.P1, defectLine.P2) && intersection.isInBox(f.LineSegment.P1, f.LineSegment.P2))
|
|
|
+ {
|
|
|
+ remove = true;
|
|
|
+ break;
|
|
|
+ }
|
|
|
|
|
|
+ Vector2D mid = (new Vector2D(d.StartPoint) + new Vector2D(d.EndPoint)) / 2.0f;
|
|
|
float dist = f.LineSegment.getDistanceTo(mid);
|
|
|
if (dist < minFingerLineDist)
|
|
|
minFingerLineDist = dist;
|
|
|
}
|
|
|
|
|
|
- if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance)
|
|
|
+ if (minFingerLineDist >= Constants.PalmMinDefectMidFingerLineDistance && !remove)
|
|
|
newDefects.Add(d);
|
|
|
}
|
|
|
convexityDefects = newDefects;
|
|
@@ -264,7 +271,9 @@ namespace bbiwarg.Detectors.Palm
|
|
|
float angle = (float) ((depth - start).getAngleBetween(depth - end) * 180 / Math.PI);
|
|
|
|
|
|
if (angle <= Constants.PalmMaxThumbDefectAngle)
|
|
|
+ {
|
|
|
return d;
|
|
|
+ }
|
|
|
}
|
|
|
return null;
|
|
|
}
|
|
@@ -288,12 +297,6 @@ namespace bbiwarg.Detectors.Palm
|
|
|
thumbDefectStartFilter.setInitialPosition(thumbDefectStart);
|
|
|
thumbDefectEndFilter.setInitialPosition(thumbDefectEnd);
|
|
|
}
|
|
|
- else if (thumbDefect == null)
|
|
|
- {
|
|
|
- thumbDefectDepth = thumbDefectDepthFilter.getPrediction();
|
|
|
- thumbDefectStart = thumbDefectStartFilter.getPrediction();
|
|
|
- thumbDefectEnd = thumbDefectEndFilter.getPrediction();
|
|
|
- }
|
|
|
else
|
|
|
{
|
|
|
thumbDefectDepth = thumbDefectDepthFilter.getCorrectedPosition(thumbDefectDepth);
|
|
@@ -301,6 +304,8 @@ namespace bbiwarg.Detectors.Palm
|
|
|
thumbDefectEnd = thumbDefectEndFilter.getCorrectedPosition(thumbDefectEnd);
|
|
|
}
|
|
|
|
|
|
+ lastThumbDefectDepth = thumbDefectDepth;
|
|
|
+
|
|
|
Vector2D handLength, handWidth;
|
|
|
if (thumbDefectDepth.getDistanceTo(thumbDefectStart) > thumbDefectDepth.getDistanceTo(thumbDefectEnd))
|
|
|
{
|
|
@@ -324,29 +329,26 @@ namespace bbiwarg.Detectors.Palm
|
|
|
}
|
|
|
|
|
|
PalmQuad = new Quadrangle(bottomLeft, topLeft, topRight, bottomRight);
|
|
|
-
|
|
|
- valid = true;
|
|
|
- }
|
|
|
- else
|
|
|
- {
|
|
|
- PalmQuad = null;
|
|
|
- valid = false;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private void draw()
|
|
|
{
|
|
|
- outputImage.drawContour(palmContour, Constants.PalmConturColor);
|
|
|
- outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
|
|
|
+
|
|
|
+
|
|
|
+ if (palmContour != null && palmContour.Count<Point>() > 0) {
|
|
|
+ outputImage.drawContour(palmContour, Constants.PalmConturColor);
|
|
|
+ outputImage.drawPoints(palmContour.GetConvexHull(Emgu.CV.CvEnum.ORIENTATION.CV_CLOCKWISE), Constants.PalmConvexHullColor);
|
|
|
+ }
|
|
|
|
|
|
if (PalmQuad != null)
|
|
|
{
|
|
|
+ outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Red);
|
|
|
+ outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Red);
|
|
|
outputImage.fillCircle(thumbDefectDepth.IntX, thumbDefectDepth.IntY, 3, Color.Red);
|
|
|
- outputImage.fillCircle(thumbDefectStart.IntX, thumbDefectStart.IntY, 3, Color.Green);
|
|
|
- outputImage.fillCircle(thumbDefectEnd.IntX, thumbDefectEnd.IntY, 3, Color.Blue);
|
|
|
|
|
|
- //outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
|
|
|
-
|
|
|
+ outputImage.drawLineSegment(new Utility.LineSegment2D(thumbDefectDepth, (thumbDefectStart + thumbDefectEnd) / 2.0f), Constants.PalmThumbDefectColor, 1);
|
|
|
+
|
|
|
Vector2D[] vertices = PalmQuad.Vertices;
|
|
|
for (int i = 0; i < 4; ++i)
|
|
|
outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(vertices[i], vertices[(i + 1) % 4]), Constants.PalmQuadColor);
|
|
@@ -357,20 +359,17 @@ namespace bbiwarg.Detectors.Palm
|
|
|
|
|
|
private void drawGrid(Vector2D a, Vector2D b, Vector2D c, Vector2D d)
|
|
|
{
|
|
|
- int numRows = 4;
|
|
|
- int numColumns = 3;
|
|
|
-
|
|
|
- Vector2D relAB = (b - a) / numRows;
|
|
|
- Vector2D relDC = (c - d) / numRows;
|
|
|
- Vector2D relBC = (c - b) / numColumns;
|
|
|
- Vector2D relAD = (d - a) / numColumns;
|
|
|
+ Vector2D relAB = (b - a) / Constants.PalmGridRows;
|
|
|
+ Vector2D relDC = (c - d) / Constants.PalmGridRows;
|
|
|
+ Vector2D relBC = (c - b) / Constants.PalmGridColumns;
|
|
|
+ Vector2D relAD = (d - a) / Constants.PalmGridColumns;
|
|
|
|
|
|
- for (int i = 1; i < numRows; i++)
|
|
|
+ for (int i = 1; i < Constants.PalmGridRows; i++)
|
|
|
{
|
|
|
outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAB, d + i * relDC), Constants.PalmGridColor);
|
|
|
}
|
|
|
|
|
|
- for (int i = 1; i < numColumns; i++)
|
|
|
+ for (int i = 1; i < Constants.PalmGridColumns; i++)
|
|
|
{
|
|
|
outputImage.drawLineSegment(new bbiwarg.Utility.LineSegment2D(a + i * relAD, b + i * relBC), Constants.PalmGridColor);
|
|
|
}
|