using System; using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Text; using System.Threading.Tasks; using MathNet.Numerics.LinearAlgebra.Single; using bbiwarg.Images; using bbiwarg.InputProviders; using bbiwarg.Detectors; using bbiwarg.Helpers; namespace bbiwarg.VideoHandles { class VideoHandle { private IInputProvider inputProvider; private ForeFingerDetection foreFingerDetection; private PalmDetection palmDetection; private DepthImage depthImage; private ConfidenceImage confidenceImage; private UVImage uvImage; private ColorImage colorImage; private DepthImage handImage; private Int16[] segmentationDepth; private int[] histogram; private float maxU; private float maxV; public VideoHandle(IInputProvider inputProvider) { this.inputProvider = inputProvider; inputProvider.init(); inputProvider.start(); inputProvider.updateFrame(); foreFingerDetection = new ForeFingerDetection(inputProvider, this); palmDetection = new PalmDetection(inputProvider, this); createImageData(); maxU = (float)(Math.Tan(inputProvider.getHFOV() / 2f)); maxV = (float)(Math.Tan(inputProvider.getVFOV() / 2f)); } ~VideoHandle() { //inputProvider.stop(); } public void nextFrame() { if (inputProvider.isActive()) { inputProvider.releaseFrame(); inputProvider.updateFrame(); createImageData(); } else { inputProvider.stop(); } } public int getWidth() { return depthImage.getWidth(); } public int getHeight() { return depthImage.getHeight(); } private void createImageData() { depthImage = inputProvider.getDepthImage(); handImage = inputProvider.getDepthImage(); confidenceImage = inputProvider.getConfidenceImage(); uvImage = inputProvider.getUVImage(); colorImage = inputProvider.getColorImage(); handImage.filterMedian(3); Int16 minDepth = handImage.getMinDepth(); handImage.thresholdDepth(minDepth, minDepth + 200); histogram = handImage.getSmoothedHistogram(); segmentationDepth = handImage.getSegmentationDepth(); handImage.thresholdBinary((Int16) (segmentationDepth[1]+minDepth)); } public Int16[] getSegementationDepth() { return segmentationDepth; } public int[] getSmoothedHistogram() { return histogram; } public short getDepth(int x, int y) { return depthImage.getDepth(x, y); } public short getConfidence(int x, int y) { return confidenceImage.getConfidence(x, y); } public Color getColor(int x, int y) { float u = uvImage.getU(x, y); float v = uvImage.getV(x, y); if (u < 0 || v < 0) return Color.Black; int colorImageWidth = colorImage.getWidth(); int colorImageHeight = colorImage.getHeight(); int xInColorImage = (int)(u * colorImageWidth) % colorImageWidth; int yInColorImage = (int)(v * colorImageHeight) % colorImageHeight; return colorImage.getColor(xInColorImage, yInColorImage); } public DepthImage getHandImage() { return handImage; } public DetectionStatus[] getFingerStatus(uint handIndex) { return inputProvider.getFingerStatus(handIndex); } public Vector[] getFingerTipPositions3D(uint handIndex) { return inputProvider.getFingerTipPositions3D(handIndex); } public Vector getPalmPosition3D(uint handIndex) { return inputProvider.getPalmPosition3D(handIndex); } public Vector getPalmNormal3D(uint handIndex) { return inputProvider.getPalmNormal3D(handIndex); } public Vector getForearmPosition3D(uint handIndex) { return inputProvider.getForearmPosition3D(handIndex); } public Vector getForeFingerPosition3D(uint handIndex) { return foreFingerDetection.getForeFingerPosition3D(handIndex); } public List getHandPoints() { return foreFingerDetection.getHandPoints(); } public Palm getPalm(uint handIndex) { return palmDetection.getPalm(handIndex); } // TODO public void createVertexArray(IntPtr vertexBuffer) { int width = depthImage.getWidth(); int height = depthImage.getHeight(); int confidenceThreshold = inputProvider.getConfidenceThreshold(); int index = 0; for (int x = 0; x < width; x++) { for (int y = 0; y < height; y++) { if (confidenceImage.getConfidence(x, y) > confidenceThreshold) { int depth = depthImage.getDepth(x, y); Color c = getColor(x, y); create3DVertexFrom2D(x, y, depth, c, index, vertexBuffer); index++; } } } } public Vector pixel2VertexPosition(Vector pixelPosition) { int width = depthImage.getWidth(); int height = depthImage.getHeight(); float convertedDepth = pixelPosition[2] / 1000f; // mm into m float u = (pixelPosition[0] / (float)width - 0.5f) * 2f; float v = ((1 - pixelPosition[1] / (float)height) - 0.5f) * 2f; float relX = (u * maxU); float relY = (v * maxV); Vector result = new DenseVector(3); result[2] = convertedDepth / (float)Math.Sqrt(1 + relX * relX + relY * relY); result[0] = relX * result[2]; result[1] = relY * result[2]; return result; } private void create3DVertexFrom2D(float pixelX, float pixelY, int depth, Color c, int index, IntPtr vertexBuffer) { Vector pixelPosition = new DenseVector(3); pixelPosition[0] = pixelX; pixelPosition[1] = pixelY; pixelPosition[2] = depth; Vector vertexPosition = pixel2VertexPosition(pixelPosition); int i4 = (3 * sizeof(float) + 4 * sizeof(byte)) / sizeof(float) * index; int i16 = (3 * sizeof(float) + 4 * sizeof(byte)) * index; unsafe { byte* vertexArrayB = (byte*)vertexBuffer.ToPointer(); float* vertexArrayF = (float*)vertexBuffer.ToPointer(); vertexArrayF[i4 + 0] = vertexPosition[0]; vertexArrayF[i4 + 1] = vertexPosition[1]; vertexArrayF[i4 + 2] = -vertexPosition[2]; vertexArrayB[i16 + 12] = c.R; vertexArrayB[i16 + 13] = c.G; vertexArrayB[i16 + 14] = c.B; vertexArrayB[i16 + 15] = c.A; } } } }