Get Depth for RGB pixel

[CharlesFr]

I'm having a hard time understanding the "MapDepthToColor" example. I'd be very grateful if you could point out how to retreive the depth of an RGB pixel?

I'm specifically trying to retrieve the depth of the user for each RGB pixel.

Thanks for sharing this library!

[CharlesFr]

By slightly changing the example I get the depth mapped to RGB space. Unfortunately, there is a strange duplication appearing. Notice how the tv appears twice in an overlap. (bottom left image)

Any ideas how to fix this?

/*
Thomas Sanchez Lengeling.
http://codigogenerativo.com/

KinectPV2, Kinect for Windows v2 library for processing

Color to fepth Example,
Color Frame is aligned to the depth frame
*/

import KinectPV2.*;

KinectPV2 kinect;

int [] depthZero;

//BUFFER ARRAY TO CLEAN DE PIXLES
PImage depthToColorImg;

void setup() {
  size(1024, 848, P3D);

  depthToColorImg = createImage(512, 424, PImage.RGB);
  depthZero    = new int[ KinectPV2.WIDTHDepth * KinectPV2.HEIGHTDepth];

  //SET THE ARRAY TO 0s
  for (int i = 0; i < KinectPV2.WIDTHDepth; i++) {
    for (int j = 0; j < KinectPV2.HEIGHTDepth; j++) {
      depthZero[424*i + j] = 0;
    }
  }

  kinect = new KinectPV2(this);
  kinect.enableDepthImg(true);
  kinect.enableColorImg(true);
  kinect.enablePointCloud(true);

  kinect.init();
}

void draw() {
  background(0);

  float [] mapDCT = kinect.getMapDepthToColor(); // 434176
  print(mapDCT.length, TAB);

  //get the raw data from depth and color
  int [] colorRaw = kinect.getRawColor(); // 434176
  println(mapDCT.length, KinectPV2.WIDTHDepth, KinectPV2.HEIGHTDepth);
  
  int [] depthRaw = kinect.getRawDepthData(); // 434176

  //clean de pixels
  PApplet.arrayCopy(depthZero, depthToColorImg.pixels);

  int count = 0;
  depthToColorImg.loadPixels();
  for (int i = 0; i < KinectPV2.WIDTHDepth; i++) {
    for (int j = 0; j < KinectPV2.HEIGHTDepth; j++) {

      //incoming pixels 512 x 424 with position in 1920 x 1080
      float valX = mapDCT[count * 2 + 0];
      float valY = mapDCT[count * 2 + 1];
      

      //maps the pixels to 512 x 424, not necessary but looks better
      int valXDepth = (int)((valX/1920.0) * 512.0);
      int valYDepth = (int)((valY/1080.0) * 424.0);

      int  valXColor = (int)(valX);
      int  valYColor = (int)(valY);

      if ( valXDepth >= 0 && valXDepth < 512 && valYDepth >= 0 && valYDepth < 424 &&
        valXColor >= 0 && valXColor < 1920 && valYColor >= 0 && valYColor < 1080) {
        //color colorPixel = colorRaw[valYColor * 1920 + valXColor];
        float col = map(depthRaw[valYDepth * 512 + valXDepth], 0, 4500, 0, 255);
        color colorPixel = color(col);
        depthToColorImg.pixels[valYDepth * 512 + valXDepth] = colorPixel;
      } 
      count++;
    }
  }
  depthToColorImg.updatePixels();

  image(depthToColorImg, 0, 424);
  image(kinect.getColorImage(), 0, 0, 512, 424);
  image(kinect.getDepthImage(), 512, 0);

  text("fps: "+frameRate, 50, 50);
}

[CharlesFr]

Here's the colution I've come up with, if anyone comes up with something more elegant then please share.

import KinectPV2.*;

KinectPV2 kinect;

ArrayList<Pair> sorted = new ArrayList<Pair>();

void setup() {
  size(1920, 1080);
  
  frameRate(60);

  kinect = new KinectPV2(this);
  kinect.enableDepthImg(true);
  kinect.enableColorImg(true);
  kinect.enablePointCloud(true); // getMapDepthToColor() doesn't work without enabling point cloud
  kinect.init();
}

void draw() {

  PImage imgDepth = kinect.getDepthImage(); 
  PImage imgColor = kinect.getColorImage();

  sorted.clear(); // We clear the array ready for a new frame

  float [] mapDCT = kinect.getMapDepthToColor(); // 434,176
  int [] rawDepth = kinect.getRawDepthData();

  for (int i = 0; i < mapDCT.length; i+=2) { // We iterate in increments of 2 as i+0 is X and i+1 is Y

    if (mapDCT[i] > 0 && mapDCT[i] < width &&
      mapDCT[i+1] > 0 && mapDCT[i+1] < height && 
      rawDepth[i/2] != 0) {
      sorted.add( new Pair(i/2, mapDCT[i], mapDCT[i+1]));
    }
  }

  PImage img = createImage(width, height, RGB);

  img.loadPixels();

  for (int i = 0; i < sorted.size(); i++) {
    img.set((int)sorted.get(i).x, (int)sorted.get(i).y, color((int)map(rawDepth[sorted.get(i).index], 0, 4500, 0, 255)));
  }

  img.updatePixels();
  //tint(255, 127);
  image(img, 0, 0);

  fill(255);
  text(frameRate, 50, 50);
}

public class Pair{
  public final int index;
  public final float x;
  public final float y;
  public final float value;

  public Pair(int index, float x, float y) {
    this.index = index;
    this.value = y * width + x;
    this.x = x;
    this.y = y;
  }
}