kinect + processing + azealia banks + mickey

I was playing around with mapping different graphics onto the skeleton using Mickey Mouse as a stand in.  I want to do more with this, use a naked body or something more interesting than Mickey.  I need to make it so the hands interpret the angles of the arms and add feet and other things.  I added the Azealia Banks video in the background just to see how that would work.  I want to make it so that the video pauses if the mickey character (the user) stops dancing.

Here’s my code, cobbled together from the examples on the syllabus and advice from Tak.

import SimpleOpenNI.*; 

Movie ab;
SimpleOpenNI kinect; 

PImage head;
PImage torso;
PImage rightHand;
PImage leftHand;

PImage resultImage;
color trans = color(0, 0, 0, 0);

int[] userMap; // declare our images 

boolean calibrated;

/* *** SKELETON 1: MICKEY *** */

void setup() {
  kinect = new SimpleOpenNI(this);

  size(640, 360);

  resultImage = new PImage(640, 480, ARGB);

  ab = new Movie(this, "ab.mp4");

  head = loadImage("head.png");
  torso = loadImage("torso.png");
  rightHand = loadImage("left-hand.png");
  leftHand = loadImage("right-hand.png");

void draw() {
  image(ab, 0, 0);
  if (!calibrated) {
    image(kinect.depthImage(), 0, 0);

  IntVector userList = new IntVector();

  if (userList.size() > 0) {
    int userId = userList.get(0);

    if ( kinect.isTrackingSkeleton(userId)) {

      userMap = kinect.getUsersPixels(SimpleOpenNI.USERS_ALL);
      for (int i =0; i < userMap.length; i++) {
        // if the pixel is part of the user
        if (userMap[i] != 0) {
          // set the pixel to the color pixel
          resultImage.pixels[i] = kinect.depthImage().pixels[i];
        else {
          //set it to the background
          resultImage.pixels[i] = trans;
      //update the pixel from the inner array to image
      image(resultImage, 0, 0);


    PVector torsoPos = new PVector();
    kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_TORSO, torsoPos);
    kinect.convertRealWorldToProjective(torsoPos, torsoPos);
    float newImageWidth = map(torsoPos.z, 500, 2000, torso.width, torso.width/2);
    float newImageHeight = map(torsoPos.z, 500, 2000, torso.width, torso.width/2);
    image(torso, torsoPos.x - newImageWidth/2, torsoPos.y - newImageHeight/3, newImageWidth, newImageHeight);

    PVector rightHandPos = new PVector();
    kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_RIGHT_HAND, rightHandPos);
    kinect.convertRealWorldToProjective(rightHandPos, rightHandPos);
    newImageWidth = map(rightHandPos.z, 500, 2000, rightHand.width, rightHand.width/2);
    newImageHeight = map(rightHandPos.z, 500, 2000, rightHand.width, rightHand.width/2);
    image(rightHand, rightHandPos.x - newImageWidth/2, rightHandPos.y - newImageHeight/2, newImageWidth, newImageHeight);

    PVector leftHandPos = new PVector();
    kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_LEFT_HAND, leftHandPos);
    kinect.convertRealWorldToProjective(leftHandPos, leftHandPos);
    newImageWidth = map(leftHandPos.z, 500, 2000, leftHand.width, leftHand.width/2);
    newImageHeight = map(leftHandPos.z, 500, 2000, leftHand.width, leftHand.width/2);
    image(leftHand, leftHandPos.x - newImageWidth/2, leftHandPos.y - newImageHeight/2, newImageWidth, newImageHeight);

    PVector headPos = new PVector();
    kinect.getJointPositionSkeleton(userId, SimpleOpenNI.SKEL_HEAD, headPos);
    kinect.convertRealWorldToProjective(headPos, headPos);
    newImageWidth = map(headPos.z, 500, 2000, head.width, head.width/2);
    newImageHeight = map(headPos.z, 500, 2000, head.width, head.width/2);
    image(head, headPos.x - newImageWidth/2 - 25, headPos.y - newImageHeight/2 - 50, newImageWidth, newImageHeight );
void drawSkeleton(int userId) {

  kinect.drawLimb(userId, SimpleOpenNI.SKEL_HEAD, SimpleOpenNI.SKEL_NECK);
  //image(head, SimpleOpenNI.SKEL_HEAD.x - head.width/2, SimpleOpenNI.SKEL_HEAD.y - head.height);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_NECK, SimpleOpenNI.SKEL_LEFT_SHOULDER);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER, SimpleOpenNI.SKEL_LEFT_ELBOW);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_ELBOW, SimpleOpenNI.SKEL_LEFT_HAND);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_NECK, SimpleOpenNI.SKEL_RIGHT_SHOULDER);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER, SimpleOpenNI.SKEL_RIGHT_ELBOW);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_ELBOW, SimpleOpenNI.SKEL_RIGHT_HAND);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER, SimpleOpenNI.SKEL_TORSO);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER, SimpleOpenNI.SKEL_TORSO);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_TORSO, SimpleOpenNI.SKEL_LEFT_HIP);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_HIP, SimpleOpenNI.SKEL_LEFT_KNEE);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_LEFT_KNEE, SimpleOpenNI.SKEL_LEFT_FOOT);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_TORSO, SimpleOpenNI.SKEL_RIGHT_HIP);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_HIP, SimpleOpenNI.SKEL_RIGHT_KNEE);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_KNEE, SimpleOpenNI.SKEL_RIGHT_FOOT);
  kinect.drawLimb(userId, SimpleOpenNI.SKEL_RIGHT_HIP, SimpleOpenNI.SKEL_LEFT_HIP);


  fill(0, 255, 0);
  drawJoint(userId, SimpleOpenNI.SKEL_HEAD);  

   drawJoint(userId, SimpleOpenNI.SKEL_NECK);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_SHOULDER);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_ELBOW);
   drawJoint(userId, SimpleOpenNI.SKEL_NECK);
   drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_SHOULDER);
   drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_ELBOW);
   drawJoint(userId, SimpleOpenNI.SKEL_TORSO);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_HIP);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_KNEE);
   drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_HIP);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_FOOT);
   drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_KNEE);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_HIP);
   drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_FOOT);
   drawJoint(userId, SimpleOpenNI.SKEL_RIGHT_HAND);
   drawJoint(userId, SimpleOpenNI.SKEL_LEFT_HAND);

float getJointDistance(int userId, int jointID1, int jointID2) {
  PVector joint1 = new PVector();
  PVector joint2 = new PVector();
  kinect.getJointPositionSkeleton(userId, jointID1, joint1);
  kinect.getJointPositionSkeleton(userId, jointID2, joint2);
  return PVector.dist(joint1, joint2);

void drawJoint(int userId, int jointID) {
  PVector joint = new PVector();
  float confidence = kinect.getJointPositionSkeleton(userId, jointID, joint);
  if (confidence < 0.5) {
  PVector convertedJoint = new PVector();
  kinect.convertRealWorldToProjective(joint, convertedJoint);
  ellipse(convertedJoint.x, convertedJoint.y, 5, 5);
// user-tracking callbacks! 
void onNewUser(int userId) {
  println("start pose detection");
  kinect.startPoseDetection("Psi", userId);

void onEndCalibration(int userId, boolean successful) {
  if (successful) {
    println(" User calibrated !!!");
    calibrated = true;
  else {
    println(" Failed to calibrate user !!!");
    kinect.startPoseDetection("Psi", userId);

void onStartPose(String pose, int userId) {
  println("Started pose for user");
  kinect.requestCalibrationSkeleton(userId, true);

void movieEvent(Movie ab) {;

Author: owen ribbit


One thought on “kinect + processing + azealia banks + mickey”

  1. Hey great work!,
    Trying to compile but it i have an error with the .mp4 file, im using a file that is the same size as the window. and its also an .mp4.
    My error is as follows:

    SimpleOpenNI Version 0.27
    Exception in thread “Animation Thread” java.lang.RuntimeException: Could not load movie file Boreal cliff vid.mp4
    at processing.core.PApplet.die(
    at processing.core.PApplet.die(
    at Person_in_video.setup(
    at processing.core.PApplet.handleDraw(
    at processing.core.PGraphicsJava2D.requestDraw(

    I would love to know your opinion on whats going on, thanks kindly!



Leave a Reply

Fill in your details below or click an icon to log in: Logo

You are commenting using your account. Log Out /  Change )

Google+ photo

You are commenting using your Google+ account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )


Connecting to %s