Kinect programing

PROGRAMMING
PROGRAMMING
PROGRAMMING

This trimester we have worked on interactive installation shown at Worm as a final outcome of thematic project "Creative Industries". For this purpose we had to work with processing and kinect as a common tool in interaction design. It was nice continuation and practical exercise of the classes we had with Evo this trim - working with kinect. The installation is an interactive test with multiple choice answers (A, B, C, D) and center of initial position. The user has to runs through 13 questions within 15 or 30 sec. per answer.( we have to work on that feedback from users - for certain people the time was not enough while for others was to slow.) At the end a random result in percentage is displayed on the screen. If no one is in the periphery of the test a slide show is displayed.
The code above is the main sketch where kinect is using library SimpleOpenNI for tracking the center of mass of the user according to the distance from kinect, the user radius and the radius of the circles marked on the floor. They are two classes - one for the questions and the second for the result state. more description follows--->

import SimpleOpenNI.*; // kinect library
import gifAnimation.*;

SimpleOpenNI context;
//********************* SlideShow 
String[] imgName;
PImage[] img;  
PImage[] resultImgs; 
Result[] results;
Question[] questions;
Question currentQuestion;

//int xPercenFont = 1000;
//int yPercenFont = 600;

int xPercenFont = 1355;
int yPercenFont = 105;

int nextimg = 0;
int startTime=3000;
int preTime;
boolean drawingKinectImage = false;
boolean drawingKinectCoords = false;
PFont fontPercentage;
PFont fontClock;
int thanksDuration = 5000; //number milliseconds thanks img
int startTimeThanks;
PImage instructionImg;
PImage gifAnimation;
PImage lastImg;
String currentPercentage;
int currentColorCircle = 0;


//********************  timer
int c;
int csec; //seconds
//***************

//******************** timer for gif animation 
Gif loopingGif;
int gifDuration = 5000;
int startTimeGif;

//Array of the number of the questions
int currentQuestionCounter = 0;
int startTimeAnswer = 30000;
//set to 30 sec

PVector com = new PVector(); // center of mass

//kinect part
int trackedUserID = 0;
PVector userPosRealWorld = new PVector(); // 3d user position
PVector userPosProjected = new PVector(); // 2d position (used to draw circle)

int NOBODY = 0;
int INSTRUCTION = 1;
int QUESTION = 2;
int RESULT = 3;
int THANKS = 4;

int width = 1920;
int height = 1080;

int currentState = NOBODY;

//*******************centerCircle

float centerX = 0;
float centerZ = 3900;
//float centerX = -135;
//float centerZ = 1700;

float radius = 150;
float userRadius = 150;
//float radius = 120;
//float userRadius = 120;


//**********************A
float centerA_X= centerX -radius;
float centerA_Z = centerZ;

//**********************B
float centerB_X= centerX + radius;
float centerB_Z = centerZ;

//**********************C
float centerC_X= centerX;
float centerC_Z = centerZ - radius;

//**********************D
float centerD_X = centerX;
float centerD_Z = centerZ + radius;


int startTimeQuestion;



void setup() {
 
 lastImg = loadImage("75.jpg");
 loopingGif = new Gif(this, "loading74.gif");
 
  results = new Result[50];
  questions = new Question[13];
 
  size(width, height);
  //********************************slide show images.
  imgName=new String[3]; 
  imgName[0]="1.jpg";
  imgName[1]="2.jpg"; 
  imgName[2]="3.jpg"; 

  img=new PImage[3];
  for (int i=0; i<=2; i++) {
    img[i] = loadImage(imgName[i]);
  }
  
  
  
 resultImgs = new PImage[4];
 int counter =0;
  for(int z = 71; z<=74; z++){
    resultImgs[counter] = loadImage(z +".jpg");
    counter++;    
 }
 
  
  preTime=millis();
  instructionImg = loadImage("4.jpg");  

results[0] = new Result("7.5%", 0); //grey

results[1] = new Result("13.9%", 0);
results[2] = new Result("21.3%", 0);
results[3] = new Result("2.2%", 0);
results[4] = new Result("24.9%", 0);
results[5] = new Result("19.2%", 0);
results[6] = new Result("15%", 0);
results[7] = new Result("8.6%", 0);
results[8] = new Result("23%", 0); 
results[9] = new Result("10.7%", 0); 
results[10] = new Result("5.7%", 0); 
results[11] = new Result("6.6%", 0); 
results[12] = new Result("9.0%", 0); 

results[13] = new Result("25.2%", 1);
results[14] = new Result("30.7%", 1);
results[15] = new Result("43.3%", 1);
results[16] = new Result("44.9%", 1);
results[17] = new Result("47.4%", 1);
results[18] = new Result("33.3%", 1);
results[19] = new Result("36.7%", 1);
results[20] = new Result("48%", 1);
results[21] = new Result("37.5%", 1); 
results[22] = new Result("40.4%", 1); 
results[23] = new Result("25%", 1);
results[24] = new Result("41%", 1); 
results[25] = new Result("29.8%", 1); 

results[26] = new Result( "51%", 2);
results[27] = new Result("57.5%", 2);
results[28] = new Result( "52.2%", 2);
results[29] = new Result("62.3%", 2);
results[30] = new Result("61.9%", 2);
results[31] = new Result("71.5%", 2);
results[32] = new Result("73%", 2);
results[33] = new Result("55.2%", 2);
results[34] = new Result("73.9%", 2); 
results[35] = new Result( "67.3%", 2);
results[36] = new Result("58.9%", 2); 
results[37] = new Result( "64.6%", 2);

results[38] = new Result( "75.5%", 3);
results[39] = new Result("90.2%", 3);
results[40] = new Result( "82.6%", 3);
results[41] = new Result("92.1%", 3);
results[42] = new Result("89.2%", 3);
results[43] = new Result("77.7%", 3);
results[44] = new Result("85", 3);
results[45] = new Result("73.6%", 3);
results[46] = new Result("85%", 3); 
results[47] = new Result( "87.3%", 3); 
results[48]= new Result("79.9%", 3); 
results[49] = new Result( "90.8%", 3); 


  // initialize the kinect        
  context = new SimpleOpenNI(this);
  context.setMirror(true);
  if (context.enableDepth() == false)
  {
    println("Can't open the depthMap, maybe the camera is not connected!"); 
    exit();
    return;
  }

  //enable skeleton generation for all joints
  context.enableUser(SimpleOpenNI.SKEL_PROFILE_ALL);

  fontPercentage = loadFont("Geogrotesque-SemiBold-60.vlw");
  fontClock = loadFont("Geogrotesque-Regular-48.vlw"); 
  

  //set the values
  // questions[0].setAnswerScores(20,10,10,20,20,10,10,20);
  int n = 5;
  for ( int i =0; i<=12;i++) {
    // here we were about to convert number dynamiclly to strings 
    questions[i] = new Question( n +".jpg", + (n+1) + ".jpg", + (n+2)+ ".jpg", (n+3) +".jpg", (n+4) +".jpg");
    n= n+5;
  }
  currentQuestion = questions[currentQuestionCounter];
}


void draw() {
  
  background(255);
  context.update();
  updateState();

  int userCount = context.getNumberOfUsers();
  // draw the center of mass
  PVector posRealWorld = new PVector();
  PVector posProjected = new PVector();

  if (currentState == INSTRUCTION) {

    for (int userId = 1; userId <= userCount; userId++) {

      // get the user's center of mass in real world coordinate system
      context.getCoM(userId, posRealWorld);

      //println("x " + posRealWorld.x + " z " + posRealWorld.z);
      // checkPosition
      if ( circle_collision(centerX, centerZ, radius, posRealWorld.x, posRealWorld.z, userRadius)) {
        // user is in the center!
        currentState = QUESTION; 
        trackedUserID = userId;
        println("CENTER! - GO TO FIRST QUESTION"); 
        startQuestion();
        break;
      }
    }
    image(instructionImg, 0, 0);
  }
  
  else if (currentState == QUESTION) {
  
    context.getCoM(trackedUserID, posRealWorld);
    PImage img = currentQuestion.getImage(posRealWorld);
    image(img, 0, 0, width, height);
    Timer();

    // build //check timer
    // if time is up determine answer based on position.
    // move on to next question 
    if (millis() - currentQuestion.getTimeToAnswer() > startTimeQuestion ) {
      println("TIME IS UP");
      String answer = currentQuestion.getAnswer(posRealWorld);


      currentQuestionCounter ++;
      
      if (currentQuestionCounter >= questions.length) {
//      if (currentQuestionCounter >=2) {
        currentState = RESULT;
        boolean allPercentagesUsed = true;
        for(int i=0; i < results.length; i++){
          int p = (int)random(results.length-1);    
           if( results[p].checkUsed() == false){
             //temporary storing what we are getting from the class 
             currentColorCircle = results[p].colorCircle();
             println("currentColorCircle="+currentColorCircle);
             currentPercentage = results[p].returnPercentage();
             allPercentagesUsed = false;
             break;
           }
        }
        
        //we have already used all the percentages, so we reset them all and start using them again 
        if(allPercentagesUsed){
           for(int i=0; i < results.length; i++){
             results[i].resetPercentage();
          }                   
             //we get a random number 
             allPercentagesUsed = false;
             int p = (int)random(results.length-1);              
             currentColorCircle = results[p].colorCircle();
             println("currentColorCircle="+currentColorCircle);
             currentPercentage = results[p].returnPercentage();             
        }
        
        ///new/////
        currentQuestionCounter = 0;
        /////////
        loopingGif.play();
        startTimeGif = millis();
      }
      
      else { 
        startQuestion();
      }
    }

    String answer = currentQuestion.getAnswer(posRealWorld);
  }

  //******************************* slide show
  else if (currentState == NOBODY) {
    doSlideShow();
    
    
  } else if (currentState == THANKS) {
   if(millis() - thanksDuration <= startTimeThanks){
      image(lastImg,0,0);
    }else{
      //we are done
      currentState = NOBODY;
      trackedUserID = 0;      
    }        
  }
  else if (currentState == RESULT) {
    if(millis() - gifDuration <= startTimeGif){
//       image(loopingGif, width/2,height/2);
       imageMode(CENTER);
       image(loopingGif,width/2, height/2);
       imageMode(CORNER);
//        textFont( fontClock, 48);
//    text("Your result will be displayed HERE", xPercenFont, yPercenFont );
    fill(0);
    }
    else {
       image(resultImgs[currentColorCircle], 0,0); 
      textFont(fontPercentage, 60);
        text(currentPercentage, 1360, 735);
    }
    
    if(millis() - 20000 >= startTimeGif){
      currentState = THANKS;
       startTimeThanks = millis();
    }
                 
  }

  if (drawingKinectImage) {
    image(context.depthImage(), 0, 0);
  }
  if (drawingKinectCoords) {    
    textFont(fontPercentage, 60);
    fill(255, 0, 0);
    text("x="+(int)posRealWorld.x+" z="+(int)posRealWorld.z, 10, 50);  
    fill(255);
  }

}

void startQuestion() {
  println("startQuestion");
  currentQuestion = questions[currentQuestionCounter];
  // set timer
  startTimeQuestion = millis();
}


void doSlideShow() {
  if (nextimg==img.length) {
    nextimg =0;
  }
  image(img[nextimg], 0, 0);

  int currentTime = millis();
  if (currentTime-preTime > startTime) {
    nextimg= nextimg +1;
    preTime=millis();
  }
}


boolean circle_collision(float x_1, float z_1, float radius_1, float x_2, float z_2, float radius_2)
{
  return dist(x_1, z_1, x_2, z_2) < radius_1 + radius_2;
}

void updateState() {
  int userCount = context.getNumberOfUsers();

  // nobody there
  if (userCount == 0) {
    currentState = NOBODY;
  }  
  // not tracking but there are users ...reset the scores
  else if (trackedUserID == 0 && userCount > 0) {
    currentState = INSTRUCTION;
 
  }  


  //println("current state " + currentState);
}


// ********************************************clock on
void Timer() {   
  
  csec=(millis()-startTimeQuestion)/1000; //for converting to seconds


  //int getAnswer = getTimeToAnswer();
  //here we are creating a virable to access to the function in the Question class
  // divede to get seconds
  int getAnswer = currentQuestion.getTimeToAnswer()/1000;
  c = getAnswer-csec; 

  if (c<=0) {
    c = 0;
  }

  if (c<10) {
        textFont( fontClock, 48);
    
//    textFont  (fontClock, 14);
//    text("time left: 00"  + " : " + "00 : "+ "0" + c +" SEC ", 1255, 105);
    text("00"  + " : " + "00 : "+ "0" + c +" SEC ", 1450, 105);
    fill(0);
  }
  else {
//    textFont (fontClock, 14);
        textFont( fontClock, 48);
//    text("time left: 00"  + " : " + "00 : "+ c +" SEC ",1255, 105);
    text("00"  + " : " + "00 : "+ c +" SEC ",1450, 105);
    fill(0);
  }
}
//****************************************************


void onNewUser(int userId){
  println("onNewUser - userId: " + userId);
 // trackedUserID = userId;
  // show intro image
  //currentState = INTRODUCTION;
  // we'll do this later
}


void keyReleased() {
  if (key == 'i') {
    drawingKinectImage = !drawingKinectImage;
  }
  if (key == 't') {
    drawingKinectCoords = !drawingKinectCoords;
  }


  if (key == 'X') {
    xPercenFont+=5;
    println("xPercenFont"+xPercenFont);
  }else   if (key == 'x') {
    xPercenFont-=5;
    println("xPercenFont"+xPercenFont);
  }

  if (key == 'Y') {
    yPercenFont+=5;
    println("yPercenFont"+yPercenFont);
  }else   if (key == 'y') {
    yPercenFont-=5;
    println("yPercenFont"+yPercenFont);
  }
  
  
}

QUESTION CLASS
This class contains the questions, answers and time to answer or the running timer on the right corner of the screen.
According to the position of the user the right image will be shown on the screen

class Question {

  int timeToAnswer = 20600;
  String answer = "";  
  PImage img;

  int sec;
  
  PImage answer0;
  PImage answerA;
  PImage answerB;
  PImage answerC;
  PImage answerD;
  int startTimeAnswerSelected =0;


 // instant of the class - object orientated  
  Question(String img0, String imgA, String imgB, String imgC, String imgD) {
    answer0 = loadImage(img0);
    answerA = loadImage(imgA);
    answerB = loadImage(imgB);
    answerC = loadImage(imgC);
    answerD = loadImage(imgD);
    img = answer0;
  }
  
  
  //this could be combined/ simplified
  
  String getAnswer(PVector pos) {
    
     if ( circle_collision(pos.x, pos.z, userRadius, centerX, centerZ, radius)) {
          if(answer!=""){ 
              answer="";
              println("NO ANSWER - CENTER");
              img = answer0;
          } 
        }
    else if (  circle_collision(pos.x, pos.z, userRadius, centerA_X, centerA_Z, radius)) {
          if(answer!="a"){ 
              answer="a";
            println("ans A");      
            img = answerA; 
            startTimeAnswerSelected = millis();
          } 
    }
    else if (  circle_collision(pos.x, pos.z, userRadius, centerB_X, centerB_Z, radius)) {
       if( answer!="b"){
          answer="b";
          println("ans B");      
           img = answerB; 
            startTimeAnswerSelected = millis();
       }
    }
    else if( circle_collision(pos.x, pos.z, userRadius, centerC_X, centerC_Z, radius)) {
        if( answer!="c"){
        answer="c";      
        println("ans C");      
        img = answerC; 
        startTimeAnswerSelected = millis();
        }
    }
    else if( circle_collision(pos.x, pos.z, userRadius, centerD_X, centerD_Z, radius)) {
        if(  answer!="d" ){
        answer="d";
        println("ans D");      
        img = answerD;
        startTimeAnswerSelected = millis();        
        } 
    }
    
    
    return answer;
   
  }

  PImage getImage(PVector pos) {
    
    // based on position determine the image
   
    return img;
  }
  
  
  
  void setTimeToAnswer(int duration) {
    
    timeToAnswer = duration;
// to set different time slots for the questions
  }
  
  
  
  int getTimeToAnswer() {
    return timeToAnswer; 
  }
}

RESULT CLASS
This class serves to prevent repetition of the preset results/ scores initiated in the main sketch.

class Result {
  String randomPercentage="";
  boolean used = false;
  int colorCircle =0;
  //0 - grey,1 - green, 2- blue, 3-red)

  Result(String randomPercentage_1, int colorCircle_1)
  {
  
    randomPercentage = randomPercentage_1;
    colorCircle = colorCircle_1;
  }
  // this f. will check if the percentage has been used or not
  //not to be repeated 
  boolean checkUsed () {
    return used;
 }

  String returnPercentage() {
    used = true;
    return randomPercentage;
  }
  
  void resetPercentage(){
       used = false;
  }

  //in order to access the virable it needs to be returned
  int colorCircle() {
    return colorCircle;
  }
}