public class Ql{
  static int goal=5;

  static int door[][] = 
     {{0,1,3,0},{1,2,4,0},{2,2,5,1},{0,4,3,3},{1,5,4,3},{0,0,0,0}}; 

  static State status[]= new State[6];

   public static void main(String[] args) {
        int now=0, next;
        double reward=0.0;


        for(int i=0; i<6; i++)
           status[i]= new State(door[i]);

	for(int count=0; count<100; count++) {
            while (now!=goal){
		next= status[now].nextStatus(now);
		status[now].changeQ(next);
                now= next;
	    }
           now=0; reward=0.0;
	}
	//report
	for( int i=0; i<6; i++) 
            System.out.println(status[i].q[0]+","+status[i].q[1]+","+status[i].q[2]+","+status[i].q[3]);
   }
}
class State {
   double q[]= new double[4];
   int neighbor[]= new int[4];
   int action;

   public State (int neigh[]) {
     neighbor= neigh;
   }
   int nextStatus(int now){
       int next;
       next=now;
       while(next==now) {
 	  action= (int)(Math.random()*100)%4;
 	  next= neighbor[action];
       }              
       return next; 
    }

    void changeQ(int next){
       double  gamma=0.9;
       double maxq, reward;
       maxq=-999;
       if(next==Ql.goal) reward=100.0;
       else reward=0.0;
       for(int i=0; i<4; i++)
	  if(maxq< Ql.status[next].q[i]) 
             maxq= Ql.status[next].q[i];
          
	  q[action]= reward + gamma*maxq;
    }
}