Highway.ipynb

{
 "metadata": {
  "name": "Highway"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def Sgenerator( ):\n",
      "    for v in range(0,3):\n",
      "        for x_b in range(0,9):\n",
      "            for y_r in range(0,9):\n",
      "                for x_r in range(0,3):\n",
      "                    yield [v,x_b,y_r,x_r]\n",
      "\n",
      "S = [s for s in Sgenerator()]\n",
      "\n",
      "A = range(0,5)\n",
      "\n",
      "def s_index( state ):\n",
      "    v = state[0]\n",
      "    x_b = state[1]\n",
      "    y_r = state[2]\n",
      "    x_r = state[3]\n",
      "    index = x_r + y_r*3 + x_b*3*9 + v*3*9*9\n",
      "    return index\n",
      "\n",
      "def sa_index( state, action ):\n",
      "    index = s_index(state) + action*3*9*9*3\n",
      "    return index\n",
      "\n",
      "def next_states( state, action ):\n",
      "    \"Returns a tuple of the next possible states given the agent is in the provided state ant takes the provided action.\"\n",
      "    v = next_v = state[0]\n",
      "    xb = next_xb = state[1]\n",
      "    yr = next_yr = state[2]\n",
      "    xr = next_xr = state[3]\n",
      "    #taking the player's action into account\n",
      "    if action == 0:\n",
      "        pass\n",
      "    elif action == 1:\n",
      "        next_v = v + 1 if v < 2 else 2\n",
      "    elif action == 2:\n",
      "        next_v = v - 1 if v > 0 else 0\n",
      "    elif action == 3:\n",
      "        next_xb = xb - 1 if xb > 0 else 0\n",
      "    elif action == 4:\n",
      "        next_xb = xb + 1 if xb < 8 else 8\n",
      "    else:\n",
      "        raise ValueError( \"Action %d does not exist\" % action )\n",
      "    #Moving the red car\n",
      "    next_yr_lst = []\n",
      "    if v == 0:\n",
      "        next_yr_lst = range(0,9)\n",
      "    elif v == 1:\n",
      "        next_yr_lst = [1,3,5,7]\n",
      "    elif v == 2:\n",
      "        next_yr_lst = [1,4,7]\n",
      "    else:\n",
      "        raise ValueError(\"Speed %d is unknown to me\"%v)\n",
      "    possible_outcomes = []        \n",
      "    try:\n",
      "        next_yr = (i for i in next_yr_lst if i > yr).next()\n",
      "        possible_outcomes.append( [next_v, next_xb, next_yr, next_xr] )\n",
      "    except StopIteration : #This means the car has reached past its final position\n",
      "        next_yr = next_yr_lst[0]\n",
      "        possible_outcomes = [ [next_v, next_xb, next_yr, i] for i in range(0,3) ]\n",
      "    return possible_outcomes\n",
      "\n",
      "def P( a ):\n",
      "    \"Returns the matrix of transition probability for action a.\"\n",
      "    P_a = zeros((3*9*9*3,3*9*9*3))\n",
      "    for state in Sgenerator():\n",
      "        current_index = s_index( state )\n",
      "        possible_outcomes = next_states( state, a )\n",
      "        #Writing the probabilities in the matrix\n",
      "        for next_s in possible_outcomes:\n",
      "            next_index = s_index( next_s )\n",
      "            P_a[ current_index, next_index ] = 1./len(possible_outcomes) #This line assumes two outcome won't share the same index\n",
      "    return P_a\n",
      "\n",
      "P = vstack([P(a) for a in A])\n",
      "savetxt(\"Highway_P.mat\",P)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def R( ):\n",
      "    reward = zeros((3*9*9*3*5,1))\n",
      "    for state in S:\n",
      "        current_indices = [sa_index(state,a) for a in A]\n",
      "        v = state[0]\n",
      "        xb = state[1]\n",
      "        yr = state[2]\n",
      "        xr = state[3]\n",
      "        lane_nb2blue_x = [[1,2,3],[3,4,5],[5,6,7]] #Coincidentally, lane_nb is xr\n",
      "        if yr in [6,7,8] and xb in lane_nb2blue_x[xr] : #Collision\n",
      "            reward[ current_indices ] = -1.\n",
      "        elif xb in [0,1,7,8]:\n",
      "            reward[ current_indices ] = -0.5\n",
      "        elif v == 2:\n",
      "            reward[ current_indices ] = 1.\n",
      "        else:\n",
      "            pass #already at 0\n",
      "    return reward\n",
      "savetxt(\"Highway_R.mat\", R())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    }
   ],
   "metadata": {}
  }
 ]
}