Simple Q Learner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A Simple Q Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "import gym\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-11-20 19:01:25,746] Making new env: FrozenLake-v0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('FrozenLake-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "States: 16\tActions: 4\n"
     ]
    }
   ],
   "source": [
    "n_states = env.observation_space.n\n",
    "n_actions = env.action_space.n\n",
    "print(f'States: {n_states:,}\\tActions: {n_actions:,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Initializing the Q-matrix\n",
    "Q = np.zeros(shape=[n_states, n_actions])\n",
    "# Learning rate\n",
    "lr = 0.1\n",
    "# Discount factor\n",
    "y = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 10,000\tTotal reward: 0.0\tTransitions: 34"
     ]
    }
   ],
   "source": [
    "# how many episodes\n",
    "episodes = 10000\n",
    "# number of times an agent performs a transition\n",
    "transitions = []\n",
    "# all total rewards\n",
    "rewards = []\n",
    "\n",
    "for episode in range(episodes):\n",
    "    done = False\n",
    "    state = env.reset()\n",
    "    n_transition = 0\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        n_transition += 1\n",
    "        action = np.argmax(Q[state])\n",
    "        state1, reward, done, _ = env.step(action)\n",
    "        # Update Q value\n",
    "        Q[state, action] += lr * (reward + y * np.max(Q[state1]) - Q[state, action])\n",
    "        # Update states\n",
    "        state = state1\n",
    "        total_reward += reward\n",
    "    # Record all transitions and rewards collected\n",
    "    transitions.append(n_transition)\n",
    "    rewards.append(total_reward)\n",
    "    sys.stdout.write(f'\\rEpisode: {episode+1:,}\\tTotal reward: {total_reward:,}\\t'\n",
    "                     f'Transitions: {n_transition:,}')\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (Left)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "\u001b[41mH\u001b[0mFFG\n"
     ]
    }
   ],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum »»» Transitions: 100\tReward: 0.0\n",
      "Minimum »»» Transitions: 3\tReward: 0.0\n"
     ]
    }
   ],
   "source": [
    "# maximum transitions & reward\n",
    "max_transition = max(transitions)\n",
    "max_reward = max(rewards)\n",
    "# minimum transitions & reward\n",
    "min_transition = min(transitions)\n",
    "min_reward = min(rewards)\n",
    "print(f'Maximum »»» Transitions: {max_transition:,}\\tReward: {max_reward:,}')\n",
    "print(f'Minimum »»» Transitions: {min_transition:,}\\tReward: {min_reward:,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode with maximum »»» transition: 2,846\t\t reward: 0\n",
      "Episode with minimum »»» transition: 0\t reward: 0\n"
     ]
    }
   ],
   "source": [
    "# Episode with max & min rewards\n",
    "reward_max_episode = rewards.index(max_reward)\n",
    "reward_min_episode = rewards.index(min_reward)\n",
    "# Episode with max & min transitions\n",
    "trans_max_episode = transitions.index(max_transition)\n",
    "trans_min_episode = transitions.index(min_transition)\n",
    "print(f'Episode with maximum »»» transition: {trans_max_episode:,}\\t\\t reward: {reward_max_episode:,}')\n",
    "print(f'Episode with minimum »»» transition: {trans_min_episode:,}\\t reward: {reward_min_episode:,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward at max transition: 0.0\n",
      "Reward at min transition: 0.0\n"
     ]
    }
   ],
   "source": [
    "# Reward @ maximum transition\n",
    "reward_max_trans = rewards[trans_max_episode]\n",
    "# Reward @ minimum transition\n",
    "reward_min_trans = rewards[trans_min_episode]\n",
    "print(f'Reward at max transition: {reward_max_trans:,}')\n",
    "print(f'Reward at min transition: {reward_min_trans:,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transition at max reward: 3\n",
      "Transition at min reward: 3\n"
     ]
    }
   ],
   "source": [
    "# Transition @ maximum reward\n",
    "trans_max_reward = transitions[reward_max_episode]\n",
    "# Transition @ minimum reward\n",
    "trans_min_reward = transitions[reward_min_episode]\n",
    "print(f'Transition at max reward: {trans_max_reward:,}')\n",
    "print(f'Transition at min reward: {trans_min_reward:,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}