fcn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from argparse import ArgumentParser\n",
    "import os\n",
    "import random\n",
    "from matplotlib import pyplot as plt\n",
    "import torch\n",
    "from torch import optim\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import models\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "from data import CityscapesDataset, num_classes, full_to_colour, train_to_full\n",
    "from model import FeatureResNet, SegResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??FeatureResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??SegResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# dimension of cityscapes : 2048 x 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setup\n",
    "parser = ArgumentParser(description='Semantic segmentation')\n",
    "parser.add_argument('--seed', type=int, default=42, help='Random seed')\n",
    "parser.add_argument('--workers', type=int, default=8, help='Data loader workers')\n",
    "parser.add_argument('--epochs', type=int, default=100, help='Training epochs')\n",
    "parser.add_argument('--crop-size', type=int, default=512, help='Training crop size')\n",
    "parser.add_argument('--lr', type=float, default=5e-5, help='Learning rate')\n",
    "parser.add_argument('--weight-decay', type=float, default=2e-4, help='Weight decay')\n",
    "parser.add_argument('--batch-size', type=int, default=4, help='Batch size')\n",
    "args = parser.parse_args()\n",
    "random.seed(args.seed)\n",
    "torch.manual_seed(args.seed)\n",
    "if not os.path.exists('results'):\n",
    "    os.makedirs('results')\n",
    "plt.switch_backend('agg')  # Allow plotting when running remotely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparmas:\n",
    "seed = 42\n",
    "workers = 8\n",
    "epochs = 2  # 100\n",
    "crop_size = 512\n",
    "lr = 5e-5\n",
    "weight_decay = 2e-4\n",
    "batch_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(seed)\n",
    "if not os.path.exists('results'):\n",
    "    os.makedirs('results')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_dataset = CityscapesDataset(split='train', crop=cropsize, flip=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_dataset = CityscapesDataset(split='val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "data.CityscapesDataset"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data\n",
    "train_dataset = CityscapesDataset(split='train', crop=crop_size, flip=True)\n",
    "val_dataset = CityscapesDataset(split='val')\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=workers, pin_memory=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=1, num_workers=workers, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.utils.data.dataloader.DataLoader"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pretrained_net = FeatureResNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??pretrained_net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FeatureResNet (\n",
      "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
      "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "  (relu): ReLU (inplace)\n",
      "  (maxpool): MaxPool2d (size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1))\n",
      "  (layer1): Sequential (\n",
      "    (0): BasicBlock (\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (1): BasicBlock (\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (2): BasicBlock (\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (layer2): Sequential (\n",
      "    (0): BasicBlock (\n",
      "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (downsample): Sequential (\n",
      "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock (\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (2): BasicBlock (\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (3): BasicBlock (\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (layer3): Sequential (\n",
      "    (0): BasicBlock (\n",
      "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (downsample): Sequential (\n",
      "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock (\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (2): BasicBlock (\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (3): BasicBlock (\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (4): BasicBlock (\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (5): BasicBlock (\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (layer4): Sequential (\n",
      "    (0): BasicBlock (\n",
      "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (downsample): Sequential (\n",
      "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock (\n",
      "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (2): BasicBlock (\n",
      "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (relu): ReLU (inplace)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (avgpool): AvgPool2d (\n",
      "  )\n",
      "  (fc): Linear (512 -> 1000)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(pretrained_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pretrained_net.load_state_dict(models.resnet34(pretrained=True).state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model.FeatureResNet"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(pretrained_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(SegResNet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net = SegResNet(num_classes, pretrained_net).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model.SegResNet"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crit = nn.BCELoss().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.loss.BCELoss"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(crit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimiser = optim.RMSprop(net.parameters(), lr=lr, weight_decay=weight_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Training/Testing\n",
    "\n",
    "# create the network structure for ResNet\n",
    "pretrained_net = FeatureResNet()\n",
    "\n",
    "# load the weights\n",
    "pretrained_net.load_state_dict(models.resnet34(pretrained=True).state_dict())\n",
    "\n",
    "# plug the ResNet into FCNN framework\n",
    "net = SegResNet(num_classes, pretrained_net).cuda()\n",
    "\n",
    "# why use Binary corss-entropy loss\n",
    "crit = nn.BCELoss().cuda()\n",
    "\n",
    "# compile the optimizer\n",
    "optimiser = optim.RMSprop(net.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "\n",
    "# to hold the score and mean score\n",
    "scores, mean_scores = [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??net.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object `cuda` not found.\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# traing steps\n",
    "# e = epoch\n",
    "def train(e):\n",
    "    # only useful for dropout / batchnorm\n",
    "    net.train()\n",
    "    #  loop over the data iterator, and feed the inputs to the network and optimize\n",
    "    for i, (input, target, _) in enumerate(train_loader):\n",
    "        # reset grad\n",
    "        optimiser.zero_grad()\n",
    "        # use GPU CUDA\n",
    "        input, target = Variable(input.cuda(async=True)), Variable(target.cuda(async=True))\n",
    "        \n",
    "        # why use sigmoid function?\n",
    "        output = F.sigmoid(net(input))\n",
    "        loss = crit(output, target)\n",
    "        print(e, i, loss.data[0])\n",
    "        # compute gradients\n",
    "        loss.backward()\n",
    "        # update params with gradients\n",
    "        optimiser.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calculates class intersections over unions\n",
    "def iou(pred, target):\n",
    "    ious = []\n",
    "    # Ignore IoU for background class, num_classes = 20\n",
    "    for cls in range(num_classes - 1):\n",
    "        pred_inds = pred == cls\n",
    "        target_inds = target == cls\n",
    "        intersection = (pred_inds[target_inds]).long().sum().data.cpu()[0]  # Cast to long to prevent overflows\n",
    "        union = pred_inds.long().sum().data.cpu()[0] + target_inds.long().sum().data.cpu()[0] - intersection\n",
    "        if union == 0:\n",
    "            ious.append(float('nan'))  # If there is no ground truth, do not include in evaluation\n",
    "        else:\n",
    "            ious.append(intersection / max(union, 1))\n",
    "    return ious"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test(e):\n",
    "    net.eval()\n",
    "    total_ious = []\n",
    "    for i, (input, _, target) in enumerate(val_loader):\n",
    "        input, target = Variable(input.cuda(async=True), volatile=True), Variable(target.cuda(async=True), volatile=True)\n",
    "        output = F.log_softmax(net(input))\n",
    "        b, _, h, w = output.size()\n",
    "        pred = output.permute(0, 2, 3, 1).contiguous().view(-1, num_classes).max(1)[1].view(b, h, w)\n",
    "        total_ious.append(iou(pred, target))\n",
    "    \n",
    "        # Save images\n",
    "        if i % 25 == 0:\n",
    "            pred = pred.data.cpu()\n",
    "            pred_remapped = pred.clone()\n",
    "            # Convert to full labels\n",
    "            for k, v in train_to_full.items():\n",
    "                pred_remapped[pred == k] = v\n",
    "            # Convert to colour image\n",
    "            pred = pred_remapped\n",
    "            pred_colour = torch.zeros(b, 3, h, w)\n",
    "            for k, v in full_to_colour.items():\n",
    "                pred_r = torch.zeros(b, 1, h, w)\n",
    "                pred_r[(pred == k)] = v[0]\n",
    "                pred_g = torch.zeros(b, 1, h, w)\n",
    "                pred_g[(pred == k)] = v[1]\n",
    "                pred_b = torch.zeros(b, 1, h, w)\n",
    "                pred_b[(pred == k)] = v[2]\n",
    "                pred_colour.add_(torch.cat((pred_r, pred_g, pred_b), 1))\n",
    "            save_image(pred_colour[0].float().div(255), os.path.join('results', str(e) + '_' + str(i) + '.png'))\n",
    "  \n",
    "    # Calculate average IoU\n",
    "    total_ious = torch.Tensor(total_ious).transpose(0, 1)\n",
    "    ious = torch.Tensor(num_classes - 1)\n",
    "    for i, class_iou in enumerate(total_ious):\n",
    "        ious[i] = class_iou[class_iou == class_iou].mean()  # Calculate mean, ignoring NaNs\n",
    "    print(ious, ious.mean())\n",
    "    scores.append(ious)\n",
    "  \n",
    "    # Save weights and scores\n",
    "    torch.save(net.state_dict(), os.path.join('results', str(e) + '_net.pth'))\n",
    "    torch.save(scores, os.path.join('results', 'scores.pth'))\n",
    "  \n",
    "    # Plot scores\n",
    "    mean_scores.append(ious.mean())\n",
    "    es = list(range(len(mean_scores)))\n",
    "    plt.plot(es, mean_scores, 'b-')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Mean IoU')\n",
    "    plt.savefig(os.path.join('results', 'ious.png'))\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0 0.6931458711624146\n",
      "1 1 0.6355560421943665\n",
      "1 2 0.6427205801010132\n",
      "1 3 0.6433953046798706\n",
      "1 4 0.6305932998657227\n",
      "1 5 0.6067423820495605\n",
      "1 6 0.5945364236831665\n",
      "1 7 0.5863555669784546\n",
      "1 8 0.5912925004959106\n",
      "1 9 0.5894274711608887\n",
      "1 10 0.5746929049491882\n",
      "1 11 0.5672615766525269\n",
      "1 12 0.5707300901412964\n",
      "1 13 0.5112751126289368\n",
      "1 14 0.48332276940345764\n",
      "1 15 0.5117632150650024\n",
      "1 16 0.47450026869773865\n",
      "1 17 0.463645875453949\n",
      "1 18 0.3759079873561859\n",
      "1 19 0.39688995480537415\n",
      "1 20 0.37869444489479065\n",
      "1 21 0.34906309843063354\n",
      "1 22 0.3234705626964569\n",
      "1 23 0.3480839133262634\n",
      "1 24 0.28830486536026\n",
      "1 25 0.2619107961654663\n",
      "1 26 0.2768684923648834\n",
      "1 27 0.24320001900196075\n",
      "1 28 0.24300476908683777\n",
      "1 29 0.23459315299987793\n",
      "1 30 0.22209849953651428\n",
      "1 31 0.20149092376232147\n",
      "1 32 0.23562505841255188\n",
      "1 33 0.2050512284040451\n",
      "1 34 0.2157030999660492\n",
      "1 35 0.20715849101543427\n",
      "1 36 0.22229750454425812\n",
      "1 37 0.23299284279346466\n",
      "1 38 0.1910499930381775\n",
      "1 39 0.19952347874641418\n",
      "1 40 0.18842554092407227\n",
      "1 41 0.20189762115478516\n",
      "1 42 0.19219741225242615\n",
      "1 43 0.17854540050029755\n",
      "1 44 0.15284980833530426\n",
      "1 45 0.1940363198518753\n",
      "1 46 0.17803288996219635\n",
      "1 47 0.19182442128658295\n",
      "1 48 0.16414675116539001\n",
      "1 49 0.1668160855770111\n",
      "1 50 0.1612926572561264\n",
      "1 51 0.17033511400222778\n",
      "1 52 0.16766560077667236\n",
      "1 53 0.1643572300672531\n",
      "1 54 0.18440815806388855\n",
      "1 55 0.16726195812225342\n",
      "1 56 0.19309869408607483\n",
      "1 57 0.16651833057403564\n",
      "1 58 0.17964795231819153\n",
      "1 59 0.1547914743423462\n",
      "1 60 0.18105176091194153\n",
      "1 61 0.16869625449180603\n",
      "1 62 0.15328668057918549\n",
      "1 63 0.15487012267112732\n",
      "1 64 0.1665942370891571\n",
      "1 65 0.16915789246559143\n",
      "1 66 0.1487732231616974\n",
      "1 67 0.16911913454532623\n",
      "1 68 0.15563912689685822\n",
      "1 69 0.1498677134513855\n",
      "1 70 0.16157183051109314\n",
      "1 71 0.1656232625246048\n",
      "1 72 0.17374232411384583\n",
      "1 73 0.2091020792722702\n",
      "1 74 0.18146008253097534\n",
      "1 75 0.1549367606639862\n",
      "1 76 0.1641767919063568\n",
      "1 77 0.16133052110671997\n",
      "1 78 0.16494932770729065\n",
      "1 79 0.1560068577528\n",
      "1 80 0.14986643195152283\n",
      "1 81 0.15431225299835205\n",
      "1 82 0.1664915382862091\n",
      "1 83 0.12613297998905182\n",
      "1 84 0.15483060479164124\n",
      "1 85 0.16204304993152618\n",
      "1 86 0.15640705823898315\n",
      "1 87 0.16710695624351501\n",
      "1 88 0.14663371443748474\n",
      "1 89 0.15489012002944946\n",
      "1 90 0.16631610691547394\n",
      "1 91 0.15615835785865784\n",
      "1 92 0.14267198741436005\n",
      "1 93 0.15034623444080353\n",
      "1 94 0.16396138072013855\n",
      "1 95 0.13304145634174347\n",
      "1 96 0.14994844794273376\n",
      "1 97 0.14825601875782013\n",
      "1 98 0.1395171731710434\n",
      "1 99 0.1284797042608261\n",
      "1 100 0.12418492883443832\n",
      "1 101 0.11098010838031769\n",
      "1 102 0.13186825811862946\n",
      "1 103 0.13881362974643707\n",
      "1 104 0.1649189442396164\n",
      "1 105 0.1346580535173416\n",
      "1 106 0.14436358213424683\n",
      "1 107 0.15564386546611786\n",
      "1 108 0.1326266974210739\n",
      "1 109 0.13711705803871155\n",
      "1 110 0.12122645229101181\n",
      "1 111 0.15734468400478363\n",
      "1 112 0.1341552585363388\n",
      "1 113 0.16575944423675537\n",
      "1 114 0.1434389352798462\n",
      "1 115 0.1593683958053589\n",
      "1 116 0.13790392875671387\n",
      "1 117 0.16792957484722137\n",
      "1 118 0.1616993099451065\n",
      "1 119 0.12340686470270157\n",
      "1 120 0.14342035353183746\n",
      "1 121 0.16016027331352234\n",
      "1 122 0.16604149341583252\n",
      "1 123 0.16290943324565887\n",
      "1 124 0.1573657989501953\n",
      "1 125 0.1630292385816574\n",
      "1 126 0.1635752022266388\n",
      "1 127 0.14692580699920654\n",
      "1 128 0.1699012964963913\n",
      "1 129 0.1593763679265976\n",
      "1 130 0.13261573016643524\n",
      "1 131 0.13866405189037323\n",
      "1 132 0.11731591075658798\n",
      "1 133 0.13090574741363525\n",
      "1 134 0.1324177235364914\n",
      "1 135 0.16416668891906738\n",
      "1 136 0.15005412697792053\n",
      "1 137 0.1486400067806244\n",
      "1 138 0.15134909749031067\n",
      "1 139 0.15778633952140808\n",
      "1 140 0.162303626537323\n",
      "1 141 0.17159512639045715\n",
      "1 142 0.13854220509529114\n",
      "1 143 0.14234831929206848\n",
      "1 144 0.16724389791488647\n",
      "1 145 0.12756416201591492\n",
      "1 146 0.1354837566614151\n",
      "1 147 0.1646767556667328\n",
      "1 148 0.13434424996376038\n",
      "1 149 0.12343690544366837\n",
      "1 150 0.15128229558467865\n",
      "1 151 0.13178910315036774\n",
      "1 152 0.16234570741653442\n",
      "1 153 0.16925792396068573\n",
      "1 154 0.18092408776283264\n",
      "1 155 0.1710461974143982\n",
      "1 156 0.17392128705978394\n",
      "1 157 0.15928372740745544\n",
      "1 158 0.14440050721168518\n",
      "1 159 0.14838242530822754\n",
      "1 160 0.15379227697849274\n",
      "1 161 0.1402808278799057\n",
      "1 162 0.15165886282920837\n",
      "1 163 0.15300466120243073\n",
      "1 164 0.13799512386322021\n",
      "1 165 0.13217408955097198\n",
      "1 166 0.16429951786994934\n",
      "1 167 0.1339888721704483\n",
      "1 168 0.15933918952941895\n",
      "1 169 0.10561563819646835\n",
      "1 170 0.15488578379154205\n",
      "1 171 0.18477816879749298\n",
      "1 172 0.1423986852169037\n",
      "1 173 0.11877189576625824\n",
      "1 174 0.1488179862499237\n",
      "1 175 0.14644578099250793\n",
      "1 176 0.14619550108909607\n",
      "1 177 0.1493832767009735\n",
      "1 178 0.14119520783424377\n",
      "1 179 0.16400478780269623\n",
      "1 180 0.16959509253501892\n",
      "1 181 0.127118781208992\n",
      "1 182 0.13506822288036346\n",
      "1 183 0.14920030534267426\n",
      "1 184 0.19442634284496307\n",
      "1 185 0.1692967414855957\n",
      "1 186 0.13048096001148224\n",
      "1 187 0.14419905841350555\n",
      "1 188 0.1854023039340973\n",
      "1 189 0.15141388773918152\n",
      "1 190 0.13705620169639587\n",
      "1 191 0.13536915183067322\n",
      "1 192 0.12113308906555176\n",
      "1 193 0.12279506027698517\n",
      "1 194 0.14262932538986206\n",
      "1 195 0.16462792456150055\n",
      "1 196 0.18474885821342468\n",
      "1 197 0.09954856336116791\n",
      "1 198 0.161396324634552\n",
      "1 199 0.136822909116745\n",
      "1 200 0.142918661236763\n",
      "1 201 0.15655222535133362\n",
      "1 202 0.12732574343681335\n",
      "1 203 0.13094958662986755\n",
      "1 204 0.15371426939964294\n",
      "1 205 0.12219341099262238\n",
      "1 206 0.15581011772155762\n",
      "1 207 0.16616742312908173\n",
      "1 208 0.13759715855121613\n",
      "1 209 0.15436366200447083\n",
      "1 210 0.1404535174369812\n",
      "1 211 0.14484953880310059\n",
      "1 212 0.1326678842306137\n",
      "1 213 0.17722009122371674\n",
      "1 214 0.12095527350902557\n",
      "1 215 0.12557187676429749\n",
      "1 216 0.13892361521720886\n",
      "1 217 0.12574553489685059\n",
      "1 218 0.1450522392988205\n",
      "1 219 0.1417316496372223\n",
      "1 220 0.12575682997703552\n",
      "1 221 0.14376071095466614\n",
      "1 222 0.12164461612701416\n",
      "1 223 0.15881486237049103\n",
      "1 224 0.16265244781970978\n",
      "1 225 0.18048201501369476\n",
      "1 226 0.13577908277511597\n",
      "1 227 0.13451026380062103\n",
      "1 228 0.13011687994003296\n",
      "1 229 0.1778828352689743\n",
      "1 230 0.10438059270381927\n",
      "1 231 0.11912320554256439\n",
      "1 232 0.13602282106876373\n",
      "1 233 0.14190194010734558\n",
      "1 234 0.1512354463338852\n",
      "1 235 0.1485920250415802\n",
      "1 236 0.1602657437324524\n",
      "1 237 0.10609447956085205\n",
      "1 238 0.1084761768579483\n",
      "1 239 0.16856878995895386\n",
      "1 240 0.12342692911624908\n",
      "1 241 0.17656122148036957\n",
      "1 242 0.10753677040338516\n",
      "1 243 0.18308421969413757\n",
      "1 244 0.12104904651641846\n",
      "1 245 0.11897329986095428\n",
      "1 246 0.15163956582546234\n",
      "1 247 0.1687266230583191\n",
      "1 248 0.12548933923244476\n",
      "1 249 0.1372339129447937\n",
      "1 250 0.150006502866745\n",
      "1 251 0.1677486002445221\n",
      "1 252 0.09666720777750015\n",
      "1 253 0.1644929200410843\n",
      "1 254 0.13481071591377258\n",
      "1 255 0.16471005976200104\n",
      "1 256 0.17804887890815735\n",
      "1 257 0.13740067183971405\n",
      "1 258 0.14651885628700256\n",
      "1 259 0.13092729449272156\n",
      "1 260 0.15551255643367767\n",
      "1 261 0.14763836562633514\n",
      "1 262 0.14224538207054138\n",
      "1 263 0.17086315155029297\n",
      "1 264 0.1464288979768753\n",
      "1 265 0.09776536375284195\n",
      "1 266 0.13239210844039917\n",
      "1 267 0.11441012471914291\n",
      "1 268 0.11541599035263062\n",
      "1 269 0.14392073452472687\n",
      "1 270 0.12432032823562622\n",
      "1 271 0.12296517938375473\n",
      "1 272 0.1570243388414383\n",
      "1 273 0.13629575073719025\n",
      "1 274 0.14303606748580933\n",
      "1 275 0.1473942995071411\n",
      "1 276 0.13196364045143127\n",
      "1 277 0.17620261013507843\n",
      "1 278 0.1674092710018158\n",
      "1 279 0.1096695214509964\n",
      "1 280 0.17281031608581543\n",
      "1 281 0.10450142621994019\n",
      "1 282 0.12414807081222534\n",
      "1 283 0.16930055618286133\n",
      "1 284 0.19070222973823547\n",
      "1 285 0.16165496408939362\n",
      "1 286 0.12887874245643616\n",
      "1 287 0.1224234476685524\n",
      "1 288 0.16642048954963684\n",
      "1 289 0.15072908997535706\n",
      "1 290 0.1493944227695465\n",
      "1 291 0.14431442320346832\n",
      "1 292 0.11686793714761734\n",
      "1 293 0.15466341376304626\n",
      "1 294 0.11606918275356293\n",
      "1 295 0.15412433445453644\n",
      "1 296 0.1463937610387802\n",
      "1 297 0.1344994306564331\n",
      "1 298 0.13760749995708466\n",
      "1 299 0.10900354385375977\n",
      "1 300 0.17856581509113312\n",
      "1 301 0.15519669651985168\n",
      "1 302 0.1279705911874771\n",
      "1 303 0.14515824615955353\n",
      "1 304 0.11704456806182861\n",
      "1 305 0.15884464979171753\n",
      "1 306 0.17881152033805847\n",
      "1 307 0.09708376228809357\n",
      "1 308 0.09786592423915863\n",
      "1 309 0.1377091407775879\n",
      "1 310 0.196144700050354\n",
      "1 311 0.11707703024148941\n",
      "1 312 0.1157291904091835\n",
      "1 313 0.12235909700393677\n",
      "1 314 0.1732623279094696\n",
      "1 315 0.1650184839963913\n",
      "1 316 0.1653953641653061\n",
      "1 317 0.18718498945236206\n",
      "1 318 0.14364702999591827\n",
      "1 319 0.1432335376739502\n",
      "1 320 0.13018305599689484\n",
      "1 321 0.12507230043411255\n",
      "1 322 0.14252814650535583\n",
      "1 323 0.202264666557312\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 324 0.16952621936798096\n",
      "1 325 0.14396785199642181\n",
      "1 326 0.11784398555755615\n",
      "1 327 0.12917423248291016\n",
      "1 328 0.18002407252788544\n",
      "1 329 0.1367807239294052\n",
      "1 330 0.12215503305196762\n",
      "1 331 0.13240770995616913\n",
      "1 332 0.18047718703746796\n",
      "1 333 0.16954916715621948\n",
      "1 334 0.15677234530448914\n",
      "1 335 0.1712571531534195\n",
      "1 336 0.15397094190120697\n",
      "1 337 0.15282964706420898\n",
      "1 338 0.12067949771881104\n",
      "1 339 0.14261645078659058\n",
      "1 340 0.13222673535346985\n",
      "1 341 0.12911447882652283\n",
      "1 342 0.14942654967308044\n",
      "1 343 0.1129477247595787\n",
      "1 344 0.15369239449501038\n",
      "1 345 0.1069754958152771\n",
      "1 346 0.19257931411266327\n",
      "1 347 0.11606863886117935\n",
      "1 348 0.10825903713703156\n",
      "1 349 0.16675244271755219\n",
      "1 350 0.12300588935613632\n",
      "1 351 0.16172607243061066\n",
      "1 352 0.1606787145137787\n",
      "1 353 0.1218864917755127\n",
      "1 354 0.11406876891851425\n",
      "1 355 0.1262023150920868\n",
      "1 356 0.12387244403362274\n",
      "1 357 0.17317430675029755\n",
      "1 358 0.1475595384836197\n",
      "1 359 0.13853588700294495\n",
      "1 360 0.1437016725540161\n",
      "1 361 0.20906205475330353\n",
      "1 362 0.1647128313779831\n",
      "1 363 0.166255384683609\n",
      "1 364 0.14535970985889435\n",
      "1 365 0.13984641432762146\n",
      "1 366 0.14082510769367218\n",
      "1 367 0.1211136132478714\n",
      "1 368 0.1377488076686859\n",
      "1 369 0.11082365363836288\n",
      "1 370 0.15836863219738007\n",
      "1 371 0.17150388658046722\n",
      "1 372 0.16211625933647156\n",
      "1 373 0.14305663108825684\n",
      "1 374 0.1598472148180008\n",
      "1 375 0.1805579960346222\n",
      "1 376 0.1474190205335617\n",
      "1 377 0.1564032882452011\n",
      "1 378 0.12001031637191772\n",
      "1 379 0.14626307785511017\n",
      "1 380 0.16064593195915222\n",
      "1 381 0.14491209387779236\n",
      "1 382 0.09206391870975494\n",
      "1 383 0.127273827791214\n",
      "1 384 0.11944365501403809\n",
      "1 385 0.11916802823543549\n",
      "1 386 0.12301798164844513\n",
      "1 387 0.18992869555950165\n",
      "1 388 0.16846367716789246\n",
      "1 389 0.13559548556804657\n",
      "1 390 0.11750010401010513\n",
      "1 391 0.15408596396446228\n",
      "1 392 0.1266932189464569\n",
      "1 393 0.10782533884048462\n",
      "1 394 0.16665588319301605\n",
      "1 395 0.1352503001689911\n",
      "1 396 0.1339409351348877\n",
      "1 397 0.1755540668964386\n",
      "1 398 0.180199533700943\n",
      "1 399 0.12404389679431915\n",
      "1 400 0.13305023312568665\n",
      "1 401 0.14409595727920532\n",
      "1 402 0.1345416158437729\n",
      "1 403 0.16438761353492737\n",
      "1 404 0.09948659688234329\n",
      "1 405 0.11979564279317856\n",
      "1 406 0.12796232104301453\n",
      "1 407 0.10587270557880402\n",
      "1 408 0.17471906542778015\n",
      "1 409 0.14628402888774872\n",
      "1 410 0.11844068765640259\n",
      "1 411 0.09184373915195465\n",
      "1 412 0.1886059045791626\n",
      "1 413 0.14003947377204895\n",
      "1 414 0.16069576144218445\n",
      "1 415 0.11616434901952744\n",
      "1 416 0.17269481718540192\n",
      "1 417 0.11948628723621368\n",
      "1 418 0.12557104229927063\n",
      "1 419 0.11587931215763092\n",
      "1 420 0.12694716453552246\n",
      "1 421 0.13034571707248688\n",
      "1 422 0.11333204805850983\n",
      "1 423 0.13873443007469177\n",
      "1 424 0.1657973825931549\n",
      "1 425 0.12788298726081848\n",
      "1 426 0.1617484837770462\n",
      "1 427 0.14837104082107544\n",
      "1 428 0.13085409998893738\n",
      "1 429 0.12766346335411072\n",
      "1 430 0.17211800813674927\n",
      "1 431 0.13438984751701355\n",
      "1 432 0.18324659764766693\n",
      "1 433 0.1817200481891632\n",
      "1 434 0.17426016926765442\n",
      "1 435 0.14909669756889343\n",
      "1 436 0.10326838493347168\n",
      "1 437 0.1428685337305069\n",
      "1 438 0.13321414589881897\n",
      "1 439 0.1530001312494278\n",
      "1 440 0.14261406660079956\n",
      "1 441 0.16155706346035004\n",
      "1 442 0.1365601271390915\n",
      "1 443 0.2148556262254715\n",
      "1 444 0.1919267773628235\n",
      "1 445 0.13549526035785675\n",
      "1 446 0.18495163321495056\n",
      "1 447 0.14255014061927795\n",
      "1 448 0.12519755959510803\n",
      "1 449 0.11625641584396362\n",
      "1 450 0.10736457258462906\n",
      "1 451 0.10918978601694107\n",
      "1 452 0.09579714387655258\n",
      "1 453 0.12511824071407318\n",
      "1 454 0.13654771447181702\n",
      "1 455 0.10393102467060089\n",
      "1 456 0.1560269594192505\n",
      "1 457 0.1596027910709381\n",
      "1 458 0.17379538714885712\n",
      "1 459 0.16079936921596527\n",
      "1 460 0.1723763793706894\n",
      "1 461 0.1513005644083023\n",
      "1 462 0.11417044699192047\n",
      "1 463 0.12065873295068741\n",
      "1 464 0.1483398675918579\n",
      "1 465 0.11697639524936676\n",
      "1 466 0.13554596900939941\n",
      "1 467 0.11342862993478775\n",
      "1 468 0.10538482666015625\n",
      "1 469 0.09492760896682739\n",
      "1 470 0.13747869431972504\n",
      "1 471 0.17031961679458618\n",
      "1 472 0.15283021330833435\n",
      "1 473 0.1146354079246521\n",
      "1 474 0.10931843519210815\n",
      "1 475 0.12005679309368134\n",
      "1 476 0.13520550727844238\n",
      "1 477 0.1651594340801239\n",
      "1 478 0.13524971902370453\n",
      "1 479 0.2480020821094513\n",
      "1 480 0.16622808575630188\n",
      "1 481 0.15450796484947205\n",
      "1 482 0.15833036601543427\n",
      "1 483 0.17657257616519928\n",
      "1 484 0.1038222536444664\n",
      "1 485 0.11941230297088623\n",
      "1 486 0.17608699202537537\n",
      "1 487 0.1425606906414032\n",
      "1 488 0.1491205096244812\n",
      "1 489 0.11659754812717438\n",
      "1 490 0.1242615133523941\n",
      "1 491 0.13639897108078003\n",
      "1 492 0.12738223373889923\n",
      "1 493 0.14447557926177979\n",
      "1 494 0.10878823697566986\n",
      "1 495 0.16090676188468933\n",
      "1 496 0.13537032902240753\n",
      "1 497 0.16850784420967102\n",
      "1 498 0.1425883024930954\n",
      "1 499 0.14632603526115417\n",
      "1 500 0.09550081193447113\n",
      "1 501 0.16405002772808075\n",
      "1 502 0.16736000776290894\n",
      "1 503 0.13690628111362457\n",
      "1 504 0.11247861385345459\n",
      "1 505 0.12185096740722656\n",
      "1 506 0.13662554323673248\n",
      "1 507 0.14110228419303894\n",
      "1 508 0.14370577037334442\n",
      "1 509 0.12293355166912079\n",
      "1 510 0.11727190017700195\n",
      "1 511 0.0978274941444397\n",
      "1 512 0.09245900809764862\n",
      "1 513 0.15484723448753357\n",
      "1 514 0.09715528786182404\n",
      "1 515 0.14691483974456787\n",
      "1 516 0.14809559285640717\n",
      "1 517 0.10613957792520523\n",
      "1 518 0.17828786373138428\n",
      "1 519 0.10871319472789764\n",
      "1 520 0.16143257915973663\n",
      "1 521 0.16931794583797455\n",
      "1 522 0.09804149717092514\n",
      "1 523 0.10148841142654419\n",
      "1 524 0.0900215357542038\n",
      "1 525 0.10132624953985214\n",
      "1 526 0.1039692759513855\n",
      "1 527 0.11243212223052979\n",
      "1 528 0.15461984276771545\n",
      "1 529 0.11269994080066681\n",
      "1 530 0.14413650333881378\n",
      "1 531 0.1651546061038971\n",
      "1 532 0.190700963139534\n",
      "1 533 0.15200074017047882\n",
      "1 534 0.1977618932723999\n",
      "1 535 0.15130549669265747\n",
      "1 536 0.14988628029823303\n",
      "1 537 0.16351985931396484\n",
      "1 538 0.14678733050823212\n",
      "1 539 0.14749357104301453\n",
      "1 540 0.11864672601222992\n",
      "1 541 0.17431120574474335\n",
      "1 542 0.1185889020562172\n",
      "1 543 0.0886697918176651\n",
      "1 544 0.10891066491603851\n",
      "1 545 0.13362181186676025\n",
      "1 546 0.18291494250297546\n",
      "1 547 0.16225476562976837\n",
      "1 548 0.16364796459674835\n",
      "1 549 0.18886011838912964\n",
      "1 550 0.11777772009372711\n",
      "1 551 0.17391902208328247\n",
      "1 552 0.11530560255050659\n",
      "1 553 0.11897531896829605\n",
      "1 554 0.10728976875543594\n",
      "1 555 0.10514797270298004\n",
      "1 556 0.11443660408258438\n",
      "1 557 0.11919578164815903\n",
      "1 558 0.13046595454216003\n",
      "1 559 0.19168803095817566\n",
      "1 560 0.15469607710838318\n",
      "1 561 0.13607515394687653\n",
      "1 562 0.16557472944259644\n",
      "1 563 0.09619106352329254\n",
      "1 564 0.10407726466655731\n",
      "1 565 0.11645732074975967\n",
      "1 566 0.12676675617694855\n",
      "1 567 0.13546285033226013\n",
      "1 568 0.1233215481042862\n",
      "1 569 0.10804080963134766\n",
      "1 570 0.10917370021343231\n",
      "1 571 0.1473306566476822\n",
      "1 572 0.08907927572727203\n",
      "1 573 0.09186793863773346\n",
      "1 574 0.09751101583242416\n",
      "1 575 0.10885173082351685\n",
      "1 576 0.17813950777053833\n",
      "1 577 0.14638739824295044\n",
      "1 578 0.14466845989227295\n",
      "1 579 0.0909239798784256\n",
      "1 580 0.14173705875873566\n",
      "1 581 0.1331818401813507\n",
      "1 582 0.2146318256855011\n",
      "1 583 0.21490879356861115\n",
      "1 584 0.15695405006408691\n",
      "1 585 0.12298727035522461\n",
      "1 586 0.16129492223262787\n",
      "1 587 0.16631369292736053\n",
      "1 588 0.16474588215351105\n",
      "1 589 0.1778210550546646\n",
      "1 590 0.1355808675289154\n",
      "1 591 0.15169039368629456\n",
      "1 592 0.11739188432693481\n",
      "1 593 0.12463631480932236\n",
      "1 594 0.157377228140831\n",
      "1 595 0.14632007479667664\n",
      "1 596 0.10072346776723862\n",
      "1 597 0.1307838410139084\n",
      "1 598 0.11382534354925156\n",
      "1 599 0.13428165018558502\n",
      "1 600 0.1365162432193756\n",
      "1 601 0.1290477216243744\n",
      "1 602 0.13948364555835724\n",
      "1 603 0.14286266267299652\n",
      "1 604 0.10778951644897461\n",
      "1 605 0.13033784925937653\n",
      "1 606 0.14104995131492615\n",
      "1 607 0.2132636308670044\n",
      "1 608 0.12107014656066895\n",
      "1 609 0.1442762315273285\n",
      "1 610 0.18233296275138855\n",
      "1 611 0.14443062245845795\n",
      "1 612 0.1194697767496109\n",
      "1 613 0.10617687553167343\n",
      "1 614 0.15161308646202087\n",
      "1 615 0.21167774498462677\n",
      "1 616 0.1384478509426117\n",
      "1 617 0.2033432275056839\n",
      "1 618 0.1220061182975769\n",
      "1 619 0.13217869400978088\n",
      "1 620 0.17133410274982452\n",
      "1 621 0.16935832798480988\n",
      "1 622 0.14270076155662537\n",
      "1 623 0.11039088666439056\n",
      "1 624 0.11061267554759979\n",
      "1 625 0.15980397164821625\n",
      "1 626 0.1211242526769638\n",
      "1 627 0.17205150425434113\n",
      "1 628 0.12007448822259903\n",
      "1 629 0.11713065207004547\n",
      "1 630 0.14153926074504852\n",
      "1 631 0.10102520138025284\n",
      "1 632 0.13266828656196594\n",
      "1 633 0.12003874778747559\n",
      "1 634 0.08520010858774185\n",
      "1 635 0.13040420413017273\n",
      "1 636 0.1229322999715805\n",
      "1 637 0.12296782433986664\n",
      "1 638 0.14459379017353058\n",
      "1 639 0.1254918873310089\n",
      "1 640 0.14519715309143066\n",
      "1 641 0.13469234108924866\n",
      "1 642 0.15887023508548737\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 643 0.1600598394870758\n",
      "1 644 0.12066410481929779\n",
      "1 645 0.18574878573417664\n",
      "1 646 0.1279158592224121\n",
      "1 647 0.1750504970550537\n",
      "1 648 0.14038851857185364\n",
      "1 649 0.15080709755420685\n",
      "1 650 0.16173292696475983\n",
      "1 651 0.09766177088022232\n",
      "1 652 0.16784359514713287\n",
      "1 653 0.13530613481998444\n",
      "1 654 0.14923451840877533\n",
      "1 655 0.14028196036815643\n",
      "1 656 0.13676327466964722\n",
      "1 657 0.15362763404846191\n",
      "1 658 0.09165014326572418\n",
      "1 659 0.12035313993692398\n",
      "1 660 0.14953011274337769\n",
      "1 661 0.10439883172512054\n",
      "1 662 0.14715030789375305\n",
      "1 663 0.11167421191930771\n",
      "1 664 0.14883002638816833\n",
      "1 665 0.13839280605316162\n",
      "1 666 0.14774355292320251\n",
      "1 667 0.12894800305366516\n",
      "1 668 0.11570147424936295\n",
      "1 669 0.14993873238563538\n",
      "1 670 0.11998595297336578\n",
      "1 671 0.12365175783634186\n",
      "1 672 0.1537283957004547\n",
      "1 673 0.148948535323143\n",
      "1 674 0.1502455770969391\n",
      "1 675 0.14124919474124908\n",
      "1 676 0.1090245246887207\n",
      "1 677 0.14162799715995789\n",
      "1 678 0.11744359880685806\n",
      "1 679 0.177715465426445\n",
      "1 680 0.14492806792259216\n",
      "1 681 0.1905486285686493\n",
      "1 682 0.10433249175548553\n",
      "1 683 0.11889801174402237\n",
      "1 684 0.12545369565486908\n",
      "1 685 0.11575688421726227\n",
      "1 686 0.12077685445547104\n",
      "1 687 0.11885862052440643\n",
      "1 688 0.12209104001522064\n",
      "1 689 0.15750382840633392\n",
      "1 690 0.11056949943304062\n",
      "1 691 0.14868345856666565\n",
      "1 692 0.1293746829032898\n",
      "1 693 0.16505512595176697\n",
      "1 694 0.17391760647296906\n",
      "1 695 0.1572631299495697\n",
      "1 696 0.16519072651863098\n",
      "1 697 0.18604867160320282\n",
      "1 698 0.12365832179784775\n",
      "1 699 0.1284170299768448\n",
      "1 700 0.22356903553009033\n",
      "1 701 0.1693061888217926\n",
      "1 702 0.1431685984134674\n",
      "1 703 0.16528856754302979\n",
      "1 704 0.12482497841119766\n",
      "1 705 0.15089789032936096\n",
      "1 706 0.18064379692077637\n",
      "1 707 0.12888136506080627\n",
      "1 708 0.13779766857624054\n",
      "1 709 0.1051279753446579\n",
      "1 710 0.10193921625614166\n",
      "1 711 0.1500769555568695\n",
      "1 712 0.15807390213012695\n",
      "1 713 0.11656055599451065\n",
      "1 714 0.15147843956947327\n",
      "1 715 0.13809452950954437\n",
      "1 716 0.18038548529148102\n",
      "1 717 0.12105168402194977\n",
      "1 718 0.15598879754543304\n",
      "1 719 0.11692202091217041\n",
      "1 720 0.13221368193626404\n",
      "1 721 0.1925094574689865\n",
      "1 722 0.14275692403316498\n",
      "1 723 0.12139048427343369\n",
      "1 724 0.18730273842811584\n",
      "1 725 0.1404605507850647\n",
      "1 726 0.1805528998374939\n",
      "1 727 0.1281818151473999\n",
      "1 728 0.15065057575702667\n",
      "1 729 0.2049913853406906\n",
      "1 730 0.14309951663017273\n",
      "1 731 0.19827024638652802\n",
      "1 732 0.14978475868701935\n",
      "1 733 0.15109920501708984\n",
      "1 734 0.11314555257558823\n",
      "1 735 0.11150858551263809\n",
      "1 736 0.1291537582874298\n",
      "1 737 0.09974298626184464\n",
      "1 738 0.19808831810951233\n",
      "1 739 0.14742784202098846\n",
      "1 740 0.1537383496761322\n",
      "1 741 0.2066711187362671\n",
      "1 742 0.14582958817481995\n",
      "1 743 0.15770553052425385\n",
      "1 744 0.17802724242210388\n",
      "1 745 0.13960430026054382\n",
      "1 746 0.13647882640361786\n",
      "1 747 0.1021183580160141\n",
      "1 748 0.1595594584941864\n",
      "1 749 0.1200634241104126\n",
      "1 750 0.17635087668895721\n",
      "1 751 0.1539316475391388\n",
      "1 752 0.13203488290309906\n",
      "1 753 0.13545359671115875\n",
      "1 754 0.1352749466896057\n",
      "1 755 0.13495798408985138\n",
      "1 756 0.17782403528690338\n",
      "1 757 0.11929038912057877\n",
      "1 758 0.18073177337646484\n",
      "1 759 0.1404433399438858\n",
      "1 760 0.13207478821277618\n",
      "1 761 0.1278296709060669\n",
      "1 762 0.15622104704380035\n",
      "1 763 0.13679611682891846\n",
      "1 764 0.14555230736732483\n",
      "1 765 0.12729540467262268\n",
      "1 766 0.1340055912733078\n",
      "1 767 0.15616914629936218\n",
      "1 768 0.14514373242855072\n",
      "1 769 0.14044426381587982\n",
      "1 770 0.1609610617160797\n",
      "1 771 0.18761634826660156\n",
      "1 772 0.14422431588172913\n",
      "1 773 0.11216899007558823\n",
      "1 774 0.14470359683036804\n",
      "1 775 0.14254894852638245\n",
      "1 776 0.11903449147939682\n",
      "1 777 0.1077650636434555\n",
      "1 778 0.1585928052663803\n",
      "1 779 0.13671955466270447\n",
      "1 780 0.1334158331155777\n",
      "1 781 0.13039354979991913\n",
      "1 782 0.1248399019241333\n",
      "1 783 0.10187399387359619\n",
      "1 784 0.18075604736804962\n",
      "1 785 0.20328457653522491\n",
      "1 786 0.10113964229822159\n",
      "1 787 0.13954201340675354\n",
      "1 788 0.1032145619392395\n",
      "1 789 0.11255331337451935\n",
      "1 790 0.14038029313087463\n",
      "1 791 0.12145060300827026\n",
      "1 792 0.1214284747838974\n",
      "1 793 0.17441673576831818\n",
      "1 794 0.11018339544534683\n",
      "1 795 0.13015815615653992\n",
      "1 796 0.10300421714782715\n",
      "1 797 0.12468498945236206\n",
      "1 798 0.17712534964084625\n",
      "1 799 0.10666154325008392\n",
      "1 800 0.1378973424434662\n",
      "1 801 0.14038985967636108\n",
      "1 802 0.12271679937839508\n",
      "1 803 0.18258899450302124\n",
      "1 804 0.2078116238117218\n",
      "1 805 0.12094746530056\n",
      "1 806 0.15570347011089325\n",
      "1 807 0.14533188939094543\n",
      "1 808 0.18984052538871765\n",
      "1 809 0.12269216775894165\n",
      "1 810 0.1364745944738388\n",
      "1 811 0.13254882395267487\n",
      "1 812 0.12496404349803925\n",
      "1 813 0.1353277564048767\n",
      "1 814 0.12527036666870117\n",
      "1 815 0.11685016006231308\n",
      "1 816 0.19748380780220032\n",
      "1 817 0.14916783571243286\n",
      "1 818 0.13935238122940063\n",
      "1 819 0.1292884647846222\n",
      "1 820 0.142043799161911\n",
      "1 821 0.10451480001211166\n",
      "1 822 0.11531047523021698\n",
      "1 823 0.14256012439727783\n",
      "1 824 0.20014521479606628\n",
      "1 825 0.12934409081935883\n",
      "1 826 0.1317559778690338\n",
      "1 827 0.14872467517852783\n",
      "1 828 0.15501292049884796\n",
      "1 829 0.15142536163330078\n",
      "1 830 0.1287001520395279\n",
      "1 831 0.14121708273887634\n",
      "1 832 0.1276460587978363\n",
      "1 833 0.1754191368818283\n",
      "1 834 0.10987379401922226\n",
      "1 835 0.1332707703113556\n",
      "1 836 0.23560956120491028\n",
      "1 837 0.16235674917697906\n",
      "1 838 0.09719796478748322\n",
      "1 839 0.1522318720817566\n",
      "1 840 0.1340325027704239\n",
      "1 841 0.13695985078811646\n",
      "1 842 0.133353129029274\n",
      "1 843 0.11721744388341904\n",
      "1 844 0.13101933896541595\n",
      "1 845 0.09553374350070953\n",
      "1 846 0.10655037313699722\n",
      "1 847 0.12474856525659561\n",
      "1 848 0.17602074146270752\n",
      "1 849 0.139863982796669\n",
      "1 850 0.12215490639209747\n",
      "1 851 0.07456028461456299\n",
      "1 852 0.09298239648342133\n",
      "1 853 0.1329711377620697\n",
      "1 854 0.15256378054618835\n",
      "1 855 0.18217912316322327\n",
      "1 856 0.19725804030895233\n",
      "1 857 0.11614613234996796\n",
      "1 858 0.18293729424476624\n",
      "1 859 0.13177569210529327\n",
      "1 860 0.11707665026187897\n",
      "1 861 0.0968041718006134\n",
      "1 862 0.14339038729667664\n",
      "1 863 0.1975955367088318\n",
      "1 864 0.14553740620613098\n",
      "1 865 0.11301183700561523\n",
      "1 866 0.11578243970870972\n",
      "1 867 0.1592961847782135\n",
      "1 868 0.12214066833257675\n",
      "1 869 0.1909780353307724\n",
      "1 870 0.18803340196609497\n",
      "1 871 0.12711918354034424\n",
      "1 872 0.15323486924171448\n",
      "1 873 0.09794531762599945\n",
      "1 874 0.1139698177576065\n",
      "1 875 0.12488144636154175\n",
      "1 876 0.15351402759552002\n",
      "1 877 0.11556142568588257\n",
      "1 878 0.12536238133907318\n",
      "1 879 0.14286336302757263\n",
      "1 880 0.2058086395263672\n",
      "1 881 0.16220960021018982\n",
      "1 882 0.12262342870235443\n",
      "1 883 0.09706517308950424\n",
      "1 884 0.14487752318382263\n",
      "1 885 0.11664428561925888\n",
      "1 886 0.11147090047597885\n",
      "1 887 0.13211879134178162\n",
      "1 888 0.12822341918945312\n",
      "1 889 0.09896433353424072\n",
      "1 890 0.11619234085083008\n",
      "1 891 0.1557367444038391\n",
      "1 892 0.1507813185453415\n",
      "1 893 0.13173173367977142\n",
      "1 894 0.16449712216854095\n",
      "1 895 0.12199129909276962\n",
      "1 896 0.18294329941272736\n",
      "1 897 0.1140679121017456\n",
      "1 898 0.1776677370071411\n",
      "1 899 0.14133673906326294\n",
      "1 900 0.11400280147790909\n",
      "1 901 0.09595118463039398\n",
      "1 902 0.10847292840480804\n",
      "1 903 0.16385066509246826\n",
      "1 904 0.152114599943161\n",
      "1 905 0.10888780653476715\n",
      "1 906 0.0814896747469902\n",
      "1 907 0.10705795139074326\n",
      "1 908 0.2336144745349884\n",
      "1 909 0.16356001794338226\n",
      "1 910 0.14408381283283234\n",
      "1 911 0.15737156569957733\n",
      "1 912 0.11134304851293564\n",
      "1 913 0.19447001814842224\n",
      "1 914 0.14648345112800598\n",
      "1 915 0.18426311016082764\n",
      "1 916 0.12034282833337784\n",
      "1 917 0.10548253357410431\n",
      "1 918 0.1429298222064972\n",
      "1 919 0.13945375382900238\n",
      "1 920 0.15918485820293427\n",
      "1 921 0.12552510201931\n",
      "1 922 0.16612407565116882\n",
      "1 923 0.18608076870441437\n",
      "1 924 0.12516722083091736\n",
      "1 925 0.12117217481136322\n",
      "1 926 0.1183851957321167\n",
      "1 927 0.09316465258598328\n",
      "1 928 0.14382018148899078\n",
      "1 929 0.18196603655815125\n",
      "1 930 0.11485247313976288\n",
      "1 931 0.11901688575744629\n",
      "1 932 0.1687716543674469\n",
      "1 933 0.13240814208984375\n",
      "1 934 0.17771098017692566\n",
      "1 935 0.15458980202674866\n",
      "1 936 0.13445547223091125\n",
      "1 937 0.13852354884147644\n",
      "1 938 0.1261061131954193\n",
      "1 939 0.130334734916687\n",
      "1 940 0.1071729063987732\n",
      "1 941 0.16229307651519775\n",
      "1 942 0.1498991847038269\n",
      "1 943 0.11452563852071762\n",
      "1 944 0.1352505385875702\n",
      "1 945 0.10304609686136246\n",
      "1 946 0.10967317968606949\n",
      "1 947 0.17172574996948242\n",
      "1 948 0.13470765948295593\n",
      "1 949 0.1385924518108368\n",
      "1 950 0.15504294633865356\n",
      "1 951 0.0807691216468811\n",
      "1 952 0.10703392326831818\n",
      "1 953 0.16196297109127045\n",
      "1 954 0.09800861030817032\n",
      "1 955 0.22421927750110626\n",
      "1 956 0.1256648153066635\n",
      "1 957 0.09628084301948547\n",
      "1 958 0.11965985596179962\n",
      "1 959 0.1045340895652771\n",
      "1 960 0.11258403211832047\n",
      "1 961 0.10635727643966675\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 962 0.13902008533477783\n",
      "1 963 0.16731010377407074\n",
      "1 964 0.17439484596252441\n",
      "1 965 0.1561526358127594\n",
      "1 966 0.09468428045511246\n",
      "1 967 0.1446932703256607\n",
      "1 968 0.13221439719200134\n",
      "1 969 0.14815539121627808\n",
      "1 970 0.17783118784427643\n",
      "1 971 0.09559211879968643\n",
      "1 972 0.1744704246520996\n",
      "1 973 0.12635818123817444\n",
      "1 974 0.10200254619121552\n",
      "1 975 0.1254597008228302\n",
      "1 976 0.11359286308288574\n",
      "1 977 0.1358216106891632\n",
      "1 978 0.14164802432060242\n",
      "1 979 0.181075319647789\n",
      "1 980 0.195900097489357\n",
      "1 981 0.14487335085868835\n",
      "1 982 0.09517328441143036\n",
      "1 983 0.13585111498832703\n",
      "1 984 0.17016127705574036\n",
      "1 985 0.17779752612113953\n",
      "1 986 0.15345928072929382\n",
      "1 987 0.13365010917186737\n",
      "1 988 0.1279253363609314\n",
      "1 989 0.2053709328174591\n",
      "1 990 0.12474668025970459\n",
      "1 991 0.15621192753314972\n",
      "1 992 0.13031716644763947\n",
      "1 993 0.1544943004846573\n",
      "1 994 0.13229431211948395\n",
      "1 995 0.17253969609737396\n",
      "1 996 0.1359737664461136\n",
      "1 997 0.09505583345890045\n",
      "1 998 0.16339127719402313\n",
      "1 999 0.148445725440979\n",
      "1 1000 0.11404011398553848\n",
      "1 1001 0.1351851522922516\n",
      "1 1002 0.10485143959522247\n",
      "1 1003 0.11509321630001068\n",
      "1 1004 0.1396155208349228\n",
      "1 1005 0.16315791010856628\n",
      "1 1006 0.19746677577495575\n",
      "1 1007 0.12780483067035675\n",
      "1 1008 0.16679351031780243\n",
      "1 1009 0.1317332684993744\n",
      "1 1010 0.12382736057043076\n",
      "1 1011 0.21156394481658936\n",
      "1 1012 0.18609820306301117\n",
      "1 1013 0.151587575674057\n",
      "1 1014 0.16077890992164612\n",
      "1 1015 0.1747046411037445\n",
      "1 1016 0.16473820805549622\n",
      "1 1017 0.19546571373939514\n",
      "1 1018 0.10497187077999115\n",
      "1 1019 0.19785316288471222\n",
      "1 1020 0.15159229934215546\n",
      "1 1021 0.16499167680740356\n",
      "1 1022 0.11430446058511734\n",
      "1 1023 0.1150474101305008\n",
      "1 1024 0.17851649224758148\n",
      "1 1025 0.08948345482349396\n",
      "1 1026 0.11827800422906876\n",
      "1 1027 0.11479780822992325\n",
      "1 1028 0.1446816623210907\n",
      "1 1029 0.16164585947990417\n",
      "1 1030 0.10815467685461044\n",
      "1 1031 0.13010826706886292\n",
      "1 1032 0.12730273604393005\n",
      "1 1033 0.13598832488059998\n",
      "1 1034 0.16638796031475067\n",
      "1 1035 0.16645339131355286\n",
      "1 1036 0.14316591620445251\n",
      "1 1037 0.13605210185050964\n",
      "1 1038 0.13698872923851013\n",
      "1 1039 0.14127673208713531\n",
      "1 1040 0.1348794400691986\n",
      "1 1041 0.1722695529460907\n",
      "1 1042 0.12023220211267471\n",
      "1 1043 0.12647773325443268\n",
      "1 1044 0.1243375912308693\n",
      "1 1045 0.13711291551589966\n",
      "1 1046 0.12505263090133667\n",
      "1 1047 0.14620906114578247\n",
      "1 1048 0.15424233675003052\n",
      "1 1049 0.10311783850193024\n",
      "1 1050 0.1411527842283249\n",
      "1 1051 0.15774980187416077\n",
      "1 1052 0.15314288437366486\n",
      "1 1053 0.14847329258918762\n",
      "1 1054 0.12879492342472076\n",
      "1 1055 0.1633273810148239\n",
      "1 1056 0.11704502999782562\n",
      "1 1057 0.1118515133857727\n",
      "1 1058 0.16078004240989685\n",
      "1 1059 0.10977701842784882\n",
      "1 1060 0.13786986470222473\n",
      "1 1061 0.1392815262079239\n",
      "1 1062 0.10886119306087494\n",
      "1 1063 0.1649141162633896\n",
      "1 1064 0.09859946370124817\n",
      "1 1065 0.11110500991344452\n",
      "1 1066 0.13736626505851746\n",
      "1 1067 0.2057168185710907\n",
      "1 1068 0.14551396667957306\n",
      "1 1069 0.181197851896286\n",
      "1 1070 0.13653632998466492\n",
      "1 1071 0.1560169756412506\n",
      "1 1072 0.12182815372943878\n",
      "1 1073 0.16286054253578186\n",
      "1 1074 0.16217537224292755\n",
      "1 1075 0.1290382444858551\n",
      "1 1076 0.110906220972538\n",
      "1 1077 0.18506455421447754\n",
      "1 1078 0.15236996114253998\n",
      "1 1079 0.11621570587158203\n",
      "1 1080 0.14981824159622192\n",
      "1 1081 0.10974041372537613\n",
      "1 1082 0.1236606016755104\n",
      "1 1083 0.16444751620292664\n",
      "1 1084 0.07592713832855225\n",
      "1 1085 0.16198790073394775\n",
      "1 1086 0.10409539937973022\n",
      "1 1087 0.14033162593841553\n",
      "1 1088 0.11317912489175797\n",
      "1 1089 0.12121126800775528\n",
      "1 1090 0.1510995775461197\n",
      "1 1091 0.15997621417045593\n",
      "1 1092 0.12956435978412628\n",
      "1 1093 0.15205810964107513\n",
      "1 1094 0.12633013725280762\n",
      "1 1095 0.2058923989534378\n",
      "1 1096 0.10266661643981934\n",
      "1 1097 0.07967809587717056\n",
      "1 1098 0.1367555409669876\n",
      "1 1099 0.12879694998264313\n",
      "1 1100 0.09946419298648834\n",
      "1 1101 0.13638265430927277\n",
      "1 1102 0.18307019770145416\n",
      "1 1103 0.15809598565101624\n",
      "1 1104 0.15658465027809143\n",
      "1 1105 0.13921178877353668\n",
      "1 1106 0.10084639489650726\n",
      "1 1107 0.10150863230228424\n",
      "1 1108 0.09914317727088928\n",
      "1 1109 0.1320861428976059\n",
      "1 1110 0.1071849837899208\n",
      "1 1111 0.15377794206142426\n",
      "1 1112 0.17168936133384705\n",
      "1 1113 0.16775518655776978\n",
      "1 1114 0.15009446442127228\n",
      "1 1115 0.13241812586784363\n",
      "1 1116 0.14416128396987915\n",
      "1 1117 0.08116912841796875\n",
      "1 1118 0.16189830005168915\n",
      "1 1119 0.12451116740703583\n",
      "1 1120 0.15570002794265747\n",
      "1 1121 0.16855590045452118\n",
      "1 1122 0.13795658946037292\n",
      "1 1123 0.12325235456228256\n",
      "1 1124 0.15801501274108887\n",
      "1 1125 0.13161543011665344\n",
      "1 1126 0.18354496359825134\n",
      "1 1127 0.1518736481666565\n",
      "1 1128 0.11149753630161285\n",
      "1 1129 0.14883099496364594\n",
      "1 1130 0.08567478507757187\n",
      "1 1131 0.09506003558635712\n",
      "1 1132 0.14719530940055847\n",
      "1 1133 0.07840278744697571\n",
      "1 1134 0.12639372050762177\n",
      "1 1135 0.18443945050239563\n",
      "1 1136 0.15946665406227112\n",
      "1 1137 0.12998318672180176\n",
      "1 1138 0.10479432344436646\n",
      "1 1139 0.10087950527667999\n",
      "1 1140 0.1756230741739273\n",
      "1 1141 0.11304457485675812\n",
      "1 1142 0.19582056999206543\n",
      "1 1143 0.16665123403072357\n",
      "1 1144 0.13447386026382446\n",
      "1 1145 0.10227175056934357\n",
      "1 1146 0.11414853483438492\n",
      "1 1147 0.11859065294265747\n",
      "1 1148 0.07137217372655869\n",
      "1 1149 0.11306866258382797\n",
      "1 1150 0.08037064224481583\n",
      "1 1151 0.16705679893493652\n",
      "1 1152 0.12108111381530762\n",
      "1 1153 0.1031048446893692\n",
      "1 1154 0.15095193684101105\n",
      "1 1155 0.13702288269996643\n",
      "1 1156 0.20246033370494843\n",
      "1 1157 0.09908808767795563\n",
      "1 1158 0.11992001533508301\n",
      "1 1159 0.15433339774608612\n",
      "1 1160 0.0986395925283432\n",
      "1 1161 0.13710638880729675\n",
      "1 1162 0.10246322304010391\n",
      "1 1163 0.12576571106910706\n",
      "1 1164 0.1618598997592926\n",
      "1 1165 0.1935243308544159\n",
      "1 1166 0.11959214508533478\n",
      "1 1167 0.14407089352607727\n",
      "1 1168 0.1516353338956833\n",
      "1 1169 0.12722167372703552\n",
      "1 1170 0.14387628436088562\n",
      "1 1171 0.155761256814003\n",
      "1 1172 0.1330328732728958\n",
      "1 1173 0.07663093507289886\n",
      "1 1174 0.13896848261356354\n",
      "1 1175 0.21365515887737274\n",
      "1 1176 0.1341876983642578\n",
      "1 1177 0.14880749583244324\n",
      "1 1178 0.11497704684734344\n",
      "1 1179 0.12128470838069916\n",
      "1 1180 0.11254511773586273\n",
      "1 1181 0.11600194126367569\n",
      "1 1182 0.16273365914821625\n",
      "1 1183 0.1370387077331543\n",
      "1 1184 0.1548033058643341\n",
      "1 1185 0.12780115008354187\n",
      "1 1186 0.13173672556877136\n",
      "1 1187 0.14665469527244568\n",
      "1 1188 0.11780519783496857\n",
      "1 1189 0.09032101929187775\n",
      "1 1190 0.08455394208431244\n",
      "1 1191 0.11256109178066254\n",
      "1 1192 0.06972645968198776\n",
      "1 1193 0.11578065156936646\n",
      "1 1194 0.14897659420967102\n",
      "1 1195 0.12465765327215195\n",
      "1 1196 0.17996546626091003\n",
      "1 1197 0.14619003236293793\n",
      "1 1198 0.12969550490379333\n",
      "1 1199 0.12915748357772827\n",
      "1 1200 0.18285812437534332\n",
      "1 1201 0.1288234293460846\n",
      "1 1202 0.09714121371507645\n",
      "1 1203 0.15038271248340607\n",
      "1 1204 0.08109728991985321\n",
      "1 1205 0.14871333539485931\n",
      "1 1206 0.14799705147743225\n",
      "1 1207 0.1392718255519867\n",
      "1 1208 0.1533525586128235\n",
      "1 1209 0.13171060383319855\n",
      "1 1210 0.10981074720621109\n",
      "1 1211 0.09481330960988998\n",
      "1 1212 0.1298857033252716\n",
      "1 1213 0.11236906051635742\n",
      "1 1214 0.14337417483329773\n",
      "1 1215 0.16695532202720642\n",
      "1 1216 0.1211772933602333\n",
      "1 1217 0.11128966510295868\n",
      "1 1218 0.17052529752254486\n",
      "1 1219 0.15327447652816772\n",
      "1 1220 0.1259317547082901\n",
      "1 1221 0.11707069724798203\n",
      "1 1222 0.10107390582561493\n",
      "1 1223 0.11975575983524323\n",
      "1 1224 0.12383250147104263\n",
      "1 1225 0.13429343700408936\n",
      "1 1226 0.1317230463027954\n",
      "1 1227 0.09942913055419922\n",
      "1 1228 0.10488524287939072\n",
      "1 1229 0.174479678273201\n",
      "1 1230 0.21892976760864258\n",
      "1 1231 0.10999862104654312\n",
      "1 1232 0.10671856254339218\n",
      "1 1233 0.112043596804142\n",
      "1 1234 0.14787325263023376\n",
      "1 1235 0.13615740835666656\n",
      "1 1236 0.1259998381137848\n",
      "1 1237 0.08533425629138947\n",
      "1 1238 0.10888370126485825\n",
      "1 1239 0.16209861636161804\n",
      "1 1240 0.12428227812051773\n",
      "1 1241 0.1558176726102829\n",
      "1 1242 0.11287547647953033\n",
      "1 1243 0.14319685101509094\n",
      "1 1244 0.13915707170963287\n",
      "1 1245 0.14238587021827698\n",
      "1 1246 0.15037354826927185\n",
      "1 1247 0.08606278151273727\n",
      "1 1248 0.09508350491523743\n",
      "1 1249 0.15159301459789276\n",
      "1 1250 0.08995786309242249\n",
      "1 1251 0.1705736368894577\n",
      "1 1252 0.08182192593812943\n",
      "1 1253 0.10752604901790619\n",
      "1 1254 0.15287408232688904\n",
      "1 1255 0.16090074181556702\n",
      "1 1256 0.18320943415164948\n",
      "1 1257 0.19206133484840393\n",
      "1 1258 0.16950154304504395\n",
      "1 1259 0.1578945517539978\n",
      "1 1260 0.2056979387998581\n",
      "1 1261 0.12346895784139633\n",
      "1 1262 0.1500442922115326\n",
      "1 1263 0.15976762771606445\n",
      "1 1264 0.13533200323581696\n",
      "1 1265 0.1326310783624649\n",
      "1 1266 0.10973048210144043\n",
      "1 1267 0.11849025636911392\n",
      "1 1268 0.11212006956338882\n",
      "1 1269 0.13274547457695007\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 1270 0.1512284129858017\n",
      "1 1271 0.14318907260894775\n",
      "1 1272 0.1095210462808609\n",
      "1 1273 0.1776197850704193\n",
      "1 1274 0.1407162845134735\n",
      "1 1275 0.1674683839082718\n",
      "1 1276 0.13640102744102478\n",
      "1 1277 0.15084335207939148\n",
      "1 1278 0.10057435184717178\n",
      "1 1279 0.11092312633991241\n",
      "1 1280 0.20158538222312927\n",
      "1 1281 0.10035575926303864\n",
      "1 1282 0.11627276986837387\n",
      "1 1283 0.0949200838804245\n",
      "1 1284 0.20306377112865448\n",
      "1 1285 0.1396486461162567\n",
      "1 1286 0.18580171465873718\n",
      "1 1287 0.1208876371383667\n",
      "1 1288 0.12712427973747253\n",
      "1 1289 0.1471814513206482\n",
      "1 1290 0.13176774978637695\n",
      "1 1291 0.1371045708656311\n",
      "1 1292 0.18106260895729065\n",
      "1 1293 0.13761486113071442\n",
      "1 1294 0.1198289543390274\n",
      "1 1295 0.13549558818340302\n",
      "1 1296 0.17132753133773804\n",
      "1 1297 0.09824708849191666\n",
      "1 1298 0.1514371633529663\n",
      "1 1299 0.13074582815170288\n",
      "1 1300 0.08754850924015045\n",
      "1 1301 0.16991600394248962\n",
      "1 1302 0.14063255488872528\n",
      "1 1303 0.14310656487941742\n",
      "1 1304 0.11840786039829254\n",
      "1 1305 0.08974780142307281\n",
      "1 1306 0.1580015867948532\n",
      "1 1307 0.14879202842712402\n",
      "1 1308 0.1426563262939453\n",
      "1 1309 0.20099060237407684\n",
      "1 1310 0.1799589842557907\n",
      "1 1311 0.12818095088005066\n",
      "1 1312 0.13594022393226624\n",
      "1 1313 0.1695362627506256\n",
      "1 1314 0.12498881667852402\n",
      "1 1315 0.21239392459392548\n",
      "1 1316 0.11920815706253052\n",
      "1 1317 0.15757156908512115\n",
      "1 1318 0.1334305703639984\n",
      "1 1319 0.09937207400798798\n",
      "1 1320 0.09239805489778519\n",
      "1 1321 0.11581182479858398\n",
      "1 1322 0.14731483161449432\n",
      "1 1323 0.1917770653963089\n",
      "1 1324 0.1409725844860077\n",
      "1 1325 0.12581893801689148\n",
      "1 1326 0.12063737958669662\n",
      "1 1327 0.1365838199853897\n",
      "1 1328 0.13421466946601868\n",
      "1 1329 0.14514419436454773\n",
      "1 1330 0.11735769361257553\n",
      "1 1331 0.1576041579246521\n",
      "1 1332 0.14368408918380737\n",
      "1 1333 0.09253402799367905\n",
      "1 1334 0.15525081753730774\n",
      "1 1335 0.11744612455368042\n",
      "1 1336 0.11340920627117157\n",
      "1 1337 0.12723296880722046\n",
      "1 1338 0.16347743570804596\n",
      "1 1339 0.18355809152126312\n",
      "1 1340 0.085425466299057\n",
      "1 1341 0.12348407506942749\n",
      "1 1342 0.1181645393371582\n",
      "1 1343 0.12071411311626434\n",
      "1 1344 0.10289641469717026\n",
      "1 1345 0.20776763558387756\n",
      "1 1346 0.1566663533449173\n",
      "1 1347 0.13558046519756317\n",
      "1 1348 0.1435493528842926\n",
      "1 1349 0.14224159717559814\n",
      "1 1350 0.0964161604642868\n",
      "1 1351 0.10736414045095444\n",
      "1 1352 0.14834758639335632\n",
      "1 1353 0.14933326840400696\n",
      "1 1354 0.0782093033194542\n",
      "1 1355 0.1309792548418045\n",
      "1 1356 0.1429898589849472\n",
      "1 1357 0.16320721805095673\n",
      "1 1358 0.14582513272762299\n",
      "1 1359 0.11950641870498657\n",
      "1 1360 0.09036330878734589\n",
      "1 1361 0.13469462096691132\n",
      "1 1362 0.0918441191315651\n",
      "1 1363 0.1181902140378952\n",
      "1 1364 0.11303649842739105\n",
      "1 1365 0.20659427344799042\n",
      "1 1366 0.20570413768291473\n",
      "1 1367 0.20051784813404083\n",
      "1 1368 0.11698269844055176\n",
      "1 1369 0.168827623128891\n",
      "1 1370 0.14990179240703583\n",
      "1 1371 0.1112140640616417\n",
      "1 1372 0.13486193120479584\n",
      "1 1373 0.1178307756781578\n",
      "1 1374 0.11347486823797226\n",
      "1 1375 0.13501764833927155\n",
      "1 1376 0.1370169222354889\n",
      "1 1377 0.09490789473056793\n",
      "1 1378 0.1389361470937729\n",
      "1 1379 0.12781743705272675\n",
      "1 1380 0.101242795586586\n",
      "1 1381 0.11836574226617813\n",
      "1 1382 0.2592769265174866\n",
      "1 1383 0.11714067310094833\n",
      "1 1384 0.1405809074640274\n",
      "1 1385 0.15203654766082764\n",
      "1 1386 0.09296265244483948\n",
      "1 1387 0.10882239043712616\n",
      "1 1388 0.12072936445474625\n",
      "1 1389 0.1650167554616928\n",
      "1 1390 0.11141426861286163\n",
      "1 1391 0.16493749618530273\n",
      "1 1392 0.18184764683246613\n",
      "1 1393 0.14513467252254486\n",
      "1 1394 0.09440852701663971\n",
      "1 1395 0.16059240698814392\n",
      "1 1396 0.10331033170223236\n",
      "1 1397 0.12016411870718002\n",
      "1 1398 0.1169423833489418\n",
      "1 1399 0.1275852471590042\n",
      "1 1400 0.15794068574905396\n",
      "1 1401 0.13748711347579956\n",
      "1 1402 0.16977159678936005\n",
      "1 1403 0.09615390747785568\n",
      "1 1404 0.12611865997314453\n",
      "1 1405 0.14950671792030334\n",
      "1 1406 0.16725483536720276\n",
      "1 1407 0.10577549040317535\n",
      "1 1408 0.14122886955738068\n",
      "1 1409 0.09508232772350311\n",
      "1 1410 0.14529775083065033\n",
      "1 1411 0.17916931211948395\n",
      "1 1412 0.23824599385261536\n",
      "1 1413 0.1629638671875\n",
      "1 1414 0.20504255592823029\n",
      "1 1415 0.19034205377101898\n",
      "1 1416 0.12097972631454468\n",
      "1 1417 0.172384113073349\n",
      "1 1418 0.21743392944335938\n",
      "1 1419 0.14613297581672668\n",
      "1 1420 0.11809792369604111\n",
      "1 1421 0.11508788168430328\n",
      "1 1422 0.15544696152210236\n",
      "1 1423 0.10389367491006851\n",
      "1 1424 0.10579165071249008\n",
      "1 1425 0.17180418968200684\n",
      "1 1426 0.10434478521347046\n",
      "1 1427 0.1398007720708847\n",
      "1 1428 0.16007234156131744\n",
      "1 1429 0.15309107303619385\n",
      "1 1430 0.16598451137542725\n",
      "1 1431 0.13868889212608337\n",
      "1 1432 0.12381911277770996\n",
      "1 1433 0.1408560574054718\n",
      "1 1434 0.08026044070720673\n",
      "1 1435 0.17024122178554535\n",
      "1 1436 0.15019330382347107\n",
      "1 1437 0.13222628831863403\n",
      "1 1438 0.14582771062850952\n",
      "1 1439 0.1420469582080841\n",
      "1 1440 0.10786058008670807\n",
      "1 1441 0.11943336576223373\n",
      "1 1442 0.13760703802108765\n",
      "1 1443 0.09640626609325409\n",
      "1 1444 0.2006000280380249\n",
      "1 1445 0.1474059820175171\n",
      "1 1446 0.09593845903873444\n",
      "1 1447 0.18273857235908508\n",
      "1 1448 0.10601826012134552\n",
      "1 1449 0.13892963528633118\n",
      "1 1450 0.13333760201931\n",
      "1 1451 0.09887222200632095\n",
      "1 1452 0.21704265475273132\n",
      "1 1453 0.10564012825489044\n",
      "1 1454 0.19088208675384521\n",
      "1 1455 0.18683834373950958\n",
      "1 1456 0.1531270295381546\n",
      "1 1457 0.14453846216201782\n",
      "1 1458 0.17131352424621582\n",
      "1 1459 0.14462950825691223\n",
      "1 1460 0.12826678156852722\n",
      "1 1461 0.12338526546955109\n",
      "1 1462 0.15462353825569153\n",
      "1 1463 0.1614144891500473\n",
      "1 1464 0.14138776063919067\n",
      "1 1465 0.12285377830266953\n",
      "1 1466 0.14009453356266022\n",
      "1 1467 0.14564171433448792\n",
      "1 1468 0.11364398151636124\n",
      "1 1469 0.1356450617313385\n",
      "1 1470 0.10010137408971786\n",
      "1 1471 0.11605104058980942\n",
      "1 1472 0.12668615579605103\n",
      "1 1473 0.11941266059875488\n",
      "1 1474 0.15283770859241486\n",
      "1 1475 0.11374469101428986\n",
      "1 1476 0.07539494335651398\n",
      "1 1477 0.13953270018100739\n",
      "1 1478 0.1183672547340393\n",
      "1 1479 0.09518557041883469\n",
      "1 1480 0.12341152131557465\n",
      "1 1481 0.12864509224891663\n",
      "1 1482 0.08746268600225449\n",
      "1 1483 0.17081108689308167\n",
      "1 1484 0.08529689162969589\n",
      "1 1485 0.14554467797279358\n",
      "1 1486 0.1263430416584015\n",
      "1 1487 0.19382090866565704\n"
     ]
    }
   ],
   "source": [
    "for e in range(1, epochs + 1):\n",
    "    train(e)\n",
    "    test(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}