Basic neural network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from torch import nn\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raw implementation of NN in numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BP thinking - for any equation, the gradient of inputs is product of gradients of\n",
    "# the loss with respect to output (how the loss changes with respect to output)\n",
    "# the output with respect to inputs (hot the output changes with respect to inputs)\n",
    "\n",
    "# Nice videos for vectorizing code\n",
    "# https://www.youtube.com/watch?v=-l9BdZsopYI\n",
    "\n",
    "class Linear:\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        # output dim is k, input dim is d\n",
    "        self.W = np.random.randn(output_dim,input_dim) *0.01# kxd matrix\n",
    "        self.b = np.random.randn(output_dim,1) *0.01 # kx1 matrix\n",
    "        self.gradW = np.random.randn(output_dim,input_dim) # kxd matrix\n",
    "        self.gradb = np.random.randn(output_dim)\n",
    "        \n",
    "    # x is 1xd single example\n",
    "    def forward(self, x):\n",
    "        # z = XWT + b\n",
    "        output = np.dot(self.W,x) + self.b\n",
    "        return output # nxk matrix\n",
    "    \n",
    "    def backward(self, x, prevGrad):\n",
    "        # this can be thought as three separate groups of variables in computation graph\n",
    "        # 1. the inputs x\n",
    "        # 2. the weights w\n",
    "        # 3. the bias b\n",
    "        # z = XWT + b\n",
    "        \n",
    "        # dL/dW = dz/dW * dL/dz\n",
    "        # this is a kxd matrix\n",
    "        self.gradW = np.dot(prevGrad, x.T)\n",
    "        \n",
    "        # dL/db = dz/db * dL/dz\n",
    "        # this is a kx1 matrix\n",
    "        self.gradb = np.copy(prevGrad)\n",
    "        \n",
    "        # this needs to be returned; as this is the value comes from previous layer\n",
    "        # dL/dX = dz/dX * dL/dz\n",
    "        # this is a dx1 matrix\n",
    "        return np.dot(self.W.T ,prevGrad)\n",
    "        \n",
    "        \n",
    "class Sigmoid:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def forward(self, x):\n",
    "        output = 1/(1+np.exp(-x))\n",
    "        return output\n",
    "    \n",
    "    def backward(self, x, prevGrad):\n",
    "        sig = 1/(1+np.exp(-x))\n",
    "        return np.multiply(np.multiply(sig, (1 - sig)), prevGrad)\n",
    "        \n",
    "class MSE:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def forward(self,prediction, label):\n",
    "        output = np.mean(np.square((prediction-label)))\n",
    "        return output\n",
    "    \n",
    "    def backward(self, prediction, label):\n",
    "        return np.mean(2*(prediction - label))\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x116fda990>]"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEbhJREFUeJzt3WGsZGV9x/Hfj12IghCRXQWBZSEQ\nWiQxyKTZSmJATANKpC9qohFDqWTfUERrY2mTSuIrX1gjL0yTmxWhcUNDkERDqoVQiWlSNtwLJAJb\nIllZWF3dK1A1xmbZ7L8vZrYO48yduec85zxznvP9JGZ3LyPnmQC/+Z///J/nOCIEAOi+k3IvAACQ\nBoEOAIUg0AGgEAQ6ABSCQAeAQhDoAFAIAh0ACkGgA0AhCHQAKMTWNi+2bdu22LlzZ5uXBIDOW1tb\n+2VEbJ/3ulYDfefOnVpdXW3zkgDQebYPLvI6Wi4AUAgCHQAKQaADQCEIdAAoBIEOAIUg0AGgEAQ6\ngF5aO/i6vv6DF7V28PXcS0mm1Tl0AFgGawdf1yf3PKGjx47rlK0nae+tu3TlBWfmXlZtVOgAeueJ\nA6/q6LHjOh7SG8eO64kDr+ZeUhIEOoDe2XXRWTpl60naYunkrSdp10Vn5V5SErRcAPTOlRecqb23\n7tITB17VrovO0pUXnKm1g6+/6c9dRKAD6KUrLzjz/4O7lJ46LRcAvVdKT51AB9ALG40pltJTp+UC\noHjzWirTeuopr91Wb55AB1C8aS2VyXAd76mn0nZvnpYLgOLlaqm03ZunQgdQvCZbKhs58UHyxrHj\nrXyQOCIavcC4wWAQPIIOQJ+k6KHbXouIwbzXUaEDwAwpwriJ3vwsBDoATNHFzUZ8KQoAU3RxsxGB\nDgBTdHGzES0XAJgi12RMHQQ6AMzQ5heaKdByAYBCEOgAUIi5gW77HttHbD879rN32H7U9o9Hv3bn\nngRA8Up8APQiFqnQ75V03cTP7pT0WERcIumx0Z8BILsT8+P/9MgL+uSeJ1oP9ZwfJnO/FI2IH9re\nOfHjGyVdPfr9fZIel/R3CdcFAJUscrJiU3JvRqraQ39XRByWpNGv75z1Qtu7ba/aXl1fX694OQBY\nTJPz4/Oq79ybkRofW4yIFUkr0vBwrqavB6DfmpofX6T6bvt0xUlVA/0Xts+JiMO2z5F0JOWiAHTf\ntIOt2np6TxPz44s+JCPnZqSqgf5dSTdL+vLo1+8kWxGAzptWzUrq3GFX42ZV35MfUjk3I80NdNv3\na/gF6DbbhyTdpWGQP2D705JelvSxJhcJoFtm9ZJzfVmZwrTqO/eXoJMWmXL5xIy/dG3itQAoxKxq\nNmd/OYXJ6jvnRM00nOUCILlZveSuHXY1T+4vQSfxCDoAqKGNL3p5BB0AtGCZTmTkcC4AKASBDiCb\nvh6i1RRaLgCS2GwvedlG/kpAoAOorUo4L9vIXwlouQCorcqhVCkP0aJ1M0SFDqC2KvPYqc49oXXz\newQ6gNqqhnOKkT9aN79HoANIItc89rLt1syJQAfQabmPrF0mBDqAzlum3Zo5MeUCAIUg0AGgEAQ6\ngKXBPHk99NABLAXmyeujQgewFKrsNsWbEegAlkLKowD6ipYLUJA2np7TlEXnybv8HptGoAOFKKEH\nPW+evIT32CRaLkAh2uxB55pGoc++MSp0oMPG2w9tnWnSdpWc4z12FYEOdNS0YG3jTJM2TzfM9R67\nikAHOmpasN52zcWNh1ybVXKu99hVBDrQUYsEaxMTIW2ebkiLZXMcEa1dbDAYxOrqamvXA0q3UWCX\nMhHCmKJkey0iBvNeR4UOdNhGY36lPMmHo3EXx9giUCh2XvZPrQrd9uck3SopJP1I0i0R8b8pFgag\nHp7k0z+VA932uZI+I+myiPid7QckfVzSvYnWBqAm2hX9UrflslXSW21vlXSqpJ/VXxIAoIrKgR4R\nP5X0FUkvSzos6VcR8UiqhQEANqdyoNs+U9KNki6U9G5Jp9m+acrrdttetb26vr5efaUAgA3Vabl8\nSNJPImI9It6Q9JCk90++KCJWImIQEYPt27fXuBzQbzyeDfPUmXJ5WdIu26dK+p2kayWxawhoQCmb\nhNCsOj30fZIelPSUhiOLJ0laSbQuAGNSHhtbpdLn7qAbas2hR8Rdku5KtBYAM6Q606RKpc/dQXew\n9R/ogFSbhKocB1DKEQJ9QKADHZFik1CVSp8TD7uD0xaBnqlyeiEnHubFaYsApqpS6XOEQDdw2iIA\nFIJABzaB8T0sM1ouwIIY38Oyo0IHFpRyc88ymXbXwZ1IN1GhAwsqcXxv2l2HJO5EOopABxY0a3NP\nl0f6Zt11sJGomwh0YBMmx/e63lefdddR2p1IXxDoQA1d3xY/666DZ5F2E4EObGBeO6WEvvq0TUNs\nJOomAh2YYZF2SqpDs2ZdnyoZm0GgAzMs2k5poprtem8eeTCHDsxwop2yxWq9nVLqzDuaRYUOzNBk\nO2WeEnrzaB/H5wJLih46TuD4XKDjmDTBZtFDBxLjHBTkQoUOJMR0CnKiQgcSYjoFORHoQEI5Rx0B\nWi5AQjlHHQECHUiM6RTkQssFAApBoAMjjBui62i5AMo/bsiuUKRAoAPK+6CK3B8mKActF0DNjhvO\na+Uwu45UalXott8uaY+kyyWFpL+KiP9KsTCgTU2NGy5SfXOyIlKp23K5W9L3I+IvbJ8i6dQEa0LP\nTfaT2+ovNzFuuEgrh9l1pFI50G2fIekDkv5SkiLiqKSjaZaFvpqsaL94w3v0pYef62x/edHqm9l1\npFCnQr9I0rqkb9p+r6Q1SXdExG+TrAy9NFnRfu/Zw9m+rExhVvXNVAuaUCfQt0p6n6TbI2Kf7bsl\n3SnpH8dfZHu3pN2StGPHjhqXQx9MVrTXX36OnnzptU73lyerb6Za0JQ6gX5I0qGI2Df684MaBvqb\nRMSKpBVp+MSiGtdDD0yraC89+/SiqtmcI5IoW+VAj4if237F9qUR8YKkayU9n25p6KvJira0/jJT\nLWhK3SmX2yXtHU24HJB0S/0lAWVjqgVNqRXoEfGMpLkPLgXwZqXddWA5sFMUncMhWsB0nOWCrDY7\nvseECDAbgY5sqoQzEyLAbLRckE2VQ6lSHqJF6waloUJHNlXG91JNiNC6QYkIdGRTNZxTTIjQukGJ\nCHRklWt8j809KBGBjl5icw9KRKCjt9jcg9Iw5QIAhSDQAaAQBDqKwEw5QA8dBWCmHBiiQkfnVdlx\nCpSIQEfnpTwOAOgyWi7o/AOLF50p7/r7BOYh0HuulP7zvJnyUt4nsBFaLj3Xdv851zQKfXb0ARV6\nD423Hto806TNKnmyvcLZLegDAr1npoVqW2eatHXC4awPDs5uQekI9J6ZFqq3XXNxKwHXVpU864OD\ns1tQOgK9ZxYN1SYmQtqqkmmvoK8cEa1dbDAYxOrqamvXw3TzwrqEiRBGFFES22sRMZj3Oir0HprX\neijhaT60V9BHjC3iD7DzEugmKnT8ASZCgG4i0DEVLQuge2i5AEAhCHQAKASBDgCFqB3otrfYftr2\nwykWhLR4NBvQHym+FL1D0n5JZyT4eyGhEjYIAVhcrQrd9nmSPiJpT5rlIKWUR8ZWqfS5OwDaVbdC\n/5qkL0g6PcFakFiqM02qVPrcHQDtq1yh275B0pGIWJvzut22V22vrq+vV70cKjixQehv/uzSWoFa\npdLngRJA++pU6FdJ+qjtD0t6i6QzbH8rIm4af1FErEhakYaHc9W4HipIsUGoSqXPiYdA+5Kctmj7\nakl/GxE3bPQ6TlvsriqnF3LiIZAGpy0iqSqVPscHAO1KEugR8bikx1P8vQAA1bBTtCWM8AFoGi2X\nFjDCB6ANVOgtKHGEb9odB3chQF5U6C0obYRv2h2HJO5CgMwI9BZMewJQl0f6Zt1xdP05pEDXEegt\nGR/h63pPfdYdR0l3IUAXEegZTKtwuxTos545ynNIgbwI9AbMa6eU0FOftmmIjURAXgR6You0U2ZV\nuKmuT5UM9BOBntii7ZQmqtmu9+YB1MMcemIn2ilbrNbbKSXOuwNYHBV6Yk22U+YpoTcPoLokx+cu\niuNzm0cPHSgPx+f2FJMmQH/RQ18SnIMCoC4q9CXAdAqAFKjQlwDTKQBSINCXQM5RRwDloOWyBHKO\nOgIoB4G+JJhOAVAXLRcAKASBngAjhwCWAS2XmnKPHLIzFMAJBHpNOR9WkfvDBMByoeVSU5Mjh/Na\nOcyvAxhHhV5TUyOHi1TfnK4IYFwxgT7ZS26zt9zEyOEirRzm1wGMKyLQJ6vZL97wHn3p4ec63Vte\ntPpmfh3ACUUE+mQ1+71nD2f7ojKVadU3Ey0ANlJEoE9Ws9dffo6efOm1zveWx6tvJloAzFM50G2f\nL+lfJJ0t6biklYi4O9XCNmNaNXvp2acXVc3mHI8E0A11KvRjkj4fEU/ZPl3Smu1HI+L5RGvblMle\ncmm9ZSZaAMxTOdAj4rCkw6Pf/8b2fknnSsoS6KVjogXAPEl66LZ3SrpC0r4pf223pN2StGPHjhSX\n663S7joApFV7p6jtt0n6tqTPRsSvJ/96RKxExCAiBtu3b697OQDADLUC3fbJGob53oh4KM2S5qt6\nuiGnIgIoWZ0pF0v6hqT9EfHVdEvaWNXxPcb+AJSuToV+laRPSfqg7WdG//twonXNVPVAqlQHWVHl\nA1hWdaZc/lOSE65lIVXH91KM/VHlA1hmndspWnV8L8XYH5t7ACyzzgW6VH18r+7YH5t7ACyzTgZ6\nLmzuAbDMCPRNYnMPgGXFI+gAoBAEOgAUgkAHgEL0OtDZJASgJL39UpRNQgBK09sKPdVRAACwLHob\n6Cc2CW2xNtwkRFsGQFf0tuWyyCYh2jIAuqQTgb528PVGdmfO2yTE2S0AumTpA73tKnn8w4OzWwB0\nydIHeptV8rQPD85uAdAVSx/obVbJ0z48brvmYoIcQCcsfaC3ecIhLRYAXeaIaO1ig8EgVldXW7te\nFU19AQsAVdlei4jBvNctfYXeNo7HBdBVvd1YBAClIdABoBAEOgAUgkAHgEIQ6ABQCAIdAArR6hy6\n7XVJByv+37dJ+mXC5XQB77kfeM/lq/t+L4iI7fNe1Gqg12F7dZHB+pLwnvuB91y+tt4vLRcAKASB\nDgCF6FKgr+ReQAa8537gPZevlffbmR46AGBjXarQAQAb6ESg277O9gu2X7R9Z+71NMn2+bZ/YHu/\n7eds35F7TW2xvcX207Yfzr2WNth+u+0Hbf/36J/3n+ZeU9Nsf2707/Wztu+3/Zbca0rN9j22j9h+\nduxn77D9qO0fj35t5EjXpQ9021skfV3S9ZIuk/QJ25flXVWjjkn6fET8saRdkm4r/P2Ou0PS/tyL\naNHdkr4fEX8k6b0q/L3bPlfSZyQNIuJySVskfTzvqhpxr6TrJn52p6THIuISSY+N/pzc0ge6pD+R\n9GJEHIiIo5L+VdKNmdfUmIg4HBFPjX7/Gw3/Iz8376qaZ/s8SR+RtCf3Wtpg+wxJH5D0DUmKiKMR\n8T95V9WKrZLeanurpFMl/SzzepKLiB9Kem3ixzdKum/0+/sk/XkT1+5CoJ8r6ZWxPx9SDwJOkmzv\nlHSFpH15V9KKr0n6gqTjuRfSkoskrUv65qjNtMf2abkX1aSI+Kmkr0h6WdJhSb+KiEfyrqo174qI\nw9KwaJP0ziYu0oVA95SfFT+aY/ttkr4t6bMR8evc62mS7RskHYmItdxradFWSe+T9M8RcYWk36qh\n2/BlMeob3yjpQknvlnSa7ZvyrqosXQj0Q5LOH/vzeSrwNm2c7ZM1DPO9EfFQ7vW04CpJH7X9koYt\ntQ/a/lbeJTXukKRDEXHi7utBDQO+ZB+S9JOIWI+INyQ9JOn9mdfUll/YPkeSRr8eaeIiXQj0JyVd\nYvtC26do+CXKdzOvqTG2rWFfdX9EfDX3etoQEX8fEedFxE4N//n+R0QUXblFxM8lvWL70tGPrpX0\nfMYlteFlSbtsnzr69/xaFf5F8JjvSrp59PubJX2niYss/UOiI+KY7b+W9O8afit+T0Q8l3lZTbpK\n0qck/cj2M6Of/UNE/FvGNaEZt0vaOypUDki6JfN6GhUR+2w/KOkpDae5nlaBO0Zt3y/paknbbB+S\ndJekL0t6wPanNfxg+1gj12anKACUoQstFwDAAgh0ACgEgQ4AhSDQAaAQBDoAFIJAB4BCEOgAUAgC\nHQAK8X+/JtbiT8e2cgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1176a7690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def createData():\n",
    "    x = np.linspace(0,10,70)\n",
    "    y = np.sin(5*x) + np.random.randn()*5 + x\n",
    "    return (x,y)\n",
    "## Create data\n",
    "# X is 1xd thing\n",
    "(X,y) = createData()\n",
    "plt.plot(X,y,'.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "n = len(X)\n",
    "d = 1\n",
    "o = 1\n",
    "h = 3\n",
    "\n",
    "Xi = X[1]\n",
    "yi = y[1]\n",
    "lin1 = Linear(d,h)\n",
    "sig = Sigmoid()\n",
    "lin2 = Linear(h,o)\n",
    "mse = MSE()\n",
    "\n",
    "z = lin1.forward(Xi) # kx1 matrix\n",
    "a = sig.forward(z)\n",
    "yhat = lin2.forward(a)\n",
    "loss = mse.forward(yhat,yi)\n",
    "\n",
    "grad1 = mse.backward(yhat,yi)\n",
    "grad2 = lin2.backward(a, grad1)\n",
    "grad3 = sig.backward(z, grad2)\n",
    "grad4 = lin1.backward(Xi, grad3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.0029194 ],\n",
       "       [ 0.00149857],\n",
       "       [ 0.01011252]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.0004231 ],\n",
       "       [ 0.00021718],\n",
       "       [ 0.00146558]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(lin1.gradb)\n",
    "display(lin1.gradW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ep = 0.00001\n",
    "for i in range (0, lin1.gradW.shape[0]):\n",
    "    for j in range(0, lin2.gradW.shape[1]):\n",
    "        loss = mse.forward(lin2.forward(a = sig.forward(lin1.forward(Xi))))\n",
    "        new_weight = np.copy(lin1.we)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stochatic gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0 = 0.627220\n",
      "Iteration 50 = 1.707527\n",
      "Iteration 100 = 1.404327\n",
      "Iteration 150 = 1.132808\n",
      "Iteration 200 = 0.904998\n",
      "Iteration 250 = 0.733552\n",
      "Iteration 300 = 0.619262\n",
      "Iteration 350 = 0.551203\n",
      "Iteration 400 = 0.514249\n",
      "Iteration 450 = 0.495494\n",
      "Iteration 500 = 0.486373\n",
      "Iteration 550 = 0.482028\n",
      "Iteration 600 = 0.479964\n",
      "Iteration 650 = 0.478973\n",
      "Iteration 700 = 0.478488\n",
      "Iteration 750 = 0.478247\n",
      "Iteration 800 = 0.478128\n",
      "Iteration 850 = 0.478073\n",
      "Iteration 900 = 0.478051\n",
      "Iteration 950 = 0.478049\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11a19d0d0>]"
      ]
     },
     "execution_count": 371,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF25JREFUeJzt3X+IpXd1x/HPyWyummhQJ2o1cbNa\nxCoZinYoebpgLm7/sLqYwtSioLE2ZYptrSuCzRRK/ghl/UNkQlqEJRoNBlvZCSglWGXMbWIZls4m\nwqqpKNbE1dWs21al2N5kPf3j3s3eeea583zn+f3j/YIwO3fvzPO9JDn33PM93/OYuwsA0H5X1L0A\nAEAxCOgA0BEEdADoCAI6AHQEAR0AOoKADgAdQUAHgI4goANARxDQAaAjDlR5sWuvvdYPHTpU5SUB\noPVOnz79U3d/SdrzKg3ohw4d0vb2dpWXBIDWM7MnQp5HyQUAOoKADgAdQUAHgI4goANARxDQAaAj\nCOgA0BEEdAD9tLUlHT8++doRlfahA0AjbG1JR45I47E0GEibm1IUJT5tNJKGw8S/bhwCOoD+GY0m\nwfzixcnX0WhXxA6M+Y1CyQVA/wyHkyi9sDD5OhzuekpSzG86MnQA/RNFk5R7pp6ydeKMRhsXNFxZ\nVLS69GzMv5ShJ8T8xiGgA+inKHq2hrJ14oyO/Omva6zXafDlsTZ1RtHqUjzmN76mTkAH0HujjQsa\n63W6qAMayzXauKBodUfMb0VNnRo6gH7Yo01xuLKogcZa0NMa6GkNVxZ3PacNNXUydADdl5JeR6tL\n2tTOGnpc5pp6hXUaAjqA7gtoU4xWlxStzv8VCfuo6Squ0xDQAXRfQS0rszX1IAFvJEUioAPovkzp\ndQEq7n0koAPoh32n12H2LJFX/EZCQAeAeVI2NINK5CW9kSQhoAPopdTmk4BoXXGJPBUBHUDvBGXW\nAdG6aeMBCOgAeicosw6I1nXttc5DQAfQO0GZdWC0jpfI46WcKue/ENAB9E5wZr3PDc14KWd9XTp2\nrLr5LwR0AL1URvNJvJSzsVHtpinDuQCgIPH7ZqyspN5Ho1CpGbqZfUrSUUlPufuN08deLOkfJR2S\n9H1Jf+ju/1XeMgFgH2oaXJ5UyllaalYN/dOS/k7SfTOP3S5p090/ama3T7//q+KXBwD7VPPg8khb\nijSSNJQUVXmuKD2gu/vDZnYo9vAtmqxWkj4jaSQCOoAmqPO0T81vJllr6C9z93OSNP360nlPNLNV\nM9s2s+3z589nvBwABAq4AXRme9wkQ1Ltd8EovcvF3U9IOiFJy8vLXvb1APRcWad9QrLvmo+OZg3o\nPzGzl7v7OTN7uaSnilwUgA5I2piMPVba3mVC4Tr3tUJKOTUfHc0a0L8o6b2SPjr9+oXCVgSg/ZKy\nWWnHY1vrp3Tk2FIl5eZCStvzsu/4O0WVu6AxIW2Ln9NkA/RaMzsr6Q5NAvnnzew2SU9KekeZiwTQ\nMvNqyTOPjTYuJD6ljOS2kH3SpOy75k3QuJAul3fN+asjBa8FQFfMy2ZnHhuuLGrwyOWnLC6WFxsL\nK23Hs++Gzc/l6D+A4s2rJc88FkVL2pw5dFNmbCyttN2w+bkEdADlSNqYVKSRoumRm91PKTM2llLa\nbtj8XAI6gEqklZsbFhvD1bgJGkdAB1CJ0K6/hsTGVmLaIoBKJB7gTDt5GfYUTJGhAyhGysmdXSUV\npbf8NawrsPEI6ADyC4y8O0oqx0epNZiGdQU2HiUXAPllGUoVMEQreM5WrC7T1zINGTqA/LL0Ywe0\ntQR1vsQ+HVQ5UqBpCOgA8svacxjQ1pL6lNing6SRAgR0ANiPinoOd+29xj4dxEcK1Hx4s1IEdACt\nkbz3Gu05UqAv2blEQAdQkjJmnc/rekkbKdAXBHQAhSurfzxp75Ve9ctoWwRQuLJurXmpunLnnZcD\nd8238WwUMnQAhcs8VTagThMvpzRsgm2tCOgACpepizFj7aS1UxpLQEAHUIp9b0zmOOff103QOGro\nAJoh+Jw/5iFDB7qkjF7BqoTWTtr8GktGQAe6ogv9e2m1ky68xhJRcgG6IqEGnTR1sJBJhHWNM6RH\ncU9k6ECbzZYfYv17W4tHdyWzUgEJbtVZ8h6vkTr7TgR0oK2SAutMDXo0WkpMZnNPIqzyrhMpr5Fy\ny04EdKCtkgLr2tqzQW6o5GQ2d4JbZZac8hqxEwEdaKuUwBpF0ub6GY02Lmi4sqgoWpJUQII7pxul\nlOYTSiz7Yu5e2cWWl5d9e3u7susBnbdXFA2sdRcRiEstq9OmKDM77e7Lac8jQwfabK82v4Bad1GB\nuNSyOsdAg9G2CHRVwMnLoroAOeTZDLkydDP7kKQ/keSSzkh6n7v/bxELA7B/O6sT6ScviypRMyCr\nGTLX0M3sOklfk/R6d/+lmX1e0oPu/ul5P0MNHShP1vIJJermq6qGfkDS88zsaUlXSfpRzt8HIKOs\ndWxK1N2RuYbu7j+U9DFJT0o6J+ln7v7lohYGYH+oYyNzQDezF0m6RdKrJL1C0tVm9u6E562a2baZ\nbZ8/fz77SgHsKen2bFnVNaoF+eSpob9D0lvc/bbp97dKusnd/2zez1BDB3KoqNgdUoun7l6tKmro\nT0q6ycyukvRLSUckEa2BMlQ4ECutFs8E2+bKU0M/JemkpEc1aVm8QtKJgtYFYFaRY2NT6imJtfiZ\nn2GCbXPl6nJx9zsk3VHQWgDMU1TDeEB6vaunXFvaGq5p9PRhDa9c0/DuuzUYLDFepYE4+g+0QVEn\ndwJ7G2dbGbfe/x0dGT+osQYajMfafOykNjeXqKE3EAEdaIsiGsYzZPoj3ayxBrqoAxrLNdLNWqN3\nvZEI6ECfZMj0h7feoMG9FzUeX9RgcIWGt95Q+jKRDQEdaLFM7YP7zPSjSNp8aIESSwsQ0IGWqrJ9\nkPEA7cD4XGA/GnSEkvZBxJGhA6EadqKGu7MhjgwdCBWYEleVxBc2uyVpwQ36JIJwZOhAqICUuOok\nPndtO2nBUqM+iSAcAR0INa/lb6bVZDSKyru3Zhnmfepo1YvAJQR0YD/iKfFW+rH4Rk8mnPepg+J8\nKxHQgRy27tv7WLzU8OrFvE8d3CC0lQjowF5S0uu0Y/HHj7egepFUiKfxvJUI6MA8ATucacfic7UW\nNrpWgyYioAPzBEwmTDsWn3lIYsN63tEOBHRgnsD0Oq06kal6kfBmsqWIhB17IqAD8xQ1gzyL2JvJ\n1uJREnakIqADeylpczC1PB57MxmNlpq/uYraEdCBigWXx2feTIaiNRzpmOUC7EPQiJOUJ2WZkljY\n3BZ0Ghk6ECgosw5pdRxmy7ZpDUcaMnQgUFBmHfAksm2UhQwdCBSUWWdsdUzaJOVcEfaLgA4ECupi\nzNDqyARbFIWADuxDUB17n8VuJtiiKAR0oGZMsEVRCOjAJTUVrZlgi6IQ0AGp9mFY0WRSiyZHiCbX\npU0R+0VAB6SgyYqlYbIiCkIfOiBdLmQvLBRftE47Xprl6CiQIFeGbmYvlHSPpBsluaQ/dve9DkUD\nzVTWZMWQ7DvXXTCAy/KWXO6S9CV3/wMzG0i6qoA1oe/im5Ox70vbu0woWue+Vkgpp84xveiUzAHd\nzK6R9CZJfyRJ7j6WNC5mWeiteEa7vi4dO3Z5Lvj6KR05tlRJubmQ0nZo9s0OKAqQp4b+aknnJd1r\nZo+Z2T1mdnVB60JfxTPajY0d3482LiSWm4OmIOZcSqbS9rzBLWUsGL2Xp+RyQNIbJX3A3U+Z2V2S\nbpf0N7NPMrNVSauSdPDgwRyXQy/EM9qVFemRR579friyqMEjOxPesppECittJw1uoasFJcgT0M9K\nOuvup6bfn9QkoO/g7icknZCk5eVlz3E99EFSPXlp6dnvo2hJm0s7//r48XI6DksrbdfZIolOyxzQ\n3f3HZvYDM3utu39b0hFJ3ypuaeitWEY7OXITPXvkJp7wltkkUkppm64WlCRvl8sHJN0/7XD5nqT3\n5V8ScFlIdaJ1TSKtWzDaIldAd/evS1ouaC3ALqHVidY1ibRuwWgDToqi0RIPcAZ0iNBEgj5ilgvq\nlXJyZ1d1Quk1GJpI0FcEdNQnMPLuqE4cH6XWYGgiQV9RckF9spzcCRiiFTxnK1aXoUyDtiNDR32y\ntO8FdIgENZHEPh1UOVIAKAsBHfXJ2r4X0CGS+pTYp4OkkQIEdLQNAR31qrB9b8f+a+zTQdJIAaBt\nCOjohd37r5GimU8HSSMFgLYhoKNxyph3nrj/Otx7pADQNgR0NEpVkxMXF+lVR/fQtohGKev2mvGx\n5BcucBtPdA8ZOhol8yDCgDpNvKTCwEN0DQEdjZKpkzFDnYaBh+giAjoaZ9+bkxnP+rMJiq6hho72\nCz7rD3QbGTrK6ROsUmj9pO2vE0hBQO+7rsyaTaufdOV1Anug5NJ3c/oE45MHC5tEWNdIw7L6IYEG\nIUPvo9nSQ0KfYDyZXV+Xjh0rILmtMkuOl1e4MTN6gIDeN0lBNVZ/Hh3fmcxubBR0w4iq7jwx742D\nPkV0HAG9b5KC6trajgAXT2ZXVqRHiphEWFWWPO+Ngz5FdBwBvW8CgmoUSZvrZzTauKDhyqKi1SUt\nFTGJMCFLLqXxhPIKesrcvbKLLS8v+/b2dmXXwxxpUTSg1l1EIC61pE6LIjrEzE67+3La88jQ+yit\n9JBS6y4qEJdaUqe8gh6ibRG7pZy8LKoDkAOeQLHI0LG7OpHSEVJUiZrGE6BY1NB7Lmv5hBI1UB1q\n6AiStY5NiRpoHmroPUcdG+gOMvSeK7KOTRkGqFfugG5mC5K2Jf3Q3Y/mXxIKleHWbFkvE1KLJ+gD\n5SkiQ/+gpMclXVPA70KRKhyGFVKLZ4ItUK5cNXQzu17S2yTdU8xyUKgiR8amjL1NrMXHfoYJtkC5\n8mbo65I+IukFBawFRSuqYTwgtd5Vi9eWtoZrGj19WMMr1xSNjms4jBixApQoc0A3s6OSnnL302Y2\n3ON5q5JWJengwYNZL4csitrxDOxtnK3Fb73/OzoyflBjDTQYj7V530lFn4g4SASUKE+GfljS283s\nrZKeK+kaM/usu7979knufkLSCWlysCjH9ZBFETueGTL9kW7WWANd1AGN5RrpZl06hEogB8qRuYbu\n7mvufr27H5L0TklfjQdzdMSlTP/OO4N3Moe33qDBc0wLdlGD51yh4a03VLBQoN/oQ++hTK2D+0yt\no0jafGiB8gpQoUICuruPJI2K+F0oV5Wtg5RXgGpx9L8qdd3tPobWQaC7KLlUoUEnarg7G9BdZOhV\nCEiLq0rgM+xvJktacEM+hQB9RYZehZS0uOoEPndtO2nBUmM+hQB9RUCvQtIBn5lWk9EoKu/emmWY\n94mjVS8C6B4CelV2HKPceSx+ePfdGgyWdiXwjZ1MOO8TB8V5oFYE9Bps3Rc7Fv/YSW1uLu1K4Btb\nwZg3UoBz/UCtCOhlSEmtk47Fr8Xq2llvDVeZpEI8jedArQjoRQtIrYe33qDBvRc1Hl/UYJB8LD5z\ne2Fj6zQAykZAL1pAah1yLD7ToMRG12kAlI2AXrTA1DqkOrHvCkbCm8mWIhJ2oCcI6EUr8q7L+xV7\nM9laPErCDvQIAb0MJW0OppbHY28mo9FSszdWARSKgN4SweXxmTeToWgNB/qEWS4VSR1zkvKELFMS\nC5vbAqAVyNArkJpdh7Q6DrNl27SGA/1Bhl6B1Ow6IP0m2waQhgy9AqnZdY5Wx/hGKeeKgP4ioFcg\ntZMxY6tjvFKzvi4dO0abItBXBPSKpNayMxS745WajY2Gz38BUCpq6C12qVKzsDD5urKy83vaFIF+\nIUMvQk2F66RKzdISNXSgrwjoedU8ECuaTGvR5BhRRJsi0GME9LzqHFzOdEUAM6ih5xUvZBdZuE47\nXprl+CiAzupVhl5Kqbus6Yoh2Xfmu2AA6KLuBPSUEzalVidihetC3jhCSjl1juoF0DjdCOgBJ2xG\noyixOlFHYh0kNPtmFxTAVDcCesAJm+Ew2hEfFxfLydgL2yNNyr451w9gD90I6PFsdmVFW6P/0+hX\nhzVc+FdFw+Gu+FhWc0qhZe3Z7JuOFgApMgd0M3ulpPsk/ZqkX0k64e53FbWwfYlF6y1FOmK3aSzT\nwFybWlCk3dWJMvYTSytr19keCaAV8mToz0j6sLs/amYvkHTazL7i7t8qaG37MxOtR8el8TMLuujS\n+Jnq9xNLKWvT0QIgReaA7u7nJJ2b/vkXZva4pOsk1RPQZ3RyP5GOFgApzN3z/xKzQ5IelnSju/88\n9nerklYl6eDBg7/1xBNP5L5eCPYPAXSFmZ129+XU5+UN6Gb2fEn/Iulv3f2BvZ67vLzs29vbua5X\nNt4IADRNaEDP1eViZldK2pB0f1owL1TWqJvyczSSAGizPF0uJumTkh53948Xt6QUWaNuwM/RSAKg\nzfIM5zos6T2S3mxmX5/+89aC1jXfnIFUaXOsQgZZBc3ZSrhQ6rUBoAJ5uly+JskKXEuYhBaWoKQ9\noPUltZEk4UJbiijTAGiE9p0UTYi6o+MBpZLAtr89WxkTsvyRds+IIaADqEP7Arq0K+oGn7nJ0Hi+\nYx814UJDcd4HQDO0M6DHVDeSPFIUu1AkzvsAaIZWBPSQLsUyTn0m7qMOI40UTe/gWd61AWC/Gh/Q\n6+wNj1dYyhq5CwBFaHxAn9dtWEWJo6qRuwBQhMYH9Lqz5CpG7gJAERof0EvNkvc5QoCBhwCarPEB\nXSopS85YnGcDFEBTtSKgzyosS6YgDqBjWhfQpYKy5NDTSMzTBdASrQzohQhJ9ZmnC6BF2hHQy8qS\n01J9yjIAWqT5Ab3qLHn2zYMbMwNokeYH9Cqz5KQ3D/oUAbRE8wN6lVly0pvH2hqBHEArND+gV3ma\nhxILgBZrfkCXqjvNw1FQAC3WjoBeJY6CAmipPDeJBgA0CAEdADqCgA4AHUFAB4COIKADQEcQ0AGg\nI8zdq7uY2XlJT2T88Wsl/bTA5bQBr7kfeM3dl/f13uDuL0l7UqUBPQ8z23b35brXUSVecz/wmruv\nqtdLyQUAOoKADgAd0aaAfqLuBdSA19wPvObuq+T1tqaGDgDYW5sydADAHloR0M3sLWb2bTP7rpnd\nXvd6ymRmrzSzh8zscTP7ppl9sO41VcXMFszsMTP7p7rXUgUze6GZnTSzf5/+++78mE8z+9D0v+tv\nmNnnzOy5da+paGb2KTN7ysy+MfPYi83sK2b2nenXF5Vx7cYHdDNbkPT3kn5P0uslvcvMXl/vqkr1\njKQPu/vrJN0k6c87/npnfVDS43UvokJ3SfqSu/+GpN9Ux1+7mV0n6S8lLbv7jZIWJL2z3lWV4tOS\n3hJ77HZJm+7+Gkmb0+8L1/iALum3JX3X3b/n7mNJ/yDplprXVBp3P+fuj07//AtN/ie/rt5Vlc/M\nrpf0Nkn31L2WKpjZNZLeJOmTkuTuY3f/73pXVYkDkp5nZgckXSXpRzWvp3Du/rCk/4w9fIukz0z/\n/BlJv1/GtdsQ0K+T9IOZ78+qBwFOkszskKQ3SDpV70oqsS7pI5J+VfdCKvJqSecl3TstM91jZlfX\nvagyufsPJX1M0pOSzkn6mbt/ud5VVeZl7n5OmiRtkl5axkXaENAt4bHOt+aY2fMlbUg65u4/r3s9\nZTKzo5KecvfTda+lQgckvVHSJ9z9DZL+RyV9DG+Kad34FkmvkvQKSVeb2bvrXVW3tCGgn5X0ypnv\nr1cHP6bNMrMrNQnm97v7A3WvpwKHJb3dzL6vSUntzWb22XqXVLqzks66+6VPXyc1CfBd9ruS/sPd\nz7v705IekPQ7Na+pKj8xs5dL0vTrU2VcpA0B/d8kvcbMXmVmA002Ub5Y85pKY2amSV31cXf/eN3r\nqYK7r7n79e5+SJN/v191905nbu7+Y0k/MLPXTh86IulbNS6pCk9KusnMrpr+d35EHd8InvFFSe+d\n/vm9kr5QxkUaf5Nod3/GzP5C0j9rsiv+KXf/Zs3LKtNhSe+RdMbMvj597K/d/cEa14RyfEDS/dNE\n5XuS3lfzekrl7qfM7KSkRzXp5npMHTwxamafkzSUdK2ZnZV0h6SPSvq8md2myRvbO0q5NidFAaAb\n2lByAQAEIKADQEcQ0AGgIwjoANARBHQA6AgCOgB0BAEdADqCgA4AHfH/rFJvJV8HY6UAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a2875d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "alpha = 0.0001\n",
    "epochs = 1000\n",
    "\n",
    "h=10000\n",
    "lin1 = Linear(d,h)\n",
    "sig = Sigmoid()\n",
    "lin2 = Linear(h,o)\n",
    "mse = MSE()\n",
    "\n",
    "for epoch in range(0,epochs):\n",
    "    loss = 0\n",
    "    Yhat = np.zeros(n)\n",
    "    for i in range(0, n):\n",
    "        Xi = X[i]\n",
    "        yi = y[i]\n",
    "\n",
    "        # forward propagation\n",
    "        z = lin1.forward(Xi)\n",
    "        a = sig.forward(z)\n",
    "        yhat = lin2.forward(a)\n",
    "        loss += mse.forward(yhat,yi)\n",
    "        Yhat[i] = yhat\n",
    "        \n",
    "        grad1 = mse.backward(yhat,yi)\n",
    "        grad2 = lin2.backward(a, grad1)\n",
    "        grad3 = sig.backward(z, grad2)\n",
    "        grad4 = lin1.backward(Xi, grad3)\n",
    "        \n",
    "        lin1.W = lin1.W - alpha*lin1.gradW\n",
    "        lin1.b = lin1.b - alpha*lin1.gradb\n",
    "        lin2.W = lin2.W - alpha*lin2.gradW\n",
    "        lin2.b = lin2.b - alpha*lin2.gradb\n",
    "\n",
    "    if epoch % 50 == 0:\n",
    "        print('Iteration %d = %.6f' % (epoch, loss / n))        \n",
    "        \n",
    "plt.plot(X,y,'.r')\n",
    "plt.plot(X,Yhat,'.b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sigmoid does not give a good solution with less nunmber of units.\n",
    "Also, notice how it cheated the loss by predicting +ve and -ve losses.\n",
    "Even I use more number of layers, i can improve a bit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rectified linear unit\n",
    "class ReLU:\n",
    "    def forward(self, x):\n",
    "        return np.multiply(np.greater(x,0),x)\n",
    "    \n",
    "    def backward(self, x, prevGrad):\n",
    "        inter = np.greater(x,0).astype(float)\n",
    "        return np.multiply(inter, prevGrad)\n",
    "        \n",
    "# Hyperbolic tangent.\n",
    "class Tanh:\n",
    "    def forward(self, x):\n",
    "        a = np.exp(x)\n",
    "        b = np.exp(-x)\n",
    "        tanh = (a-b)/(a+b)\n",
    "        return tanh\n",
    "    \n",
    "    def backward(self, x, prevGrad):\n",
    "        a = np.exp(x)\n",
    "        b = np.exp(-x)\n",
    "        tanh = (a-b)/(a+b)\n",
    "        return np.multiply((1-np.square(tanh)), prevGrad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0 = 5.066288\n",
      "Iteration 50 = 1.845312\n",
      "Iteration 100 = 0.530817\n",
      "Iteration 150 = 0.376960\n",
      "Iteration 200 = 0.376253\n",
      "Iteration 250 = 0.375795\n",
      "Iteration 300 = 0.374820\n",
      "Iteration 350 = 0.373465\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x110f40590>]"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGShJREFUeJzt3X+MZWV9x/H3d2f3KqBEHNZf4LJK\niMV400hvGo9b9er4R4tETEYTTXQptRn/aJG1NpZp0vAHace0xixJG9MNorsRbQxDqqHGQm45OJBb\n2lkgGZUaRAusLLJOWzUN3TPMfPvHzMLdu+fOPXPu+XHPuZ9XYmbvcHfOcxU/55nv832eY+6OiIhU\n366yByAiItlQoIuI1IQCXUSkJhToIiI1oUAXEakJBbqISE0o0EVEakKBLiJSEwp0EZGa2F3kxS6+\n+GLfv39/kZcUEam848eP/8Ld9w57X6GBvn//fpaXl4u8pIhI5ZnZk0nep5KLiEhNKNBFRGpCgS4i\nUhMKdBGRmlCgi4jUhAJdRKQmFOgiIkC3CwsLm1+rqtA+dBGRcdHtQhhCu735emYGoggaDeh0IAjK\nHF06CnQRmTjdLsy8d50oMhoN57rrp4giWF/fDPUwrGagq+QiIhMnPPYk0Wln3XcRnd6AZ0/SaMDU\n1OYM/cysvWoU6CIycdrcT4OIKdZosMbB191LpwO33FLdcguo5CIiEyg4eAWd268mXDtAe8+DBAcX\nIKhukJ+hQBeRyRMEBOECQRhCe2EzyXtXSSua7Ap0EZlMQc+UvNutRZuLaugiImHIOW0uFaRAF5HJ\nsN3OoXabOrS5qOQiIvU3rKQSBJvfy6OGXmBtXoEuIvUXhnRPX0W48S7ap5c2F0P7wzU4t81l5Cwu\nuDavQBeR2utOX8PMxo1ENGhsRHSmn2BYrGaSxXG1+RwDXTV0Eam9cLVJtOs81tlNtOs8wtXm8L8T\nZrBOWnBtXjN0Eam9dhsaL7Ot2bYlytUzWXxmhp4qi/Oszccwd8/1Ar1arZYvLy8Xdj0RkTPS1MO7\nR1YIF1dpz04TzA2f1efFzI67e2vY+zRDF5GJELPmub1ul+DQDEEUwVIDmuO/2Ug1dBGROBXcbKRA\nFxGJU8HNRiq5iIjEiVnQHPfzuxToIiKD9BTeq3B+l0ouIiIJVKGkPjTQzex2M3vOzL7f871Xm9m9\nZvb41teL8h2miEi50pbUtzsTLGtJSi5fBf4WONbzvZuAjrt/3sxu2nr9Z9kPT0TqpLAadA4XSrpH\nqLd3nWaz0DLN0EB39++Z2f6+b18LtLf+fBQIUaCLyDYKq0HneKFhvezdIyvMfOpyIq6kcU/EdR96\nlih6XVFHuaSuob/W3U8CbH19TXZDEpE6KqwGneeFhtRPwsVVIhqbZ8awB555ptDOx9y7XMxsDpgD\n2LdvX96XE5ExlcnZKGVeKMHMvz07TeOeiAinwRoHP7mHg83iWh3TBvrPzez17n7SzF4PPDfoje5+\nBDgCm2e5pLyeiFRNXx27sHOq8rpQgqNwg7kmHc49/6Wo9sa0gf5t4Drg81tfv5XZiESk+rpduu15\nwrUDtPfME4QLm6FOl4CQzSW4HDfq7PjglgQSzvyDuSbBXLaXTmpooJvZN9j8b/9iMzsB3MxmkH/T\nzD4JPAV8JM9Biki1dI89zkz0nc0HSkQRnWN3bj5Qoqdk0T38EDOHmmO9Uecsg2b+Y7R9NEmXy8cG\n/KOZjMciIjUR8p6exUEn5D0E4dfPKlmEi6u5PcynsJn/mG0f1dZ/Eclc++BlNL6yThSt02json3w\nMqB9VsmiPTtNY6mUtcvsFPyIuWEU6CKSuSCAzn1TfbPks0sWQdCkk0MHSKEZ227Tnfodwo0DtKce\nJCj5REYFuojkInZdsu+bJa5dZqJLwIx1iDAa5nSYGvrw6Twp0EWkVop8jGcYQvTCFOsO0QulV1wU\n6CJSP3nM/OMU+dtAEgp0ESnPGLX8pVHkbwNJKNBFJBs7Decxa/lLq6jfBpJQoIvI6NKE85i1/NWB\nnlgkIqNLc8JhzBMjUj8MosinSIwxzdBFZHRpVgf7CtBdgnQVmJqUbrKgQBeR0aVdHewpQIcLKSsw\nKt28SIEuItkYcXUwdQvguPUOlkiBLiJjIXUL4Lj1DpbI3It75kSr1fLl5eXCriciUgdmdtzdW8Pe\npy4XEZGaUKCLSCbUOVg+1dBFaqyonfWZdQ6mHHDFTxDIjAJdpKaKbM/OpHMw5YDVhv4SlVxEairN\n5s20YjZ97tyAAQ8r5RT5OcedZugiNdJbeiiyPTuTzsGYASeZfasN/SUKdJGa6HZh5r3rRJHRaDid\n+6YKbc8e+dTBmLtC7O5Rzi6Yqw39JQp0kZoIjz1JdPoS1pkiOr1GeOwE81+6LJ+Ay2sVsu+ucM7s\ne3oldso+TkfYlkmBLlITbe6nwYeJcBqs0eZ+4GD2FypwFTIIoHN4hXBxlfbsNMHq3XRPX0W48S7a\np5cIJvjcljgKdJEq65kpBwevoHP71YRrB2jveZDg4EI+1yzyMKxul+DQDEEUwVKD7g1fZ2bjRiIa\nNDYiOtNPlPpQ5nGjQBepqpiZchAubM5a2wv5hWyRq5B9N4/w0VcR7TqP9Q0j2jVFuNpUoPdQoItU\nVdxMeX5+aJCPXP4uchWy7+bRnp2msWRbL22iO1riKNBFqirJTLkvvTMrfxe1Ctl38wiCJp2mOloG\nUaCLVNWwmXJMeodhUL1nQfTdPNTRMpgCXaTKtku3mJJMux1oE06NKdBF6iqmJKNNOPU2UqCb2WeA\nPwQcWAGud/f/y2JgIjKiAemtkkV9pQ50M7sE+DTwVnd/3sy+CXwU+GpGYxORUSm9J8qopy3uBs4z\ns93A+cAzow9JRETSSB3o7v4z4AvAU8BJ4Jfufk//+8xszsyWzWz51KlT6UcqIiLbSh3oZnYRcC3w\nJuANwAVm9vH+97n7EXdvuXtr79696UcqImNDj5sbT6Msir4f+Km7nwIws7uAdwJfy2JgItInZotn\nFoce7vRn6AlB42uUQH8KeIeZnQ88D8wAy5mMSkTOFpOiXYJ0wdqT4Gl+RpFnc8nOpA50d3/IzO4E\nHgZeAB4BjmQ1MBHpEZOiISl2ffbdGMLrHiOKLhv+M3puAtqcNL5G6kN395uBmzMai4gMErNJqE2K\nQw/7bgxt7qfROLj9z+i7CQSdDp1OoM1JY0g7RUWqIGaTUECKXZ99N4bg4BV0Dg75GTG/HQTzgYJ8\nDJm7F3axVqvly8sqs4uUSquglWNmx929Nex9mqGLTJqd7h7VATCVoUAXkeF0hEAljLr1X0T61GHT\nTf9nqMNnmgSaoYvsxJD6cx3Kzf2f4fBhOHSo2p9pUijQRZIakNa9GV/JTTd9N6n+z7C4WMHPNKEU\n6CJJxaR1/07Lw4dT9IaXKeYm1W4HNHavE21AYzfMzk6xtFShzzTBFOgiScVs7glDiE476xtGdNpZ\nXbVqNYTE9Zi3oePzhByg7Q8SNBdoaiNRJSjQRZKKad9rr6zQ2LiciD00NtZoTz9BEDSrE3oxNynC\nkGD9AQK/H9antJGoQhToIjvR/wT61bvp7Ponwo130d61RLD6AaBZ3vh2alCPeaXqRnKGdoqKbGfY\nrsoEbS1ZHHFbuEoOur60U1RkVEl6EIfsohypjbHMUNVGokpSoIsMkrQHcZvwS93GOOD8c02aZTsK\ndJFB4hYMi/oRfXeC7rHHmTkaaHOPbEuBLjJIBodSpf4RfXeCkPdoc48MpUAX2U4GteRUP6LvTtDm\nMhpH1Xgi21Ogi4yrnjtBqodZyMRRoIvsQKLGk5y6U9R4IsMo0EUSStSCWIfjFqWydB66SEJxLYjp\n3qTzxSUfmqGLJJSoBTHBmzSJl7wo0EUSStSCmOBNlTwzXSpBgS6yA4kWJoe8KelmIx2nIjulQBcp\nWNwkvj+8VZaRNBToImcUOCXuncTHhbfKMpKGAl0ESp0S9z/1KAwti2NkZAKpbVEEErcb5qE9vUJj\n43mmWKOx8Tzt6ZUXyzK33KJyiySnGboIZHKy4kBDSjmDnnqknaGyUyMFupm9CrgNeBvgwB+4u7ZK\nSPVkcLJirCSlnHab4GW3EET/unUz+Ztsri0TZ9QZ+q3Ad939w2bWAM7PYEwy6eJaPopYrIyZEo98\n6SSrm3ndTGTipA50M7sQeDfw+wDuHgFRNsOSidU/oz18GA4dKmWxMpN10qSlHNVXJAOjLIq+GTgF\nfMXMHjGz28zsgozGJZOqf0a7uFja2SiZrJMOWt3UYS6Sg1FKLruBq4Ab3P0hM7sVuAn4i943mdkc\nMAewb9++ES4nE6F/Rjs7C0tLpZyNktk6af/sW7uGJCejBPoJ4IS7P7T1+k42A/0s7n4EOALQarV8\nhOvJJIirJzebZ73ur2vntQknt9K2dg1JTlIHurs/a2ZPm9lb3P1HwAzww+yGJhOrf0bb8zpuchs3\nk85qHTWX0rZ2DUlORu1yuQG4Y6vD5SfA9aMPSWSwuMnt/PzZM2kY84qGulokJyMFurs/CrQyGovI\nUIMmt70z6YWFClQ01NUiOdBOUamUIIDO4RXCxVXas9MEQfOc96iiIZNKgS7l2mmxu9slODRDEEWw\n1IDmufWUtBUNnT8uVadAl/Kkad9L2CESV9HYLrDVSSh1oECX8iQM57OCOGU9JS6wWXmpdBOuNse/\n7i4yhAJdypPqgcoBQYp6Sv+949hfn+ToP15OxJU07ok4/LknaDQuV91dKk2BLuVJ+0Dl+Z13iPTf\nO3jmJBF7WWc3Ec7qo0/T6VyuGrpUmgJdypXRA5WTXOasXvWVPRz9t4gIp8HaVseMglyqTYEuYy3L\nPThnBXbQpENP++Pcue2PIlVj7sUdr9JqtXx5ebmw64mI1IGZHXf3oZs49UxREZGaUKCLiNSEAl3q\nQQ+MENGiqIyfHW/B1zZPEUCBLmMmVTbrgREigEouMmZSPcfzTLP61JS2ecpE0wxdxuqYwVQbiZI2\nq4/R5xTJgwJ90hVdfx4Sqqk3Eg3b5qk6u0wABfqkK7L+nDBUM9mCX9STpEXGiAJ9EvWGXZGP9ykq\nVJM+SVqkZhTokyYu7Ip6YHFMqOZS1k7yJGnNzqWGFOiTZlDYFRFwfQXyLkE+Ze0kT5IWqSEF+qRJ\nWnrIqyOkJ1TDhZwqMFke0ShSIQr0SZMk7ArqCMm1rK3ZuEwgBfokGhZ2BS1eaiItki0Fupwrx6lz\nfyVHE2mR7CjQ5Vw5TZ21t0ckXwp0iZdi6tw/+9beHpFiKdAlE/2z78OH4dAh7e0RKZJOW5RY/c+L\nGPa6f/a9uBg/G+904JZbVG4RyYNm6BJbKtlutp1k9j07C0tL2tsjUqSRA93MpoBl4Gfufs3oQ5JM\nDdkgFLdQOWy2HTf7jttZ32yqJVGkSFnM0G8EHgMuzOBnSZYStJXELVQOm20nnX1rNi5SrJEC3cwu\nBT4A/CXwJ5mMSLKToK0kbqEyrmuxyQrh4irt2WmCuWay2bceKCFSKHP39H/Z7E5gAXgl8KdxJRcz\nmwPmAPbt2/dbTz75ZOrryQ4lbPwemrtpGsjVdC6SGTM77u6tYe9L3eViZtcAz7n78e3e5+5H3L3l\n7q29e/emvZykkbCtJAiGHLiY5kGfqR4OKiKjGKXkcgD4oJldDbwcuNDMvubuH89maJKJLArZaRrI\n1XQuUriRSi4v/hCzNgNKLr1arZYvLy+PfD0pQZp6uGroIplIWnJRH/oESpWzaWb6anMRKVQmge7u\nIRBm8bMkX1qrFKkvbf0vSv9e+ZJorVKkvlRyKcIYTYszW6tUfVxk7CjQizBG58ZmctT5oBuUQl6k\nVAr0IsRNiwsKv7jLjLxWOahuMya/hYhMKgV6EfqnxVBI+OVW6Ym7QY3RbyEik0qBXpTeafHCQqLw\nG/YEoGFyy9hBdRttJBIplQI9D8OSN8HKZJInAMWFfO/rXDdrxh2tmMNzSEUkOQV61pLUOWLCb9jz\nN+POIIeY0P/0OlFkNBpO576pYjNWG4lESqVAz1rSOkdP+MXdA5I8Aeic0P/yfxOdfiXrTBGdXiM8\ndoL5L12mjBWZEAr0rKWoc8TdA5I+Aeis0H9DlyXaRDgN1mhzP3Awn88pImNHgZ61FLXkQfeAYU8A\nOudSXETzO1cTrh2gvedBgoMLmX0sERl/mZy2mJROW3zJqB0siX+wiFSeTlscY4PWTTNrKVSQi0wk\nHc5VgtiNlmNyeJeIVJdm6CU4p2Y+vaJt8yIyMs3QS3DOoz5X79aZtiIyMs3QC9K/Vnl2qbutbfMi\nMjIFegGGbh7VtnkRyYACvQCJNo+qO0VERqQaegHOLIJOTamiIiL50Qw9C0M286iiIiJFUKCPKuFT\nJHKrqGhnqIhsUaCPKqZA3iUoJmPH6OHTIlI+Bfqo+nYJdaevyS5jh82+9dg3EemhQB9VX4E8DJvZ\nZGyS2XeujyQSkaqpT6CXWUvuKZC3yShjk8y+tdoqIj3qEehxs1koLOj67yWZZGzS2bf610VkSz0C\nvX82e+wYHD1ayGJhbkfhxt0Z1NEiItuoR6D3z2Yh9WLhsAdPDHuYc6brkr13BnW0iMgQqQPdzN4I\nHANeB2wAR9z91qwGtiP9s1k4e4Y+oFwRF9a9mXn4MBw6NPh13MOcc1uXVEeLiAwxygz9BeCz7v6w\nmb0SOG5m97r7DzMa28701zmGFLLjJrz9mbm4uP3rQQ9zzoU6WkRkiNSB7u4ngZNbf/61mT0GXAKU\nE+j9YgrZvTPyuAlvf2bOzsLS0uDXgx7mnNvnUUeLiGwjkxq6me0H3g48lMXPy0NcOaV/whuXmc3m\n9q8LpY4WEdnGyIFuZq8AFoFD7v6rmH8+B8wB7Nu3b9TLpdY/I19djZ/w9mfmsNciIuNipEA3sz1s\nhvkd7n5X3Hvc/QhwBKDVavko13tRiva9dhsau9eJNqCxG9rtKYWziNTKKF0uBnwZeMzdv5jdkIZI\n2b4X0KXj84QcoO0PErAAKM1FpD5GecDFAeATwPvM7NGt/1yd0bgGi1vNjNHtwsLC5tczfy9Yf4B5\n/yuC9QfSP4j5nB8sIjIeRulyeQCwDMeSTIL2vdhJfBZtf9rcIyJjrHo7RRO078XuwZnPoO1Pm3tE\nZIxVL9BhaKvJwMn4qKug2twjImOsmoE+RG57cLS5R0TGmLln00mYRKvV8uXl5Vx+tg4iFJG6MrPj\n7t4a9r5azNC1VikiMlrb4thI2MkoIlJrtQj0M2uVU1NaqxSRyVWLkkvqtUoV3kWkRmoR6JCiI1GF\ndxGpmUqWXDLZfa/Cu4jUTOVm6JlNrJNuElJZRkQqonKBntnu+ySFd5VlRKRCKhHo3SMrhIurtGen\nabeb2e2+H1Z419ktIlIhYx/o3SMrzHzqciKupHFPROfvV+h0mvlVQXpLLDq7RUQqZOwDPVxcJeJK\n1tlNhBMurjI/l9NEOa7EorNbRKQixj7Q27PTNO6JiHAarNGenc7vYnEllvl5BbmIVMLYB3ow16TD\nSzX0YK6Z38VUYhGRCqvNaYuZUZuiiIyZiTptMVOjPgRDRKQkldwpKiIi51Kgi4jUhAJdRKQmFOgi\nIjWhQBcRqQkFuohITRTah25mp4AnU/71i4FfZDicKtBnngz6zPU36ue9zN33DntToYE+CjNbTtJY\nXyf6zJNBn7n+ivq8KrmIiNSEAl1EpCaqFOhHyh5ACfSZJ4M+c/0V8nkrU0MXEZHtVWmGLiIi26hE\noJvZ75rZj8zsx2Z2U9njyZOZvdHM7jOzx8zsB2Z2Y9ljKoqZTZnZI2Z2d9ljKYKZvcrM7jSz/9j6\n37v2x3ya2We2/r3+vpl9w8xeXvaYsmZmt5vZc2b2/Z7vvdrM7jWzx7e+XpTHtcc+0M1sCvg74PeA\ntwIfM7O3ljuqXL0AfNbdrwTeAfxRzT9vrxuBx8oeRIFuBb7r7r8B/CY1/+xmdgnwaaDl7m8DpoCP\nljuqXHwV+N2+790EdNz9CqCz9TpzYx/owG8DP3b3n7h7BPwDcG3JY8qNu59094e3/vxrNv9Pfkm5\no8qfmV0KfAC4reyxFMHMLgTeDXwZwN0jd/+fckdViN3AeWa2GzgfeKbk8WTO3b8H/Ffft68Fjm79\n+SjwoTyuXYVAvwR4uuf1CSYg4ADMbD/wduChckdSiMPA54CNsgdSkDcDp4CvbJWZbjOzC8oeVJ7c\n/WfAF4CngJPAL939nnJHVZjXuvtJ2Jy0Aa/J4yJVCHSL+V7tW3PM7BXAInDI3X9V9njyZGbXAM+5\n+/Gyx1Kg3cBVwJfc/e3A/5LTr+HjYqtufC3wJuANwAVm9vFyR1UvVQj0E8Abe15fSg1/TetlZnvY\nDPM73P2ussdTgAPAB83sP9ksqb3PzL5W7pBydwI44e5nfvu6k82Ar7P3Az9191PuvgbcBbyz5DEV\n5edm9nqAra/P5XGRKgT6vwNXmNmbzKzB5iLKt0seU27MzNisqz7m7l8sezxFcPd5d7/U3fez+b/v\nv7h7rWdu7v4s8LSZvWXrWzPAD0scUhGeAt5hZudv/Xs+Q80Xgnt8G7hu68/XAd/K4yJj/5Bod3/B\nzP4Y+Gc2V8Vvd/cflDysPB0APgGsmNmjW9/7c3f/ToljknzcANyxNVH5CXB9yePJlbs/ZGZ3Ag+z\n2c31CDXcMWpm3wDawMVmdgK4Gfg88E0z+ySbN7aP5HJt7RQVEamHKpRcREQkAQW6iEhNKNBFRGpC\ngS4iUhMKdBGRmlCgi4jUhAJdRKQmFOgiIjXx/zicVIiEj71vAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x119c946d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "alpha = 0.01\n",
    "epochs = 400\n",
    "\n",
    "h=5\n",
    "lin1 = Linear(d,h)\n",
    "sig = ReLU()\n",
    "lin2 = Linear(h,o)\n",
    "mse = MSE()\n",
    "\n",
    "for epoch in range(0,epochs):\n",
    "    loss = 0\n",
    "    Yhat = np.zeros(n)\n",
    "    for i in range(0, n):\n",
    "        Xi = X[i]\n",
    "        yi = y[i]\n",
    "\n",
    "        # forward propagation\n",
    "        z = lin1.forward(Xi)\n",
    "        a = sig.forward(z)\n",
    "        yhat = lin2.forward(a)\n",
    "        loss += mse.forward(yhat,yi)\n",
    "        Yhat[i] = yhat\n",
    "        \n",
    "        grad1 = mse.backward(yhat,yi)\n",
    "        grad2 = lin2.backward(a, grad1)\n",
    "        grad3 = sig.backward(z, grad2)\n",
    "        grad4 = lin1.backward(Xi, grad3)\n",
    "        \n",
    "        lin1.W = lin1.W - alpha*lin1.gradW\n",
    "        lin1.b = lin1.b - alpha*lin1.gradb\n",
    "        lin2.W = lin2.W - alpha*lin2.gradW\n",
    "        lin2.b = lin2.b - alpha*lin2.gradb\n",
    "\n",
    "    if epoch % 50 == 0:\n",
    "        print('Iteration %d = %.6f' % (epoch, loss / n))        \n",
    "        \n",
    "plt.plot(X,y,'.r')\n",
    "plt.plot(X,Yhat,'.b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Relu gives an okayish solution even with less number of layers\n",
    "Seems like its learning the up and downward lines\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0 = 2.630389\n",
      "Iteration 50 = 0.419176\n",
      "Iteration 100 = 0.426562\n",
      "Iteration 150 = 0.433610\n",
      "Iteration 200 = 0.441109\n",
      "Iteration 250 = 0.448841\n",
      "Iteration 300 = 0.455971\n",
      "Iteration 350 = 0.462215\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11100bb10>]"
      ]
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGGlJREFUeJzt3V+MXGd5x/Hfk3VGJQQEbMy/BMeA\nLBrECoFWFQcLGFguaBIRJBcJJNiUUpmLFmJEReNKUS58sVwgtFSqqlohsBYRCMWRQGlFg6YZY6Kp\n1XWCZMBF4Z+DwRCzVQEhmtk4Ty92N5k9e2bPmfP/nPl+pMieZXbnjBJ+8+xznvd9zd0FAGi+q6q+\nAABAPgh0AGgJAh0AWoJAB4CWINABoCUIdABoCQIdAFqCQAeAliDQAaAl9pT5Ytddd53v37+/zJcE\ngMY7e/bsb9x9b9zzSg30/fv3a3V1tcyXBIDGM7MLSZ5HywUAWoJAB4CWINABoCUIdABoCQIdAFqC\nQAeAliDQAUynwUBaWtr4c+fDRip1Dh0AamEwkBYWpOFQ6nQ0WD6jhSNzWw/V60lBUPVFTo4KHcD0\n6fc3wvzKFWk4VP/k2uhD9fsbT2ta1U6FDmD6dLsbpfhmSd49NKvO6WcfqtvdUcQ3omon0AFMnyDY\nSOh+X+p2FQRz6umc+ifX1D00qyCY09KSdlTtBDoA1FEQPJfQg4GCIwsKhkPpdEea66nbDUaLeHW7\nlV5tIgQ6AIR66ur3FRwNRov42lfnEoEOYFoMBuPTOdRT3yrHR4v4JiDQAbRfxB3OgYKRfN/eU881\nxXf7IMkZgQ6g/UItlcGJx7WwEoQmWAoox0selWEOHUD7bbVUZmakTkd9vSNy7jx3Eb35IlGhA2i/\nUEulqxvVWZl8gmXi7smY3nxRzN0LfYFR8/PzzhF0AOpg0nBO3T3JoYduZmfdfT7ueVToAKZSopb5\nSBj3+0G6hUYljsoQ6AAQJVSSd5fPqNOZq/VCIwIdAKKEbmgGaw+q15ur9UIjAh0AokTc0Kz7QiMC\nHQCiFLnYqCAEOgCMU/eSPISFRQDQErGBbmb3mtmTZva9ka+9xMy+ZWaPb/754mIvEwDqqU6nGiWp\n0L8k6T2hr90pqefuByT1Nh8DQD2UlLJbk4133bXx52Cw87XLDPzYHrq7f9vM9oe+fJuk7ubfVyT1\nJf19jtcFAOmMWdJZxKaHO7ZqOXFBwUp1h0+nvSn6Mne/JEnufsnMXprjNQFAehEbYg0U5LPpYehT\nYcdko07FHj5dx0BPzMwOSzosSfv27Sv65QA0SCFbhUfMj0dtejjx60VU/kEQOtVIB6SV3Q+fLlLa\nQP+1mb1iszp/haQnxz3R3Y9LOi5tbM6V8vUANE1EWg+OP3cQs+bmitkqPGJ+vKscNj0c86mwfbIx\n0GD5zHOHTR+eU2+uvFH2tIH+DUm3S/rM5p9fz+2KADRf1AlB567Vwsdeq6FuUuehoW5/3680HL68\nmHZEaH48lzVCCbbCHQz0XM/8tNSbK3eUPTbQzewr2rgBep2ZXZR0tzaC/Gtm9lFJT0h6f5EXCaBh\nIqrZfj/QUDfpivZoKJd++Ut1Oi8vrR2ROVjHfSrksSNjTpJMuXxwzP+0kPO1AGiLiGq2O3utOg8N\nNZSro3UtfvRqLZbYjshF+FOhZjsysvQfQP4iqtkgkHo6t62/vPXUxqrZjoycWASgNgqZeilSSYdA\nc2IRgEYpKRvzVbMdGQl0ALWQy6x4FWq0IyOBDqA6Iz2WbjfIPis+5Qh0APmYtAEe6rEEvZ56vaAu\n3YtGItABZJemAR7RYwmOBgR5BhxwASC7qA2x4raN3ZpVn5mhx5ITKnQA2YUWEg1mb40v2POcEAm1\nexo3/pgTAh1AdqFw7vfnkk2s5DEhEmr3lL0HeZ0Q6ADyMRLOXeWwu2FSoXZP2XuQ1wmBDiB3BXZT\ndgq1e8reg7xOCHQAhSigm5KoFx8E5e5BXicEOoDaSrx6NGL/82kK8i2MLQKoLSYbJ0OFDqA+Qg3z\nmu19VXsEOoB6GNMwn9b2SRq0XADUQ1TDPKHYValTggodaLFGrZhMcAiztPM9NXIf9YIQ6ECbjKTd\nQEGzgi5Bw3wwkBbeeUXDoanTcfUenmnuPuoFINCBtgiVqv3bz2s4vLFZQRfTMO+fuKDhU9frimY0\nfGpd/RMX1V28kX3UN9FDB9oiVKp2dSpy5C+XfnOZTeuR1+rqlDoaakbr6mhdXZ16trA/dqwBv4UU\njAodaLLRhnKoBx0sHlBvsYB+c5lN6/AhGMvL6nVuVn/9oLpXP6JgcUnS9C4kCiPQgaaKCtZQDzrQ\n9qDLpd885ocUcgM2/Fprawr6Swr6fam7RIqHEOhAU0UdKtE9qr4CdSUl2McqXb854ocUVrRHXTDl\n+FgEOtBUSQ6VUAErLyN+SH+poEkTlopOhEAHmiruUIkTFxSsFLTyMvRDcqn8E74WxiPQgSbb7VAJ\nnSptQJtCuh4IdKAldoSqDkgr5Q1oU0hXL1Ogm9knJf21JJd0TtJH3P3/8rgwAJPbHqqUzdMmdaCb\n2fWSPiHp9e7+RzP7mqQPSPpSTtcGIKsEZXOj9nvBrrK2XPZIep6ZrUu6RtIvs18SgLKwsVW7pF76\n7+6/kPRZSU9IuiTpt+7+UF4XBqB4GXasRQ2lDnQze7Gk2yS9WtIrJT3fzD4U8bzDZrZqZquXL19O\nf6UAcscRb+2SZXOud0v6qbtfdvd1SQ9Iemv4Se5+3N3n3X1+7969GV4OmHIFbIjFxlbtkqWH/oSk\nt5jZNZL+KGlB0mouVwVguzyb3RHndk76o7iRWk+pA93dz5jZ/ZIelfS0pMckHc/rwgCMyOsUh7Qf\nDE0+OGOKZJpycfe7Jd2d07UAGGfM2vqJK+U0HwxtODhjSrBSFGiCiLX1qYrtNOd2Rh6cscgJQTVE\noANNEWp2p+rCJD23c/SDYvlWBZ1jux6cgXog0IGGSr3DYdy5nf3QB8XanIKYgzNQDwQ60FBF7XAY\n9UExULDrwRmoBwIdmESCu5BljvQVscNh+INCYnuApiDQgaQS3IVsy94oox8US0WdRoTcZVkpCkyX\nMRufjC7gbOTeKFErUEe+xvYAzUGFDiSV4HDk5eUCj2IrQtSvFNK2rwW9nnq9gKmWBiDQgaTGHY78\nlOvKM6bhU661NWvWmRLjfqUIfS04GtT/vYBAByYSPhx59pw6z7xWQ12tzjPr6s7+WEEw15zwGzf7\n2KhfM7CFQAcyCNYeVO+qf1X/mbepe9VpBWu3SJrb9pxab2Q1bvaxUb9mYIu5e2kvNj8/76urbMiI\nBolL45ixlrZMvaBaZnbW3efjnkeFDoyTJI1jVvdk2iSx1qU96ohAB8ZJmsa7rO5JvTw/4sNkICZN\nsDsCHRgndRo/J/Xy/NCHyeDE41pYCWjdYFcEOjBOTpulpFqeH/ow6esdrNZELAId2E0ojUtra4c+\nTLq6UZ0VJgmxOwIdSCjxxEpeqT/yYRKISULEI9CBhBLdIy1wTrGInRXRLmzOBSSUaJOqhLtzRe2H\nBWRFhQ4klOgeaYLJGBYboSgEOjCB2LZHgtTPtNgI2AWBDuQtJvVzGG8HIhHoQMmSjrez8h+TItCB\nLSUmaLiID780fXakQaADUqUJOhhIC++8ouHQ1Om4eg/P0GdHKowtAlKlh4H2T1zYOPXIr9LwqWfU\nP3GBczyRCoEOSAmHzFOKGTrv6pQ6GmpG6+poXV2derbPfuwY7RYkR8sFkHLbiGuHBK2cYPGAevfe\nrP76QXWvfkTB4tKzl0SQYxKZAt3MXiTpHklvkOSS/srdWfuGbKLuEJZxszIiQTO/dJJmeBAo6C8p\n6Pel7hIpjtSyVuifl/RNd/8LM+tIuiaHa8I0C1e0y8vSkSOxBz0Ukfm53CdNOnROOY4cpA50M3uh\npLdL+ktJcvehpGE+l4WpFa5oT56MPehBKmZAJZdJk3GtHIbMUYAsFfprJF2W9EUze6Oks5LucPc/\njD7JzA5LOixJ+/bty/BymArhivbQIen06V0PepCKGfHLbUVn1NA5Q+YoQJZA3yPpzZI+7u5nzOzz\nku6UdNfok9z9uKTjkjQ/P+8ZXg/TIKqinZuLPeihiKX0Rd0nZcgcRTH3dBlrZi+X9J/uvn/z8dsk\n3enut4z7nvn5eV9dXU31esCWqG5FozoYVOiYkJmddff5uOelrtDd/Vdm9nMze527/1DSgqQfpP15\nQFJR9w8bdU+xsNIf0y7rlMvHJd23OeHyE0kfyX5JwHZpqu/aV+yN+gRCU2QKdHf/rqTYXwOAtNJ0\nJ+hoYFqx9B/VilkWH7nFSprvAaYAS/9RnQSl9I7Rwdlzk39Pt7R3BFSKCh3VSVBK79ikau3Byb9n\nXLslVOlzcDOajgod1UlYSm+/f5jmeyKEfjsYLJ/RwpE5+u5oNAId1Ukzvpdh5G/b5Evot4P+yTXW\n+qDxCHRUK834Xorv2dGuX75VQefYs1/oHppV5zR9dzQbgY6psKNdvzYnLZ9R/+SauodmFRyeU2+u\n5rPrQAwCHVMh3K6fndVzPfPTUm+OtT5oPqZcUDtFTJuEJ1/W1phVR/tQoaNWilzlGa7AmVVH21Ch\no1ZSr/KcsKznEGa0ERU6aiXVKs+UZT09c7QNgY5aSTVmzoERgCQCHTU0ceXM5i2AJAIdUgM2D4+R\ntKxv+vsEYhDo067szcOLCtW4sp5N0jEFmHKZdgnHSnKZDd8K1bvu2vizyG0NwxfMJumYAlTo02i0\nSk7Qf44qbqUUhXZEqA4U5F+wR10wfXZMAQJ92kSFXUz/OZzDJ05IKyspuhehUB3M3lpMFySqGj96\nlIOZ0XoE+rQZF3a7BFy4uJVSTgmGbl72+3PFTBuOq8YZPEfLEejTJmHrYXD83PadCEeKW2l7hb71\ntUT3O0dCtauCuiAZ9kwHmszcvbQXm5+f99XV1dJeD2PEJO/g+DktfOy1Gqqjjobq/cuPFRye2/VH\npB0iYZIQiGdmZ919Pu55VOjTKKb10D+5pqFu0hXt0VCu/sk1BYd3/xFJF2uGA5wuCJAfAh07dA/N\nqvPQUEO5OlpX99Bs/Pd0d7ZP8qriASRDoGOH4PCcetreQ4/9nlDbWtoZ3my5AhSLQEek4PDcjjZL\n7PeMtE+WlnaGN6PgQLEIdBRyYzIqvBk+AYpFoE+5ovra48Kbm6BAcTIHupnNSFqV9At3vzX7JSFX\nMeV3kX1twhsoVx4V+h2Szkt6YQ4/C3lKUH7T1wbaI9Nui2Z2g6RbJN2Tz+UgVwl2GORsTaA9slbo\ny5I+LekFOVwL8paw/E7UGklz55RloECpUge6md0q6Ul3P2tm3V2ed1jSYUnat29f2pdDGnmNlaS5\nc8oqIqB0WVouByW918x+Jumrkt5lZl8OP8ndj7v7vLvP7927N8PLIZUg2LGb4sSHVaQ5HIIDJYDS\npa7Q3f2opKOStFmh/527fyin60JBUhXOae6ccrcVKB1z6FMm1ZhimtYNq4iA0uUS6O7el9TP42eh\nWKkL5zRD5QyiA6WiQi9LTSY+KJyB9iLQy1DhxEfU5wiFM9BOBHoZKto3ttDPkahPipr8FgJMKwK9\nDBVNfBT2ORL1SSExdw5UjEAvQ1TjOkU1m6QoHn1c2OfIuBlzTq8AKkWgl2W0cZ2wFzIazlJ8Uby8\nLB05sv05hdwAHfdJwdw5UCkCvQhx1XeCXkg482+/Pb4oPnly53NCi0TzMW5UhvEZoFIEet6SVN8J\neiHhzJfii+JDh6TTp65o+IzU2SN1uzPFvc+oURnGZ4BKEeh5S3InMqLCDRf14cxfXNz4Z9eiWAPN\n+VH1dVBdf0SBliQRsMC0INDzlvRO5Eg1O66oH3eE25gfIy31FVz5jgI/JV2Z4cYkMGUI9LylWIo5\nrqifuIPBhljAVCPQizBhEueWw6zrB6YagV6RcM88txzmxiQwtQj0CozrmZPDALLIdEg00olcaDnx\nMUIAsB0VegV29Mxnz7EPCoDMqNArsNUzP3ZsM7vXHuT8TQCZUaFXZHvPvMu4IYDMCPSS7Lq9C+OG\nAHJAoJcg0eaKjLkAyIgeeh5iJlTGbR8OAHmiQs8qQfld6Ip8jn0DsIlAzyrB7oqFtcgrPHwaQP0Q\n6FlFlN9RRXOqFnkOB2UAmB4Eelah8nugIJ+iOaeDMgBMj/YEepW95JHyu7+UU9Gc8qAMANOrHYEe\nVc1KlQRdbkVzioMyAEy3dgR6uJo9cUJaWSntZmEhW+FG/SAmWgDsoh2BHq5mpdJuFha6Fe7oD2Ki\nBUCM1AuLzOxVZvawmZ03s++b2R15XthEwrtdLS5uhN7MTOE3C0tbNMTqJAAxslToT0v6lLs/amYv\nkHTWzL7l7j/I6domEy6LU/Y9wl2NuMelDZow0QIgRupAd/dLki5t/v33ZnZe0vWSqgn0sAR9j6iw\nHu1qLC9LR46Mf7zV9Shl0ISJFgAxcumhm9l+SW+SdCaPn1eGqJZ0uKtx8uTuj7da86UNmjDRAmAX\nmTfnMrNrJZ2UdMTdfxfxvx82s1UzW718+XLWl8tkdA+tqJb0Vldjq/V+6NDuj+l6AKiTTBW6mV2t\njTC/z90fiHqOux+XdFyS5ufnPcvrPSvF+N5gIC2884qGQ1On41r+x5kdLemorsbc3O6PAaAuUge6\nmZmkL0g67+6fy++SYqQc3+ufuKDhU9frimY0fGpda49dVK93Y+yeK3GPAaAusrRcDkr6sKR3mdl3\nN/+5OafrGi/l+F5Xp9TRUDNaV0fr6uqUgkA6enTCgI7Z+xwAqpJlyuU7kizHa0km5fhesHhAvXtv\nVn/9oLpXP6JgcWny12ZxD4Aaa95K0YTjezva7EGgoL+koN+XukvpgpjtagHUWPMCXYptZI8tpLM2\nwFncA6DGmhnoMQorpFncA6DGWhnohRbSjLkAqKlWBjqFNIBp1MpAlyikAUyfzEv/G42ZcgAt0poK\nfeLdAJgpB9AyrQj0VNnMTDmAlmlFyyXVbgDhrRWZKQfQcK2o0FONKSYdheFgZgAN0YxAjwnV1GOK\ncaMw9NkBNEj9Az1hqOY2pjj64UGfHUCD1D/QywzVqENF2bsFQEPUP9AjGuSFtbXDHx5rayw5BdAY\n9Q/0UIN8oKC4tnbU3VWWnAJoiPoHurQtVPtLBXZg2AQGQIM1I9BHFL4lORU5gIZqRKCHe+YU0QCw\nU+0DfdzUIkEOANvVful/qmX9ADCFah/obLkCAMnUvuVCzxwAkql9oEv0zAEgidq3XAAAyRDoANAS\nBDoAtASBDgAtQaADQEsQ6ADQEubu5b2Y2WVJF1J++3WSfpPj5TQB73k68J7bL+v7vdHd98Y9qdRA\nz8LMVt19vurrKBPveTrwntuvrPdLywUAWoJAB4CWaFKgH6/6AirAe54OvOf2K+X9NqaHDgDYXZMq\ndADALhoR6Gb2HjP7oZn9yMzurPp6imRmrzKzh83svJl938zuqPqaymJmM2b2mJk9WPW1lMHMXmRm\n95vZf2/++279nqJm9snN/66/Z2ZfMbM/qfqa8mZm95rZk2b2vZGvvcTMvmVmj2/++eIiXrv2gW5m\nM5L+SdKfS3q9pA+a2eurvapCPS3pU+5+k6S3SPqblr/fUXdIOl/1RZTo85K+6e5/KumNavl7N7Pr\nJX1C0ry7v0HSjKQPVHtVhfiSpPeEvnanpJ67H5DU23ycu9oHuqQ/k/Qjd/+Juw8lfVXSbRVfU2Hc\n/ZK7P7r5999r4//k11d7VcUzsxsk3SLpnqqvpQxm9kJJb5f0BUly96G7/2+1V1WKPZKeZ2Z7JF0j\n6ZcVX0/u3P3bkv4n9OXbJK1s/n1F0vuKeO0mBPr1kn4+8viipiDgJMnM9kt6k6Qz1V5JKZYlfVrS\nM1VfSEleI+mypC9utpnuMbPnV31RRXL3X0j6rKQnJF2S9Ft3f6jaqyrNy9z9krRRtEl6aREv0oRA\nt4ivtX40x8yulXRS0hF3/13V11MkM7tV0pPufrbqaynRHklvlvTP7v4mSX9QQb+G18Vm3/g2Sa+W\n9EpJzzezD1V7Ve3ShEC/KOlVI49vUAt/TRtlZldrI8zvc/cHqr6eEhyU9F4z+5k2WmrvMrMvV3tJ\nhbso6aK7b/32db82Ar7N3i3pp+5+2d3XJT0g6a0VX1NZfm1mr5CkzT+fLOJFmhDo/yXpgJm92sw6\n2riJ8o2Kr6kwZmba6Kued/fPVX09ZXD3o+5+g7vv18a/3/9w91ZXbu7+K0k/N7PXbX5pQdIPKryk\nMjwh6S1mds3mf+cLavmN4BHfkHT75t9vl/T1Il6k9odEu/vTZva3kv5dG3fF73X371d8WUU6KOnD\nks6Z2Xc3v/YP7v5vFV4TivFxSfdtFio/kfSRiq+nUO5+xszul/SoNqa5HlMLV4ya2VckdSVdZ2YX\nJd0t6TOSvmZmH9XGB9v7C3ltVooCQDs0oeUCAEiAQAeAliDQAaAlCHQAaAkCHQBagkAHgJYg0AGg\nJQh0AGiJ/weYCl8ZZtOHLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x119e37810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Relu gives an okayish solution even with less number of layers\n",
    "# seems like its learning the up and downward lines\n",
    "\n",
    "alpha = 0.01\n",
    "epochs = 400\n",
    "\n",
    "h=50\n",
    "lin1 = Linear(d,h)\n",
    "sig = Tanh()\n",
    "lin2 = Linear(h,o)\n",
    "mse = MSE()\n",
    "\n",
    "for epoch in range(0,epochs):\n",
    "    loss = 0\n",
    "    Yhat = np.zeros(n)\n",
    "    for i in range(0, n):\n",
    "        Xi = X[i]\n",
    "        yi = y[i]\n",
    "\n",
    "        # forward propagation\n",
    "        z = lin1.forward(Xi)\n",
    "        a = sig.forward(z)\n",
    "        yhat = lin2.forward(a)\n",
    "        loss += mse.forward(yhat,yi)\n",
    "        Yhat[i] = yhat\n",
    "        \n",
    "        grad1 = mse.backward(yhat,yi)\n",
    "        grad2 = lin2.backward(a, grad1)\n",
    "        grad3 = sig.backward(z, grad2)\n",
    "        grad4 = lin1.backward(Xi, grad3)\n",
    "        \n",
    "        lin1.W = lin1.W - alpha*lin1.gradW\n",
    "        lin1.b = lin1.b - alpha*lin1.gradb\n",
    "        lin2.W = lin2.W - alpha*lin2.gradW\n",
    "        lin2.b = lin2.b - alpha*lin2.gradb\n",
    "\n",
    "    if epoch % 50 == 0:\n",
    "        print('Iteration %d = %.6f' % (epoch, loss / n))        \n",
    "        \n",
    "plt.plot(X,y,'.r')\n",
    "plt.plot(X,Yhat,'.b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TanH has similar to relu behavior, makes sense"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, hidden):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = nn.Linear(1, hidden)\n",
    "        self.fc2 = nn.Linear(hidden,1)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        z = self.fc1(x)\n",
    "        a = F.sigmoid(z)\n",
    "        yhat = self.fc2(a)\n",
    "        return yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 505,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [0]: loss = 47.200035\n",
      "Epoch [10000]: loss = 0.412635\n",
      "Epoch [20000]: loss = 0.207668\n",
      "Epoch [30000]: loss = 0.147588\n",
      "Epoch [40000]: loss = 0.133015\n",
      "Epoch [50000]: loss = 0.118666\n",
      "Epoch [60000]: loss = 0.099852\n",
      "Epoch [70000]: loss = 0.066810\n",
      "Epoch [80000]: loss = 0.038356\n",
      "Epoch [90000]: loss = 0.030612\n",
      "Epoch [100000]: loss = 0.024403\n",
      "Epoch [110000]: loss = 0.022680\n",
      "Epoch [120000]: loss = 0.021205\n",
      "Epoch [130000]: loss = 0.020583\n",
      "Epoch [140000]: loss = 0.021146\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11dad3b90>]"
      ]
     },
     "execution_count": 505,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFyRJREFUeJzt3X2MXGd1x/Hf8cRjktAI2Ji3BNsh\nisA0K0S6rbgY8IVFvISIIBkqkGBTSrW0aiFBqChbFKUoqhapCCV/VNSrkOAVERTZUUFpSoyG3CS1\nRlHXCdICLgoLOJgEYpa2AUR9N7unf8zaXs/Oembv3Je5d74fKVrveOz7jJL85tkz557H3F0AgPLb\nUvQCAADpINABoCIIdACoCAIdACqCQAeAiiDQAaAiCHQAqAgCHQAqgkAHgIq4IM+LXXrppb5r1648\nLwkApXf06NFfufv2bs/LNdB37dqlubm5PC8JAKVnZsd7eR4lFwCoCAIdACqCQAeAiiDQAaAiCHQA\nqAgCHQAqgkAHMJyaTWl6uvW1InLtQweAgdBsSuPjUhxL9brUaEhBUPSq+sYOHcDwiaJWmC8vt75G\nUdErSgWBDmD4hGFrZ16rtb6GYdErSgUlFwDDJwhaZZYoaoV5ELTKMGu/LyECHcBwCoKzwV2Rmjol\nFwBD6Zwml4rU1NmhAxgOa0oqTQXnbshvv06qPaBoZY/C2hEFJa2pE+gAqq+tpBLdcExxvPPMhnz2\n8VEdsIZimermaqim1AouOdbmCXQA1ddWUgn1kOr1iTM7dEmKn6tp2aX4udbTU8nenGvzBDqA6gtD\nNWtvPFtSmbhKjYmzG2dJOnDgbO6mVnHpVJsn0AEguaYCjbeXVIJzs7W9i1FKoVpyut899XeKzgh0\nAJUXRd1LKu0Bn0q1pFO/e4YIdACVl2SjHEVSfMq1vGKKT7miyJLlcfs7RYYIdACVl2SjHI7Mq75y\npWJtVX1lSeHIgprN0YG+mZRABzAUNrtRDhbvU2PLvylaeZPCLY9Ij09q/KbRgb6ZlDtFAaCTMFSw\n7TFN1f5RwbbHFGnvwN9Myg4dADppq9OE2ql6Fq2NKSLQAWAja+o0gTJqbUwRgQ4APcqktTFF1NAB\nVE5ex4UO2pDGrjt0M7tL0nWSnnH3q1cfe5Gkf5G0S9JPJf2pu/93dssEgN40m9L4W5YVx6Z63dV4\nsJbZrrlTf3uRJZheduhflvTOtsdultRw96skNVa/B4DCRbPHWzcE+RbFp1YUzR7P7FqnPze97bbW\nV6n1ZnLLZ1Y0/pblzH9CaNc10N39YUm/bnv4ekkHVn99QNJ7U14XACQS6iHVFaumJdW1pFAPZXq9\nIJCmplpfO72Z5FX+kZJ/KPoSd39aktz9aTN78UZPNLNJSZOStGPHjoSXA4DeBBNXqXHXtYqW9ijc\nekTBxHR6f3mXekrrzeR9iuWqa0kjv/i+xsd35vahaeZdLu4+I2lGksbGxjzr6wEYckGgIJpWEEVS\nOJ1egvbQ0tL+ZhK99ECe03MTB/ovzexlq7vzl0l6Js1FASi/5sy8okOLCveNKJgc3fCxTGQxEKuX\n2ebtbyY534yUNNC/KekGSZ9b/fqN1FYEoPSaM/Ma/9iVirVb9cOxGpqXpHWPZRrqadtoZGN7GaaH\nm5Gy0kvb4lclhZIuNbMTkm5VK8i/bmYflfSkpPdnuUgA5RIdWlSs3VrWBYrlig4tStK6x4LJghe6\nGZ1GNvZShslvem73QHf3D27wW+MprwVARYT7RlQ/HJ/5cDDcNyJJHR8rlfZ0zvmIuW649R9A6oLJ\nUTW0vl7e6bFSy/mIuW7MPb/Gk7GxMZ+bm8vtegCQuRxuDTWzo+4+1u157NABoB95Fsm7YDgXAFQE\ngQ6gOHneFz8EKLkASMdma8mDNky8Agh0AP1LEs5RpOapa1qHMJ96pHV3ZcJAH6RTg4pEoAPoX4Jw\nbo5cp/GVGxWrrvpKrMbIgpJkcZ7zzwcdgQ6gb0nCOVocVbzFtbxiirfUFC2OJgr01sjay7SsmuJT\nS4pmTygIdiZ5GaXHh6IA+tYK5wtbt/VvuVDRYvebhsJQqm8z1Wqtr0nvycl7/vkgY4cOoG+nw7lV\nQu8tnDuNRkki0/nnJcOdogBSUegHkxX/VLTXO0UJdAAYcL0GOjV0AKgIAh0AKoJABzA4GAXQF7pc\nAAwGRgH0jR06gMHQ6fQfbAo7dACDIQzVrL1R0coehbUjCgo+/aeMCHSgSkrcj91UoHFrKJapbq6G\nap1HAZT4NWaNQAeqouQ16CiS4udqWnYpfm6D85abTTXDqdW7QqcURNOleo1ZI9CBqkhxHG1XGeyS\nezlvuTn7hMbj+1tDwOJYjdmDCgj0Mwh0oMSaM/OKDi0q3DcipTSOtvtFs/lJYMPZLmvePCLtVax6\nawiYXJH2ZvMaS4pAB0qqOTOv8Y9dqVi7VT8c64b3PtuaeNjnONquOnWjpLRLXnfectubR3j7o6pv\nu1xxvKx6fYvCieEck7sRAh0oqejQomLtPrNb1VNPqb7tpZuaeJhInt0obW8eweJ9ajw4ymeiGyDQ\ngZIK942ofjhWLFddS5r46FZNjLaVLDKodffcjZKGDm8e63bxOINAB0oqmBxVQ2dr6MFk61CJc2rP\nGdS6e+pGSUmubx4VQKADJRZMjiqY3OA3M+p66aUbJS15vnlUAYEOVFRahzC3S+ukoV7k+eZRBX0F\nupl9UtJfSHJJ85I+4u7/l8bCAPQnrUOYO8mrjp3nm0cVJA50M7tM0ickvcbdf29mX5f0AUlfTmlt\nAPqQ5JzPQcSHoL3rt+RygaQLzWxJ0kWSnup/SQDSwO52+CQOdHf/uZl9XtKTkn4v6bC7H05tZQD6\nxu52uCSeh25mL5R0vaQrJL1c0sVm9qEOz5s0szkzmzt58mTylQIAzqufAy7eJukn7n7S3Zck3Svp\nDe1PcvcZdx9z97Ht27f3cTlguDVn5jX9jkjNmfmil4IB1U8N/UlJrzezi9QquYxLmktlVQDO0T63\npaH5MzcSAacl3qG7+6OSDkp6TK2WxS2SZlJaF4A1WnNbTk8Z3Kro0GLivyvRTp/Dm0uhry4Xd79V\n0q0prQXABtrntoT7RhL9PYl2+iU/OGOYcEg0UALB5Kga+xd029uPqLF/IXG5JdFOf3WEwPTy36p5\n6hoObx5g3PoPlMR557b0KMlOP6sRAkgfgQ4MkY0mNJ5PliMEkC4CHRgym93pV2WEwDAg0AGcFyME\nyoNABzYjgxOAyoARAuVAlwvQq2ZTzXBK05/5rZrhVGV6smkxrw526ECPmrNPaDy+v9XtEcdqzB5U\nUPJta7Mpjb9lWXFsqtddjQdrCoJWv/pmPjjFYCDQgR5F2rumh9sVaW/puz2i2eOKT12mZdUUn1pS\nNHtCmn+WMQMlRckF6FE4sVP1baaaLau+bYvCiZ2t3yhxzSLUQ6orVk1Lrb50PZTqmAHkix060KMg\nkBoP1s79THS1rh4t7VG4dUpBNF2qTw+DiavUuOva1fUfUTAxLc0/P5UxA8gfgQ5sQnu3R+nr6kGg\nIJpWEEVS2HozCgJt+uYjDAYCHTiPbh8OVqKu3qEnMY0xA8gfgQ5soJfJhOHETtXvXlYcL6teX1NX\nT2UBw9nzjuT4UBTYQC8fDp6uq9/2D7UzLX+pqGjPO7LFDh3YQK+TCbO4i7L0tXkUgkAHNpBkMmFa\nKlGbR+4IdOA8ivpwMNPaPCqLQAcGUMeed6ALAh1IWVpzUJhwiM0i0IEUJTqEGUgJbYtAipiDgiIR\n6ECKwn0j5w67Yg4KckTJBUhRka2OAIEOpIw5KCgKJRcAqAgCHVjVnJnX9DsiNWfmi14KkAglF0AD\n0G7IZEWkgB06oILbDZmsiJQQ6ICybTfsVso5PVnxFv+sxuP71Zx9IrVrY7j0VXIxsxdIulPS1ZJc\n0p+7O9sLlE5W7Ya9lHKYrIi09LtDv0PSt9z91ZJeK+lY/0vCsFu3o202penpzEsRweSoph4IU62d\n91LKCSd2qr7NVLNl1bcxWRHJJd6hm9klkt4s6c8kyd1jSXE6y8KwWrejXfhX6fbbV0+ln1IQTZfq\nQ8NeDslgsiLS0k/J5ZWSTkq628xeK+mopBvd/XeprAxDqbWj3X2m/DB7T00HSnxyz0alnPaJjExW\nRBr6CfQLJF0j6ePu/qiZ3SHpZkm3rH2SmU1KmpSkHTt29HE5DIP2Ha2ufKXin5e7vtx+52jhLZKo\nrH5q6CcknXD3R1e/P6hWwJ/D3Wfcfczdx7Zv397H5TAMgslRNfYv6La3H1Fj/4ImPveHlasvM5ER\nWUm8Q3f3X5jZz8zsVe7+Q0njkn6Q3tIwrNp3tFWrL/d6+DSwWf3eKfpxSfeYWV3SjyV9pP8lAeeq\nWn2ZiYzIirl7bhcbGxvzubm53K4HAFVgZkfdfazb87hTFOWTU186UDYEOgq16QmHzD0BNsS0RRQm\nSfve6bknZe1LB7LEDh2FSdK+d+7ck62KtDf5AijdoGLYoaMwSdr3womdqt+9rDheVr3eR196symN\nj0txLNXrUqNRrVYaDCUCHYVJ0r6X2tyTKFLz1DWKVt6k8NQjCqKIQEfpEegoVJIDldPoS2+OXKfx\nlRtbtfiVWI2RhdKNFADaUUPHUIoWRxVvubBVi99yoaJFbu5B+RHoGEphqNaMmFrraxgWvSKgf5Rc\nMJSCoPU5aJVmxAAEOoZW1WbEAJRcUAmbvuMUqCB26Cg9DowAWtiho/Q4MAJoIdBReuG+EdUVq6Yl\nDozAUKPkgnUHFpdNz3ecNpu0taDSOOBiyJ2tP9dVV6zG/oVShnpXq2N3o6U9CrceURBNE+ooDQ64\nQE/yrj8X1Y1yeuzuLf5Zjcf3qzn7RK7XB/JAyWUIrS2x5HlgcZ7dKO1lpHPH7roi7WV2CyqHQB8y\n60J1/4Ia+xdyqaG3fhrYfTZUDy1uejBXLzq9cYQTo+mM3QUGGIE+ZDqF6tQDYSbB2i6vnwY6vsbJ\nlMbuAgOMQB8yPYdqBh0hSeafJ7HRa+RWf1QdXS5DqGubYgU6Qsreigms1WuXC4GOdZp/Navxf37f\n2VbGvzyo4IsTRS8LGFq0LSKxVA9iBpAbAh3rhBM7W4c/2LLq2+gIAcqCD0WxTmoHMQPIFYGOjugI\nAcqHkgsAVASBDgAV0Xegm1nNzB43s/vSWBDSxdFswPBIo4Z+o6Rjki5J4e9CijiaDRgufe3Qzexy\nSe+WdGc6y0GaUh2N22xK09Otr1n+GQCJ9VtyuV3SpyWtpLAWpCy1o9lWRwFMf+a3aoZTvQV0kj8D\noC+JSy5mdp2kZ9z9qJmF53nepKRJSdqxY0fSyyGBtIZhnT4cIlZd9ThWY/aggi49jUn+DID+9FND\n3yPpPWZ2raTnSbrEzL7i7h9a+yR3n5E0I7VmufRxPSQQTI72PRo3yeEQHCgB5C9xycXdp9z9cnff\nJekDkr7THuaohiSjABgfAOSPO0XRVZJRAIwPAPLH+FwAGHCMzx00tPAByBiBngda+ADkgBp6DqrY\nwtfxiLcMziEF0DsCPQdVa+HrOFJg9LdrziGdKuU5pEDZEeg5CCd2qn73suJ4WfV6q4WvzIcYt0YK\n7D77BnVoUXr8ycr9FAKUDYGeg/YWPs2Xe2hWuG9E9cOxYvmZkQLR41dU6qcQoIwI9JysPQFo+u/X\n73D7vZszTx1HCjS17qcQAPki0DPQrZzSaYdbNu0jBbiRCCgegZ6yXmaQpzU0a6PrF1Wb5xxSoFgE\neso6fWDYqZySxtCsdhxoAQw3bixKWWozyBNI9UALAKXDDj1lWZZTuqlCbR5Acgznqpgy97cD6KzX\n4Vzs0Csmi9o8gHKghj4gmjPzmn5HpObMfNFLAVBS7NAHAN0pANLADn0A0J0CIA0E+gAostURQHVQ\nchkARbY6AqgOAn1A0J0CoF+UXACgIgj0FNByCGAQUHLpU9Eth9wZCuA0Ar1PvU5XzELRbyYABgsl\nlz5l2XLYrZRD/zqAtdih9ymrlsNedt9MVwSwVmUCvb2WnGdtOYuWw15KOfSvA1irEoHevpu9/eGH\ndNM9f1zq2nKvu2/61wGcVokaenst+dC/X1z62nIwOarG/gXd9vYjauxfOPNTB+2RADZSiR16+252\n37t+p0fuKX9tee3um44WAN0kDnQze4WkWUkvlbQiacbd70hrYZuxvpa8V6NvrlZtucj2SADl0M8O\n/TlJn3L3x8zsDyQdNbNvu/sPUlrbprTXkqtWW6ajBUA3iQPd3Z+W9PTqr39jZsckXSapkECvOjpa\nAHSTyiHRZrZL0sOSrnb3Z9t+b1LSpCTt2LHjj44fP9739QBgmPR6SHTfXS5m9nxJhyTd1B7mkuTu\nM+4+5u5j27dv7/dyAIAN9BXoZrZVrTC/x93vTWdJ3SVt36PtD0CV9dPlYpK+JOmYu38hvSWdX9L2\nPdr+AFRdPzv0PZI+LOmtZvbd1X+uTWldG0o6kCqtQVbs8gEMqn66XP5DkqW4lp4kbd9Lo+2PXT6A\nQVa6O0WTtu+l0fbHzT0ABlnpAl1KftNQvzcbcXMPgEFWykAvCjf3ABhkBPomVW2kAIDqqMT4XAAA\ngQ4AlUGgA0BFDHWgc5MQgCoZ2g9FuUkIQNUM7Q49rVEAADAohjbQw30jqitWTUvnvUmIsgyAshja\nkksvNwlRlgFQJqUI9OZMNndndrtJiNktAMpk4AM9713y2jcPZrcAKJOBD/Q8d8nr3jz2L6ixf4HZ\nLQBKYeADPc9dcqc3j6kHQsosAEph4AM9zwmHlFgAlJm5e24XGxsb87m5udyul0RWH8ACQFJmdtTd\nx7o9b+B36HljPC6AshraG4sAoGoIdACoCAIdACqCQAeAiiDQAaAiCHQAqIhc+9DN7KSk4wn/+KWS\nfpXicsqA1zwceM3V1+/r3enu27s9KddA74eZzfXSWF8lvObhwGuuvrxeLyUXAKgIAh0AKqJMgT5T\n9AIKwGseDrzm6svl9Zamhg4AOL8y7dABAOdRikA3s3ea2Q/N7EdmdnPR68mSmb3CzB40s2Nm9n0z\nu7HoNeXFzGpm9riZ3Vf0WvJgZi8ws4Nm9l+r/76DoteUNTP75Op/198zs6+a2fOKXlPazOwuM3vG\nzL635rEXmdm3zeyJ1a8vzOLaAx/oZlaT9E+S3iXpNZI+aGavKXZVmXpO0qfcfbek10v664q/3rVu\nlHSs6EXk6A5J33L3V0t6rSr+2s3sMkmfkDTm7ldLqkn6QLGrysSXJb2z7bGbJTXc/SpJjdXvUzfw\ngS7pTyT9yN1/7O6xpK9Jur7gNWXG3Z9298dWf/0btf4nv6zYVWXPzC6X9G5Jdxa9ljyY2SWS3izp\nS5Lk7rG7/0+xq8rFBZIuNLMLJF0k6amC15M6d39Y0q/bHr5e0oHVXx+Q9N4srl2GQL9M0s/WfH9C\nQxBwkmRmuyS9TtKjxa4kF7dL+rSklaIXkpNXSjop6e7VMtOdZnZx0YvKkrv/XNLnJT0p6WlJ/+vu\nh4tdVW5e4u5PS61Nm6QXZ3GRMgS6dXis8q05ZvZ8SYck3eTuzxa9niyZ2XWSnnH3o0WvJUcXSLpG\n0hfd/XWSfqeMfgwfFKt14+slXSHp5ZIuNrMPFbuqailDoJ+Q9Io131+uCv6YtpaZbVUrzO9x93uL\nXk8O9kh6j5n9VK2S2lvN7CvFLilzJySdcPfTP30dVCvgq+xtkn7i7ifdfUnSvZLeUPCa8vJLM3uZ\nJK1+fSaLi5Qh0P9T0lVmdoWZ1dX6EOWbBa8pM2ZmatVVj7n7F4peTx7cfcrdL3f3XWr9+/2Ou1d6\n5+buv5D0MzN71epD45J+UOCS8vCkpNeb2UWr/52Pq+IfBK/xTUk3rP76BknfyOIiA39ItLs/Z2Z/\nI+kBtT4Vv8vdv1/wsrK0R9KHJc2b2XdXH/s7d7+/wDUhGx+XdM/qRuXHkj5S8Hoy5e6PmtlBSY+p\n1c31uCp4x6iZfVVSKOlSMzsh6VZJn5P0dTP7qFpvbO/P5NrcKQoA1VCGkgsAoAcEOgBUBIEOABVB\noANARRDoAFARBDoAVASBDgAVQaADQEX8PyBpI8txofr/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11df19fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net = Net(20)\n",
    "net.train()\n",
    "if torch.cuda.is_available():\n",
    "    net = net.cuda()\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "tensorX = torch.FloatTensor(X)\n",
    "tensorX = tensorX.unsqueeze(1)\n",
    "varX = Variable(tensorX)    \n",
    "varY = Variable(torch.FloatTensor(y).unsqueeze(1))\n",
    "\n",
    "optimizer = optim.Adam(net.parameters(), lr=0.01)\n",
    "for epoch in range(0,150000):    \n",
    "    yhat = net(varX)\n",
    "    loss = criterion(yhat, varY)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if epoch%10000 ==0:\n",
    "        print(\"Epoch [%d]: loss = %.6f\" % (epoch, loss))\n",
    "    \n",
    "#Print paramters for diagnosis\n",
    "# for paramter in net.named_parameters():\n",
    "#     print paramter    \n",
    "\n",
    "plt.plot(X,y,'.r')\n",
    "plt.plot(X,yhat.data.squeeze().numpy(),'.b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pytorch batching is very fast and hence allows a lot of epochs quickly!\n",
    "Also adam is kind of cool and better than sgd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}