cnn/1a. Understanding Convolution Network - Part2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*********\n",
    " # Understanding convolutional networks - part 2:\n",
    "\n",
    "    1/ Use a set of weights(kernels) obtained from predefined gabor filters\n",
    "    2/ train a convolution network with the predefined filters\n",
    "    3/ Compare it with a convolutional network, with derived weights from back propagation\n",
    "*********"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.5.2 (default, Nov 23 2017, 16:37:01) \n",
      "[GCC 5.4.0 20160609]\n",
      "TF version: 1.8.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "print('Python version:',sys.version)\n",
    "print('TF version:',tf.__version__)\n",
    "SEED = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gabor(n_values=7, sigma=1.0, mean=0.0):\n",
    "    x = tf.linspace(-3.0, 3.0, n_values)\n",
    "    z = (tf.exp(tf.negative(tf.pow(x - mean, 2.0) /\n",
    "                       (2.0 * tf.pow(sigma, 2.0)))) *\n",
    "         (1.0 / (sigma * tf.sqrt(2.0 * 3.1415))))\n",
    "    gauss_kernel = tf.matmul(\n",
    "        tf.reshape(z, [n_values, 1]), tf.reshape(z, [1, n_values]))\n",
    "    x = tf.reshape(tf.sin(tf.linspace(-3.0, 3.0, n_values)), [n_values, 1])\n",
    "    y = tf.reshape(tf.ones_like(x), [1, n_values])\n",
    "    gabor_kernel = tf.multiply(tf.matmul(x, y), gauss_kernel)\n",
    "    return gabor_kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAACtlJREFUeJzt3duLXfUZxvHnMbHoJNpBtKJGqhci\niFANIVAM0lqUWCX2ohcKCi2F3NQSaUG0N8V/QOxFKQS1tXhCPICI9QAqNqGeMiZVEy0iFhMsUUJI\nJgOVxKcXsyyjDc6Ksw7b1+8HQmZPlvv9SfKdtfbeM/vnJAJQ03FjLwBAfwgcKIzAgcIIHCiMwIHC\nCBwojMCBwggcKIzAgcKW93GnU1NTmZ6e7uOuAUjav3+/5ubmvNhxvQQ+PT2tjRs39nHXACRt3ry5\n1XFcogOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UFirwG2vt/2O7Xdt39L3ogB0Y9HA\nbS+T9AdJV0q6QNJ1ti/oe2EAlq7NGXytpHeTvJfkE0kPSrqm32UB6EKbwM+S9MGC27ubzwGYcJ09\nyWZ7o+3XbL82NzfX1d0CWII2ge+RdPaC26uaz31Oks1J1iRZMzU11dX6ACxBm8BflXSe7XNtf0vS\ntZIe73dZALqw6Du6JDls+0ZJT0taJunuJG/1vjIAS9bqLZuSPCnpyZ7XAqBjfCcbUBiBA4UROFAY\ngQOFEThQGIEDhRE4UBiBA4UROFAYgQOF9bK76JgOHz482uwDBw6MNnt2dna02ZK0cuXK0WaffPLJ\no81evnyyE+IMDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBhBA4URuBAYQQOFNZmd9G7be+1/eYQ\nCwLQnTZn8D9LWt/zOgD0YNHAk7woad8AawHQMR6DA4WxfTBQWGeBs30wMHm4RAcKa/My2QOS/i7p\nfNu7bf+i/2UB6EKb/cGvG2IhALrHJTpQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4X1\ntvdpkr7u+ksdPHhwlLmStHXr1tFmz8zMjDZbklavXj3a7HXr1o02e3p6erTZbXAGBwojcKAwAgcK\nI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCmvzvuhn237e9k7bb9neNMTCACxdmx82OSzpN0lm\nbJ8kaZvtZ5Ps7HltAJaozfbBHyaZaT4+KGmXpLP6XhiApTumx+C2z5F0saSX+1gMgG61Dtz2SkmP\nSLopyYGj/DnbBwMTplXgto/XfNz3JXn0aMewfTAwedo8i25Jd0naleT2/pcEoCttzuCXSLpB0mW2\ntze/ftzzugB0oM32wVskeYC1AOgY38kGFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBh\nvW0ffNxx43ztOOWUU0aZK0kbNmz4Rs7G5OIMDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBhBA4U\nRuBAYQQOFNZm44MTbL9ie0ezffBtQywMwNK1+WGT/0i6LMlss4XRFtt/TfJSz2sDsERtNj6IpNnm\n5vHNr/S5KADdaLv54DLb2yXtlfRsErYPBr4GWgWe5EiSiyStkrTW9oVfPIbtg4HJc0zPoifZL+l5\nSeuP8mdsHwxMmDbPop9me7r5+ERJl0t6u++FAVi6Ns+inyHpHtvLNP8F4aEkT/S7LABdaPMs+j8k\nXTzAWgB0jO9kAwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCutlf3Dbst3HXS9q\nenp6lLmStG7dutFmr169erTZkjQzMzPa7C1btow2e//+/aPMbdsXZ3CgMAIHCiNwoDACBwojcKAw\nAgcKI3CgMAIHCiNwoDACBwprHXizP9nrtnlPdOBr4ljO4Jsk7eprIQC613Z30VWSrpJ0Z7/LAdCl\ntmfwOyTdLOnTHtcCoGNtNh+8WtLeJNsWOe5/2wcfOnSoswUC+OranMEvkbTB9vuSHpR0me17v3jQ\nwu2DV6xY0fEyAXwViwae5NYkq5KcI+laSc8lub73lQFYMl4HBwo7prdsSvKCpBd6WQmAznEGBwoj\ncKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCutl++AkStLHXS/qwIEDo8yVpK1bt442\ne8eOHaPNlqTZ2dnRZo/5dz7Wv/O2czmDA4UROFAYgQOFEThQGIEDhRE4UBiBA4UROFAYgQOFEThQ\nGIEDhbX6XvRm26KDko5IOpxkTZ+LAtCNY/lhkx8m+bi3lQDoHJfoQGFtA4+kZ2xvs73xaAcs3D54\nbm6uuxUC+MraXqKvS7LH9nckPWv77SQvLjwgyWZJmyXpzDPPHOeHZAF8TqszeJI9ze97JT0maW2f\niwLQjUUDt73C9kmffSzpCklv9r0wAEvX5hL9dEmP2f7s+PuTPNXrqgB0YtHAk7wn6XsDrAVAx3iZ\nDCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwnrZPnhMR44cGW32vn37vpGzMbk4gwOF\nEThQGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4W1Ctz2tO2Hbb9te5ft7/e9MABL1/aH\nTX4v6akkP7X9LUlTPa4JQEcWDdz2tyVdKulnkpTkE0mf9LssAF1oc4l+rqSPJP3J9uu272z2KPsc\ntg8GJk+bwJdLWi3pj0kulnRI0i1fPCjJ5iRrkqyZmuIKHpgEbQLfLWl3kpeb2w9rPngAE27RwJP8\nW9IHts9vPvUjSTt7XRWATrR9Fv1Xku5rnkF/T9LP+1sSgK60CjzJdklrel4LgI7xnWxAYQQOFEbg\nQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhTmJN3fqf2RpH99xf/8VEkfd7gcZjO74uzvJjltsYN6\nCXwpbL+WZJTve2c2s6vN5hIdKIzAgcImMfDNzGY2s7sxcY/BAXRnEs/gADoyUYHbXm/7Hdvv2v6/\nd27tce7dtvfafnOomQtmn237eds7bb9le9OAs0+w/YrtHc3s24aavWANy5q3435i4Lnv237D9nbb\nrw08e7CdgibmEt32Mkn/lHS55t/J9VVJ1yXp/Q0ebV8qaVbSX5Jc2Pe8L8w+Q9IZSWZsnyRpm6Sf\nDPT/bUkrkszaPl7SFkmbkrzU9+wFa/i15t8O7OQkVw84931Ja5IM/jq47Xsk/S3JnZ/tFJRkfx+z\nJukMvlbSu0nea3ZPeVDSNUMMTvKipH1DzDrK7A+TzDQfH5S0S9JZA81Oktnm5vHNr8G+4tteJekq\nSXcONXNsC3YKukua3ymor7ilyQr8LEkfLLi9WwP9Q58Uts+RdLGkl7/8yE5nLrO9XdJeSc8ueP/7\nIdwh6WZJnw448zOR9IztbbY3Dji31U5BXZmkwL/RbK+U9Iikm5IcGGpukiNJLpK0StJa24M8RLF9\ntaS9SbYNMe8o1iVZLelKSb9sHqYNodVOQV2ZpMD3SDp7we1VzefKax7/PiLpviSPjrGG5jLxeUnr\nBxp5iaQNzWPhByVdZvvegWYryZ7m972SHtP8Q8QhDLpT0CQF/qqk82yf2zzxcK2kx0deU++aJ7ru\nkrQrye0Dzz7N9nTz8Ymaf4Lz7SFmJ7k1yaok52j+7/q5JNcPMdv2iuYJTTWXx1dIGuQVlKF3Cmq7\ns0nvkhy2faOkpyUtk3R3kreGmG37AUk/kHSq7d2SfpfkriFma/5MdoOkN5rHwpL02yRPDjD7DEn3\nNK9gHCfpoSSDvlw1ktMlPTb/tVXLJd2f5KkB5w+2U9DEvEwGoHuTdIkOoGMEDhRG4EBhBA4URuBA\nYQQOFEbgQGEEDhT2X7wg8tM0yRt9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    out = gabor().eval()\n",
    "\n",
    "plt.imshow(out, cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAACq9JREFUeJzt3duLXuUZhvH7dqLoqHUUrUQj1YMi\niFATQqAYpLUosWrsQQ8UFFoKelBLpAXRnlT/AbFIKQRNa3GHuAGR1A2oWKHuEpOqSSwSLCZYooSg\naaAy8e7BLGHU0Flx1ubzyfWDkPkmK/O8Qa9Z69vM9zqJANR01NgLANAfAgcKI3CgMAIHCiNwoDAC\nBwojcKAwAgcKI3CgsCV9fNHp6enMzMz08aUBSNq3b58OHDjghY7rJfCZmRndcMMNfXxp4CuOxJdb\nr1+/vtVxXKIDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UBiBA4UROFBYq8Btr7H9ju13bd/S96IA\ndGPBwG1PSfqDpMsknSfpGtvn9b0wAIvX5gy+StK7SXYm+VTSQ5Ku6ndZALrQJvAzJb0/7/au5nMA\nJlxnD7LZvt7267ZfP3DgQFdfFsAitAl8t6Sz5t1e1nzuC5KsT7Iyycrp6emu1gdgEdoE/pqk79o+\nx/Yxkq6W9ES/ywLQhQXf0SXJrO0bJT0taUrShiRv974yAIvW6i2bkmyUtLHntQDoGK9kAwojcKAw\nAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgsF52Fz1SHYm7XE6C2dnZ0WYvWTLZCXEGBwojcKAw\nAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCmuzu+gG23tsvzXEggB0p80Z/M+S1vS8DgA9\nWDDwJC9K2jvAWgB0jPvgQGFsHwwU1lngbB8MTB4u0YHC2jxN9qCkv0s61/Yu27/of1kAutBmf/Br\nhlgIgO5xiQ4URuBAYQQOFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGG97X16JG6lu3fveO+LsX//\n/tFmj23r1q2jzV69evUocw8ePNjqOM7gQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBhBA4U\nRuBAYW3eF/0s28/b3mb7bdvrhlgYgMVr88Mms5J+k2Sz7RMlbbL9bJJtPa8NwCK12T74gySbm48/\nkbRd0pl9LwzA4h3WfXDbZ0taLumVPhYDoFutA7d9gqRHJd2U5OND/DnbBwMTplXgto/WXNz3J3ns\nUMewfTAwedo8im5J90januSO/pcEoCttzuAXSrpO0sW2tzS/ftzzugB0oM32wS9J8gBrAdAxXskG\nFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBhvW0fPPczKsMbc9visf7N0rhb6ErS5s2b\nR5t92223jTZ7LFNTU62O4wwOFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBhBA4U1mbj\ng2Ntv2p7a7N98O1DLAzA4rX5YZP/Sro4yf5mC6OXbP81ycs9rw3AIrXZ+CCS9jc3j25+jfcjWwBa\na7v54JTtLZL2SHo2CdsHA98ArQJPcjDJBZKWSVpl+/wvH8P2wcDkOaxH0ZPsk/S8pDWH+DO2DwYm\nTJtH0U+zPdN8fJykSyTt6HthABavzaPoSyXda3tKc98QHk7yZL/LAtCFNo+i/0PS8gHWAqBjvJIN\nKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCCBworLf9wccy5v7gY1q7du0RPX8sJ598\n8ihzlyxply5ncKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCmsdeLM/2Ru2eU904Bvi\ncM7g6yRt72shALrXdnfRZZIul3R3v8sB0KW2Z/A7Jd0s6bMe1wKgY202H7xC0p4kmxY4ju2DgQnT\n5gx+oaS1tt+T9JCki23f9+WD2D4YmDwLBp7k1iTLkpwt6WpJzyW5tveVAVg0ngcHCjust2xK8oKk\nF3pZCYDOcQYHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKK7d98FFHjfc965RT\nThlt9pVXXjna7LGtWLFitNl33XXXaLPb4AwOFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG\n4EBhBA4U1uq16M22RZ9IOihpNsnKPhcFoBuH88MmP0zyUW8rAdA5LtGBwtoGHknP2N5k+/pDHcD2\nwcDkaXuJvjrJbtvflvSs7R1JXpx/QJL1ktZL0hlnnJGO1wnga2h1Bk+yu/l9j6THJa3qc1EAurFg\n4LaPt33i5x9LulTSW30vDMDitblEP13S47Y/P/6BJE/1uioAnVgw8CQ7JX1vgLUA6BhPkwGFEThQ\nGIEDhRE4UBiBA4UROFAYgQOFEThQGIEDhRE4UFi57YPH1LxefxQbN24cbbYknXTSSaPN3rp162iz\n9+7dO8rc2dnZVsdxBgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwprFbjtGduP\n2N5he7vt7/e9MACL1/aHTX4v6akkP7V9jKTpHtcEoCMLBm77JEkXSfqZJCX5VNKn/S4LQBfaXKKf\nI+lDSX+y/Ybtu5s9yr6A7YOBydMm8CWSVkj6Y5Llkv4j6ZYvH5RkfZKVSVZOT3MFD0yCNoHvkrQr\nySvN7Uc0FzyACbdg4En+Lel92+c2n/qRpG29rgpAJ9o+iv4rSfc3j6DvlPTz/pYEoCutAk+yRdLK\nntcCoGO8kg0ojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcKcpPsvan8o6V9f86+fKumj\nDpfDbGZXnP2dJKctdFAvgS+G7deTjPK6d2Yzu9psLtGBwggcKGwSA1/PbGYzuxsTdx8cQHcm8QwO\noCMTFbjtNbbfsf2u7a+8c2uPczfY3mP7raFmzpt9lu3nbW+z/bbtdQPOPtb2q7a3NrNvH2r2vDVM\nNW/H/eTAc9+z/abtLbZfH3j2YDsFTcwluu0pSf+UdInm3sn1NUnXJOn9DR5tXyRpv6S/JDm/73lf\nmr1U0tIkm22fKGmTpJ8M9O+2pOOT7Ld9tKSXJK1L8nLfs+et4deaezuwbyW5YsC570lamWTw58Ft\n3yvpb0nu/nynoCT7+pg1SWfwVZLeTbKz2T3lIUlXDTE4yYuS9g4x6xCzP0iyufn4E0nbJZ050Owk\n2d/cPLr5Ndh3fNvLJF0u6e6hZo5t3k5B90hzOwX1Fbc0WYGfKen9ebd3aaD/0SeF7bMlLZf0yv8/\nstOZU7a3SNoj6dl5738/hDsl3SzpswFnfi6SnrG9yfb1A85ttVNQVyYp8COa7RMkPSrppiQfDzU3\nycEkF0haJmmV7UHuoti+QtKeJJuGmHcIq5OskHSZpF82d9OG0GqnoK5MUuC7JZ017/ay5nPlNfd/\nH5V0f5LHxlhDc5n4vKQ1A428UNLa5r7wQ5Iutn3fQLOVZHfz+x5Jj2vuLuIQBt0paJICf03Sd22f\n0zzwcLWkJ0ZeU++aB7rukbQ9yR0Dzz7N9kzz8XGae4BzxxCzk9yaZFmSszX33/q5JNcOMdv28c0D\nmmoujy+VNMgzKEPvFNR2Z5PeJZm1faOkpyVNSdqQ5O0hZtt+UNIPJJ1qe5ek3yW5Z4jZmjuTXSfp\nzea+sCT9NsnGAWYvlXRv8wzGUZIeTjLo01UjOV3S43PfW7VE0gNJnhpw/mA7BU3M02QAujdJl+gA\nOkbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGH/A1zt5RSYJEABAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from PIL import Image\n",
    "kernels = []\n",
    "for i, angle in enumerate(range(0, 180, 10)):\n",
    "    rotated = Image.Image.rotate(Image.fromarray(out), angle)\n",
    "    kernels.append(np.array(rotated))\n",
    "plt.imshow(kernels[2], cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conv2d we will convolve the input with the predefined kernel\n",
    "def conv2d_maxpool(x_tensor, kernels, pool_ksize, pool_strides):\n",
    "    \"\"\" Conv block\"\"\"\n",
    "    # convolutional layer\n",
    "    conv = tf.nn.conv2d(x_tensor, kernels, strides=[1, 1, 1, 1], padding='SAME')\n",
    "    # relu activation\n",
    "    conv = tf.nn.relu(conv)\n",
    "    # max pooling\n",
    "    conv = tf.nn.max_pool(conv, ksize = [1, pool_ksize[0], pool_ksize[1], 1],\n",
    "                         strides = [1, pool_strides[0], pool_strides[1], 1],\n",
    "                         padding = 'SAME')\n",
    "    return conv\n",
    "\n",
    "def flatten(x_tensor):\n",
    "    \"\"\" flatten the layers\"\"\"\n",
    "    size = x_tensor.get_shape().as_list()\n",
    "    x_tensor = tf.reshape(x_tensor, [-1, size[1]*size[2]*size[3]])\n",
    "    return x_tensor\n",
    "\n",
    "def fully_conn(x_tensor, num_outputs):\n",
    "    \"\"\" Fully connected layer\"\"\"\n",
    "    size = x_tensor.get_shape().as_list()\n",
    "    weights = tf.Variable(tf.truncated_normal((size[1], num_outputs), stddev=0.02, seed=SEED))\n",
    "    bias = tf.Variable(tf.zeros(num_outputs))\n",
    "    dense = tf.add(tf.matmul(x_tensor, weights), bias)\n",
    "    out = tf.nn.relu(dense)\n",
    "    return out\n",
    "\n",
    "def output(x_tensor, num_outputs):\n",
    "    size = x_tensor.get_shape().as_list()\n",
    "    weights = tf.Variable(tf.truncated_normal((size[1], num_outputs), stddev=0.02,seed=SEED))\n",
    "    bias = tf.Variable(tf.zeros(num_outputs))\n",
    "    dense = tf.add(tf.matmul(x_tensor, weights), bias)\n",
    "    return dense\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a simple convolution network\n",
    "def conv_net(x, kernels):\n",
    "    conv = conv2d_maxpool(x, kernels, (2,2), (2,2))\n",
    "    flat = flatten(conv)\n",
    "    dense = fully_conn(flat, 128)\n",
    "    out = output(dense, 10)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAADP9JREFUeJzt3VGIXPXZx/HfL9qA2CqJtctigklL\nFIpEW1apVjQlGtJQiL1QGrSmVLKCFVroRcVeVJCCFtvSGwtbDYmveW1fiKuh1NemoWgLGnYjVk1i\nEhsSu0tMKlaaothGn17Mid3GnTObmTNzZvf5fmDZmfPMmXk47G//58w5M39HhADkM6/uBgDUg/AD\nSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0jqzF6+mG0uJwS6LCI8k8d1NPLbXm17n+3XbN/VyXMB\n6C23e22/7TMk7Zd0vaQJSWOS1kXEnpJ1GPmBLuvFyH+FpNci4mBE/FPSLyWt7eD5APRQJ+G/QNJf\nptyfKJb9F9vDtsdtj3fwWgAq1vU3/CJiRNKIxG4/0E86GfknJS2ecn9RsQzALNBJ+MckLbO91PZ8\nSV+TtK2atgB0W9u7/RFxwvadkp6WdIakjRGxu7LOAHRV26f62noxjvmBruvJRT4AZi/CDyRF+IGk\nCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiB\npAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmp7im5Jsn1I0nFJ70s6ERFDVTQF\nVGHlypVNa1u2bCld99prry2t79u3r62e+klH4S98KSLerOB5APQQu/1AUp2GPyT91vYu28NVNASg\nNzrd7b86IiZtf0rSdtuvRsSzUx9Q/FPgHwPQZzoa+SNisvh9TNKopCumecxIRAzxZiDQX9oOv+2z\nbX/i5G1JqyS9UlVjALqrk93+AUmjtk8+z/9GxP9X0hWArms7/BFxUNKlFfbSVddcc01p/bzzziut\nj46OVtkOeuDyyy9vWhsbG+thJ/2JU31AUoQfSIrwA0kRfiApwg8kRfiBpKr4VN+ssGLFitL6smXL\nSuuc6us/8+aVj11Lly5tWrvwwgtL1y2uX5nTGPmBpAg/kBThB5Ii/EBShB9IivADSRF+IKk05/lv\nvfXW0vpzzz3Xo05QlcHBwdL6hg0bmtYeffTR0nVfffXVtnqaTRj5gaQIP5AU4QeSIvxAUoQfSIrw\nA0kRfiCpNOf5W332G7PPQw891Pa6Bw4cqLCT2YlEAEkRfiApwg8kRfiBpAg/kBThB5Ii/EBSLc/z\n294o6SuSjkXEJcWyhZJ+JWmJpEOSboqIv3WvzdaWL19eWh8YGOhRJ+iVc889t+11t2/fXmEns9NM\nRv5NklafsuwuSTsiYpmkHcV9ALNIy/BHxLOS3jpl8VpJm4vbmyXdUHFfALqs3WP+gYg4Utx+QxL7\n1MAs0/G1/RERtqNZ3fawpOFOXwdAtdod+Y/aHpSk4vexZg+MiJGIGIqIoTZfC0AXtBv+bZLWF7fX\nS3qymnYA9ErL8Nt+TNJzki62PWH7Nkn3Sbre9gFJ1xX3AcwiLY/5I2Jdk9LKinvpyJo1a0rrZ511\nVo86QVVaXZuxdOnStp97cnKy7XXnCq7wA5Ii/EBShB9IivADSRF+ICnCDyQ1Z766++KLL+5o/d27\nd1fUCarywAMPlNZbnQrcv39/09rx48fb6mkuYeQHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTmzHn+\nTo2NjdXdwqx0zjnnlNZXrz71i5//45Zbbildd9WqVW31dNK9997btPb222939NxzASM/kBThB5Ii\n/EBShB9IivADSRF+ICnCDyTFef7CwoULa3vtSy+9tLRuu7R+3XXXNa0tWrSodN358+eX1m+++ebS\n+rx55ePHu+++27S2c+fO0nXfe++90vqZZ5b/+e7atau0nh0jP5AU4QeSIvxAUoQfSIrwA0kRfiAp\nwg8k5Ygof4C9UdJXJB2LiEuKZfdI2iDpr8XD7o6I37R8Mbv8xTrw4IMPltZvv/320nqrz3e//vrr\np93TTC1fvry03uo8/4kTJ5rW3nnnndJ19+zZU1pvdS5+fHy8tP7MM880rR09erR03YmJidL6ggUL\nSuutrmGYqyKi/A+mMJORf5Ok6b6R4acRcVnx0zL4APpLy/BHxLOS3upBLwB6qJNj/jttv2R7o+3y\n/S8Afafd8P9c0mckXSbpiKQfN3ug7WHb47bLDw4B9FRb4Y+IoxHxfkR8IOkXkq4oeexIRAxFxFC7\nTQKoXlvhtz045e5XJb1STTsAeqXlR3ptPyZphaRP2p6Q9ANJK2xfJikkHZJUfh4NQN9pGf6IWDfN\n4oe70EtH7rjjjtL64cOHS+tXXXVVle2cllbXEDzxxBOl9b179zatPf/882311AvDw8Ol9fPPP7+0\nfvDgwSrbSYcr/ICkCD+QFOEHkiL8QFKEH0iK8ANJpfnq7vvvv7/uFnCKlStXdrT+1q1bK+okJ0Z+\nICnCDyRF+IGkCD+QFOEHkiL8QFKEH0gqzXl+zD2jo6N1tzCrMfIDSRF+ICnCDyRF+IGkCD+QFOEH\nkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUi0/z297saRHJA1ICkkjEfEz2wsl/UrS\nEkmHJN0UEX/rXqvIxnZp/aKLLiqt9/P05P1gJiP/CUnfjYjPSvqCpG/Z/qykuyTtiIhlknYU9wHM\nEi3DHxFHIuKF4vZxSXslXSBpraTNxcM2S7qhW00CqN5pHfPbXiLpc5J2ShqIiCNF6Q01DgsAzBIz\n/g4/2x+XtFXSdyLi71OPxyIibEeT9YYlDXfaKIBqzWjkt/0xNYK/JSIeLxYftT1Y1AclHZtu3YgY\niYihiBiqomEA1WgZfjeG+Icl7Y2In0wpbZO0vri9XtKT1bcHoFtmstv/RUlfl/Sy7ReLZXdLuk/S\n/9m+TdJhSTd1p0VkFTHtkeSH5s3jMpVOtAx/RPxRUrMTrp1NsA6gNvzrBJIi/EBShB9IivADSRF+\nICnCDyTFFN2Yta688srS+qZNm3rTyCzFyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSXGeH32r1Vd3\nozOM/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOf5UZunnnqqtH7jjTf2qJOcGPmBpAg/kBThB5Ii\n/EBShB9IivADSRF+ICm3mgPd9mJJj0gakBSSRiLiZ7bvkbRB0l+Lh94dEb9p8VzlLwagYxExoy9C\nmEn4ByUNRsQLtj8haZekGyTdJOkfEfHATJsi/ED3zTT8La/wi4gjko4Ut4/b3ivpgs7aA1C30zrm\nt71E0uck7SwW3Wn7JdsbbS9oss6w7XHb4x11CqBSLXf7P3yg/XFJz0j6YUQ8bntA0ptqvA9wrxqH\nBt9s8Rzs9gNdVtkxvyTZ/pikX0t6OiJ+Mk19iaRfR8QlLZ6H8ANdNtPwt9ztd+MrVB+WtHdq8Is3\nAk/6qqRXTrdJAPWZybv9V0v6g6SXJX1QLL5b0jpJl6mx239I0u3Fm4Nlz8XID3RZpbv9VSH8QPdV\nttsPYG4i/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJNXrKbrf\nlHR4yv1PFsv6Ub/21q99SfTWrip7u3CmD+zp5/k/8uL2eEQM1dZAiX7trV/7kuitXXX1xm4/kBTh\nB5KqO/wjNb9+mX7trV/7kuitXbX0VusxP4D61D3yA6hJLeG3vdr2Ptuv2b6rjh6asX3I9su2X6x7\nirFiGrRjtl+Zsmyh7e22DxS/p50mrabe7rE9WWy7F22vqam3xbZ/b3uP7d22v10sr3XblfRVy3br\n+W6/7TMk7Zd0vaQJSWOS1kXEnp420oTtQ5KGIqL2c8K2r5H0D0mPnJwNyfaPJL0VEfcV/zgXRMT3\n+qS3e3SaMzd3qbdmM0t/QzVuuypnvK5CHSP/FZJei4iDEfFPSb+UtLaGPvpeRDwr6a1TFq+VtLm4\nvVmNP56ea9JbX4iIIxHxQnH7uKSTM0vXuu1K+qpFHeG/QNJfptyfUH9N+R2Sfmt7l+3hupuZxsCU\nmZHekDRQZzPTaDlzcy+dMrN032y7dma8rhpv+H3U1RHxeUlflvStYve2L0XjmK2fTtf8XNJn1JjG\n7YikH9fZTDGz9FZJ34mIv0+t1bntpumrlu1WR/gnJS2ecn9RsawvRMRk8fuYpFE1DlP6ydGTk6QW\nv4/V3M+HIuJoRLwfER9I+oVq3HbFzNJbJW2JiMeLxbVvu+n6qmu71RH+MUnLbC+1PV/S1yRtq6GP\nj7B9dvFGjGyfLWmV+m/24W2S1he310t6ssZe/ku/zNzcbGZp1bzt+m7G64jo+Y+kNWq84/9nSd+v\no4cmfX1a0p+Kn9119ybpMTV2A/+lxnsjt0k6T9IOSQck/U7Swj7q7X/UmM35JTWCNlhTb1ersUv/\nkqQXi581dW+7kr5q2W5c4QckxRt+QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeS+jePVgFoos9Y\nrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# The images are in a single vector of 784 (28*28). need to reshape to convert it to a image\n",
    "im = np.reshape(x_train[2],[28,28])\n",
    "plt.imshow(im, cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hand coded kernels for the convolution\n",
    "\n",
    "> Here we use the handcoded gabor filters for the convolution.\n",
    "> The fully connected layers are learned through backpropagation through."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 2900, training accuracy 0.5996 \n",
      " Test accuracy 0.5997\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Feed batch data\n",
    "def get_batch(inputX, inputY, batch_size):\n",
    "    duration = len(inputX)\n",
    "    for i in range(0,duration//batch_size):\n",
    "        idx = i*batch_size\n",
    "        yield inputX[idx:idx+batch_size]/255., inputY[idx:idx+batch_size]\n",
    "        \n",
    "\n",
    "# Input placeholders\n",
    "x = tf.placeholder(tf.float32,(None, 28,28,1), name='x')\n",
    "\n",
    "y = tf.placeholder(tf.int32,(None), name='y' )\n",
    "# convert to one hot encoding\n",
    "y_one_hot = tf.one_hot(y,10)\n",
    "\n",
    "\n",
    "# The kernel weights are fixed\n",
    "kernels = np.reshape(kernels,[7,7,1,18])\n",
    "weights = tf.constant(kernels, dtype=tf.float32)\n",
    "\n",
    "# un normalized log propabilities\n",
    "logits = conv_net(x, weights)\n",
    "\n",
    "# cost and the minimizers\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y_one_hot))\n",
    "optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "\n",
    "#get the predictions\n",
    "correct_pred = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32), y)\n",
    "\n",
    "#calculate the accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# run the graph in a session\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for i in range(3000):\n",
    "        batch_x, batch_y = next(get_batch(x_train, y_train, 50))\n",
    "        optimizer.run(feed_dict={x: batch_x[...,None], y: batch_y})\n",
    "        if i%100 == 0:\n",
    "            train_accuracy = accuracy.eval(feed_dict={x:x_test[...,None], y:y_test})\n",
    "            print(\"step %d, training accuracy %g \"%(i, train_accuracy), end ='\\r', flush=True)\n",
    "    accuracy = sess.run(accuracy, feed_dict={x:x_test[...,None], y:y_test})\n",
    "    print(\"\\n Test accuracy %g\"%accuracy)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "********\n",
    "We get a accuracy of close to 0.6 with the hand-coded kernel approach.\n",
    "Now instead of predefining our kernel, let the neural network find its own kernel through backpropagration.\n",
    "********"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## kernel weights are learned using back propagation\n",
    "\n",
    "> Here all the weights of filters for the convolution are learned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv2d_maxpool_w(x_tensor, conv_num_outputs, conv_ksize, conv_strides, pool_ksize, pool_strides):\n",
    "    \n",
    "    in_channels = x_tensor.get_shape().as_list()[3]\n",
    "    \n",
    "    weight = tf.Variable(tf.truncated_normal((conv_ksize[0], conv_ksize[1], in_channels, conv_num_outputs), stddev=0.1,seed=SEED))\n",
    "\n",
    "    bias = tf.Variable(tf.zeros(conv_num_outputs))\n",
    "    \n",
    "    \n",
    "    conv = tf.nn.conv2d(x_tensor, \n",
    "                        weight, \n",
    "                        strides=[1, conv_strides[0], conv_strides[1], 1],\n",
    "                        padding = 'SAME')\n",
    "    conv = tf.nn.bias_add(conv, bias)\n",
    "    conv = tf.nn.relu(conv) \n",
    "    conv_maxpool = tf.nn.max_pool(conv,\n",
    "                                  ksize = [1, pool_ksize[0], pool_ksize[1], 1],\n",
    "                                  strides = [1, pool_strides[0], pool_strides[1], 1],\n",
    "                                  padding = 'SAME')\n",
    "    return conv_maxpool\n",
    "\n",
    "# define a simple convolution network\n",
    "def conv_net_w(x):\n",
    "    conv = conv2d_maxpool_w(x, 18, (7,7),(1,1),(2,2),(2,2))\n",
    "    flat = flatten(conv)\n",
    "    dense = fully_conn(flat, 128)\n",
    "    out = output(dense, 10)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 2900, training accuracy 0.675  \n",
      " Test accuracy 0.675\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Input placeholders\n",
    "# Input placeholders\n",
    "x = tf.placeholder(tf.float32,(None, 28,28,1), name='x')\n",
    "\n",
    "y = tf.placeholder(tf.int32,(None), name='y' )\n",
    "# convert to one hot encoding\n",
    "y_one_hot = tf.one_hot(y,10)\n",
    "\n",
    "# get the unormalized probabilities\n",
    "logits = conv_net_w(x)\n",
    "\n",
    "# cost and solvers\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y_one_hot))\n",
    "optimizer = tf.train.GradientDescentOptimizer(.5).minimize(loss)\n",
    "\n",
    "# Get the correct predictions\n",
    "correct_pred = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32), y)\n",
    "# Obtain the accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# run the graph in a session\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for i in range(3000):\n",
    "        batch_x, batch_y = next(get_batch(x_train, y_train, 50))\n",
    "        optimizer.run(feed_dict={x: batch_x[...,None], y: batch_y})\n",
    "        if i%100 == 0:\n",
    "            train_accuracy = accuracy.eval(feed_dict={x:x_test[...,None], y:y_test})\n",
    "            print(\"step %d, training accuracy %g \"%(i, train_accuracy), end ='\\r', flush=True)\n",
    "    accuracy = sess.run(accuracy, feed_dict={x:x_test[...,None], y:y_test})\n",
    "    print(\"\\n Test accuracy %g\"%accuracy)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "In this case we get a Acuracy of 0.67.\n",
    "So instead of handcoding the input features, learning the filters through backpropogation is the best way!."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-- EOF --"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}