deep_learning/space_image/BasicCNN_Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Convolutional Layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "```python\n",
    "\"\"\"\n",
    "input: N x h x w x d \n",
    "  N: batch size\n",
    "  h: height\n",
    "  w: width\n",
    "  d: depth (channel)\n",
    "\n",
    "filter: e x e x d x k\n",
    "  e: filter (or kernel) width / height, 3 x 3 or 5 x 5\n",
    "  d: depth is the same with input's d\n",
    "  k: kernel sizes\n",
    "  \n",
    "padding:\n",
    "  SAME: with zero padding\n",
    "  VALID: without padding\n",
    "\"\"\"\n",
    "tf.nn.conv2d(input=input, \\\n",
    "             filter=filter, \\\n",
    "             strides=strides, \\\n",
    "             padding=padding, \\\n",
    "             use_cudnn_on_gpu=True, \\\n",
    "             name=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Pooling Layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Max Pooling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "```python\n",
    "\"\"\"\n",
    "value: N x w x h x d\n",
    "\n",
    "ksize: [1, k, k, 1]\n",
    "  k: kernel size\n",
    "\n",
    "strides: [1, s, s, 1]\n",
    "  s: stride size\n",
    "  \n",
    "padding: SAME or VALID\n",
    "\"\"\"\n",
    "tf.nn.max_pool(value=input, \\\n",
    "               ksize=[1,k,k,1], \\\n",
    "               strides=[1,s,s,1], \\\n",
    "               padding=\"SAME\", \\\n",
    "               name=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Conv2D:0\", shape=(1, 5, 5, 7), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# SAME vs. VALID\n",
    "input = tf.Variable(tf.random_normal([1,5,5,5]))\n",
    "filter = tf.Variable(tf.random_normal([3,3,5,7]))\n",
    "conv = tf.nn.conv2d(input=input, filter=filter, strides=[1,1,1,1], padding=\"SAME\")\n",
    "print(conv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# MNIST Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def conv2d(input, weight_shape, bias_shape):\n",
    "    in_count = weight_shape[0] * weight_shape[1] * weight_shape[2]\n",
    "    weight_init = tf.random_normal_initializer(stddev=(2.0/in_count)**0.5)\n",
    "    W = tf.get_variable(\"W\", weight_shape, initializer=weight_init)\n",
    "    bias_init = tf.constant_initializer(value=0)\n",
    "    b = tf.get_variable(\"b\", bias_shape, initializer=bias_init)\n",
    "    conv_out = tf.nn.conv2d(input, W, strides=[1,1,1,1], padding=\"SAME\")\n",
    "    return tf.nn.relu(tf.nn.bias_add(conv_out, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def max_pool(input, k=2):\n",
    "    return tf.nn.max_pool(value=input, ksize=[1,k,k,1], strides=[1,k,k,1], padding=\"SAME\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def layer(input, weight_shape, bias_shape):\n",
    "    in_count = weight_shape[0] * weight_shape[1]\n",
    "    w_init = tf.random_normal_initializer(stddev=(2.0/in_count)**0.5)\n",
    "    b_init = tf.constant_initializer(value=0)\n",
    "    W = tf.get_variable(\"W\", weight_shape, initializer=w_init)\n",
    "    b = tf.get_variable(\"b\", bias_shape, initializer=b_init)\n",
    "    return tf.nn.relu(tf.matmul(input, W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def loss(output, y):\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=output, labels=y)\n",
    "    return tf.reduce_mean(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def inference(input, keep_prob=0.5):\n",
    "    x = tf.reshape(input, [-1, 28, 28, 1])\n",
    "    \n",
    "    with tf.variable_scope(\"conv_1\"):\n",
    "        conv_1 = conv2d(x, [5, 5, 1, 32], [32])                # conv_1: 28 x 28 x 32\n",
    "        pool_1 = max_pool(conv_1)                              # pool_1: 14 x 14 x 32\n",
    "    \n",
    "    with tf.variable_scope(\"conv_2\"):\n",
    "        conv_2 = conv2d(pool_1, [5, 5, 32, 64], [64])          # conv_2: 14 x 14 x 64\n",
    "        pool_2 = max_pool(conv_2)                              # pool_2: 7 x 7 x 64\n",
    "        \n",
    "    with tf.variable_scope(\"fc\"):\n",
    "        pool_2_flat = tf.reshape(pool_2, [-1, 7 * 7 * 64])     # from pool_2\n",
    "        #pool_2_flat = tf.layers.flatten()                       # similar with the above line\n",
    "        fc_1 = layer(pool_2_flat, [7 * 7 * 64, 1024], [1024])   # fc_1: 1024\n",
    "        \n",
    "        # dropout\n",
    "        fc_1_dropout = tf.nn.dropout(fc_1, keep_prob=keep_prob)\n",
    "        \n",
    "    with tf.variable_scope(\"out\"):\n",
    "        out = layer(fc_1_dropout, [1024, 10], [10])\n",
    "    \n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def training(cost, global_step):\n",
    "    tf.summary.scalar(\"cost\", cost)\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=9e-1, beta2=0.999, epsilon=1e-8, name=\"Adam\")\n",
    "    train_opt = optimizer.minimize(cost, global_step=global_step)\n",
    "    return train_opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def evaluate(output, y, name):\n",
    "    compare = tf.equal(tf.argmax(output, axis=1), tf.argmax(y, axis=1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(compare, tf.float32))\n",
    "    tf.summary.scalar(\"eval/{}\".format(str(name)), accuracy)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "full_training = True\n",
    "learning_rate = 1e-2\n",
    "training_epochs = 1000 if full_training else 1\n",
    "batch_size = 100 if full_training else 1000\n",
    "display_step = 1000 if full_training else 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-11-4e39c927863c>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /Users/jiankaiwang/Desktop/devops/tmp/MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /Users/jiankaiwang/Desktop/devops/tmp/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /Users/jiankaiwang/Desktop/devops/tmp/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /Users/jiankaiwang/Desktop/devops/tmp/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/Users/jiankaiwang/Desktop/devops/tmp/MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /Users/jiankaiwang/Desktop/devops/tmp/basic_tf/basic_cnn/model-checkpoint-1017\n",
      "Load pretrained.\n",
      "Step: 1018 Accuracy: 0.7738 Vaildation Error: 0.22619998455047607\n",
      "Step: 1019 Accuracy: 0.7694 Vaildation Error: 0.2305999994277954\n",
      "Step: 1020 Accuracy: 0.7722 Vaildation Error: 0.22780001163482666\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-01ece064aa31>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     58\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdisplay_step\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m                         \u001b[0mfeed_dict_val_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m                         \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval_opt_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict_val_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m                         \u001b[0mstep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mglobal_step\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m                         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Step:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Accuracy:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0macc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Vaildation Error:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0macc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    898\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 900\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    901\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1135\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1316\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1317\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1320\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1322\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1323\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1324\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1305\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1307\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1407\u001b[0m       return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1409\u001b[0;31m           run_metadata)\n\u001b[0m\u001b[1;32m   1410\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1411\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "with tf.Graph().as_default():\n",
    "    with tf.variable_scope(\"cnn_model\") as scope:\n",
    "        x = tf.placeholder(\"float\", [None, 784])\n",
    "        y = tf.placeholder(\"float\", [None, 10])\n",
    "        \n",
    "        output = inference(x)\n",
    "        cost = loss(output=output, y=y)\n",
    "        global_step = tf.Variable(initial_value=0, name=\"global_step\", trainable=False)\n",
    "        train_opt = training(cost=cost, global_step=global_step)\n",
    "        eval_opt_val = evaluate(output=output, y=y, name=\"valid\")\n",
    "        eval_opt_test = evaluate(output=output, y=y, name=\"test\")\n",
    "        \n",
    "        init_var = tf.global_variables_initializer()\n",
    "        summary_opt = tf.summary.merge_all()\n",
    "        saver = tf.train.Saver()\n",
    "        \n",
    "        with tf.Session() as sess:\n",
    "            \n",
    "            model_path = \"/Users/jiankaiwang/Desktop/devops/tmp/basic_tf/basic_cnn/\"\n",
    "            ckpt_log = os.path.join(model_path, \"checkpoint\")\n",
    "            if os.path.isdir(model_path) and os.path.isfile(ckpt_log):\n",
    "                #ckpt_path = \"/Users/jiankaiwang/Desktop/devops/tmp/basic_tf/basic_cnn/model-checkpoint-{}\".format(restrore_idx)\n",
    "                ckpt_path = \"\"\n",
    "                with open(ckpt_log, \"r\") as fin:\n",
    "                    for line in fin:\n",
    "                        ckpt_path += line.strip()   # only read the first line\n",
    "                        break\n",
    "                # model_checkpoint_path: \"/Users/jiankaiwang/Desktop/devops/tmp/basic_tf/basic_cnn/model-checkpoint-1020\"\n",
    "                ckpt_path = ckpt_path[24:-1]\n",
    "                \n",
    "                if os.path.isfile(ckpt_path + \".meta\"):\n",
    "                    scope.reuse_variables()\n",
    "                    var_list_opt = [\"conv_1/W\", \"conv_1/b\", \"conv_2/W\", \"conv_2/b\", \"fc/W\", \"fc/b\", \"out/W\", \"out/b\"]\n",
    "                    var_list_opt = [tf.get_variable(v) for v in var_list_opt]\n",
    "                    saver.restore(sess, ckpt_path)\n",
    "                    print(\"Load pretrained.\")\n",
    "                else:\n",
    "                    # no need to initialize if loading pretrained data\n",
    "                    sess.run(init_var)\n",
    "            else:\n",
    "                # no need to initialize if loading pretrained data\n",
    "                sess.run(init_var)\n",
    "            \n",
    "            summary_writer = tf.summary.FileWriter(\\\n",
    "                    \"/Users/jiankaiwang/Desktop/devops/tmp/basic_tf/basic_cnn\", \\\n",
    "                    graph=sess.graph)\n",
    "            \n",
    "            for epoch in range(training_epochs):\n",
    "                avg_cost = 0.\n",
    "                total_batch = int(mnist.train.num_examples / batch_size)\n",
    "                \n",
    "                for idx in range(total_batch):\n",
    "                    b_x, b_y = mnist.train.next_batch(batch_size=batch_size)\n",
    "                    \n",
    "                    feed_dict_data = {x: b_x, y: b_y}\n",
    "                    sess.run(train_opt, feed_dict=feed_dict_data)\n",
    "                    \n",
    "                    batch_cost = sess.run(cost, feed_dict=feed_dict_data)\n",
    "                    avg_cost += (batch_cost / total_batch)\n",
    "                    \n",
    "                if epoch % display_step == 0:\n",
    "                    feed_dict_val_data = {x: mnist.validation.images, y: mnist.validation.labels}\n",
    "                    acc = sess.run(eval_opt_val, feed_dict=feed_dict_val_data)\n",
    "                    step = sess.run(global_step)\n",
    "                    print(\"Step:\", step, \"Accuracy:\", acc, \"Vaildation Error:\", (1-acc))\n",
    "                    tf.summary.scalar(\"Validation_accuracy\", acc)\n",
    "\n",
    "                    summary_str = sess.run(summary_opt, feed_dict = feed_dict_data)\n",
    "                    summary_writer.add_summary(summary_str, step)\n",
    "\n",
    "                    saver.save(sess, \\\n",
    "                               \"/Users/jiankaiwang/Desktop/devops/tmp/basic_tf/basic_cnn/model-checkpoint\", \\\n",
    "                              global_step=global_step)\n",
    "            \n",
    "            print(\"Training finished.\")\n",
    "            \n",
    "            feed_dict_test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "            acc = sess.run(eval_opt_test, feed_dict=feed_dict_test_data)\n",
    "            print(\"Test accuracy:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Image Pre-processing "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "```python\n",
    "tf.image.per_image_standardization(image)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Image Data Augmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "```python\n",
    "tf.random_crop()\n",
    "tf.image.transpose_image()\n",
    "tf.image.random_flip_up_down()\n",
    "tf.image.random_flip_left_right()\n",
    "tf.image.random_brightness()\n",
    "tf.image.random_contrast()\n",
    "tf.image.random_saturation()\n",
    "tf.image.random_hue()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Batch Normalizarion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.ops import control_flow_ops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## BN for Convolutional Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "# example for control_flow_ops.cond\n",
    "# similar with if-else\n",
    "\n",
    "a = tf.constant(1)\n",
    "b = tf.constant(10)\n",
    "\n",
    "condition = tf.convert_to_tensor(False, dtype='bool')\n",
    "test = control_flow_ops.cond(condition, lambda: a, lambda: b)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def conv_batch_norm(x, n_out, phase_train):\n",
    "    beta_init = tf.constant_initializer(value=0.0, dtype=tf.float32)\n",
    "    gamma_init = tf.constant_initializer(value=1.0, dtype=tf.float32)\n",
    "    \n",
    "    beta = tf.get_variable(\"beta\", [n_out], initializer=beta_init)\n",
    "    gamma = tf.get_variable(\"alpha\", [n_out], initializer=gamma_init)\n",
    "    \n",
    "    batch_mean, batch_var = tf.nn.moments(x, len(x.shape())-1, name='moments')\n",
    "    \n",
    "    ema = tf.train.ExponentialMovingAverage(decay=0.9)\n",
    "    ema_apply_op = ema.apply([batch_mean, batch_var])\n",
    "    ema_mean, ema_var = ema.average(batch_mean), ema.average(batch_var)\n",
    "    \n",
    "    def mean_var_with_update():\n",
    "        with tf.control_dependencies([ema_apply_op]):\n",
    "            return tf.identity(batch_mean), tf.identity(batch_var)\n",
    "    \n",
    "    mean, var = control_flow_ops.cond(phase_train, mean_var_with_update, lambda: (ema_mean, ema_var))\n",
    "    \n",
    "    normed = tf.nn.batch_norm_with_global_normalization(x, mean, var, beta, gamma, 1e-3, True)\n",
    "    return normed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def BN_conv2d(input, weight_shape, bias_shape, phase_train):\n",
    "    in_count = weight_shape[0] * weight_shape[1] * weight_shape[2]\n",
    "    weight_init = tf.random_normal_initializer(stddev=(2.0/in_count)**0.5)\n",
    "    W = tf.get_variable(\"W\", weight_shape, initializer=weight_init)\n",
    "    bias_init = tf.constant_initializer(value=0)\n",
    "    b = tf.get_variable(\"b\", bias_shape, initializer=bias_init)\n",
    "    conv_out = tf.nn.conv2d(input, W, strides=[1,1,1,1], padding=\"SAME\")\n",
    "    logits = tf.nn.bias_add(conv_out, b)\n",
    "    return tf.nn.relu(conv_batch_norm(logits, weight_shape[3], phase_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## BN for Non-Convolutional Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def layer_batch_norm(x, n_out, phase_train):\n",
    "    beta_init = tf.constant_initializer(value=0.0, dtype=tf.float32)\n",
    "    gamma_init = tf.constant_initializer(value=1.0, dtype=tf.float32)\n",
    "    \n",
    "    beta = tf.get_variable(\"beta\", [n_out], initializer=beta_init)\n",
    "    gamma = tf.get_variable(\"alpha\", [n_out], initializer=gamma_init)\n",
    "    \n",
    "    batch_mean, batch_var = tf.nn.moments(x, len(x.shape())-1, name='moments')\n",
    "    \n",
    "    ema = tf.train.ExponentialMovingAverage(decay=0.9)\n",
    "    ema_apply_op = ema.apply([batch_mean, batch_var])\n",
    "    ema_mean, ema_var = ema.average(batch_mean), ema.average(batch_var)\n",
    "    \n",
    "    def mean_var_with_update():\n",
    "        with tf.control_dependencies([ema_apply_op]):\n",
    "            return tf.identity(batch_mean), tf.identity(batch_var)\n",
    "    \n",
    "    mean, var = control_flow_ops.cond(phase_train, mean_var_with_update, lambda: (ema_mean, ema_var))\n",
    "    \n",
    "    x_r = tf.reshape(x, [-1, 1, 1, n_out])\n",
    "    normed = tf.nn.batch_norm_with_global_normalization(x_r, mean, var, beta, gamma, 1e-3, True)\n",
    "    return tf.reshape(normed, [-1, n_out])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def BN_layer(input, weight_shape, bias_shape, phase_train):\n",
    "    in_count = weight_shape[0] * weight_shape[1]\n",
    "    w_init = tf.random_normal_initializer(stddev=(2.0/in_count)**0.5)\n",
    "    b_init = tf.constant_initializer(value=0)\n",
    "    W = tf.get_variable(\"W\", weight_shape, initializer=w_init)\n",
    "    b = tf.get_variable(\"b\", bias_shape, initializer=b_init)\n",
    "    logits = tf.matmul(input, W) + b\n",
    "    return tf.nn.relu(layer_batch_norm(logits, weight_shape[1], phase_train))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}