deep_learning/time_series_sequences/RNN_LSTM_Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version: 1.9.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "print(\"Tensorflow version: {}\".format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN in Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Cell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "basicRNNCell = tf.nn.rnn_cell.BasicRNNCell(num_units=num_units, input_size=None, activation=tanh)\n",
    "\n",
    "basicLSTMCell = tf.nn.rnn_cell.BasicLSTMCell(num_units=num_units, input_size=None, activation=tanh, \\\n",
    "                                             forget_bias=1.0,  state_is_tuple=True)\n",
    "\n",
    "LSTMCell = tf.nn.rnn_cell.LSTMCell(num_units=num_units, input_size=None, activation=tanh, \\\n",
    "                                  forget_bias=1.0, state_is_tuple=True, \\\n",
    "                                  use_peepholes=False, cell_clip=None, initializer=None, num_proj=None, \\\n",
    "                                  num_unit_shards=1, num_proj_shards=1)\n",
    "\n",
    "GRUCell = tf.nn.rnn_cell.GRUCell(num_units=num_units, input_size=None, activation=tanh)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "# create LSTM cell\n",
    "cell_layer_1 = tf.nn.rnn_cell.BasicLSTMCell(10)\n",
    "\n",
    "# create another LSTM Cell\n",
    "cell_layer_2 = tf.nn.rnn_cell.BasicLSTMCell(20)\n",
    "tf.nn.rnn_cell.DropoutWrapper(cell_layer_2, input_keep_prob=1.0, output_keep_prob=1.0, seed=None)   # dropout\n",
    "\n",
    "# assemble the rnn layer\n",
    "full_cell = tf.nn.rnn_cell.MultiRNNCell([cell_layer_1, cell_layer_2])\n",
    "\n",
    "# assemble the rnn network\n",
    "# input@time_major = F, inputs = [batch_size, max_time, ...], outputs = [batch_size, max_time, cell.output_size]\n",
    "#                    T, inputs = [max_time, batch_size, ...], outputs = [max_time, batch_size, cell.output_size]\n",
    "# state = [batch_size, cell.state_size]\n",
    "outputs, state = tf.nn.dynamic_rnn(full_cell, inputs=inputs, sequence_length=None, dtype=None, \\\n",
    "                                   parallel_iterations=None, swap_memory=False, time_major=False, scope=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Emotion Analyzing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tflearn in /anaconda3/lib/python3.6/site-packages (0.3.2)\n",
      "Requirement already satisfied: Pillow in /anaconda3/lib/python3.6/site-packages (from tflearn) (5.1.0)\n",
      "Requirement already satisfied: numpy in /anaconda3/lib/python3.6/site-packages (from tflearn) (1.14.3)\n",
      "Requirement already satisfied: six in /anaconda3/lib/python3.6/site-packages (from tflearn) (1.11.0)\n",
      "\u001b[33mYou are using pip version 18.0, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install tflearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tflearn                            0.3.2    \n",
      "\u001b[33mYou are using pip version 18.0, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import tflearn\n",
    "\n",
    "!pip list | grep 'tflearn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tflearn.data_utils import to_categorical, pad_sequences\n",
    "from tflearn.datasets import imdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# n_word=30000, most common used 300000 word\n",
    "if os.path.exists('/notebooks/data'):\n",
    "    train, test, _ = imdb.load_data('/notebooks/data/imdb/imdb.pkl', n_words=30000, valid_portion=0.1)\n",
    "elif os.path.exists('/Volumes/Data'):\n",
    "    train, test, _ = imdb.load_data('/Volumes/Data/imdb/imdb.pkl', n_words=30000, valid_portion=0.1)\n",
    "else:\n",
    "    raise IOError(\"no such path\")\n",
    "\n",
    "# trainX, testX: the sequence\n",
    "# trainY, testY: the emotion label (positive/negative, 1/0)\n",
    "trainX, trainY = train\n",
    "testX, testY = test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# expand sequence to length of 500 with value = 0 if the sequence length <= 500\n",
    "# truncated sequence to length of 500 if the sequence length > 500\n",
    "trainX = pad_sequences(trainX, maxlen=500, value=0.)\n",
    "testX = pad_sequences(testX, maxlen=500, value=0.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# change label to one-hot encoding\n",
    "trainY = to_categorical(trainY, nb_classes=2)\n",
    "testY = to_categorical(testY, nb_classes=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  16,  586,   32,  885,   17,   39,   68,   31, 2994, 2389,  328,\n",
       "          4,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0], dtype=int32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a simple example\n",
    "trainX[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data loading method\n",
    "class IMDBDataset():\n",
    "    def __init__(self, X, Y):\n",
    "        self.data_count = len(X)\n",
    "        self.input = X\n",
    "        self.label = Y\n",
    "        self.ptr = 0\n",
    "        \n",
    "    def minibatch(self, size):\n",
    "        ret = None\n",
    "        \n",
    "        if self.ptr + size < self.data_count:\n",
    "            ret = self.input[self.ptr:(self.ptr + size)], self.label[self.ptr:(self.ptr + size)]\n",
    "        else:\n",
    "            ret = np.concatenate((self.input[self.ptr:], self.input[:size-len(self.input[self.ptr:])])), \\\n",
    "                  np.concatenate((self.label[self.ptr:], self.label[:size-len(self.label[self.ptr:])]))\n",
    "        \n",
    "        self.ptr = (self.ptr + size) % (self.data_count)\n",
    "        return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = IMDBDataset(trainX, trainY)\n",
    "test = IMDBDataset(testX, testY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log path: /Users/jiankaiwang/devops/tmp/emotion_log\n"
     ]
    }
   ],
   "source": [
    "full_training = False\n",
    "learning_rate = 1e-2\n",
    "training_epochs = 100 if full_training else 1\n",
    "batch_size = 2000 if full_training else 1000\n",
    "display_step = 1\n",
    "\n",
    "if os.path.exists(os.path.join(\"/\",\"notebooks\",\"devops\")):\n",
    "    log_path = os.path.join(\"/\",\"notebooks\",\"devops\",\"tmp\",\"emotion_log\")\n",
    "elif os.path.exists(os.path.join(\"/\",\"Users\",\"jiankaiwang\",\"devops\")):\n",
    "    log_path = os.path.join(\"/\",\"Users\",\"jiankaiwang\",\"devops\",\"tmp\",\"emotion_log\")\n",
    "else:\n",
    "    raise IOError(\"no such log path\")\n",
    "print(\"log path: {}\".format(log_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def embedding_layer(input, weight_shape):\n",
    "    weight_init = tf.random_normal_initializer(stddev=(1.0/weight_shape[0])**0.5)\n",
    "    E = tf.get_variable(\"E\", weight_shape, initializer=weight_init)\n",
    "    incoming = tf.cast(input, tf.int32)\n",
    "    embeddings = tf.nn.embedding_lookup(E, incoming)\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm(input, hidden_dim, keep_prob, phase_train):\n",
    "    lstm = tf.nn.rnn_cell.BasicLSTMCell(hidden_dim)\n",
    "    dropout_lstm = tf.nn.rnn_cell.DropoutWrapper(lstm, input_keep_prob=keep_prob, output_keep_prob=keep_prob)\n",
    "    lstm_outputs, state = tf.nn.dynamic_rnn(dropout_lstm, inputs=input, dtype=tf.float32)\n",
    "    \n",
    "    # (batch_size, max_time, cell.output_size) -> (batch_size, 1, cell.output_size)\n",
    "    slice_output = tf.slice(lstm_outputs, \\\n",
    "            [0, tf.shape(lstm_outputs)[1]-1, 0], [tf.shape(lstm_outputs)[0], 1, tf.shape(lstm_outputs)[1]])\n",
    "    \n",
    "    # (batch_size, 1, cell.output_size) -> (batch_size, cell.output_size)\n",
    "    squeeze_output = tf.squeeze(slice_output)\n",
    "    \n",
    "    return squeeze_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dense(input, weight_shape, bias_shape, phase_train):\n",
    "    in_count = weight_shape[0] * weight_shape[1]\n",
    "    weight_init = tf.random_normal_initializer(stddev=(2/in_count)**0.5)\n",
    "    bias_init = tf.constant_initializer(value=0)\n",
    "    weight = tf.get_variable(\"W\", weight_shape, initializer=weight_init)\n",
    "    bias = tf.get_variable(\"b\", bias_shape, initializer=bias_init)\n",
    "    return tf.nn.relu(tf.nn.bias_add(tf.matmul(input, weight), bias=bias))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def inference(input, keep_prob, phase_train):\n",
    "    embedding = embedding_layer(input, [30000, 500])\n",
    "    lstm_output = lstm(embedding, 500, keep_prob, phase_train)\n",
    "    output = dense(lstm_output, [500, 2], [2], phase_train)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loss(output, y):\n",
    "    \"\"\"\n",
    "    output: the logits value from inference\n",
    "    y: the labeling data\n",
    "    \"\"\"\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=output, labels=y)\n",
    "    loss = tf.reduce_mean(cross_entropy)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def training(loss, global_step):\n",
    "    tf.summary.scalar(\"loss\", loss)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "    grads = tf.gradients(loss, tf.trainable_variables())\n",
    "    grads = list(zip(grads, tf.trainable_variables()))\n",
    "    apply_grads = optimizer.apply_gradients(grads_and_vars=grads, global_step=global_step)\n",
    "    return grads, apply_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(output, y):\n",
    "    compare = tf.equal(tf.argmax(output, 1), tf.argmax(y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(compare, tf.float32))\n",
    "    tf.summary.scalar(\"eval\", accuracy)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run 'tensorboard --logdir=/Users/jiankaiwang/devops/tmp/emotion_log' to monitor the training process.\n"
     ]
    }
   ],
   "source": [
    "print(\"Run 'tensorboard --logdir={}' to monitor the training process.\".format(log_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name LSTM/E:0 is illegal; using LSTM/E_0 instead.\n",
      "INFO:tensorflow:Summary name LSTM/rnn/basic_lstm_cell/kernel:0 is illegal; using LSTM/rnn/basic_lstm_cell/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name LSTM/rnn/basic_lstm_cell/bias:0 is illegal; using LSTM/rnn/basic_lstm_cell/bias_0 instead.\n",
      "INFO:tensorflow:Summary name LSTM/W:0 is illegal; using LSTM/W_0 instead.\n",
      "INFO:tensorflow:Summary name LSTM/b:0 is illegal; using LSTM/b_0 instead.\n",
      "INFO:tensorflow:Summary name LSTM/E:0/gradient is illegal; using LSTM/E_0/gradient instead.\n",
      "INFO:tensorflow:Summary name LSTM/rnn/basic_lstm_cell/kernel:0/gradient is illegal; using LSTM/rnn/basic_lstm_cell/kernel_0/gradient instead.\n",
      "INFO:tensorflow:Summary name LSTM/rnn/basic_lstm_cell/bias:0/gradient is illegal; using LSTM/rnn/basic_lstm_cell/bias_0/gradient instead.\n",
      "INFO:tensorflow:Summary name LSTM/W:0/gradient is illegal; using LSTM/W_0/gradient instead.\n",
      "INFO:tensorflow:Summary name LSTM/b:0/gradient is illegal; using LSTM/b_0/gradient instead.\n",
      "epoch: 0, accuracy: 0.48399999737739563\n",
      "Training finished.\n"
     ]
    }
   ],
   "source": [
    "with tf.Graph().as_default():\n",
    "    with tf.variable_scope(\"LSTM\"):\n",
    "        # input, label\n",
    "        x = tf.placeholder(tf.int32, [None, 500])\n",
    "        y = tf.placeholder(tf.int32, [None, 2])\n",
    "        keep_prob = tf.placeholder(tf.float32)\n",
    "        phase_train = tf.placeholder(tf.bool)\n",
    "        \n",
    "        # inference, training, evaluation\n",
    "        output = inference(x, keep_prob, phase_train)\n",
    "        loss_val = loss(output, y)\n",
    "        global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "        train_grads, train_opt = training(loss_val, global_step)\n",
    "        eval_opt = evaluate(output, y)\n",
    "        \n",
    "        # summary\n",
    "        for var in tf.trainable_variables():\n",
    "            tf.summary.histogram(var.name, var)\n",
    "            \n",
    "        for grad, var in train_grads:\n",
    "            tf.summary.histogram(var.name + \"/gradient\", grad)\n",
    "        \n",
    "        summary_opt = tf.summary.merge_all()\n",
    "            \n",
    "        # initialization\n",
    "        init_var = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()\n",
    "        \n",
    "        # start leanring\n",
    "        with tf.Session() as sess:\n",
    "            summary_writer = tf.summary.FileWriter(log_path, graph=sess.graph)\n",
    "            sess.run(init_var)\n",
    "            \n",
    "            batch_count = int(len(trainX) / batch_size)\n",
    "            for epoch in range(training_epochs):\n",
    "                avg_cost = 0.\n",
    "                \n",
    "                for batch in range(batch_count):\n",
    "                    b_x, b_y = train.minibatch(batch_size)\n",
    "                    feed_data = {x: b_x, y: b_y, keep_prob: 0.5, phase_train: True}\n",
    "                    grads, _ = sess.run([train_grads, train_opt], feed_dict=feed_data)\n",
    "                    \n",
    "                    batch_loss = sess.run(loss_val, feed_dict=feed_data)\n",
    "                    avg_cost += batch_loss / batch_size\n",
    "                    \n",
    "                if epoch % display_step == 0:\n",
    "                    # validation \n",
    "                    b_v_x, b_v_y = test.minibatch(batch_size)\n",
    "                    feed_val_data = {x: b_v_x, y: b_v_y, keep_prob: 1.0, phase_train: False}\n",
    "                    acc = sess.run(eval_opt, feed_dict=feed_val_data)\n",
    "                    tf.summary.scalar(\"val_acc\", acc)\n",
    "                    print(\"epoch: {}, accuracy: {}\".format(epoch, acc))\n",
    "                    \n",
    "                    # summary\n",
    "                    summary_str = sess.run(summary_opt, feed_dict=feed_val_data)\n",
    "                    summary_writer.add_summary(summary_str, sess.run(global_step))\n",
    "                    saver.save(sess, os.path.join(log_path, \"model-checkpoint\"), global_step=global_step)\n",
    "                    \n",
    "            print(\"Training finished.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}