ResNet_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import metrics\n",
    "from tensorflow.keras.layers import Dense, Conv2D, BatchNormalization, Activation, MaxPooling2D, UpSampling2D\n",
    "from tensorflow.keras.layers import AveragePooling2D, Input, Flatten\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "from tensorflow.keras.models import Model\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import cv2\n",
    "import h5py\n",
    "\n",
    "# 自動增長 GPU 記憶體用量\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "print(physical_devices)\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "from tensorflow.python.framework.ops import disable_eager_execution\n",
    "disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training parameters\n",
    "batch_size = 32  # orig paper trained all networks with batch_size=128\n",
    "epochs = 200\n",
    "data_augmentation = True\n",
    "num_classes = 10\n",
    "\n",
    "n = 3\n",
    "\n",
    "# Model version\n",
    "# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)\n",
    "version = 1\n",
    "\n",
    "# Computed depth from supplied model parameter n\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "    \n",
    "# Model name, depth and version\n",
    "model_type = 'CIFAR10_ResNet%dv%d' % (depth, version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the CIFAR10 data.\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "subtract_pixel_mean = True\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_layer(inputs,\n",
    "                 num_filters=16,\n",
    "                 kernel_size=3,\n",
    "                 strides=1,\n",
    "                 activation='relu',\n",
    "                 kernel_regularizer=l2(1e-4),\n",
    "                 batch_normalization=True,\n",
    "                 conv_first=True):\n",
    "    \"\"\"2D Convolution-Batch Normalization-Activation stack builder\n",
    "\n",
    "    # Arguments\n",
    "        inputs (tensor): input tensor from input image or previous layer\n",
    "        num_filters (int): Conv2D number of filters\n",
    "        kernel_size (int): Conv2D square kernel dimensions\n",
    "        strides (int): Conv2D square stride dimensions\n",
    "        activation (string): activation name\n",
    "        batch_normalization (bool): whether to include batch normalization\n",
    "        conv_first (bool): conv-bn-activation (True) or\n",
    "            bn-activation-conv (False)\n",
    "\n",
    "    # Returns\n",
    "        x (tensor): tensor as input to the next layer\n",
    "    \"\"\"\n",
    "    conv = Conv2D(num_filters,\n",
    "                  kernel_size=kernel_size,\n",
    "                  strides=strides,\n",
    "                  padding='same',\n",
    "                  kernel_initializer='he_normal',\n",
    "                  kernel_regularizer=kernel_regularizer)\n",
    "\n",
    "    x = inputs\n",
    "    if conv_first:\n",
    "        x = conv(x)\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "    else:\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "        x = conv(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v1(input_shape, depth, num_classes=10, kernel_regularizer=l2(1e-4)):\n",
    "    \"\"\"ResNet Version 1 Model builder [a]\n",
    "\n",
    "    Stacks of 2 x (3 x 3) Conv2D-BN-ReLU\n",
    "    Last ReLU is after the shortcut connection.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filters is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same number of filters.\n",
    "    Features maps sizes:\n",
    "    stage 0: 32x32, 16\n",
    "    stage 1: 16x16, 32\n",
    "    stage 2:  8x8,  64\n",
    "    The Number of parameters is approx the same as Table 6 of [a]:\n",
    "    ResNet20 0.27M\n",
    "    ResNet32 0.46M\n",
    "    ResNet44 0.66M\n",
    "    ResNet56 0.85M\n",
    "    ResNet110 1.7M\n",
    "\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 6 != 0:\n",
    "        raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])')\n",
    "    # Start model definition.\n",
    "    num_filters = 16\n",
    "    num_res_blocks = int((depth - 2) / 6)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = resnet_layer(inputs=inputs, kernel_regularizer=kernel_regularizer)\n",
    "    # Instantiate the stack of residual units\n",
    "    for stack in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            strides = 1\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                strides = 2  # downsample\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters,\n",
    "                             kernel_regularizer=kernel_regularizer,\n",
    "                             strides=strides)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters,\n",
    "                             kernel_regularizer=kernel_regularizer,\n",
    "                             activation=None)\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 kernel_regularizer=kernel_regularizer,\n",
    "                                 batch_normalization=False)\n",
    "            x = tf.keras.layers.add([x, y])\n",
    "            x = Activation('relu')(x)\n",
    "        num_filters *= 2\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v1 does not use BN after last shortcut connection-ReLU\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v2(input_shape, depth, num_classes=10, kernel_regularizer=l2(1e-4)):\n",
    "    \"\"\"ResNet Version 2 Model builder [b]\n",
    "\n",
    "    Stacks of (1 x 1)-(3 x 3)-(1 x 1) BN-ReLU-Conv2D or also known as\n",
    "    bottleneck layer\n",
    "    First shortcut connection per layer is 1 x 1 Conv2D.\n",
    "    Second and onwards shortcut connection is identity.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filter maps is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same filter map sizes.\n",
    "    Features maps sizes:\n",
    "    conv1  : 32x32,  16\n",
    "    stage 0: 32x32,  64\n",
    "    stage 1: 16x16, 128\n",
    "    stage 2:  8x8,  256\n",
    "\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 9 != 0:\n",
    "        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n",
    "    # Start model definition.\n",
    "    num_filters_in = 16\n",
    "    num_res_blocks = int((depth - 2) / 9)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n",
    "    x = resnet_layer(inputs=inputs,\n",
    "                     num_filters=num_filters_in,\n",
    "                     kernel_regularizer=kernel_regularizer,\n",
    "                     conv_first=True)\n",
    "\n",
    "    # Instantiate the stack of residual units\n",
    "    for stage in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            activation = 'relu'\n",
    "            batch_normalization = True\n",
    "            strides = 1\n",
    "            if stage == 0:\n",
    "                num_filters_out = num_filters_in * 4\n",
    "                if res_block == 0:  # first layer and first stage\n",
    "                    activation = None\n",
    "                    batch_normalization = False\n",
    "            else:\n",
    "                num_filters_out = num_filters_in * 2\n",
    "                if res_block == 0:  # first layer but not first stage\n",
    "                    strides = 2    # downsample\n",
    "\n",
    "            # bottleneck residual unit\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_size=1,\n",
    "                             strides=strides,\n",
    "                             activation=activation,\n",
    "                             batch_normalization=batch_normalization,\n",
    "                             kernel_regularizer=kernel_regularizer,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_regularizer=kernel_regularizer,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_out,\n",
    "                             kernel_regularizer=kernel_regularizer,\n",
    "                             kernel_size=1,\n",
    "                             conv_first=False)\n",
    "            if res_block == 0:\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters_out,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 kernel_regularizer=kernel_regularizer,\n",
    "                                 batch_normalization=False)\n",
    "            x = tf.keras.layers.add([x, y])\n",
    "\n",
    "        num_filters_in = num_filters_out\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v2 has BN-ReLU before Pooling\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert class vectors to binary class matrices.\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/z99792458/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1635: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    }
   ],
   "source": [
    "if version == 2:\n",
    "    model = resnet_v2(input_shape=input_shape, depth=depth, num_classes=num_classes, kernel_regularizer=l2(1e-4))\n",
    "else:\n",
    "    model = resnet_v1(input_shape=input_shape, depth=depth, num_classes=num_classes, kernel_regularizer=l2(1e-4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 32, 32, 16)   448         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 32, 32, 16)   64          conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 32, 32, 16)   0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   2320        activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   2320        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 32, 32, 16)   0           activation[0][0]                 \n",
      "                                                                 batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 32, 32, 16)   2320        activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 16)   64          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 32, 32, 16)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 32, 32, 16)   2320        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 32, 32, 16)   64          conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 16)   0           activation_2[0][0]               \n",
      "                                                                 batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 32, 32, 16)   0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 32, 32, 16)   2320        activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 32, 32, 16)   64          conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 32, 32, 16)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 32, 32, 16)   2320        activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 32, 32, 16)   64          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 32, 32, 16)   0           activation_4[0][0]               \n",
      "                                                                 batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 32, 32, 16)   0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 16, 16, 32)   4640        activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 16, 16, 32)   128         conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 16, 16, 32)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 16, 16, 32)   9248        activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 32)   544         activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 16, 16, 32)   128         conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 16, 16, 32)   0           conv2d_9[0][0]                   \n",
      "                                                                 batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 16, 16, 32)   0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 16, 16, 32)   9248        activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 16, 16, 32)   128         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 16, 16, 32)   0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 16, 16, 32)   9248        activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 16, 16, 32)   128         conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 16, 16, 32)   0           activation_8[0][0]               \n",
      "                                                                 batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 16, 16, 32)   0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 16, 16, 32)   9248        activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 16, 16, 32)   128         conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 16, 16, 32)   0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 16, 16, 32)   9248        activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 16, 16, 32)   128         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 16, 16, 32)   0           activation_10[0][0]              \n",
      "                                                                 batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 16, 16, 32)   0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 8, 8, 64)     18496       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 8, 8, 64)     256         conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 8, 8, 64)     0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 8, 8, 64)     36928       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 8, 8, 64)     2112        activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 8, 8, 64)     256         conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 8, 8, 64)     0           conv2d_16[0][0]                  \n",
      "                                                                 batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 8, 8, 64)     0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 8, 8, 64)     36928       activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 8, 8, 64)     256         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 8, 8, 64)     0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 8, 8, 64)     36928       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 8, 8, 64)     256         conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 8, 8, 64)     0           activation_14[0][0]              \n",
      "                                                                 batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 8, 8, 64)     0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 8, 8, 64)     36928       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 8, 8, 64)     256         conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 8, 8, 64)     0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 8, 8, 64)     36928       activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 8, 8, 64)     256         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 8, 8, 64)     0           activation_16[0][0]              \n",
      "                                                                 batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 8, 8, 64)     0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d (AveragePooli (None, 1, 1, 64)     0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 64)           0           average_pooling2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 10)           650         flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 274,442\n",
      "Trainable params: 273,066\n",
      "Non-trainable params: 1,376\n",
      "__________________________________________________________________________________________________\n",
      "CIFAR10_ResNet20v1\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(lr=lr_schedule(0)),\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "print(model_type)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare model model saving directory.\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "import datetime as dt\n",
    "model_name = 'pmon_%s_%s_model.{epoch:03d}.h5' % (model_type, dt.datetime.now().strftime(\"%Y%m%d%H\"))\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)\n",
    "\n",
    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=1,\n",
    "                             save_best_only=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "WARNING:tensorflow:From <ipython-input-14-780cfb4a8826>:63: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use Model.fit, which supports generators.\n",
      "Learning rate:  0.001\n",
      "Epoch 1/200\n",
      "1562/1563 [============================>.] - ETA: 0s - loss: 1.5649 - accuracy: 0.4901Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 3s 290us/sample - loss: 1.4639 - accuracy: 0.5944\n",
      "\n",
      "Epoch 00001: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.001.h5\n",
      "1563/1563 [==============================] - 63s 40ms/step - loss: 1.5646 - accuracy: 0.4901 - val_loss: 1.3208 - val_accuracy: 0.5944\n",
      "Learning rate:  0.001\n",
      "Epoch 2/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 1.1803 - accuracy: 0.6373Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 221us/sample - loss: 1.1259 - accuracy: 0.6467\n",
      "\n",
      "Epoch 00002: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.002.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 1.1799 - accuracy: 0.6374 - val_loss: 1.1782 - val_accuracy: 0.6467\n",
      "Learning rate:  0.001\n",
      "Epoch 3/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 1.0244 - accuracy: 0.6976Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 217us/sample - loss: 0.8959 - accuracy: 0.6946\n",
      "\n",
      "Epoch 00003: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.003.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 1.0244 - accuracy: 0.6976 - val_loss: 1.0646 - val_accuracy: 0.6946\n",
      "Learning rate:  0.001\n",
      "Epoch 4/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.9419 - accuracy: 0.7310Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 218us/sample - loss: 1.1644 - accuracy: 0.6554\n",
      "\n",
      "Epoch 00004: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.004.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 0.9418 - accuracy: 0.7310 - val_loss: 1.2077 - val_accuracy: 0.6554\n",
      "Learning rate:  0.001\n",
      "Epoch 5/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.8821 - accuracy: 0.7519Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 216us/sample - loss: 1.0092 - accuracy: 0.7275\n",
      "\n",
      "Epoch 00005: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.005.h5\n",
      "1563/1563 [==============================] - 54s 35ms/step - loss: 0.8819 - accuracy: 0.7520 - val_loss: 0.9495 - val_accuracy: 0.7275\n",
      "Learning rate:  0.001\n",
      "Epoch 6/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.8286 - accuracy: 0.7729Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 220us/sample - loss: 0.7929 - accuracy: 0.7456\n",
      "\n",
      "Epoch 00006: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.006.h5\n",
      "1563/1563 [==============================] - 52s 34ms/step - loss: 0.8288 - accuracy: 0.7729 - val_loss: 0.9411 - val_accuracy: 0.7456\n",
      "Learning rate:  0.001\n",
      "Epoch 7/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.7954 - accuracy: 0.7862Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 222us/sample - loss: 0.6567 - accuracy: 0.7838\n",
      "\n",
      "Epoch 00007: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.007.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 0.7952 - accuracy: 0.7863 - val_loss: 0.8160 - val_accuracy: 0.7838\n",
      "Learning rate:  0.001\n",
      "Epoch 8/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.7678 - accuracy: 0.7982Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 224us/sample - loss: 1.0970 - accuracy: 0.6881\n",
      "\n",
      "Epoch 00008: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.008.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 0.7679 - accuracy: 0.7982 - val_loss: 1.2752 - val_accuracy: 0.6881\n",
      "Learning rate:  0.001\n",
      "Epoch 9/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.7414 - accuracy: 0.8067Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 221us/sample - loss: 0.8472 - accuracy: 0.7742\n",
      "\n",
      "Epoch 00009: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.009.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 0.7413 - accuracy: 0.8067 - val_loss: 0.8637 - val_accuracy: 0.7742\n",
      "Learning rate:  0.001\n",
      "Epoch 10/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.7194 - accuracy: 0.8146Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 217us/sample - loss: 0.7484 - accuracy: 0.7945\n",
      "\n",
      "Epoch 00010: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.010.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 0.7193 - accuracy: 0.8147 - val_loss: 0.7884 - val_accuracy: 0.7945\n",
      "Learning rate:  0.001\n",
      "Epoch 11/200\n",
      "1562/1563 [============================>.] - ETA: 0s - loss: 0.7025 - accuracy: 0.8207Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 218us/sample - loss: 0.7336 - accuracy: 0.7913\n",
      "\n",
      "Epoch 00011: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.011.h5\n",
      "1563/1563 [==============================] - 53s 34ms/step - loss: 0.7024 - accuracy: 0.8207 - val_loss: 0.8081 - val_accuracy: 0.7913\n",
      "Learning rate:  0.001\n",
      "Epoch 12/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.6931 - accuracy: 0.8251Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 4s 425us/sample - loss: 0.7625 - accuracy: 0.7763\n",
      "\n",
      "Epoch 00012: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.012.h5\n",
      "1563/1563 [==============================] - 55s 35ms/step - loss: 0.6935 - accuracy: 0.8249 - val_loss: 0.8606 - val_accuracy: 0.7763\n",
      "Learning rate:  0.001\n",
      "Epoch 13/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.6721 - accuracy: 0.8322Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 4s 428us/sample - loss: 0.8784 - accuracy: 0.7486\n",
      "\n",
      "Epoch 00013: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.013.h5\n",
      "1563/1563 [==============================] - 61s 39ms/step - loss: 0.6721 - accuracy: 0.8322 - val_loss: 1.0364 - val_accuracy: 0.7486\n",
      "Learning rate:  0.001\n",
      "Epoch 14/200\n",
      "1562/1563 [============================>.] - ETA: 0s - loss: 0.6620 - accuracy: 0.8375Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 221us/sample - loss: 1.2250 - accuracy: 0.6754\n",
      "\n",
      "Epoch 00014: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.014.h5\n",
      "1563/1563 [==============================] - 55s 35ms/step - loss: 0.6620 - accuracy: 0.8375 - val_loss: 1.2894 - val_accuracy: 0.6754\n",
      "Learning rate:  0.001\n",
      "Epoch 15/200\n",
      "1561/1563 [============================>.] - ETA: 0s - loss: 0.6531 - accuracy: 0.8412Epoch 1/200\n",
      "10000/1563 [===============================================================================================================================================================================================] - 2s 220us/sample - loss: 0.6648 - accuracy: 0.8091\n",
      "\n",
      "Epoch 00015: saving model to /home/z99792458/nbs/detection_models_imagenet_100/resnet/saved_models/pmon_CIFAR10_ResNet20v1_2020081508_model.015.h5\n",
      "1563/1563 [==============================] - 56s 36ms/step - loss: 0.6531 - accuracy: 0.8411 - val_loss: 0.7621 - val_accuracy: 0.8091\n",
      "Learning rate:  0.001\n",
      "Epoch 16/200\n",
      "  45/1563 [..............................] - ETA: 48s - loss: 0.6179 - accuracy: 0.8569"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-780cfb4a8826>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     61\u001b[0m                         \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m                         \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m                         callbacks=callbacks)\n\u001b[0m",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m               \u001b[0;34m'in a future version'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'after %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m               instructions)\n\u001b[0;32m--> 324\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m     return tf_decorator.make_decorator(\n\u001b[1;32m    326\u001b[0m         \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'deprecated'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1304\u001b[0m         \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1305\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m         initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1308\u001b[0m   @deprecation.deprecated(\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    817\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    818\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 819\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    820\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    821\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m    602\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    603\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 604\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m    605\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    606\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m       \u001b[0mis_deferred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_compiled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 265\u001b[0;31m       \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    266\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m   1121\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_sample_weight_modes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1123\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1125\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3565\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3566\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3567\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3568\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3569\u001b[0m     output_structure = nest.pack_sequence_as(\n",
      "\u001b[0;32m~/anaconda_env/tf2/lib/python3.7/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1472\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1473\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1474\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1475\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1476\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Run training, with or without data augmentation.\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    history = model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=callbacks)\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for featurewise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    history = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs, verbose=1, workers=1,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True,\n",
    "    rotation_range=20,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    horizontal_flip=True)\n",
    "train_datagen.fit(x_train)\n",
    "validation_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True,\n",
    "    rotation_range=20,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    horizontal_flip=True)\n",
    "validation_datagen.fit(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights('./saved_models/pmon_%s_%s_model.%s.final.h5' % (model_type, dt.datetime.now().strftime(\"%Y%m%d%H\"), '22.2e-4'))  # The third string parameter is the suffix string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm \n",
    "import matplotlib.pyplot as plt \n",
    "import matplotlib.cbook as cbook \n",
    "\n",
    "plt.plot(history.history['acc']) \n",
    "plt.plot(history.history['val_acc']) \n",
    "plt.title('model accuracy') \n",
    "plt.ylabel('accuracy') \n",
    "plt.xlabel('epoch') \n",
    "plt.legend(['train', 'validation'], loc='upper left') \n",
    "plt.show()\n",
    "\n",
    "# summarize history for loss \n",
    "plt.plot(history.history['loss']) \n",
    "plt.plot(history.history['val_loss']) \n",
    "plt.title('model loss') \n",
    "plt.ylabel('loss') \n",
    "plt.xlabel('epoch') \n",
    "plt.legend(['train', 'validation'], loc='upper left') \n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 205us/sample - loss: 0.6857 - accuracy: 0.8326\n",
      "Test loss: 0.6857166467666626\n",
      "Test accuracy: 0.8326\n"
     ]
    }
   ],
   "source": [
    "# Score trained model.\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}