Caltech 256 - Dark Knowledge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.applications import VGG16\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Dropout, Input\n",
    "from keras.regularizers import l2, activity_l2,l1\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import array_to_img, img_to_array, load_img\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from PIL import Image\n",
    "from scipy import misc\n",
    "from keras.optimizers import SGD\n",
    "from os import listdir\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temperature=1\n",
    "\n",
    "def softmaxTemp(x):\n",
    "    return K.softmax(x/temperature)\n",
    "\n",
    "def getModel( output_dim):\n",
    "    # output_dim: the number of classes (int)\n",
    "    # return: compiled model (keras.engine.training.Model)\n",
    "    \n",
    "    vgg_model = VGG16( weights='imagenet', include_top=True )\n",
    "    vgg_out = vgg_model.layers[-1].output\n",
    "    \n",
    "    out = Dense( output_dim, activation='softmax')( vgg_out )\n",
    "    \n",
    "    tl_model = Model( input=vgg_model.input, output=out)\n",
    "    \n",
    "    tl_model.layers[-2].activation=softmaxTemp\n",
    "    \n",
    "    for layer in tl_model.layers[0:-1]:\n",
    "        layer.trainable = False            \n",
    "\n",
    "    tl_model.compile(loss= \"categorical_crossentropy\", optimizer=\"adagrad\", metrics=[\"accuracy\"])\n",
    "    tl_model.summary()\n",
    "    return tl_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# define functions to laod images\n",
    "def loadBatchImages(path,s, nVal = 2):\n",
    "    # return array of images\n",
    "    catList = listdir(path)\n",
    "    loadedImagesTrain = []\n",
    "    loadedLabelsTrain = []\n",
    "    loadedImagesVal = []\n",
    "    loadedLabelsVal = []\n",
    "\n",
    "\n",
    "    for cat in catList[0:256]:\n",
    "        deepPath = path+cat+\"/\"\n",
    "        # if cat == \".DS_Store\": continue\n",
    "        imageList = listdir(deepPath)\n",
    "        indx = 0\n",
    "        for images in imageList[0:s + nVal]:                \n",
    "            img = load_img(deepPath + images)\n",
    "            img = img_to_array(img)\n",
    "            img = misc.imresize(img, (224,224))\n",
    "            img = scipy.misc.imrotate(img,180)\n",
    "            if indx < s:\n",
    "                loadedLabelsTrain.append(int(images[0:3])-1)\n",
    "                loadedImagesTrain.append(img)\n",
    "            else:\n",
    "                loadedLabelsVal.append(int(images[0:3])-1)\n",
    "                loadedImagesVal.append(img)\n",
    "            indx += 1\n",
    "            \n",
    "\n",
    "    return loadedImagesTrain, np_utils.to_categorical(loadedLabelsTrain), \\\n",
    "loadedImagesVal, np_utils.to_categorical(loadedLabelsVal) \n",
    "    \n",
    "def shuffledSet(a, b):\n",
    "    # shuffle the entire dataset\n",
    "    assert np.shape(a)[0] == np.shape(b)[0]\n",
    "    p = np.random.permutation(np.shape(a)[0])\n",
    "    return (a[p], b[p])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = \"/mnt/cube/VGG_/256_ObjectCategories/\"    \n",
    "samCat = 8 # number of samples per category\n",
    "\n",
    "data, labels, dataVal, labelsVal = loadBatchImages(path,samCat, nVal = 2)\n",
    "\n",
    "data = preprocess_input(np.float64(data))\n",
    "data = data.swapaxes(1, 3).swapaxes(2, 3)\n",
    "\n",
    "dataVal = preprocess_input(np.float64(dataVal))\n",
    "dataVal = dataVal.swapaxes(1, 3).swapaxes(2, 3)\n",
    "\n",
    "train = shuffledSet(np.asarray(data),labels)\n",
    "val = shuffledSet(np.asarray(dataVal),labelsVal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# plt.imshow(train[0][0][0])\n",
    "# plt.show()\n",
    "print train[0].shape, val[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_8 (InputLayer)             (None, 3, 224, 224)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv1 (Convolution2D)     (None, 64, 224, 224)  1792        input_8[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv2 (Convolution2D)     (None, 64, 224, 224)  36928       block1_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block1_pool (MaxPooling2D)       (None, 64, 112, 112)  0           block1_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv1 (Convolution2D)     (None, 128, 112, 112) 73856       block1_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv2 (Convolution2D)     (None, 128, 112, 112) 147584      block2_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)       (None, 128, 56, 56)   0           block2_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv1 (Convolution2D)     (None, 256, 56, 56)   295168      block2_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv2 (Convolution2D)     (None, 256, 56, 56)   590080      block3_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv3 (Convolution2D)     (None, 256, 56, 56)   590080      block3_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)       (None, 256, 28, 28)   0           block3_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv1 (Convolution2D)     (None, 512, 28, 28)   1180160     block3_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv2 (Convolution2D)     (None, 512, 28, 28)   2359808     block4_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv3 (Convolution2D)     (None, 512, 28, 28)   2359808     block4_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)       (None, 512, 14, 14)   0           block4_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv1 (Convolution2D)     (None, 512, 14, 14)   2359808     block4_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv2 (Convolution2D)     (None, 512, 14, 14)   2359808     block5_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv3 (Convolution2D)     (None, 512, 14, 14)   2359808     block5_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_pool (MaxPooling2D)       (None, 512, 7, 7)     0           block5_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "flatten (Flatten)                (None, 25088)         0           block5_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "fc1 (Dense)                      (None, 4096)          102764544   flatten[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "fc2 (Dense)                      (None, 4096)          16781312    fc1[0][0]                        \n",
      "____________________________________________________________________________________________________\n",
      "predictions (Dense)              (None, 1000)          4097000     fc2[0][0]                        \n",
      "____________________________________________________________________________________________________\n",
      "dense_8 (Dense)                  (None, 256)           256256      predictions[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 138,613,800\n",
      "Trainable params: 256,256\n",
      "Non-trainable params: 138,357,544\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "output_dim = 256\n",
    "tl_model = getModel(output_dim) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2048 samples, validate on 512 samples\n",
      "Epoch 1/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.5295 - acc: 0.0264 - val_loss: 5.5011 - val_acc: 0.0586\n",
      "Epoch 2/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.4816 - acc: 0.0859 - val_loss: 5.4778 - val_acc: 0.0840\n",
      "Epoch 3/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.4542 - acc: 0.1299 - val_loss: 5.4598 - val_acc: 0.1113\n",
      "Epoch 4/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.4322 - acc: 0.1890 - val_loss: 5.4446 - val_acc: 0.1367\n",
      "Epoch 5/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.4134 - acc: 0.2334 - val_loss: 5.4313 - val_acc: 0.1523\n",
      "Epoch 6/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.3966 - acc: 0.2671 - val_loss: 5.4192 - val_acc: 0.1660\n",
      "Epoch 7/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.3813 - acc: 0.2979 - val_loss: 5.4081 - val_acc: 0.1719\n",
      "Epoch 8/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.3672 - acc: 0.3198 - val_loss: 5.3978 - val_acc: 0.1836\n",
      "Epoch 9/20\n",
      "2048/2048 [==============================] - 59s - loss: 5.3541 - acc: 0.3501 - val_loss: 5.3882 - val_acc: 0.1953\n",
      "Epoch 10/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.3417 - acc: 0.3691 - val_loss: 5.3791 - val_acc: 0.2207\n",
      "Epoch 11/20\n",
      "2048/2048 [==============================] - 59s - loss: 5.3300 - acc: 0.3862 - val_loss: 5.3705 - val_acc: 0.2266\n",
      "Epoch 12/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.3189 - acc: 0.4038 - val_loss: 5.3622 - val_acc: 0.2266\n",
      "Epoch 13/20\n",
      "2048/2048 [==============================] - 59s - loss: 5.3083 - acc: 0.4209 - val_loss: 5.3543 - val_acc: 0.2324\n",
      "Epoch 14/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.2980 - acc: 0.4385 - val_loss: 5.3467 - val_acc: 0.2441\n",
      "Epoch 15/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.2882 - acc: 0.4541 - val_loss: 5.3394 - val_acc: 0.2500\n",
      "Epoch 16/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.2787 - acc: 0.4702 - val_loss: 5.3324 - val_acc: 0.2578\n",
      "Epoch 17/20\n",
      "2048/2048 [==============================] - 59s - loss: 5.2696 - acc: 0.4873 - val_loss: 5.3255 - val_acc: 0.2637\n",
      "Epoch 18/20\n",
      "2048/2048 [==============================] - 59s - loss: 5.2607 - acc: 0.4985 - val_loss: 5.3189 - val_acc: 0.2617\n",
      "Epoch 19/20\n",
      "2048/2048 [==============================] - 59s - loss: 5.2521 - acc: 0.5059 - val_loss: 5.3125 - val_acc: 0.2695\n",
      "Epoch 20/20\n",
      "2048/2048 [==============================] - 60s - loss: 5.2437 - acc: 0.5181 - val_loss: 5.3062 - val_acc: 0.2734\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.EarlyStopping at 0x7f8391889290>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_epoch = 20\n",
    "\n",
    "history = tl_model.fit(train[0], train[1], batch_size = 16, nb_epoch = nb_epoch, validation_data = val, \n",
    "                       shuffle = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss for %d samples per category' % samCat)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='right left')\n",
    "plt.show()\n",
    "\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy for %d samples per category' % samCat)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=[2, 4, 8, 16, 64]\n",
    "Y=[19.82, 25.20, 18.36, 18.75]\n",
    "plt.plot(X,Y)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}