3. two_stream_fusion_validate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import random\n",
    "import cv2\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Input, average, concatenate, GlobalAveragePooling2D\n",
    "from keras.layers import TimeDistributed, GlobalAveragePooling1D\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "from keras.layers.core import Dense,Flatten\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup\n",
    "path = './data/hmdb51'\n",
    "path_rowframes = './data/hmdb51/rawframes/'\n",
    "path_annotations = './data/hmdb51/annotations/'\n",
    "\n",
    "# Parametri Comuni\n",
    "img_height = 224\n",
    "img_width = 224\n",
    "image_shape=(img_height, img_width)\n",
    "batch_size = 256\n",
    "num_classes = 51\n",
    "\n",
    "# Parametri del temporal batch generator\n",
    "num_of_snip=1\n",
    "opt_flow_len=10\n",
    "\n",
    "# Parametri di evaluation\n",
    "fuse_method = 'average'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch Generation Setting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tentativo di creare un generator che invii simultaneamente 1 frame di un video e 1 stacked optical flow rispettivamente ai frame spaziali e temporali"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataSet():\n",
    "    def __init__(self, \n",
    "                 num_of_snip=1, \n",
    "                 opt_flow_len=10, \n",
    "                 image_shape=(224, 224),\n",
    "                 partition='val',\n",
    "                 batch_size = 16):\n",
    "        \n",
    "    # opt_flow_len = (int) number of optical flow frames pet stacked optical flow (snip)\n",
    "\n",
    "        self.opt_flow_len = opt_flow_len\n",
    "        self.num_of_snip = num_of_snip\n",
    "        self.image_shape = image_shape\n",
    "        self.opt_flow_path = os.path.join(path_rowframes)\n",
    "        self.path_annotations = path_annotations\n",
    "        self.partition = partition\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "        # Get data\n",
    "        self.video_list = self.find_videos_and_metadata()\n",
    "        self.n_batch = len(self.video_list) // self.batch_size\n",
    "\n",
    "        \n",
    "    def find_videos_and_metadata(self):\n",
    "        if self.partition == 'val':\n",
    "            video_list = pd.read_csv(f'{self.path_annotations}/hmdb51_val_split_1_rawframes.txt', sep=\" \", header=None) #test\n",
    "            video_list.columns = [\"path\", \"num_frames_tot\", \"class\"]\n",
    "        else:\n",
    "            raise Exception(\"invalid partition\")\n",
    "        return(video_list)\n",
    "    \n",
    "    def val_generator(self):\n",
    "        video_list = self.video_list\n",
    "        idx = 0\n",
    "        print(f\"Creating validation generator with {len(self.video_list)} samples.\")\n",
    "        while 1:\n",
    "            idx +=1\n",
    "            idx = idx % self.n_batch\n",
    "            #print(f\"Generator creating batch {idx}\")\n",
    "            X_spatial_batch = []\n",
    "            X_temporal_batch = []\n",
    "            y_batch = []\n",
    "            \n",
    "            batch_list = video_list.iloc[idx * self.batch_size: (idx + 1) * self.batch_size]\n",
    "            for index, row in batch_list.iterrows():\n",
    "                # Get the stacked optical flows from disk.\n",
    "                X_spatial, X_temporal = self.find_frame_and_stacked_optical_flows(row)\n",
    "                y = row['class']\n",
    "                y = np.array(y)\n",
    "                y = np.squeeze(y) \n",
    "\n",
    "                X_spatial_batch.append(X_spatial)\n",
    "                X_temporal_batch.append(X_temporal)\n",
    "                y_batch.append(y)\n",
    "\n",
    "            X_batch = [np.array(X_spatial_batch), np.array(X_temporal_batch)]\n",
    "            y_batch = np.array(y_batch)\n",
    "            \n",
    "            yield X_batch, y_batch\n",
    "    \n",
    "    def val_generator1(self):\n",
    "        video_list = self.video_list\n",
    "        idx = 0\n",
    "        #print(f\"Creating validation generator with {len(self.video_list)} samples.\")\n",
    "        idx +=1\n",
    "        idx = idx % self.n_batch\n",
    "        #print(f\"Generator creating batch {idx}\")\n",
    "        X_spatial_batch = []\n",
    "        X_temporal_batch = []\n",
    "        y_batch = []\n",
    "        #print(video_list)\n",
    "        batch_list = video_list.iloc[idx * self.batch_size: (idx + 1) * self.batch_size]\n",
    "        #print(type(batch_list))\n",
    "        for index, row in batch_list.iterrows():\n",
    "            #print(row[0][2])\n",
    "            # Get the stacked optical flows from disk.\n",
    "            #print(row['num_frames_tot'])\n",
    "            # print(type(row))\n",
    "            X_spatial, X_temporal = self.find_frame_and_stacked_optical_flows(row)\n",
    "            y = row['class']\n",
    "            y = np.array(y)\n",
    "            y = np.squeeze(y) \n",
    "\n",
    "            X_spatial_batch.append(X_spatial)\n",
    "            X_temporal_batch.append(X_temporal)\n",
    "            y_batch.append(y)\n",
    "\n",
    "        X_batch = [np.array(X_spatial_batch), np.array(X_temporal_batch)]\n",
    "        y_batch = np.array(y_batch)\n",
    "        \n",
    "        return X_batch, y_batch\n",
    "            \n",
    "    def find_frame_and_stacked_optical_flows(self, row):\n",
    "        static_frames = []\n",
    "        opt_flow_stack = []\n",
    "        opt_flow_dir = self.opt_flow_path\n",
    "        \n",
    "        # Temporal parameters\n",
    "        total_frames = row['num_frames_tot'] # row[0][1]\n",
    "        if total_frames - self.opt_flow_len + 1 < self.num_of_snip:\n",
    "            loop = True\n",
    "            start_frame_window_len = 1\n",
    "        else:\n",
    "            loop = False\n",
    "            start_frame_window_len = (total_frames - self.opt_flow_len + 1) // self.num_of_snip # starting frame selection window length\n",
    "        '''win_len = (total_frames - self.opt_flow_len) // self.num_of_snip\n",
    "        if self.partition=='train':\n",
    "            start_frame = int(random.random() * win_len) + 1\n",
    "        else:\n",
    "            start_frame = int(0.5 * win_len) + 1\n",
    "        frames = [] # selected optical flow frames\n",
    "        for i in range(self.num_of_snip):\n",
    "            frames += range(start_frame + self.opt_flow_len * i, \n",
    "                            start_frame + self.opt_flow_len * (i + 1))  \n",
    "        if self.partition == 'train' and random.random() > 0.5:\n",
    "            flip = True\n",
    "        else:\n",
    "            flip = False'''\n",
    "        \n",
    "        # Spatial Parameter\n",
    "        img_path = None\n",
    "        img_path = row['path']\n",
    "        img_test = cv2.imread(os.path.join(f'.\\\\{img_path}' + '\\\\img_' + str(\"%05d\"%(1)) + '.jpg'), 0)\n",
    "        #print(img_test)\n",
    "        top = int((img_test.shape[0] - self.image_shape[0]) * random.random())\n",
    "        left = int((img_test.shape[1] - self.image_shape[1]) * random.random())\n",
    "        right = left + self.image_shape[1]\n",
    "        bottom = top + self.image_shape[0]\n",
    "        \n",
    "        # loop over snip\n",
    "        for i_snip in range(self.num_of_snip):\n",
    "            if loop:\n",
    "                start_frame = i_snip % (total_frames - self.opt_flow_len + 1) + 1\n",
    "            else:\n",
    "                start_frame = int(0.5 * start_frame_window_len + 0.5) + start_frame_window_len * i_snip\n",
    "\n",
    "            # Get the static frame\n",
    "            static_frame = cv2.imread(os.path.join(f'.\\\\{img_path}' + '\\\\img_' + str(\"%05d\"%(start_frame)) + '.jpg'))\n",
    "            static_frame = static_frame / 255.0\n",
    "            static_frame = cv2.resize(static_frame, self.image_shape)\n",
    "\n",
    "            static_frames.append(static_frame)\n",
    "\n",
    "            # Get the optical flow stack\n",
    "            frames = range(start_frame, start_frame + self.opt_flow_len) # selected optical flow frames\n",
    "            opt_flow_stack = []\n",
    "            for i_frame in frames:\n",
    "                # x flow\n",
    "                img = None # reset to be safe\n",
    "                temp_path = None\n",
    "                temp_path = row['path']\n",
    "                img = cv2.imread(os.path.join(f'.\\\\{temp_path}' + '\\\\flow_x_' + str(\"%05d\"%(i_frame)) + '.jpg'), 0)\n",
    "                #print(os.path.join(f'.\\\\{temp_path}' + '\\\\flow_x_' + str(\"%05d\"%(i_frame)) + '.jpg'))\n",
    "                #print(os.path.join(f'.\\\\{temp_path}' + '\\\\flow_x_' + str(\"%05d\"%(i_frame)) + '.jpg'))\n",
    "                #print(img.shape)\n",
    "                img = np.array(img)\n",
    "                # mean substraction \n",
    "                img = img - np.mean(img)\n",
    "                img = img[top : bottom, left : right]\n",
    "                img = img / 255. # normalize pixels \n",
    "                img = cv2.resize(img, self.image_shape)\n",
    "                #print(img.shape)\n",
    "                opt_flow_stack.append(img)\n",
    "                \n",
    "                # y flow\n",
    "                img2 = None # reset to be safe\n",
    "                img2 = cv2.imread(os.path.join(f'.\\\\{temp_path}' + '\\\\flow_y_' + str(\"%05d\"%(i_frame)) + '.jpg'), 0)\n",
    "                #print(img2.shape)\n",
    "                img2 = np.array(img2)\n",
    "                #img2 = np.swapaxes(img2, 0, 1)\n",
    "                img2 = img2 - np.mean(img2)\n",
    "                img2 = img2[top : bottom, left : right]\n",
    "                img2 = img2 / 255. # normalize pixels\n",
    "                img2 = cv2.resize(img2, self.image_shape)\n",
    "                #print(img2.shape)\n",
    "                opt_flow_stack.append(img2)\n",
    "                \n",
    "            opt_flow_stack = np.array(opt_flow_stack)\n",
    "            opt_flow_stack = np.swapaxes(opt_flow_stack, 0, 2)\n",
    "        # random horizontal flip for training sets\n",
    "        \n",
    "        return np.array(static_frames), np.array(opt_flow_stack)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Two Stream Model: Fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def two_stream_fuse(spatial_model, temporal_model):\n",
    "    # spatial stream (frozen)\n",
    "    cnn_spatial = spatial_model\n",
    "\n",
    "    # temporal stream (frozen)\n",
    "    cnn_temporal = temporal_model\n",
    "\n",
    "    # fused by taking average\n",
    "    outputs = average([cnn_spatial.output, cnn_temporal.output])\n",
    "\n",
    "    model = Model([cnn_spatial.input, temporal_model.input], outputs)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classe data_val\n",
    "data_val = DataSet(num_of_snip=num_of_snip, \n",
    "                  opt_flow_len=opt_flow_len, \n",
    "                  image_shape=image_shape,\n",
    "                  partition='val',\n",
    "                  batch_size = 256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creazione del generatore\n",
    "validation_generator = data_val.val_generator()\n",
    "steps = data_val.n_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importazione dei modelli\n",
    "spatial_model = load_model('./Models/spatial_spat_resnet.hdf5')\n",
    "temporal_model = load_model('./Models/model_mot.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model\n",
    "two_stream_model = two_stream_fuse(spatial_model, temporal_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling\n",
    "optimizer = Adam()\n",
    "two_stream_model.compile(loss=keras.losses.sparse_categorical_crossentropy, \n",
    "                   metrics=['sparse_categorical_accuracy','sparse_top_k_categorical_accuracy'], \n",
    "                   optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " conv2d_5_input (InputLayer)    [(None, 224, 224, 2  0           []                               \n",
      "                                0)]                                                               \n",
      "                                                                                                  \n",
      " conv2d_5 (Conv2D)              (None, 109, 109, 96  94176       ['conv2d_5_input[0][0]']         \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " batch_normalization_2 (BatchNo  (None, 109, 109, 96  384        ['conv2d_5[0][0]']               \n",
      " rmalization)                   )                                                                 \n",
      "                                                                                                  \n",
      " max_pooling2d_3 (MaxPooling2D)  (None, 55, 55, 96)  0           ['batch_normalization_2[0][0]']  \n",
      "                                                                                                  \n",
      " zero_padding2d_4 (ZeroPadding2  (None, 57, 57, 96)  0           ['max_pooling2d_3[0][0]']        \n",
      " D)                                                                                               \n",
      "                                                                                                  \n",
      " conv2d_6 (Conv2D)              (None, 27, 27, 256)  614656      ['zero_padding2d_4[0][0]']       \n",
      "                                                                                                  \n",
      " batch_normalization_3 (BatchNo  (None, 27, 27, 256)  1024       ['conv2d_6[0][0]']               \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " max_pooling2d_4 (MaxPooling2D)  (None, 14, 14, 256)  0          ['batch_normalization_3[0][0]']  \n",
      "                                                                                                  \n",
      " zero_padding2d_5 (ZeroPadding2  (None, 16, 16, 256)  0          ['max_pooling2d_4[0][0]']        \n",
      " D)                                                                                               \n",
      "                                                                                                  \n",
      " conv2d_7 (Conv2D)              (None, 14, 14, 512)  1180160     ['zero_padding2d_5[0][0]']       \n",
      "                                                                                                  \n",
      " zero_padding2d_6 (ZeroPadding2  (None, 16, 16, 512)  0          ['conv2d_7[0][0]']               \n",
      " D)                                                                                               \n",
      "                                                                                                  \n",
      " conv2d_8 (Conv2D)              (None, 14, 14, 512)  2359808     ['zero_padding2d_6[0][0]']       \n",
      "                                                                                                  \n",
      " zero_padding2d_7 (ZeroPadding2  (None, 16, 16, 512)  0          ['conv2d_8[0][0]']               \n",
      " D)                                                                                               \n",
      "                                                                                                  \n",
      " conv2d_9 (Conv2D)              (None, 14, 14, 512)  2359808     ['zero_padding2d_7[0][0]']       \n",
      "                                                                                                  \n",
      " max_pooling2d_5 (MaxPooling2D)  (None, 7, 7, 512)   0           ['conv2d_9[0][0]']               \n",
      "                                                                                                  \n",
      " flatten_1 (Flatten)            (None, 25088)        0           ['max_pooling2d_5[0][0]']        \n",
      "                                                                                                  \n",
      " resnet50_input (InputLayer)    [(None, 224, 224, 3  0           []                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " dense_3 (Dense)                (None, 4096)         102764544   ['flatten_1[0][0]']              \n",
      "                                                                                                  \n",
      " resnet50 (Functional)          (None, 2048)         23587712    ['resnet50_input[0][0]']         \n",
      "                                                                                                  \n",
      " dropout_2 (Dropout)            (None, 4096)         0           ['dense_3[0][0]']                \n",
      "                                                                                                  \n",
      " flatten (Flatten)              (None, 2048)         0           ['resnet50[0][0]']               \n",
      "                                                                                                  \n",
      " dense_4 (Dense)                (None, 2048)         8390656     ['dropout_2[0][0]']              \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 512)          1049088     ['flatten[0][0]']                \n",
      "                                                                                                  \n",
      " dropout_3 (Dropout)            (None, 2048)         0           ['dense_4[0][0]']                \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 51)           26163       ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " dense_5 (Dense)                (None, 51)           104499      ['dropout_3[0][0]']              \n",
      "                                                                                                  \n",
      " average (Average)              (None, 51)           0           ['dense_1[0][0]',                \n",
      "                                                                  'dense_5[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 142,532,678\n",
      "Trainable params: 118,944,262\n",
      "Non-trainable params: 23,588,416\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "two_stream_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "two_stream_model.fit_generator(generator=validation_generator, steps_per_epoch=steps)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.9 ('deepL37')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5f73b1806a9ec290aebb7f16dd5394fd2c9a0b8bec90a8791426b7ef5a3d9604"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}