HRNN_2-fortest2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#在该版本中测试进行 采用变长输入的改进，即读取时间长度信息并作为变长输入的依据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/uu201817092/.local/lib/python3.7/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 10010). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:109.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n"
     ]
    }
   ],
   "source": [
    "# 第二版不同之处在于使用全连接层进行特征融合\n",
    "#Step1.导入包并选定设备\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "import time\n",
    "from torch.nn.utils.rnn import pack_padded_sequence,pad_packed_sequence\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Step2.模型参数设置\n",
    "\n",
    "sequence_length = 300  #序列长度，最大帧为300，但这里还需要更改\n",
    "input_size = 75       #输入数据特征大小 3（x,y,z）*25（关节数量）\n",
    "hidden_size = 128     #隐藏层数据特征大小,即每个时间步对应的ht的维数\n",
    "num_layers = 2        #隐藏层层数\n",
    "num_classes = 60      #结果类数\n",
    "batch_size = 1000     #一个batch大小\n",
    "num_epochs = 50       #epoch数目\n",
    "learning_rate = 0.001  #学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 新增代码\n",
    "# 用于读取数据集\n",
    "import pickle\n",
    "import numpy as np\n",
    "class Feeder(torch.utils.data.Dataset):\n",
    "    \"\"\" \n",
    "    参数：\n",
    "    data_path:.npy形式的数据的路径，数据的格式需要是(N,C,T,V,M)\n",
    "        N：样本数目，C：有几个相机，T：帧数，V：有几个关节点，M：动作次数\n",
    "    label_path：标签的路径\n",
    "    random_choose：如果为真，则随机的选择输入序列中的一部分\n",
    "    random_shift:如果为真，则在序列的开始和结束时随机的填充0\n",
    "    window_size:输出序列的宽度\n",
    "    normalization:如果为真，则对序列进行标准化\n",
    "    debug:如果为真，则仅使用前100个样本\n",
    "    mmap：如果为真，则使用虚拟内存映射（因为数据集太大了，故需要虚拟内存映射）\n",
    "    \"\"\"\n",
    "\n",
    "    '''\n",
    "    使用了标准化的版本\n",
    "    def __init__(self,\n",
    "                 data_path,\n",
    "                 label_path,\n",
    "                 random_choose=False,\n",
    "                 random_move=False,\n",
    "                 window_size=-1,\n",
    "                 debug=False,\n",
    "                 mmap=True):\n",
    "        self.debug = debug\n",
    "        self.data_path = data_path\n",
    "        self.label_path = label_path\n",
    "        self.random_choose = random_choose\n",
    "        self.random_move = random_move\n",
    "        self.window_size = window_size\n",
    "        self.load_data(mmap)\n",
    "    '''\n",
    "    \n",
    "    def __init__(self,\n",
    "                 data_path,\n",
    "                 seq_len_path,\n",
    "                 label_path,\n",
    "                 window_size=-1,\n",
    "                 debug=False,\n",
    "                 mmap=True):\n",
    "        self.debug = debug\n",
    "        self.data_path = data_path\n",
    "        self.seq_len_path = seq_len_path\n",
    "        self.label_path = label_path\n",
    "        self.window_size = window_size\n",
    "        self.load_data(mmap)\n",
    "        \n",
    "    def load_data(self, mmap):\n",
    "        # data: N C V T M\n",
    "\n",
    "        # 加载标签\n",
    "        with open(self.label_path, 'rb') as f:\n",
    "            self.sample_name, self.label = pickle.load(f)\n",
    "\n",
    "        # load data\n",
    "        if mmap:\n",
    "            # 如果使用了虚拟内存映射，则使用虚拟内存映射模式加载数据\n",
    "            self.data = np.load(self.data_path, mmap_mode='r')\n",
    "            self.seq_len = np.load(self.seq_len_path, mmap_mode='r')\n",
    "        else:\n",
    "            self.data = np.load(self.data_path)\n",
    "            self.seq_len = np.load(self.seq_len_path)\n",
    "        # 如果是debug模式，则不载入全部数据,注：原来是100，为了方便观察这里改成的2\n",
    "        if self.debug:\n",
    "            self.label = self.label[0:10]\n",
    "            self.data = self.data[0:10]\n",
    "            self.seq_len = self.seq_len[0:10]\n",
    "            self.sample_name = self.sample_name[0:10]\n",
    "\n",
    "        self.N, self.C, self.T, self.V, self.M = self.data.shape\n",
    "\n",
    "    # 获取数据集大小\n",
    "    def __len__(self):\n",
    "        return len(self.label)\n",
    "\n",
    "    # 用于获取某一个数据的函数\n",
    "    def __getitem__(self, index):\n",
    "        data_numpy = np.array(self.data[index])\n",
    "        label = self.label[index]\n",
    "        seq_len = self.seq_len[index]\n",
    "        \"\"\"\n",
    "        预处理过程，后续可以在此基础上补充\n",
    "        注：预处理是在__getitem__中进行的\n",
    "        if self.random_choose:\n",
    "            data_numpy = tools.random_choose(data_numpy, self.window_size)\n",
    "        elif self.window_size > 0:\n",
    "            data_numpy = tools.auto_pading(data_numpy, self.window_size)\n",
    "        if self.random_move:\n",
    "            data_numpy = tools.random_move(data_numpy)\n",
    "        \"\"\"\n",
    "        return data_numpy, label,seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 新增代码\n",
    "# 用于读取数据\n",
    "'''\n",
    "self,\n",
    "data_path,\n",
    "label_path,\n",
    "window_size=-1,\n",
    "debug=False,\n",
    "mmap=True)\n",
    "'''\n",
    "train_set = Feeder(data_path='./cooked_data/xsub/train_data.npy',\n",
    "                  label_path='./cooked_data/xsub/train_label.pkl',\n",
    "                   seq_len_path='./cooked_data/xsub/train_data_seqlen.npy',\n",
    "                  )\n",
    "\n",
    "test_set = Feeder(data_path='./cooked_data/xsub/val_data.npy',\n",
    "                  label_path='./cooked_data/xsub/val_label.pkl',\n",
    "                  seq_len_path='./cooked_data/xsub/val_data_seqlen.npy',\n",
    "                  )\n",
    "\n",
    "# 添加了一个debug_set用于观察数据\n",
    "debug_set = Feeder(data_path='./cooked_data/xsub/train_data.npy',\n",
    "                  label_path='./cooked_data/xsub/train_label.pkl',\n",
    "                   seq_len_path='./cooked_data/xsub/train_data_seqlen.npy',\n",
    "                   debug=True,\n",
    "                  )\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_set,\n",
    "                                      batch_size = batch_size,\n",
    "                                      shuffle = True,\n",
    "                                      num_workers = 4,\n",
    "                                        drop_last=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test_set,\n",
    "                                      batch_size = batch_size,\n",
    "                                      num_workers = 4,\n",
    "                                         drop_last=True)\n",
    "\n",
    "\n",
    "debug_loader = torch.utils.data.DataLoader(debug_set,\n",
    "                                           batch_size = 10,\n",
    "                                      num_workers = 0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
      "tensor([[103.],\n",
      "        [158.],\n",
      "        [104.],\n",
      "        [ 99.],\n",
      "        [ 54.],\n",
      "        [ 75.],\n",
      "        [ 86.],\n",
      "        [ 71.],\n",
      "        [ 67.],\n",
      "        [ 69.]])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/uu201817092/.local/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py:63: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:143.)\n",
      "  return default_collate([torch.as_tensor(b) for b in batch])\n"
     ]
    }
   ],
   "source": [
    "# 第0维代表是第几个样本\n",
    "# 第一维存的是关节的x,y,z坐标\n",
    "# 第二维用于标识是哪一帧\n",
    "# 第三维用于标识是哪个关节\n",
    "# 第四维用于表示是哪个身体\n",
    "# 1*3*300*25*2->1*3*300*25*1最后只取了一个身体方便计算\n",
    "# batch_x = batch_x.view(-1,sequence_length,input_size)\n",
    "\n",
    "'''\n",
    "for batch_x,batch_y in debug_loader:\n",
    "    for i in range(5):\n",
    "        print(batch_x.size(i))\n",
    "        \n",
    "    print('\\n')\n",
    "    batch_x = batch_x[:,:,:,:,0].view(-1,300,75)\n",
    "    for i in range(3):\n",
    "        print(batch_x.size(i))\n",
    "    print(batch_y)\n",
    "'''\n",
    "for batch_x,batch_y,seq_len in debug_loader:\n",
    "    print(batch_y)\n",
    "    print(seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Step4.模型定义\n",
    "# 数据类型\n",
    "# 数据的第0维是序号即代表是哪一个样本\n",
    "# 第一维存的是关节的x,y,z坐标\n",
    "# 第二维用于标识是哪一帧\n",
    "# 第三维用于标识是哪个关节\n",
    "# 第四维用于表示是哪个身体\n",
    "class HRNN(nn.Module):\n",
    "    # 实现三层架构，即首先经过两层普通BRNN并经过全连接层融合，最后经过一层LSTM的BRNN，然后用FC表示\n",
    "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
    "        super(HRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        \n",
    "        #如果要使用反向的传递，则需令bidirectional=True\n",
    "        # batch_first代表传入数据为（batch,seq,feature)的顺序 否则Pytroch所有RNN网络默认输入结构为(seq,batch,feature)\n",
    "        # batch_first = true代表输入X为 batch_size,seq_len,input_size\n",
    "        # 为了测试循环的提升，将True改成了False\n",
    "        \n",
    "        '''\n",
    "        self.rnn1_4 = nn.RNN(4, int(hidden_size/4), num_layers, batch_first=True, bidirectional=True)\n",
    "        self.rnn1_5 = nn.RNN(5, int(hidden_size/4), num_layers, batch_first=True, bidirectional=True)\n",
    "        self.rnn1_6 = nn.RNN(6, int(hidden_size/4), num_layers, batch_first=True, bidirectional=True)\n",
    "        '''\n",
    "        \n",
    "        \n",
    "        self.rnn = nn.RNN(int(input_size/5), int(hidden_size/4), num_layers, batch_first=True, bidirectional=True)\n",
    "        self.rnn2 = nn.RNN(hidden_size, hidden_size, num_layers, batch_first=True, bidirectional=True)\n",
    "        self.lstm = nn.LSTM(int(hidden_size/4*2*2), hidden_size, num_layers, batch_first=True, bidirectional=True)\n",
    "        \n",
    "        \n",
    "        #如果使用了反向传递，则需要将hidden_size*2!\n",
    "        self.fc = nn.Linear(hidden_size*2, num_classes)\n",
    "        self.fs1 = nn.Linear(hidden_size,hidden_size)\n",
    "        self.fs2 = nn.Linear(hidden_size*4,hidden_size)\n",
    "        \n",
    "    def forward(self, x,seq_len):\n",
    "        # 输入：\n",
    "        # X为batch_size*seq_len*input_size(batch_first=true时)\n",
    "        \n",
    "        # 输出：\n",
    "        # 输出为out,(hn,cn)\n",
    "        # out(seq_len, batch_size, num_directions*hidden_size) 即为[h1,h2,...,hseq_len]\n",
    "        # 即out = torch.Size([1000, 28, 128])\n",
    "        # out: tensor of shape (batch_size, seq_length, hidden_size*2)\n",
    "        '''\n",
    "        注意减1！！！\n",
    "        x_p1为左手臂部分（24、25、12、11、10、9）6\n",
    "        x_p2为右手臂部分（22、23、8、7、6、5）6\n",
    "        x_p3为左腿部分（20、19、18、17）4\n",
    "        x_p4为右腿部分（13、14、15、16）4\n",
    "        x_p5为躯干部分（1、2、3、4、21）5\n",
    "        '''\n",
    "        seq_len = torch.squeeze(seq_len)\n",
    "        # layer1:即分成五个部分利用rnn进行分别建模 75/5\n",
    "        # step1:将五个部分分别经过rnn层\n",
    "        '''\n",
    "        x_p1 = torch.cat((x[:,:,8:12],x[:,:,23:25]),2)\n",
    "        x_p2 = torch.cat((x[:,:,4:8],x[:,:,21:23]),2)\n",
    "        x_p3 = x[:,:,16:20]\n",
    "        x_p4 = x[:,:,12:16]\n",
    "        x_p5 = torch.cat((x[:,:,0:4],x[:,:,21:22]),2)\n",
    "        '''\n",
    "        \n",
    "        \n",
    "        \n",
    "        (x_p1,x_p2,x_p3,x_p4,x_p5) = torch.chunk(x, 5, dim = 2)\n",
    "        x_p1 = pack_padded_sequence(x_p1,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        x_p2 = pack_padded_sequence(x_p2,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        x_p3 = pack_padded_sequence(x_p3,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        x_p4 = pack_padded_sequence(x_p4,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        x_p5 = pack_padded_sequence(x_p5,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        out1_p1,_ = self.rnn(x_p1)\n",
    "        out1_p2,_ = self.rnn(x_p2)\n",
    "        out1_p3,_ = self.rnn(x_p3)\n",
    "        out1_p4,_ = self.rnn(x_p4)\n",
    "        out1_p5,_ = self.rnn(x_p5)\n",
    "        \n",
    "        out1_p1,_ = pad_packed_sequence(out1_p1,batch_first = True)\n",
    "        out1_p2,_ = pad_packed_sequence(out1_p2,batch_first = True)\n",
    "        out1_p3,_ = pad_packed_sequence(out1_p3,batch_first = True)\n",
    "        out1_p4,_ = pad_packed_sequence(out1_p4,batch_first = True)\n",
    "        out1_p5,_ = pad_packed_sequence(out1_p5,batch_first = True)\n",
    "        \n",
    "\n",
    "        #print(out1_p1.shape)\n",
    "        #torch.Size([1000, 216, 64])\n",
    "        # 经过第一个RNN得到的是五个子部分的表示\n",
    "        \n",
    "        \n",
    "        # step2:利用全连接层进行特征融合\n",
    "        # 先进行特征拼接\n",
    "        temp2_p1 = torch.cat((out1_p1,out1_p2),2) #(,,128/4*2*2),第一个2为双向乘的，第二个2为两个并在一起乘的\n",
    "        temp2_p2 = torch.cat((out1_p1,out1_p3),2)\n",
    "        temp2_p3 = torch.cat((out1_p1,out1_p4),2)\n",
    "        temp2_p4 = torch.cat((out1_p1,out1_p5),2)\n",
    "        \n",
    "        # 再进行特征融合\n",
    "        seqs = temp2_p1.size(1)\n",
    "        in2_p1 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp2_p1[:,seq,:],dim=1) #删除这个维度\n",
    "            in2_p1_i = F.relu(self.fs1(temp))\n",
    "            in2_p1.append(in2_p1_i)\n",
    "        in2_p1 = torch.stack(in2_p1,dim = 1)\n",
    "        in2_p2 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp2_p2[:,seq,:],dim=1) #删除这个维度\n",
    "            in2_p2_i = F.relu(self.fs1(temp))\n",
    "            in2_p2.append(in2_p2_i)\n",
    "        in2_p2 = torch.stack(in2_p2,dim = 1)\n",
    "        in2_p3 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp2_p3[:,seq,:],dim=1) #删除这个维度\n",
    "            in2_p3_i = F.relu(self.fs1(temp))\n",
    "            in2_p3.append(in2_p3_i)\n",
    "        in2_p3 = torch.stack(in2_p3,dim = 1)\n",
    "        in2_p4 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp2_p4[:,seq,:],dim=1) #删除这个维度\n",
    "            in2_p4_i = F.relu(self.fs1(temp))\n",
    "            in2_p4.append(in2_p4_i)\n",
    "        in2_p4 = torch.stack(in2_p4,dim = 1)\n",
    "        \n",
    "        \n",
    "        # layer2:用4个部分进行输入，得到结果经过融合层变成两部分\n",
    "        # step1:四个部分分别经过第二个rnn层\n",
    "        '''\n",
    "        in2_p1 = pack_padded_sequence(in2_p1,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        in2_p2 = pack_padded_sequence(in2_p2,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        in2_p3 = pack_padded_sequence(in2_p3,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        in2_p4 = pack_padded_sequence(in2_p4,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        '''\n",
    "        \n",
    "        \n",
    "        out2_p1,_ = self.rnn2(in2_p1)\n",
    "        out2_p2,_ = self.rnn2(in2_p2)\n",
    "        out2_p3,_ = self.rnn2(in2_p3)\n",
    "        out2_p4,_ = self.rnn2(in2_p4)\n",
    "        \n",
    "        '''\n",
    "        out2_p1,_ = pad_packed_sequence(out2_p1)\n",
    "        out2_p2,_ = pad_packed_sequence(out2_p2)\n",
    "        out2_p3,_ = pad_packed_sequence(out2_p3)\n",
    "        out2_p4,_ = pad_packed_sequence(out2_p4)\n",
    "        '''\n",
    "        \n",
    "        \n",
    "        # step2:利用全连接层进行特征融合\n",
    "        temp3_p1 = torch.cat((out2_p1,out2_p2),2)\n",
    "        temp3_p2 = torch.cat((out2_p3,out2_p4),2)\n",
    "        seqs = temp3_p1.size(1)\n",
    "        in3_p1 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp3_p1[:,seq,:],dim=1) #删除这个维度\n",
    "            in3_p1_i = F.relu(self.fs2(temp))\n",
    "            in3_p1.append(in3_p1_i)\n",
    "        in3_p1 = torch.stack(in3_p1,dim = 1)\n",
    "        \n",
    "        in3_p2 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp3_p2[:,seq,:],dim=1) #删除这个维度\n",
    "            in3_p2_i = F.relu(self.fs2(temp))\n",
    "            in3_p2.append(in3_p2_i)\n",
    "        in3_p2 = torch.stack(in3_p2,dim = 1)\n",
    "        \n",
    "        # layer3:将两个部分的结果再经过rnn层最终得到一个部分的结果\n",
    "        # step1:将两个部分分别经过rnn\n",
    "        '''\n",
    "        in3_p1 = pack_padded_sequence(in3_p1,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        in3_p2 = pack_padded_sequence(in3_p2,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        '''\n",
    "        \n",
    "        \n",
    "        out3_p1,_ = self.rnn2(in3_p1)\n",
    "        out3_p2,_ = self.rnn2(in3_p2)\n",
    "        \n",
    "        '''\n",
    "        out3_p1,_ = pad_packed_sequence(out3_p1)\n",
    "        out3_p2,_ = pad_packed_sequence(out3_p2)\n",
    "        '''\n",
    "        \n",
    "        # step2.利用全连接层进行特征融合\n",
    "        temp4_p1 = torch.cat((out3_p1,out3_p2),2)\n",
    "        in4_p1 = []\n",
    "        for seq in range(seqs):\n",
    "            temp = torch.squeeze(temp4_p1[:,seq,:],dim=1) #删除这个维度\n",
    "            in4_p1_i = F.relu(self.fs2(temp))\n",
    "            in4_p1.append(in4_p1_i)\n",
    "        in4_p1 = torch.stack(in4_p1,1)\n",
    "        \n",
    "        # layer3:整体作为输入经过lstm层得到输出\n",
    "        '''\n",
    "        in4_p1 = pack_padded_sequence(in4_p1,seq_len,batch_first = True,enforce_sorted=False)\n",
    "        '''\n",
    "        \n",
    "        #out(seq_len, batch_size, num_directions*hidden_size)\n",
    "        #[237, 1000, 256]\n",
    "        \n",
    "        out4,_ = self.lstm(in4_p1)\n",
    "        \n",
    "        #print(out4.shape)\n",
    "        #torch.Size([1000, 216, 256])\n",
    "        '''\n",
    "        out4_p1,_ = pad_packed_sequence(out3_p1)\n",
    "        '''\n",
    "        \n",
    "        # 代表仅取最后一个时间步的隐状态表示作为全连接层的输入(这显然是不合理的，因为有很多都没有到最后一帧)\n",
    "        #out = self.fc(out4[:, -1, :])\n",
    "        # 尝试一：将向量展平（但这样会存在很多0）\n",
    "        #out4 = out4.reshape(out4.size(0),-1)\n",
    "        #torch.Size([1000, 62720])\n",
    "        #print(out4.shape)\n",
    "        #out = self.fc(out4)\n",
    "        \n",
    "        \n",
    "        #尝试二：隔帧采样并逐帧累加\n",
    "        out = torch.zeros(out4.size(0),num_classes)\n",
    "        for step in range(int(seqs/20)):\n",
    "            temp = torch.squeeze(out4[:,step,:],dim=1)\n",
    "            out_i = self.fc(temp)\n",
    "            out = out + out_i\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# 该模型仅用于测试\\nclass TestFC(nn.Module):\\n    def __init__(self, input_size, hidden_size, num_layers, num_classes):\\n        super(TestFC, self).__init__()\\n        self.hidden_size = hidden_size\\n        self.num_layers = num_layers\\n        self.input_size = input_size\\n        self.num_classes = num_classes\\n        \\n        \\n        self.rnn = nn.RNN(input_size, hidden_size, num_layers, batch_first=True, bidirectional=True)\\n        self.fc = nn.Linear(hidden_size*2*sequence_length,num_classes)\\n        \\n        \\n    def forward(self, x):\\n        out1,_ = self.rnn(x)\\n        #print(out1.shape)\\n        #torch.Size([1000, 300, 128])\\n        out = self.fc(out1.reshape(out1.size(0),hidden_size*2*sequence_length))\\n        return out\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# 该模型仅用于测试\n",
    "class TestFC(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
    "        super(TestFC, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.input_size = input_size\n",
    "        self.num_classes = num_classes\n",
    "        \n",
    "        \n",
    "        self.rnn = nn.RNN(input_size, hidden_size, num_layers, batch_first=True, bidirectional=True)\n",
    "        self.fc = nn.Linear(hidden_size*2*sequence_length,num_classes)\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        out1,_ = self.rnn(x)\n",
    "        #print(out1.shape)\n",
    "        #torch.Size([1000, 300, 128])\n",
    "        out = self.fc(out1.reshape(out1.size(0),hidden_size*2*sequence_length))\n",
    "        return out\n",
    "'''\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step5.定义辅助函数用于模型评估\n",
    "def eval(model,criterion,dataloader):\n",
    "    loss = 0\n",
    "    accuracy = 0\n",
    "    for batch_x, batch_y,seq_len in dataloader:\n",
    "        batch_x = batch_x[:,:,:,:,0].view(-1,sequence_length,input_size)\n",
    "        \n",
    "        # batch_y = F.one_hot(batch_y,num_class)\n",
    "        batch_x, batch_y = batch_x.to(device),batch_y.to(device)\n",
    "        seq_len = seq_len.to(device)\n",
    "        logits = model(batch_x,seq_len)\n",
    "        error = criterion(logits,batch_y)\n",
    "        loss += error.item()\n",
    "        \n",
    "        probs,pred_y = logits.data.max(dim=1)\n",
    "        accuracy += (pred_y==batch_y.data).sum().double()/batch_y.size(0)\n",
    "        \n",
    "    loss /= len(dataloader)\n",
    "    accuracy = accuracy*100.0/len(dataloader)\n",
    "    return loss,accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Step6.模型使用以及损失函数、优化函数使用\n",
    "model = HRNN(input_size, hidden_size, num_layers, num_classes).to(device)\n",
    "model.train()\n",
    "# 使用交叉熵损失函数作为目标函数\n",
    "# 使用Adam作为优化函数\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Step7.模型训练\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    since = time.time()\n",
    "    for batch_x,batch_y,seq_len in train_loader:\n",
    "        # 暂时只取了第一个身体\n",
    "        batch_x = batch_x[:,:,:,:,0].view(-1,sequence_length,input_size)\n",
    "        # batch_y = F.one_hot(batch_y,num_classes)\n",
    "        # print(batch_x.size(0))\n",
    "        batch_x,batch_y = batch_x.to(device),batch_y.to(device)\n",
    "        seq_len = seq_len.to(device)\n",
    "        # print(batch_y.shape)\n",
    "        # print(batch_x.shape)\n",
    "        optimizer.zero_grad()\n",
    "        logit = model(batch_x,seq_len)\n",
    "        # print(logit.shape)\n",
    "        # print(batch_y.shape)\n",
    "        E = criterion(logit,batch_y)\n",
    "        E.backward()\n",
    "        optimizer.step()\n",
    "    now = time.time()\n",
    "    model.eval()\n",
    "    tr_loss, tr_acc = eval(model,criterion,train_loader)\n",
    "    te_loss, te_acc = eval(model,criterion,test_loader)\n",
    "    print('[%d/%d,%.0f seconds],train error:%.1e, train acc:%.2f\\t test error: %.1e,test acc: %.2f'%(epoch+1,num_epochs,now-since,tr_loss,tr_acc,te_loss,te_acc))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# step8.用于测试\n",
    "for batch_x,batch_y in debug_loader:\n",
    "    batch_x = batch_x[:,:,:,:,0].view(-1,sequence_length,input_size)\n",
    "    zero_bone = [0.0000 for _ in range(75)]\n",
    "    #print(batch_x.shape)\n",
    "    #print(batch_x)\n",
    "    # print(zero_bone)\n",
    "    print(batch_x[0,:,:])\n",
    "    print((batch_x[0,:,:] == zero_bone).nonzero(as_tuple = True)[0])\n",
    "    \n",
    "    batch_x,batch_y = batch_x.to(device),batch_y.to(device)\n",
    "    logit = model(batch_x)\n",
    "    print(logit)\n",
    "    '''\n",
    "    pred_y = logit.data.max(dim=1)\n",
    "    print(pred_y)\n",
    "    '''\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "#Step6.模型训练、测试与保存\n",
    "total_step = len(train_loader)\n",
    "print(total_step)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.reshape(-1, sequence_length, input_size).to(device)\n",
    "        labels = labels.to(device)\n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "# Test the model\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.reshape(-1, sequence_length, input_size).to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "    print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total)) \n",
    "\n",
    "# Save the model checkpoint\n",
    "torch.save(model.state_dict(), 'model.ckpt')\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'torch' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-18bf08814031>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'torch' is not defined"
     ]
    }
   ],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}